@fluidframework/container-runtime 2.0.0-dev.1.4.6.106135 → 2.0.0-dev.2.3.0.115467

package/lib/opLifecycle/opSplitter.js

@@ -0,0 +1,57 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { DataCorruptionError, extractSafePropertiesFromMessage } from "@fluidframework/container-utils";
+import { ContainerMessageType } from "../containerRuntime";
+/**
+ * Responsible for creating and reconstructing chunked messages.
+ */
+export class OpSplitter {
+    constructor(chunks) {
+        this.chunkMap = new Map(chunks);
+    }
+    get chunks() {
+        return this.chunkMap;
+    }
+    processRemoteMessage(message) {
+        if (message.type !== ContainerMessageType.ChunkedOp) {
+            return message;
+        }
+        const clientId = message.clientId;
+        const chunkedContent = message.contents;
+        this.addChunk(clientId, chunkedContent, message);
+        if (chunkedContent.chunkId < chunkedContent.totalChunks) {
+            // We are processing the op in chunks but haven't reached
+            // the last chunk yet in order to reconstruct the original op
+            return message;
+        }
+        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+        const serializedContent = this.chunkMap.get(clientId).join("");
+        this.clearPartialChunks(clientId);
+        const newMessage = Object.assign({}, message);
+        newMessage.contents = serializedContent === "" ? undefined : JSON.parse(serializedContent);
+        newMessage.type = chunkedContent.originalType;
+        return newMessage;
+    }
+    clearPartialChunks(clientId) {
+        if (this.chunkMap.has(clientId)) {
+            this.chunkMap.delete(clientId);
+        }
+    }
+    addChunk(clientId, chunkedContent, originalMessage) {
+        let map = this.chunkMap.get(clientId);
+        if (map === undefined) {
+            map = [];
+            this.chunkMap.set(clientId, map);
+        }
+        if (chunkedContent.chunkId !== map.length + 1) {
+            // We are expecting the chunks to be processed sequentially, in the same order as they are sent.
+            // Therefore, the chunkId of the incoming op needs to match the length of the array (1-based indexing)
+            // holding the existing chunks for that particular clientId.
+            throw new DataCorruptionError("Chunk Id mismatch", Object.assign(Object.assign({}, extractSafePropertiesFromMessage(originalMessage)), { chunkMapLength: map.length, chunkId: chunkedContent.chunkId, totalChunks: chunkedContent.totalChunks }));
+        }
+        map.push(chunkedContent.contents);
+    }
+}
+//# sourceMappingURL=opSplitter.js.map

package/lib/opLifecycle/opSplitter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"opSplitter.js","sourceRoot":"","sources":["../../src/opLifecycle/opSplitter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,mBAAmB,EAAE,gCAAgC,EAAE,MAAM,iCAAiC,CAAC;AAExG,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAG3D;;GAEG;AACH,MAAM,OAAO,UAAU;IAInB,YAAY,MAA4B;QACpC,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,CAAmB,MAAM,CAAC,CAAC;IACtD,CAAC;IAED,IAAW,MAAM;QACb,OAAO,IAAI,CAAC,QAAQ,CAAC;IACzB,CAAC;IAEM,oBAAoB,CAAC,OAAkC;QAC1D,IAAI,OAAO,CAAC,IAAI,KAAK,oBAAoB,CAAC,SAAS,EAAE;YACjD,OAAO,OAAO,CAAC;SAClB;QAED,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QAClC,MAAM,cAAc,GAAG,OAAO,CAAC,QAAsB,CAAC;QACtD,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;QAEjD,IAAI,cAAc,CAAC,OAAO,GAAG,cAAc,CAAC,WAAW,EAAE;YACrD,yDAAyD;YACzD,6DAA6D;YAC7D,OAAO,OAAO,CAAC;SAClB;QAED,oEAAoE;QACpE,MAAM,iBAAiB,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChE,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QAElC,MAAM,UAAU,qBAAQ,OAAO,CAAE,CAAC;QAClC,UAAU,CAAC,QAAQ,GAAG,iBAAiB,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;QAC3F,UAAU,CAAC,IAAI,GAAG,cAAc,CAAC,YAAY,CAAC;QAC9C,OAAO,UAAU,CAAC;IACtB,CAAC;IAEM,kBAAkB,CAAC,QAAgB;QACtC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;YAC7B,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;SAClC;IACL,CAAC;IAEO,QAAQ,CAAC,QAAgB,EAAE,cAA0B,EAAE,eAA0C;QACrG,IAAI,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,GAAG,KAAK,SAAS,EAAE;YACnB,GAAG,GAAG,EAAE,CAAC;YACT,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;SACpC;QAED,IAAI,cAAc,CAAC,OAAO,KAAK,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE;YAC3C,gGAAgG;YAChG,sGAAsG;YACtG,4DAA4D;YAC5D,MAAM,IAAI,mBAAmB,CAAC,mBAAmB,kCAC1C,gCAAgC,CAAC,eAAe,CAAC,KACpD,cAAc,EAAE,GAAG,CAAC,MAAM,EAC1B,OAAO,EAAE,cAAc,CAAC,OAAO,EAC/B,WAAW,EAAE,cAAc,CAAC,WAAW,IACzC,CAAC;SACN;QAED,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IACtC,CAAC;CACJ","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { DataCorruptionError, extractSafePropertiesFromMessage } from \"@fluidframework/container-utils\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/protocol-definitions\";\nimport { ContainerMessageType } from \"../containerRuntime\";\nimport { IChunkedOp } from \"./definitions\";\n\n/**\n * Responsible for creating and reconstructing chunked messages.\n */\nexport class OpSplitter {\n    // Local copy of incomplete received chunks.\n    private readonly chunkMap: Map<string, string[]>;\n\n    constructor(chunks: [string, string[]][]) {\n        this.chunkMap = new Map<string, string[]>(chunks);\n    }\n\n    public get chunks(): ReadonlyMap<string, string[]> {\n        return this.chunkMap;\n    }\n\n    public processRemoteMessage(message: ISequencedDocumentMessage): ISequencedDocumentMessage {\n        if (message.type !== ContainerMessageType.ChunkedOp) {\n            return message;\n        }\n\n        const clientId = message.clientId;\n        const chunkedContent = message.contents as IChunkedOp;\n        this.addChunk(clientId, chunkedContent, message);\n\n        if (chunkedContent.chunkId < chunkedContent.totalChunks) {\n            // We are processing the op in chunks but haven't reached\n            // the last chunk yet in order to reconstruct the original op\n            return message;\n        }\n\n        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n        const serializedContent = this.chunkMap.get(clientId)!.join(\"\");\n        this.clearPartialChunks(clientId);\n\n        const newMessage = { ...message };\n        newMessage.contents = serializedContent === \"\" ? undefined : JSON.parse(serializedContent);\n        newMessage.type = chunkedContent.originalType;\n        return newMessage;\n    }\n\n    public clearPartialChunks(clientId: string) {\n        if (this.chunkMap.has(clientId)) {\n            this.chunkMap.delete(clientId);\n        }\n    }\n\n    private addChunk(clientId: string, chunkedContent: IChunkedOp, originalMessage: ISequencedDocumentMessage) {\n        let map = this.chunkMap.get(clientId);\n        if (map === undefined) {\n            map = [];\n            this.chunkMap.set(clientId, map);\n        }\n\n        if (chunkedContent.chunkId !== map.length + 1) {\n            // We are expecting the chunks to be processed sequentially, in the same order as they are sent.\n            // Therefore, the chunkId of the incoming op needs to match the length of the array (1-based indexing)\n            // holding the existing chunks for that particular clientId.\n            throw new DataCorruptionError(\"Chunk Id mismatch\", {\n                ...extractSafePropertiesFromMessage(originalMessage),\n                chunkMapLength: map.length,\n                chunkId: chunkedContent.chunkId,\n                totalChunks: chunkedContent.totalChunks,\n            });\n        }\n\n        map.push(chunkedContent.contents);\n    }\n}\n"]}

package/lib/opLifecycle/outbox.d.ts

@@ -0,0 +1,47 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { IContainerContext } from "@fluidframework/container-definitions";
+import { ICompressionRuntimeOptions } from "../containerRuntime";
+import { PendingStateManager } from "../pendingStateManager";
+import { BatchMessage } from "./definitions";
+import { OpCompressor } from "./opCompressor";
+export interface IOutboxConfig {
+    readonly compressionOptions: ICompressionRuntimeOptions;
+    readonly maxBatchSizeInBytes: number;
+}
+export interface IOutboxParameters {
+    readonly shouldSend: () => boolean;
+    readonly pendingStateManager: PendingStateManager;
+    readonly containerContext: IContainerContext;
+    readonly config: IOutboxConfig;
+    readonly compressor: OpCompressor;
+}
+export declare class Outbox {
+    private readonly params;
+    private readonly attachFlowBatch;
+    private readonly mainBatch;
+    private readonly defaultAttachFlowSoftLimitInBytes;
+    constructor(params: IOutboxParameters);
+    get isEmpty(): boolean;
+    submit(message: BatchMessage): void;
+    submitAttach(message: BatchMessage): void;
+    flush(): void;
+    private flushInternal;
+    private compressBatch;
+    /**
+     * Sends the batch object to the container context to be sent over the wire.
+     *
+     * @param batch - batch to be sent
+     * @returns the client sequence number of the last batched op which was sent and
+     * -1 if there are no ops or the container cannot send ops.
+     */
+    private sendBatch;
+    private persistBatch;
+    checkpoint(): {
+        mainBatch: import("./definitions").IBatchCheckpoint;
+        attachFlowBatch: import("./definitions").IBatchCheckpoint;
+    };
+}
+//# sourceMappingURL=outbox.d.ts.map

package/lib/opLifecycle/outbox.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"outbox.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/outbox.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,iBAAiB,EAAE,MAAM,uCAAuC,CAAC;AAG1E,OAAO,EAAE,0BAA0B,EAAE,MAAM,qBAAqB,CAAC;AACjE,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAE7D,OAAO,EAAE,YAAY,EAAU,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C,MAAM,WAAW,aAAa;IAC1B,QAAQ,CAAC,kBAAkB,EAAE,0BAA0B,CAAC;IAExD,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;CACxC;AAED,MAAM,WAAW,iBAAiB;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,OAAO,CAAC;IACnC,QAAQ,CAAC,mBAAmB,EAAE,mBAAmB,CAAC;IAClD,QAAQ,CAAC,gBAAgB,EAAE,iBAAiB,CAAC;IAC7C,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,YAAY,CAAC;CACrC;AAED,qBAAa,MAAM;IAKH,OAAO,CAAC,QAAQ,CAAC,MAAM;IAJnC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAe;IAC/C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAe;IACzC,OAAO,CAAC,QAAQ,CAAC,iCAAiC,CAAa;gBAElC,MAAM,EAAE,iBAAiB;IAetD,IAAW,OAAO,IAAI,OAAO,CAE5B;IAEM,MAAM,CAAC,OAAO,EAAE,YAAY;IAa5B,YAAY,CAAC,OAAO,EAAE,YAAY;IA4BlC,KAAK;IAKZ,OAAO,CAAC,aAAa;IAOrB,OAAO,CAAC,aAAa;IAyBrB;;;;;;OAMG;IACH,OAAO,CAAC,SAAS;IAuCjB,OAAO,CAAC,YAAY;IAoBb,UAAU;;;;CAMpB"}

package/lib/opLifecycle/outbox.js

@@ -0,0 +1,149 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { assert } from "@fluidframework/common-utils";
+import { GenericError } from "@fluidframework/container-utils";
+import { MessageType } from "@fluidframework/protocol-definitions";
+import { BatchManager } from "./batchManager";
+;
+export class Outbox {
+    constructor(params) {
+        this.params = params;
+        this.defaultAttachFlowSoftLimitInBytes = 64 * 1024;
+        const isCompressionEnabled = this.params.config.compressionOptions.minimumBatchSizeInBytes !== Number.POSITIVE_INFINITY;
+        // We need to allow infinite size batches if we enable compression
+        const hardLimit = isCompressionEnabled ? Infinity : this.params.config.maxBatchSizeInBytes;
+        const softLimit = isCompressionEnabled ? Infinity : this.defaultAttachFlowSoftLimitInBytes;
+        this.attachFlowBatch = new BatchManager({
+            hardLimit,
+            softLimit,
+        });
+        this.mainBatch = new BatchManager({
+            hardLimit
+        });
+    }
+    get isEmpty() {
+        return this.attachFlowBatch.length === 0 && this.mainBatch.length === 0;
+    }
+    submit(message) {
+        var _a, _b;
+        if (!this.mainBatch.push(message)) {
+            throw new GenericError("BatchTooLarge", 
+            /* error */ undefined, {
+                opSize: (_b = ((_a = message.contents) === null || _a === void 0 ? void 0 : _a.length)) !== null && _b !== void 0 ? _b : 0,
+                count: this.mainBatch.length,
+                limit: this.mainBatch.options.hardLimit,
+            });
+        }
+    }
+    submitAttach(message) {
+        var _a, _b;
+        if (!this.attachFlowBatch.push(message)) {
+            // BatchManager has two limits - soft limit & hard limit. Soft limit is only engaged
+            // when queue is not empty.
+            // Flush queue & retry. Failure on retry would mean - single message is bigger than hard limit
+            this.flushInternal(this.attachFlowBatch.popBatch());
+            if (!this.attachFlowBatch.push(message)) {
+                throw new GenericError("BatchTooLarge", 
+                /* error */ undefined, {
+                    opSize: (_b = ((_a = message.contents) === null || _a === void 0 ? void 0 : _a.length)) !== null && _b !== void 0 ? _b : 0,
+                    count: this.attachFlowBatch.length,
+                    limit: this.attachFlowBatch.options.hardLimit,
+                });
+            }
+        }
+        // If compression is enabled, we will always successfully receive
+        // attach ops and compress then send them at the next JS turn, regardless
+        // of the overall size of the accumulated ops in the batch.
+        // However, it is more efficient to flush these ops faster, preferably
+        // after they reach a size which would benefit from compression.
+        if (this.attachFlowBatch.contentSizeInBytes >= this.params.config.compressionOptions.minimumBatchSizeInBytes) {
+            this.flushInternal(this.attachFlowBatch.popBatch());
+        }
+    }
+    flush() {
+        this.flushInternal(this.attachFlowBatch.popBatch());
+        this.flushInternal(this.mainBatch.popBatch());
+    }
+    flushInternal(rawBatch) {
+        const processedBatch = this.compressBatch(rawBatch);
+        const clientSequenceNumber = this.sendBatch(processedBatch);
+        this.persistBatch(clientSequenceNumber, rawBatch.content);
+    }
+    compressBatch(batch) {
+        if (batch.content.length === 0
+            || this.params.config.compressionOptions === undefined
+            || this.params.config.compressionOptions.minimumBatchSizeInBytes > batch.contentSizeInBytes) {
+            // Nothing to do if the batch is empty or if compression is disabled or if we don't need to compress
+            return batch;
+        }
+        const compressedBatch = this.params.compressor.compressBatch(batch);
+        if (compressedBatch.contentSizeInBytes > this.params.config.maxBatchSizeInBytes) {
+            throw new GenericError("BatchTooLarge", 
+            /* error */ undefined, {
+                opSize: batch.contentSizeInBytes,
+                count: batch.content.length,
+                limit: this.params.config.maxBatchSizeInBytes,
+                compressed: true,
+            });
+        }
+        // If we don't reach the maximum supported size of a batch, it safe to be sent as is
+        return compressedBatch;
+    }
+    /**
+     * Sends the batch object to the container context to be sent over the wire.
+     *
+     * @param batch - batch to be sent
+     * @returns the client sequence number of the last batched op which was sent and
+     * -1 if there are no ops or the container cannot send ops.
+     */
+    sendBatch(batch) {
+        var _a;
+        let clientSequenceNumber = -1;
+        const length = batch.content.length;
+        // Did we disconnect in the middle of turn-based batch?
+        // If so, do nothing, as pending state manager will resubmit it correctly on reconnect.
+        if (length === 0 || !this.params.shouldSend()) {
+            return clientSequenceNumber;
+        }
+        if (this.params.containerContext.submitBatchFn === undefined) {
+            // Legacy path - supporting old loader versions. Can be removed only when LTS moves above
+            // version that has support for batches (submitBatchFn)
+            for (const message of batch.content) {
+                // Legacy path doesn't support compressed payloads and will submit uncompressed payload anyways
+                if ((_a = message.metadata) === null || _a === void 0 ? void 0 : _a.compressed) {
+                    delete message.metadata.compressed;
+                }
+                clientSequenceNumber = this.params.containerContext.submitFn(MessageType.Operation, message.deserializedContent, true, // batch
+                message.metadata);
+            }
+            this.params.containerContext.deltaManager.flush();
+        }
+        else {
+            // returns clientSequenceNumber of last message in a batch
+            clientSequenceNumber = this.params.containerContext.submitBatchFn(batch.content.map((message) => ({ contents: message.contents, metadata: message.metadata })));
+        }
+        // Convert from clientSequenceNumber of last message in the batch to clientSequenceNumber of first message.
+        clientSequenceNumber -= length - 1;
+        assert(clientSequenceNumber >= 0, 0x3d0 /* clientSequenceNumber can't be negative */);
+        return clientSequenceNumber;
+    }
+    persistBatch(initialClientSequenceNumber, batch) {
+        let clientSequenceNumber = initialClientSequenceNumber;
+        // Let the PendingStateManager know that a message was submitted.
+        // In future, need to shift toward keeping batch as a whole!
+        for (const message of batch) {
+            this.params.pendingStateManager.onSubmitMessage(message.deserializedContent.type, clientSequenceNumber, message.referenceSequenceNumber, message.deserializedContent.contents, message.localOpMetadata, message.metadata);
+            clientSequenceNumber++;
+        }
+        this.params.pendingStateManager.onFlush();
+    }
+    checkpoint() {
+        return {
+            mainBatch: this.mainBatch.checkpoint(),
+            attachFlowBatch: this.attachFlowBatch.checkpoint(),
+        };
+    }
+}
+//# sourceMappingURL=outbox.js.map

package/lib/opLifecycle/outbox.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"outbox.js","sourceRoot":"","sources":["../../src/opLifecycle/outbox.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,8BAA8B,CAAC;AAEtD,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,MAAM,sCAAsC,CAAC;AAGnE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAQ7C,CAAC;AAUF,MAAM,OAAO,MAAM;IAKf,YAA6B,MAAyB;QAAzB,WAAM,GAAN,MAAM,CAAmB;QAFrC,sCAAiC,GAAG,EAAE,GAAG,IAAI,CAAC;QAG3D,MAAM,oBAAoB,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,uBAAuB,KAAK,MAAM,CAAC,iBAAiB,CAAC;QACxH,kEAAkE;QAClE,MAAM,SAAS,GAAG,oBAAoB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,mBAAmB,CAAC;QAC3F,MAAM,SAAS,GAAG,oBAAoB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,iCAAiC,CAAC;QAE3F,IAAI,CAAC,eAAe,GAAG,IAAI,YAAY,CAAC;YACpC,SAAS;YACT,SAAS;SACZ,CAAC,CAAC;QACH,IAAI,CAAC,SAAS,GAAG,IAAI,YAAY,CAAC;YAC9B,SAAS;SACZ,CAAC,CAAC;IACP,CAAC;IAED,IAAW,OAAO;QACd,OAAO,IAAI,CAAC,eAAe,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC;IAC5E,CAAC;IAEM,MAAM,CAAC,OAAqB;;QAC/B,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;YAC/B,MAAM,IAAI,YAAY,CAClB,eAAe;YACf,WAAW,CAAC,SAAS,EACrB;gBACI,MAAM,EAAE,MAAA,CAAC,MAAA,OAAO,CAAC,QAAQ,0CAAE,MAAM,CAAC,mCAAI,CAAC;gBACvC,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM;gBAC5B,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,SAAS;aAC1C,CAAC,CAAC;SACV;IACL,CAAC;IAEM,YAAY,CAAC,OAAqB;;QACrC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;YACrC,oFAAoF;YACpF,2BAA2B;YAC3B,8FAA8F;YAC9F,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,CAAC;YACpD,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;gBACrC,MAAM,IAAI,YAAY,CAClB,eAAe;gBACf,WAAW,CAAC,SAAS,EACrB;oBACI,MAAM,EAAE,MAAA,CAAC,MAAA,OAAO,CAAC,QAAQ,0CAAE,MAAM,CAAC,mCAAI,CAAC;oBACvC,KAAK,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM;oBAClC,KAAK,EAAE,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,SAAS;iBAChD,CAAC,CAAC;aACV;SACJ;QAED,iEAAiE;QACjE,yEAAyE;QACzE,2DAA2D;QAC3D,sEAAsE;QACtE,gEAAgE;QAChE,IAAI,IAAI,CAAC,eAAe,CAAC,kBAAkB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,uBAAuB,EAAE;YAC1G,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,CAAC;SACvD;IACL,CAAC;IAEM,KAAK;QACR,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,CAAC;QACpD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;IAClD,CAAC;IAEO,aAAa,CAAC,QAAgB;QAClC,MAAM,cAAc,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QACpD,MAAM,oBAAoB,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;QAE5D,IAAI,CAAC,YAAY,CAAC,oBAAoB,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC9D,CAAC;IAEO,aAAa,CAAC,KAAa;QAC/B,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;eACvB,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,kBAAkB,KAAK,SAAS;eACnD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,uBAAuB,GAAG,KAAK,CAAC,kBAAkB,EAAE;YAC7F,oGAAoG;YACpG,OAAO,KAAK,CAAC;SAChB;QAED,MAAM,eAAe,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QACpE,IAAI,eAAe,CAAC,kBAAkB,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,mBAAmB,EAAE;YAC7E,MAAM,IAAI,YAAY,CAClB,eAAe;YACX,WAAW,CAAC,SAAS,EACzB;gBACI,MAAM,EAAE,KAAK,CAAC,kBAAkB;gBAChC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM;gBAC3B,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,mBAAmB;gBAC7C,UAAU,EAAE,IAAI;aACnB,CAAC,CAAC;SACV;QAED,oFAAoF;QACpF,OAAO,eAAe,CAAC;IAC3B,CAAC;IAED;;;;;;OAMG;IACK,SAAS,CAAC,KAAa;;QAC3B,IAAI,oBAAoB,GAAW,CAAC,CAAC,CAAC;QACtC,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;QAEpC,uDAAuD;QACvD,uFAAuF;QACvF,IAAI,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE;YAC3C,OAAO,oBAAoB,CAAC;SAC/B;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,aAAa,KAAK,SAAS,EAAE;YAC1D,yFAAyF;YACzF,uDAAuD;YACvD,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,OAAO,EAAE;gBACjC,+FAA+F;gBAC/F,IAAI,MAAA,OAAO,CAAC,QAAQ,0CAAE,UAAU,EAAE;oBAC9B,OAAO,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC;iBACtC;gBAED,oBAAoB,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,QAAQ,CACxD,WAAW,CAAC,SAAS,EACrB,OAAO,CAAC,mBAAmB,EAC3B,IAAI,EAAE,QAAQ;gBACd,OAAO,CAAC,QAAQ,CAAC,CAAC;aACzB;YAED,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;SACrD;aAAM;YACH,0DAA0D;YAC1D,oBAAoB,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,aAAa,CAC7D,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;SACrG;QAED,2GAA2G;QAC3G,oBAAoB,IAAI,MAAM,GAAG,CAAC,CAAC;QACnC,MAAM,CAAC,oBAAoB,IAAI,CAAC,EAAE,KAAK,CAAC,4CAA4C,CAAC,CAAC;QACtF,OAAO,oBAAoB,CAAC;IAChC,CAAC;IAEO,YAAY,CAAC,2BAAmC,EAAE,KAAqB;QAC3E,IAAI,oBAAoB,GAAG,2BAA2B,CAAC;QACvD,iEAAiE;QACjE,4DAA4D;QAC5D,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE;YACzB,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,eAAe,CAC3C,OAAO,CAAC,mBAAmB,CAAC,IAAI,EAChC,oBAAoB,EACpB,OAAO,CAAC,uBAAuB,EAC/B,OAAO,CAAC,mBAAmB,CAAC,QAAQ,EACpC,OAAO,CAAC,eAAe,EACvB,OAAO,CAAC,QAAQ,CACnB,CAAC;YAEF,oBAAoB,EAAE,CAAC;SAC1B;QAED,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,CAAC;IAC9C,CAAC;IAEM,UAAU;QACb,OAAO;YACH,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE;YACtC,eAAe,EAAE,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE;SACrD,CAAC;IACN,CAAC;CACJ","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { assert } from \"@fluidframework/common-utils\";\nimport { IContainerContext } from \"@fluidframework/container-definitions\";\nimport { GenericError } from \"@fluidframework/container-utils\";\nimport { MessageType } from \"@fluidframework/protocol-definitions\";\nimport { ICompressionRuntimeOptions } from \"../containerRuntime\";\nimport { PendingStateManager } from \"../pendingStateManager\";\nimport { BatchManager } from \"./batchManager\";\nimport { BatchMessage, IBatch } from \"./definitions\";\nimport { OpCompressor } from \"./opCompressor\";\n\nexport interface IOutboxConfig {\n    readonly compressionOptions: ICompressionRuntimeOptions;\n    // The maximum size of a batch that we can send over the wire.\n    readonly maxBatchSizeInBytes: number;\n};\n\nexport interface IOutboxParameters {\n    readonly shouldSend: () => boolean,\n    readonly pendingStateManager: PendingStateManager,\n    readonly containerContext: IContainerContext,\n    readonly config: IOutboxConfig,\n    readonly compressor: OpCompressor;\n}\n\nexport class Outbox {\n    private readonly attachFlowBatch: BatchManager;\n    private readonly mainBatch: BatchManager;\n    private readonly defaultAttachFlowSoftLimitInBytes = 64 * 1024;\n\n    constructor(private readonly params: IOutboxParameters) {\n        const isCompressionEnabled = this.params.config.compressionOptions.minimumBatchSizeInBytes !== Number.POSITIVE_INFINITY;\n        // We need to allow infinite size batches if we enable compression\n        const hardLimit = isCompressionEnabled ? Infinity : this.params.config.maxBatchSizeInBytes;\n        const softLimit = isCompressionEnabled ? Infinity : this.defaultAttachFlowSoftLimitInBytes;\n\n        this.attachFlowBatch = new BatchManager({\n            hardLimit,\n            softLimit,\n        });\n        this.mainBatch = new BatchManager({\n            hardLimit\n        });\n    }\n\n    public get isEmpty(): boolean {\n        return this.attachFlowBatch.length === 0 && this.mainBatch.length === 0;\n    }\n\n    public submit(message: BatchMessage) {\n        if (!this.mainBatch.push(message)) {\n            throw new GenericError(\n                \"BatchTooLarge\",\n                /* error */ undefined,\n                {\n                    opSize: (message.contents?.length) ?? 0,\n                    count: this.mainBatch.length,\n                    limit: this.mainBatch.options.hardLimit,\n                });\n        }\n    }\n\n    public submitAttach(message: BatchMessage) {\n        if (!this.attachFlowBatch.push(message)) {\n            // BatchManager has two limits - soft limit & hard limit. Soft limit is only engaged\n            // when queue is not empty.\n            // Flush queue & retry. Failure on retry would mean - single message is bigger than hard limit\n            this.flushInternal(this.attachFlowBatch.popBatch());\n            if (!this.attachFlowBatch.push(message)) {\n                throw new GenericError(\n                    \"BatchTooLarge\",\n                    /* error */ undefined,\n                    {\n                        opSize: (message.contents?.length) ?? 0,\n                        count: this.attachFlowBatch.length,\n                        limit: this.attachFlowBatch.options.hardLimit,\n                    });\n            }\n        }\n\n        // If compression is enabled, we will always successfully receive\n        // attach ops and compress then send them at the next JS turn, regardless\n        // of the overall size of the accumulated ops in the batch.\n        // However, it is more efficient to flush these ops faster, preferably\n        // after they reach a size which would benefit from compression.\n        if (this.attachFlowBatch.contentSizeInBytes >= this.params.config.compressionOptions.minimumBatchSizeInBytes) {\n            this.flushInternal(this.attachFlowBatch.popBatch());\n        }\n    }\n\n    public flush() {\n        this.flushInternal(this.attachFlowBatch.popBatch());\n        this.flushInternal(this.mainBatch.popBatch());\n    }\n\n    private flushInternal(rawBatch: IBatch) {\n        const processedBatch = this.compressBatch(rawBatch);\n        const clientSequenceNumber = this.sendBatch(processedBatch);\n\n        this.persistBatch(clientSequenceNumber, rawBatch.content);\n    }\n\n    private compressBatch(batch: IBatch): IBatch {\n        if (batch.content.length === 0\n            || this.params.config.compressionOptions === undefined\n            || this.params.config.compressionOptions.minimumBatchSizeInBytes > batch.contentSizeInBytes) {\n            // Nothing to do if the batch is empty or if compression is disabled or if we don't need to compress\n            return batch;\n        }\n\n        const compressedBatch = this.params.compressor.compressBatch(batch);\n        if (compressedBatch.contentSizeInBytes > this.params.config.maxBatchSizeInBytes) {\n            throw new GenericError(\n                \"BatchTooLarge\",\n                    /* error */ undefined,\n                {\n                    opSize: batch.contentSizeInBytes,\n                    count: batch.content.length,\n                    limit: this.params.config.maxBatchSizeInBytes,\n                    compressed: true,\n                });\n        }\n\n        // If we don't reach the maximum supported size of a batch, it safe to be sent as is\n        return compressedBatch;\n    }\n\n    /**\n     * Sends the batch object to the container context to be sent over the wire.\n     *\n     * @param batch - batch to be sent\n     * @returns the client sequence number of the last batched op which was sent and\n     * -1 if there are no ops or the container cannot send ops.\n     */\n    private sendBatch(batch: IBatch): number {\n        let clientSequenceNumber: number = -1;\n        const length = batch.content.length;\n\n        // Did we disconnect in the middle of turn-based batch?\n        // If so, do nothing, as pending state manager will resubmit it correctly on reconnect.\n        if (length === 0 || !this.params.shouldSend()) {\n            return clientSequenceNumber;\n        }\n\n        if (this.params.containerContext.submitBatchFn === undefined) {\n            // Legacy path - supporting old loader versions. Can be removed only when LTS moves above\n            // version that has support for batches (submitBatchFn)\n            for (const message of batch.content) {\n                // Legacy path doesn't support compressed payloads and will submit uncompressed payload anyways\n                if (message.metadata?.compressed) {\n                    delete message.metadata.compressed;\n                }\n\n                clientSequenceNumber = this.params.containerContext.submitFn(\n                    MessageType.Operation,\n                    message.deserializedContent,\n                    true, // batch\n                    message.metadata);\n            }\n\n            this.params.containerContext.deltaManager.flush();\n        } else {\n            // returns clientSequenceNumber of last message in a batch\n            clientSequenceNumber = this.params.containerContext.submitBatchFn(\n                batch.content.map((message) => ({ contents: message.contents, metadata: message.metadata })));\n        }\n\n        // Convert from clientSequenceNumber of last message in the batch to clientSequenceNumber of first message.\n        clientSequenceNumber -= length - 1;\n        assert(clientSequenceNumber >= 0, 0x3d0 /* clientSequenceNumber can't be negative */);\n        return clientSequenceNumber;\n    }\n\n    private persistBatch(initialClientSequenceNumber: number, batch: BatchMessage[]) {\n        let clientSequenceNumber = initialClientSequenceNumber;\n        // Let the PendingStateManager know that a message was submitted.\n        // In future, need to shift toward keeping batch as a whole!\n        for (const message of batch) {\n            this.params.pendingStateManager.onSubmitMessage(\n                message.deserializedContent.type,\n                clientSequenceNumber,\n                message.referenceSequenceNumber,\n                message.deserializedContent.contents,\n                message.localOpMetadata,\n                message.metadata,\n            );\n\n            clientSequenceNumber++;\n        }\n\n        this.params.pendingStateManager.onFlush();\n    }\n\n    public checkpoint() {\n        return {\n            mainBatch: this.mainBatch.checkpoint(),\n            attachFlowBatch: this.attachFlowBatch.checkpoint(),\n        };\n    }\n}\n"]}

package/lib/opLifecycle/remoteMessageProcessor.d.ts

@@ -0,0 +1,26 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
+import { OpDecompressor } from "./opDecompressor";
+import { OpSplitter } from "./opSplitter";
+export declare class RemoteMessageProcessor {
+    private readonly opSplitter;
+    private readonly opDecompressor;
+    constructor(opSplitter: OpSplitter, opDecompressor: OpDecompressor);
+    get partialMessages(): ReadonlyMap<string, string[]>;
+    clearPartialMessagesFor(clientId: string): void;
+    process(remoteMessage: ISequencedDocumentMessage): ISequencedDocumentMessage;
+}
+/**
+ * Unpacks runtime messages.
+ *
+ * @remarks This API makes no promises regarding backward-compatibility. This is internal API.
+ * @param message - message (as it observed in storage / service)
+ * @returns unpacked runtime message
+ *
+ * @internal
+ */
+export declare function unpackRuntimeMessage(message: ISequencedDocumentMessage): boolean;
+//# sourceMappingURL=remoteMessageProcessor.d.ts.map

package/lib/opLifecycle/remoteMessageProcessor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"remoteMessageProcessor.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/remoteMessageProcessor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,yBAAyB,EAAe,MAAM,sCAAsC,CAAC;AAE9F,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE1C,qBAAa,sBAAsB;IAE3B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,cAAc;gBADd,UAAU,EAAE,UAAU,EACtB,cAAc,EAAE,cAAc;IAGnD,IAAW,eAAe,IAAI,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAE1D;IAEM,uBAAuB,CAAC,QAAQ,EAAE,MAAM;IAIxC,OAAO,CAAC,aAAa,EAAE,yBAAyB,GAAG,yBAAyB;CAStF;AA6BD;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAkBhF"}

package/lib/opLifecycle/remoteMessageProcessor.js

@@ -0,0 +1,76 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { MessageType } from "@fluidframework/protocol-definitions";
+import { ContainerMessageType } from "../containerRuntime";
+export class RemoteMessageProcessor {
+    constructor(opSplitter, opDecompressor) {
+        this.opSplitter = opSplitter;
+        this.opDecompressor = opDecompressor;
+    }
+    get partialMessages() {
+        return this.opSplitter.chunks;
+    }
+    clearPartialMessagesFor(clientId) {
+        this.opSplitter.clearPartialChunks(clientId);
+    }
+    process(remoteMessage) {
+        let message = copy(remoteMessage);
+        message = this.opDecompressor.processMessage(message);
+        unpackRuntimeMessage(message);
+        message = this.opSplitter.processRemoteMessage(message);
+        return message;
+    }
+}
+const copy = (remoteMessage) => {
+    // Do shallow copy of message, as the processing flow will modify it.
+    // There might be multiple container instances receiving same message
+    // We do not need to make deep copy, as each layer will just replace message.content itself,
+    // but would not modify contents details
+    const message = Object.assign({}, remoteMessage);
+    // back-compat: ADO #1385: eventually should become unconditional, but only for runtime messages!
+    // System message may have no contents, or in some cases (mostly for back-compat) they may have actual objects.
+    // Old ops may contain empty string (I assume noops).
+    if (typeof message.contents === "string" && message.contents !== "") {
+        message.contents = JSON.parse(message.contents);
+    }
+    return message;
+};
+/**
+ * For a given message, it moves the nested contents and type on level up.
+ *
+ */
+const unpack = (message) => {
+    const innerContents = message.contents;
+    message.type = innerContents.type;
+    message.contents = innerContents.contents;
+};
+/**
+ * Unpacks runtime messages.
+ *
+ * @remarks This API makes no promises regarding backward-compatibility. This is internal API.
+ * @param message - message (as it observed in storage / service)
+ * @returns unpacked runtime message
+ *
+ * @internal
+ */
+export function unpackRuntimeMessage(message) {
+    if (message.type !== MessageType.Operation) {
+        // Legacy format, but it's already "unpacked",
+        // i.e. message.type is actually ContainerMessageType.
+        // Or it's non-runtime message.
+        // Nothing to do in such case.
+        return false;
+    }
+    // legacy op format?
+    if (message.contents.address !== undefined && message.contents.type === undefined) {
+        message.type = ContainerMessageType.FluidDataStoreOp;
+    }
+    else {
+        // new format
+        unpack(message);
+    }
+    return true;
+}
+//# sourceMappingURL=remoteMessageProcessor.js.map

package/lib/opLifecycle/remoteMessageProcessor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"remoteMessageProcessor.js","sourceRoot":"","sources":["../../src/opLifecycle/remoteMessageProcessor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAA6B,WAAW,EAAE,MAAM,sCAAsC,CAAC;AAC9F,OAAO,EAAE,oBAAoB,EAA2B,MAAM,qBAAqB,CAAC;AAIpF,MAAM,OAAO,sBAAsB;IAC/B,YACqB,UAAsB,EACtB,cAA8B;QAD9B,eAAU,GAAV,UAAU,CAAY;QACtB,mBAAc,GAAd,cAAc,CAAgB;IAC/C,CAAC;IAEL,IAAW,eAAe;QACtB,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;IAClC,CAAC;IAEM,uBAAuB,CAAC,QAAgB;QAC3C,IAAI,CAAC,UAAU,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IACjD,CAAC;IAEM,OAAO,CAAC,aAAwC;QACnD,IAAI,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC;QAElC,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QACtD,oBAAoB,CAAC,OAAO,CAAC,CAAC;QAC9B,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC;QAExD,OAAO,OAAO,CAAC;IACnB,CAAC;CACJ;AAED,MAAM,IAAI,GAAG,CAAC,aAAwC,EAA6B,EAAE;IACjF,qEAAqE;IACrE,qEAAqE;IACrE,4FAA4F;IAC5F,wCAAwC;IACxC,MAAM,OAAO,qBAAQ,aAAa,CAAE,CAAC;IAErC,iGAAiG;IACjG,+GAA+G;IAC/G,qDAAqD;IACrD,IAAI,OAAO,OAAO,CAAC,QAAQ,KAAK,QAAQ,IAAI,OAAO,CAAC,QAAQ,KAAK,EAAE,EAAE;QACjE,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;KACnD;IAED,OAAO,OAAO,CAAC;AACnB,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,GAAG,CAAC,OAAkC,EAAE,EAAE;IAClD,MAAM,aAAa,GAAG,OAAO,CAAC,QAAmC,CAAC;IAClE,OAAO,CAAC,IAAI,GAAG,aAAa,CAAC,IAAI,CAAC;IAClC,OAAO,CAAC,QAAQ,GAAG,aAAa,CAAC,QAAQ,CAAC;AAC9C,CAAC,CAAC;AAEF;;;;;;;;GAQG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAkC;IACnE,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,CAAC,SAAS,EAAE;QACxC,8CAA8C;QAC9C,sDAAsD;QACtD,+BAA+B;QAC/B,8BAA8B;QAC9B,OAAO,KAAK,CAAC;KAChB;IAED,oBAAoB;IACpB,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,KAAK,SAAS,EAAE;QAC/E,OAAO,CAAC,IAAI,GAAG,oBAAoB,CAAC,gBAAgB,CAAC;KACxD;SAAM;QACH,aAAa;QACb,MAAM,CAAC,OAAO,CAAC,CAAC;KACnB;IAED,OAAO,IAAI,CAAC;AAChB,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ISequencedDocumentMessage, MessageType } from \"@fluidframework/protocol-definitions\";\nimport { ContainerMessageType, ContainerRuntimeMessage } from \"../containerRuntime\";\nimport { OpDecompressor } from \"./opDecompressor\";\nimport { OpSplitter } from \"./opSplitter\";\n\nexport class RemoteMessageProcessor {\n    constructor(\n        private readonly opSplitter: OpSplitter,\n        private readonly opDecompressor: OpDecompressor,\n    ) { }\n\n    public get partialMessages(): ReadonlyMap<string, string[]> {\n        return this.opSplitter.chunks;\n    }\n\n    public clearPartialMessagesFor(clientId: string) {\n        this.opSplitter.clearPartialChunks(clientId);\n    }\n\n    public process(remoteMessage: ISequencedDocumentMessage): ISequencedDocumentMessage {\n        let message = copy(remoteMessage);\n\n        message = this.opDecompressor.processMessage(message);\n        unpackRuntimeMessage(message);\n        message = this.opSplitter.processRemoteMessage(message);\n\n        return message;\n    }\n}\n\nconst copy = (remoteMessage: ISequencedDocumentMessage): ISequencedDocumentMessage => {\n    // Do shallow copy of message, as the processing flow will modify it.\n    // There might be multiple container instances receiving same message\n    // We do not need to make deep copy, as each layer will just replace message.content itself,\n    // but would not modify contents details\n    const message = { ...remoteMessage };\n\n    // back-compat: ADO #1385: eventually should become unconditional, but only for runtime messages!\n    // System message may have no contents, or in some cases (mostly for back-compat) they may have actual objects.\n    // Old ops may contain empty string (I assume noops).\n    if (typeof message.contents === \"string\" && message.contents !== \"\") {\n        message.contents = JSON.parse(message.contents);\n    }\n\n    return message;\n};\n\n/**\n * For a given message, it moves the nested contents and type on level up.\n *\n */\nconst unpack = (message: ISequencedDocumentMessage) => {\n    const innerContents = message.contents as ContainerRuntimeMessage;\n    message.type = innerContents.type;\n    message.contents = innerContents.contents;\n};\n\n/**\n * Unpacks runtime messages.\n *\n * @remarks This API makes no promises regarding backward-compatibility. This is internal API.\n * @param message - message (as it observed in storage / service)\n * @returns unpacked runtime message\n *\n * @internal\n */\nexport function unpackRuntimeMessage(message: ISequencedDocumentMessage): boolean {\n    if (message.type !== MessageType.Operation) {\n        // Legacy format, but it's already \"unpacked\",\n        // i.e. message.type is actually ContainerMessageType.\n        // Or it's non-runtime message.\n        // Nothing to do in such case.\n        return false;\n    }\n\n    // legacy op format?\n    if (message.contents.address !== undefined && message.contents.type === undefined) {\n        message.type = ContainerMessageType.FluidDataStoreOp;\n    } else {\n        // new format\n        unpack(message);\n    }\n\n    return true;\n}\n"]}

package/lib/packageVersion.d.ts

@@ -5,5 +5,5 @@
  * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY
  */
 export declare const pkgName = "@fluidframework/container-runtime";
-export declare const pkgVersion = "2.0.0-dev.1.4.6.106135";
+export declare const pkgVersion = "2.0.0-dev.2.3.0.115467";
 //# sourceMappingURL=packageVersion.d.ts.map

package/lib/packageVersion.js

@@ -5,5 +5,5 @@
  * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY
  */
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.0.0-dev.1.4.6.106135";
+export const pkgVersion = "2.0.0-dev.2.3.0.115467";
 //# sourceMappingURL=packageVersion.js.map

package/lib/packageVersion.js.map

@@ -1 +1 @@
-{"version":3,"file":"packageVersion.js","sourceRoot":"","sources":["../src/packageVersion.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,CAAC,MAAM,OAAO,GAAG,mCAAmC,CAAC;AAC3D,MAAM,CAAC,MAAM,UAAU,GAAG,wBAAwB,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n *\n * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY\n */\n\nexport const pkgName = \"@fluidframework/container-runtime\";\nexport const pkgVersion = \"2.0.0-dev.1.4.6.106135\";\n"]}
+{"version":3,"file":"packageVersion.js","sourceRoot":"","sources":["../src/packageVersion.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,CAAC,MAAM,OAAO,GAAG,mCAAmC,CAAC;AAC3D,MAAM,CAAC,MAAM,UAAU,GAAG,wBAAwB,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n *\n * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY\n */\n\nexport const pkgName = \"@fluidframework/container-runtime\";\nexport const pkgVersion = \"2.0.0-dev.2.3.0.115467\";\n"]}

package/lib/pendingStateManager.d.ts

@@ -5,7 +5,6 @@
 import { IDisposable } from "@fluidframework/common-definitions";
 import { ICriticalContainerError } from "@fluidframework/container-definitions";
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
-import { FlushMode } from "@fluidframework/runtime-definitions";
 import { ContainerMessageType } from "./containerRuntime";
 /**
  * This represents a message that has been submitted and is added to the pending queue when `submit` is called on the
@@ -20,14 +19,6 @@ export interface IPendingMessage {
     localOpMetadata: unknown;
     opMetadata: Record<string, unknown> | undefined;
 }
-/**
- * This represents a FlushMode update and is added to the pending queue when `setFlushMode` is called on the
- * ContainerRuntime and the FlushMode changes.
- */
-export interface IPendingFlushMode {
-    type: "flushMode";
-    flushMode: FlushMode;
-}
 /**
  * This represents an explicit flush call and is added to the pending queue when flush is called on the ContainerRuntime
  * to flush pending messages.
@@ -35,7 +26,7 @@ export interface IPendingFlushMode {
 export interface IPendingFlush {
     type: "flush";
 }
-export declare type IPendingState = IPendingMessage | IPendingFlushMode | IPendingFlush;
+export declare type IPendingState = IPendingMessage | IPendingFlush;
 export interface IPendingLocalState {
     /**
      * list of pending states, including ops and batch information
@@ -45,18 +36,18 @@ export interface IPendingLocalState {
 export interface IRuntimeStateHandler {
     connected(): boolean;
     clientId(): string | undefined;
-    flushMode(): FlushMode;
-    setFlushMode(mode: FlushMode): void;
     close(error?: ICriticalContainerError): void;
     applyStashedOp: (type: ContainerMessageType, content: ISequencedDocumentMessage) => Promise<unknown>;
     flush(): void;
     reSubmit(type: ContainerMessageType, content: any, localOpMetadata: unknown, opMetadata: Record<string, unknown> | undefined): void;
+    rollback(type: ContainerMessageType, content: any, localOpMetadata: unknown): void;
+    orderSequentially(callback: () => void): void;
 }
 /**
  * PendingStateManager is responsible for maintaining the messages that have not been sent or have not yet been
  * acknowledged by the server. It also maintains the batch information for both automatically and manually flushed
  * batches along with the messages.
- * When the Container reconnects, it replays the pending states, which includes setting the FlushMode, manual flushing
+ * When the Container reconnects, it replays the pending states, which includes manual flushing
  * of messages and triggering resubmission of unacked ops.
  *
  * It verifies that all the ops are acked, are received in the right order and batch information is correct.
@@ -70,12 +61,6 @@ export declare class PendingStateManager implements IDisposable {
     get pendingMessagesCount(): number;
     private isProcessingBatch;
     private pendingBatchBeginMessage;
-    /**
-     * This tracks the flush mode for the next message in the pending state queue. When replaying messages, we need to
-     * first set the flush mode to this value and then send ops. It is important to do this info because the flush
-     * mode could have been updated.
-     */
-    private flushModeForNextMessage;
     private clientId;
     /**
      * Called to check if there are any pending messages in the pending state queue.
@@ -83,7 +68,7 @@ export declare class PendingStateManager implements IDisposable {
      */
     hasPendingMessages(): boolean;
     getLocalState(): IPendingLocalState | undefined;
-    constructor(stateHandler: IRuntimeStateHandler, initialFlushMode: FlushMode, initialLocalState: IPendingLocalState | undefined);
+    constructor(stateHandler: IRuntimeStateHandler, initialLocalState: IPendingLocalState | undefined);
     get disposed(): boolean;
     readonly dispose: () => void;
     /**
@@ -95,11 +80,6 @@ export declare class PendingStateManager implements IDisposable {
      * @param localOpMetadata - The local metadata associated with the message.
      */
     onSubmitMessage(type: ContainerMessageType, clientSequenceNumber: number, referenceSequenceNumber: number, content: any, localOpMetadata: unknown, opMetadata: Record<string, unknown> | undefined): void;
-    /**
-     * Called when the FlushMode is updated. Adds the FlushMode to the pending state queue.
-     * @param flushMode - The flushMode that was updated.
-     */
-    onFlushModeUpdated(flushMode: FlushMode): void;
     /**
      * Called when flush() is called on the ContainerRuntime to manually flush messages.
      */
@@ -131,7 +111,7 @@ export declare class PendingStateManager implements IDisposable {
     private peekNextPendingState;
     /**
      * Called when the Container's connection state changes. If the Container gets connected, it replays all the pending
-     * states in its queue. This includes setting the FlushMode and triggering resubmission of unacked ops.
+     * states in its queue. This includes triggering resubmission of unacked ops.
      */
     replayPendingStates(): void;
 }

package/lib/pendingStateManager.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pendingStateManager.d.ts","sourceRoot":"","sources":["../src/pendingStateManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,oCAAoC,CAAC;AAEjE,OAAO,EAAE,uBAAuB,EAAE,MAAM,uCAAuC,CAAC;AAEhF,OAAO,EACH,yBAAyB,EAC5B,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,qCAAqC,CAAC;AAEhE,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAG1D;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC5B,IAAI,EAAE,SAAS,CAAC;IAChB,WAAW,EAAE,oBAAoB,CAAC;IAClC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,MAAM,CAAC;IAChC,OAAO,EAAE,GAAG,CAAC;IACb,eAAe,EAAE,OAAO,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;CACnD;AAED;;;GAGG;AACH,MAAM,WAAW,iBAAiB;IAC9B,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,SAAS,CAAC;CACxB;AAED;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC1B,IAAI,EAAE,OAAO,CAAC;CACjB;AAED,oBAAY,aAAa,GAAG,eAAe,GAAG,iBAAiB,GAAG,aAAa,CAAC;AAEhF,MAAM,WAAW,kBAAkB;IAC/B;;OAEG;IACH,aAAa,EAAE,aAAa,EAAE,CAAC;CAClC;AAED,MAAM,WAAW,oBAAoB;IACjC,SAAS,IAAI,OAAO,CAAC;IACrB,QAAQ,IAAI,MAAM,GAAG,SAAS,CAAC;IAC/B,SAAS,IAAI,SAAS,CAAC;IACvB,YAAY,CAAC,IAAI,EAAE,SAAS,GAAG,IAAI,CAAC;IACpC,KAAK,CAAC,KAAK,CAAC,EAAE,uBAAuB,GAAG,IAAI,CAAC;IAC7C,cAAc,EAAE,CAAC,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,yBAAyB,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACrG,KAAK,IAAI,IAAI,CAAC;IACd,QAAQ,CACJ,IAAI,EAAE,oBAAoB,EAC1B,OAAO,EAAE,GAAG,EACZ,eAAe,EAAE,OAAO,EACxB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,GAAG,IAAI,CAAC;CAC9D;AAED;;;;;;;;GAQG;AACH,qBAAa,mBAAoB,YAAW,WAAW;IAmD/C,OAAO,CAAC,QAAQ,CAAC,YAAY;IAlDjC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA8B;IAC5D,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAuB;IACrD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAGzB;IAGH,OAAO,CAAC,qBAAqB,CAAa;IAC1C,IAAW,oBAAoB,IAAI,MAAM,CAExC;IAGD,OAAO,CAAC,iBAAiB,CAAkB;IAI3C,OAAO,CAAC,wBAAwB,CAAwC;IAExE;;;;OAIG;IACH,OAAO,CAAC,uBAAuB,CAAY;IAE3C,OAAO,CAAC,QAAQ,CAAqB;IAErC;;;OAGG;IACI,kBAAkB,IAAI,OAAO;IAI7B,aAAa,IAAI,kBAAkB,GAAG,SAAS;gBAajC,YAAY,EAAE,oBAAoB,EACnD,gBAAgB,EAAE,SAAS,EAC3B,iBAAiB,EAAE,kBAAkB,GAAG,SAAS;IAQrD,IAAW,QAAQ,YAAyC;IAC5D,SAAgB,OAAO,aAAgC;IAEvD;;;;;;;OAOG;IACI,eAAe,CAClB,IAAI,EAAE,oBAAoB,EAC1B,oBAAoB,EAAE,MAAM,EAC5B,uBAAuB,EAAE,MAAM,EAC/B,OAAO,EAAE,GAAG,EACZ,eAAe,EAAE,OAAO,EACxB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS;IAiBnD;;;OAGG;IACI,kBAAkB,CAAC,SAAS,EAAE,SAAS;IAI9C;;OAEG;IACI,OAAO;IAiBd;;;OAGG;IACU,iBAAiB,CAAC,MAAM,CAAC,EAAE,MAAM;IA8B9C;;;;OAIG;IACI,0BAA0B,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO;IAiC9E;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAuD9B;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IA0D5B;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAM5B;;;OAGG;IACI,mBAAmB;CAsD7B"}
+{"version":3,"file":"pendingStateManager.d.ts","sourceRoot":"","sources":["../src/pendingStateManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,oCAAoC,CAAC;AAEjE,OAAO,EAAE,uBAAuB,EAAE,MAAM,uCAAuC,CAAC;AAEhF,OAAO,EACH,yBAAyB,EAC5B,MAAM,sCAAsC,CAAC;AAE9C,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAG1D;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC5B,IAAI,EAAE,SAAS,CAAC;IAChB,WAAW,EAAE,oBAAoB,CAAC;IAClC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,MAAM,CAAC;IAChC,OAAO,EAAE,GAAG,CAAC;IACb,eAAe,EAAE,OAAO,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;CACnD;AAED;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC1B,IAAI,EAAE,OAAO,CAAC;CACjB;AAED,oBAAY,aAAa,GAAG,eAAe,GAAG,aAAa,CAAC;AAE5D,MAAM,WAAW,kBAAkB;IAC/B;;OAEG;IACH,aAAa,EAAE,aAAa,EAAE,CAAC;CAClC;AAED,MAAM,WAAW,oBAAoB;IACjC,SAAS,IAAI,OAAO,CAAC;IACrB,QAAQ,IAAI,MAAM,GAAG,SAAS,CAAC;IAC/B,KAAK,CAAC,KAAK,CAAC,EAAE,uBAAuB,GAAG,IAAI,CAAC;IAC7C,cAAc,EAAE,CAAC,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,yBAAyB,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACrG,KAAK,IAAI,IAAI,CAAC;IACd,QAAQ,CACJ,IAAI,EAAE,oBAAoB,EAC1B,OAAO,EAAE,GAAG,EACZ,eAAe,EAAE,OAAO,EACxB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,GAAG,IAAI,CAAC;IAC3D,QAAQ,CACJ,IAAI,EAAE,oBAAoB,EAC1B,OAAO,EAAE,GAAG,EACZ,eAAe,EAAE,OAAO,GAAG,IAAI,CAAC;IACpC,iBAAiB,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,IAAI,CAAC;CACjD;AAED;;;;;;;;GAQG;AACH,qBAAa,mBAAoB,YAAW,WAAW;IA4C/C,OAAO,CAAC,QAAQ,CAAC,YAAY;IA3CjC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA8B;IAC5D,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAuB;IACrD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAGzB;IAGH,OAAO,CAAC,qBAAqB,CAAa;IAC1C,IAAW,oBAAoB,IAAI,MAAM,CAExC;IAGD,OAAO,CAAC,iBAAiB,CAAkB;IAI3C,OAAO,CAAC,wBAAwB,CAAwC;IAExE,OAAO,CAAC,QAAQ,CAAqB;IAErC;;;OAGG;IACI,kBAAkB,IAAI,OAAO;IAI7B,aAAa,IAAI,kBAAkB,GAAG,SAAS;gBAajC,YAAY,EAAE,oBAAoB,EACnD,iBAAiB,EAAE,kBAAkB,GAAG,SAAS;IAKrD,IAAW,QAAQ,YAAyC;IAC5D,SAAgB,OAAO,aAAgC;IAEvD;;;;;;;OAOG;IACI,eAAe,CAClB,IAAI,EAAE,oBAAoB,EAC1B,oBAAoB,EAAE,MAAM,EAC5B,uBAAuB,EAAE,MAAM,EAC/B,OAAO,EAAE,GAAG,EACZ,eAAe,EAAE,OAAO,EACxB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS;IAiBnD;;OAEG;IACI,OAAO;IAWd;;;OAGG;IACU,iBAAiB,CAAC,MAAM,CAAC,EAAE,MAAM;IA8B9C;;;;OAIG;IACI,0BAA0B,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO;IAiC9E;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IA4B9B;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAgD5B;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAM5B;;;OAGG;IACI,mBAAmB;CAoF7B"}