@fluidframework/container-runtime 2.0.0-dev.2.2.0.111723 → 2.0.0-dev.2.3.0.115467

package/src/opLifecycle/outbox.ts

@@ -0,0 +1,204 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { assert } from "@fluidframework/common-utils";
+import { IContainerContext } from "@fluidframework/container-definitions";
+import { GenericError } from "@fluidframework/container-utils";
+import { MessageType } from "@fluidframework/protocol-definitions";
+import { ICompressionRuntimeOptions } from "../containerRuntime";
+import { PendingStateManager } from "../pendingStateManager";
+import { BatchManager } from "./batchManager";
+import { BatchMessage, IBatch } from "./definitions";
+import { OpCompressor } from "./opCompressor";
+
+export interface IOutboxConfig {
+    readonly compressionOptions: ICompressionRuntimeOptions;
+    // The maximum size of a batch that we can send over the wire.
+    readonly maxBatchSizeInBytes: number;
+};
+
+export interface IOutboxParameters {
+    readonly shouldSend: () => boolean,
+    readonly pendingStateManager: PendingStateManager,
+    readonly containerContext: IContainerContext,
+    readonly config: IOutboxConfig,
+    readonly compressor: OpCompressor;
+}
+
+export class Outbox {
+    private readonly attachFlowBatch: BatchManager;
+    private readonly mainBatch: BatchManager;
+    private readonly defaultAttachFlowSoftLimitInBytes = 64 * 1024;
+
+    constructor(private readonly params: IOutboxParameters) {
+        const isCompressionEnabled = this.params.config.compressionOptions.minimumBatchSizeInBytes !== Number.POSITIVE_INFINITY;
+        // We need to allow infinite size batches if we enable compression
+        const hardLimit = isCompressionEnabled ? Infinity : this.params.config.maxBatchSizeInBytes;
+        const softLimit = isCompressionEnabled ? Infinity : this.defaultAttachFlowSoftLimitInBytes;
+
+        this.attachFlowBatch = new BatchManager({
+            hardLimit,
+            softLimit,
+        });
+        this.mainBatch = new BatchManager({
+            hardLimit
+        });
+    }
+
+    public get isEmpty(): boolean {
+        return this.attachFlowBatch.length === 0 && this.mainBatch.length === 0;
+    }
+
+    public submit(message: BatchMessage) {
+        if (!this.mainBatch.push(message)) {
+            throw new GenericError(
+                "BatchTooLarge",
+                /* error */ undefined,
+                {
+                    opSize: (message.contents?.length) ?? 0,
+                    count: this.mainBatch.length,
+                    limit: this.mainBatch.options.hardLimit,
+                });
+        }
+    }
+
+    public submitAttach(message: BatchMessage) {
+        if (!this.attachFlowBatch.push(message)) {
+            // BatchManager has two limits - soft limit & hard limit. Soft limit is only engaged
+            // when queue is not empty.
+            // Flush queue & retry. Failure on retry would mean - single message is bigger than hard limit
+            this.flushInternal(this.attachFlowBatch.popBatch());
+            if (!this.attachFlowBatch.push(message)) {
+                throw new GenericError(
+                    "BatchTooLarge",
+                    /* error */ undefined,
+                    {
+                        opSize: (message.contents?.length) ?? 0,
+                        count: this.attachFlowBatch.length,
+                        limit: this.attachFlowBatch.options.hardLimit,
+                    });
+            }
+        }
+
+        // If compression is enabled, we will always successfully receive
+        // attach ops and compress then send them at the next JS turn, regardless
+        // of the overall size of the accumulated ops in the batch.
+        // However, it is more efficient to flush these ops faster, preferably
+        // after they reach a size which would benefit from compression.
+        if (this.attachFlowBatch.contentSizeInBytes >= this.params.config.compressionOptions.minimumBatchSizeInBytes) {
+            this.flushInternal(this.attachFlowBatch.popBatch());
+        }
+    }
+
+    public flush() {
+        this.flushInternal(this.attachFlowBatch.popBatch());
+        this.flushInternal(this.mainBatch.popBatch());
+    }
+
+    private flushInternal(rawBatch: IBatch) {
+        const processedBatch = this.compressBatch(rawBatch);
+        const clientSequenceNumber = this.sendBatch(processedBatch);
+
+        this.persistBatch(clientSequenceNumber, rawBatch.content);
+    }
+
+    private compressBatch(batch: IBatch): IBatch {
+        if (batch.content.length === 0
+            || this.params.config.compressionOptions === undefined
+            || this.params.config.compressionOptions.minimumBatchSizeInBytes > batch.contentSizeInBytes) {
+            // Nothing to do if the batch is empty or if compression is disabled or if we don't need to compress
+            return batch;
+        }
+
+        const compressedBatch = this.params.compressor.compressBatch(batch);
+        if (compressedBatch.contentSizeInBytes > this.params.config.maxBatchSizeInBytes) {
+            throw new GenericError(
+                "BatchTooLarge",
+                    /* error */ undefined,
+                {
+                    opSize: batch.contentSizeInBytes,
+                    count: batch.content.length,
+                    limit: this.params.config.maxBatchSizeInBytes,
+                    compressed: true,
+                });
+        }
+
+        // If we don't reach the maximum supported size of a batch, it safe to be sent as is
+        return compressedBatch;
+    }
+
+    /**
+     * Sends the batch object to the container context to be sent over the wire.
+     *
+     * @param batch - batch to be sent
+     * @returns the client sequence number of the last batched op which was sent and
+     * -1 if there are no ops or the container cannot send ops.
+     */
+    private sendBatch(batch: IBatch): number {
+        let clientSequenceNumber: number = -1;
+        const length = batch.content.length;
+
+        // Did we disconnect in the middle of turn-based batch?
+        // If so, do nothing, as pending state manager will resubmit it correctly on reconnect.
+        if (length === 0 || !this.params.shouldSend()) {
+            return clientSequenceNumber;
+        }
+
+        if (this.params.containerContext.submitBatchFn === undefined) {
+            // Legacy path - supporting old loader versions. Can be removed only when LTS moves above
+            // version that has support for batches (submitBatchFn)
+            for (const message of batch.content) {
+                // Legacy path doesn't support compressed payloads and will submit uncompressed payload anyways
+                if (message.metadata?.compressed) {
+                    delete message.metadata.compressed;
+                }
+
+                clientSequenceNumber = this.params.containerContext.submitFn(
+                    MessageType.Operation,
+                    message.deserializedContent,
+                    true, // batch
+                    message.metadata);
+            }
+
+            this.params.containerContext.deltaManager.flush();
+        } else {
+            // returns clientSequenceNumber of last message in a batch
+            clientSequenceNumber = this.params.containerContext.submitBatchFn(
+                batch.content.map((message) => ({ contents: message.contents, metadata: message.metadata })));
+        }
+
+        // Convert from clientSequenceNumber of last message in the batch to clientSequenceNumber of first message.
+        clientSequenceNumber -= length - 1;
+        assert(clientSequenceNumber >= 0, 0x3d0 /* clientSequenceNumber can't be negative */);
+        return clientSequenceNumber;
+    }
+
+    private persistBatch(initialClientSequenceNumber: number, batch: BatchMessage[]) {
+        let clientSequenceNumber = initialClientSequenceNumber;
+        // Let the PendingStateManager know that a message was submitted.
+        // In future, need to shift toward keeping batch as a whole!
+        for (const message of batch) {
+            this.params.pendingStateManager.onSubmitMessage(
+                message.deserializedContent.type,
+                clientSequenceNumber,
+                message.referenceSequenceNumber,
+                message.deserializedContent.contents,
+                message.localOpMetadata,
+                message.metadata,
+            );
+
+            clientSequenceNumber++;
+        }
+
+        this.params.pendingStateManager.onFlush();
+    }
+
+    public checkpoint() {
+        return {
+            mainBatch: this.mainBatch.checkpoint(),
+            attachFlowBatch: this.attachFlowBatch.checkpoint(),
+        };
+    }
+}

package/src/opLifecycle/remoteMessageProcessor.ts

@@ -0,0 +1,90 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { ISequencedDocumentMessage, MessageType } from "@fluidframework/protocol-definitions";
+import { ContainerMessageType, ContainerRuntimeMessage } from "../containerRuntime";
+import { OpDecompressor } from "./opDecompressor";
+import { OpSplitter } from "./opSplitter";
+
+export class RemoteMessageProcessor {
+    constructor(
+        private readonly opSplitter: OpSplitter,
+        private readonly opDecompressor: OpDecompressor,
+    ) { }
+
+    public get partialMessages(): ReadonlyMap<string, string[]> {
+        return this.opSplitter.chunks;
+    }
+
+    public clearPartialMessagesFor(clientId: string) {
+        this.opSplitter.clearPartialChunks(clientId);
+    }
+
+    public process(remoteMessage: ISequencedDocumentMessage): ISequencedDocumentMessage {
+        let message = copy(remoteMessage);
+
+        message = this.opDecompressor.processMessage(message);
+        unpackRuntimeMessage(message);
+        message = this.opSplitter.processRemoteMessage(message);
+
+        return message;
+    }
+}
+
+const copy = (remoteMessage: ISequencedDocumentMessage): ISequencedDocumentMessage => {
+    // Do shallow copy of message, as the processing flow will modify it.
+    // There might be multiple container instances receiving same message
+    // We do not need to make deep copy, as each layer will just replace message.content itself,
+    // but would not modify contents details
+    const message = { ...remoteMessage };
+
+    // back-compat: ADO #1385: eventually should become unconditional, but only for runtime messages!
+    // System message may have no contents, or in some cases (mostly for back-compat) they may have actual objects.
+    // Old ops may contain empty string (I assume noops).
+    if (typeof message.contents === "string" && message.contents !== "") {
+        message.contents = JSON.parse(message.contents);
+    }
+
+    return message;
+};
+
+/**
+ * For a given message, it moves the nested contents and type on level up.
+ *
+ */
+const unpack = (message: ISequencedDocumentMessage) => {
+    const innerContents = message.contents as ContainerRuntimeMessage;
+    message.type = innerContents.type;
+    message.contents = innerContents.contents;
+};
+
+/**
+ * Unpacks runtime messages.
+ *
+ * @remarks This API makes no promises regarding backward-compatibility. This is internal API.
+ * @param message - message (as it observed in storage / service)
+ * @returns unpacked runtime message
+ *
+ * @internal
+ */
+export function unpackRuntimeMessage(message: ISequencedDocumentMessage): boolean {
+    if (message.type !== MessageType.Operation) {
+        // Legacy format, but it's already "unpacked",
+        // i.e. message.type is actually ContainerMessageType.
+        // Or it's non-runtime message.
+        // Nothing to do in such case.
+        return false;
+    }
+
+    // legacy op format?
+    if (message.contents.address !== undefined && message.contents.type === undefined) {
+        message.type = ContainerMessageType.FluidDataStoreOp;
+    } else {
+        // new format
+        unpack(message);
+    }
+
+    return true;
+}

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.0.0-dev.2.2.0.111723";
+export const pkgVersion = "2.0.0-dev.2.3.0.115467";

package/src/summaryFormat.ts

@@ -8,7 +8,7 @@ import { IDocumentStorageService } from "@fluidframework/driver-definitions";
 import { readAndParse } from "@fluidframework/driver-utils";
 import { ISequencedDocumentMessage, ISnapshotTree, SummaryType } from "@fluidframework/protocol-definitions";
 import { channelsTreeName, ISummaryTreeWithStats } from "@fluidframework/runtime-definitions";
-import { gcTreeKey } from "./garbageCollection";
+import { gcTreeKey } from "./garbageCollectionConstants";
 
 type OmitAttributesVersions<T> = Omit<T, "snapshotFormatVersion" | "summaryFormatVersion">;
 interface IFluidDataStoreAttributes0 {

package/dist/batchManager.d.ts

@@ -1,42 +0,0 @@
-/*!
- * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
- * Licensed under the MIT License.
- */
-import { ITelemetryLogger } from "@fluidframework/common-definitions";
-import { IBatchMessage } from "@fluidframework/container-definitions";
-import { ContainerRuntimeMessage, ICompressionRuntimeOptions } from "./containerRuntime";
-/**
- * Message type used by BatchManager
- */
-export declare type BatchMessage = IBatchMessage & {
-    localOpMetadata: unknown;
-    deserializedContent: ContainerRuntimeMessage;
-    referenceSequenceNumber: number;
-};
-export interface IBatchManagerOptions {
-    readonly hardLimit: number;
-    readonly softLimit?: number;
-    readonly compressionOptions?: ICompressionRuntimeOptions;
-}
-/**
- * Helper class that manages partial batch & rollback.
- */
-export declare class BatchManager {
-    readonly logger: ITelemetryLogger;
-    readonly options: IBatchManagerOptions;
-    private readonly opCompressor;
-    private pendingBatch;
-    private batchContentSize;
-    get length(): number;
-    constructor(logger: ITelemetryLogger, options: IBatchManagerOptions);
-    push(message: BatchMessage): boolean;
-    get empty(): boolean;
-    popBatch(): BatchMessage[];
-    /**
-     * Capture the pending state at this point
-     */
-    checkpoint(): {
-        rollback: (process: (message: BatchMessage) => void) => void;
-    };
-}
-//# sourceMappingURL=batchManager.d.ts.map

package/dist/batchManager.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"batchManager.d.ts","sourceRoot":"","sources":["../src/batchManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AACtE,OAAO,EAAE,aAAa,EAAE,MAAM,uCAAuC,CAAC;AACtE,OAAO,EAAE,uBAAuB,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAGzF;;GAEG;AACH,oBAAY,YAAY,GAAG,aAAa,GAAG;IACvC,eAAe,EAAE,OAAO,CAAC;IACzB,mBAAmB,EAAE,uBAAuB,CAAC;IAC7C,uBAAuB,EAAE,MAAM,CAAC;CACnC,CAAC;AAEF,MAAM,WAAW,oBAAoB;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,kBAAkB,CAAC,EAAE,0BAA0B,CAAC;CAC5D;AAED;;GAEG;AACH,qBAAa,YAAY;aAOO,MAAM,EAAE,gBAAgB;aAAkB,OAAO,EAAE,oBAAoB;IANnG,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAe;IAC5C,OAAO,CAAC,YAAY,CAAuB;IAC3C,OAAO,CAAC,gBAAgB,CAAK;IAE7B,IAAW,MAAM,WAAuC;gBAE5B,MAAM,EAAE,gBAAgB,EAAkB,OAAO,EAAE,oBAAoB;IAI5F,IAAI,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO;IAiC3C,IAAW,KAAK,YAA6C;IAEtD,QAAQ;IAef;;OAEG;IACI,UAAU;sCAGqB,YAAY,KAAK,IAAI;;CAY9D"}

package/dist/batchManager.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"batchManager.js","sourceRoot":"","sources":["../src/batchManager.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAKH,iDAA8C;AAiB9C;;GAEG;AACH,MAAa,YAAY;IAOrB,YAA4B,MAAwB,EAAkB,OAA6B;QAAvE,WAAM,GAAN,MAAM,CAAkB;QAAkB,YAAO,GAAP,OAAO,CAAsB;QAL3F,iBAAY,GAAoB,EAAE,CAAC;QACnC,qBAAgB,GAAG,CAAC,CAAC;QAKzB,IAAI,CAAC,YAAY,GAAG,IAAI,2BAAY,CAAC,MAAM,CAAC,CAAC;IACjD,CAAC;IAJD,IAAW,MAAM,KAAK,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;IAMjD,IAAI,CAAC,OAAqB;;QAC7B,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,GAAG,CAAC,MAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,MAAM,mCAAI,CAAC,CAAC,CAAC;QAC5E,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;QAEzC,2DAA2D;QAC3D,iEAAiE;QACjE,sGAAsG;QACtG,iGAAiG;QACjG,gFAAgF;QAChF,MAAM,iBAAiB,GAAG,WAAW,GAAG,GAAG,GAAG,OAAO,CAAC;QAEtD,0DAA0D;QAC1D,wGAAwG;QACxG,0DAA0D;QAC1D,yGAAyG;QACzG,mGAAmG;QACnG,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS;eACjC,IAAI,CAAC,MAAM,GAAG,CAAC;eACf,iBAAiB,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS;eAC3C,QAAQ,KAAK,CAAC,MAAA,MAAA,IAAI,CAAC,OAAO,CAAC,kBAAkB,0CAAE,uBAAuB,mCAAI,QAAQ,CAAC,EAAE;YACxF,OAAO,KAAK,CAAC;SAChB;QAED,IAAI,iBAAiB,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS;eACxC,QAAQ,KAAK,CAAC,MAAA,MAAA,IAAI,CAAC,OAAO,CAAC,kBAAkB,0CAAE,uBAAuB,mCAAI,QAAQ,CAAC,EAAE;YACxF,OAAO,KAAK,CAAC;SAChB;QAED,IAAI,CAAC,gBAAgB,GAAG,WAAW,CAAC;QACpC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAChC,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,IAAW,KAAK,KAAK,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC;IAEtD,QAAQ;QACX,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC;QAE1B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;eACb,IAAI,CAAC,OAAO,CAAC,kBAAkB,KAAK,SAAS;eAC7C,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,uBAAuB,GAAG,IAAI,EAAE;YACnE,OAAO,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;SACvD;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;OAEG;IACI,UAAU;QACb,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;QAC5C,OAAO;YACH,QAAQ,EAAE,CAAC,OAAwC,EAAE,EAAE;;gBACnD,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,GAAG,UAAU,GAAG;oBACpD,CAAC,EAAE,CAAC;oBACJ,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;oBACrC,IAAI,CAAC,gBAAgB,IAAI,MAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,MAAM,mCAAI,CAAC,CAAC;oBACvD,OAAO,CAAC,OAAO,CAAC,CAAC;iBACpB;gBAED,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,UAAU,CAAC;YAC1C,CAAC;SACJ,CAAC;IACN,CAAC;CACJ;AA/ED,oCA+EC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { IBatchMessage } from \"@fluidframework/container-definitions\";\nimport { ContainerRuntimeMessage, ICompressionRuntimeOptions } from \"./containerRuntime\";\nimport { OpCompressor } from \"./opCompressor\";\n\n/**\n * Message type used by BatchManager\n */\nexport type BatchMessage = IBatchMessage & {\n    localOpMetadata: unknown;\n    deserializedContent: ContainerRuntimeMessage;\n    referenceSequenceNumber: number;\n};\n\nexport interface IBatchManagerOptions {\n    readonly hardLimit: number;\n    readonly softLimit?: number;\n    readonly compressionOptions?: ICompressionRuntimeOptions;\n}\n\n/**\n * Helper class that manages partial batch & rollback.\n */\nexport class BatchManager {\n    private readonly opCompressor: OpCompressor;\n    private pendingBatch: BatchMessage [] = [];\n    private batchContentSize = 0;\n\n    public get length() { return this.pendingBatch.length; }\n\n    constructor(public readonly logger: ITelemetryLogger, public readonly options: IBatchManagerOptions) {\n        this.opCompressor = new OpCompressor(logger);\n    }\n\n    public push(message: BatchMessage): boolean {\n        const contentSize = this.batchContentSize + (message.contents?.length ?? 0);\n        const opCount = this.pendingBatch.length;\n\n        // Attempt to estimate batch size, aka socket message size.\n        // Each op has pretty large envelope, estimating to be 200 bytes.\n        // Also content will be strigified, and that adds a lot of overhead due to a lot of escape characters.\n        // Not taking it into account, as compression work should help there - compressed payload will be\n        // initially stored as base64, and that requires only 2 extra escape characters.\n        const socketMessageSize = contentSize + 200 * opCount;\n\n        // If we were provided soft limit, check for exceeding it.\n        // But only if we have any ops, as the intention here is to flush existing ops (on exceeding this limit)\n        // and start over. That's not an option if we have no ops.\n        // If compression is enabled, the soft and hard limit are ignored and the message will be pushed anyways.\n        // Cases where the message is still too large will be handled by the maxConsecutiveReconnects path.\n        if (this.options.softLimit !== undefined\n            && this.length > 0\n            && socketMessageSize >= this.options.softLimit\n            && Infinity === (this.options.compressionOptions?.minimumBatchSizeInBytes ?? Infinity)) {\n            return false;\n        }\n\n        if (socketMessageSize >= this.options.hardLimit\n            && Infinity === (this.options.compressionOptions?.minimumBatchSizeInBytes ?? Infinity)) {\n            return false;\n        }\n\n        this.batchContentSize = contentSize;\n        this.pendingBatch.push(message);\n        return true;\n    }\n\n    public get empty() { return this.pendingBatch.length === 0; }\n\n    public popBatch() {\n        const batch = this.pendingBatch;\n        const size = this.batchContentSize;\n        this.pendingBatch = [];\n        this.batchContentSize = 0;\n\n        if (batch.length > 0\n            && this.options.compressionOptions !== undefined\n            && this.options.compressionOptions.minimumBatchSizeInBytes < size) {\n            return this.opCompressor.compressBatch(batch, size);\n        }\n\n        return batch;\n    }\n\n    /**\n     * Capture the pending state at this point\n     */\n    public checkpoint() {\n        const startPoint = this.pendingBatch.length;\n        return {\n            rollback: (process: (message: BatchMessage) => void) => {\n                for (let i = this.pendingBatch.length; i > startPoint;) {\n                    i--;\n                    const message = this.pendingBatch[i];\n                    this.batchContentSize -= message.contents?.length ?? 0;\n                    process(message);\n                }\n\n                this.pendingBatch.length = startPoint;\n            },\n        };\n    }\n}\n"]}

package/dist/opCompressor.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"opCompressor.d.ts","sourceRoot":"","sources":["../src/opCompressor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAItE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAG9C;;;;GAIG;AACH,qBAAa,YAAY;IACrB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;IACxB,OAAO,CAAC,oBAAoB,CAAK;gBAErB,MAAM,EAAE,gBAAgB;IAI7B,aAAa,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE,cAAc,EAAE,MAAM,GAAG,YAAY,EAAE;CAiCtF"}

package/dist/opCompressor.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"opCompressor.js","sourceRoot":"","sources":["../src/opCompressor.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAGH,+DAAyD;AACzD,qEAA8D;AAC9D,iCAAiC;AAEjC,yDAAoF;AAEpF;;;;GAIG;AACH,MAAa,YAAY;IAIrB,YAAY,MAAwB;QAF5B,yBAAoB,GAAG,CAAC,CAAC;QAG7B,IAAI,CAAC,MAAM,GAAG,6BAAW,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAC7D,CAAC;IAEM,aAAa,CAAC,KAAqB,EAAE,cAAsB;QAC9D,MAAM,WAAW,GAAmB,EAAE,CAAC;QACvC,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC5B,MAAM,eAAe,GAA8B,EAAE,CAAC;QACtD,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE;YACzB,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;SACrD;QAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACpC,MAAM,gBAAgB,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC,CAAC;QACnF,MAAM,kBAAkB,GAAG,IAAA,gBAAQ,EAAC,gBAAgB,CAAC,CAAC;QACtD,MAAM,iBAAiB,GAAG,wBAAS,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAChF,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,gBAAgB,CAAC;QAE/C,IAAI,cAAc,GAAG,MAAM,IAAI,IAAI,CAAC,oBAAoB,GAAG,GAAG,EAAE;YAC5D,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;gBAC7B,SAAS,EAAE,iBAAiB;gBAC5B,QAAQ;gBACR,qBAAqB,EAAE,cAAc;gBACrC,oBAAoB,EAAE,iBAAiB,CAAC,MAAM;aACjD,CAAC,CAAC;SACN;QAED,WAAW,CAAC,IAAI,iCAAM,KAAK,CAAC,CAAC,CAAC,KAAE,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,cAAc,EAAE,iBAAiB,EAAE,CAAC,EAC5E,QAAQ,kCAAO,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,KAAE,UAAU,EAAE,IAAI,KAClD,WAAW,EAAE,wCAAqB,CAAC,GAAG,IAAG,CAAC;QAE7D,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;YAClC,WAAW,CAAC,IAAI,iCAAM,OAAO,KAAE,QAAQ,EAAE,SAAS,IAAG,CAAC;SACzD;QAED,OAAO,WAAW,CAAC;IACvB,CAAC;CACJ;AAzCD,oCAyCC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { IsoBuffer } from \"@fluidframework/common-utils\";\nimport { ChildLogger } from \"@fluidframework/telemetry-utils\";\nimport { compress } from \"lz4js\";\nimport { BatchMessage } from \"./batchManager\";\nimport { CompressionAlgorithms, ContainerRuntimeMessage } from \"./containerRuntime\";\n\n/**\n * Compresses batches of ops. It generates a single compressed op that contains\n * the contents of each op in the batch. It then submits empty ops for each original\n * op to reserve sequence numbers.\n */\nexport class OpCompressor {\n    private readonly logger;\n    private compressedBatchCount = 0;\n\n    constructor(logger: ITelemetryLogger) {\n        this.logger = ChildLogger.create(logger, \"OpCompressor\");\n    }\n\n    public compressBatch(batch: BatchMessage[], originalLength: number): BatchMessage[] {\n        const batchToSend: BatchMessage[] = [];\n        this.compressedBatchCount++;\n        const batchedContents: ContainerRuntimeMessage[] = [];\n        for (const message of batch) {\n            batchedContents.push(message.deserializedContent);\n        }\n\n        const compressionStart = Date.now();\n        const contentsAsBuffer = new TextEncoder().encode(JSON.stringify(batchedContents));\n        const compressedContents = compress(contentsAsBuffer);\n        const compressedContent = IsoBuffer.from(compressedContents).toString(\"base64\");\n        const duration = Date.now() - compressionStart;\n\n        if (originalLength > 200000 || this.compressedBatchCount % 100) {\n            this.logger.sendPerformanceEvent({\n                eventName: \"CompressedBatch\",\n                duration,\n                sizeBeforeCompression: originalLength,\n                sizeAfterCompression: compressedContent.length,\n            });\n        }\n\n        batchToSend.push({ ...batch[0], contents: JSON.stringify({ packedContents: compressedContent }),\n                           metadata: { ...batch[0].metadata, compressed: true },\n                           compression: CompressionAlgorithms.lz4 });\n\n        for (const message of batch.slice(1)) {\n            batchToSend.push({ ...message, contents: undefined });\n        }\n\n        return batchToSend;\n    }\n}\n"]}

package/dist/opDecompressor.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"opDecompressor.d.ts","sourceRoot":"","sources":["../src/opDecompressor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,yBAAyB,EAAE,MAAM,sCAAsC,CAAC;AAIjF;;;;;;;GAOG;AACH,qBAAa,cAAc;IACvB,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,mBAAmB,CAAkB;IAC7C,OAAO,CAAC,cAAc,CAAK;IAEpB,cAAc,CAAC,OAAO,EAAE,yBAAyB,GAAG,yBAAyB;CA0DvF"}

package/dist/opDecompressor.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"opDecompressor.js","sourceRoot":"","sources":["../src/opDecompressor.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,iCAAmC;AAEnC,+DAAqF;AACrF,wBAA0C;AAE1C;;;;;;;GAOG;AACH,MAAa,cAAc;IAA3B;QACY,gBAAW,GAAG,KAAK,CAAC;QAEpB,mBAAc,GAAG,CAAC,CAAC;IA4D/B,CAAC;IA1DU,cAAc,CAAC,OAAkC;;QACpD,6EAA6E;QAC7E,0EAA0E;QAC1E,4EAA4E;QAC5E,WAAW;QACX,IAAI,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,KAAK,MAAK,IAAI;eAC7B,CAAC,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,UAAU,KAAI,OAAO,CAAC,WAAW,KAAK,SAAS,CAAC,EAAE;YACxE,kCAAkC;YAClC,IAAA,qBAAM,EAAC,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,wCAAwC,CAAC,CAAC;YAC7E,IAAI,OAAO,CAAC,WAAW,EAAE;gBACrB,0DAA0D;gBAC1D,IAAA,qBAAM,EAAC,OAAO,CAAC,WAAW,KAAK,wBAAqB,CAAC,GAAG,EAChD,2DAA2D,CAAC,CAAC;aACxE;YAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YAExB,MAAM,QAAQ,GAAG,wBAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;YAC3E,MAAM,mBAAmB,GAAG,IAAA,kBAAU,EAAC,QAAQ,CAAC,CAAC;YACjD,MAAM,UAAU,GAAG,IAAA,iCAAkB,EAAC,mBAAmB,CAAC,CAAC;YAC3D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACrC,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;YAEjC,uCAAY,OAAO,KAAE,QAAQ,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,IAAG;SACpF;QAED,IAAI,IAAI,CAAC,mBAAmB,KAAK,SAAS,IAAI,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,KAAK,MAAK,SAAS,IAAI,IAAI,CAAC,WAAW,EAAE;YACrG,mCAAmC;YACnC,uCAAY,OAAO,KAAE,QAAQ,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,IAAG;SACpF;QAED,IAAI,IAAI,CAAC,mBAAmB,KAAK,SAAS,IAAI,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,KAAK,MAAK,KAAK,EAAE;YAC7E,0BAA0B;YAC1B,MAAM,aAAa,mCAAQ,OAAO,KACV,QAAQ,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,GAAE,CAAC;YAEpF,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;YACzB,IAAI,CAAC,mBAAmB,GAAG,SAAS,CAAC;YACrC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;YAExB,OAAO,aAAa,CAAC;SACxB;QAED,IAAI,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,KAAK,MAAK,SAAS;YACrC,CAAC,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,UAAU,KAAI,OAAO,CAAC,WAAW,KAAK,wBAAqB,CAAC,GAAG,CAAC,EAAE;YACrF,4BAA4B;YAC5B,IAAA,qBAAM,EAAC,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,2DAA2D,CAAC,CAAC;YAEhG,MAAM,QAAQ,GAAG,wBAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;YAC3E,MAAM,mBAAmB,GAAG,IAAA,kBAAU,EAAC,QAAQ,CAAC,CAAC;YACjD,MAAM,UAAU,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;YACjE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAErC,uCAAY,OAAO,KAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,IAAG;SAC7C;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;CACJ;AA/DD,wCA+DC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { decompress } from \"lz4js\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/protocol-definitions\";\nimport { assert, IsoBuffer, Uint8ArrayToString } from \"@fluidframework/common-utils\";\nimport { CompressionAlgorithms } from \".\";\n\n/**\n * State machine that \"unrolls\" contents of compressed batches of ops after decompressing them.\n * This class relies on some implicit contracts defined below:\n * 1. A compressed batch will have its first message with batch metadata set to true and compressed set to true\n * 2. Messages in the middle of a compressed batch will have neither batch metadata nor the compression property set\n * 3. The final message of a batch will have batch metadata set to false\n * 4. An individually compressed op will have undefined batch metadata and compression set to true\n */\nexport class OpDecompressor {\n    private activeBatch = false;\n    private rootMessageContents: any | undefined;\n    private processedCount = 0;\n\n    public processMessage(message: ISequencedDocumentMessage): ISequencedDocumentMessage {\n        // We're checking for compression = true or top level compression property so\n        // that we can enable compression without waiting on all ordering services\n        // to pick up protocol change. Eventually only the top level property should\n        // be used.\n        if (message.metadata?.batch === true\n            && (message.metadata?.compressed || message.compression !== undefined)) {\n            // Beginning of a compressed batch\n            assert(this.activeBatch === false, \"shouldn't have multiple active batches\");\n            if (message.compression) {\n                // lz4 is the only supported compression algorithm for now\n                assert(message.compression === CompressionAlgorithms.lz4,\n                        \"lz4 is currently the only supported compression algorithm\");\n            }\n\n            this.activeBatch = true;\n\n            const contents = IsoBuffer.from(message.contents.packedContents, \"base64\");\n            const decompressedMessage = decompress(contents);\n            const intoString = Uint8ArrayToString(decompressedMessage);\n            const asObj = JSON.parse(intoString);\n            this.rootMessageContents = asObj;\n\n            return { ...message, contents: this.rootMessageContents[this.processedCount++] };\n        }\n\n        if (this.rootMessageContents !== undefined && message.metadata?.batch === undefined && this.activeBatch) {\n            // Continuation of compressed batch\n            return { ...message, contents: this.rootMessageContents[this.processedCount++] };\n        }\n\n        if (this.rootMessageContents !== undefined && message.metadata?.batch === false) {\n            // End of compressed batch\n            const returnMessage = { ...message,\n                                    contents: this.rootMessageContents[this.processedCount++] };\n\n            this.activeBatch = false;\n            this.rootMessageContents = undefined;\n            this.processedCount = 0;\n\n            return returnMessage;\n        }\n\n        if (message.metadata?.batch === undefined &&\n            (message.metadata?.compressed || message.compression === CompressionAlgorithms.lz4)) {\n            // Single compressed message\n            assert(this.activeBatch === false, \"shouldn't receive compressed message in middle of a batch\");\n\n            const contents = IsoBuffer.from(message.contents.packedContents, \"base64\");\n            const decompressedMessage = decompress(contents);\n            const intoString = new TextDecoder().decode(decompressedMessage);\n            const asObj = JSON.parse(intoString);\n\n            return { ...message, contents: asObj[0] };\n        }\n\n        return message;\n    }\n}\n"]}

package/lib/batchManager.d.ts

@@ -1,42 +0,0 @@
-/*!
- * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
- * Licensed under the MIT License.
- */
-import { ITelemetryLogger } from "@fluidframework/common-definitions";
-import { IBatchMessage } from "@fluidframework/container-definitions";
-import { ContainerRuntimeMessage, ICompressionRuntimeOptions } from "./containerRuntime";
-/**
- * Message type used by BatchManager
- */
-export declare type BatchMessage = IBatchMessage & {
-    localOpMetadata: unknown;
-    deserializedContent: ContainerRuntimeMessage;
-    referenceSequenceNumber: number;
-};
-export interface IBatchManagerOptions {
-    readonly hardLimit: number;
-    readonly softLimit?: number;
-    readonly compressionOptions?: ICompressionRuntimeOptions;
-}
-/**
- * Helper class that manages partial batch & rollback.
- */
-export declare class BatchManager {
-    readonly logger: ITelemetryLogger;
-    readonly options: IBatchManagerOptions;
-    private readonly opCompressor;
-    private pendingBatch;
-    private batchContentSize;
-    get length(): number;
-    constructor(logger: ITelemetryLogger, options: IBatchManagerOptions);
-    push(message: BatchMessage): boolean;
-    get empty(): boolean;
-    popBatch(): BatchMessage[];
-    /**
-     * Capture the pending state at this point
-     */
-    checkpoint(): {
-        rollback: (process: (message: BatchMessage) => void) => void;
-    };
-}
-//# sourceMappingURL=batchManager.d.ts.map

package/lib/batchManager.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"batchManager.d.ts","sourceRoot":"","sources":["../src/batchManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AACtE,OAAO,EAAE,aAAa,EAAE,MAAM,uCAAuC,CAAC;AACtE,OAAO,EAAE,uBAAuB,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAGzF;;GAEG;AACH,oBAAY,YAAY,GAAG,aAAa,GAAG;IACvC,eAAe,EAAE,OAAO,CAAC;IACzB,mBAAmB,EAAE,uBAAuB,CAAC;IAC7C,uBAAuB,EAAE,MAAM,CAAC;CACnC,CAAC;AAEF,MAAM,WAAW,oBAAoB;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,kBAAkB,CAAC,EAAE,0BAA0B,CAAC;CAC5D;AAED;;GAEG;AACH,qBAAa,YAAY;aAOO,MAAM,EAAE,gBAAgB;aAAkB,OAAO,EAAE,oBAAoB;IANnG,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAe;IAC5C,OAAO,CAAC,YAAY,CAAuB;IAC3C,OAAO,CAAC,gBAAgB,CAAK;IAE7B,IAAW,MAAM,WAAuC;gBAE5B,MAAM,EAAE,gBAAgB,EAAkB,OAAO,EAAE,oBAAoB;IAI5F,IAAI,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO;IAiC3C,IAAW,KAAK,YAA6C;IAEtD,QAAQ;IAef;;OAEG;IACI,UAAU;sCAGqB,YAAY,KAAK,IAAI;;CAY9D"}

package/lib/batchManager.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"batchManager.js","sourceRoot":"","sources":["../src/batchManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAiB9C;;GAEG;AACH,MAAM,OAAO,YAAY;IAOrB,YAA4B,MAAwB,EAAkB,OAA6B;QAAvE,WAAM,GAAN,MAAM,CAAkB;QAAkB,YAAO,GAAP,OAAO,CAAsB;QAL3F,iBAAY,GAAoB,EAAE,CAAC;QACnC,qBAAgB,GAAG,CAAC,CAAC;QAKzB,IAAI,CAAC,YAAY,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC;IACjD,CAAC;IAJD,IAAW,MAAM,KAAK,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;IAMjD,IAAI,CAAC,OAAqB;;QAC7B,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,GAAG,CAAC,MAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,MAAM,mCAAI,CAAC,CAAC,CAAC;QAC5E,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;QAEzC,2DAA2D;QAC3D,iEAAiE;QACjE,sGAAsG;QACtG,iGAAiG;QACjG,gFAAgF;QAChF,MAAM,iBAAiB,GAAG,WAAW,GAAG,GAAG,GAAG,OAAO,CAAC;QAEtD,0DAA0D;QAC1D,wGAAwG;QACxG,0DAA0D;QAC1D,yGAAyG;QACzG,mGAAmG;QACnG,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS;eACjC,IAAI,CAAC,MAAM,GAAG,CAAC;eACf,iBAAiB,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS;eAC3C,QAAQ,KAAK,CAAC,MAAA,MAAA,IAAI,CAAC,OAAO,CAAC,kBAAkB,0CAAE,uBAAuB,mCAAI,QAAQ,CAAC,EAAE;YACxF,OAAO,KAAK,CAAC;SAChB;QAED,IAAI,iBAAiB,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS;eACxC,QAAQ,KAAK,CAAC,MAAA,MAAA,IAAI,CAAC,OAAO,CAAC,kBAAkB,0CAAE,uBAAuB,mCAAI,QAAQ,CAAC,EAAE;YACxF,OAAO,KAAK,CAAC;SAChB;QAED,IAAI,CAAC,gBAAgB,GAAG,WAAW,CAAC;QACpC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAChC,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,IAAW,KAAK,KAAK,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC;IAEtD,QAAQ;QACX,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC;QAE1B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;eACb,IAAI,CAAC,OAAO,CAAC,kBAAkB,KAAK,SAAS;eAC7C,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,uBAAuB,GAAG,IAAI,EAAE;YACnE,OAAO,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;SACvD;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;OAEG;IACI,UAAU;QACb,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;QAC5C,OAAO;YACH,QAAQ,EAAE,CAAC,OAAwC,EAAE,EAAE;;gBACnD,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,GAAG,UAAU,GAAG;oBACpD,CAAC,EAAE,CAAC;oBACJ,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;oBACrC,IAAI,CAAC,gBAAgB,IAAI,MAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,MAAM,mCAAI,CAAC,CAAC;oBACvD,OAAO,CAAC,OAAO,CAAC,CAAC;iBACpB;gBAED,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,UAAU,CAAC;YAC1C,CAAC;SACJ,CAAC;IACN,CAAC;CACJ","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { IBatchMessage } from \"@fluidframework/container-definitions\";\nimport { ContainerRuntimeMessage, ICompressionRuntimeOptions } from \"./containerRuntime\";\nimport { OpCompressor } from \"./opCompressor\";\n\n/**\n * Message type used by BatchManager\n */\nexport type BatchMessage = IBatchMessage & {\n    localOpMetadata: unknown;\n    deserializedContent: ContainerRuntimeMessage;\n    referenceSequenceNumber: number;\n};\n\nexport interface IBatchManagerOptions {\n    readonly hardLimit: number;\n    readonly softLimit?: number;\n    readonly compressionOptions?: ICompressionRuntimeOptions;\n}\n\n/**\n * Helper class that manages partial batch & rollback.\n */\nexport class BatchManager {\n    private readonly opCompressor: OpCompressor;\n    private pendingBatch: BatchMessage [] = [];\n    private batchContentSize = 0;\n\n    public get length() { return this.pendingBatch.length; }\n\n    constructor(public readonly logger: ITelemetryLogger, public readonly options: IBatchManagerOptions) {\n        this.opCompressor = new OpCompressor(logger);\n    }\n\n    public push(message: BatchMessage): boolean {\n        const contentSize = this.batchContentSize + (message.contents?.length ?? 0);\n        const opCount = this.pendingBatch.length;\n\n        // Attempt to estimate batch size, aka socket message size.\n        // Each op has pretty large envelope, estimating to be 200 bytes.\n        // Also content will be strigified, and that adds a lot of overhead due to a lot of escape characters.\n        // Not taking it into account, as compression work should help there - compressed payload will be\n        // initially stored as base64, and that requires only 2 extra escape characters.\n        const socketMessageSize = contentSize + 200 * opCount;\n\n        // If we were provided soft limit, check for exceeding it.\n        // But only if we have any ops, as the intention here is to flush existing ops (on exceeding this limit)\n        // and start over. That's not an option if we have no ops.\n        // If compression is enabled, the soft and hard limit are ignored and the message will be pushed anyways.\n        // Cases where the message is still too large will be handled by the maxConsecutiveReconnects path.\n        if (this.options.softLimit !== undefined\n            && this.length > 0\n            && socketMessageSize >= this.options.softLimit\n            && Infinity === (this.options.compressionOptions?.minimumBatchSizeInBytes ?? Infinity)) {\n            return false;\n        }\n\n        if (socketMessageSize >= this.options.hardLimit\n            && Infinity === (this.options.compressionOptions?.minimumBatchSizeInBytes ?? Infinity)) {\n            return false;\n        }\n\n        this.batchContentSize = contentSize;\n        this.pendingBatch.push(message);\n        return true;\n    }\n\n    public get empty() { return this.pendingBatch.length === 0; }\n\n    public popBatch() {\n        const batch = this.pendingBatch;\n        const size = this.batchContentSize;\n        this.pendingBatch = [];\n        this.batchContentSize = 0;\n\n        if (batch.length > 0\n            && this.options.compressionOptions !== undefined\n            && this.options.compressionOptions.minimumBatchSizeInBytes < size) {\n            return this.opCompressor.compressBatch(batch, size);\n        }\n\n        return batch;\n    }\n\n    /**\n     * Capture the pending state at this point\n     */\n    public checkpoint() {\n        const startPoint = this.pendingBatch.length;\n        return {\n            rollback: (process: (message: BatchMessage) => void) => {\n                for (let i = this.pendingBatch.length; i > startPoint;) {\n                    i--;\n                    const message = this.pendingBatch[i];\n                    this.batchContentSize -= message.contents?.length ?? 0;\n                    process(message);\n                }\n\n                this.pendingBatch.length = startPoint;\n            },\n        };\n    }\n}\n"]}

package/lib/opCompressor.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"opCompressor.d.ts","sourceRoot":"","sources":["../src/opCompressor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAItE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAG9C;;;;GAIG;AACH,qBAAa,YAAY;IACrB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;IACxB,OAAO,CAAC,oBAAoB,CAAK;gBAErB,MAAM,EAAE,gBAAgB;IAI7B,aAAa,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE,cAAc,EAAE,MAAM,GAAG,YAAY,EAAE;CAiCtF"}

package/lib/opCompressor.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"opCompressor.js","sourceRoot":"","sources":["../src/opCompressor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,EAAE,QAAQ,EAAE,MAAM,OAAO,CAAC;AAEjC,OAAO,EAAE,qBAAqB,EAA2B,MAAM,oBAAoB,CAAC;AAEpF;;;;GAIG;AACH,MAAM,OAAO,YAAY;IAIrB,YAAY,MAAwB;QAF5B,yBAAoB,GAAG,CAAC,CAAC;QAG7B,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAC7D,CAAC;IAEM,aAAa,CAAC,KAAqB,EAAE,cAAsB;QAC9D,MAAM,WAAW,GAAmB,EAAE,CAAC;QACvC,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC5B,MAAM,eAAe,GAA8B,EAAE,CAAC;QACtD,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE;YACzB,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;SACrD;QAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACpC,MAAM,gBAAgB,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC,CAAC;QACnF,MAAM,kBAAkB,GAAG,QAAQ,CAAC,gBAAgB,CAAC,CAAC;QACtD,MAAM,iBAAiB,GAAG,SAAS,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAChF,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,gBAAgB,CAAC;QAE/C,IAAI,cAAc,GAAG,MAAM,IAAI,IAAI,CAAC,oBAAoB,GAAG,GAAG,EAAE;YAC5D,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;gBAC7B,SAAS,EAAE,iBAAiB;gBAC5B,QAAQ;gBACR,qBAAqB,EAAE,cAAc;gBACrC,oBAAoB,EAAE,iBAAiB,CAAC,MAAM;aACjD,CAAC,CAAC;SACN;QAED,WAAW,CAAC,IAAI,iCAAM,KAAK,CAAC,CAAC,CAAC,KAAE,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,cAAc,EAAE,iBAAiB,EAAE,CAAC,EAC5E,QAAQ,kCAAO,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,KAAE,UAAU,EAAE,IAAI,KAClD,WAAW,EAAE,qBAAqB,CAAC,GAAG,IAAG,CAAC;QAE7D,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;YAClC,WAAW,CAAC,IAAI,iCAAM,OAAO,KAAE,QAAQ,EAAE,SAAS,IAAG,CAAC;SACzD;QAED,OAAO,WAAW,CAAC;IACvB,CAAC;CACJ","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { IsoBuffer } from \"@fluidframework/common-utils\";\nimport { ChildLogger } from \"@fluidframework/telemetry-utils\";\nimport { compress } from \"lz4js\";\nimport { BatchMessage } from \"./batchManager\";\nimport { CompressionAlgorithms, ContainerRuntimeMessage } from \"./containerRuntime\";\n\n/**\n * Compresses batches of ops. It generates a single compressed op that contains\n * the contents of each op in the batch. It then submits empty ops for each original\n * op to reserve sequence numbers.\n */\nexport class OpCompressor {\n    private readonly logger;\n    private compressedBatchCount = 0;\n\n    constructor(logger: ITelemetryLogger) {\n        this.logger = ChildLogger.create(logger, \"OpCompressor\");\n    }\n\n    public compressBatch(batch: BatchMessage[], originalLength: number): BatchMessage[] {\n        const batchToSend: BatchMessage[] = [];\n        this.compressedBatchCount++;\n        const batchedContents: ContainerRuntimeMessage[] = [];\n        for (const message of batch) {\n            batchedContents.push(message.deserializedContent);\n        }\n\n        const compressionStart = Date.now();\n        const contentsAsBuffer = new TextEncoder().encode(JSON.stringify(batchedContents));\n        const compressedContents = compress(contentsAsBuffer);\n        const compressedContent = IsoBuffer.from(compressedContents).toString(\"base64\");\n        const duration = Date.now() - compressionStart;\n\n        if (originalLength > 200000 || this.compressedBatchCount % 100) {\n            this.logger.sendPerformanceEvent({\n                eventName: \"CompressedBatch\",\n                duration,\n                sizeBeforeCompression: originalLength,\n                sizeAfterCompression: compressedContent.length,\n            });\n        }\n\n        batchToSend.push({ ...batch[0], contents: JSON.stringify({ packedContents: compressedContent }),\n                           metadata: { ...batch[0].metadata, compressed: true },\n                           compression: CompressionAlgorithms.lz4 });\n\n        for (const message of batch.slice(1)) {\n            batchToSend.push({ ...message, contents: undefined });\n        }\n\n        return batchToSend;\n    }\n}\n"]}

package/lib/opDecompressor.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"opDecompressor.d.ts","sourceRoot":"","sources":["../src/opDecompressor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,yBAAyB,EAAE,MAAM,sCAAsC,CAAC;AAIjF;;;;;;;GAOG;AACH,qBAAa,cAAc;IACvB,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,mBAAmB,CAAkB;IAC7C,OAAO,CAAC,cAAc,CAAK;IAEpB,cAAc,CAAC,OAAO,EAAE,yBAAyB,GAAG,yBAAyB;CA0DvF"}

package/lib/opDecompressor.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"opDecompressor.js","sourceRoot":"","sources":["../src/opDecompressor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AAEnC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AACrF,OAAO,EAAE,qBAAqB,EAAE,MAAM,GAAG,CAAC;AAE1C;;;;;;;GAOG;AACH,MAAM,OAAO,cAAc;IAA3B;QACY,gBAAW,GAAG,KAAK,CAAC;QAEpB,mBAAc,GAAG,CAAC,CAAC;IA4D/B,CAAC;IA1DU,cAAc,CAAC,OAAkC;;QACpD,6EAA6E;QAC7E,0EAA0E;QAC1E,4EAA4E;QAC5E,WAAW;QACX,IAAI,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,KAAK,MAAK,IAAI;eAC7B,CAAC,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,UAAU,KAAI,OAAO,CAAC,WAAW,KAAK,SAAS,CAAC,EAAE;YACxE,kCAAkC;YAClC,MAAM,CAAC,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,wCAAwC,CAAC,CAAC;YAC7E,IAAI,OAAO,CAAC,WAAW,EAAE;gBACrB,0DAA0D;gBAC1D,MAAM,CAAC,OAAO,CAAC,WAAW,KAAK,qBAAqB,CAAC,GAAG,EAChD,2DAA2D,CAAC,CAAC;aACxE;YAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YAExB,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;YAC3E,MAAM,mBAAmB,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;YACjD,MAAM,UAAU,GAAG,kBAAkB,CAAC,mBAAmB,CAAC,CAAC;YAC3D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACrC,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;YAEjC,uCAAY,OAAO,KAAE,QAAQ,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,IAAG;SACpF;QAED,IAAI,IAAI,CAAC,mBAAmB,KAAK,SAAS,IAAI,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,KAAK,MAAK,SAAS,IAAI,IAAI,CAAC,WAAW,EAAE;YACrG,mCAAmC;YACnC,uCAAY,OAAO,KAAE,QAAQ,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,IAAG;SACpF;QAED,IAAI,IAAI,CAAC,mBAAmB,KAAK,SAAS,IAAI,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,KAAK,MAAK,KAAK,EAAE;YAC7E,0BAA0B;YAC1B,MAAM,aAAa,mCAAQ,OAAO,KACV,QAAQ,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,GAAE,CAAC;YAEpF,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;YACzB,IAAI,CAAC,mBAAmB,GAAG,SAAS,CAAC;YACrC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;YAExB,OAAO,aAAa,CAAC;SACxB;QAED,IAAI,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,KAAK,MAAK,SAAS;YACrC,CAAC,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,UAAU,KAAI,OAAO,CAAC,WAAW,KAAK,qBAAqB,CAAC,GAAG,CAAC,EAAE;YACrF,4BAA4B;YAC5B,MAAM,CAAC,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,2DAA2D,CAAC,CAAC;YAEhG,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;YAC3E,MAAM,mBAAmB,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;YACjD,MAAM,UAAU,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;YACjE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAErC,uCAAY,OAAO,KAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,IAAG;SAC7C;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;CACJ","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { decompress } from \"lz4js\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/protocol-definitions\";\nimport { assert, IsoBuffer, Uint8ArrayToString } from \"@fluidframework/common-utils\";\nimport { CompressionAlgorithms } from \".\";\n\n/**\n * State machine that \"unrolls\" contents of compressed batches of ops after decompressing them.\n * This class relies on some implicit contracts defined below:\n * 1. A compressed batch will have its first message with batch metadata set to true and compressed set to true\n * 2. Messages in the middle of a compressed batch will have neither batch metadata nor the compression property set\n * 3. The final message of a batch will have batch metadata set to false\n * 4. An individually compressed op will have undefined batch metadata and compression set to true\n */\nexport class OpDecompressor {\n    private activeBatch = false;\n    private rootMessageContents: any | undefined;\n    private processedCount = 0;\n\n    public processMessage(message: ISequencedDocumentMessage): ISequencedDocumentMessage {\n        // We're checking for compression = true or top level compression property so\n        // that we can enable compression without waiting on all ordering services\n        // to pick up protocol change. Eventually only the top level property should\n        // be used.\n        if (message.metadata?.batch === true\n            && (message.metadata?.compressed || message.compression !== undefined)) {\n            // Beginning of a compressed batch\n            assert(this.activeBatch === false, \"shouldn't have multiple active batches\");\n            if (message.compression) {\n                // lz4 is the only supported compression algorithm for now\n                assert(message.compression === CompressionAlgorithms.lz4,\n                        \"lz4 is currently the only supported compression algorithm\");\n            }\n\n            this.activeBatch = true;\n\n            const contents = IsoBuffer.from(message.contents.packedContents, \"base64\");\n            const decompressedMessage = decompress(contents);\n            const intoString = Uint8ArrayToString(decompressedMessage);\n            const asObj = JSON.parse(intoString);\n            this.rootMessageContents = asObj;\n\n            return { ...message, contents: this.rootMessageContents[this.processedCount++] };\n        }\n\n        if (this.rootMessageContents !== undefined && message.metadata?.batch === undefined && this.activeBatch) {\n            // Continuation of compressed batch\n            return { ...message, contents: this.rootMessageContents[this.processedCount++] };\n        }\n\n        if (this.rootMessageContents !== undefined && message.metadata?.batch === false) {\n            // End of compressed batch\n            const returnMessage = { ...message,\n                                    contents: this.rootMessageContents[this.processedCount++] };\n\n            this.activeBatch = false;\n            this.rootMessageContents = undefined;\n            this.processedCount = 0;\n\n            return returnMessage;\n        }\n\n        if (message.metadata?.batch === undefined &&\n            (message.metadata?.compressed || message.compression === CompressionAlgorithms.lz4)) {\n            // Single compressed message\n            assert(this.activeBatch === false, \"shouldn't receive compressed message in middle of a batch\");\n\n            const contents = IsoBuffer.from(message.contents.packedContents, \"base64\");\n            const decompressedMessage = decompress(contents);\n            const intoString = new TextDecoder().decode(decompressedMessage);\n            const asObj = JSON.parse(intoString);\n\n            return { ...message, contents: asObj[0] };\n        }\n\n        return message;\n    }\n}\n"]}