@fluidframework/container-runtime 2.0.0-internal.2.1.1 → 2.0.0-internal.2.2.0

package/src/garbageCollectionConstants.ts

@@ -0,0 +1,35 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+
+// The key for the GC tree in summary.
+export const gcTreeKey = "gc";
+// They prefix for GC blobs in the GC tree in summary.
+export const gcBlobPrefix = "__gc";
+// The key for tombstone blob in the GC tree in summary.
+export const gcTombstoneBlobKey = "__tombstones";
+
+// Feature gate key to turn GC on / off.
+export const runGCKey = "Fluid.GarbageCollection.RunGC";
+// Feature gate key to turn GC sweep on / off.
+export const runSweepKey = "Fluid.GarbageCollection.RunSweep";
+// Feature gate key to turn GC test mode on / off.
+export const gcTestModeKey = "Fluid.GarbageCollection.GCTestMode";
+// Feature gate key to expire a session after a set period of time.
+export const runSessionExpiryKey = "Fluid.GarbageCollection.RunSessionExpiry";
+// Feature gate key to write the gc blob as a handle if the data is the same.
+export const trackGCStateKey = "Fluid.GarbageCollection.TrackGCState";
+// Feature gate key to turn GC sweep log off.
+export const disableSweepLogKey = "Fluid.GarbageCollection.DisableSweepLog";
+// Feature gate key to disable the tombstone feature, i.e., tombstone information is not read / written into summary.
+export const disableTombstoneKey = "Fluid.GarbageCollection.DisableTombstone";
+// Feature gate to enable throwing an error when tombstone object is used.
+export const throwOnTombstoneUsageKey = "Fluid.GarbageCollection.ThrowOnTombstoneUsage";
+
+// One day in milliseconds.
+export const oneDayMs = 1 * 24 * 60 * 60 * 1000;
+
+export const defaultInactiveTimeoutMs = 7 * oneDayMs; // 7 days
+export const defaultSessionExpiryDurationMs = 30 * oneDayMs; // 30 days

package/src/gcSweepReadyUsageDetection.ts

@@ -11,7 +11,7 @@ import {
     LoggingError,
     MonitoringContext,
 } from "@fluidframework/telemetry-utils";
-import { oneDayMs } from "./garbageCollection";
+import { oneDayMs } from "./garbageCollectionConstants";
 
 /**
  * Feature Gate Key -

package/src/index.ts

@@ -5,7 +5,6 @@
 
 export {
     ContainerMessageType,
-    IChunkedOp,
     ContainerRuntimeMessage,
     IGCRuntimeOptions,
     ISummaryRuntimeOptions,
@@ -17,21 +16,23 @@ export {
     IRootSummaryTreeWithStats,
     isRuntimeMessage,
     RuntimeMessage,
-    unpackRuntimeMessage,
     agentSchedulerId,
     ContainerRuntime,
     RuntimeHeaders,
     ISummaryConfiguration,
     DefaultSummaryConfiguration,
     ICompressionRuntimeOptions,
+    CompressionAlgorithms,
 } from "./containerRuntime";
 export { FluidDataStoreRegistry } from "./dataStoreRegistry";
+export {
+    IGCStats,
+} from "./garbageCollection";
 export {
     gcBlobPrefix,
     gcTombstoneBlobKey,
     gcTreeKey,
-    IGCStats,
-} from "./garbageCollection";
+} from "./garbageCollectionConstants";
 export {
     IPendingFlush,
     IPendingLocalState,
@@ -81,3 +82,4 @@ export {
     SummaryCollection,
 } from "./summaryCollection";
 export { ICancellableSummarizerController, neverCancelledSummaryToken } from "./runWhileConnectedCoordinator";
+export { IChunkedOp, unpackRuntimeMessage } from "./opLifecycle";

package/src/{batchManager.ts → opLifecycle/batchManager.ts}

@@ -3,17 +3,14 @@
  * Licensed under the MIT License.
  */
 
-import { IBatchMessage } from "@fluidframework/container-definitions";
-import { ContainerRuntimeMessage } from "./containerRuntime";
+import { ICompressionRuntimeOptions } from "../containerRuntime";
+import { BatchMessage, IBatch, IBatchCheckpoint } from "./definitions";
 
-/**
- * Message type used by BatchManager
- */
-export type BatchMessage = IBatchMessage & {
-    localOpMetadata: unknown;
-    deserializedContent: ContainerRuntimeMessage;
-    referenceSequenceNumber: number;
-};
+export interface IBatchManagerOptions {
+    readonly hardLimit: number;
+    readonly softLimit?: number;
+    readonly compressionOptions?: ICompressionRuntimeOptions;
+}
 
 /**
  * Helper class that manages partial batch & rollback.
@@ -23,15 +20,12 @@ export class BatchManager {
     private batchContentSize = 0;
 
     public get length() { return this.pendingBatch.length; }
-    public get limit() { return this.hardLimit; }
+    public get contentSizeInBytes() { return this.batchContentSize; }
 
-    constructor(
-        private readonly hardLimit: number,
-        public readonly softLimit?: number,
-    ) { }
+    constructor(public readonly options: IBatchManagerOptions) { }
 
     public push(message: BatchMessage): boolean {
-        const contentSize = this.batchContentSize + message.contents.length;
+        const contentSize = this.batchContentSize + (message.contents?.length ?? 0);
         const opCount = this.pendingBatch.length;
 
         // Attempt to estimate batch size, aka socket message size.
@@ -44,11 +38,15 @@ export class BatchManager {
         // If we were provided soft limit, check for exceeding it.
         // But only if we have any ops, as the intention here is to flush existing ops (on exceeding this limit)
         // and start over. That's not an option if we have no ops.
-        if (this.softLimit !== undefined && this.length > 0 && socketMessageSize >= this.softLimit) {
+        // If compression is enabled, the soft and hard limit are ignored and the message will be pushed anyways.
+        // Cases where the message is still too large will be handled by the maxConsecutiveReconnects path.
+        if (this.options.softLimit !== undefined
+            && this.length > 0
+            && socketMessageSize >= this.options.softLimit) {
             return false;
         }
 
-        if (socketMessageSize >= this.limit) {
+        if (socketMessageSize >= this.options.hardLimit) {
             return false;
         }
 
@@ -59,24 +57,29 @@ export class BatchManager {
 
     public get empty() { return this.pendingBatch.length === 0; }
 
-    public popBatch() {
-        const batch = this.pendingBatch;
+    public popBatch(): IBatch {
+        const batch: IBatch = {
+            content: this.pendingBatch,
+            contentSizeInBytes: this.batchContentSize,
+        };
+
         this.pendingBatch = [];
         this.batchContentSize = 0;
-        return batch;
+
+        return addBatchMetadata(batch);
     }
 
     /**
      * Capture the pending state at this point
      */
-    public checkpoint() {
+    public checkpoint(): IBatchCheckpoint {
         const startPoint = this.pendingBatch.length;
         return {
             rollback: (process: (message: BatchMessage) => void) => {
                 for (let i = this.pendingBatch.length; i > startPoint;) {
                     i--;
                     const message = this.pendingBatch[i];
-                    this.batchContentSize -= message.contents.length;
+                    this.batchContentSize -= message.contents?.length ?? 0;
                     process(message);
                 }
 
@@ -85,3 +88,18 @@ export class BatchManager {
         };
     }
 }
+
+const addBatchMetadata = (batch: IBatch): IBatch => {
+    if (batch.content.length > 1) {
+        batch.content[0].metadata = {
+            ...batch.content[0].metadata,
+            batch: true
+        };
+        batch.content[batch.content.length - 1].metadata = {
+            ...batch.content[batch.content.length - 1].metadata,
+            batch: false
+        };
+    }
+
+    return batch;
+};

package/src/opLifecycle/definitions.ts

@@ -0,0 +1,44 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { IBatchMessage } from "@fluidframework/container-definitions";
+import { MessageType } from "@fluidframework/protocol-definitions";
+import { CompressionAlgorithms, ContainerMessageType, ContainerRuntimeMessage } from "..";
+
+/**
+ * Batch message type used internally by the runtime
+ */
+export type BatchMessage = IBatchMessage & {
+    localOpMetadata: unknown;
+    deserializedContent: ContainerRuntimeMessage;
+    referenceSequenceNumber: number;
+    compression?: CompressionAlgorithms;
+};
+
+/**
+ * Batch interface used internally by the runtime.
+ */
+export interface IBatch {
+    /**
+     * Sum of the in-memory content sizes of all messages in the batch.
+     * If the batch is compressed, this number reflects the post-compression size.
+     */
+    readonly contentSizeInBytes: number;
+    /**
+     * All the messages in the batch
+     */
+    readonly content: BatchMessage[];
+}
+
+export interface IBatchCheckpoint {
+    rollback: (action: (message: BatchMessage) => void) => void;
+}
+
+export interface IChunkedOp {
+    chunkId: number;
+    totalChunks: number;
+    contents: string;
+    originalType: MessageType | ContainerMessageType;
+}

package/src/opLifecycle/index.ts

@@ -0,0 +1,17 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+export { BatchManager } from "./batchManager";
+export {
+    BatchMessage,
+    IBatch,
+    IBatchCheckpoint,
+    IChunkedOp,
+} from "./definitions";
+export { Outbox } from "./outbox";
+export { OpCompressor } from "./opCompressor";
+export { OpDecompressor } from "./opDecompressor";
+export { OpSplitter } from "./opSplitter";
+export { RemoteMessageProcessor, unpackRuntimeMessage } from "./remoteMessageProcessor";

package/src/opLifecycle/opCompressor.ts

@@ -0,0 +1,64 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { ITelemetryLogger } from "@fluidframework/common-definitions";
+import { IsoBuffer } from "@fluidframework/common-utils";
+import { ChildLogger } from "@fluidframework/telemetry-utils";
+import { compress } from "lz4js";
+import { CompressionAlgorithms, ContainerRuntimeMessage } from "../containerRuntime";
+import { IBatch, BatchMessage } from "./definitions";
+
+/**
+ * Compresses batches of ops. It generates a single compressed op that contains
+ * the contents of each op in the batch. It then submits empty ops for each original
+ * op to reserve sequence numbers.
+ */
+export class OpCompressor {
+    private readonly logger;
+    private compressedBatchCount = 0;
+
+    constructor(logger: ITelemetryLogger) {
+        this.logger = ChildLogger.create(logger, "OpCompressor");
+    }
+
+    public compressBatch(batch: IBatch): IBatch {
+        const messages: BatchMessage[] = [];
+        this.compressedBatchCount++;
+        const contentToCompress: ContainerRuntimeMessage[] = [];
+        for (const message of batch.content) {
+            contentToCompress.push(message.deserializedContent);
+        }
+
+        const compressionStart = Date.now();
+        const contentsAsBuffer = new TextEncoder().encode(JSON.stringify(contentToCompress));
+        const compressedContents = compress(contentsAsBuffer);
+        const compressedContent = IsoBuffer.from(compressedContents).toString("base64");
+        const duration = Date.now() - compressionStart;
+
+        if (batch.contentSizeInBytes > 200000 || this.compressedBatchCount % 25) {
+            this.logger.sendPerformanceEvent({
+                eventName: "CompressedBatch",
+                duration,
+                sizeBeforeCompression: batch.contentSizeInBytes,
+                sizeAfterCompression: compressedContent.length,
+            });
+        }
+
+        messages.push({
+            ...batch.content[0], contents: JSON.stringify({ packedContents: compressedContent }),
+            metadata: { ...batch.content[0].metadata, compressed: true },
+            compression: CompressionAlgorithms.lz4,
+        });
+
+        for (const message of batch.content.slice(1)) {
+            messages.push({ ...message, contents: undefined });
+        }
+
+        return {
+            contentSizeInBytes: compressedContent.length,
+            content: messages,
+        };
+    }
+}

package/src/opLifecycle/opDecompressor.ts

@@ -0,0 +1,84 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { decompress } from "lz4js";
+import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
+import { assert, IsoBuffer, Uint8ArrayToString } from "@fluidframework/common-utils";
+import { CompressionAlgorithms } from "../containerRuntime";
+
+/**
+ * State machine that "unrolls" contents of compressed batches of ops after decompressing them.
+ * This class relies on some implicit contracts defined below:
+ * 1. A compressed batch will have its first message with batch metadata set to true and compressed set to true
+ * 2. Messages in the middle of a compressed batch will have neither batch metadata nor the compression property set
+ * 3. The final message of a batch will have batch metadata set to false
+ * 4. An individually compressed op will have undefined batch metadata and compression set to true
+ */
+export class OpDecompressor {
+    private activeBatch = false;
+    private rootMessageContents: any | undefined;
+    private processedCount = 0;
+
+    public processMessage(message: ISequencedDocumentMessage): ISequencedDocumentMessage {
+        // We're checking for compression = true or top level compression property so
+        // that we can enable compression without waiting on all ordering services
+        // to pick up protocol change. Eventually only the top level property should
+        // be used.
+        if (message.metadata?.batch === true
+            && (message.metadata?.compressed || message.compression !== undefined)) {
+            // Beginning of a compressed batch
+            assert(this.activeBatch === false, 0x4b8 /* shouldn't have multiple active batches */);
+            if (message.compression) {
+                // lz4 is the only supported compression algorithm for now
+                assert(message.compression === CompressionAlgorithms.lz4,
+                    0x4b9 /* lz4 is currently the only supported compression algorithm */);
+            }
+
+            this.activeBatch = true;
+
+            const contents = IsoBuffer.from(message.contents.packedContents, "base64");
+            const decompressedMessage = decompress(contents);
+            const intoString = Uint8ArrayToString(decompressedMessage);
+            const asObj = JSON.parse(intoString);
+            this.rootMessageContents = asObj;
+
+            return { ...message, contents: this.rootMessageContents[this.processedCount++] };
+        }
+
+        if (this.rootMessageContents !== undefined && message.metadata?.batch === undefined && this.activeBatch) {
+            // Continuation of compressed batch
+            return { ...message, contents: this.rootMessageContents[this.processedCount++] };
+        }
+
+        if (this.rootMessageContents !== undefined && message.metadata?.batch === false) {
+            // End of compressed batch
+            const returnMessage = {
+                ...message,
+                contents: this.rootMessageContents[this.processedCount++]
+            };
+
+            this.activeBatch = false;
+            this.rootMessageContents = undefined;
+            this.processedCount = 0;
+
+            return returnMessage;
+        }
+
+        if (message.metadata?.batch === undefined &&
+            (message.metadata?.compressed || message.compression === CompressionAlgorithms.lz4)) {
+            // Single compressed message
+            assert(this.activeBatch === false, 0x4ba /* shouldn't receive compressed message in middle of a batch */);
+
+            const contents = IsoBuffer.from(message.contents.packedContents, "base64");
+            const decompressedMessage = decompress(contents);
+            const intoString = new TextDecoder().decode(decompressedMessage);
+            const asObj = JSON.parse(intoString);
+
+            return { ...message, contents: asObj[0] };
+        }
+
+        return message;
+    }
+}

package/src/opLifecycle/opSplitter.ts

@@ -0,0 +1,78 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { DataCorruptionError, extractSafePropertiesFromMessage } from "@fluidframework/container-utils";
+import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
+import { ContainerMessageType } from "../containerRuntime";
+import { IChunkedOp } from "./definitions";
+
+/**
+ * Responsible for creating and reconstructing chunked messages.
+ */
+export class OpSplitter {
+    // Local copy of incomplete received chunks.
+    private readonly chunkMap: Map<string, string[]>;
+
+    constructor(chunks: [string, string[]][]) {
+        this.chunkMap = new Map<string, string[]>(chunks);
+    }
+
+    public get chunks(): ReadonlyMap<string, string[]> {
+        return this.chunkMap;
+    }
+
+    public processRemoteMessage(message: ISequencedDocumentMessage): ISequencedDocumentMessage {
+        if (message.type !== ContainerMessageType.ChunkedOp) {
+            return message;
+        }
+
+        const clientId = message.clientId;
+        const chunkedContent = message.contents as IChunkedOp;
+        this.addChunk(clientId, chunkedContent, message);
+
+        if (chunkedContent.chunkId < chunkedContent.totalChunks) {
+            // We are processing the op in chunks but haven't reached
+            // the last chunk yet in order to reconstruct the original op
+            return message;
+        }
+
+        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+        const serializedContent = this.chunkMap.get(clientId)!.join("");
+        this.clearPartialChunks(clientId);
+
+        const newMessage = { ...message };
+        newMessage.contents = serializedContent === "" ? undefined : JSON.parse(serializedContent);
+        newMessage.type = chunkedContent.originalType;
+        return newMessage;
+    }
+
+    public clearPartialChunks(clientId: string) {
+        if (this.chunkMap.has(clientId)) {
+            this.chunkMap.delete(clientId);
+        }
+    }
+
+    private addChunk(clientId: string, chunkedContent: IChunkedOp, originalMessage: ISequencedDocumentMessage) {
+        let map = this.chunkMap.get(clientId);
+        if (map === undefined) {
+            map = [];
+            this.chunkMap.set(clientId, map);
+        }
+
+        if (chunkedContent.chunkId !== map.length + 1) {
+            // We are expecting the chunks to be processed sequentially, in the same order as they are sent.
+            // Therefore, the chunkId of the incoming op needs to match the length of the array (1-based indexing)
+            // holding the existing chunks for that particular clientId.
+            throw new DataCorruptionError("Chunk Id mismatch", {
+                ...extractSafePropertiesFromMessage(originalMessage),
+                chunkMapLength: map.length,
+                chunkId: chunkedContent.chunkId,
+                totalChunks: chunkedContent.totalChunks,
+            });
+        }
+
+        map.push(chunkedContent.contents);
+    }
+}