@fluidframework/container-runtime 2.0.0-dev.2.2.0.111723 → 2.0.0-dev.2.3.0.115467

package/src/garbageCollection.ts

@@ -45,6 +45,21 @@ import {
 
 import { IGCRuntimeOptions, RuntimeHeaders } from "./containerRuntime";
 import { getSummaryForDatastores } from "./dataStores";
+import {
+    defaultInactiveTimeoutMs,
+    defaultSessionExpiryDurationMs,
+    disableSweepLogKey,
+    disableTombstoneKey,
+    gcBlobPrefix,
+    gcTestModeKey,
+    gcTombstoneBlobKey,
+    gcTreeKey,
+    oneDayMs,
+    runGCKey,
+    runSessionExpiryKey,
+    runSweepKey,
+    trackGCStateKey
+} from "./garbageCollectionConstants";
 import { SweepReadyUsageDetectionHandler } from "./gcSweepReadyUsageDetection";
 import {
     getGCVersion,
@@ -60,36 +75,6 @@ import {
 /** This is the current version of garbage collection. */
 const GCVersion = 1;
 
-// The key for the GC tree in summary.
-export const gcTreeKey = "gc";
-// They prefix for GC blobs in the GC tree in summary.
-export const gcBlobPrefix = "__gc";
-// The key for tombstone blob in the GC tree in summary.
-export const gcTombstoneBlobKey = "__tombstones";
-
-// Feature gate key to turn GC on / off.
-export const runGCKey = "Fluid.GarbageCollection.RunGC";
-// Feature gate key to turn GC sweep on / off.
-export const runSweepKey = "Fluid.GarbageCollection.RunSweep";
-// Feature gate key to turn GC test mode on / off.
-export const gcTestModeKey = "Fluid.GarbageCollection.GCTestMode";
-// Feature gate key to expire a session after a set period of time.
-export const runSessionExpiryKey = "Fluid.GarbageCollection.RunSessionExpiry";
-// Feature gate key to write the gc blob as a handle if the data is the same.
-export const trackGCStateKey = "Fluid.GarbageCollection.TrackGCState";
-// Feature gate key to turn GC sweep log off.
-export const disableSweepLogKey = "Fluid.GarbageCollection.DisableSweepLog";
-// Feature gate key to disable the tombstone feature, i.e., tombstone information is not read / written into summary.
-export const disableTombstoneKey = "Fluid.GarbageCollection.DisableTombstone";
-// Feature gate to enable throwing an error when tombstone object is used.
-export const throwOnTombstoneUsageKey = "Fluid.GarbageCollection.ThrowOnTombstoneUsage";
-
-// One day in milliseconds.
-export const oneDayMs = 1 * 24 * 60 * 60 * 1000;
-
-export const defaultInactiveTimeoutMs = 7 * oneDayMs; // 7 days
-export const defaultSessionExpiryDurationMs = 30 * oneDayMs; // 30 days
-
 /** The statistics of the system state after a garbage collection run. */
 export interface IGCStats {
     /** The number of nodes in the container. */
@@ -1250,8 +1235,13 @@ export class GarbageCollector implements IGarbageCollector {
 
     /**
      * Since GC runs periodically, the GC data that is generated only tells us the state of the world at that point in
-     * time. It's possible that nodes transition from `unreferenced -> referenced -> unreferenced` between two runs. The
-     * unreferenced timestamp of such nodes needs to be reset as they may have been accessed when they were referenced.
+     * time. There can be nodes that were referenced in between two runs and their unreferenced state needs to be
+     * updated. For example, in the following scenarios not updating the unreferenced timestamp can lead to deletion of
+     * these objects while there can be in-memory referenced to it:
+     * 1. A node transitions from `unreferenced -> referenced -> unreferenced` between two runs. When the reference is
+     * added, the object may have been accessed and in-memory reference to it added.
+     * 2. A reference is added from one unreferenced node to one or more unreferenced nodes. Even though the node[s] were
+     * unreferenced, they could have been accessed and in-memory reference to them added.
      *
      * This function identifies nodes that were referenced since last run and removes their unreferenced state, if any.
      * If these nodes are currently unreferenced, they will be assigned new unreferenced state by the current run.
@@ -1292,41 +1282,41 @@ export class GarbageCollector implements IGarbageCollector {
          * run, and then add the references since last run.
          *
          * Note on why we need to combine the data from previous run, current run and all references in between -
-         *
          * 1. We need data from last run because some of its references may have been deleted since then. If those
-         * references added new outbound references before getting deleted, we need to detect them.
+         * references added new outbound references before they were deleted, we need to detect them.
          *
          * 2. We need new outbound references since last run because some of them may have been deleted later. If those
-         * references added new outbound references before getting deleted, we need to detect them.
+         * references added new outbound references before they were deleted, we need to detect them.
          *
          * 3. We need data from the current run because currently we may not detect when DDSes are referenced:
-         *
-         * - We don't require DDSes handles to be stored in a referenced DDS. For this, we need GC at DDS level
-         * which is tracked by https://github.com/microsoft/FluidFramework/issues/8470.
-         *
+         * - We don't require DDSes handles to be stored in a referenced DDS.
          * - A new data store may have "root" DDSes already created and we don't detect them today.
          */
         const gcDataSuperSet = concatGarbageCollectionData(this.previousGCDataFromLastRun, currentGCData);
+        const newOutboundRoutesSinceLastRun: string[] = [];
         this.newReferencesSinceLastRun.forEach((outboundRoutes: string[], sourceNodeId: string) => {
             if (gcDataSuperSet.gcNodes[sourceNodeId] === undefined) {
                 gcDataSuperSet.gcNodes[sourceNodeId] = outboundRoutes;
             } else {
                 gcDataSuperSet.gcNodes[sourceNodeId].push(...outboundRoutes);
             }
+            newOutboundRoutesSinceLastRun.push(...outboundRoutes);
         });
 
         /**
-         * Run GC on the above reference graph to find all nodes that are referenced. For each one, if they are
+         * Run GC on the above reference graph starting with root and all new outbound routes. This will generate a
+         * list of all nodes that could have been referenced since the last run. If any of these nodes are unreferenced,
          * unreferenced, stop tracking them and remove from unreferenced list.
-         * Some of these nodes may be unreferenced now and if so, the current run will add unreferenced state for them.
+         * Note that some of these nodes may be unreferenced now and if so, the current run will mark them as
+         * unreferenced and add unreferenced state.
          */
-        const gcResult = runGarbageCollection(gcDataSuperSet.gcNodes, ["/"]);
+        const gcResult = runGarbageCollection(gcDataSuperSet.gcNodes, ["/", ...newOutboundRoutesSinceLastRun]);
         for (const nodeId of gcResult.referencedNodeIds) {
             const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
             if (nodeStateTracker !== undefined) {
                 // Stop tracking so as to clear out any running timers.
                 nodeStateTracker.stopTracking();
-                // Delete the node as we don't need to track it any more.
+                // Delete the unreferenced state as we don't need to track it any more.
                 this.unreferencedNodesState.delete(nodeId);
             }
         }

package/src/garbageCollectionConstants.ts

@@ -0,0 +1,35 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+
+// The key for the GC tree in summary.
+export const gcTreeKey = "gc";
+// They prefix for GC blobs in the GC tree in summary.
+export const gcBlobPrefix = "__gc";
+// The key for tombstone blob in the GC tree in summary.
+export const gcTombstoneBlobKey = "__tombstones";
+
+// Feature gate key to turn GC on / off.
+export const runGCKey = "Fluid.GarbageCollection.RunGC";
+// Feature gate key to turn GC sweep on / off.
+export const runSweepKey = "Fluid.GarbageCollection.RunSweep";
+// Feature gate key to turn GC test mode on / off.
+export const gcTestModeKey = "Fluid.GarbageCollection.GCTestMode";
+// Feature gate key to expire a session after a set period of time.
+export const runSessionExpiryKey = "Fluid.GarbageCollection.RunSessionExpiry";
+// Feature gate key to write the gc blob as a handle if the data is the same.
+export const trackGCStateKey = "Fluid.GarbageCollection.TrackGCState";
+// Feature gate key to turn GC sweep log off.
+export const disableSweepLogKey = "Fluid.GarbageCollection.DisableSweepLog";
+// Feature gate key to disable the tombstone feature, i.e., tombstone information is not read / written into summary.
+export const disableTombstoneKey = "Fluid.GarbageCollection.DisableTombstone";
+// Feature gate to enable throwing an error when tombstone object is used.
+export const throwOnTombstoneUsageKey = "Fluid.GarbageCollection.ThrowOnTombstoneUsage";
+
+// One day in milliseconds.
+export const oneDayMs = 1 * 24 * 60 * 60 * 1000;
+
+export const defaultInactiveTimeoutMs = 7 * oneDayMs; // 7 days
+export const defaultSessionExpiryDurationMs = 30 * oneDayMs; // 30 days

package/src/gcSweepReadyUsageDetection.ts

@@ -11,7 +11,7 @@ import {
     LoggingError,
     MonitoringContext,
 } from "@fluidframework/telemetry-utils";
-import { oneDayMs } from "./garbageCollection";
+import { oneDayMs } from "./garbageCollectionConstants";
 
 /**
  * Feature Gate Key -

package/src/index.ts

@@ -5,7 +5,6 @@
 
 export {
     ContainerMessageType,
-    IChunkedOp,
     ContainerRuntimeMessage,
     IGCRuntimeOptions,
     ISummaryRuntimeOptions,
@@ -17,7 +16,6 @@ export {
     IRootSummaryTreeWithStats,
     isRuntimeMessage,
     RuntimeMessage,
-    unpackRuntimeMessage,
     agentSchedulerId,
     ContainerRuntime,
     RuntimeHeaders,
@@ -27,12 +25,14 @@ export {
     CompressionAlgorithms,
 } from "./containerRuntime";
 export { FluidDataStoreRegistry } from "./dataStoreRegistry";
+export {
+    IGCStats,
+} from "./garbageCollection";
 export {
     gcBlobPrefix,
     gcTombstoneBlobKey,
     gcTreeKey,
-    IGCStats,
-} from "./garbageCollection";
+} from "./garbageCollectionConstants";
 export {
     IPendingFlush,
     IPendingLocalState,
@@ -82,3 +82,4 @@ export {
     SummaryCollection,
 } from "./summaryCollection";
 export { ICancellableSummarizerController, neverCancelledSummaryToken } from "./runWhileConnectedCoordinator";
+export { IChunkedOp, unpackRuntimeMessage } from "./opLifecycle";

package/src/{batchManager.ts → opLifecycle/batchManager.ts}

@@ -3,19 +3,8 @@
  * Licensed under the MIT License.
  */
 
-import { ITelemetryLogger } from "@fluidframework/common-definitions";
-import { IBatchMessage } from "@fluidframework/container-definitions";
-import { ContainerRuntimeMessage, ICompressionRuntimeOptions } from "./containerRuntime";
-import { OpCompressor } from "./opCompressor";
-
-/**
- * Message type used by BatchManager
- */
-export type BatchMessage = IBatchMessage & {
-    localOpMetadata: unknown;
-    deserializedContent: ContainerRuntimeMessage;
-    referenceSequenceNumber: number;
-};
+import { ICompressionRuntimeOptions } from "../containerRuntime";
+import { BatchMessage, IBatch, IBatchCheckpoint } from "./definitions";
 
 export interface IBatchManagerOptions {
     readonly hardLimit: number;
@@ -27,15 +16,13 @@ export interface IBatchManagerOptions {
  * Helper class that manages partial batch & rollback.
  */
 export class BatchManager {
-    private readonly opCompressor: OpCompressor;
-    private pendingBatch: BatchMessage [] = [];
+    private pendingBatch: BatchMessage[] = [];
     private batchContentSize = 0;
 
     public get length() { return this.pendingBatch.length; }
+    public get contentSizeInBytes() { return this.batchContentSize; }
 
-    constructor(public readonly logger: ITelemetryLogger, public readonly options: IBatchManagerOptions) {
-        this.opCompressor = new OpCompressor(logger);
-    }
+    constructor(public readonly options: IBatchManagerOptions) { }
 
     public push(message: BatchMessage): boolean {
         const contentSize = this.batchContentSize + (message.contents?.length ?? 0);
@@ -55,13 +42,11 @@ export class BatchManager {
         // Cases where the message is still too large will be handled by the maxConsecutiveReconnects path.
         if (this.options.softLimit !== undefined
             && this.length > 0
-            && socketMessageSize >= this.options.softLimit
-            && Infinity === (this.options.compressionOptions?.minimumBatchSizeInBytes ?? Infinity)) {
+            && socketMessageSize >= this.options.softLimit) {
             return false;
         }
 
-        if (socketMessageSize >= this.options.hardLimit
-            && Infinity === (this.options.compressionOptions?.minimumBatchSizeInBytes ?? Infinity)) {
+        if (socketMessageSize >= this.options.hardLimit) {
             return false;
         }
 
@@ -72,25 +57,22 @@ export class BatchManager {
 
     public get empty() { return this.pendingBatch.length === 0; }
 
-    public popBatch() {
-        const batch = this.pendingBatch;
-        const size = this.batchContentSize;
+    public popBatch(): IBatch {
+        const batch: IBatch = {
+            content: this.pendingBatch,
+            contentSizeInBytes: this.batchContentSize,
+        };
+
         this.pendingBatch = [];
         this.batchContentSize = 0;
 
-        if (batch.length > 0
-            && this.options.compressionOptions !== undefined
-            && this.options.compressionOptions.minimumBatchSizeInBytes < size) {
-            return this.opCompressor.compressBatch(batch, size);
-        }
-
-        return batch;
+        return addBatchMetadata(batch);
     }
 
     /**
      * Capture the pending state at this point
      */
-    public checkpoint() {
+    public checkpoint(): IBatchCheckpoint {
         const startPoint = this.pendingBatch.length;
         return {
             rollback: (process: (message: BatchMessage) => void) => {
@@ -106,3 +88,18 @@ export class BatchManager {
         };
     }
 }
+
+const addBatchMetadata = (batch: IBatch): IBatch => {
+    if (batch.content.length > 1) {
+        batch.content[0].metadata = {
+            ...batch.content[0].metadata,
+            batch: true
+        };
+        batch.content[batch.content.length - 1].metadata = {
+            ...batch.content[batch.content.length - 1].metadata,
+            batch: false
+        };
+    }
+
+    return batch;
+};

package/src/opLifecycle/definitions.ts

@@ -0,0 +1,44 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { IBatchMessage } from "@fluidframework/container-definitions";
+import { MessageType } from "@fluidframework/protocol-definitions";
+import { CompressionAlgorithms, ContainerMessageType, ContainerRuntimeMessage } from "..";
+
+/**
+ * Batch message type used internally by the runtime
+ */
+export type BatchMessage = IBatchMessage & {
+    localOpMetadata: unknown;
+    deserializedContent: ContainerRuntimeMessage;
+    referenceSequenceNumber: number;
+    compression?: CompressionAlgorithms;
+};
+
+/**
+ * Batch interface used internally by the runtime.
+ */
+export interface IBatch {
+    /**
+     * Sum of the in-memory content sizes of all messages in the batch.
+     * If the batch is compressed, this number reflects the post-compression size.
+     */
+    readonly contentSizeInBytes: number;
+    /**
+     * All the messages in the batch
+     */
+    readonly content: BatchMessage[];
+}
+
+export interface IBatchCheckpoint {
+    rollback: (action: (message: BatchMessage) => void) => void;
+}
+
+export interface IChunkedOp {
+    chunkId: number;
+    totalChunks: number;
+    contents: string;
+    originalType: MessageType | ContainerMessageType;
+}

package/src/opLifecycle/index.ts

@@ -0,0 +1,17 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+export { BatchManager } from "./batchManager";
+export {
+    BatchMessage,
+    IBatch,
+    IBatchCheckpoint,
+    IChunkedOp,
+} from "./definitions";
+export { Outbox } from "./outbox";
+export { OpCompressor } from "./opCompressor";
+export { OpDecompressor } from "./opDecompressor";
+export { OpSplitter } from "./opSplitter";
+export { RemoteMessageProcessor, unpackRuntimeMessage } from "./remoteMessageProcessor";

package/src/{opCompressor.ts → opLifecycle/opCompressor.ts}

@@ -7,8 +7,8 @@ import { ITelemetryLogger } from "@fluidframework/common-definitions";
 import { IsoBuffer } from "@fluidframework/common-utils";
 import { ChildLogger } from "@fluidframework/telemetry-utils";
 import { compress } from "lz4js";
-import { BatchMessage } from "./batchManager";
-import { CompressionAlgorithms, ContainerRuntimeMessage } from "./containerRuntime";
+import { CompressionAlgorithms, ContainerRuntimeMessage } from "../containerRuntime";
+import { IBatch, BatchMessage } from "./definitions";
 
 /**
  * Compresses batches of ops. It generates a single compressed op that contains
@@ -23,37 +23,42 @@ export class OpCompressor {
         this.logger = ChildLogger.create(logger, "OpCompressor");
     }
 
-    public compressBatch(batch: BatchMessage[], originalLength: number): BatchMessage[] {
-        const batchToSend: BatchMessage[] = [];
+    public compressBatch(batch: IBatch): IBatch {
+        const messages: BatchMessage[] = [];
         this.compressedBatchCount++;
-        const batchedContents: ContainerRuntimeMessage[] = [];
-        for (const message of batch) {
-            batchedContents.push(message.deserializedContent);
+        const contentToCompress: ContainerRuntimeMessage[] = [];
+        for (const message of batch.content) {
+            contentToCompress.push(message.deserializedContent);
         }
 
         const compressionStart = Date.now();
-        const contentsAsBuffer = new TextEncoder().encode(JSON.stringify(batchedContents));
+        const contentsAsBuffer = new TextEncoder().encode(JSON.stringify(contentToCompress));
         const compressedContents = compress(contentsAsBuffer);
         const compressedContent = IsoBuffer.from(compressedContents).toString("base64");
         const duration = Date.now() - compressionStart;
 
-        if (originalLength > 200000 || this.compressedBatchCount % 100) {
+        if (batch.contentSizeInBytes > 200000 || this.compressedBatchCount % 25) {
             this.logger.sendPerformanceEvent({
                 eventName: "CompressedBatch",
                 duration,
-                sizeBeforeCompression: originalLength,
+                sizeBeforeCompression: batch.contentSizeInBytes,
                 sizeAfterCompression: compressedContent.length,
             });
         }
 
-        batchToSend.push({ ...batch[0], contents: JSON.stringify({ packedContents: compressedContent }),
-                           metadata: { ...batch[0].metadata, compressed: true },
-                           compression: CompressionAlgorithms.lz4 });
+        messages.push({
+            ...batch.content[0], contents: JSON.stringify({ packedContents: compressedContent }),
+            metadata: { ...batch.content[0].metadata, compressed: true },
+            compression: CompressionAlgorithms.lz4,
+        });
 
-        for (const message of batch.slice(1)) {
-            batchToSend.push({ ...message, contents: undefined });
+        for (const message of batch.content.slice(1)) {
+            messages.push({ ...message, contents: undefined });
         }
 
-        return batchToSend;
+        return {
+            contentSizeInBytes: compressedContent.length,
+            content: messages,
+        };
     }
 }

package/src/{opDecompressor.ts → opLifecycle/opDecompressor.ts}

@@ -6,7 +6,7 @@
 import { decompress } from "lz4js";
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
 import { assert, IsoBuffer, Uint8ArrayToString } from "@fluidframework/common-utils";
-import { CompressionAlgorithms } from ".";
+import { CompressionAlgorithms } from "../containerRuntime";
 
 /**
  * State machine that "unrolls" contents of compressed batches of ops after decompressing them.
@@ -29,11 +29,11 @@ export class OpDecompressor {
         if (message.metadata?.batch === true
             && (message.metadata?.compressed || message.compression !== undefined)) {
             // Beginning of a compressed batch
-            assert(this.activeBatch === false, "shouldn't have multiple active batches");
+            assert(this.activeBatch === false, 0x4b8 /* shouldn't have multiple active batches */);
             if (message.compression) {
                 // lz4 is the only supported compression algorithm for now
                 assert(message.compression === CompressionAlgorithms.lz4,
-                        "lz4 is currently the only supported compression algorithm");
+                    0x4b9 /* lz4 is currently the only supported compression algorithm */);
             }
 
             this.activeBatch = true;
@@ -54,8 +54,10 @@ export class OpDecompressor {
 
         if (this.rootMessageContents !== undefined && message.metadata?.batch === false) {
             // End of compressed batch
-            const returnMessage = { ...message,
-                                    contents: this.rootMessageContents[this.processedCount++] };
+            const returnMessage = {
+                ...message,
+                contents: this.rootMessageContents[this.processedCount++]
+            };
 
             this.activeBatch = false;
             this.rootMessageContents = undefined;
@@ -67,7 +69,7 @@ export class OpDecompressor {
         if (message.metadata?.batch === undefined &&
             (message.metadata?.compressed || message.compression === CompressionAlgorithms.lz4)) {
             // Single compressed message
-            assert(this.activeBatch === false, "shouldn't receive compressed message in middle of a batch");
+            assert(this.activeBatch === false, 0x4ba /* shouldn't receive compressed message in middle of a batch */);
 
             const contents = IsoBuffer.from(message.contents.packedContents, "base64");
             const decompressedMessage = decompress(contents);

package/src/opLifecycle/opSplitter.ts

@@ -0,0 +1,78 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { DataCorruptionError, extractSafePropertiesFromMessage } from "@fluidframework/container-utils";
+import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
+import { ContainerMessageType } from "../containerRuntime";
+import { IChunkedOp } from "./definitions";
+
+/**
+ * Responsible for creating and reconstructing chunked messages.
+ */
+export class OpSplitter {
+    // Local copy of incomplete received chunks.
+    private readonly chunkMap: Map<string, string[]>;
+
+    constructor(chunks: [string, string[]][]) {
+        this.chunkMap = new Map<string, string[]>(chunks);
+    }
+
+    public get chunks(): ReadonlyMap<string, string[]> {
+        return this.chunkMap;
+    }
+
+    public processRemoteMessage(message: ISequencedDocumentMessage): ISequencedDocumentMessage {
+        if (message.type !== ContainerMessageType.ChunkedOp) {
+            return message;
+        }
+
+        const clientId = message.clientId;
+        const chunkedContent = message.contents as IChunkedOp;
+        this.addChunk(clientId, chunkedContent, message);
+
+        if (chunkedContent.chunkId < chunkedContent.totalChunks) {
+            // We are processing the op in chunks but haven't reached
+            // the last chunk yet in order to reconstruct the original op
+            return message;
+        }
+
+        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+        const serializedContent = this.chunkMap.get(clientId)!.join("");
+        this.clearPartialChunks(clientId);
+
+        const newMessage = { ...message };
+        newMessage.contents = serializedContent === "" ? undefined : JSON.parse(serializedContent);
+        newMessage.type = chunkedContent.originalType;
+        return newMessage;
+    }
+
+    public clearPartialChunks(clientId: string) {
+        if (this.chunkMap.has(clientId)) {
+            this.chunkMap.delete(clientId);
+        }
+    }
+
+    private addChunk(clientId: string, chunkedContent: IChunkedOp, originalMessage: ISequencedDocumentMessage) {
+        let map = this.chunkMap.get(clientId);
+        if (map === undefined) {
+            map = [];
+            this.chunkMap.set(clientId, map);
+        }
+
+        if (chunkedContent.chunkId !== map.length + 1) {
+            // We are expecting the chunks to be processed sequentially, in the same order as they are sent.
+            // Therefore, the chunkId of the incoming op needs to match the length of the array (1-based indexing)
+            // holding the existing chunks for that particular clientId.
+            throw new DataCorruptionError("Chunk Id mismatch", {
+                ...extractSafePropertiesFromMessage(originalMessage),
+                chunkMapLength: map.length,
+                chunkId: chunkedContent.chunkId,
+                totalChunks: chunkedContent.totalChunks,
+            });
+        }
+
+        map.push(chunkedContent.contents);
+    }
+}