@fluidframework/container-runtime 0.54.0-47413 → 0.55.0-48551

package/src/garbageCollection.ts

@@ -5,6 +5,7 @@
 
 import { ITelemetryLogger } from "@fluidframework/common-definitions";
 import { assert, LazyPromise, Timer } from "@fluidframework/common-utils";
+import { ICriticalContainerError } from "@fluidframework/container-definitions";
 import {
     cloneGCData,
     concatGarbageCollectionStates,
@@ -18,7 +19,7 @@ import {
     gcBlobKey,
     IGarbageCollectionData,
     IGarbageCollectionState,
-    IGarbageCollectionSummaryDetails,
+    IGarbageCollectionDetailsBase,
     ISummaryTreeWithStats,
 } from "@fluidframework/runtime-definitions";
 import {
@@ -77,6 +78,8 @@ export interface IGCStats {
 
 /** Defines the APIs for the runtime object to be passed to the garbage collector. */
 export interface IGarbageCollectionRuntime {
+    /** Before GC runs, called to notify the runtime to update any pending GC state. */
+    updateStateBeforeGC(): Promise<void>;
     /** Returns the garbage collection data of the runtime. */
     getGCData(fullGC?: boolean): Promise<IGarbageCollectionData>;
     /** After GC has run, called to notify the runtime of routes that are used in it. */
@@ -87,6 +90,8 @@ export interface IGarbageCollectionRuntime {
 export interface IGarbageCollector {
     /** Tells whether GC should run or not. */
     readonly shouldRunGC: boolean;
+    /** The time in ms to expire a session for a client for gc. */
+    readonly sessionExpiryTimeoutMs: number | undefined;
     /**
      * This tracks two things:
      * 1. Whether GC is enabled - If this is 0, GC is disabled. If this is greater than 0, GC is enabled.
@@ -104,11 +109,12 @@ export interface IGarbageCollector {
     /** Summarizes the GC data and returns it as a summary tree. */
     summarize(): ISummaryTreeWithStats | undefined;
     /** Returns a map of each data store id to its GC details in the base summary. */
-    getDataStoreBaseGCDetails(): Promise<Map<string, IGarbageCollectionSummaryDetails>>;
+    getDataStoreBaseGCDetails(): Promise<Map<string, IGarbageCollectionDetailsBase>>;
     /** Called when the latest summary of the system has been refreshed. */
     latestSummaryStateRefreshed(result: RefreshSummaryResult, readAndParseBlob: ReadAndParseBlob): Promise<void>;
     /** Called when a node is changed. Used to detect and log when an inactive node is changed. */
     nodeChanged(id: string): void;
+    dispose(): void;
     /** Called when a reference is added to a node. Used to identify nodes that were referenced between summaries. */
     addedOutboundReference(fromNodeId: string, toNodeId: string): void;
 }
@@ -174,6 +180,7 @@ export class GarbageCollector implements IGarbageCollector {
         gcOptions: IGCRuntimeOptions,
         deleteUnusedRoutes: (unusedRoutes: string[]) => void,
         getCurrentTimestampMs: () => number,
+        closeFn: (error?: ICriticalContainerError) => void,
         baseSnapshot: ISnapshotTree | undefined,
         readAndParseBlob: ReadAndParseBlob,
         baseLogger: ITelemetryLogger,
@@ -185,6 +192,7 @@ export class GarbageCollector implements IGarbageCollector {
             gcOptions,
             deleteUnusedRoutes,
             getCurrentTimestampMs,
+            closeFn,
             baseSnapshot,
             readAndParseBlob,
             baseLogger,
@@ -198,6 +206,11 @@ export class GarbageCollector implements IGarbageCollector {
      */
     public readonly shouldRunGC: boolean;
 
+    /**
+     * The time in ms to expire a session for a client for gc.
+     */
+    public readonly sessionExpiryTimeoutMs: number | undefined;
+
     /**
      * This tracks two things:
      * 1. Whether GC is enabled - If this is 0, GC is disabled. If this is greater than 0, GC is enabled.
@@ -251,11 +264,13 @@ export class GarbageCollector implements IGarbageCollector {
     // Promise when resolved initializes the base state of the nodes from the base summary state.
     private readonly initializeBaseStateP: Promise<void>;
     // The map of data store ids to their GC details in the base summary returned in getDataStoreGCDetails().
-    private readonly dataStoreGCDetailsP: Promise<Map<string, IGarbageCollectionSummaryDetails>>;
+    private readonly dataStoreGCDetailsP: Promise<Map<string, IGarbageCollectionDetailsBase>>;
     // The time after which an unreferenced node can be deleted. Currently, we only set the node's state to expired.
     private readonly deleteTimeoutMs: number;
     // Map of node ids to their unreferenced state tracker.
     private readonly unreferencedNodesState: Map<string, UnreferencedStateTracker> = new Map();
+    // The timeout responsible for closing the container when the session has expired
+    private sessionExpiryTimer?: ReturnType<typeof setTimeout>;
 
     protected constructor(
         private readonly provider: IGarbageCollectionRuntime,
@@ -264,6 +279,7 @@ export class GarbageCollector implements IGarbageCollector {
         private readonly deleteUnusedRoutes: (unusedRoutes: string[]) => void,
         /** Returns the current timestamp to be assigned to nodes that become unreferenced. */
         private readonly getCurrentTimestampMs: () => number,
+        private readonly closeFn: (error?: ICriticalContainerError) => void,
         baseSnapshot: ISnapshotTree | undefined,
         readAndParseBlob: ReadAndParseBlob,
         baseLogger: ITelemetryLogger,
@@ -276,17 +292,35 @@ export class GarbageCollector implements IGarbageCollector {
         this.deleteTimeoutMs = this.gcOptions.deleteTimeoutMs ?? defaultDeleteTimeoutMs;
 
         let prevSummaryGCVersion: number | undefined;
+
         // GC can only be enabled during creation. After that, it can never be enabled again. So, for existing
-        // documents, we get this information from the metadata blob.
+        // documents, we get this information from the metadata blob. Similarly the session timeout should be
+        // consistent across all clients, thus we grab it as well from the metadata blob, and set it once on creation.
         if (existing) {
             prevSummaryGCVersion = getGCVersion(metadata);
             // Existing documents which did not have metadata blob or had GC disabled have version as 0. For all
             // other exsiting documents, GC is enabled.
             this.gcEnabled = prevSummaryGCVersion > 0;
+            this.sessionExpiryTimeoutMs = metadata?.sessionExpiryTimeoutMs;
         } else {
             // For new documents, GC has to be exlicitly enabled via the gcAllowed flag in GC options.
             this.gcEnabled = gcOptions.gcAllowed === true;
+            this.sessionExpiryTimeoutMs = this.gcOptions.gcTestSessionTimeoutMs;
+        }
+
+        // If session expiry is enabled, we need to close the container when the timeout expires
+        if (this.sessionExpiryTimeoutMs !== undefined) {
+            // TODO: Change to ClientSessionExpiredError, issue https://github.com/microsoft/FluidFramework/issues/8605
+            // this.sessionExpiryTimer = setTimeout(() => this.closeFn(
+            //     new ClientSessionExpiredError(`Client expired, it has lasted ${this.sessionExpiryTimeoutMs} ms.`,
+            //     this.sessionExpiryTimeoutMs)),
+            //     this.sessionExpiryTimeoutMs);
+            this.sessionExpiryTimer = setTimeout(() => this.closeFn({
+                errorType: "clientSessionExpiredError",
+                message: `The client has reached the expiry time of ${this.sessionExpiryTimeoutMs} ms.`,
+            }), this.sessionExpiryTimeoutMs);
         }
+
         // For existing document, the latest summary is the one that we loaded from. So, use its GC version as the
         // latest tracked GC version. For new documents, we will be writing the first summary with the current version.
         this.latestSummaryGCVersion = prevSummaryGCVersion ?? this.currentGCVersion;
@@ -302,7 +336,8 @@ export class GarbageCollector implements IGarbageCollector {
         // Whether GC sweep phase should run or not. If this is false, only GC mark phase is run. Can override with
         // localStorage flag.
         this.shouldRunSweep = this.shouldRunGC &&
-            (this.mc.config.getBoolean(runSweepKey) ?? gcOptions.runSweep === true);
+            (this.mc.config.getBoolean(runSweepKey) ?? gcOptions.runSweep === true)
+            && this.sessionExpiryTimer !== undefined;
 
         // Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
         this.testMode = this.mc.config.getBoolean(gcTestModeKey) ?? gcOptions.runGCInTestMode === true;
@@ -339,7 +374,7 @@ export class GarbageCollector implements IGarbageCollector {
                     continue;
                 }
 
-                const gcSummaryDetails = await readAndParseBlob<IGarbageCollectionSummaryDetails>(blobId);
+                const gcSummaryDetails = await readAndParseBlob<IGarbageCollectionDetailsBase>(blobId);
                 // If there are no nodes for this data store, skip it.
                 if (gcSummaryDetails.gcData?.gcNodes === undefined) {
                     continue;
@@ -401,8 +436,8 @@ export class GarbageCollector implements IGarbageCollector {
         });
 
         // Get the GC details for each data store from the GC state in the base summary. This is returned in
-        // getDataStoreBaseGCDetails and is used to initialize each data store's base GC details.
-        this.dataStoreGCDetailsP = new LazyPromise<Map<string, IGarbageCollectionSummaryDetails>>(async () => {
+        // getDataStoreBaseGCDetails and is used to initialize each data store's GC state.
+        this.dataStoreGCDetailsP = new LazyPromise<Map<string, IGarbageCollectionDetailsBase>>(async () => {
             const baseState = await baseSummaryStateP;
             if (baseState === undefined) {
                 return new Map();
@@ -459,6 +494,9 @@ export class GarbageCollector implements IGarbageCollector {
         return PerformanceEvent.timedExecAsync(logger, { eventName: "GarbageCollection" }, async (event) => {
             await this.initializeBaseStateP;
 
+            // Let the runtime update its pending state before GC runs.
+            await this.provider.updateStateBeforeGC();
+
             const gcStats: {
                 deletedNodes?: number,
                 totalNodes?: number,
@@ -530,9 +568,9 @@ export class GarbageCollector implements IGarbageCollector {
 
     /**
      * Returns a map of data store ids to their base GC details generated from the base summary.This is used to
-     * initialize the data stores with their base GC state.
+     * initialize the GC state of data stores.
      */
-    public async getDataStoreBaseGCDetails(): Promise<Map<string, IGarbageCollectionSummaryDetails>> {
+    public async getDataStoreBaseGCDetails(): Promise<Map<string, IGarbageCollectionDetailsBase>> {
         return this.dataStoreGCDetailsP;
     }
 
@@ -574,6 +612,13 @@ export class GarbageCollector implements IGarbageCollector {
         );
     }
 
+    public dispose(): void {
+        if (this.sessionExpiryTimer !== undefined) {
+            clearTimeout(this.sessionExpiryTimer);
+            this.sessionExpiryTimer = undefined;
+        }
+    }
+
     /**
      * Called when an outbound reference is added to a node. This is used to identify all nodes that have been
      * referenced between summaries so that their unreferenced timestamp can be reset.
@@ -662,6 +707,9 @@ export class GarbageCollector implements IGarbageCollector {
             return;
         }
 
+        // Validate that we have identified all references correctly.
+        this.validateReferenceCorrectness(currentGCData);
+
         /**
          * Generate a super set of the GC data that contains the nodes and edges from last run, plus any new node and
          * edges that have been added since then. To do this, combine the GC data from the last run and the current
@@ -702,6 +750,56 @@ export class GarbageCollector implements IGarbageCollector {
             }
         }
     }
+
+    /**
+     * Validates that all new references are correctly identified and processed. The basic principle for validation is
+     * that we should not have new references in the reference graph (GC data) that have not been notified to the
+     * garbage collector via `referenceAdded`.
+     * We validate that the references in the current reference graph should be a subset of the references in the last
+     * run's reference graph + references since the last run.
+     * @param currentGCData - The GC data (reference graph) from the current GC run.
+     */
+    private validateReferenceCorrectness(currentGCData: IGarbageCollectionData) {
+        assert(this.gcDataFromLastRun !== undefined, 0x2b7
+            /* "Can't validate correctness without GC data from last run" */);
+
+        // Get a list of all the outbound routes (or references) in the current GC data.
+        const currentReferences: string[] = [];
+        for (const [nodeId, outboundRoutes] of Object.entries(currentGCData.gcNodes)) {
+            /**
+             * Remove routes from a child node to its parent which is added implicitly by the runtime. For instance,
+             * each adds its data store as an outbound route to mark it as referenced if the DDS is referenced.
+             * We won't get any explicit notification for these references so they must be removed before validation.
+             */
+            const explicitRoutes = outboundRoutes.filter((route) => !nodeId.startsWith(route));
+            currentReferences.push(...explicitRoutes);
+        }
+
+        // Get a list of outbound routes (or references) from the last run's GC data plus references added since the
+        // last run that were notified via `referenceAdded`.
+        const explicitReferences: string[] = [];
+        for (const [, outboundRoutes] of Object.entries(this.gcDataFromLastRun.gcNodes)) {
+            explicitReferences.push(...outboundRoutes);
+        }
+        this.referencesSinceLastRun.forEach((outboundRoutes: string[]) => {
+            explicitReferences.push(...outboundRoutes);
+        });
+
+        // Validate that the current reference graph doesn't have references that we are not already aware of. If this
+        // happens, it might indicate data corruption since we may delete objects prematurely.
+        currentReferences.forEach((route: string) => {
+            // Validate references for data stores only whose routes are of the format "/dataStoreId". Currently, layers
+            // below data stores don't have GC implemented so there is no guarantee their references will be notified.
+            if (route.split("/").length === 2 && !explicitReferences.includes(route)) {
+                // We should ideally throw a data corruption error here. However, send an error for now until we have
+                // implemented sweep and have reasonable confidence in the sweep process.
+                this.mc.logger.sendErrorEvent({
+                    eventName: "gcUnknownOutboundRoute",
+                    route,
+                });
+            }
+        });
+    }
 }
 
 /**

package/src/index.ts

@@ -11,6 +11,7 @@ export {
     ISummaryRuntimeOptions,
     IContainerRuntimeOptions,
     isRuntimeMessage,
+    RuntimeMessage,
     unpackRuntimeMessage,
     ScheduleManager,
     agentSchedulerId,

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "0.54.0-47413";
+export const pkgVersion = "0.55.0-48551";

package/src/summaryFormat.ts

@@ -69,7 +69,6 @@ export function getAttributesFormatVersion(attributes: ReadFluidDataStoreAttribu
     return 0;
 }
 
-// eslint-disable-next-line prefer-arrow/prefer-arrow-functions
 export function hasIsolatedChannels(attributes: ReadFluidDataStoreAttributes): boolean {
     return !!attributes.summaryFormatVersion && !attributes.disableIsolatedChannels;
 }
@@ -85,6 +84,8 @@ export interface IContainerRuntimeMetadata extends ICreateContainerMetadata {
     readonly gcFeature?: GCVersion;
     /** Counter of the last summary happened, increments every time we summarize */
     readonly summaryCount?: number;
+    /** If this is present, the session for this container will expire after this time and the container will close */
+    readonly sessionExpiryTimeoutMs?: number;
 }
 
 export interface ICreateContainerMetadata {
@@ -120,7 +121,6 @@ export const extractSummaryMetadataMessage = (
     type: message.type,
 };
 
-// eslint-disable-next-line prefer-arrow/prefer-arrow-functions
 export function getMetadataFormatVersion(metadata?: IContainerRuntimeMetadata): number {
     /**
      * Version 2+: Introduces runtime sequence number for data verification.
@@ -141,7 +141,6 @@ export const chunksBlobName = ".chunks";
 export const electedSummarizerBlobName = ".electedSummarizer";
 export const blobsTreeName = ".blobs";
 
-// eslint-disable-next-line prefer-arrow/prefer-arrow-functions
 export function rootHasIsolatedChannels(metadata?: IContainerRuntimeMetadata): boolean {
     return !!metadata && !metadata.disableIsolatedChannels;
 }

package/src/summaryManager.ts

@@ -141,6 +141,9 @@ export class SummaryManager extends TypedEventEmitter<ISummaryManagerEvents> imp
         this.refreshSummarizer();
     };
 
+    private static readonly isStartingOrRunning = (state: SummaryManagerState) =>
+        state === SummaryManagerState.Starting || state === SummaryManagerState.Running;
+
     private getShouldSummarizeState(): ShouldSummarizeState {
         if (!this.connectedState.connected) {
             return { shouldSummarize: false, stopReason: "parentNotConnected" };
@@ -253,7 +256,9 @@ export class SummaryManager extends TypedEventEmitter<ISummaryManagerEvents> imp
                 // Note that summarizer may keep going (like doing last summary).
                 // Ideally we await stopping process, but this code path is due to a bug
                 // that needs to be fixed either way.
-                this.stop("summarizerException");
+                if (SummaryManager.isStartingOrRunning(this.state)) {
+                    this.stop("summarizerException");
+                }
             }
         }).finally(() => {
             assert(this.state !== SummaryManagerState.Off, 0x264 /* "Expected: Not Off" */);
@@ -273,8 +278,9 @@ export class SummaryManager extends TypedEventEmitter<ISummaryManagerEvents> imp
     }
 
     private stop(reason: SummarizerStopReason) {
-        assert(this.state === SummaryManagerState.Running || this.state === SummaryManagerState.Starting,
-            0x265 /* "Expected: Starting or Running" */);
+        if (!SummaryManager.isStartingOrRunning(this.state)) {
+            return;
+        }
         this.state = SummaryManagerState.Stopping;
 
         // Stopping the running summarizer client should trigger a change