@fluidframework/container-runtime 0.56.7 → 0.57.1

package/src/garbageCollection.ts

@@ -6,7 +6,8 @@
 import { ITelemetryLogger } from "@fluidframework/common-definitions";
 import { assert, LazyPromise, Timer } from "@fluidframework/common-utils";
 import { ICriticalContainerError } from "@fluidframework/container-definitions";
-import { ClientSessionExpiredError } from "@fluidframework/container-utils";
+import { ClientSessionExpiredError, DataProcessingError } from "@fluidframework/container-utils";
+import { IRequestHeader } from "@fluidframework/core-interfaces";
 import {
     cloneGCData,
     concatGarbageCollectionStates,
@@ -33,7 +34,9 @@ import {
     loggerToMonitoringContext,
     MonitoringContext,
     PerformanceEvent,
+    TelemetryDataTag,
  } from "@fluidframework/telemetry-utils";
+import { RuntimeHeaders } from ".";
 
 import { IGCRuntimeOptions } from "./containerRuntime";
 import { getSummaryForDatastores } from "./dataStores";
@@ -62,23 +65,39 @@ const gcTestModeKey = "Fluid.GarbageCollection.GCTestMode";
 const runSweepKey = "Fluid.GarbageCollection.RunSweep";
 // Feature gate key to write GC data at the root of the summary tree.
 const writeAtRootKey = "Fluid.GarbageCollection.WriteDataAtRoot";
+// Feature gate key to expire a session after a set period of time.
+const runSessionExpiry = "Fluid.GarbageCollection.RunSessionExpiry";
 
 const defaultDeleteTimeoutMs = 7 * 24 * 60 * 60 * 1000; // 7 days
-
-/** The used state statistics of a node. */
-export interface IUsedStateStats {
-    totalNodeCount: number;
-    unusedNodeCount: number;
-}
+const defaultSessionExpiryDurationMs = 30 * 24 * 60 * 60 * 1000; // 30 days
 
 /** The statistics of the system state after a garbage collection run. */
 export interface IGCStats {
-    totalNodes: number;
-    deletedNodes: number;
-    totalDataStores: number;
-    deletedDataStores: number;
+    /** The number of nodes in the container. */
+    nodeCount: number;
+    /** The number of data stores in the container. */
+    dataStoreCount: number;
+    /** The number of unreferenced nodes in the container. */
+    unrefNodeCount: number;
+    /** The number of unreferenced data stores in the container. */
+    unrefDataStoreCount: number;
+    /** The number of nodes whose reference state updated since last GC run. */
+    updatedNodeCount: number;
+    /** The number of data stores whose reference state updated since last GC run. */
+    updatedDataStoreCount: number;
 }
 
+/** The event that is logged when unreferenced node is used after a certain time. */
+interface IUnreferencedEvent {
+    eventName: string;
+    id: string;
+    age: number;
+    timeout: number;
+    lastSummaryTime?: number;
+    externalRequest?: boolean;
+    viaHandle?: boolean;
+};
+
 /** Defines the APIs for the runtime object to be passed to the garbage collector. */
 export interface IGarbageCollectionRuntime {
     /** Before GC runs, called to notify the runtime to update any pending GC state. */
@@ -86,7 +105,9 @@ export interface IGarbageCollectionRuntime {
     /** Returns the garbage collection data of the runtime. */
     getGCData(fullGC?: boolean): Promise<IGarbageCollectionData>;
     /** After GC has run, called to notify the runtime of routes that are used in it. */
-    updateUsedRoutes(usedRoutes: string[], gcTimestamp?: number): IUsedStateStats;
+    updateUsedRoutes(usedRoutes: string[], gcTimestamp?: number): void;
+    /** Called when the runtime should close because of an error. */
+    closeFn(error?: ICriticalContainerError): void;
 }
 
 /** Defines the contract for the garbage collector. */
@@ -115,10 +136,16 @@ export interface IGarbageCollector {
     getDataStoreBaseGCDetails(): Promise<Map<string, IGarbageCollectionDetailsBase>>;
     /** Called when the latest summary of the system has been refreshed. */
     latestSummaryStateRefreshed(result: RefreshSummaryResult, readAndParseBlob: ReadAndParseBlob): Promise<void>;
-    /** Called when a node is changed. Used to detect and log when an inactive node is changed. */
-    nodeChanged(id: string): void;
+    /** Called when a node is updated. Used to detect and log when an inactive node is changed or loaded. */
+    nodeUpdated(
+        nodePath: string,
+        reason: "Loaded" | "Changed",
+        timestampMs?: number,
+        packagePath?: readonly string[],
+        requestHeaders?: IRequestHeader,
+    ): void;
     /** Called when a reference is added to a node. Used to identify nodes that were referenced between summaries. */
-    addedOutboundReference(fromNodeId: string, toNodeId: string): void;
+    addedOutboundReference(fromNodePath: string, toNodePath: string): void;
     dispose(): void;
 }
 
@@ -127,49 +154,51 @@ export interface IGarbageCollector {
  * the node's state to inactive if it remains unreferenced for a given amount of time (inactiveTimeoutMs).
  */
 class UnreferencedStateTracker {
-    private inactive: boolean = false;
-    // Keeps track of all inactive events that are logged. This is used to limit the log generation for each event to 1
-    // so that it is not noisy.
-    private readonly inactiveEventsLogged: Set<string> = new Set();
-    private readonly timer: Timer | undefined;
+    private _inactive: boolean = false;
+    public get inactive(): boolean {
+        return this._inactive;
+    }
+
+    private timer: Timer | undefined;
 
     constructor(
         public readonly unreferencedTimestampMs: number,
-        inactiveTimeoutMs: number,
+        private readonly inactiveTimeoutMs: number,
+        currentReferenceTimestampMs?: number,
     ) {
-        // If the timeout has already expired, the node should become inactive immediately. Otherwise, start a timer of
-        // inactiveTimeoutMs after which the node will become inactive.
-        if (inactiveTimeoutMs <= 0) {
-            this.inactive = true;
-        } else {
-            this.timer = new Timer(inactiveTimeoutMs, () => { this.inactive = true; });
-            this.timer.start();
+        // If there is no current reference timestamp, don't track the node's inactive state. This will happen later
+        // when updateTracking is called with a reference timestamp.
+        if (currentReferenceTimestampMs !== undefined) {
+            this.updateTracking(currentReferenceTimestampMs);
         }
     }
 
-    /** Stop tracking this node. Reset the unreferenced timer, if any, and reset inactive state. */
-    public stopTracking() {
-        this.timer?.clear();
-        this.inactive = false;
-    }
+    /**
+     * Updates the tracking state based on the provided timestamp.
+     */
+    public updateTracking(currentReferenceTimestampMs: number) {
+        const unreferencedDurationMs = currentReferenceTimestampMs - this.unreferencedTimestampMs;
+        // If the timeout has already expired, the node has become inactive.
+        if (unreferencedDurationMs > this.inactiveTimeoutMs) {
+            this._inactive = true;
+            this.timer?.clear();
+            return;
+        }
 
-    /** Logs an error with the given properties if the node  is inactive. */
-    public logIfInactive(
-        logger: ITelemetryLogger,
-        eventName: string,
-        currentTimestampMs: number,
-        deleteTimeoutMs: number,
-        inactiveNodeId: string,
-    ) {
-        if (this.inactive && !this.inactiveEventsLogged.has(eventName)) {
-            logger.sendErrorEvent({
-                eventName,
-                age: currentTimestampMs - this.unreferencedTimestampMs,
-                timeout: deleteTimeoutMs,
-                id: inactiveNodeId,
-            });
-            this.inactiveEventsLogged.add(eventName);
+        // The node isn't inactive yet. Restart a timer for the duration remaining for it to become inactive.
+        const remainingDurationMs = this.inactiveTimeoutMs - unreferencedDurationMs; 
+        if (this.timer === undefined) {
+            this.timer = new Timer(remainingDurationMs, () => { this._inactive = true; });
         }
+        this.timer.restart(remainingDurationMs);
+    }
+
+    /**
+     * Stop tracking this node. Reset the unreferenced timer, if any, and reset inactive state.
+     */
+    public stopTracking() {
+        this.timer?.clear();
+        this._inactive = false;
     }
 }
 
@@ -182,8 +211,9 @@ export class GarbageCollector implements IGarbageCollector {
         provider: IGarbageCollectionRuntime,
         gcOptions: IGCRuntimeOptions,
         deleteUnusedRoutes: (unusedRoutes: string[]) => void,
-        getCurrentTimestampMs: () => number,
-        closeFn: (error?: ICriticalContainerError) => void,
+        getNodePackagePath: (nodeId: string) => readonly string[] | undefined,
+        getCurrentReferenceTimestampMs: () => number | undefined,
+        getLastSummaryTimestampMs: () => number | undefined,
         baseSnapshot: ISnapshotTree | undefined,
         readAndParseBlob: ReadAndParseBlob,
         baseLogger: ITelemetryLogger,
@@ -194,8 +224,9 @@ export class GarbageCollector implements IGarbageCollector {
             provider,
             gcOptions,
             deleteUnusedRoutes,
-            getCurrentTimestampMs,
-            closeFn,
+            getNodePackagePath,
+            getCurrentReferenceTimestampMs,
+            getLastSummaryTimestampMs,
             baseSnapshot,
             readAndParseBlob,
             baseLogger,
@@ -287,14 +318,27 @@ export class GarbageCollector implements IGarbageCollector {
     // The timeout responsible for closing the container when the session has expired
     private sessionExpiryTimer?: ReturnType<typeof setTimeout>;
 
+    // Keeps track of unreferenced events that are logged for a node. This is used to limit the log generation to one
+    // per event per node.
+    private readonly loggedUnreferencedEvents: Set<string> = new Set();
+    // Queue for unreferenced events that should be logged the next time GC runs.
+    private pendingEventsQueue: IUnreferencedEvent[] = [];
+
     protected constructor(
         private readonly provider: IGarbageCollectionRuntime,
         private readonly gcOptions: IGCRuntimeOptions,
         /** After GC has run, called to delete objects in the runtime whose routes are unused. */
         private readonly deleteUnusedRoutes: (unusedRoutes: string[]) => void,
-        /** Returns the current timestamp to be assigned to nodes that become unreferenced. */
-        private readonly getCurrentTimestampMs: () => number,
-        private readonly closeFn: (error?: ICriticalContainerError) => void,
+        /** For a given node path, returns the node's package path. */
+        private readonly getNodePackagePath: (nodePath: string) => readonly string[] | undefined,
+        /**
+         * Returns a referenced timestamp to be used to track unreferenced nodes. This is a server generated timestamp
+         * and may not be available if there aren't any ops processed yet. If so, we skip tracking unreferenced state
+         * such as time when node becomes unreferenced or inactive.
+         */
+        private readonly getCurrentReferenceTimestampMs: () => number | undefined,
+        /** Returns the timestamp of the last summary generated for this container. */
+        private readonly getLastSummaryTimestampMs: () => number | undefined,
         baseSnapshot: ISnapshotTree | undefined,
         readAndParseBlob: ReadAndParseBlob,
         baseLogger: ITelemetryLogger,
@@ -320,14 +364,22 @@ export class GarbageCollector implements IGarbageCollector {
         } else {
             // For new documents, GC has to be exlicitly enabled via the gcAllowed flag in GC options.
             this.gcEnabled = gcOptions.gcAllowed === true;
-            this.sessionExpiryTimeoutMs = this.gcOptions.gcTestSessionTimeoutMs;
+            // Set the Session Expiry only if the flag is enabled or the test option is set.
+            if (this.mc.config.getBoolean(runSessionExpiry) && this.gcEnabled) {
+                this.sessionExpiryTimeoutMs = defaultSessionExpiryDurationMs;
+            }
         }
 
         // If session expiry is enabled, we need to close the container when the timeout expires
         if (this.sessionExpiryTimeoutMs !== undefined) {
-            const expiryMs = this.sessionExpiryTimeoutMs;
-            this.sessionExpiryTimer = setTimeout(() => this.closeFn(
-                new ClientSessionExpiredError(`Client session expired.`, expiryMs)), expiryMs);
+            const timeoutMs = this.sessionExpiryTimeoutMs;
+            setLongTimeout(timeoutMs,
+                () => {
+                    this.provider.closeFn(new ClientSessionExpiredError(`Client session expired.`, timeoutMs));
+                },
+                (timer) => {
+                    this.sessionExpiryTimer = timer;
+                });
         }
 
         // For existing document, the latest summary is the one that we loaded from. So, use its GC version as the
@@ -425,27 +477,27 @@ export class GarbageCollector implements IGarbageCollector {
             return Object.keys(gcState.gcNodes).length === 1 ? undefined : gcState;
         });
 
-        // Set up the initializer which initializes the base GC state from the base snapshot. Use lazy promise because
-        // we only do this once - the very first time we run GC.
+        /**
+         * Set up the initializer which initializes the base GC state from the base snapshot. Note that the reference
+         * timestamp maybe from old ops which were not summarized and stored in the file. So, the unreferenced state
+         * may be out of date. This is fine because the state is updated every time GC runs based on the time then.
+         */
         this.initializeBaseStateP = new LazyPromise<void>(async () => {
-            const currentTimestampMs = this.getCurrentTimestampMs();
-            const baseState =  await baseSummaryStateP;
+            const currentReferenceTimestampMs = this.getCurrentReferenceTimestampMs();
+            const baseState = await baseSummaryStateP;
             if (baseState === undefined) {
                 return;
             }
 
             const gcNodes: { [ id: string ]: string[] } = {};
             for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {
-                const unreferencedTimestampMs = nodeData.unreferencedTimestampMs;
-                if (unreferencedTimestampMs !== undefined) {
-                    // Get how long it has been since the node was unreferenced. Start a timeout for the remaining time
-                    // left for it to be eligible for deletion.
-                    const unreferencedDurationMs = currentTimestampMs - unreferencedTimestampMs;
+                if (nodeData.unreferencedTimestampMs !== undefined) {
                     this.unreferencedNodesState.set(
                         nodeId,
                         new UnreferencedStateTracker(
-                            unreferencedTimestampMs,
-                            this.deleteTimeoutMs - unreferencedDurationMs,
+                            nodeData.unreferencedTimestampMs,
+                            this.deleteTimeoutMs,
+                            currentReferenceTimestampMs,
                         ),
                     );
                 }
@@ -488,6 +540,24 @@ export class GarbageCollector implements IGarbageCollector {
             }
             return dataStoreGCDetailsMap;
         });
+
+        // Initialize the base state. The base GC data is used to detect and log when inactive / deleted objects are
+        // used in the container.
+        if (this.shouldRunGC) {
+            this.initializeBaseStateP.catch((error) => {
+                throw new DataProcessingError(
+                    error?.message,
+                    "FailedToInitializeGC",
+                    {
+                        gcEnabled: this.gcEnabled,
+                        runSweep: this.shouldRunSweep,
+                        writeAtRoot: this._writeDataAtRoot,
+                        testMode: this.testMode,
+                        sessionExpiry: this.sessionExpiryTimeoutMs,
+                    },
+                );
+            });
+        }
     }
 
     /**
@@ -516,48 +586,36 @@ export class GarbageCollector implements IGarbageCollector {
             // Let the runtime update its pending state before GC runs.
             await this.provider.updateStateBeforeGC();
 
-            const gcStats: {
-                deletedNodes?: number,
-                totalNodes?: number,
-                deletedDataStores?: number,
-                totalDataStores?: number,
-            } = {};
-
             // Get the runtime's GC data and run GC on the reference graph in it.
             const gcData = await this.provider.getGCData(fullGC);
-
-            this.updateStateSinceLatestRun(gcData);
-
             const gcResult = runGarbageCollection(
                 gcData.gcNodes,
                 [ "/" ],
                 logger,
             );
+            const gcStats = this.generateStatsAndLogEvents(gcResult);
+
+            // Update the state since the last GC run. There can be nodes that were referenced between the last and
+            // the current run. We need to identify than and update their unreferenced state if needed.
+            this.updateStateSinceLastRun(gcData);
 
-            const currentTimestampMs = this.getCurrentTimestampMs();
             // Update the current state of the system based on the GC run.
-            this.updateCurrentState(gcData, gcResult, currentTimestampMs);
+            const currentReferenceTimestampMs = this.getCurrentReferenceTimestampMs();
+            this.updateCurrentState(gcData, gcResult, currentReferenceTimestampMs);
 
-            const dataStoreUsedStateStats =
-                this.provider.updateUsedRoutes(gcResult.referencedNodeIds, currentTimestampMs);
+            this.provider.updateUsedRoutes(gcResult.referencedNodeIds, currentReferenceTimestampMs);
 
             if (runSweep) {
                 // Placeholder for running sweep logic.
             }
 
-            // Update stats to be reported in the peformance event.
-            gcStats.deletedNodes = gcResult.deletedNodeIds.length;
-            gcStats.totalNodes = gcResult.referencedNodeIds.length + gcResult.deletedNodeIds.length;
-            gcStats.deletedDataStores = dataStoreUsedStateStats.unusedNodeCount;
-            gcStats.totalDataStores = dataStoreUsedStateStats.totalNodeCount;
-
             // If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios
             // involving access to deleted data.
             if (this.testMode) {
                 this.deleteUnusedRoutes(gcResult.deletedNodeIds);
             }
-            event.end(gcStats);
-            return gcStats as IGCStats;
+            event.end({ ...gcStats });
+            return gcStats;
         },
         { end: true, cancel: "error" });
     }
@@ -621,38 +679,61 @@ export class GarbageCollector implements IGarbageCollector {
     }
 
     /**
-     * Called when a node with the given id is changed. If the node is inactive, log an error.
+     * Called when a node with the given id is updated. If the node is inactive, log an error.
+     * @param nodePath - The id of the node that changed.
+     * @param reason - Whether the node was loaded or changed.
+     * @param timestampMs - The timestamp when the node changed.
+     * @param packagePath - The package path of the node. This may not be available if the node hasn't been loaded yet.
+     * @param requestHeaders - If the node was loaded via request path, the headers in the request.
      */
-    public nodeChanged(id: string) {
-        // Prefix "/" if needed to make it relative to the root.
-        const nodeId = id.startsWith("/") ? id : `/${id}`;
-        this.unreferencedNodesState.get(nodeId)?.logIfInactive(
-            this.mc.logger,
-            "inactiveObjectChanged",
-            this.getCurrentTimestampMs(),
-            this.deleteTimeoutMs,
-            nodeId,
-        );
-    }
-
-    public dispose(): void {
-        if (this.sessionExpiryTimer !== undefined) {
-            clearTimeout(this.sessionExpiryTimer);
-            this.sessionExpiryTimer = undefined;
+    public nodeUpdated(
+        nodePath: string,
+        reason: "Loaded" | "Changed",
+        timestampMs?: number,
+        packagePath?: readonly string[],
+        requestHeaders?: IRequestHeader,
+    ) {
+        if (!this.shouldRunGC) {
+            return;
         }
+
+        this.logIfInactive(
+            reason,
+            nodePath,
+            timestampMs,
+            packagePath,
+            requestHeaders,
+        );
     }
 
     /**
      * Called when an outbound reference is added to a node. This is used to identify all nodes that have been
      * referenced between summaries so that their unreferenced timestamp can be reset.
      *
-     * @param fromNodeId - The node from which the reference is added.
-     * @param toNodeId - The node to which the reference is added.
+     * @param fromNodePath - The node from which the reference is added.
+     * @param toNodePath - The node to which the reference is added.
      */
-    public addedOutboundReference(fromNodeId: string, toNodeId: string) {
-        const outboundRoutes = this.referencesSinceLastRun.get(fromNodeId) ?? [];
-        outboundRoutes.push(toNodeId);
-        this.referencesSinceLastRun.set(fromNodeId, outboundRoutes);
+    public addedOutboundReference(fromNodePath: string, toNodePath: string) {
+        if (!this.shouldRunGC) {
+            return;
+        }
+
+        const outboundRoutes = this.referencesSinceLastRun.get(fromNodePath) ?? [];
+        outboundRoutes.push(toNodePath);
+        this.referencesSinceLastRun.set(fromNodePath, outboundRoutes);
+
+        // If the node that got referenced is inactive, log an event as that may indicate use-after-delete.
+        this.logIfInactive(
+            "Revived",
+            toNodePath,
+        );
+    }
+
+    public dispose(): void {
+        if (this.sessionExpiryTimer !== undefined) {
+            clearTimeout(this.sessionExpiryTimer);
+            this.sessionExpiryTimer = undefined;
+        }
     }
 
     /**
@@ -673,47 +754,56 @@ export class GarbageCollector implements IGarbageCollector {
      * 3. Clears tracking for nodes that were unreferenced but became referenced in this run.
      * @param gcData - The data representing the reference graph on which GC is run.
      * @param gcResult - The result of the GC run on the gcData.
-     * @param currentTimestampMs - The current timestamp to be used for unreferenced nodes' timestamp.
+     * @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
      */
-    private updateCurrentState(gcData: IGarbageCollectionData, gcResult: IGCResult, currentTimestampMs: number) {
+    private updateCurrentState(
+        gcData: IGarbageCollectionData,
+        gcResult: IGCResult,
+        currentReferenceTimestampMs?: number,
+    ) {
         this.gcDataFromLastRun = cloneGCData(gcData);
         this.referencesSinceLastRun.clear();
 
-        // Iterate through the deleted nodes and start tracking if they became unreferenced in this run.
-        for (const nodeId of gcResult.deletedNodeIds) {
-            // The time when the node became unreferenced. This is added to the current GC state.
-            let unreferencedTimestampMs: number = currentTimestampMs;
-            const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
-            if (nodeStateTracker !== undefined) {
-                unreferencedTimestampMs = nodeStateTracker.unreferencedTimestampMs;
-            } else {
-                // Start tracking this node as it became unreferenced in this run.
-                this.unreferencedNodesState.set(
-                    nodeId,
-                    new UnreferencedStateTracker(unreferencedTimestampMs, this.deleteTimeoutMs),
-                );
-            }
-        }
-
         // Iterate through the referenced nodes and stop tracking if they were unreferenced before.
         for (const nodeId of gcResult.referencedNodeIds) {
             const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
             if (nodeStateTracker !== undefined) {
-                // If this node has been unreferenced for longer than deleteTimeoutMs and is being referenced,
-                // log an error as this may mean the deleteTimeoutMs is not long enough.
-                nodeStateTracker.logIfInactive(
-                    this.mc.logger,
-                    "inactiveObjectRevived",
-                    currentTimestampMs,
-                    this.deleteTimeoutMs,
-                    nodeId,
-                );
                 // Stop tracking so as to clear out any running timers.
                 nodeStateTracker.stopTracking();
                 // Delete the node as we don't need to track it any more.
                 this.unreferencedNodesState.delete(nodeId);
             }
         }
+
+        /**
+         * If there is no current reference time, skip tracking when a node becomes unreferenced. This would happen
+         * if no ops have been processed ever and we still try to run GC. If so, there is nothing interesting to track
+         * anyway.
+         */
+        if (currentReferenceTimestampMs === undefined) {
+            return;
+        }
+
+        /**
+         * If a node became unreferenced in this run, start tracking it.
+         * If a node was already unreferenced, update its tracking information. Since the current reference time is
+         * from the ops seen, this will ensure that we keep updating the unreferenced state as time moves forward.
+         */
+        for (const nodeId of gcResult.deletedNodeIds) {
+            const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
+            if (nodeStateTracker === undefined) {
+                this.unreferencedNodesState.set(
+                    nodeId,
+                    new UnreferencedStateTracker(
+                        currentReferenceTimestampMs,
+                        this.deleteTimeoutMs,
+                        currentReferenceTimestampMs,
+                    ),
+                );
+            } else {
+                nodeStateTracker.updateTracking(currentReferenceTimestampMs);
+            }
+        }
     }
 
     /**
@@ -724,7 +814,7 @@ export class GarbageCollector implements IGarbageCollector {
      * This function identifies nodes that were referenced since last run and removes their unreferenced state, if any.
      * If these nodes are currently unreferenced, they will be assigned new unreferenced state by the current run.
      */
-    private updateStateSinceLatestRun(currentGCData: IGarbageCollectionData) {
+    private updateStateSinceLastRun(currentGCData: IGarbageCollectionData) {
         // If we haven't run GC before or no references were added since the last run, there is nothing to do.
         if (this.gcDataFromLastRun === undefined || this.referencesSinceLastRun.size === 0) {
             return;
@@ -783,8 +873,10 @@ export class GarbageCollector implements IGarbageCollector {
      * @param currentGCData - The GC data (reference graph) from the current GC run.
      */
     private validateReferenceCorrectness(currentGCData: IGarbageCollectionData) {
-        assert(this.gcDataFromLastRun !== undefined, 0x2b7
-            /* "Can't validate correctness without GC data from last run" */);
+        assert(
+            this.gcDataFromLastRun !== undefined,
+            0x2b7, /* "Can't validate correctness without GC data from last run" */
+        );
 
         // Get a list of all the outbound routes (or references) in the current GC data.
         const currentReferences: string[] = [];
@@ -811,9 +903,9 @@ export class GarbageCollector implements IGarbageCollector {
         // Validate that the current reference graph doesn't have references that we are not already aware of. If this
         // happens, it might indicate data corruption since we may delete objects prematurely.
         currentReferences.forEach((route: string) => {
-            // Validate references for data stores only whose routes are of the format "/dataStoreId". Currently, layers
-            // below data stores don't have GC implemented so there is no guarantee their references will be notified.
-            if (route.split("/").length === 2 && !explicitReferences.includes(route)) {
+            // Validate references for data stores only. Currently, layers below data stores don't have GC implemented
+            // so there is no guarantee their references will be notified.
+            if (isDataStoreNode(route) && !explicitReferences.includes(route)) {
                 /**
                  * The following log will be enabled once this issue is resolved:
                  * https://github.com/microsoft/FluidFramework/issues/8878.
@@ -827,12 +919,123 @@ export class GarbageCollector implements IGarbageCollector {
             }
         });
     }
+
+    /**
+     * Generates the stats of a garbage collection run from the given results of the run. Also, logs any pending events
+     * in the pendingEventsQueue.
+     * @param gcResult - The result of a GC run.
+     * @returns the GC stats of the GC run.
+     */
+    private generateStatsAndLogEvents(gcResult: IGCResult): IGCStats {
+        // Log pending events for unreferenced nodes after GC has run. We should have the package data available for
+        // them now since the GC run should have loaded these nodes.
+        let event = this.pendingEventsQueue.shift();
+        while (event !== undefined) {
+            const pkg = this.getNodePackagePath(event.id);
+            this.mc.logger.sendErrorEvent({
+                ...event,
+                pkg: pkg ? { value: `/${pkg.join("/")}`, tag: TelemetryDataTag.PackageData } : undefined,
+            });
+            event = this.pendingEventsQueue.shift();
+        }
+
+        const gcStats: IGCStats = {
+            nodeCount: 0,
+            dataStoreCount: 0,
+            unrefNodeCount: 0,
+            unrefDataStoreCount: 0,
+            updatedNodeCount: 0,
+            updatedDataStoreCount: 0,
+        };
+
+        for (const nodeId of gcResult.referencedNodeIds) {
+            gcStats.nodeCount++;
+            const isDataStore = isDataStoreNode(nodeId);
+            if (isDataStore) {
+                gcStats.dataStoreCount++;
+            }
+            // If a referenced node has an entry in `unreferencedNodesState`, it was previously unreferenced. So, its
+            // reference state updated from the last GC run.
+            if (this.unreferencedNodesState.has(nodeId)) {
+                gcStats.updatedNodeCount++;
+                if (isDataStore) {
+                    gcStats.updatedDataStoreCount++;
+                }
+            }
+        }
+
+        for (const nodeId of gcResult.deletedNodeIds) {
+            gcStats.nodeCount++;
+            gcStats.unrefNodeCount++;
+            const isDataStore = isDataStoreNode(nodeId);
+            if (isDataStore) {
+                gcStats.dataStoreCount++;
+                gcStats.unrefDataStoreCount++;
+            }
+            // If an unreferenced node doesn't an entry in `unreferencedNodesState`, it was previously referenced. So,
+            // its reference state updated from the last GC run.
+            if (!this.unreferencedNodesState.has(nodeId)) {
+                gcStats.updatedNodeCount++;
+                if (isDataStore) {
+                    gcStats.updatedDataStoreCount++;
+                }
+            }
+        }
+
+        return gcStats;
+    }
+
+    /**
+     * Logs an event if a node is inactive and is used.
+     */
+    private logIfInactive(
+        eventSuffix: "Changed" | "Loaded" | "Revived",
+        nodeId: string,
+        currentReferenceTimestampMs = this.getCurrentReferenceTimestampMs(),
+        packagePath?: readonly string[],
+        requestHeaders?: IRequestHeader,
+    ) {
+        // If there is no reference timestamp to work with, no ops have been processed after creation. If so, skip
+        // logging as nothing interesting would have happened worth logging.
+        if (currentReferenceTimestampMs === undefined) {
+            return;
+        }
+
+        const eventName = `inactiveObject_${eventSuffix}`;
+        // We log a particular event for a given node only once so that it is not too noisy.
+        const uniqueEventId = `${nodeId}-${eventName}`;
+        const nodeState = this.unreferencedNodesState.get(nodeId);
+        if (nodeState?.inactive && !this.loggedUnreferencedEvents.has(uniqueEventId)) {
+            this.loggedUnreferencedEvents.add(uniqueEventId);
+            const event: IUnreferencedEvent = {
+                eventName,
+                id: nodeId,
+                age: currentReferenceTimestampMs - nodeState.unreferencedTimestampMs,
+                timeout: this.deleteTimeoutMs,
+                lastSummaryTime: this.getLastSummaryTimestampMs(),
+                externalRequest: requestHeaders?.[RuntimeHeaders.externalRequest],
+                viaHandle: requestHeaders?.[RuntimeHeaders.viaHandle],
+            };
+
+            // If the package data for the node exists, log immediately. Otherwise, queue it and it will be logged the
+            // next time GC runs as the package data should be available then.
+            const pkg = packagePath ?? this.getNodePackagePath(nodeId);
+            if (pkg !== undefined) {
+                this.mc.logger.sendErrorEvent({
+                    ...event,
+                    pkg: { value: `/${pkg.join("/")}`, tag: TelemetryDataTag.PackageData },
+                });
+            } else {
+                this.pendingEventsQueue.push(event);
+            }
+        }
+    }
 }
 
 /**
  * Gets the garbage collection state from the given snapshot tree. The GC state may be written into multiple blobs.
  * Merge the GC state from all such blobs and return the merged GC state.
-*/
+ */
 async function getGCStateFromSnapshot(
     gcSnapshotTree: ISnapshotTree,
     readAndParseBlob: ReadAndParseBlob,
@@ -855,3 +1058,35 @@ async function getGCStateFromSnapshot(
     }
     return rootGCState;
 }
+
+/**
+ * setLongTimeout is used for timeouts longer than setTimeout's ~24.8 day max
+ * @param timeoutMs - the total time the timeout needs to last in ms
+ * @param timeoutFn - the function to execute when the timer ends
+ * @param setTimerFn - the function used to update your timer variable
+ */
+function setLongTimeout(
+    timeoutMs: number,
+    timeoutFn: () => void,
+    setTimerFn: (timer: ReturnType<typeof setTimeout>) => void,
+) {
+    // The setTimeout max is 24.8 days before looping occurs.
+    const maxTimeout = 2147483647;
+    let timer: ReturnType<typeof setTimeout>;
+    if (timeoutMs > maxTimeout) {
+        const newTimeoutMs = timeoutMs - maxTimeout;
+        timer = setTimeout(() => setLongTimeout(newTimeoutMs, timeoutFn, setTimerFn), maxTimeout);
+    } else {
+        timer = setTimeout(() => timeoutFn(), timeoutMs);
+    }
+    setTimerFn(timer);
+}
+
+/**
+ * Given a GC nodeId, tells whether it belongs to a data store or not.
+ */
+function isDataStoreNode(nodeId: string): boolean {
+    const pathParts = nodeId.split("/");
+    // Data store ids are in the format "/dataStoreId".
+    return pathParts.length === 2 && pathParts[1] !== "" ? true : false;
+}