@fluidframework/container-runtime 0.56.0 → 0.57.0-51086

package/lib/garbageCollection.js

@@ -3,11 +3,12 @@
  * Licensed under the MIT License.
  */
 import { assert, LazyPromise, Timer } from "@fluidframework/common-utils";
-import { ClientSessionExpiredError } from "@fluidframework/container-utils";
+import { ClientSessionExpiredError, DataProcessingError } from "@fluidframework/container-utils";
 import { cloneGCData, concatGarbageCollectionStates, concatGarbageCollectionData, runGarbageCollection, unpackChildNodesGCDetails, } from "@fluidframework/garbage-collector";
 import { gcBlobKey, } from "@fluidframework/runtime-definitions";
 import { SummaryTreeBuilder, } from "@fluidframework/runtime-utils";
-import { ChildLogger, loggerToMonitoringContext, PerformanceEvent, } from "@fluidframework/telemetry-utils";
+import { ChildLogger, loggerToMonitoringContext, PerformanceEvent, TelemetryDataTag, } from "@fluidframework/telemetry-utils";
+import { RuntimeHeaders } from ".";
 import { getSummaryForDatastores } from "./dataStores";
 import { getGCVersion, metadataBlobName, dataStoreAttributesBlobName, } from "./summaryFormat";
 /** This is the current version of garbage collection. */
@@ -24,7 +25,11 @@ const gcTestModeKey = "Fluid.GarbageCollection.GCTestMode";
 const runSweepKey = "Fluid.GarbageCollection.RunSweep";
 // Feature gate key to write GC data at the root of the summary tree.
 const writeAtRootKey = "Fluid.GarbageCollection.WriteDataAtRoot";
+// Feature gate key to expire a session after a set period of time.
+const runSessionExpiry = "Fluid.GarbageCollection.RunSessionExpiry";
 const defaultDeleteTimeoutMs = 7 * 24 * 60 * 60 * 1000; // 7 days
+const defaultSessionExpiryDurationMs = 30 * 24 * 60 * 60 * 1000; // 30 days
+;
 /**
  * Helper class that tracks the state of an unreferenced node such as the time it was unreferenced. It also sets
  * the node's state to inactive if it remains unreferenced for a given amount of time (inactiveTimeoutMs).
@@ -32,37 +37,25 @@ const defaultDeleteTimeoutMs = 7 * 24 * 60 * 60 * 1000; // 7 days
 class UnreferencedStateTracker {
     constructor(unreferencedTimestampMs, inactiveTimeoutMs) {
         this.unreferencedTimestampMs = unreferencedTimestampMs;
-        this.inactive = false;
-        // Keeps track of all inactive events that are logged. This is used to limit the log generation for each event to 1
-        // so that it is not noisy.
-        this.inactiveEventsLogged = new Set();
+        this._inactive = false;
         // If the timeout has already expired, the node should become inactive immediately. Otherwise, start a timer of
         // inactiveTimeoutMs after which the node will become inactive.
         if (inactiveTimeoutMs <= 0) {
-            this.inactive = true;
+            this._inactive = true;
         }
         else {
-            this.timer = new Timer(inactiveTimeoutMs, () => { this.inactive = true; });
+            this.timer = new Timer(inactiveTimeoutMs, () => { this._inactive = true; });
             this.timer.start();
         }
     }
+    get inactive() {
+        return this._inactive;
+    }
     /** Stop tracking this node. Reset the unreferenced timer, if any, and reset inactive state. */
     stopTracking() {
         var _a;
         (_a = this.timer) === null || _a === void 0 ? void 0 : _a.clear();
-        this.inactive = false;
-    }
-    /** Logs an error with the given properties if the node  is inactive. */
-    logIfInactive(logger, eventName, currentTimestampMs, deleteTimeoutMs, inactiveNodeId) {
-        if (this.inactive && !this.inactiveEventsLogged.has(eventName)) {
-            logger.sendErrorEvent({
-                eventName,
-                age: currentTimestampMs - this.unreferencedTimestampMs,
-                timeout: deleteTimeoutMs,
-                id: inactiveNodeId,
-            });
-            this.inactiveEventsLogged.add(eventName);
-        }
+        this._inactive = false;
     }
 }
 /**
@@ -73,12 +66,15 @@ export class GarbageCollector {
     constructor(provider, gcOptions, 
     /** After GC has run, called to delete objects in the runtime whose routes are unused. */
     deleteUnusedRoutes, 
+    /** For a given node path, returns the node's package path. */
+    getNodePackagePath, 
     /** Returns the current timestamp to be assigned to nodes that become unreferenced. */
     getCurrentTimestampMs, closeFn, baseSnapshot, readAndParseBlob, baseLogger, existing, metadata) {
         var _a, _b, _c, _d, _e;
         this.provider = provider;
         this.gcOptions = gcOptions;
         this.deleteUnusedRoutes = deleteUnusedRoutes;
+        this.getNodePackagePath = getNodePackagePath;
         this.getCurrentTimestampMs = getCurrentTimestampMs;
         this.closeFn = closeFn;
         /**
@@ -103,6 +99,11 @@ export class GarbageCollector {
         this.referencesSinceLastRun = new Map();
         // Map of node ids to their unreferenced state tracker.
         this.unreferencedNodesState = new Map();
+        // Keeps track of unreferenced events that are logged for a node. This is used to limit the log generation to one
+        // per event per node.
+        this.loggedUnreferencedEvents = new Set();
+        // Queue for unreferenced events that should be logged the next time GC runs.
+        this.pendingEventsQueue = [];
         this.mc = loggerToMonitoringContext(ChildLogger.create(baseLogger, "GarbageCollector"));
         this.deleteTimeoutMs = (_a = this.gcOptions.deleteTimeoutMs) !== null && _a !== void 0 ? _a : defaultDeleteTimeoutMs;
         let prevSummaryGCVersion;
@@ -119,12 +120,19 @@ export class GarbageCollector {
         else {
             // For new documents, GC has to be exlicitly enabled via the gcAllowed flag in GC options.
             this.gcEnabled = gcOptions.gcAllowed === true;
-            this.sessionExpiryTimeoutMs = this.gcOptions.gcTestSessionTimeoutMs;
+            // Set the Session Expiry only if the flag is enabled or the test option is set.
+            if (this.mc.config.getBoolean(runSessionExpiry) && this.gcEnabled) {
+                this.sessionExpiryTimeoutMs = defaultSessionExpiryDurationMs;
+            }
         }
         // If session expiry is enabled, we need to close the container when the timeout expires
         if (this.sessionExpiryTimeoutMs !== undefined) {
-            const expiryMs = this.sessionExpiryTimeoutMs;
-            this.sessionExpiryTimer = setTimeout(() => this.closeFn(new ClientSessionExpiredError(`Client session expired.`, expiryMs)), expiryMs);
+            const timeoutMs = this.sessionExpiryTimeoutMs;
+            setLongTimeout(timeoutMs, () => {
+                this.closeFn(new ClientSessionExpiredError(`Client session expired.`, timeoutMs));
+            }, (timer) => {
+                this.sessionExpiryTimer = timer;
+            });
         }
         // For existing document, the latest summary is the one that we loaded from. So, use its GC version as the
         // latest tracked GC version. For new documents, we will be writing the first summary with the current version.
@@ -253,9 +261,22 @@ export class GarbageCollector {
             }
             return dataStoreGCDetailsMap;
         });
+        // Initialize the base state. The base GC data is used to detect and log when inactive / deleted objects are
+        // used in the container.
+        if (this.shouldRunGC) {
+            this.initializeBaseStateP.catch((error) => {
+                throw new DataProcessingError(error === null || error === void 0 ? void 0 : error.message, "FailedToInitializeGC", {
+                    gcEnabled: this.gcEnabled,
+                    runSweep: this.shouldRunSweep,
+                    writeAtRoot: this._writeDataAtRoot,
+                    testMode: this.testMode,
+                    sessionExpiry: this.sessionExpiryTimeoutMs,
+                });
+            });
+        }
     }
-    static create(provider, gcOptions, deleteUnusedRoutes, getCurrentTimestampMs, closeFn, baseSnapshot, readAndParseBlob, baseLogger, existing, metadata) {
-        return new GarbageCollector(provider, gcOptions, deleteUnusedRoutes, getCurrentTimestampMs, closeFn, baseSnapshot, readAndParseBlob, baseLogger, existing, metadata);
+    static create(provider, gcOptions, deleteUnusedRoutes, getNodePackagePath, getCurrentTimestampMs, closeFn, baseSnapshot, readAndParseBlob, baseLogger, existing, metadata) {
+        return new GarbageCollector(provider, gcOptions, deleteUnusedRoutes, getNodePackagePath, getCurrentTimestampMs, closeFn, baseSnapshot, readAndParseBlob, baseLogger, existing, metadata);
     }
     /**
      * This tracks two things:
@@ -290,29 +311,26 @@ export class GarbageCollector {
             await this.initializeBaseStateP;
             // Let the runtime update its pending state before GC runs.
             await this.provider.updateStateBeforeGC();
-            const gcStats = {};
             // Get the runtime's GC data and run GC on the reference graph in it.
             const gcData = await this.provider.getGCData(fullGC);
-            this.updateStateSinceLatestRun(gcData);
             const gcResult = runGarbageCollection(gcData.gcNodes, ["/"], logger);
-            const currentTimestampMs = this.getCurrentTimestampMs();
+            const gcStats = this.generateStatsAndLogEvents(gcResult);
+            // Update the state since the last GC run. There can be nodes that were referenced between the last and
+            // the current run. We need to identify than and update their unreferenced state if needed.
+            this.updateStateSinceLastRun(gcData);
             // Update the current state of the system based on the GC run.
+            const currentTimestampMs = this.getCurrentTimestampMs();
             this.updateCurrentState(gcData, gcResult, currentTimestampMs);
-            const dataStoreUsedStateStats = this.provider.updateUsedRoutes(gcResult.referencedNodeIds, currentTimestampMs);
+            this.provider.updateUsedRoutes(gcResult.referencedNodeIds, currentTimestampMs);
             if (runSweep) {
                 // Placeholder for running sweep logic.
             }
-            // Update stats to be reported in the peformance event.
-            gcStats.deletedNodes = gcResult.deletedNodeIds.length;
-            gcStats.totalNodes = gcResult.referencedNodeIds.length + gcResult.deletedNodeIds.length;
-            gcStats.deletedDataStores = dataStoreUsedStateStats.unusedNodeCount;
-            gcStats.totalDataStores = dataStoreUsedStateStats.totalNodeCount;
             // If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios
             // involving access to deleted data.
             if (this.testMode) {
                 this.deleteUnusedRoutes(gcResult.deletedNodeIds);
             }
-            event.end(gcStats);
+            event.end(Object.assign({}, gcStats));
             return gcStats;
         }, { end: true, cancel: "error" });
     }
@@ -366,32 +384,41 @@ export class GarbageCollector {
         await this.updateSummaryGCVersionFromSnapshot(result.snapshot, readAndParseBlob);
     }
     /**
-     * Called when a node with the given id is changed. If the node is inactive, log an error.
+     * Called when a node with the given id is updated. If the node is inactive, log an error.
+     * @param nodePath - The id of the node that changed.
+     * @param reason - Whether the node was loaded or changed.
+     * @param packagePath - The package path of the node. This may not be available if the node hasn't been loaded yet.
+     * @param requestHeaders - If the node was loaded via request path, the headers in the request.
      */
-    nodeChanged(id) {
-        var _a;
-        // Prefix "/" if needed to make it relative to the root.
-        const nodeId = id.startsWith("/") ? id : `/${id}`;
-        (_a = this.unreferencedNodesState.get(nodeId)) === null || _a === void 0 ? void 0 : _a.logIfInactive(this.mc.logger, "inactiveObjectChanged", this.getCurrentTimestampMs(), this.deleteTimeoutMs, nodeId);
-    }
-    dispose() {
-        if (this.sessionExpiryTimer !== undefined) {
-            clearTimeout(this.sessionExpiryTimer);
-            this.sessionExpiryTimer = undefined;
+    nodeUpdated(nodePath, reason, packagePath, requestHeaders) {
+        if (!this.shouldRunGC) {
+            return;
         }
+        this.logIfInactive(reason, nodePath, this.getCurrentTimestampMs(), packagePath, requestHeaders);
     }
     /**
      * Called when an outbound reference is added to a node. This is used to identify all nodes that have been
      * referenced between summaries so that their unreferenced timestamp can be reset.
      *
-     * @param fromNodeId - The node from which the reference is added.
-     * @param toNodeId - The node to which the reference is added.
+     * @param fromNodePath - The node from which the reference is added.
+     * @param toNodePath - The node to which the reference is added.
      */
-    addedOutboundReference(fromNodeId, toNodeId) {
+    addedOutboundReference(fromNodePath, toNodePath) {
         var _a;
-        const outboundRoutes = (_a = this.referencesSinceLastRun.get(fromNodeId)) !== null && _a !== void 0 ? _a : [];
-        outboundRoutes.push(toNodeId);
-        this.referencesSinceLastRun.set(fromNodeId, outboundRoutes);
+        if (!this.shouldRunGC) {
+            return;
+        }
+        const outboundRoutes = (_a = this.referencesSinceLastRun.get(fromNodePath)) !== null && _a !== void 0 ? _a : [];
+        outboundRoutes.push(toNodePath);
+        this.referencesSinceLastRun.set(fromNodePath, outboundRoutes);
+        // If the node that got referenced is inactive, log an event as that may indicate use-after-delete.
+        this.logIfInactive("Revived", toNodePath, this.getCurrentTimestampMs(), this.getNodePackagePath(toNodePath));
+    }
+    dispose() {
+        if (this.sessionExpiryTimer !== undefined) {
+            clearTimeout(this.sessionExpiryTimer);
+            this.sessionExpiryTimer = undefined;
+        }
     }
     /**
      * Update the latest summary GC version from the metadata blob in the given snapshot.
@@ -432,9 +459,6 @@ export class GarbageCollector {
         for (const nodeId of gcResult.referencedNodeIds) {
             const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
             if (nodeStateTracker !== undefined) {
-                // If this node has been unreferenced for longer than deleteTimeoutMs and is being referenced,
-                // log an error as this may mean the deleteTimeoutMs is not long enough.
-                nodeStateTracker.logIfInactive(this.mc.logger, "inactiveObjectRevived", currentTimestampMs, this.deleteTimeoutMs, nodeId);
                 // Stop tracking so as to clear out any running timers.
                 nodeStateTracker.stopTracking();
                 // Delete the node as we don't need to track it any more.
@@ -450,7 +474,7 @@ export class GarbageCollector {
      * This function identifies nodes that were referenced since last run and removes their unreferenced state, if any.
      * If these nodes are currently unreferenced, they will be assigned new unreferenced state by the current run.
      */
-    updateStateSinceLatestRun(currentGCData) {
+    updateStateSinceLastRun(currentGCData) {
         // If we haven't run GC before or no references were added since the last run, there is nothing to do.
         if (this.gcDataFromLastRun === undefined || this.referencesSinceLastRun.size === 0) {
             return;
@@ -506,8 +530,7 @@ export class GarbageCollector {
      * @param currentGCData - The GC data (reference graph) from the current GC run.
      */
     validateReferenceCorrectness(currentGCData) {
-        assert(this.gcDataFromLastRun !== undefined, 0x2b7
-        /* "Can't validate correctness without GC data from last run" */ );
+        assert(this.gcDataFromLastRun !== undefined, 0x2b7);
         // Get a list of all the outbound routes (or references) in the current GC data.
         const currentReferences = [];
         for (const [nodeId, outboundRoutes] of Object.entries(currentGCData.gcNodes)) {
@@ -531,9 +554,9 @@ export class GarbageCollector {
         // Validate that the current reference graph doesn't have references that we are not already aware of. If this
         // happens, it might indicate data corruption since we may delete objects prematurely.
         currentReferences.forEach((route) => {
-            // Validate references for data stores only whose routes are of the format "/dataStoreId". Currently, layers
-            // below data stores don't have GC implemented so there is no guarantee their references will be notified.
-            if (route.split("/").length === 2 && !explicitReferences.includes(route)) {
+            // Validate references for data stores only. Currently, layers below data stores don't have GC implemented
+            // so there is no guarantee their references will be notified.
+            if (isDataStoreNode(route) && !explicitReferences.includes(route)) {
                 /**
                  * The following log will be enabled once this issue is resolved:
                  * https://github.com/microsoft/FluidFramework/issues/8878.
@@ -547,11 +570,95 @@ export class GarbageCollector {
             }
         });
     }
+    /**
+     * Generates the stats of a garbage collection run from the given results of the run. Also, logs any pending events
+     * in the pendingEventsQueue.
+     * @param gcResult - The result of a GC run.
+     * @returns the GC stats of the GC run.
+     */
+    generateStatsAndLogEvents(gcResult) {
+        // Log pending events for unreferenced nodes after GC has run. We should have the package data available for
+        // them now since the GC run should have loaded these nodes.
+        let event = this.pendingEventsQueue.shift();
+        while (event !== undefined) {
+            const pkg = this.getNodePackagePath(event.id);
+            this.mc.logger.sendErrorEvent(Object.assign(Object.assign({}, event), { pkg: pkg ? { value: `/${pkg.join("/")}`, tag: TelemetryDataTag.PackageData } : undefined }));
+            event = this.pendingEventsQueue.shift();
+        }
+        const gcStats = {
+            nodeCount: 0,
+            dataStoreCount: 0,
+            unrefNodeCount: 0,
+            unrefDataStoreCount: 0,
+            updatedNodeCount: 0,
+            updatedDataStoreCount: 0,
+        };
+        for (const nodeId of gcResult.referencedNodeIds) {
+            gcStats.nodeCount++;
+            const isDataStore = isDataStoreNode(nodeId);
+            if (isDataStore) {
+                gcStats.dataStoreCount++;
+            }
+            // If a referenced node has an entry in `unreferencedNodesState`, it was previously unreferenced. So, its
+            // reference state updated from the last GC run.
+            if (this.unreferencedNodesState.has(nodeId)) {
+                gcStats.updatedNodeCount++;
+                if (isDataStore) {
+                    gcStats.updatedDataStoreCount++;
+                }
+            }
+        }
+        for (const nodeId of gcResult.deletedNodeIds) {
+            gcStats.nodeCount++;
+            gcStats.unrefNodeCount++;
+            const isDataStore = isDataStoreNode(nodeId);
+            if (isDataStore) {
+                gcStats.dataStoreCount++;
+                gcStats.unrefDataStoreCount++;
+            }
+            // If an unreferenced node doesn't an entry in `unreferencedNodesState`, it was previously referenced. So,
+            // its reference state updated from the last GC run.
+            if (!this.unreferencedNodesState.has(nodeId)) {
+                gcStats.updatedNodeCount++;
+                if (isDataStore) {
+                    gcStats.updatedDataStoreCount++;
+                }
+            }
+        }
+        return gcStats;
+    }
+    /**
+     * Logs an event if a node is inactive and is used. If the package data for the node exists, log immediately. Else,
+     * queue it and it will be logged the next time GC runs as the package data should be available then.
+     */
+    logIfInactive(eventSuffix, nodeId, timestampMs, packagePath, requestHeaders) {
+        const eventName = `inactiveObject_${eventSuffix}`;
+        // We log a particular event for a given node only once so that it is not too noisy.
+        const uniqueEventId = `${nodeId}-${eventName}`;
+        const nodeState = this.unreferencedNodesState.get(nodeId);
+        if ((nodeState === null || nodeState === void 0 ? void 0 : nodeState.inactive) && !this.loggedUnreferencedEvents.has(uniqueEventId)) {
+            this.loggedUnreferencedEvents.add(uniqueEventId);
+            const event = {
+                eventName,
+                id: nodeId,
+                age: timestampMs - nodeState.unreferencedTimestampMs,
+                timeout: this.deleteTimeoutMs,
+                externalRequest: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[RuntimeHeaders.externalRequest],
+                viaHandle: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[RuntimeHeaders.viaHandle],
+            };
+            if (packagePath !== undefined) {
+                this.mc.logger.sendErrorEvent(Object.assign(Object.assign({}, event), { pkg: { value: `/${packagePath.join("/")}`, tag: TelemetryDataTag.PackageData } }));
+            }
+            else {
+                this.pendingEventsQueue.push(event);
+            }
+        }
+    }
 }
 /**
  * Gets the garbage collection state from the given snapshot tree. The GC state may be written into multiple blobs.
  * Merge the GC state from all such blobs and return the merged GC state.
-*/
+ */
 async function getGCStateFromSnapshot(gcSnapshotTree, readAndParseBlob) {
     let rootGCState = { gcNodes: {} };
     for (const key of Object.keys(gcSnapshotTree.blobs)) {
@@ -570,4 +677,31 @@ async function getGCStateFromSnapshot(gcSnapshotTree, readAndParseBlob) {
     }
     return rootGCState;
 }
+/**
+ * setLongTimeout is used for timeouts longer than setTimeout's ~24.8 day max
+ * @param timeoutMs - the total time the timeout needs to last in ms
+ * @param timeoutFn - the function to execute when the timer ends
+ * @param setTimerFn - the function used to update your timer variable
+ */
+function setLongTimeout(timeoutMs, timeoutFn, setTimerFn) {
+    // The setTimeout max is 24.8 days before looping occurs.
+    const maxTimeout = 2147483647;
+    let timer;
+    if (timeoutMs > maxTimeout) {
+        const newTimeoutMs = timeoutMs - maxTimeout;
+        timer = setTimeout(() => setLongTimeout(newTimeoutMs, timeoutFn, setTimerFn), maxTimeout);
+    }
+    else {
+        timer = setTimeout(() => timeoutFn(), timeoutMs);
+    }
+    setTimerFn(timer);
+}
+/**
+ * Given a GC nodeId, tells whether it belongs to a data store or not.
+ */
+function isDataStoreNode(nodeId) {
+    const pathParts = nodeId.split("/");
+    // Data store ids are in the format "/dataStoreId".
+    return pathParts.length === 2 && pathParts[1] !== "" ? true : false;
+}
 //# sourceMappingURL=garbageCollection.js.map