@fluidframework/container-runtime 0.59.2000-63294 → 0.59.3000-66610

package/dist/garbageCollection.js

@@ -29,7 +29,9 @@ const runSweepKey = "Fluid.GarbageCollection.RunSweep";
 // Feature gate key to write GC data at the root of the summary tree.
 const writeAtRootKey = "Fluid.GarbageCollection.WriteDataAtRoot";
 // Feature gate key to expire a session after a set period of time.
-const runSessionExpiry = "Fluid.GarbageCollection.RunSessionExpiry";
+const runSessionExpiryKey = "Fluid.GarbageCollection.RunSessionExpiry";
+// Feature gate key to disable expiring session after a set period of time, even if expiry value is present
+const disableSessionExpiryKey = "Fluid.GarbageCollection.DisableSessionExpiry";
 // Feature gate key to log error messages if GC reference validation fails.
 const logUnknownOutboundReferencesKey = "Fluid.GarbageCollection.LogUnknownOutboundReferences";
 const defaultDeleteTimeoutMs = 7 * 24 * 60 * 60 * 1000; // 7 days
@@ -38,9 +40,11 @@ exports.defaultSessionExpiryDurationMs = 30 * 24 * 60 * 60 * 1000; // 30 days
 exports.GCNodeType = {
     // Nodes that are for data stores.
     DataStore: "DataStore",
+    // Nodes that are within a data store. For example, DDS nodes.
+    SubDataStore: "SubDataStore",
     // Nodes that are for attachment blobs, i.e., blobs uploaded via BlobManager.
     Blob: "Blob",
-    // Nodes that are neither data store not blobs. For example, root node and DDS nodes.
+    // Nodes that are neither of the above. For example, root node.
     Other: "Other",
 };
 /**
@@ -94,15 +98,15 @@ class UnreferencedStateTracker {
  * its state across summaries.
  *
  * Node - represented as nodeId, it's a node on the GC graph
- * Outbound Route - a path from one node to another node, think `nodeA` -> `nodeB`
+ * Outbound Route - a path from one node to another node, think `nodeA` -\> `nodeB`
  * Graph - all nodes with their respective routes
  *             GC Graph
  *
  *               Node
  *        NodeId = "datastore1"
- *           /             \
+ *           /             \\
  *    OutboundRoute   OutboundRoute
- *         /                 \
+ *         /                 \\
  *       Node               Node
  *  NodeId = "dds1"     NodeId = "dds2"
  */
@@ -112,7 +116,7 @@ class GarbageCollector {
     getNodePackagePath, 
     /** Returns the timestamp of the last summary generated for this container. */
     getLastSummaryTimestampMs, baseSnapshot, readAndParseBlob, baseLogger, existing, metadata) {
-        var _a, _b, _c, _d, _e;
+        var _a, _b, _c, _d, _e, _f;
         this.runtime = runtime;
         this.gcOptions = gcOptions;
         this.getNodePackagePath = getNodePackagePath;
@@ -144,29 +148,46 @@ class GarbageCollector {
         this.loggedUnreferencedEvents = new Set();
         // Queue for unreferenced events that should be logged the next time GC runs.
         this.pendingEventsQueue = [];
-        this.mc = telemetry_utils_1.loggerToMonitoringContext(telemetry_utils_1.ChildLogger.create(baseLogger, "GarbageCollector"));
+        this.mc = (0, telemetry_utils_1.loggerToMonitoringContext)(telemetry_utils_1.ChildLogger.create(baseLogger, "GarbageCollector"));
         this.deleteTimeoutMs = (_a = this.gcOptions.deleteTimeoutMs) !== null && _a !== void 0 ? _a : defaultDeleteTimeoutMs;
         let prevSummaryGCVersion;
-        // GC can only be enabled during creation. After that, it can never be enabled again. So, for existing
-        // documents, we get this information from the metadata blob. Similarly the session timeout should be
-        // consistent across all clients, thus we grab it as well from the metadata blob, and set it once on creation.
+        /**
+         * The following GC state is enabled during container creation and cannot be changed throughout its lifetime:
+         * 1. Whether running GC mark phase is allowed or not.
+         * 2. Whether running GC sweep phase is allowed or not.
+         * 3. Whether GC session expiry is enabled or not.
+         * For existing containers, we get this information from the metadata blob of its summary.
+         */
         if (existing) {
-            prevSummaryGCVersion = summaryFormat_1.getGCVersion(metadata);
+            prevSummaryGCVersion = (0, summaryFormat_1.getGCVersion)(metadata);
             // Existing documents which did not have metadata blob or had GC disabled have version as 0. For all
             // other existing documents, GC is enabled.
             this.gcEnabled = prevSummaryGCVersion > 0;
+            this.sweepEnabled = (_b = metadata === null || metadata === void 0 ? void 0 : metadata.sweepEnabled) !== null && _b !== void 0 ? _b : false;
             this.sessionExpiryTimeoutMs = metadata === null || metadata === void 0 ? void 0 : metadata.sessionExpiryTimeoutMs;
         }
         else {
-            // For new documents, GC has to be explicitly enabled via the gcAllowed flag in GC options.
+            // Sweep should not be enabled without enabling GC mark phase. We could silently disable sweep in this
+            // scenario but explicitly failing makes it clearer and promotes correct usage.
+            if (gcOptions.sweepAllowed && !gcOptions.gcAllowed) {
+                throw new container_utils_1.UsageError("GC sweep phase cannot be enabled without enabling GC mark phase");
+            }
+            // For new documents, GC has to be explicitly enabled via the flags in GC options.
             this.gcEnabled = gcOptions.gcAllowed === true;
+            this.sweepEnabled = gcOptions.sweepAllowed === true;
             // Set the Session Expiry only if the flag is enabled or the test option is set.
-            if (this.mc.config.getBoolean(runSessionExpiry) && this.gcEnabled) {
+            if (this.mc.config.getBoolean(runSessionExpiryKey) && this.gcEnabled) {
                 this.sessionExpiryTimeoutMs = exports.defaultSessionExpiryDurationMs;
             }
         }
         // If session expiry is enabled, we need to close the container when the timeout expires
-        if (this.sessionExpiryTimeoutMs !== undefined) {
+        if (this.sessionExpiryTimeoutMs !== undefined
+            && this.mc.config.getBoolean(disableSessionExpiryKey) !== true) {
+            // If Test Override config is set, override Session Expiry timeout
+            const overrideSessionExpiryTimeoutMs = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.SessionExpiryMs");
+            if (overrideSessionExpiryTimeoutMs !== undefined) {
+                this.sessionExpiryTimeoutMs = overrideSessionExpiryTimeoutMs;
+            }
             const timeoutMs = this.sessionExpiryTimeoutMs;
             setLongTimeout(timeoutMs, () => {
                 this.runtime.closeFn(new container_utils_1.ClientSessionExpiredError(`Client session expired.`, timeoutMs));
@@ -177,19 +198,26 @@ class GarbageCollector {
         // For existing document, the latest summary is the one that we loaded from. So, use its GC version as the
         // latest tracked GC version. For new documents, we will be writing the first summary with the current version.
         this.latestSummaryGCVersion = prevSummaryGCVersion !== null && prevSummaryGCVersion !== void 0 ? prevSummaryGCVersion : this.currentGCVersion;
-        // Whether GC should run or not. Can override with localStorage flag.
-        this.shouldRunGC = (_b = this.mc.config.getBoolean(runGCKey)) !== null && _b !== void 0 ? _b : (
+        /**
+         * Whether GC should run or not. The following conditions have to be met to run sweep:
+         * 1. GC should be enabled for this container.
+         * 2. GC should not be disabled via disableGC GC option.
+         * These conditions can be overridden via runGCKey feature flag.
+         */
+        this.shouldRunGC = (_c = this.mc.config.getBoolean(runGCKey)) !== null && _c !== void 0 ? _c : (
         // GC must be enabled for the document.
         this.gcEnabled
             // GC must not be disabled via GC options.
             && !gcOptions.disableGC);
-        // Whether GC sweep phase should run or not. If this is false, only GC mark phase is run. Can override with
-        // localStorage flag.
-        this.shouldRunSweep = this.shouldRunGC &&
-            ((_c = this.mc.config.getBoolean(runSweepKey)) !== null && _c !== void 0 ? _c : gcOptions.runSweep === true)
-            && this.sessionExpiryTimer !== undefined;
+        /**
+         * Whether sweep should run or not. The following conditions have to be met to run sweep:
+         * 1. Overall GC or mark phase must be enabled (this.shouldRunGC).
+         * 2. Session expiry and sweep should be enabled for this container. Without session expiry we cannot safely
+         *    delete unreferenced objects. This condition (#2) can be overridden via runSweepKey feature flag.
+         */
+        this.shouldRunSweep = this.shouldRunGC && ((_d = this.mc.config.getBoolean(runSweepKey)) !== null && _d !== void 0 ? _d : (this.sessionExpiryTimeoutMs !== undefined && this.sweepEnabled));
         // Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
-        this.testMode = (_d = this.mc.config.getBoolean(gcTestModeKey)) !== null && _d !== void 0 ? _d : gcOptions.runGCInTestMode === true;
+        this.testMode = (_e = this.mc.config.getBoolean(gcTestModeKey)) !== null && _e !== void 0 ? _e : gcOptions.runGCInTestMode === true;
         /**
          * Enable resetting initial state once the following issue is resolved:
          * https://github.com/microsoft/FluidFramework/issues/8878.
@@ -201,7 +229,7 @@ class GarbageCollector {
         // this.initialStateNeedsReset = gcTreePresent ? !this.shouldRunGC : this.shouldRunGC;
         // If `writeDataAtRoot` setting is true, write the GC data into the root of the summary tree. We do this so that
         // the roll out can be staged. Once its rolled out everywhere, we will start writing at root by default.
-        this._writeDataAtRoot = (_e = this.mc.config.getBoolean(writeAtRootKey)) !== null && _e !== void 0 ? _e : this.gcOptions.writeDataAtRoot === true;
+        this._writeDataAtRoot = (_f = this.mc.config.getBoolean(writeAtRootKey)) !== null && _f !== void 0 ? _f : this.gcOptions.writeDataAtRoot === true;
         // Get the GC state from the GC blob in the base snapshot. Use LazyPromise because we only want to do
         // this once since it involves fetching blobs from storage which is expensive.
         const baseSummaryStateP = new common_utils_1.LazyPromise(async () => {
@@ -220,8 +248,8 @@ class GarbageCollector {
             // consolidate into IGarbageCollectionState format.
             // Add a node for the root node that is not present in older snapshot format.
             const gcState = { gcNodes: { "/": { outboundRoutes: [] } } };
-            const dataStoreSnapshotTree = dataStores_1.getSummaryForDatastores(baseSnapshot, metadata);
-            common_utils_1.assert(dataStoreSnapshotTree !== undefined, 0x2a8 /* "Expected data store snapshot tree in base snapshot" */);
+            const dataStoreSnapshotTree = (0, dataStores_1.getSummaryForDatastores)(baseSnapshot, metadata);
+            (0, common_utils_1.assert)(dataStoreSnapshotTree !== undefined, 0x2a8 /* "Expected data store snapshot tree in base snapshot" */);
             for (const [dsId, dsSnapshotTree] of Object.entries(dataStoreSnapshotTree.trees)) {
                 const blobId = dsSnapshotTree.blobs[runtime_definitions_1.gcBlobKey];
                 if (blobId === undefined) {
@@ -245,7 +273,7 @@ class GarbageCollector {
                     const rootId = id === "/" ? dsRootId : `${dsRootId}${id}`;
                     gcState.gcNodes[rootId] = { outboundRoutes: Array.from(outboundRoutes) };
                 }
-                common_utils_1.assert(gcState.gcNodes[dsRootId] !== undefined, 0x2a9 /* `GC nodes for data store ${dsId} not in GC blob` */);
+                (0, common_utils_1.assert)(gcState.gcNodes[dsRootId] !== undefined, 0x2a9 /* `GC nodes for data store ${dsId} not in GC blob` */);
                 gcState.gcNodes[dsRootId].unreferencedTimestampMs = gcSummaryDetails.unrefTimestamp;
             }
             // If there is only one node (root node just added above), either GC is disabled or we are loading from the
@@ -286,8 +314,8 @@ class GarbageCollector {
             // Run GC on the nodes in the base summary to get the routes used in each node in the container.
             // This is an optimization for space (vs performance) wherein we don't need to store the used routes of
             // each node in the summary.
-            const usedRoutes = garbage_collector_1.runGarbageCollection(gcNodes, ["/"], this.mc.logger).referencedNodeIds;
-            const baseGCDetailsMap = garbage_collector_1.unpackChildNodesGCDetails({ gcData: { gcNodes }, usedRoutes });
+            const usedRoutes = (0, garbage_collector_1.runGarbageCollection)(gcNodes, ["/"], this.mc.logger).referencedNodeIds;
+            const baseGCDetailsMap = (0, garbage_collector_1.unpackChildNodesGCDetails)({ gcData: { gcNodes }, usedRoutes });
             // Currently, the nodes may write the GC data. So, we need to update it's base GC details with the
             // unreferenced timestamp. Once we start writing the GC data here, we won't need to do this anymore.
             for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {
@@ -319,14 +347,6 @@ class GarbageCollector {
     static create(provider, gcOptions, getNodePackagePath, getLastSummaryTimestampMs, baseSnapshot, readAndParseBlob, baseLogger, existing, metadata) {
         return new GarbageCollector(provider, gcOptions, getNodePackagePath, getLastSummaryTimestampMs, baseSnapshot, readAndParseBlob, baseLogger, existing, metadata);
     }
-    /**
-     * This tracks two things:
-     * 1. Whether GC is enabled - If this is 0, GC is disabled. If this is greater than 0, GC is enabled.
-     * 2. If GC is enabled, the version of GC used to generate the GC data written in a summary.
-     */
-    get gcSummaryFeatureVersion() {
-        return this.gcEnabled ? this.currentGCVersion : 0;
-    }
     /**
      * Tells whether the GC state needs to be reset in the next summary. We need to do this if:
      * 1. GC was enabled and is now disabled. The GC state needs to be removed and everything becomes referenced.
@@ -354,11 +374,11 @@ class GarbageCollector {
             await this.runtime.updateStateBeforeGC();
             // Get the runtime's GC data and run GC on the reference graph in it.
             const gcData = await this.runtime.getGCData(fullGC);
-            const gcResult = garbage_collector_1.runGarbageCollection(gcData.gcNodes, ["/"], logger);
-            const gcStats = this.generateStatsAndLogEvents(gcResult);
+            const gcResult = (0, garbage_collector_1.runGarbageCollection)(gcData.gcNodes, ["/"], logger);
+            const gcStats = this.generateStatsAndLogEvents(gcResult, logger);
             // Update the state since the last GC run. There can be nodes that were referenced between the last and
             // the current run. We need to identify than and update their unreferenced state if needed.
-            this.updateStateSinceLastRun(gcData);
+            this.updateStateSinceLastRun(gcData, logger);
             // Update the current state of the system based on the GC run.
             const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
             this.updateCurrentState(gcData, gcResult, currentReferenceTimestampMs);
@@ -396,6 +416,17 @@ class GarbageCollector {
         builder.addBlob(`${exports.gcBlobPrefix}_root`, JSON.stringify(gcState));
         return builder.getSummaryTree();
     }
+    getMetadata() {
+        return {
+            /**
+             * If GC is enabled, the GC data is written using the current GC version and that is the gcFeature that goes
+             * into the metadata blob. If GC is disabled, the gcFeature is 0.
+             */
+            gcFeature: this.gcEnabled ? this.currentGCVersion : 0,
+            sessionExpiryTimeoutMs: this.sessionExpiryTimeoutMs,
+            sweepEnabled: this.sweepEnabled,
+        };
+    }
     /**
      * Returns a map of node ids to their base GC details generated from the base summary. This is used by the caller
      * to initialize the GC state of the nodes.
@@ -469,7 +500,7 @@ class GarbageCollector {
         const metadataBlobId = snapshot.blobs[summaryFormat_1.metadataBlobName];
         if (metadataBlobId) {
             const metadata = await readAndParseBlob(metadataBlobId);
-            this.latestSummaryGCVersion = summaryFormat_1.getGCVersion(metadata);
+            this.latestSummaryGCVersion = (0, summaryFormat_1.getGCVersion)(metadata);
         }
     }
     /**
@@ -482,7 +513,7 @@ class GarbageCollector {
      * @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
      */
     updateCurrentState(gcData, gcResult, currentReferenceTimestampMs) {
-        this.previousGCDataFromLastRun = garbage_collector_1.cloneGCData(gcData);
+        this.previousGCDataFromLastRun = (0, garbage_collector_1.cloneGCData)(gcData);
         this.newReferencesSinceLastRun.clear();
         // Iterate through the referenced nodes and stop tracking if they were unreferenced before.
         for (const nodeId of gcResult.referencedNodeIds) {
@@ -525,7 +556,7 @@ class GarbageCollector {
      * This function identifies nodes that were referenced since last run and removes their unreferenced state, if any.
      * If these nodes are currently unreferenced, they will be assigned new unreferenced state by the current run.
      */
-    updateStateSinceLastRun(currentGCData) {
+    updateStateSinceLastRun(currentGCData, logger) {
         // If we haven't run GC before there is nothing to do.
         if (this.previousGCDataFromLastRun === undefined) {
             return;
@@ -542,7 +573,7 @@ class GarbageCollector {
                     gcNodeId: missingExplicitReference[0],
                     gcRoutes: JSON.stringify(missingExplicitReference[1]),
                 };
-                this.mc.logger.sendPerformanceEvent(event);
+                logger.sendPerformanceEvent(event);
             });
         }
         // No references were added since the last run so we don't have to update reference states of any unreferenced
@@ -565,7 +596,7 @@ class GarbageCollector {
          *      which is tracked by https://github.com/microsoft/FluidFramework/issues/8470.
          *    - A new data store may have "root" DDSs already created and we don't detect them today.
          */
-        const gcDataSuperSet = garbage_collector_1.concatGarbageCollectionData(this.previousGCDataFromLastRun, currentGCData);
+        const gcDataSuperSet = (0, garbage_collector_1.concatGarbageCollectionData)(this.previousGCDataFromLastRun, currentGCData);
         this.newReferencesSinceLastRun.forEach((outboundRoutes, sourceNodeId) => {
             if (gcDataSuperSet.gcNodes[sourceNodeId] === undefined) {
                 gcDataSuperSet.gcNodes[sourceNodeId] = outboundRoutes;
@@ -579,7 +610,7 @@ class GarbageCollector {
          * unreferenced, stop tracking them and remove from unreferenced list.
          * Some of these nodes may be unreferenced now and if so, the current run will add unreferenced state for them.
          */
-        const gcResult = garbage_collector_1.runGarbageCollection(gcDataSuperSet.gcNodes, ["/"], this.mc.logger);
+        const gcResult = (0, garbage_collector_1.runGarbageCollection)(gcDataSuperSet.gcNodes, ["/"], logger);
         for (const nodeId of gcResult.referencedNodeIds) {
             const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
             if (nodeStateTracker !== undefined) {
@@ -604,7 +635,7 @@ class GarbageCollector {
      * @returns - a list of missing explicit references
      */
     findMissingExplicitReferences(currentGCData, previousGCData, explicitReferences) {
-        common_utils_1.assert(previousGCData !== undefined, 0x2b7);
+        (0, common_utils_1.assert)(previousGCData !== undefined, 0x2b7);
         const currentGraph = Object.entries(currentGCData.gcNodes);
         const missingExplicitReferences = [];
         currentGraph.forEach(([nodeId, currentOutboundRoutes]) => {
@@ -637,13 +668,13 @@ class GarbageCollector {
      * @param gcResult - The result of a GC run.
      * @returns the GC stats of the GC run.
      */
-    generateStatsAndLogEvents(gcResult) {
+    generateStatsAndLogEvents(gcResult, logger) {
         // Log pending events for unreferenced nodes after GC has run. We should have the package data available for
         // them now since the GC run should have loaded these nodes.
         let event = this.pendingEventsQueue.shift();
         while (event !== undefined) {
             const pkg = this.getNodePackagePath(event.id);
-            this.mc.logger.sendErrorEvent(Object.assign(Object.assign({}, event), { pkg: pkg ? { value: `/${pkg.join("/")}`, tag: telemetry_utils_1.TelemetryDataTag.PackageData } : undefined }));
+            logger.sendErrorEvent(Object.assign(Object.assign({}, event), { pkg: pkg ? { value: `/${pkg.join("/")}`, tag: telemetry_utils_1.TelemetryDataTag.PackageData } : undefined }));
             event = this.pendingEventsQueue.shift();
         }
         const gcStats = {
@@ -750,9 +781,9 @@ async function getGCStateFromSnapshot(gcSnapshotTree, readAndParseBlob) {
             continue;
         }
         const gcState = await readAndParseBlob(blobId);
-        common_utils_1.assert(gcState !== undefined, 0x2ad /* "GC blob missing from snapshot" */);
+        (0, common_utils_1.assert)(gcState !== undefined, 0x2ad /* "GC blob missing from snapshot" */);
         // Merge the GC state of this blob into the root GC state.
-        rootGCState = garbage_collector_1.concatGarbageCollectionStates(rootGCState, gcState);
+        rootGCState = (0, garbage_collector_1.concatGarbageCollectionStates)(rootGCState, gcState);
     }
     return rootGCState;
 }