@fluidframework/container-runtime 2.0.0-dev-rc.1.0.0.228517 → 2.0.0-dev-rc.2.0.0.245554

package/dist/gc/garbageCollection.js

@@ -9,17 +9,17 @@ const core_utils_1 = require("@fluidframework/core-utils");
 const runtime_definitions_1 = require("@fluidframework/runtime-definitions");
 const runtime_utils_1 = require("@fluidframework/runtime-utils");
 const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
-const blobManager_1 = require("../blobManager");
-const containerRuntime_1 = require("../containerRuntime");
-const error_1 = require("../error");
-const messageTypes_1 = require("../messageTypes");
-const gcConfigs_1 = require("./gcConfigs");
-const gcDefinitions_1 = require("./gcDefinitions");
-const gcHelpers_1 = require("./gcHelpers");
-const gcReferenceGraphAlgorithm_1 = require("./gcReferenceGraphAlgorithm");
-const gcSummaryStateTracker_1 = require("./gcSummaryStateTracker");
-const gcUnreferencedStateTracker_1 = require("./gcUnreferencedStateTracker");
-const gcTelemetry_1 = require("./gcTelemetry");
+const blobManager_js_1 = require("../blobManager.js");
+const containerRuntime_js_1 = require("../containerRuntime.js");
+const error_js_1 = require("../error.js");
+const messageTypes_js_1 = require("../messageTypes.js");
+const gcConfigs_js_1 = require("./gcConfigs.js");
+const gcDefinitions_js_1 = require("./gcDefinitions.js");
+const gcHelpers_js_1 = require("./gcHelpers.js");
+const gcReferenceGraphAlgorithm_js_1 = require("./gcReferenceGraphAlgorithm.js");
+const gcSummaryStateTracker_js_1 = require("./gcSummaryStateTracker.js");
+const gcUnreferencedStateTracker_js_1 = require("./gcUnreferencedStateTracker.js");
+const gcTelemetry_js_1 = require("./gcTelemetry.js");
 /**
  * The garbage collector for the container runtime. It consolidates the garbage collection functionality and maintains
  * its state across summaries.
@@ -76,8 +76,12 @@ class GarbageCollector {
         this.tombstones = [];
         // A list of nodes that have been deleted during sweep phase.
         this.deletedNodes = new Set();
-        // Map of node ids to their unreferenced state tracker.
-        this.unreferencedNodesState = new Map();
+        /**
+         * Map of node ids to their unreferenced state tracker
+         * NOTE: The set of keys in this map is considered as the set of unreferenced nodes
+         * as of the last GC run. So in between runs, nothing should be added or removed.
+         */
+        this.unreferencedNodesState = new gcUnreferencedStateTracker_js_1.UnreferencedStateTrackerMap();
         // The number of times GC has successfully completed on this instance of GarbageCollector.
         this.completedRuns = 0;
         this.runtime = createParams.runtime;
@@ -87,6 +91,7 @@ class GarbageCollector {
         this.submitMessage = createParams.submitMessage;
         const baseSnapshot = createParams.baseSnapshot;
         const readAndParseBlob = createParams.readAndParseBlob;
+        const pendingSessionExpiryTimerStarted = createParams.sessionExpiryTimerStarted;
         this.mc = (0, telemetry_utils_1.createChildMonitoringContext)({
             logger: createParams.baseLogger,
             namespace: "GarbageCollector",
@@ -94,19 +99,29 @@ class GarbageCollector {
                 all: { completedGCRuns: () => this.completedRuns },
             },
         });
-        this.configs = (0, gcConfigs_1.generateGCConfigs)(this.mc, createParams);
+        this.configs = (0, gcConfigs_js_1.generateGCConfigs)(this.mc, createParams);
         // If session expiry is enabled, we need to close the container when the session expiry timeout expires.
         if (this.configs.sessionExpiryTimeoutMs !== undefined) {
             // If Test Override config is set, override Session Expiry timeout.
             const overrideSessionExpiryTimeoutMs = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.SessionExpiryMs");
-            const timeoutMs = overrideSessionExpiryTimeoutMs ?? this.configs.sessionExpiryTimeoutMs;
+            let timeoutMs = this.configs.sessionExpiryTimeoutMs;
+            if (pendingSessionExpiryTimerStarted) {
+                // NOTE: This assumes the client clock hasn't been tampered with since the original session
+                const timeLapsedSincePendingTimer = Date.now() - pendingSessionExpiryTimerStarted;
+                timeoutMs -= timeLapsedSincePendingTimer;
+            }
+            timeoutMs = overrideSessionExpiryTimeoutMs ?? timeoutMs;
+            if (timeoutMs <= 0) {
+                this.runtime.closeFn(new error_js_1.ClientSessionExpiredError(`Client session expired.`, timeoutMs));
+            }
             this.sessionExpiryTimer = new core_utils_1.Timer(timeoutMs, () => {
-                this.runtime.closeFn(new error_1.ClientSessionExpiredError(`Client session expired.`, timeoutMs));
+                this.runtime.closeFn(new error_js_1.ClientSessionExpiredError(`Client session expired.`, timeoutMs));
             });
             this.sessionExpiryTimer.start();
+            this.sessionExpiryTimerStarted = Date.now();
         }
-        this.summaryStateTracker = new gcSummaryStateTracker_1.GCSummaryStateTracker(this.configs, baseSnapshot?.trees[runtime_definitions_1.gcTreeKey] !== undefined /* wasGCRunInBaseSnapshot */);
-        this.telemetryTracker = new gcTelemetry_1.GCTelemetryTracker(this.mc, this.configs, this.isSummarizerClient, createParams.createContainerMetadata, (nodeId) => this.runtime.getNodeType(nodeId), (nodeId) => this.unreferencedNodesState.get(nodeId), this.getNodePackagePath);
+        this.summaryStateTracker = new gcSummaryStateTracker_js_1.GCSummaryStateTracker(this.configs, baseSnapshot?.trees[runtime_definitions_1.gcTreeKey] !== undefined /* wasGCRunInBaseSnapshot */);
+        this.telemetryTracker = new gcTelemetry_js_1.GCTelemetryTracker(this.mc, this.configs, this.isSummarizerClient, createParams.createContainerMetadata, (nodeId) => this.runtime.getNodeType(nodeId), (nodeId) => this.unreferencedNodesState.get(nodeId), this.getNodePackagePath);
         // Get the GC data from the base snapshot. Use LazyPromise because we only want to do this once since it
         // it involves fetching blobs from storage which is expensive.
         this.baseSnapshotDataP = new core_utils_1.LazyPromise(async () => {
@@ -121,7 +136,7 @@ class GarbageCollector {
                     // incremental gc summary will not work with older gc data as well
                     return undefined;
                 }
-                const snapshotData = await (0, gcHelpers_1.getGCDataFromSnapshot)(gcSnapshotTree, readAndParseBlob);
+                const snapshotData = await (0, gcHelpers_js_1.getGCDataFromSnapshot)(gcSnapshotTree, readAndParseBlob);
                 // If the GC version in base snapshot does not match the GC version currently in effect, the GC data
                 // in the snapshot cannot be interpreted correctly. Set everything to undefined except for
                 // deletedNodes because irrespective of GC versions, these nodes have been deleted and cannot be
@@ -168,7 +183,7 @@ class GarbageCollector {
             const gcNodes = {};
             for (const [nodeId, nodeData] of Object.entries(baseSnapshotData.gcState.gcNodes)) {
                 if (nodeData.unreferencedTimestampMs !== undefined) {
-                    this.unreferencedNodesState.set(nodeId, new gcUnreferencedStateTracker_1.UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.tombstoneTimeoutMs, this.configs.sweepGracePeriodMs));
+                    this.unreferencedNodesState.set(nodeId, new gcUnreferencedStateTracker_js_1.UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.tombstoneTimeoutMs, this.configs.sweepGracePeriodMs));
                 }
                 gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
             }
@@ -188,7 +203,7 @@ class GarbageCollector {
             // Run GC on the nodes in the base summary to get the routes used in each node in the container.
             // This is an optimization for space (vs performance) wherein we don't need to store the used routes of
             // each node in the summary.
-            const usedRoutes = (0, gcReferenceGraphAlgorithm_1.runGarbageCollection)(gcNodes, ["/"]).referencedNodeIds;
+            const usedRoutes = (0, gcReferenceGraphAlgorithm_js_1.runGarbageCollection)(gcNodes, ["/"]).referencedNodeIds;
             return { gcData: { gcNodes }, usedRoutes };
         });
         // Log all the GC options and the state determined by the garbage collector.
@@ -290,7 +305,9 @@ class GarbageCollector {
      */
     async collectGarbage(options, telemetryContext) {
         const fullGC = options.fullGC ??
-            (this.configs.runFullGC === true || this.summaryStateTracker.doesSummaryStateNeedReset);
+            (this.configs.runFullGC === true ||
+                this.summaryStateTracker.autoRecovery.fullGCRequested() ||
+                this.summaryStateTracker.doesSummaryStateNeedReset);
         // Add the options that are used to run GC to the telemetry context.
         telemetryContext?.setMultiple("fluid_GC", "Options", {
             fullGC,
@@ -366,7 +383,7 @@ class GarbageCollector {
         // 1. Generate / analyze the runtime's reference graph.
         // Get the reference graph (gcData) and run GC algorithm to get referenced / unreferenced nodes.
         const gcData = await this.runtime.getGCData(fullGC);
-        const gcResult = (0, gcReferenceGraphAlgorithm_1.runGarbageCollection)(gcData.gcNodes, ["/"]);
+        const gcResult = (0, gcReferenceGraphAlgorithm_js_1.runGarbageCollection)(gcData.gcNodes, ["/"]);
         // Get all referenced nodes - References in this run + references between the previous and current runs.
         const allReferencedNodeIds = this.findAllNodesReferencedBetweenGCs(gcData, this.gcDataFromLastRun, logger) ??
             gcResult.referencedNodeIds;
@@ -377,10 +394,11 @@ class GarbageCollector {
         // It will mark nodes as referenced / unreferenced and return lists of tombstone-ready and sweep-ready nodes.
         const { tombstoneReadyNodeIds, sweepReadyNodeIds } = this.runMarkPhase(gcResult, allReferencedNodeIds, currentReferenceTimestampMs);
         // 4. Run the Sweep phase.
-        // It will tombstone any tombstone-ready nodes, and initiate the deletion of sweep-ready nodes by sending a
-        // sweep op. All clients, including this one, will delete these nodes once it processes the op.
+        // It will initiate the deletion (sending the GC Sweep op) of any sweep-ready nodes that are
+        // allowed to be deleted per config, and tombstone the rest along with the tombstone-ready nodes.
+        // Note that no nodes will be deleted until the GC Sweep op is processed.
         this.runSweepPhase(gcResult, tombstoneReadyNodeIds, sweepReadyNodeIds);
-        this.gcDataFromLastRun = (0, gcHelpers_1.cloneGCData)(gcData);
+        this.gcDataFromLastRun = (0, gcHelpers_js_1.cloneGCData)(gcData);
         // 5. Get the sweep phase stats.
         const sweepPhaseStats = this.getSweepPhaseStats(this.deletedNodes, sweepReadyNodeIds, markPhaseStats);
         return { ...markPhaseStats, ...sweepPhaseStats };
@@ -402,31 +420,25 @@ class GarbageCollector {
     runMarkPhase(gcResult, allReferencedNodeIds, currentReferenceTimestampMs) {
         // 1. Marks all referenced nodes by clearing their unreferenced tracker, if any.
         for (const nodeId of allReferencedNodeIds) {
-            const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
-            if (nodeStateTracker !== undefined) {
-                // Stop tracking so as to clear out any running timers.
-                nodeStateTracker.stopTracking();
-                // Delete the node as we don't need to track it any more.
-                this.unreferencedNodesState.delete(nodeId);
-            }
+            this.unreferencedNodesState.delete(nodeId);
         }
-        // 2. Mark unreferenced nodes in this run by starting unreferenced tracking for them.
+        // 2. Mark unreferenced nodes in this run by starting or updating unreferenced tracking for them.
         const tombstoneReadyNodeIds = new Set();
         const sweepReadyNodeIds = new Set();
         for (const nodeId of gcResult.deletedNodeIds) {
             const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
             if (nodeStateTracker === undefined) {
-                this.unreferencedNodesState.set(nodeId, new gcUnreferencedStateTracker_1.UnreferencedStateTracker(currentReferenceTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.tombstoneTimeoutMs, this.configs.sweepGracePeriodMs));
+                this.unreferencedNodesState.set(nodeId, new gcUnreferencedStateTracker_js_1.UnreferencedStateTracker(currentReferenceTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.tombstoneTimeoutMs, this.configs.sweepGracePeriodMs));
             }
             else {
                 // If a node was already unreferenced, update its tracking information. Since the current reference time
                 // is from the ops seen, this will ensure that we keep updating unreferenced state as time moves forward.
                 nodeStateTracker.updateTracking(currentReferenceTimestampMs);
                 // If a node is tombstone or sweep-ready, store it so it can be returned.
-                if (nodeStateTracker.state === gcDefinitions_1.UnreferencedState.TombstoneReady) {
+                if (nodeStateTracker.state === gcDefinitions_js_1.UnreferencedState.TombstoneReady) {
                     tombstoneReadyNodeIds.add(nodeId);
                 }
-                if (nodeStateTracker.state === gcDefinitions_1.UnreferencedState.SweepReady) {
+                if (nodeStateTracker.state === gcDefinitions_js_1.UnreferencedState.SweepReady) {
                     sweepReadyNodeIds.add(nodeId);
                 }
             }
@@ -454,45 +466,61 @@ class GarbageCollector {
          * marked as Tombstone, or deleted by Sweep.
          */
         if (this.configs.testMode) {
-            // If we are running in GC test mode, unreferenced nodes (gcResult.deletedNodeIds) are deleted.
-            this.runtime.updateUnusedRoutes(gcResult.deletedNodeIds);
+            // If we are running in GC test mode, unreferenced nodes (gcResult.deletedNodeIds) are deleted immediately.
+            this.runtime.deleteSweepReadyNodes(gcResult.deletedNodeIds);
             return;
         }
-        // If sweep is disabled, we'll tombstone both tombstone-ready and sweep-ready nodes.
+        // We'll build up the lists of nodes to be either Tombstoned or Deleted
+        // based on the configuration and the nodes' current state.
+        // We must Tombstone any sweep-ready node that Sweep won't run for.
         // This is important because a container may never load during a node's Sweep Grace Period,
         // so that node would directly become sweep-ready skipping over tombstone-ready state,
         // but should be Tombstoned since Sweep is disabled.
-        const { nodesToTombstone, nodesToDelete } = this.configs.shouldRunSweep
-            ? {
-                nodesToTombstone: [...tombstoneReadyNodes],
-                nodesToDelete: [...sweepReadyNodes],
-            }
-            : {
-                nodesToTombstone: [...tombstoneReadyNodes, ...sweepReadyNodes],
-                nodesToDelete: [],
-            };
+        const { nodesToTombstone, nodesToDelete } = {
+            nodesToTombstone: [...tombstoneReadyNodes],
+            nodesToDelete: [],
+        };
+        switch (this.configs.shouldRunSweep) {
+            case "YES":
+                nodesToDelete.push(...sweepReadyNodes);
+                break;
+            case "ONLY_BLOBS":
+                sweepReadyNodes.forEach((nodeId) => {
+                    const nodeType = this.runtime.getNodeType(nodeId);
+                    if (nodeType === gcDefinitions_js_1.GCNodeType.Blob) {
+                        nodesToDelete.push(nodeId);
+                    }
+                    else {
+                        nodesToTombstone.push(nodeId);
+                    }
+                });
+                break;
+            default: // case "NO":
+                nodesToTombstone.push(...sweepReadyNodes);
+                break;
+        }
         if (this.configs.tombstoneMode) {
             this.tombstones = nodesToTombstone;
             // If we are running in GC tombstone mode, update tombstoned routes.
             this.runtime.updateTombstonedRoutes(this.tombstones);
         }
-        if (this.configs.shouldRunSweep && nodesToDelete.length > 0) {
+        if (nodesToDelete.length > 0) {
             // Do not send DDS node ids in the GC op. This is an optimization to reduce its size. Since GC applies to
             // to data store only, all its DDSes are deleted along with it. The DDS ids will be retrieved from the
             // local state when processing the op.
             const sweepReadyDSAndBlobs = nodesToDelete.filter((nodeId) => {
                 const nodeType = this.runtime.getNodeType(nodeId);
-                return nodeType === gcDefinitions_1.GCNodeType.DataStore || nodeType === gcDefinitions_1.GCNodeType.Blob;
+                return nodeType === gcDefinitions_js_1.GCNodeType.DataStore || nodeType === gcDefinitions_js_1.GCNodeType.Blob;
             });
             const contents = {
-                type: gcDefinitions_1.GarbageCollectionMessageType.Sweep,
+                type: gcDefinitions_js_1.GarbageCollectionMessageType.Sweep,
                 deletedNodeIds: sweepReadyDSAndBlobs,
             };
             // Its fine for older clients to ignore this op because it doesn't have any functional impact. This op
             // is an optimization to ensure that all clients are in sync when it comes to deleted nodes to prevent their
             // accidental usage. The clients will sync without the delete op too but it may take longer.
             const containerGCMessage = {
-                type: messageTypes_1.ContainerMessageType.GC,
+                type: messageTypes_js_1.ContainerMessageType.GC,
                 contents,
                 compatDetails: { behavior: "Ignore" },
             };
@@ -547,7 +575,7 @@ class GarbageCollector {
          * - We don't require DDSes handles to be stored in a referenced DDS.
          * - A new data store may have "root" DDSes already created and we don't detect them today.
          */
-        const gcDataSuperSet = (0, gcHelpers_1.concatGarbageCollectionData)(previousGCData, currentGCData);
+        const gcDataSuperSet = (0, gcHelpers_js_1.concatGarbageCollectionData)(previousGCData, currentGCData);
         const newOutboundRoutesSinceLastRun = [];
         this.newReferencesSinceLastRun.forEach((outboundRoutes, sourceNodeId) => {
             if (gcDataSuperSet.gcNodes[sourceNodeId] === undefined) {
@@ -565,7 +593,7 @@ class GarbageCollector {
          * Note that some of these nodes may be unreferenced now and if so, the current run will mark them as
          * unreferenced and add unreferenced state.
          */
-        const gcResult = (0, gcReferenceGraphAlgorithm_1.runGarbageCollection)(gcDataSuperSet.gcNodes, [
+        const gcResult = (0, gcReferenceGraphAlgorithm_js_1.runGarbageCollection)(gcDataSuperSet.gcNodes, [
             "/",
             ...newOutboundRoutesSinceLastRun,
         ]);
@@ -587,7 +615,7 @@ class GarbageCollector {
                 unreferencedTimestampMs: this.unreferencedNodesState.get(nodeId)?.unreferencedTimestampMs,
             };
         }
-        return this.summaryStateTracker.summarize(fullTree, trackState, gcState, this.deletedNodes, this.tombstones);
+        return this.summaryStateTracker.summarize(trackState && !fullTree, gcState, this.deletedNodes, this.tombstones);
     }
     getMetadata() {
         return {
@@ -614,15 +642,28 @@ class GarbageCollector {
      * @param local - Whether it was send by this client.
      */
     processMessage(message, local) {
-        switch (message.contents.type) {
-            case "Sweep": {
+        const gcMessageType = message.contents.type;
+        switch (gcMessageType) {
+            case gcDefinitions_js_1.GarbageCollectionMessageType.Sweep: {
                 // Delete the nodes whose ids are present in the contents.
                 this.deleteSweepReadyNodes(message.contents.deletedNodeIds);
                 break;
             }
+            case gcDefinitions_js_1.GarbageCollectionMessageType.TombstoneLoaded: {
+                if (this.mc.config.getBoolean(gcDefinitions_js_1.disableAutoRecoveryKey) === true) {
+                    break;
+                }
+                // Mark the node as referenced to ensure it isn't Swept
+                const tombstonedNodePath = message.contents.nodePath;
+                this.addedOutboundReference("/", tombstonedNodePath, true /* autorecovery */);
+                // In case the cause of the TombstoneLoaded event is incorrect GC Data (i.e. the object is actually reachable),
+                // do fullGC on the next run to get a chance to repair (in the likely case the bug is not deterministic)
+                this.summaryStateTracker.autoRecovery.requestFullGCOnNextRun();
+                break;
+            }
             default: {
-                if (!(0, gcHelpers_1.compatBehaviorAllowsGCMessageType)(message.contents.type, message.compatDetails?.behavior)) {
-                    const error = telemetry_utils_1.DataProcessingError.create(`Garbage collection message of unknown type ${message.contents.type}`, "processMessage");
+                if (!(0, gcHelpers_js_1.compatBehaviorAllowsGCMessageType)(gcMessageType, message.compatDetails?.behavior)) {
+                    const error = telemetry_utils_1.DataProcessingError.create(`Garbage collection message of unknown type ${gcMessageType}`, "processMessage");
                     throw error;
                 }
                 break;
@@ -632,6 +673,10 @@ class GarbageCollector {
     /**
      * Delete nodes that are sweep-ready. Call the runtime to delete these nodes and clear the unreferenced state
      * tracking for nodes that are actually deleted by the runtime.
+     *
+     * Note that this doesn't check any configuration around whether Sweep is enabled.
+     * That happens before the op is submitted, and from that point, any client should execute the delete.
+     *
      * @param sweepReadyNodeIds - The ids of nodes that are ready to be deleted.
      */
     deleteSweepReadyNodes(sweepReadyNodeIds) {
@@ -657,13 +702,9 @@ class GarbageCollector {
         const deletedNodeIds = this.runtime.deleteSweepReadyNodes(allSweepReadyNodeIds);
         // Clear unreferenced state tracking for deleted nodes.
         for (const nodeId of deletedNodeIds) {
-            const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
-            if (nodeStateTracker !== undefined) {
-                // Stop tracking so as to clear out any running timers.
-                nodeStateTracker.stopTracking();
-                // Delete the node as we don't need to track it any more.
-                this.unreferencedNodesState.delete(nodeId);
-            }
+            // Usually we avoid modifying the set of unreferencedNodesState keys in between GC runs,
+            // but this is ok since this node won't exist at all in the next GC run.
+            this.unreferencedNodesState.delete(nodeId);
             this.deletedNodes.add(nodeId);
         }
     }
@@ -693,17 +734,24 @@ class GarbageCollector {
             lastSummaryTime: this.getLastSummaryTimestampMs(),
             headers: headerData,
         });
+        // Any time we log a Tombstone Loaded error (via Telemetry Tracker),
+        // we want to also trigger autorecovery to avoid the object being deleted
+        // Note: We don't need to trigger on "Changed" because any change will cause the object
+        // to be loaded by the Summarizer, and auto-recovery will be triggered then.
+        if (isTombstoned && reason === "Loaded") {
+            // Note that when a DataStore and its DDS are all loaded, each will trigger AutoRecovery for itself.
+            this.triggerAutoRecovery(nodePath);
+        }
         const nodeType = this.runtime.getNodeType(nodePath);
         // Unless this is a Loaded event for a Blob or DataStore, we're done after telemetry tracking
-        if (reason !== "Loaded" || ![gcDefinitions_1.GCNodeType.Blob, gcDefinitions_1.GCNodeType.DataStore].includes(nodeType)) {
+        if (reason !== "Loaded" || ![gcDefinitions_js_1.GCNodeType.Blob, gcDefinitions_js_1.GCNodeType.DataStore].includes(nodeType)) {
             return;
         }
         const errorRequest = request ?? { url: nodePath };
-        // If the object is tombstoned and tombstone enforcement is configured, throw an error.
         if (isTombstoned && this.throwOnTombstoneLoad && headerData?.allowTombstone !== true) {
             // The requested data store is removed by gc. Create a 404 gc response exception.
             throw (0, runtime_utils_1.responseToException)((0, runtime_utils_1.createResponseError)(404, `${nodeType} was tombstoned`, errorRequest, {
-                [containerRuntime_1.TombstoneResponseHeaderKey]: true,
+                [containerRuntime_js_1.TombstoneResponseHeaderKey]: true,
             }), errorRequest);
         }
         // If the object is inactive and inactive enforcement is configured, throw an error.
@@ -713,19 +761,45 @@ class GarbageCollector {
                 headerData?.allowInactive !== true;
             if (shouldThrowOnInactiveLoad) {
                 throw (0, runtime_utils_1.responseToException)((0, runtime_utils_1.createResponseError)(404, `${nodeType} is inactive`, errorRequest, {
-                    [containerRuntime_1.InactiveResponseHeaderKey]: true,
+                    [containerRuntime_js_1.InactiveResponseHeaderKey]: true,
                 }), errorRequest);
             }
         }
     }
+    /**
+     * The given node should have its unreferenced state reset in the next GC,
+     * even if the true GC graph shows it is unreferenced. This will
+     * prevent it from being deleted by Sweep (after the Grace Period).
+     *
+     * Submit a GC op indicating that the Tombstone with the given path has been loaded.
+     * Broadcasting this information in the op stream allows the Summarizer to reset unreferenced state
+     * before runnint GC next.
+     */
+    triggerAutoRecovery(nodePath) {
+        if (this.mc.config.getBoolean(gcDefinitions_js_1.disableAutoRecoveryKey) === true) {
+            return;
+        }
+        // Use compat behavior "Ignore" since this is an optimization to opportunistically protect
+        // objects from deletion, so it's fine for older clients to ignore this op.
+        const containerGCMessage = {
+            type: messageTypes_js_1.ContainerMessageType.GC,
+            contents: {
+                type: gcDefinitions_js_1.GarbageCollectionMessageType.TombstoneLoaded,
+                nodePath,
+            },
+            compatDetails: { behavior: "Ignore" },
+        };
+        this.submitMessage(containerGCMessage);
+    }
     /**
      * Called when an outbound reference is added to a node. This is used to identify all nodes that have been
      * referenced between summaries so that their unreferenced timestamp can be reset.
      *
      * @param fromNodePath - The node from which the reference is added.
      * @param toNodePath - The node to which the reference is added.
+     * @param autorecovery - This reference is added artificially, for autorecovery. Used for logging.
      */
-    addedOutboundReference(fromNodePath, toNodePath) {
+    addedOutboundReference(fromNodePath, toNodePath, autorecovery) {
         if (!this.configs.shouldRunGC) {
             return;
         }
@@ -751,7 +825,13 @@ class GarbageCollector {
             isTombstoned: this.tombstones.includes(toNodePath),
             lastSummaryTime: this.getLastSummaryTimestampMs(),
             fromId: fromNodePath,
+            autorecovery,
         });
+        // This node is referenced - Clear its unreferenced state
+        // But don't delete the node id from the map yet.
+        // When generating GC stats, the set of nodes in here is used as the baseline for
+        // what was unreferenced in the last GC run.
+        this.unreferencedNodesState.get(toNodePath)?.stopTracking();
     }
     /**
      * Returns whether a node with the given path has been deleted or not. This can be used by the runtime to identify
@@ -781,33 +861,33 @@ class GarbageCollector {
             updatedDataStoreCount: 0,
             updatedAttachmentBlobCount: 0,
         };
-        const updateNodeStats = (nodeId, referenced) => {
+        const updateNodeStats = (nodeId, isReferenced) => {
             markPhaseStats.nodeCount++;
             // If there is no previous GC data, every node's state is generated and is considered as updated.
             // Otherwise, find out if any node went from referenced to unreferenced or vice-versa.
-            const stateUpdated = this.gcDataFromLastRun === undefined ||
-                this.unreferencedNodesState.has(nodeId) === referenced;
+            const wasNotReferenced = this.unreferencedNodesState.has(nodeId);
+            const stateUpdated = this.gcDataFromLastRun === undefined || wasNotReferenced === isReferenced;
             if (stateUpdated) {
                 markPhaseStats.updatedNodeCount++;
             }
-            if (!referenced) {
+            if (!isReferenced) {
                 markPhaseStats.unrefNodeCount++;
             }
-            if (this.runtime.getNodeType(nodeId) === gcDefinitions_1.GCNodeType.DataStore) {
+            if (this.runtime.getNodeType(nodeId) === gcDefinitions_js_1.GCNodeType.DataStore) {
                 markPhaseStats.dataStoreCount++;
                 if (stateUpdated) {
                     markPhaseStats.updatedDataStoreCount++;
                 }
-                if (!referenced) {
+                if (!isReferenced) {
                     markPhaseStats.unrefDataStoreCount++;
                 }
             }
-            if (this.runtime.getNodeType(nodeId) === gcDefinitions_1.GCNodeType.Blob) {
+            if (this.runtime.getNodeType(nodeId) === gcDefinitions_js_1.GCNodeType.Blob) {
                 markPhaseStats.attachmentBlobCount++;
                 if (stateUpdated) {
                     markPhaseStats.updatedAttachmentBlobCount++;
                 }
-                if (!referenced) {
+                if (!isReferenced) {
                     markPhaseStats.unrefAttachmentBlobCount++;
                 }
             }
@@ -822,8 +902,9 @@ class GarbageCollector {
     }
     /**
      * Generates the stats of a garbage collection sweep phase run.
-     * @param deletedNodes - The nodes that have been deleted until this run.
-     * @param sweepReadyNodes - The nodes that are sweep-ready in this GC run.
+     * @param deletedNodes - The nodes that have already been deleted even before this run.
+     * @param sweepReadyNodes - The nodes that are sweep-ready in this GC run. These will be deleted but are not deleted yet,
+     * due to either sweep not being enabled or the Sweep Op needing to roundtrip before the delete is executed.
      * @param markPhaseStats - The stats of the mark phase run.
      * @returns the stats of the sweep phase run.
      */
@@ -842,45 +923,45 @@ class GarbageCollector {
         // be good enough because the only types that participate in GC today are data stores, DDSes and blobs.
         const getDeletedNodeType = (nodeId) => {
             const pathParts = nodeId.split("/");
-            if (pathParts[1] === blobManager_1.BlobManager.basePath) {
-                return gcDefinitions_1.GCNodeType.Blob;
+            if (pathParts[1] === blobManager_js_1.BlobManager.basePath) {
+                return gcDefinitions_js_1.GCNodeType.Blob;
             }
             if (pathParts.length === 2) {
-                return gcDefinitions_1.GCNodeType.DataStore;
+                return gcDefinitions_js_1.GCNodeType.DataStore;
             }
             if (pathParts.length === 3) {
-                return gcDefinitions_1.GCNodeType.SubDataStore;
+                return gcDefinitions_js_1.GCNodeType.SubDataStore;
             }
-            return gcDefinitions_1.GCNodeType.Other;
+            return gcDefinitions_js_1.GCNodeType.Other;
         };
         for (const nodeId of deletedNodes) {
             sweepPhaseStats.deletedNodeCount++;
             const nodeType = getDeletedNodeType(nodeId);
-            if (nodeType === gcDefinitions_1.GCNodeType.DataStore) {
+            if (nodeType === gcDefinitions_js_1.GCNodeType.DataStore) {
                 sweepPhaseStats.deletedDataStoreCount++;
             }
-            else if (nodeType === gcDefinitions_1.GCNodeType.Blob) {
+            else if (nodeType === gcDefinitions_js_1.GCNodeType.Blob) {
                 sweepPhaseStats.deletedAttachmentBlobCount++;
             }
         }
-        // If sweep is enabled, the counts from the mark phase stats do not include nodes that have been
+        // The counts from the mark phase stats do not include nodes that were
         // deleted in previous runs. So, add the deleted node counts to life time stats.
         sweepPhaseStats.lifetimeNodeCount += sweepPhaseStats.deletedNodeCount;
         sweepPhaseStats.lifetimeDataStoreCount += sweepPhaseStats.deletedDataStoreCount;
         sweepPhaseStats.lifetimeAttachmentBlobCount += sweepPhaseStats.deletedAttachmentBlobCount;
-        if (this.configs.shouldRunSweep) {
-            return sweepPhaseStats;
-        }
-        // If sweep is not enabled, the current sweep-ready node stats should be added to deleted stats since this
-        // is the final state the node will be in.
-        // If sweep is enabled, this will happen in the run after the GC op round trips back.
+        // These stats are used to estimate the impact of GC in terms of how much garbage is/will be cleaned up.
+        // So we include the current sweep-ready node stats since these nodes will be deleted eventually.
+        // - If sweep is enabled, this will happen in the run after the GC op round trips back
+        //   (they'll be in deletedNodes that time).
+        // - If sweep is not enabled, we still want to include these nodes since they
+        //   _will be_ deleted once it is enabled.
         for (const nodeId of sweepReadyNodes) {
             sweepPhaseStats.deletedNodeCount++;
             const nodeType = this.runtime.getNodeType(nodeId);
-            if (nodeType === gcDefinitions_1.GCNodeType.DataStore) {
+            if (nodeType === gcDefinitions_js_1.GCNodeType.DataStore) {
                 sweepPhaseStats.deletedDataStoreCount++;
             }
-            else if (nodeType === gcDefinitions_1.GCNodeType.Blob) {
+            else if (nodeType === gcDefinitions_js_1.GCNodeType.Blob) {
                 sweepPhaseStats.deletedAttachmentBlobCount++;
             }
         }