@fluidframework/container-runtime 2.0.0-dev-rc.1.0.0.228517 → 2.0.0-dev-rc.2.0.0.245554

package/lib/gc/{garbageCollection.d.mts → garbageCollection.d.ts}

@@ -5,10 +5,10 @@
 import { IRequest } from "@fluidframework/core-interfaces";
 import { IGarbageCollectionDetailsBase, ISummarizeResult, ITelemetryContext } from "@fluidframework/runtime-definitions";
 import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
-import { RuntimeHeaderData } from "../containerRuntime.mjs";
-import { ContainerRuntimeGCMessage } from "../messageTypes.mjs";
-import { IRefreshSummaryResult } from "../summary/index.mjs";
-import { IGarbageCollector, IGarbageCollectorCreateParams, IGCStats, IGCMetadata } from "./gcDefinitions.mjs";
+import { RuntimeHeaderData } from "../containerRuntime.js";
+import { ContainerRuntimeGCMessage } from "../messageTypes.js";
+import { IRefreshSummaryResult } from "../summary/index.js";
+import { IGarbageCollector, IGarbageCollectorCreateParams, IGCStats, IGCMetadata } from "./gcDefinitions.js";
 /**
  * The garbage collector for the container runtime. It consolidates the garbage collection functionality and maintains
  * its state across summaries.
@@ -36,6 +36,7 @@ export declare class GarbageCollector implements IGarbageCollector {
     private readonly mc;
     private readonly configs;
     get shouldRunGC(): boolean;
+    readonly sessionExpiryTimerStarted: number | undefined;
     private gcDataFromLastRun;
     private readonly newReferencesSinceLastRun;
     private tombstones;
@@ -43,6 +44,11 @@ export declare class GarbageCollector implements IGarbageCollector {
     private readonly baseSnapshotDataP;
     private readonly initializeGCStateFromBaseSnapshotP;
     private readonly baseGCDetailsP;
+    /**
+     * Map of node ids to their unreferenced state tracker
+     * NOTE: The set of keys in this map is considered as the set of unreferenced nodes
+     * as of the last GC run. So in between runs, nothing should be added or removed.
+     */
     private readonly unreferencedNodesState;
     private sessionExpiryTimer;
     private completedRuns;
@@ -178,6 +184,10 @@ export declare class GarbageCollector implements IGarbageCollector {
     /**
      * Delete nodes that are sweep-ready. Call the runtime to delete these nodes and clear the unreferenced state
      * tracking for nodes that are actually deleted by the runtime.
+     *
+     * Note that this doesn't check any configuration around whether Sweep is enabled.
+     * That happens before the op is submitted, and from that point, any client should execute the delete.
+     *
      * @param sweepReadyNodeIds - The ids of nodes that are ready to be deleted.
      */
     private deleteSweepReadyNodes;
@@ -192,14 +202,25 @@ export declare class GarbageCollector implements IGarbageCollector {
      * @param requestHeaders - If the node was loaded via request path, the headers in the request.
      */
     nodeUpdated(nodePath: string, reason: "Loaded" | "Changed", timestampMs?: number, packagePath?: readonly string[], request?: IRequest, headerData?: RuntimeHeaderData): void;
+    /**
+     * The given node should have its unreferenced state reset in the next GC,
+     * even if the true GC graph shows it is unreferenced. This will
+     * prevent it from being deleted by Sweep (after the Grace Period).
+     *
+     * Submit a GC op indicating that the Tombstone with the given path has been loaded.
+     * Broadcasting this information in the op stream allows the Summarizer to reset unreferenced state
+     * before runnint GC next.
+     */
+    private triggerAutoRecovery;
     /**
      * Called when an outbound reference is added to a node. This is used to identify all nodes that have been
      * referenced between summaries so that their unreferenced timestamp can be reset.
      *
      * @param fromNodePath - The node from which the reference is added.
      * @param toNodePath - The node to which the reference is added.
+     * @param autorecovery - This reference is added artificially, for autorecovery. Used for logging.
      */
-    addedOutboundReference(fromNodePath: string, toNodePath: string): void;
+    addedOutboundReference(fromNodePath: string, toNodePath: string, autorecovery?: true): void;
     /**
      * Returns whether a node with the given path has been deleted or not. This can be used by the runtime to identify
      * cases where objects are used after they are deleted and throw / log errors accordingly.
@@ -214,11 +235,12 @@ export declare class GarbageCollector implements IGarbageCollector {
     private getMarkPhaseStats;
     /**
      * Generates the stats of a garbage collection sweep phase run.
-     * @param deletedNodes - The nodes that have been deleted until this run.
-     * @param sweepReadyNodes - The nodes that are sweep-ready in this GC run.
+     * @param deletedNodes - The nodes that have already been deleted even before this run.
+     * @param sweepReadyNodes - The nodes that are sweep-ready in this GC run. These will be deleted but are not deleted yet,
+     * due to either sweep not being enabled or the Sweep Op needing to roundtrip before the delete is executed.
      * @param markPhaseStats - The stats of the mark phase run.
      * @returns the stats of the sweep phase run.
      */
     private getSweepPhaseStats;
 }
-//# sourceMappingURL=garbageCollection.d.mts.map
+//# sourceMappingURL=garbageCollection.d.ts.map

package/lib/gc/garbageCollection.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"garbageCollection.d.ts","sourceRoot":"","sources":["../../src/gc/garbageCollection.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAC3D,OAAO,EAGN,6BAA6B,EAC7B,gBAAgB,EAChB,iBAAiB,EACjB,MAAM,qCAAqC,CAAC;AAE7C,OAAO,EAIN,mBAAmB,EAInB,MAAM,iCAAiC,CAAC;AAEzC,OAAO,EAEN,iBAAiB,EAEjB,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAAwB,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AACrF,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAE5D,OAAO,EAEN,iBAAiB,EACjB,6BAA6B,EAG7B,QAAQ,EAER,WAAW,EAOX,MAAM,oBAAoB,CAAC;AAgB5B;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBAAa,gBAAiB,YAAW,iBAAiB;WAC3C,MAAM,CAAC,YAAY,EAAE,6BAA6B,GAAG,iBAAiB;IAIpF,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAoB;IAEvC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA2B;IAEnD,IAAW,WAAW,IAAI,OAAO,CAEhC;IAED,SAAgB,yBAAyB,EAAE,MAAM,GAAG,SAAS,CAAC;IAE9D,OAAO,CAAC,iBAAiB,CAAqC;IAG9D,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAAoC;IAE9E,OAAO,CAAC,UAAU,CAAgB;IAElC,OAAO,CAAC,YAAY,CAA0B;IAG9C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAsD;IAExF,OAAO,CAAC,QAAQ,CAAC,kCAAkC,CAAgB;IAEnE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAyC;IAExE;;;;OAIG;IACH,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CACJ;IAGnC,OAAO,CAAC,kBAAkB,CAAoB;IAG9C,OAAO,CAAC,aAAa,CAAK;IAE1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA4B;IACpD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAU;IAE7C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAwB;IAC5D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAqB;IAEtD,iFAAiF;IACjF,IAAW,2BAA2B,IAAI,OAAO,CAEhD;IACD,uEAAuE;IACvE,IAAW,oBAAoB,IAAI,OAAO,CAEzC;IACD,kEAAkE;IAClE,IAAW,qBAAqB,IAAI,OAAO,CAE1C;IAED,8DAA8D;IAC9D,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAES;IAC5C,8EAA8E;IAC9E,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAA2B;IAErE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA+C;IAE7E,IAAW,sBAAsB,IAAI,OAAO,CAE3C;IAED,sFAAsF;IACtF,IAAW,8BAA8B,IAAI,MAAM,CAElD;IAED,SAAS,aAAa,YAAY,EAAE,6BAA6B;IA4LjE;;;;;OAKG;IACU,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC;IA6BjD;;;OAGG;YACW,yBAAyB;IAmBvC;;;;;OAKG;IACI,kBAAkB,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAwBlF;;;OAGG;IACU,gBAAgB,IAAI,OAAO,CAAC,6BAA6B,CAAC;IAIvE;;;OAGG;IACU,cAAc,CAC1B,OAAO,EAAE;QACR,0CAA0C;QAC1C,MAAM,CAAC,EAAE,mBAAmB,CAAC;QAC7B,sDAAsD;QACtD,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,oCAAoC;QACpC,MAAM,CAAC,EAAE,OAAO,CAAC;KACjB,EACD,gBAAgB,CAAC,EAAE,iBAAiB,GAClC,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC;IA6EhC;;;;;;;;;;;;OAYG;YACW,KAAK;IA4CnB;;;;;;;;;;;;;OAaG;IACH,OAAO,CAAC,YAAY;IA+CpB;;;;;;;;;;OAUG;IACH,OAAO,CAAC,aAAa;IA+ErB;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,gCAAgC;IAqExC;;;;OAIG;IACI,SAAS,CACf,QAAQ,EAAE,OAAO,EACjB,UAAU,EAAE,OAAO,EACnB,gBAAgB,CAAC,EAAE,iBAAiB,GAClC,gBAAgB,GAAG,SAAS;IAsBxB,WAAW,IAAI,WAAW;IAcjC;;OAEG;IACU,oBAAoB,CAAC,MAAM,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC;IAI/E;;;;OAIG;IACI,cAAc,CAAC,OAAO,EAAE,yBAAyB,EAAE,KAAK,EAAE,OAAO;IAyCxE;;;;;;;;OAQG;IACH,OAAO,CAAC,qBAAqB;IAiC7B;;;;;;;;;OASG;IACI,WAAW,CACjB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,QAAQ,GAAG,SAAS,EAC5B,WAAW,CAAC,EAAE,MAAM,EACpB,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,EAC/B,OAAO,CAAC,EAAE,QAAQ,EAClB,UAAU,CAAC,EAAE,iBAAiB;IAiE/B;;;;;;;;OAQG;IACH,OAAO,CAAC,mBAAmB;IAkB3B;;;;;;;OAOG;IACI,sBAAsB,CAAC,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,IAAI;IAwC3F;;;OAGG;IACI,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIxC,OAAO,IAAI,IAAI;IAKtB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IA0DzB;;;;;;;OAOG;IACH,OAAO,CAAC,kBAAkB;CAkE1B"}

package/lib/gc/{garbageCollection.mjs → garbageCollection.js}

@@ -6,17 +6,17 @@ import { assert, LazyPromise, Timer } from "@fluidframework/core-utils";
 import { gcTreeKey, } from "@fluidframework/runtime-definitions";
 import { createResponseError, responseToException } from "@fluidframework/runtime-utils";
 import { createChildLogger, createChildMonitoringContext, DataProcessingError, PerformanceEvent, tagCodeArtifacts, } from "@fluidframework/telemetry-utils";
-import { BlobManager } from "../blobManager.mjs";
-import { InactiveResponseHeaderKey, TombstoneResponseHeaderKey, } from "../containerRuntime.mjs";
-import { ClientSessionExpiredError } from "../error.mjs";
-import { ContainerMessageType } from "../messageTypes.mjs";
-import { generateGCConfigs } from "./gcConfigs.mjs";
-import { GCNodeType, UnreferencedState, GarbageCollectionMessageType, } from "./gcDefinitions.mjs";
-import { cloneGCData, compatBehaviorAllowsGCMessageType, concatGarbageCollectionData, getGCDataFromSnapshot, } from "./gcHelpers.mjs";
-import { runGarbageCollection } from "./gcReferenceGraphAlgorithm.mjs";
-import { GCSummaryStateTracker } from "./gcSummaryStateTracker.mjs";
-import { UnreferencedStateTracker } from "./gcUnreferencedStateTracker.mjs";
-import { GCTelemetryTracker } from "./gcTelemetry.mjs";
+import { BlobManager } from "../blobManager.js";
+import { InactiveResponseHeaderKey, TombstoneResponseHeaderKey, } from "../containerRuntime.js";
+import { ClientSessionExpiredError } from "../error.js";
+import { ContainerMessageType } from "../messageTypes.js";
+import { generateGCConfigs } from "./gcConfigs.js";
+import { GCNodeType, UnreferencedState, GarbageCollectionMessageType, disableAutoRecoveryKey, } from "./gcDefinitions.js";
+import { cloneGCData, compatBehaviorAllowsGCMessageType, concatGarbageCollectionData, getGCDataFromSnapshot, } from "./gcHelpers.js";
+import { runGarbageCollection } from "./gcReferenceGraphAlgorithm.js";
+import { GCSummaryStateTracker } from "./gcSummaryStateTracker.js";
+import { UnreferencedStateTracker, UnreferencedStateTrackerMap, } from "./gcUnreferencedStateTracker.js";
+import { GCTelemetryTracker } from "./gcTelemetry.js";
 /**
  * The garbage collector for the container runtime. It consolidates the garbage collection functionality and maintains
  * its state across summaries.
@@ -73,8 +73,12 @@ export class GarbageCollector {
         this.tombstones = [];
         // A list of nodes that have been deleted during sweep phase.
         this.deletedNodes = new Set();
-        // Map of node ids to their unreferenced state tracker.
-        this.unreferencedNodesState = new Map();
+        /**
+         * Map of node ids to their unreferenced state tracker
+         * NOTE: The set of keys in this map is considered as the set of unreferenced nodes
+         * as of the last GC run. So in between runs, nothing should be added or removed.
+         */
+        this.unreferencedNodesState = new UnreferencedStateTrackerMap();
         // The number of times GC has successfully completed on this instance of GarbageCollector.
         this.completedRuns = 0;
         this.runtime = createParams.runtime;
@@ -84,6 +88,7 @@ export class GarbageCollector {
         this.submitMessage = createParams.submitMessage;
         const baseSnapshot = createParams.baseSnapshot;
         const readAndParseBlob = createParams.readAndParseBlob;
+        const pendingSessionExpiryTimerStarted = createParams.sessionExpiryTimerStarted;
         this.mc = createChildMonitoringContext({
             logger: createParams.baseLogger,
             namespace: "GarbageCollector",
@@ -96,11 +101,21 @@ export class GarbageCollector {
         if (this.configs.sessionExpiryTimeoutMs !== undefined) {
             // If Test Override config is set, override Session Expiry timeout.
             const overrideSessionExpiryTimeoutMs = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.SessionExpiryMs");
-            const timeoutMs = overrideSessionExpiryTimeoutMs ?? this.configs.sessionExpiryTimeoutMs;
+            let timeoutMs = this.configs.sessionExpiryTimeoutMs;
+            if (pendingSessionExpiryTimerStarted) {
+                // NOTE: This assumes the client clock hasn't been tampered with since the original session
+                const timeLapsedSincePendingTimer = Date.now() - pendingSessionExpiryTimerStarted;
+                timeoutMs -= timeLapsedSincePendingTimer;
+            }
+            timeoutMs = overrideSessionExpiryTimeoutMs ?? timeoutMs;
+            if (timeoutMs <= 0) {
+                this.runtime.closeFn(new ClientSessionExpiredError(`Client session expired.`, timeoutMs));
+            }
             this.sessionExpiryTimer = new Timer(timeoutMs, () => {
                 this.runtime.closeFn(new ClientSessionExpiredError(`Client session expired.`, timeoutMs));
             });
             this.sessionExpiryTimer.start();
+            this.sessionExpiryTimerStarted = Date.now();
         }
         this.summaryStateTracker = new GCSummaryStateTracker(this.configs, baseSnapshot?.trees[gcTreeKey] !== undefined /* wasGCRunInBaseSnapshot */);
         this.telemetryTracker = new GCTelemetryTracker(this.mc, this.configs, this.isSummarizerClient, createParams.createContainerMetadata, (nodeId) => this.runtime.getNodeType(nodeId), (nodeId) => this.unreferencedNodesState.get(nodeId), this.getNodePackagePath);
@@ -287,7 +302,9 @@ export class GarbageCollector {
      */
     async collectGarbage(options, telemetryContext) {
         const fullGC = options.fullGC ??
-            (this.configs.runFullGC === true || this.summaryStateTracker.doesSummaryStateNeedReset);
+            (this.configs.runFullGC === true ||
+                this.summaryStateTracker.autoRecovery.fullGCRequested() ||
+                this.summaryStateTracker.doesSummaryStateNeedReset);
         // Add the options that are used to run GC to the telemetry context.
         telemetryContext?.setMultiple("fluid_GC", "Options", {
             fullGC,
@@ -374,8 +391,9 @@ export class GarbageCollector {
         // It will mark nodes as referenced / unreferenced and return lists of tombstone-ready and sweep-ready nodes.
         const { tombstoneReadyNodeIds, sweepReadyNodeIds } = this.runMarkPhase(gcResult, allReferencedNodeIds, currentReferenceTimestampMs);
         // 4. Run the Sweep phase.
-        // It will tombstone any tombstone-ready nodes, and initiate the deletion of sweep-ready nodes by sending a
-        // sweep op. All clients, including this one, will delete these nodes once it processes the op.
+        // It will initiate the deletion (sending the GC Sweep op) of any sweep-ready nodes that are
+        // allowed to be deleted per config, and tombstone the rest along with the tombstone-ready nodes.
+        // Note that no nodes will be deleted until the GC Sweep op is processed.
         this.runSweepPhase(gcResult, tombstoneReadyNodeIds, sweepReadyNodeIds);
         this.gcDataFromLastRun = cloneGCData(gcData);
         // 5. Get the sweep phase stats.
@@ -399,15 +417,9 @@ export class GarbageCollector {
     runMarkPhase(gcResult, allReferencedNodeIds, currentReferenceTimestampMs) {
         // 1. Marks all referenced nodes by clearing their unreferenced tracker, if any.
         for (const nodeId of allReferencedNodeIds) {
-            const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
-            if (nodeStateTracker !== undefined) {
-                // Stop tracking so as to clear out any running timers.
-                nodeStateTracker.stopTracking();
-                // Delete the node as we don't need to track it any more.
-                this.unreferencedNodesState.delete(nodeId);
-            }
+            this.unreferencedNodesState.delete(nodeId);
         }
-        // 2. Mark unreferenced nodes in this run by starting unreferenced tracking for them.
+        // 2. Mark unreferenced nodes in this run by starting or updating unreferenced tracking for them.
         const tombstoneReadyNodeIds = new Set();
         const sweepReadyNodeIds = new Set();
         for (const nodeId of gcResult.deletedNodeIds) {
@@ -451,29 +463,45 @@ export class GarbageCollector {
          * marked as Tombstone, or deleted by Sweep.
          */
         if (this.configs.testMode) {
-            // If we are running in GC test mode, unreferenced nodes (gcResult.deletedNodeIds) are deleted.
-            this.runtime.updateUnusedRoutes(gcResult.deletedNodeIds);
+            // If we are running in GC test mode, unreferenced nodes (gcResult.deletedNodeIds) are deleted immediately.
+            this.runtime.deleteSweepReadyNodes(gcResult.deletedNodeIds);
             return;
         }
-        // If sweep is disabled, we'll tombstone both tombstone-ready and sweep-ready nodes.
+        // We'll build up the lists of nodes to be either Tombstoned or Deleted
+        // based on the configuration and the nodes' current state.
+        // We must Tombstone any sweep-ready node that Sweep won't run for.
         // This is important because a container may never load during a node's Sweep Grace Period,
         // so that node would directly become sweep-ready skipping over tombstone-ready state,
         // but should be Tombstoned since Sweep is disabled.
-        const { nodesToTombstone, nodesToDelete } = this.configs.shouldRunSweep
-            ? {
-                nodesToTombstone: [...tombstoneReadyNodes],
-                nodesToDelete: [...sweepReadyNodes],
-            }
-            : {
-                nodesToTombstone: [...tombstoneReadyNodes, ...sweepReadyNodes],
-                nodesToDelete: [],
-            };
+        const { nodesToTombstone, nodesToDelete } = {
+            nodesToTombstone: [...tombstoneReadyNodes],
+            nodesToDelete: [],
+        };
+        switch (this.configs.shouldRunSweep) {
+            case "YES":
+                nodesToDelete.push(...sweepReadyNodes);
+                break;
+            case "ONLY_BLOBS":
+                sweepReadyNodes.forEach((nodeId) => {
+                    const nodeType = this.runtime.getNodeType(nodeId);
+                    if (nodeType === GCNodeType.Blob) {
+                        nodesToDelete.push(nodeId);
+                    }
+                    else {
+                        nodesToTombstone.push(nodeId);
+                    }
+                });
+                break;
+            default: // case "NO":
+                nodesToTombstone.push(...sweepReadyNodes);
+                break;
+        }
         if (this.configs.tombstoneMode) {
             this.tombstones = nodesToTombstone;
             // If we are running in GC tombstone mode, update tombstoned routes.
             this.runtime.updateTombstonedRoutes(this.tombstones);
         }
-        if (this.configs.shouldRunSweep && nodesToDelete.length > 0) {
+        if (nodesToDelete.length > 0) {
             // Do not send DDS node ids in the GC op. This is an optimization to reduce its size. Since GC applies to
             // to data store only, all its DDSes are deleted along with it. The DDS ids will be retrieved from the
             // local state when processing the op.
@@ -584,7 +612,7 @@ export class GarbageCollector {
                 unreferencedTimestampMs: this.unreferencedNodesState.get(nodeId)?.unreferencedTimestampMs,
             };
         }
-        return this.summaryStateTracker.summarize(fullTree, trackState, gcState, this.deletedNodes, this.tombstones);
+        return this.summaryStateTracker.summarize(trackState && !fullTree, gcState, this.deletedNodes, this.tombstones);
     }
     getMetadata() {
         return {
@@ -611,15 +639,28 @@ export class GarbageCollector {
      * @param local - Whether it was send by this client.
      */
     processMessage(message, local) {
-        switch (message.contents.type) {
-            case "Sweep": {
+        const gcMessageType = message.contents.type;
+        switch (gcMessageType) {
+            case GarbageCollectionMessageType.Sweep: {
                 // Delete the nodes whose ids are present in the contents.
                 this.deleteSweepReadyNodes(message.contents.deletedNodeIds);
                 break;
             }
+            case GarbageCollectionMessageType.TombstoneLoaded: {
+                if (this.mc.config.getBoolean(disableAutoRecoveryKey) === true) {
+                    break;
+                }
+                // Mark the node as referenced to ensure it isn't Swept
+                const tombstonedNodePath = message.contents.nodePath;
+                this.addedOutboundReference("/", tombstonedNodePath, true /* autorecovery */);
+                // In case the cause of the TombstoneLoaded event is incorrect GC Data (i.e. the object is actually reachable),
+                // do fullGC on the next run to get a chance to repair (in the likely case the bug is not deterministic)
+                this.summaryStateTracker.autoRecovery.requestFullGCOnNextRun();
+                break;
+            }
             default: {
-                if (!compatBehaviorAllowsGCMessageType(message.contents.type, message.compatDetails?.behavior)) {
-                    const error = DataProcessingError.create(`Garbage collection message of unknown type ${message.contents.type}`, "processMessage");
+                if (!compatBehaviorAllowsGCMessageType(gcMessageType, message.compatDetails?.behavior)) {
+                    const error = DataProcessingError.create(`Garbage collection message of unknown type ${gcMessageType}`, "processMessage");
                     throw error;
                 }
                 break;
@@ -629,6 +670,10 @@ export class GarbageCollector {
     /**
      * Delete nodes that are sweep-ready. Call the runtime to delete these nodes and clear the unreferenced state
      * tracking for nodes that are actually deleted by the runtime.
+     *
+     * Note that this doesn't check any configuration around whether Sweep is enabled.
+     * That happens before the op is submitted, and from that point, any client should execute the delete.
+     *
      * @param sweepReadyNodeIds - The ids of nodes that are ready to be deleted.
      */
     deleteSweepReadyNodes(sweepReadyNodeIds) {
@@ -654,13 +699,9 @@ export class GarbageCollector {
         const deletedNodeIds = this.runtime.deleteSweepReadyNodes(allSweepReadyNodeIds);
         // Clear unreferenced state tracking for deleted nodes.
         for (const nodeId of deletedNodeIds) {
-            const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
-            if (nodeStateTracker !== undefined) {
-                // Stop tracking so as to clear out any running timers.
-                nodeStateTracker.stopTracking();
-                // Delete the node as we don't need to track it any more.
-                this.unreferencedNodesState.delete(nodeId);
-            }
+            // Usually we avoid modifying the set of unreferencedNodesState keys in between GC runs,
+            // but this is ok since this node won't exist at all in the next GC run.
+            this.unreferencedNodesState.delete(nodeId);
             this.deletedNodes.add(nodeId);
         }
     }
@@ -690,13 +731,20 @@ export class GarbageCollector {
             lastSummaryTime: this.getLastSummaryTimestampMs(),
             headers: headerData,
         });
+        // Any time we log a Tombstone Loaded error (via Telemetry Tracker),
+        // we want to also trigger autorecovery to avoid the object being deleted
+        // Note: We don't need to trigger on "Changed" because any change will cause the object
+        // to be loaded by the Summarizer, and auto-recovery will be triggered then.
+        if (isTombstoned && reason === "Loaded") {
+            // Note that when a DataStore and its DDS are all loaded, each will trigger AutoRecovery for itself.
+            this.triggerAutoRecovery(nodePath);
+        }
         const nodeType = this.runtime.getNodeType(nodePath);
         // Unless this is a Loaded event for a Blob or DataStore, we're done after telemetry tracking
         if (reason !== "Loaded" || ![GCNodeType.Blob, GCNodeType.DataStore].includes(nodeType)) {
             return;
         }
         const errorRequest = request ?? { url: nodePath };
-        // If the object is tombstoned and tombstone enforcement is configured, throw an error.
         if (isTombstoned && this.throwOnTombstoneLoad && headerData?.allowTombstone !== true) {
             // The requested data store is removed by gc. Create a 404 gc response exception.
             throw responseToException(createResponseError(404, `${nodeType} was tombstoned`, errorRequest, {
@@ -715,14 +763,40 @@ export class GarbageCollector {
             }
         }
     }
+    /**
+     * The given node should have its unreferenced state reset in the next GC,
+     * even if the true GC graph shows it is unreferenced. This will
+     * prevent it from being deleted by Sweep (after the Grace Period).
+     *
+     * Submit a GC op indicating that the Tombstone with the given path has been loaded.
+     * Broadcasting this information in the op stream allows the Summarizer to reset unreferenced state
+     * before runnint GC next.
+     */
+    triggerAutoRecovery(nodePath) {
+        if (this.mc.config.getBoolean(disableAutoRecoveryKey) === true) {
+            return;
+        }
+        // Use compat behavior "Ignore" since this is an optimization to opportunistically protect
+        // objects from deletion, so it's fine for older clients to ignore this op.
+        const containerGCMessage = {
+            type: ContainerMessageType.GC,
+            contents: {
+                type: GarbageCollectionMessageType.TombstoneLoaded,
+                nodePath,
+            },
+            compatDetails: { behavior: "Ignore" },
+        };
+        this.submitMessage(containerGCMessage);
+    }
     /**
      * Called when an outbound reference is added to a node. This is used to identify all nodes that have been
      * referenced between summaries so that their unreferenced timestamp can be reset.
      *
      * @param fromNodePath - The node from which the reference is added.
      * @param toNodePath - The node to which the reference is added.
+     * @param autorecovery - This reference is added artificially, for autorecovery. Used for logging.
      */
-    addedOutboundReference(fromNodePath, toNodePath) {
+    addedOutboundReference(fromNodePath, toNodePath, autorecovery) {
         if (!this.configs.shouldRunGC) {
             return;
         }
@@ -748,7 +822,13 @@ export class GarbageCollector {
             isTombstoned: this.tombstones.includes(toNodePath),
             lastSummaryTime: this.getLastSummaryTimestampMs(),
             fromId: fromNodePath,
+            autorecovery,
         });
+        // This node is referenced - Clear its unreferenced state
+        // But don't delete the node id from the map yet.
+        // When generating GC stats, the set of nodes in here is used as the baseline for
+        // what was unreferenced in the last GC run.
+        this.unreferencedNodesState.get(toNodePath)?.stopTracking();
     }
     /**
      * Returns whether a node with the given path has been deleted or not. This can be used by the runtime to identify
@@ -778,16 +858,16 @@ export class GarbageCollector {
             updatedDataStoreCount: 0,
             updatedAttachmentBlobCount: 0,
         };
-        const updateNodeStats = (nodeId, referenced) => {
+        const updateNodeStats = (nodeId, isReferenced) => {
             markPhaseStats.nodeCount++;
             // If there is no previous GC data, every node's state is generated and is considered as updated.
             // Otherwise, find out if any node went from referenced to unreferenced or vice-versa.
-            const stateUpdated = this.gcDataFromLastRun === undefined ||
-                this.unreferencedNodesState.has(nodeId) === referenced;
+            const wasNotReferenced = this.unreferencedNodesState.has(nodeId);
+            const stateUpdated = this.gcDataFromLastRun === undefined || wasNotReferenced === isReferenced;
             if (stateUpdated) {
                 markPhaseStats.updatedNodeCount++;
             }
-            if (!referenced) {
+            if (!isReferenced) {
                 markPhaseStats.unrefNodeCount++;
             }
             if (this.runtime.getNodeType(nodeId) === GCNodeType.DataStore) {
@@ -795,7 +875,7 @@ export class GarbageCollector {
                 if (stateUpdated) {
                     markPhaseStats.updatedDataStoreCount++;
                 }
-                if (!referenced) {
+                if (!isReferenced) {
                     markPhaseStats.unrefDataStoreCount++;
                 }
             }
@@ -804,7 +884,7 @@ export class GarbageCollector {
                 if (stateUpdated) {
                     markPhaseStats.updatedAttachmentBlobCount++;
                 }
-                if (!referenced) {
+                if (!isReferenced) {
                     markPhaseStats.unrefAttachmentBlobCount++;
                 }
             }
@@ -819,8 +899,9 @@ export class GarbageCollector {
     }
     /**
      * Generates the stats of a garbage collection sweep phase run.
-     * @param deletedNodes - The nodes that have been deleted until this run.
-     * @param sweepReadyNodes - The nodes that are sweep-ready in this GC run.
+     * @param deletedNodes - The nodes that have already been deleted even before this run.
+     * @param sweepReadyNodes - The nodes that are sweep-ready in this GC run. These will be deleted but are not deleted yet,
+     * due to either sweep not being enabled or the Sweep Op needing to roundtrip before the delete is executed.
      * @param markPhaseStats - The stats of the mark phase run.
      * @returns the stats of the sweep phase run.
      */
@@ -860,17 +941,17 @@ export class GarbageCollector {
                 sweepPhaseStats.deletedAttachmentBlobCount++;
             }
         }
-        // If sweep is enabled, the counts from the mark phase stats do not include nodes that have been
+        // The counts from the mark phase stats do not include nodes that were
         // deleted in previous runs. So, add the deleted node counts to life time stats.
         sweepPhaseStats.lifetimeNodeCount += sweepPhaseStats.deletedNodeCount;
         sweepPhaseStats.lifetimeDataStoreCount += sweepPhaseStats.deletedDataStoreCount;
         sweepPhaseStats.lifetimeAttachmentBlobCount += sweepPhaseStats.deletedAttachmentBlobCount;
-        if (this.configs.shouldRunSweep) {
-            return sweepPhaseStats;
-        }
-        // If sweep is not enabled, the current sweep-ready node stats should be added to deleted stats since this
-        // is the final state the node will be in.
-        // If sweep is enabled, this will happen in the run after the GC op round trips back.
+        // These stats are used to estimate the impact of GC in terms of how much garbage is/will be cleaned up.
+        // So we include the current sweep-ready node stats since these nodes will be deleted eventually.
+        // - If sweep is enabled, this will happen in the run after the GC op round trips back
+        //   (they'll be in deletedNodes that time).
+        // - If sweep is not enabled, we still want to include these nodes since they
+        //   _will be_ deleted once it is enabled.
         for (const nodeId of sweepReadyNodes) {
             sweepPhaseStats.deletedNodeCount++;
             const nodeType = this.runtime.getNodeType(nodeId);
@@ -884,4 +965,4 @@ export class GarbageCollector {
         return sweepPhaseStats;
     }
 }
-//# sourceMappingURL=garbageCollection.mjs.map
+//# sourceMappingURL=garbageCollection.js.map