@fluidframework/container-runtime 2.0.0-dev-rc.1.0.0.225277 → 2.0.0-dev-rc.1.0.0.232845

package/src/gc/garbageCollection.ts

@@ -20,7 +20,9 @@ import {
 	ITelemetryLoggerExt,
 	MonitoringContext,
 	PerformanceEvent,
+	tagCodeArtifacts,
 } from "@fluidframework/telemetry-utils";
+import { BlobManager } from "../blobManager";
 import {
 	InactiveResponseHeaderKey,
 	RuntimeHeaderData,
@@ -44,6 +46,7 @@ import {
 	ISweepPhaseStats,
 	GarbageCollectionMessage,
 	GarbageCollectionMessageType,
+	disableAutoRecoveryKey,
 } from "./gcDefinitions";
 import {
 	cloneGCData,
@@ -54,7 +57,10 @@ import {
 import { runGarbageCollection } from "./gcReferenceGraphAlgorithm";
 import { IGarbageCollectionSnapshotData, IGarbageCollectionState } from "./gcSummaryDefinitions";
 import { GCSummaryStateTracker } from "./gcSummaryStateTracker";
-import { UnreferencedStateTracker } from "./gcUnreferencedStateTracker";
+import {
+	UnreferencedStateTracker,
+	UnreferencedStateTrackerMap,
+} from "./gcUnreferencedStateTracker";
 import { GCTelemetryTracker } from "./gcTelemetry";
 
 /**
@@ -108,8 +114,15 @@ export class GarbageCollector implements IGarbageCollector {
 	private readonly initializeGCStateFromBaseSnapshotP: Promise<void>;
 	// The GC details generated from the base snapshot.
 	private readonly baseGCDetailsP: Promise<IGarbageCollectionDetailsBase>;
-	// Map of node ids to their unreferenced state tracker.
-	private readonly unreferencedNodesState: Map<string, UnreferencedStateTracker> = new Map();
+
+	/**
+	 * Map of node ids to their unreferenced state tracker
+	 * NOTE: The set of keys in this map is considered as the set of unreferenced nodes
+	 * as of the last GC run. So in between runs, nothing should be added or removed.
+	 */
+	private readonly unreferencedNodesState: UnreferencedStateTrackerMap =
+		new UnreferencedStateTrackerMap();
+
 	// The Timer responsible for closing the container when the session has expired
 	private sessionExpiryTimer: Timer | undefined;
 
@@ -261,7 +274,7 @@ export class GarbageCollector implements IGarbageCollector {
 			const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
 			assert(
 				currentReferenceTimestampMs !== undefined,
-				"Trying to initialize GC state without current timestamp",
+				0x8a4 /* Trying to initialize GC state without current timestamp */,
 			);
 
 			/**
@@ -590,16 +603,10 @@ export class GarbageCollector implements IGarbageCollector {
 	): { tombstoneReadyNodeIds: Set<string>; sweepReadyNodeIds: Set<string> } {
 		// 1. Marks all referenced nodes by clearing their unreferenced tracker, if any.
 		for (const nodeId of allReferencedNodeIds) {
-			const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
-			if (nodeStateTracker !== undefined) {
-				// Stop tracking so as to clear out any running timers.
-				nodeStateTracker.stopTracking();
-				// Delete the node as we don't need to track it any more.
-				this.unreferencedNodesState.delete(nodeId);
-			}
+			this.unreferencedNodesState.delete(nodeId);
 		}
 
-		// 2. Mark unreferenced nodes in this run by starting unreferenced tracking for them.
+		// 2. Mark unreferenced nodes in this run by starting or updating unreferenced tracking for them.
 		const tombstoneReadyNodeIds: Set<string> = new Set();
 		const sweepReadyNodeIds: Set<string> = new Set();
 		for (const nodeId of gcResult.deletedNodeIds) {
@@ -854,21 +861,33 @@ export class GarbageCollector implements IGarbageCollector {
 	 * @param local - Whether it was send by this client.
 	 */
 	public processMessage(message: ContainerRuntimeGCMessage, local: boolean) {
-		switch (message.contents.type) {
-			case "Sweep": {
+		const gcMessageType = message.contents.type;
+		switch (gcMessageType) {
+			case GarbageCollectionMessageType.Sweep: {
 				// Delete the nodes whose ids are present in the contents.
 				this.deleteSweepReadyNodes(message.contents.deletedNodeIds);
 				break;
 			}
+			case GarbageCollectionMessageType.TombstoneLoaded: {
+				if (this.mc.config.getBoolean(disableAutoRecoveryKey) === true) {
+					break;
+				}
+
+				// Mark the node as referenced to ensure it isn't Swept
+				const tombstonedNodePath = message.contents.nodePath;
+				this.addedOutboundReference("/", tombstonedNodePath);
+
+				break;
+			}
 			default: {
 				if (
 					!compatBehaviorAllowsGCMessageType(
-						message.contents.type,
+						gcMessageType,
 						message.compatDetails?.behavior,
 					)
 				) {
 					const error = DataProcessingError.create(
-						`Garbage collection message of unknown type ${message.contents.type}`,
+						`Garbage collection message of unknown type ${gcMessageType}`,
 						"processMessage",
 					);
 					throw error;
@@ -909,13 +928,9 @@ export class GarbageCollector implements IGarbageCollector {
 
 		// Clear unreferenced state tracking for deleted nodes.
 		for (const nodeId of deletedNodeIds) {
-			const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
-			if (nodeStateTracker !== undefined) {
-				// Stop tracking so as to clear out any running timers.
-				nodeStateTracker.stopTracking();
-				// Delete the node as we don't need to track it any more.
-				this.unreferencedNodesState.delete(nodeId);
-			}
+			// Usually we avoid modifying the set of unreferencedNodesState keys in between GC runs,
+			// but this is ok since this node won't exist at all in the next GC run.
+			this.unreferencedNodesState.delete(nodeId);
 			this.deletedNodes.add(nodeId);
 		}
 	}
@@ -957,6 +972,15 @@ export class GarbageCollector implements IGarbageCollector {
 			headers: headerData,
 		});
 
+		// Any time we log a Tombstone Loaded error (via Telemetry Tracker),
+		// we want to also trigger autorecovery to avoid the object being deleted
+		// Note: We don't need to trigger on "Changed" because any change will cause the object
+		// to be loaded by the Summarizer, and auto-recovery will be triggered then.
+		if (isTombstoned && reason === "Loaded") {
+			// Note that when a DataStore and its DDS are all loaded, each will trigger AutoRecovery for itself.
+			this.triggerAutoRecovery(nodePath);
+		}
+
 		const nodeType = this.runtime.getNodeType(nodePath);
 
 		// Unless this is a Loaded event for a Blob or DataStore, we're done after telemetry tracking
@@ -965,7 +989,6 @@ export class GarbageCollector implements IGarbageCollector {
 		}
 
 		const errorRequest: IRequest = request ?? { url: nodePath };
-		// If the object is tombstoned and tombstone enforcement is configured, throw an error.
 		if (isTombstoned && this.throwOnTombstoneLoad && headerData?.allowTombstone !== true) {
 			// The requested data store is removed by gc. Create a 404 gc response exception.
 			throw responseToException(
@@ -993,18 +1016,58 @@ export class GarbageCollector implements IGarbageCollector {
 		}
 	}
 
+	/**
+	 * The given node should have its unreferenced state reset in the next GC,
+	 * even if the true GC graph shows it is unreferenced. This will
+	 * prevent it from being deleted by Sweep (after the Grace Period).
+	 *
+	 * Submit a GC op indicating that the Tombstone with the given path has been loaded.
+	 * Broadcasting this information in the op stream allows the Summarizer to reset unreferenced state
+	 * before runnint GC next.
+	 */
+	private triggerAutoRecovery(nodePath: string) {
+		if (this.mc.config.getBoolean(disableAutoRecoveryKey) === true) {
+			return;
+		}
+
+		// Use compat behavior "Ignore" since this is an optimization to opportunistically protect
+		// objects from deletion, so it's fine for older clients to ignore this op.
+		const containerGCMessage: ContainerRuntimeGCMessage = {
+			type: ContainerMessageType.GC,
+			contents: {
+				type: "TombstoneLoaded",
+				nodePath,
+			},
+			compatDetails: { behavior: "Ignore" },
+		};
+		this.submitMessage(containerGCMessage);
+	}
+
 	/**
 	 * Called when an outbound reference is added to a node. This is used to identify all nodes that have been
 	 * referenced between summaries so that their unreferenced timestamp can be reset.
 	 *
 	 * @param fromNodePath - The node from which the reference is added.
 	 * @param toNodePath - The node to which the reference is added.
+	 * @param autorecovery - This reference is added artificially, for autorecovery. Used for logging.
 	 */
-	public addedOutboundReference(fromNodePath: string, toNodePath: string) {
+	public addedOutboundReference(fromNodePath: string, toNodePath: string, autorecovery?: true) {
 		if (!this.configs.shouldRunGC) {
 			return;
 		}
 
+		if (!toNodePath.startsWith("/")) {
+			// A long time ago we stored handles with relatives paths. We don't expect to see these cases though
+			// because GC was enabled only after we made the switch to always using absolute paths.
+			this.mc.logger.sendErrorEvent({
+				eventName: "InvalidRelativeOutboundRoute",
+				...tagCodeArtifacts({ fromId: fromNodePath, id: toNodePath }),
+			});
+			return;
+		}
+
+		assert(fromNodePath.startsWith("/"), 0x8a5 /* fromNodePath must be an absolute path */);
+
 		const outboundRoutes = this.newReferencesSinceLastRun.get(fromNodePath) ?? [];
 		outboundRoutes.push(toNodePath);
 		this.newReferencesSinceLastRun.set(fromNodePath, outboundRoutes);
@@ -1018,7 +1081,14 @@ export class GarbageCollector implements IGarbageCollector {
 			isTombstoned: this.tombstones.includes(toNodePath),
 			lastSummaryTime: this.getLastSummaryTimestampMs(),
 			fromId: fromNodePath,
+			autorecovery,
 		});
+
+		// This node is referenced - Clear its unreferenced state
+		// But don't delete the node id from the map yet.
+		// When generating GC stats, the set of nodes in here is used as the baseline for
+		// what was unreferenced in the last GC run.
+		this.unreferencedNodesState.get(toNodePath)?.stopTracking();
 	}
 
 	/**
@@ -1052,17 +1122,17 @@ export class GarbageCollector implements IGarbageCollector {
 			updatedAttachmentBlobCount: 0,
 		};
 
-		const updateNodeStats = (nodeId: string, referenced: boolean) => {
+		const updateNodeStats = (nodeId: string, isReferenced: boolean) => {
 			markPhaseStats.nodeCount++;
 			// If there is no previous GC data, every node's state is generated and is considered as updated.
 			// Otherwise, find out if any node went from referenced to unreferenced or vice-versa.
+			const wasNotReferenced = this.unreferencedNodesState.has(nodeId);
 			const stateUpdated =
-				this.gcDataFromLastRun === undefined ||
-				this.unreferencedNodesState.has(nodeId) === referenced;
+				this.gcDataFromLastRun === undefined || wasNotReferenced === isReferenced;
 			if (stateUpdated) {
 				markPhaseStats.updatedNodeCount++;
 			}
-			if (!referenced) {
+			if (!isReferenced) {
 				markPhaseStats.unrefNodeCount++;
 			}
 
@@ -1071,7 +1141,7 @@ export class GarbageCollector implements IGarbageCollector {
 				if (stateUpdated) {
 					markPhaseStats.updatedDataStoreCount++;
 				}
-				if (!referenced) {
+				if (!isReferenced) {
 					markPhaseStats.unrefDataStoreCount++;
 				}
 			}
@@ -1080,7 +1150,7 @@ export class GarbageCollector implements IGarbageCollector {
 				if (stateUpdated) {
 					markPhaseStats.updatedAttachmentBlobCount++;
 				}
-				if (!referenced) {
+				if (!isReferenced) {
 					markPhaseStats.unrefAttachmentBlobCount++;
 				}
 			}
@@ -1120,9 +1190,25 @@ export class GarbageCollector implements IGarbageCollector {
 			deletedAttachmentBlobCount: 0,
 		};
 
+		// The runtime can't reliably identify the type of deleted nodes. So, get the type here. This should
+		// be good enough because the only types that participate in GC today are data stores, DDSes and blobs.
+		const getDeletedNodeType = (nodeId: string): GCNodeType => {
+			const pathParts = nodeId.split("/");
+			if (pathParts[1] === BlobManager.basePath) {
+				return GCNodeType.Blob;
+			}
+			if (pathParts.length === 2) {
+				return GCNodeType.DataStore;
+			}
+			if (pathParts.length === 3) {
+				return GCNodeType.SubDataStore;
+			}
+			return GCNodeType.Other;
+		};
+
 		for (const nodeId of deletedNodes) {
 			sweepPhaseStats.deletedNodeCount++;
-			const nodeType = this.runtime.getNodeType(nodeId);
+			const nodeType = getDeletedNodeType(nodeId);
 			if (nodeType === GCNodeType.DataStore) {
 				sweepPhaseStats.deletedDataStoreCount++;
 			} else if (nodeType === GCNodeType.Blob) {

package/src/gc/gcDefinitions.ts

@@ -74,6 +74,10 @@ export const gcVersionUpgradeToV4Key = "Fluid.GarbageCollection.GCVersionUpgrade
 export const disableDatastoreSweepKey = "Fluid.GarbageCollection.DisableDataStoreSweep";
 /** Config key to disable GC sweep for attachment blobs. */
 export const disableAttachmentBlobSweepKey = "Fluid.GarbageCollection.DisableAttachmentBlobSweep";
+/** Config key to revert new paradigm of detecting outbound routes in ContainerRuntime layer (use true) */
+export const detectOutboundRoutesViaDDSKey = "Fluid.GarbageCollection.DetectOutboundRoutesViaDDS";
+/** Config key to disable auto-recovery mechanism that protects Tombstones that are loaded from being swept (use true) */
+export const disableAutoRecoveryKey = "Fluid.GarbageCollection.DisableAutoRecovery";
 
 // One day in milliseconds.
 export const oneDayMs = 1 * 24 * 60 * 60 * 1000;
@@ -251,6 +255,8 @@ export type GCNodeType = (typeof GCNodeType)[keyof typeof GCNodeType];
 export const GarbageCollectionMessageType = {
 	/** Message sent directing GC to delete the given nodes */
 	Sweep: "Sweep",
+	/** Message sent notifying GC that a Tombstoned object was Loaded */
+	TombstoneLoaded: "TombstoneLoaded",
 } as const;
 
 /**
@@ -264,16 +270,28 @@ export type GarbageCollectionMessageType =
  * @internal
  */
 export interface ISweepMessage {
-	type: "Sweep";
-	// The ids of nodes that are deleted.
+	/** @see GarbageCollectionMessageType.Sweep */
+	type: typeof GarbageCollectionMessageType.Sweep;
+	/** The ids of nodes that are deleted. */
 	deletedNodeIds: string[];
 }
 
+/**
+ * The GC TombstoneLoaded message.
+ * @internal
+ */
+export interface ITombstoneLoadedMessage {
+	/** @see GarbageCollectionMessageType.TombstoneLoaded */
+	type: typeof GarbageCollectionMessageType.TombstoneLoaded;
+	/** The id of Tombstoned node that was loaded. */
+	nodePath: string;
+}
+
 /**
  * Type for a message to be used for sending / received garbage collection messages.
  * @internal
  */
-export type GarbageCollectionMessage = ISweepMessage;
+export type GarbageCollectionMessage = ISweepMessage | ITombstoneLoadedMessage;
 
 /**
  * Defines the APIs for the runtime object to be passed to the garbage collector.

package/src/gc/gcTelemetry.ts

@@ -63,6 +63,7 @@ interface INodeUsageProps extends ICommonProps {
 	currentReferenceTimestampMs: number | undefined;
 	packagePath: readonly string[] | undefined;
 	fromId?: string;
+	autorecovery?: true;
 }
 
 /**
@@ -189,8 +190,6 @@ export class GCTelemetryTracker {
 		};
 
 		// If the node that is used is tombstoned, log a tombstone telemetry.
-		// Note that this is done before checking if "nodeStateTracker" is undefined below because unreferenced
-		// tracking may not have yet been enabled. That happens only after the client transitions to write mode.
 		if (nodeUsageProps.isTombstoned) {
 			this.logTombstoneUsageTelemetry(nodeUsageProps, unrefEventProps, nodeType, usageType);
 		}

package/src/gc/gcUnreferencedStateTracker.ts

@@ -25,6 +25,17 @@ class TimerWithNoDefaultTimeout extends Timer {
 	}
 }
 
+/** The collection of UnreferencedStateTrackers for all unreferenced nodes. Ensures stopTracking is called when deleting */
+export class UnreferencedStateTrackerMap extends Map<string, UnreferencedStateTracker> {
+	/** Delete the given key, and stop tracking if that node was actually unreferenced */
+	delete(key: string): boolean {
+		// Stop tracking so as to clear out any running timers.
+		this.get(key)?.stopTracking();
+		// Delete the node as we don't need to track it any more.
+		return super.delete(key);
+	}
+}
+
 /**
  * Helper class that tracks the state of an unreferenced node such as the time it was unreferenced and if it can
  * be tombstoned or deleted by the sweep phase.

package/src/gc/index.ts

@@ -33,10 +33,13 @@ export {
 	runSweepKey,
 	stableGCVersion,
 	disableAttachmentBlobSweepKey,
+	disableAutoRecoveryKey,
 	disableDatastoreSweepKey,
+	detectOutboundRoutesViaDDSKey,
 	UnreferencedState,
 	throwOnTombstoneLoadOverrideKey,
 	GarbageCollectionMessage,
+	ISweepMessage,
 } from "./gcDefinitions";
 export {
 	cloneGCData,

package/src/index.ts

@@ -30,9 +30,11 @@ export {
 	IContainerRuntimeMessageCompatDetails,
 	CompatModeBehavior,
 	RecentlyAddedContainerRuntimeMessageDetails,
+	UnknownContainerRuntimeMessage,
 } from "./messageTypes";
 export { IBlobManagerLoadInfo } from "./blobManager";
 export { FluidDataStoreRegistry } from "./dataStoreRegistry";
+export { detectOutboundReferences } from "./dataStores";
 export {
 	GCNodeType,
 	IGCMetadata,

package/src/messageTypes.ts

@@ -133,7 +133,7 @@ export type ContainerRuntimeGCMessage = TypedContainerRuntimeMessage<
 >;
 
 /**
- * Represents an unrecognized {@link TypedContainerRuntimeMessage}, e.g. a message from a future version of the container runtime.
+ * Represents an unrecognized TypedContainerRuntimeMessage, e.g. a message from a future version of the container runtime.
  * @internal
  */
 export interface UnknownContainerRuntimeMessage

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.0.0-dev-rc.1.0.0.225277";
+export const pkgVersion = "2.0.0-dev-rc.1.0.0.232845";