@fluidframework/container-runtime 2.0.0-internal.8.0.7 → 2.0.0-internal.8.0.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fluidframework/container-runtime",
-  "version": "2.0.0-internal.8.0.7",
+  "version": "2.0.0-internal.8.0.8",
   "description": "Fluid container runtime",
   "homepage": "https://fluidframework.com",
   "repository": {
@@ -35,19 +35,19 @@
     "temp-directory": "nyc/.nyc_output"
   },
   "dependencies": {
-    "@fluid-internal/client-utils": ">=2.0.0-internal.8.0.7 <2.0.0-internal.8.1.0",
-    "@fluidframework/container-definitions": ">=2.0.0-internal.8.0.7 <2.0.0-internal.8.1.0",
-    "@fluidframework/container-runtime-definitions": ">=2.0.0-internal.8.0.7 <2.0.0-internal.8.1.0",
-    "@fluidframework/core-interfaces": ">=2.0.0-internal.8.0.7 <2.0.0-internal.8.1.0",
-    "@fluidframework/core-utils": ">=2.0.0-internal.8.0.7 <2.0.0-internal.8.1.0",
-    "@fluidframework/datastore": ">=2.0.0-internal.8.0.7 <2.0.0-internal.8.1.0",
-    "@fluidframework/driver-definitions": ">=2.0.0-internal.8.0.7 <2.0.0-internal.8.1.0",
-    "@fluidframework/driver-utils": ">=2.0.0-internal.8.0.7 <2.0.0-internal.8.1.0",
-    "@fluidframework/id-compressor": ">=2.0.0-internal.8.0.7 <2.0.0-internal.8.1.0",
+    "@fluid-internal/client-utils": ">=2.0.0-internal.8.0.8 <2.0.0-internal.8.1.0",
+    "@fluidframework/container-definitions": ">=2.0.0-internal.8.0.8 <2.0.0-internal.8.1.0",
+    "@fluidframework/container-runtime-definitions": ">=2.0.0-internal.8.0.8 <2.0.0-internal.8.1.0",
+    "@fluidframework/core-interfaces": ">=2.0.0-internal.8.0.8 <2.0.0-internal.8.1.0",
+    "@fluidframework/core-utils": ">=2.0.0-internal.8.0.8 <2.0.0-internal.8.1.0",
+    "@fluidframework/datastore": ">=2.0.0-internal.8.0.8 <2.0.0-internal.8.1.0",
+    "@fluidframework/driver-definitions": ">=2.0.0-internal.8.0.8 <2.0.0-internal.8.1.0",
+    "@fluidframework/driver-utils": ">=2.0.0-internal.8.0.8 <2.0.0-internal.8.1.0",
+    "@fluidframework/id-compressor": ">=2.0.0-internal.8.0.8 <2.0.0-internal.8.1.0",
     "@fluidframework/protocol-definitions": "^3.0.0",
-    "@fluidframework/runtime-definitions": ">=2.0.0-internal.8.0.7 <2.0.0-internal.8.1.0",
-    "@fluidframework/runtime-utils": ">=2.0.0-internal.8.0.7 <2.0.0-internal.8.1.0",
-    "@fluidframework/telemetry-utils": ">=2.0.0-internal.8.0.7 <2.0.0-internal.8.1.0",
+    "@fluidframework/runtime-definitions": ">=2.0.0-internal.8.0.8 <2.0.0-internal.8.1.0",
+    "@fluidframework/runtime-utils": ">=2.0.0-internal.8.0.8 <2.0.0-internal.8.1.0",
+    "@fluidframework/telemetry-utils": ">=2.0.0-internal.8.0.8 <2.0.0-internal.8.1.0",
     "double-ended-queue": "^2.1.0-0",
     "events": "^3.1.0",
     "lz4js": "^0.2.0",
@@ -56,15 +56,15 @@
   },
   "devDependencies": {
     "@arethetypeswrong/cli": "^0.13.3",
-    "@fluid-private/stochastic-test-utils": ">=2.0.0-internal.8.0.7 <2.0.0-internal.8.1.0",
+    "@fluid-private/stochastic-test-utils": ">=2.0.0-internal.8.0.8 <2.0.0-internal.8.1.0",
     "@fluid-tools/benchmark": "^0.48.0",
     "@fluid-tools/build-cli": "^0.28.0",
     "@fluidframework/build-common": "^2.0.3",
     "@fluidframework/build-tools": "^0.28.0",
     "@fluidframework/container-runtime-previous": "npm:@fluidframework/container-runtime@2.0.0-internal.7.2.0",
     "@fluidframework/eslint-config-fluid": "^3.1.0",
-    "@fluidframework/mocha-test-setup": ">=2.0.0-internal.8.0.7 <2.0.0-internal.8.1.0",
-    "@fluidframework/test-runtime-utils": ">=2.0.0-internal.8.0.7 <2.0.0-internal.8.1.0",
+    "@fluidframework/mocha-test-setup": ">=2.0.0-internal.8.0.8 <2.0.0-internal.8.1.0",
+    "@fluidframework/test-runtime-utils": ">=2.0.0-internal.8.0.8 <2.0.0-internal.8.1.0",
     "@microsoft/api-extractor": "^7.38.3",
     "@types/double-ended-queue": "^2.1.0",
     "@types/events": "^3.0.0",

package/src/containerRuntime.ts

@@ -2909,6 +2909,12 @@ export class ContainerRuntime
 	 * data store or an attachment blob.
 	 */
 	public async getGCNodePackagePath(nodePath: string): Promise<readonly string[] | undefined> {
+		// GC uses "/" when adding "root" references, e.g. for Aliasing or as part of Tombstone Auto-Recovery.
+		// These have no package path so return a special value.
+		if (nodePath === "/") {
+			return ["_gcRoot"];
+		}
+
 		switch (this.getNodeType(nodePath)) {
 			case GCNodeType.Blob:
 				return [BlobManager.basePath];
@@ -3691,6 +3697,7 @@ export class ContainerRuntime
 		localOpMetadata: unknown,
 		opMetadata: Record<string, unknown> | undefined,
 	) {
+		assert(!this.isSummarizerClient, "Summarizer never reconnects so should never resubmit");
 		switch (message.type) {
 			case ContainerMessageType.FluidDataStoreOp:
 				// For Operations, call resubmitDataStoreOp which will find the right store
@@ -3712,8 +3719,8 @@ export class ContainerRuntime
 				this.submit(message);
 				break;
 			case ContainerMessageType.GC:
-				// GC op is only sent in summarizer which should never reconnect.
-				throw new LoggingError("GC op not expected to be resubmitted in summarizer");
+				this.submit(message);
+				break;
 			default: {
 				// This case should be very rare - it would imply an op was stashed from a
 				// future version of runtime code and now is being applied on an older version

package/src/gc/garbageCollection.ts

@@ -45,6 +45,7 @@ import {
 	ISweepPhaseStats,
 	GarbageCollectionMessage,
 	GarbageCollectionMessageType,
+	disableAutoRecoveryKey,
 } from "./gcDefinitions";
 import {
 	cloneGCData,
@@ -441,7 +442,9 @@ export class GarbageCollector implements IGarbageCollector {
 	): Promise<IGCStats | undefined> {
 		const fullGC =
 			options.fullGC ??
-			(this.configs.runFullGC === true || this.summaryStateTracker.doesSummaryStateNeedReset);
+			(this.configs.runFullGC === true ||
+				this.summaryStateTracker.autoRecovery.fullGCRequested() ||
+				this.summaryStateTracker.doesSummaryStateNeedReset);
 
 		// Add the options that are used to run GC to the telemetry context.
 		telemetryContext?.setMultiple("fluid_GC", "Options", {
@@ -870,21 +873,37 @@ export class GarbageCollector implements IGarbageCollector {
 	 * @param local - Whether it was send by this client.
 	 */
 	public processMessage(message: ContainerRuntimeGCMessage, local: boolean) {
-		switch (message.contents.type) {
-			case "Sweep": {
+		const gcMessageType = message.contents.type;
+		switch (gcMessageType) {
+			case GarbageCollectionMessageType.Sweep: {
 				// Delete the nodes whose ids are present in the contents.
 				this.deleteSweepReadyNodes(message.contents.deletedNodeIds);
 				break;
 			}
+			case GarbageCollectionMessageType.TombstoneLoaded: {
+				if (this.mc.config.getBoolean(disableAutoRecoveryKey) === true) {
+					break;
+				}
+
+				// Mark the node as referenced to ensure it isn't Swept
+				const tombstonedNodePath = message.contents.nodePath;
+				this.addedOutboundReference("/", tombstonedNodePath, true /* autorecovery */);
+
+				// In case the cause of the TombstoneLoaded event is incorrect GC Data (i.e. the object is actually reachable),
+				// do fullGC on the next run to get a chance to repair (in the likely case the bug is not deterministic)
+				this.summaryStateTracker.autoRecovery.requestFullGCOnNextRun();
+
+				break;
+			}
 			default: {
 				if (
 					!compatBehaviorAllowsGCMessageType(
-						message.contents.type,
+						gcMessageType,
 						message.compatDetails?.behavior,
 					)
 				) {
 					const error = DataProcessingError.create(
-						`Garbage collection message of unknown type ${message.contents.type}`,
+						`Garbage collection message of unknown type ${gcMessageType}`,
 						"processMessage",
 					);
 					throw error;
@@ -977,6 +996,15 @@ export class GarbageCollector implements IGarbageCollector {
 			headers: headerData,
 		});
 
+		// Any time we log a Tombstone Loaded error (via Telemetry Tracker),
+		// we want to also trigger autorecovery to avoid the object being deleted
+		// Note: We don't need to trigger on "Changed" because any change will cause the object
+		// to be loaded by the Summarizer, and auto-recovery will be triggered then.
+		if (isTombstoned && reason === "Loaded") {
+			// Note that when a DataStore and its DDS are all loaded, each will trigger AutoRecovery for itself.
+			this.triggerAutoRecovery(nodePath);
+		}
+
 		const nodeType = this.runtime.getNodeType(nodePath);
 
 		// Unless this is a Loaded event for a Blob or DataStore, we're done after telemetry tracking
@@ -985,7 +1013,6 @@ export class GarbageCollector implements IGarbageCollector {
 		}
 
 		const errorRequest: IRequest = request ?? { url: nodePath };
-		// If the object is tombstoned and tombstone enforcement is configured, throw an error.
 		if (isTombstoned && this.throwOnTombstoneLoad && headerData?.allowTombstone !== true) {
 			// The requested data store is removed by gc. Create a 404 gc response exception.
 			throw responseToException(
@@ -1013,14 +1040,42 @@ export class GarbageCollector implements IGarbageCollector {
 		}
 	}
 
+	/**
+	 * The given node should have its unreferenced state reset in the next GC,
+	 * even if the true GC graph shows it is unreferenced. This will
+	 * prevent it from being deleted by Sweep (after the Grace Period).
+	 *
+	 * Submit a GC op indicating that the Tombstone with the given path has been loaded.
+	 * Broadcasting this information in the op stream allows the Summarizer to reset unreferenced state
+	 * before runnint GC next.
+	 */
+	private triggerAutoRecovery(nodePath: string) {
+		if (this.mc.config.getBoolean(disableAutoRecoveryKey) === true) {
+			return;
+		}
+
+		// Use compat behavior "Ignore" since this is an optimization to opportunistically protect
+		// objects from deletion, so it's fine for older clients to ignore this op.
+		const containerGCMessage: ContainerRuntimeGCMessage = {
+			type: ContainerMessageType.GC,
+			contents: {
+				type: GarbageCollectionMessageType.TombstoneLoaded,
+				nodePath,
+			},
+			compatDetails: { behavior: "Ignore" },
+		};
+		this.submitMessage(containerGCMessage);
+	}
+
 	/**
 	 * Called when an outbound reference is added to a node. This is used to identify all nodes that have been
 	 * referenced between summaries so that their unreferenced timestamp can be reset.
 	 *
 	 * @param fromNodePath - The node from which the reference is added.
 	 * @param toNodePath - The node to which the reference is added.
+	 * @param autorecovery - This reference is added artificially, for autorecovery. Used for logging.
 	 */
-	public addedOutboundReference(fromNodePath: string, toNodePath: string) {
+	public addedOutboundReference(fromNodePath: string, toNodePath: string, autorecovery?: true) {
 		if (!this.configs.shouldRunGC) {
 			return;
 		}
@@ -1038,7 +1093,14 @@ export class GarbageCollector implements IGarbageCollector {
 			isTombstoned: this.tombstones.includes(toNodePath),
 			lastSummaryTime: this.getLastSummaryTimestampMs(),
 			fromId: fromNodePath,
+			autorecovery,
 		});
+
+		// This node is referenced - Clear its unreferenced state
+		// But don't delete the node id from the map yet.
+		// When generating GC stats, the set of nodes in here is used as the baseline for
+		// what was unreferenced in the last GC run.
+		this.unreferencedNodesState.get(toNodePath)?.stopTracking();
 	}
 
 	/**

package/src/gc/gcDefinitions.ts

@@ -78,6 +78,8 @@ export const throwOnTombstoneUsageKey = "Fluid.GarbageCollection.ThrowOnTombston
 export const gcVersionUpgradeToV4Key = "Fluid.GarbageCollection.GCVersionUpgradeToV4";
 /** Config key to disable GC sweep for datastores. They'll merely be Tombstoned. */
 export const disableDatastoreSweepKey = "Fluid.GarbageCollection.DisableDataStoreSweep";
+/** Config key to disable auto-recovery mechanism that protects Tombstones that are loaded from being swept (use true) */
+export const disableAutoRecoveryKey = "Fluid.GarbageCollection.DisableAutoRecovery";
 
 // One day in milliseconds.
 export const oneDayMs = 1 * 24 * 60 * 60 * 1000;
@@ -233,6 +235,8 @@ export type GCNodeType = (typeof GCNodeType)[keyof typeof GCNodeType];
 export const GarbageCollectionMessageType = {
 	/** Message sent directing GC to delete the given nodes */
 	Sweep: "Sweep",
+	/** Message sent notifying GC that a Tombstoned object was Loaded */
+	TombstoneLoaded: "TombstoneLoaded",
 } as const;
 
 /**
@@ -246,16 +250,28 @@ export type GarbageCollectionMessageType =
  * @internal
  */
 export interface ISweepMessage {
-	type: "Sweep";
-	// The ids of nodes that are deleted.
+	/** @see GarbageCollectionMessageType.Sweep */
+	type: typeof GarbageCollectionMessageType.Sweep;
+	/** The ids of nodes that are deleted. */
 	deletedNodeIds: string[];
 }
 
+/**
+ * The GC TombstoneLoaded message.
+ * @internal
+ */
+export interface ITombstoneLoadedMessage {
+	/** @see GarbageCollectionMessageType.TombstoneLoaded */
+	type: typeof GarbageCollectionMessageType.TombstoneLoaded;
+	/** The id of Tombstoned node that was loaded. */
+	nodePath: string;
+}
+
 /**
  * Type for a message to be used for sending / received garbage collection messages.
  * @internal
  */
-export type GarbageCollectionMessage = ISweepMessage;
+export type GarbageCollectionMessage = ISweepMessage | ITombstoneLoadedMessage;
 
 /**
  * Defines the APIs for the runtime object to be passed to the garbage collector.
@@ -335,7 +351,7 @@ export interface IGarbageCollector {
 		headerData?: RuntimeHeaderData,
 	): void;
 	/** Called when a reference is added to a node. Used to identify nodes that were referenced between summaries. */
-	addedOutboundReference(fromNodePath: string, toNodePath: string): void;
+	addedOutboundReference(fromNodePath: string, toNodePath: string, autorecovery?: true): void;
 	/** Called to process a garbage collection message. */
 	processMessage(message: ContainerRuntimeGCMessage, local: boolean): void;
 	/** Returns true if this node has been deleted by GC during sweep phase. */

package/src/gc/gcSummaryStateTracker.ts

@@ -51,6 +51,18 @@ export class GCSummaryStateTracker {
 	// to unreferenced or vice-versa.
 	public updatedDSCountSinceLastSummary: number = 0;
 
+	/** API for ensuring the correct auto-recovery mitigations */
+	public autoRecovery = {
+		requestFullGCOnNextRun: () => {
+			this.fullGCModeForAutoRecovery = true;
+		},
+		fullGCRequested: () => {
+			return this.fullGCModeForAutoRecovery;
+		},
+	};
+	/** If true, the next GC run will do fullGC mode to regenerate the GC data for each node */
+	private fullGCModeForAutoRecovery: boolean = false;
+
 	constructor(
 		// Tells whether GC should run or not.
 		private readonly configs: Pick<
@@ -267,7 +279,7 @@ export class GCSummaryStateTracker {
 	}
 
 	/**
-	 * Called to refresh the latest summary state. This happens when either a pending summary is acked.
+	 * Called to refresh the latest summary state. This happens when a pending summary is acked.
 	 */
 	public async refreshLatestSummary(result: IRefreshSummaryResult): Promise<void> {
 		if (!result.isSummaryTracked) {
@@ -287,6 +299,7 @@ export class GCSummaryStateTracker {
 		this.latestSummaryData = this.pendingSummaryData;
 		this.pendingSummaryData = undefined;
 		this.updatedDSCountSinceLastSummary = 0;
+		this.fullGCModeForAutoRecovery = false;
 		return;
 	}
 

package/src/gc/gcTelemetry.ts

@@ -63,6 +63,7 @@ interface INodeUsageProps extends ICommonProps {
 	currentReferenceTimestampMs: number | undefined;
 	packagePath: readonly string[] | undefined;
 	fromId?: string;
+	autorecovery?: true;
 }
 
 /**

package/src/gc/index.ts

@@ -32,10 +32,13 @@ export {
 	runSessionExpiryKey,
 	runSweepKey,
 	stableGCVersion,
+	disableAutoRecoveryKey,
 	disableDatastoreSweepKey,
 	UnreferencedState,
 	throwOnTombstoneLoadOverrideKey,
 	GarbageCollectionMessage,
+	GarbageCollectionMessageType,
+	ISweepMessage,
 } from "./gcDefinitions";
 export {
 	cloneGCData,

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.0.0-internal.8.0.7";
+export const pkgVersion = "2.0.0-internal.8.0.8";