@fluidframework/container-runtime 2.0.0-internal.7.1.0 → 2.0.0-internal.7.1.2

package/src/dataStoreContext.ts

@@ -78,7 +78,7 @@ import {
 	summarizerClientType,
 } from "./summary";
 import { ContainerRuntime } from "./containerRuntime";
-import { sendGCUnexpectedUsageEvent, throwOnTombstoneUsageKey } from "./gc";
+import { sendGCUnexpectedUsageEvent } from "./gc";
 
 function createAttributes(
 	pkg: readonly string[],
@@ -325,11 +325,7 @@ export abstract class FluidDataStoreContext
 			this.mc.logger,
 		);
 
-		// Tombstone should only throw when the feature flag is enabled and the client isn't a summarizer
-		this.throwOnTombstoneUsage =
-			this.mc.config.getBoolean(throwOnTombstoneUsageKey) === true &&
-			this._containerRuntime.gcTombstoneEnforcementAllowed &&
-			this.clientDetails.type !== summarizerClientType;
+		this.throwOnTombstoneUsage = this._containerRuntime.gcThrowOnTombstoneUsage;
 
 		// By default, a data store can log maximum 10 local changes telemetry in summarizer.
 		this.localChangesTelemetryCount =
@@ -486,7 +482,16 @@ export abstract class FluidDataStoreContext
 		local: boolean,
 		localOpMetadata: unknown,
 	): void {
-		this.verifyNotClosed("process", true, extractSafePropertiesFromMessage(messageArg));
+		const safeTelemetryProps = extractSafePropertiesFromMessage(messageArg);
+		// On op process, tombstone error is logged in garbage collector. So, set "checkTombstone" to false when calling
+		// "verifyNotClosed" which logs tombstone errors. Throw error if tombstoned and throwing on load is configured.
+		this.verifyNotClosed("process", false /* checkTombstone */, safeTelemetryProps);
+		if (this.tombstoned && this.throwOnTombstoneUsage) {
+			throw new DataCorruptionError(
+				"Context is tombstoned! Call site [process]",
+				safeTelemetryProps,
+			);
+		}
 
 		const innerContents = messageArg.contents as FluidDataStoreMessage;
 		const message = {
@@ -1089,7 +1094,7 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext {
 		return message;
 	}
 
-	public async getInitialSnapshotDetails(): Promise<ISnapshotDetails> {
+	private readonly initialSnapshotDetailsP = new LazyPromise<ISnapshotDetails>(async () => {
 		let snapshot = this.snapshotTree;
 		let attributes: ReadFluidDataStoreAttributes;
 		let isRootDataStore = false;
@@ -1122,6 +1127,10 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext {
 			isRootDataStore,
 			snapshot,
 		};
+	});
+
+	public async getInitialSnapshotDetails(): Promise<ISnapshotDetails> {
+		return this.initialSnapshotDetailsP;
 	}
 
 	/**

package/src/dataStores.ts

@@ -50,12 +50,7 @@ import { buildSnapshotTree } from "@fluidframework/driver-utils";
 import { assert, Lazy } from "@fluidframework/core-utils";
 import { v4 as uuid } from "uuid";
 import { DataStoreContexts } from "./dataStoreContexts";
-import {
-	ContainerRuntime,
-	defaultRuntimeHeaderData,
-	RuntimeHeaderData,
-	TombstoneResponseHeaderKey,
-} from "./containerRuntime";
+import { ContainerRuntime, defaultRuntimeHeaderData, RuntimeHeaderData } from "./containerRuntime";
 import {
 	FluidDataStoreContext,
 	RemoteFluidDataStoreContext,
@@ -65,12 +60,7 @@ import {
 } from "./dataStoreContext";
 import { StorageServiceWithAttachBlobs } from "./storageServiceWithAttachBlobs";
 import { IDataStoreAliasMessage, isDataStoreAliasMessage } from "./dataStore";
-import {
-	GCNodeType,
-	disableDatastoreSweepKey,
-	throwOnTombstoneLoadKey,
-	sendGCUnexpectedUsageEvent,
-} from "./gc";
+import { GCNodeType, disableDatastoreSweepKey, sendGCUnexpectedUsageEvent } from "./gc";
 import {
 	summarizerClientType,
 	IContainerRuntimeMetadata,
@@ -104,8 +94,6 @@ export class DataStores implements IDisposable {
 	// Stores the ids of new data stores between two GC runs. This is used to notify the garbage collector of new
 	// root data stores that are added.
 	private dataStoresSinceLastGC: string[] = [];
-	/** If true, throw an error when a tombstone data store is retrieved. */
-	private readonly throwOnTombstoneLoad: boolean;
 	// The handle to the container runtime. This is used mainly for GC purposes to represent outbound reference from
 	// the container runtime to other nodes.
 	private readonly containerRuntimeHandle: IFluidHandle;
@@ -140,12 +128,6 @@ export class DataStores implements IDisposable {
 			this.runtime.IFluidHandleContext,
 		);
 
-		// Tombstone should only throw when the feature flag is enabled and the client isn't a summarizer
-		this.throwOnTombstoneLoad =
-			this.mc.config.getBoolean(throwOnTombstoneLoadKey) === true &&
-			this.runtime.gcTombstoneEnforcementAllowed &&
-			this.runtime.clientDetails.type !== summarizerClientType;
-
 		// Extract stores stored inside the snapshot
 		const fluidDataStores = new Map<string, ISnapshotTree>();
 		if (baseSnapshot) {
@@ -455,9 +437,6 @@ export class DataStores implements IDisposable {
 			const request: IRequest = { url: id };
 			throw responseToException(create404Response(request), request);
 		}
-
-		this.validateNotTombstoned(context, requestHeaderData);
-
 		return context;
 	}
 
@@ -477,8 +456,6 @@ export class DataStores implements IDisposable {
 		if (context === undefined) {
 			return undefined;
 		}
-		// Check if the data store is tombstoned. If so, we want to log a telemetry event.
-		this.checkIfTombstoned(context, requestHeaderData);
 		return context;
 	}
 
@@ -529,61 +506,6 @@ export class DataStores implements IDisposable {
 		}
 	}
 
-	/**
-	 * Checks if the data store has not been marked as tombstone by GC or not.
-	 * @param context - the data store context in question
-	 * @param requestHeaderData - the request header information to log if the validation detects the data store has been tombstoned
-	 * @returns true if the data store is tombstoned. Otherwise, returns false.
-	 */
-	private checkIfTombstoned(
-		context: FluidDataStoreContext,
-		requestHeaderData: RuntimeHeaderData,
-	) {
-		if (!context.tombstoned) {
-			return false;
-		}
-		const logErrorEvent = this.throwOnTombstoneLoad && !requestHeaderData.allowTombstone;
-		sendGCUnexpectedUsageEvent(
-			this.mc,
-			{
-				eventName: "GC_Tombstone_DataStore_Requested",
-				category: logErrorEvent ? "error" : "generic",
-				isSummarizerClient: this.runtime.clientDetails.type === summarizerClientType,
-				id: context.id,
-				headers: JSON.stringify(requestHeaderData),
-				gcTombstoneEnforcementAllowed: this.runtime.gcTombstoneEnforcementAllowed,
-			},
-			context.isLoaded ? context.packagePath : undefined,
-		);
-		return true;
-	}
-
-	/**
-	 * Validates that the data store context requested has not been marked as tombstone by GC.
-	 * @param context - the data store context in question
-	 * @param request - the request information to log if the validation detects the data store has been tombstoned
-	 * @param requestHeaderData - the request header information to log if the validation detects the data store has been tombstoned
-	 */
-	private validateNotTombstoned(
-		context: FluidDataStoreContext,
-		requestHeaderData: RuntimeHeaderData,
-	) {
-		if (this.checkIfTombstoned(context, requestHeaderData)) {
-			// The requested data store is removed by gc. Create a 404 gc response exception.
-			const request: IRequest = { url: context.id };
-			const error = responseToException(
-				createResponseError(404, "DataStore was deleted", request, {
-					[TombstoneResponseHeaderKey]: true,
-				}),
-				request,
-			);
-			// Throw an error if configured via options and via request headers.
-			if (this.throwOnTombstoneLoad && !requestHeaderData.allowTombstone) {
-				throw error;
-			}
-		}
-	}
-
 	public processSignal(fluidDataStoreId: string, message: IInboundSignalMessage, local: boolean) {
 		this.validateNotDeleted(fluidDataStoreId);
 		const context = this.contexts.get(fluidDataStoreId);

package/src/gc/garbageCollection.ts

@@ -4,7 +4,7 @@
  */
 
 import { LazyPromise, Timer } from "@fluidframework/core-utils";
-import { IRequest, IRequestHeader } from "@fluidframework/core-interfaces";
+import { IRequest } from "@fluidframework/core-interfaces";
 import {
 	gcTreeKey,
 	IGarbageCollectionData,
@@ -23,9 +23,9 @@ import {
 } from "@fluidframework/telemetry-utils";
 
 import {
-	AllowInactiveRequestHeaderKey,
 	InactiveResponseHeaderKey,
-	RuntimeHeaders,
+	RuntimeHeaderData,
+	TombstoneResponseHeaderKey,
 } from "../containerRuntime";
 import { ClientSessionExpiredError } from "../error";
 import { IRefreshSummaryResult } from "../summary";
@@ -113,6 +113,19 @@ export class GarbageCollector implements IGarbageCollector {
 	private readonly summaryStateTracker: GCSummaryStateTracker;
 	private readonly telemetryTracker: GCTelemetryTracker;
 
+	/** If false, loading or using a Tombstoned object should merely log, not fail */
+	public get tombstoneEnforcementAllowed(): boolean {
+		return this.configs.tombstoneEnforcementAllowed;
+	}
+	/** If true, throw an error when a tombstone data store is retrieved */
+	public get throwOnTombstoneLoad(): boolean {
+		return this.configs.throwOnTombstoneLoad;
+	}
+	/** If true, throw an error when a tombstone data store is used */
+	public get throwOnTombstoneUsage(): boolean {
+		return this.configs.throwOnTombstoneUsage;
+	}
+
 	/** For a given node path, returns the node's package path. */
 	private readonly getNodePackagePath: (
 		nodePath: string,
@@ -176,7 +189,6 @@ export class GarbageCollector implements IGarbageCollector {
 			this.mc,
 			this.configs,
 			this.isSummarizerClient,
-			this.runtime.gcTombstoneEnforcementAllowed,
 			createParams.createContainerMetadata,
 			(nodeId: string) => this.runtime.getNodeType(nodeId),
 			(nodeId: string) => this.unreferencedNodesState.get(nodeId),
@@ -850,11 +862,13 @@ export class GarbageCollector implements IGarbageCollector {
 	}
 
 	/**
-	 * Called when a node with the given id is updated. If the node is inactive, log an error.
+	 * Called when a node with the given id is updated. If the node is inactive or tombstoned, this will log an error
+	 * or throw an error if failing on incorrect usage is configured.
 	 * @param nodePath - The path of the node that changed.
 	 * @param reason - Whether the node was loaded or changed.
 	 * @param timestampMs - The timestamp when the node changed.
 	 * @param packagePath - The package path of the node. This may not be available if the node hasn't been loaded yet.
+	 * @param request - The original request for loads to preserve it in telemetry.
 	 * @param requestHeaders - If the node was loaded via request path, the headers in the request.
 	 */
 	public nodeUpdated(
@@ -862,12 +876,15 @@ export class GarbageCollector implements IGarbageCollector {
 		reason: "Loaded" | "Changed",
 		timestampMs?: number,
 		packagePath?: readonly string[],
-		requestHeaders?: IRequestHeader,
+		request?: IRequest,
+		headerData?: RuntimeHeaderData,
 	) {
 		if (!this.configs.shouldRunGC) {
 			return;
 		}
 
+		const isTombstoned = this.tombstones.includes(nodePath);
+
 		// This will log if appropriate
 		this.telemetryTracker.nodeUsed({
 			id: nodePath,
@@ -876,32 +893,44 @@ export class GarbageCollector implements IGarbageCollector {
 				timestampMs ?? this.runtime.getCurrentReferenceTimestampMs(),
 			packagePath,
 			completedGCRuns: this.completedRuns,
-			isTombstoned: this.tombstones.includes(nodePath),
+			isTombstoned,
 			lastSummaryTime: this.getLastSummaryTimestampMs(),
-			viaHandle: requestHeaders?.[RuntimeHeaders.viaHandle],
+			headers: headerData,
 		});
 
-		// Unless this is a Loaded event, we're done after telemetry tracking
-		if (reason !== "Loaded") {
+		const nodeType = this.runtime.getNodeType(nodePath);
+
+		// Unless this is a Loaded event for a Blob or DataStore, we're done after telemetry tracking
+		if (reason !== "Loaded" || ![GCNodeType.Blob, GCNodeType.DataStore].includes(nodeType)) {
 			return;
 		}
 
-		// We may throw when loading an Inactive object, depending on these preconditions
-		const shouldThrowOnInactiveLoad =
-			!this.isSummarizerClient &&
-			this.configs.throwOnInactiveLoad === true &&
-			requestHeaders?.[AllowInactiveRequestHeaderKey] !== true;
-		const state = this.unreferencedNodesState.get(nodePath)?.state;
-
-		if (shouldThrowOnInactiveLoad && state === "Inactive") {
-			const request: IRequest = { url: nodePath };
-			const error = responseToException(
-				createResponseError(404, "Object is inactive", request, {
-					[InactiveResponseHeaderKey]: true,
+		const errorRequest: IRequest = request ?? { url: nodePath };
+		// If the object is tombstoned and tombstone enforcement is configured, throw an error.
+		if (isTombstoned && this.throwOnTombstoneLoad && headerData?.allowTombstone !== true) {
+			// The requested data store is removed by gc. Create a 404 gc response exception.
+			throw responseToException(
+				createResponseError(404, `${nodeType} was tombstoned`, errorRequest, {
+					[TombstoneResponseHeaderKey]: true,
 				}),
-				request,
+				errorRequest,
 			);
-			throw error;
+		}
+
+		// If the object is inactive and inactive enforcement is configured, throw an error.
+		if (this.unreferencedNodesState.get(nodePath)?.state === "Inactive") {
+			const shouldThrowOnInactiveLoad =
+				!this.isSummarizerClient &&
+				this.configs.throwOnInactiveLoad === true &&
+				headerData?.allowInactive !== true;
+			if (shouldThrowOnInactiveLoad) {
+				throw responseToException(
+					createResponseError(404, `${nodeType} is inactive`, errorRequest, {
+						[InactiveResponseHeaderKey]: true,
+					}),
+					errorRequest,
+				);
+			}
 		}
 	}
 

package/src/gc/gcConfigs.ts

@@ -24,8 +24,11 @@ import {
 	runSessionExpiryKey,
 	runSweepKey,
 	stableGCVersion,
+	throwOnTombstoneLoadOverrideKey,
+	throwOnTombstoneUsageKey,
+	gcThrowOnTombstoneLoadOptionName,
 } from "./gcDefinitions";
-import { getGCVersion, shouldAllowGcSweep } from "./gcHelpers";
+import { getGCVersion, shouldAllowGcSweep, shouldAllowGcTombstoneEnforcement } from "./gcHelpers";
 
 /**
  * Generates configurations for the Garbage Collector that it uses to determine what to run and how.
@@ -42,6 +45,7 @@ export function generateGCConfigs(
 		gcOptions: IGCRuntimeOptions;
 		metadata: IContainerRuntimeMetadata | undefined;
 		existing: boolean;
+		isSummarizerClient: boolean;
 	},
 ): IGarbageCollectorConfigs {
 	let gcEnabled: boolean;
@@ -152,8 +156,6 @@ export function generateGCConfigs(
 		throw new UsageError("inactive timeout should not be greater than the sweep timeout");
 	}
 
-	const throwOnInactiveLoad: boolean | undefined = createParams.gcOptions.throwOnInactiveLoad;
-
 	// Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
 	const testMode =
 		mc.config.getBoolean(gcTestModeKey) ?? createParams.gcOptions.runGCInTestMode === true;
@@ -162,6 +164,25 @@ export function generateGCConfigs(
 	const tombstoneMode = !shouldRunSweep && mc.config.getBoolean(disableTombstoneKey) !== true;
 	const runFullGC = createParams.gcOptions.runFullGC;
 
+	const throwOnInactiveLoad: boolean | undefined = createParams.gcOptions.throwOnInactiveLoad;
+	const tombstoneEnforcementAllowed = shouldAllowGcTombstoneEnforcement(
+		createParams.metadata?.gcFeatureMatrix?.tombstoneGeneration /* persisted */,
+		createParams.gcOptions[gcTombstoneGenerationOptionName] /* current */,
+	);
+
+	const throwOnTombstoneLoadConfig =
+		mc.config.getBoolean(throwOnTombstoneLoadOverrideKey) ??
+		createParams.gcOptions[gcThrowOnTombstoneLoadOptionName] ??
+		false;
+	const throwOnTombstoneLoad =
+		throwOnTombstoneLoadConfig &&
+		tombstoneEnforcementAllowed &&
+		!createParams.isSummarizerClient;
+	const throwOnTombstoneUsage =
+		mc.config.getBoolean(throwOnTombstoneUsageKey) === true &&
+		tombstoneEnforcementAllowed &&
+		!createParams.isSummarizerClient;
+
 	return {
 		gcEnabled,
 		sweepEnabled,
@@ -173,10 +194,13 @@ export function generateGCConfigs(
 		sessionExpiryTimeoutMs,
 		sweepTimeoutMs,
 		inactiveTimeoutMs,
-		throwOnInactiveLoad,
 		persistedGcFeatureMatrix,
 		gcVersionInBaseSnapshot,
 		gcVersionInEffect,
+		throwOnInactiveLoad,
+		tombstoneEnforcementAllowed,
+		throwOnTombstoneLoad,
+		throwOnTombstoneUsage,
 	};
 }
 

package/src/gc/gcDefinitions.ts

@@ -4,7 +4,7 @@
  */
 
 import { ICriticalContainerError } from "@fluidframework/container-definitions";
-import { IRequestHeader } from "@fluidframework/core-interfaces";
+import { IRequest } from "@fluidframework/core-interfaces";
 import { ISnapshotTree } from "@fluidframework/protocol-definitions";
 import {
 	IGarbageCollectionData,
@@ -19,6 +19,7 @@ import {
 	ICreateContainerMetadata,
 	IRefreshSummaryResult,
 } from "../summary";
+import { RuntimeHeaderData } from "../containerRuntime";
 
 export type GCVersion = number;
 
@@ -34,6 +35,13 @@ export const nextGCVersion: GCVersion = 4;
  * Otherwise, only enforce GC Tombstone if the passed in value matches the persisted value
  */
 export const gcTombstoneGenerationOptionName = "gcTombstoneGeneration";
+
+/**
+ * This undocumented GC Option (on ContainerRuntime Options) allows an app to enable throwing an error when tombstone
+ * object is loaded (requested).
+ */
+export const gcThrowOnTombstoneLoadOptionName = "gcThrowOnTombstoneLoad";
+
 /**
  * This GC Option (on ContainerRuntime Options) allows an app to disable GC Sweep on old documents by incrementing this value.
  *
@@ -55,8 +63,9 @@ export const runSessionExpiryKey = "Fluid.GarbageCollection.RunSessionExpiry";
 export const disableSweepLogKey = "Fluid.GarbageCollection.DisableSweepLog";
 /** Config key to disable the tombstone feature, i.e., tombstone information is not read / written into summary. */
 export const disableTombstoneKey = "Fluid.GarbageCollection.DisableTombstone";
-/** Config key to enable throwing an error when tombstone object is loaded (requested). */
-export const throwOnTombstoneLoadKey = "Fluid.GarbageCollection.ThrowOnTombstoneLoad";
+/** Config key to override throwing an error when tombstone object is loaded (requested). */
+export const throwOnTombstoneLoadOverrideKey =
+	"Fluid.GarbageCollection.ThrowOnTombstoneLoadOverride";
 /** Config key to enable throwing an error when tombstone object is used (e.g. outgoing or incoming ops). */
 export const throwOnTombstoneUsageKey = "Fluid.GarbageCollection.ThrowOnTombstoneUsage";
 /** Config key to enable GC version upgrade. */
@@ -193,8 +202,6 @@ export interface IGarbageCollectionRuntime {
 	getNodeType(nodePath: string): GCNodeType;
 	/** Called when the runtime should close because of an error. */
 	closeFn: (error?: ICriticalContainerError) => void;
-	/** If false, loading or using a Tombstoned object should merely log, not fail */
-	gcTombstoneEnforcementAllowed: boolean;
 }
 
 /** Defines the contract for the garbage collector. */
@@ -205,6 +212,12 @@ export interface IGarbageCollector {
 	readonly summaryStateNeedsReset: boolean;
 	/** The count of data stores whose GC state updated since the last summary. */
 	readonly updatedDSCountSinceLastSummary: number;
+	/** Tells whether tombstone feature is enabled and enforced. */
+	readonly tombstoneEnforcementAllowed: boolean;
+	/** Tells whether loading a tombstone object should fail or merely log. */
+	readonly throwOnTombstoneLoad: boolean;
+	/** Tells whether using a tombstone object should fail or merely log. */
+	readonly throwOnTombstoneUsage: boolean;
 	/** Initialize the state from the base snapshot after its creation. */
 	initializeBaseState(): Promise<void>;
 	/** Run garbage collection and update the reference / used state of the system. */
@@ -228,13 +241,17 @@ export interface IGarbageCollector {
 	getBaseGCDetails(): Promise<IGarbageCollectionDetailsBase>;
 	/** Called when the latest summary of the system has been refreshed. */
 	refreshLatestSummary(result: IRefreshSummaryResult): Promise<void>;
-	/** Called when a node is updated. Used to detect and log when an inactive node is changed or loaded. */
+	/**
+	 * Called when a node with the given path is updated. If the node is inactive or tombstoned, this will log an error
+	 * or throw an error if failing on incorrect usage is configured.
+	 */
 	nodeUpdated(
 		nodePath: string,
 		reason: "Loaded" | "Changed",
 		timestampMs?: number,
 		packagePath?: readonly string[],
-		requestHeaders?: IRequestHeader,
+		request?: IRequest,
+		headerData?: RuntimeHeaderData,
 	): void;
 	/** Called when a reference is added to a node. Used to identify nodes that were referenced between summaries. */
 	addedOutboundReference(fromNodePath: string, toNodePath: string): void;
@@ -332,8 +349,6 @@ export interface IGarbageCollectorConfigs {
 	readonly sweepTimeoutMs: number | undefined;
 	/** The time after which an unreferenced node is inactive. */
 	readonly inactiveTimeoutMs: number;
-	/** It is easier for users to diagnose InactiveObject usage if we throw on load, which this option enables */
-	readonly throwOnInactiveLoad: boolean | undefined;
 	/** Tracks whether GC should run in test mode. In this mode, unreferenced objects are deleted immediately. */
 	readonly testMode: boolean;
 	/**
@@ -349,6 +364,14 @@ export interface IGarbageCollectorConfigs {
 	readonly gcVersionInBaseSnapshot: GCVersion | undefined;
 	/** The current version of GC data in the running code */
 	readonly gcVersionInEffect: GCVersion;
+	/** It is easier for users to diagnose InactiveObject usage if we throw on load, which this option enables */
+	readonly throwOnInactiveLoad: boolean | undefined;
+	/** If false, loading or using a Tombstoned object should merely log, not fail */
+	readonly tombstoneEnforcementAllowed: boolean;
+	/** If true, throw an error when a tombstone data store is retrieved */
+	readonly throwOnTombstoneLoad: boolean;
+	/** If true, throw an error when a tombstone data store is used. */
+	readonly throwOnTombstoneUsage: boolean;
 }
 
 /** The state of node that is unreferenced. */