@fluidframework/container-runtime 2.0.0-internal.7.2.0 → 2.0.0-internal.7.2.1

package/src/dataStores.ts

@@ -50,12 +50,7 @@ import { buildSnapshotTree } from "@fluidframework/driver-utils";
 import { assert, Lazy } from "@fluidframework/core-utils";
 import { v4 as uuid } from "uuid";
 import { DataStoreContexts } from "./dataStoreContexts";
-import {
-	ContainerRuntime,
-	defaultRuntimeHeaderData,
-	RuntimeHeaderData,
-	TombstoneResponseHeaderKey,
-} from "./containerRuntime";
+import { ContainerRuntime, defaultRuntimeHeaderData, RuntimeHeaderData } from "./containerRuntime";
 import {
 	FluidDataStoreContext,
 	RemoteFluidDataStoreContext,
@@ -442,9 +437,6 @@ export class DataStores implements IDisposable {
 			const request: IRequest = { url: id };
 			throw responseToException(create404Response(request), request);
 		}
-
-		this.validateNotTombstoned(context, requestHeaderData);
-
 		return context;
 	}
 
@@ -464,8 +456,6 @@ export class DataStores implements IDisposable {
 		if (context === undefined) {
 			return undefined;
 		}
-		// Check if the data store is tombstoned. If so, we want to log a telemetry event.
-		this.checkIfTombstoned(context, requestHeaderData);
 		return context;
 	}
 
@@ -516,62 +506,6 @@ export class DataStores implements IDisposable {
 		}
 	}
 
-	/**
-	 * Checks if the data store has not been marked as tombstone by GC or not.
-	 * @param context - the data store context in question
-	 * @param requestHeaderData - the request header information to log if the validation detects the data store has been tombstoned
-	 * @returns true if the data store is tombstoned. Otherwise, returns false.
-	 */
-	private checkIfTombstoned(
-		context: FluidDataStoreContext,
-		requestHeaderData: RuntimeHeaderData,
-	) {
-		if (!context.tombstoned) {
-			return false;
-		}
-		const logErrorEvent =
-			this.runtime.gcThrowOnTombstoneLoad && !requestHeaderData.allowTombstone;
-		sendGCUnexpectedUsageEvent(
-			this.mc,
-			{
-				eventName: "GC_Tombstone_DataStore_Requested",
-				category: logErrorEvent ? "error" : "generic",
-				isSummarizerClient: this.runtime.clientDetails.type === summarizerClientType,
-				id: context.id,
-				headers: JSON.stringify(requestHeaderData),
-				gcTombstoneEnforcementAllowed: this.runtime.gcTombstoneEnforcementAllowed,
-			},
-			context.isLoaded ? context.packagePath : undefined,
-		);
-		return true;
-	}
-
-	/**
-	 * Validates that the data store context requested has not been marked as tombstone by GC.
-	 * @param context - the data store context in question
-	 * @param request - the request information to log if the validation detects the data store has been tombstoned
-	 * @param requestHeaderData - the request header information to log if the validation detects the data store has been tombstoned
-	 */
-	private validateNotTombstoned(
-		context: FluidDataStoreContext,
-		requestHeaderData: RuntimeHeaderData,
-	) {
-		if (this.checkIfTombstoned(context, requestHeaderData)) {
-			// The requested data store is removed by gc. Create a 404 gc response exception.
-			const request: IRequest = { url: context.id };
-			const error = responseToException(
-				createResponseError(404, "DataStore was deleted", request, {
-					[TombstoneResponseHeaderKey]: true,
-				}),
-				request,
-			);
-			// Throw an error if configured via options and via request headers.
-			if (this.runtime.gcThrowOnTombstoneLoad && !requestHeaderData.allowTombstone) {
-				throw error;
-			}
-		}
-	}
-
 	public processSignal(fluidDataStoreId: string, message: IInboundSignalMessage, local: boolean) {
 		this.validateNotDeleted(fluidDataStoreId);
 		const context = this.contexts.get(fluidDataStoreId);

package/src/gc/garbageCollection.ts

@@ -4,7 +4,7 @@
  */
 
 import { LazyPromise, Timer } from "@fluidframework/core-utils";
-import { IRequest, IRequestHeader } from "@fluidframework/core-interfaces";
+import { IRequest } from "@fluidframework/core-interfaces";
 import {
 	gcTreeKey,
 	IGarbageCollectionData,
@@ -23,9 +23,9 @@ import {
 } from "@fluidframework/telemetry-utils";
 
 import {
-	AllowInactiveRequestHeaderKey,
 	InactiveResponseHeaderKey,
-	RuntimeHeaders,
+	RuntimeHeaderData,
+	TombstoneResponseHeaderKey,
 } from "../containerRuntime";
 import { ClientSessionExpiredError } from "../error";
 import { IRefreshSummaryResult } from "../summary";
@@ -862,11 +862,13 @@ export class GarbageCollector implements IGarbageCollector {
 	}
 
 	/**
-	 * Called when a node with the given id is updated. If the node is inactive, log an error.
+	 * Called when a node with the given id is updated. If the node is inactive or tombstoned, this will log an error
+	 * or throw an error if failing on incorrect usage is configured.
 	 * @param nodePath - The path of the node that changed.
 	 * @param reason - Whether the node was loaded or changed.
 	 * @param timestampMs - The timestamp when the node changed.
 	 * @param packagePath - The package path of the node. This may not be available if the node hasn't been loaded yet.
+	 * @param request - The original request for loads to preserve it in telemetry.
 	 * @param requestHeaders - If the node was loaded via request path, the headers in the request.
 	 */
 	public nodeUpdated(
@@ -874,12 +876,15 @@ export class GarbageCollector implements IGarbageCollector {
 		reason: "Loaded" | "Changed",
 		timestampMs?: number,
 		packagePath?: readonly string[],
-		requestHeaders?: IRequestHeader,
+		request?: IRequest,
+		headerData?: RuntimeHeaderData,
 	) {
 		if (!this.configs.shouldRunGC) {
 			return;
 		}
 
+		const isTombstoned = this.tombstones.includes(nodePath);
+
 		// This will log if appropriate
 		this.telemetryTracker.nodeUsed({
 			id: nodePath,
@@ -888,35 +893,44 @@ export class GarbageCollector implements IGarbageCollector {
 				timestampMs ?? this.runtime.getCurrentReferenceTimestampMs(),
 			packagePath,
 			completedGCRuns: this.completedRuns,
-			isTombstoned: this.tombstones.includes(nodePath),
+			isTombstoned,
 			lastSummaryTime: this.getLastSummaryTimestampMs(),
-			viaHandle: requestHeaders?.[RuntimeHeaders.viaHandle],
+			headers: headerData,
 		});
 
+		const nodeType = this.runtime.getNodeType(nodePath);
+
 		// Unless this is a Loaded event for a Blob or DataStore, we're done after telemetry tracking
-		if (
-			reason !== "Loaded" ||
-			![GCNodeType.Blob, GCNodeType.DataStore].includes(this.runtime.getNodeType(nodePath))
-		) {
+		if (reason !== "Loaded" || ![GCNodeType.Blob, GCNodeType.DataStore].includes(nodeType)) {
 			return;
 		}
 
-		// We may throw when loading an Inactive object, depending on these preconditions
-		const shouldThrowOnInactiveLoad =
-			!this.isSummarizerClient &&
-			this.configs.throwOnInactiveLoad === true &&
-			requestHeaders?.[AllowInactiveRequestHeaderKey] !== true;
-		const state = this.unreferencedNodesState.get(nodePath)?.state;
-
-		if (shouldThrowOnInactiveLoad && state === "Inactive") {
-			const request: IRequest = { url: nodePath };
-			const error = responseToException(
-				createResponseError(404, "Object is inactive", request, {
-					[InactiveResponseHeaderKey]: true,
+		const errorRequest: IRequest = request ?? { url: nodePath };
+		// If the object is tombstoned and tombstone enforcement is configured, throw an error.
+		if (isTombstoned && this.throwOnTombstoneLoad && headerData?.allowTombstone !== true) {
+			// The requested data store is removed by gc. Create a 404 gc response exception.
+			throw responseToException(
+				createResponseError(404, `${nodeType} was tombstoned`, errorRequest, {
+					[TombstoneResponseHeaderKey]: true,
 				}),
-				request,
+				errorRequest,
 			);
-			throw error;
+		}
+
+		// If the object is inactive and inactive enforcement is configured, throw an error.
+		if (this.unreferencedNodesState.get(nodePath)?.state === "Inactive") {
+			const shouldThrowOnInactiveLoad =
+				!this.isSummarizerClient &&
+				this.configs.throwOnInactiveLoad === true &&
+				headerData?.allowInactive !== true;
+			if (shouldThrowOnInactiveLoad) {
+				throw responseToException(
+					createResponseError(404, `${nodeType} is inactive`, errorRequest, {
+						[InactiveResponseHeaderKey]: true,
+					}),
+					errorRequest,
+				);
+			}
 		}
 	}
 

package/src/gc/gcDefinitions.ts

@@ -4,7 +4,7 @@
  */
 
 import { ICriticalContainerError } from "@fluidframework/container-definitions";
-import { IRequestHeader } from "@fluidframework/core-interfaces";
+import { IRequest } from "@fluidframework/core-interfaces";
 import { ISnapshotTree } from "@fluidframework/protocol-definitions";
 import {
 	IGarbageCollectionData,
@@ -19,6 +19,7 @@ import {
 	ICreateContainerMetadata,
 	IRefreshSummaryResult,
 } from "../summary";
+import { RuntimeHeaderData } from "../containerRuntime";
 
 /**
  * @public
@@ -251,13 +252,17 @@ export interface IGarbageCollector {
 	getBaseGCDetails(): Promise<IGarbageCollectionDetailsBase>;
 	/** Called when the latest summary of the system has been refreshed. */
 	refreshLatestSummary(result: IRefreshSummaryResult): Promise<void>;
-	/** Called when a node is updated. Used to detect and log when an inactive node is changed or loaded. */
+	/**
+	 * Called when a node with the given path is updated. If the node is inactive or tombstoned, this will log an error
+	 * or throw an error if failing on incorrect usage is configured.
+	 */
 	nodeUpdated(
 		nodePath: string,
 		reason: "Loaded" | "Changed",
 		timestampMs?: number,
 		packagePath?: readonly string[],
-		requestHeaders?: IRequestHeader,
+		request?: IRequest,
+		headerData?: RuntimeHeaderData,
 	): void;
 	/** Called when a reference is added to a node. Used to identify nodes that were referenced between summaries. */
 	addedOutboundReference(fromNodePath: string, toNodePath: string): void;

package/src/gc/gcTelemetry.ts

@@ -11,6 +11,7 @@ import {
 	MonitoringContext,
 	tagCodeArtifacts,
 } from "@fluidframework/telemetry-utils";
+import { RuntimeHeaderData } from "../containerRuntime";
 import { ICreateContainerMetadata } from "../summary";
 import {
 	disableSweepLogKey,
@@ -32,7 +33,7 @@ interface ICommonProps {
 	completedGCRuns: number;
 	isTombstoned: boolean;
 	lastSummaryTime?: number;
-	viaHandle?: boolean;
+	headers?: RuntimeHeaderData;
 }
 
 /** The event that is logged when unreferenced node is used after a certain time. */
@@ -80,7 +81,11 @@ export class GCTelemetryTracker {
 		private readonly mc: MonitoringContext,
 		private readonly configs: Pick<
 			IGarbageCollectorConfigs,
-			"inactiveTimeoutMs" | "sweepTimeoutMs" | "tombstoneEnforcementAllowed"
+			| "inactiveTimeoutMs"
+			| "sweepTimeoutMs"
+			| "tombstoneEnforcementAllowed"
+			| "throwOnTombstoneLoad"
+			| "throwOnTombstoneUsage"
 		>,
 		private readonly isSummarizerClient: boolean,
 		private readonly createContainerMetadata: ICreateContainerMetadata,
@@ -94,12 +99,12 @@ export class GCTelemetryTracker {
 	) {}
 
 	/**
-	 * Returns whether an event should be logged for a node that isn't active anymore. Some scenarios where we won't log:
+	 * Returns whether an event should be logged for a node that isn't active anymore. This does not apply to
+	 * tombstoned nodes for which an event is always logged. Some scenarios where we won't log:
 	 * 1. When a DDS is changed. The corresponding data store's event will be logged instead.
 	 * 2. An event is logged only once per container instance per event per node.
 	 */
 	private shouldLogNonActiveEvent(
-		nodeId: string,
 		nodeType: GCNodeType,
 		usageType: NodeUsageType,
 		nodeStateTracker: UnreferencedStateTracker,
@@ -119,6 +124,8 @@ export class GCTelemetryTracker {
 			return false;
 		}
 
+		// Non-tombstone events are logged once per event per node. A unique id is generated by joining
+		// node state (inactive / sweep ready), node's id and usage (loaded / changed / revived).
 		if (this.loggedUnreferencedEvents.has(uniqueEventId)) {
 			return false;
 		}
@@ -126,24 +133,61 @@ export class GCTelemetryTracker {
 	}
 
 	/**
-	 * Called when a node is used. If the node is not active, log an event indicating object is used when its not active.
+	 * Called when a node is used. If the node is not active or tombstoned, log telemetry indicating object is used
+	 * when it should not have been.
 	 */
 	public nodeUsed(nodeUsageProps: INodeUsageProps) {
 		// If there is no reference timestamp to work with, no ops have been processed after creation. If so, skip
 		// logging as nothing interesting would have happened worth logging.
-		// If the node is not unreferenced, skip logging.
-		const nodeStateTracker = this.getNodeStateTracker(nodeUsageProps.id);
-		if (!nodeStateTracker || nodeUsageProps.currentReferenceTimestampMs === undefined) {
+		if (nodeUsageProps.currentReferenceTimestampMs === undefined) {
 			return;
 		}
 
-		// We log these events once per event per node. A unique id is generated by joining node state (inactive / sweep ready),
-		// node's id and usage (loaded / changed / revived).
-		const uniqueEventId = `${nodeStateTracker.state}-${nodeUsageProps.id}-${nodeUsageProps.usageType}`;
+		const nodeStateTracker = this.getNodeStateTracker(nodeUsageProps.id);
 		const nodeType = this.getNodeType(nodeUsageProps.id);
+		const {
+			usageType,
+			currentReferenceTimestampMs,
+			packagePath,
+			id: untaggedId,
+			fromId: untaggedFromId,
+			...propsToLog
+		} = nodeUsageProps;
+		const unrefEventProps: Omit<IUnreferencedEventProps, "state" | "usageType"> = {
+			type: nodeType,
+			unrefTime: nodeStateTracker?.unreferencedTimestampMs ?? -1,
+			age:
+				nodeStateTracker !== undefined
+					? nodeUsageProps.currentReferenceTimestampMs -
+					  nodeStateTracker.unreferencedTimestampMs
+					: -1,
+			timeout:
+				nodeStateTracker?.state === UnreferencedState.Inactive
+					? this.configs.inactiveTimeoutMs
+					: this.configs.sweepTimeoutMs,
+			...tagCodeArtifacts({ id: untaggedId, fromId: untaggedFromId }),
+			...propsToLog,
+			...this.createContainerMetadata,
+		};
+
+		// If the node that is used is tombstoned, log a tombstone telemetry.
+		// Note that this is done before checking if "nodeStateTracker" is undefined below because unreferenced
+		// tracking may not have yet been enabled. That happens only after the client transitions to write mode.
+		if (nodeUsageProps.isTombstoned) {
+			this.logTombstoneUsageTelemetry(nodeUsageProps, unrefEventProps, nodeType, usageType);
+		}
+
+		// After logging tombstone telemetry, if the node's unreferenced state is not tracked, there is nothing
+		// else to log.
+		if (nodeStateTracker === undefined) {
+			return;
+		}
+
+		const state = nodeStateTracker.state;
+		const uniqueEventId = `${state}-${nodeUsageProps.id}-${nodeUsageProps.usageType}`;
+
 		if (
 			!this.shouldLogNonActiveEvent(
-				nodeUsageProps.id,
 				nodeType,
 				nodeUsageProps.usageType,
 				nodeStateTracker,
@@ -153,42 +197,9 @@ export class GCTelemetryTracker {
 			return;
 		}
 
-		// Add the unique event id so that we don't generate a log for this event again in this session..
+		// Add the unique event id so that we don't generate a log for this event again in this session.
 		this.loggedUnreferencedEvents.add(uniqueEventId);
 
-		const state = nodeStateTracker.state;
-		const { usageType, currentReferenceTimestampMs, packagePath, id, fromId, ...propsToLog } =
-			nodeUsageProps;
-		const eventProps: Omit<IUnreferencedEventProps, "state" | "usageType"> = {
-			type: nodeType,
-			unrefTime: nodeStateTracker.unreferencedTimestampMs,
-			age:
-				nodeUsageProps.currentReferenceTimestampMs -
-				nodeStateTracker.unreferencedTimestampMs,
-			timeout:
-				state === UnreferencedState.Inactive
-					? this.configs.inactiveTimeoutMs
-					: this.configs.sweepTimeoutMs,
-			...tagCodeArtifacts({ id, fromId }),
-			...propsToLog,
-			...this.createContainerMetadata,
-		};
-
-		// This will log the following events:
-		// GC_Tombstone_DataStore_Revived, GC_Tombstone_SubDataStore_Revived, GC_Tombstone_Blob_Revived
-		if (nodeUsageProps.usageType === "Revived" && nodeUsageProps.isTombstoned) {
-			sendGCUnexpectedUsageEvent(
-				this.mc,
-				{
-					eventName: `GC_Tombstone_${nodeType}_Revived`,
-					category: "generic",
-					...tagCodeArtifacts({ url: id }),
-					gcTombstoneEnforcementAllowed: this.configs.tombstoneEnforcementAllowed,
-				},
-				undefined /* packagePath */,
-			);
-		}
-
 		// For summarizer client, queue the event so it is logged the next time GC runs if the event is still valid.
 		// For non-summarizer client, log the event now since GC won't run on it. This may result in false positives
 		// but it's a good signal nonetheless and we can consume it with a grain of salt.
@@ -196,7 +207,7 @@ export class GCTelemetryTracker {
 		// SweepReady errors are usages of Objects that will be deleted by GC Sweep!
 		if (this.isSummarizerClient) {
 			this.pendingEventsQueue.push({
-				...eventProps,
+				...unrefEventProps,
 				usageType: nodeUsageProps.usageType,
 				state,
 			});
@@ -206,16 +217,15 @@ export class GCTelemetryTracker {
 			// Events generated:
 			// InactiveObject_Loaded, SweepReadyObject_Loaded
 			if (nodeUsageProps.usageType === "Loaded") {
-				const { id: taggedId, fromId: taggedFromId, ...otherProps } = eventProps;
+				const { id, fromId, headers, ...detailedProps } = unrefEventProps;
 				const event = {
 					eventName: `${state}Object_${nodeUsageProps.usageType}`,
-					pkg: tagCodeArtifacts({ pkg: nodeUsageProps.packagePath?.join("/") }).pkg,
+					...tagCodeArtifacts({ pkg: nodeUsageProps.packagePath?.join("/") }),
 					stack: generateStack(),
-					id: taggedId,
-					fromId: taggedFromId,
-					details: JSON.stringify({
-						...otherProps,
-					}),
+					id,
+					fromId,
+					headers: { ...headers },
+					details: detailedProps,
 				};
 
 				// Do not log the inactive object x events as error events as they are not the best signal for
@@ -229,6 +239,44 @@ export class GCTelemetryTracker {
 		}
 	}
 
+	/**
+	 * Logs telemetry when a tombstoned object is changed, revived or loaded.
+	 */
+	private logTombstoneUsageTelemetry(
+		nodeUsageProps: INodeUsageProps,
+		unrefEventProps: Omit<IUnreferencedEventProps, "state" | "usageType">,
+		nodeType: GCNodeType,
+		usageType: NodeUsageType,
+	) {
+		// This will log the following events:
+		// GC_Tombstone_DataStore_Requested, GC_Tombstone_DataStore_Changed, GC_Tombstone_DataStore_Revived
+		// GC_Tombstone_SubDataStore_Requested, GC_Tombstone_SubDataStore_Changed, GC_Tombstone_SubDataStore_Revived
+		// GC_Tombstone_Blob_Requested, GC_Tombstone_Blob_Changed, GC_Tombstone_Blob_Revived
+		const { id, fromId, headers, ...detailedProps } = unrefEventProps;
+		const eventUsageName = usageType === "Loaded" ? "Requested" : usageType;
+		const event = {
+			eventName: `GC_Tombstone_${nodeType}_${eventUsageName}`,
+			pkg: tagCodeArtifacts({ pkg: nodeUsageProps.packagePath?.join("/") }).pkg,
+			stack: generateStack(),
+			id,
+			fromId,
+			headers: { ...headers },
+			details: detailedProps,
+			gcTombstoneEnforcementAllowed: this.configs.tombstoneEnforcementAllowed,
+		};
+
+		if (
+			(usageType === "Loaded" &&
+				this.configs.throwOnTombstoneLoad &&
+				!headers?.allowTombstone) ||
+			(usageType === "Changed" && this.configs.throwOnTombstoneUsage)
+		) {
+			this.mc.logger.sendErrorEvent(event);
+		} else {
+			this.mc.logger.sendTelemetryEvent(event);
+		}
+	}
+
 	/**
 	 * Log all new references or outbound routes in the current graph that haven't been explicitly notified to GC.
 	 * The principle is that every new reference or outbound route must be notified to GC via the

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.0.0-internal.7.2.0";
+export const pkgVersion = "2.0.0-internal.7.2.1";