@fluidframework/container-runtime 2.0.0-dev.7.2.0.204906 → 2.0.0-dev.7.3.0.206769

package/src/gc/gcTelemetry.ts

@@ -11,6 +11,7 @@ import {
 	MonitoringContext,
 	tagCodeArtifacts,
 } from "@fluidframework/telemetry-utils";
+import { RuntimeHeaderData } from "../containerRuntime";
 import { ICreateContainerMetadata } from "../summary";
 import {
 	disableSweepLogKey,
@@ -32,7 +33,7 @@ interface ICommonProps {
 	completedGCRuns: number;
 	isTombstoned: boolean;
 	lastSummaryTime?: number;
-	viaHandle?: boolean;
+	headers?: RuntimeHeaderData;
 }
 
 /** The event that is logged when unreferenced node is used after a certain time. */
@@ -80,7 +81,11 @@ export class GCTelemetryTracker {
 		private readonly mc: MonitoringContext,
 		private readonly configs: Pick<
 			IGarbageCollectorConfigs,
-			"inactiveTimeoutMs" | "sweepTimeoutMs" | "tombstoneEnforcementAllowed"
+			| "inactiveTimeoutMs"
+			| "sweepTimeoutMs"
+			| "tombstoneEnforcementAllowed"
+			| "throwOnTombstoneLoad"
+			| "throwOnTombstoneUsage"
 		>,
 		private readonly isSummarizerClient: boolean,
 		private readonly createContainerMetadata: ICreateContainerMetadata,
@@ -94,12 +99,12 @@ export class GCTelemetryTracker {
 	) {}
 
 	/**
-	 * Returns whether an event should be logged for a node that isn't active anymore. Some scenarios where we won't log:
+	 * Returns whether an event should be logged for a node that isn't active anymore. This does not apply to
+	 * tombstoned nodes for which an event is always logged. Some scenarios where we won't log:
 	 * 1. When a DDS is changed. The corresponding data store's event will be logged instead.
 	 * 2. An event is logged only once per container instance per event per node.
 	 */
 	private shouldLogNonActiveEvent(
-		nodeId: string,
 		nodeType: GCNodeType,
 		usageType: NodeUsageType,
 		nodeStateTracker: UnreferencedStateTracker,
@@ -119,6 +124,8 @@ export class GCTelemetryTracker {
 			return false;
 		}
 
+		// Non-tombstone events are logged once per event per node. A unique id is generated by joining
+		// node state (inactive / sweep ready), node's id and usage (loaded / changed / revived).
 		if (this.loggedUnreferencedEvents.has(uniqueEventId)) {
 			return false;
 		}
@@ -126,24 +133,61 @@ export class GCTelemetryTracker {
 	}
 
 	/**
-	 * Called when a node is used. If the node is not active, log an event indicating object is used when its not active.
+	 * Called when a node is used. If the node is not active or tombstoned, log telemetry indicating object is used
+	 * when it should not have been.
 	 */
 	public nodeUsed(nodeUsageProps: INodeUsageProps) {
 		// If there is no reference timestamp to work with, no ops have been processed after creation. If so, skip
 		// logging as nothing interesting would have happened worth logging.
-		// If the node is not unreferenced, skip logging.
-		const nodeStateTracker = this.getNodeStateTracker(nodeUsageProps.id);
-		if (!nodeStateTracker || nodeUsageProps.currentReferenceTimestampMs === undefined) {
+		if (nodeUsageProps.currentReferenceTimestampMs === undefined) {
 			return;
 		}
 
-		// We log these events once per event per node. A unique id is generated by joining node state (inactive / sweep ready),
-		// node's id and usage (loaded / changed / revived).
-		const uniqueEventId = `${nodeStateTracker.state}-${nodeUsageProps.id}-${nodeUsageProps.usageType}`;
+		const nodeStateTracker = this.getNodeStateTracker(nodeUsageProps.id);
 		const nodeType = this.getNodeType(nodeUsageProps.id);
+		const {
+			usageType,
+			currentReferenceTimestampMs,
+			packagePath,
+			id: untaggedId,
+			fromId: untaggedFromId,
+			...propsToLog
+		} = nodeUsageProps;
+		const unrefEventProps: Omit<IUnreferencedEventProps, "state" | "usageType"> = {
+			type: nodeType,
+			unrefTime: nodeStateTracker?.unreferencedTimestampMs ?? -1,
+			age:
+				nodeStateTracker !== undefined
+					? nodeUsageProps.currentReferenceTimestampMs -
+					  nodeStateTracker.unreferencedTimestampMs
+					: -1,
+			timeout:
+				nodeStateTracker?.state === UnreferencedState.Inactive
+					? this.configs.inactiveTimeoutMs
+					: this.configs.sweepTimeoutMs,
+			...tagCodeArtifacts({ id: untaggedId, fromId: untaggedFromId }),
+			...propsToLog,
+			...this.createContainerMetadata,
+		};
+
+		// If the node that is used is tombstoned, log a tombstone telemetry.
+		// Note that this is done before checking if "nodeStateTracker" is undefined below because unreferenced
+		// tracking may not have yet been enabled. That happens only after the client transitions to write mode.
+		if (nodeUsageProps.isTombstoned) {
+			this.logTombstoneUsageTelemetry(nodeUsageProps, unrefEventProps, nodeType, usageType);
+		}
+
+		// After logging tombstone telemetry, if the node's unreferenced state is not tracked, there is nothing
+		// else to log.
+		if (nodeStateTracker === undefined) {
+			return;
+		}
+
+		const state = nodeStateTracker.state;
+		const uniqueEventId = `${state}-${nodeUsageProps.id}-${nodeUsageProps.usageType}`;
+
 		if (
 			!this.shouldLogNonActiveEvent(
-				nodeUsageProps.id,
 				nodeType,
 				nodeUsageProps.usageType,
 				nodeStateTracker,
@@ -153,42 +197,9 @@ export class GCTelemetryTracker {
 			return;
 		}
 
-		// Add the unique event id so that we don't generate a log for this event again in this session..
+		// Add the unique event id so that we don't generate a log for this event again in this session.
 		this.loggedUnreferencedEvents.add(uniqueEventId);
 
-		const state = nodeStateTracker.state;
-		const { usageType, currentReferenceTimestampMs, packagePath, id, fromId, ...propsToLog } =
-			nodeUsageProps;
-		const eventProps: Omit<IUnreferencedEventProps, "state" | "usageType"> = {
-			type: nodeType,
-			unrefTime: nodeStateTracker.unreferencedTimestampMs,
-			age:
-				nodeUsageProps.currentReferenceTimestampMs -
-				nodeStateTracker.unreferencedTimestampMs,
-			timeout:
-				state === UnreferencedState.Inactive
-					? this.configs.inactiveTimeoutMs
-					: this.configs.sweepTimeoutMs,
-			...tagCodeArtifacts({ id, fromId }),
-			...propsToLog,
-			...this.createContainerMetadata,
-		};
-
-		// This will log the following events:
-		// GC_Tombstone_DataStore_Revived, GC_Tombstone_SubDataStore_Revived, GC_Tombstone_Blob_Revived
-		if (nodeUsageProps.usageType === "Revived" && nodeUsageProps.isTombstoned) {
-			sendGCUnexpectedUsageEvent(
-				this.mc,
-				{
-					eventName: `GC_Tombstone_${nodeType}_Revived`,
-					category: "generic",
-					...tagCodeArtifacts({ url: id }),
-					gcTombstoneEnforcementAllowed: this.configs.tombstoneEnforcementAllowed,
-				},
-				undefined /* packagePath */,
-			);
-		}
-
 		// For summarizer client, queue the event so it is logged the next time GC runs if the event is still valid.
 		// For non-summarizer client, log the event now since GC won't run on it. This may result in false positives
 		// but it's a good signal nonetheless and we can consume it with a grain of salt.
@@ -196,7 +207,7 @@ export class GCTelemetryTracker {
 		// SweepReady errors are usages of Objects that will be deleted by GC Sweep!
 		if (this.isSummarizerClient) {
 			this.pendingEventsQueue.push({
-				...eventProps,
+				...unrefEventProps,
 				usageType: nodeUsageProps.usageType,
 				state,
 			});
@@ -206,16 +217,15 @@ export class GCTelemetryTracker {
 			// Events generated:
 			// InactiveObject_Loaded, SweepReadyObject_Loaded
 			if (nodeUsageProps.usageType === "Loaded") {
-				const { id: taggedId, fromId: taggedFromId, ...otherProps } = eventProps;
+				const { id, fromId, headers, ...detailedProps } = unrefEventProps;
 				const event = {
 					eventName: `${state}Object_${nodeUsageProps.usageType}`,
-					pkg: tagCodeArtifacts({ pkg: nodeUsageProps.packagePath?.join("/") }).pkg,
+					...tagCodeArtifacts({ pkg: nodeUsageProps.packagePath?.join("/") }),
 					stack: generateStack(),
-					id: taggedId,
-					fromId: taggedFromId,
-					details: JSON.stringify({
-						...otherProps,
-					}),
+					id,
+					fromId,
+					headers: { ...headers },
+					details: detailedProps,
 				};
 
 				// Do not log the inactive object x events as error events as they are not the best signal for
@@ -229,6 +239,44 @@ export class GCTelemetryTracker {
 		}
 	}
 
+	/**
+	 * Logs telemetry when a tombstoned object is changed, revived or loaded.
+	 */
+	private logTombstoneUsageTelemetry(
+		nodeUsageProps: INodeUsageProps,
+		unrefEventProps: Omit<IUnreferencedEventProps, "state" | "usageType">,
+		nodeType: GCNodeType,
+		usageType: NodeUsageType,
+	) {
+		// This will log the following events:
+		// GC_Tombstone_DataStore_Requested, GC_Tombstone_DataStore_Changed, GC_Tombstone_DataStore_Revived
+		// GC_Tombstone_SubDataStore_Requested, GC_Tombstone_SubDataStore_Changed, GC_Tombstone_SubDataStore_Revived
+		// GC_Tombstone_Blob_Requested, GC_Tombstone_Blob_Changed, GC_Tombstone_Blob_Revived
+		const { id, fromId, headers, ...detailedProps } = unrefEventProps;
+		const eventUsageName = usageType === "Loaded" ? "Requested" : usageType;
+		const event = {
+			eventName: `GC_Tombstone_${nodeType}_${eventUsageName}`,
+			pkg: tagCodeArtifacts({ pkg: nodeUsageProps.packagePath?.join("/") }).pkg,
+			stack: generateStack(),
+			id,
+			fromId,
+			headers: { ...headers },
+			details: detailedProps,
+			gcTombstoneEnforcementAllowed: this.configs.tombstoneEnforcementAllowed,
+		};
+
+		if (
+			(usageType === "Loaded" &&
+				this.configs.throwOnTombstoneLoad &&
+				!headers?.allowTombstone) ||
+			(usageType === "Changed" && this.configs.throwOnTombstoneUsage)
+		) {
+			this.mc.logger.sendErrorEvent(event);
+		} else {
+			this.mc.logger.sendTelemetryEvent(event);
+		}
+	}
+
 	/**
 	 * Log all new references or outbound routes in the current graph that haven't been explicitly notified to GC.
 	 * The principle is that every new reference or outbound route must be notified to GC via the

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.0.0-dev.7.2.0.204906";
+export const pkgVersion = "2.0.0-dev.7.3.0.206769";