@fluidframework/container-runtime 2.0.0-internal.7.1.0 → 2.0.0-internal.7.1.1

package/src/gc/gcConfigs.ts

@@ -24,8 +24,10 @@ import {
 	runSessionExpiryKey,
 	runSweepKey,
 	stableGCVersion,
+	throwOnTombstoneLoadKey,
+	throwOnTombstoneUsageKey,
 } from "./gcDefinitions";
-import { getGCVersion, shouldAllowGcSweep } from "./gcHelpers";
+import { getGCVersion, shouldAllowGcSweep, shouldAllowGcTombstoneEnforcement } from "./gcHelpers";
 
 /**
  * Generates configurations for the Garbage Collector that it uses to determine what to run and how.
@@ -42,6 +44,7 @@ export function generateGCConfigs(
 		gcOptions: IGCRuntimeOptions;
 		metadata: IContainerRuntimeMetadata | undefined;
 		existing: boolean;
+		isSummarizerClient: boolean;
 	},
 ): IGarbageCollectorConfigs {
 	let gcEnabled: boolean;
@@ -152,8 +155,6 @@ export function generateGCConfigs(
 		throw new UsageError("inactive timeout should not be greater than the sweep timeout");
 	}
 
-	const throwOnInactiveLoad: boolean | undefined = createParams.gcOptions.throwOnInactiveLoad;
-
 	// Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
 	const testMode =
 		mc.config.getBoolean(gcTestModeKey) ?? createParams.gcOptions.runGCInTestMode === true;
@@ -162,6 +163,20 @@ export function generateGCConfigs(
 	const tombstoneMode = !shouldRunSweep && mc.config.getBoolean(disableTombstoneKey) !== true;
 	const runFullGC = createParams.gcOptions.runFullGC;
 
+	const throwOnInactiveLoad: boolean | undefined = createParams.gcOptions.throwOnInactiveLoad;
+	const tombstoneEnforcementAllowed = shouldAllowGcTombstoneEnforcement(
+		createParams.metadata?.gcFeatureMatrix?.tombstoneGeneration /* persisted */,
+		createParams.gcOptions[gcTombstoneGenerationOptionName] /* current */,
+	);
+	const throwOnTombstoneLoad =
+		mc.config.getBoolean(throwOnTombstoneLoadKey) === true &&
+		tombstoneEnforcementAllowed &&
+		!createParams.isSummarizerClient;
+	const throwOnTombstoneUsage =
+		mc.config.getBoolean(throwOnTombstoneUsageKey) === true &&
+		tombstoneEnforcementAllowed &&
+		!createParams.isSummarizerClient;
+
 	return {
 		gcEnabled,
 		sweepEnabled,
@@ -173,10 +188,13 @@ export function generateGCConfigs(
 		sessionExpiryTimeoutMs,
 		sweepTimeoutMs,
 		inactiveTimeoutMs,
-		throwOnInactiveLoad,
 		persistedGcFeatureMatrix,
 		gcVersionInBaseSnapshot,
 		gcVersionInEffect,
+		throwOnInactiveLoad,
+		tombstoneEnforcementAllowed,
+		throwOnTombstoneLoad,
+		throwOnTombstoneUsage,
 	};
 }
 

package/src/gc/gcDefinitions.ts

@@ -4,7 +4,7 @@
  */
 
 import { ICriticalContainerError } from "@fluidframework/container-definitions";
-import { IRequestHeader } from "@fluidframework/core-interfaces";
+import { IRequest } from "@fluidframework/core-interfaces";
 import { ISnapshotTree } from "@fluidframework/protocol-definitions";
 import {
 	IGarbageCollectionData,
@@ -19,6 +19,7 @@ import {
 	ICreateContainerMetadata,
 	IRefreshSummaryResult,
 } from "../summary";
+import { RuntimeHeaderData } from "../containerRuntime";
 
 export type GCVersion = number;
 
@@ -193,8 +194,6 @@ export interface IGarbageCollectionRuntime {
 	getNodeType(nodePath: string): GCNodeType;
 	/** Called when the runtime should close because of an error. */
 	closeFn: (error?: ICriticalContainerError) => void;
-	/** If false, loading or using a Tombstoned object should merely log, not fail */
-	gcTombstoneEnforcementAllowed: boolean;
 }
 
 /** Defines the contract for the garbage collector. */
@@ -205,6 +204,12 @@ export interface IGarbageCollector {
 	readonly summaryStateNeedsReset: boolean;
 	/** The count of data stores whose GC state updated since the last summary. */
 	readonly updatedDSCountSinceLastSummary: number;
+	/** Tells whether tombstone feature is enabled and enforced. */
+	readonly tombstoneEnforcementAllowed: boolean;
+	/** Tells whether loading a tombstone object should fail or merely log. */
+	readonly throwOnTombstoneLoad: boolean;
+	/** Tells whether using a tombstone object should fail or merely log. */
+	readonly throwOnTombstoneUsage: boolean;
 	/** Initialize the state from the base snapshot after its creation. */
 	initializeBaseState(): Promise<void>;
 	/** Run garbage collection and update the reference / used state of the system. */
@@ -228,13 +233,17 @@ export interface IGarbageCollector {
 	getBaseGCDetails(): Promise<IGarbageCollectionDetailsBase>;
 	/** Called when the latest summary of the system has been refreshed. */
 	refreshLatestSummary(result: IRefreshSummaryResult): Promise<void>;
-	/** Called when a node is updated. Used to detect and log when an inactive node is changed or loaded. */
+	/**
+	 * Called when a node with the given path is updated. If the node is inactive or tombstoned, this will log an error
+	 * or throw an error if failing on incorrect usage is configured.
+	 */
 	nodeUpdated(
 		nodePath: string,
 		reason: "Loaded" | "Changed",
 		timestampMs?: number,
 		packagePath?: readonly string[],
-		requestHeaders?: IRequestHeader,
+		request?: IRequest,
+		headerData?: RuntimeHeaderData,
 	): void;
 	/** Called when a reference is added to a node. Used to identify nodes that were referenced between summaries. */
 	addedOutboundReference(fromNodePath: string, toNodePath: string): void;
@@ -332,8 +341,6 @@ export interface IGarbageCollectorConfigs {
 	readonly sweepTimeoutMs: number | undefined;
 	/** The time after which an unreferenced node is inactive. */
 	readonly inactiveTimeoutMs: number;
-	/** It is easier for users to diagnose InactiveObject usage if we throw on load, which this option enables */
-	readonly throwOnInactiveLoad: boolean | undefined;
 	/** Tracks whether GC should run in test mode. In this mode, unreferenced objects are deleted immediately. */
 	readonly testMode: boolean;
 	/**
@@ -349,6 +356,14 @@ export interface IGarbageCollectorConfigs {
 	readonly gcVersionInBaseSnapshot: GCVersion | undefined;
 	/** The current version of GC data in the running code */
 	readonly gcVersionInEffect: GCVersion;
+	/** It is easier for users to diagnose InactiveObject usage if we throw on load, which this option enables */
+	readonly throwOnInactiveLoad: boolean | undefined;
+	/** If false, loading or using a Tombstoned object should merely log, not fail */
+	readonly tombstoneEnforcementAllowed: boolean;
+	/** If true, throw an error when a tombstone data store is retrieved */
+	readonly throwOnTombstoneLoad: boolean;
+	/** If true, throw an error when a tombstone data store is used. */
+	readonly throwOnTombstoneUsage: boolean;
 }
 
 /** The state of node that is unreferenced. */

package/src/gc/gcTelemetry.ts

@@ -11,6 +11,7 @@ import {
 	MonitoringContext,
 	tagCodeArtifacts,
 } from "@fluidframework/telemetry-utils";
+import { RuntimeHeaderData } from "../containerRuntime";
 import { ICreateContainerMetadata } from "../summary";
 import {
 	disableSweepLogKey,
@@ -32,7 +33,7 @@ interface ICommonProps {
 	completedGCRuns: number;
 	isTombstoned: boolean;
 	lastSummaryTime?: number;
-	viaHandle?: boolean;
+	headers?: RuntimeHeaderData;
 }
 
 /** The event that is logged when unreferenced node is used after a certain time. */
@@ -80,10 +81,13 @@ export class GCTelemetryTracker {
 		private readonly mc: MonitoringContext,
 		private readonly configs: Pick<
 			IGarbageCollectorConfigs,
-			"inactiveTimeoutMs" | "sweepTimeoutMs"
+			| "inactiveTimeoutMs"
+			| "sweepTimeoutMs"
+			| "tombstoneEnforcementAllowed"
+			| "throwOnTombstoneLoad"
+			| "throwOnTombstoneUsage"
 		>,
 		private readonly isSummarizerClient: boolean,
-		private readonly gcTombstoneEnforcementAllowed: boolean,
 		private readonly createContainerMetadata: ICreateContainerMetadata,
 		private readonly getNodeType: (nodeId: string) => GCNodeType,
 		private readonly getNodeStateTracker: (
@@ -95,12 +99,12 @@ export class GCTelemetryTracker {
 	) {}
 
 	/**
-	 * Returns whether an event should be logged for a node that isn't active anymore. Some scenarios where we won't log:
-	 * 1. When a DDS is changed or loaded. The corresponding data store's event will be logged instead.
+	 * Returns whether an event should be logged for a node that isn't active anymore. This does not apply to
+	 * tombstoned nodes for which an event is always logged. Some scenarios where we won't log:
+	 * 1. When a DDS is changed. The corresponding data store's event will be logged instead.
 	 * 2. An event is logged only once per container instance per event per node.
 	 */
 	private shouldLogNonActiveEvent(
-		nodeId: string,
 		nodeType: GCNodeType,
 		usageType: NodeUsageType,
 		nodeStateTracker: UnreferencedStateTracker,
@@ -119,6 +123,8 @@ export class GCTelemetryTracker {
 			return false;
 		}
 
+		// Non-tombstone events are logged once per event per node. A unique id is generated by joining
+		// node state (inactive / sweep ready), node's id and usage (loaded / changed / revived).
 		if (this.loggedUnreferencedEvents.has(uniqueEventId)) {
 			return false;
 		}
@@ -126,24 +132,61 @@ export class GCTelemetryTracker {
 	}
 
 	/**
-	 * Called when a node is used. If the node is not active, log an event indicating object is used when its not active.
+	 * Called when a node is used. If the node is not active or tombstoned, log telemetry indicating object is used
+	 * when it should not have been.
 	 */
 	public nodeUsed(nodeUsageProps: INodeUsageProps) {
 		// If there is no reference timestamp to work with, no ops have been processed after creation. If so, skip
 		// logging as nothing interesting would have happened worth logging.
-		// If the node is not unreferenced, skip logging.
-		const nodeStateTracker = this.getNodeStateTracker(nodeUsageProps.id);
-		if (!nodeStateTracker || nodeUsageProps.currentReferenceTimestampMs === undefined) {
+		if (nodeUsageProps.currentReferenceTimestampMs === undefined) {
 			return;
 		}
 
-		// We log these events once per event per node. A unique id is generated by joining node state (inactive / sweep ready),
-		// node's id and usage (loaded / changed / revived).
-		const uniqueEventId = `${nodeStateTracker.state}-${nodeUsageProps.id}-${nodeUsageProps.usageType}`;
+		const nodeStateTracker = this.getNodeStateTracker(nodeUsageProps.id);
 		const nodeType = this.getNodeType(nodeUsageProps.id);
+		const {
+			usageType,
+			currentReferenceTimestampMs,
+			packagePath,
+			id: untaggedId,
+			fromId: untaggedFromId,
+			...propsToLog
+		} = nodeUsageProps;
+		const unrefEventProps: Omit<IUnreferencedEventProps, "state" | "usageType"> = {
+			type: nodeType,
+			unrefTime: nodeStateTracker?.unreferencedTimestampMs ?? -1,
+			age:
+				nodeStateTracker !== undefined
+					? nodeUsageProps.currentReferenceTimestampMs -
+					  nodeStateTracker.unreferencedTimestampMs
+					: -1,
+			timeout:
+				nodeStateTracker?.state === UnreferencedState.Inactive
+					? this.configs.inactiveTimeoutMs
+					: this.configs.sweepTimeoutMs,
+			...tagCodeArtifacts({ id: untaggedId, fromId: untaggedFromId }),
+			...propsToLog,
+			...this.createContainerMetadata,
+		};
+
+		// If the node that is used is tombstoned, log a tombstone telemetry.
+		// Note that this is done before checking if "nodeStateTracker" is undefined below because unreferenced
+		// tracking may not have yet been enabled. That happens only after the client transitions to write mode.
+		if (nodeUsageProps.isTombstoned) {
+			this.logTombstoneUsageTelemetry(nodeUsageProps, unrefEventProps, nodeType, usageType);
+		}
+
+		// After logging tombstone telemetry, if the node's unreferenced state is not tracked, there is nothing
+		// else to log.
+		if (nodeStateTracker === undefined) {
+			return;
+		}
+
+		const state = nodeStateTracker.state;
+		const uniqueEventId = `${state}-${nodeUsageProps.id}-${nodeUsageProps.usageType}`;
+
 		if (
 			!this.shouldLogNonActiveEvent(
-				nodeUsageProps.id,
 				nodeType,
 				nodeUsageProps.usageType,
 				nodeStateTracker,
@@ -153,42 +196,9 @@ export class GCTelemetryTracker {
 			return;
 		}
 
-		// Add the unique event id so that we don't generate a log for this event again in this session..
+		// Add the unique event id so that we don't generate a log for this event again in this session.
 		this.loggedUnreferencedEvents.add(uniqueEventId);
 
-		const state = nodeStateTracker.state;
-		const { usageType, currentReferenceTimestampMs, packagePath, id, fromId, ...propsToLog } =
-			nodeUsageProps;
-		const eventProps: Omit<IUnreferencedEventProps, "state" | "usageType"> = {
-			type: nodeType,
-			unrefTime: nodeStateTracker.unreferencedTimestampMs,
-			age:
-				nodeUsageProps.currentReferenceTimestampMs -
-				nodeStateTracker.unreferencedTimestampMs,
-			timeout:
-				state === UnreferencedState.Inactive
-					? this.configs.inactiveTimeoutMs
-					: this.configs.sweepTimeoutMs,
-			...tagCodeArtifacts({ id, fromId }),
-			...propsToLog,
-			...this.createContainerMetadata,
-		};
-
-		// This will log the following events:
-		// GC_Tombstone_DataStore_Revived, GC_Tombstone_SubDataStore_Revived, GC_Tombstone_Blob_Revived
-		if (nodeUsageProps.usageType === "Revived" && nodeUsageProps.isTombstoned) {
-			sendGCUnexpectedUsageEvent(
-				this.mc,
-				{
-					eventName: `GC_Tombstone_${nodeType}_Revived`,
-					category: "generic",
-					...tagCodeArtifacts({ url: id }),
-					gcTombstoneEnforcementAllowed: this.gcTombstoneEnforcementAllowed,
-				},
-				undefined /* packagePath */,
-			);
-		}
-
 		// For summarizer client, queue the event so it is logged the next time GC runs if the event is still valid.
 		// For non-summarizer client, log the event now since GC won't run on it. This may result in false positives
 		// but it's a good signal nonetheless and we can consume it with a grain of salt.
@@ -196,7 +206,7 @@ export class GCTelemetryTracker {
 		// SweepReady errors are usages of Objects that will be deleted by GC Sweep!
 		if (this.isSummarizerClient) {
 			this.pendingEventsQueue.push({
-				...eventProps,
+				...unrefEventProps,
 				usageType: nodeUsageProps.usageType,
 				state,
 			});
@@ -206,16 +216,15 @@ export class GCTelemetryTracker {
 			// Events generated:
 			// InactiveObject_Loaded, SweepReadyObject_Loaded
 			if (nodeUsageProps.usageType === "Loaded") {
-				const { id: taggedId, fromId: taggedFromId, ...otherProps } = eventProps;
+				const { id, fromId, headers, ...detailedProps } = unrefEventProps;
 				const event = {
 					eventName: `${state}Object_${nodeUsageProps.usageType}`,
-					pkg: tagCodeArtifacts({ pkg: nodeUsageProps.packagePath?.join("/") }).pkg,
+					...tagCodeArtifacts({ pkg: nodeUsageProps.packagePath?.join("/") }),
 					stack: generateStack(),
-					id: taggedId,
-					fromId: taggedFromId,
-					details: JSON.stringify({
-						...otherProps,
-					}),
+					id,
+					fromId,
+					headers: { ...headers },
+					details: detailedProps,
 				};
 
 				// Do not log the inactive object x events as error events as they are not the best signal for
@@ -229,6 +238,44 @@ export class GCTelemetryTracker {
 		}
 	}
 
+	/**
+	 * Logs telemetry when a tombstoned object is changed, revived or loaded.
+	 */
+	private logTombstoneUsageTelemetry(
+		nodeUsageProps: INodeUsageProps,
+		unrefEventProps: Omit<IUnreferencedEventProps, "state" | "usageType">,
+		nodeType: GCNodeType,
+		usageType: NodeUsageType,
+	) {
+		// This will log the following events:
+		// GC_Tombstone_DataStore_Requested, GC_Tombstone_DataStore_Changed, GC_Tombstone_DataStore_Revived
+		// GC_Tombstone_SubDataStore_Requested, GC_Tombstone_SubDataStore_Changed, GC_Tombstone_SubDataStore_Revived
+		// GC_Tombstone_Blob_Requested, GC_Tombstone_Blob_Changed, GC_Tombstone_Blob_Revived
+		const { id, fromId, headers, ...detailedProps } = unrefEventProps;
+		const eventUsageName = usageType === "Loaded" ? "Requested" : usageType;
+		const event = {
+			eventName: `GC_Tombstone_${nodeType}_${eventUsageName}`,
+			pkg: tagCodeArtifacts({ pkg: nodeUsageProps.packagePath?.join("/") }).pkg,
+			stack: generateStack(),
+			id,
+			fromId,
+			headers: { ...headers },
+			details: detailedProps,
+			gcTombstoneEnforcementAllowed: this.configs.tombstoneEnforcementAllowed,
+		};
+
+		if (
+			(usageType === "Loaded" &&
+				this.configs.throwOnTombstoneLoad &&
+				!headers?.allowTombstone) ||
+			(usageType === "Changed" && this.configs.throwOnTombstoneUsage)
+		) {
+			this.mc.logger.sendErrorEvent(event);
+		} else {
+			this.mc.logger.sendTelemetryEvent(event);
+		}
+	}
+
 	/**
 	 * Log all new references or outbound routes in the current graph that haven't been explicitly notified to GC.
 	 * The principle is that every new reference or outbound route must be notified to GC via the

package/src/gc/index.ts

@@ -31,15 +31,11 @@ export {
 	stableGCVersion,
 	disableAttachmentBlobSweepKey,
 	disableDatastoreSweepKey,
-	throwOnTombstoneLoadKey,
-	throwOnTombstoneUsageKey,
 	UnreferencedState,
 } from "./gcDefinitions";
 export {
 	cloneGCData,
 	concatGarbageCollectionStates,
-	shouldAllowGcTombstoneEnforcement,
-	shouldAllowGcSweep,
 	trimLeadingAndTrailingSlashes,
 	unpackChildNodesGCDetails,
 } from "./gcHelpers";

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.0.0-internal.7.1.0";
+export const pkgVersion = "2.0.0-internal.7.1.1";