@fluidframework/container-runtime 2.0.0-internal.4.2.1 → 2.0.0-internal.4.4.0

package/src/gc/gcConfigs.ts

@@ -106,21 +106,26 @@ export function generateGCConfigs(
 		createParams.gcOptions[gcSweepGenerationOptionName] /* currentGeneration */,
 	);
 
+	// If version upgrade is not enabled, fall back to the stable GC version.
+	const gcVersionInEffect =
+		mc.config.getBoolean(gcVersionUpgradeToV3Key) === true ? currentGCVersion : stableGCVersion;
+
+	// The GC version is up-to-date if the GC version in effect is at least equal to the GC version in base snapshot.
+	// If it is not up-to-date, there is a newer version of GC out there which is more reliable than this. So, GC
+	// should not run as it may produce incorrect / unreliable state.
+	const isGCVersionUpToDate =
+		gcVersionInBaseSnapshot === undefined || gcVersionInEffect >= gcVersionInBaseSnapshot;
+
 	/**
 	 * Whether GC should run or not. The following conditions have to be met to run sweep:
-	 *
 	 * 1. GC should be enabled for this container.
-	 *
 	 * 2. GC should not be disabled via disableGC GC option.
-	 *
+	 * 3. The current GC version should be greater of equal to the GC version in the base snapshot.
 	 * These conditions can be overridden via runGCKey feature flag.
 	 */
 	const shouldRunGC =
 		mc.config.getBoolean(runGCKey) ??
-		// GC must be enabled for the document.
-		(gcEnabled &&
-			// GC must not be disabled via GC options.
-			!createParams.gcOptions.disableGC);
+		(gcEnabled && !createParams.gcOptions.disableGC && isGCVersionUpToDate);
 
 	/**
 	 * Whether sweep should run or not. The following conditions have to be met to run sweep:
@@ -156,10 +161,6 @@ export function generateGCConfigs(
 	const tombstoneMode = !shouldRunSweep && mc.config.getBoolean(disableTombstoneKey) !== true;
 	const runFullGC = createParams.gcOptions.runFullGC;
 
-	// If version upgrade is not enabled, fall back to the stable GC version.
-	const gcVersionInEffect =
-		mc.config.getBoolean(gcVersionUpgradeToV3Key) === true ? currentGCVersion : stableGCVersion;
-
 	return {
 		gcEnabled,
 		sweepEnabled,

package/src/gc/gcDefinitions.ts

@@ -204,6 +204,8 @@ export interface IGarbageCollector {
 	readonly shouldRunGC: boolean;
 	/** Tells whether the GC state in summary needs to be reset in the next summary. */
 	readonly summaryStateNeedsReset: boolean;
+	/** The count of data stores whose GC state updated since the last summary. */
+	readonly updatedDSCountSinceLastSummary: number;
 	/** Initialize the state from the base snapshot after its creation. */
 	initializeBaseState(): Promise<void>;
 	/** Run garbage collection and update the reference / used state of the system. */

package/src/gc/gcHelpers.ts

@@ -3,7 +3,6 @@
  * Licensed under the MIT License.
  */
 
-import { ITelemetryGenericEvent } from "@fluidframework/common-definitions";
 import { assert } from "@fluidframework/common-utils";
 import { ISnapshotTree } from "@fluidframework/protocol-definitions";
 import {
@@ -13,17 +12,8 @@ import {
 	IGarbageCollectionData,
 	IGarbageCollectionDetailsBase,
 } from "@fluidframework/runtime-definitions";
-import { packagePathToTelemetryProperty } from "@fluidframework/runtime-utils";
-import { MonitoringContext } from "@fluidframework/telemetry-utils";
-import {
-	disableTombstoneKey,
-	GCFeatureMatrix,
-	GCVersion,
-	IGCMetadata,
-	runSweepKey,
-	throwOnTombstoneLoadKey,
-	throwOnTombstoneUsageKey,
-} from "./gcDefinitions";
+import { TelemetryDataTag } from "@fluidframework/telemetry-utils";
+import { GCFeatureMatrix, GCVersion, IGCMetadata } from "./gcDefinitions";
 import {
 	IGarbageCollectionNodeData,
 	IGarbageCollectionSnapshotData,
@@ -38,32 +28,6 @@ export function getGCVersion(metadata?: IGCMetadata): GCVersion {
 	return metadata.gcFeature ?? 0;
 }
 
-/**
- * Consolidates info / logic for logging when we encounter unexpected usage of GC'd objects. For example, when a
- * tombstoned or deleted object is loaded.
- */
-export function sendGCUnexpectedUsageEvent(
-	mc: MonitoringContext,
-	event: ITelemetryGenericEvent & {
-		category: "error" | "generic";
-		gcTombstoneEnforcementAllowed: boolean | undefined;
-	},
-	packagePath: readonly string[] | undefined,
-	error?: unknown,
-) {
-	event.pkg = packagePathToTelemetryProperty(packagePath);
-	event.tombstoneFlags = JSON.stringify({
-		DisableTombstone: mc.config.getBoolean(disableTombstoneKey),
-		ThrowOnTombstoneUsage: mc.config.getBoolean(throwOnTombstoneUsageKey),
-		ThrowOnTombstoneLoad: mc.config.getBoolean(throwOnTombstoneLoadKey),
-	});
-	event.sweepFlags = JSON.stringify({
-		EnableSweepFlag: mc.config.getBoolean(runSweepKey),
-	});
-
-	mc.logger.sendTelemetryEvent(event, error);
-}
-
 /**
  * Indicates whether Tombstone Enforcement is allowed for this document based on the current/persisted
  * TombstoneGeneration values
@@ -188,12 +152,19 @@ export function concatGarbageCollectionStates(
 /**
  * Helper function that clones the GC data.
  * @param gcData - The GC data to clone.
+ * @param filter - Optional function to filter out node ids not to be included in the cloned GC data. Returns
+ * true to filter out nodes.
  * @returns a clone of the given GC data.
  */
-export function cloneGCData(gcData: IGarbageCollectionData): IGarbageCollectionData {
+export function cloneGCData(
+	gcData: IGarbageCollectionData,
+	filter?: (id: string) => boolean,
+): IGarbageCollectionData {
 	const clonedGCNodes: { [id: string]: string[] } = {};
 	for (const [id, outboundRoutes] of Object.entries(gcData.gcNodes)) {
-		clonedGCNodes[id] = Array.from(outboundRoutes);
+		if (filter?.(id) !== true) {
+			clonedGCNodes[id] = Array.from(outboundRoutes);
+		}
 	}
 	return {
 		gcNodes: clonedGCNodes,
@@ -333,3 +304,13 @@ export function unpackChildNodesGCDetails(gcDetails: IGarbageCollectionDetailsBa
 export function trimLeadingAndTrailingSlashes(str: string) {
 	return str.replace(/^\/+|\/+$/g, "");
 }
+
+/**
+ * Tags the passed value as a CodeArtifact and returns the tagged value.
+ */
+export function tagAsCodeArtifact(value: string) {
+	return {
+		value,
+		tag: TelemetryDataTag.CodeArtifact,
+	};
+}

package/src/gc/gcSummaryStateTracker.ts

@@ -14,7 +14,7 @@ import {
 } from "@fluidframework/runtime-definitions";
 import { mergeStats, ReadAndParseBlob, SummaryTreeBuilder } from "@fluidframework/runtime-utils";
 import { IContainerRuntimeMetadata, metadataBlobName, RefreshSummaryResult } from "../summary";
-import { GCVersion } from "./gcDefinitions";
+import { GCVersion, IGCStats } from "./gcDefinitions";
 import { getGCDataFromSnapshot, generateSortedGCState, getGCVersion } from "./gcHelpers";
 import { IGarbageCollectionSnapshotData, IGarbageCollectionState } from "./gcSummaryDefinitions";
 import { IGarbageCollectorConfigs } from ".";
@@ -37,8 +37,6 @@ export interface IGCSummaryTrackingData {
  * On summarize, it decides whether to write new state or re-use previous summary's state.
  */
 export class GCSummaryStateTracker {
-	// The current version of GC running.
-	public readonly currentGCVersion: GCVersion = this.configs.gcVersionInEffect;
 	// This is the version of GC data in the latest summary being tracked.
 	private latestSummaryGCVersion: GCVersion;
 
@@ -50,6 +48,10 @@ export class GCSummaryStateTracker {
 	// Tracks whether there was GC was run in latest summary being tracked.
 	private wasGCRunInLatestSummary: boolean;
 
+	// Tracks the count of data stores whose state updated since the last summary, i.e., they went from referenced
+	// to unreferenced or vice-versa.
+	public updatedDSCountSinceLastSummary: number = 0;
+
 	constructor(
 		// Tells whether GC should run or not.
 		private readonly configs: Pick<
@@ -62,7 +64,8 @@ export class GCSummaryStateTracker {
 		this.wasGCRunInLatestSummary = wasGCRunInBaseSnapshot;
 		// For existing document, the latest summary is the one that we loaded from. So, use its GC version as the
 		// latest tracked GC version. For new documents, we will be writing the first summary with the current version.
-		this.latestSummaryGCVersion = this.configs.gcVersionInBaseSnapshot ?? this.currentGCVersion;
+		this.latestSummaryGCVersion =
+			this.configs.gcVersionInBaseSnapshot ?? this.configs.gcVersionInEffect;
 	}
 
 	/**
@@ -101,7 +104,8 @@ export class GCSummaryStateTracker {
 	public get doesSummaryStateNeedReset(): boolean {
 		return (
 			this.doesGCStateNeedReset ||
-			(this.configs.shouldRunGC && this.latestSummaryGCVersion !== this.currentGCVersion)
+			(this.configs.shouldRunGC &&
+				this.latestSummaryGCVersion !== this.configs.gcVersionInEffect)
 		);
 	}
 
@@ -283,9 +287,10 @@ export class GCSummaryStateTracker {
 		// If the summary was tracked by this client, it was the one that generated the summary in the first place.
 		// Update latest state from pending.
 		if (result.wasSummaryTracked) {
-			this.latestSummaryGCVersion = this.currentGCVersion;
+			this.latestSummaryGCVersion = this.configs.gcVersionInEffect;
 			this.latestSummaryData = this.pendingSummaryData;
 			this.pendingSummaryData = undefined;
+			this.updatedDSCountSinceLastSummary = 0;
 			return undefined;
 		}
 
@@ -311,7 +316,7 @@ export class GCSummaryStateTracker {
 		// in the snapshot cannot be interpreted correctly. Set everything to undefined except for deletedNodes
 		// because irrespective of GC versions, these nodes have been deleted and cannot be brought back. The
 		// deletedNodes info is needed to identify when these nodes are used.
-		if (getGCVersion(metadata) !== this.currentGCVersion) {
+		if (getGCVersion(metadata) !== this.configs.gcVersionInEffect) {
 			snapshotData = {
 				gcState: undefined,
 				tombstones: undefined,
@@ -326,4 +331,11 @@ export class GCSummaryStateTracker {
 		};
 		return snapshotData;
 	}
+
+	/**
+	 * Called to update the state from a GC run's stats. Used to update the count of data stores whose state updated.
+	 */
+	public updateStateFromGCRunStats(stats: IGCStats) {
+		this.updatedDSCountSinceLastSummary += stats.updatedDataStoreCount;
+	}
 }

package/src/gc/gcTelemetry.ts

@@ -0,0 +1,408 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { ITelemetryGenericEvent, ITelemetryLogger } from "@fluidframework/common-definitions";
+import { IGarbageCollectionData } from "@fluidframework/runtime-definitions";
+import { packagePathToTelemetryProperty } from "@fluidframework/runtime-utils";
+import { generateStack, MonitoringContext } from "@fluidframework/telemetry-utils";
+import { ICreateContainerMetadata } from "../summary";
+import {
+	disableSweepLogKey,
+	GCNodeType,
+	UnreferencedState,
+	IGarbageCollectorConfigs,
+	disableTombstoneKey,
+	throwOnTombstoneUsageKey,
+	throwOnTombstoneLoadKey,
+	runSweepKey,
+} from "./gcDefinitions";
+import { UnreferencedStateTracker } from "./gcUnreferencedStateTracker";
+import { tagAsCodeArtifact } from "./gcHelpers";
+
+type NodeUsageType = "Changed" | "Loaded" | "Revived";
+
+/** Properties that are common to IUnreferencedEventProps and INodeUsageProps */
+interface ICommonProps {
+	usageType: NodeUsageType;
+	completedGCRuns: number;
+	isTombstoned: boolean;
+	lastSummaryTime?: number;
+	viaHandle?: boolean;
+}
+
+/** The event that is logged when unreferenced node is used after a certain time. */
+interface IUnreferencedEventProps extends ICreateContainerMetadata, ICommonProps {
+	state: UnreferencedState;
+	id: {
+		value: string;
+		tag: string;
+	};
+	type: GCNodeType;
+	unrefTime: number;
+	age: number;
+	timeout?: number;
+	fromId?: {
+		value: string;
+		tag: string;
+	};
+}
+
+/** Properties passed to nodeUsed function when a node is used. */
+interface INodeUsageProps extends ICommonProps {
+	id: string;
+	currentReferenceTimestampMs: number | undefined;
+	packagePath: readonly string[] | undefined;
+	fromId?: string;
+}
+
+/**
+ * Encapsulates the logic that tracks the various telemetry logged by the Garbage Collector. There are 4 types of
+ * telemetry logged:
+ * 1. inactiveObject telemetry - When an inactive node is used - A node that has been unreferenced for inactiveTimeoutMs.
+ * 2. sweepReadyObject telemetry - When a sweep ready node is used - A node that has been unreferenced for sweepTimeoutMs.
+ * 3. Tombstone telemetry - When a tombstoned node is used - A node that that has been marked as tombstone.
+ * 4. Sweep / deleted telemetry - When a node is detected as sweep ready in the sweep phase.
+ * 5. Unknown outbound reference telemetry - When a node is referenced but GC is not explicitly notified of it.
+ */
+export class GCTelemetryTracker {
+	// Keeps track of unreferenced events that are logged for a node. This is used to limit the log generation to one
+	// per event per node.
+	private readonly loggedUnreferencedEvents: Set<string> = new Set();
+	// Queue for unreferenced events that should be logged the next time GC runs.
+	private pendingEventsQueue: IUnreferencedEventProps[] = [];
+
+	constructor(
+		private readonly mc: MonitoringContext,
+		private readonly configs: Pick<
+			IGarbageCollectorConfigs,
+			"inactiveTimeoutMs" | "sweepTimeoutMs"
+		>,
+		private readonly isSummarizerClient: boolean,
+		private readonly gcTombstoneEnforcementAllowed: boolean,
+		private readonly createContainerMetadata: ICreateContainerMetadata,
+		private readonly getNodeType: (nodeId: string) => GCNodeType,
+		private readonly getNodeStateTracker: (
+			nodeId: string,
+		) => UnreferencedStateTracker | undefined,
+		private readonly getNodePackagePath: (
+			nodePath: string,
+		) => Promise<readonly string[] | undefined>,
+	) {}
+
+	/**
+	 * Returns whether an event should be logged for a node that isn't active anymore. Some scenarios where we won't log:
+	 * 1. When a DDS is changed or loaded. The corresponding data store's event will be logged instead.
+	 * 2. An event is logged only once per container instance per event per node.
+	 */
+	private shouldLogNonActiveEvent(
+		nodeId: string,
+		nodeType: GCNodeType,
+		usageType: NodeUsageType,
+		nodeStateTracker: UnreferencedStateTracker,
+		uniqueEventId: string,
+	) {
+		if (nodeStateTracker.state === UnreferencedState.Active) {
+			return false;
+		}
+
+		// For sub data store (DDS) nodes, if they are changed or loaded, its data store will also be changed or loaded,
+		// so skip logging to make the telemetry less noisy.
+		if (nodeType === GCNodeType.SubDataStore && usageType !== "Revived") {
+			return false;
+		}
+		if (nodeType === GCNodeType.Other) {
+			return false;
+		}
+
+		if (this.loggedUnreferencedEvents.has(uniqueEventId)) {
+			return false;
+		}
+		return true;
+	}
+
+	/**
+	 * Called when a node is used. If the node is not active, log an event indicating object is used when its not active.
+	 */
+	public nodeUsed(nodeUsageProps: INodeUsageProps) {
+		// If there is no reference timestamp to work with, no ops have been processed after creation. If so, skip
+		// logging as nothing interesting would have happened worth logging.
+		// If the node is not unreferenced, skip logging.
+		const nodeStateTracker = this.getNodeStateTracker(nodeUsageProps.id);
+		if (!nodeStateTracker || nodeUsageProps.currentReferenceTimestampMs === undefined) {
+			return;
+		}
+
+		// We log these events once per event per node. A unique id is generated by joining node state (inactive / sweep ready),
+		// node's id and usage (loaded / changed / revived).
+		const uniqueEventId = `${nodeStateTracker.state}-${nodeUsageProps.id}-${nodeUsageProps.usageType}`;
+		const nodeType = this.getNodeType(nodeUsageProps.id);
+		if (
+			!this.shouldLogNonActiveEvent(
+				nodeUsageProps.id,
+				nodeType,
+				nodeUsageProps.usageType,
+				nodeStateTracker,
+				uniqueEventId,
+			)
+		) {
+			return;
+		}
+
+		// Add the unique event id so that we don't generate a log for this event again in this session..
+		this.loggedUnreferencedEvents.add(uniqueEventId);
+
+		const state = nodeStateTracker.state;
+		const { usageType, currentReferenceTimestampMs, packagePath, id, fromId, ...propsToLog } =
+			nodeUsageProps;
+		const eventProps: Omit<IUnreferencedEventProps, "state" | "usageType"> = {
+			id: tagAsCodeArtifact(id),
+			type: nodeType,
+			unrefTime: nodeStateTracker.unreferencedTimestampMs,
+			age:
+				nodeUsageProps.currentReferenceTimestampMs -
+				nodeStateTracker.unreferencedTimestampMs,
+			timeout:
+				state === UnreferencedState.Inactive
+					? this.configs.inactiveTimeoutMs
+					: this.configs.sweepTimeoutMs,
+			fromId: fromId ? tagAsCodeArtifact(fromId) : undefined,
+			...propsToLog,
+			...this.createContainerMetadata,
+		};
+
+		// This will log the following events:
+		// GC_Tombstone_DataStore_Revived, GC_Tombstone_SubDataStore_Revived, GC_Tombstone_Blob_Revived
+		if (nodeUsageProps.usageType === "Revived" && nodeUsageProps.isTombstoned) {
+			sendGCUnexpectedUsageEvent(
+				this.mc,
+				{
+					eventName: `GC_Tombstone_${nodeType}_Revived`,
+					category: "generic",
+					url: tagAsCodeArtifact(id),
+					gcTombstoneEnforcementAllowed: this.gcTombstoneEnforcementAllowed,
+				},
+				undefined /* packagePath */,
+			);
+		}
+
+		// For summarizer client, queue the event so it is logged the next time GC runs if the event is still valid.
+		// For non-summarizer client, log the event now since GC won't run on it. This may result in false positives
+		// but it's a good signal nonetheless and we can consume it with a grain of salt.
+		// Inactive errors are usages of Objects that are unreferenced for at least a period of 7 days.
+		// SweepReady errors are usages of Objects that will be deleted by GC Sweep!
+		if (this.isSummarizerClient) {
+			this.pendingEventsQueue.push({
+				...eventProps,
+				usageType: nodeUsageProps.usageType,
+				state,
+			});
+		} else {
+			// For non-summarizer clients, only log "Loaded" type events since these objects may not be loaded in the
+			// summarizer clients if they are based off of user actions (such as scrolling to content for these objects)
+			// Events generated:
+			// InactiveObject_Loaded, SweepReadyObject_Loaded
+			if (nodeUsageProps.usageType === "Loaded") {
+				const { id: taggedId, fromId: taggedFromId, ...otherProps } = eventProps;
+				const event = {
+					eventName: `${state}Object_${nodeUsageProps.usageType}`,
+					pkg: packagePathToTelemetryProperty(nodeUsageProps.packagePath),
+					stack: generateStack(),
+					id: taggedId,
+					fromId: taggedFromId,
+					details: JSON.stringify({
+						...otherProps,
+					}),
+				};
+
+				// Do not log the inactive object x events as error events as they are not the best signal for
+				// detecting something wrong with GC either from the partner or from the runtime itself.
+				if (state === UnreferencedState.Inactive) {
+					this.mc.logger.sendTelemetryEvent(event);
+				} else {
+					this.mc.logger.sendErrorEvent(event);
+				}
+			}
+		}
+	}
+
+	/**
+	 * Log all new references or outbound routes in the current graph that haven't been explicitly notified to GC.
+	 * The principle is that every new reference or outbound route must be notified to GC via the
+	 * addedOutboundReference method. It it hasn't, its a bug and we want to identify these scenarios.
+	 *
+	 * In more simple terms:
+	 * Missing Explicit References = Current References - Previous References - Explicitly Added References;
+	 *
+	 * @param currentGCData - The GC data (reference graph) from the current GC run.
+	 * @param previousGCData - The GC data (reference graph) from the previous GC run.
+	 * @param explicitReferences - New references added explicity between the previous and the current run.
+	 */
+	public logIfMissingExplicitReferences(
+		currentGCData: IGarbageCollectionData,
+		previousGCData: IGarbageCollectionData,
+		explicitReferences: Map<string, string[]>,
+		logger: ITelemetryLogger,
+	) {
+		for (const [nodeId, currentOutboundRoutes] of Object.entries(currentGCData.gcNodes)) {
+			const previousRoutes = previousGCData.gcNodes[nodeId] ?? [];
+			const explicitRoutes = explicitReferences.get(nodeId) ?? [];
+
+			/**
+			 * 1. For routes in the current GC data, routes that were not present in previous GC data and did not have
+			 * explicit references should be added to missing explicit routes list.
+			 * 2. Only include data store and blob routes since GC only works for these two.
+			 * Note: Due to a bug with de-duped blobs, only adding data store routes for now.
+			 * 3. Ignore DDS routes to their parent datastores since those were added implicitly. So, there won't be
+			 * explicit routes to them.
+			 */
+			const missingExplicitRoutes: string[] = [];
+			for (const route of currentOutboundRoutes) {
+				const nodeType = this.getNodeType(route);
+				if (
+					(nodeType === GCNodeType.DataStore || nodeType === GCNodeType.Blob) &&
+					!nodeId.startsWith(route) &&
+					!previousRoutes.includes(route) &&
+					!explicitRoutes.includes(route)
+				) {
+					missingExplicitRoutes.push(route);
+				}
+			}
+
+			if (missingExplicitRoutes.length > 0) {
+				logger.sendErrorEvent({
+					eventName: "gcUnknownOutboundReferences",
+					id: tagAsCodeArtifact(nodeId),
+					routes: tagAsCodeArtifact(JSON.stringify(missingExplicitRoutes)),
+				});
+			}
+		}
+	}
+
+	/**
+	 * Log events that are pending in pendingEventsQueue. This is called after GC runs in the summarizer client
+	 * so that the state of an unreferenced node is updated.
+	 */
+	public async logPendingEvents(logger: ITelemetryLogger) {
+		// Events sent come only from the summarizer client. In between summaries, events are pushed to a queue and at
+		// summary time they are then logged.
+		// Events generated:
+		// InactiveObject_Loaded, InactiveObject_Changed, InactiveObject_Revived
+		// SweepReadyObject_Loaded, SweepReadyObject_Changed, SweepReadyObject_Revived
+		for (const eventProps of this.pendingEventsQueue) {
+			const { usageType, state, id, fromId, ...propsToLog } = eventProps;
+			/**
+			 * Revived event is logged only if the node is active. If the node is not active, the reference to it was
+			 * from another unreferenced node and this scenario is not interesting to log.
+			 * Loaded and Changed events are logged only if the node is not active. If the node is active, it was
+			 * revived and a Revived event will be logged for it.
+			 */
+			const nodeStateTracker = this.getNodeStateTracker(eventProps.id.value);
+			const active =
+				nodeStateTracker === undefined ||
+				nodeStateTracker.state === UnreferencedState.Active;
+			if ((usageType === "Revived") === active) {
+				const pkg = await this.getNodePackagePath(eventProps.id.value);
+				const fromPkg = eventProps.fromId
+					? await this.getNodePackagePath(eventProps.fromId.value)
+					: undefined;
+				const event = {
+					eventName: `${state}Object_${usageType}`,
+					details: JSON.stringify({
+						...propsToLog,
+					}),
+					id,
+					fromId,
+					pkg: pkg ? tagAsCodeArtifact(pkg.join("/")) : undefined,
+					fromPkg: fromPkg ? tagAsCodeArtifact(fromPkg.join("/")) : undefined,
+				};
+
+				if (state === UnreferencedState.Inactive) {
+					logger.sendTelemetryEvent(event);
+				} else {
+					logger.sendErrorEvent(event);
+				}
+			}
+		}
+		this.pendingEventsQueue = [];
+	}
+
+	/**
+	 * For nodes that are ready to sweep, log an event for now. Until we start running sweep which deletes objects,
+	 * this will give us a view into how much deleted content a container has.
+	 */
+	public logSweepEvents(
+		logger: ITelemetryLogger,
+		currentReferenceTimestampMs: number,
+		unreferencedNodesState: Map<string, UnreferencedStateTracker>,
+		completedGCRuns: number,
+		lastSummaryTime?: number,
+	) {
+		if (
+			this.mc.config.getBoolean(disableSweepLogKey) === true ||
+			this.configs.sweepTimeoutMs === undefined
+		) {
+			return;
+		}
+
+		const deletedNodeIds: string[] = [];
+		for (const [nodeId, nodeStateTracker] of unreferencedNodesState) {
+			if (nodeStateTracker.state !== UnreferencedState.SweepReady) {
+				return;
+			}
+
+			const nodeType = this.getNodeType(nodeId);
+			if (nodeType !== GCNodeType.DataStore && nodeType !== GCNodeType.Blob) {
+				return;
+			}
+
+			// Log deleted event for each node only once to reduce noise in telemetry.
+			const uniqueEventId = `Deleted-${nodeId}`;
+			if (this.loggedUnreferencedEvents.has(uniqueEventId)) {
+				return;
+			}
+			this.loggedUnreferencedEvents.add(uniqueEventId);
+			deletedNodeIds.push(nodeId);
+		}
+
+		if (deletedNodeIds.length > 0) {
+			logger.sendTelemetryEvent({
+				eventName: "GC_SweepReadyObjects_Delete",
+				details: JSON.stringify({
+					timeout: this.configs.sweepTimeoutMs,
+					completedGCRuns,
+					lastSummaryTime,
+					...this.createContainerMetadata,
+				}),
+				id: tagAsCodeArtifact(JSON.stringify(deletedNodeIds)),
+			});
+		}
+	}
+}
+
+/**
+ * Consolidates info / logic for logging when we encounter unexpected usage of GC'd objects. For example, when a
+ * tombstoned or deleted object is loaded.
+ */
+export function sendGCUnexpectedUsageEvent(
+	mc: MonitoringContext,
+	event: ITelemetryGenericEvent & {
+		category: "error" | "generic";
+		gcTombstoneEnforcementAllowed: boolean | undefined;
+	},
+	packagePath: readonly string[] | undefined,
+	error?: unknown,
+) {
+	event.pkg = packagePathToTelemetryProperty(packagePath);
+	event.tombstoneFlags = JSON.stringify({
+		DisableTombstone: mc.config.getBoolean(disableTombstoneKey),
+		ThrowOnTombstoneUsage: mc.config.getBoolean(throwOnTombstoneUsageKey),
+		ThrowOnTombstoneLoad: mc.config.getBoolean(throwOnTombstoneLoadKey),
+	});
+	event.sweepFlags = JSON.stringify({
+		EnableSweepFlag: mc.config.getBoolean(runSweepKey),
+	});
+
+	mc.logger.sendTelemetryEvent(event, error);
+}

package/src/gc/index.ts

@@ -39,11 +39,11 @@ export {
 	cloneGCData,
 	concatGarbageCollectionStates,
 	getGCDataFromSnapshot,
-	sendGCUnexpectedUsageEvent,
 	shouldAllowGcTombstoneEnforcement,
 	shouldAllowGcSweep,
 	trimLeadingAndTrailingSlashes,
 	unpackChildNodesGCDetails,
+	tagAsCodeArtifact,
 } from "./gcHelpers";
 export { runGarbageCollection } from "./gcReferenceGraphAlgorithm";
 export {
@@ -57,9 +57,5 @@ export {
 	GCSummaryStateTracker,
 	IGCSummaryTrackingData,
 } from "./gcSummaryStateTracker";
-export {
-	skipClosureForXDaysKey,
-	closuresMapLocalStorageKey,
-	SweepReadyUsageDetectionHandler,
-} from "./gcSweepReadyUsageDetection";
+export { GCTelemetryTracker, sendGCUnexpectedUsageEvent } from "./gcTelemetry";
 export { UnreferencedStateTracker } from "./gcUnreferencedStateTracker";

package/src/opLifecycle/README.md

@@ -13,6 +13,7 @@ By default, the runtime is configured with a max batch size of `716800` bytes, w
 -   [Introduction](#introduction)
     -   [Compression](#compression)
     -   [Grouped batching](#grouped-batching)
+        -   [Risks](#risks)
     -   [Chunking for compression](#chunking-for-compression)
     -   [Disabling in case of emergency](#disabling-in-case-of-emergency)
     -   [Example configs](#example-configs)
@@ -38,6 +39,18 @@ The purpose for enabling grouped batching on top of compression is that regular
 
 See [below](#how-grouped-batching-works) for an example.
 
+### Risks
+
+This option is experimental and should not be enabled yet in production. This option should **ONLY** be enabled after observing that 99.9% of your application sessions contains these changes (runtime version "2.0.0-internal.4.1.0" or later). Containers created with this option may not open in future versions of the framework.
+
+This option will change a couple of expectations around message structure and runtime layer expectations. Only enable this option after testing
+and verifying that the following expectation changes won't have any effects:
+
+-   batch messages observed at the runtime layer will not match messages seen at the loader layer (i.e. grouped form at loader layer, ungrouped form at runtime layer)
+-   messages within the same batch will have the same sequence number
+-   client sequence numbers on batch messages can only be used to order messages with the same sequenceNumber
+-   requires all ops to be processed by runtime layer (version "2.0.0-internal.1.2.0" or later https://github.com/microsoft/FluidFramework/pull/11832)
+
 ## Chunking for compression
 
 **Op chunking for compression targets payloads which exceed the max batch size after compression.** So, only payloads which are already compressed. By default, the feature is enabled.