@fluidframework/container-runtime 2.0.0-internal.7.1.0 → 2.0.0-internal.7.1.2

package/src/gc/gcTelemetry.ts

@@ -11,6 +11,7 @@ import {
 	MonitoringContext,
 	tagCodeArtifacts,
 } from "@fluidframework/telemetry-utils";
+import { RuntimeHeaderData } from "../containerRuntime";
 import { ICreateContainerMetadata } from "../summary";
 import {
 	disableSweepLogKey,
@@ -19,8 +20,9 @@ import {
 	IGarbageCollectorConfigs,
 	disableTombstoneKey,
 	throwOnTombstoneUsageKey,
-	throwOnTombstoneLoadKey,
+	throwOnTombstoneLoadOverrideKey,
 	runSweepKey,
+	GCFeatureMatrix,
 } from "./gcDefinitions";
 import { UnreferencedStateTracker } from "./gcUnreferencedStateTracker";
 
@@ -32,7 +34,7 @@ interface ICommonProps {
 	completedGCRuns: number;
 	isTombstoned: boolean;
 	lastSummaryTime?: number;
-	viaHandle?: boolean;
+	headers?: RuntimeHeaderData;
 }
 
 /** The event that is logged when unreferenced node is used after a certain time. */
@@ -45,6 +47,10 @@ interface IUnreferencedEventProps extends ICreateContainerMetadata, ICommonProps
 	type: GCNodeType;
 	unrefTime: number;
 	age: number;
+	// Expanding GC feature matrix. Without doing this, the configs cannot be logged in telemetry directly.
+	gcConfigs: Omit<IGarbageCollectorConfigs, "persistedGcFeatureMatrix"> & {
+		[K in keyof GCFeatureMatrix]: GCFeatureMatrix[K];
+	};
 	timeout?: number;
 	fromId?: {
 		value: string;
@@ -78,12 +84,8 @@ export class GCTelemetryTracker {
 
 	constructor(
 		private readonly mc: MonitoringContext,
-		private readonly configs: Pick<
-			IGarbageCollectorConfigs,
-			"inactiveTimeoutMs" | "sweepTimeoutMs"
-		>,
+		private readonly configs: IGarbageCollectorConfigs,
 		private readonly isSummarizerClient: boolean,
-		private readonly gcTombstoneEnforcementAllowed: boolean,
 		private readonly createContainerMetadata: ICreateContainerMetadata,
 		private readonly getNodeType: (nodeId: string) => GCNodeType,
 		private readonly getNodeStateTracker: (
@@ -95,12 +97,12 @@ export class GCTelemetryTracker {
 	) {}
 
 	/**
-	 * Returns whether an event should be logged for a node that isn't active anymore. Some scenarios where we won't log:
-	 * 1. When a DDS is changed or loaded. The corresponding data store's event will be logged instead.
+	 * Returns whether an event should be logged for a node that isn't active anymore. This does not apply to
+	 * tombstoned nodes for which an event is always logged. Some scenarios where we won't log:
+	 * 1. When a DDS is changed. The corresponding data store's event will be logged instead.
 	 * 2. An event is logged only once per container instance per event per node.
 	 */
 	private shouldLogNonActiveEvent(
-		nodeId: string,
 		nodeType: GCNodeType,
 		usageType: NodeUsageType,
 		nodeStateTracker: UnreferencedStateTracker,
@@ -119,6 +121,8 @@ export class GCTelemetryTracker {
 			return false;
 		}
 
+		// Non-tombstone events are logged once per event per node. A unique id is generated by joining
+		// node state (inactive / sweep ready), node's id and usage (loaded / changed / revived).
 		if (this.loggedUnreferencedEvents.has(uniqueEventId)) {
 			return false;
 		}
@@ -126,24 +130,63 @@ export class GCTelemetryTracker {
 	}
 
 	/**
-	 * Called when a node is used. If the node is not active, log an event indicating object is used when its not active.
+	 * Called when a node is used. If the node is not active or tombstoned, log telemetry indicating object is used
+	 * when it should not have been.
 	 */
 	public nodeUsed(nodeUsageProps: INodeUsageProps) {
 		// If there is no reference timestamp to work with, no ops have been processed after creation. If so, skip
 		// logging as nothing interesting would have happened worth logging.
-		// If the node is not unreferenced, skip logging.
-		const nodeStateTracker = this.getNodeStateTracker(nodeUsageProps.id);
-		if (!nodeStateTracker || nodeUsageProps.currentReferenceTimestampMs === undefined) {
+		if (nodeUsageProps.currentReferenceTimestampMs === undefined) {
 			return;
 		}
 
-		// We log these events once per event per node. A unique id is generated by joining node state (inactive / sweep ready),
-		// node's id and usage (loaded / changed / revived).
-		const uniqueEventId = `${nodeStateTracker.state}-${nodeUsageProps.id}-${nodeUsageProps.usageType}`;
+		const nodeStateTracker = this.getNodeStateTracker(nodeUsageProps.id);
 		const nodeType = this.getNodeType(nodeUsageProps.id);
+		const {
+			usageType,
+			currentReferenceTimestampMs,
+			packagePath,
+			id: untaggedId,
+			fromId: untaggedFromId,
+			...propsToLog
+		} = nodeUsageProps;
+		const { persistedGcFeatureMatrix, ...configs } = this.configs;
+		const unrefEventProps: Omit<IUnreferencedEventProps, "state" | "usageType"> = {
+			type: nodeType,
+			unrefTime: nodeStateTracker?.unreferencedTimestampMs ?? -1,
+			age:
+				nodeStateTracker !== undefined
+					? nodeUsageProps.currentReferenceTimestampMs -
+					  nodeStateTracker.unreferencedTimestampMs
+					: -1,
+			timeout:
+				nodeStateTracker?.state === UnreferencedState.Inactive
+					? this.configs.inactiveTimeoutMs
+					: this.configs.sweepTimeoutMs,
+			...tagCodeArtifacts({ id: untaggedId, fromId: untaggedFromId }),
+			...propsToLog,
+			...this.createContainerMetadata,
+			gcConfigs: { ...configs, ...persistedGcFeatureMatrix },
+		};
+
+		// If the node that is used is tombstoned, log a tombstone telemetry.
+		// Note that this is done before checking if "nodeStateTracker" is undefined below because unreferenced
+		// tracking may not have yet been enabled. That happens only after the client transitions to write mode.
+		if (nodeUsageProps.isTombstoned) {
+			this.logTombstoneUsageTelemetry(nodeUsageProps, unrefEventProps, nodeType, usageType);
+		}
+
+		// After logging tombstone telemetry, if the node's unreferenced state is not tracked, there is nothing
+		// else to log.
+		if (nodeStateTracker === undefined) {
+			return;
+		}
+
+		const state = nodeStateTracker.state;
+		const uniqueEventId = `${state}-${nodeUsageProps.id}-${nodeUsageProps.usageType}`;
+
 		if (
 			!this.shouldLogNonActiveEvent(
-				nodeUsageProps.id,
 				nodeType,
 				nodeUsageProps.usageType,
 				nodeStateTracker,
@@ -153,42 +196,9 @@ export class GCTelemetryTracker {
 			return;
 		}
 
-		// Add the unique event id so that we don't generate a log for this event again in this session..
+		// Add the unique event id so that we don't generate a log for this event again in this session.
 		this.loggedUnreferencedEvents.add(uniqueEventId);
 
-		const state = nodeStateTracker.state;
-		const { usageType, currentReferenceTimestampMs, packagePath, id, fromId, ...propsToLog } =
-			nodeUsageProps;
-		const eventProps: Omit<IUnreferencedEventProps, "state" | "usageType"> = {
-			type: nodeType,
-			unrefTime: nodeStateTracker.unreferencedTimestampMs,
-			age:
-				nodeUsageProps.currentReferenceTimestampMs -
-				nodeStateTracker.unreferencedTimestampMs,
-			timeout:
-				state === UnreferencedState.Inactive
-					? this.configs.inactiveTimeoutMs
-					: this.configs.sweepTimeoutMs,
-			...tagCodeArtifacts({ id, fromId }),
-			...propsToLog,
-			...this.createContainerMetadata,
-		};
-
-		// This will log the following events:
-		// GC_Tombstone_DataStore_Revived, GC_Tombstone_SubDataStore_Revived, GC_Tombstone_Blob_Revived
-		if (nodeUsageProps.usageType === "Revived" && nodeUsageProps.isTombstoned) {
-			sendGCUnexpectedUsageEvent(
-				this.mc,
-				{
-					eventName: `GC_Tombstone_${nodeType}_Revived`,
-					category: "generic",
-					...tagCodeArtifacts({ url: id }),
-					gcTombstoneEnforcementAllowed: this.gcTombstoneEnforcementAllowed,
-				},
-				undefined /* packagePath */,
-			);
-		}
-
 		// For summarizer client, queue the event so it is logged the next time GC runs if the event is still valid.
 		// For non-summarizer client, log the event now since GC won't run on it. This may result in false positives
 		// but it's a good signal nonetheless and we can consume it with a grain of salt.
@@ -196,7 +206,7 @@ export class GCTelemetryTracker {
 		// SweepReady errors are usages of Objects that will be deleted by GC Sweep!
 		if (this.isSummarizerClient) {
 			this.pendingEventsQueue.push({
-				...eventProps,
+				...unrefEventProps,
 				usageType: nodeUsageProps.usageType,
 				state,
 			});
@@ -206,16 +216,16 @@ export class GCTelemetryTracker {
 			// Events generated:
 			// InactiveObject_Loaded, SweepReadyObject_Loaded
 			if (nodeUsageProps.usageType === "Loaded") {
-				const { id: taggedId, fromId: taggedFromId, ...otherProps } = eventProps;
+				const { id, fromId, headers, gcConfigs, ...detailedProps } = unrefEventProps;
 				const event = {
 					eventName: `${state}Object_${nodeUsageProps.usageType}`,
-					pkg: tagCodeArtifacts({ pkg: nodeUsageProps.packagePath?.join("/") }).pkg,
+					...tagCodeArtifacts({ pkg: nodeUsageProps.packagePath?.join("/") }),
 					stack: generateStack(),
-					id: taggedId,
-					fromId: taggedFromId,
-					details: JSON.stringify({
-						...otherProps,
-					}),
+					id,
+					fromId,
+					headers: { ...headers },
+					details: detailedProps,
+					gcConfigs,
 				};
 
 				// Do not log the inactive object x events as error events as they are not the best signal for
@@ -229,6 +239,52 @@ export class GCTelemetryTracker {
 		}
 	}
 
+	/**
+	 * Logs telemetry when a tombstoned object is changed, revived or loaded.
+	 */
+	private logTombstoneUsageTelemetry(
+		nodeUsageProps: INodeUsageProps,
+		unrefEventProps: Omit<IUnreferencedEventProps, "state" | "usageType">,
+		nodeType: GCNodeType,
+		usageType: NodeUsageType,
+	) {
+		// This will log the following events:
+		// GC_Tombstone_DataStore_Requested, GC_Tombstone_DataStore_Changed, GC_Tombstone_DataStore_Revived
+		// GC_Tombstone_SubDataStore_Requested, GC_Tombstone_SubDataStore_Changed, GC_Tombstone_SubDataStore_Revived
+		// GC_Tombstone_Blob_Requested, GC_Tombstone_Blob_Changed, GC_Tombstone_Blob_Revived
+		const { id, fromId, headers, gcConfigs, ...detailedProps } = unrefEventProps;
+		const eventUsageName = usageType === "Loaded" ? "Requested" : usageType;
+		const event = {
+			eventName: `GC_Tombstone_${nodeType}_${eventUsageName}`,
+			pkg: tagCodeArtifacts({ pkg: nodeUsageProps.packagePath?.join("/") }).pkg,
+			stack: generateStack(),
+			id,
+			fromId,
+			headers: { ...headers },
+			details: detailedProps,
+			gcConfigs,
+			tombstoneFlags: {
+				DisableTombstone: this.mc.config.getBoolean(disableTombstoneKey),
+				ThrowOnTombstoneUsage: this.mc.config.getBoolean(throwOnTombstoneUsageKey),
+				ThrowOnTombstoneLoad: this.mc.config.getBoolean(throwOnTombstoneLoadOverrideKey),
+			},
+			sweepFlags: {
+				EnableSweepFlag: this.mc.config.getBoolean(runSweepKey),
+			},
+		};
+
+		if (
+			(usageType === "Loaded" &&
+				this.configs.throwOnTombstoneLoad &&
+				!headers?.allowTombstone) ||
+			(usageType === "Changed" && this.configs.throwOnTombstoneUsage)
+		) {
+			this.mc.logger.sendErrorEvent(event);
+		} else {
+			this.mc.logger.sendTelemetryEvent(event);
+		}
+	}
+
 	/**
 	 * Log all new references or outbound routes in the current graph that haven't been explicitly notified to GC.
 	 * The principle is that every new reference or outbound route must be notified to GC via the
@@ -295,7 +351,9 @@ export class GCTelemetryTracker {
 		// InactiveObject_Loaded, InactiveObject_Changed, InactiveObject_Revived
 		// SweepReadyObject_Loaded, SweepReadyObject_Changed, SweepReadyObject_Revived
 		for (const eventProps of this.pendingEventsQueue) {
-			const { usageType, state, id, fromId, ...propsToLog } = eventProps;
+			// const { usageType, state, id, fromId, ...propsToLog } = eventProps;
+			const { usageType, state, id, fromId, headers, gcConfigs, ...detailedProps } =
+				eventProps;
 			/**
 			 * Revived event is logged only if the node is active. If the node is not active, the reference to it was
 			 * from another unreferenced node and this scenario is not interesting to log.
@@ -313,11 +371,11 @@ export class GCTelemetryTracker {
 					: undefined;
 				const event = {
 					eventName: `${state}Object_${usageType}`,
-					details: JSON.stringify({
-						...propsToLog,
-					}),
 					id,
 					fromId,
+					headers: { ...headers },
+					details: detailedProps,
+					gcConfigs,
 					...tagCodeArtifacts({
 						pkg: pkg?.join("/"),
 						fromPkg: fromPkg?.join("/"),
@@ -404,7 +462,7 @@ export function sendGCUnexpectedUsageEvent(
 	event.tombstoneFlags = JSON.stringify({
 		DisableTombstone: mc.config.getBoolean(disableTombstoneKey),
 		ThrowOnTombstoneUsage: mc.config.getBoolean(throwOnTombstoneUsageKey),
-		ThrowOnTombstoneLoad: mc.config.getBoolean(throwOnTombstoneLoadKey),
+		ThrowOnTombstoneLoad: mc.config.getBoolean(throwOnTombstoneLoadOverrideKey),
 	});
 	event.sweepFlags = JSON.stringify({
 		EnableSweepFlag: mc.config.getBoolean(runSweepKey),

package/src/gc/index.ts

@@ -12,6 +12,7 @@ export {
 	GCNodeType,
 	gcTestModeKey,
 	gcTombstoneGenerationOptionName,
+	gcThrowOnTombstoneLoadOptionName,
 	gcSweepGenerationOptionName,
 	GCFeatureMatrix,
 	GCVersion,
@@ -31,15 +32,12 @@ export {
 	stableGCVersion,
 	disableAttachmentBlobSweepKey,
 	disableDatastoreSweepKey,
-	throwOnTombstoneLoadKey,
-	throwOnTombstoneUsageKey,
 	UnreferencedState,
+	throwOnTombstoneLoadOverrideKey,
 } from "./gcDefinitions";
 export {
 	cloneGCData,
 	concatGarbageCollectionStates,
-	shouldAllowGcTombstoneEnforcement,
-	shouldAllowGcSweep,
 	trimLeadingAndTrailingSlashes,
 	unpackChildNodesGCDetails,
 } from "./gcHelpers";

package/src/opLifecycle/opGroupingManager.ts

@@ -5,6 +5,8 @@
 
 import { assert } from "@fluidframework/core-utils";
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
+import { createChildLogger } from "@fluidframework/telemetry-utils";
+import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
 import { ContainerMessageType } from "../messageTypes";
 import { IBatch } from "./definitions";
 
@@ -26,16 +28,38 @@ function isGroupContents(opContents: any): opContents is IGroupedBatchMessageCon
 	return opContents?.type === OpGroupingManager.groupedBatchOp;
 }
 
+export interface OpGroupingManagerConfig {
+	readonly groupedBatchingEnabled: boolean;
+	readonly opCountThreshold: number;
+	readonly reentrantBatchGroupingEnabled: boolean;
+}
+
 export class OpGroupingManager {
 	static readonly groupedBatchOp = "groupedBatch";
+	private readonly logger;
 
-	constructor(private readonly groupedBatchingEnabled: boolean) {}
+	constructor(
+		private readonly config: OpGroupingManagerConfig,
+		logger: ITelemetryBaseLogger,
+	) {
+		this.logger = createChildLogger({ logger, namespace: "OpGroupingManager" });
+	}
 
 	public groupBatch(batch: IBatch): IBatch {
-		if (batch.content.length < 2 || !this.groupedBatchingEnabled) {
+		if (!this.shouldGroup(batch)) {
 			return batch;
 		}
 
+		if (batch.content.length >= 1000) {
+			this.logger.sendTelemetryEvent({
+				eventName: "GroupLargeBatch",
+				length: batch.content.length,
+				threshold: this.config.opCountThreshold,
+				reentrant: batch.hasReentrantOps,
+				referenceSequenceNumber: batch.content[0].referenceSequenceNumber,
+			});
+		}
+
 		for (const message of batch.content) {
 			if (message.metadata) {
 				const keys = Object.keys(message.metadata);
@@ -86,4 +110,15 @@ export class OpGroupingManager {
 			compression: subMessage.compression,
 		}));
 	}
+
+	public shouldGroup(batch: IBatch): boolean {
+		return (
+			// Grouped batching must be enabled
+			this.config.groupedBatchingEnabled &&
+			// The number of ops in the batch must surpass the configured threshold
+			batch.content.length >= this.config.opCountThreshold &&
+			// Support for reentrant batches must be explicitly enabled
+			(this.config.reentrantBatchGroupingEnabled || batch.hasReentrantOps !== true)
+		);
+	}
 }

package/src/opLifecycle/outbox.ts

@@ -30,7 +30,6 @@ export interface IOutboxConfig {
 	// The maximum size of a batch that we can send over the wire.
 	readonly maxBatchSizeInBytes: number;
 	readonly disablePartialFlush: boolean;
-	readonly enableGroupedBatching: boolean;
 }
 
 export interface IOutboxParameters {
@@ -270,7 +269,10 @@ export class Outbox {
 		}
 
 		const rawBatch = batchManager.popBatch();
-		if (rawBatch.hasReentrantOps === true && this.params.config.enableGroupedBatching) {
+		if (
+			rawBatch.hasReentrantOps === true &&
+			this.params.groupingManager.shouldGroup(rawBatch)
+		) {
 			assert(!this.rebasing, 0x6fa /* A rebased batch should never have reentrant ops */);
 			// If a batch contains reentrant ops (ops created as a result from processing another op)
 			// it needs to be rebased so that we can ensure consistent reference sequence numbers

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.0.0-internal.7.1.0";
+export const pkgVersion = "2.0.0-internal.7.1.2";