@fluidframework/runtime-utils 2.0.0-internal.3.1.0 → 2.0.0-internal.3.2.1

package/src/summarizerNode/summarizerNode.ts

@@ -19,8 +19,14 @@ import {
 	ISnapshotTree,
 	SummaryObject,
 } from "@fluidframework/protocol-definitions";
-import { ITelemetryLogger } from "@fluidframework/common-definitions";
+import { ITelemetryErrorEvent, ITelemetryLogger } from "@fluidframework/common-definitions";
 import { assert, unreachableCase } from "@fluidframework/common-utils";
+import {
+	ChildLogger,
+	LoggingError,
+	PerformanceEvent,
+	TelemetryDataTag,
+} from "@fluidframework/telemetry-utils";
 import { mergeStats, convertToSummaryTree, calculateStats } from "../summaryUtils";
 import { ReadAndParseBlob } from "../utils";
 import {
@@ -35,6 +41,10 @@ import {
 	SummaryNode,
 } from "./summarizerNodeUtils";
 
+/**
+ * @deprecated Internal implementation detail and will no longer be exported in an
+ * upcoming release.
+ */
 export interface IRootSummarizerNode extends ISummarizerNode, ISummarizerNodeRootContract {}
 
 /**
@@ -61,10 +71,40 @@ export class SummarizerNode implements IRootSummarizerNode {
 
 	protected readonly children = new Map<string, SummarizerNode>();
 	protected readonly pendingSummaries = new Map<string, SummaryNode>();
-	private wipReferenceSequenceNumber: number | undefined;
+	protected wipReferenceSequenceNumber: number | undefined;
 	private wipLocalPaths: { localPath: EscapedPath; additionalPath?: EscapedPath } | undefined;
 	private wipSkipRecursion = false;
 
+	protected readonly logger: ITelemetryLogger;
+
+	/**
+	 * Do not call constructor directly.
+	 * Use createRootSummarizerNode to create root node, or createChild to create child nodes.
+	 */
+	public constructor(
+		baseLogger: ITelemetryLogger,
+		private readonly summarizeInternalFn: SummarizeInternalFn,
+		config: ISummarizerNodeConfig,
+		private _changeSequenceNumber: number,
+		/** Undefined means created without summary */
+		private _latestSummary?: SummaryNode,
+		private readonly initialSummary?: IInitialSummary,
+		protected wipSummaryLogger?: ITelemetryLogger,
+		/** A unique id of this node to be logged when sending telemetry. */
+		protected telemetryNodeId?: string,
+	) {
+		this.canReuseHandle = config.canReuseHandle ?? true;
+		// All logs posted by the summarizer node should include the telemetryNodeId.
+		this.logger = ChildLogger.create(baseLogger, undefined /* namespace */, {
+			all: {
+				id: {
+					tag: TelemetryDataTag.CodeArtifact,
+					value: this.telemetryNodeId,
+				},
+			},
+		});
+	}
+
 	public startSummary(referenceSequenceNumber: number, summaryLogger: ITelemetryLogger) {
 		assert(
 			this.wipSummaryLogger === undefined,
@@ -89,7 +129,7 @@ export class SummarizerNode implements IRootSummarizerNode {
 		telemetryContext?: ITelemetryContext,
 	): Promise<ISummarizeResult> {
 		assert(
-			this.isTrackingInProgress(),
+			this.isSummaryInProgress(),
 			0x1a1 /* "summarize should not be called when not tracking the summary" */,
 		);
 		assert(
@@ -176,10 +216,21 @@ export class SummarizerNode implements IRootSummarizerNode {
 			}
 		}
 
-		// This should come from wipLocalPaths in normal cases, or from the latestSummary
-		// if parentIsFailure or parentIsReused is true.
-		// If there is no latestSummary, clearSummary and return before reaching this code.
-		assert(!!localPathsToUse, 0x1a5 /* "Tracked summary local paths not set" */);
+		/**
+		 * The absence of wip local path indicates that summarize was not called for this node. This can happen if:
+		 * 1. A child node was created after summarize was already called on the parent. For example, a data store
+		 * is realized (loaded) after summarize was called on it creating summarizer nodes for its DDSes. In this case,
+		 * parentSkipRecursion will be true and the if block above would handle it.
+		 * 2. A new node was created but summarize was never called on it. This can mean that the summary that is
+		 * generated may not have the data from this node. We should not continue, log and throw an error. This
+		 * will help us identify these cases and take appropriate action.
+		 */
+		if (localPathsToUse === undefined) {
+			this.throwUnexpectedError({
+				eventName: "NodeNotSummarized",
+				proposalHandle,
+			});
+		}
 
 		const summary = new SummaryNode({
 			...localPathsToUse,
@@ -235,66 +286,104 @@ export class SummarizerNode implements IRootSummarizerNode {
 		readAndParseBlob: ReadAndParseBlob,
 		correlatedSummaryLogger: ITelemetryLogger,
 	): Promise<RefreshSummaryResult> {
-		this.defaultLogger.sendTelemetryEvent({
-			eventName: "refreshLatestSummary_start",
+		const eventProps: {
+			proposalHandle: string | undefined;
+			summaryRefSeq: number;
+			referenceSequenceNumber: number;
+			latestSummaryUpdated?: boolean;
+			wasSummaryTracked?: boolean;
+		} = {
 			proposalHandle,
-			referenceSequenceNumber: this.referenceSequenceNumber,
 			summaryRefSeq,
-		});
+			referenceSequenceNumber: this.referenceSequenceNumber,
+		};
+		return PerformanceEvent.timedExecAsync(
+			this.logger,
+			{
+				eventName: "refreshLatestSummary",
+				...eventProps,
+			},
+			async (event) => {
+				// Refresh latest summary should not happen while a summary is in progress. If it does, it can result
+				// in inconsistent state, so, we should not continue;
+				if (this.isSummaryInProgress()) {
+					throw new LoggingError("UnexpectedRefreshDuringSummarize", {
+						inProgressSummaryRefSeq: this.wipReferenceSequenceNumber,
+					});
+				}
 
-		if (proposalHandle !== undefined) {
-			const maybeSummaryNode = this.pendingSummaries.get(proposalHandle);
+				if (proposalHandle !== undefined) {
+					const maybeSummaryNode = this.pendingSummaries.get(proposalHandle);
 
-			if (maybeSummaryNode !== undefined) {
-				this.refreshLatestSummaryFromPending(
-					proposalHandle,
-					maybeSummaryNode.referenceSequenceNumber,
-				);
-				return { latestSummaryUpdated: true, wasSummaryTracked: true, summaryRefSeq };
-			}
+					if (maybeSummaryNode !== undefined) {
+						this.refreshLatestSummaryFromPending(
+							proposalHandle,
+							maybeSummaryNode.referenceSequenceNumber,
+						);
+						eventProps.wasSummaryTracked = true;
+						eventProps.latestSummaryUpdated = true;
+						event.end(eventProps);
+						return {
+							latestSummaryUpdated: true,
+							wasSummaryTracked: true,
+							summaryRefSeq,
+						};
+					}
 
-			const props = {
-				summaryRefSeq,
-				pendingSize: this.pendingSummaries.size ?? undefined,
-			};
-			this.defaultLogger.sendTelemetryEvent({
-				eventName: "PendingSummaryNotFound",
-				proposalHandle,
-				referenceSequenceNumber: this.referenceSequenceNumber,
-				details: JSON.stringify(props),
-			});
-		}
+					const props = {
+						summaryRefSeq,
+						pendingSize: this.pendingSummaries.size ?? undefined,
+					};
+					this.logger.sendTelemetryEvent({
+						eventName: "PendingSummaryNotFound",
+						proposalHandle,
+						referenceSequenceNumber: this.referenceSequenceNumber,
+						details: JSON.stringify(props),
+					});
+				}
 
-		// If the summary for which refresh is called is older than the latest tracked summary, ignore it.
-		if (this.referenceSequenceNumber >= summaryRefSeq) {
-			return { latestSummaryUpdated: false };
-		}
+				// If the summary for which refresh is called is older than the latest tracked summary, ignore it.
+				if (this.referenceSequenceNumber >= summaryRefSeq) {
+					eventProps.latestSummaryUpdated = false;
+					event.end(eventProps);
+					return { latestSummaryUpdated: false };
+				}
 
-		// Fetch the latest snapshot and refresh state from it. Note that we need to use the reference sequence number
-		// of the fetched snapshot and not the "summaryRefSeq" that was passed in.
-		const { snapshotTree, snapshotRefSeq: fetchedSnapshotRefSeq } = await fetchLatestSnapshot();
+				// Fetch the latest snapshot and refresh state from it. Note that we need to use the reference sequence number
+				// of the fetched snapshot and not the "summaryRefSeq" that was passed in.
+				const { snapshotTree, snapshotRefSeq: fetchedSnapshotRefSeq } =
+					await fetchLatestSnapshot();
+
+				// Possible re-entrancy. We may have updated latest summary state while fetching the snapshot. If the fetched
+				// snapshot is older than the latest tracked summary, ignore it.
+				if (this.referenceSequenceNumber >= fetchedSnapshotRefSeq) {
+					eventProps.latestSummaryUpdated = false;
+					event.end(eventProps);
+					return { latestSummaryUpdated: false };
+				}
 
-		// Possible re-entrancy. We may have updated latest summary state while fetching the snapshot. If the fetched
-		// snapshot is older than the latest tracked summary, ignore it.
-		if (this.referenceSequenceNumber >= fetchedSnapshotRefSeq) {
-			return { latestSummaryUpdated: false };
-		}
+				await this.refreshLatestSummaryFromSnapshot(
+					fetchedSnapshotRefSeq,
+					snapshotTree,
+					undefined,
+					EscapedPath.create(""),
+					correlatedSummaryLogger,
+					readAndParseBlob,
+				);
 
-		await this.refreshLatestSummaryFromSnapshot(
-			fetchedSnapshotRefSeq,
-			snapshotTree,
-			undefined,
-			EscapedPath.create(""),
-			correlatedSummaryLogger,
-			readAndParseBlob,
+				eventProps.latestSummaryUpdated = true;
+				eventProps.wasSummaryTracked = false;
+				eventProps.summaryRefSeq = fetchedSnapshotRefSeq;
+				event.end(eventProps);
+				return {
+					latestSummaryUpdated: true,
+					wasSummaryTracked: false,
+					snapshotTree,
+					summaryRefSeq: fetchedSnapshotRefSeq,
+				};
+			},
+			{ start: true, end: true, cancel: "error" },
 		);
-
-		return {
-			latestSummaryUpdated: true,
-			wasSummaryTracked: false,
-			snapshotTree,
-			summaryRefSeq: fetchedSnapshotRefSeq,
-		};
 	}
 	/**
 	 * Called when we get an ack from the server for a summary we've just sent. Updates the reference state of this node
@@ -428,23 +517,6 @@ export class SummarizerNode implements IRootSummarizerNode {
 
 	protected readonly canReuseHandle: boolean;
 
-	/**
-	 * Do not call constructor directly.
-	 * Use createRootSummarizerNode to create root node, or createChild to create child nodes.
-	 */
-	public constructor(
-		protected readonly defaultLogger: ITelemetryLogger,
-		private readonly summarizeInternalFn: SummarizeInternalFn,
-		config: ISummarizerNodeConfig,
-		private _changeSequenceNumber: number,
-		/** Undefined means created without summary */
-		private _latestSummary?: SummaryNode,
-		private readonly initialSummary?: IInitialSummary,
-		protected wipSummaryLogger?: ITelemetryLogger,
-	) {
-		this.canReuseHandle = config.canReuseHandle ?? true;
-	}
-
 	public createChild(
 		/** Summarize function */
 		summarizeInternalFn: SummarizeInternalFn,
@@ -462,13 +534,14 @@ export class SummarizerNode implements IRootSummarizerNode {
 
 		const createDetails: ICreateChildDetails = this.getCreateDetailsForChild(id, createParam);
 		const child = new SummarizerNode(
-			this.defaultLogger,
+			this.logger,
 			summarizeInternalFn,
 			config,
 			createDetails.changeSequenceNumber,
 			createDetails.latestSummary,
 			createDetails.initialSummary,
 			this.wipSummaryLogger,
+			createDetails.telemetryNodeId,
 		);
 
 		// There may be additional state that has to be updated in this child. For example, if a summary is being
@@ -574,10 +647,13 @@ export class SummarizerNode implements IRootSummarizerNode {
 			}
 		}
 
+		const childtelemetryNodeId = `${this.telemetryNodeId ?? ""}/${id}`;
+
 		return {
 			initialSummary,
 			latestSummary,
 			changeSequenceNumber,
+			telemetryNodeId: childtelemetryNodeId,
 		};
 	}
 
@@ -589,9 +665,9 @@ export class SummarizerNode implements IRootSummarizerNode {
 	 * @param child - The child node whose state is to be updated.
 	 */
 	protected maybeUpdateChildState(child: SummarizerNode) {
-		// If we are tracking a summary, this child was created after the tracking started. So, we need to update the
-		// child's tracking state as well.
-		if (this.isTrackingInProgress()) {
+		// If a summary is in progress, this child was created after the summary started. So, we need to update the
+		// child's summary state as well.
+		if (this.isSummaryInProgress()) {
 			child.wipReferenceSequenceNumber = this.wipReferenceSequenceNumber;
 		}
 		// In case we have pending summaries on the parent, let's initialize it on the child.
@@ -611,12 +687,29 @@ export class SummarizerNode implements IRootSummarizerNode {
 	protected addPendingSummary(key: string, summary: SummaryNode) {
 		this.pendingSummaries.set(key, summary);
 	}
+
 	/**
 	 * Tells whether summary tracking is in progress. True if "startSummary" API is called before summarize.
 	 */
-	protected isTrackingInProgress(): boolean {
+	public isSummaryInProgress(): boolean {
 		return this.wipReferenceSequenceNumber !== undefined;
 	}
+
+	/**
+	 * Creates and throws an error due to unexpected conditions.
+	 */
+	protected throwUnexpectedError(eventProps: ITelemetryErrorEvent): never {
+		const error = new LoggingError(eventProps.eventName, {
+			...eventProps,
+			referenceSequenceNumber: this.wipReferenceSequenceNumber,
+			id: {
+				tag: TelemetryDataTag.CodeArtifact,
+				value: this.telemetryNodeId,
+			},
+		});
+		this.logger.sendErrorEvent(eventProps, error);
+		throw error;
+	}
 }
 
 /**
@@ -627,6 +720,9 @@ export class SummarizerNode implements IRootSummarizerNode {
  * @param referenceSequenceNumber - Reference sequence number of last acked summary,
  * or undefined if not loaded from summary
  * @param config - Configure behavior of summarizer node
+ *
+ * @deprecated Internal implementation detail and will no longer be exported in an
+ * upcoming release.
  */
 export const createRootSummarizerNode = (
 	logger: ITelemetryLogger,
@@ -643,4 +739,7 @@ export const createRootSummarizerNode = (
 		referenceSequenceNumber === undefined
 			? undefined
 			: SummaryNode.createForRoot(referenceSequenceNumber),
+		undefined /* initialSummary */,
+		undefined /* wipSummaryLogger */,
+		"" /* telemetryNodeId */,
 	);

package/src/summarizerNode/summarizerNodeUtils.ts

@@ -17,6 +17,9 @@ import { ReadAndParseBlob } from "../utils";
  *
  * 3. The latest summary was updated but the summary corresponding to the params was not tracked. In this case, the
  * latest snapshot is fetched and the latest summary state is updated based on it.
+ *
+ * @deprecated Internal implementation detail and will no longer be exported in an
+ * upcoming release.
  */
 export type RefreshSummaryResult =
 	| {
@@ -36,12 +39,19 @@ export type RefreshSummaryResult =
 
 /**
  * Result of snapshot fetch during refreshing latest summary state.
+ *
+ * @deprecated Internal implementation detail and will no longer be exported in an
+ * upcoming release.
  */
 export interface IFetchSnapshotResult {
 	snapshotTree: ISnapshotTree;
 	snapshotRefSeq: number;
 }
 
+/**
+ * @deprecated Internal implementation detail and will no longer be exported in an
+ * upcoming release.
+ */
 export interface ISummarizerNodeRootContract {
 	startSummary(referenceSequenceNumber: number, summaryLogger: ITelemetryLogger): void;
 	completeSummary(proposalHandle: string): void;
@@ -143,23 +153,6 @@ export class SummaryNode {
 	}
 }
 
-/**
- * Parameter to help encode summary with conditional behavior.
- * When fromSummary is true, it will contain the SummaryNode of
- * its previous summary, which it can use to point to with a handle.
- * When fromSummary is false, it will use an actual summary tree
- * as its base summary in case the first summary is a differential summary.
- */
-export type EncodeSummaryParam =
-	| {
-			fromSummary: true;
-			summaryNode: SummaryNode;
-	  }
-	| {
-			fromSummary: false;
-			initialSummary: ISummaryTreeWithStats;
-	  };
-
 /**
  * Information about the initial summary tree found from an attach op.
  */
@@ -179,6 +172,8 @@ export interface ICreateChildDetails {
 	latestSummary: SummaryNode | undefined;
 	/** Sequence number of latest known change to the node */
 	changeSequenceNumber: number;
+	/** A unique id of this child to be logged when sending telemetry. */
+	telemetryNodeId: string;
 }
 
 export interface ISubtreeInfo<T extends ISnapshotTree | SummaryObject> {

package/src/summarizerNode/summarizerNodeWithGc.ts

@@ -35,6 +35,10 @@ import {
 	SummaryNode,
 } from "./summarizerNodeUtils";
 
+/**
+ * @deprecated Internal implementation detail and will no longer be exported in an
+ * upcoming release.
+ */
 export interface IRootSummarizerNodeWithGC
 	extends ISummarizerNodeWithGC,
 		ISummarizerNodeRootContract {}
@@ -67,7 +71,7 @@ class SummaryNodeWithGC extends SummaryNode {
  * - Adds trackState param to summarize. If trackState is false, it bypasses the SummarizerNode and calls
  * directly into summarizeInternal method.
  */
-export class SummarizerNodeWithGC extends SummarizerNode implements IRootSummarizerNodeWithGC {
+class SummarizerNodeWithGC extends SummarizerNode implements IRootSummarizerNodeWithGC {
 	// Tracks the work-in-progress used routes during summary.
 	private wipSerializedUsedRoutes: string | undefined;
 
@@ -109,6 +113,8 @@ export class SummarizerNodeWithGC extends SummarizerNode implements IRootSummari
 		wipSummaryLogger?: ITelemetryLogger,
 		private readonly getGCDataFn?: (fullGC?: boolean) => Promise<IGarbageCollectionData>,
 		getBaseGCDetailsFn?: () => Promise<IGarbageCollectionDetailsBase>,
+		/** A unique id of this node to be logged when sending telemetry. */
+		telemetryId?: string,
 	) {
 		super(
 			logger,
@@ -119,6 +125,7 @@ export class SummarizerNodeWithGC extends SummarizerNode implements IRootSummari
 			latestSummary,
 			initialSummary,
 			wipSummaryLogger,
+			telemetryId,
 		);
 
 		this.gcDisabled = config.gcDisabled === true;
@@ -164,9 +171,9 @@ export class SummarizerNodeWithGC extends SummarizerNode implements IRootSummari
 		trackState: boolean = true,
 		telemetryContext?: ITelemetryContext,
 	): Promise<ISummarizeResult> {
-		// If GC is not disabled and we are tracking a summary, GC should have run and updated the used routes for this
+		// If GC is not disabled and a summary is in progress, GC should have run and updated the used routes for this
 		// summary by calling updateUsedRoutes which sets wipSerializedUsedRoutes.
-		if (!this.gcDisabled && this.isTrackingInProgress()) {
+		if (!this.gcDisabled && this.isSummaryInProgress()) {
 			assert(
 				this.wipSerializedUsedRoutes !== undefined,
 				0x1b1 /* "wip used routes should be set if tracking a summary" */,
@@ -239,10 +246,21 @@ export class SummarizerNodeWithGC extends SummarizerNode implements IRootSummari
 		// If GC is disabled, don't set wip used routes.
 		if (!this.gcDisabled) {
 			wipSerializedUsedRoutes = this.wipSerializedUsedRoutes;
-			assert(
-				wipSerializedUsedRoutes !== undefined,
-				0x1b5 /* "We should have been tracking used routes" */,
-			);
+			/**
+			 * The absence of wip used routes indicates that GC was not run on this node. This can happen if:
+			 * 1. A child node was created after GC was already run on the parent. For example, a data store
+			 * is realized (loaded) after GC was run on it creating summarizer nodes for its DDSes. In this
+			 * case, the used routes of the parent should be passed on the child nodes and it should be fine.
+			 * 2. A new node was created but GC was never run on it. This can mean that the GC data generated
+			 * during summarize is complete . We should not continue, log and throw an error. This will help us
+			 * identify these cases and take appropriate action.
+			 */
+			if (wipSerializedUsedRoutes === undefined) {
+				this.throwUnexpectedError({
+					eventName: "NodeDidNotRunGC",
+					proposalHandle,
+				});
+			}
 		}
 
 		super.completeSummaryCore(proposalHandle, parentPath, parentSkipRecursion);
@@ -436,7 +454,7 @@ export class SummarizerNodeWithGC extends SummarizerNode implements IRootSummari
 
 		const createDetails: ICreateChildDetails = this.getCreateDetailsForChild(id, createParam);
 		const child = new SummarizerNodeWithGC(
-			this.defaultLogger,
+			this.logger,
 			summarizeInternalFn,
 			{
 				...config,
@@ -449,6 +467,7 @@ export class SummarizerNodeWithGC extends SummarizerNode implements IRootSummari
 			this.wipSummaryLogger,
 			getGCDataFn,
 			async () => getChildBaseGCDetailsP,
+			createDetails.telemetryNodeId,
 		);
 
 		// There may be additional state that has to be updated in this child. For example, if a summary is being
@@ -482,9 +501,9 @@ export class SummarizerNodeWithGC extends SummarizerNode implements IRootSummari
 		// are in the same order.
 		this.usedRoutes = usedRoutes.sort();
 
-		// If GC is not disabled and we are tracking a summary, update the work-in-progress used routes so that it can
+		// If GC is not disabled and a summary is in progress, update the work-in-progress used routes so that it can
 		// be tracked for this summary.
-		if (!this.gcDisabled && this.isTrackingInProgress()) {
+		if (!this.gcDisabled && this.isSummaryInProgress()) {
 			this.wipSerializedUsedRoutes = JSON.stringify(this.usedRoutes);
 		}
 	}
@@ -531,6 +550,9 @@ export class SummarizerNodeWithGC extends SummarizerNode implements IRootSummari
  * @param config - Configure behavior of summarizer node
  * @param getGCDataFn - Function to get the GC data of this node
  * @param baseGCDetailsP - Function to get the initial GC details of this node
+ *
+ * @deprecated Internal implementation detail and will no longer be exported in an
+ * upcoming release.
  */
 export const createRootSummarizerNodeWithGC = (
 	logger: ITelemetryLogger,
@@ -553,4 +575,5 @@ export const createRootSummarizerNodeWithGC = (
 		undefined /* wipSummaryLogger */,
 		getGCDataFn,
 		getBaseGCDetailsFn,
+		"" /* telemetryId */,
 	);

package/src/summaryUtils.ts

@@ -356,6 +356,20 @@ export class TelemetryContext implements ITelemetryContext {
 		this.telemetry.set(`${prefix}${property}`, value);
 	}
 
+	/**
+	 * {@inheritDoc @fluidframework/runtime-definitions#ITelemetryContext.setMultiple}
+	 */
+	setMultiple(
+		prefix: string,
+		property: string,
+		values: Record<string, TelemetryEventPropertyType>,
+	): void {
+		// Set the values individually so that they are logged as a flat list along with other properties.
+		for (const key of Object.keys(values)) {
+			this.set(prefix, `${property}_${key}`, values[key]);
+		}
+	}
+
 	/**
 	 * {@inheritDoc @fluidframework/runtime-definitions#ITelemetryContext.get}
 	 */