@fluidframework/container-runtime 2.0.0-internal.5.1.0 → 2.0.0-internal.5.2.0

package/src/summary/index.ts

@@ -62,6 +62,8 @@ export {
 	ISummarizingWarning,
 	IUploadSummaryResult,
 	SummarizeResultPart,
+	SubmitSummaryFailureData,
+	SummaryStage,
 } from "./summarizerTypes";
 export {
 	IAckedSummary,
@@ -96,7 +98,7 @@ export {
 	wrapSummaryInChannelsTree,
 	idCompressorBlobName,
 } from "./summaryFormat";
-export { getFailMessage, SummarizeReason } from "./summaryGenerator";
+export { getFailMessage, RetriableSummaryError, SummarizeReason } from "./summaryGenerator";
 export {
 	IConnectedEvents,
 	IConnectedState,

package/src/summary/runningSummarizer.ts

@@ -3,7 +3,7 @@
  * Licensed under the MIT License.
  */
 
-import { IDisposable } from "@fluidframework/common-definitions";
+import { IDisposable } from "@fluidframework/core-interfaces";
 import {
 	ITelemetryLoggerExt,
 	ChildLogger,

package/src/summary/summarizerNode/index.ts

@@ -7,6 +7,7 @@ export {
 	IFetchSnapshotResult,
 	ISummarizerNodeRootContract,
 	RefreshSummaryResult,
+	ValidateSummaryResult,
 } from "./summarizerNodeUtils";
 export { IRootSummarizerNode, createRootSummarizerNode } from "./summarizerNode";
 export { IRootSummarizerNodeWithGC, createRootSummarizerNodeWithGC } from "./summarizerNodeWithGc";

package/src/summary/summarizerNode/summarizerNode.ts

@@ -3,6 +3,7 @@
  * Licensed under the MIT License.
  */
 
+import { ITelemetryErrorEvent } from "@fluidframework/common-definitions";
 import {
 	ISummarizerNode,
 	ISummarizerNodeConfig,
@@ -20,7 +21,6 @@ import {
 	ISnapshotTree,
 	SummaryObject,
 } from "@fluidframework/protocol-definitions";
-import { ITelemetryErrorEvent } from "@fluidframework/common-definitions";
 import {
 	ITelemetryLoggerExt,
 	ChildLogger,
@@ -45,6 +45,7 @@ import {
 	parseSummaryTreeForSubtrees,
 	RefreshSummaryResult,
 	SummaryNode,
+	ValidateSummaryResult,
 } from "./summarizerNodeUtils";
 
 export interface IRootSummarizerNode extends ISummarizerNode, ISummarizerNodeRootContract {}
@@ -193,24 +194,117 @@ export class SummarizerNode implements IRootSummarizerNode {
 	}
 
 	/**
-	 * Complete the WIP summary for the given proposalHandle
+	 * Validates that the in-progress summary is correct, i.e., summarize should have run for all non-skipped
+	 * nodes. This will only be called for the root summarizer node and is called by it recursively on all child nodes.
+	 *
+	 * @returns ValidateSummaryResult which contains a boolean success indicating whether the validation was successful.
+	 * In case of failure, additional information is returned indicating type of failure and where it was.
+	 */
+	public validateSummary(): ValidateSummaryResult {
+		return this.validateSummaryCore(false /* parentSkipRecursion */);
+	}
+
+	/**
+	 * Validates that the in-progress summary is correct for all nodes, i.e., summarize should have run for all
+	 * non-skipped nodes.
+	 * @param parentSkipRecursion - true if the parent of this node skipped recursing the child nodes when summarizing.
+	 * In that case, the children will not have work-in-progress state.
+	 *
+	 * @returns ValidateSummaryResult which contains a boolean success indicating whether the validation was successful.
+	 * In case of failure, additional information is returned indicating type of failure and where it was.
+	 */
+	protected validateSummaryCore(parentSkipRecursion: boolean): ValidateSummaryResult {
+		if (this.wasSummarizeMissed(parentSkipRecursion)) {
+			return {
+				success: false,
+				reason: "NodeDidNotSummarize",
+				id: {
+					tag: TelemetryDataTag.CodeArtifact,
+					value: this.telemetryNodeId,
+				},
+				// These errors are usually transient and should go away when summarize is retried.
+				retryAfterSeconds: 1,
+			};
+		}
+		if (parentSkipRecursion) {
+			return { success: true };
+		}
+
+		for (const child of this.children.values()) {
+			const result = child.validateSummaryCore(this.wipSkipRecursion || parentSkipRecursion);
+			// If any child fails, return the failure.
+			if (!result.success) {
+				return result;
+			}
+		}
+		return { success: true };
+	}
+
+	private wasSummarizeMissed(parentSkipRecursion: boolean): boolean {
+		assert(
+			this.wipSummaryLogger !== undefined,
+			0x6fc /* wipSummaryLogger should have been set in startSummary or ctor */,
+		);
+		assert(this.wipReferenceSequenceNumber !== undefined, 0x6fd /* Not tracking a summary */);
+
+		// If the parent node skipped recursion, it did not call summarize on this node. So, summarize was not missed
+		// but was intentionally not called.
+		// Otherwise, summarize should have been called on this node and wipLocalPaths must be set.
+		if (parentSkipRecursion || this.wipLocalPaths !== undefined) {
+			return false;
+		}
+
+		/**
+		 * The absence of wip local path indicates that summarize was not called for this node. Return failure.
+		 * This can happen if:
+		 * 1. A child node was created after summarize was already called on the parent. For example, a data store
+		 * is realized (loaded) after summarize was called on it creating summarizer nodes for its DDSes. In this case,
+		 * parentSkipRecursion will be true and the if block above would handle it.
+		 * 2. A new node was created but summarize was never called on it. This can mean that the summary that is
+		 * generated may not have the data from this node. We should not continue, log and throw an error. This
+		 * will help us identify these cases and take appropriate action.
+		 *
+		 * This happens due to scenarios such as data store created during summarize. Such errors should go away when
+		 * summarize is attempted again.
+		 */
+		return true;
+	}
+
+	/**
+	 * Called after summary has been uploaded to the server. Add the work-in-progress state to the pending summary
+	 * queue. We track this until we get an ack from the server for this summary.
+	 * @param proposalHandle - The handle of the summary that was uploaded to the server.
 	 */
-	public completeSummary(proposalHandle: string) {
-		this.completeSummaryCore(proposalHandle, undefined, false);
+	public completeSummary(proposalHandle: string, validate: boolean) {
+		this.completeSummaryCore(
+			proposalHandle,
+			undefined /* parentPath */,
+			false /* parentSkipRecursion */,
+			validate,
+		);
 	}
 
 	/**
-	 * Recursive implementation for completeSummary, with additional internal-only parameters
+	 * Recursive implementation for completeSummary, with additional internal-only parameters.
+	 * @param proposalHandle - The handle of the summary that was uploaded to the server.
+	 * @param parentPath - The path of the parent node which is used to build the path of this node.
+	 * @param parentSkipRecursion - true if the parent of this node skipped recursing the child nodes when summarizing.
+	 * In that case, the children will not have work-in-progress state.
+	 * @param validate - true to validate that the in-progress summary is correct for all nodes.
 	 */
 	protected completeSummaryCore(
 		proposalHandle: string,
 		parentPath: EscapedPath | undefined,
 		parentSkipRecursion: boolean,
+		validate: boolean,
 	) {
-		assert(
-			this.wipSummaryLogger !== undefined,
-			0x1a3 /* "wipSummaryLogger should have been set in startSummary or ctor" */,
-		);
+		if (validate && this.wasSummarizeMissed(parentSkipRecursion)) {
+			this.throwUnexpectedError({
+				eventName: "NodeDidNotSummarize",
+				proposalHandle,
+			});
+		}
+
 		assert(this.wipReferenceSequenceNumber !== undefined, 0x1a4 /* "Not tracking a summary" */);
 		let localPathsToUse = this.wipLocalPaths;
 
@@ -239,22 +333,9 @@ export class SummarizerNode implements IRootSummarizerNode {
 			}
 		}
 
-		/**
-		 * The absence of wip local path indicates that summarize was not called for this node. This can happen if:
-		 * 1. A child node was created after summarize was already called on the parent. For example, a data store
-		 * is realized (loaded) after summarize was called on it creating summarizer nodes for its DDSes. In this case,
-		 * parentSkipRecursion will be true and the if block above would handle it.
-		 * 2. A new node was created but summarize was never called on it. This can mean that the summary that is
-		 * generated may not have the data from this node. We should not continue, log and throw an error. This
-		 * will help us identify these cases and take appropriate action.
-		 */
-		if (localPathsToUse === undefined) {
-			this.throwUnexpectedError({
-				eventName: "NodeNotSummarized",
-				proposalHandle,
-			});
-		}
-
+		// If localPathsToUse is undefined, it means summarize didn't run for this node and in that case the validate
+		// step should have failed.
+		assert(localPathsToUse !== undefined, 0x6fe /* summarize didn't run for node */);
 		const summary = new SummaryNode({
 			...localPathsToUse,
 			referenceSequenceNumber: this.wipReferenceSequenceNumber,
@@ -266,6 +347,7 @@ export class SummarizerNode implements IRootSummarizerNode {
 				proposalHandle,
 				fullPathForChildren,
 				this.wipSkipRecursion || parentSkipRecursion,
+				validate,
 			);
 		}
 		// Note that this overwrites existing pending summary with

package/src/summary/summarizerNode/summarizerNodeUtils.ts

@@ -3,7 +3,7 @@
  * Licensed under the MIT License.
  */
 
-import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
+import { ITelemetryLoggerExt, TelemetryDataTag } from "@fluidframework/telemetry-utils";
 import { ISnapshotTree, ISummaryTree, SummaryObject } from "@fluidframework/protocol-definitions";
 import { channelsTreeName, ISummaryTreeWithStats } from "@fluidframework/runtime-definitions";
 import { ReadAndParseBlob } from "@fluidframework/runtime-utils";
@@ -42,9 +42,32 @@ export interface IFetchSnapshotResult {
 	snapshotRefSeq: number;
 }
 
+/**
+ * Return type of validateSummary function. In case of success, the object returned should have success: true.
+ * In case of failure, the object returned should have success: false and additional properties to indicate what
+ * the failure was, where it was, can it be retried, etc.
+ */
+export type ValidateSummaryResult =
+	| {
+			success: true;
+	  }
+	| {
+			success: false;
+			/** The failure reason */
+			reason: string;
+			/** id of the node that failed during validation */
+			id: {
+				tag: TelemetryDataTag.CodeArtifact;
+				value: string | undefined;
+			};
+			/** If the error can be retried, time to wait before retrying */
+			retryAfterSeconds?: number;
+	  };
+
 export interface ISummarizerNodeRootContract {
 	startSummary(referenceSequenceNumber: number, summaryLogger: ITelemetryLoggerExt): void;
-	completeSummary(proposalHandle: string): void;
+	validateSummary(): ValidateSummaryResult;
+	completeSummary(proposalHandle: string, validate: boolean): void;
 	clearSummary(): void;
 	refreshLatestSummary(
 		proposalHandle: string | undefined,

package/src/summary/summarizerNode/summarizerNodeWithGc.ts

@@ -38,6 +38,7 @@ import {
 	ISummarizerNodeRootContract,
 	parseSummaryForSubtrees,
 	SummaryNode,
+	ValidateSummaryResult,
 } from "./summarizerNodeUtils";
 
 export interface IRootSummarizerNodeWithGC
@@ -47,7 +48,7 @@ export interface IRootSummarizerNodeWithGC
 // Extend SummaryNode to add used routes tracking to it.
 class SummaryNodeWithGC extends SummaryNode {
 	constructor(
-		public readonly serializedUsedRoutes: string,
+		public readonly serializedUsedRoutes: string | undefined,
 		summary: {
 			readonly referenceSequenceNumber: number;
 			readonly basePath: EscapedPath | undefined;
@@ -255,45 +256,86 @@ export class SummarizerNodeWithGC extends SummarizerNode implements IRootSummari
 		super.startSummary(referenceSequenceNumber, summaryLogger);
 	}
 
+	/**
+	 * Validates that the in-progress summary is correct for all nodes, i.e., GC should have run for non-skipped nodes.
+	 * @param parentSkipRecursion - true if the parent of this node skipped recursing the child nodes when running GC.
+	 * In that case, the children will not have work-in-progress state.
+	 *
+	 * @returns ValidateSummaryResult which contains a boolean success indicating whether the validation was successful.
+	 * In case of failure, additional information is returned indicating type of failure and where it was.
+	 */
+	protected validateSummaryCore(parentSkipRecursion: boolean): ValidateSummaryResult {
+		if (this.wasGCMissed()) {
+			return {
+				success: false,
+				reason: "NodeDidNotRunGC",
+				id: {
+					tag: TelemetryDataTag.CodeArtifact,
+					value: this.telemetryNodeId,
+				},
+				// These errors are usually transient and should go away when summarize / GC is retried.
+				retryAfterSeconds: 1,
+			};
+		}
+		return super.validateSummaryCore(parentSkipRecursion);
+	}
+
+	private wasGCMissed(): boolean {
+		// If GC is disabled, it should not have run so it was not missed.
+		// Otherwise, GC should have been called on this node and wipSerializedUsedRoutes must be set.
+		if (this.gcDisabled || this.wipSerializedUsedRoutes !== undefined) {
+			return false;
+		}
+		/**
+		 * The absence of wip used routes indicates that GC was not run on this node. This can happen if:
+		 * 1. A child node was created after GC was already run on the parent. For example, a data store
+		 * is realized (loaded) after GC was run on it creating summarizer nodes for its DDSes. In this
+		 * case, the parent will pass on used routes to the child nodes and it will have wip used routes.
+		 * 2. A new node was created but GC was never run on it. This can mean that the GC data generated
+		 * during summarize is incomplete.
+		 *
+		 * This happens due to scenarios such as data store created during summarize. Such errors should go away when
+		 * summarize is attempted again.
+		 */
+		return true;
+	}
+
 	/**
 	 * Called after summary has been uploaded to the server. Add the work-in-progress state to the pending
 	 * summary queue. We track this until we get an ack from the server for this summary.
+	 * @param proposalHandle - The handle of the summary that was uploaded to the server.
+	 * @param parentPath - The path of the parent node which is used to build the path of this node.
+	 * @param parentSkipRecursion - true if the parent of this node skipped recursing the child nodes when summarizing.
+	 * In that case, the children will not have work-in-progress state.
+	 * @param validate - true to validate that the in-progress summary is correct for all nodes.
 	 */
 	protected completeSummaryCore(
 		proposalHandle: string,
 		parentPath: EscapedPath | undefined,
 		parentSkipRecursion: boolean,
+		validate: boolean,
 	) {
+		if (validate && this.wasGCMissed()) {
+			this.throwUnexpectedError({
+				eventName: "NodeDidNotRunGC",
+				proposalHandle,
+			});
+		}
+
 		let wipSerializedUsedRoutes: string | undefined;
 		// If GC is disabled, don't set wip used routes.
 		if (!this.gcDisabled) {
 			wipSerializedUsedRoutes = this.wipSerializedUsedRoutes;
-			/**
-			 * The absence of wip used routes indicates that GC was not run on this node. This can happen if:
-			 * 1. A child node was created after GC was already run on the parent. For example, a data store
-			 * is realized (loaded) after GC was run on it creating summarizer nodes for its DDSes. In this
-			 * case, the used routes of the parent should be passed on the child nodes and it should be fine.
-			 * 2. A new node was created but GC was never run on it. This can mean that the GC data generated
-			 * during summarize is complete . We should not continue, log and throw an error. This will help us
-			 * identify these cases and take appropriate action.
-			 */
-			if (wipSerializedUsedRoutes === undefined) {
-				this.throwUnexpectedError({
-					eventName: "NodeDidNotRunGC",
-					proposalHandle,
-				});
-			}
 		}
 
-		super.completeSummaryCore(proposalHandle, parentPath, parentSkipRecursion);
+		super.completeSummaryCore(proposalHandle, parentPath, parentSkipRecursion, validate);
 
 		// If GC is disabled, skip setting pending summary with GC state.
 		if (!this.gcDisabled) {
 			const summaryNode = this.pendingSummaries.get(proposalHandle);
 			if (summaryNode !== undefined) {
 				const summaryNodeWithGC = new SummaryNodeWithGC(
-					// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-					wipSerializedUsedRoutes!,
+					wipSerializedUsedRoutes,
 					summaryNode,
 				);
 				this.pendingSummaries.set(proposalHandle, summaryNodeWithGC);

package/src/summary/summarizerTypes.ts

@@ -213,6 +213,13 @@ export type SubmitSummaryResult =
 	| IUploadSummaryResult
 	| ISubmitSummaryOpResult;
 
+/** The stages of Summarize, used to describe how far progress succeeded in case of a failure at a later stage. */
+export type SummaryStage = SubmitSummaryResult["stage"] | "unknown";
+/** The data in summarizer result when submit summary stage fails. */
+export interface SubmitSummaryFailureData {
+	stage: SummaryStage;
+}
+
 export interface IBroadcastSummaryResult {
 	readonly summarizeOp: ISummaryOpMessage;
 	readonly broadcastDuration: number;
@@ -243,7 +250,9 @@ export type SummarizeResultPart<TSuccess, TFailure = undefined> =
 
 export interface ISummarizeResults {
 	/** Resolves when we generate, upload, and submit the summary. */
-	readonly summarySubmitted: Promise<SummarizeResultPart<SubmitSummaryResult>>;
+	readonly summarySubmitted: Promise<
+		SummarizeResultPart<SubmitSummaryResult, SubmitSummaryFailureData>
+	>;
 	/** Resolves when we observe our summarize op broadcast. */
 	readonly summaryOpBroadcasted: Promise<SummarizeResultPart<IBroadcastSummaryResult>>;
 	/** Resolves when we receive a summaryAck or summaryNack. */

package/src/summary/summaryCollection.ts

@@ -3,7 +3,8 @@
  * Licensed under the MIT License.
  */
 
-import { IDisposable, IEvent } from "@fluidframework/common-definitions";
+import { IEvent } from "@fluidframework/common-definitions";
+import { IDisposable } from "@fluidframework/core-interfaces";
 import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
 import { Deferred, assert, TypedEventEmitter } from "@fluidframework/common-utils";
 import { IDeltaManager } from "@fluidframework/container-definitions";

package/src/summary/summaryFormat.ts

@@ -5,7 +5,10 @@
 
 import { assert } from "@fluidframework/common-utils";
 import { IDocumentStorageService } from "@fluidframework/driver-definitions";
-import { readAndParse } from "@fluidframework/driver-utils";
+import {
+	readAndParse,
+	blobHeadersBlobName as blobNameForBlobHeaders,
+} from "@fluidframework/driver-utils";
 import {
 	ISequencedDocumentMessage,
 	ISnapshotTree,
@@ -153,6 +156,7 @@ export const chunksBlobName = ".chunks";
 export const electedSummarizerBlobName = ".electedSummarizer";
 export const blobsTreeName = ".blobs";
 export const idCompressorBlobName = ".idCompressor";
+export const blobHeadersBlobName = blobNameForBlobHeaders;
 
 export function rootHasIsolatedChannels(metadata?: IContainerRuntimeMetadata): boolean {
 	return !!metadata && !metadata.disableIsolatedChannels;

package/src/summary/summaryGenerator.ts

@@ -9,6 +9,8 @@ import {
 	LoggingError,
 	ChildLogger,
 } from "@fluidframework/telemetry-utils";
+import { ITelemetryProperties } from "@fluidframework/common-definitions";
+
 import {
 	assert,
 	Deferred,
@@ -32,6 +34,8 @@ import {
 	ISummaryCancellationToken,
 	ISummarizeTelemetryProperties,
 	SummaryGeneratorTelemetry,
+	SummaryStage,
+	SubmitSummaryFailureData,
 } from "./summarizerTypes";
 import { IClientSummaryWatcher } from "./summaryCollection";
 
@@ -129,7 +133,9 @@ export const getFailMessage = (errorCode: keyof typeof summarizeErrors) =>
 	`${errorCode}: ${summarizeErrors[errorCode]}`;
 
 export class SummarizeResultBuilder {
-	public readonly summarySubmitted = new Deferred<SummarizeResultPart<SubmitSummaryResult>>();
+	public readonly summarySubmitted = new Deferred<
+		SummarizeResultPart<SubmitSummaryResult, SubmitSummaryFailureData>
+	>();
 	public readonly summaryOpBroadcasted = new Deferred<
 		SummarizeResultPart<IBroadcastSummaryResult>
 	>();
@@ -140,6 +146,7 @@ export class SummarizeResultBuilder {
 	public fail(
 		message: string,
 		error: any,
+		stage: SummaryStage = "unknown",
 		nackSummaryResult?: INackSummaryResult,
 		retryAfterSeconds?: number,
 	) {
@@ -155,7 +162,7 @@ export class SummarizeResultBuilder {
 			error,
 			retryAfterSeconds,
 		} as const;
-		this.summarySubmitted.resolve(result);
+		this.summarySubmitted.resolve({ ...result, data: { stage } });
 		this.summaryOpBroadcasted.resolve(result);
 		this.receivedSummaryAckOrNack.resolve({ ...result, data: nackSummaryResult });
 	}
@@ -168,6 +175,20 @@ export class SummarizeResultBuilder {
 	}
 }
 
+/**
+ * Errors type for errors hit during summary that may be retriable.
+ */
+export class RetriableSummaryError extends LoggingError {
+	public readonly canRetry = this.retryAfterSeconds !== undefined;
+	constructor(
+		message: string,
+		public readonly retryAfterSeconds?: number,
+		props?: ITelemetryProperties,
+	) {
+		super(message, props);
+	}
+}
+
 /**
  * This class generates and tracks a summary attempt.
  */
@@ -242,6 +263,7 @@ export class SummaryGenerator {
 			{ start: true, end: true, cancel: "generic" },
 		);
 
+		let summaryData: SubmitSummaryResult | undefined;
 		const fail = (
 			errorCode: keyof typeof summarizeErrors,
 			error?: any,
@@ -270,14 +292,15 @@ export class SummaryGenerator {
 				},
 				error ?? reason,
 			); // disconnect & summaryAckTimeout do not have proper error.
-			resultsBuilder.fail(reason, error, nackSummaryResult, retryAfterSeconds);
+
+			// If summarize did not hit an unexpected error, summaryData would be available. Otherwise, the state is
+			// unknown.
+			const stage = summaryData?.stage ?? "unknown";
+			resultsBuilder.fail(reason, error, stage, nackSummaryResult, retryAfterSeconds);
 		};
 
 		// Wait to generate and send summary
 		this.summarizeTimer.start();
-
-		// Use record type to prevent unexpected value types
-		let summaryData: SubmitSummaryResult | undefined;
 		try {
 			// Need to save refSeqNum before we record new attempt (happens as part of submitSummaryCallback)
 			const lastAttemptRefSeqNum = this.heuristicData.lastAttempt.refSequenceNumber;
@@ -359,10 +382,10 @@ export class SummaryGenerator {
 				cancellationToken,
 			);
 			if (waitBroadcastResult.result === "cancelled") {
-				return fail("disconnect");
+				return fail("disconnect", summaryData.stage);
 			}
 			if (waitBroadcastResult.result !== "done") {
-				return fail("summaryOpWaitTimeout");
+				return fail("summaryOpWaitTimeout", summaryData.stage);
 			}
 			const summarizeOp = waitBroadcastResult.value;
 

package/src/summary/summaryManager.ts

@@ -3,7 +3,8 @@
  * Licensed under the MIT License.
  */
 
-import { IDisposable, IEvent, IEventProvider } from "@fluidframework/common-definitions";
+import { IEvent, IEventProvider } from "@fluidframework/common-definitions";
+import { IDisposable } from "@fluidframework/core-interfaces";
 import { assert } from "@fluidframework/common-utils";
 import {
 	ChildLogger,