@fluidframework/container-runtime 2.0.0-dev.3.1.0.125672 → 2.0.0-dev.4.2.0.153917

package/src/{runningSummarizer.ts → summary/runningSummarizer.ts}

@@ -6,11 +6,17 @@
 import { IDisposable, ITelemetryLogger } from "@fluidframework/common-definitions";
 import { assert, delay, Deferred, PromiseTimer } from "@fluidframework/common-utils";
 import { UsageError } from "@fluidframework/container-utils";
+import { DriverErrorType } from "@fluidframework/driver-definitions";
 import { isRuntimeMessage } from "@fluidframework/driver-utils";
 import { ISequencedDocumentMessage, MessageType } from "@fluidframework/protocol-definitions";
-import { ChildLogger } from "@fluidframework/telemetry-utils";
-import { ISummaryConfiguration } from "./containerRuntime";
-import { opSize } from "./opProperties";
+import {
+	ChildLogger,
+	isFluidError,
+	loggerToMonitoringContext,
+	MonitoringContext,
+} from "@fluidframework/telemetry-utils";
+import { ISummaryConfiguration } from "../containerRuntime";
+import { opSize } from "../opProperties";
 import { SummarizeHeuristicRunner } from "./summarizerHeuristics";
 import {
 	IEnqueueSummarizeOptions,
@@ -27,8 +33,9 @@ import {
 	ISummarizeTelemetryProperties,
 	ISummarizerRuntime,
 	ISummarizeRunnerTelemetry,
+	IRefreshSummaryAckOptions,
 } from "./summarizerTypes";
-import { IClientSummaryWatcher, SummaryCollection } from "./summaryCollection";
+import { IAckedSummary, IClientSummaryWatcher, SummaryCollection } from "./summaryCollection";
 import {
 	raceTimer,
 	SummarizeReason,
@@ -38,6 +45,9 @@ import {
 
 const maxSummarizeAckWaitTime = 10 * 60 * 1000; // 10 minutes
 
+const defaultNumberSummarizationAttempts = 2; // only up to 2 attempts
+const numberOfAttemptsOnRestartAsRecovery = 1; // Only summarize once
+
 /**
  * An instance of RunningSummarizer manages the heuristics for summarizing.
  * Until disposed, the instance of RunningSummarizer can assume that it is
@@ -51,8 +61,8 @@ export class RunningSummarizer implements IDisposable {
 		summaryWatcher: IClientSummaryWatcher,
 		configuration: ISummaryConfiguration,
 		submitSummaryCallback: (options: ISubmitSummaryOptions) => Promise<SubmitSummaryResult>,
+		refreshLatestSummaryAckCallback: (options: IRefreshSummaryAckOptions) => Promise<void>,
 		heuristicData: ISummarizeHeuristicData,
-		raiseSummarizingError: (errorMessage: string) => void,
 		summaryCollection: SummaryCollection,
 		cancellationToken: ISummaryCancellationToken,
 		stopSummarizerCallback: (reason: SummarizerStopReason) => void,
@@ -63,16 +73,26 @@ export class RunningSummarizer implements IDisposable {
 			summaryWatcher,
 			configuration,
 			submitSummaryCallback,
+			refreshLatestSummaryAckCallback,
 			heuristicData,
-			raiseSummarizingError,
 			summaryCollection,
 			cancellationToken,
 			stopSummarizerCallback,
 			runtime,
 		);
 
+		// Before doing any heuristics or proceeding with its refreshing, if there is a summary ack received while
+		// this summarizer catches up, let's refresh state before proceeding with the summarization.
+		const lastAckRefSeq = await summarizer.handleSummaryAck();
+
 		await summarizer.waitStart();
 
+		// Handle summary acks asynchronously
+		// Note: no exceptions are thrown from processIncomingSummaryAcks handler as it handles all exceptions
+		summarizer.processIncomingSummaryAcks(lastAckRefSeq).catch((error) => {
+			logger.sendErrorEvent({ eventName: "HandleSummaryAckFatalError" }, error);
+		});
+
 		// Update heuristic counts
 		// By the time we get here, there are potentially ops missing from the heuristic summary counts
 		// Examples of where this could happen:
@@ -108,12 +128,13 @@ export class RunningSummarizer implements IDisposable {
 	private stopping = false;
 	private _disposed = false;
 	private summarizingLock: Promise<void> | undefined;
-	private refreshSummaryAckLock: Promise<void> | undefined;
 	private tryWhileSummarizing = false;
 	private readonly pendingAckTimer: PromiseTimer;
 	private heuristicRunner?: ISummarizeHeuristicRunner;
 	private readonly generator: SummaryGenerator;
-	private readonly logger: ITelemetryLogger;
+	private readonly mc: MonitoringContext;
+	private readonly shouldAbortOnSummaryFailure: boolean;
+
 	private enqueuedSummary:
 		| {
 				reason: SummarizeReason;
@@ -126,6 +147,9 @@ export class RunningSummarizer implements IDisposable {
 	private totalSuccessfulAttempts = 0;
 	private initialized = false;
 
+	private readonly deltaManagerListener;
+	private readonly runtimeListener;
+
 	private constructor(
 		baseLogger: ITelemetryLogger,
 		private readonly summaryWatcher: IClientSummaryWatcher,
@@ -133,8 +157,10 @@ export class RunningSummarizer implements IDisposable {
 		private readonly submitSummaryCallback: (
 			options: ISubmitSummaryOptions,
 		) => Promise<SubmitSummaryResult>,
+		private readonly refreshLatestSummaryAckCallback: (
+			options: IRefreshSummaryAckOptions,
+		) => Promise<void>,
 		private readonly heuristicData: ISummarizeHeuristicData,
-		private readonly raiseSummarizingError: (errorMessage: string) => void,
 		private readonly summaryCollection: SummaryCollection,
 		private readonly cancellationToken: ISummaryCancellationToken,
 		private readonly stopSummarizerCallback: (reason: SummarizerStopReason) => void,
@@ -145,9 +171,15 @@ export class RunningSummarizer implements IDisposable {
 			summarizerSuccessfulAttempts: () => this.totalSuccessfulAttempts,
 		};
 
-		this.logger = ChildLogger.create(baseLogger, "Running", {
-			all: telemetryProps,
-		});
+		this.mc = loggerToMonitoringContext(
+			ChildLogger.create(baseLogger, "Running", {
+				all: telemetryProps,
+			}),
+		);
+
+		this.shouldAbortOnSummaryFailure =
+			this.mc.config.getString("Fluid.ContainerRuntime.Test.SummarizationRecoveryMethod") ===
+			"restart";
 
 		if (configuration.state !== "disableHeuristics") {
 			assert(
@@ -158,7 +190,7 @@ export class RunningSummarizer implements IDisposable {
 				heuristicData,
 				this.configuration,
 				(reason) => this.trySummarize(reason),
-				this.logger,
+				this.mc.logger,
 			);
 		}
 
@@ -173,13 +205,12 @@ export class RunningSummarizer implements IDisposable {
 		const maxAckWaitTime = Math.min(this.configuration.maxAckWaitTime, maxSummarizeAckWaitTime);
 
 		this.pendingAckTimer = new PromiseTimer(maxAckWaitTime, () => {
-			// pre-0.58 error message: summaryAckWaitTimeout
-			this.raiseSummarizingError("Pending summary ack not received in time");
 			// Note: summarizeCount (from ChildLogger definition) may be 0,
 			// since this code path is hit when RunningSummarizer first starts up,
 			// before this instance has kicked off a new summarize run.
-			this.logger.sendErrorEvent({
+			this.mc.logger.sendErrorEvent({
 				eventName: "SummaryAckWaitTimeout",
+				message: "Pending summary ack not received in time",
 				maxAckWaitTime,
 				referenceSequenceNumber: this.heuristicData.lastAttempt.refSequenceNumber,
 				summarySequenceNumber: this.heuristicData.lastAttempt.summarySequenceNumber,
@@ -189,7 +220,7 @@ export class RunningSummarizer implements IDisposable {
 		// Set up pending ack timeout by op timestamp differences for previous summaries.
 		summaryCollection.setPendingAckTimerTimeoutCallback(maxAckWaitTime, () => {
 			if (this.pendingAckTimer.hasTimer) {
-				this.logger.sendTelemetryEvent({
+				this.mc.logger.sendTelemetryEvent({
 					eventName: "MissingSummaryAckFoundByOps",
 					referenceSequenceNumber: this.heuristicData.lastAttempt.refSequenceNumber,
 					summarySequenceNumber: this.heuristicData.lastAttempt.summarySequenceNumber,
@@ -202,24 +233,127 @@ export class RunningSummarizer implements IDisposable {
 			this.pendingAckTimer,
 			this.heuristicData,
 			this.submitSummaryCallback,
-			this.raiseSummarizingError,
 			() => {
 				this.totalSuccessfulAttempts++;
 			},
 			this.summaryWatcher,
-			this.logger,
+			this.mc.logger,
 		);
 
-		// Listen for ops
-		this.runtime.deltaManager.on("op", (op) => {
-			this.handleOp(op);
-		});
+		// Listen to deltaManager for non-runtime ops
+		this.deltaManagerListener = (op) => {
+			if (!isRuntimeMessage(op)) {
+				this.handleOp(op, false);
+			}
+		};
+
+		// Listen to runtime for runtime ops
+		this.runtimeListener = (op, runtimeMessage) => {
+			if (runtimeMessage) {
+				this.handleOp(op, true);
+			}
+		};
+
+		// Purpose of listening to deltaManager is for back-compat
+		// Can remove and only listen to runtime once loader version is past 2.0.0-internal.1.2.0 (https://github.com/microsoft/FluidFramework/pull/11832)
+		// Tracked by AB#3883
+		this.runtime.deltaManager.on("op", this.deltaManagerListener);
+		this.runtime.on?.("op", this.runtimeListener);
+	}
+
+	private async handleSummaryAck(): Promise<number> {
+		const lastAck: IAckedSummary | undefined = this.summaryCollection.latestAck;
+		let refSequenceNumber = -1;
+		// In case we haven't received the lastestAck yet, just return.
+		if (lastAck !== undefined) {
+			refSequenceNumber = lastAck.summaryOp.referenceSequenceNumber;
+			const summaryLogger = this.tryGetCorrelatedLogger(refSequenceNumber) ?? this.mc.logger;
+			const summaryOpHandle = lastAck.summaryOp.contents.handle;
+			const summaryAckHandle = lastAck.summaryAck.contents.handle;
+			while (this.summarizingLock !== undefined) {
+				summaryLogger.sendTelemetryEvent({
+					eventName: "RefreshAttemptWithSummarizerRunning",
+					referenceSequenceNumber: refSequenceNumber,
+					proposalHandle: summaryOpHandle,
+					ackHandle: summaryAckHandle,
+				});
+				await this.summarizingLock;
+			}
+
+			// Make sure we block any summarizer from being executed/enqueued while
+			// executing the refreshLatestSummaryAck.
+			// https://dev.azure.com/fluidframework/internal/_workitems/edit/779
+			await this.lockedSummaryAction(
+				() => {},
+				async () =>
+					this.refreshLatestSummaryAckCallback({
+						proposalHandle: summaryOpHandle,
+						ackHandle: summaryAckHandle,
+						summaryRefSeq: refSequenceNumber,
+						summaryLogger,
+					}).catch(async (error) => {
+						// If the error is 404, so maybe the fetched version no longer exists on server. We just
+						// ignore this error in that case, as that means we will have another summaryAck for the
+						// latest version with which we will refresh the state. However in case of single commit
+						// summary, we might me missing a summary ack, so in that case we are still fine as the
+						// code in `submitSummary` function in container runtime, will refresh the latest state
+						// by calling `refreshLatestSummaryAckFromServer` and we will be fine.
+						const isIgnoredError =
+							isFluidError(error) &&
+							error.errorType === DriverErrorType.fileNotFoundOrAccessDeniedError;
+
+						summaryLogger.sendTelemetryEvent(
+							{
+								eventName: isIgnoredError
+									? "HandleSummaryAckErrorIgnored"
+									: "HandleLastSummaryAckError",
+								referenceSequenceNumber: refSequenceNumber,
+								proposalHandle: summaryOpHandle,
+								ackHandle: summaryAckHandle,
+							},
+							error,
+						);
+					}),
+				() => {},
+			);
+			refSequenceNumber++;
+		}
+		return refSequenceNumber;
+	}
+
+	/**
+	 * Responsible for receiving and processing all the summaryAcks.
+	 * In case there was a summary ack processed by the running summarizer before processIncomingSummaryAcks is called,
+	 * it will wait for the summary ack that is newer than the one indicated by the lastAckRefSeq.
+	 * @param lastAckRefSeq - Identifies the minimum reference sequence number the summarizer needs to wait for.
+	 * In case of a negative number, the summarizer will wait for ANY summary ack that is greater than the deltaManager's initial sequence number,
+	 * and, in case of a positive one, it will wait for a summary ack that is greater than this current reference sequence number.
+	 */
+	private async processIncomingSummaryAcks(lastAckRefSeq: number) {
+		let refSequenceNumber =
+			lastAckRefSeq > 0 ? lastAckRefSeq : this.runtime.deltaManager.initialSequenceNumber;
+		while (!this.disposed) {
+			const summaryLogger = this.tryGetCorrelatedLogger(refSequenceNumber) ?? this.mc.logger;
+
+			// Initialize ack with undefined if exception happens inside of waitSummaryAck on second iteration,
+			// we record undefined, not previous handles.
+			await this.summaryCollection.waitSummaryAck(refSequenceNumber);
+
+			summaryLogger.sendTelemetryEvent({
+				eventName: "processIncomingSummaryAcks",
+				referenceSequenceNumber: refSequenceNumber,
+				lastAckRefSeq,
+			});
+
+			refSequenceNumber = await this.handleSummaryAck();
+			// A valid Summary Ack must have been processed.
+			assert(refSequenceNumber >= 0, 0x58f /* Invalid ref sequence number */);
+		}
 	}
 
 	public dispose(): void {
-		this.runtime.deltaManager.off("op", (op) => {
-			this.handleOp(op);
-		});
+		this.runtime.deltaManager.off("op", this.deltaManagerListener);
+		this.runtime.off?.("op", this.runtimeListener);
 		this.summaryWatcher.dispose();
 		this.heuristicRunner?.dispose();
 		this.heuristicRunner = undefined;
@@ -238,16 +372,16 @@ export class RunningSummarizer implements IDisposable {
 	 */
 	public tryGetCorrelatedLogger = (summaryOpRefSeq) =>
 		this.heuristicData.lastAttempt.refSequenceNumber === summaryOpRefSeq
-			? this.logger
+			? this.mc.logger
 			: undefined;
 
 	/** We only want a single heuristic runner micro-task (will provide better optimized grouping of ops) */
 	private heuristicRunnerMicroTaskExists = false;
 
-	public handleOp(op: ISequencedDocumentMessage) {
+	public handleOp(op: ISequencedDocumentMessage, runtimeMessage: boolean) {
 		this.heuristicData.lastOpSequenceNumber = op.sequenceNumber;
 
-		if (isRuntimeMessage(op)) {
+		if (runtimeMessage) {
 			this.heuristicData.numRuntimeOps++;
 		} else {
 			this.heuristicData.numNonRuntimeOps++;
@@ -258,7 +392,7 @@ export class RunningSummarizer implements IDisposable {
 		// Check for enqueued on-demand summaries; Intentionally do nothing otherwise
 		if (
 			this.initialized &&
-			this.opCanTriggerSummary(op) &&
+			this.opCanTriggerSummary(op, runtimeMessage) &&
 			!this.tryRunEnqueuedSummary() &&
 			!this.heuristicRunnerMicroTaskExists
 		) {
@@ -279,14 +413,14 @@ export class RunningSummarizer implements IDisposable {
 	 * @param op - op to check
 	 * @returns true if this op can trigger a summary
 	 */
-	private opCanTriggerSummary(op: ISequencedDocumentMessage): boolean {
+	private opCanTriggerSummary(op: ISequencedDocumentMessage, runtimeMessage: boolean): boolean {
 		switch (op.type) {
 			case MessageType.Summarize:
 			case MessageType.SummaryAck:
 			case MessageType.SummaryNack:
 				return false;
 			default:
-				return isRuntimeMessage(op) || this.nonRuntimeOpCanTriggerSummary();
+				return runtimeMessage || this.nonRuntimeOpCanTriggerSummary();
 		}
 	}
 
@@ -353,41 +487,35 @@ export class RunningSummarizer implements IDisposable {
 		this.initialized = true;
 	}
 
-	/**
-	 * Blocks a new summarizer from running in case RefreshSummaryAck is being processed.
-	 * Assumes that caller checked upfront for lack of concurrent action (this.refreshSummaryAckLock)
-	 * before calling this API. I.e. caller is responsible for either erroring out or waiting on this promise.
-	 * Note: The refreshSummaryAckLock makes sure no summarizer gets enqueued or processed
-	 * until the refresh has completed. One can't rely uniquely on the summarizingLock as the
-	 * refreshLatestSummaryAck also happens during the time summarizingLock !== undefined.
-	 * Ex. Summarizer submits a summay + op and then waits for the Summary Ack to proceed
-	 * with the refreshLatestSummaryAck and complete the summary.
-	 * @param action - action to perform.
-	 * @returns - result of action.
-	 */
-	public async lockedRefreshSummaryAckAction<T>(action: () => Promise<T>) {
-		assert(
-			this.refreshSummaryAckLock === undefined,
-			0x396 /* Refresh Summary Ack - Caller is responsible for checking lock */,
-		);
+	private beforeSummaryAction() {
+		this.summarizeCount++;
+	}
 
-		const refreshSummaryAckLock = new Deferred<void>();
-		this.refreshSummaryAckLock = refreshSummaryAckLock.promise;
+	private afterSummaryAction() {
+		const retry = this.tryWhileSummarizing;
+		this.tryWhileSummarizing = false;
 
-		return action().finally(() => {
-			refreshSummaryAckLock.resolve();
-			this.refreshSummaryAckLock = undefined;
-		});
+		// After summarizing, we should check to see if we need to summarize again.
+		// Rerun the heuristics and check for enqueued summaries.
+		if (!this.stopping && !this.tryRunEnqueuedSummary() && retry) {
+			this.heuristicRunner?.run();
+		}
 	}
 
 	/**
 	 * Runs single summary action that prevents any other concurrent actions.
 	 * Assumes that caller checked upfront for lack of concurrent action (this.summarizingLock)
 	 * before calling this API. I.e. caller is responsible for either erroring out or waiting on this promise.
+	 * @param before - set of instructions to run before running the action.
 	 * @param action - action to perform.
+	 * @param after - set of instructions to run after running the action.
 	 * @returns - result of action.
 	 */
-	private async lockedSummaryAction<T>(action: () => Promise<T>) {
+	private async lockedSummaryAction<T>(
+		before: () => void,
+		action: () => Promise<T>,
+		after: () => void,
+	) {
 		assert(
 			this.summarizingLock === undefined,
 			0x25b /* "Caller is responsible for checking lock" */,
@@ -396,22 +524,12 @@ export class RunningSummarizer implements IDisposable {
 		const summarizingLock = new Deferred<void>();
 		this.summarizingLock = summarizingLock.promise;
 
-		this.summarizeCount++;
-		// Make sure the RefreshLatestSummaryAck is not being executed.
-		await this.refreshSummaryAckLock;
+		before();
 
 		return action().finally(() => {
 			summarizingLock.resolve();
 			this.summarizingLock = undefined;
-
-			const retry = this.tryWhileSummarizing;
-			this.tryWhileSummarizing = false;
-
-			// After summarizing, we should check to see if we need to summarize again.
-			// Rerun the heuristics and check for enqueued summaries.
-			if (!this.stopping && !this.tryRunEnqueuedSummary() && retry) {
-				this.heuristicRunner?.run();
-			}
+			after();
 		});
 	}
 
@@ -429,16 +547,24 @@ export class RunningSummarizer implements IDisposable {
 		cancellationToken = this.cancellationToken,
 		resultsBuilder = new SummarizeResultBuilder(),
 	): ISummarizeResults {
-		this.lockedSummaryAction(async () => {
-			const summarizeResult = this.generator.summarize(
-				summarizeProps,
-				options,
-				cancellationToken,
-				resultsBuilder,
-			);
-			// ensure we wait till the end of the process
-			return summarizeResult.receivedSummaryAckOrNack;
-		}).catch((error) => {
+		this.lockedSummaryAction(
+			() => {
+				this.beforeSummaryAction();
+			},
+			async () => {
+				const summarizeResult = this.generator.summarize(
+					summarizeProps,
+					options,
+					cancellationToken,
+					resultsBuilder,
+				);
+				// ensure we wait till the end of the process
+				return summarizeResult.receivedSummaryAckOrNack;
+			},
+			() => {
+				this.afterSummaryAction();
+			},
+		).catch((error) => {
 			// SummaryGenerator.summarize() does not throw exceptions - it converts them to failed result
 			// on resultsBuilder
 			// We do not care about exceptions on receivedSummaryAckOrNack - caller should check results
@@ -460,91 +586,127 @@ export class RunningSummarizer implements IDisposable {
 			return;
 		}
 
-		this.lockedSummaryAction(async () => {
-			const attempts: (ISummarizeOptions & { delaySeconds?: number })[] = [
-				{ refreshLatestAck: false, fullTree: false },
-				{ refreshLatestAck: true, fullTree: false },
-				{ refreshLatestAck: true, fullTree: false, delaySeconds: 2 * 60 },
-				{ refreshLatestAck: true, fullTree: true, delaySeconds: 10 * 60 },
-			];
-			let overrideDelaySeconds: number | undefined;
-			let summaryAttempts = 0;
-			let summaryAttemptsPerPhase = 0;
-
-			let lastResult: { message: string; error: any } | undefined;
-
-			for (let summaryAttemptPhase = 0; summaryAttemptPhase < attempts.length; ) {
-				if (this.cancellationToken.cancelled) {
-					return;
-				}
-
-				// We only want to attempt 1 summary when reason is "lastSummary"
-				if (++summaryAttempts > 1 && reason === "lastSummary") {
-					return;
-				}
-
-				summaryAttemptsPerPhase++;
-
-				const { delaySeconds: regularDelaySeconds = 0, ...options } =
-					attempts[summaryAttemptPhase];
-				const delaySeconds = overrideDelaySeconds ?? regularDelaySeconds;
-
-				const summarizeProps: ISummarizeTelemetryProperties = {
-					reason,
-					summaryAttempts,
-					summaryAttemptsPerPhase,
-					summaryAttemptPhase: summaryAttemptPhase + 1, // make everything 1-based
-					...options,
-				};
-
-				if (delaySeconds > 0) {
-					this.logger.sendPerformanceEvent({
-						eventName: "SummarizeAttemptDelay",
-						duration: delaySeconds,
-						summaryNackDelay: overrideDelaySeconds !== undefined,
-						...summarizeProps,
+		this.lockedSummaryAction(
+			() => {
+				this.beforeSummaryAction();
+			},
+			async () => {
+				const attempts: (ISummarizeOptions & { delaySeconds?: number })[] = [
+					{ refreshLatestAck: false, fullTree: false },
+					{ refreshLatestAck: true, fullTree: false },
+					{ refreshLatestAck: true, fullTree: false, delaySeconds: 2 * 60 },
+					{ refreshLatestAck: true, fullTree: true, delaySeconds: 10 * 60 },
+				];
+				let overrideDelaySeconds: number | undefined;
+				let summaryAttempts = 0;
+				let summaryAttemptsPerPhase = 0;
+				// Reducing the default number of attempts to defaultNumberofSummarizationAttempts.
+				let totalAttempts = this.shouldAbortOnSummaryFailure
+					? numberOfAttemptsOnRestartAsRecovery
+					: this.mc.config.getNumber("Fluid.Summarizer.Attempts") ??
+					  defaultNumberSummarizationAttempts;
+
+				if (totalAttempts > attempts.length) {
+					this.mc.logger.sendTelemetryEvent({
+						eventName: "InvalidSummarizerAttempts",
+						attempts: totalAttempts,
 					});
-					await delay(delaySeconds * 1000);
+					totalAttempts = defaultNumberSummarizationAttempts;
+				} else if (totalAttempts < 1) {
+					throw new UsageError("Invalid number of attempts.");
 				}
 
-				// Make sure the refresh Summary Ack is not being executed.
-				await this.refreshSummaryAckLock;
-
-				// Note: no need to account for cancellationToken.waitCancelled here, as
-				// this is accounted SummaryGenerator.summarizeCore that controls receivedSummaryAckOrNack.
-				const resultSummarize = this.generator.summarize(
-					summarizeProps,
-					options,
-					cancellationToken,
-				);
-				const result = await resultSummarize.receivedSummaryAckOrNack;
+				let lastResult: { message: string; error: any } | undefined;
+
+				for (let summaryAttemptPhase = 0; summaryAttemptPhase < totalAttempts; ) {
+					if (this.cancellationToken.cancelled) {
+						return;
+					}
+
+					// We only want to attempt 1 summary when reason is "lastSummary"
+					if (++summaryAttempts > 1 && reason === "lastSummary") {
+						return;
+					}
+
+					summaryAttemptsPerPhase++;
+
+					const { delaySeconds: regularDelaySeconds = 0, ...options } =
+						attempts[summaryAttemptPhase];
+
+					const summarizeProps: ISummarizeTelemetryProperties = {
+						reason,
+						summaryAttempts,
+						summaryAttemptsPerPhase,
+						summaryAttemptPhase: summaryAttemptPhase + 1, // make everything 1-based
+						...options,
+					};
+
+					// Note: no need to account for cancellationToken.waitCancelled here, as
+					// this is accounted SummaryGenerator.summarizeCore that controls receivedSummaryAckOrNack.
+					const resultSummarize = this.generator.summarize(
+						summarizeProps,
+						options,
+						cancellationToken,
+					);
+					const result = await resultSummarize.receivedSummaryAckOrNack;
+
+					if (result.success) {
+						return;
+					}
+
+					// Check for retryDelay that can come from summaryNack or upload summary flow.
+					// Retry the same step only once per retryAfter response.
+					overrideDelaySeconds = result.retryAfterSeconds;
+					if (overrideDelaySeconds === undefined || summaryAttemptsPerPhase > 1) {
+						summaryAttemptPhase++;
+						summaryAttemptsPerPhase = 0;
+					}
+					lastResult = result;
+
+					const delaySeconds = overrideDelaySeconds ?? regularDelaySeconds;
+
+					if (delaySeconds > 0) {
+						this.mc.logger.sendPerformanceEvent({
+							eventName: "SummarizeAttemptDelay",
+							duration: delaySeconds,
+							summaryNackDelay: overrideDelaySeconds !== undefined,
+							...summarizeProps,
+						});
+						await delay(delaySeconds * 1000);
+					}
+				}
 
-				if (result.success) {
+				if (this.shouldAbortOnSummaryFailure) {
+					this.mc.logger.sendTelemetryEvent(
+						{
+							eventName: "ClosingSummarizerOnSummaryStale",
+							reason,
+							message: lastResult?.message,
+						},
+						lastResult?.error,
+					);
+
+					this.stopSummarizerCallback("latestSummaryStateStale");
+					this.runtime.closeFn();
 					return;
 				}
-				// Check for retryDelay that can come from summaryNack or upload summary flow.
-				// Retry the same step only once per retryAfter response.
-				overrideDelaySeconds = result.retryAfterSeconds;
-				if (overrideDelaySeconds === undefined || summaryAttemptsPerPhase > 1) {
-					summaryAttemptPhase++;
-					summaryAttemptsPerPhase = 0;
-				}
-				lastResult = result;
-			}
-
-			// If all attempts failed, log error (with last attempt info) and close the summarizer container
-			this.logger.sendErrorEvent(
-				{
-					eventName: "FailToSummarize",
-					reason,
-					message: lastResult?.message,
-				},
-				lastResult?.error,
-			);
+				// If all attempts failed, log error (with last attempt info) and close the summarizer container
+				this.mc.logger.sendErrorEvent(
+					{
+						eventName: "FailToSummarize",
+						reason,
+						message: lastResult?.message,
+					},
+					lastResult?.error,
+				);
 
-			this.stopSummarizerCallback("failToSummarize");
-		}).catch((error) => {
-			this.logger.sendErrorEvent({ eventName: "UnexpectedSummarizeError" }, error);
+				this.stopSummarizerCallback("failToSummarize");
+			},
+			() => {
+				this.afterSummaryAction();
+			},
+		).catch((error) => {
+			this.mc.logger.sendErrorEvent({ eventName: "UnexpectedSummarizeError" }, error);
 		});
 	}