@fluidframework/container-runtime 2.0.0-dev.5.2.0.169897 → 2.0.0-dev.6.4.0.191258

package/lib/summary/runningSummarizer.js

@@ -2,28 +2,25 @@
  * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
  * Licensed under the MIT License.
  */
-var __rest = (this && this.__rest) || function (s, e) {
-    var t = {};
-    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
-        t[p] = s[p];
-    if (s != null && typeof Object.getOwnPropertySymbols === "function")
-        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
-            if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
-                t[p[i]] = s[p[i]];
-        }
-    return t;
-};
-import { ChildLogger, isFluidError, loggerToMonitoringContext, } from "@fluidframework/telemetry-utils";
-import { assert, delay, Deferred, PromiseTimer } from "@fluidframework/common-utils";
-import { UsageError } from "@fluidframework/container-utils";
-import { DriverErrorType } from "@fluidframework/driver-definitions";
-import { isRuntimeMessage } from "@fluidframework/driver-utils";
+import { isFluidError, createChildMonitoringContext, createChildLogger, UsageError, } from "@fluidframework/telemetry-utils";
+import { assert, delay, Deferred, PromiseTimer } from "@fluidframework/core-utils";
+import { TypedEventEmitter } from "@fluid-internal/client-utils";
+import { DriverErrorTypes } from "@fluidframework/driver-definitions";
 import { MessageType } from "@fluidframework/protocol-definitions";
 import { opSize } from "../opProperties";
 import { SummarizeHeuristicRunner } from "./summarizerHeuristics";
 import { raceTimer, SummarizeResultBuilder, SummaryGenerator, } from "./summaryGenerator";
 const maxSummarizeAckWaitTime = 10 * 60 * 1000; // 10 minutes
-const defaultNumberSummarizationAttempts = 2; // only up to 2 attempts
+/**
+ * The maximum number of summarization attempts that will be done by default in case of failures
+ * that can be retried.
+ */
+export const defaultMaxAttempts = 2;
+/**
+ * The default value for maximum number of summarization attempts that will be done for summarization failures where
+ * submit fails and the failure can be retried.
+ */
+export const defaultMaxAttemptsForSubmitFailures = 5;
 /**
  * An instance of RunningSummarizer manages the heuristics for summarizing.
  * Until disposed, the instance of RunningSummarizer can assume that it is
@@ -31,8 +28,9 @@ const defaultNumberSummarizationAttempts = 2; // only up to 2 attempts
  * track of summaries that it is generating as they are broadcast and acked/nacked.
  * This object is created and controlled by Summarizer object.
  */
-export class RunningSummarizer {
+export class RunningSummarizer extends TypedEventEmitter {
     constructor(baseLogger, summaryWatcher, configuration, submitSummaryCallback, refreshLatestSummaryAckCallback, heuristicData, summaryCollection, cancellationToken, stopSummarizerCallback, runtime) {
+        super();
         this.summaryWatcher = summaryWatcher;
         this.configuration = configuration;
         this.submitSummaryCallback = submitSummaryCallback;
@@ -63,9 +61,13 @@ export class RunningSummarizer {
             summarizeCount: () => this.summarizeCount,
             summarizerSuccessfulAttempts: () => this.totalSuccessfulAttempts,
         };
-        this.mc = loggerToMonitoringContext(ChildLogger.create(baseLogger, "Running", {
-            all: telemetryProps,
-        }));
+        this.mc = createChildMonitoringContext({
+            logger: baseLogger,
+            namespace: "Running",
+            properties: {
+                all: telemetryProps,
+            },
+        });
         if (configuration.state !== "disableHeuristics") {
             assert(this.configuration.state === "enabled", 0x2ea /* "Configuration state should be enabled" */);
             this.heuristicRunner = new SummarizeHeuristicRunner(heuristicData, this.configuration, (reason) => this.trySummarize(reason), this.mc.logger);
@@ -101,26 +103,21 @@ export class RunningSummarizer {
         this.generator = new SummaryGenerator(this.pendingAckTimer, this.heuristicData, this.submitSummaryCallback, () => {
             this.totalSuccessfulAttempts++;
         }, this.summaryWatcher, this.mc.logger);
-        // Listen to deltaManager for non-runtime ops
-        this.deltaManagerListener = (op) => {
-            if (!isRuntimeMessage(op)) {
-                this.handleOp(op, false);
-            }
-        };
-        // Listen to runtime for runtime ops
+        // Listen to runtime for ops
         this.runtimeListener = (op, runtimeMessage) => {
-            if (runtimeMessage) {
-                this.handleOp(op, true);
-            }
+            this.handleOp(op, runtimeMessage === true);
         };
-        // Purpose of listening to deltaManager is for back-compat
-        // Can remove and only listen to runtime once loader version is past 2.0.0-internal.1.2.0 (https://github.com/microsoft/FluidFramework/pull/11832)
-        // Tracked by AB#3883
-        this.runtime.deltaManager.on("op", this.deltaManagerListener);
         this.runtime.on("op", this.runtimeListener);
+        // The max attempts for submit failures can be overridden via a feature flag. This allows us to
+        // tweak this as per telemetry data until we arrive at a stable number.
+        // If its set to a number higher than `defaultMaxAttemptsForSubmitFailures`, it will be ignored.
+        const overrideMaxAttempts = this.mc.config.getNumber("Fluid.Summarizer.AttemptsForSubmitFailures");
+        this.maxAttemptsForSubmitFailures =
+            overrideMaxAttempts && overrideMaxAttempts < defaultMaxAttemptsForSubmitFailures
+                ? overrideMaxAttempts
+                : defaultMaxAttemptsForSubmitFailures;
     }
     static async start(logger, summaryWatcher, configuration, submitSummaryCallback, refreshLatestSummaryAckCallback, heuristicData, summaryCollection, cancellationToken, stopSummarizerCallback, runtime) {
-        var _a, _b;
         const summarizer = new RunningSummarizer(logger, summaryWatcher, configuration, submitSummaryCallback, refreshLatestSummaryAckCallback, heuristicData, summaryCollection, cancellationToken, stopSummarizerCallback, runtime);
         // Before doing any heuristics or proceeding with its refreshing, if there is a summary ack received while
         // this summarizer catches up, let's refresh state before proceeding with the summarization.
@@ -129,7 +126,7 @@ export class RunningSummarizer {
         // Handle summary acks asynchronously
         // Note: no exceptions are thrown from processIncomingSummaryAcks handler as it handles all exceptions
         summarizer.processIncomingSummaryAcks(lastAckRefSeq).catch((error) => {
-            logger.sendErrorEvent({ eventName: "HandleSummaryAckFatalError" }, error);
+            createChildLogger({ logger }).sendErrorEvent({ eventName: "HandleSummaryAckFatalError" }, error);
         });
         // Update heuristic counts
         // By the time we get here, there are potentially ops missing from the heuristic summary counts
@@ -150,21 +147,20 @@ export class RunningSummarizer {
         // Update last seq number (in case the handlers haven't processed anything yet)
         heuristicData.lastOpSequenceNumber = runtime.deltaManager.lastSequenceNumber;
         // Start heuristics
-        (_a = summarizer.heuristicRunner) === null || _a === void 0 ? void 0 : _a.start();
-        (_b = summarizer.heuristicRunner) === null || _b === void 0 ? void 0 : _b.run();
+        summarizer.heuristicRunner?.start();
+        summarizer.heuristicRunner?.run();
         return summarizer;
     }
     get disposed() {
         return this._disposed;
     }
     async handleSummaryAck() {
-        var _a;
         const lastAck = this.summaryCollection.latestAck;
         let refSequenceNumber = -1;
         // In case we haven't received the lastestAck yet, just return.
         if (lastAck !== undefined) {
             refSequenceNumber = lastAck.summaryOp.referenceSequenceNumber;
-            const summaryLogger = (_a = this.tryGetCorrelatedLogger(refSequenceNumber)) !== null && _a !== void 0 ? _a : this.mc.logger;
+            const summaryLogger = this.tryGetCorrelatedLogger(refSequenceNumber) ?? this.mc.logger;
             const summaryOpHandle = lastAck.summaryOp.contents.handle;
             const summaryAckHandle = lastAck.summaryAck.contents.handle;
             while (this.summarizingLock !== undefined) {
@@ -192,7 +188,7 @@ export class RunningSummarizer {
                 // code in `submitSummary` function in container runtime, will refresh the latest state
                 // by calling `refreshLatestSummaryAckFromServer` and we will be fine.
                 const isIgnoredError = isFluidError(error) &&
-                    error.errorType === DriverErrorType.fileNotFoundOrAccessDeniedError;
+                    error.errorType === DriverErrorTypes.fileNotFoundOrAccessDeniedError;
                 summaryLogger.sendTelemetryEvent({
                     eventName: isIgnoredError
                         ? "HandleSummaryAckErrorIgnored"
@@ -215,10 +211,9 @@ export class RunningSummarizer {
      * and, in case of a positive one, it will wait for a summary ack that is greater than this current reference sequence number.
      */
     async processIncomingSummaryAcks(lastAckRefSeq) {
-        var _a;
         let refSequenceNumber = lastAckRefSeq > 0 ? lastAckRefSeq : this.runtime.deltaManager.initialSequenceNumber;
         while (!this.disposed) {
-            const summaryLogger = (_a = this.tryGetCorrelatedLogger(refSequenceNumber)) !== null && _a !== void 0 ? _a : this.mc.logger;
+            const summaryLogger = this.tryGetCorrelatedLogger(refSequenceNumber) ?? this.mc.logger;
             // Initialize ack with undefined if exception happens inside of waitSummaryAck on second iteration,
             // we record undefined, not previous handles.
             await this.summaryCollection.waitSummaryAck(refSequenceNumber);
@@ -233,11 +228,9 @@ export class RunningSummarizer {
         }
     }
     dispose() {
-        var _a;
-        this.runtime.deltaManager.off("op", this.deltaManagerListener);
         this.runtime.off("op", this.runtimeListener);
         this.summaryWatcher.dispose();
-        (_a = this.heuristicRunner) === null || _a === void 0 ? void 0 : _a.dispose();
+        this.heuristicRunner?.dispose();
         this.heuristicRunner = undefined;
         this.generator.dispose();
         this.pendingAckTimer.clear();
@@ -262,8 +255,7 @@ export class RunningSummarizer {
             this.heuristicRunnerMicroTaskExists = true;
             Promise.resolve()
                 .then(() => {
-                var _a;
-                (_a = this.heuristicRunner) === null || _a === void 0 ? void 0 : _a.run();
+                this.heuristicRunner?.run();
             })
                 .finally(() => {
                 this.heuristicRunnerMicroTaskExists = false;
@@ -294,18 +286,17 @@ export class RunningSummarizer {
                 this.configuration.nonRuntimeHeuristicThreshold <= opsSinceLastAck));
     }
     async waitStop(allowLastSummary) {
-        var _a;
         if (this.stopping) {
             return;
         }
         this.stopping = true;
         this.disposeEnqueuedSummary();
         // This will try to run lastSummary if needed.
-        if (allowLastSummary && ((_a = this.heuristicRunner) === null || _a === void 0 ? void 0 : _a.shouldRunLastSummary())) {
+        if (allowLastSummary && this.heuristicRunner?.shouldRunLastSummary()) {
             if (this.summarizingLock === undefined) {
                 this.trySummarizeOnce(
                 // summarizeProps
-                { reason: "lastSummary" }, 
+                { summarizeReason: "lastSummary" }, 
                 // ISummarizeOptions, using defaults: { refreshLatestAck: false, fullTree: false }
                 {});
             }
@@ -338,13 +329,12 @@ export class RunningSummarizer {
         this.summarizeCount++;
     }
     afterSummaryAction() {
-        var _a;
         const retry = this.tryWhileSummarizing;
         this.tryWhileSummarizing = false;
         // After summarizing, we should check to see if we need to summarize again.
         // Rerun the heuristics and check for enqueued summaries.
         if (!this.stopping && !this.tryRunEnqueuedSummary() && retry) {
-            (_a = this.heuristicRunner) === null || _a === void 0 ? void 0 : _a.run();
+            this.heuristicRunner?.run();
         }
     }
     /**
@@ -354,7 +344,7 @@ export class RunningSummarizer {
      * @param before - set of instructions to run before running the action.
      * @param action - action to perform.
      * @param after - set of instructions to run after running the action.
-     * @returns - result of action.
+     * @returns The result of the action.
      */
     async lockedSummaryAction(before, action, after) {
         assert(this.summarizingLock === undefined, 0x25b /* "Caller is responsible for checking lock" */);
@@ -375,11 +365,20 @@ export class RunningSummarizer {
      * @param resultsBuilder - optional, result builder to use.
      * @returns ISummarizeResult - result of running a summary.
      */
-    trySummarizeOnce(summarizeProps, options, cancellationToken = this.cancellationToken, resultsBuilder = new SummarizeResultBuilder()) {
+    trySummarizeOnce(summarizeProps, options, resultsBuilder = new SummarizeResultBuilder()) {
         this.lockedSummaryAction(() => {
             this.beforeSummaryAction();
         }, async () => {
-            const summarizeResult = this.generator.summarize(summarizeProps, options, cancellationToken, resultsBuilder);
+            const summaryLogger = createChildLogger({
+                logger: this.mc.logger,
+                properties: { all: summarizeProps },
+            });
+            const summaryOptions = {
+                ...options,
+                summaryLogger,
+                cancellationToken: this.cancellationToken,
+            };
+            const summarizeResult = this.generator.summarize(summaryOptions, resultsBuilder);
             // ensure we wait till the end of the process
             return summarizeResult.receivedSummaryAckOrNack;
         }, () => {
@@ -393,7 +392,7 @@ export class RunningSummarizer {
         return resultsBuilder.build();
     }
     /** Heuristics summarize attempt. */
-    trySummarize(reason, cancellationToken = this.cancellationToken) {
+    trySummarize(reason) {
         if (this.summarizingLock !== undefined) {
             // lockedSummaryAction() will retry heuristic-based summary at the end of current attempt
             // if it's still needed
@@ -403,79 +402,207 @@ export class RunningSummarizer {
         this.lockedSummaryAction(() => {
             this.beforeSummaryAction();
         }, async () => {
-            var _a;
-            const attempts = [
-                { refreshLatestAck: false, fullTree: false },
-                { refreshLatestAck: true, fullTree: false },
-                { refreshLatestAck: true, fullTree: false, delaySeconds: 2 * 60 },
-                { refreshLatestAck: true, fullTree: true, delaySeconds: 10 * 60 },
-            ];
-            let overrideDelaySeconds;
-            let summaryAttempts = 0;
-            let summaryAttemptsPerPhase = 0;
-            // Reducing the default number of attempts to defaultNumberofSummarizationAttempts.
-            let totalAttempts = (_a = this.mc.config.getNumber("Fluid.Summarizer.Attempts")) !== null && _a !== void 0 ? _a : defaultNumberSummarizationAttempts;
-            if (totalAttempts > attempts.length) {
-                this.mc.logger.sendTelemetryEvent({
-                    eventName: "InvalidSummarizerAttempts",
-                    attempts: totalAttempts,
-                });
-                totalAttempts = defaultNumberSummarizationAttempts;
-            }
-            else if (totalAttempts < 1) {
-                throw new UsageError("Invalid number of attempts.");
-            }
-            let lastResult;
-            for (let summaryAttemptPhase = 0; summaryAttemptPhase < totalAttempts;) {
-                if (this.cancellationToken.cancelled) {
-                    return;
-                }
-                // We only want to attempt 1 summary when reason is "lastSummary"
-                if (++summaryAttempts > 1 && reason === "lastSummary") {
-                    return;
-                }
-                summaryAttemptsPerPhase++;
-                const _b = attempts[summaryAttemptPhase], { delaySeconds: regularDelaySeconds = 0 } = _b, options = __rest(_b, ["delaySeconds"]);
-                const summarizeProps = Object.assign({ reason,
-                    summaryAttempts,
-                    summaryAttemptsPerPhase, summaryAttemptPhase: summaryAttemptPhase + 1 }, options);
-                // Note: no need to account for cancellationToken.waitCancelled here, as
-                // this is accounted SummaryGenerator.summarizeCore that controls receivedSummaryAckOrNack.
-                const resultSummarize = this.generator.summarize(summarizeProps, options, cancellationToken);
-                const result = await resultSummarize.receivedSummaryAckOrNack;
-                if (result.success) {
-                    return;
-                }
-                // Check for retryDelay that can come from summaryNack or upload summary flow.
-                // Retry the same step only once per retryAfter response.
-                overrideDelaySeconds = result.retryAfterSeconds;
-                if (overrideDelaySeconds === undefined || summaryAttemptsPerPhase > 1) {
-                    summaryAttemptPhase++;
-                    summaryAttemptsPerPhase = 0;
-                }
-                lastResult = result;
-                const delaySeconds = overrideDelaySeconds !== null && overrideDelaySeconds !== void 0 ? overrideDelaySeconds : regularDelaySeconds;
-                if (delaySeconds > 0) {
-                    this.mc.logger.sendPerformanceEvent(Object.assign({ eventName: "SummarizeAttemptDelay", duration: delaySeconds, summaryNackDelay: overrideDelaySeconds !== undefined }, summarizeProps));
-                    await delay(delaySeconds * 1000);
-                }
-            }
-            // If all attempts failed, log error (with last attempt info) and close the summarizer container
-            this.mc.logger.sendErrorEvent({
-                eventName: "FailToSummarize",
-                reason,
-                message: lastResult === null || lastResult === void 0 ? void 0 : lastResult.message,
-            }, lastResult === null || lastResult === void 0 ? void 0 : lastResult.error);
-            this.stopSummarizerCallback("failToSummarize");
+            return this.mc.config.getBoolean("Fluid.Summarizer.UseDynamicRetries")
+                ? this.trySummarizeWithRetries(reason)
+                : this.trySummarizeWithStaticAttempts(reason);
         }, () => {
             this.afterSummaryAction();
         }).catch((error) => {
             this.mc.logger.sendErrorEvent({ eventName: "UnexpectedSummarizeError" }, error);
         });
     }
+    /**
+     * Tries to summarize 2 times with pre-defined summary options. If an attempt fails with "retryAfterSeconds"
+     * param, that attempt is tried once more.
+     */
+    async trySummarizeWithStaticAttempts(reason) {
+        const attemptOptions = [
+            { refreshLatestAck: false, fullTree: false },
+            { refreshLatestAck: true, fullTree: false },
+        ];
+        let summaryAttempts = 0;
+        let summaryAttemptsPerPhase = 0;
+        let summaryAttemptPhase = 0;
+        while (summaryAttemptPhase < attemptOptions.length) {
+            if (this.cancellationToken.cancelled) {
+                return;
+            }
+            // We only want to attempt 1 summary when reason is "lastSummary"
+            if (++summaryAttempts > 1 && reason === "lastSummary") {
+                return;
+            }
+            summaryAttemptsPerPhase++;
+            const summarizeOptions = attemptOptions[summaryAttemptPhase];
+            const summarizeProps = {
+                summarizeReason: reason,
+                summaryAttempts,
+                summaryAttemptsPerPhase,
+                summaryAttemptPhase: summaryAttemptPhase + 1,
+                ...summarizeOptions,
+            };
+            const summaryLogger = createChildLogger({
+                logger: this.mc.logger,
+                properties: { all: summarizeProps },
+            });
+            const summaryOptions = {
+                ...summarizeOptions,
+                summaryLogger,
+                cancellationToken: this.cancellationToken,
+            };
+            // Note: no need to account for cancellationToken.waitCancelled here, as
+            // this is accounted SummaryGenerator.summarizeCore that controls receivedSummaryAckOrNack.
+            const resultSummarize = this.generator.summarize(summaryOptions);
+            const ackNackResult = await resultSummarize.receivedSummaryAckOrNack;
+            if (ackNackResult.success) {
+                return;
+            }
+            // Check for retryDelay that can come from summaryNack, upload summary or submit summary flows.
+            // Retry the same step only once per retryAfter response.
+            const submitResult = await resultSummarize.summarySubmitted;
+            const delaySeconds = !submitResult.success
+                ? submitResult.data?.retryAfterSeconds
+                : ackNackResult.data?.retryAfterSeconds;
+            if (delaySeconds === undefined || summaryAttemptsPerPhase > 1) {
+                summaryAttemptPhase++;
+                summaryAttemptsPerPhase = 0;
+            }
+            if (delaySeconds !== undefined) {
+                this.mc.logger.sendPerformanceEvent({
+                    eventName: "SummarizeAttemptDelay",
+                    duration: delaySeconds,
+                    summaryNackDelay: ackNackResult.data?.retryAfterSeconds !== undefined,
+                    ...summarizeProps,
+                });
+                await delay(delaySeconds * 1000);
+            }
+        }
+        this.stopSummarizerCallback("failToSummarize");
+    }
+    /**
+     * Tries to summarize with retries where retry is based on the failure params.
+     * For example, summarization may be retried for failures with "retryAfterSeconds" param.
+     */
+    async trySummarizeWithRetries(reason) {
+        // Helper to set summarize options, telemetry properties and call summarize.
+        const attemptSummarize = (attemptNumber, finalAttempt) => {
+            const summarizeOptions = {
+                fullTree: false,
+            };
+            const summarizeProps = {
+                summarizeReason: reason,
+                summaryAttempts: attemptNumber,
+                ...summarizeOptions,
+                finalAttempt,
+            };
+            const summaryLogger = createChildLogger({
+                logger: this.mc.logger,
+                properties: { all: summarizeProps },
+            });
+            const summaryOptions = {
+                ...summarizeOptions,
+                summaryLogger,
+                cancellationToken: this.cancellationToken,
+                finalAttempt,
+            };
+            const summarizeResult = this.generator.summarize(summaryOptions);
+            return { summarizeProps, summarizeResult };
+        };
+        // The max number of attempts are based on the stage at which summarization failed. If it fails before it is
+        // submitted, a different value is used compared to if it fails after submission. Usually, in the former case,
+        // we would retry more often as its cheaper and retries are likely to succeed.
+        // This makes it harder to predict how many attempts would actually happen as that depends on how far an attempt
+        // made. To keep things simple, the max attempts is reset after every attempt based on where it failed. This may
+        // result in some failures not being retried depending on what happened before this attempt. That's fine because
+        // such scenarios are very unlikely and even if it happens, it would resolve when a new summarizer starts over.
+        // For example - When failure switches from one the submit failures to nack failure, only one more retry will
+        // happen irrespective of the value of `defaultMaxAttempts`.
+        let maxAttempts = defaultMaxAttempts;
+        let currentAttempt = 0;
+        let retryAfterSeconds;
+        let done = false;
+        let result = "success";
+        do {
+            currentAttempt++;
+            if (this.cancellationToken.cancelled) {
+                result = "canceled";
+                done = true;
+                break;
+            }
+            const { summarizeProps, summarizeResult } = attemptSummarize(currentAttempt, false /* finalAttempt */);
+            // Ack / nack is the final step, so if it succeeds we're done.
+            const ackNackResult = await summarizeResult.receivedSummaryAckOrNack;
+            if (ackNackResult.success) {
+                result = "success";
+                done = true;
+                break;
+            }
+            // Update max attempts and retry params from the failure result.
+            // If submit summary failed, use the params from "summarySubmitted" result. Else, use the params
+            // from "receivedSummaryAckOrNack" result.
+            // Note: Check "summarySubmitted" result first because if it fails, ack nack would fail as well.
+            const submitSummaryResult = await summarizeResult.summarySubmitted;
+            if (!submitSummaryResult.success) {
+                maxAttempts = this.maxAttemptsForSubmitFailures;
+                retryAfterSeconds = submitSummaryResult.data?.retryAfterSeconds;
+            }
+            else {
+                maxAttempts = defaultMaxAttempts;
+                retryAfterSeconds = ackNackResult.data?.retryAfterSeconds;
+            }
+            // Emit "summarize" event for this failed attempt.
+            result = "failure";
+            const eventProps = {
+                result,
+                currentAttempt,
+                maxAttempts,
+                error: ackNackResult.error,
+            };
+            this.emit("summarize", eventProps);
+            // If the failure doesn't have "retryAfterSeconds" or the max number of attempts have been done, we're done.
+            if (retryAfterSeconds === undefined || currentAttempt >= maxAttempts - 1) {
+                done = true;
+            }
+            // If the failure has "retryAfterSeconds", add a delay of that time. In this case, a final attempt will
+            // take place and we need to wait for "retryAfterSeconds" before that.
+            if (retryAfterSeconds !== undefined) {
+                this.mc.logger.sendPerformanceEvent({
+                    eventName: "SummarizeAttemptDelay",
+                    duration: retryAfterSeconds,
+                    summaryNackDelay: ackNackResult.data?.retryAfterSeconds !== undefined,
+                    stage: submitSummaryResult.data?.stage,
+                    dynamicRetries: true,
+                    ...summarizeProps,
+                });
+                await delay(retryAfterSeconds * 1000);
+            }
+        } while (!done);
+        // If summarize attempt did not fail, emit "summarize" event and return. A failed attempt may be retried below.
+        if (result !== "failure") {
+            this.emit("summarize", { result, currentAttempt, maxAttempts });
+            return;
+        }
+        // If summarization wasn't successful above and the failure contains "retryAfterSeconds", perform one last
+        // attempt. This gives a chance to the runtime to perform additional steps in the last attempt.
+        if (retryAfterSeconds !== undefined) {
+            const { summarizeResult } = attemptSummarize(++currentAttempt, true /* finalAttempt */);
+            // Ack / nack is the final step, so if it succeeds we're done.
+            const ackNackResult = await summarizeResult.receivedSummaryAckOrNack;
+            result = ackNackResult.success ? "success" : "failure";
+            const eventProps = {
+                result,
+                currentAttempt,
+                maxAttempts,
+                error: ackNackResult.success ? undefined : ackNackResult.error,
+            };
+            this.emit("summarize", eventProps);
+        }
+        // If summarization is still unsuccessful, stop the summarizer.
+        if (result === "failure") {
+            this.stopSummarizerCallback("failToSummarize");
+        }
+    }
     /** {@inheritdoc (ISummarizer:interface).summarizeOnDemand} */
-    summarizeOnDemand(resultsBuilder = new SummarizeResultBuilder(), _a) {
-        var { reason } = _a, options = __rest(_a, ["reason"]);
+    summarizeOnDemand(options, resultsBuilder = new SummarizeResultBuilder()) {
         if (this.stopping) {
             resultsBuilder.fail("RunningSummarizer stopped or disposed", undefined);
             return resultsBuilder.build();
@@ -486,13 +613,13 @@ export class RunningSummarizer {
             // The heuristics are blocking concurrent summarize attempts.
             throw new UsageError("Attempted to run an already-running summarizer on demand");
         }
-        const result = this.trySummarizeOnce({ reason: `onDemand/${reason}` }, options, this.cancellationToken, resultsBuilder);
+        const { reason, ...summarizeOptions } = options;
+        const result = this.trySummarizeOnce({ summarizeReason: `onDemand/${reason}` }, summarizeOptions, resultsBuilder);
         return result;
     }
     /** {@inheritdoc (ISummarizer:interface).enqueueSummarize} */
-    enqueueSummarize(_a) {
-        var { reason, afterSequenceNumber = 0, override = false } = _a, options = __rest(_a, ["reason", "afterSequenceNumber", "override"]);
-        const onDemandReason = `enqueue;${reason}`;
+    enqueueSummarize(options) {
+        const { reason, afterSequenceNumber = 0, override = false, ...summarizeOptions } = options;
         let overridden = false;
         if (this.enqueuedSummary !== undefined) {
             if (!override) {
@@ -504,15 +631,20 @@ export class RunningSummarizer {
             overridden = true;
         }
         this.enqueuedSummary = {
-            reason: onDemandReason,
+            reason: `enqueue;${reason}`,
             afterSequenceNumber,
-            options,
+            summarizeOptions,
             resultsBuilder: new SummarizeResultBuilder(),
         };
         const results = this.enqueuedSummary.resultsBuilder.build();
         this.tryRunEnqueuedSummary();
         return overridden
-            ? Object.assign(Object.assign({}, results), { alreadyEnqueued: true, overridden: true }) : results;
+            ? {
+                ...results,
+                alreadyEnqueued: true,
+                overridden: true,
+            }
+            : results;
     }
     tryRunEnqueuedSummary() {
         if (this.stopping) {
@@ -525,10 +657,10 @@ export class RunningSummarizer {
             // If no enqueued summary is ready or a summary is already in progress, take no action.
             return false;
         }
-        const { reason, resultsBuilder, options } = this.enqueuedSummary;
+        const { reason, resultsBuilder, summarizeOptions } = this.enqueuedSummary;
         // Set to undefined first, so that subsequent enqueue attempt while summarize will occur later.
         this.enqueuedSummary = undefined;
-        this.trySummarizeOnce({ reason: `enqueuedSummary/${reason}` }, options, this.cancellationToken, resultsBuilder);
+        this.trySummarizeOnce({ summarizeReason: `enqueuedSummary/${reason}` }, summarizeOptions, resultsBuilder);
         return true;
     }
     disposeEnqueuedSummary() {