@fluidframework/container-runtime 2.0.0-internal.6.1.1 → 2.0.0-internal.6.3.0

package/dist/summary/runningSummarizer.js

@@ -4,16 +4,26 @@
  * Licensed under the MIT License.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.RunningSummarizer = void 0;
+exports.RunningSummarizer = exports.defaultMaxAttemptsForSubmitFailures = exports.defaultMaxAttempts = void 0;
 const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
-const common_utils_1 = require("@fluidframework/common-utils");
-const container_utils_1 = require("@fluidframework/container-utils");
+const core_utils_1 = require("@fluidframework/core-utils");
+const client_utils_1 = require("@fluid-internal/client-utils");
 const driver_definitions_1 = require("@fluidframework/driver-definitions");
 const protocol_definitions_1 = require("@fluidframework/protocol-definitions");
 const opProperties_1 = require("../opProperties");
 const summarizerHeuristics_1 = require("./summarizerHeuristics");
 const summaryGenerator_1 = require("./summaryGenerator");
 const maxSummarizeAckWaitTime = 10 * 60 * 1000; // 10 minutes
+/**
+ * The maximum number of summarization attempts that will be done by default in case of failures
+ * that can be retried.
+ */
+exports.defaultMaxAttempts = 2;
+/**
+ * The default value for maximum number of summarization attempts that will be done for summarization failures where
+ * submit fails and the failure can be retried.
+ */
+exports.defaultMaxAttemptsForSubmitFailures = 5;
 /**
  * An instance of RunningSummarizer manages the heuristics for summarizing.
  * Until disposed, the instance of RunningSummarizer can assume that it is
@@ -21,8 +31,9 @@ const maxSummarizeAckWaitTime = 10 * 60 * 1000; // 10 minutes
  * track of summaries that it is generating as they are broadcast and acked/nacked.
  * This object is created and controlled by Summarizer object.
  */
-class RunningSummarizer {
+class RunningSummarizer extends client_utils_1.TypedEventEmitter {
     constructor(baseLogger, summaryWatcher, configuration, submitSummaryCallback, refreshLatestSummaryAckCallback, heuristicData, summaryCollection, cancellationToken, stopSummarizerCallback, runtime) {
+        super();
         this.summaryWatcher = summaryWatcher;
         this.configuration = configuration;
         this.submitSummaryCallback = submitSummaryCallback;
@@ -61,14 +72,14 @@ class RunningSummarizer {
             },
         });
         if (configuration.state !== "disableHeuristics") {
-            (0, common_utils_1.assert)(this.configuration.state === "enabled", 0x2ea /* "Configuration state should be enabled" */);
+            (0, core_utils_1.assert)(this.configuration.state === "enabled", 0x2ea /* "Configuration state should be enabled" */);
             this.heuristicRunner = new summarizerHeuristics_1.SummarizeHeuristicRunner(heuristicData, this.configuration, (reason) => this.trySummarize(reason), this.mc.logger);
         }
-        (0, common_utils_1.assert)(this.configuration.state !== "disabled", 0x2eb /* "Summary not supported with configuration disabled" */);
+        (0, core_utils_1.assert)(this.configuration.state !== "disabled", 0x2eb /* "Summary not supported with configuration disabled" */);
         // Cap the maximum amount of time client will wait for a summarize op ack to maxSummarizeAckWaitTime
         // configuration.maxAckWaitTime is composed from defaults, server values, and runtime overrides
         const maxAckWaitTime = Math.min(this.configuration.maxAckWaitTime, maxSummarizeAckWaitTime);
-        this.pendingAckTimer = new common_utils_1.PromiseTimer(maxAckWaitTime, () => {
+        this.pendingAckTimer = new core_utils_1.PromiseTimer(maxAckWaitTime, () => {
             // Note: summarizeCount (from ChildLogger definition) may be 0,
             // since this code path is hit when RunningSummarizer first starts up,
             // before this instance has kicked off a new summarize run.
@@ -100,6 +111,14 @@ class RunningSummarizer {
             this.handleOp(op, runtimeMessage === true);
         };
         this.runtime.on("op", this.runtimeListener);
+        // The max attempts for submit failures can be overridden via a feature flag. This allows us to
+        // tweak this as per telemetry data until we arrive at a stable number.
+        // If its set to a number higher than `defaultMaxAttemptsForSubmitFailures`, it will be ignored.
+        const overrideMaxAttempts = this.mc.config.getNumber("Fluid.Summarizer.AttemptsForSubmitFailures");
+        this.maxAttemptsForSubmitFailures =
+            overrideMaxAttempts && overrideMaxAttempts < exports.defaultMaxAttemptsForSubmitFailures
+                ? overrideMaxAttempts
+                : exports.defaultMaxAttemptsForSubmitFailures;
     }
     static async start(logger, summaryWatcher, configuration, submitSummaryCallback, refreshLatestSummaryAckCallback, heuristicData, summaryCollection, cancellationToken, stopSummarizerCallback, runtime) {
         const summarizer = new RunningSummarizer(logger, summaryWatcher, configuration, submitSummaryCallback, refreshLatestSummaryAckCallback, heuristicData, summaryCollection, cancellationToken, stopSummarizerCallback, runtime);
@@ -172,7 +191,7 @@ class RunningSummarizer {
                 // code in `submitSummary` function in container runtime, will refresh the latest state
                 // by calling `refreshLatestSummaryAckFromServer` and we will be fine.
                 const isIgnoredError = (0, telemetry_utils_1.isFluidError)(error) &&
-                    error.errorType === driver_definitions_1.DriverErrorType.fileNotFoundOrAccessDeniedError;
+                    error.errorType === driver_definitions_1.DriverErrorTypes.fileNotFoundOrAccessDeniedError;
                 summaryLogger.sendTelemetryEvent({
                     eventName: isIgnoredError
                         ? "HandleSummaryAckErrorIgnored"
@@ -208,7 +227,7 @@ class RunningSummarizer {
             });
             refSequenceNumber = await this.handleSummaryAck();
             // A valid Summary Ack must have been processed.
-            (0, common_utils_1.assert)(refSequenceNumber >= 0, 0x58f /* Invalid ref sequence number */);
+            (0, core_utils_1.assert)(refSequenceNumber >= 0, 0x58f /* Invalid ref sequence number */);
         }
     }
     dispose() {
@@ -280,7 +299,7 @@ class RunningSummarizer {
             if (this.summarizingLock === undefined) {
                 this.trySummarizeOnce(
                 // summarizeProps
-                { reason: "lastSummary" }, 
+                { summarizeReason: "lastSummary" }, 
                 // ISummarizeOptions, using defaults: { refreshLatestAck: false, fullTree: false }
                 {});
             }
@@ -331,8 +350,8 @@ class RunningSummarizer {
      * @returns - result of action.
      */
     async lockedSummaryAction(before, action, after) {
-        (0, common_utils_1.assert)(this.summarizingLock === undefined, 0x25b /* "Caller is responsible for checking lock" */);
-        const summarizingLock = new common_utils_1.Deferred();
+        (0, core_utils_1.assert)(this.summarizingLock === undefined, 0x25b /* "Caller is responsible for checking lock" */);
+        const summarizingLock = new core_utils_1.Deferred();
         this.summarizingLock = summarizingLock.promise;
         before();
         return action().finally(() => {
@@ -349,11 +368,20 @@ class RunningSummarizer {
      * @param resultsBuilder - optional, result builder to use.
      * @returns ISummarizeResult - result of running a summary.
      */
-    trySummarizeOnce(summarizeProps, options, cancellationToken = this.cancellationToken, resultsBuilder = new summaryGenerator_1.SummarizeResultBuilder()) {
+    trySummarizeOnce(summarizeProps, options, resultsBuilder = new summaryGenerator_1.SummarizeResultBuilder()) {
         this.lockedSummaryAction(() => {
             this.beforeSummaryAction();
         }, async () => {
-            const summarizeResult = this.generator.summarize(summarizeProps, options, cancellationToken, resultsBuilder);
+            const summaryLogger = (0, telemetry_utils_1.createChildLogger)({
+                logger: this.mc.logger,
+                properties: { all: summarizeProps },
+            });
+            const summaryOptions = {
+                ...options,
+                summaryLogger,
+                cancellationToken: this.cancellationToken,
+            };
+            const summarizeResult = this.generator.summarize(summaryOptions, resultsBuilder);
             // ensure we wait till the end of the process
             return summarizeResult.receivedSummaryAckOrNack;
         }, () => {
@@ -367,7 +395,7 @@ class RunningSummarizer {
         return resultsBuilder.build();
     }
     /** Heuristics summarize attempt. */
-    trySummarize(reason, cancellationToken = this.cancellationToken) {
+    trySummarize(reason) {
         if (this.summarizingLock !== undefined) {
             // lockedSummaryAction() will retry heuristic-based summary at the end of current attempt
             // if it's still needed
@@ -377,64 +405,207 @@ class RunningSummarizer {
         this.lockedSummaryAction(() => {
             this.beforeSummaryAction();
         }, async () => {
-            const attempts = [
-                { refreshLatestAck: false, fullTree: false },
-                { refreshLatestAck: true, fullTree: false },
-            ];
-            let overrideDelaySeconds;
-            let summaryAttempts = 0;
-            let summaryAttemptsPerPhase = 0;
-            let summaryAttemptPhase = 0;
-            while (summaryAttemptPhase < attempts.length) {
-                if (this.cancellationToken.cancelled) {
-                    return;
-                }
-                // We only want to attempt 1 summary when reason is "lastSummary"
-                if (++summaryAttempts > 1 && reason === "lastSummary") {
-                    return;
-                }
-                summaryAttemptsPerPhase++;
-                const summarizeOptions = attempts[summaryAttemptPhase];
-                const summarizeProps = {
-                    reason,
-                    summaryAttempts,
-                    summaryAttemptsPerPhase,
-                    summaryAttemptPhase: summaryAttemptPhase + 1,
-                    ...summarizeOptions,
-                };
-                // Note: no need to account for cancellationToken.waitCancelled here, as
-                // this is accounted SummaryGenerator.summarizeCore that controls receivedSummaryAckOrNack.
-                const resultSummarize = this.generator.summarize(summarizeProps, summarizeOptions, cancellationToken);
-                const result = await resultSummarize.receivedSummaryAckOrNack;
-                if (result.success) {
-                    return;
-                }
-                // Check for retryDelay that can come from summaryNack or upload summary flow.
-                // Retry the same step only once per retryAfter response.
-                const delaySeconds = result.retryAfterSeconds;
-                if (delaySeconds === undefined || summaryAttemptsPerPhase > 1) {
-                    summaryAttemptPhase++;
-                    summaryAttemptsPerPhase = 0;
-                }
-                if (delaySeconds !== undefined) {
-                    this.mc.logger.sendPerformanceEvent({
-                        eventName: "SummarizeAttemptDelay",
-                        duration: delaySeconds,
-                        summaryNackDelay: overrideDelaySeconds !== undefined,
-                        ...summarizeProps,
-                    });
-                    await (0, common_utils_1.delay)(delaySeconds * 1000);
-                }
-            }
-            this.stopSummarizerCallback("failToSummarize");
+            return this.mc.config.getBoolean("Fluid.Summarizer.UseDynamicRetries")
+                ? this.trySummarizeWithRetries(reason)
+                : this.trySummarizeWithStaticAttempts(reason);
         }, () => {
             this.afterSummaryAction();
         }).catch((error) => {
             this.mc.logger.sendErrorEvent({ eventName: "UnexpectedSummarizeError" }, error);
         });
     }
+    /**
+     * Tries to summarize 2 times with pre-defined summary options. If an attempt fails with "retryAfterSeconds"
+     * param, that attempt is tried once more.
+     */
+    async trySummarizeWithStaticAttempts(reason) {
+        const attemptOptions = [
+            { refreshLatestAck: false, fullTree: false },
+            { refreshLatestAck: true, fullTree: false },
+        ];
+        let summaryAttempts = 0;
+        let summaryAttemptsPerPhase = 0;
+        let summaryAttemptPhase = 0;
+        while (summaryAttemptPhase < attemptOptions.length) {
+            if (this.cancellationToken.cancelled) {
+                return;
+            }
+            // We only want to attempt 1 summary when reason is "lastSummary"
+            if (++summaryAttempts > 1 && reason === "lastSummary") {
+                return;
+            }
+            summaryAttemptsPerPhase++;
+            const summarizeOptions = attemptOptions[summaryAttemptPhase];
+            const summarizeProps = {
+                summarizeReason: reason,
+                summaryAttempts,
+                summaryAttemptsPerPhase,
+                summaryAttemptPhase: summaryAttemptPhase + 1,
+                ...summarizeOptions,
+            };
+            const summaryLogger = (0, telemetry_utils_1.createChildLogger)({
+                logger: this.mc.logger,
+                properties: { all: summarizeProps },
+            });
+            const summaryOptions = {
+                ...summarizeOptions,
+                summaryLogger,
+                cancellationToken: this.cancellationToken,
+            };
+            // Note: no need to account for cancellationToken.waitCancelled here, as
+            // this is accounted SummaryGenerator.summarizeCore that controls receivedSummaryAckOrNack.
+            const resultSummarize = this.generator.summarize(summaryOptions);
+            const ackNackResult = await resultSummarize.receivedSummaryAckOrNack;
+            if (ackNackResult.success) {
+                return;
+            }
+            // Check for retryDelay that can come from summaryNack, upload summary or submit summary flows.
+            // Retry the same step only once per retryAfter response.
+            const submitResult = await resultSummarize.summarySubmitted;
+            const delaySeconds = !submitResult.success
+                ? submitResult.data?.retryAfterSeconds
+                : ackNackResult.data?.retryAfterSeconds;
+            if (delaySeconds === undefined || summaryAttemptsPerPhase > 1) {
+                summaryAttemptPhase++;
+                summaryAttemptsPerPhase = 0;
+            }
+            if (delaySeconds !== undefined) {
+                this.mc.logger.sendPerformanceEvent({
+                    eventName: "SummarizeAttemptDelay",
+                    duration: delaySeconds,
+                    summaryNackDelay: ackNackResult.data?.retryAfterSeconds !== undefined,
+                    ...summarizeProps,
+                });
+                await (0, core_utils_1.delay)(delaySeconds * 1000);
+            }
+        }
+        this.stopSummarizerCallback("failToSummarize");
+    }
+    /**
+     * Tries to summarize with retries where retry is based on the failure params.
+     * For example, summarization may be retried for failures with "retryAfterSeconds" param.
+     */
+    async trySummarizeWithRetries(reason) {
+        // Helper to set summarize options, telemetry properties and call summarize.
+        const attemptSummarize = (attemptNumber, finalAttempt) => {
+            const summarizeOptions = {
+                fullTree: false,
+            };
+            const summarizeProps = {
+                summarizeReason: reason,
+                summaryAttempts: attemptNumber,
+                ...summarizeOptions,
+                finalAttempt,
+            };
+            const summaryLogger = (0, telemetry_utils_1.createChildLogger)({
+                logger: this.mc.logger,
+                properties: { all: summarizeProps },
+            });
+            const summaryOptions = {
+                ...summarizeOptions,
+                summaryLogger,
+                cancellationToken: this.cancellationToken,
+                finalAttempt,
+            };
+            const summarizeResult = this.generator.summarize(summaryOptions);
+            return { summarizeProps, summarizeResult };
+        };
+        // The max number of attempts are based on the stage at which summarization failed. If it fails before it is
+        // submitted, a different value is used compared to if it fails after submission. Usually, in the former case,
+        // we would retry more often as its cheaper and retries are likely to succeed.
+        // This makes it harder to predict how many attempts would actually happen as that depends on how far an attempt
+        // made. To keep things simple, the max attempts is reset after every attempt based on where it failed. This may
+        // result in some failures not being retried depending on what happened before this attempt. That's fine because
+        // such scenarios are very unlikely and even if it happens, it would resolve when a new summarizer starts over.
+        // For example - When failure switches from one the submit failures to nack failure, only one more retry will
+        // happen irrespective of the value of `defaultMaxAttempts`.
+        let maxAttempts = exports.defaultMaxAttempts;
+        let currentAttempt = 0;
+        let retryAfterSeconds;
+        let done = false;
+        let result = "success";
+        do {
+            currentAttempt++;
+            if (this.cancellationToken.cancelled) {
+                result = "canceled";
+                done = true;
+                break;
+            }
+            const { summarizeProps, summarizeResult } = attemptSummarize(currentAttempt, false /* finalAttempt */);
+            // Ack / nack is the final step, so if it succeeds we're done.
+            const ackNackResult = await summarizeResult.receivedSummaryAckOrNack;
+            if (ackNackResult.success) {
+                result = "success";
+                done = true;
+                break;
+            }
+            // Update max attempts and retry params from the failure result.
+            // If submit summary failed, use the params from "summarySubmitted" result. Else, use the params
+            // from "receivedSummaryAckOrNack" result.
+            // Note: Check "summarySubmitted" result first because if it fails, ack nack would fail as well.
+            const submitSummaryResult = await summarizeResult.summarySubmitted;
+            if (!submitSummaryResult.success) {
+                maxAttempts = this.maxAttemptsForSubmitFailures;
+                retryAfterSeconds = submitSummaryResult.data?.retryAfterSeconds;
+            }
+            else {
+                maxAttempts = exports.defaultMaxAttempts;
+                retryAfterSeconds = ackNackResult.data?.retryAfterSeconds;
+            }
+            // Emit "summarize" event for this failed attempt.
+            result = "failure";
+            const eventProps = {
+                result,
+                currentAttempt,
+                maxAttempts,
+                error: ackNackResult.error,
+            };
+            this.emit("summarize", eventProps);
+            // If the failure doesn't have "retryAfterSeconds" or the max number of attempts have been done, we're done.
+            if (retryAfterSeconds === undefined || currentAttempt >= maxAttempts - 1) {
+                done = true;
+            }
+            // If the failure has "retryAfterSeconds", add a delay of that time. In this case, a final attempt will
+            // take place and we need to wait for "retryAfterSeconds" before that.
+            if (retryAfterSeconds !== undefined) {
+                this.mc.logger.sendPerformanceEvent({
+                    eventName: "SummarizeAttemptDelay",
+                    duration: retryAfterSeconds,
+                    summaryNackDelay: ackNackResult.data?.retryAfterSeconds !== undefined,
+                    stage: submitSummaryResult.data?.stage,
+                    dynamicRetries: true,
+                    ...summarizeProps,
+                });
+                await (0, core_utils_1.delay)(retryAfterSeconds * 1000);
+            }
+        } while (!done);
+        // If summarize attempt did not fail, emit "summarize" event and return. A failed attempt may be retried below.
+        if (result !== "failure") {
+            this.emit("summarize", { result, currentAttempt, maxAttempts });
+            return;
+        }
+        // If summarization wasn't successful above and the failure contains "retryAfterSeconds", perform one last
+        // attempt. This gives a chance to the runtime to perform additional steps in the last attempt.
+        if (retryAfterSeconds !== undefined) {
+            const { summarizeResult } = attemptSummarize(++currentAttempt, true /* finalAttempt */);
+            // Ack / nack is the final step, so if it succeeds we're done.
+            const ackNackResult = await summarizeResult.receivedSummaryAckOrNack;
+            result = ackNackResult.success ? "success" : "failure";
+            const eventProps = {
+                result,
+                currentAttempt,
+                maxAttempts,
+                error: ackNackResult.success ? undefined : ackNackResult.error,
+            };
+            this.emit("summarize", eventProps);
+        }
+        // If summarization is still unsuccessful, stop the summarizer.
+        if (result === "failure") {
+            this.stopSummarizerCallback("failToSummarize");
+        }
+    }
     /** {@inheritdoc (ISummarizer:interface).summarizeOnDemand} */
-    summarizeOnDemand(resultsBuilder = new summaryGenerator_1.SummarizeResultBuilder(), { reason, ...options }) {
+    summarizeOnDemand(options, resultsBuilder = new summaryGenerator_1.SummarizeResultBuilder()) {
         if (this.stopping) {
             resultsBuilder.fail("RunningSummarizer stopped or disposed", undefined);
             return resultsBuilder.build();
@@ -443,14 +614,15 @@ class RunningSummarizer {
         // return a promise that caller can await before trying again.
         if (this.summarizingLock !== undefined) {
             // The heuristics are blocking concurrent summarize attempts.
-            throw new container_utils_1.UsageError("Attempted to run an already-running summarizer on demand");
+            throw new telemetry_utils_1.UsageError("Attempted to run an already-running summarizer on demand");
         }
-        const result = this.trySummarizeOnce({ reason: `onDemand/${reason}` }, options, this.cancellationToken, resultsBuilder);
+        const { reason, ...summarizeOptions } = options;
+        const result = this.trySummarizeOnce({ summarizeReason: `onDemand/${reason}` }, summarizeOptions, resultsBuilder);
         return result;
     }
     /** {@inheritdoc (ISummarizer:interface).enqueueSummarize} */
-    enqueueSummarize({ reason, afterSequenceNumber = 0, override = false, ...options }) {
-        const onDemandReason = `enqueue;${reason}`;
+    enqueueSummarize(options) {
+        const { reason, afterSequenceNumber = 0, override = false, ...summarizeOptions } = options;
         let overridden = false;
         if (this.enqueuedSummary !== undefined) {
             if (!override) {
@@ -462,9 +634,9 @@ class RunningSummarizer {
             overridden = true;
         }
         this.enqueuedSummary = {
-            reason: onDemandReason,
+            reason: `enqueue;${reason}`,
             afterSequenceNumber,
-            options,
+            summarizeOptions,
             resultsBuilder: new summaryGenerator_1.SummarizeResultBuilder(),
         };
         const results = this.enqueuedSummary.resultsBuilder.build();
@@ -488,10 +660,10 @@ class RunningSummarizer {
             // If no enqueued summary is ready or a summary is already in progress, take no action.
             return false;
         }
-        const { reason, resultsBuilder, options } = this.enqueuedSummary;
+        const { reason, resultsBuilder, summarizeOptions } = this.enqueuedSummary;
         // Set to undefined first, so that subsequent enqueue attempt while summarize will occur later.
         this.enqueuedSummary = undefined;
-        this.trySummarizeOnce({ reason: `enqueuedSummary/${reason}` }, options, this.cancellationToken, resultsBuilder);
+        this.trySummarizeOnce({ summarizeReason: `enqueuedSummary/${reason}` }, summarizeOptions, resultsBuilder);
         return true;
     }
     disposeEnqueuedSummary() {