@fluidframework/container-runtime 2.0.0-internal.6.1.1 → 2.0.0-internal.6.3.0

package/lib/summary/runningSummarizer.js

@@ -2,15 +2,25 @@
  * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
  * Licensed under the MIT License.
  */
-import { isFluidError, createChildMonitoringContext, createChildLogger, } from "@fluidframework/telemetry-utils";
-import { assert, delay, Deferred, PromiseTimer } from "@fluidframework/common-utils";
-import { UsageError } from "@fluidframework/container-utils";
-import { DriverErrorType } from "@fluidframework/driver-definitions";
+import { isFluidError, createChildMonitoringContext, createChildLogger, UsageError, } from "@fluidframework/telemetry-utils";
+import { assert, delay, Deferred, PromiseTimer } from "@fluidframework/core-utils";
+import { TypedEventEmitter } from "@fluid-internal/client-utils";
+import { DriverErrorTypes } from "@fluidframework/driver-definitions";
 import { MessageType } from "@fluidframework/protocol-definitions";
 import { opSize } from "../opProperties";
 import { SummarizeHeuristicRunner } from "./summarizerHeuristics";
 import { raceTimer, SummarizeResultBuilder, SummaryGenerator, } from "./summaryGenerator";
 const maxSummarizeAckWaitTime = 10 * 60 * 1000; // 10 minutes
+/**
+ * The maximum number of summarization attempts that will be done by default in case of failures
+ * that can be retried.
+ */
+export const defaultMaxAttempts = 2;
+/**
+ * The default value for maximum number of summarization attempts that will be done for summarization failures where
+ * submit fails and the failure can be retried.
+ */
+export const defaultMaxAttemptsForSubmitFailures = 5;
 /**
  * An instance of RunningSummarizer manages the heuristics for summarizing.
  * Until disposed, the instance of RunningSummarizer can assume that it is
@@ -18,8 +28,9 @@ const maxSummarizeAckWaitTime = 10 * 60 * 1000; // 10 minutes
  * track of summaries that it is generating as they are broadcast and acked/nacked.
  * This object is created and controlled by Summarizer object.
  */
-export class RunningSummarizer {
+export class RunningSummarizer extends TypedEventEmitter {
     constructor(baseLogger, summaryWatcher, configuration, submitSummaryCallback, refreshLatestSummaryAckCallback, heuristicData, summaryCollection, cancellationToken, stopSummarizerCallback, runtime) {
+        super();
         this.summaryWatcher = summaryWatcher;
         this.configuration = configuration;
         this.submitSummaryCallback = submitSummaryCallback;
@@ -97,6 +108,14 @@ export class RunningSummarizer {
             this.handleOp(op, runtimeMessage === true);
         };
         this.runtime.on("op", this.runtimeListener);
+        // The max attempts for submit failures can be overridden via a feature flag. This allows us to
+        // tweak this as per telemetry data until we arrive at a stable number.
+        // If its set to a number higher than `defaultMaxAttemptsForSubmitFailures`, it will be ignored.
+        const overrideMaxAttempts = this.mc.config.getNumber("Fluid.Summarizer.AttemptsForSubmitFailures");
+        this.maxAttemptsForSubmitFailures =
+            overrideMaxAttempts && overrideMaxAttempts < defaultMaxAttemptsForSubmitFailures
+                ? overrideMaxAttempts
+                : defaultMaxAttemptsForSubmitFailures;
     }
     static async start(logger, summaryWatcher, configuration, submitSummaryCallback, refreshLatestSummaryAckCallback, heuristicData, summaryCollection, cancellationToken, stopSummarizerCallback, runtime) {
         const summarizer = new RunningSummarizer(logger, summaryWatcher, configuration, submitSummaryCallback, refreshLatestSummaryAckCallback, heuristicData, summaryCollection, cancellationToken, stopSummarizerCallback, runtime);
@@ -169,7 +188,7 @@ export class RunningSummarizer {
                 // code in `submitSummary` function in container runtime, will refresh the latest state
                 // by calling `refreshLatestSummaryAckFromServer` and we will be fine.
                 const isIgnoredError = isFluidError(error) &&
-                    error.errorType === DriverErrorType.fileNotFoundOrAccessDeniedError;
+                    error.errorType === DriverErrorTypes.fileNotFoundOrAccessDeniedError;
                 summaryLogger.sendTelemetryEvent({
                     eventName: isIgnoredError
                         ? "HandleSummaryAckErrorIgnored"
@@ -277,7 +296,7 @@ export class RunningSummarizer {
             if (this.summarizingLock === undefined) {
                 this.trySummarizeOnce(
                 // summarizeProps
-                { reason: "lastSummary" }, 
+                { summarizeReason: "lastSummary" }, 
                 // ISummarizeOptions, using defaults: { refreshLatestAck: false, fullTree: false }
                 {});
             }
@@ -346,11 +365,20 @@ export class RunningSummarizer {
      * @param resultsBuilder - optional, result builder to use.
      * @returns ISummarizeResult - result of running a summary.
      */
-    trySummarizeOnce(summarizeProps, options, cancellationToken = this.cancellationToken, resultsBuilder = new SummarizeResultBuilder()) {
+    trySummarizeOnce(summarizeProps, options, resultsBuilder = new SummarizeResultBuilder()) {
         this.lockedSummaryAction(() => {
             this.beforeSummaryAction();
         }, async () => {
-            const summarizeResult = this.generator.summarize(summarizeProps, options, cancellationToken, resultsBuilder);
+            const summaryLogger = createChildLogger({
+                logger: this.mc.logger,
+                properties: { all: summarizeProps },
+            });
+            const summaryOptions = {
+                ...options,
+                summaryLogger,
+                cancellationToken: this.cancellationToken,
+            };
+            const summarizeResult = this.generator.summarize(summaryOptions, resultsBuilder);
             // ensure we wait till the end of the process
             return summarizeResult.receivedSummaryAckOrNack;
         }, () => {
@@ -364,7 +392,7 @@ export class RunningSummarizer {
         return resultsBuilder.build();
     }
     /** Heuristics summarize attempt. */
-    trySummarize(reason, cancellationToken = this.cancellationToken) {
+    trySummarize(reason) {
         if (this.summarizingLock !== undefined) {
             // lockedSummaryAction() will retry heuristic-based summary at the end of current attempt
             // if it's still needed
@@ -374,64 +402,207 @@ export class RunningSummarizer {
         this.lockedSummaryAction(() => {
             this.beforeSummaryAction();
         }, async () => {
-            const attempts = [
-                { refreshLatestAck: false, fullTree: false },
-                { refreshLatestAck: true, fullTree: false },
-            ];
-            let overrideDelaySeconds;
-            let summaryAttempts = 0;
-            let summaryAttemptsPerPhase = 0;
-            let summaryAttemptPhase = 0;
-            while (summaryAttemptPhase < attempts.length) {
-                if (this.cancellationToken.cancelled) {
-                    return;
-                }
-                // We only want to attempt 1 summary when reason is "lastSummary"
-                if (++summaryAttempts > 1 && reason === "lastSummary") {
-                    return;
-                }
-                summaryAttemptsPerPhase++;
-                const summarizeOptions = attempts[summaryAttemptPhase];
-                const summarizeProps = {
-                    reason,
-                    summaryAttempts,
-                    summaryAttemptsPerPhase,
-                    summaryAttemptPhase: summaryAttemptPhase + 1,
-                    ...summarizeOptions,
-                };
-                // Note: no need to account for cancellationToken.waitCancelled here, as
-                // this is accounted SummaryGenerator.summarizeCore that controls receivedSummaryAckOrNack.
-                const resultSummarize = this.generator.summarize(summarizeProps, summarizeOptions, cancellationToken);
-                const result = await resultSummarize.receivedSummaryAckOrNack;
-                if (result.success) {
-                    return;
-                }
-                // Check for retryDelay that can come from summaryNack or upload summary flow.
-                // Retry the same step only once per retryAfter response.
-                const delaySeconds = result.retryAfterSeconds;
-                if (delaySeconds === undefined || summaryAttemptsPerPhase > 1) {
-                    summaryAttemptPhase++;
-                    summaryAttemptsPerPhase = 0;
-                }
-                if (delaySeconds !== undefined) {
-                    this.mc.logger.sendPerformanceEvent({
-                        eventName: "SummarizeAttemptDelay",
-                        duration: delaySeconds,
-                        summaryNackDelay: overrideDelaySeconds !== undefined,
-                        ...summarizeProps,
-                    });
-                    await delay(delaySeconds * 1000);
-                }
-            }
-            this.stopSummarizerCallback("failToSummarize");
+            return this.mc.config.getBoolean("Fluid.Summarizer.UseDynamicRetries")
+                ? this.trySummarizeWithRetries(reason)
+                : this.trySummarizeWithStaticAttempts(reason);
         }, () => {
             this.afterSummaryAction();
         }).catch((error) => {
             this.mc.logger.sendErrorEvent({ eventName: "UnexpectedSummarizeError" }, error);
         });
     }
+    /**
+     * Tries to summarize 2 times with pre-defined summary options. If an attempt fails with "retryAfterSeconds"
+     * param, that attempt is tried once more.
+     */
+    async trySummarizeWithStaticAttempts(reason) {
+        const attemptOptions = [
+            { refreshLatestAck: false, fullTree: false },
+            { refreshLatestAck: true, fullTree: false },
+        ];
+        let summaryAttempts = 0;
+        let summaryAttemptsPerPhase = 0;
+        let summaryAttemptPhase = 0;
+        while (summaryAttemptPhase < attemptOptions.length) {
+            if (this.cancellationToken.cancelled) {
+                return;
+            }
+            // We only want to attempt 1 summary when reason is "lastSummary"
+            if (++summaryAttempts > 1 && reason === "lastSummary") {
+                return;
+            }
+            summaryAttemptsPerPhase++;
+            const summarizeOptions = attemptOptions[summaryAttemptPhase];
+            const summarizeProps = {
+                summarizeReason: reason,
+                summaryAttempts,
+                summaryAttemptsPerPhase,
+                summaryAttemptPhase: summaryAttemptPhase + 1,
+                ...summarizeOptions,
+            };
+            const summaryLogger = createChildLogger({
+                logger: this.mc.logger,
+                properties: { all: summarizeProps },
+            });
+            const summaryOptions = {
+                ...summarizeOptions,
+                summaryLogger,
+                cancellationToken: this.cancellationToken,
+            };
+            // Note: no need to account for cancellationToken.waitCancelled here, as
+            // this is accounted SummaryGenerator.summarizeCore that controls receivedSummaryAckOrNack.
+            const resultSummarize = this.generator.summarize(summaryOptions);
+            const ackNackResult = await resultSummarize.receivedSummaryAckOrNack;
+            if (ackNackResult.success) {
+                return;
+            }
+            // Check for retryDelay that can come from summaryNack, upload summary or submit summary flows.
+            // Retry the same step only once per retryAfter response.
+            const submitResult = await resultSummarize.summarySubmitted;
+            const delaySeconds = !submitResult.success
+                ? submitResult.data?.retryAfterSeconds
+                : ackNackResult.data?.retryAfterSeconds;
+            if (delaySeconds === undefined || summaryAttemptsPerPhase > 1) {
+                summaryAttemptPhase++;
+                summaryAttemptsPerPhase = 0;
+            }
+            if (delaySeconds !== undefined) {
+                this.mc.logger.sendPerformanceEvent({
+                    eventName: "SummarizeAttemptDelay",
+                    duration: delaySeconds,
+                    summaryNackDelay: ackNackResult.data?.retryAfterSeconds !== undefined,
+                    ...summarizeProps,
+                });
+                await delay(delaySeconds * 1000);
+            }
+        }
+        this.stopSummarizerCallback("failToSummarize");
+    }
+    /**
+     * Tries to summarize with retries where retry is based on the failure params.
+     * For example, summarization may be retried for failures with "retryAfterSeconds" param.
+     */
+    async trySummarizeWithRetries(reason) {
+        // Helper to set summarize options, telemetry properties and call summarize.
+        const attemptSummarize = (attemptNumber, finalAttempt) => {
+            const summarizeOptions = {
+                fullTree: false,
+            };
+            const summarizeProps = {
+                summarizeReason: reason,
+                summaryAttempts: attemptNumber,
+                ...summarizeOptions,
+                finalAttempt,
+            };
+            const summaryLogger = createChildLogger({
+                logger: this.mc.logger,
+                properties: { all: summarizeProps },
+            });
+            const summaryOptions = {
+                ...summarizeOptions,
+                summaryLogger,
+                cancellationToken: this.cancellationToken,
+                finalAttempt,
+            };
+            const summarizeResult = this.generator.summarize(summaryOptions);
+            return { summarizeProps, summarizeResult };
+        };
+        // The max number of attempts are based on the stage at which summarization failed. If it fails before it is
+        // submitted, a different value is used compared to if it fails after submission. Usually, in the former case,
+        // we would retry more often as its cheaper and retries are likely to succeed.
+        // This makes it harder to predict how many attempts would actually happen as that depends on how far an attempt
+        // made. To keep things simple, the max attempts is reset after every attempt based on where it failed. This may
+        // result in some failures not being retried depending on what happened before this attempt. That's fine because
+        // such scenarios are very unlikely and even if it happens, it would resolve when a new summarizer starts over.
+        // For example - When failure switches from one the submit failures to nack failure, only one more retry will
+        // happen irrespective of the value of `defaultMaxAttempts`.
+        let maxAttempts = defaultMaxAttempts;
+        let currentAttempt = 0;
+        let retryAfterSeconds;
+        let done = false;
+        let result = "success";
+        do {
+            currentAttempt++;
+            if (this.cancellationToken.cancelled) {
+                result = "canceled";
+                done = true;
+                break;
+            }
+            const { summarizeProps, summarizeResult } = attemptSummarize(currentAttempt, false /* finalAttempt */);
+            // Ack / nack is the final step, so if it succeeds we're done.
+            const ackNackResult = await summarizeResult.receivedSummaryAckOrNack;
+            if (ackNackResult.success) {
+                result = "success";
+                done = true;
+                break;
+            }
+            // Update max attempts and retry params from the failure result.
+            // If submit summary failed, use the params from "summarySubmitted" result. Else, use the params
+            // from "receivedSummaryAckOrNack" result.
+            // Note: Check "summarySubmitted" result first because if it fails, ack nack would fail as well.
+            const submitSummaryResult = await summarizeResult.summarySubmitted;
+            if (!submitSummaryResult.success) {
+                maxAttempts = this.maxAttemptsForSubmitFailures;
+                retryAfterSeconds = submitSummaryResult.data?.retryAfterSeconds;
+            }
+            else {
+                maxAttempts = defaultMaxAttempts;
+                retryAfterSeconds = ackNackResult.data?.retryAfterSeconds;
+            }
+            // Emit "summarize" event for this failed attempt.
+            result = "failure";
+            const eventProps = {
+                result,
+                currentAttempt,
+                maxAttempts,
+                error: ackNackResult.error,
+            };
+            this.emit("summarize", eventProps);
+            // If the failure doesn't have "retryAfterSeconds" or the max number of attempts have been done, we're done.
+            if (retryAfterSeconds === undefined || currentAttempt >= maxAttempts - 1) {
+                done = true;
+            }
+            // If the failure has "retryAfterSeconds", add a delay of that time. In this case, a final attempt will
+            // take place and we need to wait for "retryAfterSeconds" before that.
+            if (retryAfterSeconds !== undefined) {
+                this.mc.logger.sendPerformanceEvent({
+                    eventName: "SummarizeAttemptDelay",
+                    duration: retryAfterSeconds,
+                    summaryNackDelay: ackNackResult.data?.retryAfterSeconds !== undefined,
+                    stage: submitSummaryResult.data?.stage,
+                    dynamicRetries: true,
+                    ...summarizeProps,
+                });
+                await delay(retryAfterSeconds * 1000);
+            }
+        } while (!done);
+        // If summarize attempt did not fail, emit "summarize" event and return. A failed attempt may be retried below.
+        if (result !== "failure") {
+            this.emit("summarize", { result, currentAttempt, maxAttempts });
+            return;
+        }
+        // If summarization wasn't successful above and the failure contains "retryAfterSeconds", perform one last
+        // attempt. This gives a chance to the runtime to perform additional steps in the last attempt.
+        if (retryAfterSeconds !== undefined) {
+            const { summarizeResult } = attemptSummarize(++currentAttempt, true /* finalAttempt */);
+            // Ack / nack is the final step, so if it succeeds we're done.
+            const ackNackResult = await summarizeResult.receivedSummaryAckOrNack;
+            result = ackNackResult.success ? "success" : "failure";
+            const eventProps = {
+                result,
+                currentAttempt,
+                maxAttempts,
+                error: ackNackResult.success ? undefined : ackNackResult.error,
+            };
+            this.emit("summarize", eventProps);
+        }
+        // If summarization is still unsuccessful, stop the summarizer.
+        if (result === "failure") {
+            this.stopSummarizerCallback("failToSummarize");
+        }
+    }
     /** {@inheritdoc (ISummarizer:interface).summarizeOnDemand} */
-    summarizeOnDemand(resultsBuilder = new SummarizeResultBuilder(), { reason, ...options }) {
+    summarizeOnDemand(options, resultsBuilder = new SummarizeResultBuilder()) {
         if (this.stopping) {
             resultsBuilder.fail("RunningSummarizer stopped or disposed", undefined);
             return resultsBuilder.build();
@@ -442,12 +613,13 @@ export class RunningSummarizer {
             // The heuristics are blocking concurrent summarize attempts.
             throw new UsageError("Attempted to run an already-running summarizer on demand");
         }
-        const result = this.trySummarizeOnce({ reason: `onDemand/${reason}` }, options, this.cancellationToken, resultsBuilder);
+        const { reason, ...summarizeOptions } = options;
+        const result = this.trySummarizeOnce({ summarizeReason: `onDemand/${reason}` }, summarizeOptions, resultsBuilder);
         return result;
     }
     /** {@inheritdoc (ISummarizer:interface).enqueueSummarize} */
-    enqueueSummarize({ reason, afterSequenceNumber = 0, override = false, ...options }) {
-        const onDemandReason = `enqueue;${reason}`;
+    enqueueSummarize(options) {
+        const { reason, afterSequenceNumber = 0, override = false, ...summarizeOptions } = options;
         let overridden = false;
         if (this.enqueuedSummary !== undefined) {
             if (!override) {
@@ -459,9 +631,9 @@ export class RunningSummarizer {
             overridden = true;
         }
         this.enqueuedSummary = {
-            reason: onDemandReason,
+            reason: `enqueue;${reason}`,
             afterSequenceNumber,
-            options,
+            summarizeOptions,
             resultsBuilder: new SummarizeResultBuilder(),
         };
         const results = this.enqueuedSummary.resultsBuilder.build();
@@ -485,10 +657,10 @@ export class RunningSummarizer {
             // If no enqueued summary is ready or a summary is already in progress, take no action.
             return false;
         }
-        const { reason, resultsBuilder, options } = this.enqueuedSummary;
+        const { reason, resultsBuilder, summarizeOptions } = this.enqueuedSummary;
         // Set to undefined first, so that subsequent enqueue attempt while summarize will occur later.
         this.enqueuedSummary = undefined;
-        this.trySummarizeOnce({ reason: `enqueuedSummary/${reason}` }, options, this.cancellationToken, resultsBuilder);
+        this.trySummarizeOnce({ summarizeReason: `enqueuedSummary/${reason}` }, summarizeOptions, resultsBuilder);
         return true;
     }
     disposeEnqueuedSummary() {