@fluidframework/container-runtime 2.0.0-internal.6.1.0 → 2.0.0-internal.6.2.0

package/lib/summary/runningSummarizer.js

@@ -2,15 +2,24 @@
  * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
  * Licensed under the MIT License.
  */
-import { isFluidError, createChildMonitoringContext, createChildLogger, } from "@fluidframework/telemetry-utils";
-import { assert, delay, Deferred, PromiseTimer } from "@fluidframework/common-utils";
-import { UsageError } from "@fluidframework/container-utils";
+import { isFluidError, createChildMonitoringContext, createChildLogger, UsageError, } from "@fluidframework/telemetry-utils";
+import { assert, delay, Deferred, PromiseTimer, TypedEventEmitter, } from "@fluidframework/common-utils";
 import { DriverErrorType } from "@fluidframework/driver-definitions";
 import { MessageType } from "@fluidframework/protocol-definitions";
 import { opSize } from "../opProperties";
 import { SummarizeHeuristicRunner } from "./summarizerHeuristics";
 import { raceTimer, SummarizeResultBuilder, SummaryGenerator, } from "./summaryGenerator";
 const maxSummarizeAckWaitTime = 10 * 60 * 1000; // 10 minutes
+/**
+ * The maximum number of summarization attempts that will be done by default in case of failures
+ * that can be retried.
+ */
+export const defaultMaxAttempts = 2;
+/**
+ * The default value for maximum number of summarization attempts that will be done for summarization failures where
+ * submit fails and the failure can be retried.
+ */
+export const defaultMaxAttemptsForSubmitFailures = 5;
 /**
  * An instance of RunningSummarizer manages the heuristics for summarizing.
  * Until disposed, the instance of RunningSummarizer can assume that it is
@@ -18,8 +27,9 @@ const maxSummarizeAckWaitTime = 10 * 60 * 1000; // 10 minutes
  * track of summaries that it is generating as they are broadcast and acked/nacked.
  * This object is created and controlled by Summarizer object.
  */
-export class RunningSummarizer {
+export class RunningSummarizer extends TypedEventEmitter {
     constructor(baseLogger, summaryWatcher, configuration, submitSummaryCallback, refreshLatestSummaryAckCallback, heuristicData, summaryCollection, cancellationToken, stopSummarizerCallback, runtime) {
+        super();
         this.summaryWatcher = summaryWatcher;
         this.configuration = configuration;
         this.submitSummaryCallback = submitSummaryCallback;
@@ -97,6 +107,14 @@ export class RunningSummarizer {
             this.handleOp(op, runtimeMessage === true);
         };
         this.runtime.on("op", this.runtimeListener);
+        // The max attempts for submit failures can be overridden via a feature flag. This allows us to
+        // tweak this as per telemetry data until we arrive at a stable number.
+        // If its set to a number higher than `defaultMaxAttemptsForSubmitFailures`, it will be ignored.
+        const overrideMaxAttempts = this.mc.config.getNumber("Fluid.Summarizer.AttemptsForSubmitFailures");
+        this.maxAttemptsForSubmitFailures =
+            overrideMaxAttempts && overrideMaxAttempts < defaultMaxAttemptsForSubmitFailures
+                ? overrideMaxAttempts
+                : defaultMaxAttemptsForSubmitFailures;
     }
     static async start(logger, summaryWatcher, configuration, submitSummaryCallback, refreshLatestSummaryAckCallback, heuristicData, summaryCollection, cancellationToken, stopSummarizerCallback, runtime) {
         const summarizer = new RunningSummarizer(logger, summaryWatcher, configuration, submitSummaryCallback, refreshLatestSummaryAckCallback, heuristicData, summaryCollection, cancellationToken, stopSummarizerCallback, runtime);
@@ -277,7 +295,7 @@ export class RunningSummarizer {
             if (this.summarizingLock === undefined) {
                 this.trySummarizeOnce(
                 // summarizeProps
-                { reason: "lastSummary" }, 
+                { summarizeReason: "lastSummary" }, 
                 // ISummarizeOptions, using defaults: { refreshLatestAck: false, fullTree: false }
                 {});
             }
@@ -346,11 +364,20 @@ export class RunningSummarizer {
      * @param resultsBuilder - optional, result builder to use.
      * @returns ISummarizeResult - result of running a summary.
      */
-    trySummarizeOnce(summarizeProps, options, cancellationToken = this.cancellationToken, resultsBuilder = new SummarizeResultBuilder()) {
+    trySummarizeOnce(summarizeProps, options, resultsBuilder = new SummarizeResultBuilder()) {
         this.lockedSummaryAction(() => {
             this.beforeSummaryAction();
         }, async () => {
-            const summarizeResult = this.generator.summarize(summarizeProps, options, cancellationToken, resultsBuilder);
+            const summaryLogger = createChildLogger({
+                logger: this.mc.logger,
+                properties: { all: summarizeProps },
+            });
+            const summaryOptions = {
+                ...options,
+                summaryLogger,
+                cancellationToken: this.cancellationToken,
+            };
+            const summarizeResult = this.generator.summarize(summaryOptions, resultsBuilder);
             // ensure we wait till the end of the process
             return summarizeResult.receivedSummaryAckOrNack;
         }, () => {
@@ -364,7 +391,7 @@ export class RunningSummarizer {
         return resultsBuilder.build();
     }
     /** Heuristics summarize attempt. */
-    trySummarize(reason, cancellationToken = this.cancellationToken) {
+    trySummarize(reason) {
         if (this.summarizingLock !== undefined) {
             // lockedSummaryAction() will retry heuristic-based summary at the end of current attempt
             // if it's still needed
@@ -374,64 +401,207 @@ export class RunningSummarizer {
         this.lockedSummaryAction(() => {
             this.beforeSummaryAction();
         }, async () => {
-            const attempts = [
-                { refreshLatestAck: false, fullTree: false },
-                { refreshLatestAck: true, fullTree: false },
-            ];
-            let overrideDelaySeconds;
-            let summaryAttempts = 0;
-            let summaryAttemptsPerPhase = 0;
-            let summaryAttemptPhase = 0;
-            while (summaryAttemptPhase < attempts.length) {
-                if (this.cancellationToken.cancelled) {
-                    return;
-                }
-                // We only want to attempt 1 summary when reason is "lastSummary"
-                if (++summaryAttempts > 1 && reason === "lastSummary") {
-                    return;
-                }
-                summaryAttemptsPerPhase++;
-                const summarizeOptions = attempts[summaryAttemptPhase];
-                const summarizeProps = {
-                    reason,
-                    summaryAttempts,
-                    summaryAttemptsPerPhase,
-                    summaryAttemptPhase: summaryAttemptPhase + 1,
-                    ...summarizeOptions,
-                };
-                // Note: no need to account for cancellationToken.waitCancelled here, as
-                // this is accounted SummaryGenerator.summarizeCore that controls receivedSummaryAckOrNack.
-                const resultSummarize = this.generator.summarize(summarizeProps, summarizeOptions, cancellationToken);
-                const result = await resultSummarize.receivedSummaryAckOrNack;
-                if (result.success) {
-                    return;
-                }
-                // Check for retryDelay that can come from summaryNack or upload summary flow.
-                // Retry the same step only once per retryAfter response.
-                const delaySeconds = result.retryAfterSeconds;
-                if (delaySeconds === undefined || summaryAttemptsPerPhase > 1) {
-                    summaryAttemptPhase++;
-                    summaryAttemptsPerPhase = 0;
-                }
-                if (delaySeconds !== undefined) {
-                    this.mc.logger.sendPerformanceEvent({
-                        eventName: "SummarizeAttemptDelay",
-                        duration: delaySeconds,
-                        summaryNackDelay: overrideDelaySeconds !== undefined,
-                        ...summarizeProps,
-                    });
-                    await delay(delaySeconds * 1000);
-                }
-            }
-            this.stopSummarizerCallback("failToSummarize");
+            return this.mc.config.getBoolean("Fluid.Summarizer.TryDynamicRetries")
+                ? this.trySummarizeWithRetries(reason)
+                : this.trySummarizeWithStaticAttempts(reason);
         }, () => {
             this.afterSummaryAction();
         }).catch((error) => {
             this.mc.logger.sendErrorEvent({ eventName: "UnexpectedSummarizeError" }, error);
         });
     }
+    /**
+     * Tries to summarize 2 times with pre-defined summary options. If an attempt fails with "retryAfterSeconds"
+     * param, that attempt is tried once more.
+     */
+    async trySummarizeWithStaticAttempts(reason) {
+        const attemptOptions = [
+            { refreshLatestAck: false, fullTree: false },
+            { refreshLatestAck: true, fullTree: false },
+        ];
+        let summaryAttempts = 0;
+        let summaryAttemptsPerPhase = 0;
+        let summaryAttemptPhase = 0;
+        while (summaryAttemptPhase < attemptOptions.length) {
+            if (this.cancellationToken.cancelled) {
+                return;
+            }
+            // We only want to attempt 1 summary when reason is "lastSummary"
+            if (++summaryAttempts > 1 && reason === "lastSummary") {
+                return;
+            }
+            summaryAttemptsPerPhase++;
+            const summarizeOptions = attemptOptions[summaryAttemptPhase];
+            const summarizeProps = {
+                summarizeReason: reason,
+                summaryAttempts,
+                summaryAttemptsPerPhase,
+                summaryAttemptPhase: summaryAttemptPhase + 1,
+                ...summarizeOptions,
+            };
+            const summaryLogger = createChildLogger({
+                logger: this.mc.logger,
+                properties: { all: summarizeProps },
+            });
+            const summaryOptions = {
+                ...summarizeOptions,
+                summaryLogger,
+                cancellationToken: this.cancellationToken,
+            };
+            // Note: no need to account for cancellationToken.waitCancelled here, as
+            // this is accounted SummaryGenerator.summarizeCore that controls receivedSummaryAckOrNack.
+            const resultSummarize = this.generator.summarize(summaryOptions);
+            const ackNackResult = await resultSummarize.receivedSummaryAckOrNack;
+            if (ackNackResult.success) {
+                return;
+            }
+            // Check for retryDelay that can come from summaryNack, upload summary or submit summary flows.
+            // Retry the same step only once per retryAfter response.
+            const submitResult = await resultSummarize.summarySubmitted;
+            const delaySeconds = !submitResult.success
+                ? submitResult.data?.retryAfterSeconds
+                : ackNackResult.data?.retryAfterSeconds;
+            if (delaySeconds === undefined || summaryAttemptsPerPhase > 1) {
+                summaryAttemptPhase++;
+                summaryAttemptsPerPhase = 0;
+            }
+            if (delaySeconds !== undefined) {
+                this.mc.logger.sendPerformanceEvent({
+                    eventName: "SummarizeAttemptDelay",
+                    duration: delaySeconds,
+                    summaryNackDelay: ackNackResult.data?.retryAfterSeconds !== undefined,
+                    ...summarizeProps,
+                });
+                await delay(delaySeconds * 1000);
+            }
+        }
+        this.stopSummarizerCallback("failToSummarize");
+    }
+    /**
+     * Tries to summarize with retries where retry is based on the failure params.
+     * For example, summarization may be retried for failures with "retryAfterSeconds" param.
+     */
+    async trySummarizeWithRetries(reason) {
+        // Helper to set summarize options, telemetry properties and call summarize.
+        const attemptSummarize = (attemptNumber, finalAttempt) => {
+            const summarizeOptions = {
+                fullTree: false,
+            };
+            const summarizeProps = {
+                summarizeReason: reason,
+                summaryAttempts: attemptNumber,
+                ...summarizeOptions,
+                finalAttempt,
+            };
+            const summaryLogger = createChildLogger({
+                logger: this.mc.logger,
+                properties: { all: summarizeProps },
+            });
+            const summaryOptions = {
+                ...summarizeOptions,
+                summaryLogger,
+                cancellationToken: this.cancellationToken,
+                finalAttempt,
+            };
+            const summarizeResult = this.generator.summarize(summaryOptions);
+            return { summarizeProps, summarizeResult };
+        };
+        // The max number of attempts are based on the stage at which summarization failed. If it fails before it is
+        // submitted, a different value is used compared to if it fails after submission. Usually, in the former case,
+        // we would retry more often as its cheaper and retries are likely to succeed.
+        // This makes it harder to predict how many attempts would actually happen as that depends on how far an attempt
+        // made. To keep things simple, the max attempts is reset after every attempt based on where it failed. This may
+        // result in some failures not being retried depending on what happened before this attempt. That's fine because
+        // such scenarios are very unlikely and even if it happens, it would resolve when a new summarizer starts over.
+        // For example - When failure switches from one the submit failures to nack failure, only one more retry will
+        // happen irrespective of the value of `defaultMaxAttempts`.
+        let maxAttempts = defaultMaxAttempts;
+        let currentAttempt = 0;
+        let retryAfterSeconds;
+        let done = false;
+        let result = "success";
+        do {
+            currentAttempt++;
+            if (this.cancellationToken.cancelled) {
+                result = "canceled";
+                done = true;
+                break;
+            }
+            const { summarizeProps, summarizeResult } = attemptSummarize(currentAttempt, false /* finalAttempt */);
+            // Ack / nack is the final step, so if it succeeds we're done.
+            const ackNackResult = await summarizeResult.receivedSummaryAckOrNack;
+            if (ackNackResult.success) {
+                result = "success";
+                done = true;
+                break;
+            }
+            // Update max attempts and retry params from the failure result.
+            // If submit summary failed, use the params from "summarySubmitted" result. Else, use the params
+            // from "receivedSummaryAckOrNack" result.
+            // Note: Check "summarySubmitted" result first because if it fails, ack nack would fail as well.
+            const submitSummaryResult = await summarizeResult.summarySubmitted;
+            if (!submitSummaryResult.success) {
+                maxAttempts = this.maxAttemptsForSubmitFailures;
+                retryAfterSeconds = submitSummaryResult.data?.retryAfterSeconds;
+            }
+            else {
+                maxAttempts = defaultMaxAttempts;
+                retryAfterSeconds = ackNackResult.data?.retryAfterSeconds;
+            }
+            // Emit "summarize" event for this failed attempt.
+            result = "failure";
+            const eventProps = {
+                result,
+                currentAttempt,
+                maxAttempts,
+                error: ackNackResult.error,
+            };
+            this.emit("summarize", eventProps);
+            // If the failure doesn't have "retryAfterSeconds" or the max number of attempts have been done, we're done.
+            if (retryAfterSeconds === undefined || currentAttempt >= maxAttempts - 1) {
+                done = true;
+            }
+            // If the failure has "retryAfterSeconds", add a delay of that time. In this case, a final attempt will
+            // take place and we need to wait for "retryAfterSeconds" before that.
+            if (retryAfterSeconds !== undefined) {
+                this.mc.logger.sendPerformanceEvent({
+                    eventName: "SummarizeAttemptDelay",
+                    duration: retryAfterSeconds,
+                    summaryNackDelay: ackNackResult.data?.retryAfterSeconds !== undefined,
+                    stage: submitSummaryResult.data?.stage,
+                    dynamicRetries: true,
+                    ...summarizeProps,
+                });
+                await delay(retryAfterSeconds * 1000);
+            }
+        } while (!done);
+        // If summarize attempt did not fail, emit "summarize" event and return. A failed attempt may be retried below.
+        if (result !== "failure") {
+            this.emit("summarize", { result, currentAttempt, maxAttempts });
+            return;
+        }
+        // If summarization wasn't successful above and the failure contains "retryAfterSeconds", perform one last
+        // attempt. This gives a chance to the runtime to perform additional steps in the last attempt.
+        if (retryAfterSeconds !== undefined) {
+            const { summarizeResult } = attemptSummarize(++currentAttempt, true /* finalAttempt */);
+            // Ack / nack is the final step, so if it succeeds we're done.
+            const ackNackResult = await summarizeResult.receivedSummaryAckOrNack;
+            result = ackNackResult.success ? "success" : "failure";
+            const eventProps = {
+                result,
+                currentAttempt,
+                maxAttempts,
+                error: ackNackResult.success ? undefined : ackNackResult.error,
+            };
+            this.emit("summarize", eventProps);
+        }
+        // If summarization is still unsuccessful, stop the summarizer.
+        if (result === "failure") {
+            this.stopSummarizerCallback("failToSummarize");
+        }
+    }
     /** {@inheritdoc (ISummarizer:interface).summarizeOnDemand} */
-    summarizeOnDemand(resultsBuilder = new SummarizeResultBuilder(), { reason, ...options }) {
+    summarizeOnDemand(options, resultsBuilder = new SummarizeResultBuilder()) {
         if (this.stopping) {
             resultsBuilder.fail("RunningSummarizer stopped or disposed", undefined);
             return resultsBuilder.build();
@@ -442,12 +612,13 @@ export class RunningSummarizer {
             // The heuristics are blocking concurrent summarize attempts.
             throw new UsageError("Attempted to run an already-running summarizer on demand");
         }
-        const result = this.trySummarizeOnce({ reason: `onDemand/${reason}` }, options, this.cancellationToken, resultsBuilder);
+        const { reason, ...summarizeOptions } = options;
+        const result = this.trySummarizeOnce({ summarizeReason: `onDemand/${reason}` }, summarizeOptions, resultsBuilder);
         return result;
     }
     /** {@inheritdoc (ISummarizer:interface).enqueueSummarize} */
-    enqueueSummarize({ reason, afterSequenceNumber = 0, override = false, ...options }) {
-        const onDemandReason = `enqueue;${reason}`;
+    enqueueSummarize(options) {
+        const { reason, afterSequenceNumber = 0, override = false, ...summarizeOptions } = options;
         let overridden = false;
         if (this.enqueuedSummary !== undefined) {
             if (!override) {
@@ -459,9 +630,9 @@ export class RunningSummarizer {
             overridden = true;
         }
         this.enqueuedSummary = {
-            reason: onDemandReason,
+            reason: `enqueue;${reason}`,
             afterSequenceNumber,
-            options,
+            summarizeOptions,
             resultsBuilder: new SummarizeResultBuilder(),
         };
         const results = this.enqueuedSummary.resultsBuilder.build();
@@ -485,10 +656,10 @@ export class RunningSummarizer {
             // If no enqueued summary is ready or a summary is already in progress, take no action.
             return false;
         }
-        const { reason, resultsBuilder, options } = this.enqueuedSummary;
+        const { reason, resultsBuilder, summarizeOptions } = this.enqueuedSummary;
         // Set to undefined first, so that subsequent enqueue attempt while summarize will occur later.
         this.enqueuedSummary = undefined;
-        this.trySummarizeOnce({ reason: `enqueuedSummary/${reason}` }, options, this.cancellationToken, resultsBuilder);
+        this.trySummarizeOnce({ summarizeReason: `enqueuedSummary/${reason}` }, summarizeOptions, resultsBuilder);
         return true;
     }
     disposeEnqueuedSummary() {

package/lib/summary/runningSummarizer.js.map

@@ -1 +1 @@
-{"version":3,"file":"runningSummarizer.js","sourceRoot":"","sources":["../../src/summary/runningSummarizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EACN,YAAY,EAEZ,4BAA4B,EAC5B,iBAAiB,GACjB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AACrF,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAC;AAC7D,OAAO,EAAE,eAAe,EAAE,MAAM,oCAAoC,CAAC;AACrE,OAAO,EAA6B,WAAW,EAAE,MAAM,sCAAsC,CAAC;AAE9F,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AAmBlE,OAAO,EACN,SAAS,EAET,sBAAsB,EACtB,gBAAgB,GAChB,MAAM,oBAAoB,CAAC;AAE5B,MAAM,uBAAuB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,aAAa;AAE7D;;;;;;GAMG;AACH,MAAM,OAAO,iBAAiB;IAgG7B,YACC,UAAgC,EACf,cAAqC,EACrC,aAAoC,EACpC,qBAEgB,EAChB,+BAEC,EACD,aAAsC,EACtC,iBAAoC,EACpC,iBAA4C,EAC5C,sBAA8D,EAC9D,OAA2B;QAZ3B,mBAAc,GAAd,cAAc,CAAuB;QACrC,kBAAa,GAAb,aAAa,CAAuB;QACpC,0BAAqB,GAArB,qBAAqB,CAEL;QAChB,oCAA+B,GAA/B,+BAA+B,CAE9B;QACD,kBAAa,GAAb,aAAa,CAAyB;QACtC,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,sBAAiB,GAAjB,iBAAiB,CAA2B;QAC5C,2BAAsB,GAAtB,sBAAsB,CAAwC;QAC9D,YAAO,GAAP,OAAO,CAAoB;QArCrC,aAAQ,GAAG,KAAK,CAAC;QACjB,cAAS,GAAG,KAAK,CAAC;QAElB,wBAAmB,GAAG,KAAK,CAAC;QAc5B,mBAAc,GAAG,CAAC,CAAC;QACnB,4BAAuB,GAAG,CAAC,CAAC;QAC5B,gBAAW,GAAG,KAAK,CAAC;QAyM5B;;;;;WAKG;QACI,2BAAsB,GAAG,CAAC,eAAe,EAAE,EAAE,CACnD,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,iBAAiB,KAAK,eAAe;YACnE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM;YAChB,CAAC,CAAC,SAAS,CAAC;QAEd,wGAAwG;QAChG,mCAA8B,GAAG,KAAK,CAAC;QAjM9C,MAAM,cAAc,GAA8B;YACjD,cAAc,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,cAAc;YACzC,4BAA4B,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,uBAAuB;SAChE,CAAC;QAEF,IAAI,CAAC,EAAE,GAAG,4BAA4B,CAAC;YACtC,MAAM,EAAE,UAAU;YAClB,SAAS,EAAE,SAAS;YACpB,UAAU,EAAE;gBACX,GAAG,EAAE,cAAc;aACnB;SACD,CAAC,CAAC;QAEH,IAAI,aAAa,CAAC,KAAK,KAAK,mBAAmB,EAAE;YAChD,MAAM,CACL,IAAI,CAAC,aAAa,CAAC,KAAK,KAAK,SAAS,EACtC,KAAK,CAAC,6CAA6C,CACnD,CAAC;YACF,IAAI,CAAC,eAAe,GAAG,IAAI,wBAAwB,CAClD,aAAa,EACb,IAAI,CAAC,aAAa,EAClB,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,EACrC,IAAI,CAAC,EAAE,CAAC,MAAM,CACd,CAAC;SACF;QAED,MAAM,CACL,IAAI,CAAC,aAAa,CAAC,KAAK,KAAK,UAAU,EACvC,KAAK,CAAC,yDAAyD,CAC/D,CAAC;QAEF,oGAAoG;QACpG,+FAA+F;QAE/F,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,cAAc,EAAE,uBAAuB,CAAC,CAAC;QAE5F,IAAI,CAAC,eAAe,GAAG,IAAI,YAAY,CAAC,cAAc,EAAE,GAAG,EAAE;YAC5D,+DAA+D;YAC/D,sEAAsE;YACtE,2DAA2D;YAC3D,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC;gBAC7B,SAAS,EAAE,uBAAuB;gBAClC,OAAO,EAAE,0CAA0C;gBACnD,cAAc;gBACd,uBAAuB,EAAE,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,iBAAiB;gBACzE,qBAAqB,EAAE,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,qBAAqB;gBAC3E,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,WAAW;aACpE,CAAC,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,iFAAiF;QACjF,iBAAiB,CAAC,iCAAiC,CAAC,cAAc,EAAE,GAAG,EAAE;YACxE,IAAI,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE;gBAClC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,kBAAkB,CAAC;oBACjC,SAAS,EAAE,6BAA6B;oBACxC,uBAAuB,EAAE,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,iBAAiB;oBACzE,qBAAqB,EAAE,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,qBAAqB;iBAC3E,CAAC,CAAC;gBACH,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;aAC7B;QACF,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,SAAS,GAAG,IAAI,gBAAgB,CACpC,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,qBAAqB,EAC1B,GAAG,EAAE;YACJ,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAChC,CAAC,EACD,IAAI,CAAC,cAAc,EACnB,IAAI,CAAC,EAAE,CAAC,MAAM,CACd,CAAC;QAEF,4BAA4B;QAC5B,IAAI,CAAC,eAAe,GAAG,CAAC,EAA6B,EAAE,cAAwB,EAAE,EAAE;YAClF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,cAAc,KAAK,IAAI,CAAC,CAAC;QAC5C,CAAC,CAAC;QACF,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;IAC7C,CAAC;IA5LM,MAAM,CAAC,KAAK,CAAC,KAAK,CACxB,MAA4B,EAC5B,cAAqC,EACrC,aAAoC,EACpC,qBAAuF,EACvF,+BAAsF,EACtF,aAAsC,EACtC,iBAAoC,EACpC,iBAA4C,EAC5C,sBAA8D,EAC9D,OAA2B;QAE3B,MAAM,UAAU,GAAG,IAAI,iBAAiB,CACvC,MAAM,EACN,cAAc,EACd,aAAa,EACb,qBAAqB,EACrB,+BAA+B,EAC/B,aAAa,EACb,iBAAiB,EACjB,iBAAiB,EACjB,sBAAsB,EACtB,OAAO,CACP,CAAC;QAEF,0GAA0G;QAC1G,4FAA4F;QAC5F,MAAM,aAAa,GAAG,MAAM,UAAU,CAAC,gBAAgB,EAAE,CAAC;QAE1D,MAAM,UAAU,CAAC,SAAS,EAAE,CAAC;QAE7B,qCAAqC;QACrC,sGAAsG;QACtG,UAAU,CAAC,0BAA0B,CAAC,aAAa,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACpE,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,cAAc,CAC3C,EAAE,SAAS,EAAE,4BAA4B,EAAE,EAC3C,KAAK,CACL,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,0BAA0B;QAC1B,+FAA+F;QAC/F,uCAAuC;QACvC,yGAAyG;QACzG,wFAAwF;QACxF,6GAA6G;QAC7G,MAAM,IAAI,GACT,OAAO,CAAC,YAAY,CAAC,kBAAkB;YACvC,CAAC,aAAa,CAAC,qBAAqB,CAAC,iBAAiB;gBACrD,aAAa,CAAC,gBAAgB;gBAC9B,aAAa,CAAC,aAAa,CAAC,CAAC;QAC/B,aAAa,CAAC,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC;QAE1C,IAAI,aAAa,CAAC,gBAAgB,EAAE;YACnC,8DAA8D;YAC9D,aAAa,CAAC,gBAAgB,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;YACtD,aAAa,CAAC,aAAa,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;SACpD;QAED,+EAA+E;QAC/E,aAAa,CAAC,oBAAoB,GAAG,OAAO,CAAC,YAAY,CAAC,kBAAkB,CAAC;QAE7E,mBAAmB;QACnB,UAAU,CAAC,eAAe,EAAE,KAAK,EAAE,CAAC;QACpC,UAAU,CAAC,eAAe,EAAE,GAAG,EAAE,CAAC;QAElC,OAAO,UAAU,CAAC;IACnB,CAAC;IAED,IAAW,QAAQ;QAClB,OAAO,IAAI,CAAC,SAAS,CAAC;IACvB,CAAC;IAuHO,KAAK,CAAC,gBAAgB;QAC7B,MAAM,OAAO,GAA8B,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC;QAC5E,IAAI,iBAAiB,GAAG,CAAC,CAAC,CAAC;QAC3B,+DAA+D;QAC/D,IAAI,OAAO,KAAK,SAAS,EAAE;YAC1B,iBAAiB,GAAG,OAAO,CAAC,SAAS,CAAC,uBAAuB,CAAC;YAC9D,MAAM,aAAa,GAAG,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC;YACvF,MAAM,eAAe,GAAG,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC1D,MAAM,gBAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC5D,OAAO,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE;gBAC1C,aAAa,CAAC,kBAAkB,CAAC;oBAChC,SAAS,EAAE,qCAAqC;oBAChD,uBAAuB,EAAE,iBAAiB;oBAC1C,cAAc,EAAE,eAAe;oBAC/B,SAAS,EAAE,gBAAgB;iBAC3B,CAAC,CAAC;gBACH,MAAM,IAAI,CAAC,eAAe,CAAC;aAC3B;YAED,uEAAuE;YACvE,yCAAyC;YACzC,oEAAoE;YACpE,MAAM,IAAI,CAAC,mBAAmB,CAC7B,GAAG,EAAE,GAAE,CAAC,EACR,KAAK,IAAI,EAAE,CACV,IAAI,CAAC,+BAA+B,CAAC;gBACpC,cAAc,EAAE,eAAe;gBAC/B,SAAS,EAAE,gBAAgB;gBAC3B,aAAa,EAAE,iBAAiB;gBAChC,aAAa;aACb,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;gBACxB,wFAAwF;gBACxF,wFAAwF;gBACxF,wFAAwF;gBACxF,uFAAuF;gBACvF,uFAAuF;gBACvF,sEAAsE;gBACtE,MAAM,cAAc,GACnB,YAAY,CAAC,KAAK,CAAC;oBACnB,KAAK,CAAC,SAAS,KAAK,eAAe,CAAC,+BAA+B,CAAC;gBAErE,aAAa,CAAC,kBAAkB,CAC/B;oBACC,SAAS,EAAE,cAAc;wBACxB,CAAC,CAAC,8BAA8B;wBAChC,CAAC,CAAC,2BAA2B;oBAC9B,uBAAuB,EAAE,iBAAiB;oBAC1C,cAAc,EAAE,eAAe;oBAC/B,SAAS,EAAE,gBAAgB;iBAC3B,EACD,KAAK,CACL,CAAC;YACH,CAAC,CAAC,EACH,GAAG,EAAE,GAAE,CAAC,CACR,CAAC;YACF,iBAAiB,EAAE,CAAC;SACpB;QACD,OAAO,iBAAiB,CAAC;IAC1B,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,0BAA0B,CAAC,aAAqB;QAC7D,IAAI,iBAAiB,GACpB,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,qBAAqB,CAAC;QACrF,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE;YACtB,MAAM,aAAa,GAAG,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC;YAEvF,mGAAmG;YACnG,6CAA6C;YAC7C,MAAM,IAAI,CAAC,iBAAiB,CAAC,cAAc,CAAC,iBAAiB,CAAC,CAAC;YAE/D,aAAa,CAAC,kBAAkB,CAAC;gBAChC,SAAS,EAAE,4BAA4B;gBACvC,uBAAuB,EAAE,iBAAiB;gBAC1C,aAAa;aACb,CAAC,CAAC;YAEH,iBAAiB,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAClD,gDAAgD;YAChD,MAAM,CAAC,iBAAiB,IAAI,CAAC,EAAE,KAAK,CAAC,iCAAiC,CAAC,CAAC;SACxE;IACF,CAAC;IAEM,OAAO;QACb,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;QAC7C,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC;QAC9B,IAAI,CAAC,eAAe,EAAE,OAAO,EAAE,CAAC;QAChC,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;QACjC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QACzB,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC7B,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC9B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACtB,CAAC;IAgBM,QAAQ,CAAC,EAA6B,EAAE,cAAuB;QACrE,IAAI,CAAC,aAAa,CAAC,oBAAoB,GAAG,EAAE,CAAC,cAAc,CAAC;QAE5D,IAAI,cAAc,EAAE;YACnB,IAAI,CAAC,aAAa,CAAC,aAAa,EAAE,CAAC;SACnC;aAAM;YACN,IAAI,CAAC,aAAa,CAAC,gBAAgB,EAAE,CAAC;SACtC;QAED,IAAI,CAAC,aAAa,CAAC,YAAY,IAAI,MAAM,CAAC,EAAE,CAAC,CAAC;QAE9C,6EAA6E;QAC7E,IACC,IAAI,CAAC,WAAW;YAChB,IAAI,CAAC,mBAAmB,CAAC,EAAE,EAAE,cAAc,CAAC;YAC5C,CAAC,IAAI,CAAC,qBAAqB,EAAE;YAC7B,CAAC,IAAI,CAAC,8BAA8B,EACnC;YACD,IAAI,CAAC,8BAA8B,GAAG,IAAI,CAAC;YAC3C,OAAO,CAAC,OAAO,EAAE;iBACf,IAAI,CAAC,GAAG,EAAE;gBACV,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,CAAC;YAC7B,CAAC,CAAC;iBACD,OAAO,CAAC,GAAG,EAAE;gBACb,IAAI,CAAC,8BAA8B,GAAG,KAAK,CAAC;YAC7C,CAAC,CAAC,CAAC;SACJ;IACF,CAAC;IAED;;;;;OAKG;IACK,mBAAmB,CAAC,EAA6B,EAAE,cAAuB;QACjF,QAAQ,EAAE,CAAC,IAAI,EAAE;YAChB,KAAK,WAAW,CAAC,SAAS,CAAC;YAC3B,KAAK,WAAW,CAAC,UAAU,CAAC;YAC5B,KAAK,WAAW,CAAC,WAAW;gBAC3B,OAAO,KAAK,CAAC;YACd;gBACC,OAAO,cAAc,IAAI,IAAI,CAAC,6BAA6B,EAAE,CAAC;SAC/D;IACF,CAAC;IAEO,6BAA6B;QACpC,MAAM,eAAe,GACpB,IAAI,CAAC,aAAa,CAAC,oBAAoB;YACvC,IAAI,CAAC,aAAa,CAAC,qBAAqB,CAAC,iBAAiB,CAAC;QAC5D,OAAO,CACN,IAAI,CAAC,aAAa,CAAC,KAAK,KAAK,SAAS;YACtC,CAAC,IAAI,CAAC,aAAa,CAAC,4BAA4B,KAAK,SAAS;gBAC7D,IAAI,CAAC,aAAa,CAAC,4BAA4B,IAAI,eAAe,CAAC,CACpE,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,QAAQ,CAAC,gBAAyB;QAC9C,IAAI,IAAI,CAAC,QAAQ,EAAE;YAClB,OAAO;SACP;QAED,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QAErB,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAE9B,8CAA8C;QAC9C,IAAI,gBAAgB,IAAI,IAAI,CAAC,eAAe,EAAE,oBAAoB,EAAE,EAAE;YACrE,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE;gBACvC,IAAI,CAAC,gBAAgB;gBACpB,iBAAiB;gBACjB,EAAE,MAAM,EAAE,aAAa,EAAE;gBACzB,kFAAkF;gBAClF,EAAE,CACF,CAAC;aACF;SACD;QAED,+EAA+E;QAC/E,qFAAqF;QACrF,6FAA6F;QAC7F,0FAA0F;QAC1F,uEAAuE;QACvE,MAAM,IAAI,CAAC,eAAe,CAAC;IAC5B,CAAC;IAEO,KAAK,CAAC,SAAS;QACtB,kDAAkD;QAClD,MAAM,eAAe,GAAG,MAAM,SAAS,CACtC,IAAI,CAAC,cAAc,CAAC,WAAW,EAAE,EACjC,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAC5B,CAAC;QACF,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAE7B,sEAAsE;QACtE,uEAAuE;QACvE,IAAI,CAAC,iBAAiB,CAAC,mCAAmC,EAAE,CAAC;QAE7D,IAAI,eAAe,CAAC,MAAM,KAAK,MAAM,IAAI,eAAe,CAAC,KAAK,KAAK,SAAS,EAAE;YAC7E,IAAI,CAAC,aAAa,CAAC,4BAA4B,CAAC;gBAC/C,iBAAiB,EAAE,eAAe,CAAC,KAAK,CAAC,SAAS,CAAC,uBAAuB;gBAC1E,2FAA2F;gBAC3F,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;gBACvB,qBAAqB,EAAE,eAAe,CAAC,KAAK,CAAC,SAAS,CAAC,cAAc;aACrE,CAAC,CAAC;SACH;QACD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IACzB,CAAC;IAEO,mBAAmB;QAC1B,IAAI,CAAC,cAAc,EAAE,CAAC;IACvB,CAAC;IAEO,kBAAkB;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,mBAAmB,CAAC;QACvC,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;QAEjC,2EAA2E;QAC3E,yDAAyD;QACzD,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,IAAI,KAAK,EAAE;YAC7D,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,CAAC;SAC5B;IACF,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,mBAAmB,CAChC,MAAkB,EAClB,MAAwB,EACxB,KAAiB;QAEjB,MAAM,CACL,IAAI,CAAC,eAAe,KAAK,SAAS,EAClC,KAAK,CAAC,+CAA+C,CACrD,CAAC;QAEF,MAAM,eAAe,GAAG,IAAI,QAAQ,EAAQ,CAAC;QAC7C,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC,OAAO,CAAC;QAE/C,MAAM,EAAE,CAAC;QAET,OAAO,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;YAC5B,eAAe,CAAC,OAAO,EAAE,CAAC;YAC1B,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;YACjC,KAAK,EAAE,CAAC;QACT,CAAC,CAAC,CAAC;IACJ,CAAC;IAED;;;;;;;OAOG;IACK,gBAAgB,CACvB,cAA6C,EAC7C,OAA0B,EAC1B,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,EAC1C,cAAc,GAAG,IAAI,sBAAsB,EAAE;QAE7C,IAAI,CAAC,mBAAmB,CACvB,GAAG,EAAE;YACJ,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC5B,CAAC,EACD,KAAK,IAAI,EAAE;YACV,MAAM,eAAe,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAC/C,cAAc,EACd,OAAO,EACP,iBAAiB,EACjB,cAAc,CACd,CAAC;YACF,6CAA6C;YAC7C,OAAO,eAAe,CAAC,wBAAwB,CAAC;QACjD,CAAC,EACD,GAAG,EAAE;YACJ,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC3B,CAAC,CACD,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACjB,6FAA6F;YAC7F,oBAAoB;YACpB,4FAA4F;YAC5F,iCAAiC;QAClC,CAAC,CAAC,CAAC;QAEH,OAAO,cAAc,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;IAED,oCAAoC;IAC5B,YAAY,CACnB,MAAuB,EACvB,iBAAiB,GAAG,IAAI,CAAC,iBAAiB;QAE1C,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE;YACvC,yFAAyF;YACzF,uBAAuB;YACvB,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;YAChC,OAAO;SACP;QAED,IAAI,CAAC,mBAAmB,CACvB,GAAG,EAAE;YACJ,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC5B,CAAC,EACD,KAAK,IAAI,EAAE;YACV,MAAM,QAAQ,GAAwB;gBACrC,EAAE,gBAAgB,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE;gBAC5C,EAAE,gBAAgB,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE;aAC3C,CAAC;YACF,IAAI,oBAAwC,CAAC;YAC7C,IAAI,eAAe,GAAG,CAAC,CAAC;YACxB,IAAI,uBAAuB,GAAG,CAAC,CAAC;YAChC,IAAI,mBAAmB,GAAG,CAAC,CAAC;YAC5B,OAAO,mBAAmB,GAAG,QAAQ,CAAC,MAAM,EAAE;gBAC7C,IAAI,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE;oBACrC,OAAO;iBACP;gBAED,iEAAiE;gBACjE,IAAI,EAAE,eAAe,GAAG,CAAC,IAAI,MAAM,KAAK,aAAa,EAAE;oBACtD,OAAO;iBACP;gBAED,uBAAuB,EAAE,CAAC;gBAE1B,MAAM,gBAAgB,GAAG,QAAQ,CAAC,mBAAmB,CAAC,CAAC;gBACvD,MAAM,cAAc,GAAkC;oBACrD,MAAM;oBACN,eAAe;oBACf,uBAAuB;oBACvB,mBAAmB,EAAE,mBAAmB,GAAG,CAAC;oBAC5C,GAAG,gBAAgB;iBACnB,CAAC;gBAEF,wEAAwE;gBACxE,2FAA2F;gBAC3F,MAAM,eAAe,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAC/C,cAAc,EACd,gBAAgB,EAChB,iBAAiB,CACjB,CAAC;gBACF,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,wBAAwB,CAAC;gBAE9D,IAAI,MAAM,CAAC,OAAO,EAAE;oBACnB,OAAO;iBACP;gBAED,8EAA8E;gBAC9E,yDAAyD;gBACzD,MAAM,YAAY,GAAG,MAAM,CAAC,iBAAiB,CAAC;gBAC9C,IAAI,YAAY,KAAK,SAAS,IAAI,uBAAuB,GAAG,CAAC,EAAE;oBAC9D,mBAAmB,EAAE,CAAC;oBACtB,uBAAuB,GAAG,CAAC,CAAC;iBAC5B;gBAED,IAAI,YAAY,KAAK,SAAS,EAAE;oBAC/B,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC;wBACnC,SAAS,EAAE,uBAAuB;wBAClC,QAAQ,EAAE,YAAY;wBACtB,gBAAgB,EAAE,oBAAoB,KAAK,SAAS;wBACpD,GAAG,cAAc;qBACjB,CAAC,CAAC;oBACH,MAAM,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;iBACjC;aACD;YAED,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,CAAC;QAChD,CAAC,EACD,GAAG,EAAE;YACJ,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC3B,CAAC,CACD,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACjB,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,0BAA0B,EAAE,EAAE,KAAK,CAAC,CAAC;QACjF,CAAC,CAAC,CAAC;IACJ,CAAC;IAED,8DAA8D;IACvD,iBAAiB,CACvB,iBAAyC,IAAI,sBAAsB,EAAE,EACrE,EAAE,MAAM,EAAE,GAAG,OAAO,EAA6B;QAEjD,IAAI,IAAI,CAAC,QAAQ,EAAE;YAClB,cAAc,CAAC,IAAI,CAAC,uCAAuC,EAAE,SAAS,CAAC,CAAC;YACxE,OAAO,cAAc,CAAC,KAAK,EAAE,CAAC;SAC9B;QACD,0DAA0D;QAC1D,8DAA8D;QAC9D,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE;YACvC,6DAA6D;YAC7D,MAAM,IAAI,UAAU,CAAC,0DAA0D,CAAC,CAAC;SACjF;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CACnC,EAAE,MAAM,EAAE,YAAY,MAAM,EAAE,EAAE,EAChC,OAAO,EACP,IAAI,CAAC,iBAAiB,EACtB,cAAc,CACd,CAAC;QACF,OAAO,MAAM,CAAC;IACf,CAAC;IAED,6DAA6D;IACtD,gBAAgB,CAAC,EACvB,MAAM,EACN,mBAAmB,GAAG,CAAC,EACvB,QAAQ,GAAG,KAAK,EAChB,GAAG,OAAO,EACgB;QAC1B,MAAM,cAAc,GAAG,WAAW,MAAM,EAAW,CAAC;QACpD,IAAI,UAAU,GAAG,KAAK,CAAC;QACvB,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE;YACvC,IAAI,CAAC,QAAQ,EAAE;gBACd,OAAO,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;aACjC;YACD,gDAAgD;YAChD,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,IAAI,CACvC,0DAA0D,EAC1D,SAAS,CACT,CAAC;YACF,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;YACjC,UAAU,GAAG,IAAI,CAAC;SAClB;QACD,IAAI,CAAC,eAAe,GAAG;YACtB,MAAM,EAAE,cAAc;YACtB,mBAAmB;YACnB,OAAO;YACP,cAAc,EAAE,IAAI,sBAAsB,EAAE;SAC5C,CAAC;QACF,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC5D,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAC7B,OAAO,UAAU;YAChB,CAAC,CAAC;gBACA,GAAG,OAAO;gBACV,eAAe,EAAE,IAAI;gBACrB,UAAU,EAAE,IAAI;aACf;YACH,CAAC,CAAC,OAAO,CAAC;IACZ,CAAC;IAEO,qBAAqB;QAC5B,IAAI,IAAI,CAAC,QAAQ,EAAE;YAClB,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAC9B,OAAO,KAAK,CAAC;SACb;QACD,IACC,IAAI,CAAC,eAAe,KAAK,SAAS;YAClC,IAAI,CAAC,aAAa,CAAC,oBAAoB,GAAG,IAAI,CAAC,eAAe,CAAC,mBAAmB;YAClF,IAAI,CAAC,eAAe,KAAK,SAAS,EACjC;YACD,uFAAuF;YACvF,OAAO,KAAK,CAAC;SACb;QACD,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC;QACjE,+FAA+F;QAC/F,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;QACjC,IAAI,CAAC,gBAAgB,CACpB,EAAE,MAAM,EAAE,mBAAmB,MAAM,EAAE,EAAE,EACvC,OAAO,EACP,IAAI,CAAC,iBAAiB,EACtB,cAAc,CACd,CAAC;QACF,OAAO,IAAI,CAAC;IACb,CAAC;IAEO,sBAAsB;QAC7B,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE;YACvC,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,IAAI,CACvC,uCAAuC,EACvC,SAAS,CACT,CAAC;YACF,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;SACjC;IACF,CAAC;CACD","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { IDisposable, ITelemetryBaseLogger } from \"@fluidframework/core-interfaces\";\nimport {\n\tisFluidError,\n\tMonitoringContext,\n\tcreateChildMonitoringContext,\n\tcreateChildLogger,\n} from \"@fluidframework/telemetry-utils\";\nimport { assert, delay, Deferred, PromiseTimer } from \"@fluidframework/common-utils\";\nimport { UsageError } from \"@fluidframework/container-utils\";\nimport { DriverErrorType } from \"@fluidframework/driver-definitions\";\nimport { ISequencedDocumentMessage, MessageType } from \"@fluidframework/protocol-definitions\";\nimport { ISummaryConfiguration } from \"../containerRuntime\";\nimport { opSize } from \"../opProperties\";\nimport { SummarizeHeuristicRunner } from \"./summarizerHeuristics\";\nimport {\n\tIEnqueueSummarizeOptions,\n\tISummarizeOptions,\n\tISummarizeHeuristicData,\n\tISummarizeHeuristicRunner,\n\tIOnDemandSummarizeOptions,\n\tEnqueueSummarizeResult,\n\tSummarizerStopReason,\n\tISubmitSummaryOptions,\n\tSubmitSummaryResult,\n\tISummaryCancellationToken,\n\tISummarizeResults,\n\tISummarizeTelemetryProperties,\n\tISummarizerRuntime,\n\tISummarizeRunnerTelemetry,\n\tIRefreshSummaryAckOptions,\n} from \"./summarizerTypes\";\nimport { IAckedSummary, IClientSummaryWatcher, SummaryCollection } from \"./summaryCollection\";\nimport {\n\traceTimer,\n\tSummarizeReason,\n\tSummarizeResultBuilder,\n\tSummaryGenerator,\n} from \"./summaryGenerator\";\n\nconst maxSummarizeAckWaitTime = 10 * 60 * 1000; // 10 minutes\n\n/**\n * An instance of RunningSummarizer manages the heuristics for summarizing.\n * Until disposed, the instance of RunningSummarizer can assume that it is\n * in a state of running, meaning it is connected and initialized.  It keeps\n * track of summaries that it is generating as they are broadcast and acked/nacked.\n * This object is created and controlled by Summarizer object.\n */\nexport class RunningSummarizer implements IDisposable {\n\tpublic static async start(\n\t\tlogger: ITelemetryBaseLogger,\n\t\tsummaryWatcher: IClientSummaryWatcher,\n\t\tconfiguration: ISummaryConfiguration,\n\t\tsubmitSummaryCallback: (options: ISubmitSummaryOptions) => Promise<SubmitSummaryResult>,\n\t\trefreshLatestSummaryAckCallback: (options: IRefreshSummaryAckOptions) => Promise<void>,\n\t\theuristicData: ISummarizeHeuristicData,\n\t\tsummaryCollection: SummaryCollection,\n\t\tcancellationToken: ISummaryCancellationToken,\n\t\tstopSummarizerCallback: (reason: SummarizerStopReason) => void,\n\t\truntime: ISummarizerRuntime,\n\t): Promise<RunningSummarizer> {\n\t\tconst summarizer = new RunningSummarizer(\n\t\t\tlogger,\n\t\t\tsummaryWatcher,\n\t\t\tconfiguration,\n\t\t\tsubmitSummaryCallback,\n\t\t\trefreshLatestSummaryAckCallback,\n\t\t\theuristicData,\n\t\t\tsummaryCollection,\n\t\t\tcancellationToken,\n\t\t\tstopSummarizerCallback,\n\t\t\truntime,\n\t\t);\n\n\t\t// Before doing any heuristics or proceeding with its refreshing, if there is a summary ack received while\n\t\t// this summarizer catches up, let's refresh state before proceeding with the summarization.\n\t\tconst lastAckRefSeq = await summarizer.handleSummaryAck();\n\n\t\tawait summarizer.waitStart();\n\n\t\t// Handle summary acks asynchronously\n\t\t// Note: no exceptions are thrown from processIncomingSummaryAcks handler as it handles all exceptions\n\t\tsummarizer.processIncomingSummaryAcks(lastAckRefSeq).catch((error) => {\n\t\t\tcreateChildLogger({ logger }).sendErrorEvent(\n\t\t\t\t{ eventName: \"HandleSummaryAckFatalError\" },\n\t\t\t\terror,\n\t\t\t);\n\t\t});\n\n\t\t// Update heuristic counts\n\t\t// By the time we get here, there are potentially ops missing from the heuristic summary counts\n\t\t// Examples of where this could happen:\n\t\t// 1. Op is processed during the time that we are initiating the RunningSummarizer instance but before we\n\t\t//    listen for the op events (will get missed by the handlers in the current workflow)\n\t\t// 2. Op was sequenced after the last time we summarized (op sequence number > summarize ref sequence number)\n\t\tconst diff =\n\t\t\truntime.deltaManager.lastSequenceNumber -\n\t\t\t(heuristicData.lastSuccessfulSummary.refSequenceNumber +\n\t\t\t\theuristicData.numNonRuntimeOps +\n\t\t\t\theuristicData.numRuntimeOps);\n\t\theuristicData.hasMissingOpData = diff > 0;\n\n\t\tif (heuristicData.hasMissingOpData) {\n\t\t\t// Split the diff 50-50 and increment the counts appropriately\n\t\t\theuristicData.numNonRuntimeOps += Math.ceil(diff / 2);\n\t\t\theuristicData.numRuntimeOps += Math.floor(diff / 2);\n\t\t}\n\n\t\t// Update last seq number (in case the handlers haven't processed anything yet)\n\t\theuristicData.lastOpSequenceNumber = runtime.deltaManager.lastSequenceNumber;\n\n\t\t// Start heuristics\n\t\tsummarizer.heuristicRunner?.start();\n\t\tsummarizer.heuristicRunner?.run();\n\n\t\treturn summarizer;\n\t}\n\n\tpublic get disposed() {\n\t\treturn this._disposed;\n\t}\n\tprivate stopping = false;\n\tprivate _disposed = false;\n\tprivate summarizingLock: Promise<void> | undefined;\n\tprivate tryWhileSummarizing = false;\n\tprivate readonly pendingAckTimer: PromiseTimer;\n\tprivate heuristicRunner?: ISummarizeHeuristicRunner;\n\tprivate readonly generator: SummaryGenerator;\n\tprivate readonly mc: MonitoringContext;\n\n\tprivate enqueuedSummary:\n\t\t| {\n\t\t\t\treason: SummarizeReason;\n\t\t\t\tafterSequenceNumber: number;\n\t\t\t\toptions: ISummarizeOptions;\n\t\t\t\treadonly resultsBuilder: SummarizeResultBuilder;\n\t\t  }\n\t\t| undefined;\n\tprivate summarizeCount = 0;\n\tprivate totalSuccessfulAttempts = 0;\n\tprivate initialized = false;\n\n\tprivate readonly runtimeListener;\n\n\tprivate constructor(\n\t\tbaseLogger: ITelemetryBaseLogger,\n\t\tprivate readonly summaryWatcher: IClientSummaryWatcher,\n\t\tprivate readonly configuration: ISummaryConfiguration,\n\t\tprivate readonly submitSummaryCallback: (\n\t\t\toptions: ISubmitSummaryOptions,\n\t\t) => Promise<SubmitSummaryResult>,\n\t\tprivate readonly refreshLatestSummaryAckCallback: (\n\t\t\toptions: IRefreshSummaryAckOptions,\n\t\t) => Promise<void>,\n\t\tprivate readonly heuristicData: ISummarizeHeuristicData,\n\t\tprivate readonly summaryCollection: SummaryCollection,\n\t\tprivate readonly cancellationToken: ISummaryCancellationToken,\n\t\tprivate readonly stopSummarizerCallback: (reason: SummarizerStopReason) => void,\n\t\tprivate readonly runtime: ISummarizerRuntime,\n\t) {\n\t\tconst telemetryProps: ISummarizeRunnerTelemetry = {\n\t\t\tsummarizeCount: () => this.summarizeCount,\n\t\t\tsummarizerSuccessfulAttempts: () => this.totalSuccessfulAttempts,\n\t\t};\n\n\t\tthis.mc = createChildMonitoringContext({\n\t\t\tlogger: baseLogger,\n\t\t\tnamespace: \"Running\",\n\t\t\tproperties: {\n\t\t\t\tall: telemetryProps,\n\t\t\t},\n\t\t});\n\n\t\tif (configuration.state !== \"disableHeuristics\") {\n\t\t\tassert(\n\t\t\t\tthis.configuration.state === \"enabled\",\n\t\t\t\t0x2ea /* \"Configuration state should be enabled\" */,\n\t\t\t);\n\t\t\tthis.heuristicRunner = new SummarizeHeuristicRunner(\n\t\t\t\theuristicData,\n\t\t\t\tthis.configuration,\n\t\t\t\t(reason) => this.trySummarize(reason),\n\t\t\t\tthis.mc.logger,\n\t\t\t);\n\t\t}\n\n\t\tassert(\n\t\t\tthis.configuration.state !== \"disabled\",\n\t\t\t0x2eb /* \"Summary not supported with configuration disabled\" */,\n\t\t);\n\n\t\t// Cap the maximum amount of time client will wait for a summarize op ack to maxSummarizeAckWaitTime\n\t\t// configuration.maxAckWaitTime is composed from defaults, server values, and runtime overrides\n\n\t\tconst maxAckWaitTime = Math.min(this.configuration.maxAckWaitTime, maxSummarizeAckWaitTime);\n\n\t\tthis.pendingAckTimer = new PromiseTimer(maxAckWaitTime, () => {\n\t\t\t// Note: summarizeCount (from ChildLogger definition) may be 0,\n\t\t\t// since this code path is hit when RunningSummarizer first starts up,\n\t\t\t// before this instance has kicked off a new summarize run.\n\t\t\tthis.mc.logger.sendErrorEvent({\n\t\t\t\teventName: \"SummaryAckWaitTimeout\",\n\t\t\t\tmessage: \"Pending summary ack not received in time\",\n\t\t\t\tmaxAckWaitTime,\n\t\t\t\treferenceSequenceNumber: this.heuristicData.lastAttempt.refSequenceNumber,\n\t\t\t\tsummarySequenceNumber: this.heuristicData.lastAttempt.summarySequenceNumber,\n\t\t\t\ttimePending: Date.now() - this.heuristicData.lastAttempt.summaryTime,\n\t\t\t});\n\t\t});\n\t\t// Set up pending ack timeout by op timestamp differences for previous summaries.\n\t\tsummaryCollection.setPendingAckTimerTimeoutCallback(maxAckWaitTime, () => {\n\t\t\tif (this.pendingAckTimer.hasTimer) {\n\t\t\t\tthis.mc.logger.sendTelemetryEvent({\n\t\t\t\t\teventName: \"MissingSummaryAckFoundByOps\",\n\t\t\t\t\treferenceSequenceNumber: this.heuristicData.lastAttempt.refSequenceNumber,\n\t\t\t\t\tsummarySequenceNumber: this.heuristicData.lastAttempt.summarySequenceNumber,\n\t\t\t\t});\n\t\t\t\tthis.pendingAckTimer.clear();\n\t\t\t}\n\t\t});\n\n\t\tthis.generator = new SummaryGenerator(\n\t\t\tthis.pendingAckTimer,\n\t\t\tthis.heuristicData,\n\t\t\tthis.submitSummaryCallback,\n\t\t\t() => {\n\t\t\t\tthis.totalSuccessfulAttempts++;\n\t\t\t},\n\t\t\tthis.summaryWatcher,\n\t\t\tthis.mc.logger,\n\t\t);\n\n\t\t// Listen to runtime for ops\n\t\tthis.runtimeListener = (op: ISequencedDocumentMessage, runtimeMessage?: boolean) => {\n\t\t\tthis.handleOp(op, runtimeMessage === true);\n\t\t};\n\t\tthis.runtime.on(\"op\", this.runtimeListener);\n\t}\n\n\tprivate async handleSummaryAck(): Promise<number> {\n\t\tconst lastAck: IAckedSummary | undefined = this.summaryCollection.latestAck;\n\t\tlet refSequenceNumber = -1;\n\t\t// In case we haven't received the lastestAck yet, just return.\n\t\tif (lastAck !== undefined) {\n\t\t\trefSequenceNumber = lastAck.summaryOp.referenceSequenceNumber;\n\t\t\tconst summaryLogger = this.tryGetCorrelatedLogger(refSequenceNumber) ?? this.mc.logger;\n\t\t\tconst summaryOpHandle = lastAck.summaryOp.contents.handle;\n\t\t\tconst summaryAckHandle = lastAck.summaryAck.contents.handle;\n\t\t\twhile (this.summarizingLock !== undefined) {\n\t\t\t\tsummaryLogger.sendTelemetryEvent({\n\t\t\t\t\teventName: \"RefreshAttemptWithSummarizerRunning\",\n\t\t\t\t\treferenceSequenceNumber: refSequenceNumber,\n\t\t\t\t\tproposalHandle: summaryOpHandle,\n\t\t\t\t\tackHandle: summaryAckHandle,\n\t\t\t\t});\n\t\t\t\tawait this.summarizingLock;\n\t\t\t}\n\n\t\t\t// Make sure we block any summarizer from being executed/enqueued while\n\t\t\t// executing the refreshLatestSummaryAck.\n\t\t\t// https://dev.azure.com/fluidframework/internal/_workitems/edit/779\n\t\t\tawait this.lockedSummaryAction(\n\t\t\t\t() => {},\n\t\t\t\tasync () =>\n\t\t\t\t\tthis.refreshLatestSummaryAckCallback({\n\t\t\t\t\t\tproposalHandle: summaryOpHandle,\n\t\t\t\t\t\tackHandle: summaryAckHandle,\n\t\t\t\t\t\tsummaryRefSeq: refSequenceNumber,\n\t\t\t\t\t\tsummaryLogger,\n\t\t\t\t\t}).catch(async (error) => {\n\t\t\t\t\t\t// If the error is 404, so maybe the fetched version no longer exists on server. We just\n\t\t\t\t\t\t// ignore this error in that case, as that means we will have another summaryAck for the\n\t\t\t\t\t\t// latest version with which we will refresh the state. However in case of single commit\n\t\t\t\t\t\t// summary, we might me missing a summary ack, so in that case we are still fine as the\n\t\t\t\t\t\t// code in `submitSummary` function in container runtime, will refresh the latest state\n\t\t\t\t\t\t// by calling `refreshLatestSummaryAckFromServer` and we will be fine.\n\t\t\t\t\t\tconst isIgnoredError =\n\t\t\t\t\t\t\tisFluidError(error) &&\n\t\t\t\t\t\t\terror.errorType === DriverErrorType.fileNotFoundOrAccessDeniedError;\n\n\t\t\t\t\t\tsummaryLogger.sendTelemetryEvent(\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\teventName: isIgnoredError\n\t\t\t\t\t\t\t\t\t? \"HandleSummaryAckErrorIgnored\"\n\t\t\t\t\t\t\t\t\t: \"HandleLastSummaryAckError\",\n\t\t\t\t\t\t\t\treferenceSequenceNumber: refSequenceNumber,\n\t\t\t\t\t\t\t\tproposalHandle: summaryOpHandle,\n\t\t\t\t\t\t\t\tackHandle: summaryAckHandle,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\terror,\n\t\t\t\t\t\t);\n\t\t\t\t\t}),\n\t\t\t\t() => {},\n\t\t\t);\n\t\t\trefSequenceNumber++;\n\t\t}\n\t\treturn refSequenceNumber;\n\t}\n\n\t/**\n\t * Responsible for receiving and processing all the summaryAcks.\n\t * In case there was a summary ack processed by the running summarizer before processIncomingSummaryAcks is called,\n\t * it will wait for the summary ack that is newer than the one indicated by the lastAckRefSeq.\n\t * @param lastAckRefSeq - Identifies the minimum reference sequence number the summarizer needs to wait for.\n\t * In case of a negative number, the summarizer will wait for ANY summary ack that is greater than the deltaManager's initial sequence number,\n\t * and, in case of a positive one, it will wait for a summary ack that is greater than this current reference sequence number.\n\t */\n\tprivate async processIncomingSummaryAcks(lastAckRefSeq: number) {\n\t\tlet refSequenceNumber =\n\t\t\tlastAckRefSeq > 0 ? lastAckRefSeq : this.runtime.deltaManager.initialSequenceNumber;\n\t\twhile (!this.disposed) {\n\t\t\tconst summaryLogger = this.tryGetCorrelatedLogger(refSequenceNumber) ?? this.mc.logger;\n\n\t\t\t// Initialize ack with undefined if exception happens inside of waitSummaryAck on second iteration,\n\t\t\t// we record undefined, not previous handles.\n\t\t\tawait this.summaryCollection.waitSummaryAck(refSequenceNumber);\n\n\t\t\tsummaryLogger.sendTelemetryEvent({\n\t\t\t\teventName: \"processIncomingSummaryAcks\",\n\t\t\t\treferenceSequenceNumber: refSequenceNumber,\n\t\t\t\tlastAckRefSeq,\n\t\t\t});\n\n\t\t\trefSequenceNumber = await this.handleSummaryAck();\n\t\t\t// A valid Summary Ack must have been processed.\n\t\t\tassert(refSequenceNumber >= 0, 0x58f /* Invalid ref sequence number */);\n\t\t}\n\t}\n\n\tpublic dispose(): void {\n\t\tthis.runtime.off(\"op\", this.runtimeListener);\n\t\tthis.summaryWatcher.dispose();\n\t\tthis.heuristicRunner?.dispose();\n\t\tthis.heuristicRunner = undefined;\n\t\tthis.generator.dispose();\n\t\tthis.pendingAckTimer.clear();\n\t\tthis.disposeEnqueuedSummary();\n\t\tthis._disposed = true;\n\t\tthis.stopping = true;\n\t}\n\n\t/**\n\t * RunningSummarizer's logger includes the sequenced index of the current summary on each event.\n\t * If some other Summarizer code wants that event on their logs they can get it here,\n\t * but only if they're logging about that same summary.\n\t * @param summaryOpRefSeq - RefSeq number of the summary op, to ensure the log correlation will be correct\n\t */\n\tpublic tryGetCorrelatedLogger = (summaryOpRefSeq) =>\n\t\tthis.heuristicData.lastAttempt.refSequenceNumber === summaryOpRefSeq\n\t\t\t? this.mc.logger\n\t\t\t: undefined;\n\n\t/** We only want a single heuristic runner micro-task (will provide better optimized grouping of ops) */\n\tprivate heuristicRunnerMicroTaskExists = false;\n\n\tpublic handleOp(op: ISequencedDocumentMessage, runtimeMessage: boolean) {\n\t\tthis.heuristicData.lastOpSequenceNumber = op.sequenceNumber;\n\n\t\tif (runtimeMessage) {\n\t\t\tthis.heuristicData.numRuntimeOps++;\n\t\t} else {\n\t\t\tthis.heuristicData.numNonRuntimeOps++;\n\t\t}\n\n\t\tthis.heuristicData.totalOpsSize += opSize(op);\n\n\t\t// Check for enqueued on-demand summaries; Intentionally do nothing otherwise\n\t\tif (\n\t\t\tthis.initialized &&\n\t\t\tthis.opCanTriggerSummary(op, runtimeMessage) &&\n\t\t\t!this.tryRunEnqueuedSummary() &&\n\t\t\t!this.heuristicRunnerMicroTaskExists\n\t\t) {\n\t\t\tthis.heuristicRunnerMicroTaskExists = true;\n\t\t\tPromise.resolve()\n\t\t\t\t.then(() => {\n\t\t\t\t\tthis.heuristicRunner?.run();\n\t\t\t\t})\n\t\t\t\t.finally(() => {\n\t\t\t\t\tthis.heuristicRunnerMicroTaskExists = false;\n\t\t\t\t});\n\t\t}\n\t}\n\n\t/**\n\t * Can the given op trigger a summary?\n\t * # Currently always prevents summaries for Summarize and SummaryAck/Nack ops\n\t * @param op - op to check\n\t * @returns true if this op can trigger a summary\n\t */\n\tprivate opCanTriggerSummary(op: ISequencedDocumentMessage, runtimeMessage: boolean): boolean {\n\t\tswitch (op.type) {\n\t\t\tcase MessageType.Summarize:\n\t\t\tcase MessageType.SummaryAck:\n\t\t\tcase MessageType.SummaryNack:\n\t\t\t\treturn false;\n\t\t\tdefault:\n\t\t\t\treturn runtimeMessage || this.nonRuntimeOpCanTriggerSummary();\n\t\t}\n\t}\n\n\tprivate nonRuntimeOpCanTriggerSummary(): boolean {\n\t\tconst opsSinceLastAck =\n\t\t\tthis.heuristicData.lastOpSequenceNumber -\n\t\t\tthis.heuristicData.lastSuccessfulSummary.refSequenceNumber;\n\t\treturn (\n\t\t\tthis.configuration.state === \"enabled\" &&\n\t\t\t(this.configuration.nonRuntimeHeuristicThreshold === undefined ||\n\t\t\t\tthis.configuration.nonRuntimeHeuristicThreshold <= opsSinceLastAck)\n\t\t);\n\t}\n\n\tpublic async waitStop(allowLastSummary: boolean): Promise<void> {\n\t\tif (this.stopping) {\n\t\t\treturn;\n\t\t}\n\n\t\tthis.stopping = true;\n\n\t\tthis.disposeEnqueuedSummary();\n\n\t\t// This will try to run lastSummary if needed.\n\t\tif (allowLastSummary && this.heuristicRunner?.shouldRunLastSummary()) {\n\t\t\tif (this.summarizingLock === undefined) {\n\t\t\t\tthis.trySummarizeOnce(\n\t\t\t\t\t// summarizeProps\n\t\t\t\t\t{ reason: \"lastSummary\" },\n\t\t\t\t\t// ISummarizeOptions, using defaults: { refreshLatestAck: false, fullTree: false }\n\t\t\t\t\t{},\n\t\t\t\t);\n\t\t\t}\n\t\t}\n\n\t\t// Note that trySummarizeOnce() call above returns right away, without waiting.\n\t\t// So we need to wait for its completion, otherwise it would be destroyed right away.\n\t\t// That said, if summary lock was taken upfront, this wait might wait on  multiple retries to\n\t\t// submit summary. We should reconsider this flow and make summarizer move to exit faster.\n\t\t// This resolves when the current pending summary gets an ack or fails.\n\t\tawait this.summarizingLock;\n\t}\n\n\tprivate async waitStart() {\n\t\t// Wait no longer than ack timeout for all pending\n\t\tconst waitStartResult = await raceTimer(\n\t\t\tthis.summaryWatcher.waitFlushed(),\n\t\t\tthis.pendingAckTimer.start(),\n\t\t);\n\t\tthis.pendingAckTimer.clear();\n\n\t\t// Remove pending ack wait timeout by op timestamp comparison, because\n\t\t// it has race conditions with summaries submitted by this same client.\n\t\tthis.summaryCollection.unsetPendingAckTimerTimeoutCallback();\n\n\t\tif (waitStartResult.result === \"done\" && waitStartResult.value !== undefined) {\n\t\t\tthis.heuristicData.updateWithLastSummaryAckInfo({\n\t\t\t\trefSequenceNumber: waitStartResult.value.summaryOp.referenceSequenceNumber,\n\t\t\t\t// This will be the Summarizer starting point so only use timestamps from client's machine.\n\t\t\t\tsummaryTime: Date.now(),\n\t\t\t\tsummarySequenceNumber: waitStartResult.value.summaryOp.sequenceNumber,\n\t\t\t});\n\t\t}\n\t\tthis.initialized = true;\n\t}\n\n\tprivate beforeSummaryAction() {\n\t\tthis.summarizeCount++;\n\t}\n\n\tprivate afterSummaryAction() {\n\t\tconst retry = this.tryWhileSummarizing;\n\t\tthis.tryWhileSummarizing = false;\n\n\t\t// After summarizing, we should check to see if we need to summarize again.\n\t\t// Rerun the heuristics and check for enqueued summaries.\n\t\tif (!this.stopping && !this.tryRunEnqueuedSummary() && retry) {\n\t\t\tthis.heuristicRunner?.run();\n\t\t}\n\t}\n\n\t/**\n\t * Runs single summary action that prevents any other concurrent actions.\n\t * Assumes that caller checked upfront for lack of concurrent action (this.summarizingLock)\n\t * before calling this API. I.e. caller is responsible for either erroring out or waiting on this promise.\n\t * @param before - set of instructions to run before running the action.\n\t * @param action - action to perform.\n\t * @param after - set of instructions to run after running the action.\n\t * @returns - result of action.\n\t */\n\tprivate async lockedSummaryAction<T>(\n\t\tbefore: () => void,\n\t\taction: () => Promise<T>,\n\t\tafter: () => void,\n\t) {\n\t\tassert(\n\t\t\tthis.summarizingLock === undefined,\n\t\t\t0x25b /* \"Caller is responsible for checking lock\" */,\n\t\t);\n\n\t\tconst summarizingLock = new Deferred<void>();\n\t\tthis.summarizingLock = summarizingLock.promise;\n\n\t\tbefore();\n\n\t\treturn action().finally(() => {\n\t\t\tsummarizingLock.resolve();\n\t\t\tthis.summarizingLock = undefined;\n\t\t\tafter();\n\t\t});\n\t}\n\n\t/**\n\t * Runs single summarize attempt\n\t * @param summarizeProps - props to log with each telemetry event associated with this attempt\n\t * @param options - summary options\n\t * @param cancellationToken - cancellation token to use to be able to cancel this summary, if needed\n\t * @param resultsBuilder - optional, result builder to use.\n\t * @returns ISummarizeResult - result of running a summary.\n\t */\n\tprivate trySummarizeOnce(\n\t\tsummarizeProps: ISummarizeTelemetryProperties,\n\t\toptions: ISummarizeOptions,\n\t\tcancellationToken = this.cancellationToken,\n\t\tresultsBuilder = new SummarizeResultBuilder(),\n\t): ISummarizeResults {\n\t\tthis.lockedSummaryAction(\n\t\t\t() => {\n\t\t\t\tthis.beforeSummaryAction();\n\t\t\t},\n\t\t\tasync () => {\n\t\t\t\tconst summarizeResult = this.generator.summarize(\n\t\t\t\t\tsummarizeProps,\n\t\t\t\t\toptions,\n\t\t\t\t\tcancellationToken,\n\t\t\t\t\tresultsBuilder,\n\t\t\t\t);\n\t\t\t\t// ensure we wait till the end of the process\n\t\t\t\treturn summarizeResult.receivedSummaryAckOrNack;\n\t\t\t},\n\t\t\t() => {\n\t\t\t\tthis.afterSummaryAction();\n\t\t\t},\n\t\t).catch((error) => {\n\t\t\t// SummaryGenerator.summarize() does not throw exceptions - it converts them to failed result\n\t\t\t// on resultsBuilder\n\t\t\t// We do not care about exceptions on receivedSummaryAckOrNack - caller should check results\n\t\t\t// and take a appropriate action.\n\t\t});\n\n\t\treturn resultsBuilder.build();\n\t}\n\n\t/** Heuristics summarize attempt. */\n\tprivate trySummarize(\n\t\treason: SummarizeReason,\n\t\tcancellationToken = this.cancellationToken,\n\t): void {\n\t\tif (this.summarizingLock !== undefined) {\n\t\t\t// lockedSummaryAction() will retry heuristic-based summary at the end of current attempt\n\t\t\t// if it's still needed\n\t\t\tthis.tryWhileSummarizing = true;\n\t\t\treturn;\n\t\t}\n\n\t\tthis.lockedSummaryAction(\n\t\t\t() => {\n\t\t\t\tthis.beforeSummaryAction();\n\t\t\t},\n\t\t\tasync () => {\n\t\t\t\tconst attempts: ISummarizeOptions[] = [\n\t\t\t\t\t{ refreshLatestAck: false, fullTree: false },\n\t\t\t\t\t{ refreshLatestAck: true, fullTree: false },\n\t\t\t\t];\n\t\t\t\tlet overrideDelaySeconds: number | undefined;\n\t\t\t\tlet summaryAttempts = 0;\n\t\t\t\tlet summaryAttemptsPerPhase = 0;\n\t\t\t\tlet summaryAttemptPhase = 0;\n\t\t\t\twhile (summaryAttemptPhase < attempts.length) {\n\t\t\t\t\tif (this.cancellationToken.cancelled) {\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\n\t\t\t\t\t// We only want to attempt 1 summary when reason is \"lastSummary\"\n\t\t\t\t\tif (++summaryAttempts > 1 && reason === \"lastSummary\") {\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\n\t\t\t\t\tsummaryAttemptsPerPhase++;\n\n\t\t\t\t\tconst summarizeOptions = attempts[summaryAttemptPhase];\n\t\t\t\t\tconst summarizeProps: ISummarizeTelemetryProperties = {\n\t\t\t\t\t\treason,\n\t\t\t\t\t\tsummaryAttempts,\n\t\t\t\t\t\tsummaryAttemptsPerPhase,\n\t\t\t\t\t\tsummaryAttemptPhase: summaryAttemptPhase + 1, // make everything 1-based\n\t\t\t\t\t\t...summarizeOptions,\n\t\t\t\t\t};\n\n\t\t\t\t\t// Note: no need to account for cancellationToken.waitCancelled here, as\n\t\t\t\t\t// this is accounted SummaryGenerator.summarizeCore that controls receivedSummaryAckOrNack.\n\t\t\t\t\tconst resultSummarize = this.generator.summarize(\n\t\t\t\t\t\tsummarizeProps,\n\t\t\t\t\t\tsummarizeOptions,\n\t\t\t\t\t\tcancellationToken,\n\t\t\t\t\t);\n\t\t\t\t\tconst result = await resultSummarize.receivedSummaryAckOrNack;\n\n\t\t\t\t\tif (result.success) {\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\n\t\t\t\t\t// Check for retryDelay that can come from summaryNack or upload summary flow.\n\t\t\t\t\t// Retry the same step only once per retryAfter response.\n\t\t\t\t\tconst delaySeconds = result.retryAfterSeconds;\n\t\t\t\t\tif (delaySeconds === undefined || summaryAttemptsPerPhase > 1) {\n\t\t\t\t\t\tsummaryAttemptPhase++;\n\t\t\t\t\t\tsummaryAttemptsPerPhase = 0;\n\t\t\t\t\t}\n\n\t\t\t\t\tif (delaySeconds !== undefined) {\n\t\t\t\t\t\tthis.mc.logger.sendPerformanceEvent({\n\t\t\t\t\t\t\teventName: \"SummarizeAttemptDelay\",\n\t\t\t\t\t\t\tduration: delaySeconds,\n\t\t\t\t\t\t\tsummaryNackDelay: overrideDelaySeconds !== undefined,\n\t\t\t\t\t\t\t...summarizeProps,\n\t\t\t\t\t\t});\n\t\t\t\t\t\tawait delay(delaySeconds * 1000);\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tthis.stopSummarizerCallback(\"failToSummarize\");\n\t\t\t},\n\t\t\t() => {\n\t\t\t\tthis.afterSummaryAction();\n\t\t\t},\n\t\t).catch((error) => {\n\t\t\tthis.mc.logger.sendErrorEvent({ eventName: \"UnexpectedSummarizeError\" }, error);\n\t\t});\n\t}\n\n\t/** {@inheritdoc (ISummarizer:interface).summarizeOnDemand} */\n\tpublic summarizeOnDemand(\n\t\tresultsBuilder: SummarizeResultBuilder = new SummarizeResultBuilder(),\n\t\t{ reason, ...options }: IOnDemandSummarizeOptions,\n\t): ISummarizeResults {\n\t\tif (this.stopping) {\n\t\t\tresultsBuilder.fail(\"RunningSummarizer stopped or disposed\", undefined);\n\t\t\treturn resultsBuilder.build();\n\t\t}\n\t\t// Check for concurrent summary attempts. If one is found,\n\t\t// return a promise that caller can await before trying again.\n\t\tif (this.summarizingLock !== undefined) {\n\t\t\t// The heuristics are blocking concurrent summarize attempts.\n\t\t\tthrow new UsageError(\"Attempted to run an already-running summarizer on demand\");\n\t\t}\n\n\t\tconst result = this.trySummarizeOnce(\n\t\t\t{ reason: `onDemand/${reason}` },\n\t\t\toptions,\n\t\t\tthis.cancellationToken,\n\t\t\tresultsBuilder,\n\t\t);\n\t\treturn result;\n\t}\n\n\t/** {@inheritdoc (ISummarizer:interface).enqueueSummarize} */\n\tpublic enqueueSummarize({\n\t\treason,\n\t\tafterSequenceNumber = 0,\n\t\toverride = false,\n\t\t...options\n\t}: IEnqueueSummarizeOptions): EnqueueSummarizeResult {\n\t\tconst onDemandReason = `enqueue;${reason}` as const;\n\t\tlet overridden = false;\n\t\tif (this.enqueuedSummary !== undefined) {\n\t\t\tif (!override) {\n\t\t\t\treturn { alreadyEnqueued: true };\n\t\t\t}\n\t\t\t// Override existing enqueued summarize attempt.\n\t\t\tthis.enqueuedSummary.resultsBuilder.fail(\n\t\t\t\t\"Aborted; overridden by another enqueue summarize attempt\",\n\t\t\t\tundefined,\n\t\t\t);\n\t\t\tthis.enqueuedSummary = undefined;\n\t\t\toverridden = true;\n\t\t}\n\t\tthis.enqueuedSummary = {\n\t\t\treason: onDemandReason,\n\t\t\tafterSequenceNumber,\n\t\t\toptions,\n\t\t\tresultsBuilder: new SummarizeResultBuilder(),\n\t\t};\n\t\tconst results = this.enqueuedSummary.resultsBuilder.build();\n\t\tthis.tryRunEnqueuedSummary();\n\t\treturn overridden\n\t\t\t? {\n\t\t\t\t\t...results,\n\t\t\t\t\talreadyEnqueued: true,\n\t\t\t\t\toverridden: true,\n\t\t\t  }\n\t\t\t: results;\n\t}\n\n\tprivate tryRunEnqueuedSummary() {\n\t\tif (this.stopping) {\n\t\t\tthis.disposeEnqueuedSummary();\n\t\t\treturn false;\n\t\t}\n\t\tif (\n\t\t\tthis.enqueuedSummary === undefined ||\n\t\t\tthis.heuristicData.lastOpSequenceNumber < this.enqueuedSummary.afterSequenceNumber ||\n\t\t\tthis.summarizingLock !== undefined\n\t\t) {\n\t\t\t// If no enqueued summary is ready or a summary is already in progress, take no action.\n\t\t\treturn false;\n\t\t}\n\t\tconst { reason, resultsBuilder, options } = this.enqueuedSummary;\n\t\t// Set to undefined first, so that subsequent enqueue attempt while summarize will occur later.\n\t\tthis.enqueuedSummary = undefined;\n\t\tthis.trySummarizeOnce(\n\t\t\t{ reason: `enqueuedSummary/${reason}` },\n\t\t\toptions,\n\t\t\tthis.cancellationToken,\n\t\t\tresultsBuilder,\n\t\t);\n\t\treturn true;\n\t}\n\n\tprivate disposeEnqueuedSummary() {\n\t\tif (this.enqueuedSummary !== undefined) {\n\t\t\tthis.enqueuedSummary.resultsBuilder.fail(\n\t\t\t\t\"RunningSummarizer stopped or disposed\",\n\t\t\t\tundefined,\n\t\t\t);\n\t\t\tthis.enqueuedSummary = undefined;\n\t\t}\n\t}\n}\n"]}
+{"version":3,"file":"runningSummarizer.js","sourceRoot":"","sources":["../../src/summary/runningSummarizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EACN,YAAY,EAEZ,4BAA4B,EAC5B,iBAAiB,EACjB,UAAU,GACV,MAAM,iCAAiC,CAAC;AACzC,OAAO,EACN,MAAM,EACN,KAAK,EACL,QAAQ,EACR,YAAY,EACZ,iBAAiB,GACjB,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,eAAe,EAAE,MAAM,oCAAoC,CAAC;AACrE,OAAO,EAA6B,WAAW,EAAE,MAAM,sCAAsC,CAAC;AAE9F,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AAqBlE,OAAO,EACN,SAAS,EAET,sBAAsB,EACtB,gBAAgB,GAChB,MAAM,oBAAoB,CAAC;AAE5B,MAAM,uBAAuB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,aAAa;AAE7D;;;GAGG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC;AACpC;;;GAGG;AACH,MAAM,CAAC,MAAM,mCAAmC,GAAG,CAAC,CAAC;AAErD;;;;;;GAMG;AACH,MAAM,OAAO,iBAAkB,SAAQ,iBAAoC;IAmG1E,YACC,UAAgC,EACf,cAAqC,EACrC,aAAoC,EACpC,qBAEgB,EAChB,+BAEC,EACD,aAAsC,EACtC,iBAAoC,EACpC,iBAA4C,EAC5C,sBAA8D,EAC9D,OAA2B;QAE5C,KAAK,EAAE,CAAC;QAdS,mBAAc,GAAd,cAAc,CAAuB;QACrC,kBAAa,GAAb,aAAa,CAAuB;QACpC,0BAAqB,GAArB,qBAAqB,CAEL;QAChB,oCAA+B,GAA/B,+BAA+B,CAE9B;QACD,kBAAa,GAAb,aAAa,CAAyB;QACtC,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,sBAAiB,GAAjB,iBAAiB,CAA2B;QAC5C,2BAAsB,GAAtB,sBAAsB,CAAwC;QAC9D,YAAO,GAAP,OAAO,CAAoB;QAxCrC,aAAQ,GAAG,KAAK,CAAC;QACjB,cAAS,GAAG,KAAK,CAAC;QAElB,wBAAmB,GAAG,KAAK,CAAC;QAc5B,mBAAc,GAAG,CAAC,CAAC;QACnB,4BAAuB,GAAG,CAAC,CAAC;QAC5B,gBAAW,GAAG,KAAK,CAAC;QAyN5B;;;;;WAKG;QACI,2BAAsB,GAAG,CAAC,eAAe,EAAE,EAAE,CACnD,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,iBAAiB,KAAK,eAAe;YACnE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM;YAChB,CAAC,CAAC,SAAS,CAAC;QAEd,wGAAwG;QAChG,mCAA8B,GAAG,KAAK,CAAC;QA5M9C,MAAM,cAAc,GAA8B;YACjD,cAAc,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,cAAc;YACzC,4BAA4B,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,uBAAuB;SAChE,CAAC;QAEF,IAAI,CAAC,EAAE,GAAG,4BAA4B,CAAC;YACtC,MAAM,EAAE,UAAU;YAClB,SAAS,EAAE,SAAS;YACpB,UAAU,EAAE;gBACX,GAAG,EAAE,cAAc;aACnB;SACD,CAAC,CAAC;QAEH,IAAI,aAAa,CAAC,KAAK,KAAK,mBAAmB,EAAE;YAChD,MAAM,CACL,IAAI,CAAC,aAAa,CAAC,KAAK,KAAK,SAAS,EACtC,KAAK,CAAC,6CAA6C,CACnD,CAAC;YACF,IAAI,CAAC,eAAe,GAAG,IAAI,wBAAwB,CAClD,aAAa,EACb,IAAI,CAAC,aAAa,EAClB,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,EACrC,IAAI,CAAC,EAAE,CAAC,MAAM,CACd,CAAC;SACF;QAED,MAAM,CACL,IAAI,CAAC,aAAa,CAAC,KAAK,KAAK,UAAU,EACvC,KAAK,CAAC,yDAAyD,CAC/D,CAAC;QAEF,oGAAoG;QACpG,+FAA+F;QAE/F,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,cAAc,EAAE,uBAAuB,CAAC,CAAC;QAE5F,IAAI,CAAC,eAAe,GAAG,IAAI,YAAY,CAAC,cAAc,EAAE,GAAG,EAAE;YAC5D,+DAA+D;YAC/D,sEAAsE;YACtE,2DAA2D;YAC3D,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC;gBAC7B,SAAS,EAAE,uBAAuB;gBAClC,OAAO,EAAE,0CAA0C;gBACnD,cAAc;gBACd,uBAAuB,EAAE,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,iBAAiB;gBACzE,qBAAqB,EAAE,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,qBAAqB;gBAC3E,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,WAAW;aACpE,CAAC,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,iFAAiF;QACjF,iBAAiB,CAAC,iCAAiC,CAAC,cAAc,EAAE,GAAG,EAAE;YACxE,IAAI,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE;gBAClC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,kBAAkB,CAAC;oBACjC,SAAS,EAAE,6BAA6B;oBACxC,uBAAuB,EAAE,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,iBAAiB;oBACzE,qBAAqB,EAAE,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,qBAAqB;iBAC3E,CAAC,CAAC;gBACH,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;aAC7B;QACF,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,SAAS,GAAG,IAAI,gBAAgB,CACpC,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,qBAAqB,EAC1B,GAAG,EAAE;YACJ,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAChC,CAAC,EACD,IAAI,CAAC,cAAc,EACnB,IAAI,CAAC,EAAE,CAAC,MAAM,CACd,CAAC;QAEF,4BAA4B;QAC5B,IAAI,CAAC,eAAe,GAAG,CAAC,EAA6B,EAAE,cAAwB,EAAE,EAAE;YAClF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,cAAc,KAAK,IAAI,CAAC,CAAC;QAC5C,CAAC,CAAC;QACF,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;QAE5C,+FAA+F;QAC/F,uEAAuE;QACvE,gGAAgG;QAChG,MAAM,mBAAmB,GAAG,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CACnD,4CAA4C,CAC5C,CAAC;QACF,IAAI,CAAC,4BAA4B;YAChC,mBAAmB,IAAI,mBAAmB,GAAG,mCAAmC;gBAC/E,CAAC,CAAC,mBAAmB;gBACrB,CAAC,CAAC,mCAAmC,CAAC;IACzC,CAAC;IA5MM,MAAM,CAAC,KAAK,CAAC,KAAK,CACxB,MAA4B,EAC5B,cAAqC,EACrC,aAAoC,EACpC,qBAAuF,EACvF,+BAAsF,EACtF,aAAsC,EACtC,iBAAoC,EACpC,iBAA4C,EAC5C,sBAA8D,EAC9D,OAA2B;QAE3B,MAAM,UAAU,GAAG,IAAI,iBAAiB,CACvC,MAAM,EACN,cAAc,EACd,aAAa,EACb,qBAAqB,EACrB,+BAA+B,EAC/B,aAAa,EACb,iBAAiB,EACjB,iBAAiB,EACjB,sBAAsB,EACtB,OAAO,CACP,CAAC;QAEF,0GAA0G;QAC1G,4FAA4F;QAC5F,MAAM,aAAa,GAAG,MAAM,UAAU,CAAC,gBAAgB,EAAE,CAAC;QAE1D,MAAM,UAAU,CAAC,SAAS,EAAE,CAAC;QAE7B,qCAAqC;QACrC,sGAAsG;QACtG,UAAU,CAAC,0BAA0B,CAAC,aAAa,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACpE,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,cAAc,CAC3C,EAAE,SAAS,EAAE,4BAA4B,EAAE,EAC3C,KAAK,CACL,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,0BAA0B;QAC1B,+FAA+F;QAC/F,uCAAuC;QACvC,yGAAyG;QACzG,wFAAwF;QACxF,6GAA6G;QAC7G,MAAM,IAAI,GACT,OAAO,CAAC,YAAY,CAAC,kBAAkB;YACvC,CAAC,aAAa,CAAC,qBAAqB,CAAC,iBAAiB;gBACrD,aAAa,CAAC,gBAAgB;gBAC9B,aAAa,CAAC,aAAa,CAAC,CAAC;QAC/B,aAAa,CAAC,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC;QAE1C,IAAI,aAAa,CAAC,gBAAgB,EAAE;YACnC,8DAA8D;YAC9D,aAAa,CAAC,gBAAgB,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;YACtD,aAAa,CAAC,aAAa,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;SACpD;QAED,+EAA+E;QAC/E,aAAa,CAAC,oBAAoB,GAAG,OAAO,CAAC,YAAY,CAAC,kBAAkB,CAAC;QAE7E,mBAAmB;QACnB,UAAU,CAAC,eAAe,EAAE,KAAK,EAAE,CAAC;QACpC,UAAU,CAAC,eAAe,EAAE,GAAG,EAAE,CAAC;QAElC,OAAO,UAAU,CAAC;IACnB,CAAC;IAED,IAAW,QAAQ;QAClB,OAAO,IAAI,CAAC,SAAS,CAAC;IACvB,CAAC;IAuIO,KAAK,CAAC,gBAAgB;QAC7B,MAAM,OAAO,GAA8B,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC;QAC5E,IAAI,iBAAiB,GAAG,CAAC,CAAC,CAAC;QAC3B,+DAA+D;QAC/D,IAAI,OAAO,KAAK,SAAS,EAAE;YAC1B,iBAAiB,GAAG,OAAO,CAAC,SAAS,CAAC,uBAAuB,CAAC;YAC9D,MAAM,aAAa,GAAG,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC;YACvF,MAAM,eAAe,GAAG,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC1D,MAAM,gBAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC5D,OAAO,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE;gBAC1C,aAAa,CAAC,kBAAkB,CAAC;oBAChC,SAAS,EAAE,qCAAqC;oBAChD,uBAAuB,EAAE,iBAAiB;oBAC1C,cAAc,EAAE,eAAe;oBAC/B,SAAS,EAAE,gBAAgB;iBAC3B,CAAC,CAAC;gBACH,MAAM,IAAI,CAAC,eAAe,CAAC;aAC3B;YAED,uEAAuE;YACvE,yCAAyC;YACzC,oEAAoE;YACpE,MAAM,IAAI,CAAC,mBAAmB,CAC7B,GAAG,EAAE,GAAE,CAAC,EACR,KAAK,IAAI,EAAE,CACV,IAAI,CAAC,+BAA+B,CAAC;gBACpC,cAAc,EAAE,eAAe;gBAC/B,SAAS,EAAE,gBAAgB;gBAC3B,aAAa,EAAE,iBAAiB;gBAChC,aAAa;aACb,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;gBACxB,wFAAwF;gBACxF,wFAAwF;gBACxF,wFAAwF;gBACxF,uFAAuF;gBACvF,uFAAuF;gBACvF,sEAAsE;gBACtE,MAAM,cAAc,GACnB,YAAY,CAAC,KAAK,CAAC;oBACnB,KAAK,CAAC,SAAS,KAAK,eAAe,CAAC,+BAA+B,CAAC;gBAErE,aAAa,CAAC,kBAAkB,CAC/B;oBACC,SAAS,EAAE,cAAc;wBACxB,CAAC,CAAC,8BAA8B;wBAChC,CAAC,CAAC,2BAA2B;oBAC9B,uBAAuB,EAAE,iBAAiB;oBAC1C,cAAc,EAAE,eAAe;oBAC/B,SAAS,EAAE,gBAAgB;iBAC3B,EACD,KAAK,CACL,CAAC;YACH,CAAC,CAAC,EACH,GAAG,EAAE,GAAE,CAAC,CACR,CAAC;YACF,iBAAiB,EAAE,CAAC;SACpB;QACD,OAAO,iBAAiB,CAAC;IAC1B,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,0BAA0B,CAAC,aAAqB;QAC7D,IAAI,iBAAiB,GACpB,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,qBAAqB,CAAC;QACrF,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE;YACtB,MAAM,aAAa,GAAG,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC;YAEvF,mGAAmG;YACnG,6CAA6C;YAC7C,MAAM,IAAI,CAAC,iBAAiB,CAAC,cAAc,CAAC,iBAAiB,CAAC,CAAC;YAE/D,aAAa,CAAC,kBAAkB,CAAC;gBAChC,SAAS,EAAE,4BAA4B;gBACvC,uBAAuB,EAAE,iBAAiB;gBAC1C,aAAa;aACb,CAAC,CAAC;YAEH,iBAAiB,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAClD,gDAAgD;YAChD,MAAM,CAAC,iBAAiB,IAAI,CAAC,EAAE,KAAK,CAAC,iCAAiC,CAAC,CAAC;SACxE;IACF,CAAC;IAEM,OAAO;QACb,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;QAC7C,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC;QAC9B,IAAI,CAAC,eAAe,EAAE,OAAO,EAAE,CAAC;QAChC,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;QACjC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QACzB,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC7B,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC9B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACtB,CAAC;IAgBM,QAAQ,CAAC,EAA6B,EAAE,cAAuB;QACrE,IAAI,CAAC,aAAa,CAAC,oBAAoB,GAAG,EAAE,CAAC,cAAc,CAAC;QAE5D,IAAI,cAAc,EAAE;YACnB,IAAI,CAAC,aAAa,CAAC,aAAa,EAAE,CAAC;SACnC;aAAM;YACN,IAAI,CAAC,aAAa,CAAC,gBAAgB,EAAE,CAAC;SACtC;QAED,IAAI,CAAC,aAAa,CAAC,YAAY,IAAI,MAAM,CAAC,EAAE,CAAC,CAAC;QAE9C,6EAA6E;QAC7E,IACC,IAAI,CAAC,WAAW;YAChB,IAAI,CAAC,mBAAmB,CAAC,EAAE,EAAE,cAAc,CAAC;YAC5C,CAAC,IAAI,CAAC,qBAAqB,EAAE;YAC7B,CAAC,IAAI,CAAC,8BAA8B,EACnC;YACD,IAAI,CAAC,8BAA8B,GAAG,IAAI,CAAC;YAC3C,OAAO,CAAC,OAAO,EAAE;iBACf,IAAI,CAAC,GAAG,EAAE;gBACV,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,CAAC;YAC7B,CAAC,CAAC;iBACD,OAAO,CAAC,GAAG,EAAE;gBACb,IAAI,CAAC,8BAA8B,GAAG,KAAK,CAAC;YAC7C,CAAC,CAAC,CAAC;SACJ;IACF,CAAC;IAED;;;;;OAKG;IACK,mBAAmB,CAAC,EAA6B,EAAE,cAAuB;QACjF,QAAQ,EAAE,CAAC,IAAI,EAAE;YAChB,KAAK,WAAW,CAAC,SAAS,CAAC;YAC3B,KAAK,WAAW,CAAC,UAAU,CAAC;YAC5B,KAAK,WAAW,CAAC,WAAW;gBAC3B,OAAO,KAAK,CAAC;YACd;gBACC,OAAO,cAAc,IAAI,IAAI,CAAC,6BAA6B,EAAE,CAAC;SAC/D;IACF,CAAC;IAEO,6BAA6B;QACpC,MAAM,eAAe,GACpB,IAAI,CAAC,aAAa,CAAC,oBAAoB;YACvC,IAAI,CAAC,aAAa,CAAC,qBAAqB,CAAC,iBAAiB,CAAC;QAC5D,OAAO,CACN,IAAI,CAAC,aAAa,CAAC,KAAK,KAAK,SAAS;YACtC,CAAC,IAAI,CAAC,aAAa,CAAC,4BAA4B,KAAK,SAAS;gBAC7D,IAAI,CAAC,aAAa,CAAC,4BAA4B,IAAI,eAAe,CAAC,CACpE,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,QAAQ,CAAC,gBAAyB;QAC9C,IAAI,IAAI,CAAC,QAAQ,EAAE;YAClB,OAAO;SACP;QAED,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QAErB,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAE9B,8CAA8C;QAC9C,IAAI,gBAAgB,IAAI,IAAI,CAAC,eAAe,EAAE,oBAAoB,EAAE,EAAE;YACrE,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE;gBACvC,IAAI,CAAC,gBAAgB;gBACpB,iBAAiB;gBACjB,EAAE,eAAe,EAAE,aAAa,EAAE;gBAClC,kFAAkF;gBAClF,EAAE,CACF,CAAC;aACF;SACD;QAED,+EAA+E;QAC/E,qFAAqF;QACrF,6FAA6F;QAC7F,0FAA0F;QAC1F,uEAAuE;QACvE,MAAM,IAAI,CAAC,eAAe,CAAC;IAC5B,CAAC;IAEO,KAAK,CAAC,SAAS;QACtB,kDAAkD;QAClD,MAAM,eAAe,GAAG,MAAM,SAAS,CACtC,IAAI,CAAC,cAAc,CAAC,WAAW,EAAE,EACjC,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAC5B,CAAC;QACF,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAE7B,sEAAsE;QACtE,uEAAuE;QACvE,IAAI,CAAC,iBAAiB,CAAC,mCAAmC,EAAE,CAAC;QAE7D,IAAI,eAAe,CAAC,MAAM,KAAK,MAAM,IAAI,eAAe,CAAC,KAAK,KAAK,SAAS,EAAE;YAC7E,IAAI,CAAC,aAAa,CAAC,4BAA4B,CAAC;gBAC/C,iBAAiB,EAAE,eAAe,CAAC,KAAK,CAAC,SAAS,CAAC,uBAAuB;gBAC1E,2FAA2F;gBAC3F,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;gBACvB,qBAAqB,EAAE,eAAe,CAAC,KAAK,CAAC,SAAS,CAAC,cAAc;aACrE,CAAC,CAAC;SACH;QACD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IACzB,CAAC;IAEO,mBAAmB;QAC1B,IAAI,CAAC,cAAc,EAAE,CAAC;IACvB,CAAC;IAEO,kBAAkB;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,mBAAmB,CAAC;QACvC,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;QAEjC,2EAA2E;QAC3E,yDAAyD;QACzD,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,IAAI,KAAK,EAAE;YAC7D,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,CAAC;SAC5B;IACF,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,mBAAmB,CAChC,MAAkB,EAClB,MAAwB,EACxB,KAAiB;QAEjB,MAAM,CACL,IAAI,CAAC,eAAe,KAAK,SAAS,EAClC,KAAK,CAAC,+CAA+C,CACrD,CAAC;QAEF,MAAM,eAAe,GAAG,IAAI,QAAQ,EAAQ,CAAC;QAC7C,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC,OAAO,CAAC;QAE/C,MAAM,EAAE,CAAC;QAET,OAAO,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;YAC5B,eAAe,CAAC,OAAO,EAAE,CAAC;YAC1B,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;YACjC,KAAK,EAAE,CAAC;QACT,CAAC,CAAC,CAAC;IACJ,CAAC;IAED;;;;;;;OAOG;IACK,gBAAgB,CACvB,cAA6C,EAC7C,OAA0B,EAC1B,cAAc,GAAG,IAAI,sBAAsB,EAAE;QAE7C,IAAI,CAAC,mBAAmB,CACvB,GAAG,EAAE;YACJ,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC5B,CAAC,EACD,KAAK,IAAI,EAAE;YACV,MAAM,aAAa,GAAG,iBAAiB,CAAC;gBACvC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,MAAM;gBACtB,UAAU,EAAE,EAAE,GAAG,EAAE,cAAc,EAAE;aACnC,CAAC,CAAC;YACH,MAAM,cAAc,GAA0B;gBAC7C,GAAG,OAAO;gBACV,aAAa;gBACb,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;aACzC,CAAC;YACF,MAAM,eAAe,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,cAAc,EAAE,cAAc,CAAC,CAAC;YACjF,6CAA6C;YAC7C,OAAO,eAAe,CAAC,wBAAwB,CAAC;QACjD,CAAC,EACD,GAAG,EAAE;YACJ,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC3B,CAAC,CACD,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACjB,6FAA6F;YAC7F,oBAAoB;YACpB,4FAA4F;YAC5F,iCAAiC;QAClC,CAAC,CAAC,CAAC;QAEH,OAAO,cAAc,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;IAED,oCAAoC;IAC5B,YAAY,CAAC,MAAuB;QAC3C,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE;YACvC,yFAAyF;YACzF,uBAAuB;YACvB,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;YAChC,OAAO;SACP;QAED,IAAI,CAAC,mBAAmB,CACvB,GAAG,EAAE;YACJ,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC5B,CAAC,EACD,KAAK,IAAI,EAAE;YACV,OAAO,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,oCAAoC,CAAC;gBACrE,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC;gBACtC,CAAC,CAAC,IAAI,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;QAChD,CAAC,EACD,GAAG,EAAE;YACJ,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC3B,CAAC,CACD,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACjB,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,0BAA0B,EAAE,EAAE,KAAK,CAAC,CAAC;QACjF,CAAC,CAAC,CAAC;IACJ,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,8BAA8B,CAAC,MAAuB;QACnE,MAAM,cAAc,GAAwB;YAC3C,EAAE,gBAAgB,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE;YAC5C,EAAE,gBAAgB,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE;SAC3C,CAAC;QACF,IAAI,eAAe,GAAG,CAAC,CAAC;QACxB,IAAI,uBAAuB,GAAG,CAAC,CAAC;QAChC,IAAI,mBAAmB,GAAG,CAAC,CAAC;QAC5B,OAAO,mBAAmB,GAAG,cAAc,CAAC,MAAM,EAAE;YACnD,IAAI,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE;gBACrC,OAAO;aACP;YAED,iEAAiE;YACjE,IAAI,EAAE,eAAe,GAAG,CAAC,IAAI,MAAM,KAAK,aAAa,EAAE;gBACtD,OAAO;aACP;YAED,uBAAuB,EAAE,CAAC;YAE1B,MAAM,gBAAgB,GAAG,cAAc,CAAC,mBAAmB,CAAC,CAAC;YAC7D,MAAM,cAAc,GAAkC;gBACrD,eAAe,EAAE,MAAM;gBACvB,eAAe;gBACf,uBAAuB;gBACvB,mBAAmB,EAAE,mBAAmB,GAAG,CAAC;gBAC5C,GAAG,gBAAgB;aACnB,CAAC;YACF,MAAM,aAAa,GAAG,iBAAiB,CAAC;gBACvC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,MAAM;gBACtB,UAAU,EAAE,EAAE,GAAG,EAAE,cAAc,EAAE;aACnC,CAAC,CAAC;YACH,MAAM,cAAc,GAA0B;gBAC7C,GAAG,gBAAgB;gBACnB,aAAa;gBACb,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;aACzC,CAAC;YAEF,wEAAwE;YACxE,2FAA2F;YAC3F,MAAM,eAAe,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;YACjE,MAAM,aAAa,GAAG,MAAM,eAAe,CAAC,wBAAwB,CAAC;YACrE,IAAI,aAAa,CAAC,OAAO,EAAE;gBAC1B,OAAO;aACP;YAED,+FAA+F;YAC/F,yDAAyD;YACzD,MAAM,YAAY,GAAG,MAAM,eAAe,CAAC,gBAAgB,CAAC;YAC5D,MAAM,YAAY,GAAG,CAAC,YAAY,CAAC,OAAO;gBACzC,CAAC,CAAC,YAAY,CAAC,IAAI,EAAE,iBAAiB;gBACtC,CAAC,CAAC,aAAa,CAAC,IAAI,EAAE,iBAAiB,CAAC;YACzC,IAAI,YAAY,KAAK,SAAS,IAAI,uBAAuB,GAAG,CAAC,EAAE;gBAC9D,mBAAmB,EAAE,CAAC;gBACtB,uBAAuB,GAAG,CAAC,CAAC;aAC5B;YAED,IAAI,YAAY,KAAK,SAAS,EAAE;gBAC/B,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC;oBACnC,SAAS,EAAE,uBAAuB;oBAClC,QAAQ,EAAE,YAAY;oBACtB,gBAAgB,EAAE,aAAa,CAAC,IAAI,EAAE,iBAAiB,KAAK,SAAS;oBACrE,GAAG,cAAc;iBACjB,CAAC,CAAC;gBACH,MAAM,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;aACjC;SACD;QACD,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,CAAC;IAChD,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,uBAAuB,CAAC,MAAuB;QAC5D,4EAA4E;QAC5E,MAAM,gBAAgB,GAAG,CAAC,aAAqB,EAAE,YAAqB,EAAE,EAAE;YACzE,MAAM,gBAAgB,GAAsB;gBAC3C,QAAQ,EAAE,KAAK;aACf,CAAC;YACF,MAAM,cAAc,GAAkC;gBACrD,eAAe,EAAE,MAAM;gBACvB,eAAe,EAAE,aAAa;gBAC9B,GAAG,gBAAgB;gBACnB,YAAY;aACZ,CAAC;YACF,MAAM,aAAa,GAAG,iBAAiB,CAAC;gBACvC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,MAAM;gBACtB,UAAU,EAAE,EAAE,GAAG,EAAE,cAAc,EAAE;aACnC,CAAC,CAAC;YACH,MAAM,cAAc,GAA0B;gBAC7C,GAAG,gBAAgB;gBACnB,aAAa;gBACb,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;gBACzC,YAAY;aACZ,CAAC;YACF,MAAM,eAAe,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;YACjE,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,CAAC;QAC5C,CAAC,CAAC;QAEF,4GAA4G;QAC5G,8GAA8G;QAC9G,8EAA8E;QAC9E,gHAAgH;QAChH,gHAAgH;QAChH,gHAAgH;QAChH,+GAA+G;QAC/G,6GAA6G;QAC7G,4DAA4D;QAC5D,IAAI,WAAW,GAAG,kBAAkB,CAAC;QACrC,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,IAAI,iBAAqC,CAAC;QAC1C,IAAI,IAAI,GAAG,KAAK,CAAC;QACjB,IAAI,MAAM,GAAuC,SAAS,CAAC;QAC3D,GAAG;YACF,cAAc,EAAE,CAAC;YACjB,IAAI,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE;gBACrC,MAAM,GAAG,UAAU,CAAC;gBACpB,IAAI,GAAG,IAAI,CAAC;gBACZ,MAAM;aACN;YAED,MAAM,EAAE,cAAc,EAAE,eAAe,EAAE,GAAG,gBAAgB,CAC3D,cAAc,EACd,KAAK,CAAC,kBAAkB,CACxB,CAAC;YAEF,8DAA8D;YAC9D,MAAM,aAAa,GAAG,MAAM,eAAe,CAAC,wBAAwB,CAAC;YACrE,IAAI,aAAa,CAAC,OAAO,EAAE;gBAC1B,MAAM,GAAG,SAAS,CAAC;gBACnB,IAAI,GAAG,IAAI,CAAC;gBACZ,MAAM;aACN;YAED,gEAAgE;YAChE,gGAAgG;YAChG,0CAA0C;YAC1C,gGAAgG;YAChG,MAAM,mBAAmB,GAAG,MAAM,eAAe,CAAC,gBAAgB,CAAC;YACnE,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE;gBACjC,WAAW,GAAG,IAAI,CAAC,4BAA4B,CAAC;gBAChD,iBAAiB,GAAG,mBAAmB,CAAC,IAAI,EAAE,iBAAiB,CAAC;aAChE;iBAAM;gBACN,WAAW,GAAG,kBAAkB,CAAC;gBACjC,iBAAiB,GAAG,aAAa,CAAC,IAAI,EAAE,iBAAiB,CAAC;aAC1D;YAED,kDAAkD;YAClD,MAAM,GAAG,SAAS,CAAC;YACnB,MAAM,UAAU,GAAyB;gBACxC,MAAM;gBACN,cAAc;gBACd,WAAW;gBACX,KAAK,EAAE,aAAa,CAAC,KAAK;aAC1B,CAAC;YACF,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;YAEnC,4GAA4G;YAC5G,IAAI,iBAAiB,KAAK,SAAS,IAAI,cAAc,IAAI,WAAW,GAAG,CAAC,EAAE;gBACzE,IAAI,GAAG,IAAI,CAAC;aACZ;YAED,uGAAuG;YACvG,sEAAsE;YACtE,IAAI,iBAAiB,KAAK,SAAS,EAAE;gBACpC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC;oBACnC,SAAS,EAAE,uBAAuB;oBAClC,QAAQ,EAAE,iBAAiB;oBAC3B,gBAAgB,EAAE,aAAa,CAAC,IAAI,EAAE,iBAAiB,KAAK,SAAS;oBACrE,KAAK,EAAE,mBAAmB,CAAC,IAAI,EAAE,KAAK;oBACtC,cAAc,EAAE,IAAI;oBACpB,GAAG,cAAc;iBACjB,CAAC,CAAC;gBACH,MAAM,KAAK,CAAC,iBAAiB,GAAG,IAAI,CAAC,CAAC;aACtC;SACD,QAAQ,CAAC,IAAI,EAAE;QAEhB,+GAA+G;QAC/G,IAAI,MAAM,KAAK,SAAS,EAAE;YACzB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE,WAAW,EAAE,CAAC,CAAC;YAChE,OAAO;SACP;QAED,0GAA0G;QAC1G,+FAA+F;QAC/F,IAAI,iBAAiB,KAAK,SAAS,EAAE;YACpC,MAAM,EAAE,eAAe,EAAE,GAAG,gBAAgB,CAAC,EAAE,cAAc,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC;YACxF,8DAA8D;YAC9D,MAAM,aAAa,GAAG,MAAM,eAAe,CAAC,wBAAwB,CAAC;YACrE,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;YACvD,MAAM,UAAU,GAAyB;gBACxC,MAAM;gBACN,cAAc;gBACd,WAAW;gBACX,KAAK,EAAE,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK;aAC9D,CAAC;YACF,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;SACnC;QAED,+DAA+D;QAC/D,IAAI,MAAM,KAAK,SAAS,EAAE;YACzB,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,CAAC;SAC/C;IACF,CAAC;IAED,8DAA8D;IACvD,iBAAiB,CACvB,OAAkC,EAClC,iBAAyC,IAAI,sBAAsB,EAAE;QAErE,IAAI,IAAI,CAAC,QAAQ,EAAE;YAClB,cAAc,CAAC,IAAI,CAAC,uCAAuC,EAAE,SAAS,CAAC,CAAC;YACxE,OAAO,cAAc,CAAC,KAAK,EAAE,CAAC;SAC9B;QACD,0DAA0D;QAC1D,8DAA8D;QAC9D,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE;YACvC,6DAA6D;YAC7D,MAAM,IAAI,UAAU,CAAC,0DAA0D,CAAC,CAAC;SACjF;QAED,MAAM,EAAE,MAAM,EAAE,GAAG,gBAAgB,EAAE,GAAG,OAAO,CAAC;QAChD,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CACnC,EAAE,eAAe,EAAE,YAAY,MAAM,EAAE,EAAE,EACzC,gBAAgB,EAChB,cAAc,CACd,CAAC;QACF,OAAO,MAAM,CAAC;IACf,CAAC;IAED,6DAA6D;IACtD,gBAAgB,CAAC,OAAiC;QACxD,MAAM,EAAE,MAAM,EAAE,mBAAmB,GAAG,CAAC,EAAE,QAAQ,GAAG,KAAK,EAAE,GAAG,gBAAgB,EAAE,GAAG,OAAO,CAAC;QAC3F,IAAI,UAAU,GAAG,KAAK,CAAC;QACvB,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE;YACvC,IAAI,CAAC,QAAQ,EAAE;gBACd,OAAO,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;aACjC;YACD,gDAAgD;YAChD,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,IAAI,CACvC,0DAA0D,EAC1D,SAAS,CACT,CAAC;YACF,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;YACjC,UAAU,GAAG,IAAI,CAAC;SAClB;QAED,IAAI,CAAC,eAAe,GAAG;YACtB,MAAM,EAAE,WAAW,MAAM,EAAE;YAC3B,mBAAmB;YACnB,gBAAgB;YAChB,cAAc,EAAE,IAAI,sBAAsB,EAAE;SAC5C,CAAC;QACF,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC5D,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAC7B,OAAO,UAAU;YAChB,CAAC,CAAC;gBACA,GAAG,OAAO;gBACV,eAAe,EAAE,IAAI;gBACrB,UAAU,EAAE,IAAI;aACf;YACH,CAAC,CAAC,OAAO,CAAC;IACZ,CAAC;IAEO,qBAAqB;QAC5B,IAAI,IAAI,CAAC,QAAQ,EAAE;YAClB,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAC9B,OAAO,KAAK,CAAC;SACb;QACD,IACC,IAAI,CAAC,eAAe,KAAK,SAAS;YAClC,IAAI,CAAC,aAAa,CAAC,oBAAoB,GAAG,IAAI,CAAC,eAAe,CAAC,mBAAmB;YAClF,IAAI,CAAC,eAAe,KAAK,SAAS,EACjC;YACD,uFAAuF;YACvF,OAAO,KAAK,CAAC;SACb;QACD,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,gBAAgB,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC;QAC1E,+FAA+F;QAC/F,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;QACjC,IAAI,CAAC,gBAAgB,CACpB,EAAE,eAAe,EAAE,mBAAmB,MAAM,EAAE,EAAE,EAChD,gBAAgB,EAChB,cAAc,CACd,CAAC;QACF,OAAO,IAAI,CAAC;IACb,CAAC;IAEO,sBAAsB;QAC7B,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE;YACvC,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,IAAI,CACvC,uCAAuC,EACvC,SAAS,CACT,CAAC;YACF,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;SACjC;IACF,CAAC;CACD","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { IDisposable, ITelemetryBaseLogger } from \"@fluidframework/core-interfaces\";\nimport {\n\tisFluidError,\n\tMonitoringContext,\n\tcreateChildMonitoringContext,\n\tcreateChildLogger,\n\tUsageError,\n} from \"@fluidframework/telemetry-utils\";\nimport {\n\tassert,\n\tdelay,\n\tDeferred,\n\tPromiseTimer,\n\tTypedEventEmitter,\n} from \"@fluidframework/common-utils\";\nimport { DriverErrorType } from \"@fluidframework/driver-definitions\";\nimport { ISequencedDocumentMessage, MessageType } from \"@fluidframework/protocol-definitions\";\nimport { ISummaryConfiguration } from \"../containerRuntime\";\nimport { opSize } from \"../opProperties\";\nimport { SummarizeHeuristicRunner } from \"./summarizerHeuristics\";\nimport {\n\tIEnqueueSummarizeOptions,\n\tISummarizeOptions,\n\tISummarizeHeuristicData,\n\tISummarizeHeuristicRunner,\n\tIOnDemandSummarizeOptions,\n\tEnqueueSummarizeResult,\n\tSummarizerStopReason,\n\tISubmitSummaryOptions,\n\tSubmitSummaryResult,\n\tISummaryCancellationToken,\n\tISummarizeResults,\n\tISummarizeTelemetryProperties,\n\tISummarizerRuntime,\n\tISummarizeRunnerTelemetry,\n\tIRefreshSummaryAckOptions,\n\tISummarizerEvents,\n\tISummarizeEventProps,\n} from \"./summarizerTypes\";\nimport { IAckedSummary, IClientSummaryWatcher, SummaryCollection } from \"./summaryCollection\";\nimport {\n\traceTimer,\n\tSummarizeReason,\n\tSummarizeResultBuilder,\n\tSummaryGenerator,\n} from \"./summaryGenerator\";\n\nconst maxSummarizeAckWaitTime = 10 * 60 * 1000; // 10 minutes\n\n/**\n * The maximum number of summarization attempts that will be done by default in case of failures\n * that can be retried.\n */\nexport const defaultMaxAttempts = 2;\n/**\n * The default value for maximum number of summarization attempts that will be done for summarization failures where\n * submit fails and the failure can be retried.\n */\nexport const defaultMaxAttemptsForSubmitFailures = 5;\n\n/**\n * An instance of RunningSummarizer manages the heuristics for summarizing.\n * Until disposed, the instance of RunningSummarizer can assume that it is\n * in a state of running, meaning it is connected and initialized.  It keeps\n * track of summaries that it is generating as they are broadcast and acked/nacked.\n * This object is created and controlled by Summarizer object.\n */\nexport class RunningSummarizer extends TypedEventEmitter<ISummarizerEvents> implements IDisposable {\n\tpublic static async start(\n\t\tlogger: ITelemetryBaseLogger,\n\t\tsummaryWatcher: IClientSummaryWatcher,\n\t\tconfiguration: ISummaryConfiguration,\n\t\tsubmitSummaryCallback: (options: ISubmitSummaryOptions) => Promise<SubmitSummaryResult>,\n\t\trefreshLatestSummaryAckCallback: (options: IRefreshSummaryAckOptions) => Promise<void>,\n\t\theuristicData: ISummarizeHeuristicData,\n\t\tsummaryCollection: SummaryCollection,\n\t\tcancellationToken: ISummaryCancellationToken,\n\t\tstopSummarizerCallback: (reason: SummarizerStopReason) => void,\n\t\truntime: ISummarizerRuntime,\n\t): Promise<RunningSummarizer> {\n\t\tconst summarizer = new RunningSummarizer(\n\t\t\tlogger,\n\t\t\tsummaryWatcher,\n\t\t\tconfiguration,\n\t\t\tsubmitSummaryCallback,\n\t\t\trefreshLatestSummaryAckCallback,\n\t\t\theuristicData,\n\t\t\tsummaryCollection,\n\t\t\tcancellationToken,\n\t\t\tstopSummarizerCallback,\n\t\t\truntime,\n\t\t);\n\n\t\t// Before doing any heuristics or proceeding with its refreshing, if there is a summary ack received while\n\t\t// this summarizer catches up, let's refresh state before proceeding with the summarization.\n\t\tconst lastAckRefSeq = await summarizer.handleSummaryAck();\n\n\t\tawait summarizer.waitStart();\n\n\t\t// Handle summary acks asynchronously\n\t\t// Note: no exceptions are thrown from processIncomingSummaryAcks handler as it handles all exceptions\n\t\tsummarizer.processIncomingSummaryAcks(lastAckRefSeq).catch((error) => {\n\t\t\tcreateChildLogger({ logger }).sendErrorEvent(\n\t\t\t\t{ eventName: \"HandleSummaryAckFatalError\" },\n\t\t\t\terror,\n\t\t\t);\n\t\t});\n\n\t\t// Update heuristic counts\n\t\t// By the time we get here, there are potentially ops missing from the heuristic summary counts\n\t\t// Examples of where this could happen:\n\t\t// 1. Op is processed during the time that we are initiating the RunningSummarizer instance but before we\n\t\t//    listen for the op events (will get missed by the handlers in the current workflow)\n\t\t// 2. Op was sequenced after the last time we summarized (op sequence number > summarize ref sequence number)\n\t\tconst diff =\n\t\t\truntime.deltaManager.lastSequenceNumber -\n\t\t\t(heuristicData.lastSuccessfulSummary.refSequenceNumber +\n\t\t\t\theuristicData.numNonRuntimeOps +\n\t\t\t\theuristicData.numRuntimeOps);\n\t\theuristicData.hasMissingOpData = diff > 0;\n\n\t\tif (heuristicData.hasMissingOpData) {\n\t\t\t// Split the diff 50-50 and increment the counts appropriately\n\t\t\theuristicData.numNonRuntimeOps += Math.ceil(diff / 2);\n\t\t\theuristicData.numRuntimeOps += Math.floor(diff / 2);\n\t\t}\n\n\t\t// Update last seq number (in case the handlers haven't processed anything yet)\n\t\theuristicData.lastOpSequenceNumber = runtime.deltaManager.lastSequenceNumber;\n\n\t\t// Start heuristics\n\t\tsummarizer.heuristicRunner?.start();\n\t\tsummarizer.heuristicRunner?.run();\n\n\t\treturn summarizer;\n\t}\n\n\tpublic get disposed() {\n\t\treturn this._disposed;\n\t}\n\tprivate stopping = false;\n\tprivate _disposed = false;\n\tprivate summarizingLock: Promise<void> | undefined;\n\tprivate tryWhileSummarizing = false;\n\tprivate readonly pendingAckTimer: PromiseTimer;\n\tprivate heuristicRunner?: ISummarizeHeuristicRunner;\n\tprivate readonly generator: SummaryGenerator;\n\tprivate readonly mc: MonitoringContext;\n\n\tprivate enqueuedSummary:\n\t\t| {\n\t\t\t\treason: SummarizeReason;\n\t\t\t\tafterSequenceNumber: number;\n\t\t\t\tsummarizeOptions: ISummarizeOptions;\n\t\t\t\treadonly resultsBuilder: SummarizeResultBuilder;\n\t\t  }\n\t\t| undefined;\n\tprivate summarizeCount = 0;\n\tprivate totalSuccessfulAttempts = 0;\n\tprivate initialized = false;\n\n\tprivate readonly runtimeListener;\n\n\t/** The maximum number of summary attempts to do when submit summary fails. */\n\tprivate readonly maxAttemptsForSubmitFailures: number;\n\n\tprivate constructor(\n\t\tbaseLogger: ITelemetryBaseLogger,\n\t\tprivate readonly summaryWatcher: IClientSummaryWatcher,\n\t\tprivate readonly configuration: ISummaryConfiguration,\n\t\tprivate readonly submitSummaryCallback: (\n\t\t\toptions: ISubmitSummaryOptions,\n\t\t) => Promise<SubmitSummaryResult>,\n\t\tprivate readonly refreshLatestSummaryAckCallback: (\n\t\t\toptions: IRefreshSummaryAckOptions,\n\t\t) => Promise<void>,\n\t\tprivate readonly heuristicData: ISummarizeHeuristicData,\n\t\tprivate readonly summaryCollection: SummaryCollection,\n\t\tprivate readonly cancellationToken: ISummaryCancellationToken,\n\t\tprivate readonly stopSummarizerCallback: (reason: SummarizerStopReason) => void,\n\t\tprivate readonly runtime: ISummarizerRuntime,\n\t) {\n\t\tsuper();\n\n\t\tconst telemetryProps: ISummarizeRunnerTelemetry = {\n\t\t\tsummarizeCount: () => this.summarizeCount,\n\t\t\tsummarizerSuccessfulAttempts: () => this.totalSuccessfulAttempts,\n\t\t};\n\n\t\tthis.mc = createChildMonitoringContext({\n\t\t\tlogger: baseLogger,\n\t\t\tnamespace: \"Running\",\n\t\t\tproperties: {\n\t\t\t\tall: telemetryProps,\n\t\t\t},\n\t\t});\n\n\t\tif (configuration.state !== \"disableHeuristics\") {\n\t\t\tassert(\n\t\t\t\tthis.configuration.state === \"enabled\",\n\t\t\t\t0x2ea /* \"Configuration state should be enabled\" */,\n\t\t\t);\n\t\t\tthis.heuristicRunner = new SummarizeHeuristicRunner(\n\t\t\t\theuristicData,\n\t\t\t\tthis.configuration,\n\t\t\t\t(reason) => this.trySummarize(reason),\n\t\t\t\tthis.mc.logger,\n\t\t\t);\n\t\t}\n\n\t\tassert(\n\t\t\tthis.configuration.state !== \"disabled\",\n\t\t\t0x2eb /* \"Summary not supported with configuration disabled\" */,\n\t\t);\n\n\t\t// Cap the maximum amount of time client will wait for a summarize op ack to maxSummarizeAckWaitTime\n\t\t// configuration.maxAckWaitTime is composed from defaults, server values, and runtime overrides\n\n\t\tconst maxAckWaitTime = Math.min(this.configuration.maxAckWaitTime, maxSummarizeAckWaitTime);\n\n\t\tthis.pendingAckTimer = new PromiseTimer(maxAckWaitTime, () => {\n\t\t\t// Note: summarizeCount (from ChildLogger definition) may be 0,\n\t\t\t// since this code path is hit when RunningSummarizer first starts up,\n\t\t\t// before this instance has kicked off a new summarize run.\n\t\t\tthis.mc.logger.sendErrorEvent({\n\t\t\t\teventName: \"SummaryAckWaitTimeout\",\n\t\t\t\tmessage: \"Pending summary ack not received in time\",\n\t\t\t\tmaxAckWaitTime,\n\t\t\t\treferenceSequenceNumber: this.heuristicData.lastAttempt.refSequenceNumber,\n\t\t\t\tsummarySequenceNumber: this.heuristicData.lastAttempt.summarySequenceNumber,\n\t\t\t\ttimePending: Date.now() - this.heuristicData.lastAttempt.summaryTime,\n\t\t\t});\n\t\t});\n\t\t// Set up pending ack timeout by op timestamp differences for previous summaries.\n\t\tsummaryCollection.setPendingAckTimerTimeoutCallback(maxAckWaitTime, () => {\n\t\t\tif (this.pendingAckTimer.hasTimer) {\n\t\t\t\tthis.mc.logger.sendTelemetryEvent({\n\t\t\t\t\teventName: \"MissingSummaryAckFoundByOps\",\n\t\t\t\t\treferenceSequenceNumber: this.heuristicData.lastAttempt.refSequenceNumber,\n\t\t\t\t\tsummarySequenceNumber: this.heuristicData.lastAttempt.summarySequenceNumber,\n\t\t\t\t});\n\t\t\t\tthis.pendingAckTimer.clear();\n\t\t\t}\n\t\t});\n\n\t\tthis.generator = new SummaryGenerator(\n\t\t\tthis.pendingAckTimer,\n\t\t\tthis.heuristicData,\n\t\t\tthis.submitSummaryCallback,\n\t\t\t() => {\n\t\t\t\tthis.totalSuccessfulAttempts++;\n\t\t\t},\n\t\t\tthis.summaryWatcher,\n\t\t\tthis.mc.logger,\n\t\t);\n\n\t\t// Listen to runtime for ops\n\t\tthis.runtimeListener = (op: ISequencedDocumentMessage, runtimeMessage?: boolean) => {\n\t\t\tthis.handleOp(op, runtimeMessage === true);\n\t\t};\n\t\tthis.runtime.on(\"op\", this.runtimeListener);\n\n\t\t// The max attempts for submit failures can be overridden via a feature flag. This allows us to\n\t\t// tweak this as per telemetry data until we arrive at a stable number.\n\t\t// If its set to a number higher than `defaultMaxAttemptsForSubmitFailures`, it will be ignored.\n\t\tconst overrideMaxAttempts = this.mc.config.getNumber(\n\t\t\t\"Fluid.Summarizer.AttemptsForSubmitFailures\",\n\t\t);\n\t\tthis.maxAttemptsForSubmitFailures =\n\t\t\toverrideMaxAttempts && overrideMaxAttempts < defaultMaxAttemptsForSubmitFailures\n\t\t\t\t? overrideMaxAttempts\n\t\t\t\t: defaultMaxAttemptsForSubmitFailures;\n\t}\n\n\tprivate async handleSummaryAck(): Promise<number> {\n\t\tconst lastAck: IAckedSummary | undefined = this.summaryCollection.latestAck;\n\t\tlet refSequenceNumber = -1;\n\t\t// In case we haven't received the lastestAck yet, just return.\n\t\tif (lastAck !== undefined) {\n\t\t\trefSequenceNumber = lastAck.summaryOp.referenceSequenceNumber;\n\t\t\tconst summaryLogger = this.tryGetCorrelatedLogger(refSequenceNumber) ?? this.mc.logger;\n\t\t\tconst summaryOpHandle = lastAck.summaryOp.contents.handle;\n\t\t\tconst summaryAckHandle = lastAck.summaryAck.contents.handle;\n\t\t\twhile (this.summarizingLock !== undefined) {\n\t\t\t\tsummaryLogger.sendTelemetryEvent({\n\t\t\t\t\teventName: \"RefreshAttemptWithSummarizerRunning\",\n\t\t\t\t\treferenceSequenceNumber: refSequenceNumber,\n\t\t\t\t\tproposalHandle: summaryOpHandle,\n\t\t\t\t\tackHandle: summaryAckHandle,\n\t\t\t\t});\n\t\t\t\tawait this.summarizingLock;\n\t\t\t}\n\n\t\t\t// Make sure we block any summarizer from being executed/enqueued while\n\t\t\t// executing the refreshLatestSummaryAck.\n\t\t\t// https://dev.azure.com/fluidframework/internal/_workitems/edit/779\n\t\t\tawait this.lockedSummaryAction(\n\t\t\t\t() => {},\n\t\t\t\tasync () =>\n\t\t\t\t\tthis.refreshLatestSummaryAckCallback({\n\t\t\t\t\t\tproposalHandle: summaryOpHandle,\n\t\t\t\t\t\tackHandle: summaryAckHandle,\n\t\t\t\t\t\tsummaryRefSeq: refSequenceNumber,\n\t\t\t\t\t\tsummaryLogger,\n\t\t\t\t\t}).catch(async (error) => {\n\t\t\t\t\t\t// If the error is 404, so maybe the fetched version no longer exists on server. We just\n\t\t\t\t\t\t// ignore this error in that case, as that means we will have another summaryAck for the\n\t\t\t\t\t\t// latest version with which we will refresh the state. However in case of single commit\n\t\t\t\t\t\t// summary, we might me missing a summary ack, so in that case we are still fine as the\n\t\t\t\t\t\t// code in `submitSummary` function in container runtime, will refresh the latest state\n\t\t\t\t\t\t// by calling `refreshLatestSummaryAckFromServer` and we will be fine.\n\t\t\t\t\t\tconst isIgnoredError =\n\t\t\t\t\t\t\tisFluidError(error) &&\n\t\t\t\t\t\t\terror.errorType === DriverErrorType.fileNotFoundOrAccessDeniedError;\n\n\t\t\t\t\t\tsummaryLogger.sendTelemetryEvent(\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\teventName: isIgnoredError\n\t\t\t\t\t\t\t\t\t? \"HandleSummaryAckErrorIgnored\"\n\t\t\t\t\t\t\t\t\t: \"HandleLastSummaryAckError\",\n\t\t\t\t\t\t\t\treferenceSequenceNumber: refSequenceNumber,\n\t\t\t\t\t\t\t\tproposalHandle: summaryOpHandle,\n\t\t\t\t\t\t\t\tackHandle: summaryAckHandle,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\terror,\n\t\t\t\t\t\t);\n\t\t\t\t\t}),\n\t\t\t\t() => {},\n\t\t\t);\n\t\t\trefSequenceNumber++;\n\t\t}\n\t\treturn refSequenceNumber;\n\t}\n\n\t/**\n\t * Responsible for receiving and processing all the summaryAcks.\n\t * In case there was a summary ack processed by the running summarizer before processIncomingSummaryAcks is called,\n\t * it will wait for the summary ack that is newer than the one indicated by the lastAckRefSeq.\n\t * @param lastAckRefSeq - Identifies the minimum reference sequence number the summarizer needs to wait for.\n\t * In case of a negative number, the summarizer will wait for ANY summary ack that is greater than the deltaManager's initial sequence number,\n\t * and, in case of a positive one, it will wait for a summary ack that is greater than this current reference sequence number.\n\t */\n\tprivate async processIncomingSummaryAcks(lastAckRefSeq: number) {\n\t\tlet refSequenceNumber =\n\t\t\tlastAckRefSeq > 0 ? lastAckRefSeq : this.runtime.deltaManager.initialSequenceNumber;\n\t\twhile (!this.disposed) {\n\t\t\tconst summaryLogger = this.tryGetCorrelatedLogger(refSequenceNumber) ?? this.mc.logger;\n\n\t\t\t// Initialize ack with undefined if exception happens inside of waitSummaryAck on second iteration,\n\t\t\t// we record undefined, not previous handles.\n\t\t\tawait this.summaryCollection.waitSummaryAck(refSequenceNumber);\n\n\t\t\tsummaryLogger.sendTelemetryEvent({\n\t\t\t\teventName: \"processIncomingSummaryAcks\",\n\t\t\t\treferenceSequenceNumber: refSequenceNumber,\n\t\t\t\tlastAckRefSeq,\n\t\t\t});\n\n\t\t\trefSequenceNumber = await this.handleSummaryAck();\n\t\t\t// A valid Summary Ack must have been processed.\n\t\t\tassert(refSequenceNumber >= 0, 0x58f /* Invalid ref sequence number */);\n\t\t}\n\t}\n\n\tpublic dispose(): void {\n\t\tthis.runtime.off(\"op\", this.runtimeListener);\n\t\tthis.summaryWatcher.dispose();\n\t\tthis.heuristicRunner?.dispose();\n\t\tthis.heuristicRunner = undefined;\n\t\tthis.generator.dispose();\n\t\tthis.pendingAckTimer.clear();\n\t\tthis.disposeEnqueuedSummary();\n\t\tthis._disposed = true;\n\t\tthis.stopping = true;\n\t}\n\n\t/**\n\t * RunningSummarizer's logger includes the sequenced index of the current summary on each event.\n\t * If some other Summarizer code wants that event on their logs they can get it here,\n\t * but only if they're logging about that same summary.\n\t * @param summaryOpRefSeq - RefSeq number of the summary op, to ensure the log correlation will be correct\n\t */\n\tpublic tryGetCorrelatedLogger = (summaryOpRefSeq) =>\n\t\tthis.heuristicData.lastAttempt.refSequenceNumber === summaryOpRefSeq\n\t\t\t? this.mc.logger\n\t\t\t: undefined;\n\n\t/** We only want a single heuristic runner micro-task (will provide better optimized grouping of ops) */\n\tprivate heuristicRunnerMicroTaskExists = false;\n\n\tpublic handleOp(op: ISequencedDocumentMessage, runtimeMessage: boolean) {\n\t\tthis.heuristicData.lastOpSequenceNumber = op.sequenceNumber;\n\n\t\tif (runtimeMessage) {\n\t\t\tthis.heuristicData.numRuntimeOps++;\n\t\t} else {\n\t\t\tthis.heuristicData.numNonRuntimeOps++;\n\t\t}\n\n\t\tthis.heuristicData.totalOpsSize += opSize(op);\n\n\t\t// Check for enqueued on-demand summaries; Intentionally do nothing otherwise\n\t\tif (\n\t\t\tthis.initialized &&\n\t\t\tthis.opCanTriggerSummary(op, runtimeMessage) &&\n\t\t\t!this.tryRunEnqueuedSummary() &&\n\t\t\t!this.heuristicRunnerMicroTaskExists\n\t\t) {\n\t\t\tthis.heuristicRunnerMicroTaskExists = true;\n\t\t\tPromise.resolve()\n\t\t\t\t.then(() => {\n\t\t\t\t\tthis.heuristicRunner?.run();\n\t\t\t\t})\n\t\t\t\t.finally(() => {\n\t\t\t\t\tthis.heuristicRunnerMicroTaskExists = false;\n\t\t\t\t});\n\t\t}\n\t}\n\n\t/**\n\t * Can the given op trigger a summary?\n\t * # Currently always prevents summaries for Summarize and SummaryAck/Nack ops\n\t * @param op - op to check\n\t * @returns true if this op can trigger a summary\n\t */\n\tprivate opCanTriggerSummary(op: ISequencedDocumentMessage, runtimeMessage: boolean): boolean {\n\t\tswitch (op.type) {\n\t\t\tcase MessageType.Summarize:\n\t\t\tcase MessageType.SummaryAck:\n\t\t\tcase MessageType.SummaryNack:\n\t\t\t\treturn false;\n\t\t\tdefault:\n\t\t\t\treturn runtimeMessage || this.nonRuntimeOpCanTriggerSummary();\n\t\t}\n\t}\n\n\tprivate nonRuntimeOpCanTriggerSummary(): boolean {\n\t\tconst opsSinceLastAck =\n\t\t\tthis.heuristicData.lastOpSequenceNumber -\n\t\t\tthis.heuristicData.lastSuccessfulSummary.refSequenceNumber;\n\t\treturn (\n\t\t\tthis.configuration.state === \"enabled\" &&\n\t\t\t(this.configuration.nonRuntimeHeuristicThreshold === undefined ||\n\t\t\t\tthis.configuration.nonRuntimeHeuristicThreshold <= opsSinceLastAck)\n\t\t);\n\t}\n\n\tpublic async waitStop(allowLastSummary: boolean): Promise<void> {\n\t\tif (this.stopping) {\n\t\t\treturn;\n\t\t}\n\n\t\tthis.stopping = true;\n\n\t\tthis.disposeEnqueuedSummary();\n\n\t\t// This will try to run lastSummary if needed.\n\t\tif (allowLastSummary && this.heuristicRunner?.shouldRunLastSummary()) {\n\t\t\tif (this.summarizingLock === undefined) {\n\t\t\t\tthis.trySummarizeOnce(\n\t\t\t\t\t// summarizeProps\n\t\t\t\t\t{ summarizeReason: \"lastSummary\" },\n\t\t\t\t\t// ISummarizeOptions, using defaults: { refreshLatestAck: false, fullTree: false }\n\t\t\t\t\t{},\n\t\t\t\t);\n\t\t\t}\n\t\t}\n\n\t\t// Note that trySummarizeOnce() call above returns right away, without waiting.\n\t\t// So we need to wait for its completion, otherwise it would be destroyed right away.\n\t\t// That said, if summary lock was taken upfront, this wait might wait on  multiple retries to\n\t\t// submit summary. We should reconsider this flow and make summarizer move to exit faster.\n\t\t// This resolves when the current pending summary gets an ack or fails.\n\t\tawait this.summarizingLock;\n\t}\n\n\tprivate async waitStart() {\n\t\t// Wait no longer than ack timeout for all pending\n\t\tconst waitStartResult = await raceTimer(\n\t\t\tthis.summaryWatcher.waitFlushed(),\n\t\t\tthis.pendingAckTimer.start(),\n\t\t);\n\t\tthis.pendingAckTimer.clear();\n\n\t\t// Remove pending ack wait timeout by op timestamp comparison, because\n\t\t// it has race conditions with summaries submitted by this same client.\n\t\tthis.summaryCollection.unsetPendingAckTimerTimeoutCallback();\n\n\t\tif (waitStartResult.result === \"done\" && waitStartResult.value !== undefined) {\n\t\t\tthis.heuristicData.updateWithLastSummaryAckInfo({\n\t\t\t\trefSequenceNumber: waitStartResult.value.summaryOp.referenceSequenceNumber,\n\t\t\t\t// This will be the Summarizer starting point so only use timestamps from client's machine.\n\t\t\t\tsummaryTime: Date.now(),\n\t\t\t\tsummarySequenceNumber: waitStartResult.value.summaryOp.sequenceNumber,\n\t\t\t});\n\t\t}\n\t\tthis.initialized = true;\n\t}\n\n\tprivate beforeSummaryAction() {\n\t\tthis.summarizeCount++;\n\t}\n\n\tprivate afterSummaryAction() {\n\t\tconst retry = this.tryWhileSummarizing;\n\t\tthis.tryWhileSummarizing = false;\n\n\t\t// After summarizing, we should check to see if we need to summarize again.\n\t\t// Rerun the heuristics and check for enqueued summaries.\n\t\tif (!this.stopping && !this.tryRunEnqueuedSummary() && retry) {\n\t\t\tthis.heuristicRunner?.run();\n\t\t}\n\t}\n\n\t/**\n\t * Runs single summary action that prevents any other concurrent actions.\n\t * Assumes that caller checked upfront for lack of concurrent action (this.summarizingLock)\n\t * before calling this API. I.e. caller is responsible for either erroring out or waiting on this promise.\n\t * @param before - set of instructions to run before running the action.\n\t * @param action - action to perform.\n\t * @param after - set of instructions to run after running the action.\n\t * @returns - result of action.\n\t */\n\tprivate async lockedSummaryAction<T>(\n\t\tbefore: () => void,\n\t\taction: () => Promise<T>,\n\t\tafter: () => void,\n\t) {\n\t\tassert(\n\t\t\tthis.summarizingLock === undefined,\n\t\t\t0x25b /* \"Caller is responsible for checking lock\" */,\n\t\t);\n\n\t\tconst summarizingLock = new Deferred<void>();\n\t\tthis.summarizingLock = summarizingLock.promise;\n\n\t\tbefore();\n\n\t\treturn action().finally(() => {\n\t\t\tsummarizingLock.resolve();\n\t\t\tthis.summarizingLock = undefined;\n\t\t\tafter();\n\t\t});\n\t}\n\n\t/**\n\t * Runs single summarize attempt\n\t * @param summarizeProps - props to log with each telemetry event associated with this attempt\n\t * @param options - summary options\n\t * @param cancellationToken - cancellation token to use to be able to cancel this summary, if needed\n\t * @param resultsBuilder - optional, result builder to use.\n\t * @returns ISummarizeResult - result of running a summary.\n\t */\n\tprivate trySummarizeOnce(\n\t\tsummarizeProps: ISummarizeTelemetryProperties,\n\t\toptions: ISummarizeOptions,\n\t\tresultsBuilder = new SummarizeResultBuilder(),\n\t): ISummarizeResults {\n\t\tthis.lockedSummaryAction(\n\t\t\t() => {\n\t\t\t\tthis.beforeSummaryAction();\n\t\t\t},\n\t\t\tasync () => {\n\t\t\t\tconst summaryLogger = createChildLogger({\n\t\t\t\t\tlogger: this.mc.logger,\n\t\t\t\t\tproperties: { all: summarizeProps },\n\t\t\t\t});\n\t\t\t\tconst summaryOptions: ISubmitSummaryOptions = {\n\t\t\t\t\t...options,\n\t\t\t\t\tsummaryLogger,\n\t\t\t\t\tcancellationToken: this.cancellationToken,\n\t\t\t\t};\n\t\t\t\tconst summarizeResult = this.generator.summarize(summaryOptions, resultsBuilder);\n\t\t\t\t// ensure we wait till the end of the process\n\t\t\t\treturn summarizeResult.receivedSummaryAckOrNack;\n\t\t\t},\n\t\t\t() => {\n\t\t\t\tthis.afterSummaryAction();\n\t\t\t},\n\t\t).catch((error) => {\n\t\t\t// SummaryGenerator.summarize() does not throw exceptions - it converts them to failed result\n\t\t\t// on resultsBuilder\n\t\t\t// We do not care about exceptions on receivedSummaryAckOrNack - caller should check results\n\t\t\t// and take a appropriate action.\n\t\t});\n\n\t\treturn resultsBuilder.build();\n\t}\n\n\t/** Heuristics summarize attempt. */\n\tprivate trySummarize(reason: SummarizeReason): void {\n\t\tif (this.summarizingLock !== undefined) {\n\t\t\t// lockedSummaryAction() will retry heuristic-based summary at the end of current attempt\n\t\t\t// if it's still needed\n\t\t\tthis.tryWhileSummarizing = true;\n\t\t\treturn;\n\t\t}\n\n\t\tthis.lockedSummaryAction(\n\t\t\t() => {\n\t\t\t\tthis.beforeSummaryAction();\n\t\t\t},\n\t\t\tasync () => {\n\t\t\t\treturn this.mc.config.getBoolean(\"Fluid.Summarizer.TryDynamicRetries\")\n\t\t\t\t\t? this.trySummarizeWithRetries(reason)\n\t\t\t\t\t: this.trySummarizeWithStaticAttempts(reason);\n\t\t\t},\n\t\t\t() => {\n\t\t\t\tthis.afterSummaryAction();\n\t\t\t},\n\t\t).catch((error) => {\n\t\t\tthis.mc.logger.sendErrorEvent({ eventName: \"UnexpectedSummarizeError\" }, error);\n\t\t});\n\t}\n\n\t/**\n\t * Tries to summarize 2 times with pre-defined summary options. If an attempt fails with \"retryAfterSeconds\"\n\t * param, that attempt is tried once more.\n\t */\n\tprivate async trySummarizeWithStaticAttempts(reason: SummarizeReason) {\n\t\tconst attemptOptions: ISummarizeOptions[] = [\n\t\t\t{ refreshLatestAck: false, fullTree: false },\n\t\t\t{ refreshLatestAck: true, fullTree: false },\n\t\t];\n\t\tlet summaryAttempts = 0;\n\t\tlet summaryAttemptsPerPhase = 0;\n\t\tlet summaryAttemptPhase = 0;\n\t\twhile (summaryAttemptPhase < attemptOptions.length) {\n\t\t\tif (this.cancellationToken.cancelled) {\n\t\t\t\treturn;\n\t\t\t}\n\n\t\t\t// We only want to attempt 1 summary when reason is \"lastSummary\"\n\t\t\tif (++summaryAttempts > 1 && reason === \"lastSummary\") {\n\t\t\t\treturn;\n\t\t\t}\n\n\t\t\tsummaryAttemptsPerPhase++;\n\n\t\t\tconst summarizeOptions = attemptOptions[summaryAttemptPhase];\n\t\t\tconst summarizeProps: ISummarizeTelemetryProperties = {\n\t\t\t\tsummarizeReason: reason,\n\t\t\t\tsummaryAttempts,\n\t\t\t\tsummaryAttemptsPerPhase,\n\t\t\t\tsummaryAttemptPhase: summaryAttemptPhase + 1, // make everything 1-based\n\t\t\t\t...summarizeOptions,\n\t\t\t};\n\t\t\tconst summaryLogger = createChildLogger({\n\t\t\t\tlogger: this.mc.logger,\n\t\t\t\tproperties: { all: summarizeProps },\n\t\t\t});\n\t\t\tconst summaryOptions: ISubmitSummaryOptions = {\n\t\t\t\t...summarizeOptions,\n\t\t\t\tsummaryLogger,\n\t\t\t\tcancellationToken: this.cancellationToken,\n\t\t\t};\n\n\t\t\t// Note: no need to account for cancellationToken.waitCancelled here, as\n\t\t\t// this is accounted SummaryGenerator.summarizeCore that controls receivedSummaryAckOrNack.\n\t\t\tconst resultSummarize = this.generator.summarize(summaryOptions);\n\t\t\tconst ackNackResult = await resultSummarize.receivedSummaryAckOrNack;\n\t\t\tif (ackNackResult.success) {\n\t\t\t\treturn;\n\t\t\t}\n\n\t\t\t// Check for retryDelay that can come from summaryNack, upload summary or submit summary flows.\n\t\t\t// Retry the same step only once per retryAfter response.\n\t\t\tconst submitResult = await resultSummarize.summarySubmitted;\n\t\t\tconst delaySeconds = !submitResult.success\n\t\t\t\t? submitResult.data?.retryAfterSeconds\n\t\t\t\t: ackNackResult.data?.retryAfterSeconds;\n\t\t\tif (delaySeconds === undefined || summaryAttemptsPerPhase > 1) {\n\t\t\t\tsummaryAttemptPhase++;\n\t\t\t\tsummaryAttemptsPerPhase = 0;\n\t\t\t}\n\n\t\t\tif (delaySeconds !== undefined) {\n\t\t\t\tthis.mc.logger.sendPerformanceEvent({\n\t\t\t\t\teventName: \"SummarizeAttemptDelay\",\n\t\t\t\t\tduration: delaySeconds,\n\t\t\t\t\tsummaryNackDelay: ackNackResult.data?.retryAfterSeconds !== undefined,\n\t\t\t\t\t...summarizeProps,\n\t\t\t\t});\n\t\t\t\tawait delay(delaySeconds * 1000);\n\t\t\t}\n\t\t}\n\t\tthis.stopSummarizerCallback(\"failToSummarize\");\n\t}\n\n\t/**\n\t * Tries to summarize with retries where retry is based on the failure params.\n\t * For example, summarization may be retried for failures with \"retryAfterSeconds\" param.\n\t */\n\tprivate async trySummarizeWithRetries(reason: SummarizeReason) {\n\t\t// Helper to set summarize options, telemetry properties and call summarize.\n\t\tconst attemptSummarize = (attemptNumber: number, finalAttempt: boolean) => {\n\t\t\tconst summarizeOptions: ISummarizeOptions = {\n\t\t\t\tfullTree: false,\n\t\t\t};\n\t\t\tconst summarizeProps: ISummarizeTelemetryProperties = {\n\t\t\t\tsummarizeReason: reason,\n\t\t\t\tsummaryAttempts: attemptNumber,\n\t\t\t\t...summarizeOptions,\n\t\t\t\tfinalAttempt,\n\t\t\t};\n\t\t\tconst summaryLogger = createChildLogger({\n\t\t\t\tlogger: this.mc.logger,\n\t\t\t\tproperties: { all: summarizeProps },\n\t\t\t});\n\t\t\tconst summaryOptions: ISubmitSummaryOptions = {\n\t\t\t\t...summarizeOptions,\n\t\t\t\tsummaryLogger,\n\t\t\t\tcancellationToken: this.cancellationToken,\n\t\t\t\tfinalAttempt,\n\t\t\t};\n\t\t\tconst summarizeResult = this.generator.summarize(summaryOptions);\n\t\t\treturn { summarizeProps, summarizeResult };\n\t\t};\n\n\t\t// The max number of attempts are based on the stage at which summarization failed. If it fails before it is\n\t\t// submitted, a different value is used compared to if it fails after submission. Usually, in the former case,\n\t\t// we would retry more often as its cheaper and retries are likely to succeed.\n\t\t// This makes it harder to predict how many attempts would actually happen as that depends on how far an attempt\n\t\t// made. To keep things simple, the max attempts is reset after every attempt based on where it failed. This may\n\t\t// result in some failures not being retried depending on what happened before this attempt. That's fine because\n\t\t// such scenarios are very unlikely and even if it happens, it would resolve when a new summarizer starts over.\n\t\t// For example - When failure switches from one the submit failures to nack failure, only one more retry will\n\t\t// happen irrespective of the value of `defaultMaxAttempts`.\n\t\tlet maxAttempts = defaultMaxAttempts;\n\t\tlet currentAttempt = 0;\n\t\tlet retryAfterSeconds: number | undefined;\n\t\tlet done = false;\n\t\tlet result: \"success\" | \"failure\" | \"canceled\" = \"success\";\n\t\tdo {\n\t\t\tcurrentAttempt++;\n\t\t\tif (this.cancellationToken.cancelled) {\n\t\t\t\tresult = \"canceled\";\n\t\t\t\tdone = true;\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tconst { summarizeProps, summarizeResult } = attemptSummarize(\n\t\t\t\tcurrentAttempt,\n\t\t\t\tfalse /* finalAttempt */,\n\t\t\t);\n\n\t\t\t// Ack / nack is the final step, so if it succeeds we're done.\n\t\t\tconst ackNackResult = await summarizeResult.receivedSummaryAckOrNack;\n\t\t\tif (ackNackResult.success) {\n\t\t\t\tresult = \"success\";\n\t\t\t\tdone = true;\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\t// Update max attempts and retry params from the failure result.\n\t\t\t// If submit summary failed, use the params from \"summarySubmitted\" result. Else, use the params\n\t\t\t// from \"receivedSummaryAckOrNack\" result.\n\t\t\t// Note: Check \"summarySubmitted\" result first because if it fails, ack nack would fail as well.\n\t\t\tconst submitSummaryResult = await summarizeResult.summarySubmitted;\n\t\t\tif (!submitSummaryResult.success) {\n\t\t\t\tmaxAttempts = this.maxAttemptsForSubmitFailures;\n\t\t\t\tretryAfterSeconds = submitSummaryResult.data?.retryAfterSeconds;\n\t\t\t} else {\n\t\t\t\tmaxAttempts = defaultMaxAttempts;\n\t\t\t\tretryAfterSeconds = ackNackResult.data?.retryAfterSeconds;\n\t\t\t}\n\n\t\t\t// Emit \"summarize\" event for this failed attempt.\n\t\t\tresult = \"failure\";\n\t\t\tconst eventProps: ISummarizeEventProps = {\n\t\t\t\tresult,\n\t\t\t\tcurrentAttempt,\n\t\t\t\tmaxAttempts,\n\t\t\t\terror: ackNackResult.error,\n\t\t\t};\n\t\t\tthis.emit(\"summarize\", eventProps);\n\n\t\t\t// If the failure doesn't have \"retryAfterSeconds\" or the max number of attempts have been done, we're done.\n\t\t\tif (retryAfterSeconds === undefined || currentAttempt >= maxAttempts - 1) {\n\t\t\t\tdone = true;\n\t\t\t}\n\n\t\t\t// If the failure has \"retryAfterSeconds\", add a delay of that time. In this case, a final attempt will\n\t\t\t// take place and we need to wait for \"retryAfterSeconds\" before that.\n\t\t\tif (retryAfterSeconds !== undefined) {\n\t\t\t\tthis.mc.logger.sendPerformanceEvent({\n\t\t\t\t\teventName: \"SummarizeAttemptDelay\",\n\t\t\t\t\tduration: retryAfterSeconds,\n\t\t\t\t\tsummaryNackDelay: ackNackResult.data?.retryAfterSeconds !== undefined,\n\t\t\t\t\tstage: submitSummaryResult.data?.stage,\n\t\t\t\t\tdynamicRetries: true, // To differentiate this telemetry from regular retry logic\n\t\t\t\t\t...summarizeProps,\n\t\t\t\t});\n\t\t\t\tawait delay(retryAfterSeconds * 1000);\n\t\t\t}\n\t\t} while (!done);\n\n\t\t// If summarize attempt did not fail, emit \"summarize\" event and return. A failed attempt may be retried below.\n\t\tif (result !== \"failure\") {\n\t\t\tthis.emit(\"summarize\", { result, currentAttempt, maxAttempts });\n\t\t\treturn;\n\t\t}\n\n\t\t// If summarization wasn't successful above and the failure contains \"retryAfterSeconds\", perform one last\n\t\t// attempt. This gives a chance to the runtime to perform additional steps in the last attempt.\n\t\tif (retryAfterSeconds !== undefined) {\n\t\t\tconst { summarizeResult } = attemptSummarize(++currentAttempt, true /* finalAttempt */);\n\t\t\t// Ack / nack is the final step, so if it succeeds we're done.\n\t\t\tconst ackNackResult = await summarizeResult.receivedSummaryAckOrNack;\n\t\t\tresult = ackNackResult.success ? \"success\" : \"failure\";\n\t\t\tconst eventProps: ISummarizeEventProps = {\n\t\t\t\tresult,\n\t\t\t\tcurrentAttempt,\n\t\t\t\tmaxAttempts,\n\t\t\t\terror: ackNackResult.success ? undefined : ackNackResult.error,\n\t\t\t};\n\t\t\tthis.emit(\"summarize\", eventProps);\n\t\t}\n\n\t\t// If summarization is still unsuccessful, stop the summarizer.\n\t\tif (result === \"failure\") {\n\t\t\tthis.stopSummarizerCallback(\"failToSummarize\");\n\t\t}\n\t}\n\n\t/** {@inheritdoc (ISummarizer:interface).summarizeOnDemand} */\n\tpublic summarizeOnDemand(\n\t\toptions: IOnDemandSummarizeOptions,\n\t\tresultsBuilder: SummarizeResultBuilder = new SummarizeResultBuilder(),\n\t): ISummarizeResults {\n\t\tif (this.stopping) {\n\t\t\tresultsBuilder.fail(\"RunningSummarizer stopped or disposed\", undefined);\n\t\t\treturn resultsBuilder.build();\n\t\t}\n\t\t// Check for concurrent summary attempts. If one is found,\n\t\t// return a promise that caller can await before trying again.\n\t\tif (this.summarizingLock !== undefined) {\n\t\t\t// The heuristics are blocking concurrent summarize attempts.\n\t\t\tthrow new UsageError(\"Attempted to run an already-running summarizer on demand\");\n\t\t}\n\n\t\tconst { reason, ...summarizeOptions } = options;\n\t\tconst result = this.trySummarizeOnce(\n\t\t\t{ summarizeReason: `onDemand/${reason}` },\n\t\t\tsummarizeOptions,\n\t\t\tresultsBuilder,\n\t\t);\n\t\treturn result;\n\t}\n\n\t/** {@inheritdoc (ISummarizer:interface).enqueueSummarize} */\n\tpublic enqueueSummarize(options: IEnqueueSummarizeOptions): EnqueueSummarizeResult {\n\t\tconst { reason, afterSequenceNumber = 0, override = false, ...summarizeOptions } = options;\n\t\tlet overridden = false;\n\t\tif (this.enqueuedSummary !== undefined) {\n\t\t\tif (!override) {\n\t\t\t\treturn { alreadyEnqueued: true };\n\t\t\t}\n\t\t\t// Override existing enqueued summarize attempt.\n\t\t\tthis.enqueuedSummary.resultsBuilder.fail(\n\t\t\t\t\"Aborted; overridden by another enqueue summarize attempt\",\n\t\t\t\tundefined,\n\t\t\t);\n\t\t\tthis.enqueuedSummary = undefined;\n\t\t\toverridden = true;\n\t\t}\n\n\t\tthis.enqueuedSummary = {\n\t\t\treason: `enqueue;${reason}`,\n\t\t\tafterSequenceNumber,\n\t\t\tsummarizeOptions,\n\t\t\tresultsBuilder: new SummarizeResultBuilder(),\n\t\t};\n\t\tconst results = this.enqueuedSummary.resultsBuilder.build();\n\t\tthis.tryRunEnqueuedSummary();\n\t\treturn overridden\n\t\t\t? {\n\t\t\t\t\t...results,\n\t\t\t\t\talreadyEnqueued: true,\n\t\t\t\t\toverridden: true,\n\t\t\t  }\n\t\t\t: results;\n\t}\n\n\tprivate tryRunEnqueuedSummary() {\n\t\tif (this.stopping) {\n\t\t\tthis.disposeEnqueuedSummary();\n\t\t\treturn false;\n\t\t}\n\t\tif (\n\t\t\tthis.enqueuedSummary === undefined ||\n\t\t\tthis.heuristicData.lastOpSequenceNumber < this.enqueuedSummary.afterSequenceNumber ||\n\t\t\tthis.summarizingLock !== undefined\n\t\t) {\n\t\t\t// If no enqueued summary is ready or a summary is already in progress, take no action.\n\t\t\treturn false;\n\t\t}\n\t\tconst { reason, resultsBuilder, summarizeOptions } = this.enqueuedSummary;\n\t\t// Set to undefined first, so that subsequent enqueue attempt while summarize will occur later.\n\t\tthis.enqueuedSummary = undefined;\n\t\tthis.trySummarizeOnce(\n\t\t\t{ summarizeReason: `enqueuedSummary/${reason}` },\n\t\t\tsummarizeOptions,\n\t\t\tresultsBuilder,\n\t\t);\n\t\treturn true;\n\t}\n\n\tprivate disposeEnqueuedSummary() {\n\t\tif (this.enqueuedSummary !== undefined) {\n\t\t\tthis.enqueuedSummary.resultsBuilder.fail(\n\t\t\t\t\"RunningSummarizer stopped or disposed\",\n\t\t\t\tundefined,\n\t\t\t);\n\t\t\tthis.enqueuedSummary = undefined;\n\t\t}\n\t}\n}\n"]}