@fluidframework/container-runtime 1.2.3-83900 → 2.0.0-internal.1.0.0

package/dist/runningSummarizer.d.ts

@@ -5,7 +5,7 @@
 import { IDisposable, ITelemetryLogger } from "@fluidframework/common-definitions";
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
 import { ISummaryConfiguration } from "./containerRuntime";
-import { IEnqueueSummarizeOptions, ISummarizeHeuristicData, IOnDemandSummarizeOptions, EnqueueSummarizeResult, SummarizerStopReason, ISubmitSummaryOptions, SubmitSummaryResult, ISummaryCancellationToken, ISummarizeResults } from "./summarizerTypes";
+import { IEnqueueSummarizeOptions, ISummarizeHeuristicData, IOnDemandSummarizeOptions, EnqueueSummarizeResult, SummarizerStopReason, ISubmitSummaryOptions, SubmitSummaryResult, ISummaryCancellationToken, ISummarizeResults, ISummarizerRuntime } from "./summarizerTypes";
 import { IClientSummaryWatcher, SummaryCollection } from "./summaryCollection";
 import { SummarizeResultBuilder } from "./summaryGenerator";
 /**
@@ -24,11 +24,13 @@ export declare class RunningSummarizer implements IDisposable {
     private readonly summaryCollection;
     private readonly cancellationToken;
     private readonly stopSummarizerCallback;
-    static start(logger: ITelemetryLogger, summaryWatcher: IClientSummaryWatcher, configuration: ISummaryConfiguration, submitSummaryCallback: (options: ISubmitSummaryOptions) => Promise<SubmitSummaryResult>, heuristicData: ISummarizeHeuristicData, raiseSummarizingError: (errorMessage: string) => void, summaryCollection: SummaryCollection, cancellationToken: ISummaryCancellationToken, stopSummarizerCallback: (reason: SummarizerStopReason) => void): Promise<RunningSummarizer>;
+    private readonly runtime;
+    static start(logger: ITelemetryLogger, summaryWatcher: IClientSummaryWatcher, configuration: ISummaryConfiguration, submitSummaryCallback: (options: ISubmitSummaryOptions) => Promise<SubmitSummaryResult>, heuristicData: ISummarizeHeuristicData, raiseSummarizingError: (errorMessage: string) => void, summaryCollection: SummaryCollection, cancellationToken: ISummaryCancellationToken, stopSummarizerCallback: (reason: SummarizerStopReason) => void, runtime: ISummarizerRuntime): Promise<RunningSummarizer>;
     get disposed(): boolean;
     private stopping;
     private _disposed;
     private summarizingLock;
+    private refreshSummaryAckLock;
     private tryWhileSummarizing;
     private readonly pendingAckTimer;
     private heuristicRunner?;
@@ -37,6 +39,7 @@ export declare class RunningSummarizer implements IDisposable {
     private enqueuedSummary;
     private summarizeCount;
     private totalSuccessfulAttempts;
+    private initialized;
     private constructor();
     dispose(): void;
     /**
@@ -46,10 +49,31 @@ export declare class RunningSummarizer implements IDisposable {
      * @param summaryOpRefSeq - RefSeq number of the summary op, to ensure the log correlation will be correct
      */
     tryGetCorrelatedLogger: (summaryOpRefSeq: any) => ITelemetryLogger | undefined;
-    handleSystemOp(op: ISequencedDocumentMessage): void;
-    handleOp(error: any, { sequenceNumber, type, clientId, contents }: ISequencedDocumentMessage): void;
+    /** We only want a single heuristic runner micro-task (will provide better optimized grouping of ops) */
+    private heuristicRunnerMicroTaskExists;
+    handleOp(op: ISequencedDocumentMessage): void;
+    /**
+     * Can the given op trigger a summary?
+     * # Currently only prevents summaries for Summarize and SummaryAck ops
+     * @param op - op to check
+     * @returns true if this type of op can trigger a summary
+     */
+    private opCanTriggerSummary;
     waitStop(allowLastSummary: boolean): Promise<void>;
     private waitStart;
+    /**
+     * Blocks a new summarizer from running in case RefreshSummaryAck is being processed.
+     * Assumes that caller checked upfront for lack of concurrent action (this.refreshSummaryAckLock)
+     * before calling this API. I.e. caller is responsible for either erroring out or waiting on this promise.
+     * Note: The refreshSummaryAckLock makes sure no summarizer gets enqueued or processed
+     * until the refresh has completed. One can't rely uniquely on the summarizingLock as the
+     * refreshLatestSummaryAck also happens during the time summarizingLock !== undefined.
+     * Ex. Summarizer submits a summay + op and then waits for the Summary Ack to proceed
+     * with the refreshLatestSummaryAck and complete the summary.
+     * @param action - action to perform.
+     * @returns - result of action.
+     */
+    lockedRefreshSummaryAckAction<T>(action: () => Promise<T>): Promise<T>;
     /**
      * Runs single summary action that prevents any other concurrent actions.
      * Assumes that caller checked upfront for lack of concurrent action (this.summarizingLock)

package/dist/runningSummarizer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runningSummarizer.d.ts","sourceRoot":"","sources":["../src/runningSummarizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAGnF,OAAO,EACH,yBAAyB,EAE5B,MAAM,sCAAsC,CAAC;AAE9C,OAAO,EACH,qBAAqB,EACxB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACH,wBAAwB,EAExB,uBAAuB,EAEvB,yBAAyB,EACzB,sBAAsB,EACtB,oBAAoB,EACpB,qBAAqB,EACrB,mBAAmB,EACnB,yBAAyB,EACzB,iBAAiB,EAGpB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EAGH,sBAAsB,EAEzB,MAAM,oBAAoB,CAAC;AAI5B;;;;;;GAMG;AACH,qBAAa,iBAAkB,YAAW,WAAW;IAmD7C,OAAO,CAAC,QAAQ,CAAC,cAAc;IAC/B,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,qBAAqB;IACtC,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,qBAAqB;IACtC,OAAO,CAAC,QAAQ,CAAC,iBAAiB;IAClC,OAAO,CAAC,QAAQ,CAAC,iBAAiB;IAClC,OAAO,CAAC,QAAQ,CAAC,sBAAsB;WAzDvB,KAAK,CACrB,MAAM,EAAE,gBAAgB,EACxB,cAAc,EAAE,qBAAqB,EACrC,aAAa,EAAE,qBAAqB,EACpC,qBAAqB,EAAE,CAAC,OAAO,EAAE,qBAAqB,KAAK,OAAO,CAAC,mBAAmB,CAAC,EACvF,aAAa,EAAE,uBAAuB,EACtC,qBAAqB,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK,IAAI,EACrD,iBAAiB,EAAE,iBAAiB,EACpC,iBAAiB,EAAE,yBAAyB,EAC5C,sBAAsB,EAAE,CAAC,MAAM,EAAE,oBAAoB,KAAK,IAAI,GAC/D,OAAO,CAAC,iBAAiB,CAAC;IAmB7B,IAAW,QAAQ,YAA6B;IAEhD,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,eAAe,CAA4B;IACnD,OAAO,CAAC,mBAAmB,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAe;IAC/C,OAAO,CAAC,eAAe,CAAC,CAA4B;IACpD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmB;IAC1C,OAAO,CAAC,eAAe,CAKT;IACd,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,uBAAuB,CAAK;IAEpC,OAAO;IAiFA,OAAO,IAAI,IAAI;IAWtB;;;;;OAKG;IACI,sBAAsB,yDAGT;IAEb,cAAc,CAAC,EAAE,EAAE,yBAAyB;IAe5C,QAAQ,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,yBAAyB;IAYtF,QAAQ,CAAC,gBAAgB,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;YA4BjD,SAAS;IAsBvB;;;;;;OAMG;YACW,mBAAmB;IAuBjC;;;;;;;OAOG;IACH,OAAO,CAAC,gBAAgB;IAuBxB,oCAAoC;IACpC,OAAO,CAAC,YAAY;IAsFpB,8DAA8D;IACvD,iBAAiB,CACpB,cAAc,oCAAuD,EACrE,EACI,MAAM,EACN,GAAG,OAAO,EACb,EAAE,yBAAyB,GAAG,iBAAiB;IAmBpD,6DAA6D;IACtD,gBAAgB,CAAC,EACpB,MAAM,EACN,mBAAuB,EACvB,QAAgB,EAChB,GAAG,OAAO,EACb,EAAE,wBAAwB,GAAG,sBAAsB;IA8BpD,OAAO,CAAC,qBAAqB;IAwB7B,OAAO,CAAC,sBAAsB;CAMjC"}
+{"version":3,"file":"runningSummarizer.d.ts","sourceRoot":"","sources":["../src/runningSummarizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAInF,OAAO,EACH,yBAAyB,EAE5B,MAAM,sCAAsC,CAAC;AAE9C,OAAO,EACH,qBAAqB,EACxB,MAAM,oBAAoB,CAAC;AAG5B,OAAO,EACH,wBAAwB,EAExB,uBAAuB,EAEvB,yBAAyB,EACzB,sBAAsB,EACtB,oBAAoB,EACpB,qBAAqB,EACrB,mBAAmB,EACnB,yBAAyB,EACzB,iBAAiB,EAEjB,kBAAkB,EAErB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EAGH,sBAAsB,EAEzB,MAAM,oBAAoB,CAAC;AAI5B;;;;;;GAMG;AACH,qBAAa,iBAAkB,YAAW,WAAW;IA6E7C,OAAO,CAAC,QAAQ,CAAC,cAAc;IAC/B,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,qBAAqB;IACtC,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,qBAAqB;IACtC,OAAO,CAAC,QAAQ,CAAC,iBAAiB;IAClC,OAAO,CAAC,QAAQ,CAAC,iBAAiB;IAClC,OAAO,CAAC,QAAQ,CAAC,sBAAsB;IACvC,OAAO,CAAC,QAAQ,CAAC,OAAO;WApFR,KAAK,CACrB,MAAM,EAAE,gBAAgB,EACxB,cAAc,EAAE,qBAAqB,EACrC,aAAa,EAAE,qBAAqB,EACpC,qBAAqB,EAAE,CAAC,OAAO,EAAE,qBAAqB,KAAK,OAAO,CAAC,mBAAmB,CAAC,EACvF,aAAa,EAAE,uBAAuB,EACtC,qBAAqB,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK,IAAI,EACrD,iBAAiB,EAAE,iBAAiB,EACpC,iBAAiB,EAAE,yBAAyB,EAC5C,sBAAsB,EAAE,CAAC,MAAM,EAAE,oBAAoB,KAAK,IAAI,EAC9D,OAAO,EAAE,kBAAkB,GAC5B,OAAO,CAAC,iBAAiB,CAAC;IA2C7B,IAAW,QAAQ,YAA6B;IAChD,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,eAAe,CAA4B;IACnD,OAAO,CAAC,qBAAqB,CAA4B;IACzD,OAAO,CAAC,mBAAmB,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAe;IAC/C,OAAO,CAAC,eAAe,CAAC,CAA4B;IACpD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmB;IAC1C,OAAO,CAAC,eAAe,CAKT;IACd,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,uBAAuB,CAAK;IACpC,OAAO,CAAC,WAAW,CAAS;IAE5B,OAAO;IAqFA,OAAO,IAAI,IAAI;IAYtB;;;;;OAKG;IACI,sBAAsB,yDAGT;IAEpB,wGAAwG;IACxG,OAAO,CAAC,8BAA8B,CAAS;IAExC,QAAQ,CAAC,EAAE,EAAE,yBAAyB;IAyB7C;;;;;OAKG;IACH,OAAO,CAAC,mBAAmB;IAWd,QAAQ,CAAC,gBAAgB,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;YA4BjD,SAAS;IAuBvB;;;;;;;;;;;OAWG;IACU,6BAA6B,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC;IAatE;;;;;;OAMG;YACW,mBAAmB;IAyBjC;;;;;;;OAOG;IACH,OAAO,CAAC,gBAAgB;IAuBxB,oCAAoC;IACpC,OAAO,CAAC,YAAY;IA0FpB,8DAA8D;IACvD,iBAAiB,CACpB,cAAc,oCAAuD,EACrE,EACI,MAAM,EACN,GAAG,OAAO,EACb,EAAE,yBAAyB,GAAG,iBAAiB;IAoBpD,6DAA6D;IACtD,gBAAgB,CAAC,EACpB,MAAM,EACN,mBAAuB,EACvB,QAAgB,EAChB,GAAG,OAAO,EACb,EAAE,wBAAwB,GAAG,sBAAsB;IA8BpD,OAAO,CAAC,qBAAqB;IAwB7B,OAAO,CAAC,sBAAsB;CAMjC"}

package/dist/runningSummarizer.js

@@ -18,8 +18,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.RunningSummarizer = void 0;
 const common_utils_1 = require("@fluidframework/common-utils");
 const container_utils_1 = require("@fluidframework/container-utils");
+const driver_utils_1 = require("@fluidframework/driver-utils");
 const protocol_definitions_1 = require("@fluidframework/protocol-definitions");
 const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
+const opProperties_1 = require("./opProperties");
 const summarizerHeuristics_1 = require("./summarizerHeuristics");
 const summaryGenerator_1 = require("./summaryGenerator");
 const maxSummarizeAckWaitTime = 10 * 60 * 1000; // 10 minutes
@@ -31,7 +33,7 @@ const maxSummarizeAckWaitTime = 10 * 60 * 1000; // 10 minutes
  * This object is created and controlled by Summarizer object.
  */
 class RunningSummarizer {
-    constructor(baseLogger, summaryWatcher, configuration, submitSummaryCallback, heuristicData, raiseSummarizingError, summaryCollection, cancellationToken, stopSummarizerCallback) {
+    constructor(baseLogger, summaryWatcher, configuration, submitSummaryCallback, heuristicData, raiseSummarizingError, summaryCollection, cancellationToken, stopSummarizerCallback, runtime) {
         this.summaryWatcher = summaryWatcher;
         this.configuration = configuration;
         this.submitSummaryCallback = submitSummaryCallback;
@@ -40,11 +42,13 @@ class RunningSummarizer {
         this.summaryCollection = summaryCollection;
         this.cancellationToken = cancellationToken;
         this.stopSummarizerCallback = stopSummarizerCallback;
+        this.runtime = runtime;
         this.stopping = false;
         this._disposed = false;
         this.tryWhileSummarizing = false;
         this.summarizeCount = 0;
         this.totalSuccessfulAttempts = 0;
+        this.initialized = false;
         /**
          * RunningSummarizer's logger includes the sequenced index of the current summary on each event.
          * If some other Summarizer code wants that event on their logs they can get it here,
@@ -54,6 +58,8 @@ class RunningSummarizer {
         this.tryGetCorrelatedLogger = (summaryOpRefSeq) => this.heuristicData.lastAttempt.refSequenceNumber === summaryOpRefSeq
             ? this.logger
             : undefined;
+        /** We only want a single heuristic runner micro-task (will provide better optimized grouping of ops) */
+        this.heuristicRunnerMicroTaskExists = false;
         const telemetryProps = {
             summarizeCount: () => this.summarizeCount,
             summarizerSuccessfulAttempts: () => this.totalSuccessfulAttempts,
@@ -95,18 +101,39 @@ class RunningSummarizer {
             }
         });
         this.generator = new summaryGenerator_1.SummaryGenerator(this.pendingAckTimer, this.heuristicData, this.submitSummaryCallback, this.raiseSummarizingError, () => { this.totalSuccessfulAttempts++; }, this.summaryWatcher, this.logger);
+        // Listen for ops
+        this.runtime.deltaManager.on("op", (op) => { this.handleOp(op); });
     }
-    static async start(logger, summaryWatcher, configuration, submitSummaryCallback, heuristicData, raiseSummarizingError, summaryCollection, cancellationToken, stopSummarizerCallback) {
-        var _a;
-        const summarizer = new RunningSummarizer(logger, summaryWatcher, configuration, submitSummaryCallback, heuristicData, raiseSummarizingError, summaryCollection, cancellationToken, stopSummarizerCallback);
+    static async start(logger, summaryWatcher, configuration, submitSummaryCallback, heuristicData, raiseSummarizingError, summaryCollection, cancellationToken, stopSummarizerCallback, runtime) {
+        var _a, _b;
+        const summarizer = new RunningSummarizer(logger, summaryWatcher, configuration, submitSummaryCallback, heuristicData, raiseSummarizingError, summaryCollection, cancellationToken, stopSummarizerCallback, runtime);
         await summarizer.waitStart();
-        // Run the heuristics after starting
-        (_a = summarizer.heuristicRunner) === null || _a === void 0 ? void 0 : _a.run();
+        // Update heuristic counts
+        // By the time we get here, there are potentially ops missing from the heuristic summary counts
+        // Examples of where this could happen:
+        // 1. Op is processed during the time that we are initiating the RunningSummarizer instance but before we
+        //    listen for the op events (will get missed by the handlers in the current workflow)
+        // 2. Op was sequenced after the last time we summarized (op sequence number > summarize ref sequence number)
+        const diff = runtime.deltaManager.lastSequenceNumber - (heuristicData.lastSuccessfulSummary.refSequenceNumber
+            + heuristicData.numNonRuntimeOps
+            + heuristicData.numRuntimeOps);
+        heuristicData.hasMissingOpData = diff > 0;
+        if (heuristicData.hasMissingOpData) {
+            // Split the diff 50-50 and increment the counts appropriately
+            heuristicData.numNonRuntimeOps += Math.ceil(diff / 2);
+            heuristicData.numRuntimeOps += Math.floor(diff / 2);
+        }
+        // Update last seq number (in case the handlers haven't processed anything yet)
+        heuristicData.lastOpSequenceNumber = runtime.deltaManager.lastSequenceNumber;
+        // Start heuristics
+        (_a = summarizer.heuristicRunner) === null || _a === void 0 ? void 0 : _a.start();
+        (_b = summarizer.heuristicRunner) === null || _b === void 0 ? void 0 : _b.run();
         return summarizer;
     }
     get disposed() { return this._disposed; }
     dispose() {
         var _a;
+        this.runtime.deltaManager.off("op", (op) => { this.handleOp(op); });
         this.summaryWatcher.dispose();
         (_a = this.heuristicRunner) === null || _a === void 0 ? void 0 : _a.dispose();
         this.heuristicRunner = undefined;
@@ -116,29 +143,43 @@ class RunningSummarizer {
         this._disposed = true;
         this.stopping = true;
     }
-    handleSystemOp(op) {
-        switch (op.type) {
-            case protocol_definitions_1.MessageType.ClientLeave:
-            case protocol_definitions_1.MessageType.ClientJoin:
-            case protocol_definitions_1.MessageType.Propose: {
-                // Synchronously handle quorum ops like regular ops
-                this.handleOp(undefined, op);
-                return;
-            }
-            default: {
-                return;
-            }
+    handleOp(op) {
+        this.heuristicData.lastOpSequenceNumber = op.sequenceNumber;
+        if (op.type !== protocol_definitions_1.MessageType.Summarize && (0, driver_utils_1.isRuntimeMessage)(op)) {
+            this.heuristicData.numRuntimeOps++;
         }
-    }
-    handleOp(error, { sequenceNumber, type, clientId, contents }) {
-        var _a;
-        if (error !== undefined) {
-            return;
+        else {
+            this.heuristicData.numNonRuntimeOps++;
         }
-        this.heuristicData.lastOpSequenceNumber = sequenceNumber;
+        this.heuristicData.totalOpsSize += (0, opProperties_1.opSize)(op);
         // Check for enqueued on-demand summaries; Intentionally do nothing otherwise
-        if (!this.tryRunEnqueuedSummary()) {
-            (_a = this.heuristicRunner) === null || _a === void 0 ? void 0 : _a.run();
+        if (this.initialized
+            && this.opCanTriggerSummary(op)
+            && !this.tryRunEnqueuedSummary()
+            && !this.heuristicRunnerMicroTaskExists) {
+            this.heuristicRunnerMicroTaskExists = true;
+            Promise.resolve().then(() => {
+                var _a;
+                (_a = this.heuristicRunner) === null || _a === void 0 ? void 0 : _a.run();
+            }).finally(() => {
+                this.heuristicRunnerMicroTaskExists = false;
+            });
+        }
+    }
+    /**
+     * Can the given op trigger a summary?
+     * # Currently only prevents summaries for Summarize and SummaryAck ops
+     * @param op - op to check
+     * @returns true if this type of op can trigger a summary
+     */
+    opCanTriggerSummary(op) {
+        switch (op.type) {
+            case protocol_definitions_1.MessageType.Summarize:
+            case protocol_definitions_1.MessageType.SummaryAck:
+            case protocol_definitions_1.MessageType.SummaryNack:
+                return false;
+            default:
+                return true;
         }
     }
     async waitStop(allowLastSummary) {
@@ -180,6 +221,28 @@ class RunningSummarizer {
                 summarySequenceNumber: waitStartResult.value.summaryOp.sequenceNumber,
             });
         }
+        this.initialized = true;
+    }
+    /**
+     * Blocks a new summarizer from running in case RefreshSummaryAck is being processed.
+     * Assumes that caller checked upfront for lack of concurrent action (this.refreshSummaryAckLock)
+     * before calling this API. I.e. caller is responsible for either erroring out or waiting on this promise.
+     * Note: The refreshSummaryAckLock makes sure no summarizer gets enqueued or processed
+     * until the refresh has completed. One can't rely uniquely on the summarizingLock as the
+     * refreshLatestSummaryAck also happens during the time summarizingLock !== undefined.
+     * Ex. Summarizer submits a summay + op and then waits for the Summary Ack to proceed
+     * with the refreshLatestSummaryAck and complete the summary.
+     * @param action - action to perform.
+     * @returns - result of action.
+     */
+    async lockedRefreshSummaryAckAction(action) {
+        (0, common_utils_1.assert)(this.refreshSummaryAckLock === undefined, 0x396 /* Refresh Summary Ack - Caller is responsible for checking lock */);
+        const refreshSummaryAckLock = new common_utils_1.Deferred();
+        this.refreshSummaryAckLock = refreshSummaryAckLock.promise;
+        return action().finally(() => {
+            refreshSummaryAckLock.resolve();
+            this.refreshSummaryAckLock = undefined;
+        });
     }
     /**
      * Runs single summary action that prevents any other concurrent actions.
@@ -193,6 +256,8 @@ class RunningSummarizer {
         const summarizingLock = new common_utils_1.Deferred();
         this.summarizingLock = summarizingLock.promise;
         this.summarizeCount++;
+        // Make sure the RefreshLatestSummaryAck is not being executed.
+        await this.refreshSummaryAckLock;
         return action().finally(() => {
             var _a;
             summarizingLock.resolve();
@@ -264,6 +329,8 @@ class RunningSummarizer {
                     this.logger.sendPerformanceEvent(Object.assign({ eventName: "SummarizeAttemptDelay", duration: delaySeconds, summaryNackDelay: overrideDelaySeconds !== undefined }, summarizeProps));
                     await (0, common_utils_1.delay)(delaySeconds * 1000);
                 }
+                // Make sure the refresh Summary Ack is not being executed.
+                await this.refreshSummaryAckLock;
                 // Note: no need to account for cancellationToken.waitCancelled here, as
                 // this is accounted SummaryGenerator.summarizeCore that controls receivedSummaryAckOrNack.
                 const resultSummarize = this.generator.summarize(summarizeProps, options, cancellationToken);

package/dist/runningSummarizer.js.map

@@ -1 +1 @@
-{"version":3,"file":"runningSummarizer.js","sourceRoot":"","sources":["../src/runningSummarizer.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;;;;;;;;;AAGH,+DAAqF;AACrF,qEAA6D;AAC7D,+EAG8C;AAC9C,qEAA8D;AAI9D,iEAAkE;AAiBlE,yDAK4B;AAE5B,MAAM,uBAAuB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,aAAa;AAE7D;;;;;;GAMG;AACH,MAAa,iBAAiB;IAiD1B,YACI,UAA4B,EACX,cAAqC,EACrC,aAAoC,EACpC,qBAAuF,EACvF,aAAsC,EACtC,qBAAqD,EACrD,iBAAoC,EACpC,iBAA4C,EAC5C,sBAA8D;QAP9D,mBAAc,GAAd,cAAc,CAAuB;QACrC,kBAAa,GAAb,aAAa,CAAuB;QACpC,0BAAqB,GAArB,qBAAqB,CAAkE;QACvF,kBAAa,GAAb,aAAa,CAAyB;QACtC,0BAAqB,GAArB,qBAAqB,CAAgC;QACrD,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,sBAAiB,GAAjB,iBAAiB,CAA2B;QAC5C,2BAAsB,GAAtB,sBAAsB,CAAwC;QA1B3E,aAAQ,GAAG,KAAK,CAAC;QACjB,cAAS,GAAG,KAAK,CAAC;QAElB,wBAAmB,GAAG,KAAK,CAAC;QAW5B,mBAAc,GAAG,CAAC,CAAC;QACnB,4BAAuB,GAAG,CAAC,CAAC;QA8FpC;;;;;WAKG;QACI,2BAAsB,GAAG,CAAC,eAAe,EAAE,EAAE,CAChD,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,iBAAiB,KAAK,eAAe;YAChE,CAAC,CAAC,IAAI,CAAC,MAAM;YACb,CAAC,CAAC,SAAS,CAAC;QA1FhB,MAAM,cAAc,GAA8B;YAC9C,cAAc,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,cAAc;YACzC,4BAA4B,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,uBAAuB;SACnE,CAAC;QAEF,IAAI,CAAC,MAAM,GAAG,6BAAW,CAAC,MAAM,CAC5B,UAAU,EAAE,SAAS,EACrB;YACI,GAAG,EAAE,cAAc;SACtB,CACJ,CAAC;QAEF,IAAI,aAAa,CAAC,KAAK,KAAK,mBAAmB,EAAE;YAC7C,IAAA,qBAAM,EAAC,IAAI,CAAC,aAAa,CAAC,KAAK,KAAK,SAAS,EAAE,KAAK,CAAC,6CAA6C,CAAC,CAAC;YACpG,IAAI,CAAC,eAAe,GAAG,IAAI,+CAAwB,CAC/C,aAAa,EACb,IAAI,CAAC,aAAa,EAClB,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,EACrC,IAAI,CAAC,MAAM,CAAC,CAAC;SACpB;QAED,IAAA,qBAAM,EACF,IAAI,CAAC,aAAa,CAAC,KAAK,KAAK,UAAU,EACvC,KAAK,CAAC,yDAAyD,CAClE,CAAC;QAEF,oGAAoG;QACpG,+FAA+F;QAE/F,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,cAAc,EAAE,uBAAuB,CAAC,CAAC;QAE5F,IAAI,CAAC,eAAe,GAAG,IAAI,2BAAY,CACnC,cAAc,EACd,GAAG,EAAE;YACD,gDAAgD;YAChD,IAAI,CAAC,qBAAqB,CAAC,0CAA0C,CAAC,CAAC;YACvE,+DAA+D;YAC/D,sEAAsE;YACtE,2DAA2D;YAC3D,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC;gBACvB,SAAS,EAAE,uBAAuB;gBAClC,cAAc;gBACd,uBAAuB,EAAE,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,iBAAiB;gBACzE,qBAAqB,EAAE,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,qBAAqB;gBAC3E,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,WAAW;aACvE,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QACP,iFAAiF;QACjF,iBAAiB,CAAC,iCAAiC,CAAC,cAAc,EAAE,GAAG,EAAE;YACrE,IAAI,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE;gBAC/B,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC;oBAC3B,SAAS,EAAE,6BAA6B;oBACxC,uBAAuB,EAAE,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,iBAAiB;oBACzE,qBAAqB,EAAE,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,qBAAqB;iBAC9E,CAAC,CAAC;gBACH,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;aAChC;QACL,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,SAAS,GAAG,IAAI,mCAAgB,CACjC,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,qBAAqB,EAC1B,IAAI,CAAC,qBAAqB,EAC1B,GAAG,EAAE,GAAG,IAAI,CAAC,uBAAuB,EAAE,CAAC,CAAC,CAAC,EACzC,IAAI,CAAC,cAAc,EACnB,IAAI,CAAC,MAAM,CACd,CAAC;IACN,CAAC;IA/HM,MAAM,CAAC,KAAK,CAAC,KAAK,CACrB,MAAwB,EACxB,cAAqC,EACrC,aAAoC,EACpC,qBAAuF,EACvF,aAAsC,EACtC,qBAAqD,EACrD,iBAAoC,EACpC,iBAA4C,EAC5C,sBAA8D;;QAE9D,MAAM,UAAU,GAAG,IAAI,iBAAiB,CACpC,MAAM,EACN,cAAc,EACd,aAAa,EACb,qBAAqB,EACrB,aAAa,EACb,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,EACjB,sBAAsB,CAAC,CAAC;QAE5B,MAAM,UAAU,CAAC,SAAS,EAAE,CAAC;QAE7B,oCAAoC;QACpC,MAAA,UAAU,CAAC,eAAe,0CAAE,GAAG,EAAE,CAAC;QAClC,OAAO,UAAU,CAAC;IACtB,CAAC;IAED,IAAW,QAAQ,KAAK,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAoGzC,OAAO;;QACV,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC;QAC9B,MAAA,IAAI,CAAC,eAAe,0CAAE,OAAO,EAAE,CAAC;QAChC,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;QACjC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QACzB,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC7B,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC9B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACzB,CAAC;IAaM,cAAc,CAAC,EAA6B;QAC/C,QAAQ,EAAE,CAAC,IAAI,EAAE;YACb,KAAK,kCAAW,CAAC,WAAW,CAAC;YAC7B,KAAK,kCAAW,CAAC,UAAU,CAAC;YAC5B,KAAK,kCAAW,CAAC,OAAO,CAAC,CAAC;gBACtB,mDAAmD;gBACnD,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;gBAC7B,OAAO;aACV;YACD,OAAO,CAAC,CAAC;gBACL,OAAO;aACV;SACJ;IACL,CAAC;IAEM,QAAQ,CAAC,KAAU,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAA6B;;QAC/F,IAAI,KAAK,KAAK,SAAS,EAAE;YACrB,OAAO;SACV;QACD,IAAI,CAAC,aAAa,CAAC,oBAAoB,GAAG,cAAc,CAAC;QAEzD,6EAA6E;QAC7E,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,EAAE;YAC/B,MAAA,IAAI,CAAC,eAAe,0CAAE,GAAG,EAAE,CAAC;SAC/B;IACL,CAAC;IAEM,KAAK,CAAC,QAAQ,CAAC,gBAAyB;;QAC3C,IAAI,IAAI,CAAC,QAAQ,EAAE;YACf,OAAO;SACV;QAED,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QAErB,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAE9B,8CAA8C;QAC9C,IAAI,gBAAgB,KAAI,MAAA,IAAI,CAAC,eAAe,0CAAE,oBAAoB,EAAE,CAAA,EAAE;YAClE,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE;gBACpC,IAAI,CAAC,gBAAgB;gBACjB,iBAAiB;gBACjB,EAAE,MAAM,EAAE,aAAa,EAAE;gBACzB,kFAAkF;gBAClF,EAAE,CAAC,CAAC;aACX;SACJ;QAED,+EAA+E;QAC/E,qFAAqF;QACrF,6FAA6F;QAC7F,0FAA0F;QAC1F,uEAAuE;QACvE,MAAM,IAAI,CAAC,eAAe,CAAC;IAC/B,CAAC;IAEO,KAAK,CAAC,SAAS;QACnB,kDAAkD;QAClD,MAAM,eAAe,GAAG,MAAM,IAAA,4BAAS,EACnC,IAAI,CAAC,cAAc,CAAC,WAAW,EAAE,EACjC,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAC/B,CAAC;QACF,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAE7B,sEAAsE;QACtE,uEAAuE;QACvE,IAAI,CAAC,iBAAiB,CAAC,mCAAmC,EAAE,CAAC;QAE7D,IAAI,eAAe,CAAC,MAAM,KAAK,MAAM,IAAI,eAAe,CAAC,KAAK,KAAK,SAAS,EAAE;YAC1E,IAAI,CAAC,aAAa,CAAC,4BAA4B,CAAC;gBAC5C,iBAAiB,EAAE,eAAe,CAAC,KAAK,CAAC,SAAS,CAAC,uBAAuB;gBAC1E,2FAA2F;gBAC3F,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;gBACvB,qBAAqB,EAAE,eAAe,CAAC,KAAK,CAAC,SAAS,CAAC,cAAc;aACxE,CAAC,CAAC;SACN;IACL,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,mBAAmB,CAAI,MAAwB;QACzD,IAAA,qBAAM,EAAC,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE,KAAK,CAAC,+CAA+C,CAAC,CAAC;QAElG,MAAM,eAAe,GAAG,IAAI,uBAAQ,EAAQ,CAAC;QAC7C,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC,OAAO,CAAC;QAE/C,IAAI,CAAC,cAAc,EAAE,CAAC;QAEtB,OAAO,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;;YACzB,eAAe,CAAC,OAAO,EAAE,CAAC;YAC1B,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;YAEjC,MAAM,KAAK,GAAG,IAAI,CAAC,mBAAmB,CAAC;YACvC,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;YAEjC,2EAA2E;YAC3E,yDAAyD;YACzD,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,IAAI,KAAK,EAAE;gBAC1D,MAAA,IAAI,CAAC,eAAe,0CAAE,GAAG,EAAE,CAAC;aAC/B;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;OAOG;IACK,gBAAgB,CACpB,cAA6C,EAC7C,OAA0B,EAC1B,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,EAC1C,cAAc,GAAG,IAAI,yCAAsB,EAAE;QAC7C,IAAI,CAAC,mBAAmB,CAAC,KAAK,IAAI,EAAE;YAChC,MAAM,eAAe,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAC5C,cAAc,EACd,OAAO,EACP,iBAAiB,EACjB,cAAc,CAAC,CAAC;YACpB,6CAA6C;YAC7C,OAAO,eAAe,CAAC,wBAAwB,CAAC;QACpD,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACf,6FAA6F;YAC7F,oBAAoB;YACpB,4FAA4F;YAC5F,iCAAiC;QACrC,CAAC,CAAC,CAAC;QAEH,OAAO,cAAc,CAAC,KAAK,EAAE,CAAC;IAClC,CAAC;IAED,oCAAoC;IAC5B,YAAY,CAChB,MAAuB,EACvB,iBAAiB,GAAG,IAAI,CAAC,iBAAiB;QAC1C,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE;YACpC,yFAAyF;YACzF,uBAAuB;YACvB,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;YAChC,OAAO;SACV;QAED,IAAI,CAAC,mBAAmB,CAAC,KAAK,IAAI,EAAE;YAChC,MAAM,QAAQ,GAAuD;gBACjE,EAAE,gBAAgB,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE;gBAC5C,EAAE,gBAAgB,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE;gBAC3C,EAAE,gBAAgB,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,GAAG,EAAE,EAAE;gBACjE,EAAE,gBAAgB,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE,GAAG,EAAE,EAAE;aACpE,CAAC;YACF,IAAI,oBAAwC,CAAC;YAC7C,IAAI,eAAe,GAAG,CAAC,CAAC;YACxB,IAAI,uBAAuB,GAAG,CAAC,CAAC;YAEhC,IAAI,UAAwD,CAAC;YAE7D,KAAK,IAAI,mBAAmB,GAAG,CAAC,EAAE,mBAAmB,GAAG,QAAQ,CAAC,MAAM,GAAG;gBACtE,IAAI,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE;oBAClC,OAAO;iBACV;gBAED,iEAAiE;gBACjE,IAAI,EAAE,eAAe,GAAG,CAAC,IAAI,MAAM,KAAK,aAAa,EAAE;oBACnD,OAAO;iBACV;gBAED,uBAAuB,EAAE,CAAC;gBAE1B,MAAM,KAAwD,QAAQ,CAAC,mBAAmB,CAAC,EAArF,EAAE,YAAY,EAAE,mBAAmB,GAAG,CAAC,OAA8C,EAAzC,OAAO,cAAnD,gBAAqD,CAAgC,CAAC;gBAC5F,MAAM,YAAY,GAAG,oBAAoB,aAApB,oBAAoB,cAApB,oBAAoB,GAAI,mBAAmB,CAAC;gBAEjE,MAAM,cAAc,mBAChB,MAAM;oBACN,eAAe;oBACf,uBAAuB,EACvB,mBAAmB,EAAE,mBAAmB,GAAG,CAAC,IACzC,OAAO,CACb,CAAC;gBAEF,IAAI,YAAY,GAAG,CAAC,EAAE;oBAClB,IAAI,CAAC,MAAM,CAAC,oBAAoB,iBAC5B,SAAS,EAAE,uBAAuB,EAClC,QAAQ,EAAE,YAAY,EACtB,gBAAgB,EAAE,oBAAoB,KAAK,SAAS,IACjD,cAAc,EACnB,CAAC;oBACH,MAAM,IAAA,oBAAK,EAAC,YAAY,GAAG,IAAI,CAAC,CAAC;iBACpC;gBACD,wEAAwE;gBACxE,2FAA2F;gBAC3F,MAAM,eAAe,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,cAAc,EAAE,OAAO,EAAE,iBAAiB,CAAC,CAAC;gBAC7F,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,wBAAwB,CAAC;gBAE9D,IAAI,MAAM,CAAC,OAAO,EAAE;oBAChB,OAAO;iBACV;gBACD,8EAA8E;gBAC9E,yDAAyD;gBACzD,oBAAoB,GAAG,MAAM,CAAC,iBAAiB,CAAC;gBAChD,IAAI,oBAAoB,KAAK,SAAS,IAAI,uBAAuB,GAAG,CAAC,EAAE;oBACnE,mBAAmB,EAAE,CAAC;oBACtB,uBAAuB,GAAG,CAAC,CAAC;iBAC/B;gBACD,UAAU,GAAG,MAAM,CAAC;aACvB;YAED,gGAAgG;YAChG,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC;gBACvB,SAAS,EAAE,iBAAiB;gBAC5B,MAAM;gBACN,OAAO,EAAE,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,OAAO;aAC/B,EAAE,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,KAAK,CAAC,CAAC;YAEtB,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACf,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,0BAA0B,EAAE,EAAE,KAAK,CAAC,CAAC;QACjF,CAAC,CAAC,CAAC;IACP,CAAC;IAED,8DAA8D;IACvD,iBAAiB,CACpB,iBAAyC,IAAI,yCAAsB,EAAE,EACrE,EAG4B;YAH5B,EACI,MAAM,OAEkB,EADrB,OAAO,cAFd,UAGC,CADa;QAEd,IAAI,IAAI,CAAC,QAAQ,EAAE;YACf,cAAc,CAAC,IAAI,CAAC,uCAAuC,EAAE,SAAS,CAAC,CAAC;YACxE,OAAO,cAAc,CAAC,KAAK,EAAE,CAAC;SACjC;QACD,0DAA0D;QAC1D,8DAA8D;QAC9D,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE;YACpC,6DAA6D;YAC7D,MAAM,IAAI,4BAAU,CAAC,0DAA0D,CAAC,CAAC;SACpF;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAChC,EAAE,MAAM,EAAE,YAAY,MAAM,EAAE,EAAE,EAChC,OAAO,EACP,IAAI,CAAC,iBAAiB,EACtB,cAAc,CAAC,CAAC;QACpB,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,6DAA6D;IACtD,gBAAgB,CAAC,EAKG;YALH,EACpB,MAAM,EACN,mBAAmB,GAAG,CAAC,EACvB,QAAQ,GAAG,KAAK,OAEO,EADpB,OAAO,cAJU,6CAKvB,CADa;QAEV,MAAM,cAAc,GAAG,WAAW,MAAM,EAAW,CAAC;QACpD,IAAI,UAAU,GAAG,KAAK,CAAC;QACvB,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE;YACpC,IAAI,CAAC,QAAQ,EAAE;gBACX,OAAO,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;aACpC;YACD,gDAAgD;YAChD,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,IAAI,CACpC,0DAA0D,EAC1D,SAAS,CACZ,CAAC;YACF,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;YACjC,UAAU,GAAG,IAAI,CAAC;SACrB;QACD,IAAI,CAAC,eAAe,GAAG;YACnB,MAAM,EAAE,cAAc;YACtB,mBAAmB;YACnB,OAAO;YACP,cAAc,EAAE,IAAI,yCAAsB,EAAE;SAC/C,CAAC;QACF,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC5D,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAC7B,OAAO,UAAU,CAAC,CAAC,iCACZ,OAAO,KACV,eAAe,EAAE,IAAI,EACrB,UAAU,EAAE,IAAI,IAClB,CAAC,CAAC,OAAO,CAAC;IAChB,CAAC;IAEO,qBAAqB;QACzB,IAAI,IAAI,CAAC,QAAQ,EAAE;YACf,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAC9B,OAAO,KAAK,CAAC;SAChB;QACD,IACI,IAAI,CAAC,eAAe,KAAK,SAAS;eAC/B,IAAI,CAAC,aAAa,CAAC,oBAAoB,GAAG,IAAI,CAAC,eAAe,CAAC,mBAAmB;eAClF,IAAI,CAAC,eAAe,KAAK,SAAS,EACvC;YACE,uFAAuF;YACvF,OAAO,KAAK,CAAC;SAChB;QACD,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC;QACjE,+FAA+F;QAC/F,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;QACjC,IAAI,CAAC,gBAAgB,CACjB,EAAE,MAAM,EAAE,mBAAmB,MAAM,EAAE,EAAE,EACvC,OAAO,EACP,IAAI,CAAC,iBAAiB,EACtB,cAAc,CAAC,CAAC;QACpB,OAAO,IAAI,CAAC;IAChB,CAAC;IAEO,sBAAsB;QAC1B,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE;YACpC,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,IAAI,CAAC,uCAAuC,EAAE,SAAS,CAAC,CAAC;YAC7F,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;SACpC;IACL,CAAC;CACJ;AApdD,8CAodC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { IDisposable, ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { assert, delay, Deferred, PromiseTimer } from \"@fluidframework/common-utils\";\nimport { UsageError } from \"@fluidframework/container-utils\";\nimport {\n    ISequencedDocumentMessage,\n    MessageType,\n} from \"@fluidframework/protocol-definitions\";\nimport { ChildLogger } from \"@fluidframework/telemetry-utils\";\nimport {\n    ISummaryConfiguration,\n} from \"./containerRuntime\";\nimport { SummarizeHeuristicRunner } from \"./summarizerHeuristics\";\nimport {\n    IEnqueueSummarizeOptions,\n    ISummarizeOptions,\n    ISummarizeHeuristicData,\n    ISummarizeHeuristicRunner,\n    IOnDemandSummarizeOptions,\n    EnqueueSummarizeResult,\n    SummarizerStopReason,\n    ISubmitSummaryOptions,\n    SubmitSummaryResult,\n    ISummaryCancellationToken,\n    ISummarizeResults,\n    ISummarizeTelemetryProperties,\n    ISummarizeRunnerTelemetry,\n} from \"./summarizerTypes\";\nimport { IClientSummaryWatcher, SummaryCollection } from \"./summaryCollection\";\nimport {\n    raceTimer,\n    SummarizeReason,\n    SummarizeResultBuilder,\n    SummaryGenerator,\n} from \"./summaryGenerator\";\n\nconst maxSummarizeAckWaitTime = 10 * 60 * 1000; // 10 minutes\n\n/**\n * An instance of RunningSummarizer manages the heuristics for summarizing.\n * Until disposed, the instance of RunningSummarizer can assume that it is\n * in a state of running, meaning it is connected and initialized.  It keeps\n * track of summaries that it is generating as they are broadcast and acked/nacked.\n * This object is created and controlled by Summarizer object.\n */\nexport class RunningSummarizer implements IDisposable {\n    public static async start(\n        logger: ITelemetryLogger,\n        summaryWatcher: IClientSummaryWatcher,\n        configuration: ISummaryConfiguration,\n        submitSummaryCallback: (options: ISubmitSummaryOptions) => Promise<SubmitSummaryResult>,\n        heuristicData: ISummarizeHeuristicData,\n        raiseSummarizingError: (errorMessage: string) => void,\n        summaryCollection: SummaryCollection,\n        cancellationToken: ISummaryCancellationToken,\n        stopSummarizerCallback: (reason: SummarizerStopReason) => void,\n    ): Promise<RunningSummarizer> {\n        const summarizer = new RunningSummarizer(\n            logger,\n            summaryWatcher,\n            configuration,\n            submitSummaryCallback,\n            heuristicData,\n            raiseSummarizingError,\n            summaryCollection,\n            cancellationToken,\n            stopSummarizerCallback);\n\n        await summarizer.waitStart();\n\n        // Run the heuristics after starting\n        summarizer.heuristicRunner?.run();\n        return summarizer;\n    }\n\n    public get disposed() { return this._disposed; }\n\n    private stopping = false;\n    private _disposed = false;\n    private summarizingLock: Promise<void> | undefined;\n    private tryWhileSummarizing = false;\n    private readonly pendingAckTimer: PromiseTimer;\n    private heuristicRunner?: ISummarizeHeuristicRunner;\n    private readonly generator: SummaryGenerator;\n    private readonly logger: ITelemetryLogger;\n    private enqueuedSummary: {\n        reason: SummarizeReason;\n        afterSequenceNumber: number;\n        options: ISummarizeOptions;\n        readonly resultsBuilder: SummarizeResultBuilder;\n    } | undefined;\n    private summarizeCount = 0;\n    private totalSuccessfulAttempts = 0;\n\n    private constructor(\n        baseLogger: ITelemetryLogger,\n        private readonly summaryWatcher: IClientSummaryWatcher,\n        private readonly configuration: ISummaryConfiguration,\n        private readonly submitSummaryCallback: (options: ISubmitSummaryOptions) => Promise<SubmitSummaryResult>,\n        private readonly heuristicData: ISummarizeHeuristicData,\n        private readonly raiseSummarizingError: (errorMessage: string) => void,\n        private readonly summaryCollection: SummaryCollection,\n        private readonly cancellationToken: ISummaryCancellationToken,\n        private readonly stopSummarizerCallback: (reason: SummarizerStopReason) => void,\n    ) {\n        const telemetryProps: ISummarizeRunnerTelemetry = {\n            summarizeCount: () => this.summarizeCount,\n            summarizerSuccessfulAttempts: () => this.totalSuccessfulAttempts,\n        };\n\n        this.logger = ChildLogger.create(\n            baseLogger, \"Running\",\n            {\n                all: telemetryProps,\n            },\n        );\n\n        if (configuration.state !== \"disableHeuristics\") {\n            assert(this.configuration.state === \"enabled\", 0x2ea /* \"Configuration state should be enabled\" */);\n            this.heuristicRunner = new SummarizeHeuristicRunner(\n                heuristicData,\n                this.configuration,\n                (reason) => this.trySummarize(reason),\n                this.logger);\n        }\n\n        assert(\n            this.configuration.state !== \"disabled\",\n            0x2eb /* \"Summary not supported with configuration disabled\" */,\n        );\n\n        // Cap the maximum amount of time client will wait for a summarize op ack to maxSummarizeAckWaitTime\n        // configuration.maxAckWaitTime is composed from defaults, server values, and runtime overrides\n\n        const maxAckWaitTime = Math.min(this.configuration.maxAckWaitTime, maxSummarizeAckWaitTime);\n\n        this.pendingAckTimer = new PromiseTimer(\n            maxAckWaitTime,\n            () => {\n                // pre-0.58 error message: summaryAckWaitTimeout\n                this.raiseSummarizingError(\"Pending summary ack not received in time\");\n                // Note: summarizeCount (from ChildLogger definition) may be 0,\n                // since this code path is hit when RunningSummarizer first starts up,\n                // before this instance has kicked off a new summarize run.\n                this.logger.sendErrorEvent({\n                    eventName: \"SummaryAckWaitTimeout\",\n                    maxAckWaitTime,\n                    referenceSequenceNumber: this.heuristicData.lastAttempt.refSequenceNumber,\n                    summarySequenceNumber: this.heuristicData.lastAttempt.summarySequenceNumber,\n                    timePending: Date.now() - this.heuristicData.lastAttempt.summaryTime,\n                });\n            });\n        // Set up pending ack timeout by op timestamp differences for previous summaries.\n        summaryCollection.setPendingAckTimerTimeoutCallback(maxAckWaitTime, () => {\n            if (this.pendingAckTimer.hasTimer) {\n                this.logger.sendTelemetryEvent({\n                    eventName: \"MissingSummaryAckFoundByOps\",\n                    referenceSequenceNumber: this.heuristicData.lastAttempt.refSequenceNumber,\n                    summarySequenceNumber: this.heuristicData.lastAttempt.summarySequenceNumber,\n                });\n                this.pendingAckTimer.clear();\n            }\n        });\n\n        this.generator = new SummaryGenerator(\n            this.pendingAckTimer,\n            this.heuristicData,\n            this.submitSummaryCallback,\n            this.raiseSummarizingError,\n            () => { this.totalSuccessfulAttempts++; },\n            this.summaryWatcher,\n            this.logger,\n        );\n    }\n\n    public dispose(): void {\n        this.summaryWatcher.dispose();\n        this.heuristicRunner?.dispose();\n        this.heuristicRunner = undefined;\n        this.generator.dispose();\n        this.pendingAckTimer.clear();\n        this.disposeEnqueuedSummary();\n        this._disposed = true;\n        this.stopping = true;\n    }\n\n    /**\n     * RunningSummarizer's logger includes the sequenced index of the current summary on each event.\n     * If some other Summarizer code wants that event on their logs they can get it here,\n     * but only if they're logging about that same summary.\n     * @param summaryOpRefSeq - RefSeq number of the summary op, to ensure the log correlation will be correct\n     */\n    public tryGetCorrelatedLogger = (summaryOpRefSeq) =>\n        this.heuristicData.lastAttempt.refSequenceNumber === summaryOpRefSeq\n            ? this.logger\n            : undefined;\n\n    public handleSystemOp(op: ISequencedDocumentMessage) {\n        switch (op.type) {\n            case MessageType.ClientLeave:\n            case MessageType.ClientJoin:\n            case MessageType.Propose: {\n                // Synchronously handle quorum ops like regular ops\n                this.handleOp(undefined, op);\n                return;\n            }\n            default: {\n                return;\n            }\n        }\n    }\n\n    public handleOp(error: any, { sequenceNumber, type, clientId, contents }: ISequencedDocumentMessage) {\n        if (error !== undefined) {\n            return;\n        }\n        this.heuristicData.lastOpSequenceNumber = sequenceNumber;\n\n        // Check for enqueued on-demand summaries; Intentionally do nothing otherwise\n        if (!this.tryRunEnqueuedSummary()) {\n            this.heuristicRunner?.run();\n        }\n    }\n\n    public async waitStop(allowLastSummary: boolean): Promise<void> {\n        if (this.stopping) {\n            return;\n        }\n\n        this.stopping = true;\n\n        this.disposeEnqueuedSummary();\n\n        // This will try to run lastSummary if needed.\n        if (allowLastSummary && this.heuristicRunner?.shouldRunLastSummary()) {\n            if (this.summarizingLock === undefined) {\n                this.trySummarizeOnce(\n                    // summarizeProps\n                    { reason: \"lastSummary\" },\n                    // ISummarizeOptions, using defaults: { refreshLatestAck: false, fullTree: false }\n                    {});\n            }\n        }\n\n        // Note that trySummarizeOnce() call above returns right away, without waiting.\n        // So we need to wait for its completion, otherwise it would be destroyed right away.\n        // That said, if summary lock was taken upfront, this wait might wait on  multiple retries to\n        // submit summary. We should reconsider this flow and make summarizer move to exit faster.\n        // This resolves when the current pending summary gets an ack or fails.\n        await this.summarizingLock;\n    }\n\n    private async waitStart() {\n        // Wait no longer than ack timeout for all pending\n        const waitStartResult = await raceTimer(\n            this.summaryWatcher.waitFlushed(),\n            this.pendingAckTimer.start(),\n        );\n        this.pendingAckTimer.clear();\n\n        // Remove pending ack wait timeout by op timestamp comparison, because\n        // it has race conditions with summaries submitted by this same client.\n        this.summaryCollection.unsetPendingAckTimerTimeoutCallback();\n\n        if (waitStartResult.result === \"done\" && waitStartResult.value !== undefined) {\n            this.heuristicData.updateWithLastSummaryAckInfo({\n                refSequenceNumber: waitStartResult.value.summaryOp.referenceSequenceNumber,\n                // This will be the Summarizer starting point so only use timestamps from client's machine.\n                summaryTime: Date.now(),\n                summarySequenceNumber: waitStartResult.value.summaryOp.sequenceNumber,\n            });\n        }\n    }\n\n    /**\n     * Runs single summary action that prevents any other concurrent actions.\n     * Assumes that caller checked upfront for lack of concurrent action (this.summarizingLock)\n     * before calling this API. I.e. caller is responsible for either erroring out or waiting on this promise.\n     * @param action - action to perform.\n     * @returns - result of action.\n     */\n    private async lockedSummaryAction<T>(action: () => Promise<T>) {\n        assert(this.summarizingLock === undefined, 0x25b /* \"Caller is responsible for checking lock\" */);\n\n        const summarizingLock = new Deferred<void>();\n        this.summarizingLock = summarizingLock.promise;\n\n        this.summarizeCount++;\n\n        return action().finally(() => {\n            summarizingLock.resolve();\n            this.summarizingLock = undefined;\n\n            const retry = this.tryWhileSummarizing;\n            this.tryWhileSummarizing = false;\n\n            // After summarizing, we should check to see if we need to summarize again.\n            // Rerun the heuristics and check for enqueued summaries.\n            if (!this.stopping && !this.tryRunEnqueuedSummary() && retry) {\n                this.heuristicRunner?.run();\n            }\n        });\n    }\n\n    /**\n     * Runs single summarize attempt\n     * @param summarizeProps - props to log with each telemetry event associated with this attempt\n     * @param options - summary options\n     * @param cancellationToken - cancellation token to use to be able to cancel this summary, if needed\n     * @param resultsBuilder - optional, result builder to use.\n     * @returns ISummarizeResult - result of running a summary.\n     */\n    private trySummarizeOnce(\n        summarizeProps: ISummarizeTelemetryProperties,\n        options: ISummarizeOptions,\n        cancellationToken = this.cancellationToken,\n        resultsBuilder = new SummarizeResultBuilder()): ISummarizeResults {\n        this.lockedSummaryAction(async () => {\n            const summarizeResult = this.generator.summarize(\n                summarizeProps,\n                options,\n                cancellationToken,\n                resultsBuilder);\n            // ensure we wait till the end of the process\n            return summarizeResult.receivedSummaryAckOrNack;\n        }).catch((error) => {\n            // SummaryGenerator.summarize() does not throw exceptions - it converts them to failed result\n            // on resultsBuilder\n            // We do not care about exceptions on receivedSummaryAckOrNack - caller should check results\n            // and take a appropriate action.\n        });\n\n        return resultsBuilder.build();\n    }\n\n    /** Heuristics summarize attempt. */\n    private trySummarize(\n        reason: SummarizeReason,\n        cancellationToken = this.cancellationToken): void {\n        if (this.summarizingLock !== undefined) {\n            // lockedSummaryAction() will retry heuristic-based summary at the end of current attempt\n            // if it's still needed\n            this.tryWhileSummarizing = true;\n            return;\n        }\n\n        this.lockedSummaryAction(async () => {\n            const attempts: (ISummarizeOptions & { delaySeconds?: number; })[] = [\n                { refreshLatestAck: false, fullTree: false },\n                { refreshLatestAck: true, fullTree: false },\n                { refreshLatestAck: true, fullTree: false, delaySeconds: 2 * 60 },\n                { refreshLatestAck: true, fullTree: true, delaySeconds: 10 * 60 },\n            ];\n            let overrideDelaySeconds: number | undefined;\n            let summaryAttempts = 0;\n            let summaryAttemptsPerPhase = 0;\n\n            let lastResult: { message: string; error: any; } | undefined;\n\n            for (let summaryAttemptPhase = 0; summaryAttemptPhase < attempts.length;) {\n                if (this.cancellationToken.cancelled) {\n                    return;\n                }\n\n                // We only want to attempt 1 summary when reason is \"lastSummary\"\n                if (++summaryAttempts > 1 && reason === \"lastSummary\") {\n                    return;\n                }\n\n                summaryAttemptsPerPhase++;\n\n                const { delaySeconds: regularDelaySeconds = 0, ...options } = attempts[summaryAttemptPhase];\n                const delaySeconds = overrideDelaySeconds ?? regularDelaySeconds;\n\n                const summarizeProps: ISummarizeTelemetryProperties = {\n                    reason,\n                    summaryAttempts,\n                    summaryAttemptsPerPhase,\n                    summaryAttemptPhase: summaryAttemptPhase + 1, // make everything 1-based\n                    ...options,\n                };\n\n                if (delaySeconds > 0) {\n                    this.logger.sendPerformanceEvent({\n                        eventName: \"SummarizeAttemptDelay\",\n                        duration: delaySeconds,\n                        summaryNackDelay: overrideDelaySeconds !== undefined,\n                        ...summarizeProps,\n                    });\n                    await delay(delaySeconds * 1000);\n                }\n                // Note: no need to account for cancellationToken.waitCancelled here, as\n                // this is accounted SummaryGenerator.summarizeCore that controls receivedSummaryAckOrNack.\n                const resultSummarize = this.generator.summarize(summarizeProps, options, cancellationToken);\n                const result = await resultSummarize.receivedSummaryAckOrNack;\n\n                if (result.success) {\n                    return;\n                }\n                // Check for retryDelay that can come from summaryNack or upload summary flow.\n                // Retry the same step only once per retryAfter response.\n                overrideDelaySeconds = result.retryAfterSeconds;\n                if (overrideDelaySeconds === undefined || summaryAttemptsPerPhase > 1) {\n                    summaryAttemptPhase++;\n                    summaryAttemptsPerPhase = 0;\n                }\n                lastResult = result;\n            }\n\n            // If all attempts failed, log error (with last attempt info) and close the summarizer container\n            this.logger.sendErrorEvent({\n                eventName: \"FailToSummarize\",\n                reason,\n                message: lastResult?.message,\n            }, lastResult?.error);\n\n            this.stopSummarizerCallback(\"failToSummarize\");\n        }).catch((error) => {\n            this.logger.sendErrorEvent({ eventName: \"UnexpectedSummarizeError\" }, error);\n        });\n    }\n\n    /** {@inheritdoc (ISummarizer:interface).summarizeOnDemand} */\n    public summarizeOnDemand(\n        resultsBuilder: SummarizeResultBuilder = new SummarizeResultBuilder(),\n        {\n            reason,\n            ...options\n        }: IOnDemandSummarizeOptions): ISummarizeResults {\n        if (this.stopping) {\n            resultsBuilder.fail(\"RunningSummarizer stopped or disposed\", undefined);\n            return resultsBuilder.build();\n        }\n        // Check for concurrent summary attempts. If one is found,\n        // return a promise that caller can await before trying again.\n        if (this.summarizingLock !== undefined) {\n            // The heuristics are blocking concurrent summarize attempts.\n            throw new UsageError(\"Attempted to run an already-running summarizer on demand\");\n        }\n        const result = this.trySummarizeOnce(\n            { reason: `onDemand/${reason}` },\n            options,\n            this.cancellationToken,\n            resultsBuilder);\n        return result;\n    }\n\n    /** {@inheritdoc (ISummarizer:interface).enqueueSummarize} */\n    public enqueueSummarize({\n        reason,\n        afterSequenceNumber = 0,\n        override = false,\n        ...options\n    }: IEnqueueSummarizeOptions): EnqueueSummarizeResult {\n        const onDemandReason = `enqueue;${reason}` as const;\n        let overridden = false;\n        if (this.enqueuedSummary !== undefined) {\n            if (!override) {\n                return { alreadyEnqueued: true };\n            }\n            // Override existing enqueued summarize attempt.\n            this.enqueuedSummary.resultsBuilder.fail(\n                \"Aborted; overridden by another enqueue summarize attempt\",\n                undefined,\n            );\n            this.enqueuedSummary = undefined;\n            overridden = true;\n        }\n        this.enqueuedSummary = {\n            reason: onDemandReason,\n            afterSequenceNumber,\n            options,\n            resultsBuilder: new SummarizeResultBuilder(),\n        };\n        const results = this.enqueuedSummary.resultsBuilder.build();\n        this.tryRunEnqueuedSummary();\n        return overridden ? {\n            ...results,\n            alreadyEnqueued: true,\n            overridden: true,\n        } : results;\n    }\n\n    private tryRunEnqueuedSummary() {\n        if (this.stopping) {\n            this.disposeEnqueuedSummary();\n            return false;\n        }\n        if (\n            this.enqueuedSummary === undefined\n            || this.heuristicData.lastOpSequenceNumber < this.enqueuedSummary.afterSequenceNumber\n            || this.summarizingLock !== undefined\n        ) {\n            // If no enqueued summary is ready or a summary is already in progress, take no action.\n            return false;\n        }\n        const { reason, resultsBuilder, options } = this.enqueuedSummary;\n        // Set to undefined first, so that subsequent enqueue attempt while summarize will occur later.\n        this.enqueuedSummary = undefined;\n        this.trySummarizeOnce(\n            { reason: `enqueuedSummary/${reason}` },\n            options,\n            this.cancellationToken,\n            resultsBuilder);\n        return true;\n    }\n\n    private disposeEnqueuedSummary() {\n        if (this.enqueuedSummary !== undefined) {\n            this.enqueuedSummary.resultsBuilder.fail(\"RunningSummarizer stopped or disposed\", undefined);\n            this.enqueuedSummary = undefined;\n        }\n    }\n}\n"]}
+{"version":3,"file":"runningSummarizer.js","sourceRoot":"","sources":["../src/runningSummarizer.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;;;;;;;;;AAGH,+DAAqF;AACrF,qEAA6D;AAC7D,+DAAgE;AAChE,+EAG8C;AAC9C,qEAA8D;AAI9D,iDAAwC;AACxC,iEAAkE;AAkBlE,yDAK4B;AAE5B,MAAM,uBAAuB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,aAAa;AAE7D;;;;;;GAMG;AACH,MAAa,iBAAiB;IA2E1B,YACI,UAA4B,EACX,cAAqC,EACrC,aAAoC,EACpC,qBAAuF,EACvF,aAAsC,EACtC,qBAAqD,EACrD,iBAAoC,EACpC,iBAA4C,EAC5C,sBAA8D,EAC9D,OAA2B;QAR3B,mBAAc,GAAd,cAAc,CAAuB;QACrC,kBAAa,GAAb,aAAa,CAAuB;QACpC,0BAAqB,GAArB,qBAAqB,CAAkE;QACvF,kBAAa,GAAb,aAAa,CAAyB;QACtC,0BAAqB,GAArB,qBAAqB,CAAgC;QACrD,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,sBAAiB,GAAjB,iBAAiB,CAA2B;QAC5C,2BAAsB,GAAtB,sBAAsB,CAAwC;QAC9D,YAAO,GAAP,OAAO,CAAoB;QA7BxC,aAAQ,GAAG,KAAK,CAAC;QACjB,cAAS,GAAG,KAAK,CAAC;QAGlB,wBAAmB,GAAG,KAAK,CAAC;QAW5B,mBAAc,GAAG,CAAC,CAAC;QACnB,4BAAuB,GAAG,CAAC,CAAC;QAC5B,gBAAW,GAAG,KAAK,CAAC;QAmG5B;;;;;WAKG;QACI,2BAAsB,GAAG,CAAC,eAAe,EAAE,EAAE,CAChD,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,iBAAiB,KAAK,eAAe;YAChE,CAAC,CAAC,IAAI,CAAC,MAAM;YACb,CAAC,CAAC,SAAS,CAAC;QAEpB,wGAAwG;QAChG,mCAA8B,GAAG,KAAK,CAAC;QAjG3C,MAAM,cAAc,GAA8B;YAC9C,cAAc,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,cAAc;YACzC,4BAA4B,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,uBAAuB;SACnE,CAAC;QAEF,IAAI,CAAC,MAAM,GAAG,6BAAW,CAAC,MAAM,CAC5B,UAAU,EAAE,SAAS,EACrB;YACI,GAAG,EAAE,cAAc;SACtB,CACJ,CAAC;QAEF,IAAI,aAAa,CAAC,KAAK,KAAK,mBAAmB,EAAE;YAC7C,IAAA,qBAAM,EAAC,IAAI,CAAC,aAAa,CAAC,KAAK,KAAK,SAAS,EAAE,KAAK,CAAC,6CAA6C,CAAC,CAAC;YACpG,IAAI,CAAC,eAAe,GAAG,IAAI,+CAAwB,CAC/C,aAAa,EACb,IAAI,CAAC,aAAa,EAClB,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,EACrC,IAAI,CAAC,MAAM,CAAC,CAAC;SACpB;QAED,IAAA,qBAAM,EACF,IAAI,CAAC,aAAa,CAAC,KAAK,KAAK,UAAU,EACvC,KAAK,CAAC,yDAAyD,CAClE,CAAC;QAEF,oGAAoG;QACpG,+FAA+F;QAE/F,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,cAAc,EAAE,uBAAuB,CAAC,CAAC;QAE5F,IAAI,CAAC,eAAe,GAAG,IAAI,2BAAY,CACnC,cAAc,EACd,GAAG,EAAE;YACD,gDAAgD;YAChD,IAAI,CAAC,qBAAqB,CAAC,0CAA0C,CAAC,CAAC;YACvE,+DAA+D;YAC/D,sEAAsE;YACtE,2DAA2D;YAC3D,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC;gBACvB,SAAS,EAAE,uBAAuB;gBAClC,cAAc;gBACd,uBAAuB,EAAE,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,iBAAiB;gBACzE,qBAAqB,EAAE,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,qBAAqB;gBAC3E,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,WAAW;aACvE,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QACP,iFAAiF;QACjF,iBAAiB,CAAC,iCAAiC,CAAC,cAAc,EAAE,GAAG,EAAE;YACrE,IAAI,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE;gBAC/B,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC;oBAC3B,SAAS,EAAE,6BAA6B;oBACxC,uBAAuB,EAAE,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,iBAAiB;oBACzE,qBAAqB,EAAE,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,qBAAqB;iBAC9E,CAAC,CAAC;gBACH,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;aAChC;QACL,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,SAAS,GAAG,IAAI,mCAAgB,CACjC,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,qBAAqB,EAC1B,IAAI,CAAC,qBAAqB,EAC1B,GAAG,EAAE,GAAG,IAAI,CAAC,uBAAuB,EAAE,CAAC,CAAC,CAAC,EACzC,IAAI,CAAC,cAAc,EACnB,IAAI,CAAC,MAAM,CACd,CAAC;QAEF,iBAAiB;QACjB,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACvE,CAAC;IA7JM,MAAM,CAAC,KAAK,CAAC,KAAK,CACrB,MAAwB,EACxB,cAAqC,EACrC,aAAoC,EACpC,qBAAuF,EACvF,aAAsC,EACtC,qBAAqD,EACrD,iBAAoC,EACpC,iBAA4C,EAC5C,sBAA8D,EAC9D,OAA2B;;QAE3B,MAAM,UAAU,GAAG,IAAI,iBAAiB,CACpC,MAAM,EACN,cAAc,EACd,aAAa,EACb,qBAAqB,EACrB,aAAa,EACb,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,EACjB,sBAAsB,EACtB,OAAO,CAAC,CAAC;QAEb,MAAM,UAAU,CAAC,SAAS,EAAE,CAAC;QAE7B,0BAA0B;QAC1B,+FAA+F;QAC/F,uCAAuC;QACvC,yGAAyG;QACzG,wFAAwF;QACxF,6GAA6G;QAC7G,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,kBAAkB,GAAG,CACnD,aAAa,CAAC,qBAAqB,CAAC,iBAAiB;cACnD,aAAa,CAAC,gBAAgB;cAC9B,aAAa,CAAC,aAAa,CAAC,CAAC;QACnC,aAAa,CAAC,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC;QAE1C,IAAI,aAAa,CAAC,gBAAgB,EAAE;YAChC,8DAA8D;YAC9D,aAAa,CAAC,gBAAgB,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;YACtD,aAAa,CAAC,aAAa,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;SACvD;QAED,+EAA+E;QAC/E,aAAa,CAAC,oBAAoB,GAAG,OAAO,CAAC,YAAY,CAAC,kBAAkB,CAAC;QAE7E,mBAAmB;QACnB,MAAA,UAAU,CAAC,eAAe,0CAAE,KAAK,EAAE,CAAC;QACpC,MAAA,UAAU,CAAC,eAAe,0CAAE,GAAG,EAAE,CAAC;QAElC,OAAO,UAAU,CAAC;IACtB,CAAC;IAED,IAAW,QAAQ,KAAK,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAyGzC,OAAO;;QACV,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpE,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC;QAC9B,MAAA,IAAI,CAAC,eAAe,0CAAE,OAAO,EAAE,CAAC;QAChC,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;QACjC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QACzB,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC7B,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC9B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACzB,CAAC;IAgBM,QAAQ,CAAC,EAA6B;QACzC,IAAI,CAAC,aAAa,CAAC,oBAAoB,GAAG,EAAE,CAAC,cAAc,CAAC;QAE5D,IAAI,EAAE,CAAC,IAAI,KAAK,kCAAW,CAAC,SAAS,IAAI,IAAA,+BAAgB,EAAC,EAAE,CAAC,EAAE;YAC3D,IAAI,CAAC,aAAa,CAAC,aAAa,EAAE,CAAC;SACtC;aAAM;YACH,IAAI,CAAC,aAAa,CAAC,gBAAgB,EAAE,CAAC;SACzC;QAED,IAAI,CAAC,aAAa,CAAC,YAAY,IAAI,IAAA,qBAAM,EAAC,EAAE,CAAC,CAAC;QAE9C,6EAA6E;QAC7E,IAAI,IAAI,CAAC,WAAW;eACb,IAAI,CAAC,mBAAmB,CAAC,EAAE,CAAC;eAC5B,CAAC,IAAI,CAAC,qBAAqB,EAAE;eAC7B,CAAC,IAAI,CAAC,8BAA8B,EAAE;YACzC,IAAI,CAAC,8BAA8B,GAAG,IAAI,CAAC;YAC3C,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE;;gBACxB,MAAA,IAAI,CAAC,eAAe,0CAAE,GAAG,EAAE,CAAC;YAChC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE;gBACZ,IAAI,CAAC,8BAA8B,GAAG,KAAK,CAAC;YAChD,CAAC,CAAC,CAAC;SACN;IACL,CAAC;IAED;;;;;OAKG;IACK,mBAAmB,CAAC,EAA6B;QACrD,QAAQ,EAAE,CAAC,IAAI,EAAE;YACb,KAAK,kCAAW,CAAC,SAAS,CAAC;YAC3B,KAAK,kCAAW,CAAC,UAAU,CAAC;YAC5B,KAAK,kCAAW,CAAC,WAAW;gBACxB,OAAO,KAAK,CAAC;YACjB;gBACI,OAAO,IAAI,CAAC;SACnB;IACL,CAAC;IAEM,KAAK,CAAC,QAAQ,CAAC,gBAAyB;;QAC3C,IAAI,IAAI,CAAC,QAAQ,EAAE;YACf,OAAO;SACV;QAED,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QAErB,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAE9B,8CAA8C;QAC9C,IAAI,gBAAgB,KAAI,MAAA,IAAI,CAAC,eAAe,0CAAE,oBAAoB,EAAE,CAAA,EAAE;YAClE,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE;gBACpC,IAAI,CAAC,gBAAgB;gBACjB,iBAAiB;gBACjB,EAAE,MAAM,EAAE,aAAa,EAAE;gBACzB,kFAAkF;gBAClF,EAAE,CAAC,CAAC;aACX;SACJ;QAED,+EAA+E;QAC/E,qFAAqF;QACrF,6FAA6F;QAC7F,0FAA0F;QAC1F,uEAAuE;QACvE,MAAM,IAAI,CAAC,eAAe,CAAC;IAC/B,CAAC;IAEO,KAAK,CAAC,SAAS;QACnB,kDAAkD;QAClD,MAAM,eAAe,GAAG,MAAM,IAAA,4BAAS,EACnC,IAAI,CAAC,cAAc,CAAC,WAAW,EAAE,EACjC,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAC/B,CAAC;QACF,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAE7B,sEAAsE;QACtE,uEAAuE;QACvE,IAAI,CAAC,iBAAiB,CAAC,mCAAmC,EAAE,CAAC;QAE7D,IAAI,eAAe,CAAC,MAAM,KAAK,MAAM,IAAI,eAAe,CAAC,KAAK,KAAK,SAAS,EAAE;YAC1E,IAAI,CAAC,aAAa,CAAC,4BAA4B,CAAC;gBAC5C,iBAAiB,EAAE,eAAe,CAAC,KAAK,CAAC,SAAS,CAAC,uBAAuB;gBAC1E,2FAA2F;gBAC3F,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;gBACvB,qBAAqB,EAAE,eAAe,CAAC,KAAK,CAAC,SAAS,CAAC,cAAc;aACxE,CAAC,CAAC;SACN;QACD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC5B,CAAC;IAED;;;;;;;;;;;OAWG;IACI,KAAK,CAAC,6BAA6B,CAAI,MAAwB;QAClE,IAAA,qBAAM,EAAC,IAAI,CAAC,qBAAqB,KAAK,SAAS,EAC3C,KAAK,CAAC,mEAAmE,CAAC,CAAC;QAE/E,MAAM,qBAAqB,GAAG,IAAI,uBAAQ,EAAQ,CAAC;QACnD,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC,OAAO,CAAC;QAE3D,OAAO,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;YACzB,qBAAqB,CAAC,OAAO,EAAE,CAAC;YAChC,IAAI,CAAC,qBAAqB,GAAG,SAAS,CAAC;QAC3C,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,mBAAmB,CAAI,MAAwB;QACzD,IAAA,qBAAM,EAAC,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE,KAAK,CAAC,+CAA+C,CAAC,CAAC;QAElG,MAAM,eAAe,GAAG,IAAI,uBAAQ,EAAQ,CAAC;QAC7C,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC,OAAO,CAAC;QAE/C,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,+DAA+D;QAC/D,MAAM,IAAI,CAAC,qBAAqB,CAAC;QAEjC,OAAO,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;;YACzB,eAAe,CAAC,OAAO,EAAE,CAAC;YAC1B,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;YAEjC,MAAM,KAAK,GAAG,IAAI,CAAC,mBAAmB,CAAC;YACvC,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;YAEjC,2EAA2E;YAC3E,yDAAyD;YACzD,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,IAAI,KAAK,EAAE;gBAC1D,MAAA,IAAI,CAAC,eAAe,0CAAE,GAAG,EAAE,CAAC;aAC/B;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;OAOG;IACK,gBAAgB,CACpB,cAA6C,EAC7C,OAA0B,EAC1B,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,EAC1C,cAAc,GAAG,IAAI,yCAAsB,EAAE;QAC7C,IAAI,CAAC,mBAAmB,CAAC,KAAK,IAAI,EAAE;YAChC,MAAM,eAAe,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAC5C,cAAc,EACd,OAAO,EACP,iBAAiB,EACjB,cAAc,CAAC,CAAC;YACpB,6CAA6C;YAC7C,OAAO,eAAe,CAAC,wBAAwB,CAAC;QACpD,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACf,6FAA6F;YAC7F,oBAAoB;YACpB,4FAA4F;YAC5F,iCAAiC;QACrC,CAAC,CAAC,CAAC;QAEH,OAAO,cAAc,CAAC,KAAK,EAAE,CAAC;IAClC,CAAC;IAED,oCAAoC;IAC5B,YAAY,CAChB,MAAuB,EACvB,iBAAiB,GAAG,IAAI,CAAC,iBAAiB;QAC1C,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE;YACpC,yFAAyF;YACzF,uBAAuB;YACvB,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;YAChC,OAAO;SACV;QAED,IAAI,CAAC,mBAAmB,CAAC,KAAK,IAAI,EAAE;YAChC,MAAM,QAAQ,GAAuD;gBACjE,EAAE,gBAAgB,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE;gBAC5C,EAAE,gBAAgB,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE;gBAC3C,EAAE,gBAAgB,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,GAAG,EAAE,EAAE;gBACjE,EAAE,gBAAgB,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE,GAAG,EAAE,EAAE;aACpE,CAAC;YACF,IAAI,oBAAwC,CAAC;YAC7C,IAAI,eAAe,GAAG,CAAC,CAAC;YACxB,IAAI,uBAAuB,GAAG,CAAC,CAAC;YAEhC,IAAI,UAAwD,CAAC;YAE7D,KAAK,IAAI,mBAAmB,GAAG,CAAC,EAAE,mBAAmB,GAAG,QAAQ,CAAC,MAAM,GAAG;gBACtE,IAAI,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE;oBAClC,OAAO;iBACV;gBAED,iEAAiE;gBACjE,IAAI,EAAE,eAAe,GAAG,CAAC,IAAI,MAAM,KAAK,aAAa,EAAE;oBACnD,OAAO;iBACV;gBAED,uBAAuB,EAAE,CAAC;gBAE1B,MAAM,KAAwD,QAAQ,CAAC,mBAAmB,CAAC,EAArF,EAAE,YAAY,EAAE,mBAAmB,GAAG,CAAC,OAA8C,EAAzC,OAAO,cAAnD,gBAAqD,CAAgC,CAAC;gBAC5F,MAAM,YAAY,GAAG,oBAAoB,aAApB,oBAAoB,cAApB,oBAAoB,GAAI,mBAAmB,CAAC;gBAEjE,MAAM,cAAc,mBAChB,MAAM;oBACN,eAAe;oBACf,uBAAuB,EACvB,mBAAmB,EAAE,mBAAmB,GAAG,CAAC,IACzC,OAAO,CACb,CAAC;gBAEF,IAAI,YAAY,GAAG,CAAC,EAAE;oBAClB,IAAI,CAAC,MAAM,CAAC,oBAAoB,iBAC5B,SAAS,EAAE,uBAAuB,EAClC,QAAQ,EAAE,YAAY,EACtB,gBAAgB,EAAE,oBAAoB,KAAK,SAAS,IACjD,cAAc,EACnB,CAAC;oBACH,MAAM,IAAA,oBAAK,EAAC,YAAY,GAAG,IAAI,CAAC,CAAC;iBACpC;gBAED,2DAA2D;gBAC3D,MAAM,IAAI,CAAC,qBAAqB,CAAC;gBAEjC,wEAAwE;gBACxE,2FAA2F;gBAC3F,MAAM,eAAe,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,cAAc,EAAE,OAAO,EAAE,iBAAiB,CAAC,CAAC;gBAC7F,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,wBAAwB,CAAC;gBAE9D,IAAI,MAAM,CAAC,OAAO,EAAE;oBAChB,OAAO;iBACV;gBACD,8EAA8E;gBAC9E,yDAAyD;gBACzD,oBAAoB,GAAG,MAAM,CAAC,iBAAiB,CAAC;gBAChD,IAAI,oBAAoB,KAAK,SAAS,IAAI,uBAAuB,GAAG,CAAC,EAAE;oBACnE,mBAAmB,EAAE,CAAC;oBACtB,uBAAuB,GAAG,CAAC,CAAC;iBAC/B;gBACD,UAAU,GAAG,MAAM,CAAC;aACvB;YAED,gGAAgG;YAChG,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC;gBACvB,SAAS,EAAE,iBAAiB;gBAC5B,MAAM;gBACN,OAAO,EAAE,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,OAAO;aAC/B,EAAE,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,KAAK,CAAC,CAAC;YAEtB,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACf,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,0BAA0B,EAAE,EAAE,KAAK,CAAC,CAAC;QACjF,CAAC,CAAC,CAAC;IACP,CAAC;IAED,8DAA8D;IACvD,iBAAiB,CACpB,iBAAyC,IAAI,yCAAsB,EAAE,EACrE,EAG4B;YAH5B,EACI,MAAM,OAEkB,EADrB,OAAO,cAFd,UAGC,CADa;QAEd,IAAI,IAAI,CAAC,QAAQ,EAAE;YACf,cAAc,CAAC,IAAI,CAAC,uCAAuC,EAAE,SAAS,CAAC,CAAC;YACxE,OAAO,cAAc,CAAC,KAAK,EAAE,CAAC;SACjC;QACD,0DAA0D;QAC1D,8DAA8D;QAC9D,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE;YACpC,6DAA6D;YAC7D,MAAM,IAAI,4BAAU,CAAC,0DAA0D,CAAC,CAAC;SACpF;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAChC,EAAE,MAAM,EAAE,YAAY,MAAM,EAAE,EAAE,EAChC,OAAO,EACP,IAAI,CAAC,iBAAiB,EACtB,cAAc,CAAC,CAAC;QACpB,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,6DAA6D;IACtD,gBAAgB,CAAC,EAKG;YALH,EACpB,MAAM,EACN,mBAAmB,GAAG,CAAC,EACvB,QAAQ,GAAG,KAAK,OAEO,EADpB,OAAO,cAJU,6CAKvB,CADa;QAEV,MAAM,cAAc,GAAG,WAAW,MAAM,EAAW,CAAC;QACpD,IAAI,UAAU,GAAG,KAAK,CAAC;QACvB,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE;YACpC,IAAI,CAAC,QAAQ,EAAE;gBACX,OAAO,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;aACpC;YACD,gDAAgD;YAChD,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,IAAI,CACpC,0DAA0D,EAC1D,SAAS,CACZ,CAAC;YACF,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;YACjC,UAAU,GAAG,IAAI,CAAC;SACrB;QACD,IAAI,CAAC,eAAe,GAAG;YACnB,MAAM,EAAE,cAAc;YACtB,mBAAmB;YACnB,OAAO;YACP,cAAc,EAAE,IAAI,yCAAsB,EAAE;SAC/C,CAAC;QACF,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC5D,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAC7B,OAAO,UAAU,CAAC,CAAC,iCACZ,OAAO,KACV,eAAe,EAAE,IAAI,EACrB,UAAU,EAAE,IAAI,IAClB,CAAC,CAAC,OAAO,CAAC;IAChB,CAAC;IAEO,qBAAqB;QACzB,IAAI,IAAI,CAAC,QAAQ,EAAE;YACf,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAC9B,OAAO,KAAK,CAAC;SAChB;QACD,IACI,IAAI,CAAC,eAAe,KAAK,SAAS;eAC/B,IAAI,CAAC,aAAa,CAAC,oBAAoB,GAAG,IAAI,CAAC,eAAe,CAAC,mBAAmB;eAClF,IAAI,CAAC,eAAe,KAAK,SAAS,EACvC;YACE,uFAAuF;YACvF,OAAO,KAAK,CAAC;SAChB;QACD,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC;QACjE,+FAA+F;QAC/F,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;QACjC,IAAI,CAAC,gBAAgB,CACjB,EAAE,MAAM,EAAE,mBAAmB,MAAM,EAAE,EAAE,EACvC,OAAO,EACP,IAAI,CAAC,iBAAiB,EACtB,cAAc,CAAC,CAAC;QACpB,OAAO,IAAI,CAAC;IAChB,CAAC;IAEO,sBAAsB;QAC1B,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE;YACpC,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,IAAI,CAAC,uCAAuC,EAAE,SAAS,CAAC,CAAC;YAC7F,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;SACpC;IACL,CAAC;CACJ;AAtiBD,8CAsiBC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { IDisposable, ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { assert, delay, Deferred, PromiseTimer } from \"@fluidframework/common-utils\";\nimport { UsageError } from \"@fluidframework/container-utils\";\nimport { isRuntimeMessage } from \"@fluidframework/driver-utils\";\nimport {\n    ISequencedDocumentMessage,\n    MessageType,\n} from \"@fluidframework/protocol-definitions\";\nimport { ChildLogger } from \"@fluidframework/telemetry-utils\";\nimport {\n    ISummaryConfiguration,\n} from \"./containerRuntime\";\nimport { opSize } from \"./opProperties\";\nimport { SummarizeHeuristicRunner } from \"./summarizerHeuristics\";\nimport {\n    IEnqueueSummarizeOptions,\n    ISummarizeOptions,\n    ISummarizeHeuristicData,\n    ISummarizeHeuristicRunner,\n    IOnDemandSummarizeOptions,\n    EnqueueSummarizeResult,\n    SummarizerStopReason,\n    ISubmitSummaryOptions,\n    SubmitSummaryResult,\n    ISummaryCancellationToken,\n    ISummarizeResults,\n    ISummarizeTelemetryProperties,\n    ISummarizerRuntime,\n    ISummarizeRunnerTelemetry,\n} from \"./summarizerTypes\";\nimport { IClientSummaryWatcher, SummaryCollection } from \"./summaryCollection\";\nimport {\n    raceTimer,\n    SummarizeReason,\n    SummarizeResultBuilder,\n    SummaryGenerator,\n} from \"./summaryGenerator\";\n\nconst maxSummarizeAckWaitTime = 10 * 60 * 1000; // 10 minutes\n\n/**\n * An instance of RunningSummarizer manages the heuristics for summarizing.\n * Until disposed, the instance of RunningSummarizer can assume that it is\n * in a state of running, meaning it is connected and initialized.  It keeps\n * track of summaries that it is generating as they are broadcast and acked/nacked.\n * This object is created and controlled by Summarizer object.\n */\nexport class RunningSummarizer implements IDisposable {\n    public static async start(\n        logger: ITelemetryLogger,\n        summaryWatcher: IClientSummaryWatcher,\n        configuration: ISummaryConfiguration,\n        submitSummaryCallback: (options: ISubmitSummaryOptions) => Promise<SubmitSummaryResult>,\n        heuristicData: ISummarizeHeuristicData,\n        raiseSummarizingError: (errorMessage: string) => void,\n        summaryCollection: SummaryCollection,\n        cancellationToken: ISummaryCancellationToken,\n        stopSummarizerCallback: (reason: SummarizerStopReason) => void,\n        runtime: ISummarizerRuntime,\n    ): Promise<RunningSummarizer> {\n        const summarizer = new RunningSummarizer(\n            logger,\n            summaryWatcher,\n            configuration,\n            submitSummaryCallback,\n            heuristicData,\n            raiseSummarizingError,\n            summaryCollection,\n            cancellationToken,\n            stopSummarizerCallback,\n            runtime);\n\n        await summarizer.waitStart();\n\n        // Update heuristic counts\n        // By the time we get here, there are potentially ops missing from the heuristic summary counts\n        // Examples of where this could happen:\n        // 1. Op is processed during the time that we are initiating the RunningSummarizer instance but before we\n        //    listen for the op events (will get missed by the handlers in the current workflow)\n        // 2. Op was sequenced after the last time we summarized (op sequence number > summarize ref sequence number)\n        const diff = runtime.deltaManager.lastSequenceNumber - (\n            heuristicData.lastSuccessfulSummary.refSequenceNumber\n            + heuristicData.numNonRuntimeOps\n            + heuristicData.numRuntimeOps);\n        heuristicData.hasMissingOpData = diff > 0;\n\n        if (heuristicData.hasMissingOpData) {\n            // Split the diff 50-50 and increment the counts appropriately\n            heuristicData.numNonRuntimeOps += Math.ceil(diff / 2);\n            heuristicData.numRuntimeOps += Math.floor(diff / 2);\n        }\n\n        // Update last seq number (in case the handlers haven't processed anything yet)\n        heuristicData.lastOpSequenceNumber = runtime.deltaManager.lastSequenceNumber;\n\n        // Start heuristics\n        summarizer.heuristicRunner?.start();\n        summarizer.heuristicRunner?.run();\n\n        return summarizer;\n    }\n\n    public get disposed() { return this._disposed; }\n    private stopping = false;\n    private _disposed = false;\n    private summarizingLock: Promise<void> | undefined;\n    private refreshSummaryAckLock: Promise<void> | undefined;\n    private tryWhileSummarizing = false;\n    private readonly pendingAckTimer: PromiseTimer;\n    private heuristicRunner?: ISummarizeHeuristicRunner;\n    private readonly generator: SummaryGenerator;\n    private readonly logger: ITelemetryLogger;\n    private enqueuedSummary: {\n        reason: SummarizeReason;\n        afterSequenceNumber: number;\n        options: ISummarizeOptions;\n        readonly resultsBuilder: SummarizeResultBuilder;\n    } | undefined;\n    private summarizeCount = 0;\n    private totalSuccessfulAttempts = 0;\n    private initialized = false;\n\n    private constructor(\n        baseLogger: ITelemetryLogger,\n        private readonly summaryWatcher: IClientSummaryWatcher,\n        private readonly configuration: ISummaryConfiguration,\n        private readonly submitSummaryCallback: (options: ISubmitSummaryOptions) => Promise<SubmitSummaryResult>,\n        private readonly heuristicData: ISummarizeHeuristicData,\n        private readonly raiseSummarizingError: (errorMessage: string) => void,\n        private readonly summaryCollection: SummaryCollection,\n        private readonly cancellationToken: ISummaryCancellationToken,\n        private readonly stopSummarizerCallback: (reason: SummarizerStopReason) => void,\n        private readonly runtime: ISummarizerRuntime,\n    ) {\n        const telemetryProps: ISummarizeRunnerTelemetry = {\n            summarizeCount: () => this.summarizeCount,\n            summarizerSuccessfulAttempts: () => this.totalSuccessfulAttempts,\n        };\n\n        this.logger = ChildLogger.create(\n            baseLogger, \"Running\",\n            {\n                all: telemetryProps,\n            },\n        );\n\n        if (configuration.state !== \"disableHeuristics\") {\n            assert(this.configuration.state === \"enabled\", 0x2ea /* \"Configuration state should be enabled\" */);\n            this.heuristicRunner = new SummarizeHeuristicRunner(\n                heuristicData,\n                this.configuration,\n                (reason) => this.trySummarize(reason),\n                this.logger);\n        }\n\n        assert(\n            this.configuration.state !== \"disabled\",\n            0x2eb /* \"Summary not supported with configuration disabled\" */,\n        );\n\n        // Cap the maximum amount of time client will wait for a summarize op ack to maxSummarizeAckWaitTime\n        // configuration.maxAckWaitTime is composed from defaults, server values, and runtime overrides\n\n        const maxAckWaitTime = Math.min(this.configuration.maxAckWaitTime, maxSummarizeAckWaitTime);\n\n        this.pendingAckTimer = new PromiseTimer(\n            maxAckWaitTime,\n            () => {\n                // pre-0.58 error message: summaryAckWaitTimeout\n                this.raiseSummarizingError(\"Pending summary ack not received in time\");\n                // Note: summarizeCount (from ChildLogger definition) may be 0,\n                // since this code path is hit when RunningSummarizer first starts up,\n                // before this instance has kicked off a new summarize run.\n                this.logger.sendErrorEvent({\n                    eventName: \"SummaryAckWaitTimeout\",\n                    maxAckWaitTime,\n                    referenceSequenceNumber: this.heuristicData.lastAttempt.refSequenceNumber,\n                    summarySequenceNumber: this.heuristicData.lastAttempt.summarySequenceNumber,\n                    timePending: Date.now() - this.heuristicData.lastAttempt.summaryTime,\n                });\n            });\n        // Set up pending ack timeout by op timestamp differences for previous summaries.\n        summaryCollection.setPendingAckTimerTimeoutCallback(maxAckWaitTime, () => {\n            if (this.pendingAckTimer.hasTimer) {\n                this.logger.sendTelemetryEvent({\n                    eventName: \"MissingSummaryAckFoundByOps\",\n                    referenceSequenceNumber: this.heuristicData.lastAttempt.refSequenceNumber,\n                    summarySequenceNumber: this.heuristicData.lastAttempt.summarySequenceNumber,\n                });\n                this.pendingAckTimer.clear();\n            }\n        });\n\n        this.generator = new SummaryGenerator(\n            this.pendingAckTimer,\n            this.heuristicData,\n            this.submitSummaryCallback,\n            this.raiseSummarizingError,\n            () => { this.totalSuccessfulAttempts++; },\n            this.summaryWatcher,\n            this.logger,\n        );\n\n        // Listen for ops\n        this.runtime.deltaManager.on(\"op\", (op) => { this.handleOp(op); });\n    }\n\n    public dispose(): void {\n        this.runtime.deltaManager.off(\"op\", (op) => { this.handleOp(op); });\n        this.summaryWatcher.dispose();\n        this.heuristicRunner?.dispose();\n        this.heuristicRunner = undefined;\n        this.generator.dispose();\n        this.pendingAckTimer.clear();\n        this.disposeEnqueuedSummary();\n        this._disposed = true;\n        this.stopping = true;\n    }\n\n    /**\n     * RunningSummarizer's logger includes the sequenced index of the current summary on each event.\n     * If some other Summarizer code wants that event on their logs they can get it here,\n     * but only if they're logging about that same summary.\n     * @param summaryOpRefSeq - RefSeq number of the summary op, to ensure the log correlation will be correct\n     */\n    public tryGetCorrelatedLogger = (summaryOpRefSeq) =>\n        this.heuristicData.lastAttempt.refSequenceNumber === summaryOpRefSeq\n            ? this.logger\n            : undefined;\n\n    /** We only want a single heuristic runner micro-task (will provide better optimized grouping of ops) */\n    private heuristicRunnerMicroTaskExists = false;\n\n    public handleOp(op: ISequencedDocumentMessage) {\n        this.heuristicData.lastOpSequenceNumber = op.sequenceNumber;\n\n        if (op.type !== MessageType.Summarize && isRuntimeMessage(op)) {\n            this.heuristicData.numRuntimeOps++;\n        } else {\n            this.heuristicData.numNonRuntimeOps++;\n        }\n\n        this.heuristicData.totalOpsSize += opSize(op);\n\n        // Check for enqueued on-demand summaries; Intentionally do nothing otherwise\n        if (this.initialized\n            && this.opCanTriggerSummary(op)\n            && !this.tryRunEnqueuedSummary()\n            && !this.heuristicRunnerMicroTaskExists) {\n            this.heuristicRunnerMicroTaskExists = true;\n            Promise.resolve().then(() => {\n                this.heuristicRunner?.run();\n            }).finally(() => {\n                this.heuristicRunnerMicroTaskExists = false;\n            });\n        }\n    }\n\n    /**\n     * Can the given op trigger a summary?\n     * # Currently only prevents summaries for Summarize and SummaryAck ops\n     * @param op - op to check\n     * @returns true if this type of op can trigger a summary\n     */\n    private opCanTriggerSummary(op: ISequencedDocumentMessage): boolean {\n        switch (op.type) {\n            case MessageType.Summarize:\n            case MessageType.SummaryAck:\n            case MessageType.SummaryNack:\n                return false;\n            default:\n                return true;\n        }\n    }\n\n    public async waitStop(allowLastSummary: boolean): Promise<void> {\n        if (this.stopping) {\n            return;\n        }\n\n        this.stopping = true;\n\n        this.disposeEnqueuedSummary();\n\n        // This will try to run lastSummary if needed.\n        if (allowLastSummary && this.heuristicRunner?.shouldRunLastSummary()) {\n            if (this.summarizingLock === undefined) {\n                this.trySummarizeOnce(\n                    // summarizeProps\n                    { reason: \"lastSummary\" },\n                    // ISummarizeOptions, using defaults: { refreshLatestAck: false, fullTree: false }\n                    {});\n            }\n        }\n\n        // Note that trySummarizeOnce() call above returns right away, without waiting.\n        // So we need to wait for its completion, otherwise it would be destroyed right away.\n        // That said, if summary lock was taken upfront, this wait might wait on  multiple retries to\n        // submit summary. We should reconsider this flow and make summarizer move to exit faster.\n        // This resolves when the current pending summary gets an ack or fails.\n        await this.summarizingLock;\n    }\n\n    private async waitStart() {\n        // Wait no longer than ack timeout for all pending\n        const waitStartResult = await raceTimer(\n            this.summaryWatcher.waitFlushed(),\n            this.pendingAckTimer.start(),\n        );\n        this.pendingAckTimer.clear();\n\n        // Remove pending ack wait timeout by op timestamp comparison, because\n        // it has race conditions with summaries submitted by this same client.\n        this.summaryCollection.unsetPendingAckTimerTimeoutCallback();\n\n        if (waitStartResult.result === \"done\" && waitStartResult.value !== undefined) {\n            this.heuristicData.updateWithLastSummaryAckInfo({\n                refSequenceNumber: waitStartResult.value.summaryOp.referenceSequenceNumber,\n                // This will be the Summarizer starting point so only use timestamps from client's machine.\n                summaryTime: Date.now(),\n                summarySequenceNumber: waitStartResult.value.summaryOp.sequenceNumber,\n            });\n        }\n        this.initialized = true;\n    }\n\n    /**\n     * Blocks a new summarizer from running in case RefreshSummaryAck is being processed.\n     * Assumes that caller checked upfront for lack of concurrent action (this.refreshSummaryAckLock)\n     * before calling this API. I.e. caller is responsible for either erroring out or waiting on this promise.\n     * Note: The refreshSummaryAckLock makes sure no summarizer gets enqueued or processed\n     * until the refresh has completed. One can't rely uniquely on the summarizingLock as the\n     * refreshLatestSummaryAck also happens during the time summarizingLock !== undefined.\n     * Ex. Summarizer submits a summay + op and then waits for the Summary Ack to proceed\n     * with the refreshLatestSummaryAck and complete the summary.\n     * @param action - action to perform.\n     * @returns - result of action.\n     */\n    public async lockedRefreshSummaryAckAction<T>(action: () => Promise<T>) {\n        assert(this.refreshSummaryAckLock === undefined,\n            0x396 /* Refresh Summary Ack - Caller is responsible for checking lock */);\n\n        const refreshSummaryAckLock = new Deferred<void>();\n        this.refreshSummaryAckLock = refreshSummaryAckLock.promise;\n\n        return action().finally(() => {\n            refreshSummaryAckLock.resolve();\n            this.refreshSummaryAckLock = undefined;\n        });\n    }\n\n    /**\n     * Runs single summary action that prevents any other concurrent actions.\n     * Assumes that caller checked upfront for lack of concurrent action (this.summarizingLock)\n     * before calling this API. I.e. caller is responsible for either erroring out or waiting on this promise.\n     * @param action - action to perform.\n     * @returns - result of action.\n     */\n    private async lockedSummaryAction<T>(action: () => Promise<T>) {\n        assert(this.summarizingLock === undefined, 0x25b /* \"Caller is responsible for checking lock\" */);\n\n        const summarizingLock = new Deferred<void>();\n        this.summarizingLock = summarizingLock.promise;\n\n        this.summarizeCount++;\n        // Make sure the RefreshLatestSummaryAck is not being executed.\n        await this.refreshSummaryAckLock;\n\n        return action().finally(() => {\n            summarizingLock.resolve();\n            this.summarizingLock = undefined;\n\n            const retry = this.tryWhileSummarizing;\n            this.tryWhileSummarizing = false;\n\n            // After summarizing, we should check to see if we need to summarize again.\n            // Rerun the heuristics and check for enqueued summaries.\n            if (!this.stopping && !this.tryRunEnqueuedSummary() && retry) {\n                this.heuristicRunner?.run();\n            }\n        });\n    }\n\n    /**\n     * Runs single summarize attempt\n     * @param summarizeProps - props to log with each telemetry event associated with this attempt\n     * @param options - summary options\n     * @param cancellationToken - cancellation token to use to be able to cancel this summary, if needed\n     * @param resultsBuilder - optional, result builder to use.\n     * @returns ISummarizeResult - result of running a summary.\n     */\n    private trySummarizeOnce(\n        summarizeProps: ISummarizeTelemetryProperties,\n        options: ISummarizeOptions,\n        cancellationToken = this.cancellationToken,\n        resultsBuilder = new SummarizeResultBuilder()): ISummarizeResults {\n        this.lockedSummaryAction(async () => {\n            const summarizeResult = this.generator.summarize(\n                summarizeProps,\n                options,\n                cancellationToken,\n                resultsBuilder);\n            // ensure we wait till the end of the process\n            return summarizeResult.receivedSummaryAckOrNack;\n        }).catch((error) => {\n            // SummaryGenerator.summarize() does not throw exceptions - it converts them to failed result\n            // on resultsBuilder\n            // We do not care about exceptions on receivedSummaryAckOrNack - caller should check results\n            // and take a appropriate action.\n        });\n\n        return resultsBuilder.build();\n    }\n\n    /** Heuristics summarize attempt. */\n    private trySummarize(\n        reason: SummarizeReason,\n        cancellationToken = this.cancellationToken): void {\n        if (this.summarizingLock !== undefined) {\n            // lockedSummaryAction() will retry heuristic-based summary at the end of current attempt\n            // if it's still needed\n            this.tryWhileSummarizing = true;\n            return;\n        }\n\n        this.lockedSummaryAction(async () => {\n            const attempts: (ISummarizeOptions & { delaySeconds?: number; })[] = [\n                { refreshLatestAck: false, fullTree: false },\n                { refreshLatestAck: true, fullTree: false },\n                { refreshLatestAck: true, fullTree: false, delaySeconds: 2 * 60 },\n                { refreshLatestAck: true, fullTree: true, delaySeconds: 10 * 60 },\n            ];\n            let overrideDelaySeconds: number | undefined;\n            let summaryAttempts = 0;\n            let summaryAttemptsPerPhase = 0;\n\n            let lastResult: { message: string; error: any; } | undefined;\n\n            for (let summaryAttemptPhase = 0; summaryAttemptPhase < attempts.length;) {\n                if (this.cancellationToken.cancelled) {\n                    return;\n                }\n\n                // We only want to attempt 1 summary when reason is \"lastSummary\"\n                if (++summaryAttempts > 1 && reason === \"lastSummary\") {\n                    return;\n                }\n\n                summaryAttemptsPerPhase++;\n\n                const { delaySeconds: regularDelaySeconds = 0, ...options } = attempts[summaryAttemptPhase];\n                const delaySeconds = overrideDelaySeconds ?? regularDelaySeconds;\n\n                const summarizeProps: ISummarizeTelemetryProperties = {\n                    reason,\n                    summaryAttempts,\n                    summaryAttemptsPerPhase,\n                    summaryAttemptPhase: summaryAttemptPhase + 1, // make everything 1-based\n                    ...options,\n                };\n\n                if (delaySeconds > 0) {\n                    this.logger.sendPerformanceEvent({\n                        eventName: \"SummarizeAttemptDelay\",\n                        duration: delaySeconds,\n                        summaryNackDelay: overrideDelaySeconds !== undefined,\n                        ...summarizeProps,\n                    });\n                    await delay(delaySeconds * 1000);\n                }\n\n                // Make sure the refresh Summary Ack is not being executed.\n                await this.refreshSummaryAckLock;\n\n                // Note: no need to account for cancellationToken.waitCancelled here, as\n                // this is accounted SummaryGenerator.summarizeCore that controls receivedSummaryAckOrNack.\n                const resultSummarize = this.generator.summarize(summarizeProps, options, cancellationToken);\n                const result = await resultSummarize.receivedSummaryAckOrNack;\n\n                if (result.success) {\n                    return;\n                }\n                // Check for retryDelay that can come from summaryNack or upload summary flow.\n                // Retry the same step only once per retryAfter response.\n                overrideDelaySeconds = result.retryAfterSeconds;\n                if (overrideDelaySeconds === undefined || summaryAttemptsPerPhase > 1) {\n                    summaryAttemptPhase++;\n                    summaryAttemptsPerPhase = 0;\n                }\n                lastResult = result;\n            }\n\n            // If all attempts failed, log error (with last attempt info) and close the summarizer container\n            this.logger.sendErrorEvent({\n                eventName: \"FailToSummarize\",\n                reason,\n                message: lastResult?.message,\n            }, lastResult?.error);\n\n            this.stopSummarizerCallback(\"failToSummarize\");\n        }).catch((error) => {\n            this.logger.sendErrorEvent({ eventName: \"UnexpectedSummarizeError\" }, error);\n        });\n    }\n\n    /** {@inheritdoc (ISummarizer:interface).summarizeOnDemand} */\n    public summarizeOnDemand(\n        resultsBuilder: SummarizeResultBuilder = new SummarizeResultBuilder(),\n        {\n            reason,\n            ...options\n        }: IOnDemandSummarizeOptions): ISummarizeResults {\n        if (this.stopping) {\n            resultsBuilder.fail(\"RunningSummarizer stopped or disposed\", undefined);\n            return resultsBuilder.build();\n        }\n        // Check for concurrent summary attempts. If one is found,\n        // return a promise that caller can await before trying again.\n        if (this.summarizingLock !== undefined) {\n            // The heuristics are blocking concurrent summarize attempts.\n            throw new UsageError(\"Attempted to run an already-running summarizer on demand\");\n        }\n\n        const result = this.trySummarizeOnce(\n            { reason: `onDemand/${reason}` },\n            options,\n            this.cancellationToken,\n            resultsBuilder);\n        return result;\n    }\n\n    /** {@inheritdoc (ISummarizer:interface).enqueueSummarize} */\n    public enqueueSummarize({\n        reason,\n        afterSequenceNumber = 0,\n        override = false,\n        ...options\n    }: IEnqueueSummarizeOptions): EnqueueSummarizeResult {\n        const onDemandReason = `enqueue;${reason}` as const;\n        let overridden = false;\n        if (this.enqueuedSummary !== undefined) {\n            if (!override) {\n                return { alreadyEnqueued: true };\n            }\n            // Override existing enqueued summarize attempt.\n            this.enqueuedSummary.resultsBuilder.fail(\n                \"Aborted; overridden by another enqueue summarize attempt\",\n                undefined,\n            );\n            this.enqueuedSummary = undefined;\n            overridden = true;\n        }\n        this.enqueuedSummary = {\n            reason: onDemandReason,\n            afterSequenceNumber,\n            options,\n            resultsBuilder: new SummarizeResultBuilder(),\n        };\n        const results = this.enqueuedSummary.resultsBuilder.build();\n        this.tryRunEnqueuedSummary();\n        return overridden ? {\n            ...results,\n            alreadyEnqueued: true,\n            overridden: true,\n        } : results;\n    }\n\n    private tryRunEnqueuedSummary() {\n        if (this.stopping) {\n            this.disposeEnqueuedSummary();\n            return false;\n        }\n        if (\n            this.enqueuedSummary === undefined\n            || this.heuristicData.lastOpSequenceNumber < this.enqueuedSummary.afterSequenceNumber\n            || this.summarizingLock !== undefined\n        ) {\n            // If no enqueued summary is ready or a summary is already in progress, take no action.\n            return false;\n        }\n        const { reason, resultsBuilder, options } = this.enqueuedSummary;\n        // Set to undefined first, so that subsequent enqueue attempt while summarize will occur later.\n        this.enqueuedSummary = undefined;\n        this.trySummarizeOnce(\n            { reason: `enqueuedSummary/${reason}` },\n            options,\n            this.cancellationToken,\n            resultsBuilder);\n        return true;\n    }\n\n    private disposeEnqueuedSummary() {\n        if (this.enqueuedSummary !== undefined) {\n            this.enqueuedSummary.resultsBuilder.fail(\"RunningSummarizer stopped or disposed\", undefined);\n            this.enqueuedSummary = undefined;\n        }\n    }\n}\n"]}

package/dist/summarizer.d.ts

@@ -42,8 +42,6 @@ export declare class Summarizer extends EventEmitter implements ISummarizer {
     get ISummarizer(): this;
     private readonly logger;
     private runningSummarizer?;
-    private systemOpListener?;
-    private opListener?;
     private _disposed;
     private starting;
     private readonly innerHandle;

package/dist/summarizer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"summarizer.d.ts","sourceRoot":"","sources":["../src/summarizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAEtC,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AACtE,OAAO,EAAE,OAAO,EAAgB,MAAM,uCAAuC,CAAC;AAI9E,OAAO,EAAe,eAAe,EAAE,YAAY,EAAmB,MAAM,iCAAiC,CAAC;AAC9G,OAAO,EAEH,mBAAmB,EACnB,YAAY,EAEf,MAAM,iCAAiC,CAAC;AAIzC,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,gCAAgC,EAAE,MAAM,gCAAgC,CAAC;AAElF,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAGxD,OAAO,EACH,WAAW,EACX,4BAA4B,EAC5B,kBAAkB,EAClB,mBAAmB,EACnB,oBAAoB,EACvB,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,mBAAmB,EAAE,MAAM,GAAG,CAAC;AAIxC,qBAAa,kBAAmB,SAAQ,YAAa,YAAW,mBAAmB,EAAE,eAAe;IAM5F,QAAQ,CAAC,MAAM,EAAE,OAAO;IAL5B,QAAQ,CAAC,SAAS,sBAAoB;IACtC,QAAQ,CAAC,QAAQ,QAAQ;gBAGrB,YAAY,EAAE,MAAM,EACX,MAAM,GAAE,OAAe;IAKpC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,qBAAiB,EAAE,MAAM,EAAE,gBAAgB;CAI5E;AAED,eAAO,MAAM,wBAAwB,iBAClB,MAAM,UAAU,OAAO,uBAAiD,CAAC;AAE5F;;;;GAIG;AACH,qBAAa,UAAW,SAAQ,YAAa,YAAW,WAAW;IAkB3D;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,mBAAmB;IACpC;;MAEE;IACF,OAAO,CAAC,QAAQ,CAAC,iBAAiB;aAElB,iBAAiB,EAAE,iBAAiB;IACpD,OAAO,CAAC,QAAQ,CAAC,sBAAsB;IA5B3C,IAAW,cAAc,SAAmB;IAC5C,IAAW,WAAW,SAAmB;IAEzC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmB;IAC1C,OAAO,CAAC,iBAAiB,CAAC,CAAoB;IAC9C,OAAO,CAAC,gBAAgB,CAAC,CAA0C;IACnE,OAAO,CAAC,UAAU,CAAC,CAAsD;IACzE,OAAO,CAAC,SAAS,CAAkB;IACnC,OAAO,CAAC,QAAQ,CAAkB;IAElC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqB;IAEjD,IAAW,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,CAA6B;IACpE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAwC;gBAGjE,GAAG,EAAE,MAAM;IACX;;OAEG;IACc,OAAO,EAAE,kBAAkB,EAC3B,mBAAmB,EAAE,MAAM,qBAAqB;IACjE;;MAEE;IACe,iBAAiB,EAAE,4BAA4B,EAChE,aAAa,EAAE,mBAAmB,EAClB,iBAAiB,EAAE,iBAAiB,EACnC,sBAAsB,EACnC,CAAC,OAAO,EAAE,mBAAmB,KAAK,OAAO,CAAC,gCAAgC,CAAC;IAOnF;;;;;;;;OAQG;WACiB,MAAM,CACtB,MAAM,EAAE,OAAO,EACf,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAuBzB,GAAG,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAWnE;;;;OAIG;IACI,IAAI,CAAC,MAAM,EAAE,oBAAoB;IAIjC,KAAK;YAOE,OAAO;IAgDrB;;;;;OAKG;WACW,2BAA2B,CAAC,UAAU,EAAE,oBAAoB,GAAG,OAAO;IAIpF;;;;;;;;OAQG;YACW,KAAK;IAoEnB;;;;;OAKG;IACI,OAAO;IAiBd,SAAgB,iBAAiB,EAAE,WAAW,CAAC,mBAAmB,CAAC,CA+CjE;IAEF,SAAgB,gBAAgB,EAAE,WAAW,CAAC,kBAAkB,CAAC,CAK/D;YAEY,iBAAiB;CAuBlC"}
+{"version":3,"file":"summarizer.d.ts","sourceRoot":"","sources":["../src/summarizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAEtC,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AACtE,OAAO,EAAE,OAAO,EAAgB,MAAM,uCAAuC,CAAC;AAI9E,OAAO,EAEH,eAAe,EAEf,YAAY,EAEf,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAEH,mBAAmB,EACnB,YAAY,EAEf,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,gCAAgC,EAAE,MAAM,gCAAgC,CAAC;AAElF,OAAO,EAAiB,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAGvE,OAAO,EACH,WAAW,EACX,4BAA4B,EAC5B,kBAAkB,EAClB,mBAAmB,EACnB,oBAAoB,EACvB,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,mBAAmB,EAAE,MAAM,GAAG,CAAC;AAIxC,qBAAa,kBAAmB,SAAQ,YAAa,YAAW,mBAAmB,EAAE,eAAe;IAM5F,QAAQ,CAAC,MAAM,EAAE,OAAO;IAL5B,QAAQ,CAAC,SAAS,sBAAoB;IACtC,QAAQ,CAAC,QAAQ,QAAQ;gBAGrB,YAAY,EAAE,MAAM,EACX,MAAM,GAAE,OAAe;IAKpC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,qBAAiB,EAAE,MAAM,EAAE,gBAAgB;CAI5E;AAED,eAAO,MAAM,wBAAwB,iBAClB,MAAM,UAAU,OAAO,uBAAiD,CAAC;AAE5F;;;;GAIG;AACH,qBAAa,UAAW,SAAQ,YAAa,YAAW,WAAW;IAgB3D;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,mBAAmB;IACpC;;MAEE;IACF,OAAO,CAAC,QAAQ,CAAC,iBAAiB;aAElB,iBAAiB,EAAE,iBAAiB;IACpD,OAAO,CAAC,QAAQ,CAAC,sBAAsB;IA1B3C,IAAW,cAAc,SAAmB;IAC5C,IAAW,WAAW,SAAmB;IAEzC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmB;IAC1C,OAAO,CAAC,iBAAiB,CAAC,CAAoB;IAC9C,OAAO,CAAC,SAAS,CAAkB;IACnC,OAAO,CAAC,QAAQ,CAAkB;IAElC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqB;IAEjD,IAAW,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,CAA6B;IACpE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAwC;gBAGjE,GAAG,EAAE,MAAM;IACX;;OAEG;IACc,OAAO,EAAE,kBAAkB,EAC3B,mBAAmB,EAAE,MAAM,qBAAqB;IACjE;;MAEE;IACe,iBAAiB,EAAE,4BAA4B,EAChE,aAAa,EAAE,mBAAmB,EAClB,iBAAiB,EAAE,iBAAiB,EACnC,sBAAsB,EACnC,CAAC,OAAO,EAAE,mBAAmB,KAAK,OAAO,CAAC,gCAAgC,CAAC;IAOnF;;;;;;;;OAQG;WACiB,MAAM,CACtB,MAAM,EAAE,OAAO,EACf,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAuBzB,GAAG,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAWnE;;;;OAIG;IACI,IAAI,CAAC,MAAM,EAAE,oBAAoB;IAIjC,KAAK;YAOE,OAAO;IAgDrB;;;;;OAKG;WACW,2BAA2B,CAAC,UAAU,EAAE,oBAAoB,GAAG,OAAO;IAIpF;;;;;;;;OAQG;YACW,KAAK;IA8DnB;;;;;OAKG;IACI,OAAO;IAWd,SAAgB,iBAAiB,EAAE,WAAW,CAAC,mBAAmB,CAAC,CA+CjE;IAEF,SAAgB,gBAAgB,EAAE,WAAW,CAAC,kBAAkB,CAAC,CAK/D;YAEY,iBAAiB;CAqDlC"}

package/dist/summarizer.js

@@ -258,7 +258,7 @@ class Summarizer extends events_1.EventEmitter {
             if (!this._disposed) {
                 this.logger.sendErrorEvent({ eventName: "summarizingError" }, (0, exports.createSummarizingWarning)(errorMessage, true));
             }
-        }, this.summaryCollection, runCoordinator /* cancellationToken */, (reason) => runCoordinator.stop(reason));
+        }, this.summaryCollection, runCoordinator /* cancellationToken */, (reason) => runCoordinator.stop(reason), /* stopSummarizerCallback */ this.runtime);
         this.runningSummarizer = runningSummarizer;
         this.starting = false;
         // Handle summary acks
@@ -266,11 +266,6 @@ class Summarizer extends events_1.EventEmitter {
         this.handleSummaryAcks().catch((error) => {
             this.logger.sendErrorEvent({ eventName: "HandleSummaryAckFatalError" }, error);
         });
-        // Listen for ops
-        this.systemOpListener = (op) => runningSummarizer.handleSystemOp(op);
-        this.runtime.deltaManager.inbound.on("op", this.systemOpListener);
-        this.opListener = (error, op) => runningSummarizer.handleOp(error, op);
-        this.runtime.on("batchEnd", this.opListener);
         return runningSummarizer;
     }
     /**
@@ -287,27 +282,51 @@ class Summarizer extends events_1.EventEmitter {
             this.runningSummarizer.dispose();
             this.runningSummarizer = undefined;
         }
-        if (this.systemOpListener) {
-            this.runtime.deltaManager.inbound.off("op", this.systemOpListener);
-        }
-        if (this.opListener) {
-            this.runtime.removeListener("batchEnd", this.opListener);
-        }
     }
     async handleSummaryAcks() {
-        var _a;
+        var _a, _b, _c, _d, _e;
         let refSequenceNumber = this.runtime.deltaManager.initialSequenceNumber;
+        let ack;
         while (this.runningSummarizer) {
             const summaryLogger = (_a = this.runningSummarizer.tryGetCorrelatedLogger(refSequenceNumber)) !== null && _a !== void 0 ? _a : this.logger;
             try {
-                const ack = await this.summaryCollection.waitSummaryAck(refSequenceNumber);
+                // Initialize ack with undefined if exception happens inside of waitSummaryAck on second iteration,
+                // we record undefined, not previous handles.
+                ack = undefined;
+                ack = await this.summaryCollection.waitSummaryAck(refSequenceNumber);
                 refSequenceNumber = ack.summaryOp.referenceSequenceNumber;
-                await this.internalsProvider.refreshLatestSummaryAck(ack.summaryOp.contents.handle, ack.summaryAck.contents.handle, refSequenceNumber, summaryLogger);
+                const summaryOpHandle = ack.summaryOp.contents.handle;
+                const summaryAckHandle = ack.summaryAck.contents.handle;
+                // Make sure we block any summarizer from being executed/enqueued while
+                // executing the refreshLatestSummaryAck.
+                // https://dev.azure.com/fluidframework/internal/_workitems/edit/779
+                await this.runningSummarizer.lockedRefreshSummaryAckAction(async () => this.internalsProvider.refreshLatestSummaryAck(summaryOpHandle, summaryAckHandle, refSequenceNumber, summaryLogger).catch(async (error) => {
+                    // If the error is 404, so maybe the fetched version no longer exists on server. We just
+                    // ignore this error in that case, as that means we will have another summaryAck for the
+                    // latest version with which we will refresh the state. However in case of single commit
+                    // summary, we might me missing a summary ack, so in that case we are still fine as the
+                    // code in `submitSummary` function in container runtime, will refresh the latest state
+                    // by calling `refreshLatestSummaryAckFromServer` and we will be fine.
+                    if ((0, telemetry_utils_1.isFluidError)(error)
+                        && error.errorType === driver_definitions_1.DriverErrorType.fileNotFoundOrAccessDeniedError) {
+                        summaryLogger.sendTelemetryEvent({
+                            eventName: "HandleSummaryAckErrorIgnored",
+                            referenceSequenceNumber: refSequenceNumber,
+                            proposalHandle: summaryOpHandle,
+                            ackHandle: summaryAckHandle,
+                        }, error);
+                    }
+                    else {
+                        throw error;
+                    }
+                }));
             }
             catch (error) {
                 summaryLogger.sendErrorEvent({
                     eventName: "HandleSummaryAckError",
                     referenceSequenceNumber: refSequenceNumber,
+                    handle: (_c = (_b = ack === null || ack === void 0 ? void 0 : ack.summaryOp) === null || _b === void 0 ? void 0 : _b.contents) === null || _c === void 0 ? void 0 : _c.handle,
+                    ackHandle: (_e = (_d = ack === null || ack === void 0 ? void 0 : ack.summaryAck) === null || _d === void 0 ? void 0 : _d.contents) === null || _e === void 0 ? void 0 : _e.handle,
                 }, error);
             }
             refSequenceNumber++;