@fluidframework/container-runtime 0.59.2000-61729 → 0.59.2001

package/src/garbageCollection.ts

@@ -3,7 +3,7 @@
  * Licensed under the MIT License.
  */
 
-import { ITelemetryLogger } from "@fluidframework/common-definitions";
+import { ITelemetryLogger, ITelemetryPerformanceEvent } from "@fluidframework/common-definitions";
 import { assert, LazyPromise, Timer } from "@fluidframework/common-utils";
 import { ICriticalContainerError } from "@fluidframework/container-definitions";
 import { ClientSessionExpiredError, DataProcessingError } from "@fluidframework/container-utils";
@@ -66,6 +66,8 @@ const runSweepKey = "Fluid.GarbageCollection.RunSweep";
 const writeAtRootKey = "Fluid.GarbageCollection.WriteDataAtRoot";
 // Feature gate key to expire a session after a set period of time.
 const runSessionExpiry = "Fluid.GarbageCollection.RunSessionExpiry";
+// Feature gate key to log error messages if GC reference validation fails.
+const logUnknownOutboundReferencesKey = "Fluid.GarbageCollection.LogUnknownOutboundReferences";
 
 const defaultDeleteTimeoutMs = 7 * 24 * 60 * 60 * 1000; // 7 days
 export const defaultSessionExpiryDurationMs = 30 * 24 * 60 * 60 * 1000; // 30 days
@@ -227,6 +229,19 @@ class UnreferencedStateTracker {
 /**
  * The garbage collector for the container runtime. It consolidates the garbage collection functionality and maintains
  * its state across summaries.
+ *
+ * Node - represented as nodeId, it's a node on the GC graph
+ * Outbound Route - a path from one node to another node, think `nodeA` -> `nodeB`
+ * Graph - all nodes with their respective routes
+ *             GC Graph
+ *
+ *               Node
+ *        NodeId = "datastore1"
+ *           /             \
+ *    OutboundRoute   OutboundRoute
+ *         /                 \
+ *       Node               Node
+ *  NodeId = "dds1"     NodeId = "dds2"
  */
 export class GarbageCollector implements IGarbageCollector {
     public static create(
@@ -320,10 +335,10 @@ export class GarbageCollector implements IGarbageCollector {
     private latestSummaryGCVersion: GCVersion;
 
     // Keeps track of the GC state from the last run.
-    private gcDataFromLastRun: IGarbageCollectionData | undefined;
+    private previousGCDataFromLastRun: IGarbageCollectionData | undefined;
     // Keeps a list of references (edges in the GC graph) between GC runs. Each entry has a node id and a list of
     // outbound routes from that node.
-    private readonly referencesSinceLastRun: Map<string, string[]> = new Map();
+    private readonly newReferencesSinceLastRun: Map<string, string[]> = new Map();
 
     // Promise when resolved initializes the base state of the nodes from the base summary state.
     private readonly initializeBaseStateP: Promise<void>;
@@ -513,7 +528,7 @@ export class GarbageCollector implements IGarbageCollector {
                 }
                 gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
             }
-            this.gcDataFromLastRun = { gcNodes };
+            this.previousGCDataFromLastRun = { gcNodes };
         });
 
         // Get the GC details for each node from the GC state in the base summary. This is returned in getBaseGCDetails
@@ -637,12 +652,12 @@ export class GarbageCollector implements IGarbageCollector {
      * blobs. All the blob keys should start with `gcBlobPrefix`.
      */
     public summarize(): ISummaryTreeWithStats | undefined {
-        if (!this.shouldRunGC || this.gcDataFromLastRun === undefined) {
+        if (!this.shouldRunGC || this.previousGCDataFromLastRun === undefined) {
             return;
         }
 
         const gcState: IGarbageCollectionState = { gcNodes: {} };
-        for (const [nodeId, outboundRoutes] of Object.entries(this.gcDataFromLastRun.gcNodes)) {
+        for (const [nodeId, outboundRoutes] of Object.entries(this.previousGCDataFromLastRun.gcNodes)) {
             gcState.gcNodes[nodeId] = {
                 outboundRoutes,
                 unreferencedTimestampMs: this.unreferencedNodesState.get(nodeId)?.unreferencedTimestampMs,
@@ -729,9 +744,9 @@ export class GarbageCollector implements IGarbageCollector {
             return;
         }
 
-        const outboundRoutes = this.referencesSinceLastRun.get(fromNodePath) ?? [];
+        const outboundRoutes = this.newReferencesSinceLastRun.get(fromNodePath) ?? [];
         outboundRoutes.push(toNodePath);
-        this.referencesSinceLastRun.set(fromNodePath, outboundRoutes);
+        this.newReferencesSinceLastRun.set(fromNodePath, outboundRoutes);
 
         // If the node that got referenced is inactive, log an event as that may indicate use-after-delete.
         this.logIfInactive(
@@ -772,8 +787,8 @@ export class GarbageCollector implements IGarbageCollector {
         gcResult: IGCResult,
         currentReferenceTimestampMs?: number,
     ) {
-        this.gcDataFromLastRun = cloneGCData(gcData);
-        this.referencesSinceLastRun.clear();
+        this.previousGCDataFromLastRun = cloneGCData(gcData);
+        this.newReferencesSinceLastRun.clear();
 
         // Iterate through the referenced nodes and stop tracking if they were unreferenced before.
         for (const nodeId of gcResult.referencedNodeIds) {
@@ -826,13 +841,37 @@ export class GarbageCollector implements IGarbageCollector {
      * If these nodes are currently unreferenced, they will be assigned new unreferenced state by the current run.
      */
     private updateStateSinceLastRun(currentGCData: IGarbageCollectionData) {
-        // If we haven't run GC before or no references were added since the last run, there is nothing to do.
-        if (this.gcDataFromLastRun === undefined || this.referencesSinceLastRun.size === 0) {
+        // If we haven't run GC before there is nothing to do.
+        if (this.previousGCDataFromLastRun === undefined) {
             return;
         }
 
-        // Validate that we have identified all references correctly.
-        this.validateReferenceCorrectness(currentGCData);
+        // Find any references that haven't been identified correctly.
+        const missingExplicitReferences = this.findMissingExplicitReferences(
+            currentGCData,
+            this.previousGCDataFromLastRun,
+            this.newReferencesSinceLastRun,
+        );
+
+        // The following log will be enabled once this issue is resolved:
+        // https://github.com/microsoft/FluidFramework/issues/8878.
+        if(this.mc.config.getBoolean(logUnknownOutboundReferencesKey) === true
+            && missingExplicitReferences.length > 0) {
+            missingExplicitReferences.forEach((missingExplicitReference) => {
+                const event: ITelemetryPerformanceEvent = {
+                    eventName: "gcUnknownOutboundReferences",
+                    gcNodeId: missingExplicitReference[0],
+                    gcRoutes: JSON.stringify(missingExplicitReference[1]),
+                };
+                this.mc.logger.sendPerformanceEvent(event);
+            });
+        }
+
+        // No references were added since the last run so we don't have to update reference states of any unreferenced
+        // nodes
+        if (this.newReferencesSinceLastRun.size === 0) {
+            return;
+        }
 
         /**
          * Generate a super set of the GC data that contains the nodes and edges from last run, plus any new node and
@@ -849,8 +888,8 @@ export class GarbageCollector implements IGarbageCollector {
          *      which is tracked by https://github.com/microsoft/FluidFramework/issues/8470.
          *    - A new data store may have "root" DDSs already created and we don't detect them today.
          */
-        const gcDataSuperSet = concatGarbageCollectionData(this.gcDataFromLastRun, currentGCData);
-        this.referencesSinceLastRun.forEach((outboundRoutes: string[], sourceNodeId: string) => {
+        const gcDataSuperSet = concatGarbageCollectionData(this.previousGCDataFromLastRun, currentGCData);
+        this.newReferencesSinceLastRun.forEach((outboundRoutes: string[], sourceNodeId: string) => {
             if (gcDataSuperSet.gcNodes[sourceNodeId] === undefined) {
                 gcDataSuperSet.gcNodes[sourceNodeId] = outboundRoutes;
             } else {
@@ -876,59 +915,55 @@ export class GarbageCollector implements IGarbageCollector {
     }
 
     /**
-     * Validates that all new references are correctly identified and processed. The basic principle for validation is
-     * that we should not have new references in the reference graph (GC data) that have not been notified to the
-     * garbage collector via `referenceAdded`.
-     * We validate that the references in the current reference graph should be a subset of the references in the last
-     * run's reference graph + references since the last run.
+     * Finds all new references or outbound routes in the current graph that haven't been explicitly notified to GC.
+     * The principle is that every new reference or outbound route must be notified to GC via the
+     * addedOutboundReference method. It it hasn't, its a bug and we want to identify these scenarios.
+     *
+     * In more simple terms:
+     * Missing Explicit References = Current References - Previous References - Explicitly Added References;
+     *
      * @param currentGCData - The GC data (reference graph) from the current GC run.
+     * @param previousGCData - The GC data (reference graph) from the previous GC run.
+     * @param explicitReferences - New references added explicity between the previous and the current run.
+     * @returns - a list of missing explicit references
      */
-    private validateReferenceCorrectness(currentGCData: IGarbageCollectionData) {
+    private findMissingExplicitReferences(
+        currentGCData: IGarbageCollectionData,
+        previousGCData: IGarbageCollectionData,
+        explicitReferences: Map<string, string[]>,
+    ): [string, string[]][] {
         assert(
-            this.gcDataFromLastRun !== undefined,
+            previousGCData !== undefined,
             0x2b7, /* "Can't validate correctness without GC data from last run" */
         );
 
-        // Get a list of all the outbound routes (or references) in the current GC data.
-        const currentReferences: string[] = [];
-        for (const [nodeId, outboundRoutes] of Object.entries(currentGCData.gcNodes)) {
-            /**
-             * Remove routes from a child node to its parent which is added implicitly by the runtime. For instance,
-             * each adds its data store as an outbound route to mark it as referenced if the DDS is referenced.
-             * We won't get any explicit notification for these references so they must be removed before validation.
-             */
-            const explicitRoutes = outboundRoutes.filter((route) => !nodeId.startsWith(route));
-            currentReferences.push(...explicitRoutes);
-        }
-
-        // Get a list of outbound routes (or references) from the last run's GC data plus references added since the
-        // last run that were notified via `referenceAdded`.
-        const explicitReferences: string[] = [];
-        for (const [, outboundRoutes] of Object.entries(this.gcDataFromLastRun.gcNodes)) {
-            explicitReferences.push(...outboundRoutes);
-        }
-        this.referencesSinceLastRun.forEach((outboundRoutes: string[]) => {
-            explicitReferences.push(...outboundRoutes);
-        });
-
-        // Validate that the current reference graph doesn't have references that we are not already aware of. If this
-        // happens, it might indicate data corruption since we may delete objects prematurely.
-        currentReferences.forEach((route: string) => {
-            // Validate references for data stores only. Currently, layers below data stores don't have GC implemented
-            // so there is no guarantee their references will be notified.
-            if (this.runtime.getNodeType(route) === GCNodeType.DataStore && !explicitReferences.includes(route)) {
-                /**
-                 * The following log will be enabled once this issue is resolved:
-                 * https://github.com/microsoft/FluidFramework/issues/8878.
-                 */
-                // We should ideally throw a data corruption error here. However, send an error for now until we have
-                // implemented sweep and have reasonable confidence in the sweep process.
-                // this.mc.logger.sendErrorEvent({
-                //     eventName: "gcUnknownOutboundRoute",
-                //     route,
-                // });
+        const currentGraph = Object.entries(currentGCData.gcNodes);
+        const missingExplicitReferences: [string, string[]][] = [];
+        currentGraph.forEach(([nodeId, currentOutboundRoutes]) => {
+            const previousRoutes = previousGCData.gcNodes[nodeId] ?? [];
+            const explicitRoutes = explicitReferences.get(nodeId) ?? [];
+            const missingExplicitRoutes: string[] = [];
+            currentOutboundRoutes.forEach((route) => {
+                const isBlobOrDataStoreRoute =
+                    this.runtime.getNodeType(route) === GCNodeType.Blob ||
+                    this.runtime.getNodeType(route) === GCNodeType.DataStore;
+                // Ignore implicitly added DDS routes to their parent datastores
+                const notRouteFromDDSToParentDataStore = !nodeId.startsWith(route);
+                if (
+                    isBlobOrDataStoreRoute &&
+                    notRouteFromDDSToParentDataStore &&
+                    (!previousRoutes.includes(route) && !explicitRoutes.includes(route))
+                ) {
+                    missingExplicitRoutes.push(route);
+                }
+            });
+            if (missingExplicitRoutes.length > 0) {
+                missingExplicitReferences.push([nodeId, missingExplicitRoutes]);
             }
         });
+
+        // Ideally missingExplicitReferences should always have a size 0
+        return missingExplicitReferences;
     }
 
     /**

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "0.59.2000-61729";
+export const pkgVersion = "0.59.2001";

package/src/runningSummarizer.ts

@@ -27,6 +27,7 @@ import {
     ISummaryCancellationToken,
     ISummarizeResults,
     ISummarizeTelemetryProperties,
+    ISummarizeRunnerTelemetry,
 } from "./summarizerTypes";
 import { IClientSummaryWatcher, SummaryCollection } from "./summaryCollection";
 import {
@@ -108,13 +109,15 @@ export class RunningSummarizer implements IDisposable {
         private readonly stopSummarizerCallback: (reason: SummarizerStopReason) => void,
         { disableHeuristics = false }: Readonly<Partial<ISummarizerOptions>> = {},
     ) {
+        const telemetryProps: ISummarizeRunnerTelemetry = {
+            summarizeCount: () => this.summarizeCount,
+            summarizerSuccessfulAttempts: () => this.totalSuccessfulAttempts,
+        };
+
         this.logger = ChildLogger.create(
             baseLogger, "Running",
             {
-                all: {
-                    summarizeCount: () => this.summarizeCount,
-                    summarizerSuccessfulAttempts: () => this.totalSuccessfulAttempts,
-                },
+                all: telemetryProps,
             },
         );
 
@@ -231,7 +234,7 @@ export class RunningSummarizer implements IDisposable {
             if (this.summarizingLock === undefined) {
                 this.trySummarizeOnce(
                     // summarizeProps
-                    { summarizeReason: "lastSummary" },
+                    { reason: "lastSummary" },
                     // ISummarizeOptions, using defaults: { refreshLatestAck: false, fullTree: false }
                     {});
             }
@@ -330,7 +333,7 @@ export class RunningSummarizer implements IDisposable {
 
     /** Heuristics summarize attempt. */
     private trySummarize(
-        summarizeReason: SummarizeReason,
+        reason: SummarizeReason,
         cancellationToken = this.cancellationToken): void
     {
         if (this.summarizingLock !== undefined) {
@@ -365,7 +368,7 @@ export class RunningSummarizer implements IDisposable {
                 const delaySeconds = overrideDelaySeconds ?? regularDelaySeconds;
 
                 const summarizeProps: ISummarizeTelemetryProperties = {
-                    summarizeReason,
+                    reason,
                     summaryAttempts,
                     summaryAttemptsPerPhase,
                     summaryAttemptPhase: summaryAttemptPhase + 1, // make everything 1-based
@@ -402,7 +405,7 @@ export class RunningSummarizer implements IDisposable {
             // If all attempts failed, log error (with last attempt info) and close the summarizer container
             this.logger.sendErrorEvent({
                 eventName: "FailToSummarize",
-                summarizeReason,
+                reason,
                 message: lastResult?.message,
             }, lastResult?.error);
 
@@ -430,7 +433,7 @@ export class RunningSummarizer implements IDisposable {
             throw new UsageError("Attempted to run an already-running summarizer on demand");
         }
         const result = this.trySummarizeOnce(
-            { summarizeReason: `onDemand/${reason}` },
+            { reason: `onDemand/${reason}` },
             options,
             this.cancellationToken,
             resultsBuilder);
@@ -490,7 +493,7 @@ export class RunningSummarizer implements IDisposable {
         // Set to undefined first, so that subsequent enqueue attempt while summarize will occur later.
         this.enqueuedSummary = undefined;
         this.trySummarizeOnce(
-            { summarizeReason: `enqueuedSummary/${reason}` },
+            { reason: `enqueuedSummary/${reason}` },
             options,
             this.cancellationToken,
             resultsBuilder);

package/src/summarizer.ts

@@ -145,8 +145,7 @@ export class Summarizer extends EventEmitter implements ISummarizer {
             this.stop("summarizerException");
             throw SummarizingWarning.wrap(error, false /* logged */, this.logger);
         } finally {
-            this.dispose();
-            this.runtime.closeFn();
+            this.close();
         }
     }
 
@@ -159,6 +158,13 @@ export class Summarizer extends EventEmitter implements ISummarizer {
         this.stopDeferred.resolve(reason);
     }
 
+    public close() {
+        // This will result in "summarizerClientDisconnected" stop reason recorded in telemetry,
+        // unless stop() was called earlier
+        this.dispose();
+        this.runtime.closeFn();
+    }
+
     private async runCore(
         onBehalfOf: string,
         options?: Readonly<Partial<ISummarizerOptions>>): Promise<SummarizerStopReason> {
@@ -344,8 +350,7 @@ export class Summarizer extends EventEmitter implements ISummarizer {
                     const stopReason = await Promise.race([this.stopDeferred.promise, runCoordinator.waitCancelled]);
                     await runningSummarizer.waitStop(false);
                     runCoordinator.stop(stopReason);
-                    this.dispose();
-                    this.runtime.closeFn();
+                    this.close();
                 }).catch((reason) => {
                     builder.fail("Failed to start summarizer", reason);
                 });

package/src/summarizerTypes.ts

@@ -9,6 +9,7 @@ import {
     ITelemetryLogger,
     ITelemetryProperties,
 } from "@fluidframework/common-definitions";
+import { ITelemetryLoggerPropertyBag } from "@fluidframework/telemetry-utils";
 import {
     IFluidLoadable,
 } from "@fluidframework/core-interfaces";
@@ -296,8 +297,16 @@ export interface ISummarizerEvents extends IEvent {
 
 export interface ISummarizer extends
     IEventProvider<ISummarizerEvents>, IFluidLoadable, Partial<IProvideSummarizer>{
+    /*
+     * Asks summarizer to move to exit.
+     * Summarizer will finish current processes, which may take a while.
+     * For example, summarizer may complete last summary before exiting.
+     */
     stop(reason: SummarizerStopReason): void;
 
+    /* Closes summarizer. Any pending processes (summary in flight) are abandoned. */
+    close(): void;
+
     run(onBehalfOf: string, options?: Readonly<Partial<ISummarizerOptions>>): Promise<SummarizerStopReason>;
 
     /**
@@ -380,7 +389,7 @@ export interface ISummarizeHeuristicRunner {
 
 type ISummarizeTelemetryRequiredProperties =
     /** Reason code for attempting to summarize */
-    "summarizeReason";
+    "reason";
 
 type ISummarizeTelemetryOptionalProperties =
     /** Number of attempts within the last time window, used for calculating the throttle delay. */
@@ -394,3 +403,53 @@ type ISummarizeTelemetryOptionalProperties =
 export type ISummarizeTelemetryProperties =
     Pick<ITelemetryProperties, ISummarizeTelemetryRequiredProperties> &
     Partial<Pick<ITelemetryProperties, ISummarizeTelemetryOptionalProperties>>;
+
+type SummaryGeneratorRequiredTelemetryProperties =
+    /** True to generate the full tree with no handle reuse optimizations */
+    "fullTree" |
+    /** Time since we last attempted to generate a summary */
+    "timeSinceLastAttempt" |
+    /** Time since we last successfully generated a summary */
+    "timeSinceLastSummary";
+
+type SummaryGeneratorOptionalTelemetryProperties =
+    /** Reference sequence number as of the generate summary attempt. */
+    "referenceSequenceNumber" |
+    /** minimum sequence number (at the reference sequence number) */
+    "minimumSequenceNumber" |
+    /** Delta between the current reference sequence number and the reference sequence number of the last attempt */
+    "opsSinceLastAttempt" |
+    /** Delta between the current reference sequence number and the reference sequence number of the last summary */
+    "opsSinceLastSummary" |
+    /** Delta in sum of op sizes between the current reference sequence number and the reference
+     *  sequence number of the last summary */
+    "opsSizesSinceLastSummary" |
+    /** Delta between the number of non-system ops since the last summary @see isSystemMessage */
+    "nonSystemOpsSinceLastSummary" |
+    /** Time it took to generate the summary tree and stats. */
+    "generateDuration" |
+    /** The handle returned by storage pointing to the uploaded summary tree. */
+    "handle" |
+    /** Time it took to upload the summary tree to storage. */
+    "uploadDuration" |
+    /** The client sequence number of the summarize op submitted for the summary. */
+    "clientSequenceNumber" |
+    /** Time it took for this summary to be acked after it was generated */
+    "ackWaitDuration" |
+    /** Reference sequence number of the ack/nack message */
+    "ackNackSequenceNumber" |
+    /** Actual sequence number of the summary op proposal. */
+    "summarySequenceNumber" |
+    /** Optional Retry-After time in seconds. If specified, the client should wait this many seconds before retrying. */
+    "nackRetryAfter";
+
+export type SummaryGeneratorTelemetry =
+    Pick<ITelemetryProperties, SummaryGeneratorRequiredTelemetryProperties> &
+    Partial<Pick<ITelemetryProperties, SummaryGeneratorOptionalTelemetryProperties>>;
+
+export interface ISummarizeRunnerTelemetry extends ITelemetryLoggerPropertyBag {
+    /** Number of times the summarizer run. */
+    summarizeCount: () => number;
+    /** Number of successful attempts to summarize. */
+    summarizerSuccessfulAttempts: () => number;
+}

package/src/summaryGenerator.ts

@@ -3,7 +3,7 @@
  * Licensed under the MIT License.
  */
 
-import { ITelemetryLogger, ITelemetryProperties } from "@fluidframework/common-definitions";
+import { ITelemetryLogger } from "@fluidframework/common-definitions";
 import {
     assert,
     Deferred,
@@ -27,6 +27,7 @@ import {
     SummarizeResultPart,
     ISummaryCancellationToken,
     ISummarizeTelemetryProperties,
+    SummaryGeneratorTelemetry,
 } from "./summarizerTypes";
 import { IClientSummaryWatcher } from "./summaryCollection";
 
@@ -55,47 +56,6 @@ export async function raceTimer<T>(
 const maxSummarizeTimeoutTime = 20000; // 20 sec
 const maxSummarizeTimeoutCount = 5; // Double and resend 5 times
 
-type SummaryGeneratorRequiredTelemetryProperties =
-    /** True to generate the full tree with no handle reuse optimizations */
-    "fullTree" |
-    /** Time since we last attempted to generate a summary */
-    "timeSinceLastAttempt" |
-    /** Time since we last successfully generated a summary */
-    "timeSinceLastSummary";
-type SummaryGeneratorOptionalTelemetryProperties =
-    /** Reference sequence number as of the generate summary attempt. */
-    "referenceSequenceNumber" |
-    /** minimum sequence number (at the reference sequence number) */
-    "minimumSequenceNumber" |
-    /** Delta between the current reference sequence number and the reference sequence number of the last attempt */
-    "opsSinceLastAttempt" |
-    /** Delta between the current reference sequence number and the reference sequence number of the last summary */
-    "opsSinceLastSummary" |
-    /** Delta in sum of op sizes between the current reference sequence number and the reference
-     *  sequence number of the last summary */
-    "opsSizesSinceLastSummary" |
-    /** Delta between the number of non-system ops since the last summary @see isSystemMessage */
-    "nonSystemOpsSinceLastSummary" |
-    /** Time it took to generate the summary tree and stats. */
-    "generateDuration" |
-    /** The handle returned by storage pointing to the uploaded summary tree. */
-    "handle" |
-    /** Time it took to upload the summary tree to storage. */
-    "uploadDuration" |
-    /** The client sequence number of the summarize op submitted for the summary. */
-    "clientSequenceNumber" |
-    /** Time it took for this summary to be acked after it was generated */
-    "ackWaitDuration" |
-    /** Reference sequence number of the ack/nack message */
-    "ackNackSequenceNumber" |
-    /** Actual sequence number of the summary op proposal. */
-    "summarySequenceNumber" |
-    /** Optional Retry-After time in seconds. If specified, the client should wait this many seconds before retrying. */
-    "nackRetryAfter";
-type SummaryGeneratorTelemetry =
-    Pick<ITelemetryProperties, SummaryGeneratorRequiredTelemetryProperties> &
-    Partial<Pick<ITelemetryProperties, SummaryGeneratorOptionalTelemetryProperties>>;
-
 export type SummarizeReason =
     /**
      * Attempt to summarize after idle timeout has elapsed.
@@ -126,8 +86,6 @@ export type SummarizeReason =
      * stay connected long enough for summarizer client to catch up.
      */
     | "lastSummary"
-    /** Previous summary attempt failed, and we are retrying. */
-    | `retry${number}`
     /** On-demand summary requested with specified reason. */
     | `onDemand;${string}`
     /** Enqueue summarize attempt with specified reason. */

package/src/summaryManager.ts

@@ -193,8 +193,6 @@ export class SummaryManager implements IDisposable {
 
         assert(this.summarizer === undefined, 0x262 /* "Old summarizer is still working!" */);
 
-        let reason = "unknown";
-
         this.delayBeforeCreatingSummarizer().then(async (startWithInitialDelay: boolean) => {
             // Re-validate that it need to be running. Due to asynchrony, it may be not the case anymore
             // but only if creation was delayed. If it was not, then we want to ensure we always create
@@ -202,7 +200,7 @@ export class SummaryManager implements IDisposable {
             // document out of broken state if it has too many ops and ordering service keeps nacking main
             // container (and thus it goes into cycle of reconnects)
             if (startWithInitialDelay && this.getShouldSummarizeState().shouldSummarize === false) {
-                return;
+                return "early exit";
             }
 
             // We transition to Running before requesting the summarizer, because after requesting we can't predict
@@ -213,26 +211,37 @@ export class SummaryManager implements IDisposable {
             this.state = SummaryManagerState.Running;
 
             const summarizer = await this.requestSummarizerFn();
+            this.summarizer = summarizer;
 
             // Re-validate that it need to be running. Due to asynchrony, it may be not the case anymore
             const shouldSummarizeState = this.getShouldSummarizeState();
             if (shouldSummarizeState.shouldSummarize === false) {
                 this.state = SummaryManagerState.Starting;
                 summarizer.stop(shouldSummarizeState.stopReason);
-                return;
+                return "early exit after starting summarizer";
             }
 
-            this.summarizer = summarizer;
-
             // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
             const clientId = this.latestClientId!;
 
-            reason = await PerformanceEvent.timedExecAsync(
+            return PerformanceEvent.timedExecAsync(
                 this.logger,
                 { eventName: "RunningSummarizer", attempt: this.startThrottler.numAttempts },
                 async () => summarizer.run(clientId, this.summarizerOptions),
             );
+        }).then((reason: string) => {
+            this.logger.sendTelemetryEvent({
+                eventName: "EndingSummarizer",
+                reason,
+            });
         }).catch((error) => {
+            this.logger.sendTelemetryEvent(
+                {
+                    eventName: "EndingSummarizer",
+                    reason: "exception",
+                },
+                error);
+
             // Most of exceptions happen due to container being closed while loading it, due to
             // summarizer container loosing connection while load.
             // Not worth reporting such errors as errors. That said, we might miss some real errors if
@@ -252,25 +261,14 @@ export class SummaryManager implements IDisposable {
                         category,
                     },
                     error);
-
-                // Note that summarizer may keep going (like doing last summary).
-                // Ideally we await stopping process, but this code path is due to a bug
-                // that needs to be fixed either way.
-                if (SummaryManager.isStartingOrRunning(this.state)) {
-                    this.stop("summarizerException");
-                }
             }
         }).finally(() => {
             assert(this.state !== SummaryManagerState.Off, 0x264 /* "Expected: Not Off" */);
             this.state = SummaryManagerState.Off;
 
+            this.summarizer?.close();
             this.summarizer = undefined;
 
-            this.logger.sendTelemetryEvent({
-                eventName: "EndingSummarizer",
-                reason,
-            });
-
             if (this.getShouldSummarizeState().shouldSummarize) {
                 this.startSummarization();
             }