@fluidframework/container-runtime 2.0.0-internal.1.1.2 → 2.0.0-internal.1.2.0

package/src/gcSweepReadyUsageDetection.ts

@@ -0,0 +1,147 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { ITelemetryProperties } from "@fluidframework/common-definitions";
+import { ICriticalContainerError } from "@fluidframework/container-definitions";
+import {
+    IConfigProvider,
+    IFluidErrorBase,
+    LoggingError,
+    MonitoringContext,
+} from "@fluidframework/telemetry-utils";
+import { oneDayMs } from "./garbageCollection";
+
+/**
+ * Feature Gate Key -
+ * How many days between closing the container from this error (avoids locking user out of their file altogether)
+ */
+export const skipClosureForXDaysKey = "Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection.SkipClosureForXDays";
+
+/**
+ * LocalStorage key (NOT via feature gate / monitoring context)
+ * A map from docId to info about the last time we closed due to this error
+ */
+export const closuresMapLocalStorageKey = "Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection.Closures";
+
+/**
+ * Feature gate key to enable closing the container if SweepReady objects are used.
+ * Value should contain keywords "interactiveClient" and/or "summarizer" to enable detection in each container type
+ */
+const sweepReadyUsageDetectionSetting = {
+    read(config: IConfigProvider) {
+        const sweepReadyUsageDetectionKey = "Fluid.GarbageCollection.Dogfood.SweepReadyUsageDetection";
+        const value = config.getString(sweepReadyUsageDetectionKey);
+        if (value === undefined) {
+            return { interactiveClient: false, summarizer: false };
+        }
+        return {
+            interactiveClient: value.includes("interactiveClient"),
+            summarizer: value.includes("summarizer"),
+        };
+    },
+};
+
+/**
+ * Error class raised when a SweepReady object is used, indicating a bug in how
+ * references are managed in the container by the application, or a bug in how
+ * GC tracks those references.
+ *
+ * There's a chance for false positives when this error is raised by an Interactive Container,
+ * since only the Summarizer has the latest truth about unreferenced node tracking
+ */
+export class SweepReadyUsageError extends LoggingError implements IFluidErrorBase {
+    /** This errorType will be in temporary use (until Sweep is fully implemented) so don't add to any errorType type */
+    public errorType: string = "unreferencedObjectUsedAfterGarbageCollected";
+}
+
+/**
+ * This class encapsulates the logic around what to do when a SweepReady object is used.
+ * There are several tactics we plan to use in Dogfood environments to aid diagnosis of these cases:
+ * - Closing the interactive container when either the interactive or summarizer client detects this kind of violation
+ * (via sweepReadyUsageDetectionSetting above)
+ * - Throttling the frequency of these crashes via a "Skip Closure Period" per container per device
+ * (via skipClosureForXDaysKey above.  Uses localStorage and closuresMapLocalStorageKey to implement this behavior)
+ */
+export class SweepReadyUsageDetectionHandler {
+    private readonly localStorage: Pick<Storage, "getItem" | "setItem">;
+
+    constructor(
+        private readonly uniqueContainerKey: string,
+        private readonly mc: MonitoringContext,
+        private readonly closeFn: (error?: ICriticalContainerError) => void,
+        localStorageOverride?: Pick<Storage, "getItem" | "setItem">,
+    ) {
+        const noopStorage = { getItem: () => null, setItem: () => {} };
+        if (localStorageOverride !== undefined) {
+            this.localStorage = localStorageOverride;
+        } else {
+            try {
+                // localStorage is not defined in Node environment so this throws
+                this.localStorage = localStorage ?? noopStorage;
+            } catch (error) {
+                this.localStorage = noopStorage;
+            }
+        }
+
+        if (this.localStorage === noopStorage) {
+            // This means the Skip Closure Period logic will not work.
+            this.mc.logger.sendTelemetryEvent({ eventName: "SweepReadyUsageDetectionHandlerNoopStorage" });
+        }
+    }
+
+    /**
+      * If SweepReady Usage Detection is enabled, close the interactive container.
+      * If the SkipClosureForXDays setting is set, don't close the container more than once in that period.
+      *
+      * Once Sweep is fully implemented, this will be removed since the objects will be gone
+      * and errors will arise elsewhere in the runtime
+     */
+    public usageDetectedInInteractiveClient(errorProps: ITelemetryProperties) {
+        if (!sweepReadyUsageDetectionSetting.read(this.mc.config).interactiveClient) {
+            return;
+        }
+
+        // Default stance is we close every time - this reflects the severity of SweepReady Object Usage.
+        // However, we may choose to "throttle" the closures by setting the SkipClosureForXDays setting,
+        // which will only allow the container to close once during that period, to avoid locking users out.
+        let shouldClose: boolean = true;
+        let pastClosuresMap: Record<string, { lastCloseTime: number; } | undefined> = {};
+        let lastCloseTime: number | undefined;
+        const skipClosureForXDays = this.mc.config.getNumber(skipClosureForXDaysKey);
+        if (skipClosureForXDays !== undefined) {
+            // Read pastClosuresMap from localStorage then extract the lastCloseTime from the map
+            try {
+                const rawValue = this.localStorage.getItem(closuresMapLocalStorageKey);
+                const parsedValue = rawValue === null ? {} : JSON.parse(rawValue);
+                if (typeof parsedValue === "object") {
+                    pastClosuresMap = parsedValue;
+                }
+            } catch (e) {
+            }
+            lastCloseTime = pastClosuresMap[this.uniqueContainerKey]?.lastCloseTime;
+
+            // Don't close if we did already within the Skip Closure Period
+            if (lastCloseTime !== undefined && Date.now() < lastCloseTime + skipClosureForXDays * oneDayMs) {
+                shouldClose = false;
+            }
+        }
+
+        const error = new SweepReadyUsageError(
+            "SweepReady object used in Non-Summarizer Client",
+            { errorDetails: JSON.stringify({ ...errorProps, lastCloseTime, skipClosureForXDays }) },
+        );
+        if (shouldClose) {
+            // Update closures map in localStorage before closing
+            // Note there is a race condition between different tabs updating localStorage and overwriting
+            // each others' updates. If so, some tab will crash again. Just reload one at a time to get unstuck
+            pastClosuresMap[this.uniqueContainerKey] = { lastCloseTime: Date.now() };
+            this.localStorage.setItem(closuresMapLocalStorageKey, JSON.stringify(pastClosuresMap));
+
+            this.closeFn(error);
+        } else {
+            this.mc.logger.sendErrorEvent({ eventName: "SweepReadyObject_UsageAllowed" }, error);
+        }
+    }
+}

package/src/orderedClientElection.ts

@@ -208,11 +208,17 @@ export interface IOrderedClientElectionEvents extends IEvent {
 export interface ISerializedElection {
     /** Sequence number at the time of the latest election. */
     readonly electionSequenceNumber: number;
-    /** Most recently elected client id. This is either:
+
+    /**
+     * Most recently elected client id. This is either:
+     *
      * 1. the interactive elected parent client, in which case electedClientId === electedParentId,
-     *  and the SummaryManager on the elected client will spawn a summarizer client, or
-     * 2. the non-interactive summarizer client itself. */
+     * and the SummaryManager on the elected client will spawn a summarizer client, or
+     *
+     * 2. the non-interactive summarizer client itself.
+     */
     readonly electedClientId: string | undefined;
+
     /** Most recently elected parent client id. This is always an interactive client. */
     readonly electedParentId: string | undefined;
 }
@@ -221,10 +227,15 @@ export interface ISerializedElection {
 export interface IOrderedClientElection extends IEventProvider<IOrderedClientElectionEvents> {
     /** Count of eligible clients in the collection. */
     readonly eligibleCount: number;
-    /** Currently elected client. This is either:
+
+    /**
+     * Currently elected client. This is either:
+     *
      * 1. the interactive elected parent client, in which case electedClientId === electedParentId,
-     *  and the SummaryManager on the elected client will spawn a summarizer client, or
-     * 2. the non-interactive summarizer client itself. */
+     * and the SummaryManager on the elected client will spawn a summarizer client, or
+     *
+     * 2. the non-interactive summarizer client itself.
+     */
     readonly electedClient: ITrackedClient | undefined;
     /** Currently elected parent client. This is always an interactive client. */
     readonly electedParent: ITrackedClient | undefined;
@@ -283,13 +294,20 @@ export class OrderedClientElection
      * electedClient leaves the quorum.
      *
      * A typical sequence looks like this:
+     *
      * i. Begin by electing A. electedParent === A, electedClient === A.
+     *
      * ii. SummaryManager running on A spawns a summarizer client, A'. electedParent === A, electedClient === A'
+     *
      * iii. A' stops producing summaries. A new parent client, B, is elected. electedParent === B, electedClient === A'
+     *
      * iv. SummaryManager running on A detects the change to electedParent and tells the summarizer to stop, but A'
-     *      is in mid-summarization. No new summarizer is spawned, as electedParent !== electedClient.
+     * is in mid-summarization. No new summarizer is spawned, as electedParent !== electedClient.
+     *
      * v. A' completes its summary, and the summarizer and backing client are torn down.
+     *
      * vi. A' leaves the quorum, and B takes its place as electedClient. electedParent === B, electedClient === B
+     *
      * vii. SummaryManager running on B spawns a summarizer client, B'. electedParent === B, electedClient === B'
      */
     public get electedClient() {
@@ -357,7 +375,8 @@ export class OrderedClientElection
         }
     }
 
-    /** Tries changing the elected client, raising an event if it is different.
+    /**
+     * Tries changing the elected client, raising an event if it is different.
      * Note that this function does no eligibility or suitability checks. If we get here, then
      * we will set _electedClient, and we will set _electedParent if this is an interactive client.
      */
@@ -467,7 +486,8 @@ export class OrderedClientElection
         return this.orderedClientCollection.getAllClients().filter(this.isEligibleFn);
     }
 
-    /** Advance election to the next-oldest client. This is called if the current parent is leaving the quorum,
+    /**
+     * Advance election to the next-oldest client. This is called if the current parent is leaving the quorum,
      * or if the current summarizer is not responsive and we want to stop it and spawn a new one.
      */
     public incrementElectedClient(sequenceNumber: number): void {
@@ -482,7 +502,8 @@ export class OrderedClientElection
         }
     }
 
-    /** (Re-)start election with the oldest client in the quorum. This is called if we need to summarize
+    /**
+     * (Re-)start election with the oldest client in the quorum. This is called if we need to summarize
      * and no client has been elected.
      */
     public resetElectedClient(sequenceNumber: number): void {

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.0.0-internal.1.1.2";
+export const pkgVersion = "2.0.0-internal.1.2.0";

package/src/pendingStateManager.ts

@@ -11,7 +11,6 @@ import {
     ISequencedDocumentMessage,
 } from "@fluidframework/protocol-definitions";
 import { FlushMode } from "@fluidframework/runtime-definitions";
-import { wrapError } from "@fluidframework/telemetry-utils";
 import Deque from "double-ended-queue";
 import { ContainerMessageType } from "./containerRuntime";
 import { pkgVersion } from "./packageVersion";
@@ -69,10 +68,6 @@ export interface IRuntimeStateHandler{
         content: any,
         localOpMetadata: unknown,
         opMetadata: Record<string, unknown> | undefined): void;
-    rollback(
-        type: ContainerMessageType,
-        content: any,
-        localOpMetadata: unknown): void;
 }
 
 /**
@@ -233,7 +228,11 @@ export class PendingStateManager implements IDisposable {
 
             // then we push onto pendingStates which will cause PendingStateManager to resubmit when we connect
             // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-            this.pendingStates.push(this.initialStates.shift()!);
+            const firstPendingState = this.initialStates.shift()!;
+            this.pendingStates.push(firstPendingState);
+            if (firstPendingState.type === "message") {
+                this._pendingMessagesCount++;
+            }
         }
     }
 
@@ -267,6 +266,7 @@ export class PendingStateManager implements IDisposable {
         }
 
         this._pendingMessagesCount--;
+        assert(this._pendingMessagesCount >= 0, 0x3d6 /* positive */);
 
         // Post-processing part - If we are processing a batch then this could be the last message in the batch.
         this.maybeProcessBatchEnd(message);
@@ -286,9 +286,11 @@ export class PendingStateManager implements IDisposable {
 
         /**
          * We are checking if the next message is the start of a batch. It can happen in the following scenarios:
+         *
          * 1. The FlushMode was set to TurnBased before this message was sent.
+         *
          * 2. The FlushMode was already TurnBased and a flush was called before this message was sent. This essentially
-         *    means that the flush marked the end of a previous batch and beginning of a new batch.
+         * means that the flush marked the end of a previous batch and beginning of a new batch.
          *
          * Keep reading pending states from the queue until we encounter a message. It's possible that the FlushMode was
          * updated a bunch of times without sending any messages.
@@ -393,31 +395,6 @@ export class PendingStateManager implements IDisposable {
         this.isProcessingBatch = false;
     }
 
-    /**
-     * Capture the pending state at this point
-     */
-    public checkpoint() {
-        const checkpointHead = this.pendingStates.peekBack();
-        return {
-            rollback: () => {
-                try {
-                    while (this.pendingStates.peekBack() !== checkpointHead) {
-                        this.rollbackNextPendingState();
-                    }
-                } catch (err) {
-                    const error = wrapError(err, (message) => {
-                        return DataProcessingError.create(
-                            `RollbackError: ${message}`,
-                            "checkpointRollback",
-                            undefined) as DataProcessingError;
-                    });
-                    this.stateHandler.close(error);
-                    throw error;
-                }
-            },
-        };
-    }
-
     /**
      * Returns the next pending state from the pending state queue.
      */
@@ -427,31 +404,6 @@ export class PendingStateManager implements IDisposable {
         return nextPendingState;
     }
 
-    /**
-     * Undo the last pending state
-     */
-    private rollbackNextPendingState() {
-        const pendingStatesCount = this.pendingStates.length;
-        if (pendingStatesCount === 0) {
-            return;
-        }
-
-        this._pendingMessagesCount--;
-
-        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-        const pendingState = this.pendingStates.pop()!;
-        switch (pendingState.type) {
-            case "message":
-                this.stateHandler.rollback(
-                    pendingState.messageType,
-                    pendingState.content,
-                    pendingState.localOpMetadata);
-                break;
-            default:
-                throw new Error(`Can't rollback state ${pendingState.type}`);
-        }
-    }
-
     /**
      * Called when the Container's connection state changes. If the Container gets connected, it replays all the pending
      * states in its queue. This includes setting the FlushMode and triggering resubmission of unacked ops.

package/src/runningSummarizer.ts

@@ -239,7 +239,7 @@ export class RunningSummarizer implements IDisposable {
     public handleOp(op: ISequencedDocumentMessage) {
         this.heuristicData.lastOpSequenceNumber = op.sequenceNumber;
 
-        if (op.type !== MessageType.Summarize && isRuntimeMessage(op)) {
+        if (isRuntimeMessage(op)) {
             this.heuristicData.numRuntimeOps++;
         } else {
             this.heuristicData.numNonRuntimeOps++;

package/src/scheduleManager.ts

@@ -8,8 +8,12 @@ import { IDocumentMessage, ISequencedDocumentMessage } from "@fluidframework/pro
 import { ITelemetryLogger } from "@fluidframework/common-definitions";
 import { ChildLogger } from "@fluidframework/telemetry-utils";
 import { assert, performance } from "@fluidframework/common-utils";
-import { isUnpackedRuntimeMessage } from "@fluidframework/driver-utils";
-import { DataCorruptionError, extractSafePropertiesFromMessage } from "@fluidframework/container-utils";
+import { isRuntimeMessage } from "@fluidframework/driver-utils";
+import {
+    DataCorruptionError,
+    DataProcessingError,
+    extractSafePropertiesFromMessage,
+} from "@fluidframework/container-utils";
 import { DeltaScheduler } from "./deltaScheduler";
 import { pkgVersion } from "./packageVersion";
 import { latencyThreshold } from "./connectionTelemetry";
@@ -20,10 +24,12 @@ type IRuntimeMessageMetadata = undefined | {
 
 /**
  * This class has the following responsibilities:
+ *
  * 1. It tracks batches as we process ops and raises "batchBegin" and "batchEnd" events.
- *    As part of it, it validates batch correctness (i.e. no system ops in the middle of batch)
+ * As part of it, it validates batch correctness (i.e. no system ops in the middle of batch)
+ *
  * 2. It creates instance of ScheduleManagerCore that ensures we never start processing ops from batch
- *    unless all ops of the batch are in.
+ * unless all ops of the batch are in.
  */
 export class ScheduleManager {
     private readonly deltaScheduler: DeltaScheduler;
@@ -33,13 +39,14 @@ export class ScheduleManager {
     constructor(
         private readonly deltaManager: IDeltaManager<ISequencedDocumentMessage, IDocumentMessage>,
         private readonly emitter: EventEmitter,
+        readonly getClientId: () => string | undefined,
         private readonly logger: ITelemetryLogger,
     ) {
         this.deltaScheduler = new DeltaScheduler(
             this.deltaManager,
             ChildLogger.create(this.logger, "DeltaScheduler"),
         );
-        void new ScheduleManagerCore(deltaManager, logger);
+        void new ScheduleManagerCore(deltaManager, getClientId, logger);
     }
 
     public beforeOpProcessing(message: ISequencedDocumentMessage) {
@@ -52,11 +59,7 @@ export class ScheduleManager {
             this.deltaScheduler.batchBegin(message);
 
             const batch = (message?.metadata as IRuntimeMessageMetadata)?.batch;
-            if (batch) {
-                this.batchClientId = message.clientId;
-            } else {
-                this.batchClientId = undefined;
-            }
+            this.batchClientId = batch ? message.clientId : undefined;
         }
     }
 
@@ -99,6 +102,7 @@ class ScheduleManagerCore {
 
     constructor(
         private readonly deltaManager: IDeltaManager<ISequencedDocumentMessage, IDocumentMessage>,
+        private readonly getClientId: () => string | undefined,
         private readonly logger: ITelemetryLogger,
     ) {
         // Listen for delta manager sends and add batch metadata to messages
@@ -234,9 +238,22 @@ class ScheduleManagerCore {
         const batchMetadata = metadata?.batch;
 
         // Protocol messages are never part of a runtime batch of messages
-        if (!isUnpackedRuntimeMessage(message)) {
+        if (!isRuntimeMessage(message)) {
             // Protocol messages should never show up in the middle of the batch!
-            assert(this.currentBatchClientId === undefined, 0x29a /* "System message in the middle of batch!" */);
+            if (this.currentBatchClientId !== undefined) {
+                throw DataProcessingError.create(
+                    "Received a system message during batch processing", // Formerly known as assert 0x29a
+                    "trackPending",
+                    message,
+                    {
+                        runtimeVersion: pkgVersion,
+                        batchClientId: this.currentBatchClientId,
+                        pauseSequenceNumber: this.pauseSequenceNumber,
+                        localBatch: this.currentBatchClientId === this.getClientId(),
+                        messageType: message.type,
+                    });
+            }
+
             assert(batchMetadata === undefined, 0x29b /* "system op in a batch?" */);
             assert(!this.localPaused, 0x29c /* "we should be processing ops when there is no active batch" */);
             return;
@@ -258,6 +275,9 @@ class ScheduleManagerCore {
                     {
                         runtimeVersion: pkgVersion,
                         batchClientId: this.currentBatchClientId,
+                        pauseSequenceNumber: this.pauseSequenceNumber,
+                        localBatch: this.currentBatchClientId === this.getClientId(),
+                        localMessage: message.clientId === this.getClientId(),
                         ...extractSafePropertiesFromMessage(message),
                     });
             }

package/src/summarizerTypes.ts

@@ -25,16 +25,16 @@ import { SummarizeReason } from "./summaryGenerator";
 import { ISummaryConfigurationHeuristics } from ".";
 
 /**
- * @deprecated - This will be removed in a later release.
+ * @deprecated This will be removed in a later release.
  */
 export const ISummarizer: keyof IProvideSummarizer = "ISummarizer";
 
 /**
- * @deprecated - This will be removed in a later release.
+ * @deprecated This will be removed in a later release.
  */
 export interface IProvideSummarizer {
     /**
-     * @deprecated - This will be removed in a later release.
+     * @deprecated This will be removed in a later release.
      */
     readonly ISummarizer: ISummarizer;
 }
@@ -130,6 +130,7 @@ export interface IOnDemandSummarizeOptions extends ISummarizeOptions {
 export interface IEnqueueSummarizeOptions extends IOnDemandSummarizeOptions {
     /** If specified, The summarize attempt will not occur until after this sequence number. */
     readonly afterSequenceNumber?: number;
+
     /**
      * True to override the existing enqueued summarize attempt if there is one.
      * This will guarantee that this attempt gets enqueued. If override is false,
@@ -204,11 +205,16 @@ export interface ISubmitSummaryOpResult extends Omit<IUploadSummaryResult, "stag
  * The result consists of 4 possible stages, each with its own data.
  * The data is cumulative, so each stage will contain the data from the previous stages.
  * If the final "submitted" stage is not reached, the result may contain the error object.
+ *
  * Stages:
- *  1. "base" - stopped before the summary tree was even generated, and the result only contains the base data
- *  2. "generate" - the summary tree was generated, and the result will contain that tree + stats
- *  3. "upload" - the summary was uploaded to storage, and the result contains the server-provided handle
- *  4. "submit" - the summarize op was submitted, and the result contains the op client sequence number.
+ *
+ * 1. "base" - stopped before the summary tree was even generated, and the result only contains the base data
+ *
+ * 2. "generate" - the summary tree was generated, and the result will contain that tree + stats
+ *
+ * 3. "upload" - the summary was uploaded to storage, and the result contains the server-provided handle
+ *
+ * 4. "submit" - the summarize op was submitted, and the result contains the op client sequence number.
  */
 export type SubmitSummaryResult =
     | IBaseSummarizeResult
@@ -453,8 +459,10 @@ type SummaryGeneratorOptionalTelemetryProperties =
     "opsSinceLastAttempt" |
     /** Delta between the current reference sequence number and the reference sequence number of the last summary */
     "opsSinceLastSummary" |
-    /** Delta in sum of op sizes between the current reference sequence number and the reference
-     *  sequence number of the last summary */
+    /**
+     * Delta in sum of op sizes between the current reference sequence number and the reference
+     * sequence number of the last summary
+     */
     "opsSizesSinceLastSummary" |
     /** Delta between the number of non-runtime ops since the last summary */
     "nonRuntimeOpsSinceLastSummary" |

package/src/summaryCollection.ts

@@ -239,9 +239,7 @@ export class SummaryCollection extends TypedEventEmitter<ISummaryCollectionOpEve
         private readonly logger: ITelemetryLogger,
     ) {
         super();
-        this.deltaManager.on(
-            "op",
-            (op) => this.handleOp(op));
+        this.deltaManager.on("op", (op) => this.handleOp(op));
     }
 
     /**
@@ -295,24 +293,41 @@ export class SummaryCollection extends TypedEventEmitter<ISummaryCollectionOpEve
         return this.lastAck;
     }
 
+    private parseContent(op: ISequencedDocumentMessage) {
+        // back-compat: ADO #1385: Make this unconditional in the future,
+        // when Container.processRemoteMessage stops parsing contents. That said, we should move to
+        // listen for "op" events from ContainerRuntime, and parsing may not be required at all if
+        // ContainerRuntime.process() would parse it for all types of ops.
+        // Can make either of those changes only when LTS moves to a version that has no content
+        // parsing in loader layer!
+        if (typeof op.contents === "string") {
+            op.contents = JSON.parse(op.contents);
+        }
+    }
+
     /**
      * Handler for ops; only handles ops relating to summaries.
      * @param op - op message to handle
      */
-    private handleOp(op: ISequencedDocumentMessage) {
+    private handleOp(opArg: ISequencedDocumentMessage) {
+        const op = { ...opArg };
+
         switch (op.type) {
-            case MessageType.Summarize: {
-                this.handleSummaryOp(op as ISummaryOpMessage);
-                return;
-            }
-            case MessageType.SummaryAck: {
-                this.handleSummaryAck(op as ISummaryAckMessage);
-                return;
-            }
-            case MessageType.SummaryNack: {
-                this.handleSummaryNack(op as ISummaryNackMessage);
-                return;
-            }
+            case MessageType.Summarize:
+                this.parseContent(op);
+                return this.handleSummaryOp(op as ISummaryOpMessage);
+            case MessageType.SummaryAck:
+            case MessageType.SummaryNack:
+                // Old files (prior to PR #10077) may not contain this info
+                // back-compat: ADO #1385: remove cast when ISequencedDocumentMessage changes are propagated
+                if ((op as any).data !== undefined) {
+                    op.contents = JSON.parse((op as any).data);
+                } else {
+                    this.parseContent(op);
+                }
+                return op.type === MessageType.SummaryAck
+                    ? this.handleSummaryAck(op as ISummaryAckMessage)
+                    : this.handleSummaryNack(op as ISummaryNackMessage);
             default: {
                 // If the difference between timestamp of current op and last summary op is greater than
                 // the maxAckWaitTime, then we need to inform summarizer to not wait and summarize

package/src/summaryManager.ts

@@ -302,8 +302,8 @@ export class SummaryManager implements IDisposable {
 
     /**
      * Implements initial delay before creating summarizer
-     * @returns true, if creation is delayed due to heuristics (not many ops to summarize).
-     *          False if summarizer should start immediately due to too many unsummarized ops.
+     * @returns `true`, if creation is delayed due to heuristics (not many ops to summarize).
+     * `false` if summarizer should start immediately due to too many unsummarized ops.
      */
     private async delayBeforeCreatingSummarizer(): Promise<boolean> {
         // throttle creation of new summarizer containers to prevent spamming the server with websocket connections