@fluidframework/container-runtime 2.0.0-internal.6.2.0 → 2.0.0-internal.6.3.1

package/dist/containerRuntime.js

@@ -19,10 +19,10 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ContainerRuntime = exports.makeLegacySendBatchFn = exports.getDeviceSpec = exports.agentSchedulerId = exports.isRuntimeMessage = exports.RuntimeMessage = exports.CompressionAlgorithms = exports.defaultRuntimeHeaderData = exports.InactiveResponseHeaderKey = exports.TombstoneResponseHeaderKey = exports.AllowInactiveRequestHeaderKey = exports.AllowTombstoneRequestHeaderKey = exports.RuntimeHeaders = exports.DefaultSummaryConfiguration = exports.ContainerMessageType = void 0;
+exports.ContainerRuntime = exports.makeLegacySendBatchFn = exports.getDeviceSpec = exports.agentSchedulerId = exports.isRuntimeMessage = exports.RuntimeMessage = exports.defaultPendingOpsRetryDelayMs = exports.defaultPendingOpsWaitTimeoutMs = exports.CompressionAlgorithms = exports.defaultRuntimeHeaderData = exports.InactiveResponseHeaderKey = exports.TombstoneResponseHeaderKey = exports.AllowInactiveRequestHeaderKey = exports.AllowTombstoneRequestHeaderKey = exports.RuntimeHeaders = exports.DefaultSummaryConfiguration = exports.ContainerMessageType = void 0;
 const container_definitions_1 = require("@fluidframework/container-definitions");
-const common_utils_1 = require("@fluidframework/common-utils");
 const core_utils_1 = require("@fluidframework/core-utils");
+const client_utils_1 = require("@fluid-internal/client-utils");
 const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
 const driver_definitions_1 = require("@fluidframework/driver-definitions");
 const driver_utils_1 = require("@fluidframework/driver-utils");
@@ -136,6 +136,10 @@ const defaultCompressionConfig = {
     compressionAlgorithm: CompressionAlgorithms.lz4,
 };
 const defaultChunkSizeInBytes = 204800;
+/** The default time to wait for pending ops to be processed during summarization */
+exports.defaultPendingOpsWaitTimeoutMs = 1000;
+/** The default time to delay a summarization retry attempt when there are pending ops */
+exports.defaultPendingOpsRetryDelayMs = 1000;
 /**
  * Instead of refreshing from latest because we do not have 100% confidence in the state
  * of the current system, we should close the summarizer and let it recover.
@@ -201,7 +205,7 @@ exports.makeLegacySendBatchFn = makeLegacySendBatchFn;
  * Represents the runtime of the container. Contains helper functions/state of the container.
  * It will define the store level mappings.
  */
-class ContainerRuntime extends common_utils_1.TypedEventEmitter {
+class ContainerRuntime extends client_utils_1.TypedEventEmitter {
     /**
      * @internal
      */
@@ -454,12 +458,10 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         });
         // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
         this._audience = audience;
-        this.summaryStateUpdateMethod = this.mc.config.getString("Fluid.ContainerRuntime.Test.SummaryStateUpdateMethodV2");
         const closeSummarizerDelayOverride = this.mc.config.getNumber("Fluid.ContainerRuntime.Test.CloseSummarizerDelayOverrideMs");
         this.closeSummarizerDelayMs = closeSummarizerDelayOverride ?? defaultCloseSummarizerDelayMs;
         this.validateSummaryBeforeUpload =
-            this.mc.config.getBoolean("Fluid.ContainerRuntime.Test.ValidateSummaryBeforeUpload") ??
-                false;
+            this.mc.config.getBoolean("Fluid.Summarizer.ValidateSummaryBeforeUpload") ?? false;
         this.summaryCollection = new summary_1.SummaryCollection(this.deltaManager, this.logger);
         this.dirtyContainer =
             this.attachState !== container_definitions_1.AttachState.Attached || this.hasPendingMessages();
@@ -520,7 +522,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             // we accumulate ops while being in read-only state.
             // once user gets write permissions and we have active connection, flush all pending ops.
             // Note that the inner (non-proxy) delta manager is needed here to get the readonly information.
-            (0, common_utils_1.assert)(readonly === this.innerDeltaManager.readOnlyInfo.readonly, 0x124 /* "inconsistent readonly property/event state" */);
+            (0, core_utils_1.assert)(readonly === this.innerDeltaManager.readOnlyInfo.readonly, 0x124 /* "inconsistent readonly property/event state" */);
             // We need to be very careful with when we (re)send pending ops, to ensure that we only send ops
             // when we either never send an op, or attempted to send it but we know for sure it was not
             // sequenced by server and will never be sequenced (i.e. was lost)
@@ -533,7 +535,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             // can rely on same safety mechanism and resend ops only when we establish new connection.
             // This is applicable for read-only permissions (event is raised before connection is properly registered),
             // but it's an extra requirement for Container.forceReadonly() API
-            (0, common_utils_1.assert)(!readonly || !this.connected, 0x125 /* "Unsafe to transition to read-only state!" */);
+            (0, core_utils_1.assert)(!readonly || !this.connected, 0x125 /* "Unsafe to transition to read-only state!" */);
             this.replayPendingStates();
         });
         // logging hardware telemetry
@@ -557,7 +559,6 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 disableAttachReorder: this.disableAttachReorder,
                 disablePartialFlush,
                 idCompressorEnabled: this.idCompressorEnabled,
-                summaryStateUpdateMethod: this.summaryStateUpdateMethod,
                 closeSummarizerDelayOverride,
             }),
             telemetryDocumentId: this.telemetryDocumentId,
@@ -567,7 +568,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         (0, batchTracker_1.BindBatchTracker)(this, this.logger);
         this.entryPoint = new core_utils_1.LazyPromise(async () => {
             if (this.isSummarizerClient) {
-                (0, common_utils_1.assert)(this._summarizer !== undefined, 0x5bf /* Summarizer object is undefined in a summarizer client */);
+                (0, core_utils_1.assert)(this._summarizer !== undefined, 0x5bf /* Summarizer object is undefined in a summarizer client */);
                 return this._summarizer;
             }
             return initializeEntryPoint?.(this);
@@ -652,7 +653,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             if (context.baseSnapshot && blobId) {
                 // IContainerContext storage api return type still has undefined in 0.39 package version.
                 // So once we release 0.40 container-defn package we can remove this check.
-                (0, common_utils_1.assert)(context.storage !== undefined, 0x1f5 /* "Attached state should have storage" */);
+                (0, core_utils_1.assert)(context.storage !== undefined, 0x1f5 /* "Attached state should have storage" */);
                 return (0, driver_utils_1.readAndParse)(context.storage, blobId);
             }
         };
@@ -667,7 +668,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         const blobManagerSnapshot = await blobManager_1.BlobManager.load(context.baseSnapshot?.trees[summary_1.blobsTreeName], async (id) => {
             // IContainerContext storage api return type still has undefined in 0.39 package version.
             // So once we release 0.40 container-defn package we can remove this check.
-            (0, common_utils_1.assert)(context.storage !== undefined, 0x256 /* "storage undefined in attached container" */);
+            (0, core_utils_1.assert)(context.storage !== undefined, 0x256 /* "storage undefined in attached container" */);
             return (0, driver_utils_1.readAndParse)(context.storage, id);
         });
         // Verify summary runtime sequence number matches protocol sequence number.
@@ -774,7 +775,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         return this._disposed;
     }
     get summarizer() {
-        (0, common_utils_1.assert)(this._summarizer !== undefined, 0x257 /* "This is not summarizing container" */);
+        (0, core_utils_1.assert)(this._summarizer !== undefined, 0x257 /* "This is not summarizing container" */);
         return this._summarizer;
     }
     isSummariesDisabled() {
@@ -879,7 +880,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 const subRequest = requestParser.createSubRequest(1);
                 // We always expect createSubRequest to include a leading slash, but asserting here to protect against
                 // unintentionally modifying the url if that changes.
-                (0, common_utils_1.assert)(subRequest.url.startsWith("/"), 0x126 /* "Expected createSubRequest url to include a leading slash" */);
+                (0, core_utils_1.assert)(subRequest.url.startsWith("/"), 0x126 /* "Expected createSubRequest url to include a leading slash" */);
                 return dataStore.request(subRequest);
             }
             return (0, runtime_utils_1.create404Response)(request);
@@ -938,7 +939,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     addContainerStateToSummary(summaryTree, fullTree, trackState, telemetryContext) {
         this.addMetadataToSummary(summaryTree);
         if (this.idCompressorEnabled) {
-            (0, common_utils_1.assert)(this.idCompressor !== undefined, 0x67a /* IdCompressor should be defined if enabled */);
+            (0, core_utils_1.assert)(this.idCompressor !== undefined, 0x67a /* IdCompressor should be defined if enabled */);
             const idCompressorState = JSON.stringify(this.idCompressor.serialize(false));
             (0, runtime_utils_1.addBlobToSummary)(summaryTree, summary_1.idCompressorBlobName, idCompressorState);
         }
@@ -1013,7 +1014,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         // Save the old state, reset to false, disable event emit
         const oldState = this.dirtyContainer;
         this.dirtyContainer = false;
-        (0, common_utils_1.assert)(this.emitDirtyDocumentEvent, 0x127 /* "dirty document event not set on replay" */);
+        (0, core_utils_1.assert)(this.emitDirtyDocumentEvent, 0x127 /* "dirty document event not set on replay" */);
         this.emitDirtyDocumentEvent = false;
         let newState;
         try {
@@ -1047,9 +1048,9 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
      * ! Note: this format needs to be in-line with what is set in the "ContainerRuntime.submit(...)" method
      */
     parseOpContent(serializedContent) {
-        (0, common_utils_1.assert)(serializedContent !== undefined, 0x6d5 /* content must be defined */);
+        (0, core_utils_1.assert)(serializedContent !== undefined, 0x6d5 /* content must be defined */);
         const { type, contents, compatDetails } = JSON.parse(serializedContent);
-        (0, common_utils_1.assert)(type !== undefined, 0x6d6 /* incorrect op content format */);
+        (0, core_utils_1.assert)(type !== undefined, 0x6d6 /* incorrect op content format */);
         return { type, contents, compatDetails };
     }
     async applyStashedOp(op) {
@@ -1061,7 +1062,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             case ContainerMessageType.Attach:
                 return this.dataStores.applyStashedAttachOp(contents);
             case ContainerMessageType.IdAllocation:
-                (0, common_utils_1.assert)(this.idCompressor !== undefined, 0x67b /* IdCompressor should be defined if enabled */);
+                (0, core_utils_1.assert)(this.idCompressor !== undefined, 0x67b /* IdCompressor should be defined if enabled */);
                 return this.applyStashedIdAllocationOp(contents);
             case ContainerMessageType.Alias:
             case ContainerMessageType.BlobAttach:
@@ -1100,7 +1101,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         this.setConnectionStateCore(connected, clientId);
     }
     setConnectionStateCore(connected, clientId) {
-        (0, common_utils_1.assert)(!this.delayConnectClientId, 0x394 /* connect event delay must be cleared before propagating connect event */);
+        (0, core_utils_1.assert)(!this.delayConnectClientId, 0x394 /* connect event delay must be cleared before propagating connect event */);
         this.verifyNotClosed();
         // There might be no change of state due to Container calling this API after loading runtime.
         const changeOfState = this._connected !== connected;
@@ -1118,7 +1119,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             this._perfSignalData.trackingSignalSequenceNumber = undefined;
         }
         else {
-            (0, common_utils_1.assert)(this.attachState === container_definitions_1.AttachState.Attached, 0x3cd /* Connection is possible only if container exists in storage */);
+            (0, core_utils_1.assert)(this.attachState === container_definitions_1.AttachState.Attached, 0x3cd /* Connection is possible only if container exists in storage */);
         }
         // Fail while disconnected
         if (reconnection) {
@@ -1213,7 +1214,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 this.blobManager.processBlobAttachOp(message, local);
                 break;
             case ContainerMessageType.IdAllocation:
-                (0, common_utils_1.assert)(this.idCompressor !== undefined, 0x67c /* IdCompressor should be defined if enabled */);
+                (0, core_utils_1.assert)(this.idCompressor !== undefined, 0x67c /* IdCompressor should be defined if enabled */);
                 this.idCompressor.finalizeCreationRange(message.contents);
                 break;
             case ContainerMessageType.ChunkedOp:
@@ -1311,7 +1312,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         await this.dataStores.waitIfPendingAlias(id);
         const internalId = this.internalId(id);
         const context = await this.dataStores.getDataStore(internalId, { wait });
-        (0, common_utils_1.assert)(await context.isRoot(), 0x12b /* "did not get root data store" */);
+        (0, core_utils_1.assert)(await context.isRoot(), 0x12b /* "did not get root data store" */);
         return context.realize();
     }
     /**
@@ -1319,9 +1320,9 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
      * This method is expected to be called at the end of a batch.
      */
     flush() {
-        (0, common_utils_1.assert)(this._orderSequentiallyCalls === 0, 0x24c /* "Cannot call `flush()` from `orderSequentially`'s callback" */);
+        (0, core_utils_1.assert)(this._orderSequentiallyCalls === 0, 0x24c /* "Cannot call `flush()` from `orderSequentially`'s callback" */);
         this.outbox.flush();
-        (0, common_utils_1.assert)(this.outbox.isEmpty, 0x3cf /* reentrancy */);
+        (0, core_utils_1.assert)(this.outbox.isEmpty, 0x3cf /* reentrancy */);
     }
     orderSequentially(callback) {
         let checkpoint;
@@ -1481,10 +1482,10 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     }
     setAttachState(attachState) {
         if (attachState === container_definitions_1.AttachState.Attaching) {
-            (0, common_utils_1.assert)(this.attachState === container_definitions_1.AttachState.Attaching, 0x12d /* "Container Context should already be in attaching state" */);
+            (0, core_utils_1.assert)(this.attachState === container_definitions_1.AttachState.Attaching, 0x12d /* "Container Context should already be in attaching state" */);
         }
         else {
-            (0, common_utils_1.assert)(this.attachState === container_definitions_1.AttachState.Attached, 0x12e /* "Container Context should already be in attached state" */);
+            (0, core_utils_1.assert)(this.attachState === container_definitions_1.AttachState.Attached, 0x12e /* "Container Context should already be in attached state" */);
             this.emit("attached");
         }
         if (attachState === container_definitions_1.AttachState.Attached && !this.hasPendingMessages()) {
@@ -1542,7 +1543,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 await this.collectGarbage({ logger: summaryLogger, runSweep, fullGC }, telemetryContext);
             }
             const { stats, summary } = await this.summarizerNode.summarize(fullTree, trackState, telemetryContext);
-            (0, common_utils_1.assert)(summary.type === protocol_definitions_1.SummaryType.Tree, 0x12f /* "Container Runtime's summarize should always return a tree" */);
+            (0, core_utils_1.assert)(summary.type === protocol_definitions_1.SummaryType.Tree, 0x12f /* "Container Runtime's summarize should always return a tree" */);
             return { stats, summary };
         }
         finally {
@@ -1654,7 +1655,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             case gc_1.GCNodeType.SubDataStore:
                 return this.dataStores.getDataStorePackagePath(nodePath);
             default:
-                (0, common_utils_1.assert)(false, 0x2de /* "Package path requested for unsupported node type." */);
+                (0, core_utils_1.assert)(false, 0x2de /* "Package path requested for unsupported node type." */);
         }
     }
     /**
@@ -1721,7 +1722,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 all: { summaryNumber },
             },
         });
-        (0, common_utils_1.assert)(this.outbox.isEmpty, 0x3d1 /* Can't trigger summary in the middle of a batch */);
+        (0, core_utils_1.assert)(this.outbox.isEmpty, 0x3d1 /* Can't trigger summary in the middle of a batch */);
         let latestSnapshotVersionId;
         if (refreshLatestAck) {
             const latestSnapshotInfo = await this.refreshLatestSummaryAckFromServer((0, telemetry_utils_1.createChildLogger)({
@@ -1733,6 +1734,40 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             // We might need to catch up to the latest summary's reference sequence number before pausing.
             await this.waitForDeltaManagerToCatchup(latestSnapshotRefSeq, summaryNumberLogger);
         }
+        // If there are pending (unacked ops), the summary will not be eventual consistent and it may even be
+        // incorrect. So, wait for the container to be saved with a timeout. If the container is not saved
+        // within the timeout, check if it should be failed or can continue.
+        if (this.validateSummaryBeforeUpload && this.hasPendingMessages()) {
+            const countBefore = this.pendingMessagesCount;
+            // The timeout for waiting for pending ops can be overridden via configurations.
+            const pendingOpsTimeout = this.mc.config.getNumber("Fluid.Summarizer.waitForPendingOpsTimeoutMs") ??
+                exports.defaultPendingOpsWaitTimeoutMs;
+            await new Promise((resolve, reject) => {
+                const timeoutId = setTimeout(() => resolve(), pendingOpsTimeout);
+                this.once("saved", () => {
+                    clearTimeout(timeoutId);
+                    resolve();
+                });
+                this.once("dispose", () => {
+                    clearTimeout(timeoutId);
+                    reject(new Error("Runtime is disposed while summarizing"));
+                });
+            });
+            // Log that there are pending ops while summarizing. This will help us gather data on how often this
+            // happens, whether we attempted to wait for these ops to be acked and what was the result.
+            summaryNumberLogger.sendTelemetryEvent({
+                eventName: "PendingOpsWhileSummarizing",
+                saved: this.hasPendingMessages() ? false : true,
+                timeout: pendingOpsTimeout,
+                countBefore,
+                countAfter: this.pendingMessagesCount,
+            });
+            // There could still be pending ops. Check if summary should fail or continue.
+            const pendingMessagesFailResult = await this.shouldFailSummaryOnPendingOps(summaryNumberLogger, this.deltaManager.lastSequenceNumber, this.deltaManager.minimumSequenceNumber, finalAttempt, true /* beforeSummaryGeneration */);
+            if (pendingMessagesFailResult !== undefined) {
+                return pendingMessagesFailResult;
+            }
+        }
         const shouldPauseInboundSignal = this.mc.config.getBoolean("Fluid.ContainerRuntime.SubmitSummary.disableInboundSignalPause") !== true;
         let summaryRefSeqNum;
         try {
@@ -1759,7 +1794,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 // That said, we rely on submitSystemMessage() that today only works in connected state.
                 // So if we fail here, it either means that RunWhileConnectedCoordinator does not work correctly,
                 // OR that design changed and we need to remove this check and fix submitSystemMessage.
-                (0, common_utils_1.assert)(this.connected, 0x258 /* "connected" */);
+                (0, core_utils_1.assert)(this.connected, 0x258 /* "connected" */);
                 // Ensure that lastSequenceNumber has not changed after pausing.
                 // We need the summary op's reference sequence number to match our summary sequence number,
                 // otherwise we'll get the wrong sequence number stamped on the summary's .protocol attributes.
@@ -1769,7 +1804,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                         error: `lastSequenceNumber changed before uploading to storage. ${this.deltaManager.lastSequenceNumber} !== ${summaryRefSeqNum}`,
                     };
                 }
-                (0, common_utils_1.assert)(summaryRefSeqNum === this.deltaManager.lastMessage?.sequenceNumber, 0x395 /* it's one and the same thing */);
+                (0, core_utils_1.assert)(summaryRefSeqNum === this.deltaManager.lastMessage?.sequenceNumber, 0x395 /* it's one and the same thing */);
                 if (lastAck !== this.summaryCollection.latestAck) {
                     return {
                         continue: false,
@@ -1787,7 +1822,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                     error: continueResult.error,
                 };
             }
-            const trace = common_utils_1.Trace.start();
+            const trace = client_utils_1.Trace.start();
             let summarizeResult;
             // If the GC state needs to be reset, we need to force a full tree summary and update the unreferenced
             // state of all the nodes.
@@ -1822,33 +1857,9 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                         error,
                     };
                 }
-                // If there are pending messages, the summary has more data than at the summaryRefSeqNum.
-                if (this.hasPendingMessages()) {
-                    // If "SkipFailingIncorrectSummary" option is true, don't fail the summary in the last attempt.
-                    // This is a fallback to make progress in documents where there are consistently pending ops in
-                    // the summarizer.
-                    if (finalAttempt &&
-                        this.mc.config.getBoolean("Fluid.Summarizer.SkipFailingIncorrectSummary")) {
-                        const error = telemetry_utils_1.DataProcessingError.create("Pending ops during summarization", "submitSummary", undefined, { pendingMessages: this.pendingMessagesCount });
-                        summaryNumberLogger.sendErrorEvent({
-                            eventName: "SkipFailingIncorrectSummary",
-                            referenceSequenceNumber: summaryRefSeqNum,
-                            minimumSequenceNumber,
-                        }, error);
-                    }
-                    else {
-                        // Default retry delay is 1 second. This can be overridden via config so that we can adjust it
-                        // based on telemetry while we decide on a stable number.
-                        const retryDelayMs = this.mc.config.getNumber("Fluid.Summarizer.PendingOpsRetryDelayMs") ??
-                            1000;
-                        const error = new summary_1.RetriableSummaryError("PendingMessagesInSummary", retryDelayMs / 1000, { count: this.pendingMessagesCount });
-                        return {
-                            stage: "base",
-                            referenceSequenceNumber: summaryRefSeqNum,
-                            minimumSequenceNumber,
-                            error,
-                        };
-                    }
+                const pendingMessagesFailResult = await this.shouldFailSummaryOnPendingOps(summaryNumberLogger, summaryRefSeqNum, minimumSequenceNumber, finalAttempt, false /* beforeSummaryGeneration */);
+                if (pendingMessagesFailResult !== undefined) {
+                    return pendingMessagesFailResult;
                 }
             }
             const { summary: summaryTree, stats: partialStats } = summarizeResult;
@@ -1858,7 +1869,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             // Because handles are unchanged dataStores in the current logic,
             // summarized dataStore count is total dataStore count minus handle count
             const dataStoreTree = summaryTree.tree[runtime_definitions_1.channelsTreeName];
-            (0, common_utils_1.assert)(dataStoreTree.type === protocol_definitions_1.SummaryType.Tree, 0x1fc /* "summary is not a tree" */);
+            (0, core_utils_1.assert)(dataStoreTree.type === protocol_definitions_1.SummaryType.Tree, 0x1fc /* "summary is not a tree" */);
             const handleCount = Object.values(dataStoreTree.tree).filter((value) => value.type === protocol_definitions_1.SummaryType.Handle).length;
             const gcSummaryTreeStats = summaryTree.tree[runtime_definitions_1.gcTreeKey]
                 ? (0, runtime_utils_1.calculateStats)(summaryTree.tree[runtime_definitions_1.gcTreeKey])
@@ -1970,6 +1981,50 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             }
         }
     }
+    /**
+     * This helper is called during summarization. If there are pending ops, it will return a failed summarize result
+     * (IBaseSummarizeResult) unless this is the final summarize attempt and SkipFailingIncorrectSummary option is set.
+     * @param logger - The logger to be used for sending telemetry.
+     * @param referenceSequenceNumber - The reference sequence number of the summary attempt.
+     * @param minimumSequenceNumber - The minimum sequence number of the summary attempt.
+     * @param finalAttempt - Whether this is the final summary attempt.
+     * @param beforeSummaryGeneration - Whether this is called before summary generation or after.
+     * @returns failed summarize result (IBaseSummarizeResult) if summary should be failed, undefined otherwise.
+     */
+    async shouldFailSummaryOnPendingOps(logger, referenceSequenceNumber, minimumSequenceNumber, finalAttempt, beforeSummaryGeneration) {
+        if (!this.hasPendingMessages()) {
+            return;
+        }
+        // If "SkipFailingIncorrectSummary" option is true, don't fail the summary in the last attempt.
+        // This is a fallback to make progress in documents where there are consistently pending ops in
+        // the summarizer.
+        if (finalAttempt &&
+            this.mc.config.getBoolean("Fluid.Summarizer.SkipFailingIncorrectSummary")) {
+            const error = telemetry_utils_1.DataProcessingError.create("Pending ops during summarization", "submitSummary", undefined, { pendingMessages: this.pendingMessagesCount });
+            logger.sendErrorEvent({
+                eventName: "SkipFailingIncorrectSummary",
+                referenceSequenceNumber,
+                minimumSequenceNumber,
+                beforeGenerate: beforeSummaryGeneration,
+            }, error);
+        }
+        else {
+            // The retry delay when there are pending ops can be overridden via config so that we can adjust it
+            // based on telemetry while we decide on a stable number.
+            const retryDelayMs = this.mc.config.getNumber("Fluid.Summarizer.PendingOpsRetryDelayMs") ??
+                exports.defaultPendingOpsRetryDelayMs;
+            const error = new summary_1.RetriableSummaryError("PendingOpsWhileSummarizing", retryDelayMs / 1000, {
+                count: this.pendingMessagesCount,
+                beforeGenerate: beforeSummaryGeneration,
+            });
+            return {
+                stage: "base",
+                referenceSequenceNumber,
+                minimumSequenceNumber,
+                error,
+            };
+        }
+    }
     get pendingMessagesCount() {
         return this.pendingStateManager.pendingMessagesCount + this.outbox.messageCount;
     }
@@ -1978,11 +2033,11 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     }
     updateDocumentDirtyState(dirty) {
         if (this.attachState !== container_definitions_1.AttachState.Attached) {
-            (0, common_utils_1.assert)(dirty, 0x3d2 /* Non-attached container is dirty */);
+            (0, core_utils_1.assert)(dirty, 0x3d2 /* Non-attached container is dirty */);
         }
         else {
             // Other way is not true = see this.isContainerMessageDirtyable()
-            (0, common_utils_1.assert)(!dirty || this.hasPendingMessages(), 0x3d3 /* if doc is dirty, there has to be pending ops */);
+            (0, core_utils_1.assert)(!dirty || this.hasPendingMessages(), 0x3d3 /* if doc is dirty, there has to be pending ops */);
         }
         if (this.dirtyContainer === dirty) {
             return;
@@ -2015,7 +2070,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             let idAllocationBatchMessage;
             let idRange;
             if (this.idCompressorEnabled) {
-                (0, common_utils_1.assert)(this.idCompressor !== undefined, 0x67d /* IdCompressor should be defined if enabled */);
+                (0, core_utils_1.assert)(this.idCompressor !== undefined, 0x67d /* IdCompressor should be defined if enabled */);
                 idRange = this.idCompressor.takeNextCreationRange();
                 // Don't include the idRange if there weren't any Ids allocated
                 idRange = idRange?.ids !== undefined ? idRange : undefined;
@@ -2042,7 +2097,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         this.verifyNotClosed();
         this.verifyCanSubmitOps();
         // There should be no ops in detached container state!
-        (0, common_utils_1.assert)(this.attachState !== container_definitions_1.AttachState.Detached, 0x132 /* "sending ops in detached container" */);
+        (0, core_utils_1.assert)(this.attachState !== container_definitions_1.AttachState.Detached, 0x132 /* "sending ops in detached container" */);
         const serializedContent = JSON.stringify(containerRuntimeMessage);
         // Note that the real (non-proxy) delta manager is used here to get the readonly info. This is because
         // container runtime's ability to submit ops depend on the actual readonly state of the delta manager.
@@ -2142,15 +2197,15 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 setTimeout(flush, 0);
                 break;
             default:
-                (0, common_utils_1.assert)(this._orderSequentiallyCalls > 0, 0x587 /* Unreachable unless running under orderSequentially */);
+                (0, core_utils_1.assert)(this._orderSequentiallyCalls > 0, 0x587 /* Unreachable unless running under orderSequentially */);
                 break;
         }
     }
     submitSummaryMessage(contents, referenceSequenceNumber) {
         this.verifyNotClosed();
-        (0, common_utils_1.assert)(this.connected, 0x133 /* "Container disconnected when trying to submit system message" */);
+        (0, core_utils_1.assert)(this.connected, 0x133 /* "Container disconnected when trying to submit system message" */);
         // System message should not be sent in the middle of the batch.
-        (0, common_utils_1.assert)(this.outbox.isEmpty, 0x3d4 /* System op in the middle of a batch */);
+        (0, core_utils_1.assert)(this.outbox.isEmpty, 0x3d4 /* System op in the middle of a batch */);
         // back-compat: ADO #1385: Make this call unconditional in the future
         return this.submitSummaryFn !== undefined
             ? this.submitSummaryFn(contents, referenceSequenceNumber)
@@ -2288,29 +2343,22 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     /** Implementation of ISummarizerInternalsProvider.refreshLatestSummaryAck */
     async refreshLatestSummaryAck(options) {
         const { proposalHandle, ackHandle, summaryRefSeq, summaryLogger } = options;
+        // proposalHandle is always passed from RunningSummarizer.
+        (0, core_utils_1.assert)(proposalHandle !== undefined, 0x766 /* proposalHandle should be available */);
         const readAndParseBlob = async (id) => (0, driver_utils_1.readAndParse)(this.storage, id);
-        // The call to fetch the snapshot is very expensive and not always needed.
-        // It should only be done by the summarizerNode, if required.
-        // When fetching from storage we will always get the latest version and do not use the ackHandle.
-        const fetchLatestSnapshot = async () => {
-            let fetchResult = await this.fetchSnapshotFromStorageAndMaybeClose(summaryLogger, {
+        const result = await this.summarizerNode.refreshLatestSummary(proposalHandle, summaryRefSeq);
+        /**
+         * When refreshing a summary ack, this check indicates a new ack of a summary that is newer than the
+         * current summary that is tracked, but this summarizer runtime did not produce/track that summary. Thus
+         * it needs to refresh its state. Today refresh is done by fetching the latest snapshot to update the cache
+         * and then close as the current main client is likely to be re-elected as the parent summarizer again.
+         */
+        if (!result.isSummaryTracked && result.isSummaryNewer) {
+            const fetchResult = await this.fetchSnapshotFromStorage(summaryLogger, {
                 eventName: "RefreshLatestSummaryAckFetch",
                 ackHandle,
                 targetSequenceNumber: summaryRefSeq,
             }, readAndParseBlob, null);
-            /**
-             * back-compat - Older loaders and drivers (pre 2.0.0-internal.1.4) don't have fetchSource as a param in the
-             * getVersions API. So, they will not fetch the latest snapshot from network in the previous fetch call. For
-             * these scenarios, fetch the snapshot corresponding to the ack handle to have the same behavior before the
-             * change that started fetching latest snapshot always.
-             */
-            if (fetchResult.latestSnapshotRefSeq < summaryRefSeq) {
-                fetchResult = await this.fetchSnapshotFromStorageAndMaybeClose(summaryLogger, {
-                    eventName: "RefreshLatestSummaryAckFetchBackCompat",
-                    ackHandle,
-                    targetSequenceNumber: summaryRefSeq,
-                }, readAndParseBlob, ackHandle);
-            }
             /**
              * If the fetched snapshot is older than the one for which the ack was received, close the container.
              * This should never happen because an ack should be sent after the latest summary is updated in the server.
@@ -2330,17 +2378,11 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 this.disposeFn(error);
                 throw error;
             }
-            // In case we had to retrieve the latest snapshot and it is different than summaryRefSeq,
-            // wait for the delta manager to catch up before refreshing the latest Summary.
-            await this.waitForDeltaManagerToCatchup(fetchResult.latestSnapshotRefSeq, summaryLogger);
-            return {
-                snapshotTree: fetchResult.snapshotTree,
-                snapshotRefSeq: fetchResult.latestSnapshotRefSeq,
-            };
-        };
-        const result = await this.summarizerNode.refreshLatestSummary(proposalHandle, summaryRefSeq, fetchLatestSnapshot, readAndParseBlob, summaryLogger);
+            await this.closeStaleSummarizer("RefreshLatestSummaryAckFetch");
+            return;
+        }
         // Notify the garbage collector so it can update its latest summary state.
-        await this.garbageCollector.refreshLatestSummary(proposalHandle, result, readAndParseBlob);
+        await this.garbageCollector.refreshLatestSummary(result);
     }
     /**
      * Fetches the latest snapshot from storage and uses it to refresh SummarizerNode's
@@ -2350,32 +2392,38 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
      */
     async refreshLatestSummaryAckFromServer(summaryLogger) {
         const readAndParseBlob = async (id) => (0, driver_utils_1.readAndParse)(this.storage, id);
-        const { snapshotTree, versionId, latestSnapshotRefSeq } = await this.fetchSnapshotFromStorageAndMaybeClose(summaryLogger, {
+        const { versionId, latestSnapshotRefSeq } = await this.fetchSnapshotFromStorage(summaryLogger, {
             eventName: "RefreshLatestSummaryFromServerFetch",
         }, readAndParseBlob, null);
-        const fetchLatestSnapshot = {
-            snapshotTree,
-            snapshotRefSeq: latestSnapshotRefSeq,
-        };
-        const result = await this.summarizerNode.refreshLatestSummary(undefined /* proposalHandle */, latestSnapshotRefSeq, async () => fetchLatestSnapshot, readAndParseBlob, summaryLogger);
-        // Notify the garbage collector so it can update its latest summary state.
-        await this.garbageCollector.refreshLatestSummary(undefined /* proposalHandle */, result, readAndParseBlob);
+        await this.closeStaleSummarizer("RefreshLatestSummaryFromServerFetch");
         return { latestSnapshotRefSeq, latestSnapshotVersionId: versionId };
     }
+    async closeStaleSummarizer(codePath) {
+        this.mc.logger.sendTelemetryEvent({
+            eventName: "ClosingSummarizerOnSummaryStale",
+            codePath,
+            message: "Stopping fetch from storage",
+            closeSummarizerDelayMs: this.closeSummarizerDelayMs,
+        }, new telemetry_utils_1.GenericError("Restarting summarizer instead of refreshing"));
+        // Delay before restarting summarizer to prevent the summarizer from restarting too frequently.
+        await (0, core_utils_1.delay)(this.closeSummarizerDelayMs);
+        this._summarizer?.stop("latestSummaryStateStale");
+        this.disposeFn();
+    }
     /**
      * Downloads snapshot from storage with the given versionId or latest if versionId is null.
      * By default, it also closes the container after downloading the snapshot. However, this may be
      * overridden via options.
      */
-    async fetchSnapshotFromStorageAndMaybeClose(logger, event, readAndParseBlob, versionId) {
-        const snapshotResults = await telemetry_utils_1.PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
+    async fetchSnapshotFromStorage(logger, event, readAndParseBlob, versionId) {
+        return telemetry_utils_1.PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
             const stats = {};
-            const trace = common_utils_1.Trace.start();
+            const trace = client_utils_1.Trace.start();
             const versions = await this.storage.getVersions(versionId, 1, "refreshLatestSummaryAckFromServer", versionId === null ? driver_definitions_1.FetchSource.noCache : undefined);
-            (0, common_utils_1.assert)(!!versions && !!versions[0], 0x137 /* "Failed to get version from storage" */);
+            (0, core_utils_1.assert)(!!versions && !!versions[0], 0x137 /* "Failed to get version from storage" */);
             stats.getVersionDuration = trace.trace().duration;
             const maybeSnapshot = await this.storage.getSnapshotTree(versions[0]);
-            (0, common_utils_1.assert)(!!maybeSnapshot, 0x138 /* "Failed to get snapshot from storage" */);
+            (0, core_utils_1.assert)(!!maybeSnapshot, 0x138 /* "Failed to get snapshot from storage" */);
             stats.getSnapshotDuration = trace.trace().duration;
             const latestSnapshotRefSeq = await (0, runtime_utils_1.seqFromTree)(maybeSnapshot, readAndParseBlob);
             stats.snapshotRefSeq = latestSnapshotRefSeq;
@@ -2387,45 +2435,37 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 latestSnapshotRefSeq,
             };
         });
-        // We choose to close the summarizer after the snapshot cache is updated to avoid
-        // situations which the main client (which is likely to be re-elected as the leader again)
-        // loads the summarizer from cache.
-        if (this.summaryStateUpdateMethod !== "refreshFromSnapshot") {
-            this.mc.logger.sendTelemetryEvent({
-                ...event,
-                eventName: "ClosingSummarizerOnSummaryStale",
-                codePath: event.eventName,
-                message: "Stopping fetch from storage",
-                versionId: versionId != null ? versionId : undefined,
-                closeSummarizerDelayMs: this.closeSummarizerDelayMs,
-            }, new telemetry_utils_1.GenericError("Restarting summarizer instead of refreshing"));
-            // Delay before restarting summarizer to prevent the summarizer from restarting too frequently.
-            await (0, common_utils_1.delay)(this.closeSummarizerDelayMs);
-            this._summarizer?.stop("latestSummaryStateStale");
-            this.disposeFn();
-        }
-        return snapshotResults;
     }
     notifyAttaching() { } // do nothing (deprecated method)
     async getPendingLocalState(props) {
-        this.verifyNotClosed();
-        const waitBlobsToAttach = props?.notifyImminentClosure;
-        if (this._orderSequentiallyCalls !== 0) {
-            throw new telemetry_utils_1.UsageError("can't get state during orderSequentially");
-        }
-        const pendingAttachmentBlobs = await this.blobManager.getPendingBlobs(waitBlobsToAttach);
-        if (!pendingAttachmentBlobs && !this.hasPendingMessages()) {
-            return; // no pending state to save
-        }
-        // Flush pending batch.
-        // getPendingLocalState() is only exposed through Container.closeAndGetPendingLocalState(), so it's safe
-        // to close current batch.
-        this.flush();
-        const pendingState = {
-            pending: this.pendingStateManager.getLocalState(),
-            pendingAttachmentBlobs,
-        };
-        return pendingState;
+        return telemetry_utils_1.PerformanceEvent.timedExecAsync(this.mc.logger, {
+            eventName: "getPendingLocalState",
+            notifyImminentClosure: props?.notifyImminentClosure,
+        }, async (event) => {
+            this.verifyNotClosed();
+            const waitBlobsToAttach = props?.notifyImminentClosure;
+            if (this._orderSequentiallyCalls !== 0) {
+                throw new telemetry_utils_1.UsageError("can't get state during orderSequentially");
+            }
+            const pendingAttachmentBlobs = await this.blobManager.getPendingBlobs(waitBlobsToAttach);
+            const pending = this.pendingStateManager.getLocalState();
+            if (!pendingAttachmentBlobs && !this.hasPendingMessages()) {
+                return; // no pending state to save
+            }
+            // Flush pending batch.
+            // getPendingLocalState() is only exposed through Container.closeAndGetPendingLocalState(), so it's safe
+            // to close current batch.
+            this.flush();
+            const pendingState = {
+                pending,
+                pendingAttachmentBlobs,
+            };
+            event.end({
+                attachmentBlobsSize: Object.keys(pendingAttachmentBlobs ?? {}).length,
+                pendingOpsSize: pending?.pendingStates.length,
+            });
+            return pendingState;
+        });
     }
     summarizeOnDemand(options) {
         if (this.isSummarizerClient) {