@fluidframework/container-runtime 2.0.0-internal.6.1.1 → 2.0.0-internal.6.3.0

package/lib/containerRuntime.js

@@ -1,10 +1,11 @@
 import { AttachState, LoaderHeader, } from "@fluidframework/container-definitions";
-import { assert, delay, Trace, TypedEventEmitter, unreachableCase, } from "@fluidframework/common-utils";
-import { LazyPromise } from "@fluidframework/core-utils";
-import { createChildLogger, raiseConnectedEvent, PerformanceEvent, TaggedLoggerAdapter, wrapError, createChildMonitoringContext, } from "@fluidframework/telemetry-utils";
+import { assert, delay, LazyPromise } from "@fluidframework/core-utils";
+import { Trace, TypedEventEmitter } from "@fluid-internal/client-utils";
+import { createChildLogger, createChildMonitoringContext, DataCorruptionError, DataProcessingError, GenericError, raiseConnectedEvent, PerformanceEvent, 
+// eslint-disable-next-line import/no-deprecated
+TaggedLoggerAdapter, wrapError, UsageError, } from "@fluidframework/telemetry-utils";
 import { DriverHeader, FetchSource, } from "@fluidframework/driver-definitions";
 import { readAndParse } from "@fluidframework/driver-utils";
-import { DataCorruptionError, DataProcessingError, GenericError, UsageError, } from "@fluidframework/container-utils";
 import { MessageType, SummaryType, } from "@fluidframework/protocol-definitions";
 import { FlushMode, FlushModeExperimental, gcTreeKey, channelsTreeName, } from "@fluidframework/runtime-definitions";
 import { addBlobToSummary, addSummarizeResultToSummary, addTreeToSummary, RequestParser, create404Response, exceptionToResponse, GCDataBuilder, requestFluidObject, seqFromTree, calculateStats, TelemetryContext, } from "@fluidframework/runtime-utils";
@@ -45,6 +46,18 @@ export var ContainerMessageType;
      */
     ContainerMessageType["IdAllocation"] = "idAllocation";
 })(ContainerMessageType || (ContainerMessageType = {}));
+/**
+ * Utility to implement compat behaviors given an unknown message type
+ * The parameters are typed to support compile-time enforcement of handling all known types/behaviors
+ *
+ * @param _unknownContainerRuntimeMessageType - Typed as never, to ensure all known types have been
+ * handled before calling this function (e.g. in a switch statement).
+ * @param compatBehavior - Typed redundantly with CompatModeBehavior to ensure handling is added when updating that type
+ */
+function compatBehaviorAllowsMessageType(_unknownContainerRuntimeMessageType, compatBehavior) {
+    // undefined defaults to same behavior as "FailToProcess"
+    return compatBehavior === "Ignore";
+}
 export const DefaultSummaryConfiguration = {
     state: "enabled",
     minIdleTime: 0,
@@ -71,8 +84,12 @@ export var RuntimeHeaders;
 })(RuntimeHeaders || (RuntimeHeaders = {}));
 /** True if a tombstoned object should be returned without erroring */
 export const AllowTombstoneRequestHeaderKey = "allowTombstone"; // Belongs in the enum above, but avoiding the breaking change
+/** [IRRELEVANT IF throwOnInactiveLoad OPTION NOT SET] True if an inactive object should be returned without erroring */
+export const AllowInactiveRequestHeaderKey = "allowInactive"; // Belongs in the enum above, but avoiding the breaking change
 /** Tombstone error responses will have this header set to true */
 export const TombstoneResponseHeaderKey = "isTombstoned";
+/** Inactive error responses will have this header set to true */
+export const InactiveResponseHeaderKey = "isInactive";
 /** Default values for Runtime Headers */
 export const defaultRuntimeHeaderData = {
     wait: true,
@@ -99,6 +116,10 @@ const defaultCompressionConfig = {
     compressionAlgorithm: CompressionAlgorithms.lz4,
 };
 const defaultChunkSizeInBytes = 204800;
+/** The default time to wait for pending ops to be processed during summarization */
+export const defaultPendingOpsWaitTimeoutMs = 1000;
+/** The default time to delay a summarization retry attempt when there are pending ops */
+export const defaultPendingOpsRetryDelayMs = 1000;
 /**
  * Instead of refreshing from latest because we do not have 100% confidence in the state
  * of the current system, we should close the summarizer and let it recover.
@@ -200,34 +221,6 @@ export class ContainerRuntime extends TypedEventEmitter {
             signalTimestamp: 0,
             trackingSignalSequenceNumber: undefined,
         };
-        this.summarizeOnDemand = (...args) => {
-            if (this.isSummarizerClient) {
-                return this.summarizer.summarizeOnDemand(...args);
-            }
-            else if (this.summaryManager !== undefined) {
-                return this.summaryManager.summarizeOnDemand(...args);
-            }
-            else {
-                // If we're not the summarizer, and we don't have a summaryManager, we expect that
-                // disableSummaries is turned on. We are throwing instead of returning a failure here,
-                // because it is a misuse of the API rather than an expected failure.
-                throw new UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
-            }
-        };
-        this.enqueueSummarize = (...args) => {
-            if (this.isSummarizerClient) {
-                return this.summarizer.enqueueSummarize(...args);
-            }
-            else if (this.summaryManager !== undefined) {
-                return this.summaryManager.enqueueSummarize(...args);
-            }
-            else {
-                // If we're not the summarizer, and we don't have a summaryManager, we expect that
-                // generateSummaries is turned off. We are throwing instead of returning a failure here,
-                // because it is a misuse of the API rather than an expected failure.
-                throw new UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
-            }
-        };
         const { options, clientDetails, connected, baseSnapshot, submitFn, submitBatchFn, submitSummaryFn, submitSignalFn, disposeFn, closeFn, deltaManager, quorum, audience, loader, pendingLocalState, supportedFeatures, } = context;
         this.innerDeltaManager = deltaManager;
         this.deltaManager = new DeltaManagerSummarizerProxy(this.innerDeltaManager);
@@ -442,16 +435,13 @@ export class ContainerRuntime extends TypedEventEmitter {
         });
         // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
         this._audience = audience;
-        this.summaryStateUpdateMethod = this.mc.config.getString("Fluid.ContainerRuntime.Test.SummaryStateUpdateMethodV2");
         const closeSummarizerDelayOverride = this.mc.config.getNumber("Fluid.ContainerRuntime.Test.CloseSummarizerDelayOverrideMs");
         this.closeSummarizerDelayMs = closeSummarizerDelayOverride ?? defaultCloseSummarizerDelayMs;
         this.validateSummaryBeforeUpload =
-            this.mc.config.getBoolean("Fluid.ContainerRuntime.Test.ValidateSummaryBeforeUpload") ??
-                false;
+            this.mc.config.getBoolean("Fluid.Summarizer.ValidateSummaryBeforeUpload") ?? false;
         this.summaryCollection = new SummaryCollection(this.deltaManager, this.logger);
         this.dirtyContainer =
-            this.attachState !== AttachState.Attached ||
-                this.pendingStateManager.hasPendingMessages();
+            this.attachState !== AttachState.Attached || this.hasPendingMessages();
         context.updateDirtyContainerState(this.dirtyContainer);
         if (this.summariesDisabled) {
             this.mc.logger.sendTelemetryEvent({ eventName: "SummariesDisabled" });
@@ -499,6 +489,9 @@ export class ContainerRuntime extends TypedEventEmitter {
                 formExponentialFn({ coefficient: 20, initialDelay: 0 })), {
                     initialDelayMs: this.initialSummarizerDelayMs,
                 }, this.heuristicsDisabled);
+                this.summaryManager.on("summarize", (eventProps) => {
+                    this.emit("summarize", eventProps);
+                });
                 this.summaryManager.start();
             }
         }
@@ -543,7 +536,6 @@ export class ContainerRuntime extends TypedEventEmitter {
                 disableAttachReorder: this.disableAttachReorder,
                 disablePartialFlush,
                 idCompressorEnabled: this.idCompressorEnabled,
-                summaryStateUpdateMethod: this.summaryStateUpdateMethod,
                 closeSummarizerDelayOverride,
             }),
             telemetryDocumentId: this.telemetryDocumentId,
@@ -607,11 +599,21 @@ export class ContainerRuntime extends TypedEventEmitter {
      * This object should provide all the functionality that the Container is expected to provide to the loader layer.
      */
     static async loadRuntime(params) {
-        const { context, registryEntries, existing, requestHandler, runtimeOptions = {}, containerScope = {}, containerRuntimeCtor = ContainerRuntime, initializeEntryPoint, } = params;
+        const { context, registryEntries, existing, requestHandler, runtimeOptions = {}, containerScope = {}, containerRuntimeCtor = ContainerRuntime, } = params;
+        const initializeEntryPoint = params.initializeEntryPoint ??
+            (async (containerRuntime) => ({
+                get IFluidRouter() {
+                    return this;
+                },
+                async request(req) {
+                    return containerRuntime.request(req);
+                },
+            }));
         // If taggedLogger exists, use it. Otherwise, wrap the vanilla logger:
         // back-compat: Remove the TaggedLoggerAdapter fallback once all the host are using loader > 0.45
         const backCompatContext = context;
         const passLogger = backCompatContext.taggedLogger ??
+            // eslint-disable-next-line import/no-deprecated
             new TaggedLoggerAdapter(backCompatContext.logger);
         const logger = createChildLogger({
             logger: passLogger,
@@ -673,7 +675,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             idCompressor =
                 serializedIdCompressor !== undefined
                     ? IdCompressor.deserialize(serializedIdCompressor, createSessionId())
-                    : new IdCompressor(createSessionId(), logger);
+                    : IdCompressor.create(logger);
         }
         const runtime = new containerRuntimeCtor(context, registry, metadata, electedSummarizerData, chunks ?? [], aliases ?? [], {
             summaryOptions,
@@ -688,6 +690,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             enableGroupedBatching,
         }, containerScope, logger, existing, blobManagerSnapshot, context.storage, idCompressor, requestHandler, undefined, // summaryConfiguration
         initializeEntryPoint);
+        await runtime.blobManager.processStashedChanges();
         // It's possible to have ops with a reference sequence number of 0. Op sequence numbers start
         // at 1, so we won't see a replayed saved op with a sequence number of 0.
         await runtime.pendingStateManager.applyStashedOpsAt(0);
@@ -704,6 +707,7 @@ export class ContainerRuntime extends TypedEventEmitter {
     /** @deprecated - The functionality is no longer exposed publicly */
     get reSubmitFn() {
         return (type, contents, localOpMetadata, opMetadata) => this.reSubmitCore({ type, contents }, localOpMetadata, opMetadata);
+        // Note: compatDetails is not included in this deprecated API
     }
     get flushMode() {
         return this._flushMode;
@@ -854,7 +858,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 // We always expect createSubRequest to include a leading slash, but asserting here to protect against
                 // unintentionally modifying the url if that changes.
                 assert(subRequest.url.startsWith("/"), 0x126 /* "Expected createSubRequest url to include a leading slash" */);
-                return dataStore.IFluidRouter.request(subRequest);
+                return dataStore.request(subRequest);
             }
             return create404Response(request);
         }
@@ -962,7 +966,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             this.mc.logger.sendTelemetryEvent({
                 eventName: "ReconnectsWithNoProgress",
                 attempts: this.consecutiveReconnects,
-                pendingMessages: this.pendingStateManager.pendingMessagesCount,
+                pendingMessages: this.pendingMessagesCount,
             });
         }
         return this.consecutiveReconnects < this.maxConsecutiveReconnects;
@@ -1022,13 +1026,13 @@ export class ContainerRuntime extends TypedEventEmitter {
      */
     parseOpContent(serializedContent) {
         assert(serializedContent !== undefined, 0x6d5 /* content must be defined */);
-        const { type, contents } = JSON.parse(serializedContent);
+        const { type, contents, compatDetails } = JSON.parse(serializedContent);
         assert(type !== undefined, 0x6d6 /* incorrect op content format */);
-        return { type, contents };
+        return { type, contents, compatDetails };
     }
     async applyStashedOp(op) {
         // Need to parse from string for back-compat
-        const { type, contents } = this.parseOpContent(op);
+        const { type, contents, compatDetails } = this.parseOpContent(op);
         switch (type) {
             case ContainerMessageType.FluidDataStoreOp:
                 return this.dataStores.applyStashedOp(contents);
@@ -1044,8 +1048,22 @@ export class ContainerRuntime extends TypedEventEmitter {
                 throw new Error("chunkedOp not expected here");
             case ContainerMessageType.Rejoin:
                 throw new Error("rejoin not expected here");
-            default:
-                unreachableCase(type, `Unknown ContainerMessageType: ${type}`);
+            default: {
+                // This should be extremely rare for stashed ops.
+                // It would require a newer runtime stashing ops and then an older one applying them,
+                // e.g. if an app rolled back its container version
+                const compatBehavior = compatDetails?.behavior;
+                if (!compatBehaviorAllowsMessageType(type, compatBehavior)) {
+                    const error = DataProcessingError.create("Stashed runtime message of unknown type", "applyStashedOp", undefined /* sequencedMessage */, {
+                        messageDetails: JSON.stringify({
+                            type,
+                            compatBehavior,
+                        }),
+                    });
+                    this.closeFn(error);
+                    throw error;
+                }
+            }
         }
     }
     setConnectionState(connected, clientId) {
@@ -1057,24 +1075,6 @@ export class ContainerRuntime extends TypedEventEmitter {
             // Don't propagate "disconnected" event because we didn't propagate the previous "connected" event
             return;
         }
-        // If attachment blobs were added while disconnected, we need to delay
-        // propagation of the "connected" event until we have uploaded them to
-        // ensure we don't submit ops referencing a blob that has not been uploaded
-        // Note that the inner (non-proxy) delta manager is needed here to get the readonly information.
-        const connecting = connected && !this._connected && !this.innerDeltaManager.readOnlyInfo.readonly;
-        if (connecting && this.blobManager.hasPendingOfflineUploads) {
-            assert(!this.delayConnectClientId, 0x392 /* Connect event delay must be canceled before subsequent connect event */);
-            assert(!!clientId, 0x393 /* Must have clientId when connecting */);
-            this.delayConnectClientId = clientId;
-            this.blobManager.onConnected().then(() => {
-                // make sure we didn't reconnect before the promise resolved
-                if (this.delayConnectClientId === clientId && !this.disposed) {
-                    this.delayConnectClientId = undefined;
-                    this.setConnectionStateCore(connected, clientId);
-                }
-            }, (error) => this.closeFn(error));
-            return;
-        }
         this.setConnectionStateCore(connected, clientId);
     }
     setConnectionStateCore(connected, clientId) {
@@ -1105,7 +1105,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 this.closeFn(DataProcessingError.create("Runtime detected too many reconnects with no progress syncing local ops.", "setConnectionState", undefined, {
                     dataLoss: 1,
                     attempts: this.consecutiveReconnects,
-                    pendingMessages: this.pendingStateManager.pendingMessagesCount,
+                    pendingMessages: this.pendingMessagesCount,
                 }));
                 return;
             }
@@ -1122,23 +1122,23 @@ export class ContainerRuntime extends TypedEventEmitter {
     }
     process(messageArg, local) {
         this.verifyNotClosed();
-        // Whether or not the message is actually a runtime message.
+        // Whether or not the message appears to be a runtime message from an up-to-date client.
         // It may be a legacy runtime message (ie already unpacked and ContainerMessageType)
         // or something different, like a system message.
-        const runtimeMessage = messageArg.type === MessageType.Operation;
+        const modernRuntimeMessage = messageArg.type === MessageType.Operation;
         // Do shallow copy of message, as the processing flow will modify it.
         const messageCopy = { ...messageArg };
         for (const message of this.remoteMessageProcessor.process(messageCopy)) {
-            this.processCore(message, local, runtimeMessage);
+            this.processCore(message, local, modernRuntimeMessage);
         }
     }
     /**
      * Direct the message to the correct subsystem for processing, and implement other side effects
      * @param message - The unpacked message. Likely a ContainerRuntimeMessage, but could also be a system op
      * @param local - Did this client send the op?
-     * @param runtimeMessage - Does this appear like a current ContainerRuntimeMessage?  If true, certain validation will occur.
+     * @param modernRuntimeMessage - Does this appear like a current ContainerRuntimeMessage?
      */
-    processCore(message, local, runtimeMessage) {
+    processCore(message, local, modernRuntimeMessage) {
         // Surround the actual processing of the operation with messages to the schedule manager indicating
         // the beginning and end. This allows it to emit appropriate events and/or pause the processing of new
         // messages once a batch has been fully processed.
@@ -1146,7 +1146,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         this._processedClientSequenceNumber = message.clientSequenceNumber;
         try {
             let localOpMetadata;
-            if (local && runtimeMessage && message.type !== ContainerMessageType.ChunkedOp) {
+            if (local && modernRuntimeMessage && message.type !== ContainerMessageType.ChunkedOp) {
                 localOpMetadata = this.pendingStateManager.processPendingLocalMessage(message);
             }
             // If there are no more pending messages after processing a local message,
@@ -1154,43 +1154,8 @@ export class ContainerRuntime extends TypedEventEmitter {
             if (!this.hasPendingMessages()) {
                 this.updateDocumentDirtyState(false);
             }
-            const type = message.type;
-            switch (type) {
-                case ContainerMessageType.Attach:
-                    this.dataStores.processAttachMessage(message, local);
-                    break;
-                case ContainerMessageType.Alias:
-                    this.processAliasMessage(message, localOpMetadata, local);
-                    break;
-                case ContainerMessageType.FluidDataStoreOp:
-                    this.dataStores.processFluidDataStoreOp(message, local, localOpMetadata);
-                    break;
-                case ContainerMessageType.BlobAttach:
-                    this.blobManager.processBlobAttachOp(message, local);
-                    break;
-                case ContainerMessageType.IdAllocation:
-                    assert(this.idCompressor !== undefined, 0x67c /* IdCompressor should be defined if enabled */);
-                    this.idCompressor.finalizeCreationRange(message.contents);
-                    break;
-                case ContainerMessageType.ChunkedOp:
-                case ContainerMessageType.Rejoin:
-                    break;
-                default:
-                    if (runtimeMessage) {
-                        const error = DataProcessingError.create(
-                        // Former assert 0x3ce
-                        "Runtime message of unknown type", "OpProcessing", message, {
-                            local,
-                            type: message.type,
-                            contentType: typeof message.contents,
-                            batch: message.metadata?.batch,
-                            compression: message.compression,
-                        });
-                        this.closeFn(error);
-                        throw error;
-                    }
-            }
-            this.emit("op", message, runtimeMessage);
+            this.validateAndProcessRuntimeMessage(message, localOpMetadata, local, modernRuntimeMessage);
+            this.emit("op", message, modernRuntimeMessage);
             this.scheduleManager.afterOpProcessing(undefined, message);
             if (local) {
                 // If we have processed a local op, this means that the container is
@@ -1204,8 +1169,59 @@ export class ContainerRuntime extends TypedEventEmitter {
             throw e;
         }
     }
-    processAliasMessage(message, localOpMetadata, local) {
-        this.dataStores.processAliasMessage(message, localOpMetadata, local);
+    /**
+     * Assuming the given message is also a ContainerRuntimeMessage,
+     * checks its type and dispatches the message to the appropriate handler in the runtime.
+     * Throws a DataProcessingError if the message doesn't conform to the ContainerRuntimeMessage type.
+     */
+    validateAndProcessRuntimeMessage(message, localOpMetadata, local, expectRuntimeMessageType) {
+        // Optimistically extract ContainerRuntimeMessage-specific props from the message
+        const { type: maybeContainerMessageType, compatDetails } = message;
+        switch (maybeContainerMessageType) {
+            case ContainerMessageType.Attach:
+                this.dataStores.processAttachMessage(message, local);
+                break;
+            case ContainerMessageType.Alias:
+                this.dataStores.processAliasMessage(message, localOpMetadata, local);
+                break;
+            case ContainerMessageType.FluidDataStoreOp:
+                this.dataStores.processFluidDataStoreOp(message, local, localOpMetadata);
+                break;
+            case ContainerMessageType.BlobAttach:
+                this.blobManager.processBlobAttachOp(message, local);
+                break;
+            case ContainerMessageType.IdAllocation:
+                assert(this.idCompressor !== undefined, 0x67c /* IdCompressor should be defined if enabled */);
+                this.idCompressor.finalizeCreationRange(message.contents);
+                break;
+            case ContainerMessageType.ChunkedOp:
+            case ContainerMessageType.Rejoin:
+                break;
+            default: {
+                // If we didn't necessarily expect a runtime message type, then no worries - just return
+                // e.g. this case applies to system ops, or legacy ops that would have fallen into the above cases anyway.
+                if (!expectRuntimeMessageType) {
+                    return;
+                }
+                const compatBehavior = compatDetails?.behavior;
+                if (!compatBehaviorAllowsMessageType(maybeContainerMessageType, compatBehavior)) {
+                    const error = DataProcessingError.create(
+                    // Former assert 0x3ce
+                    "Runtime message of unknown type", "OpProcessing", message, {
+                        local,
+                        messageDetails: JSON.stringify({
+                            type: message.type,
+                            contentType: typeof message.contents,
+                            compatBehavior,
+                            batch: message.metadata?.batch,
+                            compression: message.compression,
+                        }),
+                    });
+                    this.closeFn(error);
+                    throw error;
+                }
+            }
+        }
     }
     /**
      * Emits the Signal event and update the perf signal data.
@@ -1673,7 +1689,7 @@ export class ContainerRuntime extends TypedEventEmitter {
      * @param options - options controlling how the summary is generated or submitted
      */
     async submitSummary(options) {
-        const { fullTree = false, refreshLatestAck, summaryLogger } = options;
+        const { fullTree = false, finalAttempt = false, refreshLatestAck, summaryLogger } = options;
         // The summary number for this summary. This will be updated during the summary process, so get it now and
         // use it for all events logged during this summary.
         const summaryNumber = this.nextSummaryNumber;
@@ -1695,6 +1711,40 @@ export class ContainerRuntime extends TypedEventEmitter {
             // We might need to catch up to the latest summary's reference sequence number before pausing.
             await this.waitForDeltaManagerToCatchup(latestSnapshotRefSeq, summaryNumberLogger);
         }
+        // If there are pending (unacked ops), the summary will not be eventual consistent and it may even be
+        // incorrect. So, wait for the container to be saved with a timeout. If the container is not saved
+        // within the timeout, check if it should be failed or can continue.
+        if (this.validateSummaryBeforeUpload && this.hasPendingMessages()) {
+            const countBefore = this.pendingMessagesCount;
+            // The timeout for waiting for pending ops can be overridden via configurations.
+            const pendingOpsTimeout = this.mc.config.getNumber("Fluid.Summarizer.waitForPendingOpsTimeoutMs") ??
+                defaultPendingOpsWaitTimeoutMs;
+            await new Promise((resolve, reject) => {
+                const timeoutId = setTimeout(() => resolve(), pendingOpsTimeout);
+                this.once("saved", () => {
+                    clearTimeout(timeoutId);
+                    resolve();
+                });
+                this.once("dispose", () => {
+                    clearTimeout(timeoutId);
+                    reject(new Error("Runtime is disposed while summarizing"));
+                });
+            });
+            // Log that there are pending ops while summarizing. This will help us gather data on how often this
+            // happens, whether we attempted to wait for these ops to be acked and what was the result.
+            summaryNumberLogger.sendTelemetryEvent({
+                eventName: "PendingOpsWhileSummarizing",
+                saved: this.hasPendingMessages() ? false : true,
+                timeout: pendingOpsTimeout,
+                countBefore,
+                countAfter: this.pendingMessagesCount,
+            });
+            // There could still be pending ops. Check if summary should fail or continue.
+            const pendingMessagesFailResult = await this.shouldFailSummaryOnPendingOps(summaryNumberLogger, this.deltaManager.lastSequenceNumber, this.deltaManager.minimumSequenceNumber, finalAttempt, true /* beforeSummaryGeneration */);
+            if (pendingMessagesFailResult !== undefined) {
+                return pendingMessagesFailResult;
+            }
+        }
         const shouldPauseInboundSignal = this.mc.config.getBoolean("Fluid.ContainerRuntime.SubmitSummary.disableInboundSignalPause") !== true;
         let summaryRefSeqNum;
         try {
@@ -1770,9 +1820,9 @@ export class ContainerRuntime extends TypedEventEmitter {
                     error,
                 };
             }
-            // If validateSummaryBeforeUpload is true, validate that the summary generated by the summarizer nodes is
-            // correct before this summary is uploaded.
+            // If validateSummaryBeforeUpload is true, validate that the summary generated is correct before uploading.
             if (this.validateSummaryBeforeUpload) {
+                // Validate that the summaries generated by summarize nodes is correct.
                 const validateResult = this.summarizerNode.validateSummary();
                 if (!validateResult.success) {
                     const { success, ...loggingProps } = validateResult;
@@ -1784,6 +1834,10 @@ export class ContainerRuntime extends TypedEventEmitter {
                         error,
                     };
                 }
+                const pendingMessagesFailResult = await this.shouldFailSummaryOnPendingOps(summaryNumberLogger, summaryRefSeqNum, minimumSequenceNumber, finalAttempt, false /* beforeSummaryGeneration */);
+                if (pendingMessagesFailResult !== undefined) {
+                    return pendingMessagesFailResult;
+                }
             }
             const { summary: summaryTree, stats: partialStats } = summarizeResult;
             // Now that we have generated the summary, update the message at last summary to the last message processed.
@@ -1904,8 +1958,55 @@ export class ContainerRuntime extends TypedEventEmitter {
             }
         }
     }
+    /**
+     * This helper is called during summarization. If there are pending ops, it will return a failed summarize result
+     * (IBaseSummarizeResult) unless this is the final summarize attempt and SkipFailingIncorrectSummary option is set.
+     * @param logger - The logger to be used for sending telemetry.
+     * @param referenceSequenceNumber - The reference sequence number of the summary attempt.
+     * @param minimumSequenceNumber - The minimum sequence number of the summary attempt.
+     * @param finalAttempt - Whether this is the final summary attempt.
+     * @param beforeSummaryGeneration - Whether this is called before summary generation or after.
+     * @returns failed summarize result (IBaseSummarizeResult) if summary should be failed, undefined otherwise.
+     */
+    async shouldFailSummaryOnPendingOps(logger, referenceSequenceNumber, minimumSequenceNumber, finalAttempt, beforeSummaryGeneration) {
+        if (!this.hasPendingMessages()) {
+            return;
+        }
+        // If "SkipFailingIncorrectSummary" option is true, don't fail the summary in the last attempt.
+        // This is a fallback to make progress in documents where there are consistently pending ops in
+        // the summarizer.
+        if (finalAttempt &&
+            this.mc.config.getBoolean("Fluid.Summarizer.SkipFailingIncorrectSummary")) {
+            const error = DataProcessingError.create("Pending ops during summarization", "submitSummary", undefined, { pendingMessages: this.pendingMessagesCount });
+            logger.sendErrorEvent({
+                eventName: "SkipFailingIncorrectSummary",
+                referenceSequenceNumber,
+                minimumSequenceNumber,
+                beforeGenerate: beforeSummaryGeneration,
+            }, error);
+        }
+        else {
+            // The retry delay when there are pending ops can be overridden via config so that we can adjust it
+            // based on telemetry while we decide on a stable number.
+            const retryDelayMs = this.mc.config.getNumber("Fluid.Summarizer.PendingOpsRetryDelayMs") ??
+                defaultPendingOpsRetryDelayMs;
+            const error = new RetriableSummaryError("PendingOpsWhileSummarizing", retryDelayMs / 1000, {
+                count: this.pendingMessagesCount,
+                beforeGenerate: beforeSummaryGeneration,
+            });
+            return {
+                stage: "base",
+                referenceSequenceNumber,
+                minimumSequenceNumber,
+                error,
+            };
+        }
+    }
+    get pendingMessagesCount() {
+        return this.pendingStateManager.pendingMessagesCount + this.outbox.messageCount;
+    }
     hasPendingMessages() {
-        return this.pendingStateManager.hasPendingMessages() || !this.outbox.isEmpty;
+        return this.pendingMessagesCount !== 0;
     }
     updateDocumentDirtyState(dirty) {
         if (this.attachState !== AttachState.Attached) {
@@ -1949,7 +2050,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 assert(this.idCompressor !== undefined, 0x67d /* IdCompressor should be defined if enabled */);
                 idRange = this.idCompressor.takeNextCreationRange();
                 // Don't include the idRange if there weren't any Ids allocated
-                idRange = idRange?.ids?.first !== undefined ? idRange : undefined;
+                idRange = idRange?.ids !== undefined ? idRange : undefined;
             }
             if (idRange !== undefined) {
                 const idAllocationMessage = {
@@ -2168,8 +2269,27 @@ export class ContainerRuntime extends TypedEventEmitter {
             case ContainerMessageType.Rejoin:
                 this.submit(message);
                 break;
-            default:
-                unreachableCase(message.type, `Unknown ContainerMessageType [type: ${message.type}]`);
+            default: {
+                // This case should be very rare - it would imply an op was stashed from a
+                // future version of runtime code and now is being applied on an older version
+                const compatBehavior = message.compatDetails?.behavior;
+                if (compatBehaviorAllowsMessageType(message.type, compatBehavior)) {
+                    this.logger.sendTelemetryEvent({
+                        eventName: "resubmitUnrecognizedMessageTypeAllowed",
+                        messageDetails: { type: message.type, compatBehavior },
+                    });
+                }
+                else {
+                    const error = DataProcessingError.create("Resubmitting runtime message of unknown type", "reSubmitCore", undefined /* sequencedMessage */, {
+                        messageDetails: JSON.stringify({
+                            type: message.type,
+                            compatBehavior,
+                        }),
+                    });
+                    this.closeFn(error);
+                    throw error;
+                }
+            }
         }
     }
     rollback(content, localOpMetadata) {
@@ -2182,6 +2302,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 this.dataStores.rollbackDataStoreOp(contents, localOpMetadata);
                 break;
             default:
+                // Don't check message.compatDetails because this is for rolling back a local op so the type will be known
                 throw new Error(`Can't rollback ${type}`);
         }
     }
@@ -2199,29 +2320,22 @@ export class ContainerRuntime extends TypedEventEmitter {
     /** Implementation of ISummarizerInternalsProvider.refreshLatestSummaryAck */
     async refreshLatestSummaryAck(options) {
         const { proposalHandle, ackHandle, summaryRefSeq, summaryLogger } = options;
+        // proposalHandle is always passed from RunningSummarizer.
+        assert(proposalHandle !== undefined, 0x766 /* proposalHandle should be available */);
         const readAndParseBlob = async (id) => readAndParse(this.storage, id);
-        // The call to fetch the snapshot is very expensive and not always needed.
-        // It should only be done by the summarizerNode, if required.
-        // When fetching from storage we will always get the latest version and do not use the ackHandle.
-        const fetchLatestSnapshot = async () => {
-            let fetchResult = await this.fetchLatestSnapshotFromStorage(summaryLogger, {
+        const result = await this.summarizerNode.refreshLatestSummary(proposalHandle, summaryRefSeq);
+        /**
+         * When refreshing a summary ack, this check indicates a new ack of a summary that is newer than the
+         * current summary that is tracked, but this summarizer runtime did not produce/track that summary. Thus
+         * it needs to refresh its state. Today refresh is done by fetching the latest snapshot to update the cache
+         * and then close as the current main client is likely to be re-elected as the parent summarizer again.
+         */
+        if (!result.isSummaryTracked && result.isSummaryNewer) {
+            const fetchResult = await this.fetchSnapshotFromStorage(summaryLogger, {
                 eventName: "RefreshLatestSummaryAckFetch",
                 ackHandle,
                 targetSequenceNumber: summaryRefSeq,
-            }, readAndParseBlob);
-            /**
-             * back-compat - Older loaders and drivers (pre 2.0.0-internal.1.4) don't have fetchSource as a param in the
-             * getVersions API. So, they will not fetch the latest snapshot from network in the previous fetch call. For
-             * these scenarios, fetch the snapshot corresponding to the ack handle to have the same behavior before the
-             * change that started fetching latest snapshot always.
-             */
-            if (fetchResult.latestSnapshotRefSeq < summaryRefSeq) {
-                fetchResult = await this.fetchSnapshotFromStorageAndClose(summaryLogger, {
-                    eventName: "RefreshLatestSummaryAckFetchBackCompat",
-                    ackHandle,
-                    targetSequenceNumber: summaryRefSeq,
-                }, readAndParseBlob, ackHandle);
-            }
+            }, readAndParseBlob, null);
             /**
              * If the fetched snapshot is older than the one for which the ack was received, close the container.
              * This should never happen because an ack should be sent after the latest summary is updated in the server.
@@ -2241,17 +2355,11 @@ export class ContainerRuntime extends TypedEventEmitter {
                 this.disposeFn(error);
                 throw error;
             }
-            // In case we had to retrieve the latest snapshot and it is different than summaryRefSeq,
-            // wait for the delta manager to catch up before refreshing the latest Summary.
-            await this.waitForDeltaManagerToCatchup(fetchResult.latestSnapshotRefSeq, summaryLogger);
-            return {
-                snapshotTree: fetchResult.snapshotTree,
-                snapshotRefSeq: fetchResult.latestSnapshotRefSeq,
-            };
-        };
-        const result = await this.summarizerNode.refreshLatestSummary(proposalHandle, summaryRefSeq, fetchLatestSnapshot, readAndParseBlob, summaryLogger);
+            await this.closeStaleSummarizer("RefreshLatestSummaryAckFetch");
+            return;
+        }
         // Notify the garbage collector so it can update its latest summary state.
-        await this.garbageCollector.refreshLatestSummary(proposalHandle, result, readAndParseBlob);
+        await this.garbageCollector.refreshLatestSummary(result);
     }
     /**
      * Fetches the latest snapshot from storage and uses it to refresh SummarizerNode's
@@ -2261,23 +2369,31 @@ export class ContainerRuntime extends TypedEventEmitter {
      */
     async refreshLatestSummaryAckFromServer(summaryLogger) {
         const readAndParseBlob = async (id) => readAndParse(this.storage, id);
-        const { snapshotTree, versionId, latestSnapshotRefSeq } = await this.fetchLatestSnapshotFromStorage(summaryLogger, {
+        const { versionId, latestSnapshotRefSeq } = await this.fetchSnapshotFromStorage(summaryLogger, {
             eventName: "RefreshLatestSummaryFromServerFetch",
-        }, readAndParseBlob);
-        const fetchLatestSnapshot = {
-            snapshotTree,
-            snapshotRefSeq: latestSnapshotRefSeq,
-        };
-        const result = await this.summarizerNode.refreshLatestSummary(undefined /* proposalHandle */, latestSnapshotRefSeq, async () => fetchLatestSnapshot, readAndParseBlob, summaryLogger);
-        // Notify the garbage collector so it can update its latest summary state.
-        await this.garbageCollector.refreshLatestSummary(undefined /* proposalHandle */, result, readAndParseBlob);
+        }, readAndParseBlob, null);
+        await this.closeStaleSummarizer("RefreshLatestSummaryFromServerFetch");
         return { latestSnapshotRefSeq, latestSnapshotVersionId: versionId };
     }
-    async fetchLatestSnapshotFromStorage(logger, event, readAndParseBlob) {
-        return this.fetchSnapshotFromStorageAndClose(logger, event, readAndParseBlob, null /* latest */);
+    async closeStaleSummarizer(codePath) {
+        this.mc.logger.sendTelemetryEvent({
+            eventName: "ClosingSummarizerOnSummaryStale",
+            codePath,
+            message: "Stopping fetch from storage",
+            closeSummarizerDelayMs: this.closeSummarizerDelayMs,
+        }, new GenericError("Restarting summarizer instead of refreshing"));
+        // Delay before restarting summarizer to prevent the summarizer from restarting too frequently.
+        await delay(this.closeSummarizerDelayMs);
+        this._summarizer?.stop("latestSummaryStateStale");
+        this.disposeFn();
     }
-    async fetchSnapshotFromStorageAndClose(logger, event, readAndParseBlob, versionId) {
-        const snapshotResults = await PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
+    /**
+     * Downloads snapshot from storage with the given versionId or latest if versionId is null.
+     * By default, it also closes the container after downloading the snapshot. However, this may be
+     * overridden via options.
+     */
+    async fetchSnapshotFromStorage(logger, event, readAndParseBlob, versionId) {
+        return PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
             const stats = {};
             const trace = Trace.start();
             const versions = await this.storage.getVersions(versionId, 1, "refreshLatestSummaryAckFromServer", versionId === null ? FetchSource.noCache : undefined);
@@ -2296,41 +2412,65 @@ export class ContainerRuntime extends TypedEventEmitter {
                 latestSnapshotRefSeq,
             };
         });
-        // We choose to close the summarizer after the snapshot cache is updated to avoid
-        // situations which the main client (which is likely to be re-elected as the leader again)
-        // loads the summarizer from cache.
-        if (this.summaryStateUpdateMethod !== "refreshFromSnapshot") {
-            this.mc.logger.sendTelemetryEvent({
-                ...event,
-                eventName: "ClosingSummarizerOnSummaryStale",
-                codePath: event.eventName,
-                message: "Stopping fetch from storage",
-                versionId: versionId != null ? versionId : undefined,
-                closeSummarizerDelayMs: this.closeSummarizerDelayMs,
-            }, new GenericError("Restarting summarizer instead of refreshing"));
-            // Delay before restarting summarizer to prevent the summarizer from restarting too frequently.
-            await delay(this.closeSummarizerDelayMs);
-            this._summarizer?.stop("latestSummaryStateStale");
-            this.disposeFn();
-        }
-        return snapshotResults;
     }
     notifyAttaching() { } // do nothing (deprecated method)
     async getPendingLocalState(props) {
-        this.verifyNotClosed();
-        const waitBlobsToAttach = props?.notifyImminentClosure;
-        if (this._orderSequentiallyCalls !== 0) {
-            throw new UsageError("can't get state during orderSequentially");
-        }
-        const pendingAttachmentBlobs = await this.blobManager.getPendingBlobs(waitBlobsToAttach);
-        // Flush pending batch.
-        // getPendingLocalState() is only exposed through Container.closeAndGetPendingLocalState(), so it's safe
-        // to close current batch.
-        this.flush();
-        return {
-            pending: this.pendingStateManager.getLocalState(),
-            pendingAttachmentBlobs,
-        };
+        return PerformanceEvent.timedExecAsync(this.mc.logger, {
+            eventName: "getPendingLocalState",
+            notifyImminentClosure: props?.notifyImminentClosure,
+        }, async (event) => {
+            this.verifyNotClosed();
+            const waitBlobsToAttach = props?.notifyImminentClosure;
+            if (this._orderSequentiallyCalls !== 0) {
+                throw new UsageError("can't get state during orderSequentially");
+            }
+            const pendingAttachmentBlobs = await this.blobManager.getPendingBlobs(waitBlobsToAttach);
+            const pending = this.pendingStateManager.getLocalState();
+            if (!pendingAttachmentBlobs && !this.hasPendingMessages()) {
+                return; // no pending state to save
+            }
+            // Flush pending batch.
+            // getPendingLocalState() is only exposed through Container.closeAndGetPendingLocalState(), so it's safe
+            // to close current batch.
+            this.flush();
+            const pendingState = {
+                pending,
+                pendingAttachmentBlobs,
+            };
+            event.end({
+                attachmentBlobsSize: Object.keys(pendingAttachmentBlobs ?? {}).length,
+                pendingOpsSize: pending?.pendingStates.length,
+            });
+            return pendingState;
+        });
+    }
+    summarizeOnDemand(options) {
+        if (this.isSummarizerClient) {
+            return this.summarizer.summarizeOnDemand(options);
+        }
+        else if (this.summaryManager !== undefined) {
+            return this.summaryManager.summarizeOnDemand(options);
+        }
+        else {
+            // If we're not the summarizer, and we don't have a summaryManager, we expect that
+            // disableSummaries is turned on. We are throwing instead of returning a failure here,
+            // because it is a misuse of the API rather than an expected failure.
+            throw new UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
+        }
+    }
+    enqueueSummarize(options) {
+        if (this.isSummarizerClient) {
+            return this.summarizer.enqueueSummarize(options);
+        }
+        else if (this.summaryManager !== undefined) {
+            return this.summaryManager.enqueueSummarize(options);
+        }
+        else {
+            // If we're not the summarizer, and we don't have a summaryManager, we expect that
+            // generateSummaries is turned off. We are throwing instead of returning a failure here,
+            // because it is a misuse of the API rather than an expected failure.
+            throw new UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
+        }
     }
     /**
      * * Forms a function that will request a Summarizer.