@fluidframework/container-runtime 2.0.0-internal.6.1.1 → 2.0.0-internal.6.3.0

package/dist/containerRuntime.js

@@ -19,14 +19,13 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ContainerRuntime = exports.makeLegacySendBatchFn = exports.getDeviceSpec = exports.agentSchedulerId = exports.isRuntimeMessage = exports.RuntimeMessage = exports.CompressionAlgorithms = exports.defaultRuntimeHeaderData = exports.TombstoneResponseHeaderKey = exports.AllowTombstoneRequestHeaderKey = exports.RuntimeHeaders = exports.DefaultSummaryConfiguration = exports.ContainerMessageType = void 0;
+exports.ContainerRuntime = exports.makeLegacySendBatchFn = exports.getDeviceSpec = exports.agentSchedulerId = exports.isRuntimeMessage = exports.RuntimeMessage = exports.defaultPendingOpsRetryDelayMs = exports.defaultPendingOpsWaitTimeoutMs = exports.CompressionAlgorithms = exports.defaultRuntimeHeaderData = exports.InactiveResponseHeaderKey = exports.TombstoneResponseHeaderKey = exports.AllowInactiveRequestHeaderKey = exports.AllowTombstoneRequestHeaderKey = exports.RuntimeHeaders = exports.DefaultSummaryConfiguration = exports.ContainerMessageType = void 0;
 const container_definitions_1 = require("@fluidframework/container-definitions");
-const common_utils_1 = require("@fluidframework/common-utils");
 const core_utils_1 = require("@fluidframework/core-utils");
+const client_utils_1 = require("@fluid-internal/client-utils");
 const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
 const driver_definitions_1 = require("@fluidframework/driver-definitions");
 const driver_utils_1 = require("@fluidframework/driver-utils");
-const container_utils_1 = require("@fluidframework/container-utils");
 const protocol_definitions_1 = require("@fluidframework/protocol-definitions");
 const runtime_definitions_1 = require("@fluidframework/runtime-definitions");
 const runtime_utils_1 = require("@fluidframework/runtime-utils");
@@ -67,6 +66,18 @@ var ContainerMessageType;
      */
     ContainerMessageType["IdAllocation"] = "idAllocation";
 })(ContainerMessageType = exports.ContainerMessageType || (exports.ContainerMessageType = {}));
+/**
+ * Utility to implement compat behaviors given an unknown message type
+ * The parameters are typed to support compile-time enforcement of handling all known types/behaviors
+ *
+ * @param _unknownContainerRuntimeMessageType - Typed as never, to ensure all known types have been
+ * handled before calling this function (e.g. in a switch statement).
+ * @param compatBehavior - Typed redundantly with CompatModeBehavior to ensure handling is added when updating that type
+ */
+function compatBehaviorAllowsMessageType(_unknownContainerRuntimeMessageType, compatBehavior) {
+    // undefined defaults to same behavior as "FailToProcess"
+    return compatBehavior === "Ignore";
+}
 exports.DefaultSummaryConfiguration = {
     state: "enabled",
     minIdleTime: 0,
@@ -93,8 +104,12 @@ var RuntimeHeaders;
 })(RuntimeHeaders = exports.RuntimeHeaders || (exports.RuntimeHeaders = {}));
 /** True if a tombstoned object should be returned without erroring */
 exports.AllowTombstoneRequestHeaderKey = "allowTombstone"; // Belongs in the enum above, but avoiding the breaking change
+/** [IRRELEVANT IF throwOnInactiveLoad OPTION NOT SET] True if an inactive object should be returned without erroring */
+exports.AllowInactiveRequestHeaderKey = "allowInactive"; // Belongs in the enum above, but avoiding the breaking change
 /** Tombstone error responses will have this header set to true */
 exports.TombstoneResponseHeaderKey = "isTombstoned";
+/** Inactive error responses will have this header set to true */
+exports.InactiveResponseHeaderKey = "isInactive";
 /** Default values for Runtime Headers */
 exports.defaultRuntimeHeaderData = {
     wait: true,
@@ -121,6 +136,10 @@ const defaultCompressionConfig = {
     compressionAlgorithm: CompressionAlgorithms.lz4,
 };
 const defaultChunkSizeInBytes = 204800;
+/** The default time to wait for pending ops to be processed during summarization */
+exports.defaultPendingOpsWaitTimeoutMs = 1000;
+/** The default time to delay a summarization retry attempt when there are pending ops */
+exports.defaultPendingOpsRetryDelayMs = 1000;
 /**
  * Instead of refreshing from latest because we do not have 100% confidence in the state
  * of the current system, we should close the summarizer and let it recover.
@@ -186,7 +205,7 @@ exports.makeLegacySendBatchFn = makeLegacySendBatchFn;
  * Represents the runtime of the container. Contains helper functions/state of the container.
  * It will define the store level mappings.
  */
-class ContainerRuntime extends common_utils_1.TypedEventEmitter {
+class ContainerRuntime extends client_utils_1.TypedEventEmitter {
     /**
      * @internal
      */
@@ -225,34 +244,6 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             signalTimestamp: 0,
             trackingSignalSequenceNumber: undefined,
         };
-        this.summarizeOnDemand = (...args) => {
-            if (this.isSummarizerClient) {
-                return this.summarizer.summarizeOnDemand(...args);
-            }
-            else if (this.summaryManager !== undefined) {
-                return this.summaryManager.summarizeOnDemand(...args);
-            }
-            else {
-                // If we're not the summarizer, and we don't have a summaryManager, we expect that
-                // disableSummaries is turned on. We are throwing instead of returning a failure here,
-                // because it is a misuse of the API rather than an expected failure.
-                throw new container_utils_1.UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
-            }
-        };
-        this.enqueueSummarize = (...args) => {
-            if (this.isSummarizerClient) {
-                return this.summarizer.enqueueSummarize(...args);
-            }
-            else if (this.summaryManager !== undefined) {
-                return this.summaryManager.enqueueSummarize(...args);
-            }
-            else {
-                // If we're not the summarizer, and we don't have a summaryManager, we expect that
-                // generateSummaries is turned off. We are throwing instead of returning a failure here,
-                // because it is a misuse of the API rather than an expected failure.
-                throw new container_utils_1.UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
-            }
-        };
         const { options, clientDetails, connected, baseSnapshot, submitFn, submitBatchFn, submitSummaryFn, submitSignalFn, disposeFn, closeFn, deltaManager, quorum, audience, loader, pendingLocalState, supportedFeatures, } = context;
         this.innerDeltaManager = deltaManager;
         this.deltaManager = new deltaManagerSummarizerProxy_1.DeltaManagerSummarizerProxy(this.innerDeltaManager);
@@ -368,7 +359,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             // This is a runtime enforcement of what's already explicit in the policy's type itself,
             // which dictates the value is either undefined or exactly 5 days in ms.
             // As long as the actual value is less than 5 days, the assumptions GC makes here are valid.
-            throw new container_utils_1.UsageError("Driver's maximumCacheDurationMs policy cannot exceed 5 days");
+            throw new telemetry_utils_1.UsageError("Driver's maximumCacheDurationMs policy cannot exceed 5 days");
         }
         this.garbageCollector = gc_1.GarbageCollector.create({
             runtime: this,
@@ -467,16 +458,13 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         });
         // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
         this._audience = audience;
-        this.summaryStateUpdateMethod = this.mc.config.getString("Fluid.ContainerRuntime.Test.SummaryStateUpdateMethodV2");
         const closeSummarizerDelayOverride = this.mc.config.getNumber("Fluid.ContainerRuntime.Test.CloseSummarizerDelayOverrideMs");
         this.closeSummarizerDelayMs = closeSummarizerDelayOverride ?? defaultCloseSummarizerDelayMs;
         this.validateSummaryBeforeUpload =
-            this.mc.config.getBoolean("Fluid.ContainerRuntime.Test.ValidateSummaryBeforeUpload") ??
-                false;
+            this.mc.config.getBoolean("Fluid.Summarizer.ValidateSummaryBeforeUpload") ?? false;
         this.summaryCollection = new summary_1.SummaryCollection(this.deltaManager, this.logger);
         this.dirtyContainer =
-            this.attachState !== container_definitions_1.AttachState.Attached ||
-                this.pendingStateManager.hasPendingMessages();
+            this.attachState !== container_definitions_1.AttachState.Attached || this.hasPendingMessages();
         context.updateDirtyContainerState(this.dirtyContainer);
         if (this.summariesDisabled) {
             this.mc.logger.sendTelemetryEvent({ eventName: "SummariesDisabled" });
@@ -524,6 +512,9 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 (0, throttler_1.formExponentialFn)({ coefficient: 20, initialDelay: 0 })), {
                     initialDelayMs: this.initialSummarizerDelayMs,
                 }, this.heuristicsDisabled);
+                this.summaryManager.on("summarize", (eventProps) => {
+                    this.emit("summarize", eventProps);
+                });
                 this.summaryManager.start();
             }
         }
@@ -531,7 +522,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             // we accumulate ops while being in read-only state.
             // once user gets write permissions and we have active connection, flush all pending ops.
             // Note that the inner (non-proxy) delta manager is needed here to get the readonly information.
-            (0, common_utils_1.assert)(readonly === this.innerDeltaManager.readOnlyInfo.readonly, 0x124 /* "inconsistent readonly property/event state" */);
+            (0, core_utils_1.assert)(readonly === this.innerDeltaManager.readOnlyInfo.readonly, 0x124 /* "inconsistent readonly property/event state" */);
             // We need to be very careful with when we (re)send pending ops, to ensure that we only send ops
             // when we either never send an op, or attempted to send it but we know for sure it was not
             // sequenced by server and will never be sequenced (i.e. was lost)
@@ -544,7 +535,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             // can rely on same safety mechanism and resend ops only when we establish new connection.
             // This is applicable for read-only permissions (event is raised before connection is properly registered),
             // but it's an extra requirement for Container.forceReadonly() API
-            (0, common_utils_1.assert)(!readonly || !this.connected, 0x125 /* "Unsafe to transition to read-only state!" */);
+            (0, core_utils_1.assert)(!readonly || !this.connected, 0x125 /* "Unsafe to transition to read-only state!" */);
             this.replayPendingStates();
         });
         // logging hardware telemetry
@@ -568,7 +559,6 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 disableAttachReorder: this.disableAttachReorder,
                 disablePartialFlush,
                 idCompressorEnabled: this.idCompressorEnabled,
-                summaryStateUpdateMethod: this.summaryStateUpdateMethod,
                 closeSummarizerDelayOverride,
             }),
             telemetryDocumentId: this.telemetryDocumentId,
@@ -578,7 +568,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         (0, batchTracker_1.BindBatchTracker)(this, this.logger);
         this.entryPoint = new core_utils_1.LazyPromise(async () => {
             if (this.isSummarizerClient) {
-                (0, common_utils_1.assert)(this._summarizer !== undefined, 0x5bf /* Summarizer object is undefined in a summarizer client */);
+                (0, core_utils_1.assert)(this._summarizer !== undefined, 0x5bf /* Summarizer object is undefined in a summarizer client */);
                 return this._summarizer;
             }
             return initializeEntryPoint?.(this);
@@ -632,11 +622,21 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
      * This object should provide all the functionality that the Container is expected to provide to the loader layer.
      */
     static async loadRuntime(params) {
-        const { context, registryEntries, existing, requestHandler, runtimeOptions = {}, containerScope = {}, containerRuntimeCtor = ContainerRuntime, initializeEntryPoint, } = params;
+        const { context, registryEntries, existing, requestHandler, runtimeOptions = {}, containerScope = {}, containerRuntimeCtor = ContainerRuntime, } = params;
+        const initializeEntryPoint = params.initializeEntryPoint ??
+            (async (containerRuntime) => ({
+                get IFluidRouter() {
+                    return this;
+                },
+                async request(req) {
+                    return containerRuntime.request(req);
+                },
+            }));
         // If taggedLogger exists, use it. Otherwise, wrap the vanilla logger:
         // back-compat: Remove the TaggedLoggerAdapter fallback once all the host are using loader > 0.45
         const backCompatContext = context;
         const passLogger = backCompatContext.taggedLogger ??
+            // eslint-disable-next-line import/no-deprecated
             new telemetry_utils_1.TaggedLoggerAdapter(backCompatContext.logger);
         const logger = (0, telemetry_utils_1.createChildLogger)({
             logger: passLogger,
@@ -653,7 +653,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             if (context.baseSnapshot && blobId) {
                 // IContainerContext storage api return type still has undefined in 0.39 package version.
                 // So once we release 0.40 container-defn package we can remove this check.
-                (0, common_utils_1.assert)(context.storage !== undefined, 0x1f5 /* "Attached state should have storage" */);
+                (0, core_utils_1.assert)(context.storage !== undefined, 0x1f5 /* "Attached state should have storage" */);
                 return (0, driver_utils_1.readAndParse)(context.storage, blobId);
             }
         };
@@ -668,7 +668,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         const blobManagerSnapshot = await blobManager_1.BlobManager.load(context.baseSnapshot?.trees[summary_1.blobsTreeName], async (id) => {
             // IContainerContext storage api return type still has undefined in 0.39 package version.
             // So once we release 0.40 container-defn package we can remove this check.
-            (0, common_utils_1.assert)(context.storage !== undefined, 0x256 /* "storage undefined in attached container" */);
+            (0, core_utils_1.assert)(context.storage !== undefined, 0x256 /* "storage undefined in attached container" */);
             return (0, driver_utils_1.readAndParse)(context.storage, id);
         });
         // Verify summary runtime sequence number matches protocol sequence number.
@@ -680,7 +680,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             if (loadSequenceNumberVerification !== "bypass" &&
                 runtimeSequenceNumber !== protocolSequenceNumber) {
                 // "Load from summary, runtime metadata sequenceNumber !== initialSequenceNumber"
-                const error = new container_utils_1.DataCorruptionError(
+                const error = new telemetry_utils_1.DataCorruptionError(
                 // pre-0.58 error message: SummaryMetadataMismatch
                 "Summary metadata mismatch", { runtimeVersion: packageVersion_1.pkgVersion, runtimeSequenceNumber, protocolSequenceNumber });
                 if (loadSequenceNumberVerification === "log") {
@@ -698,7 +698,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             idCompressor =
                 serializedIdCompressor !== undefined
                     ? IdCompressor.deserialize(serializedIdCompressor, createSessionId())
-                    : new IdCompressor(createSessionId(), logger);
+                    : IdCompressor.create(logger);
         }
         const runtime = new containerRuntimeCtor(context, registry, metadata, electedSummarizerData, chunks ?? [], aliases ?? [], {
             summaryOptions,
@@ -713,6 +713,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             enableGroupedBatching,
         }, containerScope, logger, existing, blobManagerSnapshot, context.storage, idCompressor, requestHandler, undefined, // summaryConfiguration
         initializeEntryPoint);
+        await runtime.blobManager.processStashedChanges();
         // It's possible to have ops with a reference sequence number of 0. Op sequence numbers start
         // at 1, so we won't see a replayed saved op with a sequence number of 0.
         await runtime.pendingStateManager.applyStashedOpsAt(0);
@@ -729,6 +730,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     /** @deprecated - The functionality is no longer exposed publicly */
     get reSubmitFn() {
         return (type, contents, localOpMetadata, opMetadata) => this.reSubmitCore({ type, contents }, localOpMetadata, opMetadata);
+        // Note: compatDetails is not included in this deprecated API
     }
     get flushMode() {
         return this._flushMode;
@@ -773,7 +775,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         return this._disposed;
     }
     get summarizer() {
-        (0, common_utils_1.assert)(this._summarizer !== undefined, 0x257 /* "This is not summarizing container" */);
+        (0, core_utils_1.assert)(this._summarizer !== undefined, 0x257 /* "This is not summarizing container" */);
         return this._summarizer;
     }
     isSummariesDisabled() {
@@ -878,8 +880,8 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 const subRequest = requestParser.createSubRequest(1);
                 // We always expect createSubRequest to include a leading slash, but asserting here to protect against
                 // unintentionally modifying the url if that changes.
-                (0, common_utils_1.assert)(subRequest.url.startsWith("/"), 0x126 /* "Expected createSubRequest url to include a leading slash" */);
-                return dataStore.IFluidRouter.request(subRequest);
+                (0, core_utils_1.assert)(subRequest.url.startsWith("/"), 0x126 /* "Expected createSubRequest url to include a leading slash" */);
+                return dataStore.request(subRequest);
             }
             return (0, runtime_utils_1.create404Response)(request);
         }
@@ -937,7 +939,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     addContainerStateToSummary(summaryTree, fullTree, trackState, telemetryContext) {
         this.addMetadataToSummary(summaryTree);
         if (this.idCompressorEnabled) {
-            (0, common_utils_1.assert)(this.idCompressor !== undefined, 0x67a /* IdCompressor should be defined if enabled */);
+            (0, core_utils_1.assert)(this.idCompressor !== undefined, 0x67a /* IdCompressor should be defined if enabled */);
             const idCompressorState = JSON.stringify(this.idCompressor.serialize(false));
             (0, runtime_utils_1.addBlobToSummary)(summaryTree, summary_1.idCompressorBlobName, idCompressorState);
         }
@@ -987,7 +989,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             this.mc.logger.sendTelemetryEvent({
                 eventName: "ReconnectsWithNoProgress",
                 attempts: this.consecutiveReconnects,
-                pendingMessages: this.pendingStateManager.pendingMessagesCount,
+                pendingMessages: this.pendingMessagesCount,
             });
         }
         return this.consecutiveReconnects < this.maxConsecutiveReconnects;
@@ -1012,7 +1014,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         // Save the old state, reset to false, disable event emit
         const oldState = this.dirtyContainer;
         this.dirtyContainer = false;
-        (0, common_utils_1.assert)(this.emitDirtyDocumentEvent, 0x127 /* "dirty document event not set on replay" */);
+        (0, core_utils_1.assert)(this.emitDirtyDocumentEvent, 0x127 /* "dirty document event not set on replay" */);
         this.emitDirtyDocumentEvent = false;
         let newState;
         try {
@@ -1046,21 +1048,21 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
      * ! Note: this format needs to be in-line with what is set in the "ContainerRuntime.submit(...)" method
      */
     parseOpContent(serializedContent) {
-        (0, common_utils_1.assert)(serializedContent !== undefined, 0x6d5 /* content must be defined */);
-        const { type, contents } = JSON.parse(serializedContent);
-        (0, common_utils_1.assert)(type !== undefined, 0x6d6 /* incorrect op content format */);
-        return { type, contents };
+        (0, core_utils_1.assert)(serializedContent !== undefined, 0x6d5 /* content must be defined */);
+        const { type, contents, compatDetails } = JSON.parse(serializedContent);
+        (0, core_utils_1.assert)(type !== undefined, 0x6d6 /* incorrect op content format */);
+        return { type, contents, compatDetails };
     }
     async applyStashedOp(op) {
         // Need to parse from string for back-compat
-        const { type, contents } = this.parseOpContent(op);
+        const { type, contents, compatDetails } = this.parseOpContent(op);
         switch (type) {
             case ContainerMessageType.FluidDataStoreOp:
                 return this.dataStores.applyStashedOp(contents);
             case ContainerMessageType.Attach:
                 return this.dataStores.applyStashedAttachOp(contents);
             case ContainerMessageType.IdAllocation:
-                (0, common_utils_1.assert)(this.idCompressor !== undefined, 0x67b /* IdCompressor should be defined if enabled */);
+                (0, core_utils_1.assert)(this.idCompressor !== undefined, 0x67b /* IdCompressor should be defined if enabled */);
                 return this.applyStashedIdAllocationOp(contents);
             case ContainerMessageType.Alias:
             case ContainerMessageType.BlobAttach:
@@ -1069,8 +1071,22 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 throw new Error("chunkedOp not expected here");
             case ContainerMessageType.Rejoin:
                 throw new Error("rejoin not expected here");
-            default:
-                (0, common_utils_1.unreachableCase)(type, `Unknown ContainerMessageType: ${type}`);
+            default: {
+                // This should be extremely rare for stashed ops.
+                // It would require a newer runtime stashing ops and then an older one applying them,
+                // e.g. if an app rolled back its container version
+                const compatBehavior = compatDetails?.behavior;
+                if (!compatBehaviorAllowsMessageType(type, compatBehavior)) {
+                    const error = telemetry_utils_1.DataProcessingError.create("Stashed runtime message of unknown type", "applyStashedOp", undefined /* sequencedMessage */, {
+                        messageDetails: JSON.stringify({
+                            type,
+                            compatBehavior,
+                        }),
+                    });
+                    this.closeFn(error);
+                    throw error;
+                }
+            }
         }
     }
     setConnectionState(connected, clientId) {
@@ -1082,28 +1098,10 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             // Don't propagate "disconnected" event because we didn't propagate the previous "connected" event
             return;
         }
-        // If attachment blobs were added while disconnected, we need to delay
-        // propagation of the "connected" event until we have uploaded them to
-        // ensure we don't submit ops referencing a blob that has not been uploaded
-        // Note that the inner (non-proxy) delta manager is needed here to get the readonly information.
-        const connecting = connected && !this._connected && !this.innerDeltaManager.readOnlyInfo.readonly;
-        if (connecting && this.blobManager.hasPendingOfflineUploads) {
-            (0, common_utils_1.assert)(!this.delayConnectClientId, 0x392 /* Connect event delay must be canceled before subsequent connect event */);
-            (0, common_utils_1.assert)(!!clientId, 0x393 /* Must have clientId when connecting */);
-            this.delayConnectClientId = clientId;
-            this.blobManager.onConnected().then(() => {
-                // make sure we didn't reconnect before the promise resolved
-                if (this.delayConnectClientId === clientId && !this.disposed) {
-                    this.delayConnectClientId = undefined;
-                    this.setConnectionStateCore(connected, clientId);
-                }
-            }, (error) => this.closeFn(error));
-            return;
-        }
         this.setConnectionStateCore(connected, clientId);
     }
     setConnectionStateCore(connected, clientId) {
-        (0, common_utils_1.assert)(!this.delayConnectClientId, 0x394 /* connect event delay must be cleared before propagating connect event */);
+        (0, core_utils_1.assert)(!this.delayConnectClientId, 0x394 /* connect event delay must be cleared before propagating connect event */);
         this.verifyNotClosed();
         // There might be no change of state due to Container calling this API after loading runtime.
         const changeOfState = this._connected !== connected;
@@ -1121,16 +1119,16 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             this._perfSignalData.trackingSignalSequenceNumber = undefined;
         }
         else {
-            (0, common_utils_1.assert)(this.attachState === container_definitions_1.AttachState.Attached, 0x3cd /* Connection is possible only if container exists in storage */);
+            (0, core_utils_1.assert)(this.attachState === container_definitions_1.AttachState.Attached, 0x3cd /* Connection is possible only if container exists in storage */);
         }
         // Fail while disconnected
         if (reconnection) {
             this.consecutiveReconnects++;
             if (!this.shouldContinueReconnecting()) {
-                this.closeFn(container_utils_1.DataProcessingError.create("Runtime detected too many reconnects with no progress syncing local ops.", "setConnectionState", undefined, {
+                this.closeFn(telemetry_utils_1.DataProcessingError.create("Runtime detected too many reconnects with no progress syncing local ops.", "setConnectionState", undefined, {
                     dataLoss: 1,
                     attempts: this.consecutiveReconnects,
-                    pendingMessages: this.pendingStateManager.pendingMessagesCount,
+                    pendingMessages: this.pendingMessagesCount,
                 }));
                 return;
             }
@@ -1147,23 +1145,23 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     }
     process(messageArg, local) {
         this.verifyNotClosed();
-        // Whether or not the message is actually a runtime message.
+        // Whether or not the message appears to be a runtime message from an up-to-date client.
         // It may be a legacy runtime message (ie already unpacked and ContainerMessageType)
         // or something different, like a system message.
-        const runtimeMessage = messageArg.type === protocol_definitions_1.MessageType.Operation;
+        const modernRuntimeMessage = messageArg.type === protocol_definitions_1.MessageType.Operation;
         // Do shallow copy of message, as the processing flow will modify it.
         const messageCopy = { ...messageArg };
         for (const message of this.remoteMessageProcessor.process(messageCopy)) {
-            this.processCore(message, local, runtimeMessage);
+            this.processCore(message, local, modernRuntimeMessage);
         }
     }
     /**
      * Direct the message to the correct subsystem for processing, and implement other side effects
      * @param message - The unpacked message. Likely a ContainerRuntimeMessage, but could also be a system op
      * @param local - Did this client send the op?
-     * @param runtimeMessage - Does this appear like a current ContainerRuntimeMessage?  If true, certain validation will occur.
+     * @param modernRuntimeMessage - Does this appear like a current ContainerRuntimeMessage?
      */
-    processCore(message, local, runtimeMessage) {
+    processCore(message, local, modernRuntimeMessage) {
         // Surround the actual processing of the operation with messages to the schedule manager indicating
         // the beginning and end. This allows it to emit appropriate events and/or pause the processing of new
         // messages once a batch has been fully processed.
@@ -1171,7 +1169,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         this._processedClientSequenceNumber = message.clientSequenceNumber;
         try {
             let localOpMetadata;
-            if (local && runtimeMessage && message.type !== ContainerMessageType.ChunkedOp) {
+            if (local && modernRuntimeMessage && message.type !== ContainerMessageType.ChunkedOp) {
                 localOpMetadata = this.pendingStateManager.processPendingLocalMessage(message);
             }
             // If there are no more pending messages after processing a local message,
@@ -1179,43 +1177,8 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             if (!this.hasPendingMessages()) {
                 this.updateDocumentDirtyState(false);
             }
-            const type = message.type;
-            switch (type) {
-                case ContainerMessageType.Attach:
-                    this.dataStores.processAttachMessage(message, local);
-                    break;
-                case ContainerMessageType.Alias:
-                    this.processAliasMessage(message, localOpMetadata, local);
-                    break;
-                case ContainerMessageType.FluidDataStoreOp:
-                    this.dataStores.processFluidDataStoreOp(message, local, localOpMetadata);
-                    break;
-                case ContainerMessageType.BlobAttach:
-                    this.blobManager.processBlobAttachOp(message, local);
-                    break;
-                case ContainerMessageType.IdAllocation:
-                    (0, common_utils_1.assert)(this.idCompressor !== undefined, 0x67c /* IdCompressor should be defined if enabled */);
-                    this.idCompressor.finalizeCreationRange(message.contents);
-                    break;
-                case ContainerMessageType.ChunkedOp:
-                case ContainerMessageType.Rejoin:
-                    break;
-                default:
-                    if (runtimeMessage) {
-                        const error = container_utils_1.DataProcessingError.create(
-                        // Former assert 0x3ce
-                        "Runtime message of unknown type", "OpProcessing", message, {
-                            local,
-                            type: message.type,
-                            contentType: typeof message.contents,
-                            batch: message.metadata?.batch,
-                            compression: message.compression,
-                        });
-                        this.closeFn(error);
-                        throw error;
-                    }
-            }
-            this.emit("op", message, runtimeMessage);
+            this.validateAndProcessRuntimeMessage(message, localOpMetadata, local, modernRuntimeMessage);
+            this.emit("op", message, modernRuntimeMessage);
             this.scheduleManager.afterOpProcessing(undefined, message);
             if (local) {
                 // If we have processed a local op, this means that the container is
@@ -1229,8 +1192,59 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             throw e;
         }
     }
-    processAliasMessage(message, localOpMetadata, local) {
-        this.dataStores.processAliasMessage(message, localOpMetadata, local);
+    /**
+     * Assuming the given message is also a ContainerRuntimeMessage,
+     * checks its type and dispatches the message to the appropriate handler in the runtime.
+     * Throws a DataProcessingError if the message doesn't conform to the ContainerRuntimeMessage type.
+     */
+    validateAndProcessRuntimeMessage(message, localOpMetadata, local, expectRuntimeMessageType) {
+        // Optimistically extract ContainerRuntimeMessage-specific props from the message
+        const { type: maybeContainerMessageType, compatDetails } = message;
+        switch (maybeContainerMessageType) {
+            case ContainerMessageType.Attach:
+                this.dataStores.processAttachMessage(message, local);
+                break;
+            case ContainerMessageType.Alias:
+                this.dataStores.processAliasMessage(message, localOpMetadata, local);
+                break;
+            case ContainerMessageType.FluidDataStoreOp:
+                this.dataStores.processFluidDataStoreOp(message, local, localOpMetadata);
+                break;
+            case ContainerMessageType.BlobAttach:
+                this.blobManager.processBlobAttachOp(message, local);
+                break;
+            case ContainerMessageType.IdAllocation:
+                (0, core_utils_1.assert)(this.idCompressor !== undefined, 0x67c /* IdCompressor should be defined if enabled */);
+                this.idCompressor.finalizeCreationRange(message.contents);
+                break;
+            case ContainerMessageType.ChunkedOp:
+            case ContainerMessageType.Rejoin:
+                break;
+            default: {
+                // If we didn't necessarily expect a runtime message type, then no worries - just return
+                // e.g. this case applies to system ops, or legacy ops that would have fallen into the above cases anyway.
+                if (!expectRuntimeMessageType) {
+                    return;
+                }
+                const compatBehavior = compatDetails?.behavior;
+                if (!compatBehaviorAllowsMessageType(maybeContainerMessageType, compatBehavior)) {
+                    const error = telemetry_utils_1.DataProcessingError.create(
+                    // Former assert 0x3ce
+                    "Runtime message of unknown type", "OpProcessing", message, {
+                        local,
+                        messageDetails: JSON.stringify({
+                            type: message.type,
+                            contentType: typeof message.contents,
+                            compatBehavior,
+                            batch: message.metadata?.batch,
+                            compression: message.compression,
+                        }),
+                    });
+                    this.closeFn(error);
+                    throw error;
+                }
+            }
+        }
     }
     /**
      * Emits the Signal event and update the perf signal data.
@@ -1298,7 +1312,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         await this.dataStores.waitIfPendingAlias(id);
         const internalId = this.internalId(id);
         const context = await this.dataStores.getDataStore(internalId, { wait });
-        (0, common_utils_1.assert)(await context.isRoot(), 0x12b /* "did not get root data store" */);
+        (0, core_utils_1.assert)(await context.isRoot(), 0x12b /* "did not get root data store" */);
         return context.realize();
     }
     /**
@@ -1306,9 +1320,9 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
      * This method is expected to be called at the end of a batch.
      */
     flush() {
-        (0, common_utils_1.assert)(this._orderSequentiallyCalls === 0, 0x24c /* "Cannot call `flush()` from `orderSequentially`'s callback" */);
+        (0, core_utils_1.assert)(this._orderSequentiallyCalls === 0, 0x24c /* "Cannot call `flush()` from `orderSequentially`'s callback" */);
         this.outbox.flush();
-        (0, common_utils_1.assert)(this.outbox.isEmpty, 0x3cf /* reentrancy */);
+        (0, core_utils_1.assert)(this.outbox.isEmpty, 0x3cf /* reentrancy */);
     }
     orderSequentially(callback) {
         let checkpoint;
@@ -1331,7 +1345,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 }
                 catch (err) {
                     const error2 = (0, telemetry_utils_1.wrapError)(err, (message) => {
-                        return container_utils_1.DataProcessingError.create(`RollbackError: ${message}`, "checkpointRollback", undefined);
+                        return telemetry_utils_1.DataProcessingError.create(`RollbackError: ${message}`, "checkpointRollback", undefined);
                     });
                     this.closeFn(error2);
                     throw error2;
@@ -1339,7 +1353,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             }
             else {
                 // pre-0.58 error message: orderSequentiallyCallbackException
-                this.closeFn(new container_utils_1.GenericError("orderSequentially callback exception", error));
+                this.closeFn(new telemetry_utils_1.GenericError("orderSequentially callback exception", error));
             }
             throw error; // throw the original error for the consumer of the runtime
         }
@@ -1368,13 +1382,13 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         }
         const channel = await context.realize();
         if (channel.entryPoint === undefined) {
-            throw new container_utils_1.UsageError("entryPoint must be defined on data store runtime for using getAliasedDataStoreEntryPoint");
+            throw new telemetry_utils_1.UsageError("entryPoint must be defined on data store runtime for using getAliasedDataStoreEntryPoint");
         }
         return channel.entryPoint;
     }
     createDetachedRootDataStore(pkg, rootDataStoreId) {
         if (rootDataStoreId.includes("/")) {
-            throw new container_utils_1.UsageError(`Id cannot contain slashes: '${rootDataStoreId}'`);
+            throw new telemetry_utils_1.UsageError(`Id cannot contain slashes: '${rootDataStoreId}'`);
         }
         return this.dataStores.createDetachedDataStoreCore(pkg, true, rootDataStoreId);
     }
@@ -1468,10 +1482,10 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     }
     setAttachState(attachState) {
         if (attachState === container_definitions_1.AttachState.Attaching) {
-            (0, common_utils_1.assert)(this.attachState === container_definitions_1.AttachState.Attaching, 0x12d /* "Container Context should already be in attaching state" */);
+            (0, core_utils_1.assert)(this.attachState === container_definitions_1.AttachState.Attaching, 0x12d /* "Container Context should already be in attaching state" */);
         }
         else {
-            (0, common_utils_1.assert)(this.attachState === container_definitions_1.AttachState.Attached, 0x12e /* "Container Context should already be in attached state" */);
+            (0, core_utils_1.assert)(this.attachState === container_definitions_1.AttachState.Attached, 0x12e /* "Container Context should already be in attached state" */);
             this.emit("attached");
         }
         if (attachState === container_definitions_1.AttachState.Attached && !this.hasPendingMessages()) {
@@ -1529,7 +1543,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 await this.collectGarbage({ logger: summaryLogger, runSweep, fullGC }, telemetryContext);
             }
             const { stats, summary } = await this.summarizerNode.summarize(fullTree, trackState, telemetryContext);
-            (0, common_utils_1.assert)(summary.type === protocol_definitions_1.SummaryType.Tree, 0x12f /* "Container Runtime's summarize should always return a tree" */);
+            (0, core_utils_1.assert)(summary.type === protocol_definitions_1.SummaryType.Tree, 0x12f /* "Container Runtime's summarize should always return a tree" */);
             return { stats, summary };
         }
         finally {
@@ -1641,7 +1655,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             case gc_1.GCNodeType.SubDataStore:
                 return this.dataStores.getDataStorePackagePath(nodePath);
             default:
-                (0, common_utils_1.assert)(false, 0x2de /* "Package path requested for unsupported node type." */);
+                (0, core_utils_1.assert)(false, 0x2de /* "Package path requested for unsupported node type." */);
         }
     }
     /**
@@ -1698,7 +1712,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
      * @param options - options controlling how the summary is generated or submitted
      */
     async submitSummary(options) {
-        const { fullTree = false, refreshLatestAck, summaryLogger } = options;
+        const { fullTree = false, finalAttempt = false, refreshLatestAck, summaryLogger } = options;
         // The summary number for this summary. This will be updated during the summary process, so get it now and
         // use it for all events logged during this summary.
         const summaryNumber = this.nextSummaryNumber;
@@ -1708,7 +1722,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 all: { summaryNumber },
             },
         });
-        (0, common_utils_1.assert)(this.outbox.isEmpty, 0x3d1 /* Can't trigger summary in the middle of a batch */);
+        (0, core_utils_1.assert)(this.outbox.isEmpty, 0x3d1 /* Can't trigger summary in the middle of a batch */);
         let latestSnapshotVersionId;
         if (refreshLatestAck) {
             const latestSnapshotInfo = await this.refreshLatestSummaryAckFromServer((0, telemetry_utils_1.createChildLogger)({
@@ -1720,6 +1734,40 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             // We might need to catch up to the latest summary's reference sequence number before pausing.
             await this.waitForDeltaManagerToCatchup(latestSnapshotRefSeq, summaryNumberLogger);
         }
+        // If there are pending (unacked ops), the summary will not be eventual consistent and it may even be
+        // incorrect. So, wait for the container to be saved with a timeout. If the container is not saved
+        // within the timeout, check if it should be failed or can continue.
+        if (this.validateSummaryBeforeUpload && this.hasPendingMessages()) {
+            const countBefore = this.pendingMessagesCount;
+            // The timeout for waiting for pending ops can be overridden via configurations.
+            const pendingOpsTimeout = this.mc.config.getNumber("Fluid.Summarizer.waitForPendingOpsTimeoutMs") ??
+                exports.defaultPendingOpsWaitTimeoutMs;
+            await new Promise((resolve, reject) => {
+                const timeoutId = setTimeout(() => resolve(), pendingOpsTimeout);
+                this.once("saved", () => {
+                    clearTimeout(timeoutId);
+                    resolve();
+                });
+                this.once("dispose", () => {
+                    clearTimeout(timeoutId);
+                    reject(new Error("Runtime is disposed while summarizing"));
+                });
+            });
+            // Log that there are pending ops while summarizing. This will help us gather data on how often this
+            // happens, whether we attempted to wait for these ops to be acked and what was the result.
+            summaryNumberLogger.sendTelemetryEvent({
+                eventName: "PendingOpsWhileSummarizing",
+                saved: this.hasPendingMessages() ? false : true,
+                timeout: pendingOpsTimeout,
+                countBefore,
+                countAfter: this.pendingMessagesCount,
+            });
+            // There could still be pending ops. Check if summary should fail or continue.
+            const pendingMessagesFailResult = await this.shouldFailSummaryOnPendingOps(summaryNumberLogger, this.deltaManager.lastSequenceNumber, this.deltaManager.minimumSequenceNumber, finalAttempt, true /* beforeSummaryGeneration */);
+            if (pendingMessagesFailResult !== undefined) {
+                return pendingMessagesFailResult;
+            }
+        }
         const shouldPauseInboundSignal = this.mc.config.getBoolean("Fluid.ContainerRuntime.SubmitSummary.disableInboundSignalPause") !== true;
         let summaryRefSeqNum;
         try {
@@ -1746,7 +1794,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 // That said, we rely on submitSystemMessage() that today only works in connected state.
                 // So if we fail here, it either means that RunWhileConnectedCoordinator does not work correctly,
                 // OR that design changed and we need to remove this check and fix submitSystemMessage.
-                (0, common_utils_1.assert)(this.connected, 0x258 /* "connected" */);
+                (0, core_utils_1.assert)(this.connected, 0x258 /* "connected" */);
                 // Ensure that lastSequenceNumber has not changed after pausing.
                 // We need the summary op's reference sequence number to match our summary sequence number,
                 // otherwise we'll get the wrong sequence number stamped on the summary's .protocol attributes.
@@ -1756,7 +1804,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                         error: `lastSequenceNumber changed before uploading to storage. ${this.deltaManager.lastSequenceNumber} !== ${summaryRefSeqNum}`,
                     };
                 }
-                (0, common_utils_1.assert)(summaryRefSeqNum === this.deltaManager.lastMessage?.sequenceNumber, 0x395 /* it's one and the same thing */);
+                (0, core_utils_1.assert)(summaryRefSeqNum === this.deltaManager.lastMessage?.sequenceNumber, 0x395 /* it's one and the same thing */);
                 if (lastAck !== this.summaryCollection.latestAck) {
                     return {
                         continue: false,
@@ -1774,7 +1822,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                     error: continueResult.error,
                 };
             }
-            const trace = common_utils_1.Trace.start();
+            const trace = client_utils_1.Trace.start();
             let summarizeResult;
             // If the GC state needs to be reset, we need to force a full tree summary and update the unreferenced
             // state of all the nodes.
@@ -1795,9 +1843,9 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                     error,
                 };
             }
-            // If validateSummaryBeforeUpload is true, validate that the summary generated by the summarizer nodes is
-            // correct before this summary is uploaded.
+            // If validateSummaryBeforeUpload is true, validate that the summary generated is correct before uploading.
             if (this.validateSummaryBeforeUpload) {
+                // Validate that the summaries generated by summarize nodes is correct.
                 const validateResult = this.summarizerNode.validateSummary();
                 if (!validateResult.success) {
                     const { success, ...loggingProps } = validateResult;
@@ -1809,6 +1857,10 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                         error,
                     };
                 }
+                const pendingMessagesFailResult = await this.shouldFailSummaryOnPendingOps(summaryNumberLogger, summaryRefSeqNum, minimumSequenceNumber, finalAttempt, false /* beforeSummaryGeneration */);
+                if (pendingMessagesFailResult !== undefined) {
+                    return pendingMessagesFailResult;
+                }
             }
             const { summary: summaryTree, stats: partialStats } = summarizeResult;
             // Now that we have generated the summary, update the message at last summary to the last message processed.
@@ -1817,7 +1869,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             // Because handles are unchanged dataStores in the current logic,
             // summarized dataStore count is total dataStore count minus handle count
             const dataStoreTree = summaryTree.tree[runtime_definitions_1.channelsTreeName];
-            (0, common_utils_1.assert)(dataStoreTree.type === protocol_definitions_1.SummaryType.Tree, 0x1fc /* "summary is not a tree" */);
+            (0, core_utils_1.assert)(dataStoreTree.type === protocol_definitions_1.SummaryType.Tree, 0x1fc /* "summary is not a tree" */);
             const handleCount = Object.values(dataStoreTree.tree).filter((value) => value.type === protocol_definitions_1.SummaryType.Handle).length;
             const gcSummaryTreeStats = summaryTree.tree[runtime_definitions_1.gcTreeKey]
                 ? (0, runtime_utils_1.calculateStats)(summaryTree.tree[runtime_definitions_1.gcTreeKey])
@@ -1929,16 +1981,63 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             }
         }
     }
+    /**
+     * This helper is called during summarization. If there are pending ops, it will return a failed summarize result
+     * (IBaseSummarizeResult) unless this is the final summarize attempt and SkipFailingIncorrectSummary option is set.
+     * @param logger - The logger to be used for sending telemetry.
+     * @param referenceSequenceNumber - The reference sequence number of the summary attempt.
+     * @param minimumSequenceNumber - The minimum sequence number of the summary attempt.
+     * @param finalAttempt - Whether this is the final summary attempt.
+     * @param beforeSummaryGeneration - Whether this is called before summary generation or after.
+     * @returns failed summarize result (IBaseSummarizeResult) if summary should be failed, undefined otherwise.
+     */
+    async shouldFailSummaryOnPendingOps(logger, referenceSequenceNumber, minimumSequenceNumber, finalAttempt, beforeSummaryGeneration) {
+        if (!this.hasPendingMessages()) {
+            return;
+        }
+        // If "SkipFailingIncorrectSummary" option is true, don't fail the summary in the last attempt.
+        // This is a fallback to make progress in documents where there are consistently pending ops in
+        // the summarizer.
+        if (finalAttempt &&
+            this.mc.config.getBoolean("Fluid.Summarizer.SkipFailingIncorrectSummary")) {
+            const error = telemetry_utils_1.DataProcessingError.create("Pending ops during summarization", "submitSummary", undefined, { pendingMessages: this.pendingMessagesCount });
+            logger.sendErrorEvent({
+                eventName: "SkipFailingIncorrectSummary",
+                referenceSequenceNumber,
+                minimumSequenceNumber,
+                beforeGenerate: beforeSummaryGeneration,
+            }, error);
+        }
+        else {
+            // The retry delay when there are pending ops can be overridden via config so that we can adjust it
+            // based on telemetry while we decide on a stable number.
+            const retryDelayMs = this.mc.config.getNumber("Fluid.Summarizer.PendingOpsRetryDelayMs") ??
+                exports.defaultPendingOpsRetryDelayMs;
+            const error = new summary_1.RetriableSummaryError("PendingOpsWhileSummarizing", retryDelayMs / 1000, {
+                count: this.pendingMessagesCount,
+                beforeGenerate: beforeSummaryGeneration,
+            });
+            return {
+                stage: "base",
+                referenceSequenceNumber,
+                minimumSequenceNumber,
+                error,
+            };
+        }
+    }
+    get pendingMessagesCount() {
+        return this.pendingStateManager.pendingMessagesCount + this.outbox.messageCount;
+    }
     hasPendingMessages() {
-        return this.pendingStateManager.hasPendingMessages() || !this.outbox.isEmpty;
+        return this.pendingMessagesCount !== 0;
     }
     updateDocumentDirtyState(dirty) {
         if (this.attachState !== container_definitions_1.AttachState.Attached) {
-            (0, common_utils_1.assert)(dirty, 0x3d2 /* Non-attached container is dirty */);
+            (0, core_utils_1.assert)(dirty, 0x3d2 /* Non-attached container is dirty */);
         }
         else {
             // Other way is not true = see this.isContainerMessageDirtyable()
-            (0, common_utils_1.assert)(!dirty || this.hasPendingMessages(), 0x3d3 /* if doc is dirty, there has to be pending ops */);
+            (0, core_utils_1.assert)(!dirty || this.hasPendingMessages(), 0x3d3 /* if doc is dirty, there has to be pending ops */);
         }
         if (this.dirtyContainer === dirty) {
             return;
@@ -1958,7 +2057,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     submitDataStoreAliasOp(contents, localOpMetadata) {
         const aliasMessage = contents;
         if (!(0, dataStore_1.isDataStoreAliasMessage)(aliasMessage)) {
-            throw new container_utils_1.UsageError("malformedDataStoreAliasMessage");
+            throw new telemetry_utils_1.UsageError("malformedDataStoreAliasMessage");
         }
         this.submit({ type: ContainerMessageType.Alias, contents }, localOpMetadata);
     }
@@ -1971,10 +2070,10 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             let idAllocationBatchMessage;
             let idRange;
             if (this.idCompressorEnabled) {
-                (0, common_utils_1.assert)(this.idCompressor !== undefined, 0x67d /* IdCompressor should be defined if enabled */);
+                (0, core_utils_1.assert)(this.idCompressor !== undefined, 0x67d /* IdCompressor should be defined if enabled */);
                 idRange = this.idCompressor.takeNextCreationRange();
                 // Don't include the idRange if there weren't any Ids allocated
-                idRange = idRange?.ids?.first !== undefined ? idRange : undefined;
+                idRange = idRange?.ids !== undefined ? idRange : undefined;
             }
             if (idRange !== undefined) {
                 const idAllocationMessage = {
@@ -1998,7 +2097,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         this.verifyNotClosed();
         this.verifyCanSubmitOps();
         // There should be no ops in detached container state!
-        (0, common_utils_1.assert)(this.attachState !== container_definitions_1.AttachState.Detached, 0x132 /* "sending ops in detached container" */);
+        (0, core_utils_1.assert)(this.attachState !== container_definitions_1.AttachState.Detached, 0x132 /* "sending ops in detached container" */);
         const serializedContent = JSON.stringify(containerRuntimeMessage);
         // Note that the real (non-proxy) delta manager is used here to get the readonly info. This is because
         // container runtime's ability to submit ops depend on the actual readonly state of the delta manager.
@@ -2098,15 +2197,15 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 setTimeout(flush, 0);
                 break;
             default:
-                (0, common_utils_1.assert)(this._orderSequentiallyCalls > 0, 0x587 /* Unreachable unless running under orderSequentially */);
+                (0, core_utils_1.assert)(this._orderSequentiallyCalls > 0, 0x587 /* Unreachable unless running under orderSequentially */);
                 break;
         }
     }
     submitSummaryMessage(contents, referenceSequenceNumber) {
         this.verifyNotClosed();
-        (0, common_utils_1.assert)(this.connected, 0x133 /* "Container disconnected when trying to submit system message" */);
+        (0, core_utils_1.assert)(this.connected, 0x133 /* "Container disconnected when trying to submit system message" */);
         // System message should not be sent in the middle of the batch.
-        (0, common_utils_1.assert)(this.outbox.isEmpty, 0x3d4 /* System op in the middle of a batch */);
+        (0, core_utils_1.assert)(this.outbox.isEmpty, 0x3d4 /* System op in the middle of a batch */);
         // back-compat: ADO #1385: Make this call unconditional in the future
         return this.submitSummaryFn !== undefined
             ? this.submitSummaryFn(contents, referenceSequenceNumber)
@@ -2127,7 +2226,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             if (this.opReentryCallsToReport > 0) {
                 this.mc.logger.sendTelemetryEvent({ eventName: "OpReentry" }, 
                 // We need to capture the call stack in order to inspect the source of this usage pattern
-                (0, opLifecycle_1.getLongStack)(() => new container_utils_1.UsageError(errorMessage)));
+                (0, opLifecycle_1.getLongStack)(() => new telemetry_utils_1.UsageError(errorMessage)));
                 this.opReentryCallsToReport--;
             }
             // Creating ops while processing ops can lead
@@ -2143,7 +2242,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             // The runtime must enforce op coherence by not allowing ops to be submitted
             // while ops are being processed.
             if (this.enableOpReentryCheck) {
-                throw new container_utils_1.UsageError(errorMessage);
+                throw new telemetry_utils_1.UsageError(errorMessage);
             }
         }
     }
@@ -2193,8 +2292,27 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             case ContainerMessageType.Rejoin:
                 this.submit(message);
                 break;
-            default:
-                (0, common_utils_1.unreachableCase)(message.type, `Unknown ContainerMessageType [type: ${message.type}]`);
+            default: {
+                // This case should be very rare - it would imply an op was stashed from a
+                // future version of runtime code and now is being applied on an older version
+                const compatBehavior = message.compatDetails?.behavior;
+                if (compatBehaviorAllowsMessageType(message.type, compatBehavior)) {
+                    this.logger.sendTelemetryEvent({
+                        eventName: "resubmitUnrecognizedMessageTypeAllowed",
+                        messageDetails: { type: message.type, compatBehavior },
+                    });
+                }
+                else {
+                    const error = telemetry_utils_1.DataProcessingError.create("Resubmitting runtime message of unknown type", "reSubmitCore", undefined /* sequencedMessage */, {
+                        messageDetails: JSON.stringify({
+                            type: message.type,
+                            compatBehavior,
+                        }),
+                    });
+                    this.closeFn(error);
+                    throw error;
+                }
+            }
         }
     }
     rollback(content, localOpMetadata) {
@@ -2207,6 +2325,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 this.dataStores.rollbackDataStoreOp(contents, localOpMetadata);
                 break;
             default:
+                // Don't check message.compatDetails because this is for rolling back a local op so the type will be known
                 throw new Error(`Can't rollback ${type}`);
         }
     }
@@ -2224,29 +2343,22 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     /** Implementation of ISummarizerInternalsProvider.refreshLatestSummaryAck */
     async refreshLatestSummaryAck(options) {
         const { proposalHandle, ackHandle, summaryRefSeq, summaryLogger } = options;
+        // proposalHandle is always passed from RunningSummarizer.
+        (0, core_utils_1.assert)(proposalHandle !== undefined, 0x766 /* proposalHandle should be available */);
         const readAndParseBlob = async (id) => (0, driver_utils_1.readAndParse)(this.storage, id);
-        // The call to fetch the snapshot is very expensive and not always needed.
-        // It should only be done by the summarizerNode, if required.
-        // When fetching from storage we will always get the latest version and do not use the ackHandle.
-        const fetchLatestSnapshot = async () => {
-            let fetchResult = await this.fetchLatestSnapshotFromStorage(summaryLogger, {
+        const result = await this.summarizerNode.refreshLatestSummary(proposalHandle, summaryRefSeq);
+        /**
+         * When refreshing a summary ack, this check indicates a new ack of a summary that is newer than the
+         * current summary that is tracked, but this summarizer runtime did not produce/track that summary. Thus
+         * it needs to refresh its state. Today refresh is done by fetching the latest snapshot to update the cache
+         * and then close as the current main client is likely to be re-elected as the parent summarizer again.
+         */
+        if (!result.isSummaryTracked && result.isSummaryNewer) {
+            const fetchResult = await this.fetchSnapshotFromStorage(summaryLogger, {
                 eventName: "RefreshLatestSummaryAckFetch",
                 ackHandle,
                 targetSequenceNumber: summaryRefSeq,
-            }, readAndParseBlob);
-            /**
-             * back-compat - Older loaders and drivers (pre 2.0.0-internal.1.4) don't have fetchSource as a param in the
-             * getVersions API. So, they will not fetch the latest snapshot from network in the previous fetch call. For
-             * these scenarios, fetch the snapshot corresponding to the ack handle to have the same behavior before the
-             * change that started fetching latest snapshot always.
-             */
-            if (fetchResult.latestSnapshotRefSeq < summaryRefSeq) {
-                fetchResult = await this.fetchSnapshotFromStorageAndClose(summaryLogger, {
-                    eventName: "RefreshLatestSummaryAckFetchBackCompat",
-                    ackHandle,
-                    targetSequenceNumber: summaryRefSeq,
-                }, readAndParseBlob, ackHandle);
-            }
+            }, readAndParseBlob, null);
             /**
              * If the fetched snapshot is older than the one for which the ack was received, close the container.
              * This should never happen because an ack should be sent after the latest summary is updated in the server.
@@ -2258,7 +2370,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
              * state.
              */
             if (fetchResult.latestSnapshotRefSeq < summaryRefSeq) {
-                const error = container_utils_1.DataProcessingError.create("Fetched snapshot is older than the received ack", "RefreshLatestSummaryAck", undefined /* sequencedMessage */, {
+                const error = telemetry_utils_1.DataProcessingError.create("Fetched snapshot is older than the received ack", "RefreshLatestSummaryAck", undefined /* sequencedMessage */, {
                     ackHandle,
                     summaryRefSeq,
                     fetchedSnapshotRefSeq: fetchResult.latestSnapshotRefSeq,
@@ -2266,17 +2378,11 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 this.disposeFn(error);
                 throw error;
             }
-            // In case we had to retrieve the latest snapshot and it is different than summaryRefSeq,
-            // wait for the delta manager to catch up before refreshing the latest Summary.
-            await this.waitForDeltaManagerToCatchup(fetchResult.latestSnapshotRefSeq, summaryLogger);
-            return {
-                snapshotTree: fetchResult.snapshotTree,
-                snapshotRefSeq: fetchResult.latestSnapshotRefSeq,
-            };
-        };
-        const result = await this.summarizerNode.refreshLatestSummary(proposalHandle, summaryRefSeq, fetchLatestSnapshot, readAndParseBlob, summaryLogger);
+            await this.closeStaleSummarizer("RefreshLatestSummaryAckFetch");
+            return;
+        }
         // Notify the garbage collector so it can update its latest summary state.
-        await this.garbageCollector.refreshLatestSummary(proposalHandle, result, readAndParseBlob);
+        await this.garbageCollector.refreshLatestSummary(result);
     }
     /**
      * Fetches the latest snapshot from storage and uses it to refresh SummarizerNode's
@@ -2286,30 +2392,38 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
      */
     async refreshLatestSummaryAckFromServer(summaryLogger) {
         const readAndParseBlob = async (id) => (0, driver_utils_1.readAndParse)(this.storage, id);
-        const { snapshotTree, versionId, latestSnapshotRefSeq } = await this.fetchLatestSnapshotFromStorage(summaryLogger, {
+        const { versionId, latestSnapshotRefSeq } = await this.fetchSnapshotFromStorage(summaryLogger, {
             eventName: "RefreshLatestSummaryFromServerFetch",
-        }, readAndParseBlob);
-        const fetchLatestSnapshot = {
-            snapshotTree,
-            snapshotRefSeq: latestSnapshotRefSeq,
-        };
-        const result = await this.summarizerNode.refreshLatestSummary(undefined /* proposalHandle */, latestSnapshotRefSeq, async () => fetchLatestSnapshot, readAndParseBlob, summaryLogger);
-        // Notify the garbage collector so it can update its latest summary state.
-        await this.garbageCollector.refreshLatestSummary(undefined /* proposalHandle */, result, readAndParseBlob);
+        }, readAndParseBlob, null);
+        await this.closeStaleSummarizer("RefreshLatestSummaryFromServerFetch");
         return { latestSnapshotRefSeq, latestSnapshotVersionId: versionId };
     }
-    async fetchLatestSnapshotFromStorage(logger, event, readAndParseBlob) {
-        return this.fetchSnapshotFromStorageAndClose(logger, event, readAndParseBlob, null /* latest */);
+    async closeStaleSummarizer(codePath) {
+        this.mc.logger.sendTelemetryEvent({
+            eventName: "ClosingSummarizerOnSummaryStale",
+            codePath,
+            message: "Stopping fetch from storage",
+            closeSummarizerDelayMs: this.closeSummarizerDelayMs,
+        }, new telemetry_utils_1.GenericError("Restarting summarizer instead of refreshing"));
+        // Delay before restarting summarizer to prevent the summarizer from restarting too frequently.
+        await (0, core_utils_1.delay)(this.closeSummarizerDelayMs);
+        this._summarizer?.stop("latestSummaryStateStale");
+        this.disposeFn();
     }
-    async fetchSnapshotFromStorageAndClose(logger, event, readAndParseBlob, versionId) {
-        const snapshotResults = await telemetry_utils_1.PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
+    /**
+     * Downloads snapshot from storage with the given versionId or latest if versionId is null.
+     * By default, it also closes the container after downloading the snapshot. However, this may be
+     * overridden via options.
+     */
+    async fetchSnapshotFromStorage(logger, event, readAndParseBlob, versionId) {
+        return telemetry_utils_1.PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
             const stats = {};
-            const trace = common_utils_1.Trace.start();
+            const trace = client_utils_1.Trace.start();
             const versions = await this.storage.getVersions(versionId, 1, "refreshLatestSummaryAckFromServer", versionId === null ? driver_definitions_1.FetchSource.noCache : undefined);
-            (0, common_utils_1.assert)(!!versions && !!versions[0], 0x137 /* "Failed to get version from storage" */);
+            (0, core_utils_1.assert)(!!versions && !!versions[0], 0x137 /* "Failed to get version from storage" */);
             stats.getVersionDuration = trace.trace().duration;
             const maybeSnapshot = await this.storage.getSnapshotTree(versions[0]);
-            (0, common_utils_1.assert)(!!maybeSnapshot, 0x138 /* "Failed to get snapshot from storage" */);
+            (0, core_utils_1.assert)(!!maybeSnapshot, 0x138 /* "Failed to get snapshot from storage" */);
             stats.getSnapshotDuration = trace.trace().duration;
             const latestSnapshotRefSeq = await (0, runtime_utils_1.seqFromTree)(maybeSnapshot, readAndParseBlob);
             stats.snapshotRefSeq = latestSnapshotRefSeq;
@@ -2321,41 +2435,65 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 latestSnapshotRefSeq,
             };
         });
-        // We choose to close the summarizer after the snapshot cache is updated to avoid
-        // situations which the main client (which is likely to be re-elected as the leader again)
-        // loads the summarizer from cache.
-        if (this.summaryStateUpdateMethod !== "refreshFromSnapshot") {
-            this.mc.logger.sendTelemetryEvent({
-                ...event,
-                eventName: "ClosingSummarizerOnSummaryStale",
-                codePath: event.eventName,
-                message: "Stopping fetch from storage",
-                versionId: versionId != null ? versionId : undefined,
-                closeSummarizerDelayMs: this.closeSummarizerDelayMs,
-            }, new container_utils_1.GenericError("Restarting summarizer instead of refreshing"));
-            // Delay before restarting summarizer to prevent the summarizer from restarting too frequently.
-            await (0, common_utils_1.delay)(this.closeSummarizerDelayMs);
-            this._summarizer?.stop("latestSummaryStateStale");
-            this.disposeFn();
-        }
-        return snapshotResults;
     }
     notifyAttaching() { } // do nothing (deprecated method)
     async getPendingLocalState(props) {
-        this.verifyNotClosed();
-        const waitBlobsToAttach = props?.notifyImminentClosure;
-        if (this._orderSequentiallyCalls !== 0) {
-            throw new container_utils_1.UsageError("can't get state during orderSequentially");
-        }
-        const pendingAttachmentBlobs = await this.blobManager.getPendingBlobs(waitBlobsToAttach);
-        // Flush pending batch.
-        // getPendingLocalState() is only exposed through Container.closeAndGetPendingLocalState(), so it's safe
-        // to close current batch.
-        this.flush();
-        return {
-            pending: this.pendingStateManager.getLocalState(),
-            pendingAttachmentBlobs,
-        };
+        return telemetry_utils_1.PerformanceEvent.timedExecAsync(this.mc.logger, {
+            eventName: "getPendingLocalState",
+            notifyImminentClosure: props?.notifyImminentClosure,
+        }, async (event) => {
+            this.verifyNotClosed();
+            const waitBlobsToAttach = props?.notifyImminentClosure;
+            if (this._orderSequentiallyCalls !== 0) {
+                throw new telemetry_utils_1.UsageError("can't get state during orderSequentially");
+            }
+            const pendingAttachmentBlobs = await this.blobManager.getPendingBlobs(waitBlobsToAttach);
+            const pending = this.pendingStateManager.getLocalState();
+            if (!pendingAttachmentBlobs && !this.hasPendingMessages()) {
+                return; // no pending state to save
+            }
+            // Flush pending batch.
+            // getPendingLocalState() is only exposed through Container.closeAndGetPendingLocalState(), so it's safe
+            // to close current batch.
+            this.flush();
+            const pendingState = {
+                pending,
+                pendingAttachmentBlobs,
+            };
+            event.end({
+                attachmentBlobsSize: Object.keys(pendingAttachmentBlobs ?? {}).length,
+                pendingOpsSize: pending?.pendingStates.length,
+            });
+            return pendingState;
+        });
+    }
+    summarizeOnDemand(options) {
+        if (this.isSummarizerClient) {
+            return this.summarizer.summarizeOnDemand(options);
+        }
+        else if (this.summaryManager !== undefined) {
+            return this.summaryManager.summarizeOnDemand(options);
+        }
+        else {
+            // If we're not the summarizer, and we don't have a summaryManager, we expect that
+            // disableSummaries is turned on. We are throwing instead of returning a failure here,
+            // because it is a misuse of the API rather than an expected failure.
+            throw new telemetry_utils_1.UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
+        }
+    }
+    enqueueSummarize(options) {
+        if (this.isSummarizerClient) {
+            return this.summarizer.enqueueSummarize(options);
+        }
+        else if (this.summaryManager !== undefined) {
+            return this.summaryManager.enqueueSummarize(options);
+        }
+        else {
+            // If we're not the summarizer, and we don't have a summaryManager, we expect that
+            // generateSummaries is turned off. We are throwing instead of returning a failure here,
+            // because it is a misuse of the API rather than an expected failure.
+            throw new telemetry_utils_1.UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
+        }
     }
     /**
      * * Forms a function that will request a Summarizer.
@@ -2378,7 +2516,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             const fluidObject = await (0, runtime_utils_1.requestFluidObject)(loaderRouter, request);
             const summarizer = fluidObject.ISummarizer;
             if (!summarizer) {
-                throw new container_utils_1.UsageError("Fluid object does not implement ISummarizer");
+                throw new telemetry_utils_1.UsageError("Fluid object does not implement ISummarizer");
             }
             return summarizer;
         };
@@ -2387,11 +2525,11 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         // eslint-disable-next-line no-restricted-syntax
         for (const prop in configuration) {
             if (typeof configuration[prop] === "number" && configuration[prop] < 0) {
-                throw new container_utils_1.UsageError(`Summary heuristic configuration property "${prop}" cannot be less than 0`);
+                throw new telemetry_utils_1.UsageError(`Summary heuristic configuration property "${prop}" cannot be less than 0`);
             }
         }
         if (configuration.minIdleTime > configuration.maxIdleTime) {
-            throw new container_utils_1.UsageError(`"minIdleTime" [${configuration.minIdleTime}] cannot be greater than "maxIdleTime" [${configuration.maxIdleTime}]`);
+            throw new telemetry_utils_1.UsageError(`"minIdleTime" [${configuration.minIdleTime}] cannot be greater than "maxIdleTime" [${configuration.maxIdleTime}]`);
         }
     }
     get groupedBatchingEnabled() {