@fluidframework/container-runtime 2.0.0-internal.6.1.0 → 2.0.0-internal.6.2.0

package/dist/containerRuntime.js

@@ -19,14 +19,13 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ContainerRuntime = exports.makeLegacySendBatchFn = exports.getDeviceSpec = exports.agentSchedulerId = exports.isRuntimeMessage = exports.RuntimeMessage = exports.CompressionAlgorithms = exports.defaultRuntimeHeaderData = exports.TombstoneResponseHeaderKey = exports.AllowTombstoneRequestHeaderKey = exports.RuntimeHeaders = exports.DefaultSummaryConfiguration = exports.ContainerMessageType = void 0;
+exports.ContainerRuntime = exports.makeLegacySendBatchFn = exports.getDeviceSpec = exports.agentSchedulerId = exports.isRuntimeMessage = exports.RuntimeMessage = exports.CompressionAlgorithms = exports.defaultRuntimeHeaderData = exports.InactiveResponseHeaderKey = exports.TombstoneResponseHeaderKey = exports.AllowInactiveRequestHeaderKey = exports.AllowTombstoneRequestHeaderKey = exports.RuntimeHeaders = exports.DefaultSummaryConfiguration = exports.ContainerMessageType = void 0;
 const container_definitions_1 = require("@fluidframework/container-definitions");
 const common_utils_1 = require("@fluidframework/common-utils");
 const core_utils_1 = require("@fluidframework/core-utils");
 const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
 const driver_definitions_1 = require("@fluidframework/driver-definitions");
 const driver_utils_1 = require("@fluidframework/driver-utils");
-const container_utils_1 = require("@fluidframework/container-utils");
 const protocol_definitions_1 = require("@fluidframework/protocol-definitions");
 const runtime_definitions_1 = require("@fluidframework/runtime-definitions");
 const runtime_utils_1 = require("@fluidframework/runtime-utils");
@@ -67,6 +66,18 @@ var ContainerMessageType;
      */
     ContainerMessageType["IdAllocation"] = "idAllocation";
 })(ContainerMessageType = exports.ContainerMessageType || (exports.ContainerMessageType = {}));
+/**
+ * Utility to implement compat behaviors given an unknown message type
+ * The parameters are typed to support compile-time enforcement of handling all known types/behaviors
+ *
+ * @param _unknownContainerRuntimeMessageType - Typed as never, to ensure all known types have been
+ * handled before calling this function (e.g. in a switch statement).
+ * @param compatBehavior - Typed redundantly with CompatModeBehavior to ensure handling is added when updating that type
+ */
+function compatBehaviorAllowsMessageType(_unknownContainerRuntimeMessageType, compatBehavior) {
+    // undefined defaults to same behavior as "FailToProcess"
+    return compatBehavior === "Ignore";
+}
 exports.DefaultSummaryConfiguration = {
     state: "enabled",
     minIdleTime: 0,
@@ -93,8 +104,12 @@ var RuntimeHeaders;
 })(RuntimeHeaders = exports.RuntimeHeaders || (exports.RuntimeHeaders = {}));
 /** True if a tombstoned object should be returned without erroring */
 exports.AllowTombstoneRequestHeaderKey = "allowTombstone"; // Belongs in the enum above, but avoiding the breaking change
+/** [IRRELEVANT IF throwOnInactiveLoad OPTION NOT SET] True if an inactive object should be returned without erroring */
+exports.AllowInactiveRequestHeaderKey = "allowInactive"; // Belongs in the enum above, but avoiding the breaking change
 /** Tombstone error responses will have this header set to true */
 exports.TombstoneResponseHeaderKey = "isTombstoned";
+/** Inactive error responses will have this header set to true */
+exports.InactiveResponseHeaderKey = "isInactive";
 /** Default values for Runtime Headers */
 exports.defaultRuntimeHeaderData = {
     wait: true,
@@ -225,34 +240,6 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             signalTimestamp: 0,
             trackingSignalSequenceNumber: undefined,
         };
-        this.summarizeOnDemand = (...args) => {
-            if (this.isSummarizerClient) {
-                return this.summarizer.summarizeOnDemand(...args);
-            }
-            else if (this.summaryManager !== undefined) {
-                return this.summaryManager.summarizeOnDemand(...args);
-            }
-            else {
-                // If we're not the summarizer, and we don't have a summaryManager, we expect that
-                // disableSummaries is turned on. We are throwing instead of returning a failure here,
-                // because it is a misuse of the API rather than an expected failure.
-                throw new container_utils_1.UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
-            }
-        };
-        this.enqueueSummarize = (...args) => {
-            if (this.isSummarizerClient) {
-                return this.summarizer.enqueueSummarize(...args);
-            }
-            else if (this.summaryManager !== undefined) {
-                return this.summaryManager.enqueueSummarize(...args);
-            }
-            else {
-                // If we're not the summarizer, and we don't have a summaryManager, we expect that
-                // generateSummaries is turned off. We are throwing instead of returning a failure here,
-                // because it is a misuse of the API rather than an expected failure.
-                throw new container_utils_1.UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
-            }
-        };
         const { options, clientDetails, connected, baseSnapshot, submitFn, submitBatchFn, submitSummaryFn, submitSignalFn, disposeFn, closeFn, deltaManager, quorum, audience, loader, pendingLocalState, supportedFeatures, } = context;
         this.innerDeltaManager = deltaManager;
         this.deltaManager = new deltaManagerSummarizerProxy_1.DeltaManagerSummarizerProxy(this.innerDeltaManager);
@@ -368,7 +355,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             // This is a runtime enforcement of what's already explicit in the policy's type itself,
             // which dictates the value is either undefined or exactly 5 days in ms.
             // As long as the actual value is less than 5 days, the assumptions GC makes here are valid.
-            throw new container_utils_1.UsageError("Driver's maximumCacheDurationMs policy cannot exceed 5 days");
+            throw new telemetry_utils_1.UsageError("Driver's maximumCacheDurationMs policy cannot exceed 5 days");
         }
         this.garbageCollector = gc_1.GarbageCollector.create({
             runtime: this,
@@ -475,8 +462,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 false;
         this.summaryCollection = new summary_1.SummaryCollection(this.deltaManager, this.logger);
         this.dirtyContainer =
-            this.attachState !== container_definitions_1.AttachState.Attached ||
-                this.pendingStateManager.hasPendingMessages();
+            this.attachState !== container_definitions_1.AttachState.Attached || this.hasPendingMessages();
         context.updateDirtyContainerState(this.dirtyContainer);
         if (this.summariesDisabled) {
             this.mc.logger.sendTelemetryEvent({ eventName: "SummariesDisabled" });
@@ -524,6 +510,9 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 (0, throttler_1.formExponentialFn)({ coefficient: 20, initialDelay: 0 })), {
                     initialDelayMs: this.initialSummarizerDelayMs,
                 }, this.heuristicsDisabled);
+                this.summaryManager.on("summarize", (eventProps) => {
+                    this.emit("summarize", eventProps);
+                });
                 this.summaryManager.start();
             }
         }
@@ -632,11 +621,21 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
      * This object should provide all the functionality that the Container is expected to provide to the loader layer.
      */
     static async loadRuntime(params) {
-        const { context, registryEntries, existing, requestHandler, runtimeOptions = {}, containerScope = {}, containerRuntimeCtor = ContainerRuntime, initializeEntryPoint, } = params;
+        const { context, registryEntries, existing, requestHandler, runtimeOptions = {}, containerScope = {}, containerRuntimeCtor = ContainerRuntime, } = params;
+        const initializeEntryPoint = params.initializeEntryPoint ??
+            (async (containerRuntime) => ({
+                get IFluidRouter() {
+                    return this;
+                },
+                async request(req) {
+                    return containerRuntime.request(req);
+                },
+            }));
         // If taggedLogger exists, use it. Otherwise, wrap the vanilla logger:
         // back-compat: Remove the TaggedLoggerAdapter fallback once all the host are using loader > 0.45
         const backCompatContext = context;
         const passLogger = backCompatContext.taggedLogger ??
+            // eslint-disable-next-line import/no-deprecated
             new telemetry_utils_1.TaggedLoggerAdapter(backCompatContext.logger);
         const logger = (0, telemetry_utils_1.createChildLogger)({
             logger: passLogger,
@@ -680,7 +679,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             if (loadSequenceNumberVerification !== "bypass" &&
                 runtimeSequenceNumber !== protocolSequenceNumber) {
                 // "Load from summary, runtime metadata sequenceNumber !== initialSequenceNumber"
-                const error = new container_utils_1.DataCorruptionError(
+                const error = new telemetry_utils_1.DataCorruptionError(
                 // pre-0.58 error message: SummaryMetadataMismatch
                 "Summary metadata mismatch", { runtimeVersion: packageVersion_1.pkgVersion, runtimeSequenceNumber, protocolSequenceNumber });
                 if (loadSequenceNumberVerification === "log") {
@@ -698,7 +697,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             idCompressor =
                 serializedIdCompressor !== undefined
                     ? IdCompressor.deserialize(serializedIdCompressor, createSessionId())
-                    : new IdCompressor(createSessionId(), logger);
+                    : IdCompressor.create(logger);
         }
         const runtime = new containerRuntimeCtor(context, registry, metadata, electedSummarizerData, chunks ?? [], aliases ?? [], {
             summaryOptions,
@@ -713,6 +712,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             enableGroupedBatching,
         }, containerScope, logger, existing, blobManagerSnapshot, context.storage, idCompressor, requestHandler, undefined, // summaryConfiguration
         initializeEntryPoint);
+        await runtime.blobManager.processStashedChanges();
         // It's possible to have ops with a reference sequence number of 0. Op sequence numbers start
         // at 1, so we won't see a replayed saved op with a sequence number of 0.
         await runtime.pendingStateManager.applyStashedOpsAt(0);
@@ -729,6 +729,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     /** @deprecated - The functionality is no longer exposed publicly */
     get reSubmitFn() {
         return (type, contents, localOpMetadata, opMetadata) => this.reSubmitCore({ type, contents }, localOpMetadata, opMetadata);
+        // Note: compatDetails is not included in this deprecated API
     }
     get flushMode() {
         return this._flushMode;
@@ -879,7 +880,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 // We always expect createSubRequest to include a leading slash, but asserting here to protect against
                 // unintentionally modifying the url if that changes.
                 (0, common_utils_1.assert)(subRequest.url.startsWith("/"), 0x126 /* "Expected createSubRequest url to include a leading slash" */);
-                return dataStore.IFluidRouter.request(subRequest);
+                return dataStore.request(subRequest);
             }
             return (0, runtime_utils_1.create404Response)(request);
         }
@@ -987,7 +988,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             this.mc.logger.sendTelemetryEvent({
                 eventName: "ReconnectsWithNoProgress",
                 attempts: this.consecutiveReconnects,
-                pendingMessages: this.pendingStateManager.pendingMessagesCount,
+                pendingMessages: this.pendingMessagesCount,
             });
         }
         return this.consecutiveReconnects < this.maxConsecutiveReconnects;
@@ -1047,13 +1048,13 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
      */
     parseOpContent(serializedContent) {
         (0, common_utils_1.assert)(serializedContent !== undefined, 0x6d5 /* content must be defined */);
-        const { type, contents } = JSON.parse(serializedContent);
+        const { type, contents, compatDetails } = JSON.parse(serializedContent);
         (0, common_utils_1.assert)(type !== undefined, 0x6d6 /* incorrect op content format */);
-        return { type, contents };
+        return { type, contents, compatDetails };
     }
     async applyStashedOp(op) {
         // Need to parse from string for back-compat
-        const { type, contents } = this.parseOpContent(op);
+        const { type, contents, compatDetails } = this.parseOpContent(op);
         switch (type) {
             case ContainerMessageType.FluidDataStoreOp:
                 return this.dataStores.applyStashedOp(contents);
@@ -1069,8 +1070,22 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 throw new Error("chunkedOp not expected here");
             case ContainerMessageType.Rejoin:
                 throw new Error("rejoin not expected here");
-            default:
-                (0, common_utils_1.unreachableCase)(type, `Unknown ContainerMessageType: ${type}`);
+            default: {
+                // This should be extremely rare for stashed ops.
+                // It would require a newer runtime stashing ops and then an older one applying them,
+                // e.g. if an app rolled back its container version
+                const compatBehavior = compatDetails?.behavior;
+                if (!compatBehaviorAllowsMessageType(type, compatBehavior)) {
+                    const error = telemetry_utils_1.DataProcessingError.create("Stashed runtime message of unknown type", "applyStashedOp", undefined /* sequencedMessage */, {
+                        messageDetails: JSON.stringify({
+                            type,
+                            compatBehavior,
+                        }),
+                    });
+                    this.closeFn(error);
+                    throw error;
+                }
+            }
         }
     }
     setConnectionState(connected, clientId) {
@@ -1082,24 +1097,6 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             // Don't propagate "disconnected" event because we didn't propagate the previous "connected" event
             return;
         }
-        // If attachment blobs were added while disconnected, we need to delay
-        // propagation of the "connected" event until we have uploaded them to
-        // ensure we don't submit ops referencing a blob that has not been uploaded
-        // Note that the inner (non-proxy) delta manager is needed here to get the readonly information.
-        const connecting = connected && !this._connected && !this.innerDeltaManager.readOnlyInfo.readonly;
-        if (connecting && this.blobManager.hasPendingOfflineUploads) {
-            (0, common_utils_1.assert)(!this.delayConnectClientId, 0x392 /* Connect event delay must be canceled before subsequent connect event */);
-            (0, common_utils_1.assert)(!!clientId, 0x393 /* Must have clientId when connecting */);
-            this.delayConnectClientId = clientId;
-            this.blobManager.onConnected().then(() => {
-                // make sure we didn't reconnect before the promise resolved
-                if (this.delayConnectClientId === clientId && !this.disposed) {
-                    this.delayConnectClientId = undefined;
-                    this.setConnectionStateCore(connected, clientId);
-                }
-            }, (error) => this.closeFn(error));
-            return;
-        }
         this.setConnectionStateCore(connected, clientId);
     }
     setConnectionStateCore(connected, clientId) {
@@ -1127,10 +1124,10 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         if (reconnection) {
             this.consecutiveReconnects++;
             if (!this.shouldContinueReconnecting()) {
-                this.closeFn(container_utils_1.DataProcessingError.create("Runtime detected too many reconnects with no progress syncing local ops.", "setConnectionState", undefined, {
+                this.closeFn(telemetry_utils_1.DataProcessingError.create("Runtime detected too many reconnects with no progress syncing local ops.", "setConnectionState", undefined, {
                     dataLoss: 1,
                     attempts: this.consecutiveReconnects,
-                    pendingMessages: this.pendingStateManager.pendingMessagesCount,
+                    pendingMessages: this.pendingMessagesCount,
                 }));
                 return;
             }
@@ -1147,23 +1144,23 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     }
     process(messageArg, local) {
         this.verifyNotClosed();
-        // Whether or not the message is actually a runtime message.
+        // Whether or not the message appears to be a runtime message from an up-to-date client.
         // It may be a legacy runtime message (ie already unpacked and ContainerMessageType)
         // or something different, like a system message.
-        const runtimeMessage = messageArg.type === protocol_definitions_1.MessageType.Operation;
+        const modernRuntimeMessage = messageArg.type === protocol_definitions_1.MessageType.Operation;
         // Do shallow copy of message, as the processing flow will modify it.
         const messageCopy = { ...messageArg };
         for (const message of this.remoteMessageProcessor.process(messageCopy)) {
-            this.processCore(message, local, runtimeMessage);
+            this.processCore(message, local, modernRuntimeMessage);
         }
     }
     /**
      * Direct the message to the correct subsystem for processing, and implement other side effects
      * @param message - The unpacked message. Likely a ContainerRuntimeMessage, but could also be a system op
      * @param local - Did this client send the op?
-     * @param runtimeMessage - Does this appear like a current ContainerRuntimeMessage?  If true, certain validation will occur.
+     * @param modernRuntimeMessage - Does this appear like a current ContainerRuntimeMessage?
      */
-    processCore(message, local, runtimeMessage) {
+    processCore(message, local, modernRuntimeMessage) {
         // Surround the actual processing of the operation with messages to the schedule manager indicating
         // the beginning and end. This allows it to emit appropriate events and/or pause the processing of new
         // messages once a batch has been fully processed.
@@ -1171,7 +1168,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         this._processedClientSequenceNumber = message.clientSequenceNumber;
         try {
             let localOpMetadata;
-            if (local && runtimeMessage && message.type !== ContainerMessageType.ChunkedOp) {
+            if (local && modernRuntimeMessage && message.type !== ContainerMessageType.ChunkedOp) {
                 localOpMetadata = this.pendingStateManager.processPendingLocalMessage(message);
             }
             // If there are no more pending messages after processing a local message,
@@ -1179,43 +1176,8 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             if (!this.hasPendingMessages()) {
                 this.updateDocumentDirtyState(false);
             }
-            const type = message.type;
-            switch (type) {
-                case ContainerMessageType.Attach:
-                    this.dataStores.processAttachMessage(message, local);
-                    break;
-                case ContainerMessageType.Alias:
-                    this.processAliasMessage(message, localOpMetadata, local);
-                    break;
-                case ContainerMessageType.FluidDataStoreOp:
-                    this.dataStores.processFluidDataStoreOp(message, local, localOpMetadata);
-                    break;
-                case ContainerMessageType.BlobAttach:
-                    this.blobManager.processBlobAttachOp(message, local);
-                    break;
-                case ContainerMessageType.IdAllocation:
-                    (0, common_utils_1.assert)(this.idCompressor !== undefined, 0x67c /* IdCompressor should be defined if enabled */);
-                    this.idCompressor.finalizeCreationRange(message.contents);
-                    break;
-                case ContainerMessageType.ChunkedOp:
-                case ContainerMessageType.Rejoin:
-                    break;
-                default:
-                    if (runtimeMessage) {
-                        const error = container_utils_1.DataProcessingError.create(
-                        // Former assert 0x3ce
-                        "Runtime message of unknown type", "OpProcessing", message, {
-                            local,
-                            type: message.type,
-                            contentType: typeof message.contents,
-                            batch: message.metadata?.batch,
-                            compression: message.compression,
-                        });
-                        this.closeFn(error);
-                        throw error;
-                    }
-            }
-            this.emit("op", message, runtimeMessage);
+            this.validateAndProcessRuntimeMessage(message, localOpMetadata, local, modernRuntimeMessage);
+            this.emit("op", message, modernRuntimeMessage);
             this.scheduleManager.afterOpProcessing(undefined, message);
             if (local) {
                 // If we have processed a local op, this means that the container is
@@ -1229,8 +1191,59 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             throw e;
         }
     }
-    processAliasMessage(message, localOpMetadata, local) {
-        this.dataStores.processAliasMessage(message, localOpMetadata, local);
+    /**
+     * Assuming the given message is also a ContainerRuntimeMessage,
+     * checks its type and dispatches the message to the appropriate handler in the runtime.
+     * Throws a DataProcessingError if the message doesn't conform to the ContainerRuntimeMessage type.
+     */
+    validateAndProcessRuntimeMessage(message, localOpMetadata, local, expectRuntimeMessageType) {
+        // Optimistically extract ContainerRuntimeMessage-specific props from the message
+        const { type: maybeContainerMessageType, compatDetails } = message;
+        switch (maybeContainerMessageType) {
+            case ContainerMessageType.Attach:
+                this.dataStores.processAttachMessage(message, local);
+                break;
+            case ContainerMessageType.Alias:
+                this.dataStores.processAliasMessage(message, localOpMetadata, local);
+                break;
+            case ContainerMessageType.FluidDataStoreOp:
+                this.dataStores.processFluidDataStoreOp(message, local, localOpMetadata);
+                break;
+            case ContainerMessageType.BlobAttach:
+                this.blobManager.processBlobAttachOp(message, local);
+                break;
+            case ContainerMessageType.IdAllocation:
+                (0, common_utils_1.assert)(this.idCompressor !== undefined, 0x67c /* IdCompressor should be defined if enabled */);
+                this.idCompressor.finalizeCreationRange(message.contents);
+                break;
+            case ContainerMessageType.ChunkedOp:
+            case ContainerMessageType.Rejoin:
+                break;
+            default: {
+                // If we didn't necessarily expect a runtime message type, then no worries - just return
+                // e.g. this case applies to system ops, or legacy ops that would have fallen into the above cases anyway.
+                if (!expectRuntimeMessageType) {
+                    return;
+                }
+                const compatBehavior = compatDetails?.behavior;
+                if (!compatBehaviorAllowsMessageType(maybeContainerMessageType, compatBehavior)) {
+                    const error = telemetry_utils_1.DataProcessingError.create(
+                    // Former assert 0x3ce
+                    "Runtime message of unknown type", "OpProcessing", message, {
+                        local,
+                        messageDetails: JSON.stringify({
+                            type: message.type,
+                            contentType: typeof message.contents,
+                            compatBehavior,
+                            batch: message.metadata?.batch,
+                            compression: message.compression,
+                        }),
+                    });
+                    this.closeFn(error);
+                    throw error;
+                }
+            }
+        }
     }
     /**
      * Emits the Signal event and update the perf signal data.
@@ -1331,7 +1344,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 }
                 catch (err) {
                     const error2 = (0, telemetry_utils_1.wrapError)(err, (message) => {
-                        return container_utils_1.DataProcessingError.create(`RollbackError: ${message}`, "checkpointRollback", undefined);
+                        return telemetry_utils_1.DataProcessingError.create(`RollbackError: ${message}`, "checkpointRollback", undefined);
                     });
                     this.closeFn(error2);
                     throw error2;
@@ -1339,7 +1352,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             }
             else {
                 // pre-0.58 error message: orderSequentiallyCallbackException
-                this.closeFn(new container_utils_1.GenericError("orderSequentially callback exception", error));
+                this.closeFn(new telemetry_utils_1.GenericError("orderSequentially callback exception", error));
             }
             throw error; // throw the original error for the consumer of the runtime
         }
@@ -1368,13 +1381,13 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         }
         const channel = await context.realize();
         if (channel.entryPoint === undefined) {
-            throw new container_utils_1.UsageError("entryPoint must be defined on data store runtime for using getAliasedDataStoreEntryPoint");
+            throw new telemetry_utils_1.UsageError("entryPoint must be defined on data store runtime for using getAliasedDataStoreEntryPoint");
         }
         return channel.entryPoint;
     }
     createDetachedRootDataStore(pkg, rootDataStoreId) {
         if (rootDataStoreId.includes("/")) {
-            throw new container_utils_1.UsageError(`Id cannot contain slashes: '${rootDataStoreId}'`);
+            throw new telemetry_utils_1.UsageError(`Id cannot contain slashes: '${rootDataStoreId}'`);
         }
         return this.dataStores.createDetachedDataStoreCore(pkg, true, rootDataStoreId);
     }
@@ -1698,7 +1711,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
      * @param options - options controlling how the summary is generated or submitted
      */
     async submitSummary(options) {
-        const { fullTree = false, refreshLatestAck, summaryLogger } = options;
+        const { fullTree = false, finalAttempt = false, refreshLatestAck, summaryLogger } = options;
         // The summary number for this summary. This will be updated during the summary process, so get it now and
         // use it for all events logged during this summary.
         const summaryNumber = this.nextSummaryNumber;
@@ -1795,9 +1808,9 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                     error,
                 };
             }
-            // If validateSummaryBeforeUpload is true, validate that the summary generated by the summarizer nodes is
-            // correct before this summary is uploaded.
+            // If validateSummaryBeforeUpload is true, validate that the summary generated is correct before uploading.
             if (this.validateSummaryBeforeUpload) {
+                // Validate that the summaries generated by summarize nodes is correct.
                 const validateResult = this.summarizerNode.validateSummary();
                 if (!validateResult.success) {
                     const { success, ...loggingProps } = validateResult;
@@ -1809,6 +1822,34 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                         error,
                     };
                 }
+                // If there are pending messages, the summary has more data than at the summaryRefSeqNum.
+                if (this.hasPendingMessages()) {
+                    // If "SkipFailingIncorrectSummary" option is true, don't fail the summary in the last attempt.
+                    // This is a fallback to make progress in documents where there are consistently pending ops in
+                    // the summarizer.
+                    if (finalAttempt &&
+                        this.mc.config.getBoolean("Fluid.Summarizer.SkipFailingIncorrectSummary")) {
+                        const error = telemetry_utils_1.DataProcessingError.create("Pending ops during summarization", "submitSummary", undefined, { pendingMessages: this.pendingMessagesCount });
+                        summaryNumberLogger.sendErrorEvent({
+                            eventName: "SkipFailingIncorrectSummary",
+                            referenceSequenceNumber: summaryRefSeqNum,
+                            minimumSequenceNumber,
+                        }, error);
+                    }
+                    else {
+                        // Default retry delay is 1 second. This can be overridden via config so that we can adjust it
+                        // based on telemetry while we decide on a stable number.
+                        const retryDelayMs = this.mc.config.getNumber("Fluid.Summarizer.PendingOpsRetryDelayMs") ??
+                            1000;
+                        const error = new summary_1.RetriableSummaryError("PendingMessagesInSummary", retryDelayMs / 1000, { count: this.pendingMessagesCount });
+                        return {
+                            stage: "base",
+                            referenceSequenceNumber: summaryRefSeqNum,
+                            minimumSequenceNumber,
+                            error,
+                        };
+                    }
+                }
             }
             const { summary: summaryTree, stats: partialStats } = summarizeResult;
             // Now that we have generated the summary, update the message at last summary to the last message processed.
@@ -1929,8 +1970,11 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             }
         }
     }
+    get pendingMessagesCount() {
+        return this.pendingStateManager.pendingMessagesCount + this.outbox.messageCount;
+    }
     hasPendingMessages() {
-        return this.pendingStateManager.hasPendingMessages() || !this.outbox.isEmpty;
+        return this.pendingMessagesCount !== 0;
     }
     updateDocumentDirtyState(dirty) {
         if (this.attachState !== container_definitions_1.AttachState.Attached) {
@@ -1958,7 +2002,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
     submitDataStoreAliasOp(contents, localOpMetadata) {
         const aliasMessage = contents;
         if (!(0, dataStore_1.isDataStoreAliasMessage)(aliasMessage)) {
-            throw new container_utils_1.UsageError("malformedDataStoreAliasMessage");
+            throw new telemetry_utils_1.UsageError("malformedDataStoreAliasMessage");
         }
         this.submit({ type: ContainerMessageType.Alias, contents }, localOpMetadata);
     }
@@ -1974,7 +2018,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 (0, common_utils_1.assert)(this.idCompressor !== undefined, 0x67d /* IdCompressor should be defined if enabled */);
                 idRange = this.idCompressor.takeNextCreationRange();
                 // Don't include the idRange if there weren't any Ids allocated
-                idRange = idRange?.ids?.first !== undefined ? idRange : undefined;
+                idRange = idRange?.ids !== undefined ? idRange : undefined;
             }
             if (idRange !== undefined) {
                 const idAllocationMessage = {
@@ -2127,7 +2171,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             if (this.opReentryCallsToReport > 0) {
                 this.mc.logger.sendTelemetryEvent({ eventName: "OpReentry" }, 
                 // We need to capture the call stack in order to inspect the source of this usage pattern
-                (0, opLifecycle_1.getLongStack)(() => new container_utils_1.UsageError(errorMessage)));
+                (0, opLifecycle_1.getLongStack)(() => new telemetry_utils_1.UsageError(errorMessage)));
                 this.opReentryCallsToReport--;
             }
             // Creating ops while processing ops can lead
@@ -2143,7 +2187,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             // The runtime must enforce op coherence by not allowing ops to be submitted
             // while ops are being processed.
             if (this.enableOpReentryCheck) {
-                throw new container_utils_1.UsageError(errorMessage);
+                throw new telemetry_utils_1.UsageError(errorMessage);
             }
         }
     }
@@ -2193,8 +2237,27 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             case ContainerMessageType.Rejoin:
                 this.submit(message);
                 break;
-            default:
-                (0, common_utils_1.unreachableCase)(message.type, `Unknown ContainerMessageType [type: ${message.type}]`);
+            default: {
+                // This case should be very rare - it would imply an op was stashed from a
+                // future version of runtime code and now is being applied on an older version
+                const compatBehavior = message.compatDetails?.behavior;
+                if (compatBehaviorAllowsMessageType(message.type, compatBehavior)) {
+                    this.logger.sendTelemetryEvent({
+                        eventName: "resubmitUnrecognizedMessageTypeAllowed",
+                        messageDetails: { type: message.type, compatBehavior },
+                    });
+                }
+                else {
+                    const error = telemetry_utils_1.DataProcessingError.create("Resubmitting runtime message of unknown type", "reSubmitCore", undefined /* sequencedMessage */, {
+                        messageDetails: JSON.stringify({
+                            type: message.type,
+                            compatBehavior,
+                        }),
+                    });
+                    this.closeFn(error);
+                    throw error;
+                }
+            }
         }
     }
     rollback(content, localOpMetadata) {
@@ -2207,6 +2270,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 this.dataStores.rollbackDataStoreOp(contents, localOpMetadata);
                 break;
             default:
+                // Don't check message.compatDetails because this is for rolling back a local op so the type will be known
                 throw new Error(`Can't rollback ${type}`);
         }
     }
@@ -2229,11 +2293,11 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         // It should only be done by the summarizerNode, if required.
         // When fetching from storage we will always get the latest version and do not use the ackHandle.
         const fetchLatestSnapshot = async () => {
-            let fetchResult = await this.fetchLatestSnapshotFromStorage(summaryLogger, {
+            let fetchResult = await this.fetchSnapshotFromStorageAndMaybeClose(summaryLogger, {
                 eventName: "RefreshLatestSummaryAckFetch",
                 ackHandle,
                 targetSequenceNumber: summaryRefSeq,
-            }, readAndParseBlob);
+            }, readAndParseBlob, null);
             /**
              * back-compat - Older loaders and drivers (pre 2.0.0-internal.1.4) don't have fetchSource as a param in the
              * getVersions API. So, they will not fetch the latest snapshot from network in the previous fetch call. For
@@ -2241,7 +2305,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
              * change that started fetching latest snapshot always.
              */
             if (fetchResult.latestSnapshotRefSeq < summaryRefSeq) {
-                fetchResult = await this.fetchSnapshotFromStorageAndClose(summaryLogger, {
+                fetchResult = await this.fetchSnapshotFromStorageAndMaybeClose(summaryLogger, {
                     eventName: "RefreshLatestSummaryAckFetchBackCompat",
                     ackHandle,
                     targetSequenceNumber: summaryRefSeq,
@@ -2258,7 +2322,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
              * state.
              */
             if (fetchResult.latestSnapshotRefSeq < summaryRefSeq) {
-                const error = container_utils_1.DataProcessingError.create("Fetched snapshot is older than the received ack", "RefreshLatestSummaryAck", undefined /* sequencedMessage */, {
+                const error = telemetry_utils_1.DataProcessingError.create("Fetched snapshot is older than the received ack", "RefreshLatestSummaryAck", undefined /* sequencedMessage */, {
                     ackHandle,
                     summaryRefSeq,
                     fetchedSnapshotRefSeq: fetchResult.latestSnapshotRefSeq,
@@ -2286,9 +2350,9 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
      */
     async refreshLatestSummaryAckFromServer(summaryLogger) {
         const readAndParseBlob = async (id) => (0, driver_utils_1.readAndParse)(this.storage, id);
-        const { snapshotTree, versionId, latestSnapshotRefSeq } = await this.fetchLatestSnapshotFromStorage(summaryLogger, {
+        const { snapshotTree, versionId, latestSnapshotRefSeq } = await this.fetchSnapshotFromStorageAndMaybeClose(summaryLogger, {
             eventName: "RefreshLatestSummaryFromServerFetch",
-        }, readAndParseBlob);
+        }, readAndParseBlob, null);
         const fetchLatestSnapshot = {
             snapshotTree,
             snapshotRefSeq: latestSnapshotRefSeq,
@@ -2298,10 +2362,12 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         await this.garbageCollector.refreshLatestSummary(undefined /* proposalHandle */, result, readAndParseBlob);
         return { latestSnapshotRefSeq, latestSnapshotVersionId: versionId };
     }
-    async fetchLatestSnapshotFromStorage(logger, event, readAndParseBlob) {
-        return this.fetchSnapshotFromStorageAndClose(logger, event, readAndParseBlob, null /* latest */);
-    }
-    async fetchSnapshotFromStorageAndClose(logger, event, readAndParseBlob, versionId) {
+    /**
+     * Downloads snapshot from storage with the given versionId or latest if versionId is null.
+     * By default, it also closes the container after downloading the snapshot. However, this may be
+     * overridden via options.
+     */
+    async fetchSnapshotFromStorageAndMaybeClose(logger, event, readAndParseBlob, versionId) {
         const snapshotResults = await telemetry_utils_1.PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
             const stats = {};
             const trace = common_utils_1.Trace.start();
@@ -2332,7 +2398,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
                 message: "Stopping fetch from storage",
                 versionId: versionId != null ? versionId : undefined,
                 closeSummarizerDelayMs: this.closeSummarizerDelayMs,
-            }, new container_utils_1.GenericError("Restarting summarizer instead of refreshing"));
+            }, new telemetry_utils_1.GenericError("Restarting summarizer instead of refreshing"));
             // Delay before restarting summarizer to prevent the summarizer from restarting too frequently.
             await (0, common_utils_1.delay)(this.closeSummarizerDelayMs);
             this._summarizer?.stop("latestSummaryStateStale");
@@ -2345,17 +2411,49 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         this.verifyNotClosed();
         const waitBlobsToAttach = props?.notifyImminentClosure;
         if (this._orderSequentiallyCalls !== 0) {
-            throw new container_utils_1.UsageError("can't get state during orderSequentially");
+            throw new telemetry_utils_1.UsageError("can't get state during orderSequentially");
         }
         const pendingAttachmentBlobs = await this.blobManager.getPendingBlobs(waitBlobsToAttach);
+        if (!pendingAttachmentBlobs && !this.hasPendingMessages()) {
+            return; // no pending state to save
+        }
         // Flush pending batch.
         // getPendingLocalState() is only exposed through Container.closeAndGetPendingLocalState(), so it's safe
         // to close current batch.
         this.flush();
-        return {
+        const pendingState = {
             pending: this.pendingStateManager.getLocalState(),
             pendingAttachmentBlobs,
         };
+        return pendingState;
+    }
+    summarizeOnDemand(options) {
+        if (this.isSummarizerClient) {
+            return this.summarizer.summarizeOnDemand(options);
+        }
+        else if (this.summaryManager !== undefined) {
+            return this.summaryManager.summarizeOnDemand(options);
+        }
+        else {
+            // If we're not the summarizer, and we don't have a summaryManager, we expect that
+            // disableSummaries is turned on. We are throwing instead of returning a failure here,
+            // because it is a misuse of the API rather than an expected failure.
+            throw new telemetry_utils_1.UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
+        }
+    }
+    enqueueSummarize(options) {
+        if (this.isSummarizerClient) {
+            return this.summarizer.enqueueSummarize(options);
+        }
+        else if (this.summaryManager !== undefined) {
+            return this.summaryManager.enqueueSummarize(options);
+        }
+        else {
+            // If we're not the summarizer, and we don't have a summaryManager, we expect that
+            // generateSummaries is turned off. We are throwing instead of returning a failure here,
+            // because it is a misuse of the API rather than an expected failure.
+            throw new telemetry_utils_1.UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
+        }
     }
     /**
      * * Forms a function that will request a Summarizer.
@@ -2378,7 +2476,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
             const fluidObject = await (0, runtime_utils_1.requestFluidObject)(loaderRouter, request);
             const summarizer = fluidObject.ISummarizer;
             if (!summarizer) {
-                throw new container_utils_1.UsageError("Fluid object does not implement ISummarizer");
+                throw new telemetry_utils_1.UsageError("Fluid object does not implement ISummarizer");
             }
             return summarizer;
         };
@@ -2387,11 +2485,11 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
         // eslint-disable-next-line no-restricted-syntax
         for (const prop in configuration) {
             if (typeof configuration[prop] === "number" && configuration[prop] < 0) {
-                throw new container_utils_1.UsageError(`Summary heuristic configuration property "${prop}" cannot be less than 0`);
+                throw new telemetry_utils_1.UsageError(`Summary heuristic configuration property "${prop}" cannot be less than 0`);
             }
         }
         if (configuration.minIdleTime > configuration.maxIdleTime) {
-            throw new container_utils_1.UsageError(`"minIdleTime" [${configuration.minIdleTime}] cannot be greater than "maxIdleTime" [${configuration.maxIdleTime}]`);
+            throw new telemetry_utils_1.UsageError(`"minIdleTime" [${configuration.minIdleTime}] cannot be greater than "maxIdleTime" [${configuration.maxIdleTime}]`);
         }
     }
     get groupedBatchingEnabled() {