@fluidframework/container-runtime 2.0.0-internal.6.1.0 → 2.0.0-internal.6.2.0

package/lib/containerRuntime.js

@@ -1,10 +1,11 @@
 import { AttachState, LoaderHeader, } from "@fluidframework/container-definitions";
-import { assert, delay, Trace, TypedEventEmitter, unreachableCase, } from "@fluidframework/common-utils";
+import { assert, delay, Trace, TypedEventEmitter } from "@fluidframework/common-utils";
 import { LazyPromise } from "@fluidframework/core-utils";
-import { createChildLogger, raiseConnectedEvent, PerformanceEvent, TaggedLoggerAdapter, wrapError, createChildMonitoringContext, } from "@fluidframework/telemetry-utils";
+import { createChildLogger, createChildMonitoringContext, DataCorruptionError, DataProcessingError, GenericError, raiseConnectedEvent, PerformanceEvent, 
+// eslint-disable-next-line import/no-deprecated
+TaggedLoggerAdapter, wrapError, UsageError, } from "@fluidframework/telemetry-utils";
 import { DriverHeader, FetchSource, } from "@fluidframework/driver-definitions";
 import { readAndParse } from "@fluidframework/driver-utils";
-import { DataCorruptionError, DataProcessingError, GenericError, UsageError, } from "@fluidframework/container-utils";
 import { MessageType, SummaryType, } from "@fluidframework/protocol-definitions";
 import { FlushMode, FlushModeExperimental, gcTreeKey, channelsTreeName, } from "@fluidframework/runtime-definitions";
 import { addBlobToSummary, addSummarizeResultToSummary, addTreeToSummary, RequestParser, create404Response, exceptionToResponse, GCDataBuilder, requestFluidObject, seqFromTree, calculateStats, TelemetryContext, } from "@fluidframework/runtime-utils";
@@ -45,6 +46,18 @@ export var ContainerMessageType;
      */
     ContainerMessageType["IdAllocation"] = "idAllocation";
 })(ContainerMessageType || (ContainerMessageType = {}));
+/**
+ * Utility to implement compat behaviors given an unknown message type
+ * The parameters are typed to support compile-time enforcement of handling all known types/behaviors
+ *
+ * @param _unknownContainerRuntimeMessageType - Typed as never, to ensure all known types have been
+ * handled before calling this function (e.g. in a switch statement).
+ * @param compatBehavior - Typed redundantly with CompatModeBehavior to ensure handling is added when updating that type
+ */
+function compatBehaviorAllowsMessageType(_unknownContainerRuntimeMessageType, compatBehavior) {
+    // undefined defaults to same behavior as "FailToProcess"
+    return compatBehavior === "Ignore";
+}
 export const DefaultSummaryConfiguration = {
     state: "enabled",
     minIdleTime: 0,
@@ -71,8 +84,12 @@ export var RuntimeHeaders;
 })(RuntimeHeaders || (RuntimeHeaders = {}));
 /** True if a tombstoned object should be returned without erroring */
 export const AllowTombstoneRequestHeaderKey = "allowTombstone"; // Belongs in the enum above, but avoiding the breaking change
+/** [IRRELEVANT IF throwOnInactiveLoad OPTION NOT SET] True if an inactive object should be returned without erroring */
+export const AllowInactiveRequestHeaderKey = "allowInactive"; // Belongs in the enum above, but avoiding the breaking change
 /** Tombstone error responses will have this header set to true */
 export const TombstoneResponseHeaderKey = "isTombstoned";
+/** Inactive error responses will have this header set to true */
+export const InactiveResponseHeaderKey = "isInactive";
 /** Default values for Runtime Headers */
 export const defaultRuntimeHeaderData = {
     wait: true,
@@ -200,34 +217,6 @@ export class ContainerRuntime extends TypedEventEmitter {
             signalTimestamp: 0,
             trackingSignalSequenceNumber: undefined,
         };
-        this.summarizeOnDemand = (...args) => {
-            if (this.isSummarizerClient) {
-                return this.summarizer.summarizeOnDemand(...args);
-            }
-            else if (this.summaryManager !== undefined) {
-                return this.summaryManager.summarizeOnDemand(...args);
-            }
-            else {
-                // If we're not the summarizer, and we don't have a summaryManager, we expect that
-                // disableSummaries is turned on. We are throwing instead of returning a failure here,
-                // because it is a misuse of the API rather than an expected failure.
-                throw new UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
-            }
-        };
-        this.enqueueSummarize = (...args) => {
-            if (this.isSummarizerClient) {
-                return this.summarizer.enqueueSummarize(...args);
-            }
-            else if (this.summaryManager !== undefined) {
-                return this.summaryManager.enqueueSummarize(...args);
-            }
-            else {
-                // If we're not the summarizer, and we don't have a summaryManager, we expect that
-                // generateSummaries is turned off. We are throwing instead of returning a failure here,
-                // because it is a misuse of the API rather than an expected failure.
-                throw new UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
-            }
-        };
         const { options, clientDetails, connected, baseSnapshot, submitFn, submitBatchFn, submitSummaryFn, submitSignalFn, disposeFn, closeFn, deltaManager, quorum, audience, loader, pendingLocalState, supportedFeatures, } = context;
         this.innerDeltaManager = deltaManager;
         this.deltaManager = new DeltaManagerSummarizerProxy(this.innerDeltaManager);
@@ -450,8 +439,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 false;
         this.summaryCollection = new SummaryCollection(this.deltaManager, this.logger);
         this.dirtyContainer =
-            this.attachState !== AttachState.Attached ||
-                this.pendingStateManager.hasPendingMessages();
+            this.attachState !== AttachState.Attached || this.hasPendingMessages();
         context.updateDirtyContainerState(this.dirtyContainer);
         if (this.summariesDisabled) {
             this.mc.logger.sendTelemetryEvent({ eventName: "SummariesDisabled" });
@@ -499,6 +487,9 @@ export class ContainerRuntime extends TypedEventEmitter {
                 formExponentialFn({ coefficient: 20, initialDelay: 0 })), {
                     initialDelayMs: this.initialSummarizerDelayMs,
                 }, this.heuristicsDisabled);
+                this.summaryManager.on("summarize", (eventProps) => {
+                    this.emit("summarize", eventProps);
+                });
                 this.summaryManager.start();
             }
         }
@@ -607,11 +598,21 @@ export class ContainerRuntime extends TypedEventEmitter {
      * This object should provide all the functionality that the Container is expected to provide to the loader layer.
      */
     static async loadRuntime(params) {
-        const { context, registryEntries, existing, requestHandler, runtimeOptions = {}, containerScope = {}, containerRuntimeCtor = ContainerRuntime, initializeEntryPoint, } = params;
+        const { context, registryEntries, existing, requestHandler, runtimeOptions = {}, containerScope = {}, containerRuntimeCtor = ContainerRuntime, } = params;
+        const initializeEntryPoint = params.initializeEntryPoint ??
+            (async (containerRuntime) => ({
+                get IFluidRouter() {
+                    return this;
+                },
+                async request(req) {
+                    return containerRuntime.request(req);
+                },
+            }));
         // If taggedLogger exists, use it. Otherwise, wrap the vanilla logger:
         // back-compat: Remove the TaggedLoggerAdapter fallback once all the host are using loader > 0.45
         const backCompatContext = context;
         const passLogger = backCompatContext.taggedLogger ??
+            // eslint-disable-next-line import/no-deprecated
             new TaggedLoggerAdapter(backCompatContext.logger);
         const logger = createChildLogger({
             logger: passLogger,
@@ -673,7 +674,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             idCompressor =
                 serializedIdCompressor !== undefined
                     ? IdCompressor.deserialize(serializedIdCompressor, createSessionId())
-                    : new IdCompressor(createSessionId(), logger);
+                    : IdCompressor.create(logger);
         }
         const runtime = new containerRuntimeCtor(context, registry, metadata, electedSummarizerData, chunks ?? [], aliases ?? [], {
             summaryOptions,
@@ -688,6 +689,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             enableGroupedBatching,
         }, containerScope, logger, existing, blobManagerSnapshot, context.storage, idCompressor, requestHandler, undefined, // summaryConfiguration
         initializeEntryPoint);
+        await runtime.blobManager.processStashedChanges();
         // It's possible to have ops with a reference sequence number of 0. Op sequence numbers start
         // at 1, so we won't see a replayed saved op with a sequence number of 0.
         await runtime.pendingStateManager.applyStashedOpsAt(0);
@@ -704,6 +706,7 @@ export class ContainerRuntime extends TypedEventEmitter {
     /** @deprecated - The functionality is no longer exposed publicly */
     get reSubmitFn() {
         return (type, contents, localOpMetadata, opMetadata) => this.reSubmitCore({ type, contents }, localOpMetadata, opMetadata);
+        // Note: compatDetails is not included in this deprecated API
     }
     get flushMode() {
         return this._flushMode;
@@ -854,7 +857,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 // We always expect createSubRequest to include a leading slash, but asserting here to protect against
                 // unintentionally modifying the url if that changes.
                 assert(subRequest.url.startsWith("/"), 0x126 /* "Expected createSubRequest url to include a leading slash" */);
-                return dataStore.IFluidRouter.request(subRequest);
+                return dataStore.request(subRequest);
             }
             return create404Response(request);
         }
@@ -962,7 +965,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             this.mc.logger.sendTelemetryEvent({
                 eventName: "ReconnectsWithNoProgress",
                 attempts: this.consecutiveReconnects,
-                pendingMessages: this.pendingStateManager.pendingMessagesCount,
+                pendingMessages: this.pendingMessagesCount,
             });
         }
         return this.consecutiveReconnects < this.maxConsecutiveReconnects;
@@ -1022,13 +1025,13 @@ export class ContainerRuntime extends TypedEventEmitter {
      */
     parseOpContent(serializedContent) {
         assert(serializedContent !== undefined, 0x6d5 /* content must be defined */);
-        const { type, contents } = JSON.parse(serializedContent);
+        const { type, contents, compatDetails } = JSON.parse(serializedContent);
         assert(type !== undefined, 0x6d6 /* incorrect op content format */);
-        return { type, contents };
+        return { type, contents, compatDetails };
     }
     async applyStashedOp(op) {
         // Need to parse from string for back-compat
-        const { type, contents } = this.parseOpContent(op);
+        const { type, contents, compatDetails } = this.parseOpContent(op);
         switch (type) {
             case ContainerMessageType.FluidDataStoreOp:
                 return this.dataStores.applyStashedOp(contents);
@@ -1044,8 +1047,22 @@ export class ContainerRuntime extends TypedEventEmitter {
                 throw new Error("chunkedOp not expected here");
             case ContainerMessageType.Rejoin:
                 throw new Error("rejoin not expected here");
-            default:
-                unreachableCase(type, `Unknown ContainerMessageType: ${type}`);
+            default: {
+                // This should be extremely rare for stashed ops.
+                // It would require a newer runtime stashing ops and then an older one applying them,
+                // e.g. if an app rolled back its container version
+                const compatBehavior = compatDetails?.behavior;
+                if (!compatBehaviorAllowsMessageType(type, compatBehavior)) {
+                    const error = DataProcessingError.create("Stashed runtime message of unknown type", "applyStashedOp", undefined /* sequencedMessage */, {
+                        messageDetails: JSON.stringify({
+                            type,
+                            compatBehavior,
+                        }),
+                    });
+                    this.closeFn(error);
+                    throw error;
+                }
+            }
         }
     }
     setConnectionState(connected, clientId) {
@@ -1057,24 +1074,6 @@ export class ContainerRuntime extends TypedEventEmitter {
             // Don't propagate "disconnected" event because we didn't propagate the previous "connected" event
             return;
         }
-        // If attachment blobs were added while disconnected, we need to delay
-        // propagation of the "connected" event until we have uploaded them to
-        // ensure we don't submit ops referencing a blob that has not been uploaded
-        // Note that the inner (non-proxy) delta manager is needed here to get the readonly information.
-        const connecting = connected && !this._connected && !this.innerDeltaManager.readOnlyInfo.readonly;
-        if (connecting && this.blobManager.hasPendingOfflineUploads) {
-            assert(!this.delayConnectClientId, 0x392 /* Connect event delay must be canceled before subsequent connect event */);
-            assert(!!clientId, 0x393 /* Must have clientId when connecting */);
-            this.delayConnectClientId = clientId;
-            this.blobManager.onConnected().then(() => {
-                // make sure we didn't reconnect before the promise resolved
-                if (this.delayConnectClientId === clientId && !this.disposed) {
-                    this.delayConnectClientId = undefined;
-                    this.setConnectionStateCore(connected, clientId);
-                }
-            }, (error) => this.closeFn(error));
-            return;
-        }
         this.setConnectionStateCore(connected, clientId);
     }
     setConnectionStateCore(connected, clientId) {
@@ -1105,7 +1104,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 this.closeFn(DataProcessingError.create("Runtime detected too many reconnects with no progress syncing local ops.", "setConnectionState", undefined, {
                     dataLoss: 1,
                     attempts: this.consecutiveReconnects,
-                    pendingMessages: this.pendingStateManager.pendingMessagesCount,
+                    pendingMessages: this.pendingMessagesCount,
                 }));
                 return;
             }
@@ -1122,23 +1121,23 @@ export class ContainerRuntime extends TypedEventEmitter {
     }
     process(messageArg, local) {
         this.verifyNotClosed();
-        // Whether or not the message is actually a runtime message.
+        // Whether or not the message appears to be a runtime message from an up-to-date client.
         // It may be a legacy runtime message (ie already unpacked and ContainerMessageType)
         // or something different, like a system message.
-        const runtimeMessage = messageArg.type === MessageType.Operation;
+        const modernRuntimeMessage = messageArg.type === MessageType.Operation;
         // Do shallow copy of message, as the processing flow will modify it.
         const messageCopy = { ...messageArg };
         for (const message of this.remoteMessageProcessor.process(messageCopy)) {
-            this.processCore(message, local, runtimeMessage);
+            this.processCore(message, local, modernRuntimeMessage);
         }
     }
     /**
      * Direct the message to the correct subsystem for processing, and implement other side effects
      * @param message - The unpacked message. Likely a ContainerRuntimeMessage, but could also be a system op
      * @param local - Did this client send the op?
-     * @param runtimeMessage - Does this appear like a current ContainerRuntimeMessage?  If true, certain validation will occur.
+     * @param modernRuntimeMessage - Does this appear like a current ContainerRuntimeMessage?
      */
-    processCore(message, local, runtimeMessage) {
+    processCore(message, local, modernRuntimeMessage) {
         // Surround the actual processing of the operation with messages to the schedule manager indicating
         // the beginning and end. This allows it to emit appropriate events and/or pause the processing of new
         // messages once a batch has been fully processed.
@@ -1146,7 +1145,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         this._processedClientSequenceNumber = message.clientSequenceNumber;
         try {
             let localOpMetadata;
-            if (local && runtimeMessage && message.type !== ContainerMessageType.ChunkedOp) {
+            if (local && modernRuntimeMessage && message.type !== ContainerMessageType.ChunkedOp) {
                 localOpMetadata = this.pendingStateManager.processPendingLocalMessage(message);
             }
             // If there are no more pending messages after processing a local message,
@@ -1154,43 +1153,8 @@ export class ContainerRuntime extends TypedEventEmitter {
             if (!this.hasPendingMessages()) {
                 this.updateDocumentDirtyState(false);
             }
-            const type = message.type;
-            switch (type) {
-                case ContainerMessageType.Attach:
-                    this.dataStores.processAttachMessage(message, local);
-                    break;
-                case ContainerMessageType.Alias:
-                    this.processAliasMessage(message, localOpMetadata, local);
-                    break;
-                case ContainerMessageType.FluidDataStoreOp:
-                    this.dataStores.processFluidDataStoreOp(message, local, localOpMetadata);
-                    break;
-                case ContainerMessageType.BlobAttach:
-                    this.blobManager.processBlobAttachOp(message, local);
-                    break;
-                case ContainerMessageType.IdAllocation:
-                    assert(this.idCompressor !== undefined, 0x67c /* IdCompressor should be defined if enabled */);
-                    this.idCompressor.finalizeCreationRange(message.contents);
-                    break;
-                case ContainerMessageType.ChunkedOp:
-                case ContainerMessageType.Rejoin:
-                    break;
-                default:
-                    if (runtimeMessage) {
-                        const error = DataProcessingError.create(
-                        // Former assert 0x3ce
-                        "Runtime message of unknown type", "OpProcessing", message, {
-                            local,
-                            type: message.type,
-                            contentType: typeof message.contents,
-                            batch: message.metadata?.batch,
-                            compression: message.compression,
-                        });
-                        this.closeFn(error);
-                        throw error;
-                    }
-            }
-            this.emit("op", message, runtimeMessage);
+            this.validateAndProcessRuntimeMessage(message, localOpMetadata, local, modernRuntimeMessage);
+            this.emit("op", message, modernRuntimeMessage);
             this.scheduleManager.afterOpProcessing(undefined, message);
             if (local) {
                 // If we have processed a local op, this means that the container is
@@ -1204,8 +1168,59 @@ export class ContainerRuntime extends TypedEventEmitter {
             throw e;
         }
     }
-    processAliasMessage(message, localOpMetadata, local) {
-        this.dataStores.processAliasMessage(message, localOpMetadata, local);
+    /**
+     * Assuming the given message is also a ContainerRuntimeMessage,
+     * checks its type and dispatches the message to the appropriate handler in the runtime.
+     * Throws a DataProcessingError if the message doesn't conform to the ContainerRuntimeMessage type.
+     */
+    validateAndProcessRuntimeMessage(message, localOpMetadata, local, expectRuntimeMessageType) {
+        // Optimistically extract ContainerRuntimeMessage-specific props from the message
+        const { type: maybeContainerMessageType, compatDetails } = message;
+        switch (maybeContainerMessageType) {
+            case ContainerMessageType.Attach:
+                this.dataStores.processAttachMessage(message, local);
+                break;
+            case ContainerMessageType.Alias:
+                this.dataStores.processAliasMessage(message, localOpMetadata, local);
+                break;
+            case ContainerMessageType.FluidDataStoreOp:
+                this.dataStores.processFluidDataStoreOp(message, local, localOpMetadata);
+                break;
+            case ContainerMessageType.BlobAttach:
+                this.blobManager.processBlobAttachOp(message, local);
+                break;
+            case ContainerMessageType.IdAllocation:
+                assert(this.idCompressor !== undefined, 0x67c /* IdCompressor should be defined if enabled */);
+                this.idCompressor.finalizeCreationRange(message.contents);
+                break;
+            case ContainerMessageType.ChunkedOp:
+            case ContainerMessageType.Rejoin:
+                break;
+            default: {
+                // If we didn't necessarily expect a runtime message type, then no worries - just return
+                // e.g. this case applies to system ops, or legacy ops that would have fallen into the above cases anyway.
+                if (!expectRuntimeMessageType) {
+                    return;
+                }
+                const compatBehavior = compatDetails?.behavior;
+                if (!compatBehaviorAllowsMessageType(maybeContainerMessageType, compatBehavior)) {
+                    const error = DataProcessingError.create(
+                    // Former assert 0x3ce
+                    "Runtime message of unknown type", "OpProcessing", message, {
+                        local,
+                        messageDetails: JSON.stringify({
+                            type: message.type,
+                            contentType: typeof message.contents,
+                            compatBehavior,
+                            batch: message.metadata?.batch,
+                            compression: message.compression,
+                        }),
+                    });
+                    this.closeFn(error);
+                    throw error;
+                }
+            }
+        }
     }
     /**
      * Emits the Signal event and update the perf signal data.
@@ -1673,7 +1688,7 @@ export class ContainerRuntime extends TypedEventEmitter {
      * @param options - options controlling how the summary is generated or submitted
      */
     async submitSummary(options) {
-        const { fullTree = false, refreshLatestAck, summaryLogger } = options;
+        const { fullTree = false, finalAttempt = false, refreshLatestAck, summaryLogger } = options;
         // The summary number for this summary. This will be updated during the summary process, so get it now and
         // use it for all events logged during this summary.
         const summaryNumber = this.nextSummaryNumber;
@@ -1770,9 +1785,9 @@ export class ContainerRuntime extends TypedEventEmitter {
                     error,
                 };
             }
-            // If validateSummaryBeforeUpload is true, validate that the summary generated by the summarizer nodes is
-            // correct before this summary is uploaded.
+            // If validateSummaryBeforeUpload is true, validate that the summary generated is correct before uploading.
             if (this.validateSummaryBeforeUpload) {
+                // Validate that the summaries generated by summarize nodes is correct.
                 const validateResult = this.summarizerNode.validateSummary();
                 if (!validateResult.success) {
                     const { success, ...loggingProps } = validateResult;
@@ -1784,6 +1799,34 @@ export class ContainerRuntime extends TypedEventEmitter {
                         error,
                     };
                 }
+                // If there are pending messages, the summary has more data than at the summaryRefSeqNum.
+                if (this.hasPendingMessages()) {
+                    // If "SkipFailingIncorrectSummary" option is true, don't fail the summary in the last attempt.
+                    // This is a fallback to make progress in documents where there are consistently pending ops in
+                    // the summarizer.
+                    if (finalAttempt &&
+                        this.mc.config.getBoolean("Fluid.Summarizer.SkipFailingIncorrectSummary")) {
+                        const error = DataProcessingError.create("Pending ops during summarization", "submitSummary", undefined, { pendingMessages: this.pendingMessagesCount });
+                        summaryNumberLogger.sendErrorEvent({
+                            eventName: "SkipFailingIncorrectSummary",
+                            referenceSequenceNumber: summaryRefSeqNum,
+                            minimumSequenceNumber,
+                        }, error);
+                    }
+                    else {
+                        // Default retry delay is 1 second. This can be overridden via config so that we can adjust it
+                        // based on telemetry while we decide on a stable number.
+                        const retryDelayMs = this.mc.config.getNumber("Fluid.Summarizer.PendingOpsRetryDelayMs") ??
+                            1000;
+                        const error = new RetriableSummaryError("PendingMessagesInSummary", retryDelayMs / 1000, { count: this.pendingMessagesCount });
+                        return {
+                            stage: "base",
+                            referenceSequenceNumber: summaryRefSeqNum,
+                            minimumSequenceNumber,
+                            error,
+                        };
+                    }
+                }
             }
             const { summary: summaryTree, stats: partialStats } = summarizeResult;
             // Now that we have generated the summary, update the message at last summary to the last message processed.
@@ -1904,8 +1947,11 @@ export class ContainerRuntime extends TypedEventEmitter {
             }
         }
     }
+    get pendingMessagesCount() {
+        return this.pendingStateManager.pendingMessagesCount + this.outbox.messageCount;
+    }
     hasPendingMessages() {
-        return this.pendingStateManager.hasPendingMessages() || !this.outbox.isEmpty;
+        return this.pendingMessagesCount !== 0;
     }
     updateDocumentDirtyState(dirty) {
         if (this.attachState !== AttachState.Attached) {
@@ -1949,7 +1995,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 assert(this.idCompressor !== undefined, 0x67d /* IdCompressor should be defined if enabled */);
                 idRange = this.idCompressor.takeNextCreationRange();
                 // Don't include the idRange if there weren't any Ids allocated
-                idRange = idRange?.ids?.first !== undefined ? idRange : undefined;
+                idRange = idRange?.ids !== undefined ? idRange : undefined;
             }
             if (idRange !== undefined) {
                 const idAllocationMessage = {
@@ -2168,8 +2214,27 @@ export class ContainerRuntime extends TypedEventEmitter {
             case ContainerMessageType.Rejoin:
                 this.submit(message);
                 break;
-            default:
-                unreachableCase(message.type, `Unknown ContainerMessageType [type: ${message.type}]`);
+            default: {
+                // This case should be very rare - it would imply an op was stashed from a
+                // future version of runtime code and now is being applied on an older version
+                const compatBehavior = message.compatDetails?.behavior;
+                if (compatBehaviorAllowsMessageType(message.type, compatBehavior)) {
+                    this.logger.sendTelemetryEvent({
+                        eventName: "resubmitUnrecognizedMessageTypeAllowed",
+                        messageDetails: { type: message.type, compatBehavior },
+                    });
+                }
+                else {
+                    const error = DataProcessingError.create("Resubmitting runtime message of unknown type", "reSubmitCore", undefined /* sequencedMessage */, {
+                        messageDetails: JSON.stringify({
+                            type: message.type,
+                            compatBehavior,
+                        }),
+                    });
+                    this.closeFn(error);
+                    throw error;
+                }
+            }
         }
     }
     rollback(content, localOpMetadata) {
@@ -2182,6 +2247,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 this.dataStores.rollbackDataStoreOp(contents, localOpMetadata);
                 break;
             default:
+                // Don't check message.compatDetails because this is for rolling back a local op so the type will be known
                 throw new Error(`Can't rollback ${type}`);
         }
     }
@@ -2204,11 +2270,11 @@ export class ContainerRuntime extends TypedEventEmitter {
         // It should only be done by the summarizerNode, if required.
         // When fetching from storage we will always get the latest version and do not use the ackHandle.
         const fetchLatestSnapshot = async () => {
-            let fetchResult = await this.fetchLatestSnapshotFromStorage(summaryLogger, {
+            let fetchResult = await this.fetchSnapshotFromStorageAndMaybeClose(summaryLogger, {
                 eventName: "RefreshLatestSummaryAckFetch",
                 ackHandle,
                 targetSequenceNumber: summaryRefSeq,
-            }, readAndParseBlob);
+            }, readAndParseBlob, null);
             /**
              * back-compat - Older loaders and drivers (pre 2.0.0-internal.1.4) don't have fetchSource as a param in the
              * getVersions API. So, they will not fetch the latest snapshot from network in the previous fetch call. For
@@ -2216,7 +2282,7 @@ export class ContainerRuntime extends TypedEventEmitter {
              * change that started fetching latest snapshot always.
              */
             if (fetchResult.latestSnapshotRefSeq < summaryRefSeq) {
-                fetchResult = await this.fetchSnapshotFromStorageAndClose(summaryLogger, {
+                fetchResult = await this.fetchSnapshotFromStorageAndMaybeClose(summaryLogger, {
                     eventName: "RefreshLatestSummaryAckFetchBackCompat",
                     ackHandle,
                     targetSequenceNumber: summaryRefSeq,
@@ -2261,9 +2327,9 @@ export class ContainerRuntime extends TypedEventEmitter {
      */
     async refreshLatestSummaryAckFromServer(summaryLogger) {
         const readAndParseBlob = async (id) => readAndParse(this.storage, id);
-        const { snapshotTree, versionId, latestSnapshotRefSeq } = await this.fetchLatestSnapshotFromStorage(summaryLogger, {
+        const { snapshotTree, versionId, latestSnapshotRefSeq } = await this.fetchSnapshotFromStorageAndMaybeClose(summaryLogger, {
             eventName: "RefreshLatestSummaryFromServerFetch",
-        }, readAndParseBlob);
+        }, readAndParseBlob, null);
         const fetchLatestSnapshot = {
             snapshotTree,
             snapshotRefSeq: latestSnapshotRefSeq,
@@ -2273,10 +2339,12 @@ export class ContainerRuntime extends TypedEventEmitter {
         await this.garbageCollector.refreshLatestSummary(undefined /* proposalHandle */, result, readAndParseBlob);
         return { latestSnapshotRefSeq, latestSnapshotVersionId: versionId };
     }
-    async fetchLatestSnapshotFromStorage(logger, event, readAndParseBlob) {
-        return this.fetchSnapshotFromStorageAndClose(logger, event, readAndParseBlob, null /* latest */);
-    }
-    async fetchSnapshotFromStorageAndClose(logger, event, readAndParseBlob, versionId) {
+    /**
+     * Downloads snapshot from storage with the given versionId or latest if versionId is null.
+     * By default, it also closes the container after downloading the snapshot. However, this may be
+     * overridden via options.
+     */
+    async fetchSnapshotFromStorageAndMaybeClose(logger, event, readAndParseBlob, versionId) {
         const snapshotResults = await PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
             const stats = {};
             const trace = Trace.start();
@@ -2323,14 +2391,46 @@ export class ContainerRuntime extends TypedEventEmitter {
             throw new UsageError("can't get state during orderSequentially");
         }
         const pendingAttachmentBlobs = await this.blobManager.getPendingBlobs(waitBlobsToAttach);
+        if (!pendingAttachmentBlobs && !this.hasPendingMessages()) {
+            return; // no pending state to save
+        }
         // Flush pending batch.
         // getPendingLocalState() is only exposed through Container.closeAndGetPendingLocalState(), so it's safe
         // to close current batch.
         this.flush();
-        return {
+        const pendingState = {
             pending: this.pendingStateManager.getLocalState(),
             pendingAttachmentBlobs,
         };
+        return pendingState;
+    }
+    summarizeOnDemand(options) {
+        if (this.isSummarizerClient) {
+            return this.summarizer.summarizeOnDemand(options);
+        }
+        else if (this.summaryManager !== undefined) {
+            return this.summaryManager.summarizeOnDemand(options);
+        }
+        else {
+            // If we're not the summarizer, and we don't have a summaryManager, we expect that
+            // disableSummaries is turned on. We are throwing instead of returning a failure here,
+            // because it is a misuse of the API rather than an expected failure.
+            throw new UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
+        }
+    }
+    enqueueSummarize(options) {
+        if (this.isSummarizerClient) {
+            return this.summarizer.enqueueSummarize(options);
+        }
+        else if (this.summaryManager !== undefined) {
+            return this.summaryManager.enqueueSummarize(options);
+        }
+        else {
+            // If we're not the summarizer, and we don't have a summaryManager, we expect that
+            // generateSummaries is turned off. We are throwing instead of returning a failure here,
+            // because it is a misuse of the API rather than an expected failure.
+            throw new UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
+        }
     }
     /**
      * * Forms a function that will request a Summarizer.