@fluidframework/container-runtime 2.0.0-dev-rc.4.0.0.261659 → 2.0.0-dev-rc.5.0.0.265721

package/lib/containerRuntime.js

@@ -22,7 +22,7 @@ import { ReportOpPerfTelemetry } from "./connectionTelemetry.js";
 import { ContainerFluidHandleContext } from "./containerHandleContext.js";
 import { channelToDataStore } from "./dataStore.js";
 import { FluidDataStoreRegistry } from "./dataStoreRegistry.js";
-import { DeltaManagerSummarizerProxy } from "./deltaManagerSummarizerProxy.js";
+import { DeltaManagerPendingOpsProxy, DeltaManagerSummarizerProxy } from "./deltaManagerProxies.js";
 import { GCNodeType, GarbageCollector, gcGenerationOptionName, } from "./gc/index.js";
 import { ContainerMessageType, } from "./messageTypes.js";
 import { OpCompressor, OpDecompressor, OpGroupingManager, OpSplitter, Outbox, RemoteMessageProcessor, } from "./opLifecycle/index.js";
@@ -49,13 +49,13 @@ function compatBehaviorAllowsMessageType(_unknownContainerRuntimeMessageType, co
 export const DefaultSummaryConfiguration = {
     state: "enabled",
     minIdleTime: 0,
-    maxIdleTime: 30 * 1000,
-    maxTime: 60 * 1000,
-    maxOps: 100,
+    maxIdleTime: 30 * 1000, // 30 secs.
+    maxTime: 60 * 1000, // 1 min.
+    maxOps: 100, // Summarize if 100 weighted ops received since last snapshot.
     minOpsForLastSummaryAttempt: 10,
-    maxAckWaitTime: 3 * 60 * 1000,
+    maxAckWaitTime: 3 * 60 * 1000, // 3 mins.
     maxOpsSinceLastSummary: 7000,
-    initialSummarizerDelayMs: 5 * 1000,
+    initialSummarizerDelayMs: 5 * 1000, // 5 secs.
     nonRuntimeOpWeight: 0.1,
     runtimeOpWeight: 1.0,
     nonRuntimeHeuristicThreshold: 20,
@@ -246,7 +246,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             },
         });
         const mc = loggerToMonitoringContext(logger);
-        const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", flushMode = defaultFlushMode, compressionOptions = defaultCompressionConfig, maxBatchSizeInBytes = defaultMaxBatchSizeInBytes, enableRuntimeIdCompressor, chunkSizeInBytes = defaultChunkSizeInBytes, enableGroupedBatching = false, explicitSchemaControl = false, } = runtimeOptions;
+        const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", flushMode = defaultFlushMode, compressionOptions = defaultCompressionConfig, maxBatchSizeInBytes = defaultMaxBatchSizeInBytes, enableRuntimeIdCompressor, chunkSizeInBytes = defaultChunkSizeInBytes, enableGroupedBatching = true, explicitSchemaControl = false, } = runtimeOptions;
         const registry = new FluidDataStoreRegistry(registryEntries);
         const tryFetchBlob = async (blobName) => {
             const blobId = context.baseSnapshot?.blobs[blobName];
@@ -368,23 +368,20 @@ export class ContainerRuntime extends TypedEventEmitter {
                 return createIdCompressor(compressorLogger);
             }
         };
-        const disableGroupedBatching = mc.config.getBoolean("Fluid.ContainerRuntime.DisableGroupedBatching");
         const disableCompression = mc.config.getBoolean("Fluid.ContainerRuntime.CompressionDisabled");
         const compressionLz4 = disableCompression !== true &&
             compressionOptions.minimumBatchSizeInBytes !== Infinity &&
             compressionOptions.compressionAlgorithm === "lz4";
-        const opGroupingEnabled = disableGroupedBatching !== true && enableGroupedBatching;
         const documentSchemaController = new DocumentsSchemaController(existing, protocolSequenceNumber, metadata?.documentSchema, {
             explicitSchemaControl,
             compressionLz4,
             idCompressorMode,
-            opGroupingEnabled,
+            opGroupingEnabled: enableGroupedBatching,
             disallowedVersions: [],
         }, (schema) => {
             runtime.onSchemaChange(schema);
         });
         const featureGatesForTelemetry = {
-            disableGroupedBatching,
             disableCompression,
         };
         const runtime = new containerRuntimeCtor(context, registry, metadata, electedSummarizerData, chunks ?? [], aliases ?? [], {
@@ -531,7 +528,9 @@ export class ContainerRuntime extends TypedEventEmitter {
         return this.garbageCollector.throwOnTombstoneUsage;
     }
     /***/
-    constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, logger, existing, blobManagerSnapshot, _storage, createIdCompressor, documentsSchemaController, featureGatesForTelemetry, provideEntryPoint, requestHandler, summaryConfiguration = {
+    constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, 
+    // Create a custom ITelemetryBaseLogger to output telemetry events.
+    baseLogger, existing, blobManagerSnapshot, _storage, createIdCompressor, documentsSchemaController, featureGatesForTelemetry, provideEntryPoint, requestHandler, summaryConfiguration = {
         // the defaults
         ...DefaultSummaryConfiguration,
         // the runtime configuration overrides
@@ -542,7 +541,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.metadata = metadata;
         this.runtimeOptions = runtimeOptions;
         this.containerScope = containerScope;
-        this.logger = logger;
+        this.baseLogger = baseLogger;
         this._storage = _storage;
         this.createIdCompressor = createIdCompressor;
         this.documentsSchemaController = documentsSchemaController;
@@ -573,7 +572,8 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.snapshotCacheForLoadingGroupIds = new PromiseCache({
             expiry: { policy: "absolute", durationMs: 60000 },
         });
-        const { options, clientDetails, connected, baseSnapshot, submitFn, submitBatchFn, submitSummaryFn, submitSignalFn, disposeFn, closeFn, deltaManager, quorum, audience, loader, pendingLocalState, supportedFeatures, } = context;
+        const { options, clientDetails, connected, baseSnapshot, submitFn, submitBatchFn, submitSummaryFn, submitSignalFn, disposeFn, closeFn, deltaManager, quorum, audience, loader, pendingLocalState, supportedFeatures, snapshotWithContents, } = context;
+        this.logger = createChildLogger({ logger: this.baseLogger });
         this.mc = createChildMonitoringContext({
             logger: this.logger,
             namespace: "ContainerRuntime",
@@ -589,7 +589,6 @@ export class ContainerRuntime extends TypedEventEmitter {
             compressionAlgorithm: CompressionAlgorithms.lz4,
         };
         this.innerDeltaManager = deltaManager;
-        this.deltaManager = new DeltaManagerSummarizerProxy(this.innerDeltaManager);
         // Here we could wrap/intercept on these functions to block/modify outgoing messages if needed.
         // This makes ContainerRuntime the final gatekeeper for outgoing messages.
         this.submitFn = submitFn;
@@ -663,6 +662,32 @@ export class ContainerRuntime extends TypedEventEmitter {
         }, this.mc.logger);
         const opSplitter = new OpSplitter(chunks, this.submitBatchFn, disableChunking === true ? Number.POSITIVE_INFINITY : runtimeOptions.chunkSizeInBytes, runtimeOptions.maxBatchSizeInBytes, this.mc.logger);
         this.remoteMessageProcessor = new RemoteMessageProcessor(opSplitter, new OpDecompressor(this.mc.logger), opGroupingManager);
+        const pendingRuntimeState = pendingLocalState;
+        this.pendingStateManager = new PendingStateManager({
+            applyStashedOp: this.applyStashedOp.bind(this),
+            clientId: () => this.clientId,
+            close: this.closeFn,
+            connected: () => this.connected,
+            reSubmit: (message) => {
+                this.reSubmit(message);
+                this.flush();
+            },
+            reSubmitBatch: this.reSubmitBatch.bind(this),
+            isActiveConnection: () => this.innerDeltaManager.active,
+            isAttached: () => this.attachState !== AttachState.Detached,
+        }, pendingRuntimeState?.pending, this.logger);
+        let outerDeltaManager;
+        const useDeltaManagerOpsProxy = this.mc.config.getBoolean("Fluid.ContainerRuntime.DeltaManagerOpsProxy") !== false;
+        // The summarizerDeltaManager Proxy is used to lie to the summarizer to convince it is in the right state as a summarizer client.
+        const summarizerDeltaManagerProxy = new DeltaManagerSummarizerProxy(this.innerDeltaManager);
+        outerDeltaManager = summarizerDeltaManagerProxy;
+        // The DeltaManagerPendingOpsProxy is used to control the minimum sequence number
+        // It allows us to lie to the layers below so that they can maintain enough local state for rebasing ops.
+        if (useDeltaManagerOpsProxy) {
+            const pendingOpsDeltaManagerProxy = new DeltaManagerPendingOpsProxy(summarizerDeltaManagerProxy, this.pendingStateManager);
+            outerDeltaManager = pendingOpsDeltaManagerProxy;
+        }
+        this.deltaManager = outerDeltaManager;
         this.handleContext = new ContainerFluidHandleContext("", this);
         if (this.summaryConfiguration.state === "enabled") {
             this.validateSummaryHeuristicConfiguration(this.summaryConfiguration);
@@ -682,7 +707,6 @@ export class ContainerRuntime extends TypedEventEmitter {
         else {
             this._flushMode = runtimeOptions.flushMode;
         }
-        const pendingRuntimeState = pendingLocalState;
         if (context.attachState === AttachState.Attached) {
             const maxSnapshotCacheDurationMs = this._storage?.policies?.maximumCacheDurationMs;
             if (maxSnapshotCacheDurationMs !== undefined &&
@@ -738,7 +762,14 @@ export class ContainerRuntime extends TypedEventEmitter {
             const envelope2 = this.createNewSignalEnvelope(envelope1.address, type, envelope1.contents);
             return this.submitSignalFn(envelope2, targetClientId);
         };
-        this.channelCollection = new ChannelCollection(getSummaryForDatastores(baseSnapshot, metadata), parentContext, this.mc.logger, (props) => this.garbageCollector.nodeUpdated(props), (path) => this.garbageCollector.isNodeDeleted(path), new Map(dataStoreAliasMap), async (runtime) => provideEntryPoint);
+        let snapshot = getSummaryForDatastores(baseSnapshot, metadata);
+        if (snapshot !== undefined && snapshotWithContents !== undefined) {
+            snapshot = {
+                ...snapshotWithContents,
+                snapshotTree: snapshot,
+            };
+        }
+        this.channelCollection = new ChannelCollection(snapshot, parentContext, this.mc.logger, (props) => this.garbageCollector.nodeUpdated(props), (path) => this.garbageCollector.isNodeDeleted(path), new Map(dataStoreAliasMap), async (runtime) => provideEntryPoint);
         this.blobManager = new BlobManager({
             routeContext: this.handleContext,
             snapshot: blobManagerSnapshot,
@@ -761,19 +792,6 @@ export class ContainerRuntime extends TypedEventEmitter {
             closeContainer: (error) => this.closeFn(error),
         });
         this.scheduleManager = new ScheduleManager(this.innerDeltaManager, this, () => this.clientId, createChildLogger({ logger: this.logger, namespace: "ScheduleManager" }));
-        this.pendingStateManager = new PendingStateManager({
-            applyStashedOp: this.applyStashedOp.bind(this),
-            clientId: () => this.clientId,
-            close: this.closeFn,
-            connected: () => this.connected,
-            reSubmit: (message) => {
-                this.reSubmit(message);
-                this.flush();
-            },
-            reSubmitBatch: this.reSubmitBatch.bind(this),
-            isActiveConnection: () => this.innerDeltaManager.active,
-            isAttached: () => this.attachState !== AttachState.Detached,
-        }, pendingRuntimeState?.pending, this.logger);
         const disablePartialFlush = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisablePartialFlush");
         const legacySendBatchFn = makeLegacySendBatchFn(this.submitFn, this.innerDeltaManager);
         this.outbox = new Outbox({
@@ -818,7 +836,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             let oldClientId = this.clientId;
             this.on("connected", () => {
                 const clientId = this.clientId;
-                assert(clientId !== undefined, "can't be undefined");
+                assert(clientId !== undefined, 0x975 /* can't be undefined */);
                 audience.emit("selfChanged", { clientId: oldClientId }, { clientId, client: audience.getMember(clientId) });
                 oldClientId = clientId;
             });
@@ -838,7 +856,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 namespace: "OrderedClientElection",
             });
             const orderedClientCollection = new OrderedClientCollection(orderedClientLogger, this.innerDeltaManager, this._quorum);
-            const orderedClientElectionForSummarizer = new OrderedClientElection(orderedClientLogger, orderedClientCollection, electedSummarizerData ?? this.innerDeltaManager.lastSequenceNumber, SummarizerClientElection.isClientEligible);
+            const orderedClientElectionForSummarizer = new OrderedClientElection(orderedClientLogger, orderedClientCollection, electedSummarizerData ?? this.innerDeltaManager.lastSequenceNumber, SummarizerClientElection.isClientEligible, this.mc.config.getBoolean("Fluid.ContainerRuntime.OrderedClientElection.EnablePerformanceEvents"));
             this.summarizerClientElection = new SummarizerClientElection(orderedClientLogger, this.summaryCollection, orderedClientElectionForSummarizer, this.maxOpsSinceLastSummary);
             if (this.isSummarizerClient) {
                 this._summarizer = new Summarizer(this /* ISummarizerRuntime */, () => this.summaryConfiguration, this /* ISummarizerInternalsProvider */, this.handleContext, this.summaryCollection, async (runtime) => RunWhileConnectedCoordinator.create(runtime, 
@@ -882,7 +900,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             }
         }
         // logging hardware telemetry
-        logger.sendTelemetryEvent({
+        this.logger.sendTelemetryEvent({
             eventName: "DeviceSpec",
             ...getDeviceSpec(),
         });
@@ -1045,7 +1063,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             const props = {
                 eventName: "GroupIdSnapshotCatchup",
                 loadingGroupIds: sortedLoadingGroupIds.join(","),
-                targetSequenceNumber: snapshotSeqNumber,
+                targetSequenceNumber: snapshotSeqNumber, // This is so we reuse some columns in telemetry
                 sequenceNumber: this.deltaManager.lastSequenceNumber, // This is so we reuse some columns in telemetry
             };
             const event = PerformanceEvent.start(this.mc.logger, {
@@ -1259,6 +1277,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.emitDirtyDocumentEvent = false;
         let newState;
         try {
+            this.submitIdAllocationOpIfNeeded(true);
             // replay the ops
             this.pendingStateManager.replayPendingStates();
         }
@@ -1306,8 +1325,6 @@ export class ContainerRuntime extends TypedEventEmitter {
                 return;
             case ContainerMessageType.BlobAttach:
                 return;
-            case ContainerMessageType.ChunkedOp:
-                throw new Error("chunkedOp not expected here");
             case ContainerMessageType.Rejoin:
                 throw new Error("rejoin not expected here");
             case ContainerMessageType.GC:
@@ -1319,7 +1336,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 // e.g. if an app rolled back its container version
                 const compatBehavior = opContents.compatDetails?.behavior;
                 if (!compatBehaviorAllowsMessageType(opContents.type, compatBehavior)) {
-                    const error = DataProcessingError.create("Stashed runtime message of unknown type", "applyStashedOp", undefined /* sequencedMessage */, {
+                    const error = DataProcessingError.create("Stashed runtime message of unexpected type", "applyStashedOp", undefined /* sequencedMessage */, {
                         messageDetails: JSON.stringify({
                             type: opContents.type,
                             compatBehavior,
@@ -1345,7 +1362,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 for (const range of ops) {
                     compressor.finalizeCreationRange(range);
                 }
-                assert(this.pendingIdCompressorOps.length === 0, "No new ops added");
+                assert(this.pendingIdCompressorOps.length === 0, 0x976 /* No new ops added */);
                 this._idCompressor = compressor;
             })
                 .catch((error) => {
@@ -1358,8 +1375,8 @@ export class ContainerRuntime extends TypedEventEmitter {
     setConnectionState(connected, clientId) {
         // Validate we have consistent state
         const currentClientId = this._audience.getSelf()?.clientId;
-        assert(clientId === currentClientId, "input clientId does not match Audience");
-        assert(this.clientId === currentClientId, "this.clientId does not match Audience");
+        assert(clientId === currentClientId, 0x977 /* input clientId does not match Audience */);
+        assert(this.clientId === currentClientId, 0x978 /* this.clientId does not match Audience */);
         if (connected && this.idCompressorMode === "delayed") {
             // eslint-disable-next-line @typescript-eslint/no-floating-promises
             this.loadIdCompressor();
@@ -1469,6 +1486,13 @@ export class ContainerRuntime extends TypedEventEmitter {
      */
     processCore(messageWithContext) {
         const { message, local } = messageWithContext;
+        // Intercept to reduce minimum sequence number to the delta manager's minimum sequence number.
+        // Sequence numbers are not guaranteed to follow any sort of order. Re-entrancy is one of those situations
+        if (this.deltaManager.minimumSequenceNumber <
+            messageWithContext.message.minimumSequenceNumber) {
+            messageWithContext.message.minimumSequenceNumber =
+                this.deltaManager.minimumSequenceNumber;
+        }
         // Surround the actual processing of the operation with messages to the schedule manager indicating
         // the beginning and end. This allows it to emit appropriate events and/or pause the processing of new
         // messages once a batch has been fully processed.
@@ -1535,7 +1559,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                         this.pendingIdCompressorOps.push(range);
                     }
                     else {
-                        assert(this.pendingIdCompressorOps.length === 0, "there should be no pending ops!");
+                        assert(this.pendingIdCompressorOps.length === 0, 0x979 /* there should be no pending ops! */);
                         this._idCompressor.finalizeCreationRange(range);
                     }
                 }
@@ -1896,7 +1920,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             return { stats, summary };
         }
         finally {
-            this.mc.logger.sendTelemetryEvent({
+            summaryLogger.sendTelemetryEvent({
                 eventName: "SummarizeTelemetry",
                 details: telemetryContext.serialize(),
             });
@@ -2055,7 +2079,7 @@ export class ContainerRuntime extends TypedEventEmitter {
      * @param options - options controlling how the summary is generated or submitted
      */
     async submitSummary(options) {
-        const { fullTree = false, finalAttempt = false, refreshLatestAck, summaryLogger, latestSummaryRefSeqNum, } = options;
+        const { fullTree = false, finalAttempt = false, summaryLogger, latestSummaryRefSeqNum, } = options;
         // The summary number for this summary. This will be updated during the summary process, so get it now and
         // use it for all events logged during this summary.
         const summaryNumber = this.nextSummaryNumber;
@@ -2070,13 +2094,6 @@ export class ContainerRuntime extends TypedEventEmitter {
             },
         });
         assert(this.outbox.isEmpty, 0x3d1 /* Can't trigger summary in the middle of a batch */);
-        // We close the summarizer and download a new snapshot and reload the container
-        if (refreshLatestAck === true) {
-            return this.prefetchLatestSummaryThenClose(createChildLogger({
-                logger: summaryNumberLogger,
-                properties: { all: { safeSummary: true } },
-            }));
-        }
         // If the container is dirty, i.e., there are pending unacked ops, the summary will not be eventual consistent
         // and it may even be incorrect. So, wait for the container to be saved with a timeout. If the container is not
         // saved within the timeout, check if it should be failed or can continue.
@@ -2145,7 +2162,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                         stage: "base",
                         referenceSequenceNumber: summaryRefSeqNum,
                         minimumSequenceNumber,
-                        error: new LoggingError(`Summarizer node state inconsistent with summarizer state.`),
+                        error: new RetriableSummaryError(`Summarizer node state inconsistent with summarizer state.`),
                     };
                 }
             }
@@ -2188,7 +2205,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                     stage: "base",
                     referenceSequenceNumber: summaryRefSeqNum,
                     minimumSequenceNumber,
-                    error: new LoggingError(continueResult.error),
+                    error: new RetriableSummaryError(continueResult.error),
                 };
             }
             const trace = Trace.start();
@@ -2209,7 +2226,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                     stage: "base",
                     referenceSequenceNumber: summaryRefSeqNum,
                     minimumSequenceNumber,
-                    error: wrapError(error, (msg) => new LoggingError(msg)),
+                    error: wrapError(error, (msg) => new RetriableSummaryError(msg)),
                 };
             }
             // Validate that the summary generated by summarizer nodes is correct before uploading.
@@ -2264,7 +2281,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 return {
                     stage: "generate",
                     ...generateSummaryData,
-                    error: new LoggingError(continueResult.error),
+                    error: new RetriableSummaryError(continueResult.error),
                 };
             }
             const summaryContext = lastAck === undefined
@@ -2286,7 +2303,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 return {
                     stage: "generate",
                     ...generateSummaryData,
-                    error: wrapError(error, (msg) => new LoggingError(msg)),
+                    error: wrapError(error, (msg) => new RetriableSummaryError(msg)),
                 };
             }
             const parent = summaryContext.ackHandle;
@@ -2307,7 +2324,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 return {
                     stage: "upload",
                     ...uploadData,
-                    error: new LoggingError(continueResult.error),
+                    error: new RetriableSummaryError(continueResult.error),
                 };
             }
             let clientSequenceNumber;
@@ -2318,7 +2335,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 return {
                     stage: "upload",
                     ...uploadData,
-                    error: wrapError(error, (msg) => new LoggingError(msg)),
+                    error: wrapError(error, (msg) => new RetriableSummaryError(msg)),
                 };
             }
             const submitData = {
@@ -2334,7 +2351,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 return {
                     stage: "upload",
                     ...uploadData,
-                    error: wrapError(error, (msg) => new LoggingError(msg)),
+                    error: wrapError(error, (msg) => new RetriableSummaryError(msg)),
                 };
             }
             return submitData;
@@ -2424,11 +2441,13 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.verifyNotClosed();
         return this.blobManager.createBlob(blob, signal);
     }
-    submitIdAllocationOpIfNeeded() {
+    submitIdAllocationOpIfNeeded(resubmitOutstandingRanges) {
         if (this._idCompressor) {
-            const idRange = this._idCompressor.takeNextCreationRange();
+            const idRange = resubmitOutstandingRanges
+                ? this._idCompressor.takeUnfinalizedCreationRange()
+                : this._idCompressor.takeNextCreationRange();
             // Don't include the idRange if there weren't any Ids allocated
-            if (idRange?.ids !== undefined) {
+            if (idRange.ids !== undefined) {
                 const idAllocationMessage = {
                     type: ContainerMessageType.IdAllocation,
                     contents: idRange,
@@ -2457,6 +2476,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             });
         }
         const type = containerRuntimeMessage.type;
+        assert(type !== ContainerMessageType.IdAllocation, "IdAllocation should be submitted directly to outbox.");
         const message = {
             contents: serializedContent,
             metadata,
@@ -2464,44 +2484,35 @@ export class ContainerRuntime extends TypedEventEmitter {
             referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
         };
         try {
-            // If `message` is an allocation op, then we are in the resubmit path and we must redirect the allocation
-            // op into the correct batch to avoid ranges being finalized out of order.
-            // Otherwise, submit an IdAllocation op if any IDs have been generated since the last op was submitted, as
-            // any of the other op types may contain those IDs and thus depend on the allocation op being sent first.
-            if (type === ContainerMessageType.IdAllocation) {
-                this.outbox.submitIdAllocation(message);
+            this.submitIdAllocationOpIfNeeded(false);
+            // Allow document schema controller to send a message if it needs to propose change in document schema.
+            // If it needs to send a message, it will call provided callback with payload of such message and rely
+            // on this callback to do actual sending.
+            const contents = this.documentsSchemaController.maybeSendSchemaMessage();
+            if (contents) {
+                this.logger.sendTelemetryEvent({
+                    eventName: "SchemaChangeProposal",
+                    refSeq: contents.refSeq,
+                    version: contents.version,
+                    newRuntimeSchema: JSON.stringify(contents.runtime),
+                    sessionRuntimeSchema: JSON.stringify(this.sessionSchema),
+                    oldRuntimeSchema: JSON.stringify(this.metadata?.documentSchema?.runtime),
+                });
+                const msg = {
+                    type: ContainerMessageType.DocumentSchemaChange,
+                    contents,
+                };
+                this.outbox.submit({
+                    contents: JSON.stringify(msg),
+                    referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
+                });
+            }
+            if (type === ContainerMessageType.BlobAttach) {
+                // BlobAttach ops must have their metadata visible and cannot be grouped (see opGroupingManager.ts)
+                this.outbox.submitBlobAttach(message);
             }
             else {
-                this.submitIdAllocationOpIfNeeded();
-                // Allow document schema controller to send a message if it needs to propose change in document schema.
-                // If it needs to send a message, it will call provided callback with payload of such message and rely
-                // on this callback to do actual sending.
-                const contents = this.documentsSchemaController.maybeSendSchemaMessage();
-                if (contents) {
-                    this.logger.sendTelemetryEvent({
-                        eventName: "SchemaChangeProposal",
-                        refSeq: contents.refSeq,
-                        version: contents.version,
-                        newRuntimeSchema: JSON.stringify(contents.runtime),
-                        sessionRuntimeSchema: JSON.stringify(this.sessionSchema),
-                        oldRuntimeSchema: JSON.stringify(this.metadata?.documentSchema?.runtime),
-                    });
-                    const msg = {
-                        type: ContainerMessageType.DocumentSchemaChange,
-                        contents,
-                    };
-                    this.outbox.submit({
-                        contents: JSON.stringify(msg),
-                        referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
-                    });
-                }
-                if (type === ContainerMessageType.BlobAttach) {
-                    // BlobAttach ops must have their metadata visible and cannot be grouped (see opGroupingManager.ts)
-                    this.outbox.submitBlobAttach(message);
-                }
-                else {
-                    this.outbox.submit(message);
-                }
+                this.outbox.submit(message);
             }
             if (!this.currentlyBatching()) {
                 this.flush();
@@ -2601,11 +2612,15 @@ export class ContainerRuntime extends TypedEventEmitter {
                 this.channelCollection.reSubmit(message.type, message.contents, localOpMetadata);
                 break;
             case ContainerMessageType.IdAllocation: {
-                this.submit(message, localOpMetadata);
+                // Allocation ops are never resubmitted/rebased. This is because they require special handling to
+                // avoid being submitted out of order. For example, if the pending state manager contained
+                // [idOp1, dataOp1, idOp2, dataOp2] and the resubmission of dataOp1 generated idOp3, that would be
+                // placed into the outbox in the same batch as idOp1, but before idOp2 is resubmitted.
+                // To avoid this, allocation ops are simply never resubmitted. Prior to invoking the pending state
+                // manager to replay pending ops, the runtime will always submit a new allocation range that includes
+                // all pending IDs. The resubmitted allocation ops are then ignored here.
                 break;
             }
-            case ContainerMessageType.ChunkedOp:
-                throw new Error(`chunkedOp not expected here`);
             case ContainerMessageType.BlobAttach:
                 this.blobManager.reSubmit(opMetadata);
                 break;
@@ -2632,7 +2647,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                     });
                 }
                 else {
-                    const error = DataProcessingError.create("Resubmitting runtime message of unknown type", "reSubmitCore", undefined /* sequencedMessage */, {
+                    const error = DataProcessingError.create("Resubmitting runtime message of unexpected type", "reSubmitCore", undefined /* sequencedMessage */, {
                         messageDetails: JSON.stringify({
                             type: message.type,
                             compatBehavior,
@@ -2672,51 +2687,23 @@ export class ContainerRuntime extends TypedEventEmitter {
          * and then close as the current main client is likely to be re-elected as the parent summarizer again.
          */
         if (!result.isSummaryTracked && result.isSummaryNewer) {
-            await this.fetchLatestSnapshotFromStorage(summaryLogger, {
+            await this.fetchLatestSnapshotAndClose(summaryLogger, {
                 eventName: "RefreshLatestSummaryAckFetch",
                 ackHandle,
                 targetSequenceNumber: summaryRefSeq,
             }, readAndParseBlob);
-            await this.closeStaleSummarizer();
             return;
         }
         // Notify the garbage collector so it can update its latest summary state.
         await this.garbageCollector.refreshLatestSummary(result);
     }
     /**
-     * Fetches the latest snapshot from storage to refresh the cache as a performance optimization and closes the
-     * summarizer to reload from new state.
-     * @param summaryLogger - logger to use when fetching snapshot from storage
-     * @returns a generic summarization error
+     * Fetches the latest snapshot from storage and closes the container. This is done in cases where
+     * the last known snapshot is older than the latest one. This will ensure that the latest snapshot
+     * is downloaded and we don't end up loading snapshot from cache.
      */
-    async prefetchLatestSummaryThenClose(summaryLogger) {
-        const readAndParseBlob = async (id) => readAndParse(this.storage, id);
-        // This is a performance optimization as the same parent is likely to be elected again, and would use its
-        // cache to fetch the snapshot instead of the network.
-        await this.fetchLatestSnapshotFromStorage(summaryLogger, {
-            eventName: "RefreshLatestSummaryFromServerFetch",
-        }, readAndParseBlob);
-        await this.closeStaleSummarizer();
-        return {
-            stage: "base",
-            error: new LoggingError("summary state stale - Unsupported option 'refreshLatestAck'"),
-            referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
-            minimumSequenceNumber: this.deltaManager.minimumSequenceNumber,
-        };
-    }
-    async closeStaleSummarizer() {
-        // Delay before restarting summarizer to prevent the summarizer from restarting too frequently.
-        await delay(this.closeSummarizerDelayMs);
-        this._summarizer?.stop("latestSummaryStateStale");
-        this.disposeFn();
-    }
-    /**
-     * Downloads the latest snapshot from storage.
-     * By default, it also closes the container after downloading the snapshot. However, this may be
-     * overridden via options.
-     */
-    async fetchLatestSnapshotFromStorage(logger, event, readAndParseBlob) {
-        return PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
+    async fetchLatestSnapshotAndClose(logger, event, readAndParseBlob) {
+        await PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
             const stats = {};
             const trace = Trace.start();
             const versions = await this.storage.getVersions(null, 1, "prefetchLatestSummaryBeforeClose", FetchSource.noCache);
@@ -2729,12 +2716,10 @@ export class ContainerRuntime extends TypedEventEmitter {
             stats.snapshotRefSeq = latestSnapshotRefSeq;
             stats.snapshotVersion = versions[0].id;
             perfEvent.end(stats);
-            return {
-                snapshotTree: maybeSnapshot,
-                versionId: versions[0].id,
-                latestSnapshotRefSeq,
-            };
         });
+        await delay(this.closeSummarizerDelayMs);
+        this._summarizer?.stop("latestSummaryStateStale");
+        this.disposeFn();
     }
     getPendingLocalState(props) {
         this.verifyNotClosed();