@fluidframework/container-runtime 2.1.0-276326 → 2.1.0-281041

package/dist/containerRuntime.js

@@ -17,20 +17,20 @@ const internal_6 = require("@fluidframework/runtime-utils/internal");
 const internal_7 = require("@fluidframework/telemetry-utils/internal");
 const uuid_1 = require("uuid");
 const batchTracker_js_1 = require("./batchTracker.js");
-const blobManager_js_1 = require("./blobManager.js");
+const index_js_1 = require("./blobManager/index.js");
 const channelCollection_js_1 = require("./channelCollection.js");
 const connectionTelemetry_js_1 = require("./connectionTelemetry.js");
 const containerHandleContext_js_1 = require("./containerHandleContext.js");
 const dataStore_js_1 = require("./dataStore.js");
 const dataStoreRegistry_js_1 = require("./dataStoreRegistry.js");
 const deltaManagerProxies_js_1 = require("./deltaManagerProxies.js");
-const index_js_1 = require("./gc/index.js");
+const index_js_2 = require("./gc/index.js");
 const messageTypes_js_1 = require("./messageTypes.js");
-const index_js_2 = require("./opLifecycle/index.js");
+const index_js_3 = require("./opLifecycle/index.js");
 const packageVersion_js_1 = require("./packageVersion.js");
 const pendingStateManager_js_1 = require("./pendingStateManager.js");
 const scheduleManager_js_1 = require("./scheduleManager.js");
-const index_js_3 = require("./summary/index.js");
+const index_js_4 = require("./summary/index.js");
 const throttler_js_1 = require("./throttler.js");
 /**
  * Utility to implement compat behaviors given an unknown message type
@@ -164,7 +164,7 @@ exports.getDeviceSpec = getDeviceSpec;
 const makeLegacySendBatchFn = (submitFn, deltaManager) => (batch) => {
     // Default to negative one to match Container.submitBatch behavior
     let clientSequenceNumber = -1;
-    for (const message of batch.content) {
+    for (const message of batch.messages) {
         clientSequenceNumber = submitFn(internal_3.MessageType.Operation, 
         // For back-compat (submitFn only works on deserialized content)
         message.contents === undefined ? undefined : JSON.parse(message.contents), true, // batch
@@ -184,7 +184,7 @@ async function createSummarizer(loader, url) {
             [internal_1.LoaderHeader.cache]: false,
             [internal_1.LoaderHeader.clientDetails]: {
                 capabilities: { interactive: false },
-                type: index_js_3.summarizerClientType,
+                type: index_js_4.summarizerClientType,
             },
             [internal_3.DriverHeader.summarizingClient]: true,
             [internal_1.LoaderHeader.reconnect]: false,
@@ -274,19 +274,14 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             }
         };
         const [chunks, metadata, electedSummarizerData, aliases, serializedIdCompressor] = await Promise.all([
-            tryFetchBlob(index_js_3.chunksBlobName),
-            tryFetchBlob(index_js_3.metadataBlobName),
-            tryFetchBlob(index_js_3.electedSummarizerBlobName),
-            tryFetchBlob(index_js_3.aliasBlobName),
-            tryFetchBlob(index_js_3.idCompressorBlobName),
+            tryFetchBlob(index_js_4.chunksBlobName),
+            tryFetchBlob(index_js_4.metadataBlobName),
+            tryFetchBlob(index_js_4.electedSummarizerBlobName),
+            tryFetchBlob(index_js_4.aliasBlobName),
+            tryFetchBlob(index_js_4.idCompressorBlobName),
         ]);
         // read snapshot blobs needed for BlobManager to load
-        const blobManagerSnapshot = await blobManager_js_1.BlobManager.load(context.baseSnapshot?.trees[index_js_3.blobsTreeName], async (id) => {
-            // IContainerContext storage api return type still has undefined in 0.39 package version.
-            // So once we release 0.40 container-defn package we can remove this check.
-            (0, internal_2.assert)(context.storage !== undefined, 0x256 /* "storage undefined in attached container" */);
-            return (0, internal_4.readAndParse)(context.storage, id);
-        });
+        const blobManagerSnapshot = await (0, index_js_1.loadBlobManagerLoadInfo)(context);
         const messageAtLastSummary = lastMessageFromMetadata(metadata);
         // Verify summary runtime sequence number matches protocol sequence number.
         const runtimeSequenceNumber = messageAtLastSummary?.sequenceNumber;
@@ -384,11 +379,9 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
                 return createIdCompressor(compressorLogger);
             }
         };
-        const disableCompression = mc.config.getBoolean("Fluid.ContainerRuntime.CompressionDisabled");
-        const compressionLz4 = disableCompression !== true &&
-            compressionOptions.minimumBatchSizeInBytes !== Infinity &&
+        const compressionLz4 = compressionOptions.minimumBatchSizeInBytes !== Infinity &&
             compressionOptions.compressionAlgorithm === "lz4";
-        const documentSchemaController = new index_js_3.DocumentsSchemaController(existing, protocolSequenceNumber, metadata?.documentSchema, {
+        const documentSchemaController = new index_js_4.DocumentsSchemaController(existing, protocolSequenceNumber, metadata?.documentSchema, {
             explicitSchemaControl,
             compressionLz4,
             idCompressorMode,
@@ -397,9 +390,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         }, (schema) => {
             runtime.onSchemaChange(schema);
         });
-        const featureGatesForTelemetry = {
-            disableCompression,
-        };
+        const featureGatesForTelemetry = {};
         const runtime = new containerRuntimeCtor(context, registry, metadata, electedSummarizerData, chunks ?? [], aliases ?? [], {
             summaryOptions,
             gcOptions,
@@ -615,7 +606,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         // Values are generally expected to be set from the runtime side.
         this.options = options ?? {};
         this.clientDetails = clientDetails;
-        this.isSummarizerClient = this.clientDetails.type === index_js_3.summarizerClientType;
+        this.isSummarizerClient = this.clientDetails.type === index_js_4.summarizerClientType;
         this.loadedFromVersionId = context.getLoadedFromVersion()?.id;
         this._getClientId = () => context.clientId;
         this._getAttachState = () => context.attachState;
@@ -664,30 +655,25 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             eventName: "GCFeatureMatrix",
             metadataValue: JSON.stringify(metadata?.gcFeatureMatrix),
             inputs: JSON.stringify({
-                gcOptions_gcGeneration: this.runtimeOptions.gcOptions[index_js_1.gcGenerationOptionName],
+                gcOptions_gcGeneration: this.runtimeOptions.gcOptions[index_js_2.gcGenerationOptionName],
             }),
         });
         this.telemetryDocumentId = metadata?.telemetryDocumentId ?? (0, uuid_1.v4)();
         this.disableAttachReorder = this.mc.config.getBoolean("Fluid.ContainerRuntime.disableAttachOpReorder");
-        const disableChunking = this.mc.config.getBoolean("Fluid.ContainerRuntime.CompressionChunkingDisabled");
-        const opGroupingManager = new index_js_2.OpGroupingManager({
+        const opGroupingManager = new index_js_3.OpGroupingManager({
             groupedBatchingEnabled: this.groupedBatchingEnabled,
             opCountThreshold: this.mc.config.getNumber("Fluid.ContainerRuntime.GroupedBatchingOpCount") ?? 2,
             reentrantBatchGroupingEnabled: this.mc.config.getBoolean("Fluid.ContainerRuntime.GroupedBatchingReentrancy") ??
                 true,
         }, this.mc.logger);
-        const opSplitter = new index_js_2.OpSplitter(chunks, this.submitBatchFn, disableChunking === true ? Number.POSITIVE_INFINITY : runtimeOptions.chunkSizeInBytes, runtimeOptions.maxBatchSizeInBytes, this.mc.logger);
-        this.remoteMessageProcessor = new index_js_2.RemoteMessageProcessor(opSplitter, new index_js_2.OpDecompressor(this.mc.logger), opGroupingManager);
+        const opSplitter = new index_js_3.OpSplitter(chunks, this.submitBatchFn, runtimeOptions.chunkSizeInBytes, runtimeOptions.maxBatchSizeInBytes, this.mc.logger);
+        this.remoteMessageProcessor = new index_js_3.RemoteMessageProcessor(opSplitter, new index_js_3.OpDecompressor(this.mc.logger), opGroupingManager);
         const pendingRuntimeState = pendingLocalState;
         this.pendingStateManager = new pendingStateManager_js_1.PendingStateManager({
             applyStashedOp: this.applyStashedOp.bind(this),
             clientId: () => this.clientId,
             close: this.closeFn,
             connected: () => this.connected,
-            reSubmit: (message) => {
-                this.reSubmit(message);
-                this.flush();
-            },
             reSubmitBatch: this.reSubmitBatch.bind(this),
             isActiveConnection: () => this.innerDeltaManager.active,
             isAttached: () => this.attachState !== container_definitions_1.AttachState.Detached,
@@ -733,7 +719,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
                 throw new internal_7.UsageError("Driver's maximumCacheDurationMs policy cannot exceed 5 days");
             }
         }
-        this.garbageCollector = index_js_1.GarbageCollector.create({
+        this.garbageCollector = index_js_2.GarbageCollector.create({
             runtime: this,
             gcOptions: this.runtimeOptions.gcOptions,
             baseSnapshot,
@@ -749,7 +735,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             sessionExpiryTimerStarted: pendingRuntimeState?.sessionExpiryTimerStarted,
         });
         const loadedFromSequenceNumber = this.deltaManager.initialSequenceNumber;
-        this.summarizerNode = (0, index_js_3.createRootSummarizerNodeWithGC)((0, internal_7.createChildLogger)({ logger: this.logger, namespace: "SummarizerNode" }), 
+        this.summarizerNode = (0, index_js_4.createRootSummarizerNodeWithGC)((0, internal_7.createChildLogger)({ logger: this.logger, namespace: "SummarizerNode" }), 
         // Summarize function to call when summarize is called. Summarizer node always tracks summary state.
         async (fullTree, trackState, telemetryContext) => this.summarizeInternal(fullTree, trackState, telemetryContext), 
         // Latest change sequence number, no changes since summary applied yet
@@ -770,6 +756,9 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             this.summarizerNode.updateBaseSummaryState(baseSnapshot);
         }
         const parentContext = (0, channelCollection_js_1.wrapContext)(this);
+        if (snapshotWithContents !== undefined) {
+            this.isSnapshotInstanceOfISnapshot = true;
+        }
         // Due to a mismatch between different layers in terms of
         // what is the interface of passing signals, we need the
         // downstream stores to wrap the signal.
@@ -789,7 +778,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             ...props,
             timestampMs: props.timestampMs ?? this.getCurrentReferenceTimestampMs(),
         }), (path) => this.garbageCollector.isNodeDeleted(path), new Map(dataStoreAliasMap), async (runtime) => provideEntryPoint);
-        this.blobManager = new blobManager_js_1.BlobManager({
+        this.blobManager = new index_js_1.BlobManager({
             routeContext: this.handleContext,
             snapshot: blobManagerSnapshot,
             getStorage: () => this.storage,
@@ -814,12 +803,12 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         this.scheduleManager = new scheduleManager_js_1.ScheduleManager(this.innerDeltaManager, this, () => this.clientId, (0, internal_7.createChildLogger)({ logger: this.logger, namespace: "ScheduleManager" }));
         const disablePartialFlush = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisablePartialFlush");
         const legacySendBatchFn = (0, exports.makeLegacySendBatchFn)(this.submitFn, this.innerDeltaManager);
-        this.outbox = new index_js_2.Outbox({
+        this.outbox = new index_js_3.Outbox({
             shouldSend: () => this.canSendOps(),
             pendingStateManager: this.pendingStateManager,
             submitBatchFn: this.submitBatchFn,
             legacySendBatchFn,
-            compressor: new index_js_2.OpCompressor(this.mc.logger),
+            compressor: new index_js_3.OpCompressor(this.mc.logger),
             splitter: opSplitter,
             config: {
                 compressionOptions,
@@ -864,7 +853,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         const closeSummarizerDelayOverride = this.mc.config.getNumber("Fluid.ContainerRuntime.Test.CloseSummarizerDelayOverrideMs");
         this.closeSummarizerDelayMs =
             closeSummarizerDelayOverride ?? defaultCloseSummarizerDelayMs;
-        this.summaryCollection = new index_js_3.SummaryCollection(this.deltaManager, this.logger);
+        this.summaryCollection = new index_js_4.SummaryCollection(this.deltaManager, this.logger);
         this.dirtyContainer =
             this.attachState !== container_definitions_1.AttachState.Attached || this.hasPendingMessages();
         context.updateDirtyContainerState(this.dirtyContainer);
@@ -876,16 +865,16 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
                 logger: this.logger,
                 namespace: "OrderedClientElection",
             });
-            const orderedClientCollection = new index_js_3.OrderedClientCollection(orderedClientLogger, this.innerDeltaManager, this._quorum);
-            const orderedClientElectionForSummarizer = new index_js_3.OrderedClientElection(orderedClientLogger, orderedClientCollection, electedSummarizerData ?? this.innerDeltaManager.lastSequenceNumber, index_js_3.SummarizerClientElection.isClientEligible, this.mc.config.getBoolean("Fluid.ContainerRuntime.OrderedClientElection.EnablePerformanceEvents"));
-            this.summarizerClientElection = new index_js_3.SummarizerClientElection(orderedClientLogger, this.summaryCollection, orderedClientElectionForSummarizer, this.maxOpsSinceLastSummary);
+            const orderedClientCollection = new index_js_4.OrderedClientCollection(orderedClientLogger, this.innerDeltaManager, this._quorum);
+            const orderedClientElectionForSummarizer = new index_js_4.OrderedClientElection(orderedClientLogger, orderedClientCollection, electedSummarizerData ?? this.innerDeltaManager.lastSequenceNumber, index_js_4.SummarizerClientElection.isClientEligible, this.mc.config.getBoolean("Fluid.ContainerRuntime.OrderedClientElection.EnablePerformanceEvents"));
+            this.summarizerClientElection = new index_js_4.SummarizerClientElection(orderedClientLogger, this.summaryCollection, orderedClientElectionForSummarizer, this.maxOpsSinceLastSummary);
             if (this.isSummarizerClient) {
-                this._summarizer = new index_js_3.Summarizer(this /* ISummarizerRuntime */, () => this.summaryConfiguration, this /* ISummarizerInternalsProvider */, this.handleContext, this.summaryCollection, async (runtime) => index_js_3.RunWhileConnectedCoordinator.create(runtime, 
+                this._summarizer = new index_js_4.Summarizer(this /* ISummarizerRuntime */, () => this.summaryConfiguration, this /* ISummarizerInternalsProvider */, this.handleContext, this.summaryCollection, async (runtime) => index_js_4.RunWhileConnectedCoordinator.create(runtime, 
                 // Summarization runs in summarizer client and needs access to the real (non-proxy) active
                 // information. The proxy delta manager would always return false for summarizer client.
                 () => this.innerDeltaManager.active));
             }
-            else if (index_js_3.SummarizerClientElection.clientDetailsPermitElection(this.clientDetails)) {
+            else if (index_js_4.SummarizerClientElection.clientDetailsPermitElection(this.clientDetails)) {
                 // Only create a SummaryManager and SummarizerClientElection
                 // if summaries are enabled and we are not the summarizer client.
                 const defaultAction = () => {
@@ -907,7 +896,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
                 };
                 this.summaryCollection.on("default", defaultAction);
                 // Create the SummaryManager and mark the initial state
-                this.summaryManager = new index_js_3.SummaryManager(this.summarizerClientElection, this, // IConnectedState
+                this.summaryManager = new index_js_4.SummaryManager(this.summarizerClientElection, this, // IConnectedState
                 this.summaryCollection, this.logger, this.formCreateSummarizerFn(loader), new throttler_js_1.Throttler(60 * 1000, // 60 sec delay window
                 30 * 1000, // 30 sec max delay
                 // throttling function increases exponentially (0ms, 40ms, 80ms, 160ms, etc)
@@ -939,7 +928,6 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             sessionRuntimeSchema: JSON.stringify(this.sessionSchema),
             featureGates: JSON.stringify({
                 ...featureGatesForTelemetry,
-                disableChunking,
                 disableAttachReorder: this.disableAttachReorder,
                 disablePartialFlush,
                 closeSummarizerDelayOverride,
@@ -1055,7 +1043,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             }),
         });
         // Find the snapshotTree inside the returned snapshot based on the path as given in the request.
-        const hasIsolatedChannels = (0, index_js_3.rootHasIsolatedChannels)(this.metadata);
+        const hasIsolatedChannels = (0, index_js_4.rootHasIsolatedChannels)(this.metadata);
         const snapshotTreeForPath = this.getSnapshotTreeForPath(snapshot.snapshotTree, pathParts, hasIsolatedChannels);
         (0, internal_2.assert)(snapshotTreeForPath !== undefined, 0x8ef /* no snapshotTree for the path */);
         const snapshotSeqNumber = snapshot.sequenceNumber;
@@ -1118,9 +1106,13 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         let childTree = snapshotTree;
         for (const part of pathParts) {
             if (hasIsolatedChannels) {
-                childTree = childTree?.trees[internal_5.channelsTreeName];
+                // TODO Why are we non null asserting here
+                // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+                childTree = childTree.trees[internal_5.channelsTreeName];
             }
-            childTree = childTree?.trees[part];
+            // TODO Why are we non null asserting here
+            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+            childTree = childTree.trees[part];
         }
         return childTree;
     }
@@ -1166,7 +1158,9 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
                 // eslint-disable-next-line @typescript-eslint/return-await -- Adding an await here causes test failures
                 return this.resolveHandle(requestParser.createSubRequest(1));
             }
-            if (id === blobManager_js_1.BlobManager.basePath && requestParser.isLeaf(2)) {
+            if (id === index_js_1.blobManagerBasePath && requestParser.isLeaf(2)) {
+                // TODO why are we non null asserting here?
+                // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
                 const blob = await this.blobManager.getBlob(requestParser.pathParts[1]);
                 return blob
                     ? {
@@ -1198,7 +1192,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
     addMetadataToSummary(summaryTree) {
         // The last message processed at the time of summary. If there are no new messages, use the message from the
         // last summary.
-        const message = (0, index_js_3.extractSummaryMetadataMessage)(this.deltaManager.lastMessage) ??
+        const message = (0, index_js_4.extractSummaryMetadataMessage)(this.deltaManager.lastMessage) ??
             this.messageAtLastSummary;
         const documentSchema = this.documentsSchemaController.summarizeDocumentSchema(this.deltaManager.lastSequenceNumber);
         // Is document schema explicit control on?
@@ -1221,31 +1215,31 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             lastMessage: explicitSchemaControl ? message : undefined,
             documentSchema,
         };
-        (0, internal_6.addBlobToSummary)(summaryTree, index_js_3.metadataBlobName, JSON.stringify(metadata));
+        (0, internal_6.addBlobToSummary)(summaryTree, index_js_4.metadataBlobName, JSON.stringify(metadata));
     }
     addContainerStateToSummary(summaryTree, fullTree, trackState, telemetryContext) {
         this.addMetadataToSummary(summaryTree);
         if (this._idCompressor) {
             const idCompressorState = JSON.stringify(this._idCompressor.serialize(false));
-            (0, internal_6.addBlobToSummary)(summaryTree, index_js_3.idCompressorBlobName, idCompressorState);
+            (0, internal_6.addBlobToSummary)(summaryTree, index_js_4.idCompressorBlobName, idCompressorState);
         }
         if (this.remoteMessageProcessor.partialMessages.size > 0) {
             const content = JSON.stringify([...this.remoteMessageProcessor.partialMessages]);
-            (0, internal_6.addBlobToSummary)(summaryTree, index_js_3.chunksBlobName, content);
+            (0, internal_6.addBlobToSummary)(summaryTree, index_js_4.chunksBlobName, content);
         }
         const dataStoreAliases = this.channelCollection.aliases;
         if (dataStoreAliases.size > 0) {
-            (0, internal_6.addBlobToSummary)(summaryTree, index_js_3.aliasBlobName, JSON.stringify([...dataStoreAliases]));
+            (0, internal_6.addBlobToSummary)(summaryTree, index_js_4.aliasBlobName, JSON.stringify([...dataStoreAliases]));
         }
         if (this.summarizerClientElection) {
             const electedSummarizerContent = JSON.stringify(this.summarizerClientElection?.serialize());
-            (0, internal_6.addBlobToSummary)(summaryTree, index_js_3.electedSummarizerBlobName, electedSummarizerContent);
+            (0, internal_6.addBlobToSummary)(summaryTree, index_js_4.electedSummarizerBlobName, electedSummarizerContent);
         }
         const blobManagerSummary = this.blobManager.summarize();
         // Some storage (like git) doesn't allow empty tree, so we can omit it.
         // and the blob manager can handle the tree not existing when loading
         if (Object.keys(blobManagerSummary.summary.tree).length > 0) {
-            (0, internal_6.addSummarizeResultToSummary)(summaryTree, index_js_3.blobsTreeName, blobManagerSummary);
+            (0, internal_6.addSummarizeResultToSummary)(summaryTree, index_js_1.blobsTreeName, blobManagerSummary);
         }
         const gcSummary = this.garbageCollector.summarize(fullTree, trackState, telemetryContext);
         if (gcSummary !== undefined) {
@@ -1480,25 +1474,36 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         // but will not modify the contents object (likely it will replace it on the message).
         const messageCopy = { ...messageArg };
         const savedOp = messageCopy.metadata?.savedOp;
-        for (const message of this.remoteMessageProcessor.process(messageCopy)) {
-            const msg = modernRuntimeMessage
-                ? {
-                    // Cast it since we expect it to be this based on modernRuntimeMessage computation above.
-                    // There is nothing really ensuring that anytime original message.type is Operation that
-                    // the result messages will be so. In the end modern bool being true only directs to
-                    // throw error if ultimately unrecognized without compat details saying otherwise.
-                    message: message,
+        if (modernRuntimeMessage) {
+            const processResult = this.remoteMessageProcessor.process(messageCopy);
+            if (processResult === undefined) {
+                // This means the incoming message is an incomplete part of a message or batch
+                // and we need to process more messages before the rest of the system can understand it.
+                return;
+            }
+            const batchStartCsn = processResult.batchStartCsn;
+            const batch = processResult.messages;
+            const messages = local
+                ? this.pendingStateManager.processPendingLocalBatch(batch, batchStartCsn)
+                : batch.map((message) => ({ message, localOpMetadata: undefined }));
+            messages.forEach(({ message, localOpMetadata }) => {
+                const msg = {
+                    message,
                     local,
                     modernRuntimeMessage,
-                }
-                : // Unrecognized message will be ignored.
-                    {
-                        message,
-                        local,
-                        modernRuntimeMessage,
-                    };
-            msg.savedOp = savedOp;
-            // ensure that we observe any re-entrancy, and if needed, rebase ops
+                    savedOp,
+                    localOpMetadata,
+                };
+                this.ensureNoDataModelChanges(() => this.processCore(msg));
+            });
+        }
+        else {
+            const msg = {
+                message: messageCopy,
+                local,
+                modernRuntimeMessage,
+                savedOp,
+            };
             this.ensureNoDataModelChanges(() => this.processCore(msg));
         }
     }
@@ -1506,7 +1511,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
      * Direct the message to the correct subsystem for processing, and implement other side effects
      */
     processCore(messageWithContext) {
-        const { message, local } = messageWithContext;
+        const { message, local, localOpMetadata } = messageWithContext;
         // Intercept to reduce minimum sequence number to the delta manager's minimum sequence number.
         // Sequence numbers are not guaranteed to follow any sort of order. Re-entrancy is one of those situations
         if (this.deltaManager.minimumSequenceNumber <
@@ -1520,15 +1525,8 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         this.scheduleManager.beforeOpProcessing(message);
         this._processedClientSequenceNumber = message.clientSequenceNumber;
         try {
-            // See commit that added this assert for more details.
-            // These calls should be made for all but chunked ops:
-            // 1) this.pendingStateManager.processPendingLocalMessage() below
-            // 2) this.resetReconnectCount() below
+            // RemoteMessageProcessor would have already reconstituted Chunked Ops into the original op type
             (0, internal_2.assert)(message.type !== messageTypes_js_1.ContainerMessageType.ChunkedOp, 0x93b /* we should never get here with chunked ops */);
-            let localOpMetadata;
-            if (local && messageWithContext.modernRuntimeMessage) {
-                localOpMetadata = this.pendingStateManager.processPendingLocalMessage(messageWithContext.message);
-            }
             // If there are no more pending messages after processing a local message,
             // the document is no longer dirty.
             if (!this.hasPendingMessages()) {
@@ -1687,10 +1685,12 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
     /**
      * Flush the pending ops manually.
      * This method is expected to be called at the end of a batch.
+     * @param resubmittingBatchId - If defined, indicates this is a resubmission of a batch
+     * with the given Batch ID, which must be preserved
      */
-    flush() {
+    flush(resubmittingBatchId) {
         (0, internal_2.assert)(this._orderSequentiallyCalls === 0, 0x24c /* "Cannot call `flush()` from `orderSequentially`'s callback" */);
-        this.outbox.flush();
+        this.outbox.flush(resubmittingBatchId);
         (0, internal_2.assert)(this.outbox.isEmpty, 0x3cf /* reentrancy */);
     }
     /**
@@ -1703,7 +1703,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             // Note: we are not touching any batches other than mainBatch here, for two reasons:
             // 1. It would not help, as other batches are flushed independently from main batch.
             // 2. There is no way to undo process of data store creation, blob creation, ID compressor ops, or other things tracked by other batches.
-            checkpoint = this.outbox.checkpoint().mainBatch;
+            checkpoint = this.outbox.getBatchCheckpoints().mainBatch;
         }
         try {
             this._orderSequentiallyCalls++;
@@ -1794,7 +1794,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         return (this.connected && !this.innerDeltaManager.readOnlyInfo.readonly && !this.imminentClosure);
     }
     /**
-     * Are we in the middle of batching ops together?
+     * Typically ops are batched and later flushed together, but in some cases we want to flush immediately.
      */
     currentlyBatching() {
         return this.flushMode !== internal_5.FlushMode.Immediate || this._orderSequentiallyCalls !== 0;
@@ -1899,14 +1899,14 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         }
         const summarizeResult = this.channelCollection.getAttachSummary(telemetryContext);
         // Wrap data store summaries in .channels subtree.
-        (0, index_js_3.wrapSummaryInChannelsTree)(summarizeResult);
+        (0, index_js_4.wrapSummaryInChannelsTree)(summarizeResult);
         this.addContainerStateToSummary(summarizeResult, true /* fullTree */, false /* trackState */, telemetryContext);
         return summarizeResult.summary;
     }
     async summarizeInternal(fullTree, trackState, telemetryContext) {
         const summarizeResult = await this.channelCollection.summarize(fullTree, trackState, telemetryContext);
         // Wrap data store summaries in .channels subtree.
-        (0, index_js_3.wrapSummaryInChannelsTree)(summarizeResult);
+        (0, index_js_4.wrapSummaryInChannelsTree)(summarizeResult);
         const pathPartsForChildren = [internal_5.channelsTreeName];
         // Ensure that ID compressor had a chance to load, if we are using delayed mode.
         await this.loadIdCompressor();
@@ -2011,10 +2011,10 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
      * blob manager.
      */
     getNodeType(nodePath) {
-        if (this.isBlobPath(nodePath)) {
-            return index_js_1.GCNodeType.Blob;
+        if ((0, index_js_1.isBlobPath)(nodePath)) {
+            return index_js_2.GCNodeType.Blob;
         }
-        return this.channelCollection.getGCNodeType(nodePath) ?? index_js_1.GCNodeType.Other;
+        return this.channelCollection.getGCNodeType(nodePath) ?? index_js_2.GCNodeType.Other;
     }
     /**
      * Called by GC to retrieve the package path of the node with the given path. The node should belong to a
@@ -2027,25 +2027,15 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             return ["_gcRoot"];
         }
         switch (this.getNodeType(nodePath)) {
-            case index_js_1.GCNodeType.Blob:
-                return [blobManager_js_1.BlobManager.basePath];
-            case index_js_1.GCNodeType.DataStore:
-            case index_js_1.GCNodeType.SubDataStore:
+            case index_js_2.GCNodeType.Blob:
+                return [index_js_1.blobManagerBasePath];
+            case index_js_2.GCNodeType.DataStore:
+            case index_js_2.GCNodeType.SubDataStore:
                 return this.channelCollection.getDataStorePackagePath(nodePath);
             default:
                 (0, internal_2.assert)(false, 0x2de /* "Package path requested for unsupported node type." */);
         }
     }
-    /**
-     * Returns whether a given path is for attachment blobs that are in the format - "/BlobManager.basePath/...".
-     */
-    isBlobPath(path) {
-        const pathParts = path.split("/");
-        if (pathParts.length < 2 || pathParts[1] !== blobManager_js_1.BlobManager.basePath) {
-            return false;
-        }
-        return true;
-    }
     /**
      * From a given list of routes, separate and return routes that belong to blob manager and data stores.
      * @param routes - A list of routes that can belong to data stores or blob manager.
@@ -2056,7 +2046,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         const blobManagerRoutes = [];
         const dataStoreRoutes = [];
         for (const route of routes) {
-            if (this.isBlobPath(route)) {
+            if ((0, index_js_1.isBlobPath)(route)) {
                 blobManagerRoutes.push(route);
             }
             else {
@@ -2165,7 +2155,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             summaryRefSeqNum = this.deltaManager.lastSequenceNumber;
             const minimumSequenceNumber = this.deltaManager.minimumSequenceNumber;
             const message = `Summary @${summaryRefSeqNum}:${this.deltaManager.minimumSequenceNumber}`;
-            const lastAck = this.summaryCollection.latestAck;
+            const lastAckedContext = this.lastAckedSummaryContext;
             const startSummaryResult = this.summarizerNode.startSummary(summaryRefSeqNum, summaryNumberLogger, latestSummaryRefSeqNum);
             /**
              * This was added to validate that the summarizer node tree has the same reference sequence number from the
@@ -2188,7 +2178,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
                         stage: "base",
                         referenceSequenceNumber: summaryRefSeqNum,
                         minimumSequenceNumber,
-                        error: new index_js_3.RetriableSummaryError(`Summarizer node state inconsistent with summarizer state.`),
+                        error: new index_js_4.RetriableSummaryError(`Summarizer node state inconsistent with summarizer state.`),
                     };
                 }
             }
@@ -2217,10 +2207,10 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
                     };
                 }
                 (0, internal_2.assert)(summaryRefSeqNum === this.deltaManager.lastMessage?.sequenceNumber, 0x395 /* it's one and the same thing */);
-                if (lastAck !== this.summaryCollection.latestAck) {
+                if (lastAckedContext !== this.lastAckedSummaryContext) {
                     return {
                         continue: false,
-                        error: `Last summary changed while summarizing. ${this.summaryCollection.latestAck} !== ${lastAck}`,
+                        error: `Last summary changed while summarizing. ${this.lastAckedSummaryContext} !== ${lastAckedContext}`,
                     };
                 }
                 return { continue: true };
@@ -2231,7 +2221,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
                     stage: "base",
                     referenceSequenceNumber: summaryRefSeqNum,
                     minimumSequenceNumber,
-                    error: new index_js_3.RetriableSummaryError(continueResult.error),
+                    error: new index_js_4.RetriableSummaryError(continueResult.error),
                 };
             }
             const trace = client_utils_1.Trace.start();
@@ -2249,14 +2239,14 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
                     stage: "base",
                     referenceSequenceNumber: summaryRefSeqNum,
                     minimumSequenceNumber,
-                    error: (0, internal_7.wrapError)(error, (msg) => new index_js_3.RetriableSummaryError(msg)),
+                    error: (0, internal_7.wrapError)(error, (msg) => new index_js_4.RetriableSummaryError(msg)),
                 };
             }
             // Validate that the summary generated by summarizer nodes is correct before uploading.
             const validateResult = this.summarizerNode.validateSummary();
             if (!validateResult.success) {
                 const { success, ...loggingProps } = validateResult;
-                const error = new index_js_3.RetriableSummaryError(validateResult.reason, validateResult.retryAfterSeconds, { ...loggingProps });
+                const error = new index_js_4.RetriableSummaryError(validateResult.reason, validateResult.retryAfterSeconds, { ...loggingProps });
                 return {
                     stage: "base",
                     referenceSequenceNumber: summaryRefSeqNum,
@@ -2276,6 +2266,8 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             // Counting dataStores and handles
             // Because handles are unchanged dataStores in the current logic,
             // summarized dataStore count is total dataStore count minus handle count
+            // TODO why are we non null asserting here
+            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
             const dataStoreTree = summaryTree.tree[internal_5.channelsTreeName];
             (0, internal_2.assert)(dataStoreTree.type === driver_definitions_1.SummaryType.Tree, 0x1fc /* "summary is not a tree" */);
             const handleCount = Object.values(dataStoreTree.tree).filter((value) => value.type === driver_definitions_1.SummaryType.Handle).length;
@@ -2303,20 +2295,14 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
                 return {
                     stage: "generate",
                     ...generateSummaryData,
-                    error: new index_js_3.RetriableSummaryError(continueResult.error),
+                    error: new index_js_4.RetriableSummaryError(continueResult.error),
                 };
             }
-            const summaryContext = lastAck === undefined
-                ? {
-                    proposalHandle: undefined,
-                    ackHandle: this.loadedFromVersionId,
-                    referenceSequenceNumber: summaryRefSeqNum,
-                }
-                : {
-                    proposalHandle: lastAck.summaryOp.contents.handle,
-                    ackHandle: lastAck.summaryAck.contents.handle,
-                    referenceSequenceNumber: summaryRefSeqNum,
-                };
+            const summaryContext = {
+                proposalHandle: this.lastAckedSummaryContext?.proposalHandle ?? undefined,
+                ackHandle: this.lastAckedSummaryContext?.ackHandle ?? this.loadedFromVersionId,
+                referenceSequenceNumber: summaryRefSeqNum,
+            };
             let handle;
             try {
                 handle = await this.storage.uploadSummaryWithContext(summarizeResult.summary, summaryContext);
@@ -2325,7 +2311,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
                 return {
                     stage: "generate",
                     ...generateSummaryData,
-                    error: (0, internal_7.wrapError)(error, (msg) => new index_js_3.RetriableSummaryError(msg)),
+                    error: (0, internal_7.wrapError)(error, (msg) => new index_js_4.RetriableSummaryError(msg)),
                 };
             }
             const parent = summaryContext.ackHandle;
@@ -2346,7 +2332,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
                 return {
                     stage: "upload",
                     ...uploadData,
-                    error: new index_js_3.RetriableSummaryError(continueResult.error),
+                    error: new index_js_4.RetriableSummaryError(continueResult.error),
                 };
             }
             let clientSequenceNumber;
@@ -2357,7 +2343,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
                 return {
                     stage: "upload",
                     ...uploadData,
-                    error: (0, internal_7.wrapError)(error, (msg) => new index_js_3.RetriableSummaryError(msg)),
+                    error: (0, internal_7.wrapError)(error, (msg) => new index_js_4.RetriableSummaryError(msg)),
                 };
             }
             const submitData = {
@@ -2373,7 +2359,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
                 return {
                     stage: "upload",
                     ...uploadData,
-                    error: (0, internal_7.wrapError)(error, (msg) => new index_js_3.RetriableSummaryError(msg)),
+                    error: (0, internal_7.wrapError)(error, (msg) => new index_js_4.RetriableSummaryError(msg)),
                 };
             }
             return submitData;
@@ -2422,7 +2408,7 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             // based on telemetry while we decide on a stable number.
             const retryDelayMs = this.mc.config.getNumber("Fluid.Summarizer.PendingOpsRetryDelayMs") ??
                 exports.defaultPendingOpsRetryDelayMs;
-            const error = new index_js_3.RetriableSummaryError("PendingOpsWhileSummarizing", retryDelayMs / 1000, {
+            const error = new index_js_4.RetriableSummaryError("PendingOpsWhileSummarizing", retryDelayMs / 1000, {
                 count: this.pendingMessagesCount,
                 beforeGenerate: beforeSummaryGeneration,
             });
@@ -2536,7 +2522,9 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             else {
                 this.outbox.submit(message);
             }
-            if (!this.currentlyBatching()) {
+            // Note: Technically, the system "always" batches - if this case is true we'll just have a single-message batch.
+            const flushImmediatelyOnSubmit = !this.currentlyBatching();
+            if (flushImmediatelyOnSubmit) {
                 this.flush();
             }
             else {
@@ -2603,13 +2591,16 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
             throw new Error("Runtime is closed");
         }
     }
-    reSubmitBatch(batch) {
+    reSubmitBatch(batch, batchId) {
         this.orderSequentially(() => {
             for (const message of batch) {
                 this.reSubmit(message);
             }
         });
-        this.flush();
+        // Only include Batch ID if "Offline Load" feature is enabled
+        // It's only needed to identify batches across container forks arising from misuse of offline load.
+        const includeBatchId = this.mc.config.getBoolean("Fluid.Container.enableOfflineLoad") ?? false;
+        this.flush(includeBatchId ? batchId : undefined);
     }
     reSubmit(message) {
         // Need to parse from string for back-compat
@@ -2700,45 +2691,91 @@ class ContainerRuntime extends client_utils_1.TypedEventEmitter {
         const { proposalHandle, ackHandle, summaryRefSeq, summaryLogger } = options;
         // proposalHandle is always passed from RunningSummarizer.
         (0, internal_2.assert)(proposalHandle !== undefined, 0x766 /* proposalHandle should be available */);
-        const readAndParseBlob = async (id) => (0, internal_4.readAndParse)(this.storage, id);
         const result = await this.summarizerNode.refreshLatestSummary(proposalHandle, summaryRefSeq);
+        /* eslint-disable jsdoc/check-indentation */
         /**
-         * When refreshing a summary ack, this check indicates a new ack of a summary that is newer than the
-         * current summary that is tracked, but this summarizer runtime did not produce/track that summary. Thus
-         * it needs to refresh its state. Today refresh is done by fetching the latest snapshot to update the cache
-         * and then close as the current main client is likely to be re-elected as the parent summarizer again.
+         * If the snapshot corresponding to the ack is not tracked by this client, it was submitted by another client.
+         * Take action as per the following scenarios:
+         * 1. If that snapshot is older than the one tracked by this client, ignore the ack because only the latest
+         *    snapshot is tracked.
+         * 2. If that snapshot is newer, attempt to fetch the latest snapshot and do one of the following:
+         *    2.1. If the fetched snapshot is same or newer than the one for which ack was received, close this client.
+         *         The next summarizer client will likely start from this snapshot and get out of this state. Fetching
+         *         the snapshot updates the cache for this client so if it's re-elected as summarizer, this will prevent
+         *         any thrashing.
+         *    2.2. If the fetched snapshot is older than the one for which ack was received, ignore the ack. This can
+         *         happen in scenarios where the snapshot for the ack was lost in storage (in scenarios like DB rollback,
+         *         etc.) but the summary ack is still there because it's tracked a different service. In such cases,
+         *         ignoring the ack is the correct thing to do because the latest snapshot in storage is not the one for
+         *         the ack but is still the one tracked by this client. If we were to close the summarizer like in the
+         *         previous scenario, it will result in this document stuck in this state in a loop.
          */
-        if (!result.isSummaryTracked && result.isSummaryNewer) {
-            await this.fetchLatestSnapshotAndClose(summaryLogger, {
-                eventName: "RefreshLatestSummaryAckFetch",
-                ackHandle,
-                targetSequenceNumber: summaryRefSeq,
-            }, readAndParseBlob);
+        /* eslint-enable jsdoc/check-indentation */
+        if (!result.isSummaryTracked) {
+            if (result.isSummaryNewer) {
+                await this.fetchLatestSnapshotAndMaybeClose(summaryRefSeq, ackHandle, summaryLogger);
+            }
             return;
         }
         // Notify the garbage collector so it can update its latest summary state.
         await this.garbageCollector.refreshLatestSummary(result);
+        // If we here, the ack was tracked by this client. Update the summary context of the last ack.
+        this.lastAckedSummaryContext = {
+            proposalHandle,
+            ackHandle,
+            referenceSequenceNumber: summaryRefSeq,
+        };
     }
     /**
-     * Fetches the latest snapshot from storage and closes the container. This is done in cases where
-     * the last known snapshot is older than the latest one. This will ensure that the latest snapshot
-     * is downloaded and we don't end up loading snapshot from cache.
+     * Fetches the latest snapshot from storage. If the fetched snapshot is same or newer than the one for which ack
+     * was received, close this client. Fetching the snapshot will update the cache for this client so if it's
+     * re-elected as summarizer, this will prevent any thrashing.
+     * If the fetched snapshot is older than the one for which ack was received, ignore the ack and return. This can
+     * happen in scenarios where the snapshot for the ack was lost in storage in scenarios like DB rollback, etc.
      */
-    async fetchLatestSnapshotAndClose(logger, event, readAndParseBlob) {
-        await internal_7.PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
-            const stats = {};
+    async fetchLatestSnapshotAndMaybeClose(targetRefSeq, targetAckHandle, logger) {
+        const fetchedSnapshotRefSeq = await internal_7.PerformanceEvent.timedExecAsync(logger, { eventName: "RefreshLatestSummaryAckFetch" }, async (perfEvent) => {
+            const props = { targetRefSeq, targetAckHandle };
             const trace = client_utils_1.Trace.start();
-            const versions = await this.storage.getVersions(null, 1, "prefetchLatestSummaryBeforeClose", internal_3.FetchSource.noCache);
-            (0, internal_2.assert)(!!versions && !!versions[0], 0x137 /* "Failed to get version from storage" */);
-            stats.getVersionDuration = trace.trace().duration;
-            const maybeSnapshot = await this.storage.getSnapshotTree(versions[0]);
-            (0, internal_2.assert)(!!maybeSnapshot, 0x138 /* "Failed to get snapshot from storage" */);
-            stats.getSnapshotDuration = trace.trace().duration;
-            const latestSnapshotRefSeq = await (0, internal_6.seqFromTree)(maybeSnapshot, readAndParseBlob);
-            stats.snapshotRefSeq = latestSnapshotRefSeq;
-            stats.snapshotVersion = versions[0].id;
-            perfEvent.end(stats);
+            let snapshotTree;
+            const scenarioName = "RefreshLatestSummaryAckFetch";
+            // If loader supplied us the ISnapshot when loading, the new getSnapshotApi is supported and feature gate is ON, then use the
+            // new API, otherwise it will reduce the service performance because the service will need to recalculate the full snapshot
+            // in case previously getSnapshotApi was used and now we use the getVersions API.
+            if (this.isSnapshotInstanceOfISnapshot &&
+                this.storage.getSnapshot !== undefined &&
+                this.mc.config.getBoolean("Fluid.Container.UseLoadingGroupIdForSnapshotFetch2") ===
+                    true) {
+                const snapshot = await this.storage.getSnapshot({
+                    scenarioName,
+                    fetchSource: internal_3.FetchSource.noCache,
+                });
+                const id = snapshot.snapshotTree.id;
+                (0, internal_2.assert)(id !== undefined, "id of the fetched snapshot should be defined");
+                props.snapshotVersion = id;
+                snapshotTree = snapshot.snapshotTree;
+            }
+            else {
+                const versions = await this.storage.getVersions(null, 1, scenarioName, internal_3.FetchSource.noCache);
+                (0, internal_2.assert)(!!versions && !!versions[0], 0x137 /* "Failed to get version from storage" */);
+                snapshotTree = await this.storage.getSnapshotTree(versions[0]);
+                (0, internal_2.assert)(!!snapshotTree, 0x138 /* "Failed to get snapshot from storage" */);
+                props.snapshotVersion = versions[0].id;
+            }
+            props.getSnapshotDuration = trace.trace().duration;
+            const readAndParseBlob = async (id) => (0, internal_4.readAndParse)(this.storage, id);
+            const snapshotRefSeq = await (0, internal_6.seqFromTree)(snapshotTree, readAndParseBlob);
+            props.snapshotRefSeq = snapshotRefSeq;
+            props.newerSnapshotPresent = snapshotRefSeq >= targetRefSeq;
+            perfEvent.end({ details: props });
+            return snapshotRefSeq;
         });
+        // If the snapshot that was fetched is older than the target snapshot, return. The summarizer will not be closed
+        // because the snapshot is likely deleted from storage and it so, closing the summarizer will result in the
+        // document being stuck in this state.
+        if (fetchedSnapshotRefSeq < targetRefSeq) {
+            return;
+        }
         await (0, internal_2.delay)(this.closeSummarizerDelayMs);
         this._summarizer?.stop("latestSummaryStateStale");
         this.disposeFn();