@fluidframework/container-runtime 2.1.0-276985 → 2.1.0

package/lib/containerRuntime.js

@@ -25,7 +25,7 @@ import { FluidDataStoreRegistry } from "./dataStoreRegistry.js";
 import { DeltaManagerPendingOpsProxy, DeltaManagerSummarizerProxy, } from "./deltaManagerProxies.js";
 import { GCNodeType, GarbageCollector, gcGenerationOptionName, } from "./gc/index.js";
 import { ContainerMessageType, } from "./messageTypes.js";
-import { OpCompressor, OpDecompressor, OpGroupingManager, OpSplitter, Outbox, RemoteMessageProcessor, } from "./opLifecycle/index.js";
+import { ensureContentsDeserialized, OpCompressor, OpDecompressor, OpGroupingManager, OpSplitter, Outbox, RemoteMessageProcessor, } from "./opLifecycle/index.js";
 import { pkgVersion } from "./packageVersion.js";
 import { PendingStateManager, } from "./pendingStateManager.js";
 import { ScheduleManager } from "./scheduleManager.js";
@@ -161,7 +161,7 @@ export function getDeviceSpec() {
 export const makeLegacySendBatchFn = (submitFn, deltaManager) => (batch) => {
     // Default to negative one to match Container.submitBatch behavior
     let clientSequenceNumber = -1;
-    for (const message of batch.content) {
+    for (const message of batch.messages) {
         clientSequenceNumber = submitFn(MessageType.Operation, 
         // For back-compat (submitFn only works on deserialized content)
         message.contents === undefined ? undefined : JSON.parse(message.contents), true, // batch
@@ -375,9 +375,7 @@ export class ContainerRuntime extends TypedEventEmitter {
                 return createIdCompressor(compressorLogger);
             }
         };
-        const disableCompression = mc.config.getBoolean("Fluid.ContainerRuntime.CompressionDisabled");
-        const compressionLz4 = disableCompression !== true &&
-            compressionOptions.minimumBatchSizeInBytes !== Infinity &&
+        const compressionLz4 = compressionOptions.minimumBatchSizeInBytes !== Infinity &&
             compressionOptions.compressionAlgorithm === "lz4";
         const documentSchemaController = new DocumentsSchemaController(existing, protocolSequenceNumber, metadata?.documentSchema, {
             explicitSchemaControl,
@@ -388,9 +386,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         }, (schema) => {
             runtime.onSchemaChange(schema);
         });
-        const featureGatesForTelemetry = {
-            disableCompression,
-        };
+        const featureGatesForTelemetry = {};
         const runtime = new containerRuntimeCtor(context, registry, metadata, electedSummarizerData, chunks ?? [], aliases ?? [], {
             summaryOptions,
             gcOptions,
@@ -660,14 +656,13 @@ export class ContainerRuntime extends TypedEventEmitter {
         });
         this.telemetryDocumentId = metadata?.telemetryDocumentId ?? uuid();
         this.disableAttachReorder = this.mc.config.getBoolean("Fluid.ContainerRuntime.disableAttachOpReorder");
-        const disableChunking = this.mc.config.getBoolean("Fluid.ContainerRuntime.CompressionChunkingDisabled");
         const opGroupingManager = new OpGroupingManager({
             groupedBatchingEnabled: this.groupedBatchingEnabled,
             opCountThreshold: this.mc.config.getNumber("Fluid.ContainerRuntime.GroupedBatchingOpCount") ?? 2,
             reentrantBatchGroupingEnabled: this.mc.config.getBoolean("Fluid.ContainerRuntime.GroupedBatchingReentrancy") ??
                 true,
         }, this.mc.logger);
-        const opSplitter = new OpSplitter(chunks, this.submitBatchFn, disableChunking === true ? Number.POSITIVE_INFINITY : runtimeOptions.chunkSizeInBytes, runtimeOptions.maxBatchSizeInBytes, this.mc.logger);
+        const opSplitter = new OpSplitter(chunks, this.submitBatchFn, runtimeOptions.chunkSizeInBytes, runtimeOptions.maxBatchSizeInBytes, this.mc.logger);
         this.remoteMessageProcessor = new RemoteMessageProcessor(opSplitter, new OpDecompressor(this.mc.logger), opGroupingManager);
         const pendingRuntimeState = pendingLocalState;
         this.pendingStateManager = new PendingStateManager({
@@ -675,10 +670,6 @@ export class ContainerRuntime extends TypedEventEmitter {
             clientId: () => this.clientId,
             close: this.closeFn,
             connected: () => this.connected,
-            reSubmit: (message) => {
-                this.reSubmit(message);
-                this.flush();
-            },
             reSubmitBatch: this.reSubmitBatch.bind(this),
             isActiveConnection: () => this.innerDeltaManager.active,
             isAttached: () => this.attachState !== AttachState.Detached,
@@ -761,6 +752,9 @@ export class ContainerRuntime extends TypedEventEmitter {
             this.summarizerNode.updateBaseSummaryState(baseSnapshot);
         }
         const parentContext = wrapContext(this);
+        if (snapshotWithContents !== undefined) {
+            this.isSnapshotInstanceOfISnapshot = true;
+        }
         // Due to a mismatch between different layers in terms of
         // what is the interface of passing signals, we need the
         // downstream stores to wrap the signal.
@@ -930,7 +924,6 @@ export class ContainerRuntime extends TypedEventEmitter {
             sessionRuntimeSchema: JSON.stringify(this.sessionSchema),
             featureGates: JSON.stringify({
                 ...featureGatesForTelemetry,
-                disableChunking,
                 disableAttachReorder: this.disableAttachReorder,
                 disablePartialFlush,
                 closeSummarizerDelayOverride,
@@ -1109,9 +1102,13 @@ export class ContainerRuntime extends TypedEventEmitter {
         let childTree = snapshotTree;
         for (const part of pathParts) {
             if (hasIsolatedChannels) {
-                childTree = childTree?.trees[channelsTreeName];
+                // TODO Why are we non null asserting here
+                // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+                childTree = childTree.trees[channelsTreeName];
             }
-            childTree = childTree?.trees[part];
+            // TODO Why are we non null asserting here
+            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+            childTree = childTree.trees[part];
         }
         return childTree;
     }
@@ -1158,6 +1155,8 @@ export class ContainerRuntime extends TypedEventEmitter {
                 return this.resolveHandle(requestParser.createSubRequest(1));
             }
             if (id === blobManagerBasePath && requestParser.isLeaf(2)) {
+                // TODO why are we non null asserting here?
+                // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
                 const blob = await this.blobManager.getBlob(requestParser.pathParts[1]);
                 return blob
                     ? {
@@ -1465,39 +1464,50 @@ export class ContainerRuntime extends TypedEventEmitter {
         // It may be a legacy runtime message (ie already unpacked and ContainerMessageType)
         // or something different, like a system message.
         const modernRuntimeMessage = messageArg.type === MessageType.Operation;
+        const savedOp = messageArg.metadata?.savedOp;
+        // There is some ancient back-compat code that we'd like to instrument
+        // to understand if/when it is hit.
+        const logLegacyCase = (codePath) => this.logger.sendTelemetryEvent({
+            eventName: "LegacyMessageFormat",
+            details: { codePath, type: messageArg.type },
+        });
         // Do shallow copy of message, as the processing flow will modify it.
         // There might be multiple container instances receiving the same message.
         // We do not need to make a deep copy. Each layer will just replace message.contents itself,
         // but will not modify the contents object (likely it will replace it on the message).
         const messageCopy = { ...messageArg };
-        const savedOp = messageCopy.metadata?.savedOp;
-        const processResult = this.remoteMessageProcessor.process(messageCopy);
-        if (processResult === undefined) {
-            // This means the incoming message is an incomplete part of a message or batch
-            // and we need to process more messages before the rest of the system can understand it.
-            return;
-        }
-        for (const message of processResult.messages) {
-            const msg = modernRuntimeMessage
-                ? {
-                    // Cast it since we expect it to be this based on modernRuntimeMessage computation above.
-                    // There is nothing really ensuring that anytime original message.type is Operation that
-                    // the result messages will be so. In the end modern bool being true only directs to
-                    // throw error if ultimately unrecognized without compat details saying otherwise.
-                    message: message,
+        // We expect runtime messages to have JSON contents - deserialize it in place.
+        ensureContentsDeserialized(messageCopy, modernRuntimeMessage, logLegacyCase);
+        if (modernRuntimeMessage) {
+            const processResult = this.remoteMessageProcessor.process(messageCopy, logLegacyCase);
+            if (processResult === undefined) {
+                // This means the incoming message is an incomplete part of a message or batch
+                // and we need to process more messages before the rest of the system can understand it.
+                return;
+            }
+            const batchStartCsn = processResult.batchStartCsn;
+            const batch = processResult.messages;
+            const messages = local
+                ? this.pendingStateManager.processPendingLocalBatch(batch, batchStartCsn)
+                : batch.map((message) => ({ message, localOpMetadata: undefined }));
+            messages.forEach(({ message, localOpMetadata }) => {
+                const msg = {
+                    message,
                     local,
                     modernRuntimeMessage,
-                    batchStartCsn: processResult.batchStartCsn,
-                }
-                : // Unrecognized message will be ignored.
-                    {
-                        message,
-                        local,
-                        modernRuntimeMessage,
-                        batchStartCsn: processResult.batchStartCsn,
-                    };
-            msg.savedOp = savedOp;
-            // ensure that we observe any re-entrancy, and if needed, rebase ops
+                    savedOp,
+                    localOpMetadata,
+                };
+                this.ensureNoDataModelChanges(() => this.processCore(msg));
+            });
+        }
+        else {
+            const msg = {
+                message: messageCopy,
+                local,
+                modernRuntimeMessage,
+                savedOp,
+            };
             this.ensureNoDataModelChanges(() => this.processCore(msg));
         }
     }
@@ -1505,7 +1515,7 @@ export class ContainerRuntime extends TypedEventEmitter {
      * Direct the message to the correct subsystem for processing, and implement other side effects
      */
     processCore(messageWithContext) {
-        const { message, local } = messageWithContext;
+        const { message, local, localOpMetadata } = messageWithContext;
         // Intercept to reduce minimum sequence number to the delta manager's minimum sequence number.
         // Sequence numbers are not guaranteed to follow any sort of order. Re-entrancy is one of those situations
         if (this.deltaManager.minimumSequenceNumber <
@@ -1519,15 +1529,8 @@ export class ContainerRuntime extends TypedEventEmitter {
         this.scheduleManager.beforeOpProcessing(message);
         this._processedClientSequenceNumber = message.clientSequenceNumber;
         try {
-            // See commit that added this assert for more details.
-            // These calls should be made for all but chunked ops:
-            // 1) this.pendingStateManager.processPendingLocalMessage() below
-            // 2) this.resetReconnectCount() below
+            // RemoteMessageProcessor would have already reconstituted Chunked Ops into the original op type
             assert(message.type !== ContainerMessageType.ChunkedOp, 0x93b /* we should never get here with chunked ops */);
-            let localOpMetadata;
-            if (local && messageWithContext.modernRuntimeMessage) {
-                localOpMetadata = this.pendingStateManager.processPendingLocalMessage(messageWithContext.message, messageWithContext.batchStartCsn);
-            }
             // If there are no more pending messages after processing a local message,
             // the document is no longer dirty.
             if (!this.hasPendingMessages()) {
@@ -1686,10 +1689,12 @@ export class ContainerRuntime extends TypedEventEmitter {
     /**
      * Flush the pending ops manually.
      * This method is expected to be called at the end of a batch.
+     * @param resubmittingBatchId - If defined, indicates this is a resubmission of a batch
+     * with the given Batch ID, which must be preserved
      */
-    flush() {
+    flush(resubmittingBatchId) {
         assert(this._orderSequentiallyCalls === 0, 0x24c /* "Cannot call `flush()` from `orderSequentially`'s callback" */);
-        this.outbox.flush();
+        this.outbox.flush(resubmittingBatchId);
         assert(this.outbox.isEmpty, 0x3cf /* reentrancy */);
     }
     /**
@@ -1702,7 +1707,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             // Note: we are not touching any batches other than mainBatch here, for two reasons:
             // 1. It would not help, as other batches are flushed independently from main batch.
             // 2. There is no way to undo process of data store creation, blob creation, ID compressor ops, or other things tracked by other batches.
-            checkpoint = this.outbox.checkpoint().mainBatch;
+            checkpoint = this.outbox.getBatchCheckpoints().mainBatch;
         }
         try {
             this._orderSequentiallyCalls++;
@@ -1793,7 +1798,7 @@ export class ContainerRuntime extends TypedEventEmitter {
         return (this.connected && !this.innerDeltaManager.readOnlyInfo.readonly && !this.imminentClosure);
     }
     /**
-     * Are we in the middle of batching ops together?
+     * Typically ops are batched and later flushed together, but in some cases we want to flush immediately.
      */
     currentlyBatching() {
         return this.flushMode !== FlushMode.Immediate || this._orderSequentiallyCalls !== 0;
@@ -2154,7 +2159,7 @@ export class ContainerRuntime extends TypedEventEmitter {
             summaryRefSeqNum = this.deltaManager.lastSequenceNumber;
             const minimumSequenceNumber = this.deltaManager.minimumSequenceNumber;
             const message = `Summary @${summaryRefSeqNum}:${this.deltaManager.minimumSequenceNumber}`;
-            const lastAck = this.summaryCollection.latestAck;
+            const lastAckedContext = this.lastAckedSummaryContext;
             const startSummaryResult = this.summarizerNode.startSummary(summaryRefSeqNum, summaryNumberLogger, latestSummaryRefSeqNum);
             /**
              * This was added to validate that the summarizer node tree has the same reference sequence number from the
@@ -2206,10 +2211,10 @@ export class ContainerRuntime extends TypedEventEmitter {
                     };
                 }
                 assert(summaryRefSeqNum === this.deltaManager.lastMessage?.sequenceNumber, 0x395 /* it's one and the same thing */);
-                if (lastAck !== this.summaryCollection.latestAck) {
+                if (lastAckedContext !== this.lastAckedSummaryContext) {
                     return {
                         continue: false,
-                        error: `Last summary changed while summarizing. ${this.summaryCollection.latestAck} !== ${lastAck}`,
+                        error: `Last summary changed while summarizing. ${this.lastAckedSummaryContext} !== ${lastAckedContext}`,
                     };
                 }
                 return { continue: true };
@@ -2265,6 +2270,8 @@ export class ContainerRuntime extends TypedEventEmitter {
             // Counting dataStores and handles
             // Because handles are unchanged dataStores in the current logic,
             // summarized dataStore count is total dataStore count minus handle count
+            // TODO why are we non null asserting here
+            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
             const dataStoreTree = summaryTree.tree[channelsTreeName];
             assert(dataStoreTree.type === SummaryType.Tree, 0x1fc /* "summary is not a tree" */);
             const handleCount = Object.values(dataStoreTree.tree).filter((value) => value.type === SummaryType.Handle).length;
@@ -2295,17 +2302,11 @@ export class ContainerRuntime extends TypedEventEmitter {
                     error: new RetriableSummaryError(continueResult.error),
                 };
             }
-            const summaryContext = lastAck === undefined
-                ? {
-                    proposalHandle: undefined,
-                    ackHandle: this.loadedFromVersionId,
-                    referenceSequenceNumber: summaryRefSeqNum,
-                }
-                : {
-                    proposalHandle: lastAck.summaryOp.contents.handle,
-                    ackHandle: lastAck.summaryAck.contents.handle,
-                    referenceSequenceNumber: summaryRefSeqNum,
-                };
+            const summaryContext = {
+                proposalHandle: this.lastAckedSummaryContext?.proposalHandle ?? undefined,
+                ackHandle: this.lastAckedSummaryContext?.ackHandle ?? this.loadedFromVersionId,
+                referenceSequenceNumber: summaryRefSeqNum,
+            };
             let handle;
             try {
                 handle = await this.storage.uploadSummaryWithContext(summarizeResult.summary, summaryContext);
@@ -2525,7 +2526,9 @@ export class ContainerRuntime extends TypedEventEmitter {
             else {
                 this.outbox.submit(message);
             }
-            if (!this.currentlyBatching()) {
+            // Note: Technically, the system "always" batches - if this case is true we'll just have a single-message batch.
+            const flushImmediatelyOnSubmit = !this.currentlyBatching();
+            if (flushImmediatelyOnSubmit) {
                 this.flush();
             }
             else {
@@ -2592,13 +2595,16 @@ export class ContainerRuntime extends TypedEventEmitter {
             throw new Error("Runtime is closed");
         }
     }
-    reSubmitBatch(batch) {
+    reSubmitBatch(batch, batchId) {
         this.orderSequentially(() => {
             for (const message of batch) {
                 this.reSubmit(message);
             }
         });
-        this.flush();
+        // Only include Batch ID if "Offline Load" feature is enabled
+        // It's only needed to identify batches across container forks arising from misuse of offline load.
+        const includeBatchId = this.mc.config.getBoolean("Fluid.Container.enableOfflineLoad") ?? false;
+        this.flush(includeBatchId ? batchId : undefined);
     }
     reSubmit(message) {
         // Need to parse from string for back-compat
@@ -2689,45 +2695,91 @@ export class ContainerRuntime extends TypedEventEmitter {
         const { proposalHandle, ackHandle, summaryRefSeq, summaryLogger } = options;
         // proposalHandle is always passed from RunningSummarizer.
         assert(proposalHandle !== undefined, 0x766 /* proposalHandle should be available */);
-        const readAndParseBlob = async (id) => readAndParse(this.storage, id);
         const result = await this.summarizerNode.refreshLatestSummary(proposalHandle, summaryRefSeq);
+        /* eslint-disable jsdoc/check-indentation */
         /**
-         * When refreshing a summary ack, this check indicates a new ack of a summary that is newer than the
-         * current summary that is tracked, but this summarizer runtime did not produce/track that summary. Thus
-         * it needs to refresh its state. Today refresh is done by fetching the latest snapshot to update the cache
-         * and then close as the current main client is likely to be re-elected as the parent summarizer again.
+         * If the snapshot corresponding to the ack is not tracked by this client, it was submitted by another client.
+         * Take action as per the following scenarios:
+         * 1. If that snapshot is older than the one tracked by this client, ignore the ack because only the latest
+         *    snapshot is tracked.
+         * 2. If that snapshot is newer, attempt to fetch the latest snapshot and do one of the following:
+         *    2.1. If the fetched snapshot is same or newer than the one for which ack was received, close this client.
+         *         The next summarizer client will likely start from this snapshot and get out of this state. Fetching
+         *         the snapshot updates the cache for this client so if it's re-elected as summarizer, this will prevent
+         *         any thrashing.
+         *    2.2. If the fetched snapshot is older than the one for which ack was received, ignore the ack. This can
+         *         happen in scenarios where the snapshot for the ack was lost in storage (in scenarios like DB rollback,
+         *         etc.) but the summary ack is still there because it's tracked a different service. In such cases,
+         *         ignoring the ack is the correct thing to do because the latest snapshot in storage is not the one for
+         *         the ack but is still the one tracked by this client. If we were to close the summarizer like in the
+         *         previous scenario, it will result in this document stuck in this state in a loop.
          */
-        if (!result.isSummaryTracked && result.isSummaryNewer) {
-            await this.fetchLatestSnapshotAndClose(summaryLogger, {
-                eventName: "RefreshLatestSummaryAckFetch",
-                ackHandle,
-                targetSequenceNumber: summaryRefSeq,
-            }, readAndParseBlob);
+        /* eslint-enable jsdoc/check-indentation */
+        if (!result.isSummaryTracked) {
+            if (result.isSummaryNewer) {
+                await this.fetchLatestSnapshotAndMaybeClose(summaryRefSeq, ackHandle, summaryLogger);
+            }
             return;
         }
         // Notify the garbage collector so it can update its latest summary state.
         await this.garbageCollector.refreshLatestSummary(result);
+        // If we here, the ack was tracked by this client. Update the summary context of the last ack.
+        this.lastAckedSummaryContext = {
+            proposalHandle,
+            ackHandle,
+            referenceSequenceNumber: summaryRefSeq,
+        };
     }
     /**
-     * Fetches the latest snapshot from storage and closes the container. This is done in cases where
-     * the last known snapshot is older than the latest one. This will ensure that the latest snapshot
-     * is downloaded and we don't end up loading snapshot from cache.
+     * Fetches the latest snapshot from storage. If the fetched snapshot is same or newer than the one for which ack
+     * was received, close this client. Fetching the snapshot will update the cache for this client so if it's
+     * re-elected as summarizer, this will prevent any thrashing.
+     * If the fetched snapshot is older than the one for which ack was received, ignore the ack and return. This can
+     * happen in scenarios where the snapshot for the ack was lost in storage in scenarios like DB rollback, etc.
      */
-    async fetchLatestSnapshotAndClose(logger, event, readAndParseBlob) {
-        await PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
-            const stats = {};
+    async fetchLatestSnapshotAndMaybeClose(targetRefSeq, targetAckHandle, logger) {
+        const fetchedSnapshotRefSeq = await PerformanceEvent.timedExecAsync(logger, { eventName: "RefreshLatestSummaryAckFetch" }, async (perfEvent) => {
+            const props = { targetRefSeq, targetAckHandle };
             const trace = Trace.start();
-            const versions = await this.storage.getVersions(null, 1, "prefetchLatestSummaryBeforeClose", FetchSource.noCache);
-            assert(!!versions && !!versions[0], 0x137 /* "Failed to get version from storage" */);
-            stats.getVersionDuration = trace.trace().duration;
-            const maybeSnapshot = await this.storage.getSnapshotTree(versions[0]);
-            assert(!!maybeSnapshot, 0x138 /* "Failed to get snapshot from storage" */);
-            stats.getSnapshotDuration = trace.trace().duration;
-            const latestSnapshotRefSeq = await seqFromTree(maybeSnapshot, readAndParseBlob);
-            stats.snapshotRefSeq = latestSnapshotRefSeq;
-            stats.snapshotVersion = versions[0].id;
-            perfEvent.end(stats);
+            let snapshotTree;
+            const scenarioName = "RefreshLatestSummaryAckFetch";
+            // If loader supplied us the ISnapshot when loading, the new getSnapshotApi is supported and feature gate is ON, then use the
+            // new API, otherwise it will reduce the service performance because the service will need to recalculate the full snapshot
+            // in case previously getSnapshotApi was used and now we use the getVersions API.
+            if (this.isSnapshotInstanceOfISnapshot &&
+                this.storage.getSnapshot !== undefined &&
+                this.mc.config.getBoolean("Fluid.Container.UseLoadingGroupIdForSnapshotFetch2") ===
+                    true) {
+                const snapshot = await this.storage.getSnapshot({
+                    scenarioName,
+                    fetchSource: FetchSource.noCache,
+                });
+                const id = snapshot.snapshotTree.id;
+                assert(id !== undefined, 0x9d0 /* id of the fetched snapshot should be defined */);
+                props.snapshotVersion = id;
+                snapshotTree = snapshot.snapshotTree;
+            }
+            else {
+                const versions = await this.storage.getVersions(null, 1, scenarioName, FetchSource.noCache);
+                assert(!!versions && !!versions[0], 0x137 /* "Failed to get version from storage" */);
+                snapshotTree = await this.storage.getSnapshotTree(versions[0]);
+                assert(!!snapshotTree, 0x138 /* "Failed to get snapshot from storage" */);
+                props.snapshotVersion = versions[0].id;
+            }
+            props.getSnapshotDuration = trace.trace().duration;
+            const readAndParseBlob = async (id) => readAndParse(this.storage, id);
+            const snapshotRefSeq = await seqFromTree(snapshotTree, readAndParseBlob);
+            props.snapshotRefSeq = snapshotRefSeq;
+            props.newerSnapshotPresent = snapshotRefSeq >= targetRefSeq;
+            perfEvent.end({ details: props });
+            return snapshotRefSeq;
         });
+        // If the snapshot that was fetched is older than the target snapshot, return. The summarizer will not be closed
+        // because the snapshot is likely deleted from storage and it so, closing the summarizer will result in the
+        // document being stuck in this state.
+        if (fetchedSnapshotRefSeq < targetRefSeq) {
+            return;
+        }
         await delay(this.closeSummarizerDelayMs);
         this._summarizer?.stop("latestSummaryStateStale");
         this.disposeFn();