@fluidframework/container-runtime 2.0.0-internal.6.2.0 → 2.0.0-internal.6.3.0

package/lib/containerRuntime.js

@@ -1,6 +1,6 @@
 import { AttachState, LoaderHeader, } from "@fluidframework/container-definitions";
-import { assert, delay, Trace, TypedEventEmitter } from "@fluidframework/common-utils";
-import { LazyPromise } from "@fluidframework/core-utils";
+import { assert, delay, LazyPromise } from "@fluidframework/core-utils";
+import { Trace, TypedEventEmitter } from "@fluid-internal/client-utils";
 import { createChildLogger, createChildMonitoringContext, DataCorruptionError, DataProcessingError, GenericError, raiseConnectedEvent, PerformanceEvent, 
 // eslint-disable-next-line import/no-deprecated
 TaggedLoggerAdapter, wrapError, UsageError, } from "@fluidframework/telemetry-utils";
@@ -116,6 +116,10 @@ const defaultCompressionConfig = {
     compressionAlgorithm: CompressionAlgorithms.lz4,
 };
 const defaultChunkSizeInBytes = 204800;
+/** The default time to wait for pending ops to be processed during summarization */
+export const defaultPendingOpsWaitTimeoutMs = 1000;
+/** The default time to delay a summarization retry attempt when there are pending ops */
+export const defaultPendingOpsRetryDelayMs = 1000;
 /**
  * Instead of refreshing from latest because we do not have 100% confidence in the state
  * of the current system, we should close the summarizer and let it recover.
@@ -431,12 +435,10 @@ export class ContainerRuntime extends TypedEventEmitter {
         });
         // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
         this._audience = audience;
-        this.summaryStateUpdateMethod = this.mc.config.getString("Fluid.ContainerRuntime.Test.SummaryStateUpdateMethodV2");
         const closeSummarizerDelayOverride = this.mc.config.getNumber("Fluid.ContainerRuntime.Test.CloseSummarizerDelayOverrideMs");
         this.closeSummarizerDelayMs = closeSummarizerDelayOverride ?? defaultCloseSummarizerDelayMs;
         this.validateSummaryBeforeUpload =
-            this.mc.config.getBoolean("Fluid.ContainerRuntime.Test.ValidateSummaryBeforeUpload") ??
-                false;
+            this.mc.config.getBoolean("Fluid.Summarizer.ValidateSummaryBeforeUpload") ?? false;
         this.summaryCollection = new SummaryCollection(this.deltaManager, this.logger);
         this.dirtyContainer =
             this.attachState !== AttachState.Attached || this.hasPendingMessages();
@@ -534,7 +536,6 @@ export class ContainerRuntime extends TypedEventEmitter {
                 disableAttachReorder: this.disableAttachReorder,
                 disablePartialFlush,
                 idCompressorEnabled: this.idCompressorEnabled,
-                summaryStateUpdateMethod: this.summaryStateUpdateMethod,
                 closeSummarizerDelayOverride,
             }),
             telemetryDocumentId: this.telemetryDocumentId,
@@ -1710,6 +1711,40 @@ export class ContainerRuntime extends TypedEventEmitter {
             // We might need to catch up to the latest summary's reference sequence number before pausing.
             await this.waitForDeltaManagerToCatchup(latestSnapshotRefSeq, summaryNumberLogger);
         }
+        // If there are pending (unacked ops), the summary will not be eventual consistent and it may even be
+        // incorrect. So, wait for the container to be saved with a timeout. If the container is not saved
+        // within the timeout, check if it should be failed or can continue.
+        if (this.validateSummaryBeforeUpload && this.hasPendingMessages()) {
+            const countBefore = this.pendingMessagesCount;
+            // The timeout for waiting for pending ops can be overridden via configurations.
+            const pendingOpsTimeout = this.mc.config.getNumber("Fluid.Summarizer.waitForPendingOpsTimeoutMs") ??
+                defaultPendingOpsWaitTimeoutMs;
+            await new Promise((resolve, reject) => {
+                const timeoutId = setTimeout(() => resolve(), pendingOpsTimeout);
+                this.once("saved", () => {
+                    clearTimeout(timeoutId);
+                    resolve();
+                });
+                this.once("dispose", () => {
+                    clearTimeout(timeoutId);
+                    reject(new Error("Runtime is disposed while summarizing"));
+                });
+            });
+            // Log that there are pending ops while summarizing. This will help us gather data on how often this
+            // happens, whether we attempted to wait for these ops to be acked and what was the result.
+            summaryNumberLogger.sendTelemetryEvent({
+                eventName: "PendingOpsWhileSummarizing",
+                saved: this.hasPendingMessages() ? false : true,
+                timeout: pendingOpsTimeout,
+                countBefore,
+                countAfter: this.pendingMessagesCount,
+            });
+            // There could still be pending ops. Check if summary should fail or continue.
+            const pendingMessagesFailResult = await this.shouldFailSummaryOnPendingOps(summaryNumberLogger, this.deltaManager.lastSequenceNumber, this.deltaManager.minimumSequenceNumber, finalAttempt, true /* beforeSummaryGeneration */);
+            if (pendingMessagesFailResult !== undefined) {
+                return pendingMessagesFailResult;
+            }
+        }
         const shouldPauseInboundSignal = this.mc.config.getBoolean("Fluid.ContainerRuntime.SubmitSummary.disableInboundSignalPause") !== true;
         let summaryRefSeqNum;
         try {
@@ -1799,33 +1834,9 @@ export class ContainerRuntime extends TypedEventEmitter {
                         error,
                     };
                 }
-                // If there are pending messages, the summary has more data than at the summaryRefSeqNum.
-                if (this.hasPendingMessages()) {
-                    // If "SkipFailingIncorrectSummary" option is true, don't fail the summary in the last attempt.
-                    // This is a fallback to make progress in documents where there are consistently pending ops in
-                    // the summarizer.
-                    if (finalAttempt &&
-                        this.mc.config.getBoolean("Fluid.Summarizer.SkipFailingIncorrectSummary")) {
-                        const error = DataProcessingError.create("Pending ops during summarization", "submitSummary", undefined, { pendingMessages: this.pendingMessagesCount });
-                        summaryNumberLogger.sendErrorEvent({
-                            eventName: "SkipFailingIncorrectSummary",
-                            referenceSequenceNumber: summaryRefSeqNum,
-                            minimumSequenceNumber,
-                        }, error);
-                    }
-                    else {
-                        // Default retry delay is 1 second. This can be overridden via config so that we can adjust it
-                        // based on telemetry while we decide on a stable number.
-                        const retryDelayMs = this.mc.config.getNumber("Fluid.Summarizer.PendingOpsRetryDelayMs") ??
-                            1000;
-                        const error = new RetriableSummaryError("PendingMessagesInSummary", retryDelayMs / 1000, { count: this.pendingMessagesCount });
-                        return {
-                            stage: "base",
-                            referenceSequenceNumber: summaryRefSeqNum,
-                            minimumSequenceNumber,
-                            error,
-                        };
-                    }
+                const pendingMessagesFailResult = await this.shouldFailSummaryOnPendingOps(summaryNumberLogger, summaryRefSeqNum, minimumSequenceNumber, finalAttempt, false /* beforeSummaryGeneration */);
+                if (pendingMessagesFailResult !== undefined) {
+                    return pendingMessagesFailResult;
                 }
             }
             const { summary: summaryTree, stats: partialStats } = summarizeResult;
@@ -1947,6 +1958,50 @@ export class ContainerRuntime extends TypedEventEmitter {
             }
         }
     }
+    /**
+     * This helper is called during summarization. If there are pending ops, it will return a failed summarize result
+     * (IBaseSummarizeResult) unless this is the final summarize attempt and SkipFailingIncorrectSummary option is set.
+     * @param logger - The logger to be used for sending telemetry.
+     * @param referenceSequenceNumber - The reference sequence number of the summary attempt.
+     * @param minimumSequenceNumber - The minimum sequence number of the summary attempt.
+     * @param finalAttempt - Whether this is the final summary attempt.
+     * @param beforeSummaryGeneration - Whether this is called before summary generation or after.
+     * @returns failed summarize result (IBaseSummarizeResult) if summary should be failed, undefined otherwise.
+     */
+    async shouldFailSummaryOnPendingOps(logger, referenceSequenceNumber, minimumSequenceNumber, finalAttempt, beforeSummaryGeneration) {
+        if (!this.hasPendingMessages()) {
+            return;
+        }
+        // If "SkipFailingIncorrectSummary" option is true, don't fail the summary in the last attempt.
+        // This is a fallback to make progress in documents where there are consistently pending ops in
+        // the summarizer.
+        if (finalAttempt &&
+            this.mc.config.getBoolean("Fluid.Summarizer.SkipFailingIncorrectSummary")) {
+            const error = DataProcessingError.create("Pending ops during summarization", "submitSummary", undefined, { pendingMessages: this.pendingMessagesCount });
+            logger.sendErrorEvent({
+                eventName: "SkipFailingIncorrectSummary",
+                referenceSequenceNumber,
+                minimumSequenceNumber,
+                beforeGenerate: beforeSummaryGeneration,
+            }, error);
+        }
+        else {
+            // The retry delay when there are pending ops can be overridden via config so that we can adjust it
+            // based on telemetry while we decide on a stable number.
+            const retryDelayMs = this.mc.config.getNumber("Fluid.Summarizer.PendingOpsRetryDelayMs") ??
+                defaultPendingOpsRetryDelayMs;
+            const error = new RetriableSummaryError("PendingOpsWhileSummarizing", retryDelayMs / 1000, {
+                count: this.pendingMessagesCount,
+                beforeGenerate: beforeSummaryGeneration,
+            });
+            return {
+                stage: "base",
+                referenceSequenceNumber,
+                minimumSequenceNumber,
+                error,
+            };
+        }
+    }
     get pendingMessagesCount() {
         return this.pendingStateManager.pendingMessagesCount + this.outbox.messageCount;
     }
@@ -2265,29 +2320,22 @@ export class ContainerRuntime extends TypedEventEmitter {
     /** Implementation of ISummarizerInternalsProvider.refreshLatestSummaryAck */
     async refreshLatestSummaryAck(options) {
         const { proposalHandle, ackHandle, summaryRefSeq, summaryLogger } = options;
+        // proposalHandle is always passed from RunningSummarizer.
+        assert(proposalHandle !== undefined, 0x766 /* proposalHandle should be available */);
         const readAndParseBlob = async (id) => readAndParse(this.storage, id);
-        // The call to fetch the snapshot is very expensive and not always needed.
-        // It should only be done by the summarizerNode, if required.
-        // When fetching from storage we will always get the latest version and do not use the ackHandle.
-        const fetchLatestSnapshot = async () => {
-            let fetchResult = await this.fetchSnapshotFromStorageAndMaybeClose(summaryLogger, {
+        const result = await this.summarizerNode.refreshLatestSummary(proposalHandle, summaryRefSeq);
+        /**
+         * When refreshing a summary ack, this check indicates a new ack of a summary that is newer than the
+         * current summary that is tracked, but this summarizer runtime did not produce/track that summary. Thus
+         * it needs to refresh its state. Today refresh is done by fetching the latest snapshot to update the cache
+         * and then close as the current main client is likely to be re-elected as the parent summarizer again.
+         */
+        if (!result.isSummaryTracked && result.isSummaryNewer) {
+            const fetchResult = await this.fetchSnapshotFromStorage(summaryLogger, {
                 eventName: "RefreshLatestSummaryAckFetch",
                 ackHandle,
                 targetSequenceNumber: summaryRefSeq,
             }, readAndParseBlob, null);
-            /**
-             * back-compat - Older loaders and drivers (pre 2.0.0-internal.1.4) don't have fetchSource as a param in the
-             * getVersions API. So, they will not fetch the latest snapshot from network in the previous fetch call. For
-             * these scenarios, fetch the snapshot corresponding to the ack handle to have the same behavior before the
-             * change that started fetching latest snapshot always.
-             */
-            if (fetchResult.latestSnapshotRefSeq < summaryRefSeq) {
-                fetchResult = await this.fetchSnapshotFromStorageAndMaybeClose(summaryLogger, {
-                    eventName: "RefreshLatestSummaryAckFetchBackCompat",
-                    ackHandle,
-                    targetSequenceNumber: summaryRefSeq,
-                }, readAndParseBlob, ackHandle);
-            }
             /**
              * If the fetched snapshot is older than the one for which the ack was received, close the container.
              * This should never happen because an ack should be sent after the latest summary is updated in the server.
@@ -2307,17 +2355,11 @@ export class ContainerRuntime extends TypedEventEmitter {
                 this.disposeFn(error);
                 throw error;
             }
-            // In case we had to retrieve the latest snapshot and it is different than summaryRefSeq,
-            // wait for the delta manager to catch up before refreshing the latest Summary.
-            await this.waitForDeltaManagerToCatchup(fetchResult.latestSnapshotRefSeq, summaryLogger);
-            return {
-                snapshotTree: fetchResult.snapshotTree,
-                snapshotRefSeq: fetchResult.latestSnapshotRefSeq,
-            };
-        };
-        const result = await this.summarizerNode.refreshLatestSummary(proposalHandle, summaryRefSeq, fetchLatestSnapshot, readAndParseBlob, summaryLogger);
+            await this.closeStaleSummarizer("RefreshLatestSummaryAckFetch");
+            return;
+        }
         // Notify the garbage collector so it can update its latest summary state.
-        await this.garbageCollector.refreshLatestSummary(proposalHandle, result, readAndParseBlob);
+        await this.garbageCollector.refreshLatestSummary(result);
     }
     /**
      * Fetches the latest snapshot from storage and uses it to refresh SummarizerNode's
@@ -2327,25 +2369,31 @@ export class ContainerRuntime extends TypedEventEmitter {
      */
     async refreshLatestSummaryAckFromServer(summaryLogger) {
         const readAndParseBlob = async (id) => readAndParse(this.storage, id);
-        const { snapshotTree, versionId, latestSnapshotRefSeq } = await this.fetchSnapshotFromStorageAndMaybeClose(summaryLogger, {
+        const { versionId, latestSnapshotRefSeq } = await this.fetchSnapshotFromStorage(summaryLogger, {
             eventName: "RefreshLatestSummaryFromServerFetch",
         }, readAndParseBlob, null);
-        const fetchLatestSnapshot = {
-            snapshotTree,
-            snapshotRefSeq: latestSnapshotRefSeq,
-        };
-        const result = await this.summarizerNode.refreshLatestSummary(undefined /* proposalHandle */, latestSnapshotRefSeq, async () => fetchLatestSnapshot, readAndParseBlob, summaryLogger);
-        // Notify the garbage collector so it can update its latest summary state.
-        await this.garbageCollector.refreshLatestSummary(undefined /* proposalHandle */, result, readAndParseBlob);
+        await this.closeStaleSummarizer("RefreshLatestSummaryFromServerFetch");
         return { latestSnapshotRefSeq, latestSnapshotVersionId: versionId };
     }
+    async closeStaleSummarizer(codePath) {
+        this.mc.logger.sendTelemetryEvent({
+            eventName: "ClosingSummarizerOnSummaryStale",
+            codePath,
+            message: "Stopping fetch from storage",
+            closeSummarizerDelayMs: this.closeSummarizerDelayMs,
+        }, new GenericError("Restarting summarizer instead of refreshing"));
+        // Delay before restarting summarizer to prevent the summarizer from restarting too frequently.
+        await delay(this.closeSummarizerDelayMs);
+        this._summarizer?.stop("latestSummaryStateStale");
+        this.disposeFn();
+    }
     /**
      * Downloads snapshot from storage with the given versionId or latest if versionId is null.
      * By default, it also closes the container after downloading the snapshot. However, this may be
      * overridden via options.
      */
-    async fetchSnapshotFromStorageAndMaybeClose(logger, event, readAndParseBlob, versionId) {
-        const snapshotResults = await PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
+    async fetchSnapshotFromStorage(logger, event, readAndParseBlob, versionId) {
+        return PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
             const stats = {};
             const trace = Trace.start();
             const versions = await this.storage.getVersions(versionId, 1, "refreshLatestSummaryAckFromServer", versionId === null ? FetchSource.noCache : undefined);
@@ -2364,45 +2412,37 @@ export class ContainerRuntime extends TypedEventEmitter {
                 latestSnapshotRefSeq,
             };
         });
-        // We choose to close the summarizer after the snapshot cache is updated to avoid
-        // situations which the main client (which is likely to be re-elected as the leader again)
-        // loads the summarizer from cache.
-        if (this.summaryStateUpdateMethod !== "refreshFromSnapshot") {
-            this.mc.logger.sendTelemetryEvent({
-                ...event,
-                eventName: "ClosingSummarizerOnSummaryStale",
-                codePath: event.eventName,
-                message: "Stopping fetch from storage",
-                versionId: versionId != null ? versionId : undefined,
-                closeSummarizerDelayMs: this.closeSummarizerDelayMs,
-            }, new GenericError("Restarting summarizer instead of refreshing"));
-            // Delay before restarting summarizer to prevent the summarizer from restarting too frequently.
-            await delay(this.closeSummarizerDelayMs);
-            this._summarizer?.stop("latestSummaryStateStale");
-            this.disposeFn();
-        }
-        return snapshotResults;
     }
     notifyAttaching() { } // do nothing (deprecated method)
     async getPendingLocalState(props) {
-        this.verifyNotClosed();
-        const waitBlobsToAttach = props?.notifyImminentClosure;
-        if (this._orderSequentiallyCalls !== 0) {
-            throw new UsageError("can't get state during orderSequentially");
-        }
-        const pendingAttachmentBlobs = await this.blobManager.getPendingBlobs(waitBlobsToAttach);
-        if (!pendingAttachmentBlobs && !this.hasPendingMessages()) {
-            return; // no pending state to save
-        }
-        // Flush pending batch.
-        // getPendingLocalState() is only exposed through Container.closeAndGetPendingLocalState(), so it's safe
-        // to close current batch.
-        this.flush();
-        const pendingState = {
-            pending: this.pendingStateManager.getLocalState(),
-            pendingAttachmentBlobs,
-        };
-        return pendingState;
+        return PerformanceEvent.timedExecAsync(this.mc.logger, {
+            eventName: "getPendingLocalState",
+            notifyImminentClosure: props?.notifyImminentClosure,
+        }, async (event) => {
+            this.verifyNotClosed();
+            const waitBlobsToAttach = props?.notifyImminentClosure;
+            if (this._orderSequentiallyCalls !== 0) {
+                throw new UsageError("can't get state during orderSequentially");
+            }
+            const pendingAttachmentBlobs = await this.blobManager.getPendingBlobs(waitBlobsToAttach);
+            const pending = this.pendingStateManager.getLocalState();
+            if (!pendingAttachmentBlobs && !this.hasPendingMessages()) {
+                return; // no pending state to save
+            }
+            // Flush pending batch.
+            // getPendingLocalState() is only exposed through Container.closeAndGetPendingLocalState(), so it's safe
+            // to close current batch.
+            this.flush();
+            const pendingState = {
+                pending,
+                pendingAttachmentBlobs,
+            };
+            event.end({
+                attachmentBlobsSize: Object.keys(pendingAttachmentBlobs ?? {}).length,
+                pendingOpsSize: pending?.pendingStates.length,
+            });
+            return pendingState;
+        });
     }
     summarizeOnDemand(options) {
         if (this.isSummarizerClient) {