@fluidframework/container-runtime 1.2.3-84921 → 1.2.5

package/src/containerRuntime.ts

@@ -2534,21 +2534,24 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
             },
         );
 
+        let latestSnapshotVersionId: string | undefined;
         if (refreshLatestAck) {
-            const latestSummaryRefSeq = await this.refreshLatestSummaryAckFromServer(
+            const latestSnapshotInfo = await this.refreshLatestSummaryAckFromServer(
                 ChildLogger.create(summaryNumberLogger, undefined, { all: { safeSummary: true } }));
+            const latestSnapshotRefSeq = latestSnapshotInfo.latestSnapshotRefSeq;
+            latestSnapshotVersionId = latestSnapshotInfo.latestSnapshotVersionId;
 
-            if (latestSummaryRefSeq > this.deltaManager.lastSequenceNumber) {
+            if (latestSnapshotRefSeq > this.deltaManager.lastSequenceNumber) {
                 // We need to catch up to the latest summary's reference sequence number before pausing.
                 await PerformanceEvent.timedExecAsync(
                     summaryNumberLogger,
                     {
                         eventName: "WaitingForSeq",
                         lastSequenceNumber: this.deltaManager.lastSequenceNumber,
-                        targetSequenceNumber: latestSummaryRefSeq,
+                        targetSequenceNumber: latestSnapshotRefSeq,
                         lastKnownSeqNumber: this.deltaManager.lastKnownSeqNumber,
                     },
-                    async () => waitForSeq(this.deltaManager, latestSummaryRefSeq),
+                    async () => waitForSeq(this.deltaManager, latestSnapshotRefSeq),
                     { start: true, end: true, cancel: "error" }, // definitely want start event
                 );
             }
@@ -2673,19 +2676,33 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
                 return { stage: "generate", ...generateSummaryData, error: continueResult.error };
             }
 
+            // It may happen that the lastAck it not correct due to missing summaryAck in case of single commit
+            // summary. So if the previous summarizer closes just after submitting the summary and before
+            // submitting the summaryOp then we can't rely on summaryAck. So in case we have
+            // latestSnapshotVersionId from storage and it does not match with the lastAck ackHandle, then use
+            // the one fetched from storage as parent as that is the latest.
             const lastAck = this.summaryCollection.latestAck;
-            const summaryContext: ISummaryContext =
-                lastAck === undefined
-                    ? {
-                        proposalHandle: undefined,
-                        ackHandle: this.context.getLoadedFromVersion()?.id,
-                        referenceSequenceNumber: summaryRefSeqNum,
-                    }
-                    : {
-                        proposalHandle: lastAck.summaryOp.contents.handle,
-                        ackHandle: lastAck.summaryAck.contents.handle,
-                        referenceSequenceNumber: summaryRefSeqNum,
-                    };
+            let summaryContext: ISummaryContext;
+            if (lastAck?.summaryAck.contents.handle !== latestSnapshotVersionId
+                && latestSnapshotVersionId !== undefined) {
+                summaryContext = {
+                    proposalHandle: undefined,
+                    ackHandle: latestSnapshotVersionId,
+                    referenceSequenceNumber: summaryRefSeqNum,
+                };
+            } else if (lastAck === undefined) {
+                summaryContext = {
+                    proposalHandle: undefined,
+                    ackHandle: this.context.getLoadedFromVersion()?.id,
+                    referenceSequenceNumber: summaryRefSeqNum,
+                };
+            } else {
+                summaryContext = {
+                    proposalHandle: lastAck.summaryOp.contents.handle,
+                    ackHandle: lastAck.summaryAck.contents.handle,
+                    referenceSequenceNumber: summaryRefSeqNum,
+                };
+            }
 
             let handle: string;
             try {
@@ -3039,15 +3056,20 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
         summaryLogger: ITelemetryLogger,
     ) {
         const readAndParseBlob = async <T>(id: string) => readAndParse<T>(this.storage, id);
-        const result = await this.summarizerNode.refreshLatestSummary(
-            proposalHandle,
-            summaryRefSeq,
-            async () => this.fetchSnapshotFromStorage(ackHandle, summaryLogger, {
+        const { snapshotTree } = await this.fetchSnapshotFromStorage(
+            ackHandle,
+            summaryLogger,
+            {
                 eventName: "RefreshLatestSummaryGetSnapshot",
                 ackHandle,
                 summaryRefSeq,
                 fetchLatest: false,
-            }),
+            },
+        );
+        const result = await this.summarizerNode.refreshLatestSummary(
+            proposalHandle,
+            summaryRefSeq,
+            async () => snapshotTree,
             readAndParseBlob,
             summaryLogger,
         );
@@ -3062,19 +3084,22 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
      * @param summaryLogger - logger to use when fetching snapshot from storage
      * @returns downloaded snapshot's reference sequence number
      */
-    private async refreshLatestSummaryAckFromServer(summaryLogger: ITelemetryLogger): Promise<number> {
-        const snapshot = await this.fetchSnapshotFromStorage(null, summaryLogger, {
-            eventName: "RefreshLatestSummaryGetSnapshot",
-            fetchLatest: true,
-        });
+    private async refreshLatestSummaryAckFromServer(
+        summaryLogger: ITelemetryLogger,
+    ): Promise<{ latestSnapshotRefSeq: number; latestSnapshotVersionId: string | undefined; }> {
+        const { snapshotTree, versionId } = await this.fetchSnapshotFromStorage(null, summaryLogger, {
+                eventName: "RefreshLatestSummaryGetSnapshot",
+                fetchLatest: true,
+            },
+        );
 
         const readAndParseBlob = async <T>(id: string) => readAndParse<T>(this.storage, id);
-        const snapshotRefSeq = await seqFromTree(snapshot, readAndParseBlob);
+        const latestSnapshotRefSeq = await seqFromTree(snapshotTree, readAndParseBlob);
 
         const result = await this.summarizerNode.refreshLatestSummary(
             undefined,
-            snapshotRefSeq,
-            async () => snapshot,
+            latestSnapshotRefSeq,
+            async () => snapshotTree,
             readAndParseBlob,
             summaryLogger,
         );
@@ -3082,11 +3107,14 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
         // Notify the garbage collector so it can update its latest summary state.
         await this.garbageCollector.latestSummaryStateRefreshed(result, readAndParseBlob);
 
-        return snapshotRefSeq;
+        return { latestSnapshotRefSeq, latestSnapshotVersionId: versionId };
     }
 
     private async fetchSnapshotFromStorage(
-        versionId: string | null, logger: ITelemetryLogger, event: ITelemetryGenericEvent) {
+        versionId: string | null,
+        logger: ITelemetryLogger,
+        event: ITelemetryGenericEvent,
+    ): Promise<{ snapshotTree: ISnapshotTree; versionId: string; }> {
         return PerformanceEvent.timedExecAsync(
             logger, event, async (perfEvent: {
                 end: (arg0: {
@@ -3106,7 +3134,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
             stats.getSnapshotDuration = trace.trace().duration;
 
             perfEvent.end(stats);
-            return maybeSnapshot;
+            return { snapshotTree: maybeSnapshot, versionId: versions[0].id };
         });
     }
 

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "1.2.3-84921";
+export const pkgVersion = "1.2.5";

package/src/runningSummarizer.ts

@@ -78,10 +78,10 @@ export class RunningSummarizer implements IDisposable {
     }
 
     public get disposed() { return this._disposed; }
-
     private stopping = false;
     private _disposed = false;
     private summarizingLock: Promise<void> | undefined;
+    private refreshSummaryAckLock: Promise<void> | undefined;
     private tryWhileSummarizing = false;
     private readonly pendingAckTimer: PromiseTimer;
     private heuristicRunner?: ISummarizeHeuristicRunner;
@@ -276,6 +276,31 @@ export class RunningSummarizer implements IDisposable {
         }
     }
 
+    /**
+     * Blocks a new summarizer from running in case RefreshSummaryAck is being processed.
+     * Assumes that caller checked upfront for lack of concurrent action (this.refreshSummaryAckLock)
+     * before calling this API. I.e. caller is responsible for either erroring out or waiting on this promise.
+     * Note: The refreshSummaryAckLock makes sure no summarizer gets enqueued or processed
+     * until the refresh has completed. One can't rely uniquely on the summarizingLock as the
+     * refreshLatestSummaryAck also happens during the time summarizingLock !== undefined.
+     * Ex. Summarizer submits a summay + op and then waits for the Summary Ack to proceed
+     * with the refreshLatestSummaryAck and complete the summary.
+     * @param action - action to perform.
+     * @returns - result of action.
+     */
+    public async lockedRefreshSummaryAckAction<T>(action: () => Promise<T>) {
+        assert(this.refreshSummaryAckLock === undefined,
+            "Refresh Summary Ack - Caller is responsible for checking lock");
+
+        const refreshSummaryAckLock = new Deferred<void>();
+        this.refreshSummaryAckLock = refreshSummaryAckLock.promise;
+
+        return action().finally(() => {
+            refreshSummaryAckLock.resolve();
+            this.refreshSummaryAckLock = undefined;
+        });
+    }
+
     /**
      * Runs single summary action that prevents any other concurrent actions.
      * Assumes that caller checked upfront for lack of concurrent action (this.summarizingLock)
@@ -290,6 +315,8 @@ export class RunningSummarizer implements IDisposable {
         this.summarizingLock = summarizingLock.promise;
 
         this.summarizeCount++;
+        // Make sure the RefreshLatestSummaryAck is not being executed.
+        await this.refreshSummaryAckLock;
 
         return action().finally(() => {
             summarizingLock.resolve();
@@ -393,6 +420,10 @@ export class RunningSummarizer implements IDisposable {
                     });
                     await delay(delaySeconds * 1000);
                 }
+
+                // Make sure the refresh Summary Ack is not being executed.
+                await this.refreshSummaryAckLock;
+
                 // Note: no need to account for cancellationToken.waitCancelled here, as
                 // this is accounted SummaryGenerator.summarizeCore that controls receivedSummaryAckOrNack.
                 const resultSummarize = this.generator.summarize(summarizeProps, options, cancellationToken);
@@ -441,6 +472,7 @@ export class RunningSummarizer implements IDisposable {
             // The heuristics are blocking concurrent summarize attempts.
             throw new UsageError("Attempted to run an already-running summarizer on demand");
         }
+
         const result = this.trySummarizeOnce(
             { reason: `onDemand/${reason}` },
             options,

package/src/summarizer.ts

@@ -8,9 +8,15 @@ import { Deferred } from "@fluidframework/common-utils";
 import { ITelemetryLogger } from "@fluidframework/common-definitions";
 import { ILoader, LoaderHeader } from "@fluidframework/container-definitions";
 import { UsageError } from "@fluidframework/container-utils";
-import { DriverHeader } from "@fluidframework/driver-definitions";
+import { DriverErrorType, DriverHeader } from "@fluidframework/driver-definitions";
 import { requestFluidObject } from "@fluidframework/runtime-utils";
-import { ChildLogger, IFluidErrorBase, LoggingError, wrapErrorAndLog } from "@fluidframework/telemetry-utils";
+import {
+    ChildLogger,
+    IFluidErrorBase,
+    isFluidError,
+    LoggingError,
+    wrapErrorAndLog,
+} from "@fluidframework/telemetry-utils";
 import {
     FluidObject,
     IFluidHandleContext,
@@ -23,7 +29,7 @@ import {
 import { ISummaryConfiguration } from "./containerRuntime";
 import { ICancellableSummarizerController } from "./runWhileConnectedCoordinator";
 import { summarizerClientType } from "./summarizerClientElection";
-import { SummaryCollection } from "./summaryCollection";
+import { IAckedSummary, SummaryCollection } from "./summaryCollection";
 import { SummarizerHandle } from "./summarizerHandle";
 import { RunningSummarizer } from "./runningSummarizer";
 import {
@@ -378,22 +384,52 @@ export class Summarizer extends EventEmitter implements ISummarizer {
 
     private async handleSummaryAcks() {
         let refSequenceNumber = this.runtime.deltaManager.initialSequenceNumber;
+        let ack: IAckedSummary | undefined;
         while (this.runningSummarizer) {
             const summaryLogger = this.runningSummarizer.tryGetCorrelatedLogger(refSequenceNumber) ?? this.logger;
             try {
-                const ack = await this.summaryCollection.waitSummaryAck(refSequenceNumber);
+                // Initialize ack with undefined if exception happens inside of waitSummaryAck on second iteration,
+                // we record undefined, not previous handles.
+                ack = undefined;
+                ack = await this.summaryCollection.waitSummaryAck(refSequenceNumber);
                 refSequenceNumber = ack.summaryOp.referenceSequenceNumber;
-
-                await this.internalsProvider.refreshLatestSummaryAck(
-                    ack.summaryOp.contents.handle,
-                    ack.summaryAck.contents.handle,
-                    refSequenceNumber,
-                    summaryLogger,
+                const summaryOpHandle = ack.summaryOp.contents.handle;
+                const summaryAckHandle = ack.summaryAck.contents.handle;
+                // Make sure we block any summarizer from being executed/enqueued while
+                // executing the refreshLatestSummaryAck.
+                // https://dev.azure.com/fluidframework/internal/_workitems/edit/779
+                await this.runningSummarizer.lockedRefreshSummaryAckAction(async () =>
+                    this.internalsProvider.refreshLatestSummaryAck(
+                        summaryOpHandle,
+                        summaryAckHandle,
+                        refSequenceNumber,
+                        summaryLogger,
+                    ).catch(async (error) => {
+                        // If the error is 404, so maybe the fetched version no longer exists on server. We just
+                        // ignore this error in that case, as that means we will have another summaryAck for the
+                        // latest version with which we will refresh the state. However in case of single commit
+                        // summary, we might me missing a summary ack, so in that case we are still fine as the
+                        // code in `submitSummary` function in container runtime, will refresh the latest state
+                        // by calling `refreshLatestSummaryAckFromServer` and we will be fine.
+                        if (isFluidError(error)
+                            && error.errorType === DriverErrorType.fileNotFoundOrAccessDeniedError) {
+                            summaryLogger.sendTelemetryEvent({
+                                eventName: "HandleSummaryAckErrorIgnored",
+                                referenceSequenceNumber: refSequenceNumber,
+                                proposalHandle: summaryOpHandle,
+                                ackHandle: summaryAckHandle,
+                            }, error);
+                        } else {
+                            throw error;
+                        }
+                    }),
                 );
             } catch (error) {
                 summaryLogger.sendErrorEvent({
                     eventName: "HandleSummaryAckError",
                     referenceSequenceNumber: refSequenceNumber,
+                    handle: ack?.summaryOp?.contents?.handle,
+                    ackHandle: ack?.summaryAck?.contents?.handle,
                 }, error);
             }
             refSequenceNumber++;