@fluidframework/container-loader 2.0.2 → 2.1.0-276326

package/src/serializedStateManager.ts

@@ -12,8 +12,9 @@ import type {
 	IEventProvider,
 	IEvent,
 	ITelemetryBaseLogger,
+	Tagged,
 } from "@fluidframework/core-interfaces";
-import { assert } from "@fluidframework/core-utils/internal";
+import { Timer, assert } from "@fluidframework/core-utils/internal";
 import {
 	FetchSource,
 	IDocumentStorageService,
@@ -30,6 +31,7 @@ import {
 	PerformanceEvent,
 	UsageError,
 	createChildMonitoringContext,
+	type TelemetryEventPropertyTypeExt,
 } from "@fluidframework/telemetry-utils/internal";
 
 import {
@@ -66,7 +68,9 @@ export interface SnapshotWithBlobs {
  * @internal
  */
 export interface IPendingContainerState extends SnapshotWithBlobs {
-	/** This container was attached (as opposed to IPendingDetachedContainerState.attached which is false) */
+	/**
+	 * This container was attached (as opposed to IPendingDetachedContainerState.attached which is false)
+	 */
 	attached: true;
 	/**
 	 * Runtime-specific state that will be needed to properly rehydrate
@@ -83,9 +87,13 @@ export interface IPendingContainerState extends SnapshotWithBlobs {
 	 * ops at the same sequence number at which they were made.
 	 */
 	savedOps: ISequencedDocumentMessage[];
-	/** The Container's URL in the service, needed to hook up the driver during rehydration */
+	/**
+	 * The Container's URL in the service, needed to hook up the driver during rehydration
+	 */
 	url: string;
-	/** If the Container was connected when serialized, its clientId. Used as the initial clientId upon rehydration, until reconnected. */
+	/**
+	 * If the Container was connected when serialized, its clientId. Used as the initial clientId upon rehydration, until reconnected.
+	 */
 	clientId?: string;
 }
 
@@ -95,11 +103,17 @@ export interface IPendingContainerState extends SnapshotWithBlobs {
  * @internal
  */
 export interface IPendingDetachedContainerState extends SnapshotWithBlobs {
-	/** This container was not attached (as opposed to IPendingContainerState.attached which is true) */
+	/**
+	 * This container was not attached (as opposed to IPendingContainerState.attached which is true)
+	 */
 	attached: false;
-	/** Indicates whether we expect the rehydrated container to have non-empty Detached Blob Storage */
+	/**
+	 * Indicates whether we expect the rehydrated container to have non-empty Detached Blob Storage
+	 */
 	hasAttachmentBlobs: boolean;
-	/** Used by the memory blob storage to persisted attachment blobs */
+	/**
+	 * Used by the memory blob storage to persisted attachment blobs
+	 */
 	attachmentBlobs?: string;
 	/**
 	 * Runtime-specific state that will be needed to properly rehydrate
@@ -136,8 +150,10 @@ export class SerializedStateManager {
 	private readonly mc: MonitoringContext;
 	private snapshot: ISnapshotInfo | undefined;
 	private latestSnapshot: ISnapshotInfo | undefined;
-	private refreshSnapshotP: Promise<void> | undefined;
+	private _refreshSnapshotP: Promise<number> | undefined;
 	private readonly lastSavedOpSequenceNumber: number = 0;
+	private readonly refreshTimer: Timer;
+	private readonly snapshotRefreshTimeoutMs: number = 60 * 60 * 24 * 1000;
 
 	/**
 	 * @param pendingLocalState - The pendingLocalState being rehydrated, if any (undefined when loading directly from storage)
@@ -155,12 +171,17 @@ export class SerializedStateManager {
 		containerEvent: IEventProvider<ISerializerEvent>,
 		private readonly containerDirty: () => boolean,
 		private readonly supportGetSnapshotApi: () => boolean,
+		snapshotRefreshTimeoutMs?: number,
 	) {
 		this.mc = createChildMonitoringContext({
 			logger: subLogger,
 			namespace: "serializedStateManager",
 		});
 
+		this.snapshotRefreshTimeoutMs = snapshotRefreshTimeoutMs ?? this.snapshotRefreshTimeoutMs;
+		this.refreshTimer = new Timer(this.snapshotRefreshTimeoutMs, () =>
+			this.tryRefreshSnapshot(),
+		);
 		// special case handle. Obtaining the last saved op seq num to avoid
 		// refreshing the snapshot before we have processed it. It could cause
 		// a subsequent stashing to have a newer snapshot than allowed.
@@ -169,7 +190,7 @@ export class SerializedStateManager {
 			this.lastSavedOpSequenceNumber =
 				pendingLocalState.savedOps[savedOpsSize - 1].sequenceNumber;
 		}
-		containerEvent.once("saved", () => this.updateSnapshotAndProcessedOpsMaybe());
+		containerEvent.on("saved", () => this.updateSnapshotAndProcessedOpsMaybe());
 	}
 
 	public get offlineLoadEnabled(): boolean {
@@ -178,15 +199,17 @@ export class SerializedStateManager {
 
 	/**
 	 * Promise that will resolve (or reject) once we've tried to download the latest snapshot(s) from storage
+	 * only intended to be used for testing purposes.
+	 * @returns The snapshot sequence number associated with the latest fetched snapshot
 	 */
-	public get waitForInitialRefresh(): Promise<void> | undefined {
-		return this.refreshSnapshotP;
+	public get refreshSnapshotP(): Promise<number> | undefined {
+		return this._refreshSnapshotP;
 	}
 
 	/**
 	 * Called whenever an incoming op is processed by the Container
 	 */
-	public addProcessedOp(message: ISequencedDocumentMessage) {
+	public addProcessedOp(message: ISequencedDocumentMessage): void {
 		if (this.offlineLoadEnabled) {
 			this.processedOps.push(message);
 			this.updateSnapshotAndProcessedOpsMaybe();
@@ -197,13 +220,16 @@ export class SerializedStateManager {
 	 * This wraps the basic functionality of fetching the snapshot for this container during Container load.
 	 *
 	 * If we have pendingLocalState, we get the snapshot from there.
-	 * Otherwise, fetch it from storage (according to specifiedVersion if provided)
+	 * Otherwise, fetch it from storage (according to specifiedVersion if provided).
 	 *
-	 * @param specifiedVersion - If a version is specified and we don't have pendingLocalState, fetch this version from storage
+	 * @param specifiedVersion - If a version is specified and we don't have pendingLocalState, fetch this version from storage.
 	 * @param supportGetSnapshotApi - a boolean indicating whether to use the fetchISnapshot or fetchISnapshotTree.
 	 * @returns The snapshot to boot the container from
 	 */
-	public async fetchSnapshot(specifiedVersion: string | undefined) {
+	public async fetchSnapshot(specifiedVersion: string | undefined): Promise<{
+		baseSnapshot: ISnapshot | ISnapshotTree;
+		version: IVersion | undefined;
+	}> {
 		if (this.pendingLocalState === undefined) {
 			const { baseSnapshot, version } = await getSnapshot(
 				this.mc,
@@ -224,6 +250,7 @@ export class SerializedStateManager {
 					snapshotBlobs,
 					snapshotSequenceNumber: attributes.sequenceNumber,
 				};
+				this.refreshTimer.start();
 			}
 			return { baseSnapshot, version };
 		} else {
@@ -234,21 +261,7 @@ export class SerializedStateManager {
 				snapshotBlobs,
 				snapshotSequenceNumber: attributes.sequenceNumber,
 			};
-
-			if (
-				this.refreshSnapshotP === undefined &&
-				this.mc.config.getBoolean("Fluid.Container.enableOfflineSnapshotRefresh") === true
-			) {
-				// Don't block on the refresh snapshot call - it is for the next time we serialize, not booting this incarnation
-				this.refreshSnapshotP = this.refreshLatestSnapshot(this.supportGetSnapshotApi());
-				this.refreshSnapshotP.catch((e) => {
-					this.mc.logger.sendErrorEvent({
-						eventName: "RefreshLatestSnapshotFailed",
-						error: e,
-					});
-				});
-			}
-
+			this.tryRefreshSnapshot();
 			const blobContents = new Map<string, ArrayBuffer>();
 			for (const [id, value] of Object.entries(snapshotBlobs)) {
 				blobContents.set(id, stringToBuffer(value, "utf8"));
@@ -265,13 +278,36 @@ export class SerializedStateManager {
 		}
 	}
 
+	private tryRefreshSnapshot(): void {
+		if (
+			this.mc.config.getBoolean("Fluid.Container.enableOfflineSnapshotRefresh") === true &&
+			this._refreshSnapshotP === undefined &&
+			this.latestSnapshot === undefined
+		) {
+			// Don't block on the refresh snapshot call - it is for the next time we serialize, not booting this incarnation
+			this._refreshSnapshotP = this.refreshLatestSnapshot(this.supportGetSnapshotApi());
+			this._refreshSnapshotP
+				.catch(
+					(error: TelemetryEventPropertyTypeExt | Tagged<TelemetryEventPropertyTypeExt>) => {
+						this.mc.logger.sendTelemetryEvent({
+							eventName: "RefreshLatestSnapshotFailed",
+							error,
+						});
+					},
+				)
+				.finally(() => {
+					this._refreshSnapshotP = undefined;
+				});
+		}
+	}
+
 	/**
 	 * Fetch the latest snapshot for the container, including delay-loaded groupIds if pendingLocalState was provided and contained any groupIds.
 	 * Note that this will update the StorageAdapter's cached snapshots for the groupIds (if present)
 	 *
 	 * @param supportGetSnapshotApi - a boolean indicating whether to use the fetchISnapshot or fetchISnapshotTree (must be true to fetch by groupIds)
 	 */
-	private async refreshLatestSnapshot(supportGetSnapshotApi: boolean): Promise<void> {
+	private async refreshLatestSnapshot(supportGetSnapshotApi: boolean): Promise<number> {
 		this.latestSnapshot = await getLatestSnapshotInfo(
 			this.mc,
 			this.storageAdapter,
@@ -297,15 +333,16 @@ export class SerializedStateManager {
 			assert(snapshot !== undefined, 0x973 /* Snapshot should exist */);
 		}
 
-		this.updateSnapshotAndProcessedOpsMaybe();
+		return this.updateSnapshotAndProcessedOpsMaybe();
 	}
 
 	/**
 	 * Updates class snapshot and processedOps if we have a new snapshot and it's among processedOps range.
 	 */
-	private updateSnapshotAndProcessedOpsMaybe() {
+	private updateSnapshotAndProcessedOpsMaybe(): number {
+		const snapshotSequenceNumber = this.latestSnapshot?.snapshotSequenceNumber;
 		if (
-			this.latestSnapshot === undefined ||
+			snapshotSequenceNumber === undefined ||
 			this.processedOps.length === 0 ||
 			this.processedOps[this.processedOps.length - 1].sequenceNumber <
 				this.lastSavedOpSequenceNumber ||
@@ -313,9 +350,8 @@ export class SerializedStateManager {
 		) {
 			// can't refresh latest snapshot until we have processed the ops up to it.
 			// Pending state would be behind the latest snapshot.
-			return;
+			return -1;
 		}
-		const snapshotSequenceNumber = this.latestSnapshot.snapshotSequenceNumber;
 		const firstProcessedOpSequenceNumber = this.processedOps[0].sequenceNumber;
 		const lastProcessedOpSequenceNumber =
 			this.processedOps[this.processedOps.length - 1].sequenceNumber;
@@ -331,12 +367,14 @@ export class SerializedStateManager {
 				stashedSnapshotSequenceNumber: this.snapshot?.snapshotSequenceNumber,
 			});
 			this.latestSnapshot = undefined;
+			this.refreshTimer.restart();
 		} else if (snapshotSequenceNumber <= lastProcessedOpSequenceNumber) {
 			// Snapshot seq num is between the first and last processed op.
 			// Remove the ops that are already part of the snapshot
 			this.processedOps.splice(0, snapshotSequenceNumber - firstProcessedOpSequenceNumber + 1);
 			this.snapshot = this.latestSnapshot;
 			this.latestSnapshot = undefined;
+			this.refreshTimer.restart();
 			this.mc.logger.sendTelemetryEvent({
 				eventName: "SnapshotRefreshed",
 				snapshotSequenceNumber,
@@ -345,6 +383,7 @@ export class SerializedStateManager {
 					this.processedOps.length === 0 ? undefined : this.processedOps[0].sequenceNumber,
 			});
 		}
+		return snapshotSequenceNumber;
 	}
 
 	/**
@@ -353,7 +392,7 @@ export class SerializedStateManager {
 	 * base snapshot when attaching.
 	 * @param snapshot - snapshot and blobs collected while attaching (a form of the attach summary)
 	 */
-	public setInitialSnapshot(snapshot: SnapshotWithBlobs | undefined) {
+	public setInitialSnapshot(snapshot: SnapshotWithBlobs | undefined): void {
 		if (this.offlineLoadEnabled) {
 			assert(
 				this.snapshot === undefined,
@@ -365,12 +404,19 @@ export class SerializedStateManager {
 				".protocol" in baseSnapshot.trees
 					? baseSnapshot.trees[".protocol"].blobs.attributes
 					: baseSnapshot.blobs[".attributes"];
+			// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
 			const attributes = JSON.parse(snapshotBlobs[attributesHash]);
 			assert(
+				// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
 				attributes.sequenceNumber === 0,
 				0x939 /* trying to set a non attachment snapshot */,
 			);
-			this.snapshot = { ...snapshot, snapshotSequenceNumber: attributes.sequenceNumber };
+			this.snapshot = {
+				...snapshot,
+				// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+				snapshotSequenceNumber: attributes.sequenceNumber as number,
+			};
+			this.refreshTimer.start();
 		}
 	}
 
@@ -450,11 +496,19 @@ export async function getLatestSnapshotInfo(
 		mc.logger,
 		{ eventName: "GetLatestSnapshotInfo" },
 		async () => {
+			// get the latest non cached snapshot version
+			const specifiedVersion: IVersion[] = await storageAdapter.getVersions(
+				// eslint-disable-next-line unicorn/no-null
+				null,
+				1,
+				"getLatestSnapshotInfo",
+				FetchSource.noCache,
+			);
 			const { baseSnapshot } = await getSnapshot(
 				mc,
 				storageAdapter,
 				supportGetSnapshotApi,
-				undefined,
+				specifiedVersion[0]?.id,
 			);
 
 			const baseSnapshotTree: ISnapshotTree | undefined = getSnapshotTree(baseSnapshot);
@@ -549,6 +603,8 @@ export async function fetchISnapshotTree(
 	storageAdapter: Pick<IDocumentStorageService, "getSnapshotTree" | "getVersions">,
 	specifiedVersion: string | undefined,
 ): Promise<{ snapshot?: ISnapshotTree; version?: IVersion | undefined }> {
+	// API uses null
+	// eslint-disable-next-line unicorn/no-null
 	const versions = await storageAdapter.getVersions(specifiedVersion ?? null, 1);
 	const version = versions[0];
 

package/src/utils.ts

@@ -25,7 +25,11 @@ import {
 	isCombinedAppAndProtocolSummary,
 	readAndParse,
 } from "@fluidframework/driver-utils/internal";
-import { LoggingError, UsageError } from "@fluidframework/telemetry-utils/internal";
+import {
+	LoggingError,
+	UsageError,
+	type IFluidErrorBase,
+} from "@fluidframework/telemetry-utils/internal";
 import { v4 as uuid } from "uuid";
 
 import { ISerializableBlobContents } from "./containerStorageAdapter.js";
@@ -47,6 +51,7 @@ export interface ISnapshotTreeWithBlobContents extends ISnapshotTree {
  * Interface to represent the parsed parts of IResolvedUrl.url to help
  * in getting info about different parts of the url.
  * May not be compatible or relevant for any Url Resolver
+ * @legacy
  * @alpha
  */
 export interface IParsedUrl {
@@ -76,6 +81,7 @@ export interface IParsedUrl {
  * with urls of type: protocol://<string>/.../..?<querystring>
  * @param url - This is the IResolvedUrl.url part of the resolved url.
  * @returns The IParsedUrl representing the input URL, or undefined if the format was not supported
+ * @legacy
  * @alpha
  */
 export function tryParseCompatibleResolvedUrl(url: string): IParsedUrl | undefined {
@@ -150,9 +156,10 @@ function convertSummaryToSnapshotAndBlobs(summary: ISummaryTree): SnapshotWithBl
 				blobContents = { ...blobContents, ...innerSnapshot.snapshotBlobs };
 				break;
 			}
-			case SummaryType.Attachment:
+			case SummaryType.Attachment: {
 				treeNode.blobs[key] = summaryObject.id;
 				break;
+			}
 			case SummaryType.Blob: {
 				const blobId = uuid();
 				treeNode.blobs[key] = blobId;
@@ -163,11 +170,13 @@ function convertSummaryToSnapshotAndBlobs(summary: ISummaryTree): SnapshotWithBl
 				blobContents[blobId] = contentString;
 				break;
 			}
-			case SummaryType.Handle:
+			case SummaryType.Handle: {
 				throw new LoggingError(
 					"No handles should be there in summary in detached container!!",
 				);
+			}
 			default: {
+				// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any
 				unreachableCase(summaryObject, `Unknown tree type ${(summaryObject as any).type}`);
 			}
 		}
@@ -293,12 +302,13 @@ export const combineSnapshotTreeAndSnapshotBlobs = (
 };
 
 export function isDeltaStreamConnectionForbiddenError(
-	error: any,
+	error: unknown,
 ): error is DeltaStreamConnectionForbiddenError {
 	return (
 		typeof error === "object" &&
 		error !== null &&
-		error?.errorType === DriverErrorTypes.deltaStreamConnectionForbidden
+		(error as Partial<IFluidErrorBase>)?.errorType ===
+			DriverErrorTypes.deltaStreamConnectionForbidden
 	);
 }
 
@@ -329,9 +339,12 @@ export function getDetachedContainerStateFromSerializedContainer(
 	serializedContainer: string,
 ): IPendingDetachedContainerState {
 	const hasBlobsSummaryTree = ".hasAttachmentBlobs";
+	// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
 	const parsedContainerState = JSON.parse(serializedContainer);
+	// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
 	if (isPendingDetachedContainerState(parsedContainerState)) {
 		return parsedContainerState;
+		// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
 	} else if (isCombinedAppAndProtocolSummary(parsedContainerState)) {
 		const { baseSnapshot, snapshotBlobs } =
 			getSnapshotTreeAndBlobsFromSerializedContainer(parsedContainerState);
@@ -354,16 +367,19 @@ export function getDetachedContainerStateFromSerializedContainer(
 export function getAttachedContainerStateFromSerializedContainer(
 	serializedContainer: string | undefined,
 ): IPendingContainerState | undefined {
-	return serializedContainer !== undefined
-		? (JSON.parse(serializedContainer) as IPendingContainerState)
-		: undefined;
+	return serializedContainer === undefined
+		? undefined
+		: (JSON.parse(serializedContainer) as IPendingContainerState);
 }
 
 /**
  * Ensures only a single instance of the provided async function is running.
  * If there are multiple calls they will all get the same promise to wait on.
  */
-export const runSingle = <A extends any[], R>(func: (...args: A) => Promise<R>) => {
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export const runSingle = <A extends any[], R>(
+	func: (...args: A) => Promise<R>,
+): ((...args: A) => Promise<R>) => {
 	let running:
 		| {
 				args: A;
@@ -373,7 +389,7 @@ export const runSingle = <A extends any[], R>(func: (...args: A) => Promise<R>)
 	// don't mark this function async, so we return the same promise,
 	// rather than one that is wrapped due to async
 	// eslint-disable-next-line @typescript-eslint/promise-function-async
-	return (...args: A) => {
+	return (...args: A): Promise<R> => {
 		if (running !== undefined) {
 			if (!compareArrays(running.args, args)) {
 				return Promise.reject(