@fluidframework/container-loader 2.0.0-dev-rc.3.0.0.250606 → 2.0.0-dev-rc.3.0.0.254274

package/src/protocol.ts

@@ -3,8 +3,8 @@
  * Licensed under the MIT License.
  */
 
-import { IAudienceOwner } from "@fluidframework/container-definitions";
-import { canBeCoalescedByService } from "@fluidframework/driver-utils";
+import { IAudienceOwner } from "@fluidframework/container-definitions/internal";
+import { canBeCoalescedByService } from "@fluidframework/driver-utils/internal";
 import {
 	IProtocolHandler as IBaseProtocolHandler,
 	IQuorumSnapshot,

package/src/protocolTreeDocumentStorageService.ts

@@ -4,7 +4,10 @@
  */
 
 import { IDisposable } from "@fluidframework/core-interfaces";
-import { IDocumentStorageService, ISummaryContext } from "@fluidframework/driver-definitions";
+import {
+	IDocumentStorageService,
+	ISummaryContext,
+} from "@fluidframework/driver-definitions/internal";
 import { ISummaryTree } from "@fluidframework/protocol-definitions";
 
 /**

package/src/quorum.ts

@@ -3,7 +3,7 @@
  * Licensed under the MIT License.
  */
 
-import { IFluidCodeDetails } from "@fluidframework/container-definitions";
+import { IFluidCodeDetails } from "@fluidframework/container-definitions/internal";
 import { ICommittedProposal } from "@fluidframework/protocol-definitions";
 
 export function initQuorumValuesFromCodeDetails(

package/src/retriableDocumentStorageService.ts

@@ -4,7 +4,7 @@
  */
 
 import { IDisposable } from "@fluidframework/core-interfaces";
-import { assert } from "@fluidframework/core-utils";
+import { assert } from "@fluidframework/core-utils/internal";
 import {
 	FetchSource,
 	IDocumentStorageService,
@@ -12,8 +12,8 @@ import {
 	ISnapshot,
 	ISnapshotFetchOptions,
 	ISummaryContext,
-} from "@fluidframework/driver-definitions";
-import { runWithRetry } from "@fluidframework/driver-utils";
+} from "@fluidframework/driver-definitions/internal";
+import { runWithRetry } from "@fluidframework/driver-utils/internal";
 import {
 	ICreateBlobResponse,
 	ISnapshotTree,
@@ -21,7 +21,8 @@ import {
 	ISummaryTree,
 	IVersion,
 } from "@fluidframework/protocol-definitions";
-import { GenericError, ITelemetryLoggerExt, UsageError } from "@fluidframework/telemetry-utils";
+import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
+import { GenericError, UsageError } from "@fluidframework/telemetry-utils/internal";
 
 export class RetriableDocumentStorageService implements IDocumentStorageService, IDisposable {
 	private _disposed = false;

package/src/serializedStateManager.ts

@@ -3,27 +3,33 @@
  * Licensed under the MIT License.
  */
 
-import { IGetPendingLocalStateProps, IRuntime } from "@fluidframework/container-definitions";
-import { assert } from "@fluidframework/core-utils";
+import {
+	IGetPendingLocalStateProps,
+	IRuntime,
+} from "@fluidframework/container-definitions/internal";
+import { assert } from "@fluidframework/core-utils/internal";
 import {
 	IDocumentStorageService,
 	IResolvedUrl,
 	ISnapshot,
-} from "@fluidframework/driver-definitions";
-import { isInstanceOfISnapshot } from "@fluidframework/driver-utils";
+} from "@fluidframework/driver-definitions/internal";
+import { isInstanceOfISnapshot } from "@fluidframework/driver-utils/internal";
 import {
+	type IDocumentAttributes,
 	ISequencedDocumentMessage,
 	ISnapshotTree,
 	IVersion,
 } from "@fluidframework/protocol-definitions";
+import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
 import {
-	ITelemetryLoggerExt,
 	MonitoringContext,
 	PerformanceEvent,
 	UsageError,
 	createChildMonitoringContext,
-} from "@fluidframework/telemetry-utils";
+} from "@fluidframework/telemetry-utils/internal";
+
 import { ISerializableBlobContents, getBlobContentsFromTree } from "./containerStorageAdapter.js";
+import { getDocumentAttributes } from "./utils.js";
 
 export interface SnapshotWithBlobs {
 	/**
@@ -65,10 +71,16 @@ export interface IPendingDetachedContainerState extends SnapshotWithBlobs {
 	pendingRuntimeState?: unknown;
 }
 
+export interface ISnapshotInfo extends SnapshotWithBlobs {
+	snapshotSequenceNumber: number;
+}
+
 export class SerializedStateManager {
 	private readonly processedOps: ISequencedDocumentMessage[] = [];
-	private snapshot: SnapshotWithBlobs | undefined;
+	private snapshot: ISnapshotInfo | undefined;
 	private readonly mc: MonitoringContext;
+	private latestSnapshot: ISnapshotInfo | undefined;
+	private refreshSnapshot: Promise<void> | undefined;
 
 	constructor(
 		private readonly pendingLocalState: IPendingContainerState | undefined,
@@ -78,6 +90,7 @@ export class SerializedStateManager {
 			"readBlob" | "getSnapshotTree" | "getSnapshot" | "getVersions"
 		>,
 		private readonly _offlineLoadEnabled: boolean,
+		private readonly newSnapshotFetched?: () => void,
 	) {
 		this.mc = createChildMonitoringContext({
 			logger: subLogger,
@@ -92,6 +105,7 @@ export class SerializedStateManager {
 	public addProcessedOp(message: ISequencedDocumentMessage) {
 		if (this.offlineLoadEnabled) {
 			this.processedOps.push(message);
+			this.updateSnapshotAndProcessedOpsMaybe();
 		}
 	}
 
@@ -100,36 +114,112 @@ export class SerializedStateManager {
 		supportGetSnapshotApi: boolean,
 	) {
 		if (this.pendingLocalState === undefined) {
-			const { snapshot, version } = supportGetSnapshotApi
-				? await fetchISnapshot(this.mc, this.storageAdapter, specifiedVersion)
-				: await fetchISnapshotTree(this.mc, this.storageAdapter, specifiedVersion);
-			const baseSnapshot: ISnapshotTree | undefined = isInstanceOfISnapshot(snapshot)
-				? snapshot.snapshotTree
-				: snapshot;
-			assert(baseSnapshot !== undefined, 0x8e4 /* Snapshot should exist */);
+			const { baseSnapshot, version } = await getSnapshotTree(
+				this.mc,
+				this.storageAdapter,
+				supportGetSnapshotApi,
+				specifiedVersion,
+			);
 			// non-interactive clients will not have any pending state we want to save
 			if (this.offlineLoadEnabled) {
 				const snapshotBlobs = await getBlobContentsFromTree(
 					baseSnapshot,
 					this.storageAdapter,
 				);
-				this.snapshot = { baseSnapshot, snapshotBlobs };
+				const attributes = await getDocumentAttributes(this.storageAdapter, baseSnapshot);
+				this.snapshot = {
+					baseSnapshot,
+					snapshotBlobs,
+					snapshotSequenceNumber: attributes.sequenceNumber,
+				};
 			}
 			return { baseSnapshot, version };
 		} else {
 			const { baseSnapshot, snapshotBlobs } = this.pendingLocalState;
-			this.snapshot = { baseSnapshot, snapshotBlobs };
+			const attributes = await getDocumentAttributes(this.storageAdapter, baseSnapshot);
+			this.snapshot = {
+				baseSnapshot,
+				snapshotBlobs,
+				snapshotSequenceNumber: attributes.sequenceNumber,
+			};
+			this.refreshSnapshot ??= (async () => {
+				this.latestSnapshot = await getLatestSnapshotInfo(
+					this.mc,
+					this.storageAdapter,
+					supportGetSnapshotApi,
+				);
+				this.newSnapshotFetched?.();
+				this.updateSnapshotAndProcessedOpsMaybe();
+			})();
+
 			return { baseSnapshot, version: undefined };
 		}
 	}
 
+	/**
+	 * Updates class snapshot and processedOps if we have a new snapshot and it's among processedOps range.
+	 */
+	private updateSnapshotAndProcessedOpsMaybe() {
+		if (this.latestSnapshot === undefined || this.processedOps.length === 0) {
+			// can't refresh latest snapshot until we have processed the ops up to it.
+			// Pending state would be behind the latest snapshot.
+			return;
+		}
+		const snapshotSequenceNumber = this.latestSnapshot.snapshotSequenceNumber;
+		const firstProcessedOpSequenceNumber = this.processedOps[0].sequenceNumber;
+		const lastProcessedOpSequenceNumber =
+			this.processedOps[this.processedOps.length - 1].sequenceNumber;
+
+		if (snapshotSequenceNumber < firstProcessedOpSequenceNumber) {
+			// Snapshot seq number is older than our first processed op, which could mean we're fetching
+			// the same snapshot that we already have or snapshot is too old, implicating an unexpected behavior.
+			this.mc.logger.sendTelemetryEvent({
+				eventName: "OldSnapshotFetchWhileRefreshing",
+				snapshotSequenceNumber,
+				firstProcessedOpSequenceNumber,
+				lastProcessedOpSequenceNumber,
+				stashedSnapshotSequenceNumber: this.snapshot?.snapshotSequenceNumber,
+			});
+			this.latestSnapshot = undefined;
+		} else if (snapshotSequenceNumber <= lastProcessedOpSequenceNumber) {
+			// Snapshot seq num is between the first and last processed op.
+			// Remove the ops that are already part of the snapshot
+			this.processedOps.splice(
+				0,
+				snapshotSequenceNumber - firstProcessedOpSequenceNumber + 1,
+			);
+			this.snapshot = this.latestSnapshot;
+			this.latestSnapshot = undefined;
+			this.mc.logger.sendTelemetryEvent({
+				eventName: "SnapshotRefreshed",
+				snapshotSequenceNumber,
+				firstProcessedOpSequenceNumber,
+				newFirstProcessedOpSequenceNumber:
+					this.processedOps.length === 0
+						? undefined
+						: this.processedOps[0].sequenceNumber,
+			});
+		}
+	}
+
 	/**
 	 * This method is only meant to be used by Container.attach() to set the initial
 	 * base snapshot when attaching.
 	 * @param snapshot - snapshot and blobs collected while attaching
 	 */
-	public setSnapshot(snapshot: SnapshotWithBlobs | undefined) {
-		this.snapshot = snapshot;
+	public setInitialSnapshot(snapshot: SnapshotWithBlobs | undefined) {
+		if (this.offlineLoadEnabled) {
+			assert(this.snapshot === undefined, "inital snapshot should only be defined once");
+			assert(snapshot !== undefined, "attachment snapshot should be defined");
+			const { baseSnapshot, snapshotBlobs } = snapshot;
+			const attributesHash =
+				".protocol" in baseSnapshot.trees
+					? baseSnapshot.trees[".protocol"].blobs.attributes
+					: baseSnapshot.blobs[".attributes"];
+			const attributes = JSON.parse(snapshotBlobs[attributesHash]);
+			assert(attributes.sequenceNumber === 0, "trying to set a non attachment snapshot");
+			this.snapshot = { ...snapshot, snapshotSequenceNumber: attributes.sequenceNumber };
+		}
 	}
 
 	public async getPendingLocalStateCore(
@@ -171,11 +261,84 @@ export class SerializedStateManager {
 	}
 }
 
+/**
+ * Retrieves the most recent snapshot and returns its info.
+ *
+ * @param mc - The monitoring context.
+ * @param storageAdapter - The storage adapter providing methods to retrieve the snapshot.
+ * @param supportGetSnapshotApi - a boolean indicating whether to use the fetchISnapshot or fetchISnapshotTree.
+ * @returns a SnapshotInfo object containing the snapshot tree, snapshot blobs and its sequence number.
+ */
+export async function getLatestSnapshotInfo(
+	mc: MonitoringContext,
+	storageAdapter: Pick<
+		IDocumentStorageService,
+		"getSnapshot" | "getSnapshotTree" | "getVersions" | "readBlob"
+	>,
+	supportGetSnapshotApi: boolean,
+): Promise<ISnapshotInfo | undefined> {
+	return PerformanceEvent.timedExecAsync(
+		mc.logger,
+		{ eventName: "GetLatestSnapshotInfo" },
+		async () => {
+			const { baseSnapshot } = await getSnapshotTree(
+				mc,
+				storageAdapter,
+				supportGetSnapshotApi,
+				undefined,
+			);
+			const snapshotBlobs = await getBlobContentsFromTree(baseSnapshot, storageAdapter);
+			const attributes: IDocumentAttributes = await getDocumentAttributes(
+				storageAdapter,
+				baseSnapshot,
+			);
+			const snapshotSequenceNumber = attributes.sequenceNumber;
+			return { baseSnapshot, snapshotBlobs, snapshotSequenceNumber };
+		},
+	).catch(() => undefined);
+}
+
+/**
+ * Retrieves a snapshot from the storage adapter and transforms it into an ISnapshotTree object.
+ *
+ * @param mc - The monitoring context.
+ * @param storageAdapter - The storage adapter providing methods to retrieve the snapshot.
+ * @param supportGetSnapshotApi - a boolean indicating whether to use the fetchISnapshot or fetchISnapshotTree.
+ * @param specifiedVersion - An optional version string specifying the version of the snapshot tree to fetch.
+ * @returns - An ISnapshotTree and its version.
+ */
+async function getSnapshotTree(
+	mc: MonitoringContext,
+	storageAdapter: Pick<
+		IDocumentStorageService,
+		"getSnapshot" | "getSnapshotTree" | "getVersions"
+	>,
+	supportGetSnapshotApi: boolean,
+	specifiedVersion: string | undefined,
+): Promise<{ baseSnapshot: ISnapshotTree; version?: IVersion }> {
+	const { snapshot, version } = supportGetSnapshotApi
+		? await fetchISnapshot(mc, storageAdapter, specifiedVersion)
+		: await fetchISnapshotTree(mc, storageAdapter, specifiedVersion);
+	const baseSnapshot: ISnapshotTree | undefined = isInstanceOfISnapshot(snapshot)
+		? snapshot.snapshotTree
+		: snapshot;
+	assert(baseSnapshot !== undefined, 0x8e4 /* Snapshot should exist */);
+	return { baseSnapshot, version };
+}
+
+/**
+ * Fetches an ISnapshot from a storage adapter based on the specified version.
+ *
+ * @param mc - The monitoring context.
+ * @param storageAdapter - The storage adapter providing a getSnapshot method to retrieve the ISnapshot and version.
+ * @param specifiedVersion - An optional version string specifying the version of the snapshot tree to fetch.
+ * @returns - The fetched snapshot tree and its version.
+ */
 export async function fetchISnapshot(
 	mc: MonitoringContext,
 	storageAdapter: Pick<IDocumentStorageService, "getSnapshot">,
 	specifiedVersion: string | undefined,
-): Promise<{ snapshot?: ISnapshot | ISnapshotTree; version?: IVersion }> {
+): Promise<{ snapshot?: ISnapshot; version?: IVersion }> {
 	const snapshot = await storageAdapter.getSnapshot?.({ versionId: specifiedVersion });
 	const version: IVersion | undefined =
 		snapshot?.snapshotTree.id === undefined
@@ -190,7 +353,6 @@ export async function fetchISnapshot(
 			eventName: "getSnapshotTreeFailed",
 			id: specifiedVersion,
 		});
-		// Not sure if this should be here actually
 	} else if (snapshot !== undefined && version?.id === undefined) {
 		mc.logger.sendErrorEvent({
 			eventName: "getSnapshotFetchedTreeWithoutVersionId",
@@ -201,9 +363,12 @@ export async function fetchISnapshot(
 }
 
 /**
- * Get the most recent snapshot, or a specific version.
- * @param specifiedVersion - The specific version of the snapshot to retrieve
- * @returns The snapshot requested, or the latest snapshot if no version was specified, plus version ID
+ * Fetches an ISnapshotTree from a storage adapter based on the specified version.
+ *
+ * @param mc - The monitoring context.
+ * @param storageAdapter - The storage adapter providing methods to retrieve the ISnapshotTree and version.
+ * @param specifiedVersion - An optional version string specifying the version of the snapshot tree to fetch.
+ * @returns - The fetched snapshot tree and its version.
  */
 export async function fetchISnapshotTree(
 	mc: MonitoringContext,

package/src/utils.ts

@@ -3,26 +3,32 @@
  * Licensed under the MIT License.
  */
 
-import { Uint8ArrayToString, stringToBuffer } from "@fluid-internal/client-utils";
-import { assert, compareArrays, unreachableCase } from "@fluidframework/core-utils";
-import { DriverErrorTypes, IDocumentStorageService } from "@fluidframework/driver-definitions";
+import { Uint8ArrayToString, bufferToString, stringToBuffer } from "@fluid-internal/client-utils";
+import { assert, compareArrays, unreachableCase } from "@fluidframework/core-utils/internal";
+import { DriverErrorTypes } from "@fluidframework/driver-definitions";
+import {
+	IDocumentStorageService,
+	type ISnapshot,
+} from "@fluidframework/driver-definitions/internal";
 import {
 	CombinedAppAndProtocolSummary,
 	DeltaStreamConnectionForbiddenError,
 	isCombinedAppAndProtocolSummary,
 	readAndParse,
-} from "@fluidframework/driver-utils";
+} from "@fluidframework/driver-utils/internal";
 import {
 	IDocumentAttributes,
 	ISnapshotTree,
 	ISummaryTree,
 	SummaryType,
 } from "@fluidframework/protocol-definitions";
-import { LoggingError, UsageError } from "@fluidframework/telemetry-utils";
+import { LoggingError, UsageError } from "@fluidframework/telemetry-utils/internal";
 import { v4 as uuid } from "uuid";
+
 import { ISerializableBlobContents } from "./containerStorageAdapter.js";
 import type {
 	IPendingDetachedContainerState,
+	ISnapshotInfo,
 	SnapshotWithBlobs,
 } from "./serializedStateManager.js";
 
@@ -167,6 +173,51 @@ function convertSummaryToSnapshotAndBlobs(summary: ISummaryTree): SnapshotWithBl
 	return pendingSnapshot;
 }
 
+/**
+ * Converts a snapshot to snapshotInfo with its blob contents
+ * to align detached container format with IPendingContainerState
+ *
+ * Note, this assumes the ISnapshot sequence number is defined. Otherwise an assert will be thrown
+ * @param snapshot - ISnapshot
+ */
+export function convertSnapshotToSnapshotInfo(snapshot: ISnapshot): ISnapshotInfo {
+	assert(snapshot.sequenceNumber !== undefined, "Snapshot sequence number is missing");
+	const snapshotBlobs: ISerializableBlobContents = {};
+	for (const [blobId, arrayBufferLike] of snapshot.blobContents.entries()) {
+		snapshotBlobs[blobId] = bufferToString(arrayBufferLike, "utf8");
+	}
+	return {
+		baseSnapshot: snapshot.snapshotTree,
+		snapshotBlobs,
+		snapshotSequenceNumber: snapshot.sequenceNumber,
+	};
+}
+
+/**
+ * Converts a snapshot to snapshotInfo with its blob contents
+ * to align detached container format with IPendingContainerState
+ *
+ * Note, this assumes the ISnapshot sequence number is defined. Otherwise an assert will be thrown
+ * @param snapshot - ISnapshot
+ */
+export function convertSnapshotInfoToSnapshot(
+	snapshotInfo: ISnapshotInfo,
+	snapshotSequenceNumber: number,
+): ISnapshot {
+	const blobContents = new Map<string, ArrayBufferLike>();
+	for (const [blobId, serializedContent] of Object.entries(snapshotInfo.snapshotBlobs)) {
+		blobContents.set(blobId, stringToBuffer(serializedContent, "utf8"));
+	}
+	return {
+		snapshotTree: snapshotInfo.baseSnapshot,
+		blobContents,
+		ops: [],
+		sequenceNumber: snapshotSequenceNumber,
+		latestSequenceNumber: undefined,
+		snapshotFormatV: 1,
+	};
+}
+
 /**
  * Converts summary parts into a SnapshotTree and its blob contents.
  * @param protocolSummaryTree - Protocol Summary Tree