@fluidframework/container-loader 2.0.0-rc.1.0.6 → 2.0.0-rc.2.0.1

package/src/container.ts

@@ -4,15 +4,14 @@
  */
 
 import { v4 as uuid } from "uuid";
-import { assert, unreachableCase } from "@fluidframework/core-utils";
+import { assert, unreachableCase, isPromiseLike } from "@fluidframework/core-utils";
 import { TypedEventEmitter, performance } from "@fluid-internal/client-utils";
 import {
 	IEvent,
-	ITelemetryProperties,
-	TelemetryEventCategory,
-	IRequest,
+	ITelemetryBaseProperties,
 	FluidObject,
 	LogLevel,
+	IRequest,
 } from "@fluidframework/core-interfaces";
 import {
 	AttachState,
@@ -41,6 +40,7 @@ import {
 	IDocumentServiceFactory,
 	IDocumentStorageService,
 	IResolvedUrl,
+	ISnapshot,
 	IThrottlingWarning,
 	IUrlResolver,
 } from "@fluidframework/driver-definitions";
@@ -48,9 +48,10 @@ import {
 	readAndParse,
 	OnlineStatus,
 	isOnline,
-	runWithRetry,
 	isCombinedAppAndProtocolSummary,
 	MessageType2,
+	isInstanceOfISnapshot,
+	runWithRetry,
 } from "@fluidframework/driver-utils";
 import { IQuorumSnapshot } from "@fluidframework/protocol-base";
 import {
@@ -86,42 +87,45 @@ import {
 	formatTick,
 	GenericError,
 	UsageError,
+	IFluidErrorBase,
+	type TelemetryEventCategory,
 } from "@fluidframework/telemetry-utils";
 import structuredClone from "@ungap/structured-clone";
-import { Audience } from "./audience";
-import { ContainerContext } from "./containerContext";
+import { Audience } from "./audience.js";
+import { ContainerContext } from "./containerContext.js";
 import {
 	ReconnectMode,
 	IConnectionManagerFactoryArgs,
 	getPackageName,
 	IConnectionDetailsInternal,
 	IConnectionStateChangeReason,
-} from "./contracts";
-import { DeltaManager, IConnectionArgs } from "./deltaManager";
-import { IDetachedBlobStorage, ILoaderOptions, RelativeLoader } from "./loader";
-import { pkgVersion } from "./packageVersion";
-import {
-	ContainerStorageAdapter,
-	getBlobContentsFromTree,
-	getBlobContentsFromTreeWithBlobContents,
-	ISerializableBlobContents,
-} from "./containerStorageAdapter";
-import { IConnectionStateHandler, createConnectionStateHandler } from "./connectionStateHandler";
+} from "./contracts.js";
+import { DeltaManager, IConnectionArgs } from "./deltaManager.js";
+import { IDetachedBlobStorage, ILoaderOptions, RelativeLoader } from "./loader.js";
+import { pkgVersion } from "./packageVersion.js";
+import { ContainerStorageAdapter, ISerializableBlobContents } from "./containerStorageAdapter.js";
+import { IConnectionStateHandler, createConnectionStateHandler } from "./connectionStateHandler.js";
 import {
+	ISnapshotTreeWithBlobContents,
 	combineAppAndProtocolSummary,
 	getProtocolSnapshotTree,
-	getSnapshotTreeFromSerializedContainer,
-} from "./utils";
-import { initQuorumValuesFromCodeDetails } from "./quorum";
-import { NoopHeuristic } from "./noopHeuristic";
-import { ConnectionManager } from "./connectionManager";
-import { ConnectionState } from "./connectionState";
+	getSnapshotTreeAndBlobsFromSerializedContainer,
+	combineSnapshotTreeAndSnapshotBlobs,
+	getDetachedContainerStateFromSerializedContainer,
+	runSingle,
+} from "./utils.js";
+import { initQuorumValuesFromCodeDetails } from "./quorum.js";
+import { NoopHeuristic } from "./noopHeuristic.js";
+import { ConnectionManager } from "./connectionManager.js";
+import { ConnectionState } from "./connectionState.js";
 import {
 	IProtocolHandler,
 	ProtocolHandler,
 	ProtocolHandlerBuilder,
 	protocolHandlerShouldProcessSignal,
-} from "./protocol";
+} from "./protocol.js";
+import { AttachProcessProps, AttachmentData, runRetriableAttachProcess } from "./attachment.js";
+import { SerializedStateManager } from "./serializedStateManager.js";
 
 const detachedContainerRefSeqNumber = 0;
 
@@ -130,8 +134,6 @@ const savedContainerEvent = "saved";
 
 const packageNotFactoryError = "Code package does not implement IRuntimeFactory";
 
-const hasBlobsSummaryTree = ".hasAttachmentBlobs";
-
 /**
  * @internal
  */
@@ -313,9 +315,7 @@ export async function waitContainerToCatchUp(container: IContainer) {
 	});
 }
 
-const getCodeProposal =
-	// eslint-disable-next-line @typescript-eslint/no-unsafe-return
-	(quorum: IQuorumProposals) => quorum.get("code") ?? quorum.get("code2");
+const getCodeProposal = (quorum: IQuorumProposals) => quorum.get("code") ?? quorum.get("code2");
 
 /**
  * Helper function to report to telemetry cases where operation takes longer than expected (200ms)
@@ -326,7 +326,7 @@ const getCodeProposal =
 export async function ReportIfTooLong(
 	logger: ITelemetryLoggerExt,
 	eventName: string,
-	action: () => Promise<ITelemetryProperties>,
+	action: () => Promise<ITelemetryBaseProperties>,
 ) {
 	const event = PerformanceEvent.start(logger, { eventName });
 	const props = await action();
@@ -341,6 +341,7 @@ export async function ReportIfTooLong(
  * @internal
  */
 export interface IPendingContainerState {
+	attached: true;
 	pendingRuntimeState: unknown;
 	/**
 	 * Snapshot from which container initially loaded.
@@ -361,6 +362,19 @@ export interface IPendingContainerState {
 	clientId?: string;
 }
 
+/**
+ * State saved by a container in detached state, to be used to load a new instance
+ * of the container to the same state (rehydrate)
+ * @internal
+ */
+export interface IPendingDetachedContainerState {
+	attached: false;
+	baseSnapshot: ISnapshotTree;
+	snapshotBlobs: ISerializableBlobContents;
+	hasAttachmentBlobs: boolean;
+	pendingRuntimeState?: unknown;
+}
+
 const summarizerClientType = "summarizer";
 
 interface IContainerLifecycleEvents extends IEvent {
@@ -374,7 +388,6 @@ export class Container
 {
 	/**
 	 * Load an existing container.
-	 * @internal
 	 */
 	public static async load(
 		loadProps: IContainerLoadProps,
@@ -472,12 +485,9 @@ export class Container
 			container.mc.logger,
 			{ eventName: "RehydrateDetachedFromSnapshot" },
 			async (_event) => {
-				const deserializedSummary = JSON.parse(snapshot);
-				if (!isCombinedAppAndProtocolSummary(deserializedSummary, hasBlobsSummaryTree)) {
-					throw new UsageError("Cannot rehydrate detached container. Incorrect format");
-				}
-
-				await container.rehydrateDetachedFromSnapshot(deserializedSummary);
+				const detachedContainerState: IPendingDetachedContainerState =
+					getDetachedContainerStateFromSerializedContainer(snapshot);
+				await container.rehydrateDetachedFromSnapshot(detachedContainerState);
 				return container;
 			},
 			{ start: true, end: true, cancel: "generic" },
@@ -502,7 +512,6 @@ export class Container
 
 	/**
 	 * Used by the RelativeLoader to spawn a new Container for the same document.  Used to create the summarizing client.
-	 * @internal
 	 */
 	public readonly clone: (
 		loadProps: IContainerLoadProps,
@@ -552,8 +561,6 @@ export class Container
 		return this._lifecycleState === "disposing" || this._lifecycleState === "disposed";
 	}
 
-	private _attachState = AttachState.Detached;
-
 	private readonly storageAdapter: ContainerStorageAdapter;
 
 	private readonly _deltaManager: DeltaManager<ConnectionManager>;
@@ -579,17 +586,16 @@ export class Container
 	private firstConnection = true;
 	private readonly connectionTransitionTimes: number[] = [];
 	private _loadedFromVersion: IVersion | undefined;
-	private attachStarted = false;
 	private _dirtyContainer = false;
-	private readonly savedOps: ISequencedDocumentMessage[] = [];
-	private baseSnapshot?: ISnapshotTree;
-	private baseSnapshotBlobs?: ISerializableBlobContents;
+	private attachmentData: AttachmentData = { state: AttachState.Detached };
+	private readonly serializedStateManager: SerializedStateManager;
 	private readonly _containerId: string;
 
 	private lastVisible: number | undefined;
 	private readonly visibilityEventHandler: (() => void) | undefined;
 	private readonly connectionStateHandler: IConnectionStateHandler;
 	private readonly clientsWhoShouldHaveLeft = new Set<string>();
+	private _containerMetadata: Readonly<Record<string, string>> = {};
 
 	private setAutoReconnectTime = performance.now();
 
@@ -618,6 +624,10 @@ export class Container
 		return this._deltaManager.readOnlyInfo;
 	}
 
+	public get containerMetadata(): Record<string, string> {
+		return this._containerMetadata;
+	}
+
 	/**
 	 * Sends signal to runtime (and data stores) to be read-only.
 	 * Hosts may have read only views, indicating to data stores that no edits are allowed.
@@ -661,12 +671,8 @@ export class Container
 		return this._clientId;
 	}
 
-	private get offlineLoadEnabled(): boolean {
-		const enabled =
-			this.mc.config.getBoolean("Fluid.Container.enableOfflineLoad") ??
-			this.options?.enableOfflineLoad === true;
-		// summarizer will not have any pending state we want to save
-		return enabled && this.deltaManager.clientDetails.capabilities.interactive;
+	private get isInteractiveClient(): boolean {
+		return this.deltaManager.clientDetails.capabilities.interactive;
 	}
 
 	/**
@@ -735,9 +741,6 @@ export class Container
 
 	private readonly _lifecycleEvents = new TypedEventEmitter<IContainerLifecycleEvents>();
 
-	/**
-	 * @internal
-	 */
 	constructor(
 		createProps: IContainerCreateProps,
 		loadProps?: Pick<IContainerLoadProps, "pendingLocalState">,
@@ -810,7 +813,7 @@ export class Container
 
 		this.client = Container.setupClient(
 			this._containerId,
-			this.options,
+			options.client,
 			this.clientDetailsOverride,
 		);
 
@@ -830,7 +833,7 @@ export class Container
 					clientType, // Differentiating summarizer container from main container
 					containerId: this._containerId,
 					docId: () => this.resolvedUrl?.id,
-					containerAttachState: () => this._attachState,
+					containerAttachState: () => this.attachState,
 					containerLifecycleState: () => this._lifecycleState,
 					containerConnectionState: () => ConnectionState[this.connectionState],
 					serializedContainer: pendingLocalState !== undefined,
@@ -887,7 +890,7 @@ export class Container
 				logConnectionIssue: (
 					eventName: string,
 					category: TelemetryEventCategory,
-					details?: ITelemetryProperties,
+					details?: ITelemetryBaseProperties,
 				) => {
 					const mode = this.connectionMode;
 					// We get here when socket does not receive any ops on "write" connection, including
@@ -953,6 +956,17 @@ export class Container
 			forceEnableSummarizeProtocolTree,
 		);
 
+		const offlineLoadEnabled =
+			(this.isInteractiveClient &&
+				this.mc.config.getBoolean("Fluid.Container.enableOfflineLoad")) ??
+			options.enableOfflineLoad === true;
+		this.serializedStateManager = new SerializedStateManager(
+			pendingLocalState,
+			this.subLogger,
+			this.storageAdapter,
+			offlineLoadEnabled,
+		);
+
 		const isDomAvailable =
 			typeof document === "object" &&
 			document !== null &&
@@ -1031,6 +1045,11 @@ export class Container
 
 				this._lifecycleState = "closing";
 
+				// Back-compat for Old driver
+				if (this.service?.off !== undefined) {
+					this.service?.off("metadataUpdate", this.metadataUpdateHandler);
+				}
+
 				this._protocolHandler?.close();
 
 				this.connectionStateHandler.dispose();
@@ -1128,202 +1147,175 @@ export class Container
 	}
 
 	private async getPendingLocalStateCore(props: IGetPendingLocalStateProps) {
-		return PerformanceEvent.timedExecAsync(
-			this.mc.logger,
-			{
-				eventName: "getPendingLocalState",
-				notifyImminentClosure: props.notifyImminentClosure,
-				savedOpsSize: this.savedOps.length,
-				clientId: this.clientId,
-			},
-			async () => {
-				if (!this.offlineLoadEnabled) {
-					throw new UsageError(
-						"Can't get pending local state unless offline load is enabled",
-					);
-				}
-				if (this.closed || this._disposed) {
-					throw new UsageError(
-						"Pending state cannot be retried if the container is closed or disposed",
-					);
-				}
-				assert(
-					this.attachState === AttachState.Attached,
-					0x0d1 /* "Container should be attached before close" */,
-				);
-				assert(
-					this.resolvedUrl !== undefined && this.resolvedUrl.type === "fluid",
-					0x0d2 /* "resolved url should be valid Fluid url" */,
-				);
-				assert(!!this.baseSnapshot, 0x5d4 /* no base snapshot */);
-				assert(!!this.baseSnapshotBlobs, 0x5d5 /* no snapshot blobs */);
-				const pendingRuntimeState = await this.runtime.getPendingLocalState(props);
-				const pendingState: IPendingContainerState = {
-					pendingRuntimeState,
-					baseSnapshot: this.baseSnapshot,
-					snapshotBlobs: this.baseSnapshotBlobs,
-					savedOps: this.savedOps,
-					url: this.resolvedUrl.url,
-					// no need to save this if there is no pending runtime state
-					clientId: pendingRuntimeState !== undefined ? this.clientId : undefined,
-				};
-
-				return JSON.stringify(pendingState);
-			},
+		if (this.closed || this._disposed) {
+			throw new UsageError(
+				"Pending state cannot be retried if the container is closed or disposed",
+			);
+		}
+		assert(
+			this.attachmentData.state === AttachState.Attached,
+			0x0d1 /* "Container should be attached before close" */,
+		);
+		assert(
+			this.resolvedUrl !== undefined && this.resolvedUrl.type === "fluid",
+			0x0d2 /* "resolved url should be valid Fluid url" */,
+		);
+		const pendingState = await this.serializedStateManager.getPendingLocalStateCore(
+			props,
+			this.clientId,
+			this.runtime,
+			this.resolvedUrl,
 		);
+		return pendingState;
 	}
 
 	public get attachState(): AttachState {
-		return this._attachState;
+		return this.attachmentData.state;
 	}
 
 	public serialize(): string {
-		assert(
-			this.attachState === AttachState.Detached,
-			0x0d3 /* "Should only be called in detached container" */,
-		);
-
-		const appSummary: ISummaryTree = this.runtime.createSummary();
-		const protocolSummary = this.captureProtocolSummary();
-		const combinedSummary = combineAppAndProtocolSummary(appSummary, protocolSummary);
-
-		if (this.detachedBlobStorage && this.detachedBlobStorage.size > 0) {
-			combinedSummary.tree[hasBlobsSummaryTree] = {
-				type: SummaryType.Blob,
-				content: "true",
-			};
+		if (this.attachmentData.state === AttachState.Attached || this.closed) {
+			throw new UsageError("Container must not be attached or closed.");
 		}
-		return JSON.stringify(combinedSummary);
-	}
 
-	public async attach(
-		request: IRequest,
-		attachProps?: { deltaConnection?: "none" | "delayed" },
-	): Promise<void> {
-		await PerformanceEvent.timedExecAsync(
-			this.mc.logger,
-			{ eventName: "Attach" },
-			async () => {
-				if (this._lifecycleState !== "loaded") {
-					// pre-0.58 error message: containerNotValidForAttach
-					throw new UsageError(
-						`The Container is not in a valid state for attach [${this._lifecycleState}]`,
-					);
-				}
+		const attachingData =
+			this.attachmentData.state === AttachState.Attaching ? this.attachmentData : undefined;
 
-				// If container is already attached or attach is in progress, throw an error.
-				assert(
-					this._attachState === AttachState.Detached && !this.attachStarted,
-					0x205 /* "attach() called more than once" */,
-				);
-				this.attachStarted = true;
+		const combinedSummary =
+			attachingData?.summary ??
+			combineAppAndProtocolSummary(
+				this.runtime.createSummary(),
+				this.captureProtocolSummary(),
+			);
 
-				// If attachment blobs were uploaded in detached state we will go through a different attach flow
-				const hasAttachmentBlobs =
-					this.detachedBlobStorage !== undefined && this.detachedBlobStorage.size > 0;
+		const { tree: snapshot, blobs } =
+			getSnapshotTreeAndBlobsFromSerializedContainer(combinedSummary);
 
-				try {
-					assert(
-						this.deltaManager.inbound.length === 0,
-						0x0d6 /* "Inbound queue should be empty when attaching" */,
-					);
+		const pendingRuntimeState =
+			attachingData !== undefined ? this.runtime.getPendingLocalState() : undefined;
+		assert(!isPromiseLike(pendingRuntimeState), 0x8e3 /* should not be a promise */);
 
-					let summary: ISummaryTree;
-					if (!hasAttachmentBlobs) {
-						// Get the document state post attach - possibly can just call attach but we need to change the
-						// semantics around what the attach means as far as async code goes.
-						const appSummary: ISummaryTree = this.runtime.createSummary();
-						const protocolSummary = this.captureProtocolSummary();
-						summary = combineAppAndProtocolSummary(appSummary, protocolSummary);
-
-						// Set the state as attaching as we are starting the process of attaching container.
-						// This should be fired after taking the summary because it is the place where we are
-						// starting to attach the container to storage.
-						// Also, this should only be fired in detached container.
-						this._attachState = AttachState.Attaching;
-						this.runtime.setAttachState(AttachState.Attaching);
-						this.emit("attaching");
-						if (this.offlineLoadEnabled) {
-							const snapshot = getSnapshotTreeFromSerializedContainer(summary);
-							this.baseSnapshot = snapshot;
-							this.baseSnapshotBlobs =
-								getBlobContentsFromTreeWithBlobContents(snapshot);
-						}
-					}
-
-					// Actually go and create the resolved document
-					if (this.service === undefined) {
-						const createNewResolvedUrl = await this.urlResolver.resolve(request);
-						assert(
-							this.client.details.type !== summarizerClientType &&
-								createNewResolvedUrl !== undefined,
-							0x2c4 /* "client should not be summarizer before container is created" */,
-						);
-						this.service = await runWithRetry(
-							async () =>
-								this.serviceFactory.createContainer(
-									summary,
-									createNewResolvedUrl,
-									this.subLogger,
-									false, // clientIsSummarizer
-								),
-							"containerAttach",
-							this.mc.logger,
-							{
-								cancel: this._deltaManager.closeAbortController.signal,
-							}, // progress
+		const detachedContainerState: IPendingDetachedContainerState = {
+			attached: false,
+			baseSnapshot: snapshot,
+			snapshotBlobs: blobs,
+			pendingRuntimeState,
+			hasAttachmentBlobs: !!this.detachedBlobStorage && this.detachedBlobStorage.size > 0,
+		};
+		return JSON.stringify(detachedContainerState);
+	}
+
+	public readonly attach = runSingle(
+		async (
+			request: IRequest,
+			attachProps?: { deltaConnection?: "none" | "delayed" },
+		): Promise<void> => {
+			await PerformanceEvent.timedExecAsync(
+				this.mc.logger,
+				{ eventName: "Attach" },
+				async () => {
+					if (
+						this._lifecycleState !== "loaded" ||
+						this.attachmentData.state === AttachState.Attached
+					) {
+						// pre-0.58 error message: containerNotValidForAttach
+						throw new UsageError(
+							`The Container is not in a valid state for attach [${this._lifecycleState}] and [${this.attachState}]`,
 						);
 					}
-					this.storageAdapter.connectToService(this.service);
 
-					if (hasAttachmentBlobs) {
-						// upload blobs to storage
-						assert(
-							!!this.detachedBlobStorage,
-							0x24e /* "assertion for type narrowing" */,
-						);
+					const normalizeErrorAndClose = (error: unknown): IFluidErrorBase => {
+						const newError = normalizeError(error);
+						this.close(newError);
+						// add resolved URL on error object so that host has the ability to find this document and delete it
+						newError.addTelemetryProperties({
+							resolvedUrl: this.service?.resolvedUrl?.url,
+						});
+						return newError;
+					};
 
-						// build a table mapping IDs assigned locally to IDs assigned by storage and pass it to runtime to
-						// support blob handles that only know about the local IDs
-						const redirectTable = new Map<string, string>();
-						// if new blobs are added while uploading, upload them too
-						while (redirectTable.size < this.detachedBlobStorage.size) {
-							const newIds = this.detachedBlobStorage
-								.getBlobIds()
-								.filter((id) => !redirectTable.has(id));
-							for (const id of newIds) {
-								const blob = await this.detachedBlobStorage.readBlob(id);
-								const response = await this.storageAdapter.createBlob(blob);
-								redirectTable.set(id, response.id);
+					const setAttachmentData: AttachProcessProps["setAttachmentData"] = (
+						attachmentData,
+					) => {
+						const previousState = this.attachmentData.state;
+						this.attachmentData = attachmentData;
+						const state = this.attachmentData.state;
+						if (state !== previousState && state !== AttachState.Detached) {
+							try {
+								this.runtime.setAttachState(state);
+								this.emit(state.toLocaleLowerCase());
+							} catch (error) {
+								throw normalizeErrorAndClose(error);
 							}
 						}
+					};
 
-						// take summary and upload
-						const appSummary: ISummaryTree = this.runtime.createSummary(redirectTable);
-						const protocolSummary = this.captureProtocolSummary();
-						summary = combineAppAndProtocolSummary(appSummary, protocolSummary);
-
-						this._attachState = AttachState.Attaching;
-						this.runtime.setAttachState(AttachState.Attaching);
-						this.emit("attaching");
-						if (this.offlineLoadEnabled) {
-							const snapshot = getSnapshotTreeFromSerializedContainer(summary);
-							this.baseSnapshot = snapshot;
-							this.baseSnapshotBlobs =
-								getBlobContentsFromTreeWithBlobContents(snapshot);
+					const createAttachmentSummary: AttachProcessProps["createAttachmentSummary"] = (
+						redirectTable?: Map<string, string>,
+					) => {
+						try {
+							assert(
+								this.deltaManager.inbound.length === 0,
+								0x0d6 /* "Inbound queue should be empty when attaching" */,
+							);
+							return combineAppAndProtocolSummary(
+								this.runtime.createSummary(redirectTable),
+								this.captureProtocolSummary(),
+							);
+						} catch (error) {
+							throw normalizeErrorAndClose(error);
 						}
+					};
+
+					const createOrGetStorageService: AttachProcessProps["createOrGetStorageService"] =
+						async (summary) => {
+							// Actually go and create the resolved document
+							if (this.service === undefined) {
+								const createNewResolvedUrl =
+									await this.urlResolver.resolve(request);
+								assert(
+									this.client.details.type !== summarizerClientType &&
+										createNewResolvedUrl !== undefined,
+									0x2c4 /* "client should not be summarizer before container is created" */,
+								);
+								this.service = await runWithRetry(
+									async () =>
+										this.serviceFactory.createContainer(
+											summary,
+											createNewResolvedUrl,
+											this.subLogger,
+											false, // clientIsSummarizer
+										),
+									"containerAttach",
+									this.mc.logger,
+									{
+										cancel: this._deltaManager.closeAbortController.signal,
+									}, // progress
+								);
+							}
+							this.storageAdapter.connectToService(this.service);
+							return this.storageAdapter;
+						};
+
+					let attachP = runRetriableAttachProcess({
+						initialAttachmentData: this.attachmentData,
+						offlineLoadEnabled: this.serializedStateManager.offlineLoadEnabled,
+						detachedBlobStorage: this.detachedBlobStorage,
+						setAttachmentData,
+						createAttachmentSummary,
+						createOrGetStorageService,
+					});
 
-						await this.storageAdapter.uploadSummaryWithContext(summary, {
-							referenceSequenceNumber: 0,
-							ackHandle: undefined,
-							proposalHandle: undefined,
+					// only enable the new behavior if the config is set
+					if (
+						this.mc.config.getBoolean("Fluid.Container.RetryOnAttachFailure") !== true
+					) {
+						attachP = attachP.catch((error) => {
+							throw normalizeErrorAndClose(error);
 						});
 					}
 
-					this._attachState = AttachState.Attached;
-					this.runtime.setAttachState(AttachState.Attached);
-					this.emit("attached");
-
+					this.serializedStateManager.setSnapshot(await attachP);
 					if (!this.closed) {
 						this.handleDeltaConnectionArg(
 							{
@@ -1333,17 +1325,11 @@ export class Container
 							attachProps?.deltaConnection,
 						);
 					}
-				} catch (error) {
-					// add resolved URL on error object so that host has the ability to find this document and delete it
-					const newError = normalizeError(error);
-					newError.addTelemetryProperties({ resolvedUrl: this.resolvedUrl?.url });
-					this.close(newError);
-					throw newError;
-				}
-			},
-			{ start: true, end: true, cancel: "generic" },
-		);
-	}
+				},
+				{ start: true, end: true, cancel: "generic" },
+			);
+		},
+	);
 
 	private setAutoReconnectInternal(mode: ReconnectMode, reason: IConnectionStateChangeReason) {
 		const currentMode = this._deltaManager.connectionManager.reconnectMode;
@@ -1370,7 +1356,7 @@ export class Container
 	public connect() {
 		if (this.closed) {
 			throw new UsageError(`The Container is closed and cannot be connected`);
-		} else if (this._attachState !== AttachState.Attached) {
+		} else if (this.attachState !== AttachState.Attached) {
 			throw new UsageError(`The Container is not attached and cannot be connected`);
 		} else if (!this.connected) {
 			// Note: no need to fetch ops as we do it preemptively as part of DeltaManager.attachOpHandler().
@@ -1386,7 +1372,7 @@ export class Container
 	private connectInternal(args: IConnectionArgs) {
 		assert(!this.closed, 0x2c5 /* "Attempting to connect() a closed Container" */);
 		assert(
-			this._attachState === AttachState.Attached,
+			this.attachState === AttachState.Attached,
 			0x2c6 /* "Attempting to connect() a container that is not attached" */,
 		);
 
@@ -1521,11 +1507,6 @@ export class Container
 		return true;
 	}
 
-	private async getVersion(version: string | null): Promise<IVersion | undefined> {
-		const versions = await this.storageAdapter.getVersions(version, 1);
-		return versions[0];
-	}
-
 	private connectToDeltaStream(args: IConnectionArgs) {
 		// All agents need "write" access, including summarizer.
 		if (!this._canReconnect || !this.client.details.capabilities.interactive) {
@@ -1535,6 +1516,22 @@ export class Container
 		this._deltaManager.connect(args);
 	}
 
+	private readonly metadataUpdateHandler = (metadata: Record<string, string>) => {
+		this._containerMetadata = { ...this._containerMetadata, ...metadata };
+		this.emit("metadataUpdate", metadata);
+	};
+
+	private async createDocumentService(
+		serviceProvider: () => Promise<IDocumentService>,
+	): Promise<IDocumentService> {
+		const service = await serviceProvider();
+		// Back-compat for Old driver
+		if (service.on !== undefined) {
+			service.on("metadataUpdate", this.metadataUpdateHandler);
+		}
+		return service;
+	}
+
 	/**
 	 * Load container.
 	 *
@@ -1548,10 +1545,12 @@ export class Container
 		loadToSequenceNumber: number | undefined,
 	) {
 		const timings: Record<string, number> = { phase1: performance.now() };
-		this.service = await this.serviceFactory.createDocumentService(
-			resolvedUrl,
-			this.subLogger,
-			this.client.details.type === summarizerClientType,
+		this.service = await this.createDocumentService(async () =>
+			this.serviceFactory.createDocumentService(
+				resolvedUrl,
+				this.subLogger,
+				this.client.details.type === summarizerClientType,
+			),
 		);
 
 		// Except in cases where it has stashed ops or requested by feature gate, the container will connect in "read" mode
@@ -1574,33 +1573,20 @@ export class Container
 
 		this.storageAdapter.connectToService(this.service);
 
-		this._attachState = AttachState.Attached;
+		this.attachmentData = {
+			state: AttachState.Attached,
+		};
 
 		timings.phase2 = performance.now();
 		// Fetch specified snapshot.
-		const { snapshot, versionId } =
-			pendingLocalState === undefined
-				? await this.fetchSnapshotTree(specifiedVersion)
-				: { snapshot: pendingLocalState.baseSnapshot, versionId: undefined };
-
-		if (pendingLocalState) {
-			this.baseSnapshot = pendingLocalState.baseSnapshot;
-			this.baseSnapshotBlobs = pendingLocalState.snapshotBlobs;
-		} else {
-			assert(snapshot !== undefined, 0x237 /* "Snapshot should exist" */);
-			if (this.offlineLoadEnabled) {
-				this.baseSnapshot = snapshot;
-				// Save contents of snapshot now, otherwise closeAndGetPendingLocalState() must be async
-				this.baseSnapshotBlobs = await getBlobContentsFromTree(
-					snapshot,
-					this.storageAdapter,
-				);
-			}
-		}
-
+		const { snapshotTree, version } = await this.serializedStateManager.fetchSnapshot(
+			specifiedVersion,
+			this.service?.policies?.supportGetSnapshotApi,
+		);
+		this._loadedFromVersion = version;
 		const attributes: IDocumentAttributes = await this.getDocumentAttributes(
 			this.storageAdapter,
-			snapshot,
+			snapshotTree,
 		);
 
 		// If we saved ops, we will replay them and don't need DeltaManager to fetch them
@@ -1687,15 +1673,20 @@ export class Container
 
 		// ...load in the existing quorum
 		// Initialize the protocol handler
-		await this.initializeProtocolStateFromSnapshot(attributes, this.storageAdapter, snapshot);
+		await this.initializeProtocolStateFromSnapshot(
+			attributes,
+			this.storageAdapter,
+			snapshotTree,
+		);
 
 		timings.phase3 = performance.now();
 		const codeDetails = this.getCodeDetailsFromQuorum();
 		await this.instantiateRuntime(
 			codeDetails,
-			snapshot,
+			snapshotTree,
 			// give runtime a dummy value so it knows we're loading from a stash blob
 			pendingLocalState ? pendingLocalState?.pendingRuntimeState ?? {} : undefined,
+			isInstanceOfISnapshot(snapshotTree) ? snapshotTree : undefined,
 		);
 
 		// replay saved ops
@@ -1778,7 +1769,7 @@ export class Container
 		);
 		return {
 			sequenceNumber: attributes.sequenceNumber,
-			version: versionId,
+			version: version?.id,
 			dmLastProcessedSeqNumber: this._deltaManager.lastSequenceNumber,
 			dmLastKnownSeqNumber: this._deltaManager.lastKnownSeqNumber,
 		};
@@ -1808,24 +1799,30 @@ export class Container
 		this.setLoaded();
 	}
 
-	private async rehydrateDetachedFromSnapshot(detachedContainerSnapshot: ISummaryTree) {
-		if (detachedContainerSnapshot.tree[hasBlobsSummaryTree] !== undefined) {
+	private async rehydrateDetachedFromSnapshot({
+		baseSnapshot,
+		snapshotBlobs,
+		hasAttachmentBlobs,
+		pendingRuntimeState,
+	}: IPendingDetachedContainerState) {
+		if (hasAttachmentBlobs) {
 			assert(
 				!!this.detachedBlobStorage && this.detachedBlobStorage.size > 0,
 				0x250 /* "serialized container with attachment blobs must be rehydrated with detached blob storage" */,
 			);
-			// eslint-disable-next-line @typescript-eslint/no-dynamic-delete
-			delete detachedContainerSnapshot.tree[hasBlobsSummaryTree];
 		}
-
-		const snapshotTree = getSnapshotTreeFromSerializedContainer(detachedContainerSnapshot);
-		this.storageAdapter.loadSnapshotForRehydratingContainer(snapshotTree);
-		const attributes = await this.getDocumentAttributes(this.storageAdapter, snapshotTree);
+		const snapshotTreeWithBlobContents: ISnapshotTreeWithBlobContents =
+			combineSnapshotTreeAndSnapshotBlobs(baseSnapshot, snapshotBlobs);
+		this.storageAdapter.loadSnapshotFromSnapshotBlobs(snapshotBlobs);
+		const attributes = await this.getDocumentAttributes(
+			this.storageAdapter,
+			snapshotTreeWithBlobContents,
+		);
 
 		await this.attachDeltaManagerOpHandler(attributes);
 
 		// Initialize the protocol handler
-		const baseTree = getProtocolSnapshotTree(snapshotTree);
+		const baseTree = getProtocolSnapshotTree(snapshotTreeWithBlobContents);
 		const qValues = await readAndParse<[string, ICommittedProposal][]>(
 			this.storageAdapter,
 			baseTree.blobs.quorumValues,
@@ -1840,7 +1837,11 @@ export class Container
 		);
 		const codeDetails = this.getCodeDetailsFromQuorum();
 
-		await this.instantiateRuntime(codeDetails, snapshotTree);
+		await this.instantiateRuntime(
+			codeDetails,
+			snapshotTreeWithBlobContents,
+			pendingRuntimeState,
+		);
 
 		this.setLoaded();
 	}
@@ -1984,13 +1985,12 @@ export class Container
 
 	private static setupClient(
 		containerId: string,
-		options?: ILoaderOptions,
+		loaderOptionsClient?: IClient,
 		clientDetailsOverride?: IClientDetails,
 	): IClient {
-		const loaderOptionsClient = structuredClone(options?.client);
 		const client: IClient =
 			loaderOptionsClient !== undefined
-				? (loaderOptionsClient as IClient)
+				? structuredClone(loaderOptionsClient)
 				: {
 						details: {
 							capabilities: { interactive: true },
@@ -2068,10 +2068,10 @@ export class Container
 			this.connectionStateHandler.cancelEstablishingConnection(reason);
 		});
 
-		deltaManager.on("disconnect", (reason: IConnectionStateChangeReason) => {
+		deltaManager.on("disconnect", (text, error) => {
 			this.noopHeuristic?.notifyDisconnect();
 			if (!this.closed) {
-				this.connectionStateHandler.receivedDisconnectEvent(reason);
+				this.connectionStateHandler.receivedDisconnectEvent({ text, error });
 			}
 		});
 
@@ -2295,9 +2295,6 @@ export class Container
 	}
 
 	private processRemoteMessage(message: ISequencedDocumentMessage) {
-		if (this.offlineLoadEnabled) {
-			this.savedOps.push(message);
-		}
 		const local = this.clientId === message.clientId;
 
 		// Allow the protocol handler to process the message
@@ -2305,7 +2302,7 @@ export class Container
 
 		// Forward messages to the loaded runtime for processing
 		this.runtime.process(message, local);
-
+		this.serializedStateManager.addProcessedOp(message);
 		// Inactive (not in quorum or not writers) clients don't take part in the minimum sequence number calculation.
 		if (this.activeConnection()) {
 			if (this.noopHeuristic === undefined) {
@@ -2358,36 +2355,11 @@ export class Container
 		}
 	}
 
-	/**
-	 * Get the most recent snapshot, or a specific version.
-	 * @param specifiedVersion - The specific version of the snapshot to retrieve
-	 * @returns The snapshot requested, or the latest snapshot if no version was specified, plus version ID
-	 */
-	private async fetchSnapshotTree(
-		specifiedVersion: string | undefined,
-	): Promise<{ snapshot?: ISnapshotTree; versionId?: string }> {
-		const version = await this.getVersion(specifiedVersion ?? null);
-
-		if (version === undefined && specifiedVersion !== undefined) {
-			// We should have a defined version to load from if specified version requested
-			this.mc.logger.sendErrorEvent({
-				eventName: "NoVersionFoundWhenSpecified",
-				id: specifiedVersion,
-			});
-		}
-		this._loadedFromVersion = version;
-		const snapshot = (await this.storageAdapter.getSnapshotTree(version)) ?? undefined;
-
-		if (snapshot === undefined && version !== undefined) {
-			this.mc.logger.sendErrorEvent({ eventName: "getSnapshotTreeFailed", id: version.id });
-		}
-		return { snapshot, versionId: version?.id };
-	}
-
 	private async instantiateRuntime(
 		codeDetails: IFluidCodeDetails,
-		snapshot: ISnapshotTree | undefined,
+		snapshotTree: ISnapshotTree | undefined,
 		pendingLocalState?: unknown,
+		snapshot?: ISnapshot,
 	) {
 		assert(this._runtime?.disposed !== false, 0x0dd /* "Existing runtime not disposed" */);
 
@@ -2421,12 +2393,12 @@ export class Container
 			(this.protocolHandler.quorum.get("code") ??
 				this.protocolHandler.quorum.get("code2")) as IFluidCodeDetails | undefined;
 
-		const existing = snapshot !== undefined;
+		const existing = snapshotTree !== undefined;
 
 		const context = new ContainerContext(
 			this.options,
 			this.scope,
-			snapshot,
+			snapshotTree,
 			this._loadedFromVersion,
 			this._deltaManager,
 			this.storageAdapter,
@@ -2453,6 +2425,7 @@ export class Container
 			existing,
 			this.subLogger,
 			pendingLocalState,
+			snapshot,
 		);
 
 		this._runtime = await PerformanceEvent.timedExecAsync(