@fluidframework/container-loader 2.0.0-rc.4.0.0 → 2.0.0-rc.4.0.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fluidframework/container-loader",
-  "version": "2.0.0-rc.4.0.0",
+  "version": "2.0.0-rc.4.0.10",
   "description": "Fluid container loader",
   "homepage": "https://fluidframework.com",
   "repository": {
@@ -15,11 +15,11 @@
   "exports": {
     ".": {
       "import": {
-        "types": "./lib/public.d.ts",
+        "types": "./lib/index.d.ts",
         "default": "./lib/index.js"
       },
       "require": {
-        "types": "./dist/public.d.ts",
+        "types": "./dist/index.d.ts",
         "default": "./dist/index.js"
       }
     },
@@ -95,7 +95,7 @@
     }
   },
   "main": "lib/index.js",
-  "types": "lib/public.d.ts",
+  "types": "lib/index.d.ts",
   "c8": {
     "all": true,
     "cache-dir": "nyc/.cache",
@@ -117,15 +117,15 @@
     "temp-directory": "nyc/.nyc_output"
   },
   "dependencies": {
-    "@fluid-internal/client-utils": ">=2.0.0-rc.4.0.0 <2.0.0-rc.4.1.0",
-    "@fluidframework/container-definitions": ">=2.0.0-rc.4.0.0 <2.0.0-rc.4.1.0",
-    "@fluidframework/core-interfaces": ">=2.0.0-rc.4.0.0 <2.0.0-rc.4.1.0",
-    "@fluidframework/core-utils": ">=2.0.0-rc.4.0.0 <2.0.0-rc.4.1.0",
-    "@fluidframework/driver-definitions": ">=2.0.0-rc.4.0.0 <2.0.0-rc.4.1.0",
-    "@fluidframework/driver-utils": ">=2.0.0-rc.4.0.0 <2.0.0-rc.4.1.0",
+    "@fluid-internal/client-utils": ">=2.0.0-rc.4.0.10 <2.0.0-rc.4.1.0",
+    "@fluidframework/container-definitions": ">=2.0.0-rc.4.0.10 <2.0.0-rc.4.1.0",
+    "@fluidframework/core-interfaces": ">=2.0.0-rc.4.0.10 <2.0.0-rc.4.1.0",
+    "@fluidframework/core-utils": ">=2.0.0-rc.4.0.10 <2.0.0-rc.4.1.0",
+    "@fluidframework/driver-definitions": ">=2.0.0-rc.4.0.10 <2.0.0-rc.4.1.0",
+    "@fluidframework/driver-utils": ">=2.0.0-rc.4.0.10 <2.0.0-rc.4.1.0",
     "@fluidframework/protocol-base": "^4.0.0",
     "@fluidframework/protocol-definitions": "^3.2.0",
-    "@fluidframework/telemetry-utils": ">=2.0.0-rc.4.0.0 <2.0.0-rc.4.1.0",
+    "@fluidframework/telemetry-utils": ">=2.0.0-rc.4.0.10 <2.0.0-rc.4.1.0",
     "@ungap/structured-clone": "^1.2.0",
     "debug": "^4.3.4",
     "double-ended-queue": "^2.1.0-0",
@@ -134,8 +134,8 @@
   "devDependencies": {
     "@arethetypeswrong/cli": "^0.15.2",
     "@biomejs/biome": "^1.6.2",
-    "@fluid-internal/mocha-test-setup": ">=2.0.0-rc.4.0.0 <2.0.0-rc.4.1.0",
-    "@fluid-private/test-loader-utils": ">=2.0.0-rc.4.0.0 <2.0.0-rc.4.1.0",
+    "@fluid-internal/mocha-test-setup": ">=2.0.0-rc.4.0.10 <2.0.0-rc.4.1.0",
+    "@fluid-private/test-loader-utils": ">=2.0.0-rc.4.0.10 <2.0.0-rc.4.1.0",
     "@fluid-tools/build-cli": "^0.38.0",
     "@fluidframework/build-common": "^2.0.3",
     "@fluidframework/build-tools": "^0.38.0",

package/src/attachment.ts

@@ -9,6 +9,7 @@ import { IDocumentStorageService } from "@fluidframework/driver-definitions/inte
 import { CombinedAppAndProtocolSummary } from "@fluidframework/driver-utils/internal";
 import { ISummaryTree } from "@fluidframework/protocol-definitions";
 
+// eslint-disable-next-line import/no-deprecated
 import { IDetachedBlobStorage } from "./loader.js";
 import type { SnapshotWithBlobs } from "./serializedStateManager.js";
 import { getSnapshotTreeAndBlobsFromSerializedContainer } from "./utils.js";
@@ -110,6 +111,7 @@ export interface AttachProcessProps {
 	/**
 	 * The detached blob storage if it exists.
 	 */
+	// eslint-disable-next-line import/no-deprecated
 	readonly detachedBlobStorage?: Pick<IDetachedBlobStorage, "getBlobIds" | "readBlob" | "size">;
 
 	/**

package/src/connectionStateHandler.ts

@@ -11,8 +11,8 @@ import { IClient, ISequencedClient } from "@fluidframework/protocol-definitions"
 import {
 	type TelemetryEventCategory,
 	ITelemetryLoggerExt,
+	MonitoringContext,
 	PerformanceEvent,
-	loggerToMonitoringContext,
 } from "@fluidframework/telemetry-utils/internal";
 
 import { CatchUpMonitor, ICatchUpMonitor } from "./catchUpMonitor.js";
@@ -31,6 +31,7 @@ const JoinSignalTimeoutMs = 10000;
 /** Constructor parameter type for passing in dependencies needed by the ConnectionStateHandler */
 export interface IConnectionStateHandlerInputs {
 	logger: ITelemetryLoggerExt;
+	mc: MonitoringContext;
 	/** Log to telemetry any change in state, included to Connecting */
 	connectionStateChanged: (
 		value: ConnectionState,
@@ -92,10 +93,10 @@ export function createConnectionStateHandler(
 	deltaManager: IDeltaManager<any, any>,
 	clientId?: string,
 ) {
-	const mc = loggerToMonitoringContext(inputs.logger);
+	const config = inputs.mc.config;
 	return createConnectionStateHandlerCore(
-		mc.config.getBoolean("Fluid.Container.DisableCatchUpBeforeDeclaringConnected") !== true, // connectedRaisedWhenCaughtUp
-		mc.config.getBoolean("Fluid.Container.DisableJoinSignalWait") !== true, // readClientsWaitForJoinSignal
+		config.getBoolean("Fluid.Container.DisableCatchUpBeforeDeclaringConnected") !== true, // connectedRaisedWhenCaughtUp
+		config.getBoolean("Fluid.Container.DisableJoinSignalWait") !== true, // readClientsWaitForJoinSignal
 		inputs,
 		deltaManager,
 		clientId,
@@ -189,6 +190,9 @@ class ConnectionStateHandlerPassThrough
 	public get logger() {
 		return this.inputs.logger;
 	}
+	public get mc() {
+		return this.inputs.mc;
+	}
 	public connectionStateChanged(
 		value: ConnectionState,
 		oldState: ConnectionState,
@@ -469,18 +473,6 @@ class ConnectionStateHandler implements IConnectionStateHandler {
 				});
 			}
 			this.applyForConnectedState("addMemberEvent");
-		} else if (clientId === this.clientId) {
-			// If we see our clientId and it's not also our pending ID, it's our own join op
-			// being replayed, so start the timer in case our previous client is still in quorum
-			assert(
-				!this.waitingForLeaveOp,
-				0x5d2 /* Unexpected join op with current clientId while waiting */,
-			);
-			assert(
-				this.connectionState !== ConnectionState.Connected,
-				0x5d3 /* Unexpected join op with current clientId while connected */,
-			);
-			this.prevClientLeftTimer.restart();
 		}
 	}
 
@@ -528,7 +520,7 @@ class ConnectionStateHandler implements IConnectionStateHandler {
 
 	private receivedRemoveMemberEvent(clientId: string) {
 		// If the client which has left was us, then finish the timer.
-		if (this.clientId === clientId) {
+		if (this.clientId === clientId && this.waitingForLeaveOp) {
 			this.prevClientLeftTimer.clear();
 			this.applyForConnectedState("removeMemberEvent");
 		}
@@ -737,8 +729,24 @@ class ConnectionStateHandler implements IConnectionStateHandler {
 			this.receivedAddMemberEvent(this.pendingClientId!);
 		}
 
+		assert(
+			!this.waitingForLeaveOp,
+			"leave timer can't be set as we have not had access to quorum",
+		);
+
+		// This check is required for scenario of loading container from pending state, and ensuring there is no way
+		// old clientId is still in the quorum (very unlikely, but you never know)
 		// if we have a clientId from a previous container we need to wait for its leave message
-		if (this.clientId !== undefined && this.hasMember(this.clientId)) {
+		// This mimicks check in setConnectionState()
+		// Note that we are not consulting this.handler.shouldClientJoinWrite() here
+		// It could produce wrong results for stashed ops were never sent to Loader yet, and if this check
+		// makes determination only on that (and not uses "dirty" events), then it can produce wrong result.
+		// In most cases it does not matter, as this client already left quorum. But in really unfortunate case,
+		// we might wait even if we could avoid such wait.
+		if (
+			this._clientId !== undefined &&
+			protocol.quorum?.getMember(this._clientId) !== undefined
+		) {
 			this.prevClientLeftTimer.restart();
 		}
 	}

package/src/container.ts

@@ -92,6 +92,7 @@ import {
 	normalizeError,
 	raiseConnectedEvent,
 	wrapError,
+	loggerToMonitoringContext,
 } from "@fluidframework/telemetry-utils/internal";
 import structuredClone from "@ungap/structured-clone";
 import { v4 as uuid } from "uuid";
@@ -111,6 +112,7 @@ import {
 	getPackageName,
 } from "./contracts.js";
 import { DeltaManager, IConnectionArgs } from "./deltaManager.js";
+// eslint-disable-next-line import/no-deprecated
 import { IDetachedBlobStorage, ILoaderOptions, RelativeLoader } from "./loader.js";
 import { NoopHeuristic } from "./noopHeuristic.js";
 import { pkgVersion } from "./packageVersion.js";
@@ -136,6 +138,11 @@ import {
 	getSnapshotTreeAndBlobsFromSerializedContainer,
 	runSingle,
 } from "./utils.js";
+import {
+	serializeMemoryDetachedBlobStorage,
+	createMemoryDetachedBlobStorage,
+	tryInitializeMemoryDetachedBlobStorage,
+} from "./memoryBlobStorage.js";
 
 const detachedContainerRefSeqNumber = 0;
 
@@ -218,6 +225,7 @@ export interface IContainerCreateProps {
 	/**
 	 * Blobs storage for detached containers.
 	 */
+	// eslint-disable-next-line import/no-deprecated
 	readonly detachedBlobStorage?: IDetachedBlobStorage;
 
 	/**
@@ -465,6 +473,7 @@ export class Container
 	private readonly options: ILoaderOptions;
 	private readonly scope: FluidObject;
 	private readonly subLogger: ITelemetryLoggerExt;
+	// eslint-disable-next-line import/no-deprecated
 	private readonly detachedBlobStorage: IDetachedBlobStorage | undefined;
 	private readonly protocolHandlerBuilder: ProtocolHandlerBuilder;
 	private readonly client: IClient;
@@ -517,6 +526,15 @@ export class Container
 				0x969 /* not connected yet */,
 			);
 
+			// Track membership changes and update connection state accordingly
+			// We do this call here, instead of doing it in initializeProtocolState() due to pendingLocalState.
+			// When we load from stashed state, we let connectionStateHandler know about clientId from previous container instance.
+			// But we will play trailing ops from snapshot, including potentially playing join & leave ops for that same clientId!
+			// In other words, if connectionStateHandler has access to Quorum early in load sequence, it will see events (in stashed ops mode)
+			// in the order that is not possible in real life, that it may not expect.
+			// Ideally, we should supply pendingLocalState?.clientId here as well, not in constructor, but it does not matter (at least today)
+			this.connectionStateHandler.initProtocol(this.protocolHandler);
+
 			// Propagate current connection state through the system.
 			const readonly = this.readOnlyInfo.readonly ?? false;
 			// This call does not look like needed any more, with delaying all connection-related events past loaded phase.
@@ -767,7 +785,6 @@ export class Container
 		// Tracking alternative ways to handle this in AB#4129.
 		this.options = { ...options };
 		this.scope = scope;
-		this.detachedBlobStorage = detachedBlobStorage;
 		this.protocolHandlerBuilder =
 			protocolHandlerBuilder ??
 			((
@@ -856,6 +873,9 @@ export class Container
 		this.connectionStateHandler = createConnectionStateHandler(
 			{
 				logger: this.mc.logger,
+				// WARNING: logger on this context should not including getters like containerConnectionState above (on this.subLogger),
+				// as that will result in attempt to dereference this.connectionStateHandler from this call while it's still undefined.
+				mc: loggerToMonitoringContext(subLogger),
 				connectionStateChanged: (value, oldState, reason) => {
 					this.logConnectionStateChangeTelemetry(value, oldState, reason);
 					if (this.loaded) {
@@ -943,8 +963,14 @@ export class Container
 			this.mc.config.getBoolean("Fluid.Container.summarizeProtocolTree2") ??
 			options.summarizeProtocolTree;
 
+		this.detachedBlobStorage =
+			detachedBlobStorage ??
+			(this.mc.config.getBoolean("Fluid.Container.MemoryBlobStorageEnabled") === true
+				? createMemoryDetachedBlobStorage()
+				: undefined);
+
 		this.storageAdapter = new ContainerStorageAdapter(
-			detachedBlobStorage,
+			this.detachedBlobStorage,
 			this.mc.logger,
 			pendingLocalState?.snapshotBlobs,
 			pendingLocalState?.loadedGroupIdSnapshots,
@@ -1208,7 +1234,9 @@ export class Container
 			baseSnapshot,
 			snapshotBlobs,
 			pendingRuntimeState,
-			hasAttachmentBlobs: !!this.detachedBlobStorage && this.detachedBlobStorage.size > 0,
+			hasAttachmentBlobs:
+				this.detachedBlobStorage !== undefined && this.detachedBlobStorage.size > 0,
+			attachmentBlobs: serializeMemoryDetachedBlobStorage(this.detachedBlobStorage),
 		};
 		return JSON.stringify(detachedContainerState);
 	}
@@ -1328,6 +1356,7 @@ export class Container
 					this.serializedStateManager.setInitialSnapshot(snapshotWithBlobs);
 
 					if (!this.closed) {
+						this.detachedBlobStorage?.dispose?.();
 						this.handleDeltaConnectionArg(attachProps?.deltaConnection, {
 							fetchOpsFromStorage: false,
 							reason: { text: "createDetached" },
@@ -1760,11 +1789,15 @@ export class Container
 		baseSnapshot,
 		snapshotBlobs,
 		hasAttachmentBlobs,
+		attachmentBlobs,
 		pendingRuntimeState,
 	}: IPendingDetachedContainerState) {
 		if (hasAttachmentBlobs) {
+			if (attachmentBlobs !== undefined) {
+				tryInitializeMemoryDetachedBlobStorage(this.detachedBlobStorage, attachmentBlobs);
+			}
 			assert(
-				!!this.detachedBlobStorage && this.detachedBlobStorage.size > 0,
+				this.detachedBlobStorage !== undefined && this.detachedBlobStorage.size > 0,
 				0x250 /* "serialized container with attachment blobs must be rehydrated with detached blob storage" */,
 			);
 		}
@@ -1853,9 +1886,6 @@ export class Container
 			protocolLogger.sendErrorEvent(error);
 		});
 
-		// Track membership changes and update connection state accordingly
-		this.connectionStateHandler.initProtocol(protocol);
-
 		protocol.quorum.on("addProposal", (proposal: ISequencedProposal) => {
 			if (proposal.key === "code" || proposal.key === "code2") {
 				this.emit("codeDetailsProposed", proposal.value, proposal);

package/src/containerStorageAdapter.ts

@@ -26,6 +26,7 @@ import {
 } from "@fluidframework/protocol-definitions";
 import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils/internal";
 
+// eslint-disable-next-line import/no-deprecated
 import { IDetachedBlobStorage } from "./loader.js";
 import { ProtocolTreeStorageService } from "./protocolTreeDocumentStorageService.js";
 import { RetriableDocumentStorageService } from "./retriableDocumentStorageService.js";
@@ -79,6 +80,7 @@ export class ContainerStorageAdapter
 	 * @param forceEnableSummarizeProtocolTree - Enforce uploading a protocol summary regardless of the service's policy
 	 */
 	public constructor(
+		// eslint-disable-next-line import/no-deprecated
 		detachedBlobStorage: IDetachedBlobStorage | undefined,
 		private readonly logger: ITelemetryLoggerExt,
 		/**
@@ -233,6 +235,7 @@ export class ContainerStorageAdapter
  */
 class BlobOnlyStorage implements IDocumentStorageService {
 	constructor(
+		// eslint-disable-next-line import/no-deprecated
 		private readonly detachedStorage: IDetachedBlobStorage | undefined,
 		private readonly logger: ITelemetryLoggerExt,
 	) {}
@@ -245,6 +248,7 @@ class BlobOnlyStorage implements IDocumentStorageService {
 		return this.verifyStorage().readBlob(blobId);
 	}
 
+	// eslint-disable-next-line import/no-deprecated
 	private verifyStorage(): IDetachedBlobStorage {
 		if (this.detachedStorage === undefined) {
 			throw new UsageError("Real storage calls not allowed in Unattached container");

package/src/loader.ts

@@ -227,6 +227,7 @@ export interface ILoaderServices {
 
 	/**
 	 * Blobs storage for detached containers.
+	 * @deprecated - IDetachedBlobStorage will be removed in a future release without a replacement. Blobs created while detached will be stored in memory to align with attached container behavior. AB#8049
 	 */
 	readonly detachedBlobStorage?: IDetachedBlobStorage;
 
@@ -241,6 +242,8 @@ export interface ILoaderServices {
  * Subset of IDocumentStorageService which only supports createBlob() and readBlob(). This is used to support
  * blobs in detached containers.
  * @alpha
+ *
+ * @deprecated - IDetachedBlobStorage will be removed in a future release without a replacement. Blobs created while detached will be stored in memory to align with attached container behavior. AB#8049
  */
 export type IDetachedBlobStorage = Pick<IDocumentStorageService, "createBlob" | "readBlob"> & {
 	size: number;
@@ -248,6 +251,11 @@ export type IDetachedBlobStorage = Pick<IDocumentStorageService, "createBlob" |
 	 * Return an array of all blob IDs present in storage
 	 */
 	getBlobIds(): string[];
+
+	/**
+	 * After the container is attached, the detached blob storage is no longer needed and will be disposed.
+	 */
+	dispose?(): void;
 };
 
 /**

package/src/memoryBlobStorage.ts

@@ -0,0 +1,83 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import type { ICreateBlobResponse } from "@fluidframework/protocol-definitions";
+import { bufferToString, stringToBuffer } from "@fluid-internal/client-utils";
+import { assert, isObject } from "@fluidframework/core-utils/internal";
+// eslint-disable-next-line import/no-deprecated
+import type { IDetachedBlobStorage } from "./loader.js";
+
+const MemoryDetachedBlobStorageIdentifier = Symbol();
+
+// eslint-disable-next-line import/no-deprecated
+interface MemoryDetachedBlobStorage extends IDetachedBlobStorage {
+	[MemoryDetachedBlobStorageIdentifier]: typeof MemoryDetachedBlobStorageIdentifier;
+	initialize(attachmentBlobs: string[]): void;
+	serialize(): string | undefined;
+}
+
+function isMemoryDetachedBlobStorage(
+	// eslint-disable-next-line import/no-deprecated
+	detachedStorage: IDetachedBlobStorage | undefined,
+): detachedStorage is MemoryDetachedBlobStorage {
+	return (
+		isObject(detachedStorage) &&
+		MemoryDetachedBlobStorageIdentifier in detachedStorage &&
+		detachedStorage[MemoryDetachedBlobStorageIdentifier] === MemoryDetachedBlobStorageIdentifier
+	);
+}
+
+export function serializeMemoryDetachedBlobStorage(
+	// eslint-disable-next-line import/no-deprecated
+	detachedStorage: IDetachedBlobStorage | undefined,
+): string | undefined {
+	if (
+		detachedStorage !== undefined &&
+		detachedStorage.size > 0 &&
+		isMemoryDetachedBlobStorage(detachedStorage)
+	) {
+		return detachedStorage.serialize();
+	}
+}
+
+export function tryInitializeMemoryDetachedBlobStorage(
+	// eslint-disable-next-line import/no-deprecated
+	detachedStorage: IDetachedBlobStorage | undefined,
+	attachmentBlobs: string,
+) {
+	if (!isMemoryDetachedBlobStorage(detachedStorage)) {
+		throw new Error(
+			"DetachedBlobStorage was not provided to the loader during serialize so cannot be provided during rehydrate.",
+		);
+	}
+
+	assert(detachedStorage.size === 0, "Blob storage already initialized");
+	const maybeAttachmentBlobs = JSON.parse(attachmentBlobs);
+	assert(Array.isArray(maybeAttachmentBlobs), "Invalid attachmentBlobs");
+
+	detachedStorage.initialize(maybeAttachmentBlobs);
+}
+
+// eslint-disable-next-line import/no-deprecated
+export function createMemoryDetachedBlobStorage(): IDetachedBlobStorage {
+	const blobs: ArrayBufferLike[] = [];
+	const storage: MemoryDetachedBlobStorage = {
+		[MemoryDetachedBlobStorageIdentifier]: MemoryDetachedBlobStorageIdentifier,
+		createBlob: async (file: ArrayBufferLike): Promise<ICreateBlobResponse> => ({
+			id: `${blobs.push(file) - 1}`,
+		}),
+		readBlob: async (id: string): Promise<ArrayBufferLike> =>
+			blobs[Number(id)] ?? Promise.reject(new Error(`Blob not found: ${id}`)),
+		get size() {
+			return blobs.length;
+		},
+		getBlobIds: (): string[] => blobs.map((_, i) => `${i}`),
+		dispose: () => blobs.splice(0),
+		serialize: () => JSON.stringify(blobs.map((b) => bufferToString(b, "utf-8"))),
+		initialize: (attachmentBlobs: string[]) =>
+			blobs.push(...attachmentBlobs.map((maybeBlob) => stringToBuffer(maybeBlob, "utf-8"))),
+	};
+	return storage;
+}

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-loader";
-export const pkgVersion = "2.0.0-rc.4.0.0";
+export const pkgVersion = "2.0.0-rc.4.0.10";

package/src/serializedStateManager.ts

@@ -93,6 +93,8 @@ export interface IPendingDetachedContainerState extends SnapshotWithBlobs {
 	attached: false;
 	/** Indicates whether we expect the rehydrated container to have non-empty Detached Blob Storage */
 	hasAttachmentBlobs: boolean;
+	/** Used by the memory blob storage to persisted attachment blobs */
+	attachmentBlobs?: string;
 	/**
 	 * Runtime-specific state that will be needed to properly rehydrate
 	 * (it's included in ContainerContext passed to instantiateRuntime)

package/dist/public.d.ts

@@ -1,14 +0,0 @@
-/*!
- * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
- * Licensed under the MIT License.
- */
-
-/*
- * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
- * Generated by "flub generate entrypoints" in @fluidframework/build-tools.
- */
-
-export {
-	// @public APIs
-	ConnectionState
-} from "./index.js";

package/lib/public.d.ts

@@ -1,14 +0,0 @@
-/*!
- * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
- * Licensed under the MIT License.
- */
-
-/*
- * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
- * Generated by "flub generate entrypoints" in @fluidframework/build-tools.
- */
-
-export {
-	// @public APIs
-	ConnectionState
-} from "./index.js";