@fluidframework/container-runtime 2.33.2 → 2.40.0

package/src/blobManager/blobManager.ts

@@ -3,22 +3,22 @@
  * Licensed under the MIT License.
  */
 
-import {
-	TypedEventEmitter,
-	bufferToString,
-	stringToBuffer,
-} from "@fluid-internal/client-utils";
+import { bufferToString, createEmitter, stringToBuffer } from "@fluid-internal/client-utils";
 import { AttachState } from "@fluidframework/container-definitions";
 import {
 	IContainerRuntime,
 	IContainerRuntimeEvents,
 } from "@fluidframework/container-runtime-definitions/internal";
 import type {
-	IEvent,
+	IEmitter,
 	IEventProvider,
 	IFluidHandleContext,
 	IFluidHandleInternal,
 	IFluidHandleInternalPayloadPending,
+	ILocalFluidHandle,
+	ILocalFluidHandleEvents,
+	Listenable,
+	PayloadState,
 } from "@fluidframework/core-interfaces/internal";
 import { assert, Deferred } from "@fluidframework/core-utils/internal";
 import {
@@ -65,7 +65,9 @@ import {
  */
 export class BlobHandle
 	extends FluidHandleBase<ArrayBufferLike>
-	implements IFluidHandleInternalPayloadPending<ArrayBufferLike>
+	implements
+		ILocalFluidHandle<ArrayBufferLike>,
+		IFluidHandleInternalPayloadPending<ArrayBufferLike>
 {
 	private attached: boolean = false;
 
@@ -73,9 +75,30 @@ export class BlobHandle
 		return this.routeContext.isAttached && this.attached;
 	}
 
+	private _events:
+		| (Listenable<ILocalFluidHandleEvents> & IEmitter<ILocalFluidHandleEvents>)
+		| undefined;
+	public get events(): Listenable<ILocalFluidHandleEvents> {
+		return (this._events ??= createEmitter<ILocalFluidHandleEvents>());
+	}
+
+	private _state: PayloadState = "pending";
+	public get payloadState(): PayloadState {
+		return this._state;
+	}
+
+	/**
+	 * The error property starts undefined, signalling that there has been no error yet.
+	 * If an error occurs, the property will contain the error.
+	 */
+	private _payloadShareError: unknown;
+	public get payloadShareError(): unknown {
+		return this._payloadShareError;
+	}
+
 	public readonly absolutePath: string;
 
-	constructor(
+	public constructor(
 		public readonly path: string,
 		public readonly routeContext: IFluidHandleContext,
 		public get: () => Promise<ArrayBufferLike>,
@@ -86,6 +109,16 @@ export class BlobHandle
 		this.absolutePath = generateHandleContextPath(path, this.routeContext);
 	}
 
+	public readonly notifyShared = (): void => {
+		this._state = "shared";
+		this._events?.emit("payloadShared");
+	};
+
+	public readonly notifyFailed = (error: unknown): void => {
+		this._payloadShareError = error;
+		this._events?.emit("payloadShareFailed", error);
+	};
+
 	public attachGraph(): void {
 		if (!this.attached) {
 			this.attached = true;
@@ -93,6 +126,10 @@ export class BlobHandle
 		}
 	}
 
+	// eslint-disable-next-line jsdoc/require-description
+	/**
+	 * @deprecated No replacement provided. Arbitrary handles may not serve as a bind source.
+	 */
 	public bind(handle: IFluidHandleInternal): void {
 		throw new Error("Cannot bind to blob handle");
 	}
@@ -104,7 +141,7 @@ export type IBlobManagerRuntime = Pick<
 	IContainerRuntime,
 	"attachState" | "connected" | "baseLogger" | "clientDetails" | "disposed"
 > &
-	TypedEventEmitter<IContainerRuntimeEvents>;
+	IEventProvider<IContainerRuntimeEvents>;
 
 type ICreateBlobResponseWithTTL = ICreateBlobResponse &
 	Partial<Record<"minTTLInSeconds", number>>;
@@ -133,13 +170,14 @@ export interface IPendingBlobs {
 	};
 }
 
-export interface IBlobManagerEvents extends IEvent {
-	(event: "noPendingBlobs", listener: () => void);
+export interface IBlobManagerEvents {
+	noPendingBlobs: () => void;
 }
 
-interface IBlobManagerInternalEvents extends IEvent {
-	(event: "handleAttached", listener: (pending: PendingBlob) => void);
-	(event: "processedBlobAttach", listener: (localId: string, storageId: string) => void);
+interface IBlobManagerInternalEvents {
+	uploadFailed: (localId: string, error: unknown) => void;
+	handleAttached: (pending: PendingBlob) => void;
+	processedBlobAttach: (localId: string, storageId: string) => void;
 }
 
 const stashedPendingBlobOverrides: Pick<
@@ -157,11 +195,11 @@ export const blobManagerBasePath = "_blobs" as const;
 export class BlobManager {
 	private readonly mc: MonitoringContext;
 
-	private readonly publicEvents = new TypedEventEmitter<IBlobManagerEvents>();
-	public get events(): IEventProvider<IBlobManagerEvents> {
+	private readonly publicEvents = createEmitter<IBlobManagerEvents>();
+	public get events(): Listenable<IBlobManagerEvents> {
 		return this.publicEvents;
 	}
-	private readonly internalEvents = new TypedEventEmitter<IBlobManagerInternalEvents>();
+	private readonly internalEvents = createEmitter<IBlobManagerInternalEvents>();
 
 	/**
 	 * Map of local IDs to storage IDs. Contains identity entries (storageId → storageId) for storage IDs. All requested IDs should
@@ -201,6 +239,8 @@ export class BlobManager {
 		new Map();
 	public readonly stashedBlobsUploadP: Promise<(void | ICreateBlobResponse)[]>;
 
+	private readonly createBlobPayloadPending: boolean;
+
 	public constructor(props: {
 		readonly routeContext: IFluidHandleContext;
 
@@ -238,6 +278,7 @@ export class BlobManager {
 			runtime,
 			stashedBlobs,
 			localBlobIdGenerator,
+			createBlobPayloadPending,
 		} = props;
 		this.routeContext = routeContext;
 		this.storage = storage;
@@ -245,6 +286,7 @@ export class BlobManager {
 		this.isBlobDeleted = isBlobDeleted;
 		this.runtime = runtime;
 		this.localBlobIdGenerator = localBlobIdGenerator ?? uuid;
+		this.createBlobPayloadPending = createBlobPayloadPending;
 
 		this.mc = createChildMonitoringContext({
 			logger: this.runtime.baseLogger,
@@ -362,6 +404,12 @@ export class BlobManager {
 		return this.redirectTable.get(blobId) !== undefined;
 	}
 
+	/**
+	 * Retrieve the blob with the given local blob id.
+	 * @param blobId - The local blob id.  Likely coming from a handle.
+	 * @param payloadPending - Whether we suspect the payload may be pending and not available yet.
+	 * @returns A promise which resolves to the blob contents
+	 */
 	public async getBlob(blobId: string, payloadPending: boolean): Promise<ArrayBufferLike> {
 		// Verify that the blob is not deleted, i.e., it has not been garbage collected. If it is, this will throw
 		// an error, failing the call.
@@ -386,7 +434,9 @@ export class BlobManager {
 		} else {
 			const attachedStorageId = this.redirectTable.get(blobId);
 			if (!payloadPending) {
-				assert(!!attachedStorageId, 0x11f /* "requesting unknown blobs" */);
+				// Only blob handles explicitly marked with pending payload are permitted to exist without
+				// yet knowing their storage id. Otherwise they must already be associated with a storage id.
+				assert(attachedStorageId !== undefined, 0x11f /* "requesting unknown blobs" */);
 			}
 			// If we didn't find it in the redirectTable, assume the attach op is coming eventually and wait.
 			// We do this even if the local client doesn't have the blob payloadPending flag enabled, in case a
@@ -428,11 +478,11 @@ export class BlobManager {
 			0x384 /* requesting handle for unknown blob */,
 		);
 		const pending = this.pendingBlobs.get(localId);
-		// Create a callback function for once the blob has been attached
+		// Create a callback function for once the handle has been attached
 		const callback = pending
 			? () => {
 					pending.attached = true;
-					// Notify listeners (e.g. serialization process) that blob has been attached
+					// Notify listeners (e.g. serialization process) that handle has been attached
 					this.internalEvents.emit("handleAttached", pending);
 					this.deletePendingBlobMaybe(localId);
 				}
@@ -448,7 +498,7 @@ export class BlobManager {
 
 	private async createBlobDetached(
 		blob: ArrayBufferLike,
-	): Promise<IFluidHandleInternal<ArrayBufferLike>> {
+	): Promise<IFluidHandleInternalPayloadPending<ArrayBufferLike>> {
 		// Blobs created while the container is detached are stored in IDetachedBlobStorage.
 		// The 'IDocumentStorageService.createBlob()' call below will respond with a localId.
 		const response = await this.storage.createBlob(blob);
@@ -459,7 +509,7 @@ export class BlobManager {
 	public async createBlob(
 		blob: ArrayBufferLike,
 		signal?: AbortSignal,
-	): Promise<IFluidHandleInternal<ArrayBufferLike>> {
+	): Promise<IFluidHandleInternalPayloadPending<ArrayBufferLike>> {
 		if (this.runtime.attachState === AttachState.Detached) {
 			return this.createBlobDetached(blob);
 		}
@@ -473,6 +523,15 @@ export class BlobManager {
 			0x385 /* For clarity and paranoid defense against adding future attachment states */,
 		);
 
+		return this.createBlobPayloadPending
+			? this.createBlobWithPayloadPending(blob)
+			: this.createBlobLegacy(blob, signal);
+	}
+
+	private async createBlobLegacy(
+		blob: ArrayBufferLike,
+		signal?: AbortSignal,
+	): Promise<IFluidHandleInternalPayloadPending<ArrayBufferLike>> {
 		if (signal?.aborted) {
 			throw this.createAbortError();
 		}
@@ -503,6 +562,48 @@ export class BlobManager {
 		});
 	}
 
+	private createBlobWithPayloadPending(
+		blob: ArrayBufferLike,
+	): IFluidHandleInternalPayloadPending<ArrayBufferLike> {
+		const localId = this.localBlobIdGenerator();
+
+		const blobHandle = new BlobHandle(
+			getGCNodePathFromBlobId(localId),
+			this.routeContext,
+			async () => blob,
+			true, // payloadPending
+			() => {
+				const pendingEntry: PendingBlob = {
+					blob,
+					handleP: new Deferred(),
+					uploadP: this.uploadBlob(localId, blob),
+					attached: true,
+					acked: false,
+					opsent: false,
+				};
+				this.pendingBlobs.set(localId, pendingEntry);
+			},
+		);
+
+		const onProcessedBlobAttach = (_localId: string, _storageId: string): void => {
+			if (_localId === localId) {
+				this.internalEvents.off("processedBlobAttach", onProcessedBlobAttach);
+				blobHandle.notifyShared();
+			}
+		};
+		this.internalEvents.on("processedBlobAttach", onProcessedBlobAttach);
+
+		const onUploadFailed = (_localId: string, error: unknown): void => {
+			if (_localId === localId) {
+				this.internalEvents.off("uploadFailed", onUploadFailed);
+				blobHandle.notifyFailed(error);
+			}
+		};
+		this.internalEvents.on("uploadFailed", onUploadFailed);
+
+		return blobHandle;
+	}
+
 	private async uploadBlob(
 		localId: string,
 		blob: ArrayBufferLike,
@@ -546,6 +647,7 @@ export class BlobManager {
 				// the promise but not throw any error outside.
 				this.pendingBlobs.get(localId)?.handleP.reject(error);
 				this.deletePendingBlob(localId);
+				this.internalEvents.emit("uploadFailed", localId, error);
 			},
 		);
 	}
@@ -619,7 +721,9 @@ export class BlobManager {
 			// an existing blob, we don't have to wait for the op to be ack'd since this step has already
 			// happened before and so, the server won't delete it.
 			this.setRedirection(localId, response.id);
-			entry.handleP.resolve(this.getBlobHandle(localId));
+			const blobHandle = this.getBlobHandle(localId);
+			blobHandle.notifyShared();
+			entry.handleP.resolve(blobHandle);
 			this.deletePendingBlobMaybe(localId);
 		} else {
 			// If there is already an op for this storage ID, append the local ID to the list. Once any op for
@@ -682,8 +786,8 @@ export class BlobManager {
 		// set identity (id -> id) entry
 		this.setRedirection(blobId, blobId);
 
+		assert(localId !== undefined, 0x50e /* local ID not present in blob attach message */);
 		if (local) {
-			assert(localId !== undefined, 0x50e /* local ID not present in blob attach message */);
 			const waitingBlobs = this.opsInFlight.get(blobId);
 			if (waitingBlobs !== undefined) {
 				// For each op corresponding to this storage ID that we are waiting for, resolve the pending blob.
@@ -697,7 +801,9 @@ export class BlobManager {
 					);
 					this.setRedirection(pendingLocalId, blobId);
 					entry.acked = true;
-					entry.handleP.resolve(this.getBlobHandle(pendingLocalId));
+					const blobHandle = this.getBlobHandle(pendingLocalId);
+					blobHandle.notifyShared();
+					entry.handleP.resolve(blobHandle);
 					this.deletePendingBlobMaybe(pendingLocalId);
 				}
 				this.opsInFlight.delete(blobId);
@@ -705,7 +811,9 @@ export class BlobManager {
 			const localEntry = this.pendingBlobs.get(localId);
 			if (localEntry) {
 				localEntry.acked = true;
-				localEntry.handleP.resolve(this.getBlobHandle(localId));
+				const blobHandle = this.getBlobHandle(localId);
+				blobHandle.notifyShared();
+				localEntry.handleP.resolve(blobHandle);
 				this.deletePendingBlobMaybe(localId);
 			}
 		}
@@ -875,7 +983,7 @@ export class BlobManager {
 				const localBlobs = new Set<PendingBlob>();
 				// This while is used to stash blobs created while attaching and getting blobs
 				while (localBlobs.size < this.pendingBlobs.size) {
-					const attachBlobsP: Promise<void>[] = [];
+					const attachHandlesP: Promise<void>[] = [];
 					for (const [localId, entry] of this.pendingBlobs) {
 						if (!localBlobs.has(entry)) {
 							localBlobs.add(entry);
@@ -890,8 +998,8 @@ export class BlobManager {
 							// original createBlob call) and let them attach the blob. This is a lie we told since the upload
 							// hasn't finished yet, but it's fine since we will retry on rehydration.
 							entry.handleP.resolve(this.getBlobHandle(localId));
-							// Array of promises that will resolve when blobs get attached.
-							attachBlobsP.push(
+							// Array of promises that will resolve when handles get attached.
+							attachHandlesP.push(
 								new Promise<void>((resolve, reject) => {
 									stopBlobAttachingSignal?.addEventListener(
 										"abort",
@@ -901,16 +1009,16 @@ export class BlobManager {
 										},
 										{ once: true },
 									);
-									const onBlobAttached = (attachedEntry: PendingBlob): void => {
+									const onHandleAttached = (attachedEntry: PendingBlob): void => {
 										if (attachedEntry === entry) {
-											this.internalEvents.off("handleAttached", onBlobAttached);
+											this.internalEvents.off("handleAttached", onHandleAttached);
 											resolve();
 										}
 									};
 									if (entry.attached) {
 										resolve();
 									} else {
-										this.internalEvents.on("handleAttached", onBlobAttached);
+										this.internalEvents.on("handleAttached", onHandleAttached);
 									}
 								}),
 							);
@@ -918,7 +1026,7 @@ export class BlobManager {
 					}
 					// Wait for all blobs to be attached. This is important, otherwise serialized container
 					// could send the blobAttach op without any op that references the blob, making it useless.
-					await Promise.allSettled(attachBlobsP);
+					await Promise.allSettled(attachHandlesP);
 				}
 
 				for (const [localId, entry] of this.pendingBlobs) {

package/src/channelCollection.ts

@@ -120,11 +120,22 @@ interface FluidDataStoreMessage {
 	type: string;
 }
 
+/**
+ * This version of the interface is private to this the package. it should never be exported under any tag.
+ * It is used to manage interactions within the container-runtime package. If something is needed
+ * cross package, it is likely it is also being used cross layer (ContainerRuntime * DataStoreRuntime).
+ * If that is the case, the change likely needs to be staged directly on IFluidParentContext. Changes
+ * being staged on IFluidParentContext can be added here as well, likely with optionality removed,
+ * to ease interactions within this package.
+ */
+export interface IFluidParentContextPrivate extends Omit<IFluidParentContext, "isReadOnly"> {
+	readonly isReadOnly: () => boolean;
+}
+
 /**
  * Creates a shallow wrapper of {@link IFluidParentContext}. The wrapper can then have its methods overwritten as needed
  */
-export function wrapContext(context: IFluidParentContext): IFluidParentContext {
-	const isReadOnly = context.isReadOnly;
+export function wrapContext(context: IFluidParentContextPrivate): IFluidParentContextPrivate {
 	return {
 		get IFluidDataStoreRegistry() {
 			return context.IFluidDataStoreRegistry;
@@ -150,7 +161,7 @@ export function wrapContext(context: IFluidParentContext): IFluidParentContext {
 		get attachState() {
 			return context.attachState;
 		},
-		isReadOnly: isReadOnly === undefined ? undefined : () => isReadOnly(),
+		isReadOnly: () => context.isReadOnly(),
 		containerRuntime: context.containerRuntime,
 		scope: context.scope,
 		gcThrowOnTombstoneUsage: context.gcThrowOnTombstoneUsage,
@@ -201,8 +212,8 @@ export function wrapContext(context: IFluidParentContext): IFluidParentContext {
  */
 function wrapContextForInnerChannel(
 	id: string,
-	parentContext: IFluidParentContext,
-): IFluidParentContext {
+	parentContext: IFluidParentContextPrivate,
+): IFluidParentContextPrivate {
 	const context = wrapContext(parentContext);
 
 	context.submitMessage = (type: string, content: unknown, localOpMetadata: unknown) => {
@@ -274,7 +285,7 @@ export class ChannelCollection implements IFluidDataStoreChannel, IDisposable {
 
 	constructor(
 		protected readonly baseSnapshot: ISnapshotTree | ISnapshot | undefined,
-		public readonly parentContext: IFluidParentContext,
+		public readonly parentContext: IFluidParentContextPrivate,
 		baseLogger: ITelemetryBaseLogger,
 		private readonly gcNodeUpdated: (props: IGCNodeUpdatedProps) => void,
 		private readonly isDataStoreDeleted: (nodePath: string) => boolean,
@@ -372,7 +383,7 @@ export class ChannelCollection implements IFluidDataStoreChannel, IDisposable {
 	 */
 	private shouldSendAttachLog = true;
 
-	protected wrapContextForInnerChannel(id: string): IFluidParentContext {
+	protected wrapContextForInnerChannel(id: string): IFluidParentContextPrivate {
 		return wrapContextForInnerChannel(id, this.parentContext);
 	}
 
@@ -692,7 +703,12 @@ export class ChannelCollection implements IFluidDataStoreChannel, IDisposable {
 	}
 	public readonly dispose = (): void => this.disposeOnce.value;
 
-	public reSubmit(type: string, content: unknown, localOpMetadata: unknown): void {
+	public reSubmit(
+		type: string,
+		content: unknown,
+		localOpMetadata: unknown,
+		squash: boolean,
+	): void {
 		switch (type) {
 			case ContainerMessageType.Attach:
 			case ContainerMessageType.Alias: {
@@ -700,7 +716,7 @@ export class ChannelCollection implements IFluidDataStoreChannel, IDisposable {
 				return;
 			}
 			case ContainerMessageType.FluidDataStoreOp: {
-				return this.reSubmitChannelOp(type, content, localOpMetadata);
+				return this.reSubmitChannelOp(type, content, localOpMetadata, squash);
 			}
 			default: {
 				assert(false, 0x907 /* unknown op type */);
@@ -708,7 +724,12 @@ export class ChannelCollection implements IFluidDataStoreChannel, IDisposable {
 		}
 	}
 
-	protected reSubmitChannelOp(type: string, content: unknown, localOpMetadata: unknown): void {
+	protected reSubmitChannelOp(
+		type: string,
+		content: unknown,
+		localOpMetadata: unknown,
+		squash: boolean,
+	): void {
 		const envelope = content as IEnvelope;
 		const context = this.contexts.get(envelope.address);
 		// If the data store has been deleted, log an error and throw an error. If there are local changes for a
@@ -723,7 +744,7 @@ export class ChannelCollection implements IFluidDataStoreChannel, IDisposable {
 		}
 		assert(!!context, 0x160 /* "There should be a store context for the op" */);
 		const innerContents = envelope.contents as FluidDataStoreMessage;
-		context.reSubmit(innerContents.type, innerContents.content, localOpMetadata);
+		context.reSubmit(innerContents.type, innerContents.content, localOpMetadata, squash);
 	}
 
 	public rollback(type: string, content: unknown, localOpMetadata: unknown): void {
@@ -1124,7 +1145,7 @@ export class ChannelCollection implements IFluidDataStoreChannel, IDisposable {
 	public notifyReadOnlyState(readonly: boolean): void {
 		for (const [fluidDataStoreId, context] of this.contexts) {
 			try {
-				context.notifyReadOnlyState(readonly);
+				context.notifyReadOnlyState();
 			} catch (error) {
 				this.mc.logger.sendErrorEvent(
 					{
@@ -1133,7 +1154,7 @@ export class ChannelCollection implements IFluidDataStoreChannel, IDisposable {
 							fluidDataStoreId,
 						}),
 						details: {
-							runtimeReadonly: this.parentContext.isReadOnly?.(),
+							runtimeReadonly: this.parentContext.isReadOnly(),
 							readonly,
 						},
 					},
@@ -1143,6 +1164,32 @@ export class ChannelCollection implements IFluidDataStoreChannel, IDisposable {
 		}
 	}
 
+	/**
+	 * Notifies all data store contexts about the current staging mode state.
+	 *
+	 * @param staging - A boolean indicating whether the container is in staging mode.
+	 */
+	public notifyStagingMode(staging: boolean): void {
+		for (const [fluidDataStoreId, context] of this.contexts) {
+			try {
+				context.notifyStagingMode(staging);
+			} catch (error) {
+				this.mc.logger.sendErrorEvent(
+					{
+						eventName: "notifyStagingModeError",
+						...tagCodeArtifacts({
+							fluidDataStoreId,
+						}),
+						details: {
+							staging,
+						},
+					},
+					error,
+				);
+			}
+		}
+	}
+
 	public setAttachState(attachState: AttachState.Attaching | AttachState.Attached): void {
 		for (const [, context] of this.contexts) {
 			// Fire only for bounded stores.
@@ -1622,9 +1669,7 @@ export function detectOutboundReferences(
 /**
  * @internal
  */
-export class ChannelCollectionFactory<T extends ChannelCollection = ChannelCollection>
-	implements IFluidDataStoreFactory
-{
+export class ChannelCollectionFactory implements IFluidDataStoreFactory {
 	public readonly type = "ChannelCollectionChannel";
 
 	public IFluidDataStoreRegistry: IFluidDataStoreRegistry;
@@ -1635,12 +1680,11 @@ export class ChannelCollectionFactory<T extends ChannelCollection = ChannelColle
 		private readonly provideEntryPoint: (
 			runtime: IFluidDataStoreChannel,
 		) => Promise<FluidObject>,
-		private readonly ctor: (...args: ConstructorParameters<typeof ChannelCollection>) => T,
 	) {
 		this.IFluidDataStoreRegistry = new FluidDataStoreRegistry(registryEntries);
 	}
 
-	public get IFluidDataStoreFactory(): ChannelCollectionFactory<T> {
+	public get IFluidDataStoreFactory(): ChannelCollectionFactory {
 		return this;
 	}
 
@@ -1648,7 +1692,21 @@ export class ChannelCollectionFactory<T extends ChannelCollection = ChannelColle
 		context: IFluidDataStoreContext,
 		_existing: boolean,
 	): Promise<IFluidDataStoreChannel> {
-		const runtime = this.ctor(
+		// when work is done to complete nested datastores
+		// this class should move to a new package
+		// and IFluidDataStoreContext will need to support
+		// all the cross layer needs of the channel context,
+		// and the below assert should be removed.
+		//
+		// for now this will continue to work if both
+		// this factory and the container runtime are
+		// from the same package.
+		assert(
+			context instanceof FluidDataStoreContext,
+			0xb8f /* we don't support the layer boundary here today */,
+		);
+
+		const runtime = new ChannelCollection(
 			context.baseSnapshot,
 			context, // parentContext
 			context.baseLogger,