@fluidframework/container-runtime 2.0.0-internal.6.1.1 → 2.0.0-internal.6.3.0

package/src/blobManager.ts

@@ -17,21 +17,17 @@ import {
 	responseToException,
 	SummaryTreeBuilder,
 } from "@fluidframework/runtime-utils";
-import {
-	assert,
-	bufferToString,
-	Deferred,
-	stringToBuffer,
-	TypedEventEmitter,
-} from "@fluidframework/common-utils";
+import { assert, Deferred } from "@fluidframework/core-utils";
+import { bufferToString, stringToBuffer, TypedEventEmitter } from "@fluid-internal/client-utils";
 import {
 	IContainerRuntime,
 	IContainerRuntimeEvents,
 } from "@fluidframework/container-runtime-definitions";
 import { AttachState, ICriticalContainerError } from "@fluidframework/container-definitions";
 import {
-	LoggingError,
 	createChildMonitoringContext,
+	GenericError,
+	LoggingError,
 	MonitoringContext,
 	PerformanceEvent,
 } from "@fluidframework/telemetry-utils";
@@ -40,7 +36,7 @@ import {
 	ISummaryTreeWithStats,
 	ITelemetryContext,
 } from "@fluidframework/runtime-definitions";
-import { GenericError } from "@fluidframework/container-utils";
+
 import { ContainerRuntime, TombstoneResponseHeaderKey } from "./containerRuntime";
 import { sendGCUnexpectedUsageEvent, sweepAttachmentBlobsKey, throwOnTombstoneLoadKey } from "./gc";
 import { Throttler, formExponentialFn, IThrottler } from "./throttler";
@@ -122,20 +118,12 @@ export type IBlobManagerRuntime = Pick<
 	Pick<ContainerRuntime, "gcTombstoneEnforcementAllowed"> &
 	TypedEventEmitter<IContainerRuntimeEvents>;
 
-// Note that while offline we "submit" an op before uploading the blob, but we always
-// expect blobs to be uploaded before we actually see the op round-trip
-export enum PendingBlobStatus {
-	OnlinePendingUpload,
-	OnlinePendingOp,
-	OfflinePendingUpload,
-	OfflinePendingOp,
-}
-
 type ICreateBlobResponseWithTTL = ICreateBlobResponse & Partial<Record<"minTTLInSeconds", number>>;
 
 interface PendingBlob {
 	blob: ArrayBufferLike;
-	status: PendingBlobStatus;
+	uploading?: boolean;
+	opsent?: boolean;
 	storageId?: string;
 	handleP: Deferred<BlobHandle>;
 	uploadP?: Promise<ICreateBlobResponse | void>;
@@ -144,7 +132,6 @@ interface PendingBlob {
 	attached?: boolean;
 	acked?: boolean;
 	abortSignal?: AbortSignal;
-	opsent?: boolean;
 }
 
 export interface IPendingBlobs {
@@ -177,9 +164,7 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 	private readonly redirectTable: Map<string, string | undefined>;
 
 	/**
-	 * Blobs which have not been uploaded or for which we have not yet seen a BlobAttach op round-trip.
-	 * Until we see the op round-trip, there is a possibility we may need to re-upload the blob, so
-	 * we must save it. This is true for both the online and offline flow.
+	 * Blobs which we have not yet seen a BlobAttach op round-trip and not yet attached to a DDS.
 	 */
 	private readonly pendingBlobs: Map<string, PendingBlob> = new Map();
 
@@ -245,8 +230,6 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 			this.runtime.gcTombstoneEnforcementAllowed &&
 			this.runtime.clientDetails.type !== summarizerClientType;
 
-		this.runtime.on("disconnected", () => this.onDisconnected());
-
 		this.redirectTable = this.load(snapshot);
 
 		// Begin uploading stashed blobs from previous container instance
@@ -261,7 +244,8 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 				if (entry.minTTLInSeconds - timeLapseSinceLocalUpload > entry.minTTLInSeconds / 2) {
 					this.pendingBlobs.set(localId, {
 						blob,
-						status: PendingBlobStatus.OfflinePendingOp,
+						uploading: false,
+						opsent: true,
 						handleP: new Deferred(),
 						storageId,
 						uploadP: undefined,
@@ -275,11 +259,12 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 			}
 			this.pendingBlobs.set(localId, {
 				blob,
-				status: PendingBlobStatus.OfflinePendingUpload,
+				uploading: true,
 				handleP: new Deferred(),
 				uploadP: this.uploadBlob(localId, blob),
 				attached,
 				acked,
+				opsent: true,
 			});
 		});
 
@@ -289,13 +274,11 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 				pendingEntry !== undefined,
 				0x725 /* Must have pending blob entry for upcoming op */,
 			);
-			pendingEntry.opsent = true;
 			if (pendingEntry?.uploadTime && pendingEntry?.minTTLInSeconds) {
 				const secondsSinceUpload = (Date.now() - pendingEntry.uploadTime) / 1000;
 				const expired = pendingEntry.minTTLInSeconds - secondsSinceUpload < 0;
 				this.mc.logger.sendTelemetryEvent({
 					eventName: "sendBlobAttach",
-					entryStatus: pendingEntry.status,
 					secondsSinceUpload,
 					minTTLInSeconds: pendingEntry.minTTLInSeconds,
 					expired,
@@ -309,27 +292,17 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 							{
 								localId,
 								blobId,
-								entryStatus: pendingEntry.status,
 								secondsSinceUpload,
 							},
 						),
 					);
 				}
 			}
+			pendingEntry.opsent = true;
 			return sendBlobAttachOp(localId, blobId);
 		};
 	}
 
-	private get pendingOfflineUploads() {
-		return Array.from(this.pendingBlobs.values()).filter(
-			(e) => e.status === PendingBlobStatus.OfflinePendingUpload,
-		);
-	}
-
-	public get hasPendingOfflineUploads(): boolean {
-		return this.pendingOfflineUploads.length > 0;
-	}
-
 	public get allBlobsAttached(): boolean {
 		for (const [, entry] of this.pendingBlobs) {
 			if (entry.attached === false) {
@@ -349,20 +322,21 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 	private createAbortError(pending?: PendingBlob) {
 		return new LoggingError("uploadBlob aborted", {
 			acked: pending?.acked,
-			status: pending?.status,
 			uploadTime: pending?.uploadTime,
 		});
 	}
 	/**
 	 * Upload blobs added while offline. This must be completed before connecting and resubmitting ops.
 	 */
-	public async onConnected() {
+	public async processStashedChanges() {
 		this.retryThrottler.cancel();
-		const pendingUploads = this.pendingOfflineUploads.map(async (e) => e.uploadP);
+		const pendingUploads = Array.from(this.pendingBlobs.values())
+			.filter((e) => e.uploading === true)
+			.map(async (e) => e.uploadP);
 		await PerformanceEvent.timedExecAsync(
 			this.mc.logger,
 			{
-				eventName: "BlobUploadOnConnected",
+				eventName: "BlobUploadProcessStashedChanges",
 				count: pendingUploads.length,
 			},
 			async () => Promise.all(pendingUploads),
@@ -370,42 +344,6 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 		);
 	}
 
-	/**
-	 * Transition online blobs waiting for BlobAttach op round-trip since we will not see the op until we are connected
-	 * again
-	 */
-	private onDisconnected() {
-		for (const [localId, entry] of this.pendingBlobs) {
-			if (entry.status === PendingBlobStatus.OnlinePendingOp) {
-				// This will submit another BlobAttach op for this blob. This is necessary because the one we sent
-				// already didn't have the local ID.
-				this.transitionToOffline(localId);
-			}
-		}
-	}
-
-	private async shutdownPendingBlobs(): Promise<void> {
-		for (const [localId, entry] of this.pendingBlobs) {
-			if (entry.status === PendingBlobStatus.OnlinePendingUpload) {
-				this.sendBlobAttachOp(localId, entry.storageId);
-				entry.status = PendingBlobStatus.OfflinePendingUpload;
-				entry.handleP.resolve(this.getBlobHandle(localId));
-			} else if (entry.status === PendingBlobStatus.OnlinePendingOp) {
-				entry.status = PendingBlobStatus.OfflinePendingOp;
-				entry.handleP.resolve(this.getBlobHandle(localId));
-			}
-		}
-		return new Promise<void>((resolve) => {
-			if (this.allBlobsAttached) {
-				resolve();
-			} else {
-				this.once("allBlobsAttached", () => {
-					resolve();
-				});
-			}
-		});
-	}
-
 	/**
 	 * Set of actual storage IDs (i.e., IDs that can be requested from storage). This will be empty if the container is
 	 * detached or there are no (non-pending) attachment blobs in the document
@@ -471,10 +409,8 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 		const callback = pending
 			? () => {
 					pending.attached = true;
+					this.emit("blobAttached", pending);
 					this.deletePendingBlobMaybe(id);
-					if (this.allBlobsAttached) {
-						this.emit("allBlobsAttached");
-					}
 			  }
 			: undefined;
 		return new BlobHandle(
@@ -521,7 +457,7 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 		const localId = uuid();
 		const pendingEntry: PendingBlob = {
 			blob,
-			status: PendingBlobStatus.OnlinePendingUpload,
+			uploading: true,
 			handleP: new Deferred(),
 			uploadP: this.uploadBlob(localId, blob),
 			attached: false,
@@ -589,50 +525,37 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 			return;
 		}
 		assert(
-			entry.status === PendingBlobStatus.OnlinePendingUpload ||
-				entry.status === PendingBlobStatus.OfflinePendingUpload,
+			entry.uploading === true,
 			0x386 /* Must have pending blob entry for uploaded blob */,
 		);
 		entry.storageId = response.id;
 		entry.uploadTime = Date.now();
 		entry.minTTLInSeconds = response.minTTLInSeconds;
-		if (this.runtime.connected) {
-			if (entry.status === PendingBlobStatus.OnlinePendingUpload) {
-				// Send a blob attach op. This serves two purposes:
-				// 1. If its a new blob, i.e., it isn't de-duped, the server will keep the blob alive if it sees this op
-				//    until its storage ID is added to the next summary.
-				// 2. It will create a local ID to storage ID mapping in all clients which is needed to retrieve the
-				//    blob from the server via the storage ID.
-				this.sendBlobAttachOp(localId, response.id);
-				if (this.storageIds.has(response.id)) {
-					// The blob is de-duped. Set up a local ID to storage ID mapping and return the blob. Since this is
-					// an existing blob, we don't have to wait for the op to be ack'd since this step has already
-					// happened before and so, the server won't delete it.
-					this.setRedirection(localId, response.id);
-					entry.handleP.resolve(this.getBlobHandle(localId));
-					this.deletePendingBlobMaybe(localId);
-				} else {
-					// If there is already an op for this storage ID, append the local ID to the list. Once any op for
-					// this storage ID is ack'd, all pending blobs for it can be resolved since the op will keep the
-					// blob alive in storage.
-					this.opsInFlight.set(
-						response.id,
-						(this.opsInFlight.get(response.id) ?? []).concat(localId),
-					);
-					entry.status = PendingBlobStatus.OnlinePendingOp;
-				}
-			} else if (entry.status === PendingBlobStatus.OfflinePendingUpload) {
-				// We already submitted a BlobAttach op for this blob when it was transitioned to offline flow
-				entry.status = PendingBlobStatus.OfflinePendingOp;
-			}
+		// Send a blob attach op. This serves two purposes:
+		// 1. If its a new blob, i.e., it isn't de-duped, the server will keep the blob alive if it sees this op
+		//    until its storage ID is added to the next summary.
+		// 2. It will create a local ID to storage ID mapping in all clients which is needed to retrieve the
+		//    blob from the server via the storage ID.
+		if (!entry.opsent) {
+			this.sendBlobAttachOp(localId, response.id);
+		}
+		if (this.storageIds.has(response.id)) {
+			// The blob is de-duped. Set up a local ID to storage ID mapping and return the blob. Since this is
+			// an existing blob, we don't have to wait for the op to be ack'd since this step has already
+			// happened before and so, the server won't delete it.
+			this.setRedirection(localId, response.id);
+			entry.handleP.resolve(this.getBlobHandle(localId));
+			this.deletePendingBlobMaybe(localId);
 		} else {
-			// connected to storage but not ordering service?
-			this.mc.logger.sendTelemetryEvent({ eventName: "BlobUploadSuccessWhileDisconnected" });
-			if (entry.status === PendingBlobStatus.OnlinePendingUpload) {
-				this.transitionToOffline(localId);
-			}
-			entry.status = PendingBlobStatus.OfflinePendingOp;
+			// If there is already an op for this storage ID, append the local ID to the list. Once any op for
+			// this storage ID is ack'd, all pending blobs for it can be resolved since the op will keep the
+			// blob alive in storage.
+			this.opsInFlight.set(
+				response.id,
+				(this.opsInFlight.get(response.id) ?? []).concat(localId),
+			);
 		}
+
 		return response;
 	}
 
@@ -644,9 +567,6 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 			return;
 		}
 		if (!this.runtime.connected) {
-			if (entry.status === PendingBlobStatus.OnlinePendingUpload) {
-				this.transitionToOffline(localId);
-			}
 			// we are probably not connected to storage but start another upload request in case we are
 			entry.uploadP = this.retryThrottler
 				.getDelay()
@@ -658,39 +578,6 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 		}
 	}
 
-	private transitionToOffline(localId: string) {
-		assert(
-			!this.runtime.connected,
-			0x388 /* Must only transition to offline flow while runtime is disconnected */,
-		);
-		const entry = this.pendingBlobs.get(localId);
-		assert(!!entry, 0x389 /* No pending blob entry */);
-		assert(
-			[PendingBlobStatus.OnlinePendingUpload, PendingBlobStatus.OnlinePendingOp].includes(
-				entry.status,
-			),
-			0x38a /* Blob must be in online flow to transition to offline flow */,
-		);
-
-		/**
-		 * If we haven't already submitted a BlobAttach op for this entry, send it before returning the blob handle.
-		 * This will make sure that the BlobAttach op is sequenced prior to any ops referencing the handle. Otherwise,
-		 * an invalid handle could be added to the document.
-		 * storageId may be undefined but since we are not connected we will have a chance to add it when reSubmit()
-		 * is called on reconnection.
-		 */
-		if (entry.status !== PendingBlobStatus.OnlinePendingOp) {
-			this.sendBlobAttachOp(localId, entry.storageId);
-		}
-
-		entry.status =
-			entry.status === PendingBlobStatus.OnlinePendingUpload
-				? PendingBlobStatus.OfflinePendingUpload
-				: PendingBlobStatus.OfflinePendingOp;
-
-		entry.handleP.resolve(this.getBlobHandle(localId));
-	}
-
 	/**
 	 * Resubmit a BlobAttach op. Used to add storage IDs to ops that were
 	 * submitted to runtime while disconnected.
@@ -705,8 +592,7 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 		if (!blobId) {
 			// We submitted this op while offline. The blob should have been uploaded by now.
 			assert(
-				pendingEntry?.status === PendingBlobStatus.OfflinePendingOp &&
-					!!pendingEntry?.storageId,
+				pendingEntry?.opsent === true && !!pendingEntry?.storageId,
 				0x38d /* blob must be uploaded before resubmitting BlobAttach op */,
 			);
 			return this.sendBlobAttachOp(localId, pendingEntry?.storageId);
@@ -750,22 +636,17 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 						entry !== undefined,
 						0x38f /* local online BlobAttach op with no pending blob entry */,
 					);
-
-					// It's possible we transitioned to offline flow while waiting for this op.
-					if (entry.status === PendingBlobStatus.OnlinePendingOp) {
-						this.setRedirection(pendingLocalId, blobId);
-						entry.acked = true;
-						entry.handleP.resolve(this.getBlobHandle(pendingLocalId));
-						this.deletePendingBlobMaybe(pendingLocalId);
-					}
+					this.setRedirection(pendingLocalId, blobId);
+					entry.acked = true;
+					entry.handleP.resolve(this.getBlobHandle(pendingLocalId));
+					this.deletePendingBlobMaybe(pendingLocalId);
 				});
 				this.opsInFlight.delete(blobId);
 			}
-			// offline flow does not resolve the handle (since it was already resolved)
-			// but we still need to delete the entry in case is acked and attached.
 			const localEntry = this.pendingBlobs.get(localId);
 			if (localEntry) {
 				localEntry.acked = true;
+				localEntry.handleP.resolve(this.getBlobHandle(localId));
 				this.deletePendingBlobMaybe(localId);
 			}
 		}
@@ -1037,22 +918,61 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
 		}
 	}
 
-	public async getPendingBlobs(waitBlobsToAttach?: boolean): Promise<IPendingBlobs> {
-		if (waitBlobsToAttach) {
-			await this.shutdownPendingBlobs();
-		}
-		const blobs = {};
-		for (const [key, entry] of this.pendingBlobs) {
-			blobs[key] = {
-				blob: bufferToString(entry.blob, "base64"),
-				storageId: entry.storageId,
-				attached: entry.attached,
-				acked: entry.acked,
-				minTTLInSeconds: entry.minTTLInSeconds,
-				uploadTime: entry.uploadTime,
-			};
-		}
-		return blobs;
+	public async getPendingBlobs(waitBlobsToAttach?: boolean): Promise<IPendingBlobs | undefined> {
+		return PerformanceEvent.timedExecAsync(
+			this.mc.logger,
+			{ eventName: "GetPendingBlobs" },
+			async () => {
+				if (this.pendingBlobs.size === 0) {
+					return;
+				}
+				const blobs = {};
+				const localBlobs = new Set<PendingBlob>();
+				while (localBlobs.size < this.pendingBlobs.size) {
+					const attachBlobsP: Promise<void>[] = [];
+					for (const [id, entry] of this.pendingBlobs) {
+						if (!localBlobs.has(entry)) {
+							localBlobs.add(entry);
+							if (waitBlobsToAttach) {
+								if (!entry.opsent) {
+									this.sendBlobAttachOp(id, entry.storageId);
+								}
+								entry.handleP.resolve(this.getBlobHandle(id));
+								attachBlobsP.push(
+									new Promise<void>((resolve) => {
+										const onBlobAttached = (attachedEntry) => {
+											if (attachedEntry === entry) {
+												this.off("blobAttached", onBlobAttached);
+												resolve();
+											}
+										};
+										if (!entry.attached) {
+											this.on("blobAttached", onBlobAttached);
+										} else {
+											resolve();
+										}
+									}),
+								);
+							}
+						}
+					}
+					await Promise.all(attachBlobsP);
+				}
+				// another for is needed to correctly mark attach state
+				// future optimization won't add unattached blobs to the list
+				for (const [id, entry] of this.pendingBlobs) {
+					blobs[id] = {
+						blob: bufferToString(entry.blob, "base64"),
+						storageId: entry.storageId,
+						attached: entry.attached,
+						acked: entry.acked,
+						minTTLInSeconds: entry.minTTLInSeconds,
+						uploadTime: entry.uploadTime,
+					};
+				}
+				return blobs;
+			},
+		);
 	}
 }
 

package/src/connectionTelemetry.ts

@@ -14,7 +14,8 @@ import {
 	ISequencedDocumentMessage,
 	MessageType,
 } from "@fluidframework/protocol-definitions";
-import { assert, performance } from "@fluidframework/common-utils";
+import { assert } from "@fluidframework/core-utils";
+import { performance } from "@fluid-internal/client-utils";
 
 /**
  * We report various latency-related errors when waiting for op roundtrip takes longer than that amout of time.