@fluidframework/container-runtime 2.0.0-internal.6.1.1 → 2.0.0-internal.6.3.0

package/lib/blobManager.js

@@ -4,10 +4,10 @@
  */
 import { v4 as uuid } from "uuid";
 import { createResponseError, generateHandleContextPath, responseToException, SummaryTreeBuilder, } from "@fluidframework/runtime-utils";
-import { assert, bufferToString, Deferred, stringToBuffer, TypedEventEmitter, } from "@fluidframework/common-utils";
+import { assert, Deferred } from "@fluidframework/core-utils";
+import { bufferToString, stringToBuffer, TypedEventEmitter } from "@fluid-internal/client-utils";
 import { AttachState } from "@fluidframework/container-definitions";
-import { LoggingError, createChildMonitoringContext, PerformanceEvent, } from "@fluidframework/telemetry-utils";
-import { GenericError } from "@fluidframework/container-utils";
+import { createChildMonitoringContext, GenericError, LoggingError, PerformanceEvent, } from "@fluidframework/telemetry-utils";
 import { TombstoneResponseHeaderKey } from "./containerRuntime";
 import { sendGCUnexpectedUsageEvent, sweepAttachmentBlobsKey, throwOnTombstoneLoadKey } from "./gc";
 import { Throttler, formExponentialFn } from "./throttler";
@@ -60,15 +60,6 @@ class CancellableThrottler {
         this.cancelP = new Deferred();
     }
 }
-// Note that while offline we "submit" an op before uploading the blob, but we always
-// expect blobs to be uploaded before we actually see the op round-trip
-export var PendingBlobStatus;
-(function (PendingBlobStatus) {
-    PendingBlobStatus[PendingBlobStatus["OnlinePendingUpload"] = 0] = "OnlinePendingUpload";
-    PendingBlobStatus[PendingBlobStatus["OnlinePendingOp"] = 1] = "OnlinePendingOp";
-    PendingBlobStatus[PendingBlobStatus["OfflinePendingUpload"] = 2] = "OfflinePendingUpload";
-    PendingBlobStatus[PendingBlobStatus["OfflinePendingOp"] = 3] = "OfflinePendingOp";
-})(PendingBlobStatus || (PendingBlobStatus = {}));
 export class BlobManager extends TypedEventEmitter {
     constructor(routeContext, snapshot, getStorage, 
     /**
@@ -96,9 +87,7 @@ export class BlobManager extends TypedEventEmitter {
         this.runtime = runtime;
         this.closeContainer = closeContainer;
         /**
-         * Blobs which have not been uploaded or for which we have not yet seen a BlobAttach op round-trip.
-         * Until we see the op round-trip, there is a possibility we may need to re-upload the blob, so
-         * we must save it. This is true for both the online and offline flow.
+         * Blobs which we have not yet seen a BlobAttach op round-trip and not yet attached to a DDS.
          */
         this.pendingBlobs = new Map();
         /**
@@ -125,7 +114,6 @@ export class BlobManager extends TypedEventEmitter {
             this.mc.config.getBoolean(throwOnTombstoneLoadKey) === true &&
                 this.runtime.gcTombstoneEnforcementAllowed &&
                 this.runtime.clientDetails.type !== summarizerClientType;
-        this.runtime.on("disconnected", () => this.onDisconnected());
         this.redirectTable = this.load(snapshot);
         // Begin uploading stashed blobs from previous container instance
         Object.entries(stashedBlobs).forEach(([localId, entry]) => {
@@ -139,7 +127,8 @@ export class BlobManager extends TypedEventEmitter {
                 if (entry.minTTLInSeconds - timeLapseSinceLocalUpload > entry.minTTLInSeconds / 2) {
                     this.pendingBlobs.set(localId, {
                         blob,
-                        status: PendingBlobStatus.OfflinePendingOp,
+                        uploading: false,
+                        opsent: true,
                         handleP: new Deferred(),
                         storageId,
                         uploadP: undefined,
@@ -153,23 +142,22 @@ export class BlobManager extends TypedEventEmitter {
             }
             this.pendingBlobs.set(localId, {
                 blob,
-                status: PendingBlobStatus.OfflinePendingUpload,
+                uploading: true,
                 handleP: new Deferred(),
                 uploadP: this.uploadBlob(localId, blob),
                 attached,
                 acked,
+                opsent: true,
             });
         });
         this.sendBlobAttachOp = (localId, blobId) => {
             const pendingEntry = this.pendingBlobs.get(localId);
             assert(pendingEntry !== undefined, 0x725 /* Must have pending blob entry for upcoming op */);
-            pendingEntry.opsent = true;
             if (pendingEntry?.uploadTime && pendingEntry?.minTTLInSeconds) {
                 const secondsSinceUpload = (Date.now() - pendingEntry.uploadTime) / 1000;
                 const expired = pendingEntry.minTTLInSeconds - secondsSinceUpload < 0;
                 this.mc.logger.sendTelemetryEvent({
                     eventName: "sendBlobAttach",
-                    entryStatus: pendingEntry.status,
                     secondsSinceUpload,
                     minTTLInSeconds: pendingEntry.minTTLInSeconds,
                     expired,
@@ -179,20 +167,14 @@ export class BlobManager extends TypedEventEmitter {
                     this.closeContainer(new GenericError("Trying to submit a BlobAttach for expired blob", undefined, {
                         localId,
                         blobId,
-                        entryStatus: pendingEntry.status,
                         secondsSinceUpload,
                     }));
                 }
             }
+            pendingEntry.opsent = true;
             return sendBlobAttachOp(localId, blobId);
         };
     }
-    get pendingOfflineUploads() {
-        return Array.from(this.pendingBlobs.values()).filter((e) => e.status === PendingBlobStatus.OfflinePendingUpload);
-    }
-    get hasPendingOfflineUploads() {
-        return this.pendingOfflineUploads.length > 0;
-    }
     get allBlobsAttached() {
         for (const [, entry] of this.pendingBlobs) {
             if (entry.attached === false) {
@@ -208,57 +190,22 @@ export class BlobManager extends TypedEventEmitter {
     createAbortError(pending) {
         return new LoggingError("uploadBlob aborted", {
             acked: pending?.acked,
-            status: pending?.status,
             uploadTime: pending?.uploadTime,
         });
     }
     /**
      * Upload blobs added while offline. This must be completed before connecting and resubmitting ops.
      */
-    async onConnected() {
+    async processStashedChanges() {
         this.retryThrottler.cancel();
-        const pendingUploads = this.pendingOfflineUploads.map(async (e) => e.uploadP);
+        const pendingUploads = Array.from(this.pendingBlobs.values())
+            .filter((e) => e.uploading === true)
+            .map(async (e) => e.uploadP);
         await PerformanceEvent.timedExecAsync(this.mc.logger, {
-            eventName: "BlobUploadOnConnected",
+            eventName: "BlobUploadProcessStashedChanges",
             count: pendingUploads.length,
         }, async () => Promise.all(pendingUploads), { start: true, end: true });
     }
-    /**
-     * Transition online blobs waiting for BlobAttach op round-trip since we will not see the op until we are connected
-     * again
-     */
-    onDisconnected() {
-        for (const [localId, entry] of this.pendingBlobs) {
-            if (entry.status === PendingBlobStatus.OnlinePendingOp) {
-                // This will submit another BlobAttach op for this blob. This is necessary because the one we sent
-                // already didn't have the local ID.
-                this.transitionToOffline(localId);
-            }
-        }
-    }
-    async shutdownPendingBlobs() {
-        for (const [localId, entry] of this.pendingBlobs) {
-            if (entry.status === PendingBlobStatus.OnlinePendingUpload) {
-                this.sendBlobAttachOp(localId, entry.storageId);
-                entry.status = PendingBlobStatus.OfflinePendingUpload;
-                entry.handleP.resolve(this.getBlobHandle(localId));
-            }
-            else if (entry.status === PendingBlobStatus.OnlinePendingOp) {
-                entry.status = PendingBlobStatus.OfflinePendingOp;
-                entry.handleP.resolve(this.getBlobHandle(localId));
-            }
-        }
-        return new Promise((resolve) => {
-            if (this.allBlobsAttached) {
-                resolve();
-            }
-            else {
-                this.once("allBlobsAttached", () => {
-                    resolve();
-                });
-            }
-        });
-    }
     /**
      * Set of actual storage IDs (i.e., IDs that can be requested from storage). This will be empty if the container is
      * detached or there are no (non-pending) attachment blobs in the document
@@ -305,10 +252,8 @@ export class BlobManager extends TypedEventEmitter {
         const callback = pending
             ? () => {
                 pending.attached = true;
+                this.emit("blobAttached", pending);
                 this.deletePendingBlobMaybe(id);
-                if (this.allBlobsAttached) {
-                    this.emit("allBlobsAttached");
-                }
             }
             : undefined;
         return new BlobHandle(`${BlobManager.basePath}/${id}`, this.routeContext, async () => this.getBlob(id), callback);
@@ -338,7 +283,7 @@ export class BlobManager extends TypedEventEmitter {
         const localId = uuid();
         const pendingEntry = {
             blob,
-            status: PendingBlobStatus.OnlinePendingUpload,
+            uploading: true,
             handleP: new Deferred(),
             uploadP: this.uploadBlob(localId, blob),
             attached: false,
@@ -387,47 +332,31 @@ export class BlobManager extends TypedEventEmitter {
             this.deletePendingBlob(localId);
             return;
         }
-        assert(entry.status === PendingBlobStatus.OnlinePendingUpload ||
-            entry.status === PendingBlobStatus.OfflinePendingUpload, 0x386 /* Must have pending blob entry for uploaded blob */);
+        assert(entry.uploading === true, 0x386 /* Must have pending blob entry for uploaded blob */);
         entry.storageId = response.id;
         entry.uploadTime = Date.now();
         entry.minTTLInSeconds = response.minTTLInSeconds;
-        if (this.runtime.connected) {
-            if (entry.status === PendingBlobStatus.OnlinePendingUpload) {
-                // Send a blob attach op. This serves two purposes:
-                // 1. If its a new blob, i.e., it isn't de-duped, the server will keep the blob alive if it sees this op
-                //    until its storage ID is added to the next summary.
-                // 2. It will create a local ID to storage ID mapping in all clients which is needed to retrieve the
-                //    blob from the server via the storage ID.
-                this.sendBlobAttachOp(localId, response.id);
-                if (this.storageIds.has(response.id)) {
-                    // The blob is de-duped. Set up a local ID to storage ID mapping and return the blob. Since this is
-                    // an existing blob, we don't have to wait for the op to be ack'd since this step has already
-                    // happened before and so, the server won't delete it.
-                    this.setRedirection(localId, response.id);
-                    entry.handleP.resolve(this.getBlobHandle(localId));
-                    this.deletePendingBlobMaybe(localId);
-                }
-                else {
-                    // If there is already an op for this storage ID, append the local ID to the list. Once any op for
-                    // this storage ID is ack'd, all pending blobs for it can be resolved since the op will keep the
-                    // blob alive in storage.
-                    this.opsInFlight.set(response.id, (this.opsInFlight.get(response.id) ?? []).concat(localId));
-                    entry.status = PendingBlobStatus.OnlinePendingOp;
-                }
-            }
-            else if (entry.status === PendingBlobStatus.OfflinePendingUpload) {
-                // We already submitted a BlobAttach op for this blob when it was transitioned to offline flow
-                entry.status = PendingBlobStatus.OfflinePendingOp;
-            }
+        // Send a blob attach op. This serves two purposes:
+        // 1. If its a new blob, i.e., it isn't de-duped, the server will keep the blob alive if it sees this op
+        //    until its storage ID is added to the next summary.
+        // 2. It will create a local ID to storage ID mapping in all clients which is needed to retrieve the
+        //    blob from the server via the storage ID.
+        if (!entry.opsent) {
+            this.sendBlobAttachOp(localId, response.id);
+        }
+        if (this.storageIds.has(response.id)) {
+            // The blob is de-duped. Set up a local ID to storage ID mapping and return the blob. Since this is
+            // an existing blob, we don't have to wait for the op to be ack'd since this step has already
+            // happened before and so, the server won't delete it.
+            this.setRedirection(localId, response.id);
+            entry.handleP.resolve(this.getBlobHandle(localId));
+            this.deletePendingBlobMaybe(localId);
         }
         else {
-            // connected to storage but not ordering service?
-            this.mc.logger.sendTelemetryEvent({ eventName: "BlobUploadSuccessWhileDisconnected" });
-            if (entry.status === PendingBlobStatus.OnlinePendingUpload) {
-                this.transitionToOffline(localId);
-            }
-            entry.status = PendingBlobStatus.OfflinePendingOp;
+            // If there is already an op for this storage ID, append the local ID to the list. Once any op for
+            // this storage ID is ack'd, all pending blobs for it can be resolved since the op will keep the
+            // blob alive in storage.
+            this.opsInFlight.set(response.id, (this.opsInFlight.get(response.id) ?? []).concat(localId));
         }
         return response;
     }
@@ -439,9 +368,6 @@ export class BlobManager extends TypedEventEmitter {
             return;
         }
         if (!this.runtime.connected) {
-            if (entry.status === PendingBlobStatus.OnlinePendingUpload) {
-                this.transitionToOffline(localId);
-            }
             // we are probably not connected to storage but start another upload request in case we are
             entry.uploadP = this.retryThrottler
                 .getDelay()
@@ -453,27 +379,6 @@ export class BlobManager extends TypedEventEmitter {
             throw error;
         }
     }
-    transitionToOffline(localId) {
-        assert(!this.runtime.connected, 0x388 /* Must only transition to offline flow while runtime is disconnected */);
-        const entry = this.pendingBlobs.get(localId);
-        assert(!!entry, 0x389 /* No pending blob entry */);
-        assert([PendingBlobStatus.OnlinePendingUpload, PendingBlobStatus.OnlinePendingOp].includes(entry.status), 0x38a /* Blob must be in online flow to transition to offline flow */);
-        /**
-         * If we haven't already submitted a BlobAttach op for this entry, send it before returning the blob handle.
-         * This will make sure that the BlobAttach op is sequenced prior to any ops referencing the handle. Otherwise,
-         * an invalid handle could be added to the document.
-         * storageId may be undefined but since we are not connected we will have a chance to add it when reSubmit()
-         * is called on reconnection.
-         */
-        if (entry.status !== PendingBlobStatus.OnlinePendingOp) {
-            this.sendBlobAttachOp(localId, entry.storageId);
-        }
-        entry.status =
-            entry.status === PendingBlobStatus.OnlinePendingUpload
-                ? PendingBlobStatus.OfflinePendingUpload
-                : PendingBlobStatus.OfflinePendingOp;
-        entry.handleP.resolve(this.getBlobHandle(localId));
-    }
     /**
      * Resubmit a BlobAttach op. Used to add storage IDs to ops that were
      * submitted to runtime while disconnected.
@@ -486,8 +391,7 @@ export class BlobManager extends TypedEventEmitter {
         const pendingEntry = this.pendingBlobs.get(localId);
         if (!blobId) {
             // We submitted this op while offline. The blob should have been uploaded by now.
-            assert(pendingEntry?.status === PendingBlobStatus.OfflinePendingOp &&
-                !!pendingEntry?.storageId, 0x38d /* blob must be uploaded before resubmitting BlobAttach op */);
+            assert(pendingEntry?.opsent === true && !!pendingEntry?.storageId, 0x38d /* blob must be uploaded before resubmitting BlobAttach op */);
             return this.sendBlobAttachOp(localId, pendingEntry?.storageId);
         }
         return this.sendBlobAttachOp(localId, blobId);
@@ -522,21 +426,17 @@ export class BlobManager extends TypedEventEmitter {
                 waitingBlobs.forEach((pendingLocalId) => {
                     const entry = this.pendingBlobs.get(pendingLocalId);
                     assert(entry !== undefined, 0x38f /* local online BlobAttach op with no pending blob entry */);
-                    // It's possible we transitioned to offline flow while waiting for this op.
-                    if (entry.status === PendingBlobStatus.OnlinePendingOp) {
-                        this.setRedirection(pendingLocalId, blobId);
-                        entry.acked = true;
-                        entry.handleP.resolve(this.getBlobHandle(pendingLocalId));
-                        this.deletePendingBlobMaybe(pendingLocalId);
-                    }
+                    this.setRedirection(pendingLocalId, blobId);
+                    entry.acked = true;
+                    entry.handleP.resolve(this.getBlobHandle(pendingLocalId));
+                    this.deletePendingBlobMaybe(pendingLocalId);
                 });
                 this.opsInFlight.delete(blobId);
             }
-            // offline flow does not resolve the handle (since it was already resolved)
-            // but we still need to delete the entry in case is acked and attached.
             const localEntry = this.pendingBlobs.get(localId);
             if (localEntry) {
                 localEntry.acked = true;
+                localEntry.handleP.resolve(this.getBlobHandle(localId));
                 this.deletePendingBlobMaybe(localId);
             }
         }
@@ -756,21 +656,55 @@ export class BlobManager extends TypedEventEmitter {
         }
     }
     async getPendingBlobs(waitBlobsToAttach) {
-        if (waitBlobsToAttach) {
-            await this.shutdownPendingBlobs();
-        }
-        const blobs = {};
-        for (const [key, entry] of this.pendingBlobs) {
-            blobs[key] = {
-                blob: bufferToString(entry.blob, "base64"),
-                storageId: entry.storageId,
-                attached: entry.attached,
-                acked: entry.acked,
-                minTTLInSeconds: entry.minTTLInSeconds,
-                uploadTime: entry.uploadTime,
-            };
-        }
-        return blobs;
+        return PerformanceEvent.timedExecAsync(this.mc.logger, { eventName: "GetPendingBlobs" }, async () => {
+            if (this.pendingBlobs.size === 0) {
+                return;
+            }
+            const blobs = {};
+            const localBlobs = new Set();
+            while (localBlobs.size < this.pendingBlobs.size) {
+                const attachBlobsP = [];
+                for (const [id, entry] of this.pendingBlobs) {
+                    if (!localBlobs.has(entry)) {
+                        localBlobs.add(entry);
+                        if (waitBlobsToAttach) {
+                            if (!entry.opsent) {
+                                this.sendBlobAttachOp(id, entry.storageId);
+                            }
+                            entry.handleP.resolve(this.getBlobHandle(id));
+                            attachBlobsP.push(new Promise((resolve) => {
+                                const onBlobAttached = (attachedEntry) => {
+                                    if (attachedEntry === entry) {
+                                        this.off("blobAttached", onBlobAttached);
+                                        resolve();
+                                    }
+                                };
+                                if (!entry.attached) {
+                                    this.on("blobAttached", onBlobAttached);
+                                }
+                                else {
+                                    resolve();
+                                }
+                            }));
+                        }
+                    }
+                }
+                await Promise.all(attachBlobsP);
+            }
+            // another for is needed to correctly mark attach state
+            // future optimization won't add unattached blobs to the list
+            for (const [id, entry] of this.pendingBlobs) {
+                blobs[id] = {
+                    blob: bufferToString(entry.blob, "base64"),
+                    storageId: entry.storageId,
+                    attached: entry.attached,
+                    acked: entry.acked,
+                    minTTLInSeconds: entry.minTTLInSeconds,
+                    uploadTime: entry.uploadTime,
+                };
+            }
+            return blobs;
+        });
     }
 }
 BlobManager.basePath = "_blobs";