@fluidframework/container-runtime 2.0.0-internal.2.2.1 → 2.0.0-internal.2.3.1

package/src/blobManager.ts

@@ -24,7 +24,8 @@ import {
 } from "@fluidframework/runtime-definitions";
 import { Throttler, formExponentialFn, IThrottler } from "./throttler";
 import { summarizerClientType } from "./summarizerClientElection";
-import { throwOnTombstoneUsageKey } from "./garbageCollectionConstants";
+import { throwOnTombstoneLoadKey } from "./garbageCollectionConstants";
+import { sendGCTombstoneEvent } from "./garbageCollectionTombstoneUtils";
 
 /**
  * This class represents blob (long string)
@@ -110,16 +111,21 @@ interface PendingBlob {
 
 export interface IPendingBlobs { [id: string]: { blob: string; }; }
 
-export class BlobManager {
+export interface IBlobManagerEvents {
+    (event: "noPendingBlobs", listener: () => void);
+}
+
+export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
     public static readonly basePath = "_blobs";
     private static readonly redirectTableBlobName = ".redirectTable";
     private readonly mc: MonitoringContext;
 
     /**
-     * Map of local (offline/detached) IDs to storage IDs. Contains identity entries
-     * (id → id) for storage IDs, so all requested IDs should be a key in this map.
-     * Blobs created while the container is detached are stored in IDetachedBlobStorage
-     * which gives local IDs; the storage IDs are filled in at attach time.
+     * Map of local IDs to storage IDs. Contains identity entries (id → id) for storage IDs. All requested IDs should
+     * be a key in this map. Blobs created while the container is detached are stored in IDetachedBlobStorage which
+     * gives local IDs; the storage IDs are filled in at attach time.
+     * Note: It contains mappings from all clients, i.e., from remote clients as well. local ID comes from the client
+     * that uploaded the blob but its mapping to storage ID is needed in all clients in order to retrieve the blob.
      */
     private readonly redirectTable: Map<string, string | undefined>;
 
@@ -131,8 +137,9 @@ export class BlobManager {
     private readonly pendingBlobs: Map<string, PendingBlob> = new Map();
 
     /**
-     * Track ops in flight for online flow. Used to avoid searching pendingBlobs since BlobAttach ops
-     * don't include local ID in online flow.
+     * Track ops in flight for online flow. This is used for optimizations where if we receive an ack for a storage ID,
+     * we can resolve all pending blobs with the same storage ID even though they may have different local IDs. That's
+     * because we know that the server will not delete the blob corresponding to that storage ID.
      */
     private readonly opsInFlight: Map<string, string[]> = new Map();
 
@@ -144,9 +151,9 @@ export class BlobManager {
     ));
 
     /** If true, throw an error when a tombstone attachment blob is retrieved. */
-    private readonly throwOnTombstoneUsage: boolean;
+    private readonly throwOnTombstoneLoad: boolean;
     /**
-     * This stores ides of tombstoned blobs.
+     * This stores IDs of tombstoned blobs.
      * Tombstone is a temporary feature that imitates a blob getting swept by garbage collection.
      */
     private readonly tombstonedBlobs: Set<string> = new Set();
@@ -156,26 +163,28 @@ export class BlobManager {
         snapshot: IBlobManagerLoadInfo,
         private readonly getStorage: () => IDocumentStorageService,
         /**
-         * Submit a BlobAttach op. When a blob is uploaded, there is a short grace period before which
-         * the blob is deleted. The BlobAttach op notifies the server that blob is in use. The server
-         * will then not delete the blob as long as it is listed as referenced in future summaries.
-         * The summarizing client will know to include the storage ID in the summary when it sees the op.
+         * Submit a BlobAttach op. When a blob is uploaded, there is a short grace period before which the blob is
+         * deleted. The BlobAttach op notifies the server that blob is in use. The server will then not delete the
+         * the blob as long as it is listed as referenced in future summaries. The summarizing client will know to
+         * include the storage ID in the summary when it sees the op.
          *
-         * The op may also include a local ID to inform all clients of the relation to the storage
-         * ID, without knowledge of which they cannot request the blob from storage. This is also
-         * included in the redirect table in the summary.
+         * The op will also include a local ID to inform all clients of the relation to the storage ID, without
+         * knowledge of which they cannot request the blob from storage. It's important that this op is sequenced
+         * before any ops that reference the local ID, otherwise, an invalid handle could be added to the document.
          */
-        private readonly sendBlobAttachOp: (storageId?: string, localId?: string) => void,
+        private readonly sendBlobAttachOp: (localId: string, storageId?: string) => void,
         // To be called when a blob node is requested. blobPath is the path of the blob's node in GC's graph. It's
         // of the format `/<BlobManager.basePath>/<blobId>`.
-        private readonly gcNodeUpdated: (blobPath: string) => void,
+        private readonly blobRequested: (blobPath: string) => void,
+        private readonly addedBlobReference: (fromNodePath: string, toNodePath: string) => void,
         private readonly runtime: IBlobManagerRuntime,
         stashedBlobs: IPendingBlobs = {},
     ) {
+        super();
         this.mc = loggerToMonitoringContext(ChildLogger.create(this.runtime.logger, "BlobManager"));
         // Read the feature flag that tells whether to throw when a tombstone blob is requested.
-        this.throwOnTombstoneUsage =
-            this.mc.config.getBoolean(throwOnTombstoneUsageKey) === true &&
+        this.throwOnTombstoneLoad =
+            this.mc.config.getBoolean(throwOnTombstoneLoadKey) === true &&
             this.runtime.clientDetails.type !== summarizerClientType;
 
         this.runtime.on("disconnected", () => this.onDisconnected());
@@ -202,6 +211,11 @@ export class BlobManager {
         return this.pendingOfflineUploads.length > 0;
     }
 
+    public get hasPendingBlobs(): boolean {
+        return (this.runtime.attachState !== AttachState.Attached && this.redirectTable.size > 0)
+        || this.pendingBlobs.size > 0;
+    }
+
     /**
      * Upload blobs added while offline. This must be completed before connecting and resubmitting ops.
      */
@@ -260,12 +274,18 @@ export class BlobManager {
     public async getBlob(blobId: string): Promise<ArrayBufferLike> {
         const request = { url: blobId };
         if (this.tombstonedBlobs.has(blobId) ) {
-            const error = responseToException(createResponseError(404, "Blob removed by gc", request), request);
-            this.mc.logger.sendErrorEvent({
-                eventName: "GC_Tombstone_Blob_Requested",
-                url: request.url,
-            }, error);
-            if (this.throwOnTombstoneUsage) {
+            const error = responseToException(createResponseError(404, "Blob was deleted", request), request);
+            sendGCTombstoneEvent(
+                this.mc,
+                {
+                    eventName: "GC_Tombstone_Blob_Requested",
+                    category: this.throwOnTombstoneLoad ? "error" : "generic",
+                    isSummarizerClient: this.runtime.clientDetails.type === summarizerClientType,
+                },
+                [BlobManager.basePath],
+                error,
+            );
+            if (this.throwOnTombstoneLoad) {
                 throw error;
             }
         }
@@ -274,7 +294,7 @@ export class BlobManager {
         if (pending) {
             return pending.blob;
         }
-        let storageId;
+        let storageId: string;
         if (this.runtime.attachState === AttachState.Detached) {
             assert(this.redirectTable.has(blobId), 0x383 /* requesting unknown blobs */);
 
@@ -282,12 +302,13 @@ export class BlobManager {
             // The 'IDocumentStorageService.readBlob()' call below will retrieve these via localId.
             storageId = blobId;
         } else {
-            storageId = this.redirectTable.get(blobId);
-            assert(!!storageId, 0x11f /* "requesting unknown blobs" */);
+            const attachedStorageId = this.redirectTable.get(blobId);
+            assert(!!attachedStorageId, 0x11f /* "requesting unknown blobs" */);
+            storageId = attachedStorageId;
         }
 
-        // When a GC-able (not pending) blob is retrieved, let runtime know that the corresponding GC node got updated.
-        this.gcNodeUpdated(this.getBlobGCNodePath(blobId));
+        // Let runtime know that the corresponding GC node was requested.
+        this.blobRequested(this.getBlobGCNodePath(blobId));
 
         return PerformanceEvent.timedExecAsync(
             this.mc.logger,
@@ -313,7 +334,7 @@ export class BlobManager {
         // Blobs created while the container is detached are stored in IDetachedBlobStorage.
         // The 'IDocumentStorageService.createBlob()' call below will respond with a localId.
         const response = await this.getStorage().createBlob(blob);
-        this.redirectTable.set(response.id, undefined);
+        this.setRedirection(response.id, undefined);
         return this.getBlobHandle(response.id);
     }
 
@@ -329,8 +350,8 @@ export class BlobManager {
         assert(this.runtime.attachState === AttachState.Attached,
             0x385 /* For clarity and paranoid defense against adding future attachment states */);
 
-        // Create a local ID for each blob. This is used to support blobs if/when the client goes
-        // offline since we don't have the ID from storage yet. If online flow succeeds this won't be used.
+        // Create a local ID for the blob. After uploading it to storage and before returning it, a local ID to
+        // storage ID mapping is created.
         const localId = uuid();
         const pendingEntry: PendingBlob = {
             blob,
@@ -355,6 +376,28 @@ export class BlobManager {
         );
     }
 
+    /**
+     * Set up a mapping in the redirect table from fromId to toId. Also, notify the runtime that a reference is added
+     * which is required for GC.
+     */
+    private setRedirection(fromId: string, toId: string | undefined) {
+        this.redirectTable.set(fromId, toId);
+        // Notify runtime of a reference added if toId is not undefined. It can be undefined when a blob is uploaded in
+        // detached mode. In this case, the entry will be updated when the blob is updated.
+        if (toId !== undefined) {
+            this.addedBlobReference(this.getBlobGCNodePath(fromId), this.getBlobGCNodePath(toId));
+        }
+    }
+
+    private deleteAndEmitsIfEmpty(id: string) {
+        if (this.pendingBlobs.has(id)) {
+            this.pendingBlobs.delete(id);
+            if (!this.hasPendingBlobs) {
+                this.emit("noPendingBlobs");
+            }
+        }
+    }
+
     private onUploadResolve(localId: string, response: ICreateBlobResponse) {
         const entry = this.pendingBlobs.get(localId);
         assert(entry?.status === PendingBlobStatus.OnlinePendingUpload ||
@@ -363,16 +406,23 @@ export class BlobManager {
         entry.storageId = response.id;
         if (this.runtime.connected) {
             if (entry.status === PendingBlobStatus.OnlinePendingUpload) {
+                // Send a blob attach op. This serves two purposes:
+                // 1. If its a new blob, i.e., it isn't de-duped, the server will keep the blob alive if it sees this op
+                //    until its storage ID is added to the next summary.
+                // 2. It will create a local ID to storage ID mapping in all clients which is needed to retrieve the
+                //    blob from the server via the storage ID.
+                this.sendBlobAttachOp(localId, response.id);
                 if (this.storageIds.has(response.id)) {
-                    // Storage may dedupe blobs and give us an ID we already know
-                    // no need to submit BlobAttach op in this case
-                    entry.handleP.resolve(this.getBlobHandle(response.id));
-                    this.pendingBlobs.delete(localId);
+                    // The blob is de-duped. Set up a local ID to storage ID mapping and return the blob. Since this is
+                    // an existing blob, we don't have to wait for the op to be ack'd since this step has already
+                    // happened before and so, the server won't delete it.
+                    this.setRedirection(localId, response.id);
+                    entry.handleP.resolve(this.getBlobHandle(localId));
+                    this.deleteAndEmitsIfEmpty(localId);
                 } else {
-                    // Check for still-pending duplicates too; if an op is already in flight we can wait for that one
-                    if (!this.opsInFlight.has(response.id)) {
-                        this.sendBlobAttachOp(response.id);
-                    }
+                    // If there is already an op for this storage ID, append the local ID to the list. Once any op for
+                    // this storage ID is ack'd, all pending blobs for it can be resolved since the op will keep the
+                    // blob alive in storage.
                     this.opsInFlight.set(response.id, (this.opsInFlight.get(response.id) ?? []).concat(localId));
                     entry.status = PendingBlobStatus.OnlinePendingOp;
                 }
@@ -414,16 +464,21 @@ export class BlobManager {
         assert([PendingBlobStatus.OnlinePendingUpload, PendingBlobStatus.OnlinePendingOp].includes(entry.status),
             0x38a /* Blob must be in online flow to transition to offline flow */);
 
+        /**
+         * If we haven't already submitted a BlobAttach op for this entry, send it before returning the blob handle.
+         * This will make sure that the BlobAttach op is sequenced prior to any ops referencing the handle. Otherwise,
+         * an invalid handle could be added to the document.
+         * storageId may be undefined but since we are not connected we will have a chance to add it when reSubmit()
+         * is called on reconnection.
+         */
+        if (entry.status !== PendingBlobStatus.OnlinePendingOp) {
+            this.sendBlobAttachOp(localId, entry.storageId);
+        }
+
         entry.status = entry.status === PendingBlobStatus.OnlinePendingUpload
             ? PendingBlobStatus.OfflinePendingUpload
             : PendingBlobStatus.OfflinePendingOp;
 
-        // Submit a BlobAttach op. It's important we submit this op now before returning the blob handle so the
-        // BlobAttach op is sequenced prior to any ops referencing the handle. Otherwise an invalid handle could be
-        // added to the document if the ops are not all successfully submitted upon reconnection.
-        // storageId may be undefined but since we are not connected we will have a chance to add it when reSubmit()
-        // is called
-        this.sendBlobAttachOp(entry.storageId, localId);
         entry.handleP.resolve(this.getBlobHandle(localId));
     }
 
@@ -434,33 +489,42 @@ export class BlobManager {
      */
     public reSubmit(metadata: Record<string, unknown> | undefined) {
         assert(!!metadata, 0x38b /* Resubmitted ops must have metadata */);
-        const { blobId, localId }: { blobId?: string; localId?: string; } = metadata;
+        const { localId, blobId }: { localId?: string; blobId?: string } = metadata;
+        assert(localId !== undefined, 0x50d /* local ID not available on reSubmit */);
         if (!blobId) {
-            assert(!!localId, 0x38c /* Submitted BlobAttach ops must have a blobId or localId */);
             // We submitted this op while offline. The blob should have been uploaded by now.
             const pendingEntry = this.pendingBlobs.get(localId);
             assert(pendingEntry?.status === PendingBlobStatus.OfflinePendingOp &&
                 !!pendingEntry?.storageId, 0x38d /* blob must be uploaded before resubmitting BlobAttach op */);
-            return this.sendBlobAttachOp(pendingEntry.storageId, localId);
+            return this.sendBlobAttachOp(localId, pendingEntry.storageId);
         }
-        return this.sendBlobAttachOp(blobId, localId);
+        return this.sendBlobAttachOp(localId, blobId);
     }
 
     public processBlobAttachOp(message: ISequencedDocumentMessage, local: boolean) {
-        assert(message?.metadata?.blobId, 0x12a /* "Missing blob id on metadata" */);
-        if (message.metadata.localId !== undefined) {
-            this.redirectTable.set(message.metadata.localId, message.metadata.blobId);
+        const localId = message.metadata?.localId;
+        const blobId = message.metadata?.blobId;
+        assert(blobId !== undefined, 0x12a /* "Missing blob id on metadata" */);
+
+        // Set up a mapping from local ID to storage ID. This is crucial since without this the blob cannot be
+        // requested from the server.
+        // Note: The check for undefined is needed for back-compat when localId was not part of the BlobAttach op that
+        // was sent when online.
+        if (localId !== undefined) {
+            this.setRedirection(localId, blobId);
         }
         // set identity (id -> id) entry
-        this.redirectTable.set(message.metadata.blobId, message.metadata.blobId);
+        this.setRedirection(blobId, blobId);
 
         if (local) {
-            if (message.metadata.localId === undefined) {
-                // Since there is no local ID, we know this op was submitted while online.
-                const waitingBlobs = this.opsInFlight.get(message.metadata.blobId);
-                assert(!!waitingBlobs, 0x38e /* local online BlobAttach op with no pending blob */);
-                waitingBlobs.forEach((localId) => {
-                    const pendingBlobEntry = this.pendingBlobs.get(localId);
+            assert(localId !== undefined, 0x50e /* local ID not present in blob attach message */);
+            const waitingBlobs = this.opsInFlight.get(blobId);
+            if (waitingBlobs !== undefined) {
+                // For each op corresponding to this storage ID that we are waiting for, resolve the pending blob.
+                // This is safe because the server will keep the blob alive and the op containing the local ID to
+                // storage ID is already in flight and any op containing this local ID will be sequenced after that.
+                waitingBlobs.forEach((pendingLocalId) => {
+                    const pendingBlobEntry = this.pendingBlobs.get(pendingLocalId);
                     assert(
                         pendingBlobEntry !== undefined,
                         0x38f, /* local online BlobAttach op with no pending blob entry */
@@ -468,16 +532,15 @@ export class BlobManager {
 
                     // It's possible we transitioned to offline flow while waiting for this op.
                     if (pendingBlobEntry.status === PendingBlobStatus.OnlinePendingOp) {
-                        pendingBlobEntry.handleP.resolve(this.getBlobHandle(message.metadata.blobId));
-                        this.pendingBlobs.delete(localId);
+                        this.setRedirection(pendingLocalId, blobId);
+                        pendingBlobEntry.handleP.resolve(this.getBlobHandle(pendingLocalId));
+                        this.deleteAndEmitsIfEmpty(pendingLocalId);
                     }
                 });
-            } else {
-                // Each local ID is unique; get the pending blob entry and delete it
-                assert(this.pendingBlobs.get(message.metadata.localId)?.status === PendingBlobStatus.OfflinePendingOp,
-                    0x1f8 /* "local BlobAttach op with no pending blob" */);
-                this.pendingBlobs.delete(message.metadata.localId);
+                this.opsInFlight.delete(blobId);
             }
+            // For blobs that were transitioned to offline flow while waiting for this op, the entry should be deleted.
+            this.deleteAndEmitsIfEmpty(localId);
         }
     }
 
@@ -531,75 +594,60 @@ export class BlobManager {
      */
     public getGCData(fullGC: boolean = false): IGarbageCollectionData {
         const gcData: IGarbageCollectionData = { gcNodes: {} };
-        /**
-          * The node path is of the format `/_blobs/blobId`. This path must match the path of the blob handle returned
-          * by the createBlob API because blobs are marked referenced by storing these handles in a referenced DDS.
-          */
-        this.storageIds.forEach((blobId: string) => {
-            gcData.gcNodes[this.getBlobGCNodePath(blobId)] = [];
-        });
-
-        // For some blobs, the handle returned on creation is based off of the localId. So, these
-        // nodes can be referenced by storing the localId handle. When that happens, the corresponding storageId node
-        // must also be marked referenced. So, we add a route from the localId node to the storageId node.
-        // Note that because of de-duping, there can be multiple localIds that all redirect to the same storageId or
-        // a blob may be referenced via its storageId handle.
         for (const [localId, storageId] of this.redirectTable) {
             assert(!!storageId, 0x390 /* Must be attached to get GC data */);
-            // Add node for the localId and add a route to the storageId node. The storageId node will have been
-            // added above when adding nodes for this.blobIds.
             gcData.gcNodes[this.getBlobGCNodePath(localId)] = [this.getBlobGCNodePath(storageId)];
         }
-
         return gcData;
     }
 
     /**
-     * This is called to update blobs whose routes are used. The used blobs are removed from the tombstone list.
-     * @param usedRoutes - The routes of the blob nodes that are used.
+     * This is called to update blobs whose routes are unused. The unused blobs are deleted.
+     * @param unusedRoutes - The routes of the blob nodes that are unused.
      */
-    public updateUsedRoutes(usedRoutes: string[]) {
+    public updateUnusedRoutes(unusedRoutes: string[]): void {
         // The routes or blob node paths are in the same format as returned in getGCData -
         // `/<BlobManager.basePath>/<blobId>`.
-        for (const route of usedRoutes) {
+        for (const route of unusedRoutes) {
             const pathParts = route.split("/");
             assert(
                 pathParts.length === 3 && pathParts[1] === BlobManager.basePath,
-                0x4bc /* Invalid blob node id in used routes. */,
+                0x2d5 /* "Invalid blob node id in unused routes." */,
             );
             const blobId = pathParts[2];
-            // Un-tombstone the blob if it was marked tombstone.
-            this.tombstonedBlobs.delete(blobId);
+            this.redirectTable.delete(blobId);
         }
     }
 
     /**
-     * This is called to update blobs whose routes are unused. The unused blobs are either deleted or marked as
-     * tombstones.
-     * @param unusedRoutes - The routes of the blob nodes that are unused.
-     * @param tombstone - if true, the objects corresponding to unused routes are marked tombstones. Otherwise, they
-     * are deleted.
+     * This is called to update blobs whose routes are tombstones. Tombstoned blobs enable testing scenarios with
+     * accessing deleted content without actually deleting content from summaries.
+     * @param tombstonedRoutes - The routes of blob nodes that are tombstones.
      */
-    public updateUnusedRoutes(unusedRoutes: string[], tombstone: boolean): void {
+    public updateTombstonedRoutes(tombstonedRoutes: string[]) {
+        const tombstonedBlobsSet: Set<string> = new Set();
         // The routes or blob node paths are in the same format as returned in getGCData -
         // `/<BlobManager.basePath>/<blobId>`.
-        for (const route of unusedRoutes) {
+        for (const route of tombstonedRoutes) {
             const pathParts = route.split("/");
             assert(
                 pathParts.length === 3 && pathParts[1] === BlobManager.basePath,
-                0x2d5 /* "Invalid blob node id in unused routes." */,
+                0x50f /* Invalid blob node id in tombstoned routes. */,
             );
-            const blobId = pathParts[2];
+            tombstonedBlobsSet.add(pathParts[2]);
+        }
 
-            if (tombstone) {
-                // If tombstone is set, add this blob to the tombstone list.
-                this.tombstonedBlobs.add(blobId);
-            } else {
-                // The unused blobId could be a localId. If so, remove it from the redirect table and continue. The
-                // corresponding storageId may still be used either directly or via other localIds.
-                this.redirectTable.delete(blobId);
+        // Remove blobs from the tombstone list that were tombstoned but aren't anymore as per the tombstoneRoutes.
+        for (const blobId of this.tombstonedBlobs) {
+            if (!tombstonedBlobsSet.has(blobId)) {
+                this.tombstonedBlobs.delete(blobId);
             }
         }
+
+        // Mark blobs that are now tombstoned by adding them to the tombstone list.
+        for (const blobId of tombstonedBlobsSet) {
+            this.tombstonedBlobs.add(blobId);
+        }
     }
 
     public summarize(telemetryContext?: ITelemetryContext): ISummaryTreeWithStats {
@@ -630,9 +678,9 @@ export class BlobManager {
             0x391 /* Redirect table size must match BlobManager's local ID count */);
         for (const [localId, storageId] of table) {
             assert(this.redirectTable.has(localId), 0x254 /* "unrecognized id in redirect table" */);
-            this.redirectTable.set(localId, storageId);
+            this.setRedirection(localId, storageId);
             // set identity (id -> id) entry
-            this.redirectTable.set(storageId, storageId);
+            this.setRedirection(storageId, storageId);
         }
     }