@fluidframework/container-runtime 2.0.0-internal.2.2.1 → 2.0.0-internal.2.3.1

package/dist/blobManager.js

@@ -13,6 +13,7 @@ const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
 const throttler_1 = require("./throttler");
 const summarizerClientElection_1 = require("./summarizerClientElection");
 const garbageCollectionConstants_1 = require("./garbageCollectionConstants");
+const garbageCollectionTombstoneUtils_1 = require("./garbageCollectionTombstoneUtils");
 /**
  * This class represents blob (long string)
  * This object is used only when creating (writing) new blob and serialization purposes.
@@ -65,26 +66,28 @@ var PendingBlobStatus;
     PendingBlobStatus[PendingBlobStatus["OfflinePendingUpload"] = 2] = "OfflinePendingUpload";
     PendingBlobStatus[PendingBlobStatus["OfflinePendingOp"] = 3] = "OfflinePendingOp";
 })(PendingBlobStatus || (PendingBlobStatus = {}));
-class BlobManager {
+class BlobManager extends common_utils_1.TypedEventEmitter {
     constructor(routeContext, snapshot, getStorage, 
     /**
-     * Submit a BlobAttach op. When a blob is uploaded, there is a short grace period before which
-     * the blob is deleted. The BlobAttach op notifies the server that blob is in use. The server
-     * will then not delete the blob as long as it is listed as referenced in future summaries.
-     * The summarizing client will know to include the storage ID in the summary when it sees the op.
+     * Submit a BlobAttach op. When a blob is uploaded, there is a short grace period before which the blob is
+     * deleted. The BlobAttach op notifies the server that blob is in use. The server will then not delete the
+     * the blob as long as it is listed as referenced in future summaries. The summarizing client will know to
+     * include the storage ID in the summary when it sees the op.
      *
-     * The op may also include a local ID to inform all clients of the relation to the storage
-     * ID, without knowledge of which they cannot request the blob from storage. This is also
-     * included in the redirect table in the summary.
+     * The op will also include a local ID to inform all clients of the relation to the storage ID, without
+     * knowledge of which they cannot request the blob from storage. It's important that this op is sequenced
+     * before any ops that reference the local ID, otherwise, an invalid handle could be added to the document.
      */
     sendBlobAttachOp, 
     // To be called when a blob node is requested. blobPath is the path of the blob's node in GC's graph. It's
     // of the format `/<BlobManager.basePath>/<blobId>`.
-    gcNodeUpdated, runtime, stashedBlobs = {}) {
+    blobRequested, addedBlobReference, runtime, stashedBlobs = {}) {
+        super();
         this.routeContext = routeContext;
         this.getStorage = getStorage;
         this.sendBlobAttachOp = sendBlobAttachOp;
-        this.gcNodeUpdated = gcNodeUpdated;
+        this.blobRequested = blobRequested;
+        this.addedBlobReference = addedBlobReference;
         this.runtime = runtime;
         /**
          * Blobs which have not been uploaded or for which we have not yet seen a BlobAttach op round-trip.
@@ -93,8 +96,9 @@ class BlobManager {
          */
         this.pendingBlobs = new Map();
         /**
-         * Track ops in flight for online flow. Used to avoid searching pendingBlobs since BlobAttach ops
-         * don't include local ID in online flow.
+         * Track ops in flight for online flow. This is used for optimizations where if we receive an ack for a storage ID,
+         * we can resolve all pending blobs with the same storage ID even though they may have different local IDs. That's
+         * because we know that the server will not delete the blob corresponding to that storage ID.
          */
         this.opsInFlight = new Map();
         this.retryThrottler = new CancellableThrottler(new throttler_1.Throttler(60 * 1000, // 60 sec delay window
@@ -102,14 +106,14 @@ class BlobManager {
         // throttling function increases exponentially (0ms, 40ms, 80ms, 160ms, etc)
         (0, throttler_1.formExponentialFn)({ coefficient: 20, initialDelay: 0 })));
         /**
-         * This stores ides of tombstoned blobs.
+         * This stores IDs of tombstoned blobs.
          * Tombstone is a temporary feature that imitates a blob getting swept by garbage collection.
          */
         this.tombstonedBlobs = new Set();
         this.mc = (0, telemetry_utils_1.loggerToMonitoringContext)(telemetry_utils_1.ChildLogger.create(this.runtime.logger, "BlobManager"));
         // Read the feature flag that tells whether to throw when a tombstone blob is requested.
-        this.throwOnTombstoneUsage =
-            this.mc.config.getBoolean(garbageCollectionConstants_1.throwOnTombstoneUsageKey) === true &&
+        this.throwOnTombstoneLoad =
+            this.mc.config.getBoolean(garbageCollectionConstants_1.throwOnTombstoneLoadKey) === true &&
                 this.runtime.clientDetails.type !== summarizerClientElection_1.summarizerClientType;
         this.runtime.on("disconnected", () => this.onDisconnected());
         this.redirectTable = this.load(snapshot);
@@ -131,6 +135,10 @@ class BlobManager {
     get hasPendingOfflineUploads() {
         return this.pendingOfflineUploads.length > 0;
     }
+    get hasPendingBlobs() {
+        return (this.runtime.attachState !== container_definitions_1.AttachState.Attached && this.redirectTable.size > 0)
+            || this.pendingBlobs.size > 0;
+    }
     /**
      * Upload blobs added while offline. This must be completed before connecting and resubmitting ops.
      */
@@ -179,12 +187,13 @@ class BlobManager {
     async getBlob(blobId) {
         const request = { url: blobId };
         if (this.tombstonedBlobs.has(blobId)) {
-            const error = (0, runtime_utils_1.responseToException)((0, runtime_utils_1.createResponseError)(404, "Blob removed by gc", request), request);
-            this.mc.logger.sendErrorEvent({
+            const error = (0, runtime_utils_1.responseToException)((0, runtime_utils_1.createResponseError)(404, "Blob was deleted", request), request);
+            (0, garbageCollectionTombstoneUtils_1.sendGCTombstoneEvent)(this.mc, {
                 eventName: "GC_Tombstone_Blob_Requested",
-                url: request.url,
-            }, error);
-            if (this.throwOnTombstoneUsage) {
+                category: this.throwOnTombstoneLoad ? "error" : "generic",
+                isSummarizerClient: this.runtime.clientDetails.type === summarizerClientElection_1.summarizerClientType,
+            }, [BlobManager.basePath], error);
+            if (this.throwOnTombstoneLoad) {
                 throw error;
             }
         }
@@ -200,11 +209,12 @@ class BlobManager {
             storageId = blobId;
         }
         else {
-            storageId = this.redirectTable.get(blobId);
-            (0, common_utils_1.assert)(!!storageId, 0x11f /* "requesting unknown blobs" */);
+            const attachedStorageId = this.redirectTable.get(blobId);
+            (0, common_utils_1.assert)(!!attachedStorageId, 0x11f /* "requesting unknown blobs" */);
+            storageId = attachedStorageId;
         }
-        // When a GC-able (not pending) blob is retrieved, let runtime know that the corresponding GC node got updated.
-        this.gcNodeUpdated(this.getBlobGCNodePath(blobId));
+        // Let runtime know that the corresponding GC node was requested.
+        this.blobRequested(this.getBlobGCNodePath(blobId));
         return telemetry_utils_1.PerformanceEvent.timedExecAsync(this.mc.logger, { eventName: "AttachmentReadBlob", id: storageId }, async () => {
             return this.getStorage().readBlob(storageId);
         }, { end: true, cancel: "error" });
@@ -217,7 +227,7 @@ class BlobManager {
         // Blobs created while the container is detached are stored in IDetachedBlobStorage.
         // The 'IDocumentStorageService.createBlob()' call below will respond with a localId.
         const response = await this.getStorage().createBlob(blob);
-        this.redirectTable.set(response.id, undefined);
+        this.setRedirection(response.id, undefined);
         return this.getBlobHandle(response.id);
     }
     async createBlob(blob) {
@@ -230,8 +240,8 @@ class BlobManager {
             await new Promise((resolve) => this.runtime.once("attached", resolve));
         }
         (0, common_utils_1.assert)(this.runtime.attachState === container_definitions_1.AttachState.Attached, 0x385 /* For clarity and paranoid defense against adding future attachment states */);
-        // Create a local ID for each blob. This is used to support blobs if/when the client goes
-        // offline since we don't have the ID from storage yet. If online flow succeeds this won't be used.
+        // Create a local ID for the blob. After uploading it to storage and before returning it, a local ID to
+        // storage ID mapping is created.
         const localId = (0, uuid_1.v4)();
         const pendingEntry = {
             blob,
@@ -245,6 +255,26 @@ class BlobManager {
     async uploadBlob(localId, blob) {
         return telemetry_utils_1.PerformanceEvent.timedExecAsync(this.mc.logger, { eventName: "createBlob" }, async () => this.getStorage().createBlob(blob), { end: true, cancel: this.runtime.connected ? "error" : "generic" }).then((response) => this.onUploadResolve(localId, response), async (err) => this.onUploadReject(localId, err));
     }
+    /**
+     * Set up a mapping in the redirect table from fromId to toId. Also, notify the runtime that a reference is added
+     * which is required for GC.
+     */
+    setRedirection(fromId, toId) {
+        this.redirectTable.set(fromId, toId);
+        // Notify runtime of a reference added if toId is not undefined. It can be undefined when a blob is uploaded in
+        // detached mode. In this case, the entry will be updated when the blob is updated.
+        if (toId !== undefined) {
+            this.addedBlobReference(this.getBlobGCNodePath(fromId), this.getBlobGCNodePath(toId));
+        }
+    }
+    deleteAndEmitsIfEmpty(id) {
+        if (this.pendingBlobs.has(id)) {
+            this.pendingBlobs.delete(id);
+            if (!this.hasPendingBlobs) {
+                this.emit("noPendingBlobs");
+            }
+        }
+    }
     onUploadResolve(localId, response) {
         var _a;
         const entry = this.pendingBlobs.get(localId);
@@ -253,17 +283,24 @@ class BlobManager {
         entry.storageId = response.id;
         if (this.runtime.connected) {
             if (entry.status === PendingBlobStatus.OnlinePendingUpload) {
+                // Send a blob attach op. This serves two purposes:
+                // 1. If its a new blob, i.e., it isn't de-duped, the server will keep the blob alive if it sees this op
+                //    until its storage ID is added to the next summary.
+                // 2. It will create a local ID to storage ID mapping in all clients which is needed to retrieve the
+                //    blob from the server via the storage ID.
+                this.sendBlobAttachOp(localId, response.id);
                 if (this.storageIds.has(response.id)) {
-                    // Storage may dedupe blobs and give us an ID we already know
-                    // no need to submit BlobAttach op in this case
-                    entry.handleP.resolve(this.getBlobHandle(response.id));
-                    this.pendingBlobs.delete(localId);
+                    // The blob is de-duped. Set up a local ID to storage ID mapping and return the blob. Since this is
+                    // an existing blob, we don't have to wait for the op to be ack'd since this step has already
+                    // happened before and so, the server won't delete it.
+                    this.setRedirection(localId, response.id);
+                    entry.handleP.resolve(this.getBlobHandle(localId));
+                    this.deleteAndEmitsIfEmpty(localId);
                 }
                 else {
-                    // Check for still-pending duplicates too; if an op is already in flight we can wait for that one
-                    if (!this.opsInFlight.has(response.id)) {
-                        this.sendBlobAttachOp(response.id);
-                    }
+                    // If there is already an op for this storage ID, append the local ID to the list. Once any op for
+                    // this storage ID is ack'd, all pending blobs for it can be resolved since the op will keep the
+                    // blob alive in storage.
                     this.opsInFlight.set(response.id, ((_a = this.opsInFlight.get(response.id)) !== null && _a !== void 0 ? _a : []).concat(localId));
                     entry.status = PendingBlobStatus.OnlinePendingOp;
                 }
@@ -304,15 +341,19 @@ class BlobManager {
         const entry = this.pendingBlobs.get(localId);
         (0, common_utils_1.assert)(!!entry, 0x389 /* No pending blob entry */);
         (0, common_utils_1.assert)([PendingBlobStatus.OnlinePendingUpload, PendingBlobStatus.OnlinePendingOp].includes(entry.status), 0x38a /* Blob must be in online flow to transition to offline flow */);
+        /**
+         * If we haven't already submitted a BlobAttach op for this entry, send it before returning the blob handle.
+         * This will make sure that the BlobAttach op is sequenced prior to any ops referencing the handle. Otherwise,
+         * an invalid handle could be added to the document.
+         * storageId may be undefined but since we are not connected we will have a chance to add it when reSubmit()
+         * is called on reconnection.
+         */
+        if (entry.status !== PendingBlobStatus.OnlinePendingOp) {
+            this.sendBlobAttachOp(localId, entry.storageId);
+        }
         entry.status = entry.status === PendingBlobStatus.OnlinePendingUpload
             ? PendingBlobStatus.OfflinePendingUpload
             : PendingBlobStatus.OfflinePendingOp;
-        // Submit a BlobAttach op. It's important we submit this op now before returning the blob handle so the
-        // BlobAttach op is sequenced prior to any ops referencing the handle. Otherwise an invalid handle could be
-        // added to the document if the ops are not all successfully submitted upon reconnection.
-        // storageId may be undefined but since we are not connected we will have a chance to add it when reSubmit()
-        // is called
-        this.sendBlobAttachOp(entry.storageId, localId);
         entry.handleP.resolve(this.getBlobHandle(localId));
     }
     /**
@@ -322,45 +363,52 @@ class BlobManager {
      */
     reSubmit(metadata) {
         (0, common_utils_1.assert)(!!metadata, 0x38b /* Resubmitted ops must have metadata */);
-        const { blobId, localId } = metadata;
+        const { localId, blobId } = metadata;
+        (0, common_utils_1.assert)(localId !== undefined, 0x50d /* local ID not available on reSubmit */);
         if (!blobId) {
-            (0, common_utils_1.assert)(!!localId, 0x38c /* Submitted BlobAttach ops must have a blobId or localId */);
             // We submitted this op while offline. The blob should have been uploaded by now.
             const pendingEntry = this.pendingBlobs.get(localId);
             (0, common_utils_1.assert)((pendingEntry === null || pendingEntry === void 0 ? void 0 : pendingEntry.status) === PendingBlobStatus.OfflinePendingOp &&
                 !!(pendingEntry === null || pendingEntry === void 0 ? void 0 : pendingEntry.storageId), 0x38d /* blob must be uploaded before resubmitting BlobAttach op */);
-            return this.sendBlobAttachOp(pendingEntry.storageId, localId);
+            return this.sendBlobAttachOp(localId, pendingEntry.storageId);
         }
-        return this.sendBlobAttachOp(blobId, localId);
+        return this.sendBlobAttachOp(localId, blobId);
     }
     processBlobAttachOp(message, local) {
         var _a, _b;
-        (0, common_utils_1.assert)((_a = message === null || message === void 0 ? void 0 : message.metadata) === null || _a === void 0 ? void 0 : _a.blobId, 0x12a /* "Missing blob id on metadata" */);
-        if (message.metadata.localId !== undefined) {
-            this.redirectTable.set(message.metadata.localId, message.metadata.blobId);
+        const localId = (_a = message.metadata) === null || _a === void 0 ? void 0 : _a.localId;
+        const blobId = (_b = message.metadata) === null || _b === void 0 ? void 0 : _b.blobId;
+        (0, common_utils_1.assert)(blobId !== undefined, 0x12a /* "Missing blob id on metadata" */);
+        // Set up a mapping from local ID to storage ID. This is crucial since without this the blob cannot be
+        // requested from the server.
+        // Note: The check for undefined is needed for back-compat when localId was not part of the BlobAttach op that
+        // was sent when online.
+        if (localId !== undefined) {
+            this.setRedirection(localId, blobId);
         }
         // set identity (id -> id) entry
-        this.redirectTable.set(message.metadata.blobId, message.metadata.blobId);
+        this.setRedirection(blobId, blobId);
         if (local) {
-            if (message.metadata.localId === undefined) {
-                // Since there is no local ID, we know this op was submitted while online.
-                const waitingBlobs = this.opsInFlight.get(message.metadata.blobId);
-                (0, common_utils_1.assert)(!!waitingBlobs, 0x38e /* local online BlobAttach op with no pending blob */);
-                waitingBlobs.forEach((localId) => {
-                    const pendingBlobEntry = this.pendingBlobs.get(localId);
+            (0, common_utils_1.assert)(localId !== undefined, 0x50e /* local ID not present in blob attach message */);
+            const waitingBlobs = this.opsInFlight.get(blobId);
+            if (waitingBlobs !== undefined) {
+                // For each op corresponding to this storage ID that we are waiting for, resolve the pending blob.
+                // This is safe because the server will keep the blob alive and the op containing the local ID to
+                // storage ID is already in flight and any op containing this local ID will be sequenced after that.
+                waitingBlobs.forEach((pendingLocalId) => {
+                    const pendingBlobEntry = this.pendingBlobs.get(pendingLocalId);
                     (0, common_utils_1.assert)(pendingBlobEntry !== undefined, 0x38f);
                     // It's possible we transitioned to offline flow while waiting for this op.
                     if (pendingBlobEntry.status === PendingBlobStatus.OnlinePendingOp) {
-                        pendingBlobEntry.handleP.resolve(this.getBlobHandle(message.metadata.blobId));
-                        this.pendingBlobs.delete(localId);
+                        this.setRedirection(pendingLocalId, blobId);
+                        pendingBlobEntry.handleP.resolve(this.getBlobHandle(pendingLocalId));
+                        this.deleteAndEmitsIfEmpty(pendingLocalId);
                     }
                 });
+                this.opsInFlight.delete(blobId);
             }
-            else {
-                // Each local ID is unique; get the pending blob entry and delete it
-                (0, common_utils_1.assert)(((_b = this.pendingBlobs.get(message.metadata.localId)) === null || _b === void 0 ? void 0 : _b.status) === PendingBlobStatus.OfflinePendingOp, 0x1f8 /* "local BlobAttach op with no pending blob" */);
-                this.pendingBlobs.delete(message.metadata.localId);
-            }
+            // For blobs that were transitioned to offline flow while waiting for this op, the entry should be deleted.
+            this.deleteAndEmitsIfEmpty(localId);
         }
     }
     /**
@@ -409,65 +457,50 @@ class BlobManager {
      */
     getGCData(fullGC = false) {
         const gcData = { gcNodes: {} };
-        /**
-          * The node path is of the format `/_blobs/blobId`. This path must match the path of the blob handle returned
-          * by the createBlob API because blobs are marked referenced by storing these handles in a referenced DDS.
-          */
-        this.storageIds.forEach((blobId) => {
-            gcData.gcNodes[this.getBlobGCNodePath(blobId)] = [];
-        });
-        // For some blobs, the handle returned on creation is based off of the localId. So, these
-        // nodes can be referenced by storing the localId handle. When that happens, the corresponding storageId node
-        // must also be marked referenced. So, we add a route from the localId node to the storageId node.
-        // Note that because of de-duping, there can be multiple localIds that all redirect to the same storageId or
-        // a blob may be referenced via its storageId handle.
         for (const [localId, storageId] of this.redirectTable) {
             (0, common_utils_1.assert)(!!storageId, 0x390 /* Must be attached to get GC data */);
-            // Add node for the localId and add a route to the storageId node. The storageId node will have been
-            // added above when adding nodes for this.blobIds.
             gcData.gcNodes[this.getBlobGCNodePath(localId)] = [this.getBlobGCNodePath(storageId)];
         }
         return gcData;
     }
     /**
-     * This is called to update blobs whose routes are used. The used blobs are removed from the tombstone list.
-     * @param usedRoutes - The routes of the blob nodes that are used.
+     * This is called to update blobs whose routes are unused. The unused blobs are deleted.
+     * @param unusedRoutes - The routes of the blob nodes that are unused.
      */
-    updateUsedRoutes(usedRoutes) {
+    updateUnusedRoutes(unusedRoutes) {
         // The routes or blob node paths are in the same format as returned in getGCData -
         // `/<BlobManager.basePath>/<blobId>`.
-        for (const route of usedRoutes) {
+        for (const route of unusedRoutes) {
             const pathParts = route.split("/");
-            (0, common_utils_1.assert)(pathParts.length === 3 && pathParts[1] === BlobManager.basePath, 0x4bc /* Invalid blob node id in used routes. */);
+            (0, common_utils_1.assert)(pathParts.length === 3 && pathParts[1] === BlobManager.basePath, 0x2d5 /* "Invalid blob node id in unused routes." */);
             const blobId = pathParts[2];
-            // Un-tombstone the blob if it was marked tombstone.
-            this.tombstonedBlobs.delete(blobId);
+            this.redirectTable.delete(blobId);
         }
     }
     /**
-     * This is called to update blobs whose routes are unused. The unused blobs are either deleted or marked as
-     * tombstones.
-     * @param unusedRoutes - The routes of the blob nodes that are unused.
-     * @param tombstone - if true, the objects corresponding to unused routes are marked tombstones. Otherwise, they
-     * are deleted.
+     * This is called to update blobs whose routes are tombstones. Tombstoned blobs enable testing scenarios with
+     * accessing deleted content without actually deleting content from summaries.
+     * @param tombstonedRoutes - The routes of blob nodes that are tombstones.
      */
-    updateUnusedRoutes(unusedRoutes, tombstone) {
+    updateTombstonedRoutes(tombstonedRoutes) {
+        const tombstonedBlobsSet = new Set();
         // The routes or blob node paths are in the same format as returned in getGCData -
         // `/<BlobManager.basePath>/<blobId>`.
-        for (const route of unusedRoutes) {
+        for (const route of tombstonedRoutes) {
             const pathParts = route.split("/");
-            (0, common_utils_1.assert)(pathParts.length === 3 && pathParts[1] === BlobManager.basePath, 0x2d5 /* "Invalid blob node id in unused routes." */);
-            const blobId = pathParts[2];
-            if (tombstone) {
-                // If tombstone is set, add this blob to the tombstone list.
-                this.tombstonedBlobs.add(blobId);
-            }
-            else {
-                // The unused blobId could be a localId. If so, remove it from the redirect table and continue. The
-                // corresponding storageId may still be used either directly or via other localIds.
-                this.redirectTable.delete(blobId);
+            (0, common_utils_1.assert)(pathParts.length === 3 && pathParts[1] === BlobManager.basePath, 0x50f /* Invalid blob node id in tombstoned routes. */);
+            tombstonedBlobsSet.add(pathParts[2]);
+        }
+        // Remove blobs from the tombstone list that were tombstoned but aren't anymore as per the tombstoneRoutes.
+        for (const blobId of this.tombstonedBlobs) {
+            if (!tombstonedBlobsSet.has(blobId)) {
+                this.tombstonedBlobs.delete(blobId);
             }
         }
+        // Mark blobs that are now tombstoned by adding them to the tombstone list.
+        for (const blobId of tombstonedBlobsSet) {
+            this.tombstonedBlobs.add(blobId);
+        }
     }
     summarize(telemetryContext) {
         // if storageIds is empty, it means we are detached and have only local IDs, or that there are no blobs attached
@@ -490,9 +523,9 @@ class BlobManager {
         (0, common_utils_1.assert)(this.redirectTable.size === table.size, 0x391 /* Redirect table size must match BlobManager's local ID count */);
         for (const [localId, storageId] of table) {
             (0, common_utils_1.assert)(this.redirectTable.has(localId), 0x254 /* "unrecognized id in redirect table" */);
-            this.redirectTable.set(localId, storageId);
+            this.setRedirection(localId, storageId);
             // set identity (id -> id) entry
-            this.redirectTable.set(storageId, storageId);
+            this.setRedirection(storageId, storageId);
         }
     }
     getPendingBlobs() {