@fluidframework/container-runtime 2.0.0-dev.2.3.0.115467 → 2.0.0-dev.3.1.0.125672

package/lib/blobManager.js

@@ -4,12 +4,14 @@
  */
 import { v4 as uuid } from "uuid";
 import { createResponseError, generateHandleContextPath, responseToException, SummaryTreeBuilder, } from "@fluidframework/runtime-utils";
-import { assert, bufferToString, Deferred, stringToBuffer } from "@fluidframework/common-utils";
+import { assert, bufferToString, Deferred, stringToBuffer, TypedEventEmitter, } from "@fluidframework/common-utils";
 import { AttachState } from "@fluidframework/container-definitions";
-import { ChildLogger, loggerToMonitoringContext, PerformanceEvent } from "@fluidframework/telemetry-utils";
+import { ChildLogger, loggerToMonitoringContext, PerformanceEvent, } from "@fluidframework/telemetry-utils";
+import { TombstoneResponseHeaderKey } from "./containerRuntime";
 import { Throttler, formExponentialFn } from "./throttler";
 import { summarizerClientType } from "./summarizerClientElection";
-import { throwOnTombstoneUsageKey } from "./garbageCollectionConstants";
+import { throwOnTombstoneLoadKey } from "./garbageCollectionConstants";
+import { sendGCUnexpectedUsageEvent } from "./garbageCollectionHelpers";
 /**
  * This class represents blob (long string)
  * This object is used only when creating (writing) new blob and serialization purposes.
@@ -25,7 +27,9 @@ export class BlobHandle {
         this.attached = false;
         this.absolutePath = generateHandleContextPath(path, this.routeContext);
     }
-    get IFluidHandle() { return this; }
+    get IFluidHandle() {
+        return this;
+    }
     get isAttached() {
         return this.attached;
     }
@@ -61,27 +65,38 @@ var PendingBlobStatus;
     PendingBlobStatus[PendingBlobStatus["OfflinePendingUpload"] = 2] = "OfflinePendingUpload";
     PendingBlobStatus[PendingBlobStatus["OfflinePendingOp"] = 3] = "OfflinePendingOp";
 })(PendingBlobStatus || (PendingBlobStatus = {}));
-export class BlobManager {
+export class BlobManager extends TypedEventEmitter {
     constructor(routeContext, snapshot, getStorage, 
     /**
-     * Submit a BlobAttach op. When a blob is uploaded, there is a short grace period before which
-     * the blob is deleted. The BlobAttach op notifies the server that blob is in use. The server
-     * will then not delete the blob as long as it is listed as referenced in future summaries.
-     * The summarizing client will know to include the storage ID in the summary when it sees the op.
+     * Submit a BlobAttach op. When a blob is uploaded, there is a short grace period before which the blob is
+     * deleted. The BlobAttach op notifies the server that blob is in use. The server will then not delete the
+     * the blob as long as it is listed as referenced in future summaries. The summarizing client will know to
+     * include the storage ID in the summary when it sees the op.
      *
-     * The op may also include a local ID to inform all clients of the relation to the storage
-     * ID, without knowledge of which they cannot request the blob from storage. This is also
-     * included in the redirect table in the summary.
+     * The op will also include a local ID to inform all clients of the relation to the storage ID, without
+     * knowledge of which they cannot request the blob from storage. It's important that this op is sequenced
+     * before any ops that reference the local ID, otherwise, an invalid handle could be added to the document.
      */
     sendBlobAttachOp, 
-    // To be called when a blob node is requested. blobPath is the path of the blob's node in GC's graph. It's
-    // of the format `/<BlobManager.basePath>/<blobId>`.
-    gcNodeUpdated, runtime, stashedBlobs = {}) {
+    // Called when a blob node is requested. blobPath is the path of the blob's node in GC's graph.
+    // blobPath's format - `/<BlobManager.basePath>/<blobId>`.
+    blobRequested, 
+    // Called when a reference is added to a blob. For instance, when creating a localId / storageId to storageId
+    // mapping in the redirect table.
+    // Node path formats - `/<BlobManager.basePath>/<blobId>`.
+    addedBlobReference, 
+    // Called to check if a blob has been deleted by GC.
+    // blobPath's format - `/<BlobManager.basePath>/<blobId>`.
+    isBlobDeleted, runtime, stashedBlobs = {}, getCurrentReferenceTimestampMs) {
+        super();
         this.routeContext = routeContext;
         this.getStorage = getStorage;
         this.sendBlobAttachOp = sendBlobAttachOp;
-        this.gcNodeUpdated = gcNodeUpdated;
+        this.blobRequested = blobRequested;
+        this.addedBlobReference = addedBlobReference;
+        this.isBlobDeleted = isBlobDeleted;
         this.runtime = runtime;
+        this.getCurrentReferenceTimestampMs = getCurrentReferenceTimestampMs;
         /**
          * Blobs which have not been uploaded or for which we have not yet seen a BlobAttach op round-trip.
          * Until we see the op round-trip, there is a possibility we may need to re-upload the blob, so
@@ -89,8 +104,9 @@ export class BlobManager {
          */
         this.pendingBlobs = new Map();
         /**
-         * Track ops in flight for online flow. Used to avoid searching pendingBlobs since BlobAttach ops
-         * don't include local ID in online flow.
+         * Track ops in flight for online flow. This is used for optimizations where if we receive an ack for a storage ID,
+         * we can resolve all pending blobs with the same storage ID even though they may have different local IDs. That's
+         * because we know that the server will not delete the blob corresponding to that storage ID.
          */
         this.opsInFlight = new Map();
         this.retryThrottler = new CancellableThrottler(new Throttler(60 * 1000, // 60 sec delay window
@@ -98,14 +114,14 @@ export class BlobManager {
         // throttling function increases exponentially (0ms, 40ms, 80ms, 160ms, etc)
         formExponentialFn({ coefficient: 20, initialDelay: 0 })));
         /**
-         * This stores ides of tombstoned blobs.
+         * This stores IDs of tombstoned blobs.
          * Tombstone is a temporary feature that imitates a blob getting swept by garbage collection.
          */
         this.tombstonedBlobs = new Set();
         this.mc = loggerToMonitoringContext(ChildLogger.create(this.runtime.logger, "BlobManager"));
         // Read the feature flag that tells whether to throw when a tombstone blob is requested.
-        this.throwOnTombstoneUsage =
-            this.mc.config.getBoolean(throwOnTombstoneUsageKey) === true &&
+        this.throwOnTombstoneLoad =
+            this.mc.config.getBoolean(throwOnTombstoneLoadKey) === true &&
                 this.runtime.clientDetails.type !== summarizerClientType;
         this.runtime.on("disconnected", () => this.onDisconnected());
         this.redirectTable = this.load(snapshot);
@@ -121,12 +137,15 @@ export class BlobManager {
         });
     }
     get pendingOfflineUploads() {
-        return Array.from(this.pendingBlobs.values())
-            .filter((e) => e.status === PendingBlobStatus.OfflinePendingUpload);
+        return Array.from(this.pendingBlobs.values()).filter((e) => e.status === PendingBlobStatus.OfflinePendingUpload);
     }
     get hasPendingOfflineUploads() {
         return this.pendingOfflineUploads.length > 0;
     }
+    get hasPendingBlobs() {
+        return ((this.runtime.attachState !== AttachState.Attached && this.redirectTable.size > 0) ||
+            this.pendingBlobs.size > 0);
+    }
     /**
      * Upload blobs added while offline. This must be completed before connecting and resubmitting ops.
      */
@@ -169,21 +188,14 @@ export class BlobManager {
         const undefinedValueInTable = ids.delete(undefined);
         // For a detached container, entries are inserted into the redirect table with an undefined storage ID.
         // For an attached container, entries are inserted w/storage ID after the BlobAttach op round-trips.
-        assert(!undefinedValueInTable || this.runtime.attachState === AttachState.Detached && ids.size === 0, 0x382 /* 'redirectTable' must contain only undefined while detached / defined values while attached */);
+        assert(!undefinedValueInTable ||
+            (this.runtime.attachState === AttachState.Detached && ids.size === 0), 0x382 /* 'redirectTable' must contain only undefined while detached / defined values while attached */);
         return ids;
     }
     async getBlob(blobId) {
-        const request = { url: blobId };
-        if (this.tombstonedBlobs.has(blobId)) {
-            const error = responseToException(createResponseError(404, "Blob removed by gc", request), request);
-            this.mc.logger.sendErrorEvent({
-                eventName: "GC_Tombstone_Blob_Requested",
-                url: request.url,
-            }, error);
-            if (this.throwOnTombstoneUsage) {
-                throw error;
-            }
-        }
+        // Verify that the blob is valid, i.e., it has not been garbage collected. If it is, this will throw an error,
+        // failing the call.
+        this.verifyBlobValidity(blobId);
         const pending = this.pendingBlobs.get(blobId);
         if (pending) {
             return pending.blob;
@@ -196,11 +208,12 @@ export class BlobManager {
             storageId = blobId;
         }
         else {
-            storageId = this.redirectTable.get(blobId);
-            assert(!!storageId, 0x11f /* "requesting unknown blobs" */);
+            const attachedStorageId = this.redirectTable.get(blobId);
+            assert(!!attachedStorageId, 0x11f /* "requesting unknown blobs" */);
+            storageId = attachedStorageId;
         }
-        // When a GC-able (not pending) blob is retrieved, let runtime know that the corresponding GC node got updated.
-        this.gcNodeUpdated(this.getBlobGCNodePath(blobId));
+        // Let runtime know that the corresponding GC node was requested.
+        this.blobRequested(this.getBlobGCNodePath(blobId));
         return PerformanceEvent.timedExecAsync(this.mc.logger, { eventName: "AttachmentReadBlob", id: storageId }, async () => {
             return this.getStorage().readBlob(storageId);
         }, { end: true, cancel: "error" });
@@ -213,7 +226,7 @@ export class BlobManager {
         // Blobs created while the container is detached are stored in IDetachedBlobStorage.
         // The 'IDocumentStorageService.createBlob()' call below will respond with a localId.
         const response = await this.getStorage().createBlob(blob);
-        this.redirectTable.set(response.id, undefined);
+        this.setRedirection(response.id, undefined);
         return this.getBlobHandle(response.id);
     }
     async createBlob(blob) {
@@ -226,8 +239,8 @@ export class BlobManager {
             await new Promise((resolve) => this.runtime.once("attached", resolve));
         }
         assert(this.runtime.attachState === AttachState.Attached, 0x385 /* For clarity and paranoid defense against adding future attachment states */);
-        // Create a local ID for each blob. This is used to support blobs if/when the client goes
-        // offline since we don't have the ID from storage yet. If online flow succeeds this won't be used.
+        // Create a local ID for the blob. After uploading it to storage and before returning it, a local ID to
+        // storage ID mapping is created.
         const localId = uuid();
         const pendingEntry = {
             blob,
@@ -241,25 +254,56 @@ export class BlobManager {
     async uploadBlob(localId, blob) {
         return PerformanceEvent.timedExecAsync(this.mc.logger, { eventName: "createBlob" }, async () => this.getStorage().createBlob(blob), { end: true, cancel: this.runtime.connected ? "error" : "generic" }).then((response) => this.onUploadResolve(localId, response), async (err) => this.onUploadReject(localId, err));
     }
+    /**
+     * Set up a mapping in the redirect table from fromId to toId. Also, notify the runtime that a reference is added
+     * which is required for GC.
+     */
+    setRedirection(fromId, toId) {
+        this.redirectTable.set(fromId, toId);
+        // Notify runtime of a reference added if toId is not undefined. It can be undefined when a blob is uploaded in
+        // detached mode. In this case, the entry will be updated when the blob is updated.
+        if (toId !== undefined) {
+            this.addedBlobReference(this.getBlobGCNodePath(fromId), this.getBlobGCNodePath(toId));
+        }
+    }
+    deleteAndEmitsIfEmpty(id) {
+        if (this.pendingBlobs.has(id)) {
+            this.pendingBlobs.delete(id);
+            if (!this.hasPendingBlobs) {
+                this.emit("noPendingBlobs");
+            }
+        }
+    }
     onUploadResolve(localId, response) {
         var _a;
         const entry = this.pendingBlobs.get(localId);
         assert((entry === null || entry === void 0 ? void 0 : entry.status) === PendingBlobStatus.OnlinePendingUpload ||
             (entry === null || entry === void 0 ? void 0 : entry.status) === PendingBlobStatus.OfflinePendingUpload, 0x386 /* Must have pending blob entry for uploaded blob */);
         entry.storageId = response.id;
+        entry.localUploadTime = Date.now();
+        entry.minTTLInSeconds = response.minTTLInSeconds;
+        entry.serverUploadTime = this.getCurrentReferenceTimestampMs();
         if (this.runtime.connected) {
             if (entry.status === PendingBlobStatus.OnlinePendingUpload) {
+                // Send a blob attach op. This serves two purposes:
+                // 1. If its a new blob, i.e., it isn't de-duped, the server will keep the blob alive if it sees this op
+                //    until its storage ID is added to the next summary.
+                // 2. It will create a local ID to storage ID mapping in all clients which is needed to retrieve the
+                //    blob from the server via the storage ID.
+                this.logTimeInfo(entry, "sendBlobAttachResolveTTL");
+                this.sendBlobAttachOp(localId, response.id);
                 if (this.storageIds.has(response.id)) {
-                    // Storage may dedupe blobs and give us an ID we already know
-                    // no need to submit BlobAttach op in this case
-                    entry.handleP.resolve(this.getBlobHandle(response.id));
-                    this.pendingBlobs.delete(localId);
+                    // The blob is de-duped. Set up a local ID to storage ID mapping and return the blob. Since this is
+                    // an existing blob, we don't have to wait for the op to be ack'd since this step has already
+                    // happened before and so, the server won't delete it.
+                    this.setRedirection(localId, response.id);
+                    entry.handleP.resolve(this.getBlobHandle(localId));
+                    this.deleteAndEmitsIfEmpty(localId);
                 }
                 else {
-                    // Check for still-pending duplicates too; if an op is already in flight we can wait for that one
-                    if (!this.opsInFlight.has(response.id)) {
-                        this.sendBlobAttachOp(response.id);
-                    }
+                    // If there is already an op for this storage ID, append the local ID to the list. Once any op for
+                    // this storage ID is ack'd, all pending blobs for it can be resolved since the op will keep the
+                    // blob alive in storage.
                     this.opsInFlight.set(response.id, ((_a = this.opsInFlight.get(response.id)) !== null && _a !== void 0 ? _a : []).concat(localId));
                     entry.status = PendingBlobStatus.OnlinePendingOp;
                 }
@@ -287,7 +331,9 @@ export class BlobManager {
                 this.transitionToOffline(localId);
             }
             // we are probably not connected to storage but start another upload request in case we are
-            entry.uploadP = this.retryThrottler.getDelay().then(async () => this.uploadBlob(localId, entry.blob));
+            entry.uploadP = this.retryThrottler
+                .getDelay()
+                .then(async () => this.uploadBlob(localId, entry.blob));
             return entry.uploadP;
         }
         else {
@@ -300,15 +346,21 @@ export class BlobManager {
         const entry = this.pendingBlobs.get(localId);
         assert(!!entry, 0x389 /* No pending blob entry */);
         assert([PendingBlobStatus.OnlinePendingUpload, PendingBlobStatus.OnlinePendingOp].includes(entry.status), 0x38a /* Blob must be in online flow to transition to offline flow */);
-        entry.status = entry.status === PendingBlobStatus.OnlinePendingUpload
-            ? PendingBlobStatus.OfflinePendingUpload
-            : PendingBlobStatus.OfflinePendingOp;
-        // Submit a BlobAttach op. It's important we submit this op now before returning the blob handle so the
-        // BlobAttach op is sequenced prior to any ops referencing the handle. Otherwise an invalid handle could be
-        // added to the document if the ops are not all successfully submitted upon reconnection.
-        // storageId may be undefined but since we are not connected we will have a chance to add it when reSubmit()
-        // is called
-        this.sendBlobAttachOp(entry.storageId, localId);
+        /**
+         * If we haven't already submitted a BlobAttach op for this entry, send it before returning the blob handle.
+         * This will make sure that the BlobAttach op is sequenced prior to any ops referencing the handle. Otherwise,
+         * an invalid handle could be added to the document.
+         * storageId may be undefined but since we are not connected we will have a chance to add it when reSubmit()
+         * is called on reconnection.
+         */
+        if (entry.status !== PendingBlobStatus.OnlinePendingOp) {
+            this.logTimeInfo(entry, "sendBlobAttachTransitionOfflineTTL");
+            this.sendBlobAttachOp(localId, entry.storageId);
+        }
+        entry.status =
+            entry.status === PendingBlobStatus.OnlinePendingUpload
+                ? PendingBlobStatus.OfflinePendingUpload
+                : PendingBlobStatus.OfflinePendingOp;
         entry.handleP.resolve(this.getBlobHandle(localId));
     }
     /**
@@ -318,45 +370,81 @@ export class BlobManager {
      */
     reSubmit(metadata) {
         assert(!!metadata, 0x38b /* Resubmitted ops must have metadata */);
-        const { blobId, localId } = metadata;
+        const { localId, blobId } = metadata;
+        assert(localId !== undefined, 0x50d /* local ID not available on reSubmit */);
+        const pendingEntry = this.pendingBlobs.get(localId);
+        if (pendingEntry) {
+            this.logTimeInfo(pendingEntry, "sendBlobAttachResubmitTTL");
+        }
         if (!blobId) {
-            assert(!!localId, 0x38c /* Submitted BlobAttach ops must have a blobId or localId */);
             // We submitted this op while offline. The blob should have been uploaded by now.
-            const pendingEntry = this.pendingBlobs.get(localId);
             assert((pendingEntry === null || pendingEntry === void 0 ? void 0 : pendingEntry.status) === PendingBlobStatus.OfflinePendingOp &&
                 !!(pendingEntry === null || pendingEntry === void 0 ? void 0 : pendingEntry.storageId), 0x38d /* blob must be uploaded before resubmitting BlobAttach op */);
-            return this.sendBlobAttachOp(pendingEntry.storageId, localId);
+            return this.sendBlobAttachOp(localId, pendingEntry.storageId);
+        }
+        return this.sendBlobAttachOp(localId, blobId);
+    }
+    logTimeInfo(pendingEntry, eventName) {
+        var _a, _b;
+        let timeLapseSinceLocalUpload = 0;
+        let timeLapseSinceServerUpload = 0;
+        let expiredUsingLocalTime;
+        let expiredUsingServerTime;
+        if (pendingEntry.localUploadTime) {
+            timeLapseSinceLocalUpload = (Date.now() - pendingEntry.localUploadTime) / 1000;
+            expiredUsingLocalTime =
+                ((_a = pendingEntry.minTTLInSeconds) !== null && _a !== void 0 ? _a : 0) - timeLapseSinceLocalUpload < 0 ? true : false;
+        }
+        if (pendingEntry.serverUploadTime) {
+            timeLapseSinceServerUpload = (Date.now() - pendingEntry.serverUploadTime) / 1000;
+            expiredUsingServerTime =
+                ((_b = pendingEntry.minTTLInSeconds) !== null && _b !== void 0 ? _b : 0) - timeLapseSinceServerUpload < 0 ? true : false;
         }
-        return this.sendBlobAttachOp(blobId, localId);
+        this.mc.logger.sendTelemetryEvent({
+            eventName,
+            entryStatus: pendingEntry.status,
+            timeLapseSinceLocalUpload,
+            timeLapseSinceServerUpload,
+            minTTLInSeconds: pendingEntry.minTTLInSeconds,
+            expiredUsingLocalTime,
+            expiredUsingServerTime,
+        });
     }
     processBlobAttachOp(message, local) {
         var _a, _b;
-        assert((_a = message === null || message === void 0 ? void 0 : message.metadata) === null || _a === void 0 ? void 0 : _a.blobId, 0x12a /* "Missing blob id on metadata" */);
-        if (message.metadata.localId !== undefined) {
-            this.redirectTable.set(message.metadata.localId, message.metadata.blobId);
+        const localId = (_a = message.metadata) === null || _a === void 0 ? void 0 : _a.localId;
+        const blobId = (_b = message.metadata) === null || _b === void 0 ? void 0 : _b.blobId;
+        assert(blobId !== undefined, 0x12a /* "Missing blob id on metadata" */);
+        // Set up a mapping from local ID to storage ID. This is crucial since without this the blob cannot be
+        // requested from the server.
+        // Note: The check for undefined is needed for back-compat when localId was not part of the BlobAttach op that
+        // was sent when online.
+        if (localId !== undefined) {
+            this.setRedirection(localId, blobId);
         }
         // set identity (id -> id) entry
-        this.redirectTable.set(message.metadata.blobId, message.metadata.blobId);
+        this.setRedirection(blobId, blobId);
         if (local) {
-            if (message.metadata.localId === undefined) {
-                // Since there is no local ID, we know this op was submitted while online.
-                const waitingBlobs = this.opsInFlight.get(message.metadata.blobId);
-                assert(!!waitingBlobs, 0x38e /* local online BlobAttach op with no pending blob */);
-                waitingBlobs.forEach((localId) => {
-                    const pendingBlobEntry = this.pendingBlobs.get(localId);
-                    assert(pendingBlobEntry !== undefined, 0x38f);
+            assert(localId !== undefined, 0x50e /* local ID not present in blob attach message */);
+            const waitingBlobs = this.opsInFlight.get(blobId);
+            if (waitingBlobs !== undefined) {
+                // For each op corresponding to this storage ID that we are waiting for, resolve the pending blob.
+                // This is safe because the server will keep the blob alive and the op containing the local ID to
+                // storage ID is already in flight and any op containing this local ID will be sequenced after that.
+                waitingBlobs.forEach((pendingLocalId) => {
+                    const pendingBlobEntry = this.pendingBlobs.get(pendingLocalId);
+                    assert(pendingBlobEntry !== undefined, 0x38f /* local online BlobAttach op with no pending blob entry */);
                     // It's possible we transitioned to offline flow while waiting for this op.
                     if (pendingBlobEntry.status === PendingBlobStatus.OnlinePendingOp) {
-                        pendingBlobEntry.handleP.resolve(this.getBlobHandle(message.metadata.blobId));
-                        this.pendingBlobs.delete(localId);
+                        this.setRedirection(pendingLocalId, blobId);
+                        pendingBlobEntry.handleP.resolve(this.getBlobHandle(pendingLocalId));
+                        this.deleteAndEmitsIfEmpty(pendingLocalId);
                     }
                 });
+                this.opsInFlight.delete(blobId);
             }
-            else {
-                // Each local ID is unique; get the pending blob entry and delete it
-                assert(((_b = this.pendingBlobs.get(message.metadata.localId)) === null || _b === void 0 ? void 0 : _b.status) === PendingBlobStatus.OfflinePendingOp, 0x1f8 /* "local BlobAttach op with no pending blob" */);
-                this.pendingBlobs.delete(message.metadata.localId);
-            }
+            // For blobs that were transitioned to offline flow while waiting for this op, the entry should be deleted.
+            this.deleteAndEmitsIfEmpty(localId);
         }
     }
     /**
@@ -375,7 +463,8 @@ export class BlobManager {
             redirectTable = await tryFetchBlob(tableId);
         }
         const ids = Object.entries(blobsTree.blobs)
-            .filter(([k, _]) => k !== this.redirectTableBlobName).map(([_, v]) => v);
+            .filter(([k, _]) => k !== this.redirectTableBlobName)
+            .map(([_, v]) => v);
         return { ids, redirectTable };
     }
     /**
@@ -405,69 +494,93 @@ export class BlobManager {
      */
     getGCData(fullGC = false) {
         const gcData = { gcNodes: {} };
-        /**
-          * The node path is of the format `/_blobs/blobId`. This path must match the path of the blob handle returned
-          * by the createBlob API because blobs are marked referenced by storing these handles in a referenced DDS.
-          */
-        this.storageIds.forEach((blobId) => {
-            gcData.gcNodes[this.getBlobGCNodePath(blobId)] = [];
-        });
-        // For some blobs, the handle returned on creation is based off of the localId. So, these
-        // nodes can be referenced by storing the localId handle. When that happens, the corresponding storageId node
-        // must also be marked referenced. So, we add a route from the localId node to the storageId node.
-        // Note that because of de-duping, there can be multiple localIds that all redirect to the same storageId or
-        // a blob may be referenced via its storageId handle.
         for (const [localId, storageId] of this.redirectTable) {
             assert(!!storageId, 0x390 /* Must be attached to get GC data */);
-            // Add node for the localId and add a route to the storageId node. The storageId node will have been
-            // added above when adding nodes for this.blobIds.
             gcData.gcNodes[this.getBlobGCNodePath(localId)] = [this.getBlobGCNodePath(storageId)];
         }
         return gcData;
     }
     /**
-     * This is called to update blobs whose routes are used. The used blobs are removed from the tombstone list.
-     * @param usedRoutes - The routes of the blob nodes that are used.
+     * This is called to update blobs whose routes are unused. The unused blobs are deleted.
+     * @param unusedRoutes - The routes of the blob nodes that are unused.
      */
-    updateUsedRoutes(usedRoutes) {
+    updateUnusedRoutes(unusedRoutes) {
         // The routes or blob node paths are in the same format as returned in getGCData -
         // `/<BlobManager.basePath>/<blobId>`.
-        for (const route of usedRoutes) {
+        for (const route of unusedRoutes) {
             const pathParts = route.split("/");
-            assert(pathParts.length === 3 && pathParts[1] === BlobManager.basePath, 0x4bc /* Invalid blob node id in used routes. */);
+            assert(pathParts.length === 3 && pathParts[1] === BlobManager.basePath, 0x2d5 /* "Invalid blob node id in unused routes." */);
             const blobId = pathParts[2];
-            // Un-tombstone the blob if it was marked tombstone.
-            this.tombstonedBlobs.delete(blobId);
+            this.redirectTable.delete(blobId);
         }
     }
     /**
-     * This is called to update blobs whose routes are unused. The unused blobs are either deleted or marked as
-     * tombstones.
-     * @param unusedRoutes - The routes of the blob nodes that are unused.
-     * @param tombstone - if true, the objects corresponding to unused routes are marked tombstones. Otherwise, they
-     * are deleted.
+     * This is called to update blobs whose routes are tombstones. Tombstoned blobs enable testing scenarios with
+     * accessing deleted content without actually deleting content from summaries.
+     * @param tombstonedRoutes - The routes of blob nodes that are tombstones.
      */
-    updateUnusedRoutes(unusedRoutes, tombstone) {
+    updateTombstonedRoutes(tombstonedRoutes) {
+        const tombstonedBlobsSet = new Set();
         // The routes or blob node paths are in the same format as returned in getGCData -
         // `/<BlobManager.basePath>/<blobId>`.
-        for (const route of unusedRoutes) {
+        for (const route of tombstonedRoutes) {
             const pathParts = route.split("/");
-            assert(pathParts.length === 3 && pathParts[1] === BlobManager.basePath, 0x2d5 /* "Invalid blob node id in unused routes." */);
-            const blobId = pathParts[2];
-            if (tombstone) {
-                // If tombstone is set, add this blob to the tombstone list.
-                this.tombstonedBlobs.add(blobId);
-            }
-            else {
-                // The unused blobId could be a localId. If so, remove it from the redirect table and continue. The
-                // corresponding storageId may still be used either directly or via other localIds.
-                this.redirectTable.delete(blobId);
+            assert(pathParts.length === 3 && pathParts[1] === BlobManager.basePath, 0x50f /* Invalid blob node id in tombstoned routes. */);
+            tombstonedBlobsSet.add(pathParts[2]);
+        }
+        // Remove blobs from the tombstone list that were tombstoned but aren't anymore as per the tombstoneRoutes.
+        for (const blobId of this.tombstonedBlobs) {
+            if (!tombstonedBlobsSet.has(blobId)) {
+                this.tombstonedBlobs.delete(blobId);
             }
         }
+        // Mark blobs that are now tombstoned by adding them to the tombstone list.
+        for (const blobId of tombstonedBlobsSet) {
+            this.tombstonedBlobs.add(blobId);
+        }
+    }
+    /**
+     * Verifies that the blob with given id is valid, i.e., it has not been garbage collected. If the blob is GC'd,
+     * log an error and throw if necessary.
+     */
+    verifyBlobValidity(blobId) {
+        /**
+         * A blob can be in one of the following states:
+         * 1. "deleted" - It has been deleted by garbage collection sweep phase.
+         * 2. "tombstoned" - It is ready for deletion but sweep phase isn't enabled and tombstone feature is enabled.
+         * 3. "valid" - It has not been deleted or tombstoned.
+         */
+        let state = "valid";
+        if (this.isBlobDeleted(this.getBlobGCNodePath(blobId))) {
+            state = "deleted";
+        }
+        else if (this.tombstonedBlobs.has(blobId)) {
+            state = "tombstoned";
+        }
+        if (state === "valid") {
+            return;
+        }
+        // If the blob is deleted or throw on tombstone load is enabled, throw an error which will fail any attempt
+        // to load the blob.
+        const shouldFail = state === "deleted" || this.throwOnTombstoneLoad;
+        const request = { url: blobId };
+        const error = responseToException(createResponseError(404, "Blob was deleted", request, state === "tombstoned" ? { [TombstoneResponseHeaderKey]: true } : undefined), request);
+        sendGCUnexpectedUsageEvent(this.mc, {
+            eventName: state === "tombstoned"
+                ? "GC_Tombstone_Blob_Requested"
+                : "GC_Deleted_Blob_Requested",
+            category: shouldFail ? "error" : "generic",
+            isSummarizerClient: this.runtime.clientDetails.type === summarizerClientType,
+        }, [BlobManager.basePath], error);
+        if (shouldFail) {
+            throw error;
+        }
     }
     summarize(telemetryContext) {
         // if storageIds is empty, it means we are detached and have only local IDs, or that there are no blobs attached
-        const blobIds = this.storageIds.size > 0 ? Array.from(this.storageIds) : Array.from(this.redirectTable.keys());
+        const blobIds = this.storageIds.size > 0
+            ? Array.from(this.storageIds)
+            : Array.from(this.redirectTable.keys());
         const builder = new SummaryTreeBuilder();
         blobIds.forEach((blobId) => {
             builder.addAttachment(blobId);
@@ -476,8 +589,7 @@ export class BlobManager {
         if (this.redirectTable.size > blobIds.length) {
             builder.addBlob(BlobManager.redirectTableBlobName, 
             // filter out identity entries
-            JSON.stringify(Array.from(this.redirectTable.entries())
-                .filter(([localId, storageId]) => localId !== storageId)));
+            JSON.stringify(Array.from(this.redirectTable.entries()).filter(([localId, storageId]) => localId !== storageId)));
         }
         return builder.getSummaryTree();
     }
@@ -486,9 +598,9 @@ export class BlobManager {
         assert(this.redirectTable.size === table.size, 0x391 /* Redirect table size must match BlobManager's local ID count */);
         for (const [localId, storageId] of table) {
             assert(this.redirectTable.has(localId), 0x254 /* "unrecognized id in redirect table" */);
-            this.redirectTable.set(localId, storageId);
+            this.setRedirection(localId, storageId);
             // set identity (id -> id) entry
-            this.redirectTable.set(storageId, storageId);
+            this.setRedirection(storageId, storageId);
         }
     }
     getPendingBlobs() {