@fluidframework/container-runtime 2.53.1 → 2.61.0-355054

package/lib/blobManager/blobManager.js

@@ -5,9 +5,8 @@
 import { createEmitter } from "@fluid-internal/client-utils";
 import { AttachState, } from "@fluidframework/container-definitions/internal";
 import { assert, Deferred } from "@fluidframework/core-utils/internal";
-import { canRetryOnError, runWithRetry } from "@fluidframework/driver-utils/internal";
 import { FluidHandleBase, createResponseError, generateHandleContextPath, responseToException, } from "@fluidframework/runtime-utils/internal";
-import { LoggingError, PerformanceEvent, UsageError, createChildMonitoringContext, wrapError, } from "@fluidframework/telemetry-utils/internal";
+import { LoggingError, PerformanceEvent, UsageError, createChildMonitoringContext, } from "@fluidframework/telemetry-utils/internal";
 import { v4 as uuid } from "uuid";
 import { isBlobMetadata } from "../metadata.js";
 import { getStorageIds, summarizeBlobManagerState, toRedirectTable, } from "./blobManagerSnapSum.js";
@@ -75,20 +74,20 @@ export class BlobManager {
          * because we know that the server will not delete the blob corresponding to that storage ID.
          */
         this.opsInFlight = new Map();
-        const { routeContext, blobManagerLoadInfo, storage, sendBlobAttachOp, blobRequested, isBlobDeleted, runtime, localBlobIdGenerator, createBlobPayloadPending, } = props;
+        const { routeContext, blobManagerLoadInfo, storage, sendBlobAttachOp, blobRequested, isBlobDeleted, runtime, localIdGenerator, createBlobPayloadPending, } = props;
         this.routeContext = routeContext;
         this.storage = storage;
         this.blobRequested = blobRequested;
         this.isBlobDeleted = isBlobDeleted;
         this.runtime = runtime;
-        this.localBlobIdGenerator = localBlobIdGenerator ?? uuid;
+        this.localIdGenerator = localIdGenerator ?? uuid;
         this.createBlobPayloadPending = createBlobPayloadPending;
         this.mc = createChildMonitoringContext({
             logger: this.runtime.baseLogger,
             namespace: "BlobManager",
         });
-        this.redirectTable = toRedirectTable(blobManagerLoadInfo, this.mc.logger, this.runtime.attachState);
-        this.sendBlobAttachOp = (localId, blobId) => {
+        this.redirectTable = toRedirectTable(blobManagerLoadInfo, this.mc.logger);
+        this.sendBlobAttachOp = (localId, storageId) => {
             const pendingEntry = this.pendingBlobs.get(localId);
             assert(pendingEntry !== undefined, 0x725 /* Must have pending blob entry for upcoming op */);
             if (pendingEntry?.uploadTime && pendingEntry?.minTTLInSeconds) {
@@ -114,7 +113,7 @@ export class BlobManager {
                 }
             }
             pendingEntry.opsent = true;
-            sendBlobAttachOp(localId, blobId);
+            sendBlobAttachOp(localId, storageId);
         };
     }
     get allBlobsAttached() {
@@ -135,56 +134,46 @@ export class BlobManager {
             uploadTime: pending?.uploadTime,
         });
     }
-    hasBlob(blobId) {
-        return this.redirectTable.get(blobId) !== undefined;
+    hasBlob(localId) {
+        return this.redirectTable.get(localId) !== undefined;
     }
     /**
      * Retrieve the blob with the given local blob id.
-     * @param blobId - The local blob id.  Likely coming from a handle.
+     * @param localId - The local blob id.  Likely coming from a handle.
      * @param payloadPending - Whether we suspect the payload may be pending and not available yet.
      * @returns A promise which resolves to the blob contents
      */
-    async getBlob(blobId, payloadPending) {
+    async getBlob(localId, payloadPending) {
         // Verify that the blob is not deleted, i.e., it has not been garbage collected. If it is, this will throw
         // an error, failing the call.
-        this.verifyBlobNotDeleted(blobId);
+        this.verifyBlobNotDeleted(localId);
         // Let runtime know that the corresponding GC node was requested.
         // Note that this will throw if the blob is inactive or tombstoned and throwing on incorrect usage
         // is configured.
-        this.blobRequested(getGCNodePathFromBlobId(blobId));
-        const pending = this.pendingBlobs.get(blobId);
+        this.blobRequested(getGCNodePathFromLocalId(localId));
+        const pending = this.pendingBlobs.get(localId);
         if (pending) {
             return pending.blob;
         }
-        let storageId;
-        if (this.runtime.attachState === AttachState.Detached) {
-            assert(this.redirectTable.has(blobId), 0x383 /* requesting unknown blobs */);
-            // Blobs created while the container is detached are stored in IDetachedBlobStorage.
-            // The 'IContainerStorageService.readBlob()' call below will retrieve these via localId.
-            storageId = blobId;
-        }
-        else {
-            const attachedStorageId = this.redirectTable.get(blobId);
-            if (!payloadPending) {
-                // Only blob handles explicitly marked with pending payload are permitted to exist without
-                // yet knowing their storage id. Otherwise they must already be associated with a storage id.
-                assert(attachedStorageId !== undefined, 0x11f /* "requesting unknown blobs" */);
-            }
-            // If we didn't find it in the redirectTable, assume the attach op is coming eventually and wait.
-            // We do this even if the local client doesn't have the blob payloadPending flag enabled, in case a
-            // remote client does have it enabled. This wait may be infinite if the uploading client failed
-            // the upload and doesn't exist anymore.
-            storageId =
-                attachedStorageId ??
-                    (await new Promise((resolve) => {
-                        const onProcessBlobAttach = (localId, _storageId) => {
-                            if (localId === blobId) {
-                                this.internalEvents.off("processedBlobAttach", onProcessBlobAttach);
-                                resolve(_storageId);
-                            }
-                        };
-                        this.internalEvents.on("processedBlobAttach", onProcessBlobAttach);
-                    }));
+        let storageId = this.redirectTable.get(localId);
+        if (storageId === undefined) {
+            // Only blob handles explicitly marked with pending payload are permitted to exist without
+            // yet knowing their storage id. Otherwise they must already be associated with a storage id.
+            // Handles for detached blobs are not payload pending.
+            assert(payloadPending, 0x11f /* "requesting unknown blobs" */);
+            // If we didn't find it in the redirectTable and it's payloadPending, assume the attach op is coming
+            // eventually and wait. We do this even if the local client doesn't have the blob payloadPending flag
+            // enabled, in case a remote client does have it enabled. This wait may be infinite if the uploading
+            // client failed the upload and doesn't exist anymore.
+            storageId = await new Promise((resolve) => {
+                const onProcessBlobAttach = (_localId, _storageId) => {
+                    if (_localId === localId) {
+                        this.internalEvents.off("processedBlobAttach", onProcessBlobAttach);
+                        resolve(_storageId);
+                    }
+                };
+                this.internalEvents.on("processedBlobAttach", onProcessBlobAttach);
+            });
         }
         return PerformanceEvent.timedExecAsync(this.mc.logger, { eventName: "AttachmentReadBlob", id: storageId }, async (event) => {
             return this.storage.readBlob(storageId).catch((error) => {
@@ -208,15 +197,17 @@ export class BlobManager {
                 this.deletePendingBlobMaybe(localId);
             }
             : undefined;
-        return new BlobHandle(getGCNodePathFromBlobId(localId), this.routeContext, async () => this.getBlob(localId, false), false, // payloadPending
+        return new BlobHandle(getGCNodePathFromLocalId(localId), this.routeContext, async () => this.getBlob(localId, false), false, // payloadPending
         callback);
     }
     async createBlobDetached(blob) {
+        const localId = this.localIdGenerator();
         // Blobs created while the container is detached are stored in IDetachedBlobStorage.
-        // The 'IContainerStorageService.createBlob()' call below will respond with a localId.
-        const response = await this.storage.createBlob(blob);
-        this.setRedirection(response.id, undefined);
-        return this.getBlobHandle(response.id);
+        // The 'IContainerStorageService.createBlob()' call below will respond with a pseudo storage ID.
+        // That pseudo storage ID will be replaced with the real storage ID at attach time.
+        const { id: detachedStorageId } = await this.storage.createBlob(blob);
+        this.setRedirection(localId, detachedStorageId);
+        return this.getBlobHandle(localId);
     }
     async createBlob(blob, signal) {
         if (this.runtime.attachState === AttachState.Detached) {
@@ -238,7 +229,7 @@ export class BlobManager {
         }
         // Create a local ID for the blob. After uploading it to storage and before returning it, a local ID to
         // storage ID mapping is created.
-        const localId = this.localBlobIdGenerator();
+        const localId = this.localIdGenerator();
         const pendingEntry = {
             blob,
             handleP: new Deferred(),
@@ -260,8 +251,8 @@ export class BlobManager {
         });
     }
     createBlobWithPayloadPending(blob) {
-        const localId = this.localBlobIdGenerator();
-        const blobHandle = new BlobHandle(getGCNodePathFromBlobId(localId), this.routeContext, async () => blob, true, // payloadPending
+        const localId = this.localIdGenerator();
+        const blobHandle = new BlobHandle(getGCNodePathFromLocalId(localId), this.routeContext, async () => blob, true, // payloadPending
         () => {
             const pendingEntry = {
                 blob,
@@ -289,36 +280,49 @@ export class BlobManager {
         this.internalEvents.on("uploadFailed", onUploadFailed);
         return blobHandle;
     }
+    /**
+     * Upload a blob to the storage service.
+     * @returns A promise that resolves when the upload is complete and a blob attach op has been sent (but not ack'd).
+     *
+     * @privateRemarks This method must not reject, as there is no error handling for it in current tracking.
+     */
     async uploadBlob(localId, blob) {
-        return runWithRetry(async () => {
-            try {
-                return await this.storage.createBlob(blob);
-            }
-            catch (error) {
-                const entry = this.pendingBlobs.get(localId);
-                assert(!!entry, 0x387 /* Must have pending blob entry for blob which failed to upload */);
-                if (entry.opsent && !canRetryOnError(error)) {
-                    throw wrapError(error, () => new LoggingError(`uploadBlob error`, { canRetry: true }));
-                }
-                throw error;
-            }
-        }, "createBlob", this.mc.logger, {
-            cancel: this.pendingBlobs.get(localId)?.abortSignal,
-        }).then((response) => this.onUploadResolve(localId, response), (error) => {
+        let response;
+        try {
+            response = await this.storage.createBlob(blob);
+        }
+        catch (error) {
+            const entry = this.pendingBlobs.get(localId);
             this.mc.logger.sendTelemetryEvent({
                 eventName: "UploadBlobReject",
                 // TODO: better typing
-                // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
-                error,
+                // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any
+                error: error,
+                message: entry === undefined ? "Missing pendingBlob" : undefined,
                 localId,
             });
-            // it will only reject if we haven't sent an op
-            // and is a non-retriable error. It will only reject
-            // the promise but not throw any error outside.
-            this.pendingBlobs.get(localId)?.handleP.reject(error);
-            this.deletePendingBlob(localId);
+            // We probably should assert the pendingBlobs entry here, but we don't currently have any error handling
+            // for the uploadP - a promise rejection would be unhandled anyway. For now we can detect this with the
+            // message on the UploadBlobReject telemetry.
+            if (entry !== undefined) {
+                entry.handleP.reject(error);
+                this.deletePendingBlob(localId);
+            }
             this.internalEvents.emit("uploadFailed", localId, error);
-        });
+            return;
+        }
+        try {
+            this.onUploadResolve(localId, response);
+        }
+        catch (error) {
+            this.mc.logger.sendTelemetryEvent({
+                eventName: "OnUploadResolveError",
+                // TODO: better typing
+                // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any
+                error: error,
+                localId,
+            });
+        }
     }
     /**
      * Set up a mapping in the redirect table from fromId to toId. Also, notify the runtime that a reference is added
@@ -363,7 +367,7 @@ export class BlobManager {
         if (!entry.opsent) {
             this.sendBlobAttachOp(localId, response.id);
         }
-        const storageIds = getStorageIds(this.redirectTable, this.runtime.attachState);
+        const storageIds = getStorageIds(this.redirectTable);
         if (storageIds.has(response.id)) {
             // The blob is de-duped. Set up a local ID to storage ID mapping and return the blob. Since this is
             // an existing blob, we don't have to wait for the op to be ack'd since this step has already
@@ -396,7 +400,7 @@ export class BlobManager {
      */
     reSubmit(metadata) {
         assert(isBlobMetadata(metadata), 0xc01 /* Expected blob metadata for a BlobAttach op */);
-        const { localId, blobId } = metadata;
+        const { localId, blobId: storageId } = metadata;
         // Any blob that we're actively trying to advance to attached state must have a
         // pendingBlobs entry. Decline to resubmit for anything else.
         // For example, we might be asked to resubmit stashed ops for blobs that never had
@@ -404,22 +408,22 @@ export class BlobManager {
         // try to attach them since they won't be accessible to the customer and would just
         // be considered garbage immediately.
         if (this.pendingBlobs.has(localId)) {
-            this.sendBlobAttachOp(localId, blobId);
+            this.sendBlobAttachOp(localId, storageId);
         }
     }
     processBlobAttachMessage(message, local) {
         assert(isBlobMetadata(message.metadata), 0xc02 /* Expected blob metadata for a BlobAttach op */);
-        const { localId, blobId } = message.metadata;
+        const { localId, blobId: storageId } = message.metadata;
         const pendingEntry = this.pendingBlobs.get(localId);
         if (pendingEntry?.abortSignal?.aborted) {
             this.deletePendingBlob(localId);
             return;
         }
-        this.setRedirection(localId, blobId);
+        this.setRedirection(localId, storageId);
         // set identity (id -> id) entry
-        this.setRedirection(blobId, blobId);
+        this.setRedirection(storageId, storageId);
         if (local) {
-            const waitingBlobs = this.opsInFlight.get(blobId);
+            const waitingBlobs = this.opsInFlight.get(storageId);
             if (waitingBlobs !== undefined) {
                 // For each op corresponding to this storage ID that we are waiting for, resolve the pending blob.
                 // This is safe because the server will keep the blob alive and the op containing the local ID to
@@ -427,14 +431,14 @@ export class BlobManager {
                 for (const pendingLocalId of waitingBlobs) {
                     const entry = this.pendingBlobs.get(pendingLocalId);
                     assert(entry !== undefined, 0x38f /* local online BlobAttach op with no pending blob entry */);
-                    this.setRedirection(pendingLocalId, blobId);
+                    this.setRedirection(pendingLocalId, storageId);
                     entry.acked = true;
                     const blobHandle = this.getBlobHandle(pendingLocalId);
                     blobHandle.notifyShared();
                     entry.handleP.resolve(blobHandle);
                     this.deletePendingBlobMaybe(pendingLocalId);
                 }
-                this.opsInFlight.delete(blobId);
+                this.opsInFlight.delete(storageId);
             }
             const localEntry = this.pendingBlobs.get(localId);
             if (localEntry) {
@@ -445,10 +449,10 @@ export class BlobManager {
                 this.deletePendingBlobMaybe(localId);
             }
         }
-        this.internalEvents.emit("processedBlobAttach", localId, blobId);
+        this.internalEvents.emit("processedBlobAttach", localId, storageId);
     }
     summarize(telemetryContext) {
-        return summarizeBlobManagerState(this.redirectTable, this.runtime.attachState);
+        return summarizeBlobManagerState(this.redirectTable);
     }
     /**
      * Generates data used for garbage collection. Each blob uploaded represents a node in the GC graph as it can be
@@ -459,13 +463,12 @@ export class BlobManager {
     getGCData(fullGC = false) {
         const gcData = { gcNodes: {} };
         for (const [localId, storageId] of this.redirectTable) {
-            assert(!!storageId, 0x390 /* Must be attached to get GC data */);
-            // Only return local ids as GC nodes because a blob can only be referenced via its local id. The storage
-            // id entries have the same key and value, ignore them.
-            // The outbound routes are empty because a blob node cannot reference other nodes. It can only be referenced
-            // by adding its handle to a referenced DDS.
+            // Don't report the identity mappings to GC - these exist to service old handles that referenced the storage
+            // IDs directly. We'll implicitly clean them up if all of their localId references get GC'd first.
             if (localId !== storageId) {
-                gcData.gcNodes[getGCNodePathFromBlobId(localId)] = [];
+                // The outbound routes are empty because a blob node cannot reference other nodes. It can only be referenced
+                // by adding its handle to a referenced DDS.
+                gcData.gcNodes[getGCNodePathFromLocalId(localId)] = [];
             }
         }
         return gcData;
@@ -484,55 +487,53 @@ export class BlobManager {
      * Delete blobs with the given routes from the redirect table.
      *
      * @remarks
-     * The routes are GC nodes paths of format -`/<blobManagerBasePath>/<blobId>`. The blob ids are all local ids.
+     * The routes are GC nodes paths of format -`/<blobManagerBasePath>/<localId>`.
      * Deleting the blobs involves 2 steps:
      *
      * 1. The redirect table entry for the local ids are deleted.
      *
-     * 2. If the storage ids corresponding to the deleted local ids are not in-use anymore, the redirect table entries
-     * for the storage ids are deleted as well.
+     * 2. If the storage ids corresponding to the deleted local ids are not referenced by any further local ids, the
+     * identity mappings in the redirect table are deleted as well.
      *
      * Note that this does not delete the blobs from storage service immediately. Deleting the blobs from redirect table
-     * will remove them the next summary. The service would them delete them some time in the future.
+     * will ensure we don't create an attachment blob for them at the next summary. The service would then delete them
+     * some time in the future.
      */
     deleteBlobsFromRedirectTable(blobRoutes) {
-        if (blobRoutes.length === 0) {
-            return;
-        }
-        // This tracks the storage ids of local ids that are deleted. After the local ids have been deleted, if any of
-        // these storage ids are unused, they will be deleted as well.
+        // maybeUnusedStorageIds is used to compute the set of storage IDs that *used to have a local ID*, but that
+        // local ID is being deleted.
         const maybeUnusedStorageIds = new Set();
         for (const route of blobRoutes) {
-            const blobId = getBlobIdFromGCNodePath(route);
+            const localId = getLocalIdFromGCNodePath(route);
             // If the blob hasn't already been deleted, log an error because this should never happen.
             // If the blob has already been deleted, log a telemetry event. This can happen because multiple GC
             // sweep ops can contain the same data store. It would be interesting to track how often this happens.
             const alreadyDeleted = this.isBlobDeleted(route);
-            if (!this.redirectTable.has(blobId)) {
+            const storageId = this.redirectTable.get(localId);
+            if (storageId === undefined) {
                 this.mc.logger.sendTelemetryEvent({
                     eventName: "DeletedAttachmentBlobNotFound",
                     category: alreadyDeleted ? "generic" : "error",
-                    blobId,
+                    blobId: localId,
                     details: { alreadyDeleted },
                 });
                 continue;
             }
-            const storageId = this.redirectTable.get(blobId);
-            assert(!!storageId, 0x5bb /* Must be attached to run GC */);
             maybeUnusedStorageIds.add(storageId);
-            this.redirectTable.delete(blobId);
+            this.redirectTable.delete(localId);
         }
-        // Find out storage ids that are in-use and remove them from maybeUnusedStorageIds. A storage id is in-use if
-        // the redirect table has a local id -> storage id entry for it.
-        for (const [localId, storageId] of this.redirectTable.entries()) {
-            assert(!!storageId, 0x5bc /* Must be attached to run GC */);
-            // For every storage id, the redirect table has a id -> id entry. These do not make the storage id in-use.
-            if (maybeUnusedStorageIds.has(storageId) && localId !== storageId) {
+        // Remove any storage IDs that still have local IDs referring to them (excluding the identity mapping).
+        for (const [localId, storageId] of this.redirectTable) {
+            if (localId !== storageId) {
                 maybeUnusedStorageIds.delete(storageId);
             }
         }
-        // For unused storage ids, delete their id -> id entries from the redirect table.
-        // This way they'll be absent from the next summary, and the service is free to delete them from storage.
+        // Now delete any identity mappings (storage ID -> storage ID) from the redirect table that used to be
+        // referenced by a distinct local ID. This way they'll be absent from the next summary, and the service
+        // is free to delete them from storage.
+        // WARNING: This can potentially delete identity mappings that are still referenced, if storage deduping
+        // has let us add a local ID -> storage ID mapping that is later deleted.  AB#47337 tracks this issue
+        // and possible solutions.
         for (const storageId of maybeUnusedStorageIds) {
             this.redirectTable.delete(storageId);
         }
@@ -541,11 +542,11 @@ export class BlobManager {
      * Verifies that the blob with given id is not deleted, i.e., it has not been garbage collected. If the blob is GC'd,
      * log an error and throw if necessary.
      */
-    verifyBlobNotDeleted(blobId) {
-        if (!this.isBlobDeleted(getGCNodePathFromBlobId(blobId))) {
+    verifyBlobNotDeleted(localId) {
+        if (!this.isBlobDeleted(getGCNodePathFromLocalId(localId))) {
             return;
         }
-        const request = { url: blobId };
+        const request = { url: localId };
         const error = responseToException(createResponseError(404, `Blob was deleted`, request), request);
         // Only log deleted events. Tombstone events are logged by garbage collector.
         this.mc.logger.sendErrorEvent({
@@ -554,14 +555,28 @@ export class BlobManager {
         }, error);
         throw error;
     }
-    setRedirectTable(table) {
+    /**
+     * Called in detached state just prior to attaching, this will update the redirect table by
+     * converting the pseudo storage IDs into real storage IDs using the provided detachedStorageTable.
+     * The provided table must have exactly the same set of pseudo storage IDs as are found in the redirect table.
+     * @param detachedStorageTable - A map of pseudo storage IDs to real storage IDs.
+     */
+    patchRedirectTable(detachedStorageTable) {
         assert(this.runtime.attachState === AttachState.Detached, 0x252 /* "redirect table can only be set in detached container" */);
-        assert(this.redirectTable.size === table.size, 0x391 /* Redirect table size must match BlobManager's local ID count */);
-        for (const [localId, storageId] of table) {
-            assert(this.redirectTable.has(localId), 0x254 /* "unrecognized id in redirect table" */);
-            this.setRedirection(localId, storageId);
+        // The values of the redirect table are the pseudo storage IDs, which are the keys of the
+        // detachedStorageTable. We expect to have a many:1 mapping from local IDs to pseudo
+        // storage IDs (many in the case that the storage dedupes the blob).
+        assert(new Set(this.redirectTable.values()).size === detachedStorageTable.size, 0x391 /* Redirect table size must match BlobManager's local ID count */);
+        // Taking a snapshot of the redirect table entries before iterating, because
+        // we will be adding identity mappings to the the redirect table as we iterate
+        // and we don't want to include those in the iteration.
+        const redirectTableEntries = [...this.redirectTable.entries()];
+        for (const [localId, detachedStorageId] of redirectTableEntries) {
+            const newStorageId = detachedStorageTable.get(detachedStorageId);
+            assert(newStorageId !== undefined, "Couldn't find a matching storage ID");
+            this.setRedirection(localId, newStorageId);
             // set identity (id -> id) entry
-            this.setRedirection(storageId, storageId);
+            this.setRedirection(newStorageId, newStorageId);
         }
     }
     /**
@@ -593,15 +608,15 @@ export class BlobManager {
     }
 }
 /**
- * For a blobId, returns its path in GC's graph. The node path is of the format `/<blobManagerBasePath>/<blobId>`.
+ * For a localId, returns its path in GC's graph. The node path is of the format `/<blobManagerBasePath>/<localId>`.
  * This path must match the path of the blob handle returned by the createBlob API because blobs are marked
  * referenced by storing these handles in a referenced DDS.
  */
-const getGCNodePathFromBlobId = (blobId) => `/${blobManagerBasePath}/${blobId}`;
+const getGCNodePathFromLocalId = (localId) => `/${blobManagerBasePath}/${localId}`;
 /**
- * For a given GC node path, return the blobId. The node path is of the format `/<basePath>/<blobId>`.
+ * For a given GC node path, return the localId. The node path is of the format `/<basePath>/<localId>`.
  */
-const getBlobIdFromGCNodePath = (nodePath) => {
+const getLocalIdFromGCNodePath = (nodePath) => {
     const pathParts = nodePath.split("/");
     assert(areBlobPathParts(pathParts), 0x5bd /* Invalid blob node path */);
     return pathParts[2];