@fluidframework/container-runtime 2.0.0-dev.3.1.0.125672 → 2.0.0-dev.4.2.0.153917

package/dist/blobManager.js

@@ -10,11 +10,11 @@ const runtime_utils_1 = require("@fluidframework/runtime-utils");
 const common_utils_1 = require("@fluidframework/common-utils");
 const container_definitions_1 = require("@fluidframework/container-definitions");
 const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
+const container_utils_1 = require("@fluidframework/container-utils");
 const containerRuntime_1 = require("./containerRuntime");
+const gc_1 = require("./gc");
 const throttler_1 = require("./throttler");
-const summarizerClientElection_1 = require("./summarizerClientElection");
-const garbageCollectionConstants_1 = require("./garbageCollectionConstants");
-const garbageCollectionHelpers_1 = require("./garbageCollectionHelpers");
+const summary_1 = require("./summary");
 /**
  * This class represents blob (long string)
  * This object is used only when creating (writing) new blob and serialization purposes.
@@ -85,22 +85,16 @@ class BlobManager extends common_utils_1.TypedEventEmitter {
     // Called when a blob node is requested. blobPath is the path of the blob's node in GC's graph.
     // blobPath's format - `/<BlobManager.basePath>/<blobId>`.
     blobRequested, 
-    // Called when a reference is added to a blob. For instance, when creating a localId / storageId to storageId
-    // mapping in the redirect table.
-    // Node path formats - `/<BlobManager.basePath>/<blobId>`.
-    addedBlobReference, 
     // Called to check if a blob has been deleted by GC.
     // blobPath's format - `/<BlobManager.basePath>/<blobId>`.
-    isBlobDeleted, runtime, stashedBlobs = {}, getCurrentReferenceTimestampMs) {
+    isBlobDeleted, runtime, stashedBlobs = {}, closeContainer) {
         super();
         this.routeContext = routeContext;
         this.getStorage = getStorage;
-        this.sendBlobAttachOp = sendBlobAttachOp;
         this.blobRequested = blobRequested;
-        this.addedBlobReference = addedBlobReference;
         this.isBlobDeleted = isBlobDeleted;
         this.runtime = runtime;
-        this.getCurrentReferenceTimestampMs = getCurrentReferenceTimestampMs;
+        this.closeContainer = closeContainer;
         /**
          * Blobs which have not been uploaded or for which we have not yet seen a BlobAttach op round-trip.
          * Until we see the op round-trip, there is a possibility we may need to re-upload the blob, so
@@ -125,13 +119,29 @@ class BlobManager extends common_utils_1.TypedEventEmitter {
         this.mc = (0, telemetry_utils_1.loggerToMonitoringContext)(telemetry_utils_1.ChildLogger.create(this.runtime.logger, "BlobManager"));
         // Read the feature flag that tells whether to throw when a tombstone blob is requested.
         this.throwOnTombstoneLoad =
-            this.mc.config.getBoolean(garbageCollectionConstants_1.throwOnTombstoneLoadKey) === true &&
-                this.runtime.clientDetails.type !== summarizerClientElection_1.summarizerClientType;
+            this.mc.config.getBoolean(gc_1.throwOnTombstoneLoadKey) === true &&
+                this.runtime.gcTombstoneEnforcementAllowed &&
+                this.runtime.clientDetails.type !== summary_1.summarizerClientType;
         this.runtime.on("disconnected", () => this.onDisconnected());
         this.redirectTable = this.load(snapshot);
         // Begin uploading stashed blobs from previous container instance
         Object.entries(stashedBlobs).forEach(([localId, entry]) => {
             const blob = (0, common_utils_1.stringToBuffer)(entry.blob, "base64");
+            if (entry.minTTLInSeconds && entry.uploadTime) {
+                const timeLapseSinceLocalUpload = (Date.now() - entry.uploadTime) / 1000;
+                // stashed entries with more than half-life in storage will not be reuploaded
+                if (entry.minTTLInSeconds - timeLapseSinceLocalUpload > entry.minTTLInSeconds / 2) {
+                    this.pendingBlobs.set(localId, {
+                        blob,
+                        status: PendingBlobStatus.OfflinePendingOp,
+                        handleP: new common_utils_1.Deferred(),
+                        uploadP: undefined,
+                        uploadTime: entry.uploadTime,
+                        minTTLInSeconds: entry.minTTLInSeconds,
+                    });
+                    return;
+                }
+            }
             this.pendingBlobs.set(localId, {
                 blob,
                 status: PendingBlobStatus.OfflinePendingUpload,
@@ -139,6 +149,30 @@ class BlobManager extends common_utils_1.TypedEventEmitter {
                 uploadP: this.uploadBlob(localId, blob),
             });
         });
+        this.sendBlobAttachOp = (localId, blobId) => {
+            const pendingEntry = this.pendingBlobs.get(localId);
+            if ((pendingEntry === null || pendingEntry === void 0 ? void 0 : pendingEntry.uploadTime) && (pendingEntry === null || pendingEntry === void 0 ? void 0 : pendingEntry.minTTLInSeconds)) {
+                const secondsSinceUpload = (Date.now() - pendingEntry.uploadTime) / 1000;
+                const expired = pendingEntry.minTTLInSeconds - secondsSinceUpload < 0;
+                this.mc.logger.sendTelemetryEvent({
+                    eventName: "sendBlobAttach",
+                    entryStatus: pendingEntry.status,
+                    secondsSinceUpload,
+                    minTTLInSeconds: pendingEntry.minTTLInSeconds,
+                    expired,
+                });
+                if (expired) {
+                    // we want to avoid submitting ops with broken handles
+                    this.closeContainer(new container_utils_1.GenericError("Trying to submit a BlobAttach for expired blob", undefined, {
+                        localId,
+                        blobId,
+                        entryStatus: pendingEntry.status,
+                        secondsSinceUpload,
+                    }));
+                }
+            }
+            return sendBlobAttachOp(localId, blobId);
+        };
     }
     get pendingOfflineUploads() {
         return Array.from(this.pendingBlobs.values()).filter((e) => e.status === PendingBlobStatus.OfflinePendingUpload);
@@ -174,14 +208,6 @@ class BlobManager extends common_utils_1.TypedEventEmitter {
             }
         }
     }
-    /**
-     * For a blobId, returns its path in GC's graph. The node path is of the format `/<BlobManager.basePath>/<blobId>`
-     * This path must match the path of the blob handle returned by the createBlob API because blobs are marked
-     * referenced by storing these handles in a referenced DDS.
-     */
-    getBlobGCNodePath(blobId) {
-        return `/${BlobManager.basePath}/${blobId}`;
-    }
     /**
      * Set of actual storage IDs (i.e., IDs that can be requested from storage). This will be empty if the container is
      * detached or there are no (non-pending) attachment blobs in the document
@@ -217,7 +243,7 @@ class BlobManager extends common_utils_1.TypedEventEmitter {
             storageId = attachedStorageId;
         }
         // Let runtime know that the corresponding GC node was requested.
-        this.blobRequested(this.getBlobGCNodePath(blobId));
+        this.blobRequested(getGCNodePathFromBlobId(blobId));
         return telemetry_utils_1.PerformanceEvent.timedExecAsync(this.mc.logger, { eventName: "AttachmentReadBlob", id: storageId }, async () => {
             return this.getStorage().readBlob(storageId);
         }, { end: true, cancel: "error" });
@@ -264,11 +290,6 @@ class BlobManager extends common_utils_1.TypedEventEmitter {
      */
     setRedirection(fromId, toId) {
         this.redirectTable.set(fromId, toId);
-        // Notify runtime of a reference added if toId is not undefined. It can be undefined when a blob is uploaded in
-        // detached mode. In this case, the entry will be updated when the blob is updated.
-        if (toId !== undefined) {
-            this.addedBlobReference(this.getBlobGCNodePath(fromId), this.getBlobGCNodePath(toId));
-        }
     }
     deleteAndEmitsIfEmpty(id) {
         if (this.pendingBlobs.has(id)) {
@@ -284,9 +305,8 @@ class BlobManager extends common_utils_1.TypedEventEmitter {
         (0, common_utils_1.assert)((entry === null || entry === void 0 ? void 0 : entry.status) === PendingBlobStatus.OnlinePendingUpload ||
             (entry === null || entry === void 0 ? void 0 : entry.status) === PendingBlobStatus.OfflinePendingUpload, 0x386 /* Must have pending blob entry for uploaded blob */);
         entry.storageId = response.id;
-        entry.localUploadTime = Date.now();
+        entry.uploadTime = Date.now();
         entry.minTTLInSeconds = response.minTTLInSeconds;
-        entry.serverUploadTime = this.getCurrentReferenceTimestampMs();
         if (this.runtime.connected) {
             if (entry.status === PendingBlobStatus.OnlinePendingUpload) {
                 // Send a blob attach op. This serves two purposes:
@@ -294,7 +314,6 @@ class BlobManager extends common_utils_1.TypedEventEmitter {
                 //    until its storage ID is added to the next summary.
                 // 2. It will create a local ID to storage ID mapping in all clients which is needed to retrieve the
                 //    blob from the server via the storage ID.
-                this.logTimeInfo(entry, "sendBlobAttachResolveTTL");
                 this.sendBlobAttachOp(localId, response.id);
                 if (this.storageIds.has(response.id)) {
                     // The blob is de-duped. Set up a local ID to storage ID mapping and return the blob. Since this is
@@ -358,7 +377,6 @@ class BlobManager extends common_utils_1.TypedEventEmitter {
          * is called on reconnection.
          */
         if (entry.status !== PendingBlobStatus.OnlinePendingOp) {
-            this.logTimeInfo(entry, "sendBlobAttachTransitionOfflineTTL");
             this.sendBlobAttachOp(localId, entry.storageId);
         }
         entry.status =
@@ -377,9 +395,6 @@ class BlobManager extends common_utils_1.TypedEventEmitter {
         const { localId, blobId } = metadata;
         (0, common_utils_1.assert)(localId !== undefined, 0x50d /* local ID not available on reSubmit */);
         const pendingEntry = this.pendingBlobs.get(localId);
-        if (pendingEntry) {
-            this.logTimeInfo(pendingEntry, "sendBlobAttachResubmitTTL");
-        }
         if (!blobId) {
             // We submitted this op while offline. The blob should have been uploaded by now.
             (0, common_utils_1.assert)((pendingEntry === null || pendingEntry === void 0 ? void 0 : pendingEntry.status) === PendingBlobStatus.OfflinePendingOp &&
@@ -388,32 +403,6 @@ class BlobManager extends common_utils_1.TypedEventEmitter {
         }
         return this.sendBlobAttachOp(localId, blobId);
     }
-    logTimeInfo(pendingEntry, eventName) {
-        var _a, _b;
-        let timeLapseSinceLocalUpload = 0;
-        let timeLapseSinceServerUpload = 0;
-        let expiredUsingLocalTime;
-        let expiredUsingServerTime;
-        if (pendingEntry.localUploadTime) {
-            timeLapseSinceLocalUpload = (Date.now() - pendingEntry.localUploadTime) / 1000;
-            expiredUsingLocalTime =
-                ((_a = pendingEntry.minTTLInSeconds) !== null && _a !== void 0 ? _a : 0) - timeLapseSinceLocalUpload < 0 ? true : false;
-        }
-        if (pendingEntry.serverUploadTime) {
-            timeLapseSinceServerUpload = (Date.now() - pendingEntry.serverUploadTime) / 1000;
-            expiredUsingServerTime =
-                ((_b = pendingEntry.minTTLInSeconds) !== null && _b !== void 0 ? _b : 0) - timeLapseSinceServerUpload < 0 ? true : false;
-        }
-        this.mc.logger.sendTelemetryEvent({
-            eventName,
-            entryStatus: pendingEntry.status,
-            timeLapseSinceLocalUpload,
-            timeLapseSinceServerUpload,
-            minTTLInSeconds: pendingEntry.minTTLInSeconds,
-            expiredUsingLocalTime,
-            expiredUsingServerTime,
-        });
-    }
     processBlobAttachOp(message, local) {
         var _a, _b;
         const localId = (_a = message.metadata) === null || _a === void 0 ? void 0 : _a.localId;
@@ -490,6 +479,23 @@ class BlobManager extends common_utils_1.TypedEventEmitter {
         }
         return table;
     }
+    summarize(telemetryContext) {
+        // if storageIds is empty, it means we are detached and have only local IDs, or that there are no blobs attached
+        const blobIds = this.storageIds.size > 0
+            ? Array.from(this.storageIds)
+            : Array.from(this.redirectTable.keys());
+        const builder = new runtime_utils_1.SummaryTreeBuilder();
+        blobIds.forEach((blobId) => {
+            builder.addAttachment(blobId);
+        });
+        // Any non-identity entries in the table need to be saved in the summary
+        if (this.redirectTable.size > blobIds.length) {
+            builder.addBlob(BlobManager.redirectTableBlobName, 
+            // filter out identity entries
+            JSON.stringify(Array.from(this.redirectTable.entries()).filter(([localId, storageId]) => localId !== storageId)));
+        }
+        return builder.getSummaryTree();
+    }
     /**
      * Generates data used for garbage collection. Each blob uploaded represents a node in the GC graph as it can be
      * individually referenced by storing its handle in a referenced DDS. Returns the list of blob ids as GC nodes.
@@ -500,23 +506,83 @@ class BlobManager extends common_utils_1.TypedEventEmitter {
         const gcData = { gcNodes: {} };
         for (const [localId, storageId] of this.redirectTable) {
             (0, common_utils_1.assert)(!!storageId, 0x390 /* Must be attached to get GC data */);
-            gcData.gcNodes[this.getBlobGCNodePath(localId)] = [this.getBlobGCNodePath(storageId)];
+            // Only return local ids as GC nodes because a blob can only be referenced via its local id. The storage
+            // id entries have the same key and value, ignore them.
+            // The outbound routes are empty because a blob node cannot reference other nodes. It can only be referenced
+            // by adding its handle to a referenced DDS.
+            if (localId !== storageId) {
+                gcData.gcNodes[getGCNodePathFromBlobId(localId)] = [];
+            }
         }
         return gcData;
     }
     /**
      * This is called to update blobs whose routes are unused. The unused blobs are deleted.
-     * @param unusedRoutes - The routes of the blob nodes that are unused.
+     * @param unusedRoutes - The routes of the blob nodes that are unused. These routes will be based off of local ids.
      */
     updateUnusedRoutes(unusedRoutes) {
-        // The routes or blob node paths are in the same format as returned in getGCData -
-        // `/<BlobManager.basePath>/<blobId>`.
-        for (const route of unusedRoutes) {
-            const pathParts = route.split("/");
-            (0, common_utils_1.assert)(pathParts.length === 3 && pathParts[1] === BlobManager.basePath, 0x2d5 /* "Invalid blob node id in unused routes." */);
-            const blobId = pathParts[2];
+        this.deleteBlobsFromRedirectTable(unusedRoutes);
+    }
+    /**
+     * Delete attachment blobs that are sweep ready.
+     * @param sweepReadyBlobRoutes - The routes of blobs that are sweep ready and should be deleted. These routes will
+     * be based off of local ids.
+     * @returns - The routes of blobs that were deleted.
+     */
+    deleteSweepReadyNodes(sweepReadyBlobRoutes) {
+        // If sweep for attachment blobs is not enabled, return empty list indicating nothing is deleted.
+        if (this.mc.config.getBoolean(gc_1.sweepAttachmentBlobsKey) !== true) {
+            return [];
+        }
+        this.deleteBlobsFromRedirectTable(sweepReadyBlobRoutes);
+        return Array.from(sweepReadyBlobRoutes);
+    }
+    /**
+     * Delete blobs with the given routes from the redirect table.
+     * The routes are GC nodes paths of format -`/<BlobManager.basePath>/<blobId>`. The blob ids are all local ids.
+     * Deleting the blobs involves 2 steps:
+     * 1. The redirect table entry for the local ids are deleted.
+     * 2. If the storage ids corresponding to the deleted local ids are not in-use anymore, the redirect table entries
+     * for the storage ids are deleted as well.
+     *
+     * Note that this does not delete the blobs from storage service immediately. Deleting the blobs from redirect table
+     * will remove them the next summary. The service would them delete them some time in the future.
+     */
+    deleteBlobsFromRedirectTable(blobRoutes) {
+        if (blobRoutes.length === 0) {
+            return;
+        }
+        // This tracks the storage ids of local ids that are deleted. After the local ids have been deleted, if any of
+        // these storage ids are unused, they will be deleted as well.
+        const maybeUnusedStorageIds = new Set();
+        for (const route of blobRoutes) {
+            const blobId = getBlobIdFromGCNodePath(route);
+            if (!this.redirectTable.has(blobId)) {
+                this.mc.logger.sendErrorEvent({
+                    eventName: "DeletedAttachmentBlobNotFound",
+                    blobId,
+                });
+                continue;
+            }
+            const storageId = this.redirectTable.get(blobId);
+            (0, common_utils_1.assert)(!!storageId, 0x5bb /* Must be attached to run GC */);
+            maybeUnusedStorageIds.add(storageId);
             this.redirectTable.delete(blobId);
         }
+        // Find out storage ids that are in-use and remove them from maybeUnusedStorageIds. A storage id is in-use if
+        // the redirect table has a local id -> storage id entry for it.
+        for (const [localId, storageId] of this.redirectTable.entries()) {
+            (0, common_utils_1.assert)(!!storageId, 0x5bc /* Must be attached to run GC */);
+            // For every storage id, the redirect table has a id -> id entry. These do not make the storage id in-use.
+            if (maybeUnusedStorageIds.has(storageId) && localId !== storageId) {
+                maybeUnusedStorageIds.delete(storageId);
+            }
+        }
+        // For unused storage ids, delete their id -> id entries from the redirect table.
+        // This way they'll be absent from the next summary, and the service is free to delete them from storage.
+        for (const storageId of maybeUnusedStorageIds) {
+            this.redirectTable.delete(storageId);
+        }
     }
     /**
      * This is called to update blobs whose routes are tombstones. Tombstoned blobs enable testing scenarios with
@@ -528,9 +594,8 @@ class BlobManager extends common_utils_1.TypedEventEmitter {
         // The routes or blob node paths are in the same format as returned in getGCData -
         // `/<BlobManager.basePath>/<blobId>`.
         for (const route of tombstonedRoutes) {
-            const pathParts = route.split("/");
-            (0, common_utils_1.assert)(pathParts.length === 3 && pathParts[1] === BlobManager.basePath, 0x50f /* Invalid blob node id in tombstoned routes. */);
-            tombstonedBlobsSet.add(pathParts[2]);
+            const blobId = getBlobIdFromGCNodePath(route);
+            tombstonedBlobsSet.add(blobId);
         }
         // Remove blobs from the tombstone list that were tombstoned but aren't anymore as per the tombstoneRoutes.
         for (const blobId of this.tombstonedBlobs) {
@@ -555,7 +620,7 @@ class BlobManager extends common_utils_1.TypedEventEmitter {
          * 3. "valid" - It has not been deleted or tombstoned.
          */
         let state = "valid";
-        if (this.isBlobDeleted(this.getBlobGCNodePath(blobId))) {
+        if (this.isBlobDeleted(getGCNodePathFromBlobId(blobId))) {
             state = "deleted";
         }
         else if (this.tombstonedBlobs.has(blobId)) {
@@ -569,34 +634,17 @@ class BlobManager extends common_utils_1.TypedEventEmitter {
         const shouldFail = state === "deleted" || this.throwOnTombstoneLoad;
         const request = { url: blobId };
         const error = (0, runtime_utils_1.responseToException)((0, runtime_utils_1.createResponseError)(404, "Blob was deleted", request, state === "tombstoned" ? { [containerRuntime_1.TombstoneResponseHeaderKey]: true } : undefined), request);
-        (0, garbageCollectionHelpers_1.sendGCUnexpectedUsageEvent)(this.mc, {
+        (0, gc_1.sendGCUnexpectedUsageEvent)(this.mc, {
             eventName: state === "tombstoned"
                 ? "GC_Tombstone_Blob_Requested"
                 : "GC_Deleted_Blob_Requested",
             category: shouldFail ? "error" : "generic",
-            isSummarizerClient: this.runtime.clientDetails.type === summarizerClientElection_1.summarizerClientType,
+            gcTombstoneEnforcementAllowed: this.runtime.gcTombstoneEnforcementAllowed,
         }, [BlobManager.basePath], error);
         if (shouldFail) {
             throw error;
         }
     }
-    summarize(telemetryContext) {
-        // if storageIds is empty, it means we are detached and have only local IDs, or that there are no blobs attached
-        const blobIds = this.storageIds.size > 0
-            ? Array.from(this.storageIds)
-            : Array.from(this.redirectTable.keys());
-        const builder = new runtime_utils_1.SummaryTreeBuilder();
-        blobIds.forEach((blobId) => {
-            builder.addAttachment(blobId);
-        });
-        // Any non-identity entries in the table need to be saved in the summary
-        if (this.redirectTable.size > blobIds.length) {
-            builder.addBlob(BlobManager.redirectTableBlobName, 
-            // filter out identity entries
-            JSON.stringify(Array.from(this.redirectTable.entries()).filter(([localId, storageId]) => localId !== storageId)));
-        }
-        return builder.getSummaryTree();
-    }
     setRedirectTable(table) {
         (0, common_utils_1.assert)(this.runtime.attachState === container_definitions_1.AttachState.Detached, 0x252 /* "redirect table can only be set in detached container" */);
         (0, common_utils_1.assert)(this.redirectTable.size === table.size, 0x391 /* Redirect table size must match BlobManager's local ID count */);
@@ -610,7 +658,13 @@ class BlobManager extends common_utils_1.TypedEventEmitter {
     getPendingBlobs() {
         const blobs = {};
         for (const [key, entry] of this.pendingBlobs) {
-            blobs[key] = { blob: (0, common_utils_1.bufferToString)(entry.blob, "base64") };
+            blobs[key] = entry.minTTLInSeconds
+                ? {
+                    blob: (0, common_utils_1.bufferToString)(entry.blob, "base64"),
+                    uploadTime: entry.uploadTime,
+                    minTTLInSeconds: entry.minTTLInSeconds,
+                }
+                : { blob: (0, common_utils_1.bufferToString)(entry.blob, "base64") };
         }
         return blobs;
     }
@@ -618,4 +672,20 @@ class BlobManager extends common_utils_1.TypedEventEmitter {
 exports.BlobManager = BlobManager;
 BlobManager.basePath = "_blobs";
 BlobManager.redirectTableBlobName = ".redirectTable";
+/**
+ * For a blobId, returns its path in GC's graph. The node path is of the format `/<BlobManager.basePath>/<blobId>`.
+ * This path must match the path of the blob handle returned by the createBlob API because blobs are marked
+ * referenced by storing these handles in a referenced DDS.
+ */
+function getGCNodePathFromBlobId(blobId) {
+    return `/${BlobManager.basePath}/${blobId}`;
+}
+/**
+ * For a given GC node path, return the blobId. The node path is of the format `/<BlobManager.basePath>/<blobId>`.
+ */
+function getBlobIdFromGCNodePath(nodePath) {
+    const pathParts = nodePath.split("/");
+    (0, common_utils_1.assert)(pathParts.length === 3 && pathParts[1] === BlobManager.basePath, 0x5bd /* Invalid blob node path */);
+    return pathParts[2];
+}
 //# sourceMappingURL=blobManager.js.map