@fluidframework/container-runtime 2.0.0-dev.2.3.0.115467 → 2.0.0-dev.4.1.0.148229

package/dist/dataStores.js

@@ -14,26 +14,26 @@ const container_definitions_1 = require("@fluidframework/container-definitions")
 const driver_utils_1 = require("@fluidframework/driver-utils");
 const common_utils_1 = require("@fluidframework/common-utils");
 const uuid_1 = require("uuid");
-const garbage_collector_1 = require("@fluidframework/garbage-collector");
 const dataStoreContexts_1 = require("./dataStoreContexts");
+const containerRuntime_1 = require("./containerRuntime");
 const dataStoreContext_1 = require("./dataStoreContext");
-const summaryFormat_1 = require("./summaryFormat");
+const storageServiceWithAttachBlobs_1 = require("./storageServiceWithAttachBlobs");
 const dataStore_1 = require("./dataStore");
-const garbageCollection_1 = require("./garbageCollection");
-const garbageCollectionConstants_1 = require("./garbageCollectionConstants");
-const summarizerClientElection_1 = require("./summarizerClientElection");
+const gc_1 = require("./gc");
+const summary_1 = require("./summary");
 /**
  * This class encapsulates data store handling. Currently it is only used by the container runtime,
  * but eventually could be hosted on any channel once we formalize the channel api boundary.
  */
 class DataStores {
-    constructor(baseSnapshot, runtime, submitAttachFn, getCreateChildSummarizerNodeFn, deleteChildSummarizerNodeFn, baseLogger, getBaseGCDetails, gcNodeUpdated, aliasMap, contexts = new dataStoreContexts_1.DataStoreContexts(baseLogger)) {
+    constructor(baseSnapshot, runtime, submitAttachFn, getCreateChildSummarizerNodeFn, deleteChildSummarizerNodeFn, baseLogger, gcNodeUpdated, isDataStoreDeleted, aliasMap, contexts = new dataStoreContexts_1.DataStoreContexts(baseLogger)) {
         this.baseSnapshot = baseSnapshot;
         this.runtime = runtime;
         this.submitAttachFn = submitAttachFn;
         this.getCreateChildSummarizerNodeFn = getCreateChildSummarizerNodeFn;
         this.deleteChildSummarizerNodeFn = deleteChildSummarizerNodeFn;
         this.gcNodeUpdated = gcNodeUpdated;
+        this.isDataStoreDeleted = isDataStoreDeleted;
         this.aliasMap = aliasMap;
         this.contexts = contexts;
         // Stores tracked by the Domain
@@ -48,18 +48,11 @@ class DataStores {
         this.dispose = () => this.disposeOnce.value;
         this.mc = (0, telemetry_utils_1.loggerToMonitoringContext)(telemetry_utils_1.ChildLogger.create(baseLogger));
         this.containerRuntimeHandle = new datastore_1.FluidObjectHandle(this.runtime, "/", this.runtime.IFluidHandleContext);
-        const baseGCDetailsP = new common_utils_1.LazyPromise(async () => {
-            return getBaseGCDetails();
-        });
-        // Returns the base GC details for the data store with the given id.
-        const dataStoreBaseGCDetails = async (dataStoreId) => {
-            const baseGCDetails = await baseGCDetailsP;
-            return baseGCDetails.get(dataStoreId);
-        };
         // Tombstone should only throw when the feature flag is enabled and the client isn't a summarizer
-        this.throwOnTombstoneUsage =
-            this.mc.config.getBoolean(garbageCollectionConstants_1.throwOnTombstoneUsageKey) === true &&
-                this.runtime.clientDetails.type !== summarizerClientElection_1.summarizerClientType;
+        this.throwOnTombstoneLoad =
+            this.mc.config.getBoolean(gc_1.throwOnTombstoneLoadKey) === true &&
+                this.runtime.gcTombstoneEnforcementAllowed &&
+                this.runtime.clientDetails.type !== summary_1.summarizerClientType;
         // Extract stores stored inside the snapshot
         const fluidDataStores = new Map();
         if (baseSnapshot) {
@@ -80,11 +73,12 @@ class DataStores {
                 dataStoreContext = new dataStoreContext_1.RemoteFluidDataStoreContext({
                     id: key,
                     snapshotTree: value,
-                    getBaseGCDetails: async () => dataStoreBaseGCDetails(key),
                     runtime: this.runtime,
                     storage: this.runtime.storage,
                     scope: this.runtime.scope,
-                    createSummarizerNodeFn: this.getCreateChildSummarizerNodeFn(key, { type: runtime_definitions_1.CreateSummarizerNodeSource.FromSummary }),
+                    createSummarizerNodeFn: this.getCreateChildSummarizerNodeFn(key, {
+                        type: runtime_definitions_1.CreateSummarizerNodeSource.FromSummary,
+                    }),
                 });
             }
             else {
@@ -98,7 +92,9 @@ class DataStores {
                     runtime: this.runtime,
                     storage: this.runtime.storage,
                     scope: this.runtime.scope,
-                    createSummarizerNodeFn: this.getCreateChildSummarizerNodeFn(key, { type: runtime_definitions_1.CreateSummarizerNodeSource.FromSummary }),
+                    createSummarizerNodeFn: this.getCreateChildSummarizerNodeFn(key, {
+                        type: runtime_definitions_1.CreateSummarizerNodeSource.FromSummary,
+                    }),
                     makeLocallyVisibleFn: () => this.makeDataStoreLocallyVisible(key),
                     snapshotTree,
                     isRootDataStore: undefined,
@@ -143,10 +139,10 @@ class DataStores {
                 } }));
             throw error;
         }
-        const flatBlobs = new Map();
+        const flatAttachBlobs = new Map();
         let snapshotTree;
         if (attachMessage.snapshot) {
-            snapshotTree = (0, driver_utils_1.buildSnapshotTree)(attachMessage.snapshot.entries, flatBlobs);
+            snapshotTree = (0, driver_utils_1.buildSnapshotTree)(attachMessage.snapshot.entries, flatAttachBlobs);
         }
         // Include the type of attach message which is the pkg of the store to be
         // used by RemoteFluidDataStoreContext in case it is not in the snapshot.
@@ -154,10 +150,8 @@ class DataStores {
         const remoteFluidDataStoreContext = new dataStoreContext_1.RemoteFluidDataStoreContext({
             id: attachMessage.id,
             snapshotTree,
-            // New data stores begin with empty GC details since GC hasn't run on them yet.
-            getBaseGCDetails: async () => { return {}; },
             runtime: this.runtime,
-            storage: new driver_utils_1.BlobCacheStorageService(this.runtime.storage, flatBlobs),
+            storage: new storageServiceWithAttachBlobs_1.StorageServiceWithAttachBlobs(this.runtime.storage, flatAttachBlobs),
             scope: this.runtime.scope,
             createSummarizerNodeFn: this.getCreateChildSummarizerNodeFn(attachMessage.id, {
                 type: runtime_definitions_1.CreateSummarizerNodeSource.FromAttach,
@@ -232,7 +226,9 @@ class DataStores {
             runtime: this.runtime,
             storage: this.runtime.storage,
             scope: this.runtime.scope,
-            createSummarizerNodeFn: this.getCreateChildSummarizerNodeFn(id, { type: runtime_definitions_1.CreateSummarizerNodeSource.Local }),
+            createSummarizerNodeFn: this.getCreateChildSummarizerNodeFn(id, {
+                type: runtime_definitions_1.CreateSummarizerNodeSource.Local,
+            }),
             makeLocallyVisibleFn: () => this.makeDataStoreLocallyVisible(id),
             snapshotTree: undefined,
             isRootDataStore: isRoot,
@@ -248,7 +244,9 @@ class DataStores {
             runtime: this.runtime,
             storage: this.runtime.storage,
             scope: this.runtime.scope,
-            createSummarizerNodeFn: this.getCreateChildSummarizerNodeFn(id, { type: runtime_definitions_1.CreateSummarizerNodeSource.Local }),
+            createSummarizerNodeFn: this.getCreateChildSummarizerNodeFn(id, {
+                type: runtime_definitions_1.CreateSummarizerNodeSource.Local,
+            }),
             makeLocallyVisibleFn: () => this.makeDataStoreLocallyVisible(id),
             snapshotTree: undefined,
             isRootDataStore: false,
@@ -257,7 +255,9 @@ class DataStores {
         this.contexts.addUnbound(context);
         return context;
     }
-    get disposed() { return this.disposeOnce.evaluated; }
+    get disposed() {
+        return this.disposeOnce.evaluated;
+    }
     resubmitDataStoreOp(content, localOpMetadata) {
         const envelope = content;
         const context = this.contexts.get(envelope.address);
@@ -284,6 +284,8 @@ class DataStores {
     processFluidDataStoreOp(message, local, localMessageMetadata) {
         const envelope = message.contents;
         const transformed = Object.assign(Object.assign({}, message), { contents: envelope.contents });
+        const request = { url: envelope.address };
+        this.validateNotDeleted(envelope.address, request);
         const context = this.contexts.get(envelope.address);
         (0, common_utils_1.assert)(!!context, 0x162 /* "There should be a store context for the op" */);
         context.process(transformed, local, localMessageMetadata);
@@ -291,32 +293,70 @@ class DataStores {
         // being used.
         this.gcNodeUpdated(`/${envelope.address}`, message.timestamp, context.isLoaded ? context.packagePath : undefined);
     }
-    async getDataStore(id, wait, viaHandle) {
-        const context = await this.contexts.getBoundOrRemoted(id, wait);
+    async getDataStore(id, requestHeaderData) {
+        const headerData = Object.assign(Object.assign({}, containerRuntime_1.defaultRuntimeHeaderData), requestHeaderData);
         const request = { url: id };
+        this.validateNotDeleted(id, request, headerData);
+        const context = await this.contexts.getBoundOrRemoted(id, headerData.wait);
         if (context === undefined) {
             // The requested data store does not exits. Throw a 404 response exception.
             throw (0, runtime_utils_1.responseToException)((0, runtime_utils_1.create404Response)(request), request);
         }
+        this.validateNotTombstoned(context, request, requestHeaderData);
+        return context;
+    }
+    /**
+     * Validate that the data store had not been deleted by GC.
+     *
+     * @param id - data store id
+     * @param request - the request information to log if the validation detects the data store has been deleted
+     * @param requestHeaderData - the request header information to log if the validation detects the data store has been deleted
+     */
+    validateNotDeleted(id, request, requestHeaderData) {
+        const dataStoreNodePath = `/${id}`;
+        if (this.isDataStoreDeleted(dataStoreNodePath)) {
+            (0, common_utils_1.assert)(!this.contexts.has(id), 0x570 /* Inconsistent state! GC says the data store is deleted, but the data store is not deleted from the runtime. */);
+            // The requested data store is removed by gc. Create a 404 gc response exception.
+            const error = (0, runtime_utils_1.responseToException)((0, runtime_utils_1.createResponseError)(404, "DataStore was deleted", request), request);
+            (0, gc_1.sendGCUnexpectedUsageEvent)(this.mc, {
+                eventName: "GC_Deleted_DataStore_Requested",
+                category: "error",
+                isSummarizerClient: this.runtime.clientDetails.type === summary_1.summarizerClientType,
+                headers: JSON.stringify(requestHeaderData),
+                gcTombstoneEnforcementAllowed: this.runtime.gcTombstoneEnforcementAllowed,
+            }, undefined /** packagePath */, error);
+            throw error;
+        }
+    }
+    /**
+     * Validates that the data store context requested has not been marked as tombstone by GC.
+     *
+     * @param context - the data store context in question
+     * @param request - the request information to log if the validation detects the data store has been tombstoned
+     * @param headerData - the request header information to log if the validation detects the data store has been tombstoned
+     */
+    validateNotTombstoned(context, request, headerData) {
         if (context.tombstoned) {
+            const shouldFail = this.throwOnTombstoneLoad && !headerData.allowTombstone;
             // The requested data store is removed by gc. Create a 404 gc response exception.
-            const error = (0, runtime_utils_1.responseToException)((0, runtime_utils_1.createResponseError)(404, "Datastore removed by gc", request), request);
-            // Note: if a user writes a request to look like it's viaHandle, we will also send this telemetry event
-            this.mc.logger.sendErrorEvent({
+            const error = (0, runtime_utils_1.responseToException)((0, runtime_utils_1.createResponseError)(404, "DataStore was deleted", request, {
+                [containerRuntime_1.TombstoneResponseHeaderKey]: true,
+            }), request);
+            (0, gc_1.sendGCUnexpectedUsageEvent)(this.mc, {
                 eventName: "GC_Tombstone_DataStore_Requested",
-                url: request.url,
-                pkg: (0, runtime_utils_1.packagePathToTelemetryProperty)(context.isLoaded ? context.packagePath : undefined),
-                viaHandle,
-            }, error);
-            // Always log an error when tombstoned data store is used. However, throw an error only if
-            // throwOnTombstoneUsage is set.
-            if (this.throwOnTombstoneUsage) {
+                category: shouldFail ? "error" : "generic",
+                isSummarizerClient: this.runtime.clientDetails.type === summary_1.summarizerClientType,
+                headers: JSON.stringify(headerData),
+                gcTombstoneEnforcementAllowed: this.runtime.gcTombstoneEnforcementAllowed,
+            }, context.isLoaded ? context.packagePath : undefined, error);
+            if (shouldFail) {
                 throw error;
             }
         }
-        return context;
     }
     processSignal(address, message, local) {
+        const request = { url: address };
+        this.validateNotDeleted(address, request);
         const context = this.contexts.get(address);
         if (!context) {
             // Attach message may not have been processed yet
@@ -342,6 +382,10 @@ class DataStores {
                     eventName: "SetConnectionStateError",
                     clientId,
                     fluidDataStore,
+                    details: JSON.stringify({
+                        runtimeConnected: this.runtime.connected,
+                        connected,
+                    }),
                 }, error);
             }
         }
@@ -363,10 +407,12 @@ class DataStores {
         // Iterate over each store and ask it to snapshot
         await Promise.all(Array.from(this.contexts)
             .filter(([_, context]) => {
-            // Summarizer works only with clients with no local changes!
-            (0, common_utils_1.assert)(context.attachState !== container_definitions_1.AttachState.Attaching, 0x165 /* "Summarizer cannot work if client has local changes" */);
+            // Summarizer works only with clients with no local changes. A data store in attaching
+            // state indicates an op was sent to attach a local data store.
+            (0, common_utils_1.assert)(context.attachState !== container_definitions_1.AttachState.Attaching, 0x588 /* Local data store detected in attaching state during summarize */);
             return context.attachState === container_definitions_1.AttachState.Attached;
-        }).map(async ([contextId, context]) => {
+        })
+            .map(async ([contextId, context]) => {
             const contextSummary = await context.summarize(fullTree, trackState, telemetryContext);
             summaryBuilder.addWithStats(contextId, contextSummary);
         }));
@@ -386,9 +432,9 @@ class DataStores {
             // Take summary of bounded data stores only, make sure we haven't summarized them already
             // and no attach op has been fired for that data store because for loader versions <= 0.24
             // we set attach state as "attaching" before taking createNew summary.
-            !(this.contexts.isNotBound(key)
-                || builderTree[key]
-                || this.attachOpFiredForDataStore.has(key)))
+            !(this.contexts.isNotBound(key) ||
+                builderTree[key] ||
+                this.attachOpFiredForDataStore.has(key)))
                 .map(([key, value]) => {
                 let dataStoreSummary;
                 if (value.isLoaded) {
@@ -437,14 +483,16 @@ class DataStores {
      * @param fullGC - true to bypass optimizations and force full generation of GC data.
      */
     async getGCData(fullGC = false) {
-        const builder = new garbage_collector_1.GCDataBuilder();
+        const builder = new runtime_utils_1.GCDataBuilder();
         // Iterate over each store and get their GC data.
         await Promise.all(Array.from(this.contexts)
             .filter(([_, context]) => {
-            // Get GC data only for attached contexts. Detached contexts are not connected in the GC reference
-            // graph so any references they might have won't be connected as well.
+            // Summarizer client and hence GC works only with clients with no local changes. A data store in
+            // attaching state indicates an op was sent to attach a local data store.
+            (0, common_utils_1.assert)(context.attachState !== container_definitions_1.AttachState.Attaching, 0x589 /* Local data store detected in attaching state while running GC */);
             return context.attachState === container_definitions_1.AttachState.Attached;
-        }).map(async ([contextId, context]) => {
+        })
+            .map(async ([contextId, context]) => {
             const contextGCData = await context.getGCData(fullGC);
             // Prefix the child's id to the ids of its GC nodes so they can be identified as belonging to the child.
             // This also gradually builds the id of each node to be a path from the root.
@@ -461,14 +509,10 @@ class DataStores {
     updateUsedRoutes(usedRoutes) {
         var _a;
         // Get a map of data store ids to routes used in it.
-        const usedDataStoreRoutes = (0, garbage_collector_1.unpackChildNodesUsedRoutes)(usedRoutes);
+        const usedDataStoreRoutes = (0, runtime_utils_1.unpackChildNodesUsedRoutes)(usedRoutes);
         // Verify that the used routes are correct.
         for (const [id] of usedDataStoreRoutes) {
             (0, common_utils_1.assert)(this.contexts.has(id), 0x167 /* "Used route does not belong to any known data store" */);
-            // Revive datastores regardless of whether or not tombstone the tombstone flag is flipped
-            const dataStore = this.contexts.get(id);
-            (0, common_utils_1.assert)(dataStore !== undefined, 0x46e /* No data store retrieved with specified id */);
-            dataStore.setTombstone(false /* tombstone */);
         }
         // Update the used routes in each data store. Used routes is empty for unused data stores.
         for (const [contextId, context] of this.contexts) {
@@ -476,13 +520,10 @@ class DataStores {
         }
     }
     /**
-     * This is called to update objects whose routes are unused. The unused objects are either deleted or marked as
-     * tombstones.
+     * This is called to update objects whose routes are unused. The unused objects are deleted.
      * @param unusedRoutes - The routes that are unused in all data stores in this Container.
-     * @param tombstone - if true, the objects corresponding to unused routes are marked tombstones. Otherwise, they
-     * are deleted.
      */
-    updateUnusedRoutes(unusedRoutes, tombstone) {
+    updateUnusedRoutes(unusedRoutes) {
         for (const route of unusedRoutes) {
             const pathParts = route.split("/");
             // Delete data store only if its route (/datastoreId) is in unusedRoutes. We don't want to delete a data
@@ -492,23 +533,70 @@ class DataStores {
             }
             const dataStoreId = pathParts[1];
             (0, common_utils_1.assert)(this.contexts.has(dataStoreId), 0x2d7 /* No data store with specified id */);
-            /**
-             * When running GC in tombstone mode, datastore contexts are tombstoned. Tombstoned datastore contexts
-             * enable testing scenarios with accessing deleted content without actually deleting content from
-             * summaries.
-             */
-            if (tombstone) {
-                const dataStore = this.contexts.get(dataStoreId);
-                (0, common_utils_1.assert)(dataStore !== undefined, 0x442 /* No data store retrieved with specified id */);
-                dataStore.setTombstone(true /* tombstone */);
-                continue;
-            }
             // Delete the contexts of unused data stores.
             this.contexts.delete(dataStoreId);
             // Delete the summarizer node of the unused data stores.
             this.deleteChildSummarizerNodeFn(dataStoreId);
         }
     }
+    /**
+     * Delete data stores and its objects that are sweep ready.
+     * @param sweepReadyDataStoreRoutes - The routes of data stores and its objects that are sweep ready and should
+     * be deleted.
+     * @returns - The routes of data stores and its objects that were deleted.
+     */
+    deleteSweepReadyNodes(sweepReadyDataStoreRoutes) {
+        // If sweep for data stores is not enabled, return empty list indicating nothing is deleted.
+        if (this.mc.config.getBoolean(gc_1.sweepDatastoresKey) !== true) {
+            return [];
+        }
+        for (const route of sweepReadyDataStoreRoutes) {
+            const pathParts = route.split("/");
+            const dataStoreId = pathParts[1];
+            // TODO: GC:Validation - Skip any routes already deleted
+            // Ignore sub-data store routes because a data store and its sub-routes are deleted together, so, we only
+            // need to delete the data store.
+            if (pathParts.length > 2) {
+                continue;
+            }
+            if (!this.contexts.has(dataStoreId)) {
+                this.mc.logger.sendErrorEvent({
+                    eventName: "DeletedDataStoreNotFound",
+                    dataStoreId,
+                });
+            }
+            const dataStore = this.contexts.get(dataStoreId);
+            (0, common_utils_1.assert)(dataStore !== undefined, 0x571 /* Attempting to delete unknown dataStore */);
+            dataStore.delete();
+            // Delete the contexts of sweep ready data stores.
+            this.contexts.delete(dataStoreId);
+            // Delete the summarizer node of the sweep ready data stores.
+            this.deleteChildSummarizerNodeFn(dataStoreId);
+        }
+        return Array.from(sweepReadyDataStoreRoutes);
+    }
+    /**
+     * This is called to update objects whose routes are tombstones. Tombstoned datastore contexts enable testing
+     * scenarios with accessing deleted content without actually deleting content from summaries.
+     * @param tombstonedRoutes - The routes that are tombstones in all data stores in this Container.
+     */
+    updateTombstonedRoutes(tombstonedRoutes) {
+        const tombstonedDataStoresSet = new Set();
+        for (const route of tombstonedRoutes) {
+            const pathParts = route.split("/");
+            // Tombstone data store only if its route (/datastoreId) is directly in tombstoneRoutes.
+            if (pathParts.length > 2) {
+                continue;
+            }
+            const dataStoreId = pathParts[1];
+            (0, common_utils_1.assert)(this.contexts.has(dataStoreId), 0x510 /* No data store with specified id */);
+            tombstonedDataStoresSet.add(dataStoreId);
+        }
+        // Update the used routes in each data store. Used routes is empty for unused data stores.
+        for (const [contextId, context] of this.contexts) {
+            context.setTombstone(tombstonedDataStoresSet.has(contextId));
+        }
+    }
     /**
      * Returns the outbound routes of this channel. Only root data stores are considered referenced and their paths are
      * part of outbound routes.
@@ -545,9 +633,9 @@ class DataStores {
         // Data stores paths are of the format "/dataStoreId".
         // Sub data store paths are of the format "/dataStoreId/subPath/...".
         if (pathParts.length === 2) {
-            return garbageCollection_1.GCNodeType.DataStore;
+            return gc_1.GCNodeType.DataStore;
         }
-        return garbageCollection_1.GCNodeType.SubDataStore;
+        return gc_1.GCNodeType.SubDataStore;
     }
 }
 exports.DataStores = DataStores;
@@ -555,7 +643,7 @@ function getSummaryForDatastores(snapshot, metadata) {
     if (!snapshot) {
         return undefined;
     }
-    if ((0, summaryFormat_1.rootHasIsolatedChannels)(metadata)) {
+    if ((0, summary_1.rootHasIsolatedChannels)(metadata)) {
         const datastoresSnapshot = snapshot.trees[runtime_definitions_1.channelsTreeName];
         (0, common_utils_1.assert)(!!datastoresSnapshot, 0x168 /* Expected tree in snapshot not found */);
         return datastoresSnapshot;
@@ -564,7 +652,7 @@ function getSummaryForDatastores(snapshot, metadata) {
         // back-compat: strip out all non-datastore paths before giving to DataStores object.
         const datastoresTrees = {};
         for (const [key, value] of Object.entries(snapshot.trees)) {
-            if (!summaryFormat_1.nonDataStorePaths.includes(key)) {
+            if (!summary_1.nonDataStorePaths.includes(key)) {
                 datastoresTrees[key] = value;
             }
         }