@fluidframework/container-runtime 2.0.0-dev.2.3.0.115467 → 2.0.0-dev.4.1.0.148229

package/lib/dataStores.js

@@ -2,35 +2,35 @@
  * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
  * Licensed under the MIT License.
  */
-import { DataCorruptionError, extractSafePropertiesFromMessage } from "@fluidframework/container-utils";
+import { DataCorruptionError, extractSafePropertiesFromMessage, } from "@fluidframework/container-utils";
 import { FluidObjectHandle } from "@fluidframework/datastore";
 import { channelsTreeName, CreateSummarizerNodeSource, } from "@fluidframework/runtime-definitions";
-import { convertSnapshotTreeToSummaryTree, convertToSummaryTree, create404Response, createResponseError, responseToException, packagePathToTelemetryProperty, SummaryTreeBuilder, } from "@fluidframework/runtime-utils";
-import { ChildLogger, loggerToMonitoringContext, LoggingError, TelemetryDataTag } from "@fluidframework/telemetry-utils";
+import { convertSnapshotTreeToSummaryTree, convertToSummaryTree, create404Response, createResponseError, GCDataBuilder, responseToException, SummaryTreeBuilder, unpackChildNodesUsedRoutes, } from "@fluidframework/runtime-utils";
+import { ChildLogger, loggerToMonitoringContext, LoggingError, TelemetryDataTag, } from "@fluidframework/telemetry-utils";
 import { AttachState } from "@fluidframework/container-definitions";
-import { BlobCacheStorageService, buildSnapshotTree } from "@fluidframework/driver-utils";
-import { assert, Lazy, LazyPromise } from "@fluidframework/common-utils";
+import { buildSnapshotTree } from "@fluidframework/driver-utils";
+import { assert, Lazy } from "@fluidframework/common-utils";
 import { v4 as uuid } from "uuid";
-import { GCDataBuilder, unpackChildNodesUsedRoutes } from "@fluidframework/garbage-collector";
 import { DataStoreContexts } from "./dataStoreContexts";
+import { defaultRuntimeHeaderData, TombstoneResponseHeaderKey, } from "./containerRuntime";
 import { RemoteFluidDataStoreContext, LocalFluidDataStoreContext, createAttributesBlob, LocalDetachedFluidDataStoreContext, } from "./dataStoreContext";
-import { nonDataStorePaths, rootHasIsolatedChannels } from "./summaryFormat";
+import { StorageServiceWithAttachBlobs } from "./storageServiceWithAttachBlobs";
 import { isDataStoreAliasMessage } from "./dataStore";
-import { GCNodeType } from "./garbageCollection";
-import { throwOnTombstoneUsageKey } from "./garbageCollectionConstants";
-import { summarizerClientType } from "./summarizerClientElection";
+import { GCNodeType, sweepDatastoresKey, throwOnTombstoneLoadKey, sendGCUnexpectedUsageEvent, } from "./gc";
+import { summarizerClientType, nonDataStorePaths, rootHasIsolatedChannels, } from "./summary";
 /**
  * This class encapsulates data store handling. Currently it is only used by the container runtime,
  * but eventually could be hosted on any channel once we formalize the channel api boundary.
  */
 export class DataStores {
-    constructor(baseSnapshot, runtime, submitAttachFn, getCreateChildSummarizerNodeFn, deleteChildSummarizerNodeFn, baseLogger, getBaseGCDetails, gcNodeUpdated, aliasMap, contexts = new DataStoreContexts(baseLogger)) {
+    constructor(baseSnapshot, runtime, submitAttachFn, getCreateChildSummarizerNodeFn, deleteChildSummarizerNodeFn, baseLogger, gcNodeUpdated, isDataStoreDeleted, aliasMap, contexts = new DataStoreContexts(baseLogger)) {
         this.baseSnapshot = baseSnapshot;
         this.runtime = runtime;
         this.submitAttachFn = submitAttachFn;
         this.getCreateChildSummarizerNodeFn = getCreateChildSummarizerNodeFn;
         this.deleteChildSummarizerNodeFn = deleteChildSummarizerNodeFn;
         this.gcNodeUpdated = gcNodeUpdated;
+        this.isDataStoreDeleted = isDataStoreDeleted;
         this.aliasMap = aliasMap;
         this.contexts = contexts;
         // Stores tracked by the Domain
@@ -45,17 +45,10 @@ export class DataStores {
         this.dispose = () => this.disposeOnce.value;
         this.mc = loggerToMonitoringContext(ChildLogger.create(baseLogger));
         this.containerRuntimeHandle = new FluidObjectHandle(this.runtime, "/", this.runtime.IFluidHandleContext);
-        const baseGCDetailsP = new LazyPromise(async () => {
-            return getBaseGCDetails();
-        });
-        // Returns the base GC details for the data store with the given id.
-        const dataStoreBaseGCDetails = async (dataStoreId) => {
-            const baseGCDetails = await baseGCDetailsP;
-            return baseGCDetails.get(dataStoreId);
-        };
         // Tombstone should only throw when the feature flag is enabled and the client isn't a summarizer
-        this.throwOnTombstoneUsage =
-            this.mc.config.getBoolean(throwOnTombstoneUsageKey) === true &&
+        this.throwOnTombstoneLoad =
+            this.mc.config.getBoolean(throwOnTombstoneLoadKey) === true &&
+                this.runtime.gcTombstoneEnforcementAllowed &&
                 this.runtime.clientDetails.type !== summarizerClientType;
         // Extract stores stored inside the snapshot
         const fluidDataStores = new Map();
@@ -77,11 +70,12 @@ export class DataStores {
                 dataStoreContext = new RemoteFluidDataStoreContext({
                     id: key,
                     snapshotTree: value,
-                    getBaseGCDetails: async () => dataStoreBaseGCDetails(key),
                     runtime: this.runtime,
                     storage: this.runtime.storage,
                     scope: this.runtime.scope,
-                    createSummarizerNodeFn: this.getCreateChildSummarizerNodeFn(key, { type: CreateSummarizerNodeSource.FromSummary }),
+                    createSummarizerNodeFn: this.getCreateChildSummarizerNodeFn(key, {
+                        type: CreateSummarizerNodeSource.FromSummary,
+                    }),
                 });
             }
             else {
@@ -95,7 +89,9 @@ export class DataStores {
                     runtime: this.runtime,
                     storage: this.runtime.storage,
                     scope: this.runtime.scope,
-                    createSummarizerNodeFn: this.getCreateChildSummarizerNodeFn(key, { type: CreateSummarizerNodeSource.FromSummary }),
+                    createSummarizerNodeFn: this.getCreateChildSummarizerNodeFn(key, {
+                        type: CreateSummarizerNodeSource.FromSummary,
+                    }),
                     makeLocallyVisibleFn: () => this.makeDataStoreLocallyVisible(key),
                     snapshotTree,
                     isRootDataStore: undefined,
@@ -140,10 +136,10 @@ export class DataStores {
                 } }));
             throw error;
         }
-        const flatBlobs = new Map();
+        const flatAttachBlobs = new Map();
         let snapshotTree;
         if (attachMessage.snapshot) {
-            snapshotTree = buildSnapshotTree(attachMessage.snapshot.entries, flatBlobs);
+            snapshotTree = buildSnapshotTree(attachMessage.snapshot.entries, flatAttachBlobs);
         }
         // Include the type of attach message which is the pkg of the store to be
         // used by RemoteFluidDataStoreContext in case it is not in the snapshot.
@@ -151,10 +147,8 @@ export class DataStores {
         const remoteFluidDataStoreContext = new RemoteFluidDataStoreContext({
             id: attachMessage.id,
             snapshotTree,
-            // New data stores begin with empty GC details since GC hasn't run on them yet.
-            getBaseGCDetails: async () => { return {}; },
             runtime: this.runtime,
-            storage: new BlobCacheStorageService(this.runtime.storage, flatBlobs),
+            storage: new StorageServiceWithAttachBlobs(this.runtime.storage, flatAttachBlobs),
             scope: this.runtime.scope,
             createSummarizerNodeFn: this.getCreateChildSummarizerNodeFn(attachMessage.id, {
                 type: CreateSummarizerNodeSource.FromAttach,
@@ -229,7 +223,9 @@ export class DataStores {
             runtime: this.runtime,
             storage: this.runtime.storage,
             scope: this.runtime.scope,
-            createSummarizerNodeFn: this.getCreateChildSummarizerNodeFn(id, { type: CreateSummarizerNodeSource.Local }),
+            createSummarizerNodeFn: this.getCreateChildSummarizerNodeFn(id, {
+                type: CreateSummarizerNodeSource.Local,
+            }),
             makeLocallyVisibleFn: () => this.makeDataStoreLocallyVisible(id),
             snapshotTree: undefined,
             isRootDataStore: isRoot,
@@ -245,7 +241,9 @@ export class DataStores {
             runtime: this.runtime,
             storage: this.runtime.storage,
             scope: this.runtime.scope,
-            createSummarizerNodeFn: this.getCreateChildSummarizerNodeFn(id, { type: CreateSummarizerNodeSource.Local }),
+            createSummarizerNodeFn: this.getCreateChildSummarizerNodeFn(id, {
+                type: CreateSummarizerNodeSource.Local,
+            }),
             makeLocallyVisibleFn: () => this.makeDataStoreLocallyVisible(id),
             snapshotTree: undefined,
             isRootDataStore: false,
@@ -254,7 +252,9 @@ export class DataStores {
         this.contexts.addUnbound(context);
         return context;
     }
-    get disposed() { return this.disposeOnce.evaluated; }
+    get disposed() {
+        return this.disposeOnce.evaluated;
+    }
     resubmitDataStoreOp(content, localOpMetadata) {
         const envelope = content;
         const context = this.contexts.get(envelope.address);
@@ -281,6 +281,8 @@ export class DataStores {
     processFluidDataStoreOp(message, local, localMessageMetadata) {
         const envelope = message.contents;
         const transformed = Object.assign(Object.assign({}, message), { contents: envelope.contents });
+        const request = { url: envelope.address };
+        this.validateNotDeleted(envelope.address, request);
         const context = this.contexts.get(envelope.address);
         assert(!!context, 0x162 /* "There should be a store context for the op" */);
         context.process(transformed, local, localMessageMetadata);
@@ -288,32 +290,70 @@ export class DataStores {
         // being used.
         this.gcNodeUpdated(`/${envelope.address}`, message.timestamp, context.isLoaded ? context.packagePath : undefined);
     }
-    async getDataStore(id, wait, viaHandle) {
-        const context = await this.contexts.getBoundOrRemoted(id, wait);
+    async getDataStore(id, requestHeaderData) {
+        const headerData = Object.assign(Object.assign({}, defaultRuntimeHeaderData), requestHeaderData);
         const request = { url: id };
+        this.validateNotDeleted(id, request, headerData);
+        const context = await this.contexts.getBoundOrRemoted(id, headerData.wait);
         if (context === undefined) {
             // The requested data store does not exits. Throw a 404 response exception.
             throw responseToException(create404Response(request), request);
         }
+        this.validateNotTombstoned(context, request, requestHeaderData);
+        return context;
+    }
+    /**
+     * Validate that the data store had not been deleted by GC.
+     *
+     * @param id - data store id
+     * @param request - the request information to log if the validation detects the data store has been deleted
+     * @param requestHeaderData - the request header information to log if the validation detects the data store has been deleted
+     */
+    validateNotDeleted(id, request, requestHeaderData) {
+        const dataStoreNodePath = `/${id}`;
+        if (this.isDataStoreDeleted(dataStoreNodePath)) {
+            assert(!this.contexts.has(id), 0x570 /* Inconsistent state! GC says the data store is deleted, but the data store is not deleted from the runtime. */);
+            // The requested data store is removed by gc. Create a 404 gc response exception.
+            const error = responseToException(createResponseError(404, "DataStore was deleted", request), request);
+            sendGCUnexpectedUsageEvent(this.mc, {
+                eventName: "GC_Deleted_DataStore_Requested",
+                category: "error",
+                isSummarizerClient: this.runtime.clientDetails.type === summarizerClientType,
+                headers: JSON.stringify(requestHeaderData),
+                gcTombstoneEnforcementAllowed: this.runtime.gcTombstoneEnforcementAllowed,
+            }, undefined /** packagePath */, error);
+            throw error;
+        }
+    }
+    /**
+     * Validates that the data store context requested has not been marked as tombstone by GC.
+     *
+     * @param context - the data store context in question
+     * @param request - the request information to log if the validation detects the data store has been tombstoned
+     * @param headerData - the request header information to log if the validation detects the data store has been tombstoned
+     */
+    validateNotTombstoned(context, request, headerData) {
         if (context.tombstoned) {
+            const shouldFail = this.throwOnTombstoneLoad && !headerData.allowTombstone;
             // The requested data store is removed by gc. Create a 404 gc response exception.
-            const error = responseToException(createResponseError(404, "Datastore removed by gc", request), request);
-            // Note: if a user writes a request to look like it's viaHandle, we will also send this telemetry event
-            this.mc.logger.sendErrorEvent({
+            const error = responseToException(createResponseError(404, "DataStore was deleted", request, {
+                [TombstoneResponseHeaderKey]: true,
+            }), request);
+            sendGCUnexpectedUsageEvent(this.mc, {
                 eventName: "GC_Tombstone_DataStore_Requested",
-                url: request.url,
-                pkg: packagePathToTelemetryProperty(context.isLoaded ? context.packagePath : undefined),
-                viaHandle,
-            }, error);
-            // Always log an error when tombstoned data store is used. However, throw an error only if
-            // throwOnTombstoneUsage is set.
-            if (this.throwOnTombstoneUsage) {
+                category: shouldFail ? "error" : "generic",
+                isSummarizerClient: this.runtime.clientDetails.type === summarizerClientType,
+                headers: JSON.stringify(headerData),
+                gcTombstoneEnforcementAllowed: this.runtime.gcTombstoneEnforcementAllowed,
+            }, context.isLoaded ? context.packagePath : undefined, error);
+            if (shouldFail) {
                 throw error;
             }
         }
-        return context;
     }
     processSignal(address, message, local) {
+        const request = { url: address };
+        this.validateNotDeleted(address, request);
         const context = this.contexts.get(address);
         if (!context) {
             // Attach message may not have been processed yet
@@ -339,6 +379,10 @@ export class DataStores {
                     eventName: "SetConnectionStateError",
                     clientId,
                     fluidDataStore,
+                    details: JSON.stringify({
+                        runtimeConnected: this.runtime.connected,
+                        connected,
+                    }),
                 }, error);
             }
         }
@@ -360,10 +404,12 @@ export class DataStores {
         // Iterate over each store and ask it to snapshot
         await Promise.all(Array.from(this.contexts)
             .filter(([_, context]) => {
-            // Summarizer works only with clients with no local changes!
-            assert(context.attachState !== AttachState.Attaching, 0x165 /* "Summarizer cannot work if client has local changes" */);
+            // Summarizer works only with clients with no local changes. A data store in attaching
+            // state indicates an op was sent to attach a local data store.
+            assert(context.attachState !== AttachState.Attaching, 0x588 /* Local data store detected in attaching state during summarize */);
             return context.attachState === AttachState.Attached;
-        }).map(async ([contextId, context]) => {
+        })
+            .map(async ([contextId, context]) => {
             const contextSummary = await context.summarize(fullTree, trackState, telemetryContext);
             summaryBuilder.addWithStats(contextId, contextSummary);
         }));
@@ -383,9 +429,9 @@ export class DataStores {
             // Take summary of bounded data stores only, make sure we haven't summarized them already
             // and no attach op has been fired for that data store because for loader versions <= 0.24
             // we set attach state as "attaching" before taking createNew summary.
-            !(this.contexts.isNotBound(key)
-                || builderTree[key]
-                || this.attachOpFiredForDataStore.has(key)))
+            !(this.contexts.isNotBound(key) ||
+                builderTree[key] ||
+                this.attachOpFiredForDataStore.has(key)))
                 .map(([key, value]) => {
                 let dataStoreSummary;
                 if (value.isLoaded) {
@@ -438,10 +484,12 @@ export class DataStores {
         // Iterate over each store and get their GC data.
         await Promise.all(Array.from(this.contexts)
             .filter(([_, context]) => {
-            // Get GC data only for attached contexts. Detached contexts are not connected in the GC reference
-            // graph so any references they might have won't be connected as well.
+            // Summarizer client and hence GC works only with clients with no local changes. A data store in
+            // attaching state indicates an op was sent to attach a local data store.
+            assert(context.attachState !== AttachState.Attaching, 0x589 /* Local data store detected in attaching state while running GC */);
             return context.attachState === AttachState.Attached;
-        }).map(async ([contextId, context]) => {
+        })
+            .map(async ([contextId, context]) => {
             const contextGCData = await context.getGCData(fullGC);
             // Prefix the child's id to the ids of its GC nodes so they can be identified as belonging to the child.
             // This also gradually builds the id of each node to be a path from the root.
@@ -462,10 +510,6 @@ export class DataStores {
         // Verify that the used routes are correct.
         for (const [id] of usedDataStoreRoutes) {
             assert(this.contexts.has(id), 0x167 /* "Used route does not belong to any known data store" */);
-            // Revive datastores regardless of whether or not tombstone the tombstone flag is flipped
-            const dataStore = this.contexts.get(id);
-            assert(dataStore !== undefined, 0x46e /* No data store retrieved with specified id */);
-            dataStore.setTombstone(false /* tombstone */);
         }
         // Update the used routes in each data store. Used routes is empty for unused data stores.
         for (const [contextId, context] of this.contexts) {
@@ -473,13 +517,10 @@ export class DataStores {
         }
     }
     /**
-     * This is called to update objects whose routes are unused. The unused objects are either deleted or marked as
-     * tombstones.
+     * This is called to update objects whose routes are unused. The unused objects are deleted.
      * @param unusedRoutes - The routes that are unused in all data stores in this Container.
-     * @param tombstone - if true, the objects corresponding to unused routes are marked tombstones. Otherwise, they
-     * are deleted.
      */
-    updateUnusedRoutes(unusedRoutes, tombstone) {
+    updateUnusedRoutes(unusedRoutes) {
         for (const route of unusedRoutes) {
             const pathParts = route.split("/");
             // Delete data store only if its route (/datastoreId) is in unusedRoutes. We don't want to delete a data
@@ -489,23 +530,70 @@ export class DataStores {
             }
             const dataStoreId = pathParts[1];
             assert(this.contexts.has(dataStoreId), 0x2d7 /* No data store with specified id */);
-            /**
-             * When running GC in tombstone mode, datastore contexts are tombstoned. Tombstoned datastore contexts
-             * enable testing scenarios with accessing deleted content without actually deleting content from
-             * summaries.
-             */
-            if (tombstone) {
-                const dataStore = this.contexts.get(dataStoreId);
-                assert(dataStore !== undefined, 0x442 /* No data store retrieved with specified id */);
-                dataStore.setTombstone(true /* tombstone */);
-                continue;
-            }
             // Delete the contexts of unused data stores.
             this.contexts.delete(dataStoreId);
             // Delete the summarizer node of the unused data stores.
             this.deleteChildSummarizerNodeFn(dataStoreId);
         }
     }
+    /**
+     * Delete data stores and its objects that are sweep ready.
+     * @param sweepReadyDataStoreRoutes - The routes of data stores and its objects that are sweep ready and should
+     * be deleted.
+     * @returns - The routes of data stores and its objects that were deleted.
+     */
+    deleteSweepReadyNodes(sweepReadyDataStoreRoutes) {
+        // If sweep for data stores is not enabled, return empty list indicating nothing is deleted.
+        if (this.mc.config.getBoolean(sweepDatastoresKey) !== true) {
+            return [];
+        }
+        for (const route of sweepReadyDataStoreRoutes) {
+            const pathParts = route.split("/");
+            const dataStoreId = pathParts[1];
+            // TODO: GC:Validation - Skip any routes already deleted
+            // Ignore sub-data store routes because a data store and its sub-routes are deleted together, so, we only
+            // need to delete the data store.
+            if (pathParts.length > 2) {
+                continue;
+            }
+            if (!this.contexts.has(dataStoreId)) {
+                this.mc.logger.sendErrorEvent({
+                    eventName: "DeletedDataStoreNotFound",
+                    dataStoreId,
+                });
+            }
+            const dataStore = this.contexts.get(dataStoreId);
+            assert(dataStore !== undefined, 0x571 /* Attempting to delete unknown dataStore */);
+            dataStore.delete();
+            // Delete the contexts of sweep ready data stores.
+            this.contexts.delete(dataStoreId);
+            // Delete the summarizer node of the sweep ready data stores.
+            this.deleteChildSummarizerNodeFn(dataStoreId);
+        }
+        return Array.from(sweepReadyDataStoreRoutes);
+    }
+    /**
+     * This is called to update objects whose routes are tombstones. Tombstoned datastore contexts enable testing
+     * scenarios with accessing deleted content without actually deleting content from summaries.
+     * @param tombstonedRoutes - The routes that are tombstones in all data stores in this Container.
+     */
+    updateTombstonedRoutes(tombstonedRoutes) {
+        const tombstonedDataStoresSet = new Set();
+        for (const route of tombstonedRoutes) {
+            const pathParts = route.split("/");
+            // Tombstone data store only if its route (/datastoreId) is directly in tombstoneRoutes.
+            if (pathParts.length > 2) {
+                continue;
+            }
+            const dataStoreId = pathParts[1];
+            assert(this.contexts.has(dataStoreId), 0x510 /* No data store with specified id */);
+            tombstonedDataStoresSet.add(dataStoreId);
+        }
+        // Update the used routes in each data store. Used routes is empty for unused data stores.
+        for (const [contextId, context] of this.contexts) {
+            context.setTombstone(tombstonedDataStoresSet.has(contextId));
+        }
+    }
     /**
      * Returns the outbound routes of this channel. Only root data stores are considered referenced and their paths are
      * part of outbound routes.