@fluidframework/container-runtime 2.0.0-internal.7.2.2 → 2.0.0-internal.7.4.0

package/src/dataStores.ts

@@ -40,6 +40,7 @@ import {
 import {
 	createChildMonitoringContext,
 	DataCorruptionError,
+	DataProcessingError,
 	extractSafePropertiesFromMessage,
 	LoggingError,
 	MonitoringContext,
@@ -60,13 +61,8 @@ import {
 } from "./dataStoreContext";
 import { StorageServiceWithAttachBlobs } from "./storageServiceWithAttachBlobs";
 import { IDataStoreAliasMessage, isDataStoreAliasMessage } from "./dataStore";
-import { GCNodeType, disableDatastoreSweepKey, sendGCUnexpectedUsageEvent } from "./gc";
-import {
-	summarizerClientType,
-	IContainerRuntimeMetadata,
-	nonDataStorePaths,
-	rootHasIsolatedChannels,
-} from "./summary";
+import { GCNodeType, disableDatastoreSweepKey } from "./gc";
+import { IContainerRuntimeMetadata, nonDataStorePaths, rootHasIsolatedChannels } from "./summary";
 
 type PendingAliasResolve = (success: boolean) => void;
 
@@ -280,6 +276,19 @@ export class DataStores implements IDisposable {
 		}
 
 		const context = this.contexts.get(aliasMessage.internalId);
+		// If the data store has been deleted, log an error and ignore this message. This helps prevent document
+		// corruption in case a deleted data store accidentally submitted a signal.
+		if (
+			this.checkAndLogIfDeleted(
+				aliasMessage.internalId,
+				context,
+				"Changed",
+				"processAliasMessageCore",
+			)
+		) {
+			return false;
+		}
+
 		if (context === undefined) {
 			this.mc.logger.sendErrorEvent({
 				eventName: "AliasFluidDataStoreNotFound",
@@ -381,18 +390,43 @@ export class DataStores implements IDisposable {
 
 	public resubmitDataStoreOp(envelope: IEnvelope, localOpMetadata: unknown) {
 		const context = this.contexts.get(envelope.address);
+		// If the data store has been deleted, log an error and throw an error. If there are local changes for a
+		// deleted data store, it can otherwise lead to inconsistent state when compared to other clients.
+		if (
+			this.checkAndLogIfDeleted(envelope.address, context, "Changed", "resubmitDataStoreOp")
+		) {
+			throw new DataCorruptionError("Context is deleted!", {
+				callSite: "resubmitDataStoreOp",
+				...tagCodeArtifacts({ id: envelope.address }),
+			});
+		}
 		assert(!!context, 0x160 /* "There should be a store context for the op" */);
 		context.reSubmit(envelope.contents, localOpMetadata);
 	}
 
 	public rollbackDataStoreOp(envelope: IEnvelope, localOpMetadata: unknown) {
 		const context = this.contexts.get(envelope.address);
+		// If the data store has been deleted, log an error and throw an error. If there are local changes for a
+		// deleted data store, it can otherwise lead to inconsistent state when compared to other clients.
+		if (
+			this.checkAndLogIfDeleted(envelope.address, context, "Changed", "rollbackDataStoreOp")
+		) {
+			throw new DataCorruptionError("Context is deleted!", {
+				callSite: "rollbackDataStoreOp",
+				...tagCodeArtifacts({ id: envelope.address }),
+			});
+		}
 		assert(!!context, 0x2e8 /* "There should be a store context for the op" */);
 		context.rollback(envelope.contents, localOpMetadata);
 	}
 
 	public async applyStashedOp(envelope: IEnvelope): Promise<unknown> {
 		const context = this.contexts.get(envelope.address);
+		// If the data store has been deleted, log an error and ignore this message. This helps prevent document
+		// corruption in case the data store that stashed the op is deleted.
+		if (this.checkAndLogIfDeleted(envelope.address, context, "Changed", "applyStashedOp")) {
+			return undefined;
+		}
 		assert(!!context, 0x161 /* "There should be a store context for the op" */);
 		return context.applyStashedOp(envelope.contents);
 	}
@@ -410,8 +444,21 @@ export class DataStores implements IDisposable {
 	) {
 		const envelope = message.contents as IEnvelope;
 		const transformed = { ...message, contents: envelope.contents };
-		this.validateNotDeleted(envelope.address);
 		const context = this.contexts.get(envelope.address);
+
+		// If the data store has been deleted, log an error and ignore this message. This helps prevent document
+		// corruption in case a deleted data store accidentally submitted an op.
+		if (
+			this.checkAndLogIfDeleted(
+				envelope.address,
+				context,
+				"Changed",
+				"processFluidDataStoreOp",
+			)
+		) {
+			return;
+		}
+
 		assert(!!context, 0x162 /* "There should be a store context for the op" */);
 		context.process(transformed, local, localMessageMetadata);
 
@@ -429,7 +476,22 @@ export class DataStores implements IDisposable {
 		requestHeaderData: RuntimeHeaderData,
 	): Promise<FluidDataStoreContext> {
 		const headerData = { ...defaultRuntimeHeaderData, ...requestHeaderData };
-		this.validateNotDeleted(id, headerData);
+		if (
+			this.checkAndLogIfDeleted(
+				id,
+				this.contexts.get(id),
+				"Requested",
+				"getDataStore",
+				requestHeaderData,
+			)
+		) {
+			// The requested data store has been deleted by gc. Create a 404 response exception.
+			const request: IRequest = { url: id };
+			throw responseToException(
+				createResponseError(404, "DataStore was deleted", request),
+				request,
+			);
+		}
 
 		const context = await this.contexts.getBoundOrRemoted(id, headerData.wait);
 		if (context === undefined) {
@@ -447,8 +509,16 @@ export class DataStores implements IDisposable {
 		id: string,
 		requestHeaderData: RuntimeHeaderData,
 	): Promise<FluidDataStoreContext | undefined> {
-		// If the data store has been deleted, return undefined.
-		if (this.checkIfDeleted(id, requestHeaderData)) {
+		// If the data store has been deleted, log an error and return undefined.
+		if (
+			this.checkAndLogIfDeleted(
+				id,
+				this.contexts.get(id),
+				"Requested",
+				"getDataStoreIfAvailable",
+				requestHeaderData,
+			)
+		) {
 			return undefined;
 		}
 		const headerData = { ...defaultRuntimeHeaderData, ...requestHeaderData };
@@ -460,55 +530,43 @@ export class DataStores implements IDisposable {
 	}
 
 	/**
-	 * Checks if the data store has been deleted by GC.
-	 * @param id - data store id
-	 * @param request - the request information to log if the validation detects the data store has been deleted
-	 * @param requestHeaderData - the request header information to log if the validation detects the data store has been deleted
+	 * Checks if the data store has been deleted by GC. If so, log an error.
+	 * @param id - The data store's id.
+	 * @param context - The data store context.
+	 * @param callSite - The function name this is called from.
+	 * @param requestHeaderData - The request header information to log if the data store is deleted.
 	 * @returns true if the data store is deleted. Otherwise, returns false.
 	 */
-	private checkIfDeleted(id: string, requestHeaderData?: RuntimeHeaderData) {
+	private checkAndLogIfDeleted(
+		id: string,
+		context: FluidDataStoreContext | undefined,
+		deletedLogSuffix: string,
+		callSite: string,
+		requestHeaderData?: RuntimeHeaderData,
+	) {
 		const dataStoreNodePath = `/${id}`;
 		if (!this.isDataStoreDeleted(dataStoreNodePath)) {
 			return false;
 		}
-		assert(
-			!this.contexts.has(id),
-			0x570 /* Inconsistent state! GC says the data store is deleted, but the data store is not deleted from the runtime. */,
-		);
-		sendGCUnexpectedUsageEvent(
-			this.mc,
-			{
-				eventName: "GC_Deleted_DataStore_Requested",
-				category: "error",
-				isSummarizerClient: this.runtime.clientDetails.type === summarizerClientType,
-				id,
-				headers: JSON.stringify(requestHeaderData),
-				gcTombstoneEnforcementAllowed: this.runtime.gcTombstoneEnforcementAllowed,
-			},
-			undefined /* packagePath */,
-		);
-		return true;
-	}
 
-	/**
-	 * Validate that the data store had not been deleted by GC.
-	 * @param id - data store id
-	 * @param requestHeaderData - the request header information to log if the validation detects the data store has been deleted
-	 */
-	private validateNotDeleted(id: string, requestHeaderData?: RuntimeHeaderData) {
-		if (this.checkIfDeleted(id, requestHeaderData)) {
-			// The requested data store is removed by gc. Create a 404 gc response exception.
-			const request: IRequest = { url: id };
-			throw responseToException(
-				createResponseError(404, "DataStore was deleted", request),
-				request,
-			);
-		}
+		this.mc.logger.sendErrorEvent({
+			eventName: `GC_Deleted_DataStore_${deletedLogSuffix}`,
+			...tagCodeArtifacts({ id }),
+			callSite,
+			headers: JSON.stringify(requestHeaderData),
+			exists: context !== undefined,
+		});
+		return true;
 	}
 
 	public processSignal(fluidDataStoreId: string, message: IInboundSignalMessage, local: boolean) {
-		this.validateNotDeleted(fluidDataStoreId);
 		const context = this.contexts.get(fluidDataStoreId);
+		// If the data store has been deleted, log an error and ignore this message. This helps prevent document
+		// corruption in case a deleted data store accidentally submitted a signal.
+		if (this.checkAndLogIfDeleted(fluidDataStoreId, context, "Changed", "processSignal")) {
+			return;
+		}
+
 		if (!context) {
 			// Attach message may not have been processed yet
 			assert(!local, 0x163 /* "Missing datastore for local signal" */);
@@ -573,11 +631,16 @@ export class DataStores implements IDisposable {
 			Array.from(this.contexts)
 				.filter(([_, context]) => {
 					// Summarizer works only with clients with no local changes. A data store in attaching
-					// state indicates an op was sent to attach a local data store.
-					assert(
-						context.attachState !== AttachState.Attaching,
-						0x588 /* Local data store detected in attaching state during summarize */,
-					);
+					// state indicates an op was sent to attach a local data store, and the the attach op
+					// had not yet round tripped back to the client.
+					if (context.attachState === AttachState.Attaching) {
+						// Formerly assert 0x588
+						const error = DataProcessingError.create(
+							"Local data store detected in attaching state during summarize",
+							"summarize",
+						);
+						throw error;
+					}
 					return context.attachState === AttachState.Attached;
 				})
 				.map(async ([contextId, context]) => {
@@ -675,11 +738,17 @@ export class DataStores implements IDisposable {
 			Array.from(this.contexts)
 				.filter(([_, context]) => {
 					// Summarizer client and hence GC works only with clients with no local changes. A data store in
-					// attaching state indicates an op was sent to attach a local data store.
-					assert(
-						context.attachState !== AttachState.Attaching,
-						0x589 /* Local data store detected in attaching state while running GC */,
-					);
+					// attaching state indicates an op was sent to attach a local data store, and the the attach op
+					// had not yet round tripped back to the client.
+					// Formerly assert 0x589
+					if (context.attachState === AttachState.Attaching) {
+						const error = DataProcessingError.create(
+							"Local data store detected in attaching state while running GC",
+							"getGCData",
+						);
+						throw error;
+					}
+
 					return context.attachState === AttachState.Attached;
 				})
 				.map(async ([contextId, context]) => {
@@ -699,7 +768,7 @@ export class DataStores implements IDisposable {
 	 * After GC has run, called to notify this Container's data stores of routes that are used in it.
 	 * @param usedRoutes - The routes that are used in all data stores in this Container.
 	 */
-	public updateUsedRoutes(usedRoutes: string[]) {
+	public updateUsedRoutes(usedRoutes: readonly string[]) {
 		// Get a map of data store ids to routes used in it.
 		const usedDataStoreRoutes = unpackChildNodesUsedRoutes(usedRoutes);
 
@@ -721,7 +790,7 @@ export class DataStores implements IDisposable {
 	 * This is called to update objects whose routes are unused. The unused objects are deleted.
 	 * @param unusedRoutes - The routes that are unused in all data stores in this Container.
 	 */
-	public updateUnusedRoutes(unusedRoutes: string[]) {
+	public updateUnusedRoutes(unusedRoutes: readonly string[]) {
 		for (const route of unusedRoutes) {
 			const pathParts = route.split("/");
 			// Delete data store only if its route (/datastoreId) is in unusedRoutes. We don't want to delete a data
@@ -744,7 +813,7 @@ export class DataStores implements IDisposable {
 	 * be deleted.
 	 * @returns The routes of data stores and its objects that were deleted.
 	 */
-	public deleteSweepReadyNodes(sweepReadyDataStoreRoutes: string[]): string[] {
+	public deleteSweepReadyNodes(sweepReadyDataStoreRoutes: readonly string[]): readonly string[] {
 		// If sweep for data stores is not enabled, return empty list indicating nothing is deleted.
 		if (this.mc.config.getBoolean(disableDatastoreSweepKey) === true) {
 			return [];
@@ -753,23 +822,25 @@ export class DataStores implements IDisposable {
 			const pathParts = route.split("/");
 			const dataStoreId = pathParts[1];
 
-			// TODO: GC:Validation - Skip any routes already deleted
 			// Ignore sub-data store routes because a data store and its sub-routes are deleted together, so, we only
 			// need to delete the data store.
 			if (pathParts.length > 2) {
 				continue;
 			}
 
-			if (!this.contexts.has(dataStoreId)) {
+			const dataStoreContext = this.contexts.get(dataStoreId);
+			if (dataStoreContext === undefined) {
 				this.mc.logger.sendErrorEvent({
 					eventName: "DeletedDataStoreNotFound",
-					dataStoreId,
+					...tagCodeArtifacts({ id: dataStoreId }),
+					details: {
+						alreadyDeleted: this.isDataStoreDeleted(dataStoreId),
+					},
 				});
+				continue;
 			}
 
-			const dataStore = this.contexts.get(dataStoreId);
-			assert(dataStore !== undefined, 0x571 /* Attempting to delete unknown dataStore */);
-			dataStore.delete();
+			dataStoreContext.delete();
 
 			// Delete the contexts of sweep ready data stores.
 			this.contexts.delete(dataStoreId);
@@ -784,7 +855,7 @@ export class DataStores implements IDisposable {
 	 * scenarios with accessing deleted content without actually deleting content from summaries.
 	 * @param tombstonedRoutes - The routes that are tombstones in all data stores in this Container.
 	 */
-	public updateTombstonedRoutes(tombstonedRoutes: string[]) {
+	public updateTombstonedRoutes(tombstonedRoutes: readonly string[]) {
 		const tombstonedDataStoresSet: Set<string> = new Set();
 		for (const route of tombstonedRoutes) {
 			const pathParts = route.split("/");

package/src/gc/garbageCollection.md

@@ -50,16 +50,19 @@ Mark phase is enabled by default for a container. It is enabled during creation
 
 ### Sweep phase
 
-In this phase, the GC algorithm identifies all Fluid objects that have been unreferenced for a specific amount of time (typically 30-40 days) and deletes them.
-Objects are only swept once the GC system is sure that they could never be referenced again by any active clients, i.e., clients that have the object in memory and could reference it.
-The Fluid Runtime enforces a maximum session length (configurable) in order to guarantee an object is safe to delete after sufficient time has elapsed.
+In this phase, the GC algorithm deletes any Fluid object that has been unreferenced for a sufficient time to guarantee
+they could never be referenced again by any active clients, i.e., clients that have the object in memory and could reference it again.
+The Fluid Runtime enforces a maximum session length (configurable) in order to guarantee all in-memory objects are cleared before
+it concludes an object is safe to delete.
 
-GC sweep phase has not been enabled by default yet. A "soft" version of Sweep called "Tombstone Mode" is enabled by default
-as part of the Mark Phase when Sweep is disabled. In this mode, any object that GC determines is ready to be deleted is
-marked as a "Tombstone", which triggers certain logging events and/or behavior changes if/when that Tombstoned object is
-accessed by the application.
+GC sweep phase runs in two stages:
 
-Tombstone is intended for use by early adopters of GC and is documented in more detail [here](./gcEarlyAdoption.md).
+-   The first stage is the "Tombstone" stage, where objects are marked as Tombstones, meaning GC believes they will
+    never be referenced again and are safe to delete. They are not yet deleted at this point, but any attempt to
+    load them will fail. This way, there's a chance to recover a Tombstoned object in case we detect it's still being used.
+-   The second stage is the "Sweep" or "Delete" stage, where the objects are fully deleted.
+    This occurs after a configurable delay called the "Sweep Grace Period", to give time for application teams
+    to monitor for Tombstone-related errors and react before delete occurs.
 
 ## GC Configuration
 
@@ -67,7 +70,8 @@ The default configuration for GC today is:
 
 -   GC Mark Phase is **enabled**, including Tombstone Mode
 -   Session Expiry is **enabled**
--   GC Sweep Phase is **disabled**
+-   The "Tombstone" stage of Sweep Phase is **enabled** (attempting to load a tombstoned object will fail)
+-   The "Delete" stage of Sweep Phase is **disabled**
     -   Note: Once enabled, Sweep will only run for documents created from that point forward
 
 ### Techniques used for configuration
@@ -94,12 +98,7 @@ covered in the [Advanced Configuration](./gcEarlyAdoption.md#more-advanced-confi
 
 ### Enabling Sweep Phase
 
-To enable Sweep Phase for new documents, you must set the `gcSweepGeneration` GC Option to a number, e.g. 0 to start.
-The full semantics of this GC Option are discussed [here](./gcEarlyAdoption.md#more-about-gcsweepgeneration-and-gctombstonegeneration).
-Note that this will disabled Tombstone Mode.
-
-A full treatment of Tombstone and Sweep configuration can be found in
-[this companion document geared towards early adopters of GC](./gcEarlyAdoption.md).
+To enable the Sweep Phase for new documents, you must set the `enableGCSweep` GC Option to true.
 
 ### More Advanced Configuration