@fluidframework/container-runtime 2.0.0-internal.3.1.0 → 2.0.0-internal.3.2.0

package/src/dataStoreContext.ts

@@ -29,7 +29,6 @@ import {
 } from "@fluidframework/protocol-definitions";
 import { IContainerRuntime } from "@fluidframework/container-runtime-definitions";
 import {
-	BindState,
 	channelsTreeName,
 	CreateChildSummarizerNodeFn,
 	CreateChildSummarizerNodeParam,
@@ -49,10 +48,16 @@ import {
 	ISummarizerNodeWithGC,
 	SummarizeInternalFn,
 	ITelemetryContext,
+	VisibilityState,
 } from "@fluidframework/runtime-definitions";
-import { addBlobToSummary, convertSummaryTreeToITree } from "@fluidframework/runtime-utils";
+import {
+	addBlobToSummary,
+	convertSummaryTreeToITree,
+	packagePathToTelemetryProperty,
+} from "@fluidframework/runtime-utils";
 import {
 	ChildLogger,
+	generateStack,
 	loggerToMonitoringContext,
 	LoggingError,
 	MonitoringContext,
@@ -65,7 +70,6 @@ import {
 	extractSafePropertiesFromMessage,
 } from "@fluidframework/container-utils";
 
-import { ContainerRuntime } from "./containerRuntime";
 import {
 	dataStoreAttributesBlobName,
 	hasIsolatedChannels,
@@ -74,10 +78,10 @@ import {
 	WriteFluidDataStoreAttributes,
 	getAttributesFormatVersion,
 	getFluidDataStoreAttributes,
-} from "./summaryFormat";
-import { throwOnTombstoneUsageKey } from "./garbageCollectionConstants";
-import { sendGCUnexpectedUsageEvent } from "./garbageCollectionHelpers";
-import { summarizerClientType } from "./summarizerClientElection";
+	summarizerClientType,
+} from "./summary";
+import { ContainerRuntime } from "./containerRuntime";
+import { sendGCUnexpectedUsageEvent, throwOnTombstoneUsageKey } from "./gc";
 
 function createAttributes(
 	pkg: readonly string[],
@@ -131,7 +135,6 @@ export interface ILocalFluidDataStoreContextProps extends IFluidDataStoreContext
 /** Properties necessary for creating a remote FluidDataStoreContext */
 export interface IRemoteFluidDataStoreContextProps extends IFluidDataStoreContextProps {
 	readonly snapshotTree: ISnapshotTree | undefined;
-	readonly getBaseGCDetails: () => Promise<IGarbageCollectionDetailsBase | undefined>;
 }
 
 /**
@@ -241,8 +244,6 @@ export abstract class FluidDataStoreContext
 	protected registry: IFluidDataStoreRegistry | undefined;
 
 	protected detachedRuntimeCreation = false;
-	/** @deprecated - To be replaced by calling makeLocallyVisible directly  */
-	public readonly bindToContext: () => void;
 	protected channel: IFluidDataStoreChannel | undefined;
 	private loaded = false;
 	protected pending: ISequencedDocumentMessage[] | undefined = [];
@@ -255,6 +256,13 @@ export abstract class FluidDataStoreContext
 	private readonly thresholdOpsCounter: ThresholdCounter;
 	private static readonly pendingOpsCountThreshold = 1000;
 
+	/**
+	 * If the summarizer makes local changes, a telemetry event is logged. This has the potential to be very noisy.
+	 * So, adding a count of how many telemetry events are logged per data store context. This can be
+	 * controlled via feature flags.
+	 */
+	private localChangesTelemetryCount: number;
+
 	// The used routes of this node as per the last GC run. This is used to update the used routes of the channel
 	// if it realizes after GC is run.
 	private lastUsedRoutes: string[] | undefined;
@@ -268,7 +276,6 @@ export abstract class FluidDataStoreContext
 	constructor(
 		props: IFluidDataStoreContextProps,
 		private readonly existing: boolean,
-		private bindState: BindState, // Used to assert for state tracking purposes
 		public readonly isLocalDataStore: boolean,
 		private readonly makeLocallyVisibleFn: () => void,
 	) {
@@ -289,20 +296,6 @@ export abstract class FluidDataStoreContext
 				? this.containerRuntime.attachState
 				: AttachState.Detached;
 
-		this.bindToContext = () => {
-			assert(
-				this.bindState === BindState.NotBound,
-				0x13b /* "datastore context is already in bound state" */,
-			);
-			this.bindState = BindState.Binding;
-			assert(
-				this.channel !== undefined,
-				0x13c /* "undefined channel on datastore context" */,
-			);
-			this.makeLocallyVisible();
-			this.bindState = BindState.Bound;
-		};
-
 		const thisSummarizeInternal = async (
 			fullTree: boolean,
 			trackState: boolean,
@@ -312,7 +305,6 @@ export abstract class FluidDataStoreContext
 		this.summarizerNode = props.createSummarizerNodeFn(
 			thisSummarizeInternal,
 			async (fullGC?: boolean) => this.getGCDataInternal(fullGC),
-			async () => this.getBaseGCDetails(),
 		);
 
 		this.mc = loggerToMonitoringContext(
@@ -328,6 +320,10 @@ export abstract class FluidDataStoreContext
 			this.mc.config.getBoolean(throwOnTombstoneUsageKey) === true &&
 			this._containerRuntime.gcTombstoneEnforcementAllowed &&
 			this.clientDetails.type !== summarizerClientType;
+
+		// By default, a data store can log maximum 10 local changes telemetry in summarizer.
+		this.localChangesTelemetryCount =
+			this.mc.config.getNumber("Fluid.Telemetry.LocalChangesTelemetryCount") ?? 10;
 	}
 
 	public dispose(): void {
@@ -364,9 +360,14 @@ export abstract class FluidDataStoreContext
 		this._tombstoned = tombstone;
 	}
 
-	private rejectDeferredRealize(reason: string, packageName?: string): never {
+	private rejectDeferredRealize(
+		reason: string,
+		failedPkgPath?: string,
+		fullPackageName?: readonly string[],
+	): never {
 		throw new LoggingError(reason, {
-			packageName: { value: packageName, tag: TelemetryDataTag.CodeArtifact },
+			failedPkgPath: { value: failedPkgPath, tag: TelemetryDataTag.CodeArtifact },
+			fullPackageName: packagePathToTelemetryProperty(fullPackageName),
 		});
 	}
 
@@ -384,6 +385,7 @@ export abstract class FluidDataStoreContext
 						value: this.id,
 						tag: TelemetryDataTag.CodeArtifact,
 					},
+					packageName: packagePathToTelemetryProperty(this.pkg),
 				});
 				this.channelDeferred?.reject(errorWrapped);
 				this.logger.sendErrorEvent({ eventName: "RealizeError" }, errorWrapped);
@@ -404,18 +406,22 @@ export abstract class FluidDataStoreContext
 		let lastPkg: string | undefined;
 		for (const pkg of packages) {
 			if (!registry) {
-				this.rejectDeferredRealize("No registry for package", lastPkg);
+				this.rejectDeferredRealize("No registry for package", lastPkg, packages);
 			}
 			lastPkg = pkg;
 			entry = await registry.get(pkg);
 			if (!entry) {
-				this.rejectDeferredRealize("Registry does not contain entry for the package", pkg);
+				this.rejectDeferredRealize(
+					"Registry does not contain entry for the package",
+					pkg,
+					packages,
+				);
 			}
 			registry = entry.IFluidDataStoreRegistry;
 		}
 		const factory = entry?.IFluidDataStoreFactory;
 		if (factory === undefined) {
-			this.rejectDeferredRealize("Can't find factory for package", lastPkg);
+			this.rejectDeferredRealize("Can't find factory for package", lastPkg, packages);
 		}
 
 		return { factory, registry };
@@ -671,6 +677,10 @@ export abstract class FluidDataStoreContext
 			content,
 			type,
 		};
+
+		// Summarizer clients should not submit messages.
+		this.identifyLocalChangeInSummarizer("DataStoreMessageSubmittedInSummarizer", type);
+
 		this._containerRuntime.submitDataStoreOp(this.id, fluidDataStoreContent, localOpMetadata);
 	}
 
@@ -711,6 +721,15 @@ export abstract class FluidDataStoreContext
 	 */
 	public makeLocallyVisible() {
 		assert(this.channel !== undefined, 0x2cf /* "undefined channel on datastore context" */);
+		assert(
+			this.channel.visibilityState === VisibilityState.LocallyVisible,
+			0x590 /* Channel must be locally visible */,
+		);
+		this.makeLocallyVisibleFn();
+	}
+
+	/** @deprecated - To be replaced by calling makeLocallyVisible directly  */
+	public bindToContext() {
 		this.makeLocallyVisibleFn();
 	}
 
@@ -792,7 +811,12 @@ export abstract class FluidDataStoreContext
 		this._isInMemoryRoot = true;
 	}
 
-	public abstract getBaseGCDetails(): Promise<IGarbageCollectionDetailsBase>;
+	/**
+	 * @deprecated - The functionality to get base GC details has been moved to summarizer node.
+	 */
+	public async getBaseGCDetails(): Promise<IGarbageCollectionDetailsBase> {
+		return {};
+	}
 
 	public reSubmit(contents: any, localOpMetadata: unknown) {
 		assert(!!this.channel, 0x14b /* "Channel must exist when resubmitting ops" */);
@@ -865,11 +889,40 @@ export abstract class FluidDataStoreContext
 		}
 	}
 
+	/**
+	 * Summarizer client should not have local changes. These changes can become part of the summary and can break
+	 * eventual consistency. For example, the next summary (say at ref seq# 100) may contain these changes whereas
+	 * other clients that are up-to-date till seq# 100 may not have them yet.
+	 */
+	protected identifyLocalChangeInSummarizer(eventName: string, type?: string) {
+		if (
+			this.clientDetails.type !== summarizerClientType ||
+			this.localChangesTelemetryCount <= 0
+		) {
+			return;
+		}
+
+		// Log a telemetry if there are local changes in the summarizer. This will give us data on how often
+		// this is happening and which data stores do this. The eventual goal is to disallow local changes
+		// in the summarizer and the data will help us plan this.
+		this.mc.logger.sendTelemetryEvent({
+			eventName,
+			type,
+			fluidDataStoreId: {
+				value: this.id,
+				tag: TelemetryDataTag.CodeArtifact,
+			},
+			packageName: packagePathToTelemetryProperty(this.pkg),
+			isSummaryInProgress: this.summarizerNode.isSummaryInProgress?.(),
+			stack: generateStack(),
+		});
+		this.localChangesTelemetryCount--;
+	}
+
 	public getCreateChildSummarizerNodeFn(id: string, createParam: CreateChildSummarizerNodeParam) {
 		return (
 			summarizeInternal: SummarizeInternalFn,
 			getGCDataFn: (fullGC?: boolean) => Promise<IGarbageCollectionData>,
-			getBaseGCDetailsFn?: () => Promise<IGarbageCollectionDetailsBase>,
 		) =>
 			this.summarizerNode.createChild(
 				summarizeInternal,
@@ -878,7 +931,6 @@ export abstract class FluidDataStoreContext
 				// DDS will not create failure summaries
 				{ throwOnFailure: true },
 				getGCDataFn,
-				getBaseGCDetailsFn,
 			);
 	}
 
@@ -889,17 +941,13 @@ export abstract class FluidDataStoreContext
 
 export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
 	private readonly initSnapshotValue: ISnapshotTree | undefined;
-	private readonly baseGCDetailsP: Promise<IGarbageCollectionDetailsBase>;
 
 	constructor(props: IRemoteFluidDataStoreContextProps) {
-		super(props, true /* existing */, BindState.Bound, false /* isLocalDataStore */, () => {
+		super(props, true /* existing */, false /* isLocalDataStore */, () => {
 			throw new Error("Already attached");
 		});
 
 		this.initSnapshotValue = props.snapshotTree;
-		this.baseGCDetailsP = new LazyPromise<IGarbageCollectionDetailsBase>(async () => {
-			return (await props.getBaseGCDetails()) ?? {};
-		});
 
 		if (props.snapshotTree !== undefined) {
 			this.summarizerNode.updateBaseSummaryState(props.snapshotTree);
@@ -959,10 +1007,6 @@ export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
 		return this.initialSnapshotDetailsP;
 	}
 
-	public async getBaseGCDetails(): Promise<IGarbageCollectionDetailsBase> {
-		return this.baseGCDetailsP;
-	}
-
 	public generateAttachMessage(): IAttachMessage {
 		throw new Error("Cannot attach remote store");
 	}
@@ -982,11 +1026,13 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext {
 		super(
 			props,
 			props.snapshotTree !== undefined ? true : false /* existing */,
-			props.snapshotTree ? BindState.Bound : BindState.NotBound,
 			true /* isLocalDataStore */,
 			props.makeLocallyVisibleFn,
 		);
 
+		// Summarizer client should not create local data stores.
+		this.identifyLocalChangeInSummarizer("DataStoreCreatedInSummarizer");
+
 		this.snapshotTree = props.snapshotTree;
 		if (props.isRootDataStore === true) {
 			this.setInMemoryRoot();
@@ -1078,11 +1124,6 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext {
 		};
 	}
 
-	public async getBaseGCDetails(): Promise<IGarbageCollectionDetailsBase> {
-		// Local data store does not have initial summary.
-		return {};
-	}
-
 	/**
 	 * A context should only be marked as deleted when its a remote context.
 	 * Session Expiry at the runtime level should have closed the container creating the local data store context

package/src/dataStores.ts

@@ -21,7 +21,6 @@ import {
 	IEnvelope,
 	IFluidDataStoreContextDetached,
 	IGarbageCollectionData,
-	IGarbageCollectionDetailsBase,
 	IInboundSignalMessage,
 	InboundAttachMessage,
 	ISummarizeResult,
@@ -45,13 +44,9 @@ import {
 } from "@fluidframework/telemetry-utils";
 import { AttachState } from "@fluidframework/container-definitions";
 import { BlobCacheStorageService, buildSnapshotTree } from "@fluidframework/driver-utils";
-import { assert, Lazy, LazyPromise } from "@fluidframework/common-utils";
+import { assert, Lazy } from "@fluidframework/common-utils";
 import { v4 as uuid } from "uuid";
-import {
-	GCDataBuilder,
-	unpackChildNodesGCDetails,
-	unpackChildNodesUsedRoutes,
-} from "@fluidframework/garbage-collector";
+import { GCDataBuilder, unpackChildNodesUsedRoutes } from "@fluidframework/garbage-collector";
 import { DataStoreContexts } from "./dataStoreContexts";
 import {
 	ContainerRuntime,
@@ -66,16 +61,19 @@ import {
 	createAttributesBlob,
 	LocalDetachedFluidDataStoreContext,
 } from "./dataStoreContext";
+import { IDataStoreAliasMessage, isDataStoreAliasMessage } from "./dataStore";
+import {
+	GCNodeType,
+	sweepDatastoresKey,
+	throwOnTombstoneLoadKey,
+	sendGCUnexpectedUsageEvent,
+} from "./gc";
 import {
+	summarizerClientType,
 	IContainerRuntimeMetadata,
 	nonDataStorePaths,
 	rootHasIsolatedChannels,
-} from "./summaryFormat";
-import { IDataStoreAliasMessage, isDataStoreAliasMessage } from "./dataStore";
-import { GCNodeType } from "./garbageCollection";
-import { sweepDatastoresKey, throwOnTombstoneLoadKey } from "./garbageCollectionConstants";
-import { summarizerClientType } from "./summarizerClientElection";
-import { sendGCUnexpectedUsageEvent } from "./garbageCollectionHelpers";
+} from "./summary";
 
 type PendingAliasResolve = (success: boolean) => void;
 
@@ -123,7 +121,6 @@ export class DataStores implements IDisposable {
 		) => CreateChildSummarizerNodeFn,
 		private readonly deleteChildSummarizerNodeFn: (id: string) => void,
 		baseLogger: ITelemetryBaseLogger,
-		getBaseGCDetails: () => Promise<IGarbageCollectionDetailsBase>,
 		private readonly gcNodeUpdated: (
 			nodePath: string,
 			timestampMs: number,
@@ -140,15 +137,6 @@ export class DataStores implements IDisposable {
 			this.runtime.IFluidHandleContext,
 		);
 
-		const baseGCDetailsP = new LazyPromise(async () => {
-			const baseGCDetails = await getBaseGCDetails();
-			return unpackChildNodesGCDetails(baseGCDetails);
-		});
-		// Returns the base GC details for the data store with the given id.
-		const dataStoreBaseGCDetails = async (dataStoreId: string) => {
-			const baseGCDetails = await baseGCDetailsP;
-			return baseGCDetails.get(dataStoreId);
-		};
 		// Tombstone should only throw when the feature flag is enabled and the client isn't a summarizer
 		this.throwOnTombstoneLoad =
 			this.mc.config.getBoolean(throwOnTombstoneLoadKey) === true &&
@@ -177,7 +165,6 @@ export class DataStores implements IDisposable {
 				dataStoreContext = new RemoteFluidDataStoreContext({
 					id: key,
 					snapshotTree: value,
-					getBaseGCDetails: async () => dataStoreBaseGCDetails(key),
 					runtime: this.runtime,
 					storage: this.runtime.storage,
 					scope: this.runtime.scope,
@@ -270,10 +257,6 @@ export class DataStores implements IDisposable {
 		const remoteFluidDataStoreContext = new RemoteFluidDataStoreContext({
 			id: attachMessage.id,
 			snapshotTree,
-			// New data stores begin with empty GC details since GC hasn't run on them yet.
-			getBaseGCDetails: async () => {
-				return {};
-			},
 			runtime: this.runtime,
 			storage: new BlobCacheStorageService(this.runtime.storage, flatBlobs),
 			scope: this.runtime.scope,
@@ -448,6 +431,8 @@ export class DataStores implements IDisposable {
 	) {
 		const envelope = message.contents as IEnvelope;
 		const transformed = { ...message, contents: envelope.contents };
+		const request = { url: envelope.address };
+		this.validateNotDeleted(envelope.address, request);
 		const context = this.contexts.get(envelope.address);
 		assert(!!context, 0x162 /* "There should be a store context for the op" */);
 		context.process(transformed, local, localMessageMetadata);
@@ -491,7 +476,7 @@ export class DataStores implements IDisposable {
 	private validateNotDeleted(
 		id: string,
 		request: IRequest,
-		requestHeaderData: RuntimeHeaderData,
+		requestHeaderData?: RuntimeHeaderData,
 	) {
 		const dataStoreNodePath = `/${id}`;
 		if (this.isDataStoreDeleted(dataStoreNodePath)) {
@@ -562,6 +547,8 @@ export class DataStores implements IDisposable {
 	}
 
 	public processSignal(address: string, message: IInboundSignalMessage, local: boolean) {
+		const request = { url: address };
+		this.validateNotDeleted(address, request);
 		const context = this.contexts.get(address);
 		if (!context) {
 			// Attach message may not have been processed yet
@@ -621,10 +608,11 @@ export class DataStores implements IDisposable {
 		await Promise.all(
 			Array.from(this.contexts)
 				.filter(([_, context]) => {
-					// Summarizer works only with clients with no local changes!
+					// Summarizer works only with clients with no local changes. A data store in attaching
+					// state indicates an op was sent to attach a local data store.
 					assert(
 						context.attachState !== AttachState.Attaching,
-						0x165 /* "Summarizer cannot work if client has local changes" */,
+						0x588 /* Local data store detected in attaching state during summarize */,
 					);
 					return context.attachState === AttachState.Attached;
 				})
@@ -722,8 +710,12 @@ export class DataStores implements IDisposable {
 		await Promise.all(
 			Array.from(this.contexts)
 				.filter(([_, context]) => {
-					// Get GC data only for attached contexts. Detached contexts are not connected in the GC reference
-					// graph so any references they might have won't be connected as well.
+					// Summarizer client and hence GC works only with clients with no local changes. A data store in
+					// attaching state indicates an op was sent to attach a local data store.
+					assert(
+						context.attachState !== AttachState.Attaching,
+						0x589 /* Local data store detected in attaching state while running GC */,
+					);
 					return context.attachState === AttachState.Attached;
 				})
 				.map(async ([contextId, context]) => {
@@ -783,43 +775,44 @@ export class DataStores implements IDisposable {
 	}
 
 	/**
-	 * This is called to delete unused nodes and returns that list so that other
-	 * systems can remove those nodes from their states (i.e. garbage collection when sweep is run)
-	 * @param unusedRoutes - The routes of data stores and DDSes that should be deleted
-	 * @returns - routes of deleted nodes such that garbage collection can delete those nodes from its reference graph
-	 * and other state
+	 * Delete data stores and its objects that are sweep ready.
+	 * @param sweepReadyDataStoreRoutes - The routes of data stores and its objects that are sweep ready and should
+	 * be deleted.
+	 * @returns - The routes of data stores and its objects that were deleted.
 	 */
-	public deleteUnusedNodes(unusedRoutes: string[]): string[] {
+	public deleteSweepReadyNodes(sweepReadyDataStoreRoutes: string[]): string[] {
+		// If sweep for data stores is not enabled, return empty list indicating nothing is deleted.
 		if (this.mc.config.getBoolean(sweepDatastoresKey) !== true) {
 			return [];
 		}
-		const deletedRoutes = new Set<string>();
-
-		for (const route of unusedRoutes) {
+		for (const route of sweepReadyDataStoreRoutes) {
 			const pathParts = route.split("/");
 			const dataStoreId = pathParts[1];
 
 			// TODO: GC:Validation - Skip any routes already deleted
-
-			// Push all deleted DataStore (/datastoreId) and sub DataStore (/datastoreId/...) routes to deleted routes
-			deletedRoutes.add(route);
-
-			// Ignore sub-data store routes because a data store and its sub-routes are deleted together, so, we only need to delete the data store.
+			// Ignore sub-data store routes because a data store and its sub-routes are deleted together, so, we only
+			// need to delete the data store.
 			if (pathParts.length > 2) {
 				continue;
 			}
 
+			if (!this.contexts.has(dataStoreId)) {
+				this.mc.logger.sendErrorEvent({
+					eventName: "DeletedDataStoreNotFound",
+					dataStoreId,
+				});
+			}
+
 			const dataStore = this.contexts.get(dataStoreId);
 			assert(dataStore !== undefined, 0x571 /* Attempting to delete unknown dataStore */);
 			dataStore.delete();
 
-			// Delete the contexts of unused data stores.
+			// Delete the contexts of sweep ready data stores.
 			this.contexts.delete(dataStoreId);
-			// Delete the summarizer node of the unused data stores.
+			// Delete the summarizer node of the sweep ready data stores.
 			this.deleteChildSummarizerNodeFn(dataStoreId);
 		}
-
-		return Array.from(deletedRoutes);
+		return Array.from(sweepReadyDataStoreRoutes);
 	}
 
 	/**

package/{garbageCollection.md → src/gc/garbageCollection.md}

@@ -18,7 +18,7 @@ Currently, the only Fluid objects that are eligible for GC are data stores and a
 
 There are 2 ways to reference a data store:
 
--   Store the data stores's handle (see [IFluidHandle](../../../common/lib/core-interfaces/src/handles.ts)) in a referenced DDS that supports handle in its data. For example, a data store's handle can be stored in a referenced `SharedMap` DDS.
+-   Store the data stores's handle (see [IFluidHandle](../../../../../packages/common/core-interfaces/src/handles.ts) in a referenced DDS that supports handle in its data. For example, a data store's handle can be stored in a referenced `SharedMap` DDS.
 
     Note that storing a handle of any of a data store's DDS will also mark the data store as referenced.
 
@@ -50,7 +50,7 @@ Mark phase is enabled by default for a container. It is enabled during creation
 
 If you wish to disable this, set the `gcAllowed` option to `false` in `IGCRuntimeOptions`. These options are under `IContainerRuntimeOptions` and are passed to the container runtime during its creation. Note that this will disable GC permanently (including the sweep phase) for the container during its lifetime.
 
-See `IGCRuntimeOptions` in [containerRuntime.ts](./src/containerRuntime.ts) for more options to control GC behavior.
+See `IGCRuntimeOptions` in [containerRuntime.ts](../containerRuntime.ts) for more options to control GC behavior.
 
 ### Sweep phase