@fluidframework/container-runtime 2.0.0-dev-rc.3.0.0.254674 → 2.0.0-dev-rc.4.0.0.261659

package/src/dataStore.ts

@@ -11,8 +11,11 @@ import {
 	IDataStore,
 	IFluidDataStoreChannel,
 } from "@fluidframework/runtime-definitions/internal";
-import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
-import { TelemetryDataTag, UsageError } from "@fluidframework/telemetry-utils/internal";
+import {
+	ITelemetryLoggerExt,
+	TelemetryDataTag,
+	UsageError,
+} from "@fluidframework/telemetry-utils/internal";
 
 import { ChannelCollection } from "./channelCollection.js";
 import { ContainerMessageType } from "./messageTypes.js";

package/src/dataStoreContext.ts

@@ -219,6 +219,7 @@ export abstract class FluidDataStoreContext
 		return this._containerRuntime;
 	}
 
+	// back-compat, to be removed in 2.0
 	public ensureNoDataModelChanges<T>(callback: () => T): T {
 		return this.parentContext.ensureNoDataModelChanges(callback);
 	}

package/src/dataStoreContexts.ts

@@ -5,8 +5,7 @@
 
 import { IDisposable, ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
 import { assert, Deferred, Lazy } from "@fluidframework/core-utils/internal";
-import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
-import { createChildLogger } from "@fluidframework/telemetry-utils/internal";
+import { ITelemetryLoggerExt, createChildLogger } from "@fluidframework/telemetry-utils/internal";
 
 import { FluidDataStoreContext, LocalFluidDataStoreContext } from "./dataStoreContext.js";
 
@@ -83,9 +82,21 @@ export class DataStoreContexts implements Iterable<[string, FluidDataStoreContex
 	public delete(id: string): boolean {
 		this.deferredContexts.delete(id);
 		this.notBoundContexts.delete(id);
+
+		// Stash the context here in case it's requested in this session, we can log some details about it
+		const context = this._contexts.get(id);
+		this._recentlyDeletedContexts.set(id, context);
+
 		return this._contexts.delete(id);
 	}
 
+	private readonly _recentlyDeletedContexts: Map<string, FluidDataStoreContext | undefined> =
+		new Map();
+
+	public getRecentlyDeletedContext(id: string) {
+		return this._recentlyDeletedContexts.get(id);
+	}
+
 	/**
 	 * Return the unbound local context with the given id,
 	 * or undefined if it's not found or not unbound.

package/src/deltaManagerSummarizerProxy.ts

@@ -25,11 +25,12 @@ import {
 import { summarizerClientType } from "./summary/index.js";
 
 /**
- * Proxy to the real IDeltaManager for restricting certain access to layers below container runtime in summarizer clients:
- * - Summarizer client should be read-only to layers below the container runtime to restrict local changes.
- * - Summarizer client should not be active to layers below the container runtime to restrict local changes.
+ * Base class for DeltaManager proxy that proxy's access to the real DeltaManager.
+ *
+ * This class allows us to build proxy functionality without actually having to implement all the methods
+ * of the DeltaManager.
  */
-export class DeltaManagerSummarizerProxy
+export abstract class BaseDeltaManagerProxy
 	extends TypedEventEmitter<IDeltaManagerEvents>
 	implements IDeltaManager<ISequencedDocumentMessage, IDocumentMessage>
 {
@@ -90,35 +91,20 @@ export class DeltaManagerSummarizerProxy
 	}
 
 	public get active(): boolean {
-		// Summarize clients should not be active. There shouldn't be any local changes (writes) in the summarizer
-		// except for the SummarizeOp which is generated by the runtime.
-		return !this.isSummarizerClient && this.deltaManager.active;
+		return this.deltaManager.active;
 	}
 
 	public get readOnlyInfo(): ReadOnlyInfo {
-		// Summarizer clients should be read-only as far as the runtime and layers below are concerned. There shouldn't
-		// be any local changes (writes) in the summarizer except for the summarize op which is generated by the runtime.
-		if (this.isSummarizerClient) {
-			return {
-				readonly: true,
-				forced: false,
-				permissions: undefined,
-				storageOnly: false,
-			};
-		}
 		return this.deltaManager.readOnlyInfo;
 	}
 
-	private readonly isSummarizerClient: boolean;
-
 	constructor(
-		private readonly deltaManager: IDeltaManager<ISequencedDocumentMessage, IDocumentMessage>,
+		protected readonly deltaManager: IDeltaManager<ISequencedDocumentMessage, IDocumentMessage>,
 	) {
 		super();
 
 		// We are expecting this class to have many listeners, so we suppress noisy "MaxListenersExceededWarning" logging.
 		super.setMaxListeners(0);
-		this.isSummarizerClient = this.deltaManager.clientDetails.type === summarizerClientType;
 
 		this.deltaManager.on("prepareSend", this.onPrepareSend);
 		this.deltaManager.on("submitOp", this.onSubmitOp);
@@ -172,3 +158,39 @@ export class DeltaManagerSummarizerProxy
 		this.emit("readonly", readonly, readonlyConnectionReason);
 	};
 }
+
+/**
+ * Proxy to the real IDeltaManager for restricting certain access to layers below container runtime in summarizer clients:
+ * - Summarizer client should be read-only to layers below the container runtime to restrict local changes.
+ * - Summarizer client should not be active to layers below the container runtime to restrict local changes.
+ */
+export class DeltaManagerSummarizerProxy extends BaseDeltaManagerProxy {
+	public get active(): boolean {
+		// Summarize clients should not be active. There shouldn't be any local changes (writes) in the summarizer
+		// except for the SummarizeOp which is generated by the runtime.
+		return !this.isSummarizerClient && this.deltaManager.active;
+	}
+
+	public get readOnlyInfo(): ReadOnlyInfo {
+		// Summarizer clients should be read-only as far as the runtime and layers below are concerned. There shouldn't
+		// be any local changes (writes) in the summarizer except for the summarize op which is generated by the runtime.
+		if (this.isSummarizerClient) {
+			return {
+				readonly: true,
+				forced: false,
+				permissions: undefined,
+				storageOnly: false,
+			};
+		}
+		return this.deltaManager.readOnlyInfo;
+	}
+
+	private readonly isSummarizerClient: boolean;
+
+	constructor(
+		protected readonly deltaManager: IDeltaManager<ISequencedDocumentMessage, IDocumentMessage>,
+	) {
+		super(deltaManager);
+		this.isSummarizerClient = this.deltaManager.clientDetails.type === summarizerClientType;
+	}
+}

package/src/deltaScheduler.ts

@@ -6,8 +6,7 @@
 import { performance } from "@fluid-internal/client-utils";
 import { IDeltaManager } from "@fluidframework/container-definitions";
 import { IDocumentMessage, ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
-import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
-import { formatTick } from "@fluidframework/telemetry-utils/internal";
+import { ITelemetryLoggerExt, formatTick } from "@fluidframework/telemetry-utils/internal";
 
 /**
  * DeltaScheduler is responsible for the scheduling of inbound delta queue in cases where there

package/src/gc/garbageCollection.ts

@@ -12,8 +12,8 @@ import {
 	gcTreeKey,
 } from "@fluidframework/runtime-definitions/internal";
 import { createResponseError, responseToException } from "@fluidframework/runtime-utils/internal";
-import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
 import {
+	ITelemetryLoggerExt,
 	DataProcessingError,
 	MonitoringContext,
 	PerformanceEvent,
@@ -23,11 +23,7 @@ import {
 } from "@fluidframework/telemetry-utils/internal";
 
 import { BlobManager } from "../blobManager.js";
-import {
-	InactiveResponseHeaderKey,
-	RuntimeHeaderData,
-	TombstoneResponseHeaderKey,
-} from "../containerRuntime.js";
+import { InactiveResponseHeaderKey, TombstoneResponseHeaderKey } from "../containerRuntime.js";
 import { ClientSessionExpiredError } from "../error.js";
 import { ContainerMessageType, ContainerRuntimeGCMessage } from "../messageTypes.js";
 import { IRefreshSummaryResult } from "../summary/index.js";
@@ -48,12 +44,15 @@ import {
 	ISweepPhaseStats,
 	UnreferencedState,
 	disableAutoRecoveryKey,
+	type IGCNodeUpdatedProps,
 } from "./gcDefinitions.js";
 import {
 	cloneGCData,
 	compatBehaviorAllowsGCMessageType,
 	concatGarbageCollectionData,
+	dataStoreNodePathOnly,
 	getGCDataFromSnapshot,
+	urlToGCNodePath,
 } from "./gcHelpers.js";
 import { runGarbageCollection } from "./gcReferenceGraphAlgorithm.js";
 import { IGarbageCollectionSnapshotData, IGarbageCollectionState } from "./gcSummaryDefinitions.js";
@@ -401,17 +400,27 @@ export class GarbageCollector implements IGarbageCollector {
 			return;
 		}
 
-		// If the GC state hasn't been initialized yet, initialize it and return.
-		if (this.gcDataFromLastRun === undefined) {
-			await this.initializeGCStateFromBaseSnapshotP;
-			return;
-		}
+		const initialized = this.gcDataFromLastRun !== undefined;
+		await PerformanceEvent.timedExecAsync(
+			this.mc.logger,
+			{
+				eventName: "InitializeOrUpdateGCState",
+				details: { initialized, unrefNodeCount: this.unreferencedNodesState.size },
+			},
+			async () => {
+				// If the GC state hasn't been initialized yet, initialize it and return.
+				if (!initialized) {
+					await this.initializeGCStateFromBaseSnapshotP;
+					return;
+				}
 
-		// If the GC state has been initialized, update the tracking of unreferenced nodes as per the current
-		// reference timestamp.
-		for (const [, nodeStateTracker] of this.unreferencedNodesState) {
-			nodeStateTracker.updateTracking(currentReferenceTimestampMs);
-		}
+				// If the GC state has been initialized, update the tracking of unreferenced nodes as per the current
+				// reference timestamp.
+				for (const [, nodeStateTracker] of this.unreferencedNodesState) {
+					nodeStateTracker.updateTracking(currentReferenceTimestampMs);
+				}
+			},
+		);
 	}
 
 	/**
@@ -978,30 +987,30 @@ export class GarbageCollector implements IGarbageCollector {
 	/**
 	 * Called when a node with the given id is updated. If the node is inactive or tombstoned, this will log an error
 	 * or throw an error if failing on incorrect usage is configured.
-	 * @param nodePath - The path of the node that changed.
-	 * @param reason - Whether the node was loaded or changed.
-	 * @param timestampMs - The timestamp when the node changed.
-	 * @param packagePath - The package path of the node. This may not be available if the node hasn't been loaded yet.
-	 * @param request - The original request for loads to preserve it in telemetry.
-	 * @param requestHeaders - If the node was loaded via request path, the headers in the request.
+	 * @param IGCNodeUpdatedProps - Details about the node and how it was updated
 	 */
-	public nodeUpdated(
-		nodePath: string,
-		reason: "Loaded" | "Changed",
-		timestampMs?: number,
-		packagePath?: readonly string[],
-		request?: IRequest,
-		headerData?: RuntimeHeaderData,
-	) {
+	public nodeUpdated({
+		node,
+		reason,
+		timestampMs,
+		packagePath,
+		request,
+		headerData,
+	}: IGCNodeUpdatedProps) {
 		if (!this.configs.shouldRunGC) {
 			return;
 		}
 
-		const isTombstoned = this.tombstones.includes(nodePath);
+		// trackedId will be either DataStore or Blob ID (not sub-DataStore ID, since some of those are unrecognized by GC)
+		const trackedId = node.path;
+		const isTombstoned = this.tombstones.includes(trackedId);
+		const isInactive = this.unreferencedNodesState.get(trackedId)?.state === "Inactive";
+
+		const fullPath = request !== undefined ? urlToGCNodePath(request.url) : trackedId;
 
 		// This will log if appropriate
-		this.telemetryTracker.nodeUsed({
-			id: nodePath,
+		this.telemetryTracker.nodeUsed(trackedId, {
+			id: fullPath,
 			usageType: reason,
 			currentReferenceTimestampMs:
 				timestampMs ?? this.runtime.getCurrentReferenceTimestampMs(),
@@ -1010,6 +1019,8 @@ export class GarbageCollector implements IGarbageCollector {
 			isTombstoned,
 			lastSummaryTime: this.getLastSummaryTimestampMs(),
 			headers: headerData,
+			requestUrl: request?.url,
+			requestHeaders: JSON.stringify(request?.headers),
 		});
 
 		// Any time we log a Tombstone Loaded error (via Telemetry Tracker),
@@ -1018,17 +1029,20 @@ export class GarbageCollector implements IGarbageCollector {
 		// to be loaded by the Summarizer, and auto-recovery will be triggered then.
 		if (isTombstoned && reason === "Loaded") {
 			// Note that when a DataStore and its DDS are all loaded, each will trigger AutoRecovery for itself.
-			this.triggerAutoRecovery(nodePath);
+			this.triggerAutoRecovery(fullPath);
 		}
 
-		const nodeType = this.runtime.getNodeType(nodePath);
+		const nodeType = this.runtime.getNodeType(fullPath);
 
 		// Unless this is a Loaded event for a Blob or DataStore, we're done after telemetry tracking
-		if (reason !== "Loaded" || ![GCNodeType.Blob, GCNodeType.DataStore].includes(nodeType)) {
+		const loadedBlobOrDataStore =
+			reason === "Loaded" &&
+			(nodeType === GCNodeType.Blob || nodeType === GCNodeType.DataStore);
+		if (!loadedBlobOrDataStore) {
 			return;
 		}
 
-		const errorRequest: IRequest = request ?? { url: nodePath };
+		const errorRequest: IRequest = request ?? { url: fullPath };
 		if (isTombstoned && this.throwOnTombstoneLoad && headerData?.allowTombstone !== true) {
 			// The requested data store is removed by gc. Create a 404 gc response exception.
 			throw responseToException(
@@ -1040,7 +1054,7 @@ export class GarbageCollector implements IGarbageCollector {
 		}
 
 		// If the object is inactive and inactive enforcement is configured, throw an error.
-		if (this.unreferencedNodesState.get(nodePath)?.state === "Inactive") {
+		if (isInactive) {
 			const shouldThrowOnInactiveLoad =
 				!this.isSummarizerClient &&
 				this.configs.throwOnInactiveLoad === true &&
@@ -1112,22 +1126,30 @@ export class GarbageCollector implements IGarbageCollector {
 		outboundRoutes.push(toNodePath);
 		this.newReferencesSinceLastRun.set(fromNodePath, outboundRoutes);
 
-		this.telemetryTracker.nodeUsed({
+		// GC won't recognize some subDataStore paths that we encounter (e.g. a path suited for a custom request handler)
+		// So for subDataStore paths we need to check the parent dataStore for current tombstone/inactive status.
+		const trackedId =
+			this.runtime.getNodeType(toNodePath) === "SubDataStore"
+				? dataStoreNodePathOnly(toNodePath)
+				: toNodePath;
+		this.telemetryTracker.nodeUsed(trackedId, {
 			id: toNodePath,
+			fromId: fromNodePath,
 			usageType: "Revived",
 			currentReferenceTimestampMs: this.runtime.getCurrentReferenceTimestampMs(),
 			packagePath: undefined,
 			completedGCRuns: this.completedRuns,
-			isTombstoned: this.tombstones.includes(toNodePath),
+			isTombstoned: this.tombstones.includes(trackedId),
 			lastSummaryTime: this.getLastSummaryTimestampMs(),
-			fromId: fromNodePath,
 			autorecovery,
 		});
 
-		// This node is referenced - Clear its unreferenced state
+		// This node is referenced - Clear its unreferenced state if present
 		// But don't delete the node id from the map yet.
 		// When generating GC stats, the set of nodes in here is used as the baseline for
 		// what was unreferenced in the last GC run.
+		// NOTE: We use toNodePath not trackedId even though it may be an unrecognized subDataStore route (hence no-op),
+		// because a reference to such a path is not sufficient to consider the DataStore referenced.
 		this.unreferencedNodesState.get(toNodePath)?.stopTracking();
 	}
 

package/src/gc/gcDefinitions.ts

@@ -12,7 +12,7 @@ import {
 	ISummarizeResult,
 } from "@fluidframework/runtime-definitions/internal";
 import { ReadAndParseBlob } from "@fluidframework/runtime-utils/internal";
-import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
+import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils/internal";
 
 import { RuntimeHeaderData } from "../containerRuntime.js";
 import { ContainerRuntimeGCMessage } from "../messageTypes.js";
@@ -245,7 +245,7 @@ export const GCNodeType = {
 	Blob: "Blob",
 	// Nodes that are neither of the above. For example, root node.
 	Other: "Other",
-};
+} as const;
 
 /**
  * @alpha
@@ -370,14 +370,7 @@ export interface IGarbageCollector {
 	 * Called when a node with the given path is updated. If the node is inactive or tombstoned, this will log an error
 	 * or throw an error if failing on incorrect usage is configured.
 	 */
-	nodeUpdated(
-		nodePath: string,
-		reason: "Loaded" | "Changed",
-		timestampMs?: number,
-		packagePath?: readonly string[],
-		request?: IRequest,
-		headerData?: RuntimeHeaderData,
-	): void;
+	nodeUpdated(props: IGCNodeUpdatedProps): void;
 	/** Called when a reference is added to a node. Used to identify nodes that were referenced between summaries. */
 	addedOutboundReference(fromNodePath: string, toNodePath: string, autorecovery?: true): void;
 	/** Called to process a garbage collection message. */
@@ -388,6 +381,25 @@ export interface IGarbageCollector {
 	dispose(): void;
 }
 
+/**
+ * Info needed by GC when notified that a node was updated (loaded or changed)
+ * @internal
+ */
+export interface IGCNodeUpdatedProps {
+	/** Type and path of the updated node */
+	node: { type: (typeof GCNodeType)["DataStore" | "Blob"]; path: string };
+	/** Whether the node (or a subpath) was loaded or changed. */
+	reason: "Loaded" | "Changed";
+	/** The op-based timestamp when the node changed, if applicable */
+	timestampMs?: number;
+	/** The package path of the node. This may not be available if the node hasn't been loaded yet */
+	packagePath?: readonly string[];
+	/** The original request for loads to preserve it in telemetry */
+	request?: IRequest;
+	/** If the node was loaded via request path, the header data. May be modified from the original request */
+	headerData?: RuntimeHeaderData;
+}
+
 /** Parameters necessary for creating a GarbageCollector. */
 export interface IGarbageCollectorCreateParams {
 	readonly runtime: IGarbageCollectionRuntime;

package/src/gc/gcHelpers.ts

@@ -280,10 +280,23 @@ export function unpackChildNodesGCDetails(gcDetails: IGarbageCollectionDetailsBa
  * @param str - A string that may contain leading and / or trailing slashes.
  * @returns A new string without leading and trailing slashes.
  */
-export function trimLeadingAndTrailingSlashes(str: string) {
+function trimLeadingAndTrailingSlashes(str: string) {
 	return str.replace(/^\/+|\/+$/g, "");
 }
 
+/** Reformats a request URL to match expected format for a GC node path */
+export function urlToGCNodePath(url: string): string {
+	return `/${trimLeadingAndTrailingSlashes(url.split("?")[0])}`;
+}
+
+/**
+ * Pulls out the first path segment and formats it as a GC Node path
+ * e.g. "/dataStoreId/ddsId" yields "/dataStoreId"
+ */
+export function dataStoreNodePathOnly(subDataStorePath: string): string {
+	return `/${subDataStorePath.split("/")[1]}`;
+}
+
 /**
  * Utility to implement compat behaviors given an unknown message type
  * The parameters are typed to support compile-time enforcement of handling all known types/behaviors