@fluidframework/container-runtime 2.0.0-dev.7.4.0.215747 → 2.0.0-dev.7.4.0.216897

package/src/gc/garbageCollection.ts

@@ -40,6 +40,8 @@ import {
 	UnreferencedState,
 	IGCMetadata,
 	IGarbageCollectorConfigs,
+	IMarkPhaseStats,
+	ISweepPhaseStats,
 } from "./gcDefinitions";
 import { cloneGCData, concatGarbageCollectionData, getGCDataFromSnapshot } from "./gcHelpers";
 import { runGarbageCollection } from "./gcReferenceGraphAlgorithm";
@@ -305,6 +307,7 @@ export class GarbageCollector implements IGarbageCollector {
 			eventName: "GarbageCollectorLoaded",
 			gcConfigs: JSON.stringify(this.configs),
 			gcOptions: JSON.stringify(createParams.gcOptions),
+			...createParams.createContainerMetadata,
 		});
 	}
 
@@ -520,7 +523,11 @@ export class GarbageCollector implements IGarbageCollector {
 
 				/** GC step */
 				const gcStats = await this.runGC(fullGC, currentReferenceTimestampMs, logger);
-				event.end({ ...gcStats, timestamp: currentReferenceTimestampMs });
+				event.end({
+					...gcStats,
+					timestamp: currentReferenceTimestampMs,
+					sweep: this.configs.shouldRunSweep,
+				});
 
 				/** Post-GC steps */
 				// Log pending unreferenced events such as a node being used after inactive. This is done after GC runs and
@@ -541,9 +548,10 @@ export class GarbageCollector implements IGarbageCollector {
 	/**
 	 * Runs garbage collection. It does the following:
 	 * 1. It generates / analyzes the runtime's reference graph.
-	 * 2. Generates stats for the GC run based on previous / current GC state.
+	 * 2. Generates mark phase stats.
 	 * 3. Runs Mark phase.
 	 * 4. Runs Sweep phase.
+	 * 5. Generates sweep phase stats.
 	 */
 	private async runGC(
 		fullGC: boolean,
@@ -559,9 +567,9 @@ export class GarbageCollector implements IGarbageCollector {
 			this.findAllNodesReferencedBetweenGCs(gcData, this.gcDataFromLastRun, logger) ??
 			gcResult.referencedNodeIds;
 
-		// 2. Generate stats based on the previous / current GC state.
-		// Must happen before running Mark / Sweep phase because previous GC state will be updated in these stages.
-		const gcStats = this.generateStats(gcResult);
+		// 2. Get the mark phase stats based on the previous / current GC state.
+		// This is done before running mark phase because we need the previous GC state before it is updated.
+		const markPhaseStats = this.getMarkPhaseStats(gcResult);
 
 		// 3. Run the Mark phase.
 		// It will mark nodes as referenced / unreferenced and return a list of node ids that are ready to be swept.
@@ -573,18 +581,21 @@ export class GarbageCollector implements IGarbageCollector {
 
 		// 4. Run the Sweep phase.
 		// It will delete sweep ready nodes and return a list of deleted node ids.
-		const deletedNodeIds = this.runSweepPhase(
-			gcResult,
-			sweepReadyNodeIds,
-			currentReferenceTimestampMs,
-			logger,
-		);
+		const deletedNodeIds = this.runSweepPhase(gcResult, sweepReadyNodeIds);
 
 		this.gcDataFromLastRun = cloneGCData(
 			gcData,
 			(id: string) => deletedNodeIds.includes(id) /* filter out deleted nodes */,
 		);
-		return gcStats;
+
+		// 5. Get the sweep phase stats.
+		const sweepPhaseStats = this.getSweepPhaseStats(
+			this.configs.shouldRunSweep ? this.deletedNodes : sweepReadyNodeIds,
+			new Set(deletedNodeIds),
+			markPhaseStats,
+		);
+
+		return { ...markPhaseStats, ...sweepPhaseStats };
 	}
 
 	/**
@@ -605,7 +616,7 @@ export class GarbageCollector implements IGarbageCollector {
 		gcResult: IGCResult,
 		allReferencedNodeIds: string[],
 		currentReferenceTimestampMs: number,
-	): string[] {
+	): Set<string> {
 		// 1. Marks all referenced nodes by clearing their unreferenced tracker, if any.
 		for (const nodeId of allReferencedNodeIds) {
 			const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
@@ -618,7 +629,7 @@ export class GarbageCollector implements IGarbageCollector {
 		}
 
 		// 2. Mark unreferenced nodes in this run by starting unreferenced tracking for them.
-		const sweepReadyNodeIds: string[] = [];
+		const sweepReadyNodeIds: Set<string> = new Set();
 		for (const nodeId of gcResult.deletedNodeIds) {
 			const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
 			if (nodeStateTracker === undefined) {
@@ -638,7 +649,7 @@ export class GarbageCollector implements IGarbageCollector {
 
 				// If a node is sweep ready, store it so it can be returned.
 				if (nodeStateTracker.state === UnreferencedState.SweepReady) {
-					sweepReadyNodeIds.push(nodeId);
+					sweepReadyNodeIds.add(nodeId);
 				}
 			}
 		}
@@ -660,22 +671,7 @@ export class GarbageCollector implements IGarbageCollector {
 	 * @param logger - The logger to be used to log any telemetry.
 	 * @returns A list of nodes that have been deleted.
 	 */
-	private runSweepPhase(
-		gcResult: IGCResult,
-		sweepReadyNodes: string[],
-		currentReferenceTimestampMs: number,
-		logger: ITelemetryLoggerExt,
-	): string[] {
-		// Log events for objects that are ready to be deleted by sweep. This will give us data on sweep when
-		// its not enabled.
-		this.telemetryTracker.logSweepEvents(
-			logger,
-			currentReferenceTimestampMs,
-			this.unreferencedNodesState,
-			this.completedRuns,
-			this.getLastSummaryTimestampMs(),
-		);
-
+	private runSweepPhase(gcResult: IGCResult, sweepReadyNodes: Set<string>): string[] {
 		/**
 		 * Currently, there are 3 modes for sweep:
 		 * Test mode - Unreferenced nodes are immediately deleted without waiting for them to be sweep ready.
@@ -692,7 +688,7 @@ export class GarbageCollector implements IGarbageCollector {
 		}
 
 		if (this.configs.tombstoneMode) {
-			this.tombstones = sweepReadyNodes;
+			this.tombstones = Array.from(sweepReadyNodes);
 			// If we are running in GC tombstone mode, update tombstoned routes. This enables testing scenarios
 			// involving access to "deleted" data without actually deleting the data from summaries.
 			this.runtime.updateTombstonedRoutes(this.tombstones);
@@ -706,7 +702,7 @@ export class GarbageCollector implements IGarbageCollector {
 		// 1. Call the runtime to delete sweep ready nodes. The runtime returns a list of nodes it deleted.
 		// TODO: GC:Validation - validate that removed routes are not double delete and that the child routes of
 		// removed routes are deleted as well.
-		const deletedNodeIds = this.runtime.deleteSweepReadyNodes(sweepReadyNodes);
+		const deletedNodeIds = this.runtime.deleteSweepReadyNodes(Array.from(sweepReadyNodes));
 
 		// 2. Clear unreferenced state tracking for deleted nodes.
 		for (const nodeId of deletedNodeIds) {
@@ -976,12 +972,12 @@ export class GarbageCollector implements IGarbageCollector {
 	}
 
 	/**
-	 * Generates the stats of a garbage collection run from the given results of the run.
-	 * @param gcResult - The result of a GC run.
-	 * @returns the GC stats of the GC run.
+	 * Generates the stats of a garbage collection mark phase run.
+	 * @param gcResult - The result of the current GC run.
+	 * @returns the stats of the mark phase run.
 	 */
-	private generateStats(gcResult: IGCResult): IGCStats {
-		const gcStats: IGCStats = {
+	private getMarkPhaseStats(gcResult: IGCResult): IMarkPhaseStats {
+		const markPhaseStats: IMarkPhaseStats = {
 			nodeCount: 0,
 			dataStoreCount: 0,
 			attachmentBlobCount: 0,
@@ -994,35 +990,35 @@ export class GarbageCollector implements IGarbageCollector {
 		};
 
 		const updateNodeStats = (nodeId: string, referenced: boolean) => {
-			gcStats.nodeCount++;
+			markPhaseStats.nodeCount++;
 			// If there is no previous GC data, every node's state is generated and is considered as updated.
 			// Otherwise, find out if any node went from referenced to unreferenced or vice-versa.
 			const stateUpdated =
 				this.gcDataFromLastRun === undefined ||
 				this.unreferencedNodesState.has(nodeId) === referenced;
 			if (stateUpdated) {
-				gcStats.updatedNodeCount++;
+				markPhaseStats.updatedNodeCount++;
 			}
 			if (!referenced) {
-				gcStats.unrefNodeCount++;
+				markPhaseStats.unrefNodeCount++;
 			}
 
 			if (this.runtime.getNodeType(nodeId) === GCNodeType.DataStore) {
-				gcStats.dataStoreCount++;
+				markPhaseStats.dataStoreCount++;
 				if (stateUpdated) {
-					gcStats.updatedDataStoreCount++;
+					markPhaseStats.updatedDataStoreCount++;
 				}
 				if (!referenced) {
-					gcStats.unrefDataStoreCount++;
+					markPhaseStats.unrefDataStoreCount++;
 				}
 			}
 			if (this.runtime.getNodeType(nodeId) === GCNodeType.Blob) {
-				gcStats.attachmentBlobCount++;
+				markPhaseStats.attachmentBlobCount++;
 				if (stateUpdated) {
-					gcStats.updatedAttachmentBlobCount++;
+					markPhaseStats.updatedAttachmentBlobCount++;
 				}
 				if (!referenced) {
-					gcStats.unrefAttachmentBlobCount++;
+					markPhaseStats.unrefAttachmentBlobCount++;
 				}
 			}
 		};
@@ -1035,6 +1031,63 @@ export class GarbageCollector implements IGarbageCollector {
 			updateNodeStats(nodeId, false /* referenced */);
 		}
 
-		return gcStats;
+		return markPhaseStats;
+	}
+
+	/**
+	 * Generates the stats of a garbage collection sweep phase run.
+	 * @param allDeletedNodes - All the nodes that have been deleted across all GC runs.
+	 * @param currentDeletedNodes - The nodes that have been deleted in this GC run.
+	 * @param markPhaseStats - The stats of the mark phase run.
+	 * @returns the stats of the sweep phase run.
+	 */
+	private getSweepPhaseStats(
+		allDeletedNodes: Set<string>,
+		currentDeletedNodes: Set<string>,
+		markPhaseStats: IMarkPhaseStats,
+	): ISweepPhaseStats {
+		// Initialize the life time node counts to the mark phase node counts. If sweep is not enabled,
+		// these will be the life time node count for this container.
+		const sweepPhaseStats: ISweepPhaseStats = {
+			lifetimeNodeCount: markPhaseStats.nodeCount,
+			lifetimeDataStoreCount: markPhaseStats.dataStoreCount,
+			lifetimeAttachmentBlobCount: markPhaseStats.attachmentBlobCount,
+			deletedNodeCount: 0,
+			deletedDataStoreCount: 0,
+			deletedAttachmentBlobCount: 0,
+		};
+
+		for (const nodeId of allDeletedNodes) {
+			sweepPhaseStats.deletedNodeCount++;
+			const nodeType = this.runtime.getNodeType(nodeId);
+			if (nodeType === GCNodeType.DataStore) {
+				sweepPhaseStats.deletedDataStoreCount++;
+			} else if (nodeType === GCNodeType.Blob) {
+				sweepPhaseStats.deletedAttachmentBlobCount++;
+			}
+		}
+
+		if (!this.configs.shouldRunSweep) {
+			return sweepPhaseStats;
+		}
+
+		// If sweep is enabled, the counts from the mark phase stats do not include nodes that have been
+		// deleted in previous runs. Add the deleted node counts to life time stats.
+		sweepPhaseStats.lifetimeNodeCount += sweepPhaseStats.deletedNodeCount;
+		sweepPhaseStats.lifetimeDataStoreCount += sweepPhaseStats.deletedDataStoreCount;
+		sweepPhaseStats.lifetimeAttachmentBlobCount += sweepPhaseStats.deletedAttachmentBlobCount;
+
+		// The node deleted in current run are counted twice - once in allDeletedNodes and again in
+		// markPhaseStats. So, remove them from the life time stats.
+		for (const nodeId of currentDeletedNodes) {
+			sweepPhaseStats.lifetimeNodeCount--;
+			const nodeType = this.runtime.getNodeType(nodeId);
+			if (nodeType === GCNodeType.DataStore) {
+				sweepPhaseStats.lifetimeDataStoreCount--;
+			} else if (nodeType === GCNodeType.Blob) {
+				sweepPhaseStats.lifetimeAttachmentBlobCount--;
+			}
+		}
+		return sweepPhaseStats;
 	}
 }

package/src/gc/gcDefinitions.ts

@@ -22,7 +22,7 @@ import {
 import { RuntimeHeaderData } from "../containerRuntime";
 
 /**
- * @public
+ * @internal
  */
 export type GCVersion = number;
 
@@ -62,8 +62,6 @@ export const runSweepKey = "Fluid.GarbageCollection.RunSweep";
 export const gcTestModeKey = "Fluid.GarbageCollection.GCTestMode";
 /** Config key to expire a session after a set period of time. Defaults to true. */
 export const runSessionExpiryKey = "Fluid.GarbageCollection.RunSessionExpiry";
-/** Config key to turn GC sweep log off. */
-export const disableSweepLogKey = "Fluid.GarbageCollection.DisableSweepLog";
 /** Config key to disable the tombstone feature, i.e., tombstone information is not read / written into summary. */
 export const disableTombstoneKey = "Fluid.GarbageCollection.DisableTombstone";
 /** Config key to override throwing an error when tombstone object is loaded (requested). */
@@ -94,7 +92,7 @@ export const defaultSessionExpiryDurationMs = 30 * oneDayMs; // 30 days
 
 /**
  * @see IGCMetadata.gcFeatureMatrix
- * @public
+ * @internal
  */
 export interface GCFeatureMatrix {
 	/**
@@ -112,7 +110,7 @@ export interface GCFeatureMatrix {
 }
 
 /**
- * @public
+ * @internal
  */
 export interface IGCMetadata {
 	/**
@@ -151,10 +149,10 @@ export interface IGCMetadata {
 }
 
 /**
- * The statistics of the system state after a garbage collection run.
- * @public
+ * The statistics of the system state after a garbage collection mark phase run.
+ * @internal
  */
-export interface IGCStats {
+export interface IMarkPhaseStats {
 	/** The number of nodes in the container. */
 	nodeCount: number;
 	/** The number of data stores in the container. */
@@ -175,9 +173,34 @@ export interface IGCStats {
 	updatedAttachmentBlobCount: number;
 }
 
+/**
+ * The statistics of the system state after a garbage collection sweep phase run.
+ * @internal
+ */
+export interface ISweepPhaseStats {
+	/** The number of nodes in the lifetime of the container. */
+	lifetimeNodeCount: number;
+	/** The number of data stores in the lifetime of the container. */
+	lifetimeDataStoreCount: number;
+	/** The number of attachment blobs in the lifetime of the container. */
+	lifetimeAttachmentBlobCount: number;
+	/** The number of deleted nodes in the container. */
+	deletedNodeCount: number;
+	/** The number of deleted data stores in the container. */
+	deletedDataStoreCount: number;
+	/** The number of deleted attachment blobs in the container. */
+	deletedAttachmentBlobCount: number;
+}
+
+/**
+ * The statistics of the system state after a garbage collection run.
+ * @internal
+ */
+export interface IGCStats extends IMarkPhaseStats, ISweepPhaseStats {}
+
 /**
  * The types of GC nodes in the GC reference graph.
- * @public
+ * @internal
  */
 export const GCNodeType = {
 	// Nodes that are for data stores.
@@ -191,7 +214,7 @@ export const GCNodeType = {
 };
 
 /**
- * @public
+ * @internal
  */
 export type GCNodeType = (typeof GCNodeType)[keyof typeof GCNodeType];
 
@@ -297,7 +320,7 @@ export interface IGarbageCollectorCreateParams {
 }
 
 /**
- * @public
+ * @internal
  */
 export interface IGCRuntimeOptions {
 	/**

package/src/gc/gcTelemetry.ts

@@ -14,7 +14,6 @@ import {
 import { RuntimeHeaderData } from "../containerRuntime";
 import { ICreateContainerMetadata } from "../summary";
 import {
-	disableSweepLogKey,
 	GCNodeType,
 	UnreferencedState,
 	IGarbageCollectorConfigs,
@@ -392,58 +391,6 @@ export class GCTelemetryTracker {
 		}
 		this.pendingEventsQueue = [];
 	}
-
-	/**
-	 * For nodes that are ready to sweep, log an event for now. Until we start running sweep which deletes objects,
-	 * this will give us a view into how much deleted content a container has.
-	 */
-	public logSweepEvents(
-		logger: ITelemetryLoggerExt,
-		currentReferenceTimestampMs: number,
-		unreferencedNodesState: Map<string, UnreferencedStateTracker>,
-		completedGCRuns: number,
-		lastSummaryTime?: number,
-	) {
-		if (
-			this.mc.config.getBoolean(disableSweepLogKey) === true ||
-			this.configs.sweepTimeoutMs === undefined
-		) {
-			return;
-		}
-
-		const deletedNodeIds: string[] = [];
-		for (const [nodeId, nodeStateTracker] of unreferencedNodesState) {
-			if (nodeStateTracker.state !== UnreferencedState.SweepReady) {
-				return;
-			}
-
-			const nodeType = this.getNodeType(nodeId);
-			if (nodeType !== GCNodeType.DataStore && nodeType !== GCNodeType.Blob) {
-				return;
-			}
-
-			// Log deleted event for each node only once to reduce noise in telemetry.
-			const uniqueEventId = `Deleted-${nodeId}`;
-			if (this.loggedUnreferencedEvents.has(uniqueEventId)) {
-				return;
-			}
-			this.loggedUnreferencedEvents.add(uniqueEventId);
-			deletedNodeIds.push(nodeId);
-		}
-
-		if (deletedNodeIds.length > 0) {
-			logger.sendTelemetryEvent({
-				eventName: "GC_SweepReadyObjects_Delete",
-				details: JSON.stringify({
-					timeout: this.configs.sweepTimeoutMs,
-					completedGCRuns,
-					lastSummaryTime,
-					...this.createContainerMetadata,
-				}),
-				...tagCodeArtifacts({ id: JSON.stringify(deletedNodeIds) }),
-			});
-		}
-	}
 }
 
 /**

package/src/gc/index.ts

@@ -8,7 +8,6 @@ export {
 	nextGCVersion,
 	defaultInactiveTimeoutMs,
 	defaultSessionExpiryDurationMs,
-	disableSweepLogKey,
 	GCNodeType,
 	gcTestModeKey,
 	gcTombstoneGenerationOptionName,
@@ -24,6 +23,8 @@ export {
 	IGCMetadata,
 	IGCResult,
 	IGCRuntimeOptions,
+	IMarkPhaseStats,
+	ISweepPhaseStats,
 	IGCStats,
 	oneDayMs,
 	runGCKey,

package/src/id-compressor/utilities.ts

@@ -35,7 +35,7 @@ export function createSessionId(): SessionId {
 
 /**
  * Asserts that the given string is a stable ID.
- * @public
+ * @internal
  */
 export function assertIsStableId(stableId: string): StableId {
 	assert(isStableId(stableId), 0x4a3 /* Expected a StableId */);
@@ -52,7 +52,7 @@ export function assertIsSessionId(stableId: string): SessionId {
 
 /**
  * Generate a random stable ID
- * @public
+ * @internal
  */
 export function generateStableId(): StableId {
 	return assertIsStableId(v4());
@@ -61,7 +61,7 @@ export function generateStableId(): StableId {
 /**
  * Returns true iff the given string is a valid Version 4, variant 2 UUID
  * 'xxxxxxxx-xxxx-4xxx-vxxx-xxxxxxxxxxxx'
- * @public
+ * @internal
  */
 export function isStableId(str: string): str is StableId {
 	if (str.length !== 36) {

package/src/index.ts

@@ -40,6 +40,8 @@ export {
 	GCFeatureMatrix,
 	GCVersion,
 	IGCRuntimeOptions,
+	IMarkPhaseStats,
+	ISweepPhaseStats,
 	IGCStats,
 } from "./gc";
 export {

package/src/messageTypes.ts

@@ -14,7 +14,7 @@ import { IDataStoreAliasMessage } from "./dataStore";
 import { IChunkedOp } from "./opLifecycle";
 
 /**
- * @public
+ * @internal
  */
 export enum ContainerMessageType {
 	// An op to be delivered to store
@@ -71,7 +71,8 @@ export interface IContainerRuntimeMessageCompatDetails {
  * IMPORTANT: when creating one to be serialized, set the properties in the order they appear here.
  * This way stringified values can be compared.
  */
-interface TypedContainerRuntimeMessage<TType extends ContainerMessageType, TContents> {
+interface TypedContainerRuntimeMessage<TType extends ContainerMessageType, TContents>
+	extends Partial<RecentlyAddedContainerRuntimeMessageDetails> {
 	/** Type of the op, within the ContainerRuntime's domain */
 	type: TType;
 	/** Domain-specific contents, interpreted according to the type */

package/src/opLifecycle/definitions.ts

@@ -53,7 +53,7 @@ export interface IBatchCheckpoint {
 }
 
 /**
- * @public
+ * @internal
  */
 export interface IChunkedOp {
 	chunkId: number;

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.0.0-dev.7.4.0.215747";
+export const pkgVersion = "2.0.0-dev.7.4.0.216897";

package/src/summary/orderedClientElection.ts

@@ -227,7 +227,7 @@ export interface IOrderedClientElectionEvents extends IEvent {
 
 /**
  * Serialized state of IOrderedClientElection.
- * @public
+ * @internal
  */
 export interface ISerializedElection {
 	/** Sequence number at the time of the latest election. */

package/src/summary/runWhileConnectedCoordinator.ts

@@ -12,7 +12,7 @@ import {
 
 /**
  * Similar to AbortController, but using promise instead of events
- * @public
+ * @internal
  */
 export interface ICancellableSummarizerController extends ISummaryCancellationToken {
 	stop(reason: SummarizerStopReason): void;
@@ -21,7 +21,7 @@ export interface ICancellableSummarizerController extends ISummaryCancellationTo
 /**
  * Can be useful in testing as well as in places where caller does not use cancellation.
  * This object implements ISummaryCancellationToken interface but cancellation is never leveraged.
- * @public
+ * @internal
  */
 export const neverCancelledSummaryToken: ISummaryCancellationToken = {
 	cancelled: false,

package/src/summary/summarizer.ts

@@ -69,7 +69,7 @@ export const createSummarizingWarning = (errorMessage: string, logged: boolean)
  * Summarizer is responsible for coordinating when to generate and send summaries.
  * It is the main entry point for summary work.
  * It is created only by summarizing container (i.e. one with clientType === "summarizer")
- * @public
+ * @internal
  */
 export class Summarizer extends TypedEventEmitter<ISummarizerEvents> implements ISummarizer {
 	public get ISummarizer() {