@fluidframework/container-runtime 2.0.0-rc.3.0.0 → 2.0.0-rc.3.0.10

package/src/gc/gcTelemetry.ts

@@ -14,6 +14,7 @@ import {
 	tagCodeArtifacts,
 } from "@fluidframework/telemetry-utils/internal";
 
+import type { Tagged } from "@fluidframework/core-interfaces";
 import { RuntimeHeaderData } from "../containerRuntime.js";
 import { ICreateContainerMetadata } from "../summary/index.js";
 
@@ -43,11 +44,12 @@ interface ICommonProps {
 
 /** The event that is logged when unreferenced node is used after a certain time. */
 interface IUnreferencedEventProps extends ICreateContainerMetadata, ICommonProps {
+	/** The id that GC uses to track the node. May or may not match id */
+	trackedId: string;
 	state: UnreferencedState;
-	id: {
-		value: string;
-		tag: string;
-	};
+	/** The full path (in GC Path format) to the node in question */
+	id: Tagged<string>;
+	fromId?: Tagged<string>;
 	type: GCNodeType;
 	unrefTime: number;
 	age: number;
@@ -56,19 +58,24 @@ interface IUnreferencedEventProps extends ICreateContainerMetadata, ICommonProps
 		[K in keyof GCFeatureMatrix]: GCFeatureMatrix[K];
 	};
 	timeout?: number;
-	fromId?: {
-		value: string;
-		tag: string;
-	};
 }
 
 /** Properties passed to nodeUsed function when a node is used. */
 interface INodeUsageProps extends ICommonProps {
+	/** The full path (in GC Path format) to the node in question */
 	id: string;
+	/** Latest timestamp received from the server, as a baseline for computing GC state/age */
 	currentReferenceTimestampMs: number | undefined;
+	/** The package path of the node. This may not be available if the node hasn't been loaded yet */
 	packagePath: readonly string[] | undefined;
+	/** In case of Revived - what node added the reference? */
 	fromId?: string;
+	/** In case of Revived - was it revived due to autorecovery? */
 	autorecovery?: true;
+	/** URL (including query string) if this usage came from a request */
+	requestUrl?: string;
+	/** Original request headers if this usage came from a request or handle.get */
+	requestHeaders?: string;
 }
 
 /**
@@ -143,18 +150,34 @@ export class GCTelemetryTracker {
 	}
 
 	/**
-	 * Called when a node is used. If the node is not active or tombstoned, log telemetry indicating object is used
+	 * Called when a node is used. If the node is inactive or tombstoned, log telemetry indicating object is used
 	 * when it should not have been.
+	 * @param trackedId - The id that GC uses to track the node. For SubDataStore nodes, this should be the DataStore ID.
+	 * @param INodeUsageProps - All kind of details about this event to be logged
 	 */
-	public nodeUsed(nodeUsageProps: INodeUsageProps) {
+	public nodeUsed(
+		trackedId: string,
+		{
+			usageType,
+			currentReferenceTimestampMs,
+			packagePath,
+			id: untaggedId,
+			fromId: untaggedFromId,
+			isTombstoned,
+			...otherNodeUsageProps
+		}: INodeUsageProps,
+	) {
 		// If there is no reference timestamp to work with, no ops have been processed after creation. If so, skip
 		// logging as nothing interesting would have happened worth logging.
-		if (nodeUsageProps.currentReferenceTimestampMs === undefined) {
+		if (currentReferenceTimestampMs === undefined) {
 			return;
 		}
 
-		const nodeStateTracker = this.getNodeStateTracker(nodeUsageProps.id);
-		const nodeType = this.getNodeType(nodeUsageProps.id);
+		// Note: For SubDataStore Load usage, trackedId will be the DataStore's id, not the full path in question.
+		// This is necessary because the SubDataStore path may be unrecognized by GC (if suited for a custom request handler)
+		const nodeStateTracker = this.getNodeStateTracker(trackedId);
+		const nodeType = this.getNodeType(untaggedId);
+
 		const timeout = (() => {
 			switch (nodeStateTracker?.state) {
 				case UnreferencedState.Inactive:
@@ -170,33 +193,27 @@ export class GCTelemetryTracker {
 					return undefined;
 			}
 		})();
-		const {
-			usageType,
-			currentReferenceTimestampMs,
-			packagePath,
-			id: untaggedId,
-			fromId: untaggedFromId,
-			...propsToLog
-		} = nodeUsageProps;
 		const { persistedGcFeatureMatrix, ...configs } = this.configs;
-		const unrefEventProps: Omit<IUnreferencedEventProps, "state" | "usageType"> = {
+		const unrefEventProps = {
+			trackedId,
 			type: nodeType,
 			unrefTime: nodeStateTracker?.unreferencedTimestampMs ?? -1,
 			age:
 				nodeStateTracker !== undefined
-					? nodeUsageProps.currentReferenceTimestampMs -
-					  nodeStateTracker.unreferencedTimestampMs
+					? currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs
 					: -1,
 			timeout,
+			isTombstoned,
 			...tagCodeArtifacts({ id: untaggedId, fromId: untaggedFromId }),
-			...propsToLog,
+			...otherNodeUsageProps,
 			...this.createContainerMetadata,
 			gcConfigs: { ...configs, ...persistedGcFeatureMatrix },
-		};
+		} satisfies Omit<IUnreferencedEventProps, "state" | "usageType"> &
+			typeof otherNodeUsageProps;
 
 		// If the node that is used is tombstoned, log a tombstone telemetry.
-		if (nodeUsageProps.isTombstoned) {
-			this.logTombstoneUsageTelemetry(nodeUsageProps, unrefEventProps, nodeType, usageType);
+		if (isTombstoned) {
+			this.logTombstoneUsageTelemetry(unrefEventProps, nodeType, usageType, packagePath);
 		}
 
 		// After logging tombstone telemetry, if the node's unreferenced state is not tracked, there is nothing
@@ -206,16 +223,9 @@ export class GCTelemetryTracker {
 		}
 
 		const state = nodeStateTracker.state;
-		const uniqueEventId = `${state}-${nodeUsageProps.id}-${nodeUsageProps.usageType}`;
+		const uniqueEventId = `${state}-${untaggedId}-${usageType}`;
 
-		if (
-			!this.shouldLogNonActiveEvent(
-				nodeType,
-				nodeUsageProps.usageType,
-				nodeStateTracker,
-				uniqueEventId,
-			)
-		) {
+		if (!this.shouldLogNonActiveEvent(nodeType, usageType, nodeStateTracker, uniqueEventId)) {
 			return;
 		}
 
@@ -229,8 +239,8 @@ export class GCTelemetryTracker {
 		// SweepReady errors are usages of Objects that will be deleted by GC Sweep!
 		if (this.isSummarizerClient) {
 			this.pendingEventsQueue.push({
-				...unrefEventProps,
-				usageType: nodeUsageProps.usageType,
+				...unrefEventProps, // Note: Contains some properties from INodeUsageProps as well
+				usageType,
 				state,
 			});
 		} else {
@@ -238,11 +248,11 @@ export class GCTelemetryTracker {
 			// summarizer clients if they are based off of user actions (such as scrolling to content for these objects)
 			// Events generated:
 			// InactiveObject_Loaded, SweepReadyObject_Loaded
-			if (nodeUsageProps.usageType === "Loaded") {
+			if (usageType === "Loaded") {
 				const { id, fromId, headers, gcConfigs, ...detailedProps } = unrefEventProps;
 				const event = {
-					eventName: `${state}Object_${nodeUsageProps.usageType}`,
-					...tagCodeArtifacts({ pkg: nodeUsageProps.packagePath?.join("/") }),
+					eventName: `${state}Object_${usageType}`,
+					...tagCodeArtifacts({ pkg: packagePath?.join("/") }),
 					stack: generateStack(),
 					id,
 					fromId,
@@ -262,10 +272,10 @@ export class GCTelemetryTracker {
 	 * Logs telemetry when a tombstoned object is changed, revived or loaded.
 	 */
 	private logTombstoneUsageTelemetry(
-		nodeUsageProps: INodeUsageProps,
 		unrefEventProps: Omit<IUnreferencedEventProps, "state" | "usageType">,
 		nodeType: GCNodeType,
 		usageType: NodeUsageType,
+		packagePath?: readonly string[],
 	) {
 		// This will log the following events:
 		// GC_Tombstone_DataStore_Requested, GC_Tombstone_DataStore_Changed, GC_Tombstone_DataStore_Revived
@@ -275,12 +285,12 @@ export class GCTelemetryTracker {
 		const eventUsageName = usageType === "Loaded" ? "Requested" : usageType;
 		const event = {
 			eventName: `GC_Tombstone_${nodeType}_${eventUsageName}`,
-			pkg: tagCodeArtifacts({ pkg: nodeUsageProps.packagePath?.join("/") }).pkg,
+			...tagCodeArtifacts({ pkg: packagePath?.join("/") }),
 			stack: generateStack(),
 			id,
 			fromId,
 			headers: { ...headers },
-			details: detailedProps,
+			details: detailedProps, // Also includes some properties from INodeUsageProps type
 			gcConfigs,
 			tombstoneFlags: {
 				DisableTombstone: this.mc.config.getBoolean(disableTombstoneKey),
@@ -372,7 +382,6 @@ export class GCTelemetryTracker {
 		// InactiveObject_Loaded, InactiveObject_Changed, InactiveObject_Revived
 		// SweepReadyObject_Loaded, SweepReadyObject_Changed, SweepReadyObject_Revived
 		for (const eventProps of this.pendingEventsQueue) {
-			// const { usageType, state, id, fromId, ...propsToLog } = eventProps;
 			const { usageType, state, id, fromId, headers, gcConfigs, ...detailedProps } =
 				eventProps;
 			/**
@@ -381,7 +390,7 @@ export class GCTelemetryTracker {
 			 * Loaded and Changed events are logged only if the node is not active. If the node is active, it was
 			 * revived and a Revived event will be logged for it.
 			 */
-			const nodeStateTracker = this.getNodeStateTracker(eventProps.id.value);
+			const nodeStateTracker = this.getNodeStateTracker(detailedProps.trackedId); // Note: This is never SubDataStore path
 			const active =
 				nodeStateTracker === undefined ||
 				nodeStateTracker.state === UnreferencedState.Active;

package/src/gc/index.ts

@@ -23,6 +23,7 @@ export {
 	IGarbageCollectorCreateParams,
 	IGCMetadata,
 	IGCMetadata_Deprecated,
+	IGCNodeUpdatedProps,
 	IGCResult,
 	IGCRuntimeOptions,
 	IMarkPhaseStats,
@@ -46,8 +47,8 @@ export {
 	cloneGCData,
 	concatGarbageCollectionStates,
 	getGCVersionInEffect,
-	trimLeadingAndTrailingSlashes,
 	unpackChildNodesGCDetails,
+	urlToGCNodePath,
 } from "./gcHelpers.js";
 export { runGarbageCollection } from "./gcReferenceGraphAlgorithm.js";
 export {

package/src/index.ts

@@ -13,6 +13,7 @@ export {
 	isRuntimeMessage,
 	agentSchedulerId,
 	ContainerRuntime,
+	DeletedResponseHeaderKey,
 	TombstoneResponseHeaderKey,
 	InactiveResponseHeaderKey,
 	ISummaryConfiguration,
@@ -47,6 +48,7 @@ export {
 	IGCRuntimeOptions,
 	IMarkPhaseStats,
 	ISweepPhaseStats,
+	IGCNodeUpdatedProps,
 	IGCStats,
 } from "./gc/index.js";
 export {

package/src/metadata.ts

@@ -19,8 +19,8 @@ export interface IBlobMetadata {
 }
 
 /**
- * The IdCompressor needs to know if this is a replayed savedOp as those need to be skipped in stashed ops scenarios.
+ * ContainerRuntime needs to know if this is a replayed savedOp as those need to be skipped in stashed ops scenarios.
  */
-export interface IIdAllocationMetadata {
+export interface ISavedOpMetadata {
 	savedOp?: boolean;
 }

package/src/opLifecycle/README.md

@@ -339,19 +339,19 @@ stateDiagram-v2
 	state "Store original (uncompressed, unchunked, ungrouped) batch locally" as store
     state if_compression <<choice>>
 	[*] --> ContainerRuntime.submit
-	ContainerRuntime.submit --> outbox.submitAttach
+	ContainerRuntime.submit --> outbox.submitIdAllocation
 	ContainerRuntime.submit --> outbox.submitBlobAttach
 	ContainerRuntime.submit --> outbox.submit
 	outbox.submit --> scheduleFlush
-	outbox.submitAttach --> scheduleFlush
+	outbox.submitIdAllocation --> scheduleFlush
 	outbox.submitBlobAttach --> scheduleFlush
 	scheduleFlush --> jsTurn
 	jsTurn --> flush
 	flush --> outbox.flushInternalMain
-	flush --> outbox.flushInternalAttach
+	flush --> outbox.flushInternalIdAllocation
 	flush --> outbox.flushInternalBlobAttach
 	outbox.flushInternalMain --> flushInternal
-	outbox.flushInternalAttach --> flushInternal
+	outbox.flushInternalIdAllocation --> flushInternal
 	outbox.flushInternalBlobAttach --> flushInternal
 	flushInternal --> ContainerRuntime.reSubmit: if batch has reentrant ops and should group
 	ContainerRuntime.reSubmit --> flushInternal

package/src/opLifecycle/batchManager.ts

@@ -9,8 +9,12 @@ import { BatchMessage, IBatch, IBatchCheckpoint } from "./definitions.js";
 
 export interface IBatchManagerOptions {
 	readonly hardLimit: number;
-	readonly softLimit?: number;
 	readonly compressionOptions?: ICompressionRuntimeOptions;
+
+	/**
+	 * If true, the outbox is allowed to rebase the batch during flushing.
+	 */
+	readonly canRebase: boolean;
 }
 
 export interface BatchSequenceNumbers {
@@ -72,19 +76,6 @@ export class BatchManager {
 		// initially stored as base64, and that requires only 2 extra escape characters.
 		const socketMessageSize = contentSize + opOverhead * opCount;
 
-		// If we were provided soft limit, check for exceeding it.
-		// But only if we have any ops, as the intention here is to flush existing ops (on exceeding this limit)
-		// and start over. That's not an option if we have no ops.
-		// If compression is enabled, the soft and hard limit are ignored and the message will be pushed anyways.
-		// Cases where the message is still too large will be handled by the maxConsecutiveReconnects path.
-		if (
-			this.options.softLimit !== undefined &&
-			this.length > 0 &&
-			socketMessageSize >= this.options.softLimit
-		) {
-			return false;
-		}
-
 		if (socketMessageSize >= this.options.hardLimit) {
 			return false;
 		}

package/src/opLifecycle/outbox.ts

@@ -89,11 +89,9 @@ export function getLongStack<T>(action: () => T, length: number = 50): T {
 
 export class Outbox {
 	private readonly mc: MonitoringContext;
-	private readonly attachFlowBatch: BatchManager;
 	private readonly mainBatch: BatchManager;
 	private readonly blobAttachBatch: BatchManager;
 	private readonly idAllocationBatch: BatchManager;
-	private readonly defaultAttachFlowSoftLimitInBytes = 320 * 1024;
 	private batchRebasesToReport = 5;
 	private rebasing = false;
 
@@ -113,21 +111,14 @@ export class Outbox {
 			Number.POSITIVE_INFINITY;
 		// We need to allow infinite size batches if we enable compression
 		const hardLimit = isCompressionEnabled ? Infinity : this.params.config.maxBatchSizeInBytes;
-		const softLimit = isCompressionEnabled ? Infinity : this.defaultAttachFlowSoftLimitInBytes;
 
-		this.attachFlowBatch = new BatchManager({ hardLimit, softLimit });
-		this.mainBatch = new BatchManager({ hardLimit });
-		this.blobAttachBatch = new BatchManager({ hardLimit });
-		this.idAllocationBatch = new BatchManager({ hardLimit });
+		this.mainBatch = new BatchManager({ hardLimit, canRebase: true });
+		this.blobAttachBatch = new BatchManager({ hardLimit, canRebase: true });
+		this.idAllocationBatch = new BatchManager({ hardLimit, canRebase: false });
 	}
 
 	public get messageCount(): number {
-		return (
-			this.attachFlowBatch.length +
-			this.mainBatch.length +
-			this.blobAttachBatch.length +
-			this.idAllocationBatch.length
-		);
+		return this.mainBatch.length + this.blobAttachBatch.length + this.idAllocationBatch.length;
 	}
 
 	public get isEmpty(): boolean {
@@ -142,13 +133,11 @@ export class Outbox {
 	 */
 	private maybeFlushPartialBatch() {
 		const mainBatchSeqNums = this.mainBatch.sequenceNumbers;
-		const attachFlowBatchSeqNums = this.attachFlowBatch.sequenceNumbers;
 		const blobAttachSeqNums = this.blobAttachBatch.sequenceNumbers;
 		const idAllocSeqNums = this.idAllocationBatch.sequenceNumbers;
 		assert(
 			this.params.config.disablePartialFlush ||
-				(sequenceNumbersMatch(mainBatchSeqNums, attachFlowBatchSeqNums) &&
-					sequenceNumbersMatch(mainBatchSeqNums, blobAttachSeqNums) &&
+				(sequenceNumbersMatch(mainBatchSeqNums, blobAttachSeqNums) &&
 					sequenceNumbersMatch(mainBatchSeqNums, idAllocSeqNums)),
 			0x58d /* Reference sequence numbers from both batches must be in sync */,
 		);
@@ -157,7 +146,6 @@ export class Outbox {
 
 		if (
 			sequenceNumbersMatch(mainBatchSeqNums, currentSequenceNumbers) &&
-			sequenceNumbersMatch(attachFlowBatchSeqNums, currentSequenceNumbers) &&
 			sequenceNumbersMatch(blobAttachSeqNums, currentSequenceNumbers) &&
 			sequenceNumbersMatch(idAllocSeqNums, currentSequenceNumbers)
 		) {
@@ -172,8 +160,6 @@ export class Outbox {
 					eventName: "ReferenceSequenceNumberMismatch",
 					mainReferenceSequenceNumber: mainBatchSeqNums.referenceSequenceNumber,
 					mainClientSequenceNumber: mainBatchSeqNums.clientSequenceNumber,
-					attachReferenceSequenceNumber: attachFlowBatchSeqNums.referenceSequenceNumber,
-					attachClientSequenceNumber: attachFlowBatchSeqNums.clientSequenceNumber,
 					blobAttachReferenceSequenceNumber: blobAttachSeqNums.referenceSequenceNumber,
 					blobAttachClientSequenceNumber: blobAttachSeqNums.clientSequenceNumber,
 					currentReferenceSequenceNumber: currentSequenceNumbers.referenceSequenceNumber,
@@ -194,37 +180,6 @@ export class Outbox {
 		this.addMessageToBatchManager(this.mainBatch, message);
 	}
 
-	public submitAttach(message: BatchMessage) {
-		this.maybeFlushPartialBatch();
-
-		if (
-			!this.attachFlowBatch.push(
-				message,
-				this.isContextReentrant(),
-				this.params.getCurrentSequenceNumbers().clientSequenceNumber,
-			)
-		) {
-			// BatchManager has two limits - soft limit & hard limit. Soft limit is only engaged
-			// when queue is not empty.
-			// Flush queue & retry. Failure on retry would mean - single message is bigger than hard limit
-			this.flushInternal(this.attachFlowBatch);
-
-			this.addMessageToBatchManager(this.attachFlowBatch, message);
-		}
-
-		// If compression is enabled, we will always successfully receive
-		// attach ops and compress then send them at the next JS turn, regardless
-		// of the overall size of the accumulated ops in the batch.
-		// However, it is more efficient to flush these ops faster, preferably
-		// after they reach a size which would benefit from compression.
-		if (
-			this.attachFlowBatch.contentSizeInBytes >=
-			this.params.config.compressionOptions.minimumBatchSizeInBytes
-		) {
-			this.flushInternal(this.attachFlowBatch);
-		}
-	}
-
 	public submitBlobAttach(message: BatchMessage) {
 		this.maybeFlushPartialBatch();
 
@@ -303,7 +258,6 @@ export class Outbox {
 
 	private flushAll() {
 		this.flushInternal(this.idAllocationBatch);
-		this.flushInternal(this.attachFlowBatch);
 		this.flushInternal(this.blobAttachBatch, true /* disableGroupedBatching */);
 		this.flushInternal(this.mainBatch);
 	}
@@ -316,7 +270,7 @@ export class Outbox {
 		const rawBatch = batchManager.popBatch();
 		const shouldGroup =
 			!disableGroupedBatching && this.params.groupingManager.shouldGroup(rawBatch);
-		if (rawBatch.hasReentrantOps === true && shouldGroup) {
+		if (batchManager.options.canRebase && rawBatch.hasReentrantOps === true && shouldGroup) {
 			assert(!this.rebasing, 0x6fa /* A rebased batch should never have reentrant ops */);
 			// If a batch contains reentrant ops (ops created as a result from processing another op)
 			// it needs to be rebased so that we can ensure consistent reference sequence numbers
@@ -346,6 +300,7 @@ export class Outbox {
 	 */
 	private rebase(rawBatch: IBatch, batchManager: BatchManager) {
 		assert(!this.rebasing, 0x6fb /* Reentrancy */);
+		assert(batchManager.options.canRebase, "BatchManager does not support rebase");
 
 		this.rebasing = true;
 		for (const message of rawBatch.content) {
@@ -479,7 +434,6 @@ export class Outbox {
 		const mainBatch: IBatchCheckpoint = this.mainBatch.checkpoint();
 		return {
 			mainBatch,
-			attachFlowBatch: this.attachFlowBatch.checkpoint(),
 			blobAttachBatch: this.blobAttachBatch.checkpoint(),
 		};
 	}

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.0.0-rc.3.0.0";
+export const pkgVersion = "2.0.0-rc.3.0.10";

package/src/summary/documentSchema.ts

@@ -552,6 +552,7 @@ export class DocumentsSchemaController {
 	 */
 	public maybeSendSchemaMessage(): IDocumentSchemaChangeMessage | undefined {
 		if (this.sendOp && this.futureSchema !== undefined) {
+			this.sendOp = false;
 			assert(
 				this.explicitSchemaControl &&
 					this.futureSchema.runtime.explicitSchemaControl === true,
@@ -562,7 +563,6 @@ export class DocumentsSchemaController {
 				refSeq: this.documentSchema.refSeq,
 			};
 		}
-		this.sendOp = false;
 	}
 
 	/**

package/dist/public.d.ts

@@ -1,12 +0,0 @@
-/*!
- * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
- * Licensed under the MIT License.
- */
-
-/*
- * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
- * Generated by "flub generate entrypoints" in @fluidframework/build-tools.
- */
-
-export {}
-

package/lib/public.d.ts

@@ -1,12 +0,0 @@
-/*!
- * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
- * Licensed under the MIT License.
- */
-
-/*
- * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
- * Generated by "flub generate entrypoints" in @fluidframework/build-tools.
- */
-
-export {}
-