@fluidframework/container-runtime 2.101.0 → 2.102.0

package/src/opLifecycle/duplicateBatchDetector.ts

@@ -9,6 +9,41 @@ import type { ITelemetryContext } from "@fluidframework/runtime-definitions/inte
 import { getEffectiveBatchId } from "./batchManager.js";
 import type { BatchStartInfo } from "./remoteMessageProcessor.js";
 
+/**
+ * Identifying info for a previously-recorded batch that we can include in DuplicateBatch telemetry
+ * to help diagnose where the duplicate came from.
+ *
+ * @remarks `batchIdExplicit` distinguishes the two main duplicate-source scenarios:
+ * - `true`: the batchId was stamped on the wire as explicit metadata, indicating a resubmit (PendingStateManager.replayPendingStates).
+ * - `false`: the batchId was derived from the wire `clientId` and `batchStartCsn`, indicating a fresh, non-resubmit batch.
+ */
+export interface RecordedBatchInfo {
+	/**
+	 * Wire clientId on the message that started the batch (NOT necessarily the `originalClientId`
+	 * encoded in the batchId for resubmits).
+	 */
+	readonly clientId: string;
+	/**
+	 * Wire client sequence number at the start of the batch.
+	 */
+	readonly batchStartCsn: number;
+	/**
+	 * True if the batchId came from explicit metadata on the wire (i.e. a resubmit),
+	 * false if it was derived from clientId + batchStartCsn (i.e. a fresh submit).
+	 */
+	readonly batchIdExplicit: boolean;
+}
+
+interface RecordedBatch {
+	readonly batchId: string;
+	/**
+	 * Identifying info for the batch as observed at runtime.
+	 * `undefined` if the batch was loaded from a summary snapshot (where only the
+	 * `[seqNum, batchId]` pair is persisted).
+	 */
+	readonly info: RecordedBatchInfo | undefined;
+}
+
 /**
  * Detects duplicate batches that can arise from the "parallel fork" scenario:
  * Container 1 is serialized, and Containers 2 and 3 are rehydrated from that state.
@@ -24,9 +59,11 @@ export class DuplicateBatchDetector {
 	private readonly seqNumByBatchId = new Map<string, number>();
 
 	/**
-	 * We map from sequenceNumber to batchId to find which ones we can stop tracking as MSN advances
+	 * Map from sequenceNumber to the recorded batch info. Used to clear out old entries as MSN
+	 * advances, and to report identifying info about the original occurrence when a duplicate
+	 * is detected.
 	 */
-	private readonly batchIdsBySeqNum = new Map<number, string>();
+	private readonly batchesBySeqNum = new Map<number, RecordedBatch>();
 
 	/**
 	 * Number of inbound batches processed since the last summary. Reset by getRecentBatchInfoForSummary.
@@ -44,22 +81,29 @@ export class DuplicateBatchDetector {
 	constructor(batchIdsFromSnapshot: [number, string][] | undefined) {
 		if (batchIdsFromSnapshot) {
 			for (const [seqNum, batchId] of batchIdsFromSnapshot) {
-				this.batchIdsBySeqNum.set(seqNum, batchId);
+				// Entries loaded from a snapshot don't carry the original clientId/csn/explicit-bit;
+				// we record them with `info: undefined` so duplicate telemetry can indicate that.
+				this.batchesBySeqNum.set(seqNum, { batchId, info: undefined });
 				this.seqNumByBatchId.set(batchId, seqNum);
 			}
-			this.peakTrackedBatchCount = this.batchIdsBySeqNum.size;
+			this.peakTrackedBatchCount = this.batchesBySeqNum.size;
 		}
 	}
 
 	/**
 	 * Records this batch's batchId, and checks if it's a duplicate of a batch we've already seen.
-	 * If it's a duplicate, also return the sequence number of the other batch for logging.
+	 * If it's a duplicate, also return the sequence number of the other batch (and identifying info,
+	 * if the other batch was seen during this container session rather than loaded from snapshot) for logging.
 	 *
 	 * @remarks We also use the minimumSequenceNumber to clear out old batchIds that are no longer at risk for duplicates.
 	 */
-	public processInboundBatch(
-		batchStart: BatchStartInfo,
-	): { duplicate: true; otherSequenceNumber: number } | { duplicate: false } {
+	public processInboundBatch(batchStart: BatchStartInfo):
+		| {
+				duplicate: true;
+				otherSequenceNumber: number;
+				otherBatchInfo: RecordedBatchInfo | undefined;
+		  }
+		| { duplicate: false } {
 		const { sequenceNumber, minimumSequenceNumber } = batchStart.keyMessage;
 		this.processedBatchCount++;
 
@@ -76,24 +120,36 @@ export class DuplicateBatchDetector {
 		// O(1) duplicate check + get otherSequenceNumber in one lookup
 		const otherSequenceNumber = this.seqNumByBatchId.get(batchId);
 		if (otherSequenceNumber !== undefined) {
+			const other = this.batchesBySeqNum.get(otherSequenceNumber);
 			assert(
-				this.batchIdsBySeqNum.get(otherSequenceNumber) === batchId,
+				other?.batchId === batchId,
 				0xce0 /* batchIdToSeqNum and seqNumToBatchId should be in sync for duplicate */,
 			);
-			return { duplicate: true, otherSequenceNumber };
+			return {
+				duplicate: true,
+				otherSequenceNumber,
+				otherBatchInfo: other.info,
+			};
 		}
 
 		// Now we know it's not a duplicate, so add it to the tracked batchIds and return.
 		assert(
-			!this.batchIdsBySeqNum.has(sequenceNumber),
+			!this.batchesBySeqNum.has(sequenceNumber),
 			0xce1 /* seqNumToBatchId and batchIdToSeqNum should be in sync */,
 		);
 
-		// Add new batch
-		this.batchIdsBySeqNum.set(sequenceNumber, batchId);
+		// Add new batch. Record identifying info so we can report it if a future duplicate matches us.
+		const info: RecordedBatchInfo | undefined = {
+			clientId: batchStart.clientId,
+			batchStartCsn: batchStart.batchStartCsn,
+			// True iff the wire carried explicit batchId metadata (resubmit path).
+			// False indicates the batchId was derived from clientId + batchStartCsn (fresh submit).
+			batchIdExplicit: batchStart.batchId !== undefined,
+		};
+		this.batchesBySeqNum.set(sequenceNumber, { batchId, info });
 		this.seqNumByBatchId.set(batchId, sequenceNumber);
-		if (this.batchIdsBySeqNum.size > this.peakTrackedBatchCount) {
-			this.peakTrackedBatchCount = this.batchIdsBySeqNum.size;
+		if (this.batchesBySeqNum.size > this.peakTrackedBatchCount) {
+			this.peakTrackedBatchCount = this.batchesBySeqNum.size;
 		}
 
 		return { duplicate: false };
@@ -104,10 +160,10 @@ export class DuplicateBatchDetector {
 	 * since the batch start has been processed by all clients, and local batches are deduped and the forked client would close.
 	 */
 	private clearOldBatchIds(msn: number): void {
-		for (const [sequenceNumber, batchId] of this.batchIdsBySeqNum) {
+		for (const [sequenceNumber, recorded] of this.batchesBySeqNum) {
 			if (sequenceNumber < msn) {
-				this.batchIdsBySeqNum.delete(sequenceNumber);
-				this.seqNumByBatchId.delete(batchId);
+				this.batchesBySeqNum.delete(sequenceNumber);
+				this.seqNumByBatchId.delete(recorded.batchId);
 			} else {
 				break;
 			}
@@ -125,7 +181,7 @@ export class DuplicateBatchDetector {
 	): [number, string][] | undefined {
 		if (telemetryContext !== undefined) {
 			const prefix = "fluid_DuplicateBatchDetector_";
-			telemetryContext.set(prefix, "recentBatchCount", this.batchIdsBySeqNum.size);
+			telemetryContext.set(prefix, "recentBatchCount", this.batchesBySeqNum.size);
 			telemetryContext.set(prefix, "peakRecentBatchCount", this.peakTrackedBatchCount);
 			telemetryContext.set(prefix, "processedBatchCount", this.processedBatchCount);
 		}
@@ -133,12 +189,15 @@ export class DuplicateBatchDetector {
 		// Reset per-window perf counters so each summary covers only the activity since the
 		// previous one. Peak resets to the current size (the floor for the next window).
 		this.processedBatchCount = 0;
-		this.peakTrackedBatchCount = this.batchIdsBySeqNum.size;
+		this.peakTrackedBatchCount = this.batchesBySeqNum.size;
 
-		if (this.batchIdsBySeqNum.size === 0) {
+		if (this.batchesBySeqNum.size === 0) {
 			return undefined;
 		}
 
-		return [...this.batchIdsBySeqNum.entries()];
+		return [...this.batchesBySeqNum.entries()].map(([seqNum, recorded]) => [
+			seqNum,
+			recorded.batchId,
+		]);
 	}
 }

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.101.0";
+export const pkgVersion = "2.102.0";

package/src/summary/documentSchema.ts

@@ -303,6 +303,58 @@ const documentSchemaSupportedConfigs = {
 	disallowedVersions: new CheckVersions(),
 };
 
+/**
+ * Entry in {@link retiredDocumentSchemaFeatures}.
+ *
+ * - `handler` validates/merges the persisted value (same as a "non-retired" feature).
+ * - `value` is the hardcoded value that this runtime will always use for the desired schema.
+ */
+interface IRetiredFeatureEntry {
+	handler: IProperty;
+	value: DocumentSchemaValueType;
+}
+
+/**
+ * Retired runtime features. A retired feature is one this runtime version no longer toggles
+ * via {@link IDocumentSchemaFeatures}, but that older documents may still carry in their
+ * persisted schema. Each entry bundles the property handler with the hardcoded value for the feature.
+ *
+ * Retired features participate in the normal merge / schema-change-op flow exactly like
+ * non-retired features — the only difference is that their values are hardcoded.
+ */
+const retiredDocumentSchemaFeatures = {
+	// Note: There are currently no retired retired features. To retire a feature, remove it from IDocumentSchemaFeatures
+	// and documentSchemaSupportedConfigs and add an entry here, e.g.:
+	//   featureFoo: { handler: new TrueOrUndefined(), value: true },
+} satisfies Record<string, IRetiredFeatureEntry> & {
+	// This ensures that retiredDocumentSchemaFeatures and IDocumentSchemaFeatures are mutually exclusive.
+	[K in keyof IDocumentSchemaFeatures]?: never;
+};
+
+/**
+ * Looks up the validator/merger for a given runtime property name across both supported and
+ * retired configs. Returns `undefined` if the property is unknown to this runtime.
+ */
+function getRuntimeConfigHandler(name: string): IProperty | undefined {
+	return (
+		(documentSchemaSupportedConfigs as Record<string, IProperty>)[name] ??
+		(retiredDocumentSchemaFeatures as Record<string, IRetiredFeatureEntry>)[name]?.handler
+	);
+}
+
+/**
+ * Builds the `{ key: value }` for retired features (for building the desired schema).
+ */
+function retiredFeatureValues(): Record<string, DocumentSchemaValueType> {
+	const result: Record<string, DocumentSchemaValueType> = {};
+	for (const [key, entry] of Object.entries(
+		retiredDocumentSchemaFeatures as Record<string, IRetiredFeatureEntry>,
+	)) {
+		result[key] = entry.value;
+	}
+	return result;
+}
+
 /**
  * Checks if a given schema is compatible with current code, i.e. if current code can understand all the features of that schema.
  * If schema is not compatible with current code, it throws an exception.
@@ -343,7 +395,7 @@ function checkRuntimeCompatibility(
 		unknownProperty = "runtime";
 	} else {
 		for (const [name, value] of Object.entries(documentSchema.runtime)) {
-			const validator = documentSchemaSupportedConfigs[name] as IProperty | undefined;
+			const validator = getRuntimeConfigHandler(name);
 			if (!(validator?.validate(value) ?? false)) {
 				unknownProperty = `runtime/${name}`;
 			}
@@ -377,7 +429,7 @@ function and(
 		...Object.keys(persistedSchema.runtime),
 		...Object.keys(providedSchema.runtime),
 	])) {
-		runtime[key] = (documentSchemaSupportedConfigs[key] as IProperty).and(
+		runtime[key] = (getRuntimeConfigHandler(key) as IProperty).and(
 			persistedSchema.runtime[key],
 			providedSchema.runtime[key],
 		);
@@ -405,7 +457,7 @@ function or(
 		...Object.keys(persistedSchema.runtime),
 		...Object.keys(providedSchema.runtime),
 	])) {
-		runtime[key] = (documentSchemaSupportedConfigs[key] as IProperty).or(
+		runtime[key] = (getRuntimeConfigHandler(key) as IProperty).or(
 			persistedSchema.runtime[key],
 			providedSchema.runtime[key],
 		);
@@ -625,6 +677,7 @@ export class DocumentsSchemaController {
 				opGroupingEnabled: boolToProp(features.opGroupingEnabled),
 				createBlobPayloadPending: features.createBlobPayloadPending,
 				disallowedVersions: arrayToProp(features.disallowedVersions),
+				...retiredFeatureValues(),
 			},
 		};