@fluidframework/container-runtime 2.4.0-297385 → 2.4.0-299707

package/src/containerRuntime.ts

@@ -102,7 +102,10 @@ import {
 	responseToException,
 	seqFromTree,
 } from "@fluidframework/runtime-utils/internal";
-import type { ITelemetryGenericEventExt } from "@fluidframework/telemetry-utils/internal";
+import type {
+	IFluidErrorBase,
+	ITelemetryGenericEventExt,
+} from "@fluidframework/telemetry-utils/internal";
 import {
 	ITelemetryLoggerExt,
 	DataCorruptionError,
@@ -167,7 +170,7 @@ import {
 	type OutboundContainerRuntimeMessage,
 	type UnknownContainerRuntimeMessage,
 } from "./messageTypes.js";
-import { IBatchMetadata, ISavedOpMetadata } from "./metadata.js";
+import { ISavedOpMetadata } from "./metadata.js";
 import {
 	BatchId,
 	BatchMessage,
@@ -236,19 +239,32 @@ import {
 import { Throttler, formExponentialFn } from "./throttler.js";
 
 /**
- * Utility to implement compat behaviors given an unknown message type
+ * Creates an error object to be thrown / passed to Container's close fn in case of an unknown message type.
  * The parameters are typed to support compile-time enforcement of handling all known types/behaviors
  *
- * @param _unknownContainerRuntimeMessageType - Typed as something unexpected, to ensure all known types have been
+ * @param unknownContainerRuntimeMessageType - Typed as something unexpected, to ensure all known types have been
  * handled before calling this function (e.g. in a switch statement).
- * @param compatBehavior - Typed redundantly with CompatModeBehavior to ensure handling is added when updating that type
+ *
+ * @param codePath - The code path where the unexpected message type was encountered.
+ *
+ * @param sequencedMessage - The sequenced message that contained the unexpected message type.
+ *
  */
-function compatBehaviorAllowsMessageType(
-	_unknownContainerRuntimeMessageType: UnknownContainerRuntimeMessage["type"],
-	compatBehavior: "Ignore" | "FailToProcess" | undefined,
-): boolean {
-	// undefined defaults to same behavior as "FailToProcess"
-	return compatBehavior === "Ignore";
+function getUnknownMessageTypeError(
+	unknownContainerRuntimeMessageType: UnknownContainerRuntimeMessage["type"],
+	codePath: string,
+	sequencedMessage?: ISequencedDocumentMessage,
+): IFluidErrorBase {
+	return DataProcessingError.create(
+		"Runtime message of unknown type",
+		codePath,
+		sequencedMessage,
+		{
+			messageDetails: {
+				type: unknownContainerRuntimeMessageType,
+			},
+		},
+	);
 }
 
 /**
@@ -1148,6 +1164,9 @@ export class ContainerRuntime
 		summaryOp: ISummaryContent,
 		referenceSequenceNumber?: number,
 	) => number;
+	/**
+	 * Do not call directly - use submitAddressesSignal
+	 */
 	private readonly submitSignalFn: (content: ISignalEnvelope, targetClientId?: string) => void;
 	public readonly disposeFn: (error?: ICriticalContainerError) => void;
 	public readonly closeFn: (error?: ICriticalContainerError) => void;
@@ -1755,7 +1774,7 @@ export class ContainerRuntime
 				type,
 				envelope1.contents,
 			);
-			return this.submitSignalFn(envelope2, targetClientId);
+			return this.submitEnvelopedSignal(envelope2, targetClientId);
 		};
 
 		let snapshot: ISnapshot | ISnapshotTree | undefined = getSummaryForDatastores(
@@ -2524,27 +2543,12 @@ export class ContainerRuntime
 				// GC op is only sent in summarizer which should never have stashed ops.
 				throw new LoggingError("GC op not expected to be stashed in summarizer");
 			default: {
-				// This should be extremely rare for stashed ops.
-				// It would require a newer runtime stashing ops and then an older one applying them,
-				// e.g. if an app rolled back its container version
-				const compatBehavior = opContents.compatDetails?.behavior;
-				if (!compatBehaviorAllowsMessageType(opContents.type, compatBehavior)) {
-					const error = DataProcessingError.create(
-						"Stashed runtime message of unexpected type",
-						"applyStashedOp",
-						undefined /* sequencedMessage */,
-						{
-							messageDetails: JSON.stringify({
-								type: opContents.type,
-								compatBehavior,
-							}),
-						},
-					);
-					this.closeFn(error);
-					throw error;
-				}
-				// Note: Even if its compat behavior allows it, we don't know how to apply this stashed op.
-				// All we can do is ignore it (similar to on process).
+				const error = getUnknownMessageTypeError(
+					opContents.type,
+					"applyStashedOp" /* codePath */,
+				);
+				this.closeFn(error);
+				throw error;
 			}
 		}
 	}
@@ -2975,27 +2979,13 @@ export class ContainerRuntime
 				);
 				break;
 			default: {
-				const compatBehavior = message.compatDetails?.behavior;
-				if (!compatBehaviorAllowsMessageType(message.type, compatBehavior)) {
-					const error = DataProcessingError.create(
-						// Former assert 0x3ce
-						"Runtime message of unknown type",
-						"OpProcessing",
-						message,
-						{
-							local,
-							messageDetails: JSON.stringify({
-								type: message.type,
-								contentType: typeof message.contents,
-								compatBehavior,
-								batch: (message.metadata as IBatchMetadata | undefined)?.batch,
-								compression: message.compression,
-							}),
-						},
-					);
-					this.closeFn(error);
-					throw error;
-				}
+				const error = getUnknownMessageTypeError(
+					message.type,
+					"validateAndProcessRuntimeMessage" /* codePath */,
+					message,
+				);
+				this.closeFn(error);
+				throw error;
 			}
 		}
 
@@ -3021,6 +3011,72 @@ export class ContainerRuntime
 		this._signalTracking.totalSignalsSentInLatencyWindow = 0;
 	}
 
+	/**
+	 * Updates signal telemetry including emitting telemetry events.
+	 */
+	private processSignalForTelemetry(envelope: ISignalEnvelope): void {
+		const { clientBroadcastSignalSequenceNumber } = envelope;
+		if (clientBroadcastSignalSequenceNumber === undefined) {
+			return;
+		}
+
+		if (
+			this._signalTracking.trackingSignalSequenceNumber === undefined ||
+			this._signalTracking.minimumTrackingSignalSequenceNumber === undefined
+		) {
+			return;
+		}
+
+		if (
+			clientBroadcastSignalSequenceNumber >= this._signalTracking.trackingSignalSequenceNumber
+		) {
+			// Calculate the number of signals lost and log the event.
+			const signalsLost =
+				clientBroadcastSignalSequenceNumber -
+				this._signalTracking.trackingSignalSequenceNumber;
+			if (signalsLost > 0) {
+				this._signalTracking.signalsLost += signalsLost;
+				this.mc.logger.sendErrorEvent({
+					eventName: "SignalLost",
+					signalsLost, // Number of lost signals detected.
+					trackingSequenceNumber: this._signalTracking.trackingSignalSequenceNumber, // The next expected signal sequence number.
+					clientBroadcastSignalSequenceNumber, // Actual signal sequence number received.
+				});
+			}
+			// Update the tracking signal sequence number to the next expected signal in the sequence.
+			this._signalTracking.trackingSignalSequenceNumber =
+				clientBroadcastSignalSequenceNumber + 1;
+		} else if (
+			// Check if this is a signal in range of interest.
+			clientBroadcastSignalSequenceNumber >=
+			this._signalTracking.minimumTrackingSignalSequenceNumber
+		) {
+			this._signalTracking.signalsOutOfOrder++;
+			this.mc.logger.sendTelemetryEvent({
+				eventName: "SignalOutOfOrder",
+				type: envelope.contents.type, // Type of signal that was received out of order.
+				trackingSequenceNumber: this._signalTracking.trackingSignalSequenceNumber, // The next expected signal sequence number.
+				clientBroadcastSignalSequenceNumber, // Sequence number of the out of order signal.
+			});
+		}
+		if (
+			this._signalTracking.roundTripSignalSequenceNumber !== undefined &&
+			clientBroadcastSignalSequenceNumber >= this._signalTracking.roundTripSignalSequenceNumber
+		) {
+			if (
+				clientBroadcastSignalSequenceNumber ===
+				this._signalTracking.roundTripSignalSequenceNumber
+			) {
+				// Latency tracked signal has been received.
+				// We now log the roundtrip duration of the tracked signal.
+				// This telemetry event also logs metrics for signals sent, signals lost, and out of order signals received.
+				// These metrics are reset after logging the telemetry event.
+				this.sendSignalTelemetryEvent();
+			}
+			this._signalTracking.roundTripSignalSequenceNumber = undefined;
+		}
+	}
+
 	public processSignal(message: ISignalMessage, local: boolean) {
 		const envelope = message.content as ISignalEnvelope;
 		const transformed: IInboundSignalMessage = {
@@ -3033,64 +3089,15 @@ export class ContainerRuntime
 		// Only collect signal telemetry for broadcast messages sent by the current client.
 		if (
 			message.clientId === this.clientId &&
-			this.connected &&
-			envelope.clientBroadcastSignalSequenceNumber !== undefined
+			// jason-ha: This `connected` check seems incorrect. Signals that come through
+			// here must have been received while connected to service and there is no need
+			// to avoid processing when connection has dropped. Because container runtime's
+			// `connected` (and `_connected`) state also reflects some ops state, it may
+			//  easily be false when newly connected and signal tracking may very well
+			// complain lost signals (that were simply skipped per this check).
+			this.connected
 		) {
-			if (
-				this._signalTracking.trackingSignalSequenceNumber !== undefined &&
-				this._signalTracking.minimumTrackingSignalSequenceNumber !== undefined
-			) {
-				if (
-					envelope.clientBroadcastSignalSequenceNumber >=
-					this._signalTracking.trackingSignalSequenceNumber
-				) {
-					// Calculate the number of signals lost and log the event.
-					const signalsLost =
-						envelope.clientBroadcastSignalSequenceNumber -
-						this._signalTracking.trackingSignalSequenceNumber;
-					if (signalsLost > 0) {
-						this._signalTracking.signalsLost += signalsLost;
-						this.mc.logger.sendErrorEvent({
-							eventName: "SignalLost",
-							signalsLost, // Number of lost signals detected.
-							trackingSequenceNumber: this._signalTracking.trackingSignalSequenceNumber, // The next expected signal sequence number.
-							clientBroadcastSignalSequenceNumber:
-								envelope.clientBroadcastSignalSequenceNumber, // Actual signal sequence number received.
-						});
-					}
-					// Update the tracking signal sequence number to the next expected signal in the sequence.
-					this._signalTracking.trackingSignalSequenceNumber =
-						envelope.clientBroadcastSignalSequenceNumber + 1;
-				} else if (
-					envelope.clientBroadcastSignalSequenceNumber >=
-					this._signalTracking.minimumTrackingSignalSequenceNumber
-				) {
-					this._signalTracking.signalsOutOfOrder++;
-					this.mc.logger.sendTelemetryEvent({
-						eventName: "SignalOutOfOrder",
-						type: envelope.contents.type, // Type of signal that was received out of order.
-						trackingSequenceNumber: this._signalTracking.trackingSignalSequenceNumber, // The next expected signal sequence number.
-						clientBroadcastSignalSequenceNumber: envelope.clientBroadcastSignalSequenceNumber, // Sequence number of the out of order signal.
-					});
-				}
-				if (
-					this._signalTracking.roundTripSignalSequenceNumber !== undefined &&
-					envelope.clientBroadcastSignalSequenceNumber >=
-						this._signalTracking.roundTripSignalSequenceNumber
-				) {
-					if (
-						envelope.clientBroadcastSignalSequenceNumber ===
-						this._signalTracking.roundTripSignalSequenceNumber
-					) {
-						// Latency tracked signal has been received.
-						// We now log the roundtrip duration of the tracked signal.
-						// This telemetry event also logs metrics for signals sent, signals lost, and out of order signals received.
-						// These metrics are reset after logging the telemetry event.
-						this.sendSignalTelemetryEvent();
-					}
-					this._signalTracking.roundTripSignalSequenceNumber = undefined;
-				}
-			}
+			this.processSignalForTelemetry(envelope);
 		}
 
 		if (envelope.address === undefined) {
@@ -3337,19 +3344,24 @@ export class ContainerRuntime
 		address: string | undefined,
 		type: string,
 		content: any,
-		targetClientId?: string,
-	): ISignalEnvelope {
-		const newEnvelope: ISignalEnvelope = {
+	): Omit<ISignalEnvelope, "broadcastSignalSequenceNumber"> {
+		const newEnvelope: Omit<ISignalEnvelope, "broadcastSignalSequenceNumber"> = {
 			address,
 			contents: { type, content },
 		};
 
+		return newEnvelope;
+	}
+
+	private submitEnvelopedSignal(envelope: ISignalEnvelope, targetClientId?: string) {
 		const isBroadcastSignal = targetClientId === undefined;
 
 		if (isBroadcastSignal) {
 			const clientBroadcastSignalSequenceNumber = ++this._signalTracking
 				.broadcastSignalSequenceNumber;
-			newEnvelope.clientBroadcastSignalSequenceNumber = clientBroadcastSignalSequenceNumber;
+			// Stamp with the broadcast signal sequence number.
+			envelope.clientBroadcastSignalSequenceNumber = clientBroadcastSignalSequenceNumber;
+
 			this._signalTracking.signalsSentSinceLastLatencyMeasurement++;
 
 			if (
@@ -3378,7 +3390,7 @@ export class ContainerRuntime
 			}
 		}
 
-		return newEnvelope;
+		this.submitSignalFn(envelope, targetClientId);
 	}
 
 	/**
@@ -3395,13 +3407,8 @@ export class ContainerRuntime
 	 */
 	public submitSignal(type: string, content: unknown, targetClientId?: string) {
 		this.verifyNotClosed();
-		const envelope = this.createNewSignalEnvelope(
-			undefined /* address */,
-			type,
-			content,
-			targetClientId,
-		);
-		return this.submitSignalFn(envelope, targetClientId);
+		const envelope = this.createNewSignalEnvelope(undefined /* address */, type, content);
+		return this.submitEnvelopedSignal(envelope, targetClientId);
 	}
 
 	public setAttachState(attachState: AttachState.Attaching | AttachState.Attached): void {
@@ -4442,30 +4449,9 @@ export class ContainerRuntime
 				// send any ops, as some other client already changed schema.
 				break;
 			default: {
-				// This case should be very rare - it would imply an op was stashed from a
-				// future version of runtime code and now is being applied on an older version.
-				const compatBehavior = message.compatDetails?.behavior;
-				if (compatBehaviorAllowsMessageType(message.type, compatBehavior)) {
-					// We do not ultimately resubmit it, to be consistent with this version of the code.
-					this.logger.sendTelemetryEvent({
-						eventName: "resubmitUnrecognizedMessageTypeAllowed",
-						messageDetails: { type: message.type, compatBehavior },
-					});
-				} else {
-					const error = DataProcessingError.create(
-						"Resubmitting runtime message of unexpected type",
-						"reSubmitCore",
-						undefined /* sequencedMessage */,
-						{
-							messageDetails: JSON.stringify({
-								type: message.type,
-								compatBehavior,
-							}),
-						},
-					);
-					this.closeFn(error);
-					throw error;
-				}
+				const error = getUnknownMessageTypeError(message.type, "reSubmitCore" /* codePath */);
+				this.closeFn(error);
+				throw error;
 			}
 		}
 	}

package/src/gc/garbageCollection.ts

@@ -51,7 +51,6 @@ import {
 } from "./gcDefinitions.js";
 import {
 	cloneGCData,
-	compatBehaviorAllowsGCMessageType,
 	concatGarbageCollectionData,
 	dataStoreNodePathOnly,
 	getGCDataFromSnapshot,
@@ -896,16 +895,10 @@ export class GarbageCollector implements IGarbageCollector {
 				break;
 			}
 			default: {
-				if (
-					!compatBehaviorAllowsGCMessageType(gcMessageType, message.compatDetails?.behavior)
-				) {
-					const error = DataProcessingError.create(
-						`Garbage collection message of unknown type ${gcMessageType}`,
-						"processMessage",
-					);
-					throw error;
-				}
-				break;
+				throw DataProcessingError.create(
+					`Garbage collection message of unknown type ${gcMessageType}`,
+					"processMessage",
+				);
 			}
 		}
 	}
@@ -1034,7 +1027,7 @@ export class GarbageCollector implements IGarbageCollector {
 	 *
 	 * Submit a GC op indicating that the Tombstone with the given path has been loaded.
 	 * Broadcasting this information in the op stream allows the Summarizer to reset unreferenced state
-	 * before runnint GC next.
+	 * before running GC next.
 	 */
 	private triggerAutoRecovery(nodePath: string) {
 		// If sweep isn't enabled, auto-recovery isn't needed since its purpose is to prevent this object from being

package/src/opLifecycle/batchManager.ts

@@ -20,6 +20,9 @@ export interface IBatchManagerOptions {
 	 * If true, the outbox is allowed to rebase the batch during flushing.
 	 */
 	readonly canRebase: boolean;
+
+	/** If true, don't compare batchID of incoming batches to this. e.g. ID Allocation Batch IDs should be ignored */
+	readonly ignoreBatchId?: boolean;
 }
 
 export interface BatchSequenceNumbers {

package/src/opLifecycle/outbox.ts

@@ -123,7 +123,11 @@ export class Outbox {
 
 		this.mainBatch = new BatchManager({ hardLimit, canRebase: true });
 		this.blobAttachBatch = new BatchManager({ hardLimit, canRebase: true });
-		this.idAllocationBatch = new BatchManager({ hardLimit, canRebase: false });
+		this.idAllocationBatch = new BatchManager({
+			hardLimit,
+			canRebase: false,
+			ignoreBatchId: true,
+		});
 	}
 
 	public get messageCount(): number {
@@ -251,17 +255,20 @@ export class Outbox {
 	}
 
 	private flushAll(resubmittingBatchId?: BatchId) {
-		// Don't use resubmittingBatchId for idAllocationBatch.
-		// ID Allocation messages are not directly resubmitted so we don't want to reuse the batch ID.
-		this.flushInternal(this.idAllocationBatch);
-		// We need to flush an empty batch if the main batch *becomes* empty on resubmission.
-		// When resubmitting the main batch, the blobAttach batch will always be empty since we don't resubmit them simultaneously.
-		// And conversely, the blobAttach will never *become* empty on resubmit.
-		// So if both blobAttachBatch and mainBatch are empty, we must submit an empty main batch.
-		if (resubmittingBatchId && this.blobAttachBatch.empty && this.mainBatch.empty) {
+		// If we're resubmitting and all batches are empty, we need to flush an empty batch.
+		// Note that we currently resubmit one batch at a time, so on resubmit, 2 of the 3 batches will *always* be empty.
+		// It's theoretically possible that we don't *need* to resubmit this empty batch, and in those cases, it'll safely be ignored
+		// by the rest of the system, including remote clients.
+		// In some cases we *must* resubmit the empty batch (to match up with a non-empty version tracked locally by a container fork), so we do it always.
+		const allBatchesEmpty =
+			this.idAllocationBatch.empty && this.blobAttachBatch.empty && this.mainBatch.empty;
+		if (resubmittingBatchId && allBatchesEmpty) {
 			this.flushEmptyBatch(resubmittingBatchId);
 			return;
 		}
+		// Don't use resubmittingBatchId for idAllocationBatch.
+		// ID Allocation messages are not directly resubmitted so we don't want to reuse the batch ID.
+		this.flushInternal(this.idAllocationBatch);
 		this.flushInternal(
 			this.blobAttachBatch,
 			true /* disableGroupedBatching */,
@@ -332,7 +339,11 @@ export class Outbox {
 			);
 		}
 
-		this.params.pendingStateManager.onFlushBatch(rawBatch.messages, clientSequenceNumber);
+		this.params.pendingStateManager.onFlushBatch(
+			rawBatch.messages,
+			clientSequenceNumber,
+			batchManager.options.ignoreBatchId,
+		);
 	}
 
 	/**

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.4.0-297385";
+export const pkgVersion = "2.4.0-299707";

package/src/pendingStateManager.ts

@@ -41,7 +41,10 @@ export interface IPendingMessage {
 	localOpMetadata: unknown;
 	opMetadata: Record<string, unknown> | undefined;
 	sequenceNumber?: number;
-	/** Info about the batch this pending message belongs to, for validation and for computing the batchId on reconnect */
+	/**
+	 * Info about the batch this pending message belongs to, for validation and for computing the batchId on reconnect
+	 * We don't include batchId itself to avoid redundancy, because that's stamped on opMetadata above
+	 */
 	batchInfo: {
 		/**
 		 * The Batch's original clientId, from when it was first flushed to be submitted.
@@ -55,6 +58,8 @@ export interface IPendingMessage {
 		batchStartCsn: number;
 		/** length of the batch (how many runtime messages here) */
 		length: number;
+		/** If true, don't compare batchID of incoming batches to this. e.g. ID Allocation Batch IDs should be ignored */
+		ignoreBatchId?: boolean;
 	};
 }
 
@@ -239,8 +244,13 @@ export class PendingStateManager implements IDisposable {
 	 * @param batch - The batch that was flushed
 	 * @param clientSequenceNumber - The CSN of the first message in the batch,
 	 * or undefined if the batch was not yet sent (e.g. by the time we flushed we lost the connection)
+	 * @param ignoreBatchId - Whether to ignore the batchId in the batchStartInfo
 	 */
-	public onFlushBatch(batch: BatchMessage[], clientSequenceNumber: number | undefined) {
+	public onFlushBatch(
+		batch: BatchMessage[],
+		clientSequenceNumber: number | undefined,
+		ignoreBatchId?: boolean,
+	) {
 		// clientId and batchStartCsn are used for generating the batchId so we can detect container forks
 		// where this batch was submitted by two different clients rehydrating from the same local state.
 		// In the typical case where the batch was actually sent, use the clientId and clientSequenceNumber.
@@ -269,7 +279,7 @@ export class PendingStateManager implements IDisposable {
 				localOpMetadata,
 				opMetadata,
 				// Note: We only will read this off the first message, but put it on all for simplicity
-				batchInfo: { clientId, batchStartCsn, length: batch.length },
+				batchInfo: { clientId, batchStartCsn, length: batch.length, ignoreBatchId },
 			};
 			this.pendingMessages.push(pendingMessage);
 		}
@@ -328,15 +338,23 @@ export class PendingStateManager implements IDisposable {
 	 * @returns whether the batch IDs match
 	 */
 	private remoteBatchMatchesPendingBatch(remoteBatchStart: BatchStartInfo): boolean {
-		// We may have no pending changes - if so, no match, no problem.
-		const pendingMessage = this.pendingMessages.peekFront();
-		if (pendingMessage === undefined) {
+		// Find the first pending message that uses Batch ID, to compare to the incoming remote batch.
+		// If there is no such message, then the incoming remote batch doesn't have a match here and we can return.
+		const firstIndexUsingBatchId = Array.from({
+			length: this.pendingMessages.length,
+		}).findIndex((_, i) => this.pendingMessages.get(i)?.batchInfo.ignoreBatchId !== true);
+		const pendingMessageUsingBatchId =
+			firstIndexUsingBatchId === -1
+				? undefined
+				: this.pendingMessages.get(firstIndexUsingBatchId);
+
+		if (pendingMessageUsingBatchId === undefined) {
 			return false;
 		}
 
 		// We must compare the effective batch IDs, since one of these ops
 		// may have been the original, not resubmitted, so wouldn't have its batch ID stamped yet.
-		const pendingBatchId = getEffectiveBatchId(pendingMessage);
+		const pendingBatchId = getEffectiveBatchId(pendingMessageUsingBatchId);
 		const inboundBatchId = getEffectiveBatchId(remoteBatchStart);
 
 		return pendingBatchId === inboundBatchId;
@@ -488,7 +506,7 @@ export class PendingStateManager implements IDisposable {
 			0xa21 /* No pending message found as we start processing this remote batch */,
 		);
 
-		// If this batch became empty on resubmit, batch.messages will be empty (so firstMessage undefined)
+		// If this batch became empty on resubmit, batch.messages will be empty (but keyMessage is always set)
 		// and the next pending message should be an empty batch marker.
 		// More Info: We must submit empty batches and track them in case a different fork
 		// of this container also submitted the same batch (and it may not be empty for that fork).