@fluidframework/container-runtime 2.4.0-299374 → 2.4.0-299707

package/src/containerRuntime.ts

@@ -102,7 +102,10 @@ import {
 	responseToException,
 	seqFromTree,
 } from "@fluidframework/runtime-utils/internal";
-import type { ITelemetryGenericEventExt } from "@fluidframework/telemetry-utils/internal";
+import type {
+	IFluidErrorBase,
+	ITelemetryGenericEventExt,
+} from "@fluidframework/telemetry-utils/internal";
 import {
 	ITelemetryLoggerExt,
 	DataCorruptionError,
@@ -167,7 +170,7 @@ import {
 	type OutboundContainerRuntimeMessage,
 	type UnknownContainerRuntimeMessage,
 } from "./messageTypes.js";
-import { IBatchMetadata, ISavedOpMetadata } from "./metadata.js";
+import { ISavedOpMetadata } from "./metadata.js";
 import {
 	BatchId,
 	BatchMessage,
@@ -236,19 +239,32 @@ import {
 import { Throttler, formExponentialFn } from "./throttler.js";
 
 /**
- * Utility to implement compat behaviors given an unknown message type
+ * Creates an error object to be thrown / passed to Container's close fn in case of an unknown message type.
  * The parameters are typed to support compile-time enforcement of handling all known types/behaviors
  *
- * @param _unknownContainerRuntimeMessageType - Typed as something unexpected, to ensure all known types have been
+ * @param unknownContainerRuntimeMessageType - Typed as something unexpected, to ensure all known types have been
  * handled before calling this function (e.g. in a switch statement).
- * @param compatBehavior - Typed redundantly with CompatModeBehavior to ensure handling is added when updating that type
+ *
+ * @param codePath - The code path where the unexpected message type was encountered.
+ *
+ * @param sequencedMessage - The sequenced message that contained the unexpected message type.
+ *
  */
-function compatBehaviorAllowsMessageType(
-	_unknownContainerRuntimeMessageType: UnknownContainerRuntimeMessage["type"],
-	compatBehavior: "Ignore" | "FailToProcess" | undefined,
-): boolean {
-	// undefined defaults to same behavior as "FailToProcess"
-	return compatBehavior === "Ignore";
+function getUnknownMessageTypeError(
+	unknownContainerRuntimeMessageType: UnknownContainerRuntimeMessage["type"],
+	codePath: string,
+	sequencedMessage?: ISequencedDocumentMessage,
+): IFluidErrorBase {
+	return DataProcessingError.create(
+		"Runtime message of unknown type",
+		codePath,
+		sequencedMessage,
+		{
+			messageDetails: {
+				type: unknownContainerRuntimeMessageType,
+			},
+		},
+	);
 }
 
 /**
@@ -2527,27 +2543,12 @@ export class ContainerRuntime
 				// GC op is only sent in summarizer which should never have stashed ops.
 				throw new LoggingError("GC op not expected to be stashed in summarizer");
 			default: {
-				// This should be extremely rare for stashed ops.
-				// It would require a newer runtime stashing ops and then an older one applying them,
-				// e.g. if an app rolled back its container version
-				const compatBehavior = opContents.compatDetails?.behavior;
-				if (!compatBehaviorAllowsMessageType(opContents.type, compatBehavior)) {
-					const error = DataProcessingError.create(
-						"Stashed runtime message of unexpected type",
-						"applyStashedOp",
-						undefined /* sequencedMessage */,
-						{
-							messageDetails: JSON.stringify({
-								type: opContents.type,
-								compatBehavior,
-							}),
-						},
-					);
-					this.closeFn(error);
-					throw error;
-				}
-				// Note: Even if its compat behavior allows it, we don't know how to apply this stashed op.
-				// All we can do is ignore it (similar to on process).
+				const error = getUnknownMessageTypeError(
+					opContents.type,
+					"applyStashedOp" /* codePath */,
+				);
+				this.closeFn(error);
+				throw error;
 			}
 		}
 	}
@@ -2978,27 +2979,13 @@ export class ContainerRuntime
 				);
 				break;
 			default: {
-				const compatBehavior = message.compatDetails?.behavior;
-				if (!compatBehaviorAllowsMessageType(message.type, compatBehavior)) {
-					const error = DataProcessingError.create(
-						// Former assert 0x3ce
-						"Runtime message of unknown type",
-						"OpProcessing",
-						message,
-						{
-							local,
-							messageDetails: JSON.stringify({
-								type: message.type,
-								contentType: typeof message.contents,
-								compatBehavior,
-								batch: (message.metadata as IBatchMetadata | undefined)?.batch,
-								compression: message.compression,
-							}),
-						},
-					);
-					this.closeFn(error);
-					throw error;
-				}
+				const error = getUnknownMessageTypeError(
+					message.type,
+					"validateAndProcessRuntimeMessage" /* codePath */,
+					message,
+				);
+				this.closeFn(error);
+				throw error;
 			}
 		}
 
@@ -4462,30 +4449,9 @@ export class ContainerRuntime
 				// send any ops, as some other client already changed schema.
 				break;
 			default: {
-				// This case should be very rare - it would imply an op was stashed from a
-				// future version of runtime code and now is being applied on an older version.
-				const compatBehavior = message.compatDetails?.behavior;
-				if (compatBehaviorAllowsMessageType(message.type, compatBehavior)) {
-					// We do not ultimately resubmit it, to be consistent with this version of the code.
-					this.logger.sendTelemetryEvent({
-						eventName: "resubmitUnrecognizedMessageTypeAllowed",
-						messageDetails: { type: message.type, compatBehavior },
-					});
-				} else {
-					const error = DataProcessingError.create(
-						"Resubmitting runtime message of unexpected type",
-						"reSubmitCore",
-						undefined /* sequencedMessage */,
-						{
-							messageDetails: JSON.stringify({
-								type: message.type,
-								compatBehavior,
-							}),
-						},
-					);
-					this.closeFn(error);
-					throw error;
-				}
+				const error = getUnknownMessageTypeError(message.type, "reSubmitCore" /* codePath */);
+				this.closeFn(error);
+				throw error;
 			}
 		}
 	}

package/src/gc/garbageCollection.ts

@@ -51,7 +51,6 @@ import {
 } from "./gcDefinitions.js";
 import {
 	cloneGCData,
-	compatBehaviorAllowsGCMessageType,
 	concatGarbageCollectionData,
 	dataStoreNodePathOnly,
 	getGCDataFromSnapshot,
@@ -896,16 +895,10 @@ export class GarbageCollector implements IGarbageCollector {
 				break;
 			}
 			default: {
-				if (
-					!compatBehaviorAllowsGCMessageType(gcMessageType, message.compatDetails?.behavior)
-				) {
-					const error = DataProcessingError.create(
-						`Garbage collection message of unknown type ${gcMessageType}`,
-						"processMessage",
-					);
-					throw error;
-				}
-				break;
+				throw DataProcessingError.create(
+					`Garbage collection message of unknown type ${gcMessageType}`,
+					"processMessage",
+				);
 			}
 		}
 	}
@@ -1034,7 +1027,7 @@ export class GarbageCollector implements IGarbageCollector {
 	 *
 	 * Submit a GC op indicating that the Tombstone with the given path has been loaded.
 	 * Broadcasting this information in the op stream allows the Summarizer to reset unreferenced state
-	 * before runnint GC next.
+	 * before running GC next.
 	 */
 	private triggerAutoRecovery(nodePath: string) {
 		// If sweep isn't enabled, auto-recovery isn't needed since its purpose is to prevent this object from being

package/src/opLifecycle/batchManager.ts

@@ -20,6 +20,9 @@ export interface IBatchManagerOptions {
 	 * If true, the outbox is allowed to rebase the batch during flushing.
 	 */
 	readonly canRebase: boolean;
+
+	/** If true, don't compare batchID of incoming batches to this. e.g. ID Allocation Batch IDs should be ignored */
+	readonly ignoreBatchId?: boolean;
 }
 
 export interface BatchSequenceNumbers {

package/src/opLifecycle/outbox.ts

@@ -123,7 +123,11 @@ export class Outbox {
 
 		this.mainBatch = new BatchManager({ hardLimit, canRebase: true });
 		this.blobAttachBatch = new BatchManager({ hardLimit, canRebase: true });
-		this.idAllocationBatch = new BatchManager({ hardLimit, canRebase: false });
+		this.idAllocationBatch = new BatchManager({
+			hardLimit,
+			canRebase: false,
+			ignoreBatchId: true,
+		});
 	}
 
 	public get messageCount(): number {
@@ -251,17 +255,20 @@ export class Outbox {
 	}
 
 	private flushAll(resubmittingBatchId?: BatchId) {
-		// Don't use resubmittingBatchId for idAllocationBatch.
-		// ID Allocation messages are not directly resubmitted so we don't want to reuse the batch ID.
-		this.flushInternal(this.idAllocationBatch);
-		// We need to flush an empty batch if the main batch *becomes* empty on resubmission.
-		// When resubmitting the main batch, the blobAttach batch will always be empty since we don't resubmit them simultaneously.
-		// And conversely, the blobAttach will never *become* empty on resubmit.
-		// So if both blobAttachBatch and mainBatch are empty, we must submit an empty main batch.
-		if (resubmittingBatchId && this.blobAttachBatch.empty && this.mainBatch.empty) {
+		// If we're resubmitting and all batches are empty, we need to flush an empty batch.
+		// Note that we currently resubmit one batch at a time, so on resubmit, 2 of the 3 batches will *always* be empty.
+		// It's theoretically possible that we don't *need* to resubmit this empty batch, and in those cases, it'll safely be ignored
+		// by the rest of the system, including remote clients.
+		// In some cases we *must* resubmit the empty batch (to match up with a non-empty version tracked locally by a container fork), so we do it always.
+		const allBatchesEmpty =
+			this.idAllocationBatch.empty && this.blobAttachBatch.empty && this.mainBatch.empty;
+		if (resubmittingBatchId && allBatchesEmpty) {
 			this.flushEmptyBatch(resubmittingBatchId);
 			return;
 		}
+		// Don't use resubmittingBatchId for idAllocationBatch.
+		// ID Allocation messages are not directly resubmitted so we don't want to reuse the batch ID.
+		this.flushInternal(this.idAllocationBatch);
 		this.flushInternal(
 			this.blobAttachBatch,
 			true /* disableGroupedBatching */,
@@ -332,7 +339,11 @@ export class Outbox {
 			);
 		}
 
-		this.params.pendingStateManager.onFlushBatch(rawBatch.messages, clientSequenceNumber);
+		this.params.pendingStateManager.onFlushBatch(
+			rawBatch.messages,
+			clientSequenceNumber,
+			batchManager.options.ignoreBatchId,
+		);
 	}
 
 	/**

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.4.0-299374";
+export const pkgVersion = "2.4.0-299707";

package/src/pendingStateManager.ts

@@ -41,7 +41,10 @@ export interface IPendingMessage {
 	localOpMetadata: unknown;
 	opMetadata: Record<string, unknown> | undefined;
 	sequenceNumber?: number;
-	/** Info about the batch this pending message belongs to, for validation and for computing the batchId on reconnect */
+	/**
+	 * Info about the batch this pending message belongs to, for validation and for computing the batchId on reconnect
+	 * We don't include batchId itself to avoid redundancy, because that's stamped on opMetadata above
+	 */
 	batchInfo: {
 		/**
 		 * The Batch's original clientId, from when it was first flushed to be submitted.
@@ -55,6 +58,8 @@ export interface IPendingMessage {
 		batchStartCsn: number;
 		/** length of the batch (how many runtime messages here) */
 		length: number;
+		/** If true, don't compare batchID of incoming batches to this. e.g. ID Allocation Batch IDs should be ignored */
+		ignoreBatchId?: boolean;
 	};
 }
 
@@ -239,8 +244,13 @@ export class PendingStateManager implements IDisposable {
 	 * @param batch - The batch that was flushed
 	 * @param clientSequenceNumber - The CSN of the first message in the batch,
 	 * or undefined if the batch was not yet sent (e.g. by the time we flushed we lost the connection)
+	 * @param ignoreBatchId - Whether to ignore the batchId in the batchStartInfo
 	 */
-	public onFlushBatch(batch: BatchMessage[], clientSequenceNumber: number | undefined) {
+	public onFlushBatch(
+		batch: BatchMessage[],
+		clientSequenceNumber: number | undefined,
+		ignoreBatchId?: boolean,
+	) {
 		// clientId and batchStartCsn are used for generating the batchId so we can detect container forks
 		// where this batch was submitted by two different clients rehydrating from the same local state.
 		// In the typical case where the batch was actually sent, use the clientId and clientSequenceNumber.
@@ -269,7 +279,7 @@ export class PendingStateManager implements IDisposable {
 				localOpMetadata,
 				opMetadata,
 				// Note: We only will read this off the first message, but put it on all for simplicity
-				batchInfo: { clientId, batchStartCsn, length: batch.length },
+				batchInfo: { clientId, batchStartCsn, length: batch.length, ignoreBatchId },
 			};
 			this.pendingMessages.push(pendingMessage);
 		}
@@ -328,15 +338,23 @@ export class PendingStateManager implements IDisposable {
 	 * @returns whether the batch IDs match
 	 */
 	private remoteBatchMatchesPendingBatch(remoteBatchStart: BatchStartInfo): boolean {
-		// We may have no pending changes - if so, no match, no problem.
-		const pendingMessage = this.pendingMessages.peekFront();
-		if (pendingMessage === undefined) {
+		// Find the first pending message that uses Batch ID, to compare to the incoming remote batch.
+		// If there is no such message, then the incoming remote batch doesn't have a match here and we can return.
+		const firstIndexUsingBatchId = Array.from({
+			length: this.pendingMessages.length,
+		}).findIndex((_, i) => this.pendingMessages.get(i)?.batchInfo.ignoreBatchId !== true);
+		const pendingMessageUsingBatchId =
+			firstIndexUsingBatchId === -1
+				? undefined
+				: this.pendingMessages.get(firstIndexUsingBatchId);
+
+		if (pendingMessageUsingBatchId === undefined) {
 			return false;
 		}
 
 		// We must compare the effective batch IDs, since one of these ops
 		// may have been the original, not resubmitted, so wouldn't have its batch ID stamped yet.
-		const pendingBatchId = getEffectiveBatchId(pendingMessage);
+		const pendingBatchId = getEffectiveBatchId(pendingMessageUsingBatchId);
 		const inboundBatchId = getEffectiveBatchId(remoteBatchStart);
 
 		return pendingBatchId === inboundBatchId;
@@ -488,7 +506,7 @@ export class PendingStateManager implements IDisposable {
 			0xa21 /* No pending message found as we start processing this remote batch */,
 		);
 
-		// If this batch became empty on resubmit, batch.messages will be empty (so firstMessage undefined)
+		// If this batch became empty on resubmit, batch.messages will be empty (but keyMessage is always set)
 		// and the next pending message should be an empty batch marker.
 		// More Info: We must submit empty batches and track them in case a different fork
 		// of this container also submitted the same batch (and it may not be empty for that fork).