@fluidframework/container-runtime 2.5.0-302463 → 2.5.0

package/src/containerRuntime.ts

@@ -89,6 +89,7 @@ import {
 	channelsTreeName,
 	gcTreeKey,
 	IInboundSignalMessage,
+	type IRuntimeMessagesContent,
 } from "@fluidframework/runtime-definitions/internal";
 import {
 	GCDataBuilder,
@@ -160,6 +161,7 @@ import {
 	IGCStats,
 	IGarbageCollector,
 	gcGenerationOptionName,
+	type GarbageCollectionMessage,
 } from "./gc/index.js";
 import {
 	ContainerMessageType,
@@ -202,6 +204,7 @@ import {
 	IConnectableRuntime,
 	IContainerRuntimeMetadata,
 	ICreateContainerMetadata,
+	type IDocumentSchemaChangeMessage,
 	type IDocumentSchemaCurrent,
 	IEnqueueSummarizeOptions,
 	IGenerateSummaryTreeResult,
@@ -558,6 +561,13 @@ export interface RuntimeHeaderData {
 	allowTombstone?: boolean;
 }
 
+/** Default values for Runtime Headers */
+export const defaultRuntimeHeaderData: Required<RuntimeHeaderData> = {
+	wait: true,
+	viaHandle: false,
+	allowTombstone: false,
+};
+
 /**
  * Available compression algorithms for op compression.
  * @legacy
@@ -2804,6 +2814,9 @@ export class ContainerRuntime
 				local,
 				savedOp,
 				runtimeBatch,
+				inboundResult.type === "fullBatch"
+					? inboundResult.groupedBatch
+					: false /* groupedBatch */,
 			);
 		} else {
 			if (!runtimeBatch) {
@@ -2823,6 +2836,7 @@ export class ContainerRuntime
 				local,
 				savedOp,
 				runtimeBatch,
+				false /* groupedBatch */,
 			);
 		}
 
@@ -2838,14 +2852,15 @@ export class ContainerRuntime
 
 	/**
 	 * Processes inbound message(s). It calls schedule manager according to the messages' location in the batch.
-	 * @param messages - messages to process.
+	 * @param messagesWithMetadata - messages to process along with their metadata.
 	 * @param locationInBatch - Are we processing the start and/or end of a batch?
 	 * @param local - true if the messages were originally generated by the client receiving it.
 	 * @param savedOp - true if the message is a replayed saved op.
 	 * @param runtimeBatch - true if these are runtime messages.
+	 * @param groupedBatch - true if these messages are part of a grouped op batch.
 	 */
 	private processInboundMessages(
-		messages: {
+		messagesWithMetadata: {
 			message: ISequencedDocumentMessage;
 			localOpMetadata?: unknown;
 		}[],
@@ -2853,35 +2868,110 @@ export class ContainerRuntime
 		local: boolean,
 		savedOp: boolean | undefined,
 		runtimeBatch: boolean,
+		groupedBatch: boolean,
 	) {
 		if (locationInBatch.batchStart) {
-			const firstMessage = messages[0]?.message;
+			const firstMessage = messagesWithMetadata[0]?.message;
 			assert(firstMessage !== undefined, 0xa31 /* Batch must have at least one message */);
 			this.scheduleManager.batchBegin(firstMessage);
 		}
 
 		let error: unknown;
 		try {
-			messages.forEach(({ message, localOpMetadata }) => {
-				this.ensureNoDataModelChanges(() => {
-					if (runtimeBatch) {
-						this.validateAndProcessRuntimeMessage({
-							message: message as InboundSequencedContainerRuntimeMessage,
+			if (!runtimeBatch) {
+				messagesWithMetadata.forEach(({ message }) => {
+					this.ensureNoDataModelChanges(() => {
+						this.observeNonRuntimeMessage(message);
+					});
+				});
+				return;
+			}
+
+			// Helper that updates a message's minimum sequence number to the minimum sequence number that container
+			// runtime is tracking and sets _processedClientSequenceNumber. It returns the updated message.
+			const updateSequenceNumbers = (message: ISequencedDocumentMessage) => {
+				// Set the minimum sequence number to the containerRuntime's understanding of minimum sequence number.
+				message.minimumSequenceNumber =
+					this.useDeltaManagerOpsProxy &&
+					this.deltaManager.minimumSequenceNumber < message.minimumSequenceNumber
+						? this.deltaManager.minimumSequenceNumber
+						: message.minimumSequenceNumber;
+				this._processedClientSequenceNumber = message.clientSequenceNumber;
+				return message as InboundSequencedContainerRuntimeMessage;
+			};
+
+			// Non-grouped batch messages are processed one at a time.
+			if (!groupedBatch) {
+				for (const { message, localOpMetadata } of messagesWithMetadata) {
+					updateSequenceNumbers(message);
+					this.ensureNoDataModelChanges(() => {
+						this.validateAndProcessRuntimeMessages(
+							message as InboundSequencedContainerRuntimeMessage,
+							[
+								{
+									contents: message.contents,
+									localOpMetadata,
+									clientSequenceNumber: message.clientSequenceNumber,
+								},
+							],
 							local,
 							savedOp,
-							localOpMetadata,
-						});
-					} else {
-						this.observeNonRuntimeMessage(message);
-					}
+						);
+						this.emit("op", message, true /* runtimeMessage */);
+					});
+				}
+				return;
+			}
+
+			let bunchedMessagesContent: IRuntimeMessagesContent[] = [];
+			let previousMessage: InboundSequencedContainerRuntimeMessage | undefined;
+
+			// Helper that processes the previous bunch of messages.
+			const sendBunchedMessages = () => {
+				assert(previousMessage !== undefined, 0xa67 /* previous message must exist */);
+				this.ensureNoDataModelChanges(() => {
+					this.validateAndProcessRuntimeMessages(
+						// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+						previousMessage!,
+						bunchedMessagesContent,
+						local,
+						savedOp,
+					);
 				});
-			});
+				bunchedMessagesContent = [];
+			};
+
+			/**
+			 * For grouped batch messages, bunch contiguous messages of the same type and process them together.
+			 * This is an optimization mainly for DDSes, where it can process a bunch of ops together. DDSes
+			 * like merge tree or shared tree can process ops more efficiently when they are bunched together.
+			 */
+			for (const { message, localOpMetadata } of messagesWithMetadata) {
+				const currentMessage = updateSequenceNumbers(message);
+				if (previousMessage && previousMessage.type !== currentMessage.type) {
+					sendBunchedMessages();
+				}
+				previousMessage = currentMessage;
+				bunchedMessagesContent.push({
+					contents: message.contents,
+					localOpMetadata,
+					clientSequenceNumber: message.clientSequenceNumber,
+				});
+			}
+
+			// Process the last bunch of messages.
+			sendBunchedMessages();
+
+			// Send the "op" events for the messages now that the ops have been processed.
+			for (const { message } of messagesWithMetadata) {
+				this.emit("op", message, true /* runtimeMessage */);
+			}
 		} catch (e) {
 			error = e;
 			throw error;
 		} finally {
 			if (locationInBatch.batchEnd) {
-				const lastMessage = messages[messages.length - 1]?.message;
+				const lastMessage = messagesWithMetadata[messagesWithMetadata.length - 1]?.message;
 				assert(lastMessage !== undefined, 0xa32 /* Batch must have at least one message */);
 				this.scheduleManager.batchEnd(error, lastMessage);
 			}
@@ -2910,80 +3000,62 @@ export class ContainerRuntime
 	}
 
 	/**
-	 * Assuming the given message is also a TypedContainerRuntimeMessage,
-	 * checks its type and dispatches the message to the appropriate handler in the runtime.
+	 * Process runtime messages. The messages here are contiguous messages in a batch.
+	 * Assuming the messages in the given bunch are also a TypedContainerRuntimeMessage, checks its type and dispatch
+	 * the messages to the appropriate handler in the runtime.
 	 * Throws a DataProcessingError if the message looks like but doesn't conform to a known TypedContainerRuntimeMessage type.
+	 * @param message - The core message with common properties for all the messages.
+	 * @param messageContents - The contents, local metadata and clientSequenceNumbers of the messages.
+	 * @param local - true if the messages were originally generated by the client receiving it.
+	 * @param savedOp - true if the message is a replayed saved op.
+	 *
 	 */
-
-	private validateAndProcessRuntimeMessage(messageWithContext: {
-		message: InboundSequencedContainerRuntimeMessage;
-		local: boolean;
-		savedOp?: boolean;
-		localOpMetadata?: unknown;
-	}): void {
-		const { local, message, savedOp, localOpMetadata } = messageWithContext;
-
-		// Set the minimum sequence number to the containerRuntime's understanding of minimum sequence number.
-		if (
-			this.useDeltaManagerOpsProxy &&
-			this.deltaManager.minimumSequenceNumber < message.minimumSequenceNumber
-		) {
-			message.minimumSequenceNumber = this.deltaManager.minimumSequenceNumber;
-		}
-
-		this._processedClientSequenceNumber = message.clientSequenceNumber;
-
+	private validateAndProcessRuntimeMessages(
+		message: Omit<InboundSequencedContainerRuntimeMessage, "contents">,
+		messagesContent: IRuntimeMessagesContent[],
+		local: boolean,
+		savedOp?: boolean,
+	): void {
 		// If there are no more pending messages after processing a local message,
 		// the document is no longer dirty.
 		if (!this.hasPendingMessages()) {
 			this.updateDocumentDirtyState(false);
 		}
 
+		// Get the contents without the localOpMetadata because not all message types know about localOpMetadata.
+		const contents = messagesContent.map((c) => c.contents);
+
 		switch (message.type) {
+			case ContainerMessageType.FluidDataStoreOp:
 			case ContainerMessageType.Attach:
 			case ContainerMessageType.Alias:
-			case ContainerMessageType.FluidDataStoreOp:
-				this.channelCollection.process(message, local, localOpMetadata);
+				// Remove the metadata from the message before sending it to the channel collection. The metadata
+				// is added by the container runtime and is not part of the message that the channel collection and
+				// layers below it expect.
+				this.channelCollection.processMessages({ envelope: message, messagesContent, local });
 				break;
 			case ContainerMessageType.BlobAttach:
-				this.blobManager.processBlobAttachOp(message, local);
+				this.blobManager.processBlobAttachMessage(message, local);
 				break;
 			case ContainerMessageType.IdAllocation:
-				// Don't re-finalize the range if we're processing a "savedOp" in
-				// stashed ops flow. The compressor is stashed with these ops already processed.
-				// That said, in idCompressorMode === "delayed", we might not serialize ID compressor, and
-				// thus we need to process all the ops.
-				if (!(this.skipSavedCompressorOps && savedOp === true)) {
-					const range = message.contents;
-					// Some other client turned on the id compressor. If we have not turned it on,
-					// put it in a pending queue and delay finalization.
-					if (this._idCompressor === undefined) {
-						assert(
-							this.idCompressorMode !== undefined,
-							0x93c /* id compressor should be enabled */,
-						);
-						this.pendingIdCompressorOps.push(range);
-					} else {
-						assert(
-							this.pendingIdCompressorOps.length === 0,
-							0x979 /* there should be no pending ops! */,
-						);
-						this._idCompressor.finalizeCreationRange(range);
-					}
-				}
+				this.processIdCompressorMessages(contents as IdCreationRange[], savedOp);
 				break;
 			case ContainerMessageType.GC:
-				this.garbageCollector.processMessage(message, message.timestamp, local);
+				this.garbageCollector.processMessages(
+					contents as GarbageCollectionMessage[],
+					message.timestamp,
+					local,
+				);
 				break;
 			case ContainerMessageType.ChunkedOp:
-				// From observability POV, we should not exppse the rest of the system (including "op" events on object) to these messages.
+				// From observability POV, we should not expose the rest of the system (including "op" events on object) to these messages.
 				// Also resetReconnectCount() would be wrong - see comment that was there before this change was made.
 				assert(false, 0x93d /* should not even get here */);
 			case ContainerMessageType.Rejoin:
 				break;
 			case ContainerMessageType.DocumentSchemaChange:
-				this.documentsSchemaController.processDocumentSchemaOp(
-					message.contents,
+				this.documentsSchemaController.processDocumentSchemaMessages(
+					contents as IDocumentSchemaChangeMessage[],
 					local,
 					message.sequenceNumber,
 				);
@@ -2992,14 +3064,38 @@ export class ContainerRuntime
 				const error = getUnknownMessageTypeError(
 					message.type,
 					"validateAndProcessRuntimeMessage" /* codePath */,
-					message,
+					message as ISequencedDocumentMessage,
 				);
 				this.closeFn(error);
 				throw error;
 			}
 		}
+	}
 
-		this.emit("op", message, true /* runtimeMessage */);
+	private processIdCompressorMessages(messageContents: IdCreationRange[], savedOp?: boolean) {
+		for (const range of messageContents) {
+			// Don't re-finalize the range if we're processing a "savedOp" in
+			// stashed ops flow. The compressor is stashed with these ops already processed.
+			// That said, in idCompressorMode === "delayed", we might not serialize ID compressor, and
+			// thus we need to process all the ops.
+			if (!(this.skipSavedCompressorOps && savedOp === true)) {
+				// Some other client turned on the id compressor. If we have not turned it on,
+				// put it in a pending queue and delay finalization.
+				if (this._idCompressor === undefined) {
+					assert(
+						this.idCompressorMode !== undefined,
+						0x93c /* id compressor should be enabled */,
+					);
+					this.pendingIdCompressorOps.push(range);
+				} else {
+					assert(
+						this.pendingIdCompressorOps.length === 0,
+						0x979 /* there should be no pending ops! */,
+					);
+					this._idCompressor.finalizeCreationRange(range);
+				}
+			}
+		}
 	}
 
 	/**
@@ -3780,7 +3876,22 @@ export class ContainerRuntime
 			},
 		});
 
-		assert(this.outbox.isEmpty, 0x3d1 /* Can't trigger summary in the middle of a batch */);
+		// legacy: assert 0x3d1
+		if (!this.outbox.isEmpty) {
+			throw DataProcessingError.create(
+				"Can't trigger summary in the middle of a batch",
+				"submitSummary",
+				undefined,
+				{
+					summaryNumber,
+					pendingMessages: this.pendingMessagesCount,
+					outboxLength: this.outbox.messageCount,
+					mainBatchLength: this.outbox.mainBatchMessageCount,
+					blobAttachBatchLength: this.outbox.blobAttachBatchMessageCount,
+					idAllocationBatchLength: this.outbox.idAllocationBatchMessageCount,
+				},
+			);
+		}
 
 		// If the container is dirty, i.e., there are pending unacked ops, the summary will not be eventual consistent
 		// and it may even be incorrect. So, wait for the container to be saved with a timeout. If the container is not

package/src/dataStoreContext.ts

@@ -53,6 +53,8 @@ import {
 	SummarizeInternalFn,
 	channelsTreeName,
 	IInboundSignalMessage,
+	type IPendingMessagesState,
+	type IRuntimeMessageCollection,
 } from "@fluidframework/runtime-definitions/internal";
 import {
 	addBlobToSummary,
@@ -313,7 +315,7 @@ export abstract class FluidDataStoreContext
 	 * Returns the count of pending messages that are stored until the data store is realized.
 	 */
 	public get pendingCount(): number {
-		return this.pending?.length ?? 0;
+		return this.pendingMessagesState?.pendingCount ?? 0;
 	}
 
 	protected registry: IFluidDataStoreRegistry | undefined;
@@ -321,7 +323,11 @@ export abstract class FluidDataStoreContext
 	protected detachedRuntimeCreation = false;
 	protected channel: IFluidDataStoreChannel | undefined;
 	private loaded = false;
-	protected pending: ISequencedDocumentMessage[] | undefined = [];
+	/** Tracks the messages for this data store that are sent while it's not loaded */
+	private pendingMessagesState: IPendingMessagesState | undefined = {
+		messageCollections: [],
+		pendingCount: 0,
+	};
 	protected channelP: Promise<IFluidDataStoreChannel> | undefined;
 	protected _baseSnapshot: ISnapshotTree | undefined;
 	protected _attachState: AttachState;
@@ -561,25 +567,61 @@ export abstract class FluidDataStoreContext
 		this.channel!.setConnectionState(connected, clientId);
 	}
 
-	public process(
-		message: ISequencedDocumentMessage,
-		local: boolean,
-		localOpMetadata: unknown,
-	): void {
-		const safeTelemetryProps = extractSafePropertiesFromMessage(message);
-		// On op process, tombstone error is logged in garbage collector. So, set "checkTombstone" to false when calling
+	/**
+	 * back-compat ADO 21575: This is temporary and will be removed once the compat requirement across Runtime and
+	 * Datastore boundary is satisfied.
+	 * Process the messages to maintain backwards compatibility. The `processMessages` function is added to
+	 * IFluidDataStoreChannel in 2.5.0. For channels before that, call `process` for each message.
+	 */
+	private processMessagesCompat(
+		channel: IFluidDataStoreChannel,
+		messageCollection: IRuntimeMessageCollection,
+	) {
+		if (channel.processMessages !== undefined) {
+			channel.processMessages(messageCollection);
+		} else {
+			const { envelope, messagesContent, local } = messageCollection;
+			for (const { contents, localOpMetadata, clientSequenceNumber } of messagesContent) {
+				channel.process(
+					{ ...envelope, contents, clientSequenceNumber },
+					local,
+					localOpMetadata,
+				);
+			}
+		}
+	}
+
+	/**
+	 * Process messages for this data store. The messages here are contiguous messages for this data store in a batch.
+	 * @param messageCollection - The collection of messages to process.
+	 */
+	public processMessages(messageCollection: IRuntimeMessageCollection): void {
+		const { envelope, messagesContent, local } = messageCollection;
+		const safeTelemetryProps = extractSafePropertiesFromMessage(envelope);
+		// Tombstone error is logged in garbage collector. So, set "checkTombstone" to false when calling
 		// "verifyNotClosed" which logs tombstone errors.
 		this.verifyNotClosed("process", false /* checkTombstone */, safeTelemetryProps);
 
-		this.summarizerNode.recordChange(message);
+		this.summarizerNode.recordChange(envelope as ISequencedDocumentMessage);
 
 		if (this.loaded) {
-			return this.channel?.process(message, local, localOpMetadata);
+			assert(this.channel !== undefined, 0xa68 /* Channel is not loaded */);
+			this.processMessagesCompat(this.channel, messageCollection);
 		} else {
 			assert(!local, 0x142 /* "local store channel is not loaded" */);
-			assert(this.pending !== undefined, 0x23d /* "pending is undefined" */);
-			this.pending.push(message);
-			this.thresholdOpsCounter.sendIfMultiple("StorePendingOps", this.pending.length);
+			assert(
+				this.pendingMessagesState !== undefined,
+				0xa69 /* pending messages queue is undefined */,
+			);
+			this.pendingMessagesState.messageCollections.push({
+				...messageCollection,
+				messagesContent: Array.from(messagesContent),
+			});
+			this.pendingMessagesState.pendingCount += messagesContent.length;
+			this.thresholdOpsCounter.sendIfMultiple(
+				"StorePendingOps",
+				this.pendingMessagesState.pendingCount,
+			);
 		}
 	}
 
@@ -788,20 +830,21 @@ export abstract class FluidDataStoreContext
 	}
 
 	protected processPendingOps(channel: IFluidDataStoreChannel) {
-		// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-		const pending = this.pending!;
+		const baseSequenceNumber = this.baseSnapshotSequenceNumber ?? -1;
 
-		// Apply all pending ops
-		for (const op of pending) {
+		assert(
+			this.pendingMessagesState !== undefined,
+			0xa6a /* pending messages queue is undefined */,
+		);
+		for (const messageCollection of this.pendingMessagesState.messageCollections) {
 			// Only process ops whose seq number is greater than snapshot sequence number from which it loaded.
-			const seqNumber = this.baseSnapshotSequenceNumber ?? -1;
-			if (op.sequenceNumber > seqNumber) {
-				channel.process(op, false, undefined /* localOpMetadata */);
+			if (messageCollection.envelope.sequenceNumber > baseSequenceNumber) {
+				this.processMessagesCompat(channel, messageCollection);
 			}
 		}
-		this.pending = undefined;
 
-		this.thresholdOpsCounter.send("ProcessPendingOps", pending.length);
+		this.thresholdOpsCounter.send("ProcessPendingOps", this.pendingMessagesState.pendingCount);
+		this.pendingMessagesState = undefined;
 	}
 
 	protected completeBindingRuntime(channel: IFluidDataStoreChannel) {

package/src/dataStoreContexts.ts

@@ -7,7 +7,6 @@ import { IDisposable, ITelemetryBaseLogger } from "@fluidframework/core-interfac
 import { assert, Deferred, Lazy } from "@fluidframework/core-utils/internal";
 import {
 	ITelemetryLoggerExt,
-	PerformanceEvent,
 	createChildLogger,
 } from "@fluidframework/telemetry-utils/internal";
 
@@ -41,7 +40,7 @@ export class DataStoreContexts
 				.catch((contextError) => {
 					this._logger.sendErrorEvent(
 						{
-							eventName: "DisposeError",
+							eventName: "FluidDataStoreContextDisposeError",
 							fluidDataStoreId,
 						},
 						contextError,
@@ -53,10 +52,7 @@ export class DataStoreContexts
 	private readonly _logger: ITelemetryLoggerExt;
 
 	constructor(baseLogger: ITelemetryBaseLogger) {
-		this._logger = createChildLogger({
-			namespace: "FluidDataStoreContexts",
-			logger: baseLogger,
-		});
+		this._logger = createChildLogger({ logger: baseLogger });
 	}
 
 	[Symbol.iterator](): Iterator<[string, FluidDataStoreContext]> {
@@ -144,20 +140,11 @@ export class DataStoreContexts
 	): Promise<FluidDataStoreContext | undefined> {
 		const deferredContext = this.ensureDeferred(id);
 
-		const existing = deferredContext.isCompleted;
-		return PerformanceEvent.timedExecAsync(
-			this._logger,
-			{
-				eventName: "GetBoundOrRemoted",
-				wait,
-				existing,
-			},
-			async () => (!wait && !existing ? undefined : deferredContext.promise),
-			{
-				start: true,
-				end: true,
-			},
-		);
+		if (!wait && !deferredContext.isCompleted) {
+			return undefined;
+		}
+
+		return deferredContext.promise;
 	}
 
 	private ensureDeferred(id: string): Deferred<FluidDataStoreContext> {

package/src/gc/garbageCollection.ts

@@ -861,44 +861,44 @@ export class GarbageCollector implements IGarbageCollector {
 	}
 
 	/**
-	 * Process a GC message.
-	 * @param message - The GC message from the container runtime.
-	 * @param messageTimestampMs - The timestamp of the message.
+	 * Process GC messages.
+	 * @param messageContents - The contents of the messages.
+	 * @param messageTimestampMs - The timestamp of the messages.
 	 * @param local - Whether it was send by this client.
 	 */
-	public processMessage(
-		message: ContainerRuntimeGCMessage,
+	public processMessages(
+		messageContents: GarbageCollectionMessage[],
 		messageTimestampMs: number,
 		local: boolean,
 	) {
-		const gcMessageType = message.contents.type;
-		switch (gcMessageType) {
-			case GarbageCollectionMessageType.Sweep: {
-				// Delete the nodes whose ids are present in the contents.
-				this.deleteSweepReadyNodes(message.contents.deletedNodeIds);
-				break;
-			}
-			case GarbageCollectionMessageType.TombstoneLoaded: {
-				// Mark the node as referenced to ensure it isn't Swept
-				const tombstonedNodePath = message.contents.nodePath;
-				this.addedOutboundReference(
-					"/",
-					tombstonedNodePath,
-					messageTimestampMs,
-					true /* autorecovery */,
-				);
-
-				// In case the cause of the TombstoneLoaded event is incorrect GC Data (i.e. the object is actually reachable),
-				// do fullGC on the next run to get a chance to repair (in the likely case the bug is not deterministic)
-				this.summaryStateTracker.autoRecovery.requestFullGCOnNextRun();
+		for (const gcMessage of messageContents) {
+			const gcMessageType = gcMessage.type;
+			switch (gcMessageType) {
+				case GarbageCollectionMessageType.Sweep: {
+					// Delete the nodes whose ids are present in the contents.
+					this.deleteSweepReadyNodes(gcMessage.deletedNodeIds);
+					break;
+				}
+				case GarbageCollectionMessageType.TombstoneLoaded: {
+					// Mark the node as referenced to ensure it isn't Swept
+					const tombstonedNodePath = gcMessage.nodePath;
+					this.addedOutboundReference(
+						"/",
+						tombstonedNodePath,
+						messageTimestampMs,
+						true /* autorecovery */,
+					);
 
-				break;
-			}
-			default: {
-				throw DataProcessingError.create(
-					`Garbage collection message of unknown type ${gcMessageType}`,
-					"processMessage",
-				);
+					// In case the cause of the TombstoneLoaded event is incorrect GC Data (i.e. the object is actually reachable),
+					// do fullGC on the next run to get a chance to repair (in the likely case the bug is not deterministic)
+					this.summaryStateTracker.autoRecovery.requestFullGCOnNextRun();
+					break;
+				}
+				default:
+					throw DataProcessingError.create(
+						`Garbage collection message of unknown type ${gcMessageType}`,
+						"processMessage",
+					);
 			}
 		}
 	}

package/src/gc/gcDefinitions.ts

@@ -352,9 +352,9 @@ export interface IGarbageCollector {
 		timestampMs: number,
 		autorecovery?: true,
 	): void;
-	/** Called to process a garbage collection message. */
-	processMessage(
-		message: ContainerRuntimeGCMessage,
+	/** Called to process garbage collection messages */
+	processMessages(
+		messageContents: GarbageCollectionMessage[],
 		messageTimestampMs: number,
 		local: boolean,
 	): void;

package/src/opLifecycle/outbox.ts

@@ -145,6 +145,18 @@ export class Outbox {
 		return this.mainBatch.length + this.blobAttachBatch.length + this.idAllocationBatch.length;
 	}
 
+	public get mainBatchMessageCount(): number {
+		return this.mainBatch.length;
+	}
+
+	public get blobAttachBatchMessageCount(): number {
+		return this.blobAttachBatch.length;
+	}
+
+	public get idAllocationBatchMessageCount(): number {
+		return this.idAllocationBatch.length;
+	}
+
 	public get isEmpty(): boolean {
 		return this.messageCount === 0;
 	}