@fluidframework/container-runtime 2.4.0 → 2.5.0

package/src/channelCollection.ts

@@ -39,11 +39,13 @@ import {
 	IFluidDataStoreRegistry,
 	IFluidParentContext,
 	ISummarizeResult,
-	InboundAttachMessage,
 	NamedFluidDataStoreRegistryEntries,
 	channelsTreeName,
 	IInboundSignalMessage,
 	gcDataBlobKey,
+	type IRuntimeMessagesContent,
+	type InboundAttachMessage,
+	type IRuntimeMessageCollection,
 } from "@fluidframework/runtime-definitions/internal";
 import {
 	GCDataBuilder,
@@ -391,115 +393,119 @@ export class ChannelCollection implements IFluidDataStoreChannel, IDisposable {
 		this.parentContext.makeLocallyVisible();
 	}
 
-	private processAttachMessage(message: ISequencedDocumentMessage, local: boolean) {
-		const attachMessage = message.contents as InboundAttachMessage;
-
-		// We need to process the GC Data for both local and remote attach messages
-		const foundGCData = processAttachMessageGCData(
-			attachMessage.snapshot,
-			(nodeId, toPath) => {
-				// nodeId is the relative path under the node being attached. Always starts with "/", but no trailing "/" after an id
-				const fromPath = `/${attachMessage.id}${nodeId === "/" ? "" : nodeId}`;
-				this.parentContext.addedGCOutboundRoute(fromPath, toPath, message.timestamp);
-			},
-		);
-
-		// Only log once per container to avoid noise/cost.
-		// Allows longitudinal tracking of various state (e.g. foundGCData), and some sampled details
-		if (this.shouldSendAttachLog) {
-			this.shouldSendAttachLog = false;
-			this.mc.logger.sendTelemetryEvent({
-				eventName: "dataStoreAttachMessage_sampled",
-				...tagCodeArtifacts({ id: attachMessage.id, pkg: attachMessage.type }),
-				details: {
-					local,
-					snapshot: !!attachMessage.snapshot,
-					foundGCData,
+	private processAttachMessages(messageCollection: IRuntimeMessageCollection) {
+		const { envelope, messagesContent, local } = messageCollection;
+		for (const { contents } of messagesContent) {
+			const attachMessage = contents as InboundAttachMessage;
+			// We need to process the GC Data for both local and remote attach messages
+			const foundGCData = processAttachMessageGCData(
+				attachMessage.snapshot,
+				(nodeId, toPath) => {
+					// nodeId is the relative path under the node being attached. Always starts with "/", but no trailing "/" after an id
+					const fromPath = `/${attachMessage.id}${nodeId === "/" ? "" : nodeId}`;
+					this.parentContext.addedGCOutboundRoute(fromPath, toPath, envelope.timestamp);
 				},
-				...extractSafePropertiesFromMessage(message),
-			});
-		}
-
-		// The local object has already been attached
-		if (local) {
-			assert(
-				this.pendingAttach.has(attachMessage.id),
-				0x15e /* "Local object does not have matching attach message id" */,
 			);
-			this.contexts.get(attachMessage.id)?.setAttachState(AttachState.Attached);
-			this.pendingAttach.delete(attachMessage.id);
-			return;
-		}
 
-		// If a non-local operation then go and create the object, otherwise mark it as officially attached.
-		if (this.alreadyProcessed(attachMessage.id)) {
-			// TODO: dataStoreId may require a different tag from PackageData #7488
-			const error = new DataCorruptionError(
-				// pre-0.58 error message: duplicateDataStoreCreatedWithExistingId
-				"Duplicate DataStore created with existing id",
-				{
-					...extractSafePropertiesFromMessage(message),
-					...tagCodeArtifacts({ dataStoreId: attachMessage.id }),
-				},
-			);
-			throw error;
-		}
+			// Only log once per container to avoid noise/cost.
+			// Allows longitudinal tracking of various state (e.g. foundGCData), and some sampled details
+			if (this.shouldSendAttachLog) {
+				this.shouldSendAttachLog = false;
+				this.mc.logger.sendTelemetryEvent({
+					eventName: "dataStoreAttachMessage_sampled",
+					...tagCodeArtifacts({ id: attachMessage.id, pkg: attachMessage.type }),
+					details: {
+						local,
+						snapshot: !!attachMessage.snapshot,
+						foundGCData,
+					},
+					...extractSafePropertiesFromMessage(envelope),
+				});
+			}
 
-		const flatAttachBlobs = new Map<string, ArrayBufferLike>();
-		let snapshot: ISnapshotTree | ISnapshot | undefined;
-		if (attachMessage.snapshot) {
-			snapshot = buildSnapshotTree(attachMessage.snapshot.entries, flatAttachBlobs);
-			if (isInstanceOfISnapshot(this.baseSnapshot)) {
-				snapshot = { ...this.baseSnapshot, snapshotTree: snapshot };
+			// The local object has already been attached
+			if (local) {
+				assert(
+					this.pendingAttach.has(attachMessage.id),
+					0x15e /* "Local object does not have matching attach message id" */,
+				);
+				this.contexts.get(attachMessage.id)?.setAttachState(AttachState.Attached);
+				this.pendingAttach.delete(attachMessage.id);
+				continue;
 			}
-		}
 
-		// Include the type of attach message which is the pkg of the store to be
-		// used by RemoteFluidDataStoreContext in case it is not in the snapshot.
-		const pkg = [attachMessage.type];
-		const remoteFluidDataStoreContext = new RemoteFluidDataStoreContext({
-			id: attachMessage.id,
-			snapshot,
-			parentContext: this.wrapContextForInnerChannel(attachMessage.id),
-			storage: new StorageServiceWithAttachBlobs(this.parentContext.storage, flatAttachBlobs),
-			scope: this.parentContext.scope,
-			loadingGroupId: attachMessage.snapshot?.groupId,
-			createSummarizerNodeFn: this.parentContext.getCreateChildSummarizerNodeFn(
-				attachMessage.id,
-				{
-					type: CreateSummarizerNodeSource.FromAttach,
-					sequenceNumber: message.sequenceNumber,
-					snapshot: attachMessage.snapshot ?? {
-						entries: [createAttributesBlob(pkg, true /* isRootDataStore */)],
+			// If a non-local operation then go and create the object, otherwise mark it as officially attached.
+			if (this.alreadyProcessed(attachMessage.id)) {
+				// TODO: dataStoreId may require a different tag from PackageData #7488
+				const error = new DataCorruptionError(
+					// pre-0.58 error message: duplicateDataStoreCreatedWithExistingId
+					"Duplicate DataStore created with existing id",
+					{
+						...extractSafePropertiesFromMessage(envelope),
+						...tagCodeArtifacts({ dataStoreId: attachMessage.id }),
 					},
-				},
-			),
-			pkg,
-		});
+				);
+				throw error;
+			}
 
-		this.contexts.addBoundOrRemoted(remoteFluidDataStoreContext);
-	}
+			const flatAttachBlobs = new Map<string, ArrayBufferLike>();
+			let snapshot: ISnapshotTree | ISnapshot | undefined;
+			if (attachMessage.snapshot) {
+				snapshot = buildSnapshotTree(attachMessage.snapshot.entries, flatAttachBlobs);
+				if (isInstanceOfISnapshot(this.baseSnapshot)) {
+					snapshot = { ...this.baseSnapshot, snapshotTree: snapshot };
+				}
+			}
 
-	private processAliasMessage(
-		message: ISequencedDocumentMessage,
-		localOpMetadata: unknown,
-		local: boolean,
-	): void {
-		const aliasMessage = message.contents as IDataStoreAliasMessage;
-		if (!isDataStoreAliasMessage(aliasMessage)) {
-			throw new DataCorruptionError("malformedDataStoreAliasMessage", {
-				...extractSafePropertiesFromMessage(message),
+			// Include the type of attach message which is the pkg of the store to be
+			// used by RemoteFluidDataStoreContext in case it is not in the snapshot.
+			const pkg = [attachMessage.type];
+			const remoteFluidDataStoreContext = new RemoteFluidDataStoreContext({
+				id: attachMessage.id,
+				snapshot,
+				parentContext: this.wrapContextForInnerChannel(attachMessage.id),
+				storage: new StorageServiceWithAttachBlobs(
+					this.parentContext.storage,
+					flatAttachBlobs,
+				),
+				scope: this.parentContext.scope,
+				loadingGroupId: attachMessage.snapshot?.groupId,
+				createSummarizerNodeFn: this.parentContext.getCreateChildSummarizerNodeFn(
+					attachMessage.id,
+					{
+						type: CreateSummarizerNodeSource.FromAttach,
+						sequenceNumber: envelope.sequenceNumber,
+						snapshot: attachMessage.snapshot ?? {
+							entries: [createAttributesBlob(pkg, true /* isRootDataStore */)],
+						},
+					},
+				),
+				pkg,
 			});
+
+			this.contexts.addBoundOrRemoted(remoteFluidDataStoreContext);
 		}
+	}
 
-		const resolve = localOpMetadata as PendingAliasResolve;
-		const aliasResult = this.processAliasMessageCore(
-			aliasMessage.internalId,
-			aliasMessage.alias,
-			message.timestamp,
-		);
-		if (local) {
-			resolve(aliasResult);
+	private processAliasMessages(messageCollection: IRuntimeMessageCollection): void {
+		const { envelope, messagesContent, local } = messageCollection;
+		for (const { contents, localOpMetadata } of messagesContent) {
+			const aliasMessage = contents as IDataStoreAliasMessage;
+			if (!isDataStoreAliasMessage(aliasMessage)) {
+				throw new DataCorruptionError("malformedDataStoreAliasMessage", {
+					...extractSafePropertiesFromMessage(envelope),
+				});
+			}
+
+			const resolve = localOpMetadata as PendingAliasResolve;
+			const aliasResult = this.processAliasMessageCore(
+				aliasMessage.internalId,
+				aliasMessage.alias,
+				envelope.timestamp,
+			);
+			if (local) {
+				resolve(aliasResult);
+			}
 		}
 	}
 
@@ -819,84 +825,145 @@ export class ChannelCollection implements IFluidDataStoreChannel, IDisposable {
 		}
 	}
 
-	public process(
-		message: ISequencedDocumentMessage,
-		local: boolean,
-		localMessageMetadata: unknown,
-	) {
-		switch (message.type) {
+	/**
+	 * Process messages for this channel collection. The messages here are contiguous messages in a batch.
+	 * @param messageCollection - The collection of messages to process.
+	 */
+	public processMessages(messageCollection: IRuntimeMessageCollection): void {
+		switch (messageCollection.envelope.type) {
+			case ContainerMessageType.FluidDataStoreOp:
+				this.processChannelMessages(messageCollection);
+				break;
 			case ContainerMessageType.Attach:
-				this.processAttachMessage(message, local);
-				return;
+				this.processAttachMessages(messageCollection);
+				break;
 			case ContainerMessageType.Alias:
-				this.processAliasMessage(message, localMessageMetadata, local);
-				return;
-			case ContainerMessageType.FluidDataStoreOp: {
-				const envelope = message.contents as IEnvelope;
-				const innerContents = envelope.contents as FluidDataStoreMessage;
-				const transformed = {
-					...message,
-					type: innerContents.type,
-					contents: innerContents.content,
-				};
-
-				this.processChannelOp(envelope.address, transformed, local, localMessageMetadata);
-
-				// Notify GC of any outbound references that were added by this op.
-				detectOutboundReferences(
-					envelope.address,
-					transformed.contents,
-					(fromPath: string, toPath: string) =>
-						this.parentContext.addedGCOutboundRoute(fromPath, toPath, message.timestamp),
-				);
+				this.processAliasMessages(messageCollection);
 				break;
-			}
 			default:
 				assert(false, 0x8e9 /* unreached */);
 		}
 	}
 
-	protected processChannelOp(
-		address: string,
+	/**
+	 * This is still here for back-compat purposes because channel collection implements
+	 * IFluidDataStoreChannel. Once it is removed from the interface, this method can be removed.
+	 * Container runtime calls `processMessages` instead.
+	 */
+	public process(
 		message: ISequencedDocumentMessage,
 		local: boolean,
-		localMessageMetadata: unknown,
+		localOpMetadata: unknown,
 	) {
-		const context = this.contexts.get(address);
-
-		// If the data store has been deleted, log an error and ignore this message. This helps prevent document
-		// corruption in case a deleted data store accidentally submitted an op.
-		if (this.checkAndLogIfDeleted(address, context, "Changed", "processFluidDataStoreOp")) {
-			return;
-		}
-
-		if (context === undefined) {
-			// Former assert 0x162
-			throw DataProcessingError.create(
-				"No context for op",
-				"processFluidDataStoreOp",
-				message,
+		this.processMessages({
+			envelope: message,
+			messagesContent: [
 				{
-					local,
-					messageDetails: JSON.stringify({
-						type: message.type,
-						contentType: typeof message.contents,
-					}),
-					...tagCodeArtifacts({ address }),
+					contents: message.contents,
+					localOpMetadata,
+					clientSequenceNumber: message.clientSequenceNumber,
 				},
+			],
+			local,
+		});
+	}
+
+	/**
+	 * Process channel messages. The messages here are contiguous channel type messages in a batch. Bunch
+	 * of contiguous messages for a data store should be sent to it together.
+	 * @param messageCollection - The collection of messages to process.
+	 */
+	private processChannelMessages(messageCollection: IRuntimeMessageCollection): void {
+		const { messagesContent, local } = messageCollection;
+		let currentMessageState: { address: string; type: string } | undefined;
+		let currentMessagesContent: IRuntimeMessagesContent[] = [];
+
+		// Helper that sends the current bunch of messages to the data store. It validates that the data stores exists.
+		const sendBunchedMessages = () => {
+			// Current message state will be undefined for the first message in the list.
+			if (currentMessageState === undefined) {
+				return;
+			}
+			const currentContext = this.contexts.get(currentMessageState.address);
+			assert(!!currentContext, 0xa66 /* Context not found */);
+
+			currentContext.processMessages({
+				envelope: { ...messageCollection.envelope, type: currentMessageState.type },
+				messagesContent: currentMessagesContent,
+				local,
+			});
+			currentMessagesContent = [];
+		};
+
+		/**
+		 * Bunch contiguous messages for the same data store and send them together.
+		 * This is an optimization mainly for DDSes, where it can process a bunch of ops together. DDSes
+		 * like merge tree or shared tree can process ops more efficiently when they are bunched together.
+		 */
+		for (const { contents, ...restOfMessagesContent } of messagesContent) {
+			const contentsEnvelope = contents as IEnvelope;
+			const address = contentsEnvelope.address;
+			const context = this.contexts.get(address);
+
+			// If the data store has been deleted, log an error and ignore this message. This helps prevent document
+			// corruption in case a deleted data store accidentally submitted an op.
+			if (this.checkAndLogIfDeleted(address, context, "Changed", "processFluidDataStoreOp")) {
+				continue;
+			}
+
+			if (context === undefined) {
+				// Former assert 0x162
+				throw DataProcessingError.create(
+					"No context for op",
+					"processFluidDataStoreOp",
+					messageCollection.envelope as ISequencedDocumentMessage,
+					{
+						local,
+						messageDetails: JSON.stringify({
+							type: messageCollection.envelope.type,
+							contentType: typeof contents,
+						}),
+						...tagCodeArtifacts({ address }),
+					},
+				);
+			}
+
+			const { type: contextType, content: contextContents } =
+				contentsEnvelope.contents as FluidDataStoreMessage;
+			// If the address or type of the message changes while processing the message, send the current bunch.
+			if (
+				currentMessageState?.address !== address ||
+				currentMessageState?.type !== contextType
+			) {
+				sendBunchedMessages();
+			}
+			currentMessagesContent.push({
+				contents: contextContents,
+				...restOfMessagesContent,
+			});
+			currentMessageState = { address, type: contextType };
+
+			// Notify that a GC node for the data store changed. This is used to detect if a deleted data store is
+			// being used.
+			this.gcNodeUpdated({
+				node: { type: "DataStore", path: `/${address}` },
+				reason: "Changed",
+				timestampMs: messageCollection.envelope.timestamp,
+				packagePath: context.isLoaded ? context.packagePath : undefined,
+			});
+
+			detectOutboundReferences(address, contextContents, (fromPath: string, toPath: string) =>
+				this.parentContext.addedGCOutboundRoute(
+					fromPath,
+					toPath,
+					messageCollection.envelope.timestamp,
+				),
 			);
 		}
 
-		context.process(message, local, localMessageMetadata);
-
-		// Notify that a GC node for the data store changed. This is used to detect if a deleted data store is
-		// being used.
-		this.gcNodeUpdated({
-			node: { type: "DataStore", path: `/${address}` },
-			reason: "Changed",
-			timestampMs: message.timestamp,
-			packagePath: context.isLoaded ? context.packagePath : undefined,
-		});
+		// Process the last bunch of messages, if any. Note that there may not be any messages in case all of them are
+		// ignored because the data store is deleted.
+		sendBunchedMessages();
 	}
 
 	private async getDataStore(