@fluidframework/azure-end-to-end-tests 2.70.0-361248 → 2.70.0

package/src/test/multiprocess/childClient.tool.ts

@@ -14,6 +14,7 @@ import {
 } from "@fluidframework/azure-client";
 import { AttachState } from "@fluidframework/container-definitions";
 import { ConnectionState } from "@fluidframework/container-loader";
+import type { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
 import { LogLevel } from "@fluidframework/core-interfaces";
 import type { ScopeType } from "@fluidframework/driver-definitions/legacy";
 import type { ContainerSchema, IFluidContainer } from "@fluidframework/fluid-static";
@@ -32,10 +33,13 @@ import { timeoutPromise } from "@fluidframework/test-utils/internal";
 import { createAzureTokenProvider } from "../AzureTokenFactory.js";
 import { TestDataObject } from "../TestDataObject.js";
 
-import type { MessageFromChild, MessageToChild, UserIdAndName } from "./messageTypes.js";
+import type {
+	MessageFromChild as MessageToParent,
+	MessageToChild as MessageFromParent,
+	UserIdAndName,
+	EventEntry,
+} from "./messageTypes.js";
 
-type MessageFromParent = MessageToChild;
-type MessageToParent = Required<MessageFromChild>;
 const connectTimeoutMs = 10_000;
 // Identifier given to child process
 const process_id = process.argv[2];
@@ -50,41 +54,57 @@ if (useAzure && endPoint === undefined) {
 	throw new Error("Azure Fluid Relay service endpoint is missing");
 }
 
+const containerSchema = {
+	initialObjects: {
+		// A DataObject is added as otherwise fluid-static complains "Container cannot be initialized without any DataTypes"
+		_unused: TestDataObject,
+	},
+} as const satisfies ContainerSchema;
+
+function telemetryEventInterestLevel(eventName: string): "none" | "basic" | "details" {
+	if (eventName.includes(":Signal") || eventName.includes(":Join")) {
+		return "details";
+	} else if (eventName.includes(":Container:") || eventName.includes(":Presence:")) {
+		return "basic";
+	}
+	return "none";
+}
+
 function selectiveVerboseLog(event: ITelemetryBaseEvent, logLevel?: LogLevel): void {
-	if (event.eventName.includes(":Signal") || event.eventName.includes(":Join")) {
-		console.log(`[${process_id}] [${logLevel ?? LogLevel.default}]`, {
-			eventName: event.eventName,
-			details: event.details,
-			containerConnectionState: event.containerConnectionState,
-		});
-	} else if (
-		event.eventName.includes(":Container:") ||
-		event.eventName.includes(":Presence:")
-	) {
-		console.log(`[${process_id}] [${logLevel ?? LogLevel.default}]`, {
-			eventName: event.eventName,
-			containerConnectionState: event.containerConnectionState,
-		});
+	const interest = telemetryEventInterestLevel(event.eventName);
+	if (interest === "none") {
+		return;
 	}
+	const content: Record<string, unknown> = {
+		eventName: event.eventName,
+		containerConnectionState: event.containerConnectionState,
+	};
+	if (interest === "details") {
+		content.details = event.details;
+	}
+	console.log(`[${process_id}] [${logLevel ?? LogLevel.default}]`, content);
 }
 
 /**
- * Get or create a Fluid container with Presence in initialObjects.
+ * Get or create a Fluid container.
  */
-const getOrCreatePresenceContainer = async (
-	id: string | undefined,
-	user: UserIdAndName,
-	scopes?: ScopeType[],
-	createScopes?: ScopeType[],
-): Promise<{
-	container: IFluidContainer;
-	presence: Presence;
+const getOrCreateContainer = async (params: {
+	logger: ITelemetryBaseLogger;
+	onDisconnected: () => void;
+	containerId?: string;
+	user: UserIdAndName;
+	scopes?: ScopeType[];
+	createScopes?: ScopeType[];
+}): Promise<{
+	container: IFluidContainer<typeof containerSchema>;
 	services: AzureContainerServices;
 	client: AzureClient;
 	containerId: string;
+	connected: Promise<void>;
 }> => {
-	let container: IFluidContainer;
-	let containerId: string;
+	let container: IFluidContainer<typeof containerSchema>;
+	let { containerId } = params;
+	const { logger, onDisconnected, user, scopes, createScopes } = params;
 	const connectionProps: AzureRemoteConnectionConfig | AzureLocalConnectionConfig = useAzure
 		? {
 				tenantId,
@@ -104,46 +124,40 @@ const getOrCreatePresenceContainer = async (
 			};
 	const client = new AzureClient({
 		connection: connectionProps,
-		logger: {
-			send: verbosity.includes("telem") ? selectiveVerboseLog : () => {},
-		},
+		logger,
 	});
-	const schema: ContainerSchema = {
-		initialObjects: {
-			// A DataObject is added as otherwise fluid-static complains "Container cannot be initialized without any DataTypes"
-			_unused: TestDataObject,
-		},
-	};
 	let services: AzureContainerServices;
-	if (id === undefined) {
-		({ container, services } = await client.createContainer(schema, "2"));
+	if (containerId === undefined) {
+		({ container, services } = await client.createContainer(containerSchema, "2"));
 		containerId = await container.attach();
 	} else {
-		containerId = id;
-		({ container, services } = await client.getContainer(containerId, schema, "2"));
-	}
-	// wait for 'ConnectionState.Connected' so we return with client connected to container
-	if (container.connectionState !== ConnectionState.Connected) {
-		await timeoutPromise((resolve) => container.once("connected", () => resolve()), {
-			durationMs: connectTimeoutMs,
-			errorMsg: "container connect() timeout",
-		});
+		({ container, services } = await client.getContainer(containerId, containerSchema, "2"));
 	}
+	container.on("disconnected", onDisconnected);
+
+	const connected =
+		container.connectionState === ConnectionState.Connected
+			? Promise.resolve()
+			: timeoutPromise((resolve) => container.once("connected", () => resolve()), {
+					durationMs: connectTimeoutMs,
+					errorMsg: "container connect() timeout",
+				});
+
 	assert.strictEqual(
 		container.attachState,
 		AttachState.Attached,
 		"Container is not attached after attach is called",
 	);
 
-	const presence = getPresence(container);
 	return {
 		client,
 		container,
-		presence,
 		services,
 		containerId,
+		connected,
 	};
 };
+
 function createSendFunction(): (msg: MessageToParent) => void {
 	if (process.send) {
 		const sendFn = process.send.bind(process);
@@ -160,23 +174,6 @@ function createSendFunction(): (msg: MessageToParent) => void {
 
 const send = createSendFunction();
 
-function sendAttendeeConnected(attendee: Attendee): void {
-	send({
-		event: "attendeeConnected",
-		attendeeId: attendee.attendeeId,
-	});
-}
-function sendAttendeeDisconnected(attendee: Attendee): void {
-	send({
-		event: "attendeeDisconnected",
-		attendeeId: attendee.attendeeId,
-	});
-}
-
-function isConnected(container: IFluidContainer | undefined): boolean {
-	return container !== undefined && container.connectionState === ConnectionState.Connected;
-}
-
 function isStringOrNumberRecord(value: unknown): value is Record<string, string | number> {
 	if (value === null || typeof value !== "object" || Array.isArray(value)) {
 		return false;
@@ -216,17 +213,79 @@ type WorkspaceSchema = {
 const WorkspaceSchema: WorkspaceSchema = {};
 
 class MessageHandler {
-	public presence: Presence | undefined;
-	public container: IFluidContainer | undefined;
-	public containerId: string | undefined;
+	private readonly log: EventEntry[] = [];
+	private msgQueue: undefined | Exclude<MessageFromParent, { command: "ping" | "connect" }>[];
+	private container: IFluidContainer | undefined;
+	private presence: Presence | undefined;
 	private readonly workspaces = new Map<string, StatesWorkspace<WorkspaceSchema>>();
 
+	private send(msg: MessageToParent): void {
+		this.log.push({
+			timestamp: Date.now(),
+			agentId: process_id,
+			eventCategory: "messageSent",
+			eventName: msg.event,
+			details:
+				msg.event === "debugReportComplete" && msg.log
+					? JSON.stringify({ logLength: msg.log.length })
+					: JSON.stringify(msg),
+		});
+		send(msg);
+	}
+
+	private readonly sendAttendeeConnected = (attendee: Attendee): void => {
+		this.send({
+			event: "attendeeConnected",
+			attendeeId: attendee.attendeeId,
+		});
+	};
+	private readonly sendAttendeeDisconnected = (attendee: Attendee): void => {
+		this.send({
+			event: "attendeeDisconnected",
+			attendeeId: attendee.attendeeId,
+		});
+	};
+
+	private readonly logger: ITelemetryBaseLogger = {
+		send: (event: ITelemetryBaseEvent, logLevel?: LogLevel) => {
+			// Special case unexpected telemetry event
+			if (event.eventName.endsWith(":JoinResponseWhenAlone")) {
+				this.send({
+					event: "error",
+					error: `Unexpected ClientJoin response. Details: ${JSON.stringify(event.details)}`,
+				});
+				// Keep going
+			}
+
+			const interest = telemetryEventInterestLevel(event.eventName);
+			if (interest === "none") {
+				return;
+			}
+			this.log.push({
+				timestamp: Date.now(),
+				agentId: process_id,
+				eventCategory: "telemetry",
+				eventName: event.eventName,
+				details:
+					typeof event.details === "string" ? event.details : JSON.stringify(event.details),
+			});
+			if (verbosity.includes("telem")) {
+				selectiveVerboseLog(event, logLevel);
+			}
+		},
+	};
+
+	private readonly onDisconnected = (): void => {
+		// Test state is a bit fragile and does not account for reconnections.
+		this.send({ event: "error", error: `${process_id}: Container disconnected` });
+	};
+
 	private registerWorkspace(
 		workspaceId: string,
 		options: { latest?: boolean; latestMap?: boolean },
 	): void {
 		if (!this.presence) {
-			send({ event: "error", error: `${process_id} is not connected to presence` });
+			this.send({ event: "error", error: `${process_id} is not connected to presence` });
 			return;
 		}
 		const { latest, latestMap } = options;
@@ -244,7 +303,7 @@ class MessageHandler {
 			// TODO: AB#47518
 			const latestState = workspace.states.latest as LatestRaw<{ value: string }>;
 			latestState.events.on("remoteUpdated", (update) => {
-				send({
+				this.send({
 					event: "latestValueUpdated",
 					workspaceId,
 					attendeeId: update.attendee.attendeeId,
@@ -252,7 +311,7 @@ class MessageHandler {
 				});
 			});
 			for (const remote of latestState.getRemotes()) {
-				send({
+				this.send({
 					event: "latestValueUpdated",
 					workspaceId,
 					attendeeId: remote.attendee.attendeeId,
@@ -276,7 +335,7 @@ class MessageHandler {
 			>;
 			latestMapState.events.on("remoteUpdated", (update) => {
 				for (const [key, valueWithMetadata] of update.items) {
-					send({
+					this.send({
 						event: "latestMapValueUpdated",
 						workspaceId,
 						attendeeId: update.attendee.attendeeId,
@@ -287,7 +346,7 @@ class MessageHandler {
 			});
 			for (const remote of latestMapState.getRemotes()) {
 				for (const [key, valueWithMetadata] of remote.items) {
-					send({
+					this.send({
 						event: "latestMapValueUpdated",
 						workspaceId,
 						attendeeId: remote.attendee.attendeeId,
@@ -299,7 +358,7 @@ class MessageHandler {
 		}
 
 		this.workspaces.set(workspaceId, workspace);
-		send({
+		this.send({
 			event: "workspaceRegistered",
 			workspaceId,
 			latest: latest ?? false,
@@ -309,15 +368,40 @@ class MessageHandler {
 
 	public async onMessage(msg: MessageFromParent): Promise<void> {
 		if (verbosity.includes("msgs")) {
+			this.log.push({
+				timestamp: Date.now(),
+				agentId: process_id,
+				eventCategory: "messageReceived",
+				eventName: msg.command,
+			});
 			console.log(`[${process_id}] Received`, msg);
 		}
+
+		if (msg.command === "ping") {
+			this.handlePing();
+			return;
+		}
+
+		if (msg.command === "connect") {
+			await this.handleConnect(msg);
+			return;
+		}
+
+		// All other message must wait if connect is in progress
+		if (this.msgQueue !== undefined) {
+			this.msgQueue.push(msg);
+			return;
+		}
+
+		this.processMessage(msg);
+	}
+
+	private processMessage(
+		msg: Exclude<MessageFromParent, { command: "ping" | "connect" }>,
+	): void {
 		switch (msg.command) {
-			case "ping": {
-				this.handlePing();
-				break;
-			}
-			case "connect": {
-				await this.handleConnect(msg);
+			case "debugReport": {
+				this.handleDebugReport(msg);
 				break;
 			}
 			case "disconnectSelf": {
@@ -348,68 +432,118 @@ class MessageHandler {
 				break;
 			}
 			default: {
-				console.error(`${process_id}: Unknown command`);
-				send({ event: "error", error: `${process_id} Unknown command` });
+				console.error(`${process_id}: Unknown command:`, msg);
+				this.send({
+					event: "error",
+					error: `${process_id} Unknown command: ${JSON.stringify(msg)}`,
+				});
 			}
 		}
 	}
 
 	private handlePing(): void {
-		send({ event: "ack" });
+		this.send({ event: "ack" });
 	}
 
 	private async handleConnect(
 		msg: Extract<MessageFromParent, { command: "connect" }>,
 	): Promise<void> {
 		if (!msg.user) {
-			send({ event: "error", error: `${process_id}: No azure user information given` });
+			this.send({ event: "error", error: `${process_id}: No azure user information given` });
 			return;
 		}
-		if (isConnected(this.container)) {
-			send({ event: "error", error: `${process_id}: Already connected to container` });
+		if (this.container) {
+			this.send({ event: "error", error: `${process_id}: Container already loaded` });
 			return;
 		}
-		const { container, presence, containerId } = await getOrCreatePresenceContainer(
-			msg.containerId,
-			msg.user,
-			msg.scopes,
-			msg.createScopes,
-		);
-		this.container = container;
-		this.presence = presence;
-		this.containerId = containerId;
-
-		// Acknowledge connection before sending current attendee information
-		send({
-			event: "connected",
-			containerId,
-			attendeeId: presence.attendees.getMyself().attendeeId,
-		});
 
-		// Send existing attendees excluding self to parent/orchestrator
-		const self = presence.attendees.getMyself();
-		for (const attendee of presence.attendees.getAttendees()) {
-			if (attendee !== self) {
-				sendAttendeeConnected(attendee);
+		// Prevent reentrance. Queue messages until after connect is fully processed.
+		this.msgQueue = [];
+
+		try {
+			const { container, containerId, connected } = await getOrCreateContainer({
+				...msg,
+				logger: this.logger,
+				onDisconnected: this.onDisconnected,
+			});
+			this.container = container;
+			const presence = getPresence(container);
+			this.presence = presence;
+
+			// wait for 'ConnectionState.Connected'
+			await connected;
+
+			// Acknowledge connection before sending current attendee information
+			this.send({
+				event: "connected",
+				containerId,
+				attendeeId: presence.attendees.getMyself().attendeeId,
+			});
+
+			// Send existing attendees excluding self to parent/orchestrator
+			const self = presence.attendees.getMyself();
+			for (const attendee of presence.attendees.getAttendees()) {
+				if (attendee !== self && attendee.getConnectionStatus() === "Connected") {
+					this.sendAttendeeConnected(attendee);
+				}
+			}
+
+			// Listen for presence events to notify parent/orchestrator when a new attendee joins or leaves the session.
+			presence.attendees.events.on("attendeeConnected", this.sendAttendeeConnected);
+			presence.attendees.events.on("attendeeDisconnected", this.sendAttendeeDisconnected);
+		} finally {
+			// Process any queued messages received while connecting
+			for (const queuedMsg of this.msgQueue) {
+				this.processMessage(queuedMsg);
+			}
+			this.msgQueue = undefined;
+		}
+	}
+
+	private handleDebugReport(
+		msg: Extract<MessageFromParent, { command: "debugReport" }>,
+	): void {
+		if (msg.reportAttendees) {
+			if (this.presence) {
+				const attendees = this.presence.attendees.getAttendees();
+				let connectedCount = 0;
+				for (const attendee of attendees) {
+					if (attendee.getConnectionStatus() === "Connected") {
+						connectedCount++;
+					}
+				}
+				console.log(
+					`[${process_id}] Report: ${attendees.size} attendees, ${connectedCount} connected`,
+				);
+			} else {
+				this.send({ event: "error", error: `${process_id} is not connected to presence` });
 			}
 		}
 
-		// Listen for presence events to notify parent/orchestrator when a new attendee joins or leaves the session.
-		presence.attendees.events.on("attendeeConnected", sendAttendeeConnected);
-		presence.attendees.events.on("attendeeDisconnected", sendAttendeeDisconnected);
+		const debugReport: Extract<MessageToParent, { event: "debugReportComplete" }> = {
+			event: "debugReportComplete",
+		};
+		if (msg.sendEventLog) {
+			debugReport.log = this.log;
+		}
+		this.send(debugReport);
 	}
 
 	private handleDisconnectSelf(): void {
 		if (!this.container) {
-			send({ event: "error", error: `${process_id} is not connected to container` });
+			this.send({ event: "error", error: `${process_id} is not connected to container` });
 			return;
 		}
+		// There are no current scenarios where disconnect without presence is expected.
 		if (!this.presence) {
-			send({ event: "error", error: `${process_id} is not connected to presence` });
+			this.send({ event: "error", error: `${process_id} is not connected to presence` });
 			return;
 		}
+		// Disconnect event is treated as an error in normal handling.
+		// Remove listener as this disconnect is intentional.
+		this.container.off("disconnected", this.onDisconnected);
 		this.container.disconnect();
-		send({
+		this.send({
 			event: "disconnectedSelf",
 			attendeeId: this.presence.attendees.getMyself().attendeeId,
 		});
@@ -419,19 +553,22 @@ class MessageHandler {
 		msg: Extract<MessageFromParent, { command: "setLatestValue" }>,
 	): void {
 		if (!this.presence) {
-			send({ event: "error", error: `${process_id} is not connected to presence` });
+			this.send({ event: "error", error: `${process_id} is not connected to presence` });
 			return;
 		}
 		const workspace = this.workspaces.get(msg.workspaceId);
 		if (!workspace) {
-			send({ event: "error", error: `${process_id} workspace ${msg.workspaceId} not found` });
+			this.send({
+				event: "error",
+				error: `${process_id} workspace ${msg.workspaceId} not found`,
+			});
 			return;
 		}
 		// Cast required due to optional keys in WorkspaceSchema
 		// TODO: AB#47518
 		const latestState = workspace.states.latest as LatestRaw<{ value: string }> | undefined;
 		if (!latestState) {
-			send({
+			this.send({
 				event: "error",
 				error: `${process_id} latest state not registered for workspace ${msg.workspaceId}`,
 			});
@@ -447,16 +584,19 @@ class MessageHandler {
 		msg: Extract<MessageFromParent, { command: "setLatestMapValue" }>,
 	): void {
 		if (!this.presence) {
-			send({ event: "error", error: `${process_id} is not connected to presence` });
+			this.send({ event: "error", error: `${process_id} is not connected to presence` });
 			return;
 		}
 		if (typeof msg.key !== "string") {
-			send({ event: "error", error: `${process_id} invalid key type` });
+			this.send({ event: "error", error: `${process_id} invalid key type` });
 			return;
 		}
 		const workspace = this.workspaces.get(msg.workspaceId);
 		if (!workspace) {
-			send({ event: "error", error: `${process_id} workspace ${msg.workspaceId} not found` });
+			this.send({
+				event: "error",
+				error: `${process_id} workspace ${msg.workspaceId} not found`,
+			});
 			return;
 		}
 		// Cast required due to optional keys in WorkspaceSchema
@@ -465,7 +605,7 @@ class MessageHandler {
 			| LatestMapRaw<{ value: Record<string, string | number> }, string>
 			| undefined;
 		if (!latestMapState) {
-			send({
+			this.send({
 				event: "error",
 				error: `${process_id} latestMap state not registered for workspace ${msg.workspaceId}`,
 			});
@@ -481,19 +621,22 @@ class MessageHandler {
 		msg: Extract<MessageFromParent, { command: "getLatestValue" }>,
 	): void {
 		if (!this.presence) {
-			send({ event: "error", error: `${process_id} is not connected to presence` });
+			this.send({ event: "error", error: `${process_id} is not connected to presence` });
 			return;
 		}
 		const workspace = this.workspaces.get(msg.workspaceId);
 		if (!workspace) {
-			send({ event: "error", error: `${process_id} workspace ${msg.workspaceId} not found` });
+			this.send({
+				event: "error",
+				error: `${process_id} workspace ${msg.workspaceId} not found`,
+			});
 			return;
 		}
 		// Cast required due to optional keys in WorkspaceSchema
 		// TODO: AB#47518
 		const latestState = workspace.states.latest as LatestRaw<{ value: string }> | undefined;
 		if (!latestState) {
-			send({
+			this.send({
 				event: "error",
 				error: `${process_id} latest state not registered for workspace ${msg.workspaceId}`,
 			});
@@ -507,7 +650,7 @@ class MessageHandler {
 		} else {
 			value = latestState.local;
 		}
-		send({
+		this.send({
 			event: "latestValueGetResponse",
 			workspaceId: msg.workspaceId,
 			attendeeId: msg.attendeeId,
@@ -519,16 +662,19 @@ class MessageHandler {
 		msg: Extract<MessageFromParent, { command: "getLatestMapValue" }>,
 	): void {
 		if (!this.presence) {
-			send({ event: "error", error: `${process_id} is not connected to presence` });
+			this.send({ event: "error", error: `${process_id} is not connected to presence` });
 			return;
 		}
 		if (typeof msg.key !== "string") {
-			send({ event: "error", error: `${process_id} invalid key type` });
+			this.send({ event: "error", error: `${process_id} invalid key type` });
 			return;
 		}
 		const workspace = this.workspaces.get(msg.workspaceId);
 		if (!workspace) {
-			send({ event: "error", error: `${process_id} workspace ${msg.workspaceId} not found` });
+			this.send({
+				event: "error",
+				error: `${process_id} workspace ${msg.workspaceId} not found`,
+			});
 			return;
 		}
 		// Cast required due to optional keys in WorkspaceSchema
@@ -537,7 +683,7 @@ class MessageHandler {
 			| LatestMapRaw<{ value: Record<string, string | number> }, string>
 			| undefined;
 		if (!latestMapState) {
-			send({
+			this.send({
 				event: "error",
 				error: `${process_id} latestMap state not registered for workspace ${msg.workspaceId}`,
 			});
@@ -552,7 +698,7 @@ class MessageHandler {
 		} else {
 			value = latestMapState.local.get(msg.key);
 		}
-		send({
+		this.send({
 			event: "latestMapValueGetResponse",
 			workspaceId: msg.workspaceId,
 			attendeeId: msg.attendeeId,