@fluidframework/azure-end-to-end-tests 2.70.0-361248 → 2.70.0

package/src/test/multiprocess/messageTypes.ts

@@ -13,11 +13,20 @@ export interface UserIdAndName {
 	name: string;
 }
 
+export interface EventEntry {
+	timestamp: number;
+	agentId: string;
+	eventCategory: string;
+	eventName: string;
+	details?: string;
+}
+
 /**
  * Message types sent from the orchestrator to the child processes
  */
 export type MessageToChild =
 	| ConnectCommand
+	| DebugReportCommand
 	| DisconnectSelfCommand
 	| RegisterWorkspaceCommand
 	| GetLatestValueCommand
@@ -50,6 +59,24 @@ export interface ConnectCommand {
 	containerId?: string;
 }
 
+/**
+ * Instructs a child process to report debug information.
+ *
+ * @privateRemarks
+ * This can be expanded over time to include more options.
+ */
+interface DebugReportCommand {
+	command: "debugReport";
+	/**
+	 * Send event log entries.
+	 */
+	sendEventLog?: true;
+	/**
+	 * Send basic attendee statistics (like count of connected).
+	 */
+	reportAttendees?: true;
+}
+
 /**
  * Instructs a child process to disconnect from a Fluid container.
  * A {@link DisconnectedSelfEvent} should be expected in response.
@@ -127,6 +154,7 @@ export type MessageFromChild =
 	| AttendeeConnectedEvent
 	| AttendeeDisconnectedEvent
 	| ConnectedEvent
+	| DebugReportCompleteEvent
 	| DisconnectedSelfEvent
 	| ErrorEvent
 	| LatestMapValueGetResponseEvent
@@ -167,6 +195,14 @@ interface ConnectedEvent {
 	attendeeId: AttendeeId;
 }
 
+/**
+ * Sent from the child processes to the orchestrator in response to a {@link DebugReportCommand}.
+ */
+interface DebugReportCompleteEvent {
+	event: "debugReportComplete";
+	log?: EventEntry[];
+}
+
 /**
  * Sent from the child processes to the orchestrator in response to a {@link DisconnectSelfCommand}.
  */

package/src/test/multiprocess/orchestratorUtils.ts

@@ -3,7 +3,8 @@
  * Licensed under the MIT License.
  */
 
-import { fork, type ChildProcess } from "node:child_process";
+import { fork } from "node:child_process";
+import type { ChildProcess as AnyChildProcess } from "node:child_process";
 
 import { ScopeType } from "@fluidframework/driver-definitions/legacy";
 import type { AttendeeId } from "@fluidframework/presence/beta";
@@ -16,6 +17,8 @@ import type {
 	LatestMapValueUpdatedEvent,
 	LatestValueGetResponseEvent,
 	LatestMapValueGetResponseEvent,
+	MessageToChild,
+	EventEntry,
 } from "./messageTypes.js";
 
 /**
@@ -39,6 +42,10 @@ export const testConsole = {
 	error: console.error,
 };
 
+interface ChildProcess extends AnyChildProcess {
+	send(message: MessageToChild): boolean;
+}
+
 /**
  * Fork child processes to simulate multiple Fluid clients.
  *
@@ -85,9 +92,12 @@ export async function forkChildProcesses(
 		});
 		childReadyPromises.push(readyPromise);
 		const errorPromise = new Promise<never>((_resolve, reject) => {
-			child.on("error", (error) => {
+			child.once("error", (error) => {
 				reject(new Error(`Child${i} process errored: ${error.message}`));
 			});
+			child.once("exit", (code, signal) => {
+				reject(new Error(`Child${i} process exited: code ${code}, signal ${signal}`));
+			});
 		});
 		childErrorPromises.push(errorPromise);
 		child.send({ command: "ping" });
@@ -98,6 +108,42 @@ export async function forkChildProcesses(
 	return { children, childErrorPromise };
 }
 
+/**
+ * Instructs all listed child processes to send debug reports and then the
+ * collection is output sorted by timestamp. Report content is up to the child
+ * processes, but typically includes messages sent and some telemetry events.
+ */
+export async function executeDebugReports(
+	childrenRequestedToReport: ChildProcess[],
+): Promise<void> {
+	const debugReportPromises: Promise<EventEntry[]>[] = [];
+	for (const child of childrenRequestedToReport) {
+		const debugReportPromise = new Promise<EventEntry[]>((resolve) => {
+			const handler = (msg: MessageFromChild): void => {
+				if (msg.event === "debugReportComplete") {
+					resolve(msg.log ?? []);
+					child.off("message", handler);
+				}
+			};
+			child.on("message", handler);
+		});
+		debugReportPromises.push(debugReportPromise);
+		child.send({ command: "debugReport", sendEventLog: true, reportAttendees: true });
+	}
+
+	const logs = await Promise.all(debugReportPromises);
+	const combinedLogs = logs.flat().sort((a, b) => a.timestamp - b.timestamp);
+	for (const entry of combinedLogs) {
+		testConsole.log(
+			`[${new Date(entry.timestamp).toISOString()}] [${entry.agentId}] [${entry.eventCategory}] ${entry.eventName}${
+				entry.details
+					? ` - ${typeof entry.details === "string" ? entry.details : JSON.stringify(entry.details)}`
+					: ""
+			}`,
+		);
+	}
+}
+
 /**
  * Creates a {@link ConnectCommand} for a test user with a deterministic id and name.
  *
@@ -118,6 +164,47 @@ function composeConnectMessage(
 	};
 }
 
+/**
+ * Listens for a "connected" response from a child process
+ * allowing/handling subset of other expected messages.
+ */
+function listenForConnectedResponse({
+	child,
+	childId,
+	onConnected,
+	reject,
+}: {
+	child: ChildProcess;
+	childId: number | string;
+	/**
+	 * Will be called up to once when a "connected" message is received.
+	 */
+	onConnected: (msg: Extract<MessageFromChild, { event: "connected" }>) => void;
+	/**
+	 * Callback to reject for unexpected messages or child errors.
+	 */
+	reject: (reason?: unknown) => void;
+}): void {
+	const listener = (msg: MessageFromChild): void => {
+		if (msg.event === "connected") {
+			child.off("message", listener);
+			onConnected(msg);
+		} else if (msg.event === "error") {
+			child.off("message", listener);
+			reject(new Error(`Child ${childId} process error: ${msg.error}`));
+		} else if (msg.event !== "ack") {
+			child.off("message", listener);
+			// This is not strictly required, but is current expectation.
+			reject(
+				new Error(
+					`Unexpected message from child ${childId} while connecting: ${JSON.stringify(msg)}`,
+				),
+			);
+		}
+	};
+	child.on("message", listener);
+}
+
 interface CreatorAttendeeIdAndAttendeePromises {
 	containerCreatorAttendeeId: AttendeeId;
 	attendeeIdPromises: Promise<AttendeeId>[];
@@ -140,18 +227,27 @@ export async function connectChildProcesses(
 	const containerReadyPromise = new Promise<{
 		containerCreatorAttendeeId: AttendeeId;
 		containerId: string;
-	}>((resolve, reject) => {
-		firstChild.once("message", (msg: MessageFromChild) => {
-			if (msg.event === "connected" && msg.containerId) {
-				resolve({
-					containerCreatorAttendeeId: msg.attendeeId,
-					containerId: msg.containerId,
-				});
-			} else {
-				reject(new Error(`Non-connected message from child0: ${JSON.stringify(msg)}`));
-			}
-		});
-	});
+	}>((resolve, reject) =>
+		listenForConnectedResponse({
+			child: firstChild,
+			childId: 0,
+			onConnected: (msg) => {
+				if (msg.containerId) {
+					resolve({
+						containerCreatorAttendeeId: msg.attendeeId,
+						containerId: msg.containerId,
+					});
+				} else {
+					reject(
+						new Error(
+							`Child 0 (creator) connected without containerId: ${JSON.stringify(msg)}`,
+						),
+					);
+				}
+			},
+			reject,
+		}),
+	);
 	{
 		// Note that DocWrite is used to have this attendee be the "leader".
 		// DocRead would also be valid as DocWrite is specified for attach when there
@@ -182,15 +278,16 @@ export async function connectChildProcesses(
 		);
 		message.containerId = containerId;
 		attendeeIdPromises.push(
-			new Promise<AttendeeId>((resolve, reject) => {
-				child.once("message", (msg: MessageFromChild) => {
-					if (msg.event === "connected") {
+			new Promise<AttendeeId>((resolve, reject) =>
+				listenForConnectedResponse({
+					child,
+					childId: index,
+					onConnected: (msg) => {
 						resolve(msg.attendeeId);
-					} else if (msg.event === "error") {
-						reject(new Error(`Child process error: ${msg.error}`));
-					}
-				});
-			}),
+					},
+					reject,
+				}),
+			),
 		);
 		child.send(message);
 	}
@@ -236,6 +333,7 @@ export async function connectAndListenForAttendees(
 					if (msg.event === "attendeeConnected") {
 						attendeesJoinedEvents++;
 						if (attendeesJoinedEvents >= attendeeCountRequired) {
+							child.off("message", listenForAttendees);
 							resolve();
 						}
 					}
@@ -323,7 +421,7 @@ export async function connectAndWaitForAttendees(
 export async function registerWorkspaceOnChildren(
 	children: ChildProcess[],
 	workspaceId: string,
-	options: { latest?: boolean; latestMap?: boolean; timeoutMs: number },
+	options: { latest?: true; latestMap?: true; timeoutMs: number },
 ): Promise<void> {
 	const { latest, latestMap, timeoutMs } = options;
 	const promises = children.map(async (child, index) => {

package/src/test/multiprocess/presenceTest.spec.ts

@@ -15,6 +15,7 @@ import {
 	connectAndListenForAttendees,
 	connectAndWaitForAttendees,
 	connectChildProcesses,
+	executeDebugReports,
 	forkChildProcesses,
 	getLatestMapValueResponses,
 	getLatestValueResponses,
@@ -24,6 +25,14 @@ import {
 	waitForLatestValueUpdates,
 } from "./orchestratorUtils.js";
 
+/**
+ * When true, slower (long running time) tests will be run.
+ * Otherwise, those test will not appear. Console output is used to show that
+ * they exist. (They could be skipped, though skipped test are often an
+ * indication of a problem.)
+ */
+const shouldRunScaleTests = process.env.FLUID_TEST_SCALE !== undefined;
+
 const useAzure = process.env.FLUID_CLIENT === "azure";
 
 /**
@@ -46,7 +55,7 @@ const timeoutMultiplier = debuggerAttached ? 1000 : useAzure ? 3 : 1;
  * @param context - The Mocha test context.
  * @param duration - The duration in milliseconds to set the timeout to. Zero disables the timeout.
  */
-function setTimeout(context: Mocha.Context, duration: number): void {
+function setTestTimeout(context: Mocha.Context, duration: number): void {
 	const currentTimeout = context.timeout();
 	const newTimeout =
 		debuggerAttached || currentTimeout === 0 || duration === 0
@@ -86,11 +95,6 @@ function setTimeout(context: Mocha.Context, duration: number): void {
  * - Send response messages including any relevant data back to the orchestrator to verify expected behavior.
  */
 
-/**
- * This particular test suite tests the following E2E functionality for Presence:
- * - Announce 'attendeeConnected' when remote client joins session.
- * - Announce 'attendeeDisconnected' when remote client disconnects.
- */
 describe(`Presence with AzureClient`, () => {
 	const afterCleanUp: (() => void)[] = [];
 
@@ -102,131 +106,212 @@ describe(`Presence with AzureClient`, () => {
 		afterCleanUp.length = 0;
 	});
 
-	// Note that on slower systems 50+ clients may take too long to join.
-	const numClientsForAttendeeTests = [5, 20, 50, 100];
-	// TODO: AB#45620: "Presence: perf: update Join pattern for scale" may help, then remove .slice.
-	for (const numClients of numClientsForAttendeeTests.slice(0, 2)) {
-		assert(numClients > 1, "Must have at least two clients");
-		/**
-		 * Timeout for child processes to connect to container ({@link ConnectedEvent})
-		 */
-		const childConnectTimeoutMs = 1000 * numClients * timeoutMultiplier;
-		/**
-		 * Timeout for presence attendees to connect {@link AttendeeConnectedEvent}
-		 */
-		const allAttendeesJoinedTimeoutMs = (1000 + 200 * numClients) * timeoutMultiplier;
+	describe("`attendees` support", () => {
+		const numClientsForAttendeeTests = [5, 40, 100, 250];
+		for (const numClients of numClientsForAttendeeTests) {
+			if (numClients > 50 && !shouldRunScaleTests) {
+				testConsole.log(
+					`skipping Presence attendee scale tests with ${numClients} clients (set FLUID_TEST_SCALE=true to run)`,
+				);
+				continue;
+			}
+
+			assert(numClients > 1, "Must have at least two clients");
+			/**
+			 * Timeout for child processes to connect to container ({@link ConnectedEvent})
+			 */
+			const childConnectTimeoutMs = 1000 * numClients * timeoutMultiplier;
+			/**
+			 * Timeout for presence attendees to join per first child perspective {@link AttendeeConnectedEvent}
+			 */
+			const allAttendeesJoinedTimeoutMs = (1000 + 200 * numClients) * timeoutMultiplier;
+			/**
+			 * Timeout for presence attendees to fully join (everyone knows about everyone) {@link AttendeeConnectedEvent}
+			 */
+			const allAttendeesFullyJoinedTimeoutMs = (2000 + 300 * numClients) * timeoutMultiplier;
+
+			for (const writeClients of [numClients, 1]) {
+				it(`announces 'attendeeConnected' when remote client joins session [${numClients} clients, ${writeClients} writers]`, async function () {
+					setTestTimeout(this, childConnectTimeoutMs + allAttendeesJoinedTimeoutMs + 1000);
+
+					// Setup
+					const { children, childErrorPromise } = await forkChildProcesses(
+						numClients,
+						afterCleanUp,
+					);
+
+					// Further Setup with Act and Verify
+					await connectAndWaitForAttendees(
+						children,
+						{
+							writeClients,
+							attendeeCountRequired: numClients - 1,
+							childConnectTimeoutMs,
+							allAttendeesJoinedTimeoutMs,
+						},
+						childErrorPromise,
+					);
+				});
 
-		for (const writeClients of [numClients, 1]) {
-			it(`announces 'attendeeConnected' when remote client joins session [${numClients} clients, ${writeClients} writers]`, async function () {
-				// AB#48866: Fix intermittently failing presence tests
-				if (useAzure) {
-					this.skip();
-				}
+				it(`announces 'attendeeDisconnected' when remote client disconnects [${numClients} clients, ${writeClients} writers]`, async function () {
+					if (useAzure && numClients > 50) {
+						// Even with increased timeouts, more than 50 clients can be too large for AFR.
+						// This may be due to slow responses/inactivity from the clients that are
+						// creating pressure on ADO agent.
+						this.skip();
+					}
 
-				setTimeout(this, childConnectTimeoutMs + allAttendeesJoinedTimeoutMs + 1000);
+					const childDisconnectTimeoutMs = 10_000 * timeoutMultiplier;
 
-				// Setup
-				const { children, childErrorPromise } = await forkChildProcesses(
-					numClients,
-					afterCleanUp,
-				);
+					setTestTimeout(
+						this,
+						childConnectTimeoutMs +
+							allAttendeesFullyJoinedTimeoutMs +
+							childDisconnectTimeoutMs +
+							1000,
+					);
+
+					// Setup
+					const { children, childErrorPromise } = await forkChildProcesses(
+						numClients,
+						afterCleanUp,
+					);
 
-				// Further Setup with Act and Verify
-				await connectAndWaitForAttendees(
-					children,
-					{
+					const startConnectAndFullJoin = performance.now();
+					const connectResult = await connectAndListenForAttendees(children, {
 						writeClients,
 						attendeeCountRequired: numClients - 1,
 						childConnectTimeoutMs,
-						allAttendeesJoinedTimeoutMs,
-					},
-					childErrorPromise,
-				);
-			});
-
-			it(`announces 'attendeeDisconnected' when remote client disconnects [${numClients} clients, ${writeClients} writers]`, async function () {
-				// AB#48866: Fix intermittently failing presence tests
-				if (useAzure) {
-					this.skip();
-				}
-				// TODO: AB#45620: "Presence: perf: update Join pattern for scale" can handle
-				// larger counts of read-only attendees. Without protocol changes tests with
-				// 20+ attendees exceed current limits.
-				if (numClients >= 20 && writeClients === 1) {
-					this.skip();
-				}
-
-				const childDisconnectTimeoutMs = 10_000 * timeoutMultiplier;
-
-				setTimeout(
-					this,
-					childConnectTimeoutMs +
-						allAttendeesJoinedTimeoutMs +
-						childDisconnectTimeoutMs +
-						1000,
-				);
-
-				// Setup
-				const { children, childErrorPromise } = await forkChildProcesses(
-					numClients,
-					afterCleanUp,
-				);
+					});
+					// eslint-disable-next-line @typescript-eslint/no-floating-promises
+					connectResult.attendeeCountRequiredPromises[0].then(() =>
+						testConsole.log(
+							`[${new Date().toISOString()}] All attendees joined per child 0 after ${performance.now() - startConnectAndFullJoin}ms`,
+						),
+					);
 
-				const connectResult = await connectAndListenForAttendees(children, {
-					writeClients,
-					attendeeCountRequired: numClients - 1,
-					childConnectTimeoutMs,
-				});
+					// Wait for all attendees to be fully joined
+					// Keep a tally for debuggability
+					let childrenFullyJoined = 0;
+					const setNotFullyJoined = new Set<number>();
+					for (let i = 0; i < children.length; i++) {
+						setNotFullyJoined.add(i);
+					}
+					const allAttendeesFullyJoined = Promise.all(
+						connectResult.attendeeCountRequiredPromises.map(
+							async (attendeeFullyJoinedPromise, index) => {
+								await attendeeFullyJoinedPromise;
+								childrenFullyJoined++;
+								setNotFullyJoined.delete(index);
+							},
+						),
+					);
+					let timedout = true;
+					const allFullyJoinedOrChildError = Promise.race([
+						allAttendeesFullyJoined,
+						childErrorPromise,
+					]).finally(() => (timedout = false));
+					await timeoutAwait(allFullyJoinedOrChildError, {
+						durationMs: allAttendeesFullyJoinedTimeoutMs,
+						errorMsg: "Not all attendees fully joined",
+					}).catch(async (error) => {
+						// Ideally this information would just be in the timeout error message, but that
+						// must be a resolved string (not dynamic). So, just log it separately.
+						testConsole.log(
+							`[${new Date().toISOString()}] ${childrenFullyJoined} attendees fully joined before error...`,
+						);
+						if (timedout) {
+							// Gather additional timing data if timed out to understand what increased
+							// timeout could work. Test will still fail if this secondary wait succeeds.
+							const startAdditionalWait = performance.now();
+							try {
+								await timeoutAwait(allFullyJoinedOrChildError, {
+									durationMs: allAttendeesFullyJoinedTimeoutMs,
+								});
+								testConsole.log(
+									`[${new Date().toISOString()}] All attendees fully joined after additional wait (${performance.now() - startAdditionalWait}ms)`,
+								);
+							} catch (secondaryError) {
+								testConsole.log(
+									`[${new Date().toISOString()}] Secondary await resulted in`,
+									secondaryError,
+								);
+							}
+						}
+
+						// Gather and report debug info from children
+						// If there are less than 10 children, get all reports.
+						// Otherwise, just child 0 and those not fully joined.
+						setTestTimeout(this, 0); // Disable test timeout. Will throw within 20s below.
+						const childrenRequestedToReport =
+							children.length <= 10
+								? children
+								: // Just those not fully joined
+									children.filter((_, index) => index === 0 || setNotFullyJoined.has(index));
+						await timeoutAwait(
+							Promise.race([
+								executeDebugReports(childrenRequestedToReport),
+								childErrorPromise,
+							]),
+							{ durationMs: 20_000, errorMsg: "Debug report timeout" },
+						).catch((debugAwaitError) => {
+							testConsole.error("Debug report await resulted in", debugAwaitError);
+						});
 
-				// Wait for all attendees to be fully joined
-				// Keep a tally for debuggability
-				let childrenFullyJoined = 0;
-				const allAttendeesFullyJoined = Promise.all(
-					// eslint-disable-next-line @typescript-eslint/promise-function-async
-					connectResult.attendeeCountRequiredPromises.map((attendeeFullyJoinedPromise) =>
-						attendeeFullyJoinedPromise.then(() => childrenFullyJoined++),
-					),
-				);
-				await timeoutAwait(allAttendeesFullyJoined, {
-					durationMs: allAttendeesJoinedTimeoutMs,
-					errorMsg: "Not all attendees fully joined",
-				}).catch((error) => {
-					// Ideally this information would just be in the timeout error message, but that
-					// must be a resolved string (not dynamic). So, just log it separately.
-					testConsole.log(`${childrenFullyJoined} attendees fully joined before error...`);
-					throw error;
-				});
+						throw error;
+					});
+					testConsole.log(
+						`[${new Date().toISOString()}] All attendees fully joined after ${performance.now() - startConnectAndFullJoin}ms`,
+					);
 
-				const waitForDisconnected = children.map(async (child, index) =>
-					index === 0
-						? Promise.resolve()
-						: timeoutPromise(
-								(resolve) => {
-									child.on("message", (msg: MessageFromChild) => {
-										if (
-											msg.event === "attendeeDisconnected" &&
-											msg.attendeeId === connectResult.containerCreatorAttendeeId
-										) {
-											console.log(`Child[${index}] saw creator disconnect`);
-											resolve();
-										}
+					let child0ReportRequested = false;
+					const waitForDisconnected = children.map(async (child, index) =>
+						index === 0
+							? Promise.resolve()
+							: timeoutPromise(
+									(resolve) => {
+										child.on("message", (msg: MessageFromChild) => {
+											if (
+												msg.event === "attendeeDisconnected" &&
+												msg.attendeeId === connectResult.containerCreatorAttendeeId
+											) {
+												console.log(`Child[${index}] saw creator disconnect`);
+												resolve();
+											}
+										});
+									},
+									{
+										durationMs: childDisconnectTimeoutMs,
+										errorMsg: `Attendee[${index}] Disconnected Timeout`,
+									},
+								).catch(async (error) => {
+									const childrenRequestedToReport = [child];
+									if (!child0ReportRequested) {
+										childrenRequestedToReport.unshift(children[0]);
+										child0ReportRequested = true;
+									}
+									await timeoutAwait(
+										Promise.race([
+											executeDebugReports(childrenRequestedToReport),
+											childErrorPromise,
+										]),
+										{ durationMs: 20_000, errorMsg: "Debug report timeout" },
+									).catch((debugAwaitError) => {
+										testConsole.error("Debug report await resulted in", debugAwaitError);
 									});
-								},
-								{
-									durationMs: childDisconnectTimeoutMs,
-									errorMsg: `Attendee[${index}] Disconnected Timeout`,
-								},
-							),
-				);
+									throw error;
+								}),
+					);
 
-				// Act - disconnect first child process
-				children[0].send({ command: "disconnectSelf" });
+					// Act - disconnect first child process
+					children[0].send({ command: "disconnectSelf" });
 
-				// Verify - wait for all 'attendeeDisconnected' events
-				await Promise.race([Promise.all(waitForDisconnected), childErrorPromise]);
-			});
+					// Verify - wait for all 'attendeeDisconnected' events
+					await Promise.race([Promise.all(waitForDisconnected), childErrorPromise]);
+				});
+			}
 		}
-	}
+	});
 
 	{
 		/**
@@ -279,10 +364,6 @@ describe(`Presence with AzureClient`, () => {
 				});
 
 				it(`allows clients to read Latest state from other clients [${numClients} clients]`, async function () {
-					// AB#48866: Fix intermittently failing presence tests
-					if (useAzure) {
-						this.skip();
-					}
 					// Setup
 					const updateEventsPromise = waitForLatestValueUpdates(
 						remoteClients,
@@ -421,10 +502,6 @@ describe(`Presence with AzureClient`, () => {
 				});
 
 				it(`returns per-key values on read [${numClients} clients]`, async function () {
-					// AB#48866: Fix intermittently failing presence tests
-					if (useAzure) {
-						this.skip();
-					}
 					// Setup
 					const allAttendeeIds = await Promise.all(attendeeIdPromises);
 					const attendee0Id = containerCreatorAttendeeId;