@fluidframework/azure-end-to-end-tests 2.70.0-361092 → 2.70.0-361788

package/src/test/multiprocess/messageTypes.ts

@@ -13,11 +13,20 @@ export interface UserIdAndName {
 	name: string;
 }
 
+export interface EventEntry {
+	timestamp: number;
+	agentId: string;
+	eventCategory: string;
+	eventName: string;
+	details?: string;
+}
+
 /**
  * Message types sent from the orchestrator to the child processes
  */
 export type MessageToChild =
 	| ConnectCommand
+	| DebugReportCommand
 	| DisconnectSelfCommand
 	| RegisterWorkspaceCommand
 	| GetLatestValueCommand
@@ -50,6 +59,24 @@ export interface ConnectCommand {
 	containerId?: string;
 }
 
+/**
+ * Instructs a child process to report debug information.
+ *
+ * @privateRemarks
+ * This can be expanded over time to include more options.
+ */
+interface DebugReportCommand {
+	command: "debugReport";
+	/**
+	 * Send event log entries.
+	 */
+	sendEventLog?: true;
+	/**
+	 * Send basic attendee statistics (like count of connected).
+	 */
+	reportAttendees?: true;
+}
+
 /**
  * Instructs a child process to disconnect from a Fluid container.
  * A {@link DisconnectedSelfEvent} should be expected in response.
@@ -127,6 +154,7 @@ export type MessageFromChild =
 	| AttendeeConnectedEvent
 	| AttendeeDisconnectedEvent
 	| ConnectedEvent
+	| DebugReportCompleteEvent
 	| DisconnectedSelfEvent
 	| ErrorEvent
 	| LatestMapValueGetResponseEvent
@@ -167,6 +195,14 @@ interface ConnectedEvent {
 	attendeeId: AttendeeId;
 }
 
+/**
+ * Sent from the child processes to the orchestrator in response to a {@link DebugReportCommand}.
+ */
+interface DebugReportCompleteEvent {
+	event: "debugReportComplete";
+	log?: EventEntry[];
+}
+
 /**
  * Sent from the child processes to the orchestrator in response to a {@link DisconnectSelfCommand}.
  */

package/src/test/multiprocess/orchestratorUtils.ts

@@ -3,7 +3,8 @@
  * Licensed under the MIT License.
  */
 
-import { fork, type ChildProcess } from "node:child_process";
+import { fork } from "node:child_process";
+import type { ChildProcess as AnyChildProcess } from "node:child_process";
 
 import { ScopeType } from "@fluidframework/driver-definitions/legacy";
 import type { AttendeeId } from "@fluidframework/presence/beta";
@@ -16,6 +17,8 @@ import type {
 	LatestMapValueUpdatedEvent,
 	LatestValueGetResponseEvent,
 	LatestMapValueGetResponseEvent,
+	MessageToChild,
+	EventEntry,
 } from "./messageTypes.js";
 
 /**
@@ -39,6 +42,10 @@ export const testConsole = {
 	error: console.error,
 };
 
+interface ChildProcess extends AnyChildProcess {
+	send(message: MessageToChild): boolean;
+}
+
 /**
  * Fork child processes to simulate multiple Fluid clients.
  *
@@ -85,9 +92,12 @@ export async function forkChildProcesses(
 		});
 		childReadyPromises.push(readyPromise);
 		const errorPromise = new Promise<never>((_resolve, reject) => {
-			child.on("error", (error) => {
+			child.once("error", (error) => {
 				reject(new Error(`Child${i} process errored: ${error.message}`));
 			});
+			child.once("exit", (code, signal) => {
+				reject(new Error(`Child${i} process exited: code ${code}, signal ${signal}`));
+			});
 		});
 		childErrorPromises.push(errorPromise);
 		child.send({ command: "ping" });
@@ -98,6 +108,42 @@ export async function forkChildProcesses(
 	return { children, childErrorPromise };
 }
 
+/**
+ * Instructs all listed child processes to send debug reports and then the
+ * collection is output sorted by timestamp. Report content is up to the child
+ * processes, but typically includes messages sent and some telemetry events.
+ */
+export async function executeDebugReports(
+	childrenRequestedToReport: ChildProcess[],
+): Promise<void> {
+	const debugReportPromises: Promise<EventEntry[]>[] = [];
+	for (const child of childrenRequestedToReport) {
+		const debugReportPromise = new Promise<EventEntry[]>((resolve) => {
+			const handler = (msg: MessageFromChild): void => {
+				if (msg.event === "debugReportComplete") {
+					resolve(msg.log ?? []);
+					child.off("message", handler);
+				}
+			};
+			child.on("message", handler);
+		});
+		debugReportPromises.push(debugReportPromise);
+		child.send({ command: "debugReport", sendEventLog: true, reportAttendees: true });
+	}
+
+	const logs = await Promise.all(debugReportPromises);
+	const combinedLogs = logs.flat().sort((a, b) => a.timestamp - b.timestamp);
+	for (const entry of combinedLogs) {
+		testConsole.log(
+			`[${new Date(entry.timestamp).toISOString()}] [${entry.agentId}] [${entry.eventCategory}] ${entry.eventName}${
+				entry.details
+					? ` - ${typeof entry.details === "string" ? entry.details : JSON.stringify(entry.details)}`
+					: ""
+			}`,
+		);
+	}
+}
+
 /**
  * Creates a {@link ConnectCommand} for a test user with a deterministic id and name.
  *
@@ -118,6 +164,47 @@ function composeConnectMessage(
 	};
 }
 
+/**
+ * Listens for a "connected" response from a child process
+ * allowing/handling subset of other expected messages.
+ */
+function listenForConnectedResponse({
+	child,
+	childId,
+	onConnected,
+	reject,
+}: {
+	child: ChildProcess;
+	childId: number | string;
+	/**
+	 * Will be called up to once when a "connected" message is received.
+	 */
+	onConnected: (msg: Extract<MessageFromChild, { event: "connected" }>) => void;
+	/**
+	 * Callback to reject for unexpected messages or child errors.
+	 */
+	reject: (reason?: unknown) => void;
+}): void {
+	const listener = (msg: MessageFromChild): void => {
+		if (msg.event === "connected") {
+			child.off("message", listener);
+			onConnected(msg);
+		} else if (msg.event === "error") {
+			child.off("message", listener);
+			reject(new Error(`Child ${childId} process error: ${msg.error}`));
+		} else if (msg.event !== "ack") {
+			child.off("message", listener);
+			// This is not strictly required, but is current expectation.
+			reject(
+				new Error(
+					`Unexpected message from child ${childId} while connecting: ${JSON.stringify(msg)}`,
+				),
+			);
+		}
+	};
+	child.on("message", listener);
+}
+
 interface CreatorAttendeeIdAndAttendeePromises {
 	containerCreatorAttendeeId: AttendeeId;
 	attendeeIdPromises: Promise<AttendeeId>[];
@@ -140,18 +227,27 @@ export async function connectChildProcesses(
 	const containerReadyPromise = new Promise<{
 		containerCreatorAttendeeId: AttendeeId;
 		containerId: string;
-	}>((resolve, reject) => {
-		firstChild.once("message", (msg: MessageFromChild) => {
-			if (msg.event === "connected" && msg.containerId) {
-				resolve({
-					containerCreatorAttendeeId: msg.attendeeId,
-					containerId: msg.containerId,
-				});
-			} else {
-				reject(new Error(`Non-connected message from child0: ${JSON.stringify(msg)}`));
-			}
-		});
-	});
+	}>((resolve, reject) =>
+		listenForConnectedResponse({
+			child: firstChild,
+			childId: 0,
+			onConnected: (msg) => {
+				if (msg.containerId) {
+					resolve({
+						containerCreatorAttendeeId: msg.attendeeId,
+						containerId: msg.containerId,
+					});
+				} else {
+					reject(
+						new Error(
+							`Child 0 (creator) connected without containerId: ${JSON.stringify(msg)}`,
+						),
+					);
+				}
+			},
+			reject,
+		}),
+	);
 	{
 		// Note that DocWrite is used to have this attendee be the "leader".
 		// DocRead would also be valid as DocWrite is specified for attach when there
@@ -182,15 +278,16 @@ export async function connectChildProcesses(
 		);
 		message.containerId = containerId;
 		attendeeIdPromises.push(
-			new Promise<AttendeeId>((resolve, reject) => {
-				child.once("message", (msg: MessageFromChild) => {
-					if (msg.event === "connected") {
+			new Promise<AttendeeId>((resolve, reject) =>
+				listenForConnectedResponse({
+					child,
+					childId: index,
+					onConnected: (msg) => {
 						resolve(msg.attendeeId);
-					} else if (msg.event === "error") {
-						reject(new Error(`Child process error: ${msg.error}`));
-					}
-				});
-			}),
+					},
+					reject,
+				}),
+			),
 		);
 		child.send(message);
 	}
@@ -236,6 +333,7 @@ export async function connectAndListenForAttendees(
 					if (msg.event === "attendeeConnected") {
 						attendeesJoinedEvents++;
 						if (attendeesJoinedEvents >= attendeeCountRequired) {
+							child.off("message", listenForAttendees);
 							resolve();
 						}
 					}
@@ -323,7 +421,7 @@ export async function connectAndWaitForAttendees(
 export async function registerWorkspaceOnChildren(
 	children: ChildProcess[],
 	workspaceId: string,
-	options: { latest?: boolean; latestMap?: boolean; timeoutMs: number },
+	options: { latest?: true; latestMap?: true; timeoutMs: number },
 ): Promise<void> {
 	const { latest, latestMap, timeoutMs } = options;
 	const promises = children.map(async (child, index) => {

package/src/test/multiprocess/presenceTest.spec.ts

@@ -15,6 +15,7 @@ import {
 	connectAndListenForAttendees,
 	connectAndWaitForAttendees,
 	connectChildProcesses,
+	executeDebugReports,
 	forkChildProcesses,
 	getLatestMapValueResponses,
 	getLatestValueResponses,
@@ -46,7 +47,7 @@ const timeoutMultiplier = debuggerAttached ? 1000 : useAzure ? 3 : 1;
  * @param context - The Mocha test context.
  * @param duration - The duration in milliseconds to set the timeout to. Zero disables the timeout.
  */
-function setTimeout(context: Mocha.Context, duration: number): void {
+function setTestTimeout(context: Mocha.Context, duration: number): void {
 	const currentTimeout = context.timeout();
 	const newTimeout =
 		debuggerAttached || currentTimeout === 0 || duration === 0
@@ -86,11 +87,6 @@ function setTimeout(context: Mocha.Context, duration: number): void {
  * - Send response messages including any relevant data back to the orchestrator to verify expected behavior.
  */
 
-/**
- * This particular test suite tests the following E2E functionality for Presence:
- * - Announce 'attendeeConnected' when remote client joins session.
- * - Announce 'attendeeDisconnected' when remote client disconnects.
- */
 describe(`Presence with AzureClient`, () => {
 	const afterCleanUp: (() => void)[] = [];
 
@@ -112,9 +108,13 @@ describe(`Presence with AzureClient`, () => {
 		 */
 		const childConnectTimeoutMs = 1000 * numClients * timeoutMultiplier;
 		/**
-		 * Timeout for presence attendees to connect {@link AttendeeConnectedEvent}
+		 * Timeout for presence attendees to join per first child perspective {@link AttendeeConnectedEvent}
 		 */
 		const allAttendeesJoinedTimeoutMs = (1000 + 200 * numClients) * timeoutMultiplier;
+		/**
+		 * Timeout for presence attendees to fully join (everyone knows about everyone) {@link AttendeeConnectedEvent}
+		 */
+		const allAttendeesFullyJoinedTimeoutMs = (2000 + 300 * numClients) * timeoutMultiplier;
 
 		for (const writeClients of [numClients, 1]) {
 			it(`announces 'attendeeConnected' when remote client joins session [${numClients} clients, ${writeClients} writers]`, async function () {
@@ -123,7 +123,7 @@ describe(`Presence with AzureClient`, () => {
 					this.skip();
 				}
 
-				setTimeout(this, childConnectTimeoutMs + allAttendeesJoinedTimeoutMs + 1000);
+				setTestTimeout(this, childConnectTimeoutMs + allAttendeesJoinedTimeoutMs + 1000);
 
 				// Setup
 				const { children, childErrorPromise } = await forkChildProcesses(
@@ -144,11 +144,8 @@ describe(`Presence with AzureClient`, () => {
 				);
 			});
 
-			it(`announces 'attendeeDisconnected' when remote client disconnects [${numClients} clients, ${writeClients} writers]`, async function () {
-				// AB#48866: Fix intermittently failing presence tests
-				if (useAzure) {
-					this.skip();
-				}
+			// Even at 5 clients reaching fully connected state is unreliable with current implementation.
+			it.skip(`announces 'attendeeDisconnected' when remote client disconnects [${numClients} clients, ${writeClients} writers]`, async function () {
 				// TODO: AB#45620: "Presence: perf: update Join pattern for scale" can handle
 				// larger counts of read-only attendees. Without protocol changes tests with
 				// 20+ attendees exceed current limits.
@@ -156,12 +153,19 @@ describe(`Presence with AzureClient`, () => {
 					this.skip();
 				}
 
+				if (useAzure && numClients > 50) {
+					// Even with increased timeouts, more than 50 clients can be too large for AFR.
+					// This may be due to slow responses/inactivity from the clients that are
+					// creating pressure on ADO agent.
+					this.skip();
+				}
+
 				const childDisconnectTimeoutMs = 10_000 * timeoutMultiplier;
 
-				setTimeout(
+				setTestTimeout(
 					this,
 					childConnectTimeoutMs +
-						allAttendeesJoinedTimeoutMs +
+						allAttendeesFullyJoinedTimeoutMs +
 						childDisconnectTimeoutMs +
 						1000,
 				);
@@ -172,31 +176,91 @@ describe(`Presence with AzureClient`, () => {
 					afterCleanUp,
 				);
 
+				const startConnectAndFullJoin = performance.now();
 				const connectResult = await connectAndListenForAttendees(children, {
 					writeClients,
 					attendeeCountRequired: numClients - 1,
 					childConnectTimeoutMs,
 				});
+				// eslint-disable-next-line @typescript-eslint/no-floating-promises
+				connectResult.attendeeCountRequiredPromises[0].then(() =>
+					testConsole.log(
+						`[${new Date().toISOString()}] All attendees joined per child 0 after ${performance.now() - startConnectAndFullJoin}ms`,
+					),
+				);
 
 				// Wait for all attendees to be fully joined
 				// Keep a tally for debuggability
 				let childrenFullyJoined = 0;
+				const setNotFullyJoined = new Set<number>();
+				for (let i = 0; i < children.length; i++) {
+					setNotFullyJoined.add(i);
+				}
 				const allAttendeesFullyJoined = Promise.all(
-					// eslint-disable-next-line @typescript-eslint/promise-function-async
-					connectResult.attendeeCountRequiredPromises.map((attendeeFullyJoinedPromise) =>
-						attendeeFullyJoinedPromise.then(() => childrenFullyJoined++),
+					connectResult.attendeeCountRequiredPromises.map(
+						async (attendeeFullyJoinedPromise, index) => {
+							await attendeeFullyJoinedPromise;
+							childrenFullyJoined++;
+							setNotFullyJoined.delete(index);
+						},
 					),
 				);
-				await timeoutAwait(allAttendeesFullyJoined, {
-					durationMs: allAttendeesJoinedTimeoutMs,
+				let timedout = true;
+				const allFullyJoinedOrChildError = Promise.race([
+					allAttendeesFullyJoined,
+					childErrorPromise,
+				]).finally(() => (timedout = false));
+				await timeoutAwait(allFullyJoinedOrChildError, {
+					durationMs: allAttendeesFullyJoinedTimeoutMs,
 					errorMsg: "Not all attendees fully joined",
-				}).catch((error) => {
+				}).catch(async (error) => {
 					// Ideally this information would just be in the timeout error message, but that
 					// must be a resolved string (not dynamic). So, just log it separately.
-					testConsole.log(`${childrenFullyJoined} attendees fully joined before error...`);
+					testConsole.log(
+						`[${new Date().toISOString()}] ${childrenFullyJoined} attendees fully joined before error...`,
+					);
+					if (timedout) {
+						// Gather additional timing data if timed out to understand what increased
+						// timeout could work. Test will still fail if this secondary wait succeeds.
+						const startAdditionalWait = performance.now();
+						try {
+							await timeoutAwait(allFullyJoinedOrChildError, {
+								durationMs: allAttendeesFullyJoinedTimeoutMs,
+							});
+							testConsole.log(
+								`[${new Date().toISOString()}] All attendees fully joined after additional wait (${performance.now() - startAdditionalWait}ms)`,
+							);
+						} catch (secondaryError) {
+							testConsole.log(
+								`[${new Date().toISOString()}] Secondary await resulted in`,
+								secondaryError,
+							);
+						}
+					}
+
+					// Gather and report debug info from children
+					// If there are less than 10 children, get all reports.
+					// Otherwise, just child 0 and those not fully joined.
+					setTestTimeout(this, 0); // Disable test timeout. Will throw within 20s below.
+					const childrenRequestedToReport =
+						children.length <= 10
+							? children
+							: // Just those not fully joined
+								children.filter((_, index) => index === 0 || setNotFullyJoined.has(index));
+					await timeoutAwait(
+						Promise.race([executeDebugReports(childrenRequestedToReport), childErrorPromise]),
+						{ durationMs: 20_000, errorMsg: "Debug report timeout" },
+					).catch((debugAwaitError) => {
+						testConsole.error("Debug report await resulted in", debugAwaitError);
+					});
+
 					throw error;
 				});
+				testConsole.log(
+					`[${new Date().toISOString()}] All attendees fully joined after ${performance.now() - startConnectAndFullJoin}ms`,
+				);
 
+				let child0ReportRequested = false;
 				const waitForDisconnected = children.map(async (child, index) =>
 					index === 0
 						? Promise.resolve()
@@ -216,7 +280,23 @@ describe(`Presence with AzureClient`, () => {
 									durationMs: childDisconnectTimeoutMs,
 									errorMsg: `Attendee[${index}] Disconnected Timeout`,
 								},
-							),
+							).catch(async (error) => {
+								const childrenRequestedToReport = [child];
+								if (!child0ReportRequested) {
+									childrenRequestedToReport.unshift(children[0]);
+									child0ReportRequested = true;
+								}
+								await timeoutAwait(
+									Promise.race([
+										executeDebugReports(childrenRequestedToReport),
+										childErrorPromise,
+									]),
+									{ durationMs: 20_000, errorMsg: "Debug report timeout" },
+								).catch((debugAwaitError) => {
+									testConsole.error("Debug report await resulted in", debugAwaitError);
+								});
+								throw error;
+							}),
 				);
 
 				// Act - disconnect first child process

package/src/test/utils.ts

@@ -59,7 +59,7 @@ export const configProvider = (
  * Currently, there is only a test-set for Durable containers and one for Ephemeral containers.
  * The Ephemeral container tests will not run for local tests.
  *
- * @returns - The test matrix
+ * @returns The test matrix
  */
 export function getTestMatrix(): { variant: string; options: { isEphemeral: boolean } }[] {
 	const testMatrix = [