@fluidframework/azure-end-to-end-tests 2.70.0-361248 → 2.70.0-361788

package/lib/test/multiprocess/childClient.tool.js

@@ -24,28 +24,42 @@ const endPoint = process.env.azure__fluid__relay__service__endpoint;
 if (useAzure && endPoint === undefined) {
     throw new Error("Azure Fluid Relay service endpoint is missing");
 }
+const containerSchema = {
+    initialObjects: {
+        // A DataObject is added as otherwise fluid-static complains "Container cannot be initialized without any DataTypes"
+        _unused: TestDataObject,
+    },
+};
+function telemetryEventInterestLevel(eventName) {
+    if (eventName.includes(":Signal") || eventName.includes(":Join")) {
+        return "details";
+    }
+    else if (eventName.includes(":Container:") || eventName.includes(":Presence:")) {
+        return "basic";
+    }
+    return "none";
+}
 function selectiveVerboseLog(event, logLevel) {
-    if (event.eventName.includes(":Signal") || event.eventName.includes(":Join")) {
-        console.log(`[${process_id}] [${logLevel ?? LogLevel.default}]`, {
-            eventName: event.eventName,
-            details: event.details,
-            containerConnectionState: event.containerConnectionState,
-        });
+    const interest = telemetryEventInterestLevel(event.eventName);
+    if (interest === "none") {
+        return;
     }
-    else if (event.eventName.includes(":Container:") ||
-        event.eventName.includes(":Presence:")) {
-        console.log(`[${process_id}] [${logLevel ?? LogLevel.default}]`, {
-            eventName: event.eventName,
-            containerConnectionState: event.containerConnectionState,
-        });
+    const content = {
+        eventName: event.eventName,
+        containerConnectionState: event.containerConnectionState,
+    };
+    if (interest === "details") {
+        content.details = event.details;
     }
+    console.log(`[${process_id}] [${logLevel ?? LogLevel.default}]`, content);
 }
 /**
- * Get or create a Fluid container with Presence in initialObjects.
+ * Get or create a Fluid container.
  */
-const getOrCreatePresenceContainer = async (id, user, scopes, createScopes) => {
+const getOrCreateContainer = async (params) => {
     let container;
-    let containerId;
+    let { containerId } = params;
+    const { logger, onDisconnected, user, scopes, createScopes } = params;
     const connectionProps = useAzure
         ? {
             tenantId,
@@ -60,40 +74,30 @@ const getOrCreatePresenceContainer = async (id, user, scopes, createScopes) => {
         };
     const client = new AzureClient({
         connection: connectionProps,
-        logger: {
-            send: verbosity.includes("telem") ? selectiveVerboseLog : () => { },
-        },
+        logger,
     });
-    const schema = {
-        initialObjects: {
-            // A DataObject is added as otherwise fluid-static complains "Container cannot be initialized without any DataTypes"
-            _unused: TestDataObject,
-        },
-    };
     let services;
-    if (id === undefined) {
-        ({ container, services } = await client.createContainer(schema, "2"));
+    if (containerId === undefined) {
+        ({ container, services } = await client.createContainer(containerSchema, "2"));
         containerId = await container.attach();
     }
     else {
-        containerId = id;
-        ({ container, services } = await client.getContainer(containerId, schema, "2"));
+        ({ container, services } = await client.getContainer(containerId, containerSchema, "2"));
     }
-    // wait for 'ConnectionState.Connected' so we return with client connected to container
-    if (container.connectionState !== ConnectionState.Connected) {
-        await timeoutPromise((resolve) => container.once("connected", () => resolve()), {
+    container.on("disconnected", onDisconnected);
+    const connected = container.connectionState === ConnectionState.Connected
+        ? Promise.resolve()
+        : timeoutPromise((resolve) => container.once("connected", () => resolve()), {
             durationMs: connectTimeoutMs,
             errorMsg: "container connect() timeout",
         });
-    }
     assert.strictEqual(container.attachState, AttachState.Attached, "Container is not attached after attach is called");
-    const presence = getPresence(container);
     return {
         client,
         container,
-        presence,
         services,
         containerId,
+        connected,
     };
 };
 function createSendFunction() {
@@ -110,21 +114,6 @@ function createSendFunction() {
     throw new Error("process.send is not defined");
 }
 const send = createSendFunction();
-function sendAttendeeConnected(attendee) {
-    send({
-        event: "attendeeConnected",
-        attendeeId: attendee.attendeeId,
-    });
-}
-function sendAttendeeDisconnected(attendee) {
-    send({
-        event: "attendeeDisconnected",
-        attendeeId: attendee.attendeeId,
-    });
-}
-function isConnected(container) {
-    return container !== undefined && container.connectionState === ConnectionState.Connected;
-}
 function isStringOrNumberRecord(value) {
     if (value === null || typeof value !== "object" || Array.isArray(value)) {
         return false;
@@ -145,11 +134,58 @@ function isStringOrNumberRecord(value) {
 const WorkspaceSchema = {};
 class MessageHandler {
     constructor() {
+        this.log = [];
         this.workspaces = new Map();
+        this.sendAttendeeConnected = (attendee) => {
+            this.send({
+                event: "attendeeConnected",
+                attendeeId: attendee.attendeeId,
+            });
+        };
+        this.sendAttendeeDisconnected = (attendee) => {
+            this.send({
+                event: "attendeeDisconnected",
+                attendeeId: attendee.attendeeId,
+            });
+        };
+        this.logger = {
+            send: (event, logLevel) => {
+                const interest = telemetryEventInterestLevel(event.eventName);
+                if (interest === "none") {
+                    return;
+                }
+                this.log.push({
+                    timestamp: Date.now(),
+                    agentId: process_id,
+                    eventCategory: "telemetry",
+                    eventName: event.eventName,
+                    details: typeof event.details === "string" ? event.details : JSON.stringify(event.details),
+                });
+                if (verbosity.includes("telem")) {
+                    selectiveVerboseLog(event, logLevel);
+                }
+            },
+        };
+        this.onDisconnected = () => {
+            // Test state is a bit fragile and does not account for reconnections.
+            this.send({ event: "error", error: `${process_id}: Container disconnected` });
+        };
+    }
+    send(msg) {
+        this.log.push({
+            timestamp: Date.now(),
+            agentId: process_id,
+            eventCategory: "messageSent",
+            eventName: msg.event,
+            details: msg.event === "debugReportComplete" && msg.log
+                ? JSON.stringify({ logLength: msg.log.length })
+                : JSON.stringify(msg),
+        });
+        send(msg);
     }
     registerWorkspace(workspaceId, options) {
         if (!this.presence) {
-            send({ event: "error", error: `${process_id} is not connected to presence` });
+            this.send({ event: "error", error: `${process_id} is not connected to presence` });
             return;
         }
         const { latest, latestMap } = options;
@@ -160,7 +196,7 @@ class MessageHandler {
             // TODO: AB#47518
             const latestState = workspace.states.latest;
             latestState.events.on("remoteUpdated", (update) => {
-                send({
+                this.send({
                     event: "latestValueUpdated",
                     workspaceId,
                     attendeeId: update.attendee.attendeeId,
@@ -168,7 +204,7 @@ class MessageHandler {
                 });
             });
             for (const remote of latestState.getRemotes()) {
-                send({
+                this.send({
                     event: "latestValueUpdated",
                     workspaceId,
                     attendeeId: remote.attendee.attendeeId,
@@ -185,7 +221,7 @@ class MessageHandler {
             const latestMapState = workspace.states.latestMap;
             latestMapState.events.on("remoteUpdated", (update) => {
                 for (const [key, valueWithMetadata] of update.items) {
-                    send({
+                    this.send({
                         event: "latestMapValueUpdated",
                         workspaceId,
                         attendeeId: update.attendee.attendeeId,
@@ -196,7 +232,7 @@ class MessageHandler {
             });
             for (const remote of latestMapState.getRemotes()) {
                 for (const [key, valueWithMetadata] of remote.items) {
-                    send({
+                    this.send({
                         event: "latestMapValueUpdated",
                         workspaceId,
                         attendeeId: remote.attendee.attendeeId,
@@ -207,7 +243,7 @@ class MessageHandler {
             }
         }
         this.workspaces.set(workspaceId, workspace);
-        send({
+        this.send({
             event: "workspaceRegistered",
             workspaceId,
             latest: latest ?? false,
@@ -216,15 +252,33 @@ class MessageHandler {
     }
     async onMessage(msg) {
         if (verbosity.includes("msgs")) {
+            this.log.push({
+                timestamp: Date.now(),
+                agentId: process_id,
+                eventCategory: "messageReceived",
+                eventName: msg.command,
+            });
             console.log(`[${process_id}] Received`, msg);
         }
+        if (msg.command === "ping") {
+            this.handlePing();
+            return;
+        }
+        if (msg.command === "connect") {
+            await this.handleConnect(msg);
+            return;
+        }
+        // All other message must wait if connect is in progress
+        if (this.msgQueue !== undefined) {
+            this.msgQueue.push(msg);
+            return;
+        }
+        this.processMessage(msg);
+    }
+    processMessage(msg) {
         switch (msg.command) {
-            case "ping": {
-                this.handlePing();
-                break;
-            }
-            case "connect": {
-                await this.handleConnect(msg);
+            case "debugReport": {
+                this.handleDebugReport(msg);
                 break;
             }
             case "disconnectSelf": {
@@ -255,74 +309,125 @@ class MessageHandler {
                 break;
             }
             default: {
-                console.error(`${process_id}: Unknown command`);
-                send({ event: "error", error: `${process_id} Unknown command` });
+                console.error(`${process_id}: Unknown command:`, msg);
+                this.send({
+                    event: "error",
+                    error: `${process_id} Unknown command: ${JSON.stringify(msg)}`,
+                });
             }
         }
     }
     handlePing() {
-        send({ event: "ack" });
+        this.send({ event: "ack" });
     }
     async handleConnect(msg) {
         if (!msg.user) {
-            send({ event: "error", error: `${process_id}: No azure user information given` });
+            this.send({ event: "error", error: `${process_id}: No azure user information given` });
             return;
         }
-        if (isConnected(this.container)) {
-            send({ event: "error", error: `${process_id}: Already connected to container` });
+        if (this.container) {
+            this.send({ event: "error", error: `${process_id}: Container already loaded` });
             return;
         }
-        const { container, presence, containerId } = await getOrCreatePresenceContainer(msg.containerId, msg.user, msg.scopes, msg.createScopes);
-        this.container = container;
-        this.presence = presence;
-        this.containerId = containerId;
-        // Acknowledge connection before sending current attendee information
-        send({
-            event: "connected",
-            containerId,
-            attendeeId: presence.attendees.getMyself().attendeeId,
-        });
-        // Send existing attendees excluding self to parent/orchestrator
-        const self = presence.attendees.getMyself();
-        for (const attendee of presence.attendees.getAttendees()) {
-            if (attendee !== self) {
-                sendAttendeeConnected(attendee);
+        // Prevent reentrance. Queue messages until after connect is fully processed.
+        this.msgQueue = [];
+        try {
+            const { container, containerId, connected } = await getOrCreateContainer({
+                ...msg,
+                logger: this.logger,
+                onDisconnected: this.onDisconnected,
+            });
+            this.container = container;
+            const presence = getPresence(container);
+            this.presence = presence;
+            // wait for 'ConnectionState.Connected'
+            await connected;
+            // Acknowledge connection before sending current attendee information
+            this.send({
+                event: "connected",
+                containerId,
+                attendeeId: presence.attendees.getMyself().attendeeId,
+            });
+            // Send existing attendees excluding self to parent/orchestrator
+            const self = presence.attendees.getMyself();
+            for (const attendee of presence.attendees.getAttendees()) {
+                if (attendee !== self && attendee.getConnectionStatus() === "Connected") {
+                    this.sendAttendeeConnected(attendee);
+                }
+            }
+            // Listen for presence events to notify parent/orchestrator when a new attendee joins or leaves the session.
+            presence.attendees.events.on("attendeeConnected", this.sendAttendeeConnected);
+            presence.attendees.events.on("attendeeDisconnected", this.sendAttendeeDisconnected);
+        }
+        finally {
+            // Process any queued messages received while connecting
+            for (const queuedMsg of this.msgQueue) {
+                this.processMessage(queuedMsg);
+            }
+            this.msgQueue = undefined;
+        }
+    }
+    handleDebugReport(msg) {
+        if (msg.reportAttendees) {
+            if (this.presence) {
+                const attendees = this.presence.attendees.getAttendees();
+                let connectedCount = 0;
+                for (const attendee of attendees) {
+                    if (attendee.getConnectionStatus() === "Connected") {
+                        connectedCount++;
+                    }
+                }
+                console.log(`[${process_id}] Report: ${attendees.size} attendees, ${connectedCount} connected`);
+            }
+            else {
+                this.send({ event: "error", error: `${process_id} is not connected to presence` });
             }
         }
-        // Listen for presence events to notify parent/orchestrator when a new attendee joins or leaves the session.
-        presence.attendees.events.on("attendeeConnected", sendAttendeeConnected);
-        presence.attendees.events.on("attendeeDisconnected", sendAttendeeDisconnected);
+        const debugReport = {
+            event: "debugReportComplete",
+        };
+        if (msg.sendEventLog) {
+            debugReport.log = this.log;
+        }
+        this.send(debugReport);
     }
     handleDisconnectSelf() {
         if (!this.container) {
-            send({ event: "error", error: `${process_id} is not connected to container` });
+            this.send({ event: "error", error: `${process_id} is not connected to container` });
             return;
         }
+        // There are no current scenarios where disconnect without presence is expected.
         if (!this.presence) {
-            send({ event: "error", error: `${process_id} is not connected to presence` });
+            this.send({ event: "error", error: `${process_id} is not connected to presence` });
             return;
         }
+        // Disconnect event is treated as an error in normal handling.
+        // Remove listener as this disconnect is intentional.
+        this.container.off("disconnected", this.onDisconnected);
         this.container.disconnect();
-        send({
+        this.send({
             event: "disconnectedSelf",
             attendeeId: this.presence.attendees.getMyself().attendeeId,
         });
     }
     handleSetLatestValue(msg) {
         if (!this.presence) {
-            send({ event: "error", error: `${process_id} is not connected to presence` });
+            this.send({ event: "error", error: `${process_id} is not connected to presence` });
             return;
         }
         const workspace = this.workspaces.get(msg.workspaceId);
         if (!workspace) {
-            send({ event: "error", error: `${process_id} workspace ${msg.workspaceId} not found` });
+            this.send({
+                event: "error",
+                error: `${process_id} workspace ${msg.workspaceId} not found`,
+            });
             return;
         }
         // Cast required due to optional keys in WorkspaceSchema
         // TODO: AB#47518
         const latestState = workspace.states.latest;
         if (!latestState) {
-            send({
+            this.send({
                 event: "error",
                 error: `${process_id} latest state not registered for workspace ${msg.workspaceId}`,
             });
@@ -335,23 +440,26 @@ class MessageHandler {
     }
     handleSetLatestMapValue(msg) {
         if (!this.presence) {
-            send({ event: "error", error: `${process_id} is not connected to presence` });
+            this.send({ event: "error", error: `${process_id} is not connected to presence` });
             return;
         }
         if (typeof msg.key !== "string") {
-            send({ event: "error", error: `${process_id} invalid key type` });
+            this.send({ event: "error", error: `${process_id} invalid key type` });
             return;
         }
         const workspace = this.workspaces.get(msg.workspaceId);
         if (!workspace) {
-            send({ event: "error", error: `${process_id} workspace ${msg.workspaceId} not found` });
+            this.send({
+                event: "error",
+                error: `${process_id} workspace ${msg.workspaceId} not found`,
+            });
             return;
         }
         // Cast required due to optional keys in WorkspaceSchema
         // TODO: AB#47518
         const latestMapState = workspace.states.latestMap;
         if (!latestMapState) {
-            send({
+            this.send({
                 event: "error",
                 error: `${process_id} latestMap state not registered for workspace ${msg.workspaceId}`,
             });
@@ -364,19 +472,22 @@ class MessageHandler {
     }
     handleGetLatestValue(msg) {
         if (!this.presence) {
-            send({ event: "error", error: `${process_id} is not connected to presence` });
+            this.send({ event: "error", error: `${process_id} is not connected to presence` });
             return;
         }
         const workspace = this.workspaces.get(msg.workspaceId);
         if (!workspace) {
-            send({ event: "error", error: `${process_id} workspace ${msg.workspaceId} not found` });
+            this.send({
+                event: "error",
+                error: `${process_id} workspace ${msg.workspaceId} not found`,
+            });
             return;
         }
         // Cast required due to optional keys in WorkspaceSchema
         // TODO: AB#47518
         const latestState = workspace.states.latest;
         if (!latestState) {
-            send({
+            this.send({
                 event: "error",
                 error: `${process_id} latest state not registered for workspace ${msg.workspaceId}`,
             });
@@ -391,7 +502,7 @@ class MessageHandler {
         else {
             value = latestState.local;
         }
-        send({
+        this.send({
             event: "latestValueGetResponse",
             workspaceId: msg.workspaceId,
             attendeeId: msg.attendeeId,
@@ -400,23 +511,26 @@ class MessageHandler {
     }
     handleGetLatestMapValue(msg) {
         if (!this.presence) {
-            send({ event: "error", error: `${process_id} is not connected to presence` });
+            this.send({ event: "error", error: `${process_id} is not connected to presence` });
             return;
         }
         if (typeof msg.key !== "string") {
-            send({ event: "error", error: `${process_id} invalid key type` });
+            this.send({ event: "error", error: `${process_id} invalid key type` });
             return;
         }
         const workspace = this.workspaces.get(msg.workspaceId);
         if (!workspace) {
-            send({ event: "error", error: `${process_id} workspace ${msg.workspaceId} not found` });
+            this.send({
+                event: "error",
+                error: `${process_id} workspace ${msg.workspaceId} not found`,
+            });
             return;
         }
         // Cast required due to optional keys in WorkspaceSchema
         // TODO: AB#47518
         const latestMapState = workspace.states.latestMap;
         if (!latestMapState) {
-            send({
+            this.send({
                 event: "error",
                 error: `${process_id} latestMap state not registered for workspace ${msg.workspaceId}`,
             });
@@ -432,7 +546,7 @@ class MessageHandler {
         else {
             value = latestMapState.local.get(msg.key);
         }
-        send({
+        this.send({
             event: "latestMapValueGetResponse",
             workspaceId: msg.workspaceId,
             attendeeId: msg.attendeeId,