@memberjunction/server 6.2.0-edge.3 → 6.2.0-edge.4

package/src/resolvers/RealtimeBridgeResolver.ts

@@ -7,8 +7,13 @@ import {
   LiveKitTokenService,
   LiveKitAgentRoomCoordinator,
   LiveKitEgressService,
+  LiveKitParticipantService,
   LiveKitUserIdentity,
   RoomHandoffEngine,
+  type AgentRoomHostOptions,
+  type AgentRoomSession,
+  type ParticipantUpdateResult,
+  type StartAgentRoomSessionParams,
 } from '@memberjunction/livekit-room-server';
 import { AppContext } from '../types.js';
 import { ResolverBase } from '../generic/ResolverBase.js';
@@ -18,7 +23,8 @@ import { AIBridgeEngine } from '@memberjunction/ai-bridge-server';
 import { SessionManager } from '../agentSessions/SessionManager.js';
 import { NotificationEngine } from '@memberjunction/notifications';
 import { RegisterMeetingRecordingFile, CorrelateRecordingStart } from './meetingRecordingRegistration.js';
-import { RoomAuthorizationService } from '@memberjunction/livekit-room-server';
+import { WriteAgentVisionConsentAudit } from './agentVisionConsentAudit.js';
+import { RoomAuthorizationService, type RoomAuthorizationResult } from '@memberjunction/livekit-room-server';
 
 /**
  * Binds the agent realtime-session factory onto the LiveKit room coordinator's model-session creation seam.
@@ -44,6 +50,23 @@ LiveKitAgentRoomCoordinator.Instance.SetTurnToolBinder((session, handler) => {
   runtime.SetLocalToolHandler(handler);
 });
 
+/**
+ * Binds whether an agent watches meetings (same module-load rationale as the factory above). The coordinator asks per
+ * model session; the answer is the agent's `realtime.video.watchMeetings` setting, which the session runtime read from
+ * its effective configuration (`@memberjunction/ai-agents`). The coordinator also requires a model that takes video.
+ */
+LiveKitAgentRoomCoordinator.Instance.SetAgentVisionResolver((session) => GetBridgeRealtimeRuntime(session)?.WatchesMeetingVideo === true);
+
+/**
+ * Binds how a failed agent start is recorded on the agent's co-agent run (same module-load rationale as the factory
+ * above). The coordinator closes the model session of a start that fails before the bridge engine takes it; that close
+ * alone finalizes the run as Completed, so the coordinator first has the session's runtime finalize it as Failed, with the
+ * start's error.
+ */
+LiveKitAgentRoomCoordinator.Instance.SetFailedStartRecorder(async (session, error) => {
+  await GetBridgeRealtimeRuntime(session)?.Finalize(false, error);
+});
+
 /**
  * Binds the co-agent run finalizer onto the bridge engine (same module-load rationale as the factory above).
  * Lets the engine finalize a session's dangling co-agent observability run when it reaps a bridge WITHOUT a
@@ -233,7 +256,30 @@ export class LiveKitRecordingResult {
   RecordingFileID?: string;
 }
 
-/** A selectable provider-native voice for the dev voice picker. */
+/** A person's choice of whether agents may see their camera and shared screen in a room. */
+@InputType()
+export class SetLiveKitAgentVisionInput {
+  @Field(() => String)
+  RoomName: string;
+
+  /** `true` to let agents see, `false` to stop. */
+  @Field(() => Boolean)
+  Allow: boolean;
+}
+
+@ObjectType()
+export class LiveKitAgentVisionResult {
+  @Field(() => Boolean)
+  Success: boolean;
+
+  @Field(() => String, { nullable: true })
+  ErrorMessage?: string;
+}
+
+/**
+ * A selectable provider-native voice for the dev voice picker: one per persona, or one per voice only the driver declares.
+ * Two personas can share an `ID` (a voice, and the same voice with a face); `PersonaID` tells them apart.
+ */
 @ObjectType()
 export class RealtimeVoiceOptionResult {
   @Field(() => String)
@@ -241,6 +287,22 @@ export class RealtimeVoiceOptionResult {
 
   @Field(() => String)
   Name: string;
+
+  /** The persona this voice belongs to (`MJ: AI Personas.ID`). Null for a voice only the driver declares. */
+  @Field(() => String, { nullable: true })
+  PersonaID?: string;
+
+  /**
+   * The avatar that comes with this voice: the `APIName` of its persona's preset Video binding on the model's vendor,
+   * which is what `realtime.video.avatarId` takes. Null for a voice without one, and for every voice only the driver
+   * declares.
+   */
+  @Field(() => String, { nullable: true })
+  AvatarID?: string;
+
+  /** The persona's preview image URL, or null when it has none. */
+  @Field(() => String, { nullable: true })
+  PreviewImageURL?: string;
 }
 
 /** An active Realtime model with the voices its driver supports — feeds the dev model/voice picker. */
@@ -300,7 +362,9 @@ export class RealtimeBridgeResolver extends ResolverBase {
 
   /**
    * Starts (or reuses) an agent's presence in a LiveKit room and returns a client token so the calling
-   * user can immediately join the same room.
+   * user can immediately join the same room. The token is minted before anything starts, so a start that can't give the
+   * user one fails with nothing started. When no `AgentSessionID` is passed, an agent session is created for the
+   * agent; if the agent then fails to start, that session is closed (`Error`) before the failure is returned.
    */
   @Mutation(() => LiveKitAgentRoomSessionResult)
   async StartLiveKitAgentRoomSession(
@@ -331,18 +395,22 @@ export class RealtimeBridgeResolver extends ResolverBase {
         }
       }
 
+      // The user's token first: a start that can't let the user into the room fails before anything starts there. Minted
+      // after the agent joined, a failure left the agent in a room the user couldn't enter and gave the UI no bridge id to
+      // stop it by.
+      const clientToken = await new LiveKitTokenService().MintClientToken(roomName, this.participantIdentity(user), user.Name ?? user.Email);
+
       // Opt-in: an agent that can bring a person, a number or another agent into this room (the same room, so a visitor
       // is escalated without leaving it). Falls through to the standard start when the server cannot do handoffs.
       const handoffSessionBridgeID = await this.startWithHandoff(input, roomName, user, provider);
       if (handoffSessionBridgeID) {
-        const handoffToken = await new LiveKitTokenService().MintClientToken(roomName, this.participantIdentity(user), user.Name ?? user.Email);
         return {
           Success: true,
           SessionBridgeID: handoffSessionBridgeID,
           RoomName: roomName,
-          ServerUrl: handoffToken.ServerUrl,
-          ClientToken: handoffToken.Token,
-          Identity: handoffToken.Identity,
+          ServerUrl: clientToken.ServerUrl,
+          ClientToken: clientToken.Token,
+          Identity: clientToken.Identity,
         };
       }
 
@@ -351,6 +419,10 @@ export class RealtimeBridgeResolver extends ResolverBase {
       // freshly-created one. Previously this minted a bare random UUID with no backing row, so the
       // bridge INSERT failed the FK_AIAgentSessionBridge_Session constraint.
       let agentSessionID = input.AgentSessionID?.trim();
+      // A session created here exists for this agent's time in the room alone, so it closes when the agent leaves, however
+      // it leaves, or at once if the agent fails to start (see startAgentInRoom). A session the caller supplied is the
+      // caller's to close.
+      let createdSessionID: string | undefined;
       if (!agentSessionID) {
         if (!input.AgentID?.trim()) {
           return failure('An AgentID is required to start an agent room session.', roomName);
@@ -361,9 +433,10 @@ export class RealtimeBridgeResolver extends ResolverBase {
           provider,
         );
         agentSessionID = createdSession.ID;
+        createdSessionID = createdSession.ID;
       }
 
-      const session = await LiveKitAgentRoomCoordinator.Instance.StartAgentRoomSession({
+      const session = await this.startAgentInRoom(createdSessionID, user, provider, {
         AgentSessionID: agentSessionID,
         RoomName: roomName,
         AgentID: input.AgentID,
@@ -377,9 +450,6 @@ export class RealtimeBridgeResolver extends ResolverBase {
         MetadataProvider: provider,
       });
 
-      const tokenService = new LiveKitTokenService();
-      const clientToken = await tokenService.MintClientToken(roomName, this.participantIdentity(user), user.Name ?? user.Email);
-
       return {
         Success: true,
         SessionBridgeID: session.SessionBridgeID,
@@ -399,6 +469,7 @@ export class RealtimeBridgeResolver extends ResolverBase {
    * Stops one agent's presence in a room (the bot leaves) — the remove half of in-room agent management.
    * Identified by the `SessionBridgeID` returned from {@link StartLiveKitAgentRoomSession}. Returns `true`
    * when the bridge was stopped. Best-effort: a missing/already-stopped bridge or any error resolves `false`.
+   * The agent session the start created for the agent is closed (`Explicit`) before this returns.
    *
    * @param sessionBridgeID The `MJ: AI Agent Session Bridges` row id of the agent to remove.
    */
@@ -433,6 +504,7 @@ export class RealtimeBridgeResolver extends ResolverBase {
    * coordinator's server-side roster). This is the "End meeting" half of the Zoom-style leave control —
    * usable by any participant, including one who only *joined* the room and never tracked the bridge ids.
    * Returns `true` when the teardown ran (even if the room held zero agents). Best-effort: any error → `false`.
+   * The agent sessions the starts created for the agents are closed (`Explicit`) as each agent leaves.
    *
    * @param roomName The LiveKit room to end.
    */
@@ -672,6 +744,46 @@ export class RealtimeBridgeResolver extends ResolverBase {
     }
   }
 
+  /**
+   * Records whether the current user lets agents see their camera and shared screen in a room. The participant is
+   * worked out from the authenticated user, never sent by the client, so a person can only answer for themselves. To
+   * allow, the room must be one they may enter ({@link RoomAuthorizationService}, as for every room mutation). A
+   * withdrawal skips that check: a cancelled meeting or a failed lookup must not keep an agent watching someone who is
+   * still in the room, and the identity is always the caller's own. LiveKit refuses either way when they are not in it. The choice is set as their `mj.agentCanSee` attribute through LiveKit's server SDK (a
+   * participant's token can't change its own attributes) and audited, applied or not.
+   */
+  @Mutation(() => LiveKitAgentVisionResult)
+  async SetLiveKitAgentVision(
+    @Arg('input', () => SetLiveKitAgentVisionInput) input: SetLiveKitAgentVisionInput,
+    @Ctx() context: AppContext = {} as AppContext,
+  ): Promise<LiveKitAgentVisionResult> {
+    try {
+      const user = this.GetUserFromPayload(context.userPayload);
+      if (!user) {
+        return { Success: false, ErrorMessage: 'Unable to determine current user.' };
+      }
+      const provider = GetReadWriteProvider(context.providers) as unknown as IMetadataProvider;
+      const auth: RoomAuthorizationResult = input.Allow
+        ? await RoomAuthorizationService.Instance.AuthorizeRoomAccess(input.RoomName, user, provider)
+        : { Authorized: true };
+      const result: ParticipantUpdateResult = auth.Authorized
+        ? await new LiveKitParticipantService().SetAgentVision(input.RoomName, this.participantIdentity(user), input.Allow)
+        : { Success: false, ErrorMessage: auth.Reason ?? 'Unauthorized room access.' };
+      await WriteAgentVisionConsentAudit(
+        { User: user, RoomName: input.RoomName, Allow: input.Allow, Applied: result.Success, ErrorMessage: result.ErrorMessage },
+        provider,
+      );
+      if (result.Success) {
+        return { Success: true };
+      }
+      return { Success: false, ErrorMessage: result.NotInRoom ? 'You are not in this room.' : result.ErrorMessage };
+    } catch (error) {
+      const msg = error instanceof Error ? error.message : String(error);
+      LogError(`SetLiveKitAgentVision failed: ${msg}`);
+      return { Success: false, ErrorMessage: msg };
+    }
+  }
+
   /**
    * Starts the agent through the handoff-capable room starter when the caller asked for it and the server can do it.
    * Returns the bridge id, or `undefined` to mean "use the standard start" (not requested, no starter, no agent named).
@@ -704,6 +816,46 @@ export class RealtimeBridgeResolver extends ResolverBase {
     return started.SessionBridgeID;
   }
 
+  /**
+   * Starts the agent in the room through the coordinator. An agent session created for this agent's time in the room
+   * (`createdSessionID`) is this start's to close: once the agent has joined, through the end-of-session hook from
+   * {@link closeSessionWhenAgentLeaves}; when the start fails, at once, as `Error`, through
+   * {@link SessionManager.CloseSessionForFailedStart}, before the error propagates, since the agent never joined and that
+   * hook never runs (#5308). By then the coordinator has closed any model session the start opened, its co-agent run
+   * finalized as failed. A session the caller supplied gets neither: it is the caller's.
+   */
+  private async startAgentInRoom(
+    createdSessionID: string | undefined,
+    user: UserInfo,
+    provider: IMetadataProvider,
+    params: Omit<StartAgentRoomSessionParams, 'Host'>,
+  ): Promise<AgentRoomSession> {
+    const coordinator = LiveKitAgentRoomCoordinator.Instance;
+    if (!createdSessionID) {
+      return coordinator.StartAgentRoomSession(params);
+    }
+    try {
+      return await coordinator.StartAgentRoomSession({ ...params, Host: this.closeSessionWhenAgentLeaves(createdSessionID, user, provider) });
+    } catch (error) {
+      await this.sessionManager.CloseSessionForFailedStart(createdSessionID, user, provider);
+      throw error;
+    }
+  }
+
+  /**
+   * What the coordinator is given for an agent session created for one agent's time in a room: its end-of-session hook
+   * closes the session once the agent's bridge has ended, whatever ended it (the agent stopped, the meeting ended, everyone
+   * left, the model session lost), through {@link SessionManager.CloseSessionForEndedBridge}. The close runs as the user
+   * who started the agent, whoever stops it: the session is theirs.
+   */
+  private closeSessionWhenAgentLeaves(agentSessionID: string, user: UserInfo, provider: IMetadataProvider): AgentRoomHostOptions {
+    return {
+      OnSessionEnded: async (reason) => {
+        await this.sessionManager.CloseSessionForEndedBridge(agentSessionID, reason, user, provider);
+      },
+    };
+  }
+
   /** Builds a stable, lowercased participant identity from the authenticated user (shared with the handoff engine, which watches for it). */
   private participantIdentity(user: UserInfo): string {
     return LiveKitUserIdentity(user.ID);