@memberjunction/ai-agents 6.2.0-edge.2 → 6.2.0-edge.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. package/README.md +6 -0
  2. package/dist/AgentDecisionService.d.ts +8 -1
  3. package/dist/AgentDecisionService.d.ts.map +1 -1
  4. package/dist/AgentDecisionService.js +4 -0
  5. package/dist/AgentDecisionService.js.map +1 -1
  6. package/dist/AgentRunner.d.ts +51 -2
  7. package/dist/AgentRunner.d.ts.map +1 -1
  8. package/dist/AgentRunner.js +121 -23
  9. package/dist/AgentRunner.js.map +1 -1
  10. package/dist/ConversationCompactionManager.d.ts +9 -1
  11. package/dist/ConversationCompactionManager.d.ts.map +1 -1
  12. package/dist/ConversationCompactionManager.js +6 -3
  13. package/dist/ConversationCompactionManager.js.map +1 -1
  14. package/dist/ConversationToolManager.d.ts +13 -2
  15. package/dist/ConversationToolManager.d.ts.map +1 -1
  16. package/dist/ConversationToolManager.js +19 -3
  17. package/dist/ConversationToolManager.js.map +1 -1
  18. package/dist/KnowledgeAgent.d.ts.map +1 -1
  19. package/dist/KnowledgeAgent.js +6 -1
  20. package/dist/KnowledgeAgent.js.map +1 -1
  21. package/dist/MJAIAgentRequestEntityServer.d.ts +6 -0
  22. package/dist/MJAIAgentRequestEntityServer.d.ts.map +1 -1
  23. package/dist/MJAIAgentRequestEntityServer.js +26 -1
  24. package/dist/MJAIAgentRequestEntityServer.js.map +1 -1
  25. package/dist/PayloadFeedbackManager.d.ts +3 -0
  26. package/dist/PayloadFeedbackManager.d.ts.map +1 -1
  27. package/dist/PayloadFeedbackManager.js +2 -1
  28. package/dist/PayloadFeedbackManager.js.map +1 -1
  29. package/dist/agent-run-watchdog.d.ts +65 -5
  30. package/dist/agent-run-watchdog.d.ts.map +1 -1
  31. package/dist/agent-run-watchdog.js +143 -28
  32. package/dist/agent-run-watchdog.js.map +1 -1
  33. package/dist/agent-types/loop-agent-prompt-params.d.ts +7 -0
  34. package/dist/agent-types/loop-agent-prompt-params.d.ts.map +1 -1
  35. package/dist/agent-types/loop-agent-prompt-params.js +1 -0
  36. package/dist/agent-types/loop-agent-prompt-params.js.map +1 -1
  37. package/dist/agent-types/loop-agent-type.d.ts +20 -0
  38. package/dist/agent-types/loop-agent-type.d.ts.map +1 -1
  39. package/dist/agent-types/loop-agent-type.js +73 -2
  40. package/dist/agent-types/loop-agent-type.js.map +1 -1
  41. package/dist/artifact-tools/SearchResultSetToolLibrary.js +4 -4
  42. package/dist/artifact-tools/SearchResultSetToolLibrary.js.map +1 -1
  43. package/dist/base-agent.d.ts +310 -26
  44. package/dist/base-agent.d.ts.map +1 -1
  45. package/dist/base-agent.js +867 -244
  46. package/dist/base-agent.js.map +1 -1
  47. package/dist/client-tool-results.d.ts +45 -0
  48. package/dist/client-tool-results.d.ts.map +1 -0
  49. package/dist/client-tool-results.js +94 -0
  50. package/dist/client-tool-results.js.map +1 -0
  51. package/dist/index.d.ts +9 -0
  52. package/dist/index.d.ts.map +1 -1
  53. package/dist/index.js +9 -0
  54. package/dist/index.js.map +1 -1
  55. package/dist/native-tools/action-tool-builder.d.ts +14 -0
  56. package/dist/native-tools/action-tool-builder.d.ts.map +1 -1
  57. package/dist/native-tools/action-tool-builder.js +62 -7
  58. package/dist/native-tools/action-tool-builder.js.map +1 -1
  59. package/dist/native-tools/control-tools.d.ts +17 -1
  60. package/dist/native-tools/control-tools.d.ts.map +1 -1
  61. package/dist/native-tools/control-tools.js +40 -3
  62. package/dist/native-tools/control-tools.js.map +1 -1
  63. package/dist/prior-turn-tool-result-cache.d.ts +31 -12
  64. package/dist/prior-turn-tool-result-cache.d.ts.map +1 -1
  65. package/dist/prior-turn-tool-result-cache.js +59 -16
  66. package/dist/prior-turn-tool-result-cache.js.map +1 -1
  67. package/dist/realtime/bridge-host-params.d.ts +22 -0
  68. package/dist/realtime/bridge-host-params.d.ts.map +1 -0
  69. package/dist/realtime/bridge-host-params.js +53 -0
  70. package/dist/realtime/bridge-host-params.js.map +1 -0
  71. package/dist/realtime/bridge-realtime-session-factory.d.ts +63 -3
  72. package/dist/realtime/bridge-realtime-session-factory.d.ts.map +1 -1
  73. package/dist/realtime/bridge-realtime-session-factory.js +83 -22
  74. package/dist/realtime/bridge-realtime-session-factory.js.map +1 -1
  75. package/dist/realtime/bridge-realtime-usage-recorder.d.ts +81 -0
  76. package/dist/realtime/bridge-realtime-usage-recorder.d.ts.map +1 -0
  77. package/dist/realtime/bridge-realtime-usage-recorder.js +194 -0
  78. package/dist/realtime/bridge-realtime-usage-recorder.js.map +1 -0
  79. package/dist/realtime/bridge-session-transcript-sink.d.ts +37 -0
  80. package/dist/realtime/bridge-session-transcript-sink.d.ts.map +1 -0
  81. package/dist/realtime/bridge-session-transcript-sink.js +58 -0
  82. package/dist/realtime/bridge-session-transcript-sink.js.map +1 -0
  83. package/dist/realtime/client-only-channel-server.d.ts +83 -0
  84. package/dist/realtime/client-only-channel-server.d.ts.map +1 -0
  85. package/dist/realtime/client-only-channel-server.js +107 -0
  86. package/dist/realtime/client-only-channel-server.js.map +1 -0
  87. package/dist/realtime/identity-verification-channel-server.d.ts +50 -0
  88. package/dist/realtime/identity-verification-channel-server.d.ts.map +1 -0
  89. package/dist/realtime/identity-verification-channel-server.js +66 -0
  90. package/dist/realtime/identity-verification-channel-server.js.map +1 -0
  91. package/dist/realtime/interactive-component-channel-server.d.ts +39 -0
  92. package/dist/realtime/interactive-component-channel-server.d.ts.map +1 -0
  93. package/dist/realtime/interactive-component-channel-server.js +55 -0
  94. package/dist/realtime/interactive-component-channel-server.js.map +1 -0
  95. package/dist/realtime/realtime-avatar-resolution.d.ts +194 -0
  96. package/dist/realtime/realtime-avatar-resolution.d.ts.map +1 -0
  97. package/dist/realtime/realtime-avatar-resolution.js +208 -0
  98. package/dist/realtime/realtime-avatar-resolution.js.map +1 -0
  99. package/dist/realtime/realtime-channel-policy.d.ts +81 -0
  100. package/dist/realtime/realtime-channel-policy.d.ts.map +1 -0
  101. package/dist/realtime/realtime-channel-policy.js +91 -0
  102. package/dist/realtime/realtime-channel-policy.js.map +1 -0
  103. package/dist/realtime/realtime-channel-server-host.d.ts +14 -0
  104. package/dist/realtime/realtime-channel-server-host.d.ts.map +1 -1
  105. package/dist/realtime/realtime-channel-server-host.js +35 -0
  106. package/dist/realtime/realtime-channel-server-host.js.map +1 -1
  107. package/dist/realtime/realtime-client-session-service.d.ts +493 -30
  108. package/dist/realtime/realtime-client-session-service.d.ts.map +1 -1
  109. package/dist/realtime/realtime-client-session-service.js +825 -98
  110. package/dist/realtime/realtime-client-session-service.js.map +1 -1
  111. package/dist/realtime/realtime-coagent-config.d.ts +94 -15
  112. package/dist/realtime/realtime-coagent-config.d.ts.map +1 -1
  113. package/dist/realtime/realtime-coagent-config.js +109 -4
  114. package/dist/realtime/realtime-coagent-config.js.map +1 -1
  115. package/dist/realtime/realtime-coagent-resolution.d.ts +90 -0
  116. package/dist/realtime/realtime-coagent-resolution.d.ts.map +1 -0
  117. package/dist/realtime/realtime-coagent-resolution.js +186 -0
  118. package/dist/realtime/realtime-coagent-resolution.js.map +1 -0
  119. package/dist/realtime/realtime-delegation-narrator.d.ts +97 -0
  120. package/dist/realtime/realtime-delegation-narrator.d.ts.map +1 -0
  121. package/dist/realtime/realtime-delegation-narrator.js +162 -0
  122. package/dist/realtime/realtime-delegation-narrator.js.map +1 -0
  123. package/dist/realtime/realtime-recording-store.d.ts +136 -16
  124. package/dist/realtime/realtime-recording-store.d.ts.map +1 -1
  125. package/dist/realtime/realtime-recording-store.js +505 -39
  126. package/dist/realtime/realtime-recording-store.js.map +1 -1
  127. package/dist/realtime/realtime-session-runner.d.ts +2 -82
  128. package/dist/realtime/realtime-session-runner.d.ts.map +1 -1
  129. package/dist/realtime/realtime-session-runner.js +16 -191
  130. package/dist/realtime/realtime-session-runner.js.map +1 -1
  131. package/dist/realtime/realtime-vendor-resolution.d.ts +9 -0
  132. package/dist/realtime/realtime-vendor-resolution.d.ts.map +1 -1
  133. package/dist/realtime/realtime-vendor-resolution.js +16 -5
  134. package/dist/realtime/realtime-vendor-resolution.js.map +1 -1
  135. package/dist/realtime/realtime-video-output-gate.d.ts +83 -0
  136. package/dist/realtime/realtime-video-output-gate.d.ts.map +1 -0
  137. package/dist/realtime/realtime-video-output-gate.js +79 -0
  138. package/dist/realtime/realtime-video-output-gate.js.map +1 -0
  139. package/dist/rubric-evaluation-agent-runner.js +17 -8
  140. package/dist/rubric-evaluation-agent-runner.js.map +1 -1
  141. package/dist/self-check.d.ts +6 -3
  142. package/dist/self-check.d.ts.map +1 -1
  143. package/dist/self-check.js +3 -3
  144. package/dist/self-check.js.map +1 -1
  145. package/dist/tool-result-format.d.ts +14 -0
  146. package/dist/tool-result-format.d.ts.map +1 -1
  147. package/dist/tool-result-format.js.map +1 -1
  148. package/package.json +19 -19
@@ -29,19 +29,89 @@
29
29
  * @module @memberjunction/ai-agents
30
30
  * @author MemberJunction.com
31
31
  */
32
- import { LogError, LogStatus, RunView } from '@memberjunction/core';
33
- import { MJGlobal, MJLruCache, UUIDsEqual, EscapeSQLString } from '@memberjunction/global';
32
+ import { LogError, LogStatus, RunView, DatabaseProviderBase } from '@memberjunction/core';
33
+ import { MJGlobal, MJLruCache, UUIDsEqual, EscapeSQLString, NormalizeUUID } from '@memberjunction/global';
34
34
  import { ActionEngineServer } from '@memberjunction/actions';
35
35
  import { RunActionParams } from '@memberjunction/actions-base';
36
- import { BaseRealtimeModel, GetAIAPIKey } from '@memberjunction/ai';
37
- import { FormatAppContextNote, ResolvePromptRunUserID } from '@memberjunction/ai-core-plus';
36
+ import { BaseRealtimeModel, GetAIAPIKey, CredentialScopeAllows, IsZeroDataRetention } from '@memberjunction/ai';
37
+ import { FormatAppContextNote, ResolvePromptRunUserID, ClientToolMetadataFromDefinition, ParseAgentSettings, ResolveAppClientToolMetadata, ResolveClientTools } from '@memberjunction/ai-core-plus';
38
38
  import { AIEngine } from '@memberjunction/aiengine';
39
+ import { RealtimeSessionCanShowAvatar, ResolveBridgedAvatarStatus, ResolveRealtimeAvatar, ResolveRealtimeAvatarStatus, WithoutUnseenAvatar, } from './realtime-avatar-resolution.js';
40
+ import { ChooseRealtimeVendorForAvatar, ReadRealtimeVideoOutputRow, RealtimeModelShowsAvatar } from './realtime-video-output-gate.js';
39
41
  import { AgentMemoryContextBuilder } from '../agent-memory-context-builder.js';
40
42
  import { AgentRunner } from '../AgentRunner.js';
43
+ import { AgentRunWatchdog } from '../agent-run-watchdog.js';
44
+ import { DelegationNarrator } from './realtime-delegation-narrator.js';
45
+ import { FilterAllowedAgentsByCanRun } from './realtime-coagent-resolution.js';
46
+ import { BridgeRealtimeUsageRecorder } from './bridge-realtime-usage-recorder.js';
41
47
  import { RealtimeToolBroker, INVOKE_TARGET_AGENT_TOOL_NAME, INVOKE_TARGET_AGENT_DESCRIPTION, BuildRealtimeAgentFraming } from './realtime-tool-broker.js';
42
48
  import { NARRATION_PROMPT_NAME, LEGACY_NARRATION_PROMPT_NAME, ResolveNarrationInstructionsTemplate } from './realtime-narration.js';
43
- import { BuildVoiceMannerSection, BuildAppRealtimeOverridesJson, DeepMergeConfigs, GetModelCatalogSessionSettings, GetDisclosureForTarget, GetNarrationPaceMs, GetProviderVoiceSettings, GetSessionTuningSettings, ResolveEffectiveRealtimeConfig, MatchProviderVoiceSettings, GetDirectActionsConfig, IsActionAllowedForDirectInvocation } from './realtime-coagent-config.js';
44
- import { SelectRealtimeVendorForModel } from './realtime-vendor-resolution.js';
49
+ import { BuildVoiceMannerSection, BuildAppRealtimeOverridesJson, DeepMergeConfigs, FindDeprecatedRealtimeVideoKeys, GetModelCatalogSessionSettings, GetDisclosureForTarget, GetNarrationPaceMs, GetProviderVoiceSettings, GetSessionTuningSettings, GetWatchesMeetingVideo, ResolveEffectiveRealtimeConfig, MatchProviderVoiceSettings, GetDirectActionsConfig, IsActionAllowedForDirectInvocation } from './realtime-coagent-config.js';
50
+ import { ListRealtimeVendorsForModel, SelectRealtimeVendorForModel } from './realtime-vendor-resolution.js';
51
+ import { AIEngineBase, MergeRealtimeUsageRecord } from '@memberjunction/ai-engine-base';
52
+ import { BuildSessionChannelPolicy } from './realtime-channel-policy.js';
53
+ /**
54
+ * How many seconds of avatar video a run may store beyond its elapsed time: usage is client-reported, so the stored
55
+ * output video seconds are capped at the run's elapsed time plus this grace (relays land a few seconds late).
56
+ */
57
+ const VIDEO_SECONDS_GRACE = 30;
58
+ /**
59
+ * How long closing a bridged (server-held) session waits for its model session to close before the session's runs are
60
+ * finalized anyway, in milliseconds. A model session reports its last usage while it closes (a Gemini avatar session,
61
+ * the seconds of video it has not reported yet), and the runs are priced from what was stored when they are finalized.
62
+ */
63
+ export const BRIDGE_SESSION_CLOSE_WAIT_MS = 5_000;
64
+ /**
65
+ * The co-agent and target pairs already named in the deprecated `realtime.video` keys line, so the line is logged once
66
+ * per pair per process rather than on every session.
67
+ */
68
+ const deprecatedVideoKeysLogged = new Set();
69
+ /**
70
+ * How many output video tokens one second of avatar video counts: Gemini counts 258 tokens a frame at 24 frames a
71
+ * second, the rate the per-minute avatar price is derived from. The stored output video tokens are capped at the
72
+ * seconds cap at this rate, because pricing uses them: they leave the cost row's output bucket, and a per-token avatar
73
+ * price prices them. Raise it for a model that counts more.
74
+ */
75
+ const VIDEO_TOKENS_PER_SECOND = 6192;
76
+ /** Runtime handles by their realtime session, so the layer that only holds the session can reach its runtime. */
77
+ const bridgeRuntimes = new WeakMap();
78
+ /**
79
+ * Returns the runtime wired onto a bridged realtime session by
80
+ * {@link RealtimeClientSessionService.WireBridgeRealtimeSession}, or `undefined` for a session that was never
81
+ * wired. Lets a host that only holds the {@link IRealtimeSession} (the telephony services) cancel delegations on
82
+ * barge-in and install its local tool handler.
83
+ */
84
+ export function GetBridgeRealtimeRuntime(session) {
85
+ return bridgeRuntimes.get(session);
86
+ }
87
+ /**
88
+ * Closes a bridged model session, waiting for it up to `waitMs`. Resolves when the close finishes or the wait runs out;
89
+ * rejects when the close fails within the wait. A close still running when the wait runs out is logged (what the
90
+ * session reports from then on is not stored), and so is its failure, should it fail later.
91
+ *
92
+ * @param close The session's own `Close()`.
93
+ * @param waitMs How long to wait for it.
94
+ * @param label Names the session in the log lines (its co-agent prompt run).
95
+ */
96
+ async function closeBridgedSessionWithin(close, waitMs, label) {
97
+ const closing = Promise.resolve(close());
98
+ let timer;
99
+ const waitRanOut = new Promise((resolve) => {
100
+ timer = setTimeout(() => resolve(true), waitMs);
101
+ timer.unref?.();
102
+ });
103
+ try {
104
+ const ranOut = await Promise.race([closing.then(() => false), waitRanOut]);
105
+ if (ranOut) {
106
+ LogError(`[Bridged usage] ${label}: the model session took more than ${waitMs} ms to close; its runs are finalized ` +
107
+ `without waiting, so usage it reports while closing is not stored.`);
108
+ closing.catch((error) => LogError(`[Bridged usage] ${label}: closing the model session failed: ${error instanceof Error ? error.message : String(error)}`));
109
+ }
110
+ }
111
+ finally {
112
+ clearTimeout(timer);
113
+ }
114
+ }
45
115
  /**
46
116
  * Logs when a configuration authored per-provider voice settings but NONE matched the vendor that
47
117
  * actually ran — those settings are dropped, and pre-#3530 that happened with no trace at all: the
@@ -249,7 +319,7 @@ export class RealtimeClientSessionService {
249
319
  if (!prep.Success || !prep.CoAgent || !prep.Resolution || !prep.SessionParams || !prep.EffectiveConfig) {
250
320
  return { Success: false, ErrorMessage: prep.ErrorMessage };
251
321
  }
252
- const { CoAgent: coAgent, Resolution: resolution, SessionParams: sessionParams, EffectiveConfig: effectiveConfig } = prep;
322
+ const { CoAgent: coAgent, Resolution: resolution, SessionParams: sessionParams, EffectiveConfig: effectiveConfig, ClientPolicy: clientPolicy } = prep;
253
323
  if (!resolution.Model.SupportsClientDirect) {
254
324
  return {
255
325
  Success: false,
@@ -264,6 +334,7 @@ export class RealtimeClientSessionService {
264
334
  const message = error instanceof Error ? error.message : String(error);
265
335
  return { Success: false, ErrorMessage: `Failed to mint client realtime session: ${message}` };
266
336
  }
337
+ const avatarStatus = this.resolveMintedAvatarStatus(prep, clientConfig);
267
338
  // Best-effort observability: create a server-side co-agent run (+ prompt run) so the voice
268
339
  // session is visible in the agent-run timeline and delegated runs can nest under it. A
269
340
  // failure here never fails the prepare — we just omit the ids.
@@ -281,9 +352,30 @@ export class RealtimeClientSessionService {
281
352
  DriverClass: resolution.DriverClass,
282
353
  NarrationInstructionsTemplate: this.resolveNarrationInstructionsTemplate() ?? undefined,
283
354
  EffectiveConfig: effectiveConfig,
355
+ ClientPolicy: clientPolicy,
284
356
  NarrationPaceMs: GetNarrationPaceMs(effectiveConfig) ?? undefined,
357
+ ...(avatarStatus ? { AvatarStatus: avatarStatus } : {}),
285
358
  };
286
359
  }
360
+ /**
361
+ * The call's avatar status once the driver has minted the session (see {@link ResolveRealtimeAvatarStatus}), with one
362
+ * log line saying whether the avatar shows. `undefined`, and no line, when the session asked for no avatar.
363
+ */
364
+ resolveMintedAvatarStatus(prep, clientConfig) {
365
+ const resolution = prep.Resolution;
366
+ const apiName = resolution?.APIName ?? clientConfig.Model;
367
+ const status = ResolveRealtimeAvatarStatus({
368
+ Resolution: prep.AvatarResolution ?? {},
369
+ ModelSupportsAvatarOutput: resolution
370
+ ? RealtimeModelShowsAvatar({ ModelID: resolution.ModelID, APIName: apiName, Model: resolution.Model }, AIEngine.Instance)
371
+ : false,
372
+ DriverStatus: clientConfig.AvatarStatus,
373
+ });
374
+ if (status) {
375
+ console.log(`[RealtimeCoAgent] mint avatar model=${apiName} shown=${status.Granted} reason=${status.Reason ?? 'none'}`);
376
+ }
377
+ return status;
378
+ }
287
379
  /**
288
380
  * Wires a **server long-lived (bridged)** realtime session onto the SAME core machinery the
289
381
  * client-direct path uses — so a LiveKit (or future Zoom/Teams) agent does real work and is tracked
@@ -294,12 +386,22 @@ export class RealtimeClientSessionService {
294
386
  * under the co-agent run, supports barge-in cancel + paused-run resume — all of it, for free).
295
387
  *
296
388
  * Responsibilities, in order:
389
+ * 0. Report the avatar status the driver had nothing to say about: a prep that asked the driver for no avatar (a
390
+ * phone call, no face for the model's vendor, an unknown avatar, a Video/Output row that turns video off) leaves
391
+ * the driver nothing to report, and a driver that renders no avatar ignores a request, so the session's
392
+ * `AvatarStatus` says why instead of saying nothing ({@link ResolveBridgedAvatarStatus}).
297
393
  * 1. Create the co-agent observability run (+ prompt run + step) so the voice session shows up in the
298
394
  * agent-run timeline and delegated runs nest under it (best-effort; a failure just omits the ids).
299
- * 2. Wire `session.OnToolCall` → `ExecuteRelayedTool` → `session.SendToolResult`.
300
- * 3. Guarantee finalize-once: wrap `session.Close()` and listen for an unexpected drop (`OnClose`), both
395
+ * 2. Record the session's usage on the co-agent prompt run, as client-direct calls do: `session.OnUsage` →
396
+ * {@link BridgeRealtimeUsageRecorder} → {@link AccumulatePromptRunUsage}, every 10 s and once more at finalize.
397
+ * 3. Wire `session.OnToolCall` → `ExecuteRelayedTool` → `session.SendToolResult`.
398
+ * 4. Guarantee finalize-once: wrap `session.Close()` and listen for an unexpected drop (`OnClose`), both
301
399
  * routed through one idempotent finalizer. The bridge teardown calls `Close()`, so the run finalizes
302
- * on graceful end; a dropped socket finalizes via `OnClose`.
400
+ * on graceful end; a dropped socket finalizes via `OnClose`, and a `Close()` after it waits for that
401
+ * finalize. The wrapped `Close()` closes the model session first (waiting up to
402
+ * {@link BRIDGE_SESSION_CLOSE_WAIT_MS}), so the usage it reports while closing is recorded; the
403
+ * finalizer then writes the session's last usage before it finalizes the runs, so the prompt run is
404
+ * priced from its final counts.
303
405
  *
304
406
  * @param session The live realtime session the bridge owns (from `model.StartSession`).
305
407
  * @param input The same prep input used to build the session (carries AgentSessionID, TargetAgentID, …).
@@ -309,34 +411,126 @@ export class RealtimeClientSessionService {
309
411
  * @returns A {@link BridgeRealtimeRuntime} the bridge holds for the session lifetime.
310
412
  */
311
413
  async WireBridgeRealtimeSession(session, input, prep, contextUser, provider) {
414
+ this.reportBridgedAvatarStatus(session, input, prep);
312
415
  const coAgent = prep.CoAgent;
313
416
  const resolution = prep.Resolution;
314
417
  if (!coAgent || !resolution) {
315
418
  // Prep must have succeeded before wiring; degrade to a tool-error fallback rather than throw.
316
419
  return this.wireBridgeFallbackRuntime(session);
317
420
  }
421
+ // The delegation set the model may reach, narrowed to what THIS run-as user may run (the browser path
422
+ // applies the same filter; without it a bridged call would reach colleagues the caller cannot).
423
+ const allowedAgents = await FilterAllowedAgentsByCanRun(prep.EffectiveConfig?.realtime?.allowedAgents, contextUser);
424
+ // Spoken progress while delegated work runs — the same pacing/wording the generic session runner uses.
425
+ const narrator = new DelegationNarrator({
426
+ GetSession: () => session,
427
+ NarrationInstructionsTemplate: this.resolveNarrationInstructionsTemplate(),
428
+ NarrationPaceMs: GetNarrationPaceMs(prep.EffectiveConfig) ?? undefined,
429
+ });
430
+ let localToolHandler;
318
431
  const promptID = this.resolveCoAgentSystemPrompt(coAgent).PromptID;
319
432
  const obs = await this.createCoAgentObservabilityRun(coAgent, promptID, resolution.ModelID, resolution.VendorID, input.UserID || contextUser?.ID, input.AgentSessionID, contextUser, provider, input.ConversationID);
320
- let finalized = false;
321
- const finalize = async (success) => {
322
- if (finalized) {
323
- return;
324
- }
325
- finalized = true;
326
- await this.FinalizeCoAgentRun(obs?.CoAgentRunID ?? null, obs?.PromptRunID ?? null, contextUser, provider, success, obs?.CoAgentRunStepID ?? null);
433
+ const usage = this.wireBridgeUsage(session, obs?.PromptRunID, contextUser, provider);
434
+ const finalize = this.createBridgeFinalizer(obs, usage, contextUser, provider);
435
+ this.wireBridgeToolCalls(session, {
436
+ Input: input,
437
+ Prep: prep,
438
+ ParentRunID: obs?.CoAgentRunID,
439
+ AllowedAgents: allowedAgents,
440
+ Narrator: narrator,
441
+ LocalToolHandler: () => localToolHandler,
442
+ ContextUser: contextUser,
443
+ Provider: provider,
444
+ });
445
+ this.finalizeOnBridgeClose(session, narrator, finalize, obs?.PromptRunID);
446
+ const runtime = {
447
+ CoAgentRunID: obs?.CoAgentRunID,
448
+ PromptRunID: obs?.PromptRunID,
449
+ Finalize: finalize,
450
+ CancelInFlightDelegations: () => {
451
+ narrator.Cancel(); // a stale "still working on it" line must not be spoken over the caller
452
+ return this.CancelInFlightDelegations(input.AgentSessionID);
453
+ },
454
+ CancelPendingNarration: () => narrator.Cancel(),
455
+ SetLocalToolHandler: (handler) => { localToolHandler = handler; },
456
+ WatchesMeetingVideo: GetWatchesMeetingVideo(prep.EffectiveConfig),
327
457
  };
328
- // Tool calls → the shared delegation entry point, then hand the serialized result back to the model.
458
+ bridgeRuntimes.set(session, runtime);
459
+ return runtime;
460
+ }
461
+ /**
462
+ * Gives a bridged session the avatar status its driver had nothing to say about ({@link ResolveBridgedAvatarStatus}),
463
+ * with one log line: a prep that asked the driver for no avatar (a phone call, no face for the model's vendor, an
464
+ * unknown avatar, a Video/Output row that turns video off) leaves the driver nothing to report, and a driver that
465
+ * renders no avatar ignores a request and reports nothing. A host reads it like any driver's status (the LiveKit room
466
+ * puts it on the agent's bot, so the room shows its notice). A status the driver reported stands.
467
+ */
468
+ reportBridgedAvatarStatus(session, input, prep) {
469
+ const model = prep.Resolution;
470
+ const status = ResolveBridgedAvatarStatus({
471
+ Resolution: prep.AvatarResolution,
472
+ RoomDelivery: input.AvatarDelivery === 'room',
473
+ ModelShowsAvatar: () => (model ? RealtimeModelShowsAvatar(model, AIEngine.Instance) : false),
474
+ DriverStatus: session.AvatarStatus,
475
+ });
476
+ if (!status || status === session.AvatarStatus) {
477
+ return;
478
+ }
479
+ session.AvatarStatus = status;
480
+ const asked = prep.AvatarResolution?.Reason ? 'the driver was asked for none' : 'the driver was asked for one and reported nothing';
481
+ LogStatus(`[RealtimeCoAgent] bridged session avatar: audio only (${status.Reason}); ${asked}.`);
482
+ }
483
+ /**
484
+ * Records a bridged session's usage on its co-agent prompt run. The recorder is the session's only `OnUsage`
485
+ * subscriber: drivers keep one usage handler. Without a prompt run there is nowhere to record, so nothing subscribes
486
+ * and one line is logged, as the client-direct relay logs a dropped delta.
487
+ */
488
+ wireBridgeUsage(session, promptRunID, contextUser, provider) {
489
+ if (!promptRunID) {
490
+ LogStatus('WireBridgeRealtimeSession: the session has no co-agent prompt run, so its usage is not recorded.');
491
+ return undefined;
492
+ }
493
+ const recorder = new BridgeRealtimeUsageRecorder((write) => this.AccumulatePromptRunUsage(promptRunID, write.InputTokens, write.OutputTokens, contextUser, provider, write.Details), `prompt run ${promptRunID}`);
494
+ session.OnUsage((usage) => recorder.Add(usage));
495
+ return recorder;
496
+ }
497
+ /**
498
+ * The bridged session's finalizer. It closes the usage recorder first (no more updates; the unwritten usage is
499
+ * written), so the prompt run's cost, computed when finalize stamps `CompletedAt`, covers the whole session. On a
500
+ * graceful close it runs after the model session closed (see `finalizeOnBridgeClose`).
501
+ *
502
+ * It runs once, and every call returns the first call's promise: whichever path calls second (a `Close()` after a
503
+ * dropped connection's `OnClose` started the finalize) resolves only once the runs are finalized and priced. The
504
+ * first call decides the outcome (see {@link BridgeRealtimeRuntime.Finalize}).
505
+ */
506
+ createBridgeFinalizer(obs, usage, contextUser, provider) {
507
+ let finalizing;
508
+ const finalizeRuns = async (success, errorMessage) => {
509
+ await usage?.Close();
510
+ await this.FinalizeCoAgentRun(obs?.CoAgentRunID ?? null, obs?.PromptRunID ?? null, contextUser, provider, success, obs?.CoAgentRunStepID ?? null, errorMessage);
511
+ };
512
+ return (success, errorMessage) => {
513
+ finalizing ??= finalizeRuns(success, errorMessage);
514
+ return finalizing;
515
+ };
516
+ }
517
+ /** Tool calls → the host's local handler or the shared delegation entry point, then the result back to the model. */
518
+ wireBridgeToolCalls(session, wiring) {
329
519
  session.OnToolCall(async (call) => {
330
520
  try {
331
- const result = await this.ExecuteRelayedTool({
332
- AgentSessionID: input.AgentSessionID,
333
- ParentRunID: obs?.CoAgentRunID,
334
- TargetAgentID: input.TargetAgentID,
335
- AllowedAgents: prep.EffectiveConfig?.realtime?.allowedAgents,
336
- DirectActions: prep.EffectiveConfig?.realtime?.directActions,
337
- Call: call,
338
- }, contextUser, provider);
339
- await session.SendToolResult(call.CallID, result.ResultJson);
521
+ const localHandler = wiring.LocalToolHandler();
522
+ const resultJson = localHandler?.Handles(call.ToolName)
523
+ ? await localHandler.Execute(call)
524
+ : (await wiring.Narrator.Track(() => this.ExecuteRelayedTool({
525
+ AgentSessionID: wiring.Input.AgentSessionID,
526
+ ParentRunID: wiring.ParentRunID,
527
+ TargetAgentID: wiring.Input.TargetAgentID,
528
+ AllowedAgents: wiring.AllowedAgents,
529
+ DirectActions: wiring.Prep.EffectiveConfig?.realtime?.directActions,
530
+ OnProgress: (progress) => wiring.Narrator.HandleProgress(progress),
531
+ Call: call,
532
+ }, wiring.ContextUser, wiring.Provider))).ResultJson;
533
+ await session.SendToolResult(call.CallID, resultJson);
340
534
  }
341
535
  catch (error) {
342
536
  const message = error instanceof Error ? error.message : String(error);
@@ -344,15 +538,34 @@ export class RealtimeClientSessionService {
344
538
  await session.SendToolResult(call.CallID, JSON.stringify({ success: false, error: message }));
345
539
  }
346
540
  });
347
- // Finalize on graceful teardown (the bridge calls Close()) and on an unexpected drop. Both routed
348
- // through the idempotent finalizer, so double-fire is harmless.
541
+ }
542
+ /**
543
+ * Finalizes on graceful teardown (the bridge calls `Close()`) and on an unexpected drop. Both go through the one
544
+ * finalizer, which runs once: a double fire finalizes once, and a `Close()` after a drop waits for the finalize the
545
+ * drop started, so it resolves only once the runs are finalized. Both finalize as completed: a host whose bridge
546
+ * start failed calls {@link BridgeRealtimeRuntime.Finalize} with `false` and the start's error before it closes the
547
+ * session.
548
+ *
549
+ * `Close()` closes the model session before it finalizes: a session reports its last usage while it closes (a
550
+ * Gemini avatar session, the seconds of video not reported yet), and the finalizer stores only what was reported
551
+ * before it ran. The close is waited for up to {@link BRIDGE_SESSION_CLOSE_WAIT_MS}. The runs are finalized whether
552
+ * the close finished, failed or is still running; a failed close still rejects, after the finalize.
553
+ *
554
+ * @param promptRunID The session's co-agent prompt run, which names the session in a log line.
555
+ */
556
+ finalizeOnBridgeClose(session, narrator, finalize, promptRunID) {
349
557
  const originalClose = session.Close.bind(session);
558
+ const label = promptRunID ? `prompt run ${promptRunID}` : 'a session with no co-agent prompt run';
350
559
  session.Close = async () => {
351
- await finalize(true);
352
- await originalClose();
560
+ narrator.Cancel(); // nothing more is spoken on a session that is closing
561
+ try {
562
+ await closeBridgedSessionWithin(originalClose, BRIDGE_SESSION_CLOSE_WAIT_MS, label);
563
+ }
564
+ finally {
565
+ await finalize(true);
566
+ }
353
567
  };
354
568
  session.OnClose?.(() => { void finalize(true); });
355
- return { CoAgentRunID: obs?.CoAgentRunID, PromptRunID: obs?.PromptRunID, Finalize: finalize };
356
569
  }
357
570
  /**
358
571
  * Degenerate {@link BridgeRealtimeRuntime} for the rare case wiring is attempted without a resolved
@@ -363,7 +576,15 @@ export class RealtimeClientSessionService {
363
576
  session.OnToolCall((call) => {
364
577
  void session.SendToolResult(call.CallID, JSON.stringify({ success: false, error: 'Tool execution is unavailable — the co-agent did not resolve. Let the user know.' }));
365
578
  });
366
- return { Finalize: async () => { } };
579
+ const runtime = {
580
+ Finalize: async () => { },
581
+ CancelInFlightDelegations: () => 0,
582
+ CancelPendingNarration: () => { },
583
+ SetLocalToolHandler: () => { },
584
+ WatchesMeetingVideo: false,
585
+ };
586
+ bridgeRuntimes.set(session, runtime);
587
+ return runtime;
367
588
  }
368
589
  /**
369
590
  * **The single source of truth for realtime session prep.** Builds the {@link RealtimeSessionParams}
@@ -397,13 +618,157 @@ export class RealtimeClientSessionService {
397
618
  const targetAgent = this.resolveTargetAgent(input.TargetAgentID);
398
619
  const appSettingsJson = await this.resolveAppRealtimeOverrides(input.ApplicationID, contextUser, provider);
399
620
  const effectiveConfig = this.resolveEffectiveConfig(coAgent, input.ConfigOverridesJson, targetAgent, appSettingsJson);
621
+ this.logDeprecatedVideoKeys(effectiveConfig, coAgent, targetAgent);
400
622
  const outcome = await this.resolveModelForSession(input, coAgent, effectiveConfig);
401
623
  if (!outcome.Resolution) {
402
624
  return { Success: false, ErrorMessage: outcome.ErrorMessage ?? this.noModelMessage() };
403
625
  }
404
626
  const resolution = outcome.Resolution;
405
- const sessionParams = await this.buildSessionParams(input, coAgent, resolution.APIName, contextUser, provider, effectiveConfig, resolution.DriverClass, resolution.ModelID, resolution.ModelVendorID);
406
- return { Success: true, CoAgent: coAgent, Resolution: resolution, EffectiveConfig: effectiveConfig, SessionParams: sessionParams };
627
+ let hostTools = input.HostTools;
628
+ if (input.ResolveHostTools) {
629
+ const dynamicHostTools = input.ResolveHostTools({
630
+ ModelID: resolution.ModelID,
631
+ ModelVendorID: resolution.ModelVendorID,
632
+ DriverClass: resolution.DriverClass,
633
+ });
634
+ if (dynamicHostTools !== undefined) {
635
+ hostTools = dynamicHostTools;
636
+ }
637
+ }
638
+ const effectiveInput = hostTools !== input.HostTools ? { ...input, HostTools: hostTools } : input;
639
+ // One answer for the whole session: it lowers channel exposure here and keeps the driver from
640
+ // turning on provider features that store session data (Gemini session resumption).
641
+ const zeroDataRetention = this.modelHasZeroDataRetention(resolution.ModelID, resolution.ModelVendorID);
642
+ // Channel scoping + client-tool tiers: narrows the declared tools to the scope's decision and
643
+ // folds the app tier into the capability manifest the prompt renders. The scoped input is what
644
+ // the prompt/tool builders see, so a vetoed channel is absent from the framing as well as the tools.
645
+ const scoped = await this.scopeSessionInput(effectiveInput, effectiveConfig, contextUser, provider, zeroDataRetention);
646
+ // A session nobody would see an avatar in (a phone call, an app without agent video) asks the driver for none
647
+ // and keeps the persona's voice.
648
+ const avatar = WithoutUnseenAvatar(this.ResolveSessionAvatar(scoped.Input, coAgent, effectiveConfig, resolution.ModelID, resolution.ModelVendorID), scoped.Input);
649
+ const sessionParams = {
650
+ ...(await this.buildSessionParams(scoped.Input, coAgent, resolution.APIName, contextUser, provider, effectiveConfig, resolution.DriverClass, resolution.ModelID, resolution.ModelVendorID, avatar)),
651
+ ZeroDataRetention: zeroDataRetention,
652
+ };
653
+ return {
654
+ Success: true, CoAgent: coAgent, Resolution: resolution, EffectiveConfig: effectiveConfig, SessionParams: sessionParams,
655
+ ClientPolicy: scoped.ClientPolicy, AvatarResolution: avatar,
656
+ };
657
+ }
658
+ /**
659
+ * Applies channel scoping and the client-tool tiers to a prepare input.
660
+ *
661
+ * - **App tier into the prompt.** `Application.AgentSettings.ClientTools` is resolved to metadata
662
+ * and layered, through the unified {@link ResolveClientTools}, beneath the live surface's own
663
+ * manifest in the app-context snapshot — so the co-agent is told about tools the app declares
664
+ * even when the active surface did not republish them. (Only the HOST can run them; the browser
665
+ * reports "declared but not registered" for one it has no handler for.) The agent's static
666
+ * junction tools are deliberately NOT added to the voice prompt — voice prompts are
667
+ * token-sensitive and a tool with no browser handler is noise — but they ride the returned policy
668
+ * so the browser can describe them.
669
+ * - **Channel scoping.** When the browser reported candidates, they are scoped against the
670
+ * registry and the cascade's `channels` section, and the declared tools narrowed to match
671
+ * ({@link BuildSessionChannelPolicy}).
672
+ *
673
+ * @returns The (possibly adjusted) input, and the policy to hand back when candidates were reported.
674
+ */
675
+ async scopeSessionInput(input, effectiveConfig, contextUser, provider, modelHasZeroDataRetention = false) {
676
+ const tiers = await this.resolveSessionClientToolTiers(input.ApplicationID, input.TargetAgentID, contextUser);
677
+ const appContext = this.withAppToolTier(input.AppContext, tiers.App, input.TargetAgentID);
678
+ const candidates = input.ChannelCandidates ?? [];
679
+ if (candidates.length === 0) {
680
+ return { Input: appContext === input.AppContext ? input : { ...input, AppContext: appContext } };
681
+ }
682
+ const outcome = BuildSessionChannelPolicy({
683
+ Candidates: candidates,
684
+ ChannelsConfig: effectiveConfig.realtime?.channels ?? null,
685
+ Registry: this.readChannelRegistry(provider),
686
+ ClientTools: input.ExtraTools,
687
+ ClientToolTiers: tiers,
688
+ ModelHasZeroDataRetention: modelHasZeroDataRetention,
689
+ });
690
+ return {
691
+ Input: { ...input, ExtraTools: outcome.ClientTools, AppContext: appContext },
692
+ ClientPolicy: outcome.Policy,
693
+ };
694
+ }
695
+ /**
696
+ * Whether the session model's effective catalog configuration (type < model < vendor < model-vendor)
697
+ * declares `Privacy.ZeroDataRetention: true`. Fails closed: a model or vendor row that cannot be
698
+ * resolved, or a catalog lookup that throws, reads as "not declared", so an agent that REQUIRES zero
699
+ * data retention loses exposure rather than gaining it. Overridable seam.
700
+ *
701
+ * @param modelID The resolved `MJ: AI Models` id.
702
+ * @param modelVendorID The resolved model-vendor row id (the cascade's most specific layer).
703
+ */
704
+ modelHasZeroDataRetention(modelID, modelVendorID) {
705
+ if (!modelID) {
706
+ return false;
707
+ }
708
+ try {
709
+ return IsZeroDataRetention(AIEngine.Instance.GetEffectiveModelConfiguration(modelID, modelVendorID));
710
+ }
711
+ catch (error) {
712
+ LogError(`RealtimeClientSessionService.modelHasZeroDataRetention failed for model '${modelID}': ${error instanceof Error ? error.message : String(error)}`);
713
+ return false;
714
+ }
715
+ }
716
+ /**
717
+ * Layers the app's client tools beneath the surface's manifest in the app-context snapshot, through
718
+ * the unified resolver (`session > app`, first match by name wins). Returns the SAME snapshot object
719
+ * when nothing changes, so callers can detect "no change" by identity.
720
+ */
721
+ withAppToolTier(appContext, appTools, targetAgentId) {
722
+ if (!appContext || !appTools || appTools.length === 0) {
723
+ return appContext;
724
+ }
725
+ const surfaceTools = appContext.Capabilities?.Tools;
726
+ const tools = ResolveClientTools({ agentId: targetAgentId, sessionTools: surfaceTools, appTools: [...appTools] });
727
+ if (surfaceTools && tools.length === surfaceTools.length) {
728
+ return appContext; // the surface already covered every app tool
729
+ }
730
+ return { ...appContext, Capabilities: { ...appContext.Capabilities, Tools: tools } };
731
+ }
732
+ /**
733
+ * The server's view of the channel registry (`MJ: AI Agent Channels`), read from the request
734
+ * provider's cached {@link AIEngineBase} rows (already configured by the prepare). Overridable seam.
735
+ * Tolerant: an unloaded cache yields an empty registry (every candidate then reads as having no
736
+ * row), never a throw.
737
+ */
738
+ readChannelRegistry(provider) {
739
+ try {
740
+ const engine = AIEngineBase.GetProviderInstance(provider, AIEngineBase);
741
+ return (engine.AgentChannels ?? []).map(c => ({ Name: c.Name, IsActive: c.IsActive }));
742
+ }
743
+ catch (error) {
744
+ LogError(`RealtimeClientSessionService.readChannelRegistry failed: ${error instanceof Error ? error.message : String(error)}`);
745
+ return [];
746
+ }
747
+ }
748
+ /**
749
+ * Resolves the client-tool tiers the browser is told about at mint: the APP tier
750
+ * (`Application.AgentSettings.ClientTools` resolved against the tool-definition catalog) and the
751
+ * STATIC tier (the target agent's `MJ: AI Agent Client Tools` junction). Overridable seam; tolerant —
752
+ * any failure yields empty tiers and never fails the mint.
753
+ */
754
+ async resolveSessionClientToolTiers(applicationId, targetAgentId, contextUser) {
755
+ const tiers = {};
756
+ try {
757
+ const settings = await this.loadAppAgentSettings(applicationId, contextUser);
758
+ const definitions = AIEngine.Instance.ClientToolDefinitions ?? [];
759
+ const app = ResolveAppClientToolMetadata(settings?.ClientTools, definitions, ref => LogStatus(`RealtimeClientSessionService: the app's client tool '${ref.Name ?? ref.ClientToolDefinitionID}' matches no tool definition — skipping it.`));
760
+ if (app.length > 0) {
761
+ tiers.App = app;
762
+ }
763
+ const stat = targetAgentId ? AIEngine.Instance.GetClientToolsForAgent(targetAgentId).map(ClientToolMetadataFromDefinition) : [];
764
+ if (stat.length > 0) {
765
+ tiers.Static = stat;
766
+ }
767
+ }
768
+ catch (error) {
769
+ LogError(`RealtimeClientSessionService.resolveSessionClientToolTiers failed: ${error instanceof Error ? error.message : String(error)}`);
770
+ }
771
+ return tiers;
407
772
  }
408
773
  /**
409
774
  * Resolves the EFFECTIVE realtime configuration via the surface-agnostic precedence cascade:
@@ -422,6 +787,27 @@ export class RealtimeClientSessionService {
422
787
  resolveEffectiveConfig(coAgent, overridesJson, targetAgent, appSettingsJson) {
423
788
  return ResolveEffectiveRealtimeConfig(this.getAgentTypeDefaultConfiguration(coAgent), coAgent.TypeConfiguration ?? null, overridesJson ?? null, targetAgent?.TypeConfiguration ?? null, appSettingsJson ?? null);
424
789
  }
790
+ /**
791
+ * Logs one line, once per co-agent and target pair per process, when the effective configuration sets
792
+ * `realtime.video.provider` or `realtime.video.providers`. Both are deprecated and nothing reads them, so an author
793
+ * who set them would otherwise get nothing and no word of it.
794
+ *
795
+ * @param config The effective configuration.
796
+ * @param coAgent The co-agent.
797
+ * @param targetAgent The voiced agent, when there is one.
798
+ */
799
+ logDeprecatedVideoKeys(config, coAgent, targetAgent) {
800
+ const keys = FindDeprecatedRealtimeVideoKeys(config);
801
+ const pair = `${NormalizeUUID(coAgent.ID)}|${NormalizeUUID(targetAgent?.ID)}`;
802
+ if (keys.length === 0 || deprecatedVideoKeysLogged.has(pair)) {
803
+ return;
804
+ }
805
+ deprecatedVideoKeysLogged.add(pair);
806
+ const voiced = targetAgent && !UUIDsEqual(targetAgent.ID, coAgent.ID) ? `'${targetAgent.Name}' (co-agent '${coAgent.Name}')` : `'${coAgent.Name}'`;
807
+ const what = keys.length === 1 ? `${keys[0]}, which is deprecated and does nothing` : `${keys.join(' and ')}, which are deprecated and do nothing`;
808
+ LogStatus(`[RealtimeCoAgent] The realtime configuration for ${voiced} sets ${what}. Choose the model with ` +
809
+ "realtime.modelPreference, and set an avatar's settings on its persona's Video binding (VendorSettings.Avatar).");
810
+ }
425
811
  /**
426
812
  * Resolves the APP cascade layer JSON from `Application.AgentSettings`: the `Realtime` overrides
427
813
  * (persona/disclosure/model) plus `RelevantAgents` mapped to the union-accumulated allowed-agent
@@ -438,6 +824,30 @@ export class RealtimeClientSessionService {
438
824
  async resolveAppRealtimeOverrides(applicationId, contextUser,
439
825
  // eslint-disable-next-line @typescript-eslint/no-unused-vars
440
826
  provider) {
827
+ const settings = await this.loadAppAgentSettings(applicationId, contextUser);
828
+ if (!settings) {
829
+ return null;
830
+ }
831
+ const relevantAgents = (settings.RelevantAgents ?? []).map(r => {
832
+ const entry = { agentId: r.AgentID };
833
+ if (r.Label) {
834
+ entry.label = r.Label;
835
+ }
836
+ if (r.Disclosure === 'silent' || r.Disclosure === 'mention' || r.Disclosure === 'hand-voice') {
837
+ entry.disclosure = r.Disclosure;
838
+ }
839
+ return entry;
840
+ });
841
+ return BuildAppRealtimeOverridesJson(settings.Realtime ?? null, relevantAgents);
842
+ }
843
+ /**
844
+ * Reads and parses an application's `AgentSettings` (one tiny by-ID read, served from the provider's
845
+ * RunView cache on repeat). The RAW column is parsed with {@link ParseAgentSettings} rather than read
846
+ * through the CodeGen-generated `AgentSettingsObject` accessor, so a field added to the settings
847
+ * interface works the moment the code reading it ships, before CodeGen next refreshes the copy.
848
+ * Tolerant — a blank id, a failed read, a missing app, or an app with no usable settings returns `null`.
849
+ */
850
+ async loadAppAgentSettings(applicationId, contextUser) {
441
851
  const appId = applicationId?.trim();
442
852
  if (!appId) {
443
853
  return null;
@@ -453,24 +863,10 @@ export class RealtimeClientSessionService {
453
863
  if (!result.Success || !result.Results || result.Results.length === 0) {
454
864
  return null;
455
865
  }
456
- const settings = result.Results[0].AgentSettingsObject;
457
- if (!settings) {
458
- return null;
459
- }
460
- const relevantAgents = (settings.RelevantAgents ?? []).map(r => {
461
- const entry = { agentId: r.AgentID };
462
- if (r.Label) {
463
- entry.label = r.Label;
464
- }
465
- if (r.Disclosure === 'silent' || r.Disclosure === 'mention' || r.Disclosure === 'hand-voice') {
466
- entry.disclosure = r.Disclosure;
467
- }
468
- return entry;
469
- });
470
- return BuildAppRealtimeOverridesJson(settings.Realtime ?? null, relevantAgents);
866
+ return ParseAgentSettings(result.Results[0].AgentSettings);
471
867
  }
472
868
  catch (error) {
473
- LogError(`RealtimeClientSessionService.resolveAppRealtimeOverrides failed for app '${appId}': ${error instanceof Error ? error.message : String(error)}`);
869
+ LogError(`RealtimeClientSessionService.loadAppAgentSettings failed for app '${appId}': ${error instanceof Error ? error.message : String(error)}`);
474
870
  return null;
475
871
  }
476
872
  }
@@ -544,11 +940,30 @@ export class RealtimeClientSessionService {
544
940
  run.UserID = userID;
545
941
  }
546
942
  if (await run.Save()) {
943
+ this.KeepCoAgentRunAlive(run.ID, provider, contextUser);
547
944
  return run.ID;
548
945
  }
549
946
  LogError(`RealtimeClientSessionService.createCoAgentRun save failed: ${run.LatestResult?.CompleteMessage ?? 'unknown error'}`);
550
947
  return null;
551
948
  }
949
+ /**
950
+ * Keeps a voice session's co-agent run alive as far as the {@link AgentRunWatchdog} is concerned. The
951
+ * run spans the whole call, but no agent loop owns it, so nothing stamped its heartbeat: the watchdog
952
+ * force-failed every call that ran past ~5 minutes ("no liveness heartbeat … owning process presumed
953
+ * dead") while the call carried on. Called when the run is created and again on each persisted
954
+ * session heartbeat (`SessionManager`), so whichever server instance the session is talking to keeps
955
+ * it fresh; the watchdog drops it once it is finalized. Only a database provider can stamp heartbeats —
956
+ * any other provider is a no-op, as for every agent run.
957
+ *
958
+ * @param coAgentRunID The session's co-agent run id (from its `Config`), or nothing.
959
+ * @param provider The request-scoped metadata provider.
960
+ * @param contextUser The user the heartbeat writes run as.
961
+ */
962
+ KeepCoAgentRunAlive(coAgentRunID, provider, contextUser) {
963
+ if (coAgentRunID && provider instanceof DatabaseProviderBase) {
964
+ AgentRunWatchdog.Instance.Track(coAgentRunID, provider, contextUser);
965
+ }
966
+ }
552
967
  /**
553
968
  * Creates the co-agent `AIPromptRun` row (Status `Running`) linked to the co-agent run via
554
969
  * `AgentRunID` AND to the co-agent itself via `AgentID` — so the run shows up both on the
@@ -626,11 +1041,59 @@ export class RealtimeClientSessionService {
626
1041
  * @param provider The request-scoped metadata provider.
627
1042
  * @param success Whether the session ended successfully (controls Completed vs Failed).
628
1043
  * @param coAgentRunStepID The co-agent run's single `MJ: AI Agent Run Steps` row id, or `null` to skip.
1044
+ * @param errorMessage Why the session failed (for example the bridge start's error), stamped as `ErrorMessage` on each
1045
+ * record it fails. Ignored when `success` is true. Without it the step gets a generic line and the runs none.
629
1046
  */
630
- async FinalizeCoAgentRun(coAgentRunID, promptRunID, contextUser, provider, success = true, coAgentRunStepID = null) {
631
- await this.finalizeAgentRun(coAgentRunID, contextUser, provider, success);
632
- await this.finalizePromptRun(promptRunID, contextUser, provider, success);
633
- await this.finalizeRunStep(coAgentRunStepID, contextUser, provider, success);
1047
+ async FinalizeCoAgentRun(coAgentRunID, promptRunID, contextUser, provider, success = true, coAgentRunStepID = null, errorMessage) {
1048
+ const failure = success ? undefined : errorMessage?.trim() || undefined;
1049
+ try {
1050
+ await this.finalizeAgentRun(coAgentRunID, contextUser, provider, success, failure);
1051
+ await this.finalizePromptRun(promptRunID, contextUser, provider, success, failure);
1052
+ await this.finalizeRunStep(coAgentRunStepID, contextUser, provider, success, failure);
1053
+ }
1054
+ finally {
1055
+ // Even when a finalize step throws, the run still owes its cost, and the watchdog must stop
1056
+ // treating it as alive — or a run stuck at Running would be kept fresh indefinitely.
1057
+ await this.rollUpCoAgentRunUsage(coAgentRunID, promptRunID, contextUser, provider);
1058
+ if (coAgentRunID) {
1059
+ AgentRunWatchdog.Instance.Untrack(coAgentRunID);
1060
+ }
1061
+ }
1062
+ }
1063
+ /**
1064
+ * Copies the co-agent prompt run's tokens and cost onto the co-agent run. The realtime model's usage
1065
+ * accumulates on the prompt run ({@link AccumulatePromptRunUsage}), which prices itself; the run's
1066
+ * own `TotalCost` / `Total*TokensUsed` stayed 0, so everything that sums agent runs — the realtime
1067
+ * analytics dashboard's per-session cost among them — left out the voice model entirely and showed
1068
+ * only the delegated runs. Mirrors how an agent loop derives its run totals from its prompt runs.
1069
+ *
1070
+ * Applied whatever the run's status: a run the watchdog already failed, or one a shutdown cancelled,
1071
+ * still owes its cost. Runs after {@link finalizePromptRun}, which waits for in-flight usage writes,
1072
+ * so the copy sees the final counts. Tolerant: logs, never throws.
1073
+ */
1074
+ async rollUpCoAgentRunUsage(coAgentRunID, promptRunID, contextUser, provider) {
1075
+ if (!coAgentRunID || !promptRunID) {
1076
+ return;
1077
+ }
1078
+ try {
1079
+ const promptRun = await provider.GetEntityObject('MJ: AI Prompt Runs', contextUser);
1080
+ const run = await provider.GetEntityObject('MJ: AI Agent Runs', contextUser);
1081
+ if (!(await promptRun.Load(promptRunID)) || !(await run.Load(coAgentRunID))) {
1082
+ return;
1083
+ }
1084
+ const promptTokens = promptRun.TokensPrompt ?? 0;
1085
+ const completionTokens = promptRun.TokensCompletion ?? 0;
1086
+ run.TotalPromptTokensUsed = promptTokens;
1087
+ run.TotalCompletionTokensUsed = completionTokens;
1088
+ run.TotalTokensUsed = promptRun.TokensUsed ?? promptTokens + completionTokens;
1089
+ run.TotalCost = promptRun.TotalCost ?? promptRun.Cost ?? 0;
1090
+ if (run.Dirty && !(await run.Save())) {
1091
+ LogError(`RealtimeClientSessionService.rollUpCoAgentRunUsage save failed: ${run.LatestResult?.CompleteMessage ?? 'unknown error'}`);
1092
+ }
1093
+ }
1094
+ catch (error) {
1095
+ LogError(`RealtimeClientSessionService.rollUpCoAgentRunUsage failed: ${error instanceof Error ? error.message : String(error)}`);
1096
+ }
634
1097
  }
635
1098
  /**
636
1099
  * Finalizes the **co-agent observability run(s)** for an agent session that were left `Running` because
@@ -717,8 +1180,11 @@ export class RealtimeClientSessionService {
717
1180
  }
718
1181
  return { PromptRunID: promptRuns.Results[0]?.ID ?? null, StepID: stepID };
719
1182
  }
720
- /** Loads + finalizes the co-agent `AIAgentRun` if still `Running`. Tolerant: logs, never throws. */
721
- async finalizeAgentRun(coAgentRunID, contextUser, provider, success) {
1183
+ /**
1184
+ * Loads + finalizes the co-agent `AIAgentRun` if still `Running`, with `failure` as its `ErrorMessage` when it fails.
1185
+ * Tolerant: logs, never throws.
1186
+ */
1187
+ async finalizeAgentRun(coAgentRunID, contextUser, provider, success, failure) {
722
1188
  if (!coAgentRunID) {
723
1189
  return;
724
1190
  }
@@ -729,16 +1195,20 @@ export class RealtimeClientSessionService {
729
1195
  run.Status = success ? 'Completed' : 'Failed';
730
1196
  run.CompletedAt = new Date();
731
1197
  run.Success = success;
1198
+ if (failure) {
1199
+ run.ErrorMessage = failure;
1200
+ }
732
1201
  if (!(await run.Save())) {
733
1202
  LogError(`RealtimeClientSessionService.finalizeAgentRun save failed: ${run.LatestResult?.CompleteMessage ?? 'unknown error'}`);
734
1203
  }
735
1204
  }
736
1205
  /**
737
1206
  * Loads + finalizes the co-agent run's single system-prompt `MJ: AI Agent Run Steps` row if
738
- * still `Running` (Status `Completed`/`Failed`, `CompletedAt`, `Success`). Tolerant: a
739
- * missing/already-finalized step is a no-op; a load/save failure is logged, never thrown.
1207
+ * still `Running` (Status `Completed`/`Failed`, `CompletedAt`, `Success`; a failed step's `ErrorMessage` is
1208
+ * `failure`, or a generic line without one). Tolerant: a missing/already-finalized step is a no-op; a
1209
+ * load/save failure is logged, never thrown.
740
1210
  */
741
- async finalizeRunStep(coAgentRunStepID, contextUser, provider, success) {
1211
+ async finalizeRunStep(coAgentRunStepID, contextUser, provider, success, failure) {
742
1212
  if (!coAgentRunStepID) {
743
1213
  return;
744
1214
  }
@@ -751,7 +1221,7 @@ export class RealtimeClientSessionService {
751
1221
  step.CompletedAt = new Date();
752
1222
  step.Success = success;
753
1223
  if (!success) {
754
- step.ErrorMessage = 'The realtime session ended in an error state.';
1224
+ step.ErrorMessage = failure ?? 'The realtime session ended in an error state.';
755
1225
  }
756
1226
  if (!(await step.Save())) {
757
1227
  LogError(`RealtimeClientSessionService.finalizeRunStep save failed: ${step.LatestResult?.CompleteMessage ?? 'unknown error'}`);
@@ -761,8 +1231,11 @@ export class RealtimeClientSessionService {
761
1231
  LogError(`RealtimeClientSessionService.finalizeRunStep failed: ${error instanceof Error ? error.message : String(error)}`);
762
1232
  }
763
1233
  }
764
- /** Loads + finalizes the co-agent `AIPromptRun` if still `Running`. Tolerant: logs, never throws. */
765
- async finalizePromptRun(promptRunID, contextUser, provider, success) {
1234
+ /**
1235
+ * Loads + finalizes the co-agent `AIPromptRun` if still `Running`, with `failure` as its `ErrorMessage` when it fails.
1236
+ * Tolerant: logs, never throws.
1237
+ */
1238
+ async finalizePromptRun(promptRunID, contextUser, provider, success, failure) {
766
1239
  if (!promptRunID) {
767
1240
  return;
768
1241
  }
@@ -776,6 +1249,9 @@ export class RealtimeClientSessionService {
776
1249
  run.Status = success ? 'Completed' : 'Failed';
777
1250
  run.CompletedAt = new Date();
778
1251
  run.Success = success;
1252
+ if (failure) {
1253
+ run.ErrorMessage = failure;
1254
+ }
779
1255
  if (!(await run.Save())) {
780
1256
  LogError(`RealtimeClientSessionService.finalizePromptRun save failed: ${run.LatestResult?.CompleteMessage ?? 'unknown error'}`);
781
1257
  }
@@ -839,12 +1315,19 @@ export class RealtimeClientSessionService {
839
1315
  * high-frequency usage checkpoint never overwrites freshly-appended transcript turns (and vice-versa).
840
1316
  * Best-effort: load/save failures log and return `false`, never throw.
841
1317
  *
1318
+ * The per-modality details add into the usage record the run keeps in `ModelSpecificResponseDetails`
1319
+ * (`RealtimeUsage.Input` / `.Output`, which pricing reads at finalize); every other key there is kept. The
1320
+ * stored output video seconds never exceed the run's elapsed time plus 30 seconds, and the stored output video
1321
+ * tokens never exceed 6,192 a second of that: usage is client-reported, and avatar video carries a price, per
1322
+ * minute or per token.
1323
+ *
842
1324
  * @param promptRunID The co-agent observability prompt run.
843
1325
  * @param inputDelta Input-token delta to add (caller clamps to >= 0).
844
1326
  * @param outputDelta Output-token delta to add (caller clamps to >= 0).
1327
+ * @param details Per-modality usage to add (input and output blocks, avatar video seconds included).
845
1328
  * @returns `true` when the accumulated usage was persisted.
846
1329
  */
847
- async AccumulatePromptRunUsage(promptRunID, inputDelta, outputDelta, contextUser, provider) {
1330
+ async AccumulatePromptRunUsage(promptRunID, inputDelta, outputDelta, contextUser, provider, details) {
848
1331
  return this.serializePromptRunWrite(promptRunID, async () => {
849
1332
  try {
850
1333
  const promptRun = await provider.GetEntityObject('MJ: AI Prompt Runs', contextUser);
@@ -855,6 +1338,9 @@ export class RealtimeClientSessionService {
855
1338
  promptRun.TokensPrompt = (promptRun.TokensPrompt ?? 0) + inputDelta;
856
1339
  promptRun.TokensCompletion = (promptRun.TokensCompletion ?? 0) + outputDelta;
857
1340
  promptRun.TokensUsed = (promptRun.TokensPrompt ?? 0) + (promptRun.TokensCompletion ?? 0);
1341
+ if (details) {
1342
+ this.mergeUsageDetails(promptRun, details);
1343
+ }
858
1344
  if (!(await promptRun.Save())) {
859
1345
  LogError(`AccumulatePromptRunUsage: prompt run ${promptRunID} save failed: ${promptRun.LatestResult?.CompleteMessage ?? 'unknown error'}`);
860
1346
  return false;
@@ -867,6 +1353,37 @@ export class RealtimeClientSessionService {
867
1353
  }
868
1354
  });
869
1355
  }
1356
+ /**
1357
+ * Adds relayed per-modality usage into the run's usage record, with the output video seconds capped at the run's
1358
+ * elapsed time plus {@link VIDEO_SECONDS_GRACE}, and the output video tokens at that many seconds of
1359
+ * {@link VIDEO_TOKENS_PER_SECOND}. Details that are not a JSON object are left alone (logged).
1360
+ */
1361
+ mergeUsageDetails(promptRun, details) {
1362
+ const runAt = promptRun.RunAt ? new Date(promptRun.RunAt).getTime() : Number.NaN;
1363
+ const elapsedSeconds = Math.max(0, (Date.now() - runAt) / 1000);
1364
+ const maxVideoSeconds = Number.isFinite(elapsedSeconds) ? elapsedSeconds + VIDEO_SECONDS_GRACE : undefined;
1365
+ const merged = MergeRealtimeUsageRecord(promptRun.ModelSpecificResponseDetails, details, {
1366
+ MaxOutputVideoSeconds: maxVideoSeconds,
1367
+ MaxOutputVideoTokens: maxVideoSeconds === undefined ? undefined : maxVideoSeconds * VIDEO_TOKENS_PER_SECOND,
1368
+ });
1369
+ if (!merged) {
1370
+ LogError(`AccumulatePromptRunUsage: prompt run ${promptRun.ID} has details that are not a JSON object — usage details dropped.`);
1371
+ return;
1372
+ }
1373
+ this.logClampedVideo(promptRun.ID, merged);
1374
+ promptRun.ModelSpecificResponseDetails = merged.Details;
1375
+ }
1376
+ /** One log line for each output video quantity the merge capped. */
1377
+ logClampedVideo(promptRunID, merged) {
1378
+ if (merged.ClampedVideoSeconds > 0) {
1379
+ LogStatus(`AccumulatePromptRunUsage: prompt run ${promptRunID} reported ${merged.ClampedVideoSeconds} s more avatar video than ` +
1380
+ `the run's elapsed time plus ${VIDEO_SECONDS_GRACE} s allows; the stored seconds are capped.`);
1381
+ }
1382
+ if (merged.ClampedVideoTokens > 0) {
1383
+ LogStatus(`AccumulatePromptRunUsage: prompt run ${promptRunID} reported ${merged.ClampedVideoTokens} more avatar video tokens than ` +
1384
+ `${VIDEO_TOKENS_PER_SECOND} a second of the run's elapsed time plus ${VIDEO_SECONDS_GRACE} s allows; the stored tokens are capped.`);
1385
+ }
1386
+ }
870
1387
  /** Parses the prompt run's `Messages` JSON into a mutable chat-message array (tolerant: `[]` on empty/malformed). */
871
1388
  parsePromptRunMessages(raw) {
872
1389
  if (!raw || !raw.trim()) {
@@ -1030,12 +1547,19 @@ export class RealtimeClientSessionService {
1030
1547
  * {@link resolveConfiguredModelPreference}. METADATA preferences degrade gracefully — an
1031
1548
  * unsatisfiable preference logs and FALLS THROUGH to the default (mirroring the co-agent
1032
1549
  * resolution chain's tolerant metadata steps), it never breaks calls.
1033
- * - Without either: the existing default behavior via {@link resolveRealtimeModel}
1034
- * (highest-PowerRank active Realtime model), with the generic {@link noModelMessage} on failure.
1550
+ * - Without either: the default walk ({@link resolveDefaultRealtimeModel}): for a co-agent with video on, in a session
1551
+ * that could show an avatar (a browser session, or a server-side one with {@link PrepareClientSessionInput.AvatarDelivery}
1552
+ * `'room'`), the first candidate that shows one ({@link ResolveAvatarRealtimeModel}); else the existing default via
1553
+ * {@link resolveRealtimeModel} (highest-PowerRank active Realtime model), with the generic
1554
+ * {@link noModelMessage} on failure. The two explicit choices above win over the video preference.
1555
+ *
1556
+ * On every path the chosen model's vendor follows whether the session will show an avatar (the video setting is on
1557
+ * and the session could show one): among the model's keyed vendors, one that renders avatars (Vertex AI for Gemini
1558
+ * 3.8 Live) only when it will, else one that renders none; see {@link resolveVendorAndInstantiate}.
1035
1559
  *
1036
1560
  * @param input The prepare-session input (carries the optional preferred model id).
1037
1561
  * @param coAgent The resolved co-agent (threaded to the default-resolution seam).
1038
- * @param effectiveConfig The resolved effective configuration (carries `modelPreference`).
1562
+ * @param effectiveConfig The resolved effective configuration (carries `modelPreference` and the video setting).
1039
1563
  * @returns The resolution outcome (resolution or failure reason).
1040
1564
  */
1041
1565
  async resolveModelForSession(input, coAgent, effectiveConfig) {
@@ -1046,16 +1570,62 @@ export class RealtimeClientSessionService {
1046
1570
  // A resolver that fell back to the environment itself would answer before the seam, so a
1047
1571
  // subclass that overrides the seam would lose to AI_VENDOR_API_KEY__<driver>.
1048
1572
  const resolveRunKey = this.buildRunKeyResolver(input.APIKeys);
1573
+ const scope = input.CredentialScope ?? 'Any';
1574
+ const preferAvatar = effectiveConfig?.realtime?.video?.enabled === true && RealtimeSessionCanShowAvatar(input);
1049
1575
  if (input.PreferredModelID) {
1050
- return this.resolvePreferredRealtimeModel(input.PreferredModelID, resolveRunKey);
1576
+ return this.resolvePreferredRealtimeModel(input.PreferredModelID, resolveRunKey, scope, preferAvatar);
1051
1577
  }
1052
- const fromConfig = this.resolveConfiguredModelPreference(effectiveConfig, resolveRunKey);
1578
+ const fromConfig = this.resolveConfiguredModelPreference(effectiveConfig, resolveRunKey, scope, preferAvatar);
1053
1579
  if (fromConfig) {
1054
1580
  return { Resolution: fromConfig };
1055
1581
  }
1056
- const resolution = await this.resolveRealtimeModel(coAgent, resolveRunKey);
1582
+ const resolution = await this.resolveDefaultRealtimeModel(coAgent, preferAvatar, resolveRunKey, scope);
1057
1583
  return resolution ? { Resolution: resolution } : { ErrorMessage: this.noModelMessage() };
1058
1584
  }
1585
+ /**
1586
+ * The default walk. With `preferAvatar` (the co-agent's video setting is on and the session could show an avatar),
1587
+ * it takes the first candidate that shows one ({@link ResolveAvatarRealtimeModel}). When none does, or without it, it
1588
+ * takes {@link resolveRealtimeModel}'s choice; when one was preferred, one log line says the call stays audio only.
1589
+ */
1590
+ async resolveDefaultRealtimeModel(coAgent, preferAvatar, resolve, credentialScope) {
1591
+ const withAvatar = preferAvatar ? this.ResolveAvatarRealtimeModel(coAgent, resolve, credentialScope) : null;
1592
+ if (withAvatar) {
1593
+ return withAvatar;
1594
+ }
1595
+ const resolution = await this.resolveRealtimeModel(coAgent, resolve, credentialScope, preferAvatar);
1596
+ if (preferAvatar && resolution) {
1597
+ LogStatus('[RealtimeCoAgent] Video is on, but no realtime model with a usable key shows an avatar (its Video/Output row ' +
1598
+ `and its endpoint): using '${resolution.ModelName ?? resolution.APIName}' on ${resolution.DriverClass ?? 'its driver'}, audio only.`);
1599
+ }
1600
+ return resolution;
1601
+ }
1602
+ /**
1603
+ * For a co-agent with video on: the first candidate that shows an avatar and supports client-direct sessions, in the
1604
+ * default walk's order (highest PowerRank first; a model's Active vendor rows by Priority, each with a usable key).
1605
+ * A candidate shows an avatar when its model's Video/Output row allows video or it has none, and its driver renders
1606
+ * avatars for the vendor's API name on its endpoint ({@link RealtimeModelShowsAvatar}). Every keyed vendor of a model
1607
+ * is tried, not only the first. **Overridable seam.**
1608
+ *
1609
+ * @param coAgent The co-agent (reserved for future per-agent model preference).
1610
+ * @param resolve The session's run-scoped key resolver (run keys only; see {@link resolveVendorAndInstantiate}).
1611
+ * @param credentialScope The session's credential scope; see {@link resolveVendorAndInstantiate}.
1612
+ * @returns The resolution, or `null` when no candidate shows an avatar.
1613
+ */
1614
+ ResolveAvatarRealtimeModel(coAgent, resolve, credentialScope = 'Any') {
1615
+ const resolveKey = this.buildKeyChain(resolve, credentialScope);
1616
+ for (const model of this.selectRealtimeModelCandidates(coAgent)) {
1617
+ if (ReadRealtimeVideoOutputRow(model.ID, AIEngine.Instance) === 'unsupported') {
1618
+ continue;
1619
+ }
1620
+ for (const vendor of ListRealtimeVendorsForModel(model.ID)) {
1621
+ const resolution = this.instantiateRealtimeVendor(model, vendor, resolveKey);
1622
+ if (resolution?.Model.SupportsClientDirect && RealtimeModelShowsAvatar(resolution, AIEngine.Instance)) {
1623
+ return resolution;
1624
+ }
1625
+ }
1626
+ }
1627
+ return null;
1628
+ }
1059
1629
  /**
1060
1630
  * The session's run-scoped keys as a resolver: the key for a driver class when the run carries
1061
1631
  * one, else `undefined` — no platform fallback. The fallback belongs to
@@ -1077,9 +1647,11 @@ export class RealtimeClientSessionService {
1077
1647
  *
1078
1648
  * @param effectiveConfig The resolved effective configuration.
1079
1649
  * @param resolve The session's run-scoped key resolver (run keys only; see {@link resolveVendorAndInstantiate}).
1650
+ * @param credentialScope The session's credential scope; see {@link resolveVendorAndInstantiate}.
1651
+ * @param showsAvatar Whether the session will show an avatar; see {@link resolveVendorAndInstantiate}.
1080
1652
  * @returns The resolution, or `null` when no preference is configured or it can't be satisfied.
1081
1653
  */
1082
- resolveConfiguredModelPreference(effectiveConfig, resolve) {
1654
+ resolveConfiguredModelPreference(effectiveConfig, resolve, credentialScope = 'Any', showsAvatar = false) {
1083
1655
  const preference = effectiveConfig?.realtime?.modelPreference;
1084
1656
  if (!preference) {
1085
1657
  return null;
@@ -1095,7 +1667,7 @@ export class RealtimeClientSessionService {
1095
1667
  `'${model.Name}' but it is not an Active Realtime model — falling through to default resolution.`);
1096
1668
  return null;
1097
1669
  }
1098
- const resolution = this.resolveVendorAndInstantiate(model, resolve);
1670
+ const resolution = this.resolveVendorAndInstantiate(model, resolve, credentialScope, showsAvatar);
1099
1671
  if (!resolution) {
1100
1672
  LogError(`RealtimeClientSessionService: configured realtime model preference '${model.Name}' has no usable ` +
1101
1673
  'vendor DriverClass/API key — falling through to default resolution.');
@@ -1128,9 +1700,11 @@ export class RealtimeClientSessionService {
1128
1700
  *
1129
1701
  * @param preferredModelID The `MJ: AI Models.ID` the user chose.
1130
1702
  * @param resolve The session's run-scoped key resolver (run keys only; see {@link resolveVendorAndInstantiate}).
1703
+ * @param credentialScope The session's credential scope; see {@link resolveVendorAndInstantiate}.
1704
+ * @param showsAvatar Whether the session will show an avatar; see {@link resolveVendorAndInstantiate}.
1131
1705
  * @returns The resolution outcome (resolution or a specific failure reason).
1132
1706
  */
1133
- resolvePreferredRealtimeModel(preferredModelID, resolve) {
1707
+ resolvePreferredRealtimeModel(preferredModelID, resolve, credentialScope = 'Any', showsAvatar = false) {
1134
1708
  const model = this.findModelByID(preferredModelID);
1135
1709
  if (!model) {
1136
1710
  return { ErrorMessage: `The requested realtime model (id '${preferredModelID}') was not found in AI model metadata.` };
@@ -1141,7 +1715,7 @@ export class RealtimeClientSessionService {
1141
1715
  if (!this.isRealtimeModel(model)) {
1142
1716
  return { ErrorMessage: `The requested model '${model.Name}' is not a Realtime model (its type is '${model.AIModelType}').` };
1143
1717
  }
1144
- const resolution = this.resolveVendorAndInstantiate(model, resolve);
1718
+ const resolution = this.resolveVendorAndInstantiate(model, resolve, credentialScope, showsAvatar);
1145
1719
  if (!resolution) {
1146
1720
  return {
1147
1721
  ErrorMessage: `The requested model '${model.Name}' has no active vendor with a usable DriverClass/API key ` +
@@ -1174,9 +1748,11 @@ export class RealtimeClientSessionService {
1174
1748
  *
1175
1749
  * @param coAgent The co-agent being voiced (reserved for future per-agent model preference).
1176
1750
  * @param resolve The session's run-scoped key resolver (run keys only; see {@link resolveVendorAndInstantiate}).
1751
+ * @param credentialScope The session's credential scope; see {@link resolveVendorAndInstantiate}.
1752
+ * @param showsAvatar Whether the session will show an avatar; see {@link resolveVendorAndInstantiate}.
1177
1753
  * @returns The resolved model + identifiers, or `null`.
1178
1754
  */
1179
- async resolveRealtimeModel(coAgent, resolve) {
1755
+ async resolveRealtimeModel(coAgent, resolve, credentialScope = 'Any', showsAvatar = false) {
1180
1756
  // Walk candidates in descending PowerRank, returning the FIRST that fully resolves to a usable
1181
1757
  // client-direct driver (active vendor + API key + ClassFactory driver + SupportsClientDirect).
1182
1758
  // Single-pick dead-ended whenever the highest-power model lacked a key or client-direct support
@@ -1184,7 +1760,7 @@ export class RealtimeClientSessionService {
1184
1760
  // surfaced "No usable Realtime model" instead of falling through to a model that works.
1185
1761
  const candidates = this.selectRealtimeModelCandidates(coAgent);
1186
1762
  for (const model of candidates) {
1187
- const resolution = this.resolveVendorAndInstantiate(model, resolve);
1763
+ const resolution = this.resolveVendorAndInstantiate(model, resolve, credentialScope, showsAvatar);
1188
1764
  if (resolution && resolution.Model.SupportsClientDirect) {
1189
1765
  return resolution;
1190
1766
  }
@@ -1203,16 +1779,43 @@ export class RealtimeClientSessionService {
1203
1779
  * @param resolve The session's run-scoped key resolver. Expected to answer with run keys only:
1204
1780
  * one that falls back to the environment itself answers before {@link getAPIKeyForDriver} and
1205
1781
  * so bypasses an override of it.
1782
+ * @param credentialScope A scope that rules out the `'Environment'` source (`'RuntimeOnly'`) drops the
1783
+ * {@link getAPIKeyForDriver} seam: the run's keys are the whole chain, so a vendor they do not key
1784
+ * is never selected on the platform's key.
1785
+ * @param showsAvatar Whether the session will show an avatar. The highest-Priority keyed vendor stands when it
1786
+ * renders avatars ({@link RealtimeModelShowsAvatar}) exactly when the session will show one; otherwise the first
1787
+ * other keyed vendor that does is taken, when there is one. So a vendor that renders avatars (Vertex AI for Gemini
1788
+ * 3.8 Live, with its own billing and limits) takes a session only when it will show an avatar, or when it is the
1789
+ * only keyed vendor; and a session that will show one goes to such a vendor whatever the Priority order.
1206
1790
  * @returns The full resolution, or `null` when no vendor/key/driver can be satisfied.
1207
1791
  */
1208
- resolveVendorAndInstantiate(model, resolve) {
1209
- // The run's keys first, then this service's own seam (which subclasses and tests override) —
1210
- // so a run-scoped credential wins without taking that seam away from anyone who replaced it.
1211
- const resolveKey = (driverClass) => resolve?.(driverClass) ?? this.getAPIKeyForDriver(driverClass);
1792
+ resolveVendorAndInstantiate(model, resolve, credentialScope = 'Any', showsAvatar = false) {
1793
+ const resolveKey = this.buildKeyChain(resolve, credentialScope);
1212
1794
  const vendor = this.selectRealtimeVendor(model.ID, resolveKey);
1213
- if (!vendor) {
1214
- return null;
1215
- }
1795
+ const first = vendor ? this.instantiateRealtimeVendor(model, vendor, resolveKey) : null;
1796
+ if (!vendor || !first) {
1797
+ return first;
1798
+ }
1799
+ return ChooseRealtimeVendorForAvatar(vendor, first, ListRealtimeVendorsForModel(model.ID), showsAvatar, {
1800
+ Instantiate: (other) => this.instantiateRealtimeVendor(model, other, resolveKey),
1801
+ RendersAvatar: (resolution) => RealtimeModelShowsAvatar(resolution, AIEngine.Instance),
1802
+ // A stand-in does client-direct sessions when the first one does.
1803
+ Accepts: (resolution) => resolution.Model.SupportsClientDirect || !first.Model.SupportsClientDirect,
1804
+ });
1805
+ }
1806
+ /**
1807
+ * The session's complete key chain for vendor selection and the mint (see {@link resolveVendorAndInstantiate}): the
1808
+ * run's keys first, then this service's own seam (which subclasses and tests override), so a run-scoped credential
1809
+ * wins without taking that seam away from anyone who replaced it. A scope that rules out the environment has no second
1810
+ * step: the seam's default is the platform key.
1811
+ */
1812
+ buildKeyChain(resolve, credentialScope) {
1813
+ return CredentialScopeAllows(credentialScope, 'Environment')
1814
+ ? (driverClass) => resolve?.(driverClass) ?? this.getAPIKeyForDriver(driverClass)
1815
+ : (driverClass) => resolve?.(driverClass);
1816
+ }
1817
+ /** Instantiates a vendor's realtime driver with its key from the chain; `null` when the key or the driver is missing. */
1818
+ instantiateRealtimeVendor(model, vendor, resolveKey) {
1216
1819
  const apiKey = resolveKey(vendor.DriverClass);
1217
1820
  if (!apiKey) {
1218
1821
  return null;
@@ -1257,7 +1860,9 @@ export class RealtimeClientSessionService {
1257
1860
  }
1258
1861
  /**
1259
1862
  * The active models of AIModelType `Realtime`, sorted highest-PowerRank first — the candidate
1260
- * list {@link resolveRealtimeModel} walks until one yields a usable client-direct driver.
1863
+ * list {@link resolveRealtimeModel} walks until one yields a usable client-direct driver, and
1864
+ * {@link ResolveAvatarRealtimeModel} walks first for a co-agent with video on. Equal PowerRanks
1865
+ * keep the engine's cached order (a stable sort).
1261
1866
  * Returns ALL candidates (not just the top pick) so a keyless or non-client-direct top model
1262
1867
  * falls through to the next usable one instead of dead-ending the whole resolution.
1263
1868
  *
@@ -1308,20 +1913,24 @@ export class RealtimeClientSessionService {
1308
1913
  * @param driverClass The resolved vendor's DriverClass — matches per-provider voice settings.
1309
1914
  * @param modelID The resolved `MJ: AI Models` id — keys the model-catalog `ModelConfiguration` cascade.
1310
1915
  * @param modelVendorID The resolved `MJ: AI Model Vendors` ROW id — the cascade's most-specific layer.
1916
+ * @param resolvedAvatar The session's avatar resolution, when the caller already made it; resolved here otherwise.
1311
1917
  * @returns The assembled session params.
1312
1918
  */
1313
- async buildSessionParams(input, coAgent, modelApiName, contextUser, provider, effectiveConfig, driverClass, modelID, modelVendorID) {
1314
- const directTools = this.BuildDirectActionTools(input.TargetAgentID, effectiveConfig, driverClass, input.AgentSessionID);
1919
+ async buildSessionParams(input, coAgent, modelApiName, contextUser, provider, effectiveConfig, driverClass, modelID, modelVendorID, resolvedAvatar) {
1920
+ const directTools = this.projectDirectActions(input, effectiveConfig, driverClass);
1315
1921
  const hasDirectTools = directTools.length > 0 || (input.ExtraTools != null && input.ExtraTools.length > 0);
1316
1922
  const systemPrompt = await this.buildCompanionSystemPrompt(input, coAgent, contextUser, provider, effectiveConfig, hasDirectTools);
1317
1923
  const memoryContext = await this.assembleMemoryContext(input, coAgent, contextUser, provider);
1318
1924
  const combinedExtra = directTools.length > 0
1319
1925
  ? [...(input.ExtraTools ?? []), ...directTools]
1320
1926
  : input.ExtraTools;
1321
- const tools = this.buildStableToolSet(combinedExtra);
1927
+ const tools = this.appendHostTools(this.buildStableToolSet(combinedExtra), input.HostTools);
1322
1928
  // Hoisted (rather than built inline at the return) so the mint log below can report the voice
1323
- // that ACTUALLY reached the driver — see the `voice=` field. Same bag, built once.
1324
- const configBag = this.buildSessionConfigBag(input, effectiveConfig, driverClass, modelID, modelVendorID);
1929
+ // that ACTUALLY reached the driver — see the `voice=` field. Same bag, built once. A phone call, or
1930
+ // an app without agent video, carries no avatar request whoever resolved it (WithoutUnseenAvatar is
1931
+ // idempotent).
1932
+ const avatar = WithoutUnseenAvatar(resolvedAvatar ?? this.ResolveSessionAvatar(input, coAgent, effectiveConfig, modelID, modelVendorID), input);
1933
+ const configBag = this.withAvatarVoice(this.buildSessionConfigBag(input, effectiveConfig, driverClass, modelID, modelVendorID), avatar, input);
1325
1934
  WarnOnUnmatchedProviderVoice(effectiveConfig, driverClass, 'RealtimeClientSessionService');
1326
1935
  // One line per mint: confirms which tools + whether the channel-direct framing actually reach
1327
1936
  // the model — settles "why does the co-agent delegate instead of calling browser_*" without
@@ -1330,6 +1939,7 @@ export class RealtimeClientSessionService {
1330
1939
  // actually reached the driver legible — the whole failure mode in #3530 was that it was not.
1331
1940
  console.log(`[RealtimeCoAgent] mint model=${modelApiName} driver=${driverClass ?? 'unknown'} ` +
1332
1941
  `voice=${typeof configBag?.['voice'] === 'string' ? configBag['voice'] : 'none'} ` +
1942
+ `avatar=${avatar.Avatar ? avatar.Avatar.AvatarID : avatar.Reason ? `none(${avatar.Reason})` : 'none'} ` +
1333
1943
  `tools=[${tools.map(t => t.Name).join(', ')}] ` +
1334
1944
  `channelExceptionInPrompt=${systemPrompt.includes('interactive-surface')}`);
1335
1945
  return {
@@ -1344,8 +1954,58 @@ export class RealtimeClientSessionService {
1344
1954
  MaxSessionSeconds: input.MaxSessionSeconds,
1345
1955
  UserID: contextUser?.ID,
1346
1956
  HasToolFraming: true,
1957
+ ...(avatar.Avatar ? { Avatar: input.AvatarDelivery ? { ...avatar.Avatar, Delivery: input.AvatarDelivery } : avatar.Avatar } : {}),
1347
1958
  };
1348
1959
  }
1960
+ /**
1961
+ * The live avatar this session asks for, if any: from the voiced agent's persona, or `realtime.video.avatarId`,
1962
+ * when the agent's video setting is on (see `ResolveRealtimeAvatar`). A call that picked an avatar (its runtime
1963
+ * override names `realtime.video.avatarId`) while that setting is off asks for none, with reason `video-off`. A model
1964
+ * whose `MJ: AI Model Modalities`
1965
+ * Video/Output row turns video off asks for none (reason `endpoint`, one log line); otherwise whether the session can
1966
+ * render it is the driver's call. A seam so tests can supply a resolution without the engine's persona metadata.
1967
+ *
1968
+ * @param input The prepare-session input (the voiced agent).
1969
+ * @param coAgent The co-agent.
1970
+ * @param effectiveConfig The effective configuration.
1971
+ * @param modelID The resolved model.
1972
+ * @param modelVendorID The resolved model-vendor row; its vendor scopes the persona bindings.
1973
+ */
1974
+ ResolveSessionAvatar(input, coAgent, effectiveConfig, modelID, modelVendorID) {
1975
+ const videoOn = effectiveConfig?.realtime?.video?.enabled === true;
1976
+ const avatarPicked = !!ResolveEffectiveRealtimeConfig(null, null, input.ConfigOverridesJson).realtime?.video?.avatarId;
1977
+ if (!modelID || (!videoOn && !avatarPicked)) {
1978
+ return {};
1979
+ }
1980
+ const vendorID = modelVendorID ? AIEngine.Instance.ModelVendors.find((mv) => UUIDsEqual(mv.ID, modelVendorID))?.VendorID : undefined;
1981
+ const videoOutputRow = ReadRealtimeVideoOutputRow(modelID, AIEngine.Instance);
1982
+ if (videoOn && videoOutputRow === 'unsupported') {
1983
+ LogStatus(`[RealtimeCoAgent] No avatar asked for: the Video/Output modality row of '${this.findModelByID(modelID)?.Name ?? modelID}' ` +
1984
+ 'turns video off (IsSupported false). The call is audio only. Reason: endpoint.');
1985
+ }
1986
+ return ResolveRealtimeAvatar({
1987
+ EffectiveConfig: effectiveConfig,
1988
+ TargetAgentID: input.TargetAgentID,
1989
+ CoAgentID: coAgent.ID,
1990
+ ModelID: modelID,
1991
+ VendorID: vendorID,
1992
+ VideoOutputRow: videoOutputRow,
1993
+ AvatarPicked: avatarPicked,
1994
+ }, AIEngine.Instance);
1995
+ }
1996
+ /**
1997
+ * The config bag with the avatar persona's voice, so the face and the voice match, unless a voice was picked in this
1998
+ * call (a runtime override's `realtime.voice.default.voice`), which wins. A session nobody would see the avatar in (a
1999
+ * phone call, an app without agent video) keeps the voice without the face, so the agent sounds the same as in a call
2000
+ * that shows it.
2001
+ */
2002
+ withAvatarVoice(bag, avatar, input) {
2003
+ if (!avatar.Voice) {
2004
+ return bag;
2005
+ }
2006
+ const picked = ResolveEffectiveRealtimeConfig(null, null, input.ConfigOverridesJson).realtime?.voice?.default?.voice;
2007
+ return picked ? bag : { ...(bag ?? {}), voice: avatar.Voice };
2008
+ }
1349
2009
  /**
1350
2010
  * Builds the provider-pact `Config` bag for the session: the effective config's resolved voice
1351
2011
  * settings (per {@link GetProviderVoiceSettings} — the persona's agnostic wire-level slots
@@ -1413,6 +2073,7 @@ export class RealtimeClientSessionService {
1413
2073
  const colleagues = this.buildColleaguesFromConfig(effectiveConfig, input.TargetAgentID);
1414
2074
  const framing = BuildRealtimeAgentFraming(targetName, this.buildInteractiveSurfaceFraming(input.ExtraTools), colleagues, hasDirectTools);
1415
2075
  const meetingFraming = this.buildMeetingFraming(input);
2076
+ const hostFraming = input.HostFraming?.trim() ?? '';
1416
2077
  const coAgentPrompt = this.getCoAgentSystemPromptText(coAgent);
1417
2078
  const voiceManner = BuildVoiceMannerSection(effectiveConfig);
1418
2079
  const targetIdentity = this.formatTargetIdentity(target);
@@ -1420,7 +2081,7 @@ export class RealtimeClientSessionService {
1420
2081
  const priorTranscript = this.formatPriorTranscript(input.PriorTranscript);
1421
2082
  const history = this.formatConversationHistory(input.ConversationMessages);
1422
2083
  const memoryContext = await this.assembleMemoryContext(input, coAgent, contextUser, provider);
1423
- return [framing, meetingFraming, coAgentPrompt, voiceManner, targetIdentity, appContextSection, priorTranscript, history, memoryContext]
2084
+ return [framing, meetingFraming, hostFraming, coAgentPrompt, voiceManner, targetIdentity, appContextSection, priorTranscript, history, memoryContext]
1424
2085
  .filter(part => part && part.trim().length > 0)
1425
2086
  .join('\n\n');
1426
2087
  }
@@ -1542,7 +2203,9 @@ export class RealtimeClientSessionService {
1542
2203
  `Knowledge Hub" → ContextTool with action 'NavigateToApp'; "open this record"; "switch to the X tab"). ` +
1543
2204
  `Do NOT route in-app navigation/actions through '${INVOKE_TARGET_AGENT_TOOL_NAME}', and NEVER say you ` +
1544
2205
  `need an active session id or a screen snapshot to do them — calling 'ContextTool' is all that's ` +
1545
- `needed. Reserve '${INVOKE_TARGET_AGENT_TOOL_NAME}' for actual analysis / data work, not navigation.`;
2206
+ `needed. Reserve '${INVOKE_TARGET_AGENT_TOOL_NAME}' for actual analysis / data work, not navigation. ` +
2207
+ `When a channel note lists interactive channels you can use, 'ContextTool' also addresses ONE of ` +
2208
+ `them: add "target": { "channel": "<channel name>" } and use that channel's action names.`;
1546
2209
  }
1547
2210
  return clause;
1548
2211
  }
@@ -1746,6 +2409,21 @@ export class RealtimeClientSessionService {
1746
2409
  }
1747
2410
  return result;
1748
2411
  }
2412
+ /** Appends the host-declared tools to a stable tool set, dropping any whose name is already taken. */
2413
+ appendHostTools(tools, hostTools) {
2414
+ if (!hostTools || hostTools.length === 0) {
2415
+ return tools;
2416
+ }
2417
+ const taken = new Set(tools.map((t) => t.Name.toLowerCase()));
2418
+ const merged = [...tools];
2419
+ for (const tool of hostTools) {
2420
+ if (!taken.has(tool.Name.toLowerCase())) {
2421
+ taken.add(tool.Name.toLowerCase());
2422
+ merged.push(tool);
2423
+ }
2424
+ }
2425
+ return merged;
2426
+ }
1749
2427
  /**
1750
2428
  * Builds the {@link RealtimeToolBroker} for a relayed tool call, wiring `DelegateToTarget` to a
1751
2429
  * target-agent run and `ExecuteTool` to direct-action execution or a structured "not available" placeholder.
@@ -1934,11 +2612,13 @@ export class RealtimeClientSessionService {
1934
2612
  /**
1935
2613
  * Runs (or resumes) the target agent for a delegation. Threads the combined abort signal, parent
1936
2614
  * run linkage, session id, and the `OnProgress` callback so the resolver can stream progress.
2615
+ * The browser session id becomes the run's `sessionID` (client-tool channel) and the app-context
2616
+ * snapshot rides `data.appContext` (prompt context and surface tools); both are optional.
1937
2617
  * When {@link ExecuteRelayedToolInput.ResumeRunID} is set, resumes that paused run via
1938
2618
  * `lastRunId` + `autoPopulateLastRunPayload` (the user's answer continues the same interactive
1939
2619
  * run) instead of starting fresh.
1940
2620
  *
1941
- * @param input The relayed tool input (linkage, progress callback, optional resume id).
2621
+ * @param input The relayed tool input (linkage, progress callback, optional resume id, browser session, app context).
1942
2622
  * @param request The broker's delegation request (call id + arguments + abort signal).
1943
2623
  * @param target The resolved target agent.
1944
2624
  * @param contextUser The calling user.
@@ -1961,6 +2641,8 @@ export class RealtimeClientSessionService {
1961
2641
  cancellationToken: this.combineSignals(request.AbortSignal, input.AbortSignal),
1962
2642
  parentRun: parentRun ?? undefined,
1963
2643
  agentSessionID: input.AgentSessionID,
2644
+ sessionID: input.BrowserSessionID,
2645
+ data: input.AppContext ? { appContext: input.AppContext } : undefined,
1964
2646
  onProgress: input.OnProgress,
1965
2647
  lastRunId: input.ResumeRunID,
1966
2648
  autoPopulateLastRunPayload: input.ResumeRunID ? true : undefined
@@ -2046,11 +2728,25 @@ export class RealtimeClientSessionService {
2046
2728
  const agentActions = (AIEngine.Instance.AgentActions ?? []).filter(aa => UUIDsEqual(aa.AgentID, targetAgentID) && aa.Status === 'Active');
2047
2729
  return (ActionEngineServer.Instance.Actions ?? []).filter(a => agentActions.some(aa => UUIDsEqual(aa.ActionID, a.ID)) && a.Status === 'Active');
2048
2730
  }
2731
+ /**
2732
+ * The session's direct-action tools. When the input excludes direct actions, none, and the empty
2733
+ * projection is recorded so the session's relays run none.
2734
+ */
2735
+ projectDirectActions(input, effectiveConfig, driverClass) {
2736
+ if (input.ExcludeDirectActions) {
2737
+ this.recordEmptyProjection(input.AgentSessionID);
2738
+ return [];
2739
+ }
2740
+ return this.BuildDirectActionTools(input.TargetAgentID, effectiveConfig, driverClass, input.AgentSessionID);
2741
+ }
2049
2742
  /**
2050
2743
  * Builds the projected direct action tool definitions for a target agent if supported by the driver.
2051
2744
  * Gated by driver capability (`SupportsDynamicToolSet`) and explicit configuration opt-in
2052
2745
  * (`Configuration.realtime.directActions.enabled = true`).
2053
2746
  *
2747
+ * With `agentSessionID`, the projection is recorded for the session, even when it is empty: relayed
2748
+ * direct-action calls for the session may run only the projected actions.
2749
+ *
2054
2750
  * @param targetAgentID The target agent ID being voiced.
2055
2751
  * @param effectiveConfig The resolved effective configuration.
2056
2752
  * @param driverClass The resolved vendor's DriverClass.
@@ -2058,10 +2754,12 @@ export class RealtimeClientSessionService {
2058
2754
  */
2059
2755
  BuildDirectActionTools(targetAgentID, effectiveConfig, driverClass, agentSessionID) {
2060
2756
  if (!targetAgentID || !this.driverSupportsDynamicToolSet(driverClass)) {
2757
+ this.recordEmptyProjection(agentSessionID);
2061
2758
  return [];
2062
2759
  }
2063
2760
  const directConfig = GetDirectActionsConfig(effectiveConfig);
2064
2761
  if (!directConfig || !directConfig.enabled) {
2762
+ this.recordEmptyProjection(agentSessionID);
2065
2763
  return [];
2066
2764
  }
2067
2765
  const candidateActions = this.getTargetAgentActions(targetAgentID);
@@ -2080,6 +2778,12 @@ export class RealtimeClientSessionService {
2080
2778
  }
2081
2779
  return tools;
2082
2780
  }
2781
+ /** Records that a session projected no direct actions, so its relays run none. */
2782
+ recordEmptyProjection(agentSessionID) {
2783
+ if (agentSessionID) {
2784
+ this.sessionWireActionMaps.Set(agentSessionID, new Map());
2785
+ }
2786
+ }
2083
2787
  /** @deprecated Use {@link BuildDirectActionTools}. */
2084
2788
  buildDirectActionTools(targetAgentID, effectiveConfig, driverClass, agentSessionID) {
2085
2789
  return this.BuildDirectActionTools(targetAgentID, effectiveConfig, driverClass, agentSessionID);
@@ -2135,6 +2839,13 @@ export class RealtimeClientSessionService {
2135
2839
  * configured timeout. If the tool is unrecognized, disallowed, or target resolution fails,
2136
2840
  * returns a structured "not available" result.
2137
2841
  *
2842
+ * When this service holds the session's start-time projection, only the projected actions may
2843
+ * run, whatever target the input names. When it holds none, the target agent's own metadata
2844
+ * decides: another process prepared the session, the entry was evicted or lost on a restart, or
2845
+ * the session was never started through this service (for example a row created through the
2846
+ * entity API). That fallback skips the driver-capability gate but is still limited to the
2847
+ * target's own direct-actions allowlist.
2848
+ *
2138
2849
  * @param call The non-target tool call.
2139
2850
  * @param input The optional relayed tool input context.
2140
2851
  * @param contextUser The calling user context.
@@ -2156,11 +2867,9 @@ export class RealtimeClientSessionService {
2156
2867
  Output: `Target agent is not available to execute '${call.ToolName}'.`
2157
2868
  };
2158
2869
  }
2159
- const candidateActions = this.getTargetAgentActions(target.ID);
2160
- const candidateWireMap = this.BuildWireActionMap(candidateActions);
2161
- const action = (input?.AgentSessionID ? this.sessionWireActionMaps.Get(input.AgentSessionID)?.get(call.ToolName) : undefined)
2162
- ?? candidateWireMap.get(call.ToolName)
2163
- ?? Array.from(candidateWireMap.entries()).find(([w]) => w.toLowerCase() === call.ToolName.trim().toLowerCase())?.[1];
2870
+ const projection = input?.AgentSessionID ? this.sessionWireActionMaps.Get(input.AgentSessionID) : undefined;
2871
+ const candidateWireMap = this.BuildWireActionMap(this.getTargetAgentActions(target.ID));
2872
+ const action = this.findWireAction(projection, call.ToolName) ?? this.findWireAction(candidateWireMap, call.ToolName);
2164
2873
  if (!action) {
2165
2874
  return {
2166
2875
  CallID: call.CallID,
@@ -2171,7 +2880,7 @@ export class RealtimeClientSessionService {
2171
2880
  const directConfig = input?.DirectActions
2172
2881
  ?? (input?.AgentSessionID ? this.sessionDirectConfigs.Get(input.AgentSessionID) : undefined)
2173
2882
  ?? GetDirectActionsConfig(this.resolveEffectiveConfig(target, undefined, target));
2174
- if (!IsActionAllowedForDirectInvocation(action.Name, directConfig)) {
2883
+ if (!this.isInProjection(projection, action) || !IsActionAllowedForDirectInvocation(action.Name, directConfig)) {
2175
2884
  return {
2176
2885
  CallID: call.CallID,
2177
2886
  Success: false,
@@ -2232,6 +2941,24 @@ export class RealtimeClientSessionService {
2232
2941
  };
2233
2942
  }
2234
2943
  }
2944
+ /**
2945
+ * Finds the action a relayed tool name refers to: the exact wire name first, then a
2946
+ * case-insensitive match.
2947
+ */
2948
+ findWireAction(wireMap, toolName) {
2949
+ if (!wireMap) {
2950
+ return undefined;
2951
+ }
2952
+ const wanted = toolName.trim().toLowerCase();
2953
+ return wireMap.get(toolName) ?? Array.from(wireMap.entries()).find(([wireName]) => wireName.toLowerCase() === wanted)?.[1];
2954
+ }
2955
+ /**
2956
+ * True when the action is in the session's start-time projection, or when this service holds no
2957
+ * projection for the session (see {@link executeNonTargetTool} for when that happens).
2958
+ */
2959
+ isInProjection(projection, action) {
2960
+ return !projection || Array.from(projection.values()).some(projected => UUIDsEqual(projected.ID, action.ID));
2961
+ }
2235
2962
  /**
2236
2963
  * Parses JSON arguments string into a key-value record of parameters.
2237
2964
  * Distinguishes "no arguments" (valid for zero-param action) from "unparseable JSON".