@memberjunction/ai-agents 6.2.0-edge.3 → 6.2.0-edge.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/dist/AgentRunner.d.ts +49 -2
- package/dist/AgentRunner.d.ts.map +1 -1
- package/dist/AgentRunner.js +114 -21
- package/dist/AgentRunner.js.map +1 -1
- package/dist/ConversationCompactionManager.d.ts +2 -0
- package/dist/ConversationCompactionManager.d.ts.map +1 -1
- package/dist/ConversationCompactionManager.js +2 -2
- package/dist/ConversationCompactionManager.js.map +1 -1
- package/dist/ConversationToolManager.d.ts +13 -2
- package/dist/ConversationToolManager.d.ts.map +1 -1
- package/dist/ConversationToolManager.js +19 -3
- package/dist/ConversationToolManager.js.map +1 -1
- package/dist/KnowledgeAgent.d.ts.map +1 -1
- package/dist/KnowledgeAgent.js +6 -1
- package/dist/KnowledgeAgent.js.map +1 -1
- package/dist/MJAIAgentRequestEntityServer.d.ts +6 -0
- package/dist/MJAIAgentRequestEntityServer.d.ts.map +1 -1
- package/dist/MJAIAgentRequestEntityServer.js +26 -1
- package/dist/MJAIAgentRequestEntityServer.js.map +1 -1
- package/dist/agent-run-watchdog.d.ts +65 -5
- package/dist/agent-run-watchdog.d.ts.map +1 -1
- package/dist/agent-run-watchdog.js +143 -28
- package/dist/agent-run-watchdog.js.map +1 -1
- package/dist/agent-types/loop-agent-prompt-params.d.ts +7 -0
- package/dist/agent-types/loop-agent-prompt-params.d.ts.map +1 -1
- package/dist/agent-types/loop-agent-prompt-params.js +1 -0
- package/dist/agent-types/loop-agent-prompt-params.js.map +1 -1
- package/dist/artifact-tools/SearchResultSetToolLibrary.js +4 -4
- package/dist/artifact-tools/SearchResultSetToolLibrary.js.map +1 -1
- package/dist/base-agent.d.ts +275 -18
- package/dist/base-agent.d.ts.map +1 -1
- package/dist/base-agent.js +761 -216
- package/dist/base-agent.js.map +1 -1
- package/dist/client-tool-results.d.ts +45 -0
- package/dist/client-tool-results.d.ts.map +1 -0
- package/dist/client-tool-results.js +94 -0
- package/dist/client-tool-results.js.map +1 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -1
- package/dist/prior-turn-tool-result-cache.d.ts +31 -12
- package/dist/prior-turn-tool-result-cache.d.ts.map +1 -1
- package/dist/prior-turn-tool-result-cache.js +59 -16
- package/dist/prior-turn-tool-result-cache.js.map +1 -1
- package/dist/realtime/bridge-realtime-session-factory.d.ts +44 -2
- package/dist/realtime/bridge-realtime-session-factory.d.ts.map +1 -1
- package/dist/realtime/bridge-realtime-session-factory.js +68 -20
- package/dist/realtime/bridge-realtime-session-factory.js.map +1 -1
- package/dist/realtime/bridge-realtime-usage-recorder.d.ts +81 -0
- package/dist/realtime/bridge-realtime-usage-recorder.d.ts.map +1 -0
- package/dist/realtime/bridge-realtime-usage-recorder.js +194 -0
- package/dist/realtime/bridge-realtime-usage-recorder.js.map +1 -0
- package/dist/realtime/client-only-channel-server.d.ts +83 -0
- package/dist/realtime/client-only-channel-server.d.ts.map +1 -0
- package/dist/realtime/client-only-channel-server.js +107 -0
- package/dist/realtime/client-only-channel-server.js.map +1 -0
- package/dist/realtime/identity-verification-channel-server.d.ts +50 -0
- package/dist/realtime/identity-verification-channel-server.d.ts.map +1 -0
- package/dist/realtime/identity-verification-channel-server.js +66 -0
- package/dist/realtime/identity-verification-channel-server.js.map +1 -0
- package/dist/realtime/interactive-component-channel-server.d.ts +39 -0
- package/dist/realtime/interactive-component-channel-server.d.ts.map +1 -0
- package/dist/realtime/interactive-component-channel-server.js +55 -0
- package/dist/realtime/interactive-component-channel-server.js.map +1 -0
- package/dist/realtime/realtime-avatar-resolution.d.ts +194 -0
- package/dist/realtime/realtime-avatar-resolution.d.ts.map +1 -0
- package/dist/realtime/realtime-avatar-resolution.js +208 -0
- package/dist/realtime/realtime-avatar-resolution.js.map +1 -0
- package/dist/realtime/realtime-channel-policy.d.ts +81 -0
- package/dist/realtime/realtime-channel-policy.d.ts.map +1 -0
- package/dist/realtime/realtime-channel-policy.js +91 -0
- package/dist/realtime/realtime-channel-policy.js.map +1 -0
- package/dist/realtime/realtime-channel-server-host.d.ts +14 -0
- package/dist/realtime/realtime-channel-server-host.d.ts.map +1 -1
- package/dist/realtime/realtime-channel-server-host.js +35 -0
- package/dist/realtime/realtime-channel-server-host.js.map +1 -1
- package/dist/realtime/realtime-client-session-service.d.ts +430 -30
- package/dist/realtime/realtime-client-session-service.d.ts.map +1 -1
- package/dist/realtime/realtime-client-session-service.js +766 -111
- package/dist/realtime/realtime-client-session-service.js.map +1 -1
- package/dist/realtime/realtime-coagent-config.d.ts +94 -15
- package/dist/realtime/realtime-coagent-config.d.ts.map +1 -1
- package/dist/realtime/realtime-coagent-config.js +109 -4
- package/dist/realtime/realtime-coagent-config.js.map +1 -1
- package/dist/realtime/realtime-recording-store.d.ts +136 -16
- package/dist/realtime/realtime-recording-store.d.ts.map +1 -1
- package/dist/realtime/realtime-recording-store.js +505 -39
- package/dist/realtime/realtime-recording-store.js.map +1 -1
- package/dist/realtime/realtime-session-runner.js +4 -4
- package/dist/realtime/realtime-session-runner.js.map +1 -1
- package/dist/realtime/realtime-vendor-resolution.d.ts +9 -0
- package/dist/realtime/realtime-vendor-resolution.d.ts.map +1 -1
- package/dist/realtime/realtime-vendor-resolution.js +16 -5
- package/dist/realtime/realtime-vendor-resolution.js.map +1 -1
- package/dist/realtime/realtime-video-output-gate.d.ts +83 -0
- package/dist/realtime/realtime-video-output-gate.d.ts.map +1 -0
- package/dist/realtime/realtime-video-output-gate.js +79 -0
- package/dist/realtime/realtime-video-output-gate.js.map +1 -0
- package/dist/tool-result-format.d.ts +14 -0
- package/dist/tool-result-format.d.ts.map +1 -1
- package/dist/tool-result-format.js.map +1 -1
- package/package.json +19 -19
|
@@ -29,21 +29,50 @@
|
|
|
29
29
|
* @module @memberjunction/ai-agents
|
|
30
30
|
* @author MemberJunction.com
|
|
31
31
|
*/
|
|
32
|
-
import { LogError, LogStatus, RunView } from '@memberjunction/core';
|
|
33
|
-
import { MJGlobal, MJLruCache, UUIDsEqual, EscapeSQLString } from '@memberjunction/global';
|
|
32
|
+
import { LogError, LogStatus, RunView, DatabaseProviderBase } from '@memberjunction/core';
|
|
33
|
+
import { MJGlobal, MJLruCache, UUIDsEqual, EscapeSQLString, NormalizeUUID } from '@memberjunction/global';
|
|
34
34
|
import { ActionEngineServer } from '@memberjunction/actions';
|
|
35
35
|
import { RunActionParams } from '@memberjunction/actions-base';
|
|
36
|
-
import { BaseRealtimeModel, GetAIAPIKey, CredentialScopeAllows } from '@memberjunction/ai';
|
|
37
|
-
import { FormatAppContextNote, ResolvePromptRunUserID } from '@memberjunction/ai-core-plus';
|
|
36
|
+
import { BaseRealtimeModel, GetAIAPIKey, CredentialScopeAllows, IsZeroDataRetention } from '@memberjunction/ai';
|
|
37
|
+
import { FormatAppContextNote, ResolvePromptRunUserID, ClientToolMetadataFromDefinition, ParseAgentSettings, ResolveAppClientToolMetadata, ResolveClientTools } from '@memberjunction/ai-core-plus';
|
|
38
38
|
import { AIEngine } from '@memberjunction/aiengine';
|
|
39
|
+
import { RealtimeSessionCanShowAvatar, ResolveBridgedAvatarStatus, ResolveRealtimeAvatar, ResolveRealtimeAvatarStatus, WithoutUnseenAvatar, } from './realtime-avatar-resolution.js';
|
|
40
|
+
import { ChooseRealtimeVendorForAvatar, ReadRealtimeVideoOutputRow, RealtimeModelShowsAvatar } from './realtime-video-output-gate.js';
|
|
39
41
|
import { AgentMemoryContextBuilder } from '../agent-memory-context-builder.js';
|
|
40
42
|
import { AgentRunner } from '../AgentRunner.js';
|
|
43
|
+
import { AgentRunWatchdog } from '../agent-run-watchdog.js';
|
|
41
44
|
import { DelegationNarrator } from './realtime-delegation-narrator.js';
|
|
42
45
|
import { FilterAllowedAgentsByCanRun } from './realtime-coagent-resolution.js';
|
|
46
|
+
import { BridgeRealtimeUsageRecorder } from './bridge-realtime-usage-recorder.js';
|
|
43
47
|
import { RealtimeToolBroker, INVOKE_TARGET_AGENT_TOOL_NAME, INVOKE_TARGET_AGENT_DESCRIPTION, BuildRealtimeAgentFraming } from './realtime-tool-broker.js';
|
|
44
48
|
import { NARRATION_PROMPT_NAME, LEGACY_NARRATION_PROMPT_NAME, ResolveNarrationInstructionsTemplate } from './realtime-narration.js';
|
|
45
|
-
import { BuildVoiceMannerSection, BuildAppRealtimeOverridesJson, DeepMergeConfigs, GetModelCatalogSessionSettings, GetDisclosureForTarget, GetNarrationPaceMs, GetProviderVoiceSettings, GetSessionTuningSettings, ResolveEffectiveRealtimeConfig, MatchProviderVoiceSettings, GetDirectActionsConfig, IsActionAllowedForDirectInvocation } from './realtime-coagent-config.js';
|
|
46
|
-
import { SelectRealtimeVendorForModel } from './realtime-vendor-resolution.js';
|
|
49
|
+
import { BuildVoiceMannerSection, BuildAppRealtimeOverridesJson, DeepMergeConfigs, FindDeprecatedRealtimeVideoKeys, GetModelCatalogSessionSettings, GetDisclosureForTarget, GetNarrationPaceMs, GetProviderVoiceSettings, GetSessionTuningSettings, GetWatchesMeetingVideo, ResolveEffectiveRealtimeConfig, MatchProviderVoiceSettings, GetDirectActionsConfig, IsActionAllowedForDirectInvocation } from './realtime-coagent-config.js';
|
|
50
|
+
import { ListRealtimeVendorsForModel, SelectRealtimeVendorForModel } from './realtime-vendor-resolution.js';
|
|
51
|
+
import { AIEngineBase, MergeRealtimeUsageRecord } from '@memberjunction/ai-engine-base';
|
|
52
|
+
import { BuildSessionChannelPolicy } from './realtime-channel-policy.js';
|
|
53
|
+
/**
|
|
54
|
+
* How many seconds of avatar video a run may store beyond its elapsed time: usage is client-reported, so the stored
|
|
55
|
+
* output video seconds are capped at the run's elapsed time plus this grace (relays land a few seconds late).
|
|
56
|
+
*/
|
|
57
|
+
const VIDEO_SECONDS_GRACE = 30;
|
|
58
|
+
/**
|
|
59
|
+
* How long closing a bridged (server-held) session waits for its model session to close before the session's runs are
|
|
60
|
+
* finalized anyway, in milliseconds. A model session reports its last usage while it closes (a Gemini avatar session,
|
|
61
|
+
* the seconds of video it has not reported yet), and the runs are priced from what was stored when they are finalized.
|
|
62
|
+
*/
|
|
63
|
+
export const BRIDGE_SESSION_CLOSE_WAIT_MS = 5_000;
|
|
64
|
+
/**
|
|
65
|
+
* The co-agent and target pairs already named in the deprecated `realtime.video` keys line, so the line is logged once
|
|
66
|
+
* per pair per process rather than on every session.
|
|
67
|
+
*/
|
|
68
|
+
const deprecatedVideoKeysLogged = new Set();
|
|
69
|
+
/**
|
|
70
|
+
* How many output video tokens one second of avatar video counts: Gemini counts 258 tokens a frame at 24 frames a
|
|
71
|
+
* second, the rate the per-minute avatar price is derived from. The stored output video tokens are capped at the
|
|
72
|
+
* seconds cap at this rate, because pricing uses them: they leave the cost row's output bucket, and a per-token avatar
|
|
73
|
+
* price prices them. Raise it for a model that counts more.
|
|
74
|
+
*/
|
|
75
|
+
const VIDEO_TOKENS_PER_SECOND = 6192;
|
|
47
76
|
/** Runtime handles by their realtime session, so the layer that only holds the session can reach its runtime. */
|
|
48
77
|
const bridgeRuntimes = new WeakMap();
|
|
49
78
|
/**
|
|
@@ -55,6 +84,34 @@ const bridgeRuntimes = new WeakMap();
|
|
|
55
84
|
export function GetBridgeRealtimeRuntime(session) {
|
|
56
85
|
return bridgeRuntimes.get(session);
|
|
57
86
|
}
|
|
87
|
+
/**
|
|
88
|
+
* Closes a bridged model session, waiting for it up to `waitMs`. Resolves when the close finishes or the wait runs out;
|
|
89
|
+
* rejects when the close fails within the wait. A close still running when the wait runs out is logged (what the
|
|
90
|
+
* session reports from then on is not stored), and so is its failure, should it fail later.
|
|
91
|
+
*
|
|
92
|
+
* @param close The session's own `Close()`.
|
|
93
|
+
* @param waitMs How long to wait for it.
|
|
94
|
+
* @param label Names the session in the log lines (its co-agent prompt run).
|
|
95
|
+
*/
|
|
96
|
+
async function closeBridgedSessionWithin(close, waitMs, label) {
|
|
97
|
+
const closing = Promise.resolve(close());
|
|
98
|
+
let timer;
|
|
99
|
+
const waitRanOut = new Promise((resolve) => {
|
|
100
|
+
timer = setTimeout(() => resolve(true), waitMs);
|
|
101
|
+
timer.unref?.();
|
|
102
|
+
});
|
|
103
|
+
try {
|
|
104
|
+
const ranOut = await Promise.race([closing.then(() => false), waitRanOut]);
|
|
105
|
+
if (ranOut) {
|
|
106
|
+
LogError(`[Bridged usage] ${label}: the model session took more than ${waitMs} ms to close; its runs are finalized ` +
|
|
107
|
+
`without waiting, so usage it reports while closing is not stored.`);
|
|
108
|
+
closing.catch((error) => LogError(`[Bridged usage] ${label}: closing the model session failed: ${error instanceof Error ? error.message : String(error)}`));
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
finally {
|
|
112
|
+
clearTimeout(timer);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
58
115
|
/**
|
|
59
116
|
* Logs when a configuration authored per-provider voice settings but NONE matched the vendor that
|
|
60
117
|
* actually ran — those settings are dropped, and pre-#3530 that happened with no trace at all: the
|
|
@@ -262,7 +319,7 @@ export class RealtimeClientSessionService {
|
|
|
262
319
|
if (!prep.Success || !prep.CoAgent || !prep.Resolution || !prep.SessionParams || !prep.EffectiveConfig) {
|
|
263
320
|
return { Success: false, ErrorMessage: prep.ErrorMessage };
|
|
264
321
|
}
|
|
265
|
-
const { CoAgent: coAgent, Resolution: resolution, SessionParams: sessionParams, EffectiveConfig: effectiveConfig } = prep;
|
|
322
|
+
const { CoAgent: coAgent, Resolution: resolution, SessionParams: sessionParams, EffectiveConfig: effectiveConfig, ClientPolicy: clientPolicy } = prep;
|
|
266
323
|
if (!resolution.Model.SupportsClientDirect) {
|
|
267
324
|
return {
|
|
268
325
|
Success: false,
|
|
@@ -277,6 +334,7 @@ export class RealtimeClientSessionService {
|
|
|
277
334
|
const message = error instanceof Error ? error.message : String(error);
|
|
278
335
|
return { Success: false, ErrorMessage: `Failed to mint client realtime session: ${message}` };
|
|
279
336
|
}
|
|
337
|
+
const avatarStatus = this.resolveMintedAvatarStatus(prep, clientConfig);
|
|
280
338
|
// Best-effort observability: create a server-side co-agent run (+ prompt run) so the voice
|
|
281
339
|
// session is visible in the agent-run timeline and delegated runs can nest under it. A
|
|
282
340
|
// failure here never fails the prepare — we just omit the ids.
|
|
@@ -294,9 +352,30 @@ export class RealtimeClientSessionService {
|
|
|
294
352
|
DriverClass: resolution.DriverClass,
|
|
295
353
|
NarrationInstructionsTemplate: this.resolveNarrationInstructionsTemplate() ?? undefined,
|
|
296
354
|
EffectiveConfig: effectiveConfig,
|
|
355
|
+
ClientPolicy: clientPolicy,
|
|
297
356
|
NarrationPaceMs: GetNarrationPaceMs(effectiveConfig) ?? undefined,
|
|
357
|
+
...(avatarStatus ? { AvatarStatus: avatarStatus } : {}),
|
|
298
358
|
};
|
|
299
359
|
}
|
|
360
|
+
/**
|
|
361
|
+
* The call's avatar status once the driver has minted the session (see {@link ResolveRealtimeAvatarStatus}), with one
|
|
362
|
+
* log line saying whether the avatar shows. `undefined`, and no line, when the session asked for no avatar.
|
|
363
|
+
*/
|
|
364
|
+
resolveMintedAvatarStatus(prep, clientConfig) {
|
|
365
|
+
const resolution = prep.Resolution;
|
|
366
|
+
const apiName = resolution?.APIName ?? clientConfig.Model;
|
|
367
|
+
const status = ResolveRealtimeAvatarStatus({
|
|
368
|
+
Resolution: prep.AvatarResolution ?? {},
|
|
369
|
+
ModelSupportsAvatarOutput: resolution
|
|
370
|
+
? RealtimeModelShowsAvatar({ ModelID: resolution.ModelID, APIName: apiName, Model: resolution.Model }, AIEngine.Instance)
|
|
371
|
+
: false,
|
|
372
|
+
DriverStatus: clientConfig.AvatarStatus,
|
|
373
|
+
});
|
|
374
|
+
if (status) {
|
|
375
|
+
console.log(`[RealtimeCoAgent] mint avatar model=${apiName} shown=${status.Granted} reason=${status.Reason ?? 'none'}`);
|
|
376
|
+
}
|
|
377
|
+
return status;
|
|
378
|
+
}
|
|
300
379
|
/**
|
|
301
380
|
* Wires a **server long-lived (bridged)** realtime session onto the SAME core machinery the
|
|
302
381
|
* client-direct path uses — so a LiveKit (or future Zoom/Teams) agent does real work and is tracked
|
|
@@ -307,12 +386,22 @@ export class RealtimeClientSessionService {
|
|
|
307
386
|
* under the co-agent run, supports barge-in cancel + paused-run resume — all of it, for free).
|
|
308
387
|
*
|
|
309
388
|
* Responsibilities, in order:
|
|
389
|
+
* 0. Report the avatar status the driver had nothing to say about: a prep that asked the driver for no avatar (a
|
|
390
|
+
* phone call, no face for the model's vendor, an unknown avatar, a Video/Output row that turns video off) leaves
|
|
391
|
+
* the driver nothing to report, and a driver that renders no avatar ignores a request, so the session's
|
|
392
|
+
* `AvatarStatus` says why instead of saying nothing ({@link ResolveBridgedAvatarStatus}).
|
|
310
393
|
* 1. Create the co-agent observability run (+ prompt run + step) so the voice session shows up in the
|
|
311
394
|
* agent-run timeline and delegated runs nest under it (best-effort; a failure just omits the ids).
|
|
312
|
-
* 2.
|
|
313
|
-
*
|
|
395
|
+
* 2. Record the session's usage on the co-agent prompt run, as client-direct calls do: `session.OnUsage` →
|
|
396
|
+
* {@link BridgeRealtimeUsageRecorder} → {@link AccumulatePromptRunUsage}, every 10 s and once more at finalize.
|
|
397
|
+
* 3. Wire `session.OnToolCall` → `ExecuteRelayedTool` → `session.SendToolResult`.
|
|
398
|
+
* 4. Guarantee finalize-once: wrap `session.Close()` and listen for an unexpected drop (`OnClose`), both
|
|
314
399
|
* routed through one idempotent finalizer. The bridge teardown calls `Close()`, so the run finalizes
|
|
315
|
-
* on graceful end; a dropped socket finalizes via `OnClose
|
|
400
|
+
* on graceful end; a dropped socket finalizes via `OnClose`, and a `Close()` after it waits for that
|
|
401
|
+
* finalize. The wrapped `Close()` closes the model session first (waiting up to
|
|
402
|
+
* {@link BRIDGE_SESSION_CLOSE_WAIT_MS}), so the usage it reports while closing is recorded; the
|
|
403
|
+
* finalizer then writes the session's last usage before it finalizes the runs, so the prompt run is
|
|
404
|
+
* priced from its final counts.
|
|
316
405
|
*
|
|
317
406
|
* @param session The live realtime session the bridge owns (from `model.StartSession`).
|
|
318
407
|
* @param input The same prep input used to build the session (carries AgentSessionID, TargetAgentID, …).
|
|
@@ -322,6 +411,7 @@ export class RealtimeClientSessionService {
|
|
|
322
411
|
* @returns A {@link BridgeRealtimeRuntime} the bridge holds for the session lifetime.
|
|
323
412
|
*/
|
|
324
413
|
async WireBridgeRealtimeSession(session, input, prep, contextUser, provider) {
|
|
414
|
+
this.reportBridgedAvatarStatus(session, input, prep);
|
|
325
415
|
const coAgent = prep.CoAgent;
|
|
326
416
|
const resolution = prep.Resolution;
|
|
327
417
|
if (!coAgent || !resolution) {
|
|
@@ -340,28 +430,106 @@ export class RealtimeClientSessionService {
|
|
|
340
430
|
let localToolHandler;
|
|
341
431
|
const promptID = this.resolveCoAgentSystemPrompt(coAgent).PromptID;
|
|
342
432
|
const obs = await this.createCoAgentObservabilityRun(coAgent, promptID, resolution.ModelID, resolution.VendorID, input.UserID || contextUser?.ID, input.AgentSessionID, contextUser, provider, input.ConversationID);
|
|
343
|
-
|
|
344
|
-
const finalize =
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
433
|
+
const usage = this.wireBridgeUsage(session, obs?.PromptRunID, contextUser, provider);
|
|
434
|
+
const finalize = this.createBridgeFinalizer(obs, usage, contextUser, provider);
|
|
435
|
+
this.wireBridgeToolCalls(session, {
|
|
436
|
+
Input: input,
|
|
437
|
+
Prep: prep,
|
|
438
|
+
ParentRunID: obs?.CoAgentRunID,
|
|
439
|
+
AllowedAgents: allowedAgents,
|
|
440
|
+
Narrator: narrator,
|
|
441
|
+
LocalToolHandler: () => localToolHandler,
|
|
442
|
+
ContextUser: contextUser,
|
|
443
|
+
Provider: provider,
|
|
444
|
+
});
|
|
445
|
+
this.finalizeOnBridgeClose(session, narrator, finalize, obs?.PromptRunID);
|
|
446
|
+
const runtime = {
|
|
447
|
+
CoAgentRunID: obs?.CoAgentRunID,
|
|
448
|
+
PromptRunID: obs?.PromptRunID,
|
|
449
|
+
Finalize: finalize,
|
|
450
|
+
CancelInFlightDelegations: () => {
|
|
451
|
+
narrator.Cancel(); // a stale "still working on it" line must not be spoken over the caller
|
|
452
|
+
return this.CancelInFlightDelegations(input.AgentSessionID);
|
|
453
|
+
},
|
|
454
|
+
CancelPendingNarration: () => narrator.Cancel(),
|
|
455
|
+
SetLocalToolHandler: (handler) => { localToolHandler = handler; },
|
|
456
|
+
WatchesMeetingVideo: GetWatchesMeetingVideo(prep.EffectiveConfig),
|
|
457
|
+
};
|
|
458
|
+
bridgeRuntimes.set(session, runtime);
|
|
459
|
+
return runtime;
|
|
460
|
+
}
|
|
461
|
+
/**
|
|
462
|
+
* Gives a bridged session the avatar status its driver had nothing to say about ({@link ResolveBridgedAvatarStatus}),
|
|
463
|
+
* with one log line: a prep that asked the driver for no avatar (a phone call, no face for the model's vendor, an
|
|
464
|
+
* unknown avatar, a Video/Output row that turns video off) leaves the driver nothing to report, and a driver that
|
|
465
|
+
* renders no avatar ignores a request and reports nothing. A host reads it like any driver's status (the LiveKit room
|
|
466
|
+
* puts it on the agent's bot, so the room shows its notice). A status the driver reported stands.
|
|
467
|
+
*/
|
|
468
|
+
reportBridgedAvatarStatus(session, input, prep) {
|
|
469
|
+
const model = prep.Resolution;
|
|
470
|
+
const status = ResolveBridgedAvatarStatus({
|
|
471
|
+
Resolution: prep.AvatarResolution,
|
|
472
|
+
RoomDelivery: input.AvatarDelivery === 'room',
|
|
473
|
+
ModelShowsAvatar: () => (model ? RealtimeModelShowsAvatar(model, AIEngine.Instance) : false),
|
|
474
|
+
DriverStatus: session.AvatarStatus,
|
|
475
|
+
});
|
|
476
|
+
if (!status || status === session.AvatarStatus) {
|
|
477
|
+
return;
|
|
478
|
+
}
|
|
479
|
+
session.AvatarStatus = status;
|
|
480
|
+
const asked = prep.AvatarResolution?.Reason ? 'the driver was asked for none' : 'the driver was asked for one and reported nothing';
|
|
481
|
+
LogStatus(`[RealtimeCoAgent] bridged session avatar: audio only (${status.Reason}); ${asked}.`);
|
|
482
|
+
}
|
|
483
|
+
/**
|
|
484
|
+
* Records a bridged session's usage on its co-agent prompt run. The recorder is the session's only `OnUsage`
|
|
485
|
+
* subscriber: drivers keep one usage handler. Without a prompt run there is nowhere to record, so nothing subscribes
|
|
486
|
+
* and one line is logged, as the client-direct relay logs a dropped delta.
|
|
487
|
+
*/
|
|
488
|
+
wireBridgeUsage(session, promptRunID, contextUser, provider) {
|
|
489
|
+
if (!promptRunID) {
|
|
490
|
+
LogStatus('WireBridgeRealtimeSession: the session has no co-agent prompt run, so its usage is not recorded.');
|
|
491
|
+
return undefined;
|
|
492
|
+
}
|
|
493
|
+
const recorder = new BridgeRealtimeUsageRecorder((write) => this.AccumulatePromptRunUsage(promptRunID, write.InputTokens, write.OutputTokens, contextUser, provider, write.Details), `prompt run ${promptRunID}`);
|
|
494
|
+
session.OnUsage((usage) => recorder.Add(usage));
|
|
495
|
+
return recorder;
|
|
496
|
+
}
|
|
497
|
+
/**
|
|
498
|
+
* The bridged session's finalizer. It closes the usage recorder first (no more updates; the unwritten usage is
|
|
499
|
+
* written), so the prompt run's cost, computed when finalize stamps `CompletedAt`, covers the whole session. On a
|
|
500
|
+
* graceful close it runs after the model session closed (see `finalizeOnBridgeClose`).
|
|
501
|
+
*
|
|
502
|
+
* It runs once, and every call returns the first call's promise: whichever path calls second (a `Close()` after a
|
|
503
|
+
* dropped connection's `OnClose` started the finalize) resolves only once the runs are finalized and priced. The
|
|
504
|
+
* first call decides the outcome (see {@link BridgeRealtimeRuntime.Finalize}).
|
|
505
|
+
*/
|
|
506
|
+
createBridgeFinalizer(obs, usage, contextUser, provider) {
|
|
507
|
+
let finalizing;
|
|
508
|
+
const finalizeRuns = async (success, errorMessage) => {
|
|
509
|
+
await usage?.Close();
|
|
510
|
+
await this.FinalizeCoAgentRun(obs?.CoAgentRunID ?? null, obs?.PromptRunID ?? null, contextUser, provider, success, obs?.CoAgentRunStepID ?? null, errorMessage);
|
|
350
511
|
};
|
|
351
|
-
|
|
512
|
+
return (success, errorMessage) => {
|
|
513
|
+
finalizing ??= finalizeRuns(success, errorMessage);
|
|
514
|
+
return finalizing;
|
|
515
|
+
};
|
|
516
|
+
}
|
|
517
|
+
/** Tool calls → the host's local handler or the shared delegation entry point, then the result back to the model. */
|
|
518
|
+
wireBridgeToolCalls(session, wiring) {
|
|
352
519
|
session.OnToolCall(async (call) => {
|
|
353
520
|
try {
|
|
354
|
-
const
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
521
|
+
const localHandler = wiring.LocalToolHandler();
|
|
522
|
+
const resultJson = localHandler?.Handles(call.ToolName)
|
|
523
|
+
? await localHandler.Execute(call)
|
|
524
|
+
: (await wiring.Narrator.Track(() => this.ExecuteRelayedTool({
|
|
525
|
+
AgentSessionID: wiring.Input.AgentSessionID,
|
|
526
|
+
ParentRunID: wiring.ParentRunID,
|
|
527
|
+
TargetAgentID: wiring.Input.TargetAgentID,
|
|
528
|
+
AllowedAgents: wiring.AllowedAgents,
|
|
529
|
+
DirectActions: wiring.Prep.EffectiveConfig?.realtime?.directActions,
|
|
530
|
+
OnProgress: (progress) => wiring.Narrator.HandleProgress(progress),
|
|
363
531
|
Call: call,
|
|
364
|
-
},
|
|
532
|
+
}, wiring.ContextUser, wiring.Provider))).ResultJson;
|
|
365
533
|
await session.SendToolResult(call.CallID, resultJson);
|
|
366
534
|
}
|
|
367
535
|
catch (error) {
|
|
@@ -370,28 +538,34 @@ export class RealtimeClientSessionService {
|
|
|
370
538
|
await session.SendToolResult(call.CallID, JSON.stringify({ success: false, error: message }));
|
|
371
539
|
}
|
|
372
540
|
});
|
|
373
|
-
|
|
374
|
-
|
|
541
|
+
}
|
|
542
|
+
/**
|
|
543
|
+
* Finalizes on graceful teardown (the bridge calls `Close()`) and on an unexpected drop. Both go through the one
|
|
544
|
+
* finalizer, which runs once: a double fire finalizes once, and a `Close()` after a drop waits for the finalize the
|
|
545
|
+
* drop started, so it resolves only once the runs are finalized. Both finalize as completed: a host whose bridge
|
|
546
|
+
* start failed calls {@link BridgeRealtimeRuntime.Finalize} with `false` and the start's error before it closes the
|
|
547
|
+
* session.
|
|
548
|
+
*
|
|
549
|
+
* `Close()` closes the model session before it finalizes: a session reports its last usage while it closes (a
|
|
550
|
+
* Gemini avatar session, the seconds of video not reported yet), and the finalizer stores only what was reported
|
|
551
|
+
* before it ran. The close is waited for up to {@link BRIDGE_SESSION_CLOSE_WAIT_MS}. The runs are finalized whether
|
|
552
|
+
* the close finished, failed or is still running; a failed close still rejects, after the finalize.
|
|
553
|
+
*
|
|
554
|
+
* @param promptRunID The session's co-agent prompt run, which names the session in a log line.
|
|
555
|
+
*/
|
|
556
|
+
finalizeOnBridgeClose(session, narrator, finalize, promptRunID) {
|
|
375
557
|
const originalClose = session.Close.bind(session);
|
|
558
|
+
const label = promptRunID ? `prompt run ${promptRunID}` : 'a session with no co-agent prompt run';
|
|
376
559
|
session.Close = async () => {
|
|
377
|
-
narrator.Cancel();
|
|
378
|
-
|
|
379
|
-
|
|
560
|
+
narrator.Cancel(); // nothing more is spoken on a session that is closing
|
|
561
|
+
try {
|
|
562
|
+
await closeBridgedSessionWithin(originalClose, BRIDGE_SESSION_CLOSE_WAIT_MS, label);
|
|
563
|
+
}
|
|
564
|
+
finally {
|
|
565
|
+
await finalize(true);
|
|
566
|
+
}
|
|
380
567
|
};
|
|
381
568
|
session.OnClose?.(() => { void finalize(true); });
|
|
382
|
-
const runtime = {
|
|
383
|
-
CoAgentRunID: obs?.CoAgentRunID,
|
|
384
|
-
PromptRunID: obs?.PromptRunID,
|
|
385
|
-
Finalize: finalize,
|
|
386
|
-
CancelInFlightDelegations: () => {
|
|
387
|
-
narrator.Cancel(); // a stale "still working on it" line must not be spoken over the caller
|
|
388
|
-
return this.CancelInFlightDelegations(input.AgentSessionID);
|
|
389
|
-
},
|
|
390
|
-
CancelPendingNarration: () => narrator.Cancel(),
|
|
391
|
-
SetLocalToolHandler: (handler) => { localToolHandler = handler; },
|
|
392
|
-
};
|
|
393
|
-
bridgeRuntimes.set(session, runtime);
|
|
394
|
-
return runtime;
|
|
395
569
|
}
|
|
396
570
|
/**
|
|
397
571
|
* Degenerate {@link BridgeRealtimeRuntime} for the rare case wiring is attempted without a resolved
|
|
@@ -407,6 +581,7 @@ export class RealtimeClientSessionService {
|
|
|
407
581
|
CancelInFlightDelegations: () => 0,
|
|
408
582
|
CancelPendingNarration: () => { },
|
|
409
583
|
SetLocalToolHandler: () => { },
|
|
584
|
+
WatchesMeetingVideo: false,
|
|
410
585
|
};
|
|
411
586
|
bridgeRuntimes.set(session, runtime);
|
|
412
587
|
return runtime;
|
|
@@ -443,13 +618,157 @@ export class RealtimeClientSessionService {
|
|
|
443
618
|
const targetAgent = this.resolveTargetAgent(input.TargetAgentID);
|
|
444
619
|
const appSettingsJson = await this.resolveAppRealtimeOverrides(input.ApplicationID, contextUser, provider);
|
|
445
620
|
const effectiveConfig = this.resolveEffectiveConfig(coAgent, input.ConfigOverridesJson, targetAgent, appSettingsJson);
|
|
621
|
+
this.logDeprecatedVideoKeys(effectiveConfig, coAgent, targetAgent);
|
|
446
622
|
const outcome = await this.resolveModelForSession(input, coAgent, effectiveConfig);
|
|
447
623
|
if (!outcome.Resolution) {
|
|
448
624
|
return { Success: false, ErrorMessage: outcome.ErrorMessage ?? this.noModelMessage() };
|
|
449
625
|
}
|
|
450
626
|
const resolution = outcome.Resolution;
|
|
451
|
-
|
|
452
|
-
|
|
627
|
+
let hostTools = input.HostTools;
|
|
628
|
+
if (input.ResolveHostTools) {
|
|
629
|
+
const dynamicHostTools = input.ResolveHostTools({
|
|
630
|
+
ModelID: resolution.ModelID,
|
|
631
|
+
ModelVendorID: resolution.ModelVendorID,
|
|
632
|
+
DriverClass: resolution.DriverClass,
|
|
633
|
+
});
|
|
634
|
+
if (dynamicHostTools !== undefined) {
|
|
635
|
+
hostTools = dynamicHostTools;
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
const effectiveInput = hostTools !== input.HostTools ? { ...input, HostTools: hostTools } : input;
|
|
639
|
+
// One answer for the whole session: it lowers channel exposure here and keeps the driver from
|
|
640
|
+
// turning on provider features that store session data (Gemini session resumption).
|
|
641
|
+
const zeroDataRetention = this.modelHasZeroDataRetention(resolution.ModelID, resolution.ModelVendorID);
|
|
642
|
+
// Channel scoping + client-tool tiers: narrows the declared tools to the scope's decision and
|
|
643
|
+
// folds the app tier into the capability manifest the prompt renders. The scoped input is what
|
|
644
|
+
// the prompt/tool builders see, so a vetoed channel is absent from the framing as well as the tools.
|
|
645
|
+
const scoped = await this.scopeSessionInput(effectiveInput, effectiveConfig, contextUser, provider, zeroDataRetention);
|
|
646
|
+
// A session nobody would see an avatar in (a phone call, an app without agent video) asks the driver for none
|
|
647
|
+
// and keeps the persona's voice.
|
|
648
|
+
const avatar = WithoutUnseenAvatar(this.ResolveSessionAvatar(scoped.Input, coAgent, effectiveConfig, resolution.ModelID, resolution.ModelVendorID), scoped.Input);
|
|
649
|
+
const sessionParams = {
|
|
650
|
+
...(await this.buildSessionParams(scoped.Input, coAgent, resolution.APIName, contextUser, provider, effectiveConfig, resolution.DriverClass, resolution.ModelID, resolution.ModelVendorID, avatar)),
|
|
651
|
+
ZeroDataRetention: zeroDataRetention,
|
|
652
|
+
};
|
|
653
|
+
return {
|
|
654
|
+
Success: true, CoAgent: coAgent, Resolution: resolution, EffectiveConfig: effectiveConfig, SessionParams: sessionParams,
|
|
655
|
+
ClientPolicy: scoped.ClientPolicy, AvatarResolution: avatar,
|
|
656
|
+
};
|
|
657
|
+
}
|
|
658
|
+
/**
|
|
659
|
+
* Applies channel scoping and the client-tool tiers to a prepare input.
|
|
660
|
+
*
|
|
661
|
+
* - **App tier into the prompt.** `Application.AgentSettings.ClientTools` is resolved to metadata
|
|
662
|
+
* and layered, through the unified {@link ResolveClientTools}, beneath the live surface's own
|
|
663
|
+
* manifest in the app-context snapshot — so the co-agent is told about tools the app declares
|
|
664
|
+
* even when the active surface did not republish them. (Only the HOST can run them; the browser
|
|
665
|
+
* reports "declared but not registered" for one it has no handler for.) The agent's static
|
|
666
|
+
* junction tools are deliberately NOT added to the voice prompt — voice prompts are
|
|
667
|
+
* token-sensitive and a tool with no browser handler is noise — but they ride the returned policy
|
|
668
|
+
* so the browser can describe them.
|
|
669
|
+
* - **Channel scoping.** When the browser reported candidates, they are scoped against the
|
|
670
|
+
* registry and the cascade's `channels` section, and the declared tools narrowed to match
|
|
671
|
+
* ({@link BuildSessionChannelPolicy}).
|
|
672
|
+
*
|
|
673
|
+
* @returns The (possibly adjusted) input, and the policy to hand back when candidates were reported.
|
|
674
|
+
*/
|
|
675
|
+
async scopeSessionInput(input, effectiveConfig, contextUser, provider, modelHasZeroDataRetention = false) {
|
|
676
|
+
const tiers = await this.resolveSessionClientToolTiers(input.ApplicationID, input.TargetAgentID, contextUser);
|
|
677
|
+
const appContext = this.withAppToolTier(input.AppContext, tiers.App, input.TargetAgentID);
|
|
678
|
+
const candidates = input.ChannelCandidates ?? [];
|
|
679
|
+
if (candidates.length === 0) {
|
|
680
|
+
return { Input: appContext === input.AppContext ? input : { ...input, AppContext: appContext } };
|
|
681
|
+
}
|
|
682
|
+
const outcome = BuildSessionChannelPolicy({
|
|
683
|
+
Candidates: candidates,
|
|
684
|
+
ChannelsConfig: effectiveConfig.realtime?.channels ?? null,
|
|
685
|
+
Registry: this.readChannelRegistry(provider),
|
|
686
|
+
ClientTools: input.ExtraTools,
|
|
687
|
+
ClientToolTiers: tiers,
|
|
688
|
+
ModelHasZeroDataRetention: modelHasZeroDataRetention,
|
|
689
|
+
});
|
|
690
|
+
return {
|
|
691
|
+
Input: { ...input, ExtraTools: outcome.ClientTools, AppContext: appContext },
|
|
692
|
+
ClientPolicy: outcome.Policy,
|
|
693
|
+
};
|
|
694
|
+
}
|
|
695
|
+
/**
|
|
696
|
+
* Whether the session model's effective catalog configuration (type < model < vendor < model-vendor)
|
|
697
|
+
* declares `Privacy.ZeroDataRetention: true`. Fails closed: a model or vendor row that cannot be
|
|
698
|
+
* resolved, or a catalog lookup that throws, reads as "not declared", so an agent that REQUIRES zero
|
|
699
|
+
* data retention loses exposure rather than gaining it. Overridable seam.
|
|
700
|
+
*
|
|
701
|
+
* @param modelID The resolved `MJ: AI Models` id.
|
|
702
|
+
* @param modelVendorID The resolved model-vendor row id (the cascade's most specific layer).
|
|
703
|
+
*/
|
|
704
|
+
modelHasZeroDataRetention(modelID, modelVendorID) {
|
|
705
|
+
if (!modelID) {
|
|
706
|
+
return false;
|
|
707
|
+
}
|
|
708
|
+
try {
|
|
709
|
+
return IsZeroDataRetention(AIEngine.Instance.GetEffectiveModelConfiguration(modelID, modelVendorID));
|
|
710
|
+
}
|
|
711
|
+
catch (error) {
|
|
712
|
+
LogError(`RealtimeClientSessionService.modelHasZeroDataRetention failed for model '${modelID}': ${error instanceof Error ? error.message : String(error)}`);
|
|
713
|
+
return false;
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
/**
|
|
717
|
+
* Layers the app's client tools beneath the surface's manifest in the app-context snapshot, through
|
|
718
|
+
* the unified resolver (`session > app`, first match by name wins). Returns the SAME snapshot object
|
|
719
|
+
* when nothing changes, so callers can detect "no change" by identity.
|
|
720
|
+
*/
|
|
721
|
+
withAppToolTier(appContext, appTools, targetAgentId) {
|
|
722
|
+
if (!appContext || !appTools || appTools.length === 0) {
|
|
723
|
+
return appContext;
|
|
724
|
+
}
|
|
725
|
+
const surfaceTools = appContext.Capabilities?.Tools;
|
|
726
|
+
const tools = ResolveClientTools({ agentId: targetAgentId, sessionTools: surfaceTools, appTools: [...appTools] });
|
|
727
|
+
if (surfaceTools && tools.length === surfaceTools.length) {
|
|
728
|
+
return appContext; // the surface already covered every app tool
|
|
729
|
+
}
|
|
730
|
+
return { ...appContext, Capabilities: { ...appContext.Capabilities, Tools: tools } };
|
|
731
|
+
}
|
|
732
|
+
/**
|
|
733
|
+
* The server's view of the channel registry (`MJ: AI Agent Channels`), read from the request
|
|
734
|
+
* provider's cached {@link AIEngineBase} rows (already configured by the prepare). Overridable seam.
|
|
735
|
+
* Tolerant: an unloaded cache yields an empty registry (every candidate then reads as having no
|
|
736
|
+
* row), never a throw.
|
|
737
|
+
*/
|
|
738
|
+
readChannelRegistry(provider) {
|
|
739
|
+
try {
|
|
740
|
+
const engine = AIEngineBase.GetProviderInstance(provider, AIEngineBase);
|
|
741
|
+
return (engine.AgentChannels ?? []).map(c => ({ Name: c.Name, IsActive: c.IsActive }));
|
|
742
|
+
}
|
|
743
|
+
catch (error) {
|
|
744
|
+
LogError(`RealtimeClientSessionService.readChannelRegistry failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
745
|
+
return [];
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
/**
|
|
749
|
+
* Resolves the client-tool tiers the browser is told about at mint: the APP tier
|
|
750
|
+
* (`Application.AgentSettings.ClientTools` resolved against the tool-definition catalog) and the
|
|
751
|
+
* STATIC tier (the target agent's `MJ: AI Agent Client Tools` junction). Overridable seam; tolerant —
|
|
752
|
+
* any failure yields empty tiers and never fails the mint.
|
|
753
|
+
*/
|
|
754
|
+
async resolveSessionClientToolTiers(applicationId, targetAgentId, contextUser) {
|
|
755
|
+
const tiers = {};
|
|
756
|
+
try {
|
|
757
|
+
const settings = await this.loadAppAgentSettings(applicationId, contextUser);
|
|
758
|
+
const definitions = AIEngine.Instance.ClientToolDefinitions ?? [];
|
|
759
|
+
const app = ResolveAppClientToolMetadata(settings?.ClientTools, definitions, ref => LogStatus(`RealtimeClientSessionService: the app's client tool '${ref.Name ?? ref.ClientToolDefinitionID}' matches no tool definition — skipping it.`));
|
|
760
|
+
if (app.length > 0) {
|
|
761
|
+
tiers.App = app;
|
|
762
|
+
}
|
|
763
|
+
const stat = targetAgentId ? AIEngine.Instance.GetClientToolsForAgent(targetAgentId).map(ClientToolMetadataFromDefinition) : [];
|
|
764
|
+
if (stat.length > 0) {
|
|
765
|
+
tiers.Static = stat;
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
catch (error) {
|
|
769
|
+
LogError(`RealtimeClientSessionService.resolveSessionClientToolTiers failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
770
|
+
}
|
|
771
|
+
return tiers;
|
|
453
772
|
}
|
|
454
773
|
/**
|
|
455
774
|
* Resolves the EFFECTIVE realtime configuration via the surface-agnostic precedence cascade:
|
|
@@ -468,6 +787,27 @@ export class RealtimeClientSessionService {
|
|
|
468
787
|
resolveEffectiveConfig(coAgent, overridesJson, targetAgent, appSettingsJson) {
|
|
469
788
|
return ResolveEffectiveRealtimeConfig(this.getAgentTypeDefaultConfiguration(coAgent), coAgent.TypeConfiguration ?? null, overridesJson ?? null, targetAgent?.TypeConfiguration ?? null, appSettingsJson ?? null);
|
|
470
789
|
}
|
|
790
|
+
/**
|
|
791
|
+
* Logs one line, once per co-agent and target pair per process, when the effective configuration sets
|
|
792
|
+
* `realtime.video.provider` or `realtime.video.providers`. Both are deprecated and nothing reads them, so an author
|
|
793
|
+
* who set them would otherwise get nothing and no word of it.
|
|
794
|
+
*
|
|
795
|
+
* @param config The effective configuration.
|
|
796
|
+
* @param coAgent The co-agent.
|
|
797
|
+
* @param targetAgent The voiced agent, when there is one.
|
|
798
|
+
*/
|
|
799
|
+
logDeprecatedVideoKeys(config, coAgent, targetAgent) {
|
|
800
|
+
const keys = FindDeprecatedRealtimeVideoKeys(config);
|
|
801
|
+
const pair = `${NormalizeUUID(coAgent.ID)}|${NormalizeUUID(targetAgent?.ID)}`;
|
|
802
|
+
if (keys.length === 0 || deprecatedVideoKeysLogged.has(pair)) {
|
|
803
|
+
return;
|
|
804
|
+
}
|
|
805
|
+
deprecatedVideoKeysLogged.add(pair);
|
|
806
|
+
const voiced = targetAgent && !UUIDsEqual(targetAgent.ID, coAgent.ID) ? `'${targetAgent.Name}' (co-agent '${coAgent.Name}')` : `'${coAgent.Name}'`;
|
|
807
|
+
const what = keys.length === 1 ? `${keys[0]}, which is deprecated and does nothing` : `${keys.join(' and ')}, which are deprecated and do nothing`;
|
|
808
|
+
LogStatus(`[RealtimeCoAgent] The realtime configuration for ${voiced} sets ${what}. Choose the model with ` +
|
|
809
|
+
"realtime.modelPreference, and set an avatar's settings on its persona's Video binding (VendorSettings.Avatar).");
|
|
810
|
+
}
|
|
471
811
|
/**
|
|
472
812
|
* Resolves the APP cascade layer JSON from `Application.AgentSettings`: the `Realtime` overrides
|
|
473
813
|
* (persona/disclosure/model) plus `RelevantAgents` mapped to the union-accumulated allowed-agent
|
|
@@ -484,6 +824,30 @@ export class RealtimeClientSessionService {
|
|
|
484
824
|
async resolveAppRealtimeOverrides(applicationId, contextUser,
|
|
485
825
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
486
826
|
provider) {
|
|
827
|
+
const settings = await this.loadAppAgentSettings(applicationId, contextUser);
|
|
828
|
+
if (!settings) {
|
|
829
|
+
return null;
|
|
830
|
+
}
|
|
831
|
+
const relevantAgents = (settings.RelevantAgents ?? []).map(r => {
|
|
832
|
+
const entry = { agentId: r.AgentID };
|
|
833
|
+
if (r.Label) {
|
|
834
|
+
entry.label = r.Label;
|
|
835
|
+
}
|
|
836
|
+
if (r.Disclosure === 'silent' || r.Disclosure === 'mention' || r.Disclosure === 'hand-voice') {
|
|
837
|
+
entry.disclosure = r.Disclosure;
|
|
838
|
+
}
|
|
839
|
+
return entry;
|
|
840
|
+
});
|
|
841
|
+
return BuildAppRealtimeOverridesJson(settings.Realtime ?? null, relevantAgents);
|
|
842
|
+
}
|
|
843
|
+
/**
|
|
844
|
+
* Reads and parses an application's `AgentSettings` (one tiny by-ID read, served from the provider's
|
|
845
|
+
* RunView cache on repeat). The RAW column is parsed with {@link ParseAgentSettings} rather than read
|
|
846
|
+
* through the CodeGen-generated `AgentSettingsObject` accessor, so a field added to the settings
|
|
847
|
+
* interface works the moment the code reading it ships, before CodeGen next refreshes the copy.
|
|
848
|
+
* Tolerant — a blank id, a failed read, a missing app, or an app with no usable settings returns `null`.
|
|
849
|
+
*/
|
|
850
|
+
async loadAppAgentSettings(applicationId, contextUser) {
|
|
487
851
|
const appId = applicationId?.trim();
|
|
488
852
|
if (!appId) {
|
|
489
853
|
return null;
|
|
@@ -499,24 +863,10 @@ export class RealtimeClientSessionService {
|
|
|
499
863
|
if (!result.Success || !result.Results || result.Results.length === 0) {
|
|
500
864
|
return null;
|
|
501
865
|
}
|
|
502
|
-
|
|
503
|
-
if (!settings) {
|
|
504
|
-
return null;
|
|
505
|
-
}
|
|
506
|
-
const relevantAgents = (settings.RelevantAgents ?? []).map(r => {
|
|
507
|
-
const entry = { agentId: r.AgentID };
|
|
508
|
-
if (r.Label) {
|
|
509
|
-
entry.label = r.Label;
|
|
510
|
-
}
|
|
511
|
-
if (r.Disclosure === 'silent' || r.Disclosure === 'mention' || r.Disclosure === 'hand-voice') {
|
|
512
|
-
entry.disclosure = r.Disclosure;
|
|
513
|
-
}
|
|
514
|
-
return entry;
|
|
515
|
-
});
|
|
516
|
-
return BuildAppRealtimeOverridesJson(settings.Realtime ?? null, relevantAgents);
|
|
866
|
+
return ParseAgentSettings(result.Results[0].AgentSettings);
|
|
517
867
|
}
|
|
518
868
|
catch (error) {
|
|
519
|
-
LogError(`RealtimeClientSessionService.
|
|
869
|
+
LogError(`RealtimeClientSessionService.loadAppAgentSettings failed for app '${appId}': ${error instanceof Error ? error.message : String(error)}`);
|
|
520
870
|
return null;
|
|
521
871
|
}
|
|
522
872
|
}
|
|
@@ -590,11 +940,30 @@ export class RealtimeClientSessionService {
|
|
|
590
940
|
run.UserID = userID;
|
|
591
941
|
}
|
|
592
942
|
if (await run.Save()) {
|
|
943
|
+
this.KeepCoAgentRunAlive(run.ID, provider, contextUser);
|
|
593
944
|
return run.ID;
|
|
594
945
|
}
|
|
595
946
|
LogError(`RealtimeClientSessionService.createCoAgentRun save failed: ${run.LatestResult?.CompleteMessage ?? 'unknown error'}`);
|
|
596
947
|
return null;
|
|
597
948
|
}
|
|
949
|
+
/**
|
|
950
|
+
* Keeps a voice session's co-agent run alive as far as the {@link AgentRunWatchdog} is concerned. The
|
|
951
|
+
* run spans the whole call, but no agent loop owns it, so nothing stamped its heartbeat: the watchdog
|
|
952
|
+
* force-failed every call that ran past ~5 minutes ("no liveness heartbeat … owning process presumed
|
|
953
|
+
* dead") while the call carried on. Called when the run is created and again on each persisted
|
|
954
|
+
* session heartbeat (`SessionManager`), so whichever server instance the session is talking to keeps
|
|
955
|
+
* it fresh; the watchdog drops it once it is finalized. Only a database provider can stamp heartbeats —
|
|
956
|
+
* any other provider is a no-op, as for every agent run.
|
|
957
|
+
*
|
|
958
|
+
* @param coAgentRunID The session's co-agent run id (from its `Config`), or nothing.
|
|
959
|
+
* @param provider The request-scoped metadata provider.
|
|
960
|
+
* @param contextUser The user the heartbeat writes run as.
|
|
961
|
+
*/
|
|
962
|
+
KeepCoAgentRunAlive(coAgentRunID, provider, contextUser) {
|
|
963
|
+
if (coAgentRunID && provider instanceof DatabaseProviderBase) {
|
|
964
|
+
AgentRunWatchdog.Instance.Track(coAgentRunID, provider, contextUser);
|
|
965
|
+
}
|
|
966
|
+
}
|
|
598
967
|
/**
|
|
599
968
|
* Creates the co-agent `AIPromptRun` row (Status `Running`) linked to the co-agent run via
|
|
600
969
|
* `AgentRunID` AND to the co-agent itself via `AgentID` — so the run shows up both on the
|
|
@@ -672,11 +1041,59 @@ export class RealtimeClientSessionService {
|
|
|
672
1041
|
* @param provider The request-scoped metadata provider.
|
|
673
1042
|
* @param success Whether the session ended successfully (controls Completed vs Failed).
|
|
674
1043
|
* @param coAgentRunStepID The co-agent run's single `MJ: AI Agent Run Steps` row id, or `null` to skip.
|
|
1044
|
+
* @param errorMessage Why the session failed (for example the bridge start's error), stamped as `ErrorMessage` on each
|
|
1045
|
+
* record it fails. Ignored when `success` is true. Without it the step gets a generic line and the runs none.
|
|
675
1046
|
*/
|
|
676
|
-
async FinalizeCoAgentRun(coAgentRunID, promptRunID, contextUser, provider, success = true, coAgentRunStepID = null) {
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
1047
|
+
async FinalizeCoAgentRun(coAgentRunID, promptRunID, contextUser, provider, success = true, coAgentRunStepID = null, errorMessage) {
|
|
1048
|
+
const failure = success ? undefined : errorMessage?.trim() || undefined;
|
|
1049
|
+
try {
|
|
1050
|
+
await this.finalizeAgentRun(coAgentRunID, contextUser, provider, success, failure);
|
|
1051
|
+
await this.finalizePromptRun(promptRunID, contextUser, provider, success, failure);
|
|
1052
|
+
await this.finalizeRunStep(coAgentRunStepID, contextUser, provider, success, failure);
|
|
1053
|
+
}
|
|
1054
|
+
finally {
|
|
1055
|
+
// Even when a finalize step throws, the run still owes its cost, and the watchdog must stop
|
|
1056
|
+
// treating it as alive — or a run stuck at Running would be kept fresh indefinitely.
|
|
1057
|
+
await this.rollUpCoAgentRunUsage(coAgentRunID, promptRunID, contextUser, provider);
|
|
1058
|
+
if (coAgentRunID) {
|
|
1059
|
+
AgentRunWatchdog.Instance.Untrack(coAgentRunID);
|
|
1060
|
+
}
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1063
|
+
/**
|
|
1064
|
+
* Copies the co-agent prompt run's tokens and cost onto the co-agent run. The realtime model's usage
|
|
1065
|
+
* accumulates on the prompt run ({@link AccumulatePromptRunUsage}), which prices itself; the run's
|
|
1066
|
+
* own `TotalCost` / `Total*TokensUsed` stayed 0, so everything that sums agent runs — the realtime
|
|
1067
|
+
* analytics dashboard's per-session cost among them — left out the voice model entirely and showed
|
|
1068
|
+
* only the delegated runs. Mirrors how an agent loop derives its run totals from its prompt runs.
|
|
1069
|
+
*
|
|
1070
|
+
* Applied whatever the run's status: a run the watchdog already failed, or one a shutdown cancelled,
|
|
1071
|
+
* still owes its cost. Runs after {@link finalizePromptRun}, which waits for in-flight usage writes,
|
|
1072
|
+
* so the copy sees the final counts. Tolerant: logs, never throws.
|
|
1073
|
+
*/
|
|
1074
|
+
async rollUpCoAgentRunUsage(coAgentRunID, promptRunID, contextUser, provider) {
|
|
1075
|
+
if (!coAgentRunID || !promptRunID) {
|
|
1076
|
+
return;
|
|
1077
|
+
}
|
|
1078
|
+
try {
|
|
1079
|
+
const promptRun = await provider.GetEntityObject('MJ: AI Prompt Runs', contextUser);
|
|
1080
|
+
const run = await provider.GetEntityObject('MJ: AI Agent Runs', contextUser);
|
|
1081
|
+
if (!(await promptRun.Load(promptRunID)) || !(await run.Load(coAgentRunID))) {
|
|
1082
|
+
return;
|
|
1083
|
+
}
|
|
1084
|
+
const promptTokens = promptRun.TokensPrompt ?? 0;
|
|
1085
|
+
const completionTokens = promptRun.TokensCompletion ?? 0;
|
|
1086
|
+
run.TotalPromptTokensUsed = promptTokens;
|
|
1087
|
+
run.TotalCompletionTokensUsed = completionTokens;
|
|
1088
|
+
run.TotalTokensUsed = promptRun.TokensUsed ?? promptTokens + completionTokens;
|
|
1089
|
+
run.TotalCost = promptRun.TotalCost ?? promptRun.Cost ?? 0;
|
|
1090
|
+
if (run.Dirty && !(await run.Save())) {
|
|
1091
|
+
LogError(`RealtimeClientSessionService.rollUpCoAgentRunUsage save failed: ${run.LatestResult?.CompleteMessage ?? 'unknown error'}`);
|
|
1092
|
+
}
|
|
1093
|
+
}
|
|
1094
|
+
catch (error) {
|
|
1095
|
+
LogError(`RealtimeClientSessionService.rollUpCoAgentRunUsage failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1096
|
+
}
|
|
680
1097
|
}
|
|
681
1098
|
/**
|
|
682
1099
|
* Finalizes the **co-agent observability run(s)** for an agent session that were left `Running` because
|
|
@@ -763,8 +1180,11 @@ export class RealtimeClientSessionService {
|
|
|
763
1180
|
}
|
|
764
1181
|
return { PromptRunID: promptRuns.Results[0]?.ID ?? null, StepID: stepID };
|
|
765
1182
|
}
|
|
766
|
-
/**
|
|
767
|
-
|
|
1183
|
+
/**
|
|
1184
|
+
* Loads + finalizes the co-agent `AIAgentRun` if still `Running`, with `failure` as its `ErrorMessage` when it fails.
|
|
1185
|
+
* Tolerant: logs, never throws.
|
|
1186
|
+
*/
|
|
1187
|
+
async finalizeAgentRun(coAgentRunID, contextUser, provider, success, failure) {
|
|
768
1188
|
if (!coAgentRunID) {
|
|
769
1189
|
return;
|
|
770
1190
|
}
|
|
@@ -775,16 +1195,20 @@ export class RealtimeClientSessionService {
|
|
|
775
1195
|
run.Status = success ? 'Completed' : 'Failed';
|
|
776
1196
|
run.CompletedAt = new Date();
|
|
777
1197
|
run.Success = success;
|
|
1198
|
+
if (failure) {
|
|
1199
|
+
run.ErrorMessage = failure;
|
|
1200
|
+
}
|
|
778
1201
|
if (!(await run.Save())) {
|
|
779
1202
|
LogError(`RealtimeClientSessionService.finalizeAgentRun save failed: ${run.LatestResult?.CompleteMessage ?? 'unknown error'}`);
|
|
780
1203
|
}
|
|
781
1204
|
}
|
|
782
1205
|
/**
|
|
783
1206
|
* Loads + finalizes the co-agent run's single system-prompt `MJ: AI Agent Run Steps` row if
|
|
784
|
-
* still `Running` (Status `Completed`/`Failed`, `CompletedAt`, `Success`
|
|
785
|
-
* missing/already-finalized step is a no-op; a
|
|
1207
|
+
* still `Running` (Status `Completed`/`Failed`, `CompletedAt`, `Success`; a failed step's `ErrorMessage` is
|
|
1208
|
+
* `failure`, or a generic line without one). Tolerant: a missing/already-finalized step is a no-op; a
|
|
1209
|
+
* load/save failure is logged, never thrown.
|
|
786
1210
|
*/
|
|
787
|
-
async finalizeRunStep(coAgentRunStepID, contextUser, provider, success) {
|
|
1211
|
+
async finalizeRunStep(coAgentRunStepID, contextUser, provider, success, failure) {
|
|
788
1212
|
if (!coAgentRunStepID) {
|
|
789
1213
|
return;
|
|
790
1214
|
}
|
|
@@ -797,7 +1221,7 @@ export class RealtimeClientSessionService {
|
|
|
797
1221
|
step.CompletedAt = new Date();
|
|
798
1222
|
step.Success = success;
|
|
799
1223
|
if (!success) {
|
|
800
|
-
step.ErrorMessage = 'The realtime session ended in an error state.';
|
|
1224
|
+
step.ErrorMessage = failure ?? 'The realtime session ended in an error state.';
|
|
801
1225
|
}
|
|
802
1226
|
if (!(await step.Save())) {
|
|
803
1227
|
LogError(`RealtimeClientSessionService.finalizeRunStep save failed: ${step.LatestResult?.CompleteMessage ?? 'unknown error'}`);
|
|
@@ -807,8 +1231,11 @@ export class RealtimeClientSessionService {
|
|
|
807
1231
|
LogError(`RealtimeClientSessionService.finalizeRunStep failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
808
1232
|
}
|
|
809
1233
|
}
|
|
810
|
-
/**
|
|
811
|
-
|
|
1234
|
+
/**
|
|
1235
|
+
* Loads + finalizes the co-agent `AIPromptRun` if still `Running`, with `failure` as its `ErrorMessage` when it fails.
|
|
1236
|
+
* Tolerant: logs, never throws.
|
|
1237
|
+
*/
|
|
1238
|
+
async finalizePromptRun(promptRunID, contextUser, provider, success, failure) {
|
|
812
1239
|
if (!promptRunID) {
|
|
813
1240
|
return;
|
|
814
1241
|
}
|
|
@@ -822,6 +1249,9 @@ export class RealtimeClientSessionService {
|
|
|
822
1249
|
run.Status = success ? 'Completed' : 'Failed';
|
|
823
1250
|
run.CompletedAt = new Date();
|
|
824
1251
|
run.Success = success;
|
|
1252
|
+
if (failure) {
|
|
1253
|
+
run.ErrorMessage = failure;
|
|
1254
|
+
}
|
|
825
1255
|
if (!(await run.Save())) {
|
|
826
1256
|
LogError(`RealtimeClientSessionService.finalizePromptRun save failed: ${run.LatestResult?.CompleteMessage ?? 'unknown error'}`);
|
|
827
1257
|
}
|
|
@@ -885,12 +1315,19 @@ export class RealtimeClientSessionService {
|
|
|
885
1315
|
* high-frequency usage checkpoint never overwrites freshly-appended transcript turns (and vice-versa).
|
|
886
1316
|
* Best-effort: load/save failures log and return `false`, never throw.
|
|
887
1317
|
*
|
|
1318
|
+
* The per-modality details add into the usage record the run keeps in `ModelSpecificResponseDetails`
|
|
1319
|
+
* (`RealtimeUsage.Input` / `.Output`, which pricing reads at finalize); every other key there is kept. The
|
|
1320
|
+
* stored output video seconds never exceed the run's elapsed time plus 30 seconds, and the stored output video
|
|
1321
|
+
* tokens never exceed 6,192 a second of that: usage is client-reported, and avatar video carries a price, per
|
|
1322
|
+
* minute or per token.
|
|
1323
|
+
*
|
|
888
1324
|
* @param promptRunID The co-agent observability prompt run.
|
|
889
1325
|
* @param inputDelta Input-token delta to add (caller clamps to >= 0).
|
|
890
1326
|
* @param outputDelta Output-token delta to add (caller clamps to >= 0).
|
|
1327
|
+
* @param details Per-modality usage to add (input and output blocks, avatar video seconds included).
|
|
891
1328
|
* @returns `true` when the accumulated usage was persisted.
|
|
892
1329
|
*/
|
|
893
|
-
async AccumulatePromptRunUsage(promptRunID, inputDelta, outputDelta, contextUser, provider) {
|
|
1330
|
+
async AccumulatePromptRunUsage(promptRunID, inputDelta, outputDelta, contextUser, provider, details) {
|
|
894
1331
|
return this.serializePromptRunWrite(promptRunID, async () => {
|
|
895
1332
|
try {
|
|
896
1333
|
const promptRun = await provider.GetEntityObject('MJ: AI Prompt Runs', contextUser);
|
|
@@ -901,6 +1338,9 @@ export class RealtimeClientSessionService {
|
|
|
901
1338
|
promptRun.TokensPrompt = (promptRun.TokensPrompt ?? 0) + inputDelta;
|
|
902
1339
|
promptRun.TokensCompletion = (promptRun.TokensCompletion ?? 0) + outputDelta;
|
|
903
1340
|
promptRun.TokensUsed = (promptRun.TokensPrompt ?? 0) + (promptRun.TokensCompletion ?? 0);
|
|
1341
|
+
if (details) {
|
|
1342
|
+
this.mergeUsageDetails(promptRun, details);
|
|
1343
|
+
}
|
|
904
1344
|
if (!(await promptRun.Save())) {
|
|
905
1345
|
LogError(`AccumulatePromptRunUsage: prompt run ${promptRunID} save failed: ${promptRun.LatestResult?.CompleteMessage ?? 'unknown error'}`);
|
|
906
1346
|
return false;
|
|
@@ -913,6 +1353,37 @@ export class RealtimeClientSessionService {
|
|
|
913
1353
|
}
|
|
914
1354
|
});
|
|
915
1355
|
}
|
|
1356
|
+
/**
|
|
1357
|
+
* Adds relayed per-modality usage into the run's usage record, with the output video seconds capped at the run's
|
|
1358
|
+
* elapsed time plus {@link VIDEO_SECONDS_GRACE}, and the output video tokens at that many seconds of
|
|
1359
|
+
* {@link VIDEO_TOKENS_PER_SECOND}. Details that are not a JSON object are left alone (logged).
|
|
1360
|
+
*/
|
|
1361
|
+
mergeUsageDetails(promptRun, details) {
|
|
1362
|
+
const runAt = promptRun.RunAt ? new Date(promptRun.RunAt).getTime() : Number.NaN;
|
|
1363
|
+
const elapsedSeconds = Math.max(0, (Date.now() - runAt) / 1000);
|
|
1364
|
+
const maxVideoSeconds = Number.isFinite(elapsedSeconds) ? elapsedSeconds + VIDEO_SECONDS_GRACE : undefined;
|
|
1365
|
+
const merged = MergeRealtimeUsageRecord(promptRun.ModelSpecificResponseDetails, details, {
|
|
1366
|
+
MaxOutputVideoSeconds: maxVideoSeconds,
|
|
1367
|
+
MaxOutputVideoTokens: maxVideoSeconds === undefined ? undefined : maxVideoSeconds * VIDEO_TOKENS_PER_SECOND,
|
|
1368
|
+
});
|
|
1369
|
+
if (!merged) {
|
|
1370
|
+
LogError(`AccumulatePromptRunUsage: prompt run ${promptRun.ID} has details that are not a JSON object — usage details dropped.`);
|
|
1371
|
+
return;
|
|
1372
|
+
}
|
|
1373
|
+
this.logClampedVideo(promptRun.ID, merged);
|
|
1374
|
+
promptRun.ModelSpecificResponseDetails = merged.Details;
|
|
1375
|
+
}
|
|
1376
|
+
/** One log line for each output video quantity the merge capped. */
|
|
1377
|
+
logClampedVideo(promptRunID, merged) {
|
|
1378
|
+
if (merged.ClampedVideoSeconds > 0) {
|
|
1379
|
+
LogStatus(`AccumulatePromptRunUsage: prompt run ${promptRunID} reported ${merged.ClampedVideoSeconds} s more avatar video than ` +
|
|
1380
|
+
`the run's elapsed time plus ${VIDEO_SECONDS_GRACE} s allows; the stored seconds are capped.`);
|
|
1381
|
+
}
|
|
1382
|
+
if (merged.ClampedVideoTokens > 0) {
|
|
1383
|
+
LogStatus(`AccumulatePromptRunUsage: prompt run ${promptRunID} reported ${merged.ClampedVideoTokens} more avatar video tokens than ` +
|
|
1384
|
+
`${VIDEO_TOKENS_PER_SECOND} a second of the run's elapsed time plus ${VIDEO_SECONDS_GRACE} s allows; the stored tokens are capped.`);
|
|
1385
|
+
}
|
|
1386
|
+
}
|
|
916
1387
|
/** Parses the prompt run's `Messages` JSON into a mutable chat-message array (tolerant: `[]` on empty/malformed). */
|
|
917
1388
|
parsePromptRunMessages(raw) {
|
|
918
1389
|
if (!raw || !raw.trim()) {
|
|
@@ -1076,12 +1547,19 @@ export class RealtimeClientSessionService {
|
|
|
1076
1547
|
* {@link resolveConfiguredModelPreference}. METADATA preferences degrade gracefully — an
|
|
1077
1548
|
* unsatisfiable preference logs and FALLS THROUGH to the default (mirroring the co-agent
|
|
1078
1549
|
* resolution chain's tolerant metadata steps), it never breaks calls.
|
|
1079
|
-
* - Without either: the
|
|
1080
|
-
* (
|
|
1550
|
+
* - Without either: the default walk ({@link resolveDefaultRealtimeModel}): for a co-agent with video on, in a session
|
|
1551
|
+
* that could show an avatar (a browser session, or a server-side one with {@link PrepareClientSessionInput.AvatarDelivery}
|
|
1552
|
+
* `'room'`), the first candidate that shows one ({@link ResolveAvatarRealtimeModel}); else the existing default via
|
|
1553
|
+
* {@link resolveRealtimeModel} (highest-PowerRank active Realtime model), with the generic
|
|
1554
|
+
* {@link noModelMessage} on failure. The two explicit choices above win over the video preference.
|
|
1555
|
+
*
|
|
1556
|
+
* On every path the chosen model's vendor follows whether the session will show an avatar (the video setting is on
|
|
1557
|
+
* and the session could show one): among the model's keyed vendors, one that renders avatars (Vertex AI for Gemini
|
|
1558
|
+
* 3.8 Live) only when it will, else one that renders none; see {@link resolveVendorAndInstantiate}.
|
|
1081
1559
|
*
|
|
1082
1560
|
* @param input The prepare-session input (carries the optional preferred model id).
|
|
1083
1561
|
* @param coAgent The resolved co-agent (threaded to the default-resolution seam).
|
|
1084
|
-
* @param effectiveConfig The resolved effective configuration (carries `modelPreference`).
|
|
1562
|
+
* @param effectiveConfig The resolved effective configuration (carries `modelPreference` and the video setting).
|
|
1085
1563
|
* @returns The resolution outcome (resolution or failure reason).
|
|
1086
1564
|
*/
|
|
1087
1565
|
async resolveModelForSession(input, coAgent, effectiveConfig) {
|
|
@@ -1093,16 +1571,61 @@ export class RealtimeClientSessionService {
|
|
|
1093
1571
|
// subclass that overrides the seam would lose to AI_VENDOR_API_KEY__<driver>.
|
|
1094
1572
|
const resolveRunKey = this.buildRunKeyResolver(input.APIKeys);
|
|
1095
1573
|
const scope = input.CredentialScope ?? 'Any';
|
|
1574
|
+
const preferAvatar = effectiveConfig?.realtime?.video?.enabled === true && RealtimeSessionCanShowAvatar(input);
|
|
1096
1575
|
if (input.PreferredModelID) {
|
|
1097
|
-
return this.resolvePreferredRealtimeModel(input.PreferredModelID, resolveRunKey, scope);
|
|
1576
|
+
return this.resolvePreferredRealtimeModel(input.PreferredModelID, resolveRunKey, scope, preferAvatar);
|
|
1098
1577
|
}
|
|
1099
|
-
const fromConfig = this.resolveConfiguredModelPreference(effectiveConfig, resolveRunKey, scope);
|
|
1578
|
+
const fromConfig = this.resolveConfiguredModelPreference(effectiveConfig, resolveRunKey, scope, preferAvatar);
|
|
1100
1579
|
if (fromConfig) {
|
|
1101
1580
|
return { Resolution: fromConfig };
|
|
1102
1581
|
}
|
|
1103
|
-
const resolution = await this.
|
|
1582
|
+
const resolution = await this.resolveDefaultRealtimeModel(coAgent, preferAvatar, resolveRunKey, scope);
|
|
1104
1583
|
return resolution ? { Resolution: resolution } : { ErrorMessage: this.noModelMessage() };
|
|
1105
1584
|
}
|
|
1585
|
+
/**
|
|
1586
|
+
* The default walk. With `preferAvatar` (the co-agent's video setting is on and the session could show an avatar),
|
|
1587
|
+
* it takes the first candidate that shows one ({@link ResolveAvatarRealtimeModel}). When none does, or without it, it
|
|
1588
|
+
* takes {@link resolveRealtimeModel}'s choice; when one was preferred, one log line says the call stays audio only.
|
|
1589
|
+
*/
|
|
1590
|
+
async resolveDefaultRealtimeModel(coAgent, preferAvatar, resolve, credentialScope) {
|
|
1591
|
+
const withAvatar = preferAvatar ? this.ResolveAvatarRealtimeModel(coAgent, resolve, credentialScope) : null;
|
|
1592
|
+
if (withAvatar) {
|
|
1593
|
+
return withAvatar;
|
|
1594
|
+
}
|
|
1595
|
+
const resolution = await this.resolveRealtimeModel(coAgent, resolve, credentialScope, preferAvatar);
|
|
1596
|
+
if (preferAvatar && resolution) {
|
|
1597
|
+
LogStatus('[RealtimeCoAgent] Video is on, but no realtime model with a usable key shows an avatar (its Video/Output row ' +
|
|
1598
|
+
`and its endpoint): using '${resolution.ModelName ?? resolution.APIName}' on ${resolution.DriverClass ?? 'its driver'}, audio only.`);
|
|
1599
|
+
}
|
|
1600
|
+
return resolution;
|
|
1601
|
+
}
|
|
1602
|
+
/**
|
|
1603
|
+
* For a co-agent with video on: the first candidate that shows an avatar and supports client-direct sessions, in the
|
|
1604
|
+
* default walk's order (highest PowerRank first; a model's Active vendor rows by Priority, each with a usable key).
|
|
1605
|
+
* A candidate shows an avatar when its model's Video/Output row allows video or it has none, and its driver renders
|
|
1606
|
+
* avatars for the vendor's API name on its endpoint ({@link RealtimeModelShowsAvatar}). Every keyed vendor of a model
|
|
1607
|
+
* is tried, not only the first. **Overridable seam.**
|
|
1608
|
+
*
|
|
1609
|
+
* @param coAgent The co-agent (reserved for future per-agent model preference).
|
|
1610
|
+
* @param resolve The session's run-scoped key resolver (run keys only; see {@link resolveVendorAndInstantiate}).
|
|
1611
|
+
* @param credentialScope The session's credential scope; see {@link resolveVendorAndInstantiate}.
|
|
1612
|
+
* @returns The resolution, or `null` when no candidate shows an avatar.
|
|
1613
|
+
*/
|
|
1614
|
+
ResolveAvatarRealtimeModel(coAgent, resolve, credentialScope = 'Any') {
|
|
1615
|
+
const resolveKey = this.buildKeyChain(resolve, credentialScope);
|
|
1616
|
+
for (const model of this.selectRealtimeModelCandidates(coAgent)) {
|
|
1617
|
+
if (ReadRealtimeVideoOutputRow(model.ID, AIEngine.Instance) === 'unsupported') {
|
|
1618
|
+
continue;
|
|
1619
|
+
}
|
|
1620
|
+
for (const vendor of ListRealtimeVendorsForModel(model.ID)) {
|
|
1621
|
+
const resolution = this.instantiateRealtimeVendor(model, vendor, resolveKey);
|
|
1622
|
+
if (resolution?.Model.SupportsClientDirect && RealtimeModelShowsAvatar(resolution, AIEngine.Instance)) {
|
|
1623
|
+
return resolution;
|
|
1624
|
+
}
|
|
1625
|
+
}
|
|
1626
|
+
}
|
|
1627
|
+
return null;
|
|
1628
|
+
}
|
|
1106
1629
|
/**
|
|
1107
1630
|
* The session's run-scoped keys as a resolver: the key for a driver class when the run carries
|
|
1108
1631
|
* one, else `undefined` — no platform fallback. The fallback belongs to
|
|
@@ -1125,9 +1648,10 @@ export class RealtimeClientSessionService {
|
|
|
1125
1648
|
* @param effectiveConfig The resolved effective configuration.
|
|
1126
1649
|
* @param resolve The session's run-scoped key resolver (run keys only; see {@link resolveVendorAndInstantiate}).
|
|
1127
1650
|
* @param credentialScope The session's credential scope; see {@link resolveVendorAndInstantiate}.
|
|
1651
|
+
* @param showsAvatar Whether the session will show an avatar; see {@link resolveVendorAndInstantiate}.
|
|
1128
1652
|
* @returns The resolution, or `null` when no preference is configured or it can't be satisfied.
|
|
1129
1653
|
*/
|
|
1130
|
-
resolveConfiguredModelPreference(effectiveConfig, resolve, credentialScope = 'Any') {
|
|
1654
|
+
resolveConfiguredModelPreference(effectiveConfig, resolve, credentialScope = 'Any', showsAvatar = false) {
|
|
1131
1655
|
const preference = effectiveConfig?.realtime?.modelPreference;
|
|
1132
1656
|
if (!preference) {
|
|
1133
1657
|
return null;
|
|
@@ -1143,7 +1667,7 @@ export class RealtimeClientSessionService {
|
|
|
1143
1667
|
`'${model.Name}' but it is not an Active Realtime model — falling through to default resolution.`);
|
|
1144
1668
|
return null;
|
|
1145
1669
|
}
|
|
1146
|
-
const resolution = this.resolveVendorAndInstantiate(model, resolve, credentialScope);
|
|
1670
|
+
const resolution = this.resolveVendorAndInstantiate(model, resolve, credentialScope, showsAvatar);
|
|
1147
1671
|
if (!resolution) {
|
|
1148
1672
|
LogError(`RealtimeClientSessionService: configured realtime model preference '${model.Name}' has no usable ` +
|
|
1149
1673
|
'vendor DriverClass/API key — falling through to default resolution.');
|
|
@@ -1177,9 +1701,10 @@ export class RealtimeClientSessionService {
|
|
|
1177
1701
|
* @param preferredModelID The `MJ: AI Models.ID` the user chose.
|
|
1178
1702
|
* @param resolve The session's run-scoped key resolver (run keys only; see {@link resolveVendorAndInstantiate}).
|
|
1179
1703
|
* @param credentialScope The session's credential scope; see {@link resolveVendorAndInstantiate}.
|
|
1704
|
+
* @param showsAvatar Whether the session will show an avatar; see {@link resolveVendorAndInstantiate}.
|
|
1180
1705
|
* @returns The resolution outcome (resolution or a specific failure reason).
|
|
1181
1706
|
*/
|
|
1182
|
-
resolvePreferredRealtimeModel(preferredModelID, resolve, credentialScope = 'Any') {
|
|
1707
|
+
resolvePreferredRealtimeModel(preferredModelID, resolve, credentialScope = 'Any', showsAvatar = false) {
|
|
1183
1708
|
const model = this.findModelByID(preferredModelID);
|
|
1184
1709
|
if (!model) {
|
|
1185
1710
|
return { ErrorMessage: `The requested realtime model (id '${preferredModelID}') was not found in AI model metadata.` };
|
|
@@ -1190,7 +1715,7 @@ export class RealtimeClientSessionService {
|
|
|
1190
1715
|
if (!this.isRealtimeModel(model)) {
|
|
1191
1716
|
return { ErrorMessage: `The requested model '${model.Name}' is not a Realtime model (its type is '${model.AIModelType}').` };
|
|
1192
1717
|
}
|
|
1193
|
-
const resolution = this.resolveVendorAndInstantiate(model, resolve, credentialScope);
|
|
1718
|
+
const resolution = this.resolveVendorAndInstantiate(model, resolve, credentialScope, showsAvatar);
|
|
1194
1719
|
if (!resolution) {
|
|
1195
1720
|
return {
|
|
1196
1721
|
ErrorMessage: `The requested model '${model.Name}' has no active vendor with a usable DriverClass/API key ` +
|
|
@@ -1224,9 +1749,10 @@ export class RealtimeClientSessionService {
|
|
|
1224
1749
|
* @param coAgent The co-agent being voiced (reserved for future per-agent model preference).
|
|
1225
1750
|
* @param resolve The session's run-scoped key resolver (run keys only; see {@link resolveVendorAndInstantiate}).
|
|
1226
1751
|
* @param credentialScope The session's credential scope; see {@link resolveVendorAndInstantiate}.
|
|
1752
|
+
* @param showsAvatar Whether the session will show an avatar; see {@link resolveVendorAndInstantiate}.
|
|
1227
1753
|
* @returns The resolved model + identifiers, or `null`.
|
|
1228
1754
|
*/
|
|
1229
|
-
async resolveRealtimeModel(coAgent, resolve, credentialScope = 'Any') {
|
|
1755
|
+
async resolveRealtimeModel(coAgent, resolve, credentialScope = 'Any', showsAvatar = false) {
|
|
1230
1756
|
// Walk candidates in descending PowerRank, returning the FIRST that fully resolves to a usable
|
|
1231
1757
|
// client-direct driver (active vendor + API key + ClassFactory driver + SupportsClientDirect).
|
|
1232
1758
|
// Single-pick dead-ended whenever the highest-power model lacked a key or client-direct support
|
|
@@ -1234,7 +1760,7 @@ export class RealtimeClientSessionService {
|
|
|
1234
1760
|
// surfaced "No usable Realtime model" instead of falling through to a model that works.
|
|
1235
1761
|
const candidates = this.selectRealtimeModelCandidates(coAgent);
|
|
1236
1762
|
for (const model of candidates) {
|
|
1237
|
-
const resolution = this.resolveVendorAndInstantiate(model, resolve, credentialScope);
|
|
1763
|
+
const resolution = this.resolveVendorAndInstantiate(model, resolve, credentialScope, showsAvatar);
|
|
1238
1764
|
if (resolution && resolution.Model.SupportsClientDirect) {
|
|
1239
1765
|
return resolution;
|
|
1240
1766
|
}
|
|
@@ -1256,19 +1782,40 @@ export class RealtimeClientSessionService {
|
|
|
1256
1782
|
* @param credentialScope A scope that rules out the `'Environment'` source (`'RuntimeOnly'`) drops the
|
|
1257
1783
|
* {@link getAPIKeyForDriver} seam: the run's keys are the whole chain, so a vendor they do not key
|
|
1258
1784
|
* is never selected on the platform's key.
|
|
1785
|
+
* @param showsAvatar Whether the session will show an avatar. The highest-Priority keyed vendor stands when it
|
|
1786
|
+
* renders avatars ({@link RealtimeModelShowsAvatar}) exactly when the session will show one; otherwise the first
|
|
1787
|
+
* other keyed vendor that does is taken, when there is one. So a vendor that renders avatars (Vertex AI for Gemini
|
|
1788
|
+
* 3.8 Live, with its own billing and limits) takes a session only when it will show an avatar, or when it is the
|
|
1789
|
+
* only keyed vendor; and a session that will show one goes to such a vendor whatever the Priority order.
|
|
1259
1790
|
* @returns The full resolution, or `null` when no vendor/key/driver can be satisfied.
|
|
1260
1791
|
*/
|
|
1261
|
-
resolveVendorAndInstantiate(model, resolve, credentialScope = 'Any') {
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1792
|
+
resolveVendorAndInstantiate(model, resolve, credentialScope = 'Any', showsAvatar = false) {
|
|
1793
|
+
const resolveKey = this.buildKeyChain(resolve, credentialScope);
|
|
1794
|
+
const vendor = this.selectRealtimeVendor(model.ID, resolveKey);
|
|
1795
|
+
const first = vendor ? this.instantiateRealtimeVendor(model, vendor, resolveKey) : null;
|
|
1796
|
+
if (!vendor || !first) {
|
|
1797
|
+
return first;
|
|
1798
|
+
}
|
|
1799
|
+
return ChooseRealtimeVendorForAvatar(vendor, first, ListRealtimeVendorsForModel(model.ID), showsAvatar, {
|
|
1800
|
+
Instantiate: (other) => this.instantiateRealtimeVendor(model, other, resolveKey),
|
|
1801
|
+
RendersAvatar: (resolution) => RealtimeModelShowsAvatar(resolution, AIEngine.Instance),
|
|
1802
|
+
// A stand-in does client-direct sessions when the first one does.
|
|
1803
|
+
Accepts: (resolution) => resolution.Model.SupportsClientDirect || !first.Model.SupportsClientDirect,
|
|
1804
|
+
});
|
|
1805
|
+
}
|
|
1806
|
+
/**
|
|
1807
|
+
* The session's complete key chain for vendor selection and the mint (see {@link resolveVendorAndInstantiate}): the
|
|
1808
|
+
* run's keys first, then this service's own seam (which subclasses and tests override), so a run-scoped credential
|
|
1809
|
+
* wins without taking that seam away from anyone who replaced it. A scope that rules out the environment has no second
|
|
1810
|
+
* step: the seam's default is the platform key.
|
|
1811
|
+
*/
|
|
1812
|
+
buildKeyChain(resolve, credentialScope) {
|
|
1813
|
+
return CredentialScopeAllows(credentialScope, 'Environment')
|
|
1266
1814
|
? (driverClass) => resolve?.(driverClass) ?? this.getAPIKeyForDriver(driverClass)
|
|
1267
1815
|
: (driverClass) => resolve?.(driverClass);
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
}
|
|
1816
|
+
}
|
|
1817
|
+
/** Instantiates a vendor's realtime driver with its key from the chain; `null` when the key or the driver is missing. */
|
|
1818
|
+
instantiateRealtimeVendor(model, vendor, resolveKey) {
|
|
1272
1819
|
const apiKey = resolveKey(vendor.DriverClass);
|
|
1273
1820
|
if (!apiKey) {
|
|
1274
1821
|
return null;
|
|
@@ -1313,7 +1860,9 @@ export class RealtimeClientSessionService {
|
|
|
1313
1860
|
}
|
|
1314
1861
|
/**
|
|
1315
1862
|
* The active models of AIModelType `Realtime`, sorted highest-PowerRank first — the candidate
|
|
1316
|
-
* list {@link resolveRealtimeModel} walks until one yields a usable client-direct driver
|
|
1863
|
+
* list {@link resolveRealtimeModel} walks until one yields a usable client-direct driver, and
|
|
1864
|
+
* {@link ResolveAvatarRealtimeModel} walks first for a co-agent with video on. Equal PowerRanks
|
|
1865
|
+
* keep the engine's cached order (a stable sort).
|
|
1317
1866
|
* Returns ALL candidates (not just the top pick) so a keyless or non-client-direct top model
|
|
1318
1867
|
* falls through to the next usable one instead of dead-ending the whole resolution.
|
|
1319
1868
|
*
|
|
@@ -1364,10 +1913,11 @@ export class RealtimeClientSessionService {
|
|
|
1364
1913
|
* @param driverClass The resolved vendor's DriverClass — matches per-provider voice settings.
|
|
1365
1914
|
* @param modelID The resolved `MJ: AI Models` id — keys the model-catalog `ModelConfiguration` cascade.
|
|
1366
1915
|
* @param modelVendorID The resolved `MJ: AI Model Vendors` ROW id — the cascade's most-specific layer.
|
|
1916
|
+
* @param resolvedAvatar The session's avatar resolution, when the caller already made it; resolved here otherwise.
|
|
1367
1917
|
* @returns The assembled session params.
|
|
1368
1918
|
*/
|
|
1369
|
-
async buildSessionParams(input, coAgent, modelApiName, contextUser, provider, effectiveConfig, driverClass, modelID, modelVendorID) {
|
|
1370
|
-
const directTools = this.
|
|
1919
|
+
async buildSessionParams(input, coAgent, modelApiName, contextUser, provider, effectiveConfig, driverClass, modelID, modelVendorID, resolvedAvatar) {
|
|
1920
|
+
const directTools = this.projectDirectActions(input, effectiveConfig, driverClass);
|
|
1371
1921
|
const hasDirectTools = directTools.length > 0 || (input.ExtraTools != null && input.ExtraTools.length > 0);
|
|
1372
1922
|
const systemPrompt = await this.buildCompanionSystemPrompt(input, coAgent, contextUser, provider, effectiveConfig, hasDirectTools);
|
|
1373
1923
|
const memoryContext = await this.assembleMemoryContext(input, coAgent, contextUser, provider);
|
|
@@ -1376,8 +1926,11 @@ export class RealtimeClientSessionService {
|
|
|
1376
1926
|
: input.ExtraTools;
|
|
1377
1927
|
const tools = this.appendHostTools(this.buildStableToolSet(combinedExtra), input.HostTools);
|
|
1378
1928
|
// Hoisted (rather than built inline at the return) so the mint log below can report the voice
|
|
1379
|
-
// that ACTUALLY reached the driver — see the `voice=` field. Same bag, built once.
|
|
1380
|
-
|
|
1929
|
+
// that ACTUALLY reached the driver — see the `voice=` field. Same bag, built once. A phone call, or
|
|
1930
|
+
// an app without agent video, carries no avatar request whoever resolved it (WithoutUnseenAvatar is
|
|
1931
|
+
// idempotent).
|
|
1932
|
+
const avatar = WithoutUnseenAvatar(resolvedAvatar ?? this.ResolveSessionAvatar(input, coAgent, effectiveConfig, modelID, modelVendorID), input);
|
|
1933
|
+
const configBag = this.withAvatarVoice(this.buildSessionConfigBag(input, effectiveConfig, driverClass, modelID, modelVendorID), avatar, input);
|
|
1381
1934
|
WarnOnUnmatchedProviderVoice(effectiveConfig, driverClass, 'RealtimeClientSessionService');
|
|
1382
1935
|
// One line per mint: confirms which tools + whether the channel-direct framing actually reach
|
|
1383
1936
|
// the model — settles "why does the co-agent delegate instead of calling browser_*" without
|
|
@@ -1386,6 +1939,7 @@ export class RealtimeClientSessionService {
|
|
|
1386
1939
|
// actually reached the driver legible — the whole failure mode in #3530 was that it was not.
|
|
1387
1940
|
console.log(`[RealtimeCoAgent] mint model=${modelApiName} driver=${driverClass ?? 'unknown'} ` +
|
|
1388
1941
|
`voice=${typeof configBag?.['voice'] === 'string' ? configBag['voice'] : 'none'} ` +
|
|
1942
|
+
`avatar=${avatar.Avatar ? avatar.Avatar.AvatarID : avatar.Reason ? `none(${avatar.Reason})` : 'none'} ` +
|
|
1389
1943
|
`tools=[${tools.map(t => t.Name).join(', ')}] ` +
|
|
1390
1944
|
`channelExceptionInPrompt=${systemPrompt.includes('interactive-surface')}`);
|
|
1391
1945
|
return {
|
|
@@ -1400,8 +1954,58 @@ export class RealtimeClientSessionService {
|
|
|
1400
1954
|
MaxSessionSeconds: input.MaxSessionSeconds,
|
|
1401
1955
|
UserID: contextUser?.ID,
|
|
1402
1956
|
HasToolFraming: true,
|
|
1957
|
+
...(avatar.Avatar ? { Avatar: input.AvatarDelivery ? { ...avatar.Avatar, Delivery: input.AvatarDelivery } : avatar.Avatar } : {}),
|
|
1403
1958
|
};
|
|
1404
1959
|
}
|
|
1960
|
+
/**
|
|
1961
|
+
* The live avatar this session asks for, if any: from the voiced agent's persona, or `realtime.video.avatarId`,
|
|
1962
|
+
* when the agent's video setting is on (see `ResolveRealtimeAvatar`). A call that picked an avatar (its runtime
|
|
1963
|
+
* override names `realtime.video.avatarId`) while that setting is off asks for none, with reason `video-off`. A model
|
|
1964
|
+
* whose `MJ: AI Model Modalities`
|
|
1965
|
+
* Video/Output row turns video off asks for none (reason `endpoint`, one log line); otherwise whether the session can
|
|
1966
|
+
* render it is the driver's call. A seam so tests can supply a resolution without the engine's persona metadata.
|
|
1967
|
+
*
|
|
1968
|
+
* @param input The prepare-session input (the voiced agent).
|
|
1969
|
+
* @param coAgent The co-agent.
|
|
1970
|
+
* @param effectiveConfig The effective configuration.
|
|
1971
|
+
* @param modelID The resolved model.
|
|
1972
|
+
* @param modelVendorID The resolved model-vendor row; its vendor scopes the persona bindings.
|
|
1973
|
+
*/
|
|
1974
|
+
ResolveSessionAvatar(input, coAgent, effectiveConfig, modelID, modelVendorID) {
|
|
1975
|
+
const videoOn = effectiveConfig?.realtime?.video?.enabled === true;
|
|
1976
|
+
const avatarPicked = !!ResolveEffectiveRealtimeConfig(null, null, input.ConfigOverridesJson).realtime?.video?.avatarId;
|
|
1977
|
+
if (!modelID || (!videoOn && !avatarPicked)) {
|
|
1978
|
+
return {};
|
|
1979
|
+
}
|
|
1980
|
+
const vendorID = modelVendorID ? AIEngine.Instance.ModelVendors.find((mv) => UUIDsEqual(mv.ID, modelVendorID))?.VendorID : undefined;
|
|
1981
|
+
const videoOutputRow = ReadRealtimeVideoOutputRow(modelID, AIEngine.Instance);
|
|
1982
|
+
if (videoOn && videoOutputRow === 'unsupported') {
|
|
1983
|
+
LogStatus(`[RealtimeCoAgent] No avatar asked for: the Video/Output modality row of '${this.findModelByID(modelID)?.Name ?? modelID}' ` +
|
|
1984
|
+
'turns video off (IsSupported false). The call is audio only. Reason: endpoint.');
|
|
1985
|
+
}
|
|
1986
|
+
return ResolveRealtimeAvatar({
|
|
1987
|
+
EffectiveConfig: effectiveConfig,
|
|
1988
|
+
TargetAgentID: input.TargetAgentID,
|
|
1989
|
+
CoAgentID: coAgent.ID,
|
|
1990
|
+
ModelID: modelID,
|
|
1991
|
+
VendorID: vendorID,
|
|
1992
|
+
VideoOutputRow: videoOutputRow,
|
|
1993
|
+
AvatarPicked: avatarPicked,
|
|
1994
|
+
}, AIEngine.Instance);
|
|
1995
|
+
}
|
|
1996
|
+
/**
|
|
1997
|
+
* The config bag with the avatar persona's voice, so the face and the voice match, unless a voice was picked in this
|
|
1998
|
+
* call (a runtime override's `realtime.voice.default.voice`), which wins. A session nobody would see the avatar in (a
|
|
1999
|
+
* phone call, an app without agent video) keeps the voice without the face, so the agent sounds the same as in a call
|
|
2000
|
+
* that shows it.
|
|
2001
|
+
*/
|
|
2002
|
+
withAvatarVoice(bag, avatar, input) {
|
|
2003
|
+
if (!avatar.Voice) {
|
|
2004
|
+
return bag;
|
|
2005
|
+
}
|
|
2006
|
+
const picked = ResolveEffectiveRealtimeConfig(null, null, input.ConfigOverridesJson).realtime?.voice?.default?.voice;
|
|
2007
|
+
return picked ? bag : { ...(bag ?? {}), voice: avatar.Voice };
|
|
2008
|
+
}
|
|
1405
2009
|
/**
|
|
1406
2010
|
* Builds the provider-pact `Config` bag for the session: the effective config's resolved voice
|
|
1407
2011
|
* settings (per {@link GetProviderVoiceSettings} — the persona's agnostic wire-level slots
|
|
@@ -1599,7 +2203,9 @@ export class RealtimeClientSessionService {
|
|
|
1599
2203
|
`Knowledge Hub" → ContextTool with action 'NavigateToApp'; "open this record"; "switch to the X tab"). ` +
|
|
1600
2204
|
`Do NOT route in-app navigation/actions through '${INVOKE_TARGET_AGENT_TOOL_NAME}', and NEVER say you ` +
|
|
1601
2205
|
`need an active session id or a screen snapshot to do them — calling 'ContextTool' is all that's ` +
|
|
1602
|
-
`needed. Reserve '${INVOKE_TARGET_AGENT_TOOL_NAME}' for actual analysis / data work, not navigation
|
|
2206
|
+
`needed. Reserve '${INVOKE_TARGET_AGENT_TOOL_NAME}' for actual analysis / data work, not navigation. ` +
|
|
2207
|
+
`When a channel note lists interactive channels you can use, 'ContextTool' also addresses ONE of ` +
|
|
2208
|
+
`them: add "target": { "channel": "<channel name>" } and use that channel's action names.`;
|
|
1603
2209
|
}
|
|
1604
2210
|
return clause;
|
|
1605
2211
|
}
|
|
@@ -2006,11 +2612,13 @@ export class RealtimeClientSessionService {
|
|
|
2006
2612
|
/**
|
|
2007
2613
|
* Runs (or resumes) the target agent for a delegation. Threads the combined abort signal, parent
|
|
2008
2614
|
* run linkage, session id, and the `OnProgress` callback so the resolver can stream progress.
|
|
2615
|
+
* The browser session id becomes the run's `sessionID` (client-tool channel) and the app-context
|
|
2616
|
+
* snapshot rides `data.appContext` (prompt context and surface tools); both are optional.
|
|
2009
2617
|
* When {@link ExecuteRelayedToolInput.ResumeRunID} is set, resumes that paused run via
|
|
2010
2618
|
* `lastRunId` + `autoPopulateLastRunPayload` (the user's answer continues the same interactive
|
|
2011
2619
|
* run) instead of starting fresh.
|
|
2012
2620
|
*
|
|
2013
|
-
* @param input The relayed tool input (linkage, progress callback, optional resume id).
|
|
2621
|
+
* @param input The relayed tool input (linkage, progress callback, optional resume id, browser session, app context).
|
|
2014
2622
|
* @param request The broker's delegation request (call id + arguments + abort signal).
|
|
2015
2623
|
* @param target The resolved target agent.
|
|
2016
2624
|
* @param contextUser The calling user.
|
|
@@ -2033,6 +2641,8 @@ export class RealtimeClientSessionService {
|
|
|
2033
2641
|
cancellationToken: this.combineSignals(request.AbortSignal, input.AbortSignal),
|
|
2034
2642
|
parentRun: parentRun ?? undefined,
|
|
2035
2643
|
agentSessionID: input.AgentSessionID,
|
|
2644
|
+
sessionID: input.BrowserSessionID,
|
|
2645
|
+
data: input.AppContext ? { appContext: input.AppContext } : undefined,
|
|
2036
2646
|
onProgress: input.OnProgress,
|
|
2037
2647
|
lastRunId: input.ResumeRunID,
|
|
2038
2648
|
autoPopulateLastRunPayload: input.ResumeRunID ? true : undefined
|
|
@@ -2118,11 +2728,25 @@ export class RealtimeClientSessionService {
|
|
|
2118
2728
|
const agentActions = (AIEngine.Instance.AgentActions ?? []).filter(aa => UUIDsEqual(aa.AgentID, targetAgentID) && aa.Status === 'Active');
|
|
2119
2729
|
return (ActionEngineServer.Instance.Actions ?? []).filter(a => agentActions.some(aa => UUIDsEqual(aa.ActionID, a.ID)) && a.Status === 'Active');
|
|
2120
2730
|
}
|
|
2731
|
+
/**
|
|
2732
|
+
* The session's direct-action tools. When the input excludes direct actions, none, and the empty
|
|
2733
|
+
* projection is recorded so the session's relays run none.
|
|
2734
|
+
*/
|
|
2735
|
+
projectDirectActions(input, effectiveConfig, driverClass) {
|
|
2736
|
+
if (input.ExcludeDirectActions) {
|
|
2737
|
+
this.recordEmptyProjection(input.AgentSessionID);
|
|
2738
|
+
return [];
|
|
2739
|
+
}
|
|
2740
|
+
return this.BuildDirectActionTools(input.TargetAgentID, effectiveConfig, driverClass, input.AgentSessionID);
|
|
2741
|
+
}
|
|
2121
2742
|
/**
|
|
2122
2743
|
* Builds the projected direct action tool definitions for a target agent if supported by the driver.
|
|
2123
2744
|
* Gated by driver capability (`SupportsDynamicToolSet`) and explicit configuration opt-in
|
|
2124
2745
|
* (`Configuration.realtime.directActions.enabled = true`).
|
|
2125
2746
|
*
|
|
2747
|
+
* With `agentSessionID`, the projection is recorded for the session, even when it is empty: relayed
|
|
2748
|
+
* direct-action calls for the session may run only the projected actions.
|
|
2749
|
+
*
|
|
2126
2750
|
* @param targetAgentID The target agent ID being voiced.
|
|
2127
2751
|
* @param effectiveConfig The resolved effective configuration.
|
|
2128
2752
|
* @param driverClass The resolved vendor's DriverClass.
|
|
@@ -2130,10 +2754,12 @@ export class RealtimeClientSessionService {
|
|
|
2130
2754
|
*/
|
|
2131
2755
|
BuildDirectActionTools(targetAgentID, effectiveConfig, driverClass, agentSessionID) {
|
|
2132
2756
|
if (!targetAgentID || !this.driverSupportsDynamicToolSet(driverClass)) {
|
|
2757
|
+
this.recordEmptyProjection(agentSessionID);
|
|
2133
2758
|
return [];
|
|
2134
2759
|
}
|
|
2135
2760
|
const directConfig = GetDirectActionsConfig(effectiveConfig);
|
|
2136
2761
|
if (!directConfig || !directConfig.enabled) {
|
|
2762
|
+
this.recordEmptyProjection(agentSessionID);
|
|
2137
2763
|
return [];
|
|
2138
2764
|
}
|
|
2139
2765
|
const candidateActions = this.getTargetAgentActions(targetAgentID);
|
|
@@ -2152,6 +2778,12 @@ export class RealtimeClientSessionService {
|
|
|
2152
2778
|
}
|
|
2153
2779
|
return tools;
|
|
2154
2780
|
}
|
|
2781
|
+
/** Records that a session projected no direct actions, so its relays run none. */
|
|
2782
|
+
recordEmptyProjection(agentSessionID) {
|
|
2783
|
+
if (agentSessionID) {
|
|
2784
|
+
this.sessionWireActionMaps.Set(agentSessionID, new Map());
|
|
2785
|
+
}
|
|
2786
|
+
}
|
|
2155
2787
|
/** @deprecated Use {@link BuildDirectActionTools}. */
|
|
2156
2788
|
buildDirectActionTools(targetAgentID, effectiveConfig, driverClass, agentSessionID) {
|
|
2157
2789
|
return this.BuildDirectActionTools(targetAgentID, effectiveConfig, driverClass, agentSessionID);
|
|
@@ -2207,6 +2839,13 @@ export class RealtimeClientSessionService {
|
|
|
2207
2839
|
* configured timeout. If the tool is unrecognized, disallowed, or target resolution fails,
|
|
2208
2840
|
* returns a structured "not available" result.
|
|
2209
2841
|
*
|
|
2842
|
+
* When this service holds the session's start-time projection, only the projected actions may
|
|
2843
|
+
* run, whatever target the input names. When it holds none, the target agent's own metadata
|
|
2844
|
+
* decides: another process prepared the session, the entry was evicted or lost on a restart, or
|
|
2845
|
+
* the session was never started through this service (for example a row created through the
|
|
2846
|
+
* entity API). That fallback skips the driver-capability gate but is still limited to the
|
|
2847
|
+
* target's own direct-actions allowlist.
|
|
2848
|
+
*
|
|
2210
2849
|
* @param call The non-target tool call.
|
|
2211
2850
|
* @param input The optional relayed tool input context.
|
|
2212
2851
|
* @param contextUser The calling user context.
|
|
@@ -2228,11 +2867,9 @@ export class RealtimeClientSessionService {
|
|
|
2228
2867
|
Output: `Target agent is not available to execute '${call.ToolName}'.`
|
|
2229
2868
|
};
|
|
2230
2869
|
}
|
|
2231
|
-
const
|
|
2232
|
-
const candidateWireMap = this.BuildWireActionMap(
|
|
2233
|
-
const action = (
|
|
2234
|
-
?? candidateWireMap.get(call.ToolName)
|
|
2235
|
-
?? Array.from(candidateWireMap.entries()).find(([w]) => w.toLowerCase() === call.ToolName.trim().toLowerCase())?.[1];
|
|
2870
|
+
const projection = input?.AgentSessionID ? this.sessionWireActionMaps.Get(input.AgentSessionID) : undefined;
|
|
2871
|
+
const candidateWireMap = this.BuildWireActionMap(this.getTargetAgentActions(target.ID));
|
|
2872
|
+
const action = this.findWireAction(projection, call.ToolName) ?? this.findWireAction(candidateWireMap, call.ToolName);
|
|
2236
2873
|
if (!action) {
|
|
2237
2874
|
return {
|
|
2238
2875
|
CallID: call.CallID,
|
|
@@ -2243,7 +2880,7 @@ export class RealtimeClientSessionService {
|
|
|
2243
2880
|
const directConfig = input?.DirectActions
|
|
2244
2881
|
?? (input?.AgentSessionID ? this.sessionDirectConfigs.Get(input.AgentSessionID) : undefined)
|
|
2245
2882
|
?? GetDirectActionsConfig(this.resolveEffectiveConfig(target, undefined, target));
|
|
2246
|
-
if (!IsActionAllowedForDirectInvocation(action.Name, directConfig)) {
|
|
2883
|
+
if (!this.isInProjection(projection, action) || !IsActionAllowedForDirectInvocation(action.Name, directConfig)) {
|
|
2247
2884
|
return {
|
|
2248
2885
|
CallID: call.CallID,
|
|
2249
2886
|
Success: false,
|
|
@@ -2304,6 +2941,24 @@ export class RealtimeClientSessionService {
|
|
|
2304
2941
|
};
|
|
2305
2942
|
}
|
|
2306
2943
|
}
|
|
2944
|
+
/**
|
|
2945
|
+
* Finds the action a relayed tool name refers to: the exact wire name first, then a
|
|
2946
|
+
* case-insensitive match.
|
|
2947
|
+
*/
|
|
2948
|
+
findWireAction(wireMap, toolName) {
|
|
2949
|
+
if (!wireMap) {
|
|
2950
|
+
return undefined;
|
|
2951
|
+
}
|
|
2952
|
+
const wanted = toolName.trim().toLowerCase();
|
|
2953
|
+
return wireMap.get(toolName) ?? Array.from(wireMap.entries()).find(([wireName]) => wireName.toLowerCase() === wanted)?.[1];
|
|
2954
|
+
}
|
|
2955
|
+
/**
|
|
2956
|
+
* True when the action is in the session's start-time projection, or when this service holds no
|
|
2957
|
+
* projection for the session (see {@link executeNonTargetTool} for when that happens).
|
|
2958
|
+
*/
|
|
2959
|
+
isInProjection(projection, action) {
|
|
2960
|
+
return !projection || Array.from(projection.values()).some(projected => UUIDsEqual(projected.ID, action.ID));
|
|
2961
|
+
}
|
|
2307
2962
|
/**
|
|
2308
2963
|
* Parses JSON arguments string into a key-value record of parameters.
|
|
2309
2964
|
* Distinguishes "no arguments" (valid for zero-param action) from "unparseable JSON".
|