@memberjunction/ai-agents 6.2.0-edge.2 → 6.2.0-edge.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/dist/AgentDecisionService.d.ts +8 -1
- package/dist/AgentDecisionService.d.ts.map +1 -1
- package/dist/AgentDecisionService.js +4 -0
- package/dist/AgentDecisionService.js.map +1 -1
- package/dist/AgentRunner.d.ts +51 -2
- package/dist/AgentRunner.d.ts.map +1 -1
- package/dist/AgentRunner.js +121 -23
- package/dist/AgentRunner.js.map +1 -1
- package/dist/ConversationCompactionManager.d.ts +9 -1
- package/dist/ConversationCompactionManager.d.ts.map +1 -1
- package/dist/ConversationCompactionManager.js +6 -3
- package/dist/ConversationCompactionManager.js.map +1 -1
- package/dist/ConversationToolManager.d.ts +13 -2
- package/dist/ConversationToolManager.d.ts.map +1 -1
- package/dist/ConversationToolManager.js +19 -3
- package/dist/ConversationToolManager.js.map +1 -1
- package/dist/KnowledgeAgent.d.ts.map +1 -1
- package/dist/KnowledgeAgent.js +6 -1
- package/dist/KnowledgeAgent.js.map +1 -1
- package/dist/MJAIAgentRequestEntityServer.d.ts +6 -0
- package/dist/MJAIAgentRequestEntityServer.d.ts.map +1 -1
- package/dist/MJAIAgentRequestEntityServer.js +26 -1
- package/dist/MJAIAgentRequestEntityServer.js.map +1 -1
- package/dist/PayloadFeedbackManager.d.ts +3 -0
- package/dist/PayloadFeedbackManager.d.ts.map +1 -1
- package/dist/PayloadFeedbackManager.js +2 -1
- package/dist/PayloadFeedbackManager.js.map +1 -1
- package/dist/agent-run-watchdog.d.ts +65 -5
- package/dist/agent-run-watchdog.d.ts.map +1 -1
- package/dist/agent-run-watchdog.js +143 -28
- package/dist/agent-run-watchdog.js.map +1 -1
- package/dist/agent-types/loop-agent-prompt-params.d.ts +7 -0
- package/dist/agent-types/loop-agent-prompt-params.d.ts.map +1 -1
- package/dist/agent-types/loop-agent-prompt-params.js +1 -0
- package/dist/agent-types/loop-agent-prompt-params.js.map +1 -1
- package/dist/agent-types/loop-agent-type.d.ts +20 -0
- package/dist/agent-types/loop-agent-type.d.ts.map +1 -1
- package/dist/agent-types/loop-agent-type.js +73 -2
- package/dist/agent-types/loop-agent-type.js.map +1 -1
- package/dist/artifact-tools/SearchResultSetToolLibrary.js +4 -4
- package/dist/artifact-tools/SearchResultSetToolLibrary.js.map +1 -1
- package/dist/base-agent.d.ts +310 -26
- package/dist/base-agent.d.ts.map +1 -1
- package/dist/base-agent.js +867 -244
- package/dist/base-agent.js.map +1 -1
- package/dist/client-tool-results.d.ts +45 -0
- package/dist/client-tool-results.d.ts.map +1 -0
- package/dist/client-tool-results.js +94 -0
- package/dist/client-tool-results.js.map +1 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -0
- package/dist/index.js.map +1 -1
- package/dist/native-tools/action-tool-builder.d.ts +14 -0
- package/dist/native-tools/action-tool-builder.d.ts.map +1 -1
- package/dist/native-tools/action-tool-builder.js +62 -7
- package/dist/native-tools/action-tool-builder.js.map +1 -1
- package/dist/native-tools/control-tools.d.ts +17 -1
- package/dist/native-tools/control-tools.d.ts.map +1 -1
- package/dist/native-tools/control-tools.js +40 -3
- package/dist/native-tools/control-tools.js.map +1 -1
- package/dist/prior-turn-tool-result-cache.d.ts +31 -12
- package/dist/prior-turn-tool-result-cache.d.ts.map +1 -1
- package/dist/prior-turn-tool-result-cache.js +59 -16
- package/dist/prior-turn-tool-result-cache.js.map +1 -1
- package/dist/realtime/bridge-host-params.d.ts +22 -0
- package/dist/realtime/bridge-host-params.d.ts.map +1 -0
- package/dist/realtime/bridge-host-params.js +53 -0
- package/dist/realtime/bridge-host-params.js.map +1 -0
- package/dist/realtime/bridge-realtime-session-factory.d.ts +63 -3
- package/dist/realtime/bridge-realtime-session-factory.d.ts.map +1 -1
- package/dist/realtime/bridge-realtime-session-factory.js +83 -22
- package/dist/realtime/bridge-realtime-session-factory.js.map +1 -1
- package/dist/realtime/bridge-realtime-usage-recorder.d.ts +81 -0
- package/dist/realtime/bridge-realtime-usage-recorder.d.ts.map +1 -0
- package/dist/realtime/bridge-realtime-usage-recorder.js +194 -0
- package/dist/realtime/bridge-realtime-usage-recorder.js.map +1 -0
- package/dist/realtime/bridge-session-transcript-sink.d.ts +37 -0
- package/dist/realtime/bridge-session-transcript-sink.d.ts.map +1 -0
- package/dist/realtime/bridge-session-transcript-sink.js +58 -0
- package/dist/realtime/bridge-session-transcript-sink.js.map +1 -0
- package/dist/realtime/client-only-channel-server.d.ts +83 -0
- package/dist/realtime/client-only-channel-server.d.ts.map +1 -0
- package/dist/realtime/client-only-channel-server.js +107 -0
- package/dist/realtime/client-only-channel-server.js.map +1 -0
- package/dist/realtime/identity-verification-channel-server.d.ts +50 -0
- package/dist/realtime/identity-verification-channel-server.d.ts.map +1 -0
- package/dist/realtime/identity-verification-channel-server.js +66 -0
- package/dist/realtime/identity-verification-channel-server.js.map +1 -0
- package/dist/realtime/interactive-component-channel-server.d.ts +39 -0
- package/dist/realtime/interactive-component-channel-server.d.ts.map +1 -0
- package/dist/realtime/interactive-component-channel-server.js +55 -0
- package/dist/realtime/interactive-component-channel-server.js.map +1 -0
- package/dist/realtime/realtime-avatar-resolution.d.ts +194 -0
- package/dist/realtime/realtime-avatar-resolution.d.ts.map +1 -0
- package/dist/realtime/realtime-avatar-resolution.js +208 -0
- package/dist/realtime/realtime-avatar-resolution.js.map +1 -0
- package/dist/realtime/realtime-channel-policy.d.ts +81 -0
- package/dist/realtime/realtime-channel-policy.d.ts.map +1 -0
- package/dist/realtime/realtime-channel-policy.js +91 -0
- package/dist/realtime/realtime-channel-policy.js.map +1 -0
- package/dist/realtime/realtime-channel-server-host.d.ts +14 -0
- package/dist/realtime/realtime-channel-server-host.d.ts.map +1 -1
- package/dist/realtime/realtime-channel-server-host.js +35 -0
- package/dist/realtime/realtime-channel-server-host.js.map +1 -1
- package/dist/realtime/realtime-client-session-service.d.ts +493 -30
- package/dist/realtime/realtime-client-session-service.d.ts.map +1 -1
- package/dist/realtime/realtime-client-session-service.js +825 -98
- package/dist/realtime/realtime-client-session-service.js.map +1 -1
- package/dist/realtime/realtime-coagent-config.d.ts +94 -15
- package/dist/realtime/realtime-coagent-config.d.ts.map +1 -1
- package/dist/realtime/realtime-coagent-config.js +109 -4
- package/dist/realtime/realtime-coagent-config.js.map +1 -1
- package/dist/realtime/realtime-coagent-resolution.d.ts +90 -0
- package/dist/realtime/realtime-coagent-resolution.d.ts.map +1 -0
- package/dist/realtime/realtime-coagent-resolution.js +186 -0
- package/dist/realtime/realtime-coagent-resolution.js.map +1 -0
- package/dist/realtime/realtime-delegation-narrator.d.ts +97 -0
- package/dist/realtime/realtime-delegation-narrator.d.ts.map +1 -0
- package/dist/realtime/realtime-delegation-narrator.js +162 -0
- package/dist/realtime/realtime-delegation-narrator.js.map +1 -0
- package/dist/realtime/realtime-recording-store.d.ts +136 -16
- package/dist/realtime/realtime-recording-store.d.ts.map +1 -1
- package/dist/realtime/realtime-recording-store.js +505 -39
- package/dist/realtime/realtime-recording-store.js.map +1 -1
- package/dist/realtime/realtime-session-runner.d.ts +2 -82
- package/dist/realtime/realtime-session-runner.d.ts.map +1 -1
- package/dist/realtime/realtime-session-runner.js +16 -191
- package/dist/realtime/realtime-session-runner.js.map +1 -1
- package/dist/realtime/realtime-vendor-resolution.d.ts +9 -0
- package/dist/realtime/realtime-vendor-resolution.d.ts.map +1 -1
- package/dist/realtime/realtime-vendor-resolution.js +16 -5
- package/dist/realtime/realtime-vendor-resolution.js.map +1 -1
- package/dist/realtime/realtime-video-output-gate.d.ts +83 -0
- package/dist/realtime/realtime-video-output-gate.d.ts.map +1 -0
- package/dist/realtime/realtime-video-output-gate.js +79 -0
- package/dist/realtime/realtime-video-output-gate.js.map +1 -0
- package/dist/rubric-evaluation-agent-runner.js +17 -8
- package/dist/rubric-evaluation-agent-runner.js.map +1 -1
- package/dist/self-check.d.ts +6 -3
- package/dist/self-check.d.ts.map +1 -1
- package/dist/self-check.js +3 -3
- package/dist/self-check.js.map +1 -1
- package/dist/tool-result-format.d.ts +14 -0
- package/dist/tool-result-format.d.ts.map +1 -1
- package/dist/tool-result-format.js.map +1 -1
- package/package.json +19 -19
|
@@ -29,19 +29,89 @@
|
|
|
29
29
|
* @module @memberjunction/ai-agents
|
|
30
30
|
* @author MemberJunction.com
|
|
31
31
|
*/
|
|
32
|
-
import { LogError, LogStatus, RunView } from '@memberjunction/core';
|
|
33
|
-
import { MJGlobal, MJLruCache, UUIDsEqual, EscapeSQLString } from '@memberjunction/global';
|
|
32
|
+
import { LogError, LogStatus, RunView, DatabaseProviderBase } from '@memberjunction/core';
|
|
33
|
+
import { MJGlobal, MJLruCache, UUIDsEqual, EscapeSQLString, NormalizeUUID } from '@memberjunction/global';
|
|
34
34
|
import { ActionEngineServer } from '@memberjunction/actions';
|
|
35
35
|
import { RunActionParams } from '@memberjunction/actions-base';
|
|
36
|
-
import { BaseRealtimeModel, GetAIAPIKey } from '@memberjunction/ai';
|
|
37
|
-
import { FormatAppContextNote, ResolvePromptRunUserID } from '@memberjunction/ai-core-plus';
|
|
36
|
+
import { BaseRealtimeModel, GetAIAPIKey, CredentialScopeAllows, IsZeroDataRetention } from '@memberjunction/ai';
|
|
37
|
+
import { FormatAppContextNote, ResolvePromptRunUserID, ClientToolMetadataFromDefinition, ParseAgentSettings, ResolveAppClientToolMetadata, ResolveClientTools } from '@memberjunction/ai-core-plus';
|
|
38
38
|
import { AIEngine } from '@memberjunction/aiengine';
|
|
39
|
+
import { RealtimeSessionCanShowAvatar, ResolveBridgedAvatarStatus, ResolveRealtimeAvatar, ResolveRealtimeAvatarStatus, WithoutUnseenAvatar, } from './realtime-avatar-resolution.js';
|
|
40
|
+
import { ChooseRealtimeVendorForAvatar, ReadRealtimeVideoOutputRow, RealtimeModelShowsAvatar } from './realtime-video-output-gate.js';
|
|
39
41
|
import { AgentMemoryContextBuilder } from '../agent-memory-context-builder.js';
|
|
40
42
|
import { AgentRunner } from '../AgentRunner.js';
|
|
43
|
+
import { AgentRunWatchdog } from '../agent-run-watchdog.js';
|
|
44
|
+
import { DelegationNarrator } from './realtime-delegation-narrator.js';
|
|
45
|
+
import { FilterAllowedAgentsByCanRun } from './realtime-coagent-resolution.js';
|
|
46
|
+
import { BridgeRealtimeUsageRecorder } from './bridge-realtime-usage-recorder.js';
|
|
41
47
|
import { RealtimeToolBroker, INVOKE_TARGET_AGENT_TOOL_NAME, INVOKE_TARGET_AGENT_DESCRIPTION, BuildRealtimeAgentFraming } from './realtime-tool-broker.js';
|
|
42
48
|
import { NARRATION_PROMPT_NAME, LEGACY_NARRATION_PROMPT_NAME, ResolveNarrationInstructionsTemplate } from './realtime-narration.js';
|
|
43
|
-
import { BuildVoiceMannerSection, BuildAppRealtimeOverridesJson, DeepMergeConfigs, GetModelCatalogSessionSettings, GetDisclosureForTarget, GetNarrationPaceMs, GetProviderVoiceSettings, GetSessionTuningSettings, ResolveEffectiveRealtimeConfig, MatchProviderVoiceSettings, GetDirectActionsConfig, IsActionAllowedForDirectInvocation } from './realtime-coagent-config.js';
|
|
44
|
-
import { SelectRealtimeVendorForModel } from './realtime-vendor-resolution.js';
|
|
49
|
+
import { BuildVoiceMannerSection, BuildAppRealtimeOverridesJson, DeepMergeConfigs, FindDeprecatedRealtimeVideoKeys, GetModelCatalogSessionSettings, GetDisclosureForTarget, GetNarrationPaceMs, GetProviderVoiceSettings, GetSessionTuningSettings, GetWatchesMeetingVideo, ResolveEffectiveRealtimeConfig, MatchProviderVoiceSettings, GetDirectActionsConfig, IsActionAllowedForDirectInvocation } from './realtime-coagent-config.js';
|
|
50
|
+
import { ListRealtimeVendorsForModel, SelectRealtimeVendorForModel } from './realtime-vendor-resolution.js';
|
|
51
|
+
import { AIEngineBase, MergeRealtimeUsageRecord } from '@memberjunction/ai-engine-base';
|
|
52
|
+
import { BuildSessionChannelPolicy } from './realtime-channel-policy.js';
|
|
53
|
+
/**
|
|
54
|
+
* How many seconds of avatar video a run may store beyond its elapsed time: usage is client-reported, so the stored
|
|
55
|
+
* output video seconds are capped at the run's elapsed time plus this grace (relays land a few seconds late).
|
|
56
|
+
*/
|
|
57
|
+
const VIDEO_SECONDS_GRACE = 30;
|
|
58
|
+
/**
|
|
59
|
+
* How long closing a bridged (server-held) session waits for its model session to close before the session's runs are
|
|
60
|
+
* finalized anyway, in milliseconds. A model session reports its last usage while it closes (a Gemini avatar session,
|
|
61
|
+
* the seconds of video it has not reported yet), and the runs are priced from what was stored when they are finalized.
|
|
62
|
+
*/
|
|
63
|
+
export const BRIDGE_SESSION_CLOSE_WAIT_MS = 5_000;
|
|
64
|
+
/**
|
|
65
|
+
* The co-agent and target pairs already named in the deprecated `realtime.video` keys line, so the line is logged once
|
|
66
|
+
* per pair per process rather than on every session.
|
|
67
|
+
*/
|
|
68
|
+
const deprecatedVideoKeysLogged = new Set();
|
|
69
|
+
/**
|
|
70
|
+
* How many output video tokens one second of avatar video counts: Gemini counts 258 tokens a frame at 24 frames a
|
|
71
|
+
* second, the rate the per-minute avatar price is derived from. The stored output video tokens are capped at the
|
|
72
|
+
* seconds cap at this rate, because pricing uses them: they leave the cost row's output bucket, and a per-token avatar
|
|
73
|
+
* price prices them. Raise it for a model that counts more.
|
|
74
|
+
*/
|
|
75
|
+
const VIDEO_TOKENS_PER_SECOND = 6192;
|
|
76
|
+
/** Runtime handles by their realtime session, so the layer that only holds the session can reach its runtime. */
|
|
77
|
+
const bridgeRuntimes = new WeakMap();
|
|
78
|
+
/**
|
|
79
|
+
* Returns the runtime wired onto a bridged realtime session by
|
|
80
|
+
* {@link RealtimeClientSessionService.WireBridgeRealtimeSession}, or `undefined` for a session that was never
|
|
81
|
+
* wired. Lets a host that only holds the {@link IRealtimeSession} (the telephony services) cancel delegations on
|
|
82
|
+
* barge-in and install its local tool handler.
|
|
83
|
+
*/
|
|
84
|
+
export function GetBridgeRealtimeRuntime(session) {
|
|
85
|
+
return bridgeRuntimes.get(session);
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Closes a bridged model session, waiting for it up to `waitMs`. Resolves when the close finishes or the wait runs out;
|
|
89
|
+
* rejects when the close fails within the wait. A close still running when the wait runs out is logged (what the
|
|
90
|
+
* session reports from then on is not stored), and so is its failure, should it fail later.
|
|
91
|
+
*
|
|
92
|
+
* @param close The session's own `Close()`.
|
|
93
|
+
* @param waitMs How long to wait for it.
|
|
94
|
+
* @param label Names the session in the log lines (its co-agent prompt run).
|
|
95
|
+
*/
|
|
96
|
+
async function closeBridgedSessionWithin(close, waitMs, label) {
|
|
97
|
+
const closing = Promise.resolve(close());
|
|
98
|
+
let timer;
|
|
99
|
+
const waitRanOut = new Promise((resolve) => {
|
|
100
|
+
timer = setTimeout(() => resolve(true), waitMs);
|
|
101
|
+
timer.unref?.();
|
|
102
|
+
});
|
|
103
|
+
try {
|
|
104
|
+
const ranOut = await Promise.race([closing.then(() => false), waitRanOut]);
|
|
105
|
+
if (ranOut) {
|
|
106
|
+
LogError(`[Bridged usage] ${label}: the model session took more than ${waitMs} ms to close; its runs are finalized ` +
|
|
107
|
+
`without waiting, so usage it reports while closing is not stored.`);
|
|
108
|
+
closing.catch((error) => LogError(`[Bridged usage] ${label}: closing the model session failed: ${error instanceof Error ? error.message : String(error)}`));
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
finally {
|
|
112
|
+
clearTimeout(timer);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
45
115
|
/**
|
|
46
116
|
* Logs when a configuration authored per-provider voice settings but NONE matched the vendor that
|
|
47
117
|
* actually ran — those settings are dropped, and pre-#3530 that happened with no trace at all: the
|
|
@@ -249,7 +319,7 @@ export class RealtimeClientSessionService {
|
|
|
249
319
|
if (!prep.Success || !prep.CoAgent || !prep.Resolution || !prep.SessionParams || !prep.EffectiveConfig) {
|
|
250
320
|
return { Success: false, ErrorMessage: prep.ErrorMessage };
|
|
251
321
|
}
|
|
252
|
-
const { CoAgent: coAgent, Resolution: resolution, SessionParams: sessionParams, EffectiveConfig: effectiveConfig } = prep;
|
|
322
|
+
const { CoAgent: coAgent, Resolution: resolution, SessionParams: sessionParams, EffectiveConfig: effectiveConfig, ClientPolicy: clientPolicy } = prep;
|
|
253
323
|
if (!resolution.Model.SupportsClientDirect) {
|
|
254
324
|
return {
|
|
255
325
|
Success: false,
|
|
@@ -264,6 +334,7 @@ export class RealtimeClientSessionService {
|
|
|
264
334
|
const message = error instanceof Error ? error.message : String(error);
|
|
265
335
|
return { Success: false, ErrorMessage: `Failed to mint client realtime session: ${message}` };
|
|
266
336
|
}
|
|
337
|
+
const avatarStatus = this.resolveMintedAvatarStatus(prep, clientConfig);
|
|
267
338
|
// Best-effort observability: create a server-side co-agent run (+ prompt run) so the voice
|
|
268
339
|
// session is visible in the agent-run timeline and delegated runs can nest under it. A
|
|
269
340
|
// failure here never fails the prepare — we just omit the ids.
|
|
@@ -281,9 +352,30 @@ export class RealtimeClientSessionService {
|
|
|
281
352
|
DriverClass: resolution.DriverClass,
|
|
282
353
|
NarrationInstructionsTemplate: this.resolveNarrationInstructionsTemplate() ?? undefined,
|
|
283
354
|
EffectiveConfig: effectiveConfig,
|
|
355
|
+
ClientPolicy: clientPolicy,
|
|
284
356
|
NarrationPaceMs: GetNarrationPaceMs(effectiveConfig) ?? undefined,
|
|
357
|
+
...(avatarStatus ? { AvatarStatus: avatarStatus } : {}),
|
|
285
358
|
};
|
|
286
359
|
}
|
|
360
|
+
/**
|
|
361
|
+
* The call's avatar status once the driver has minted the session (see {@link ResolveRealtimeAvatarStatus}), with one
|
|
362
|
+
* log line saying whether the avatar shows. `undefined`, and no line, when the session asked for no avatar.
|
|
363
|
+
*/
|
|
364
|
+
resolveMintedAvatarStatus(prep, clientConfig) {
|
|
365
|
+
const resolution = prep.Resolution;
|
|
366
|
+
const apiName = resolution?.APIName ?? clientConfig.Model;
|
|
367
|
+
const status = ResolveRealtimeAvatarStatus({
|
|
368
|
+
Resolution: prep.AvatarResolution ?? {},
|
|
369
|
+
ModelSupportsAvatarOutput: resolution
|
|
370
|
+
? RealtimeModelShowsAvatar({ ModelID: resolution.ModelID, APIName: apiName, Model: resolution.Model }, AIEngine.Instance)
|
|
371
|
+
: false,
|
|
372
|
+
DriverStatus: clientConfig.AvatarStatus,
|
|
373
|
+
});
|
|
374
|
+
if (status) {
|
|
375
|
+
console.log(`[RealtimeCoAgent] mint avatar model=${apiName} shown=${status.Granted} reason=${status.Reason ?? 'none'}`);
|
|
376
|
+
}
|
|
377
|
+
return status;
|
|
378
|
+
}
|
|
287
379
|
/**
|
|
288
380
|
* Wires a **server long-lived (bridged)** realtime session onto the SAME core machinery the
|
|
289
381
|
* client-direct path uses — so a LiveKit (or future Zoom/Teams) agent does real work and is tracked
|
|
@@ -294,12 +386,22 @@ export class RealtimeClientSessionService {
|
|
|
294
386
|
* under the co-agent run, supports barge-in cancel + paused-run resume — all of it, for free).
|
|
295
387
|
*
|
|
296
388
|
* Responsibilities, in order:
|
|
389
|
+
* 0. Report the avatar status the driver had nothing to say about: a prep that asked the driver for no avatar (a
|
|
390
|
+
* phone call, no face for the model's vendor, an unknown avatar, a Video/Output row that turns video off) leaves
|
|
391
|
+
* the driver nothing to report, and a driver that renders no avatar ignores a request, so the session's
|
|
392
|
+
* `AvatarStatus` says why instead of saying nothing ({@link ResolveBridgedAvatarStatus}).
|
|
297
393
|
* 1. Create the co-agent observability run (+ prompt run + step) so the voice session shows up in the
|
|
298
394
|
* agent-run timeline and delegated runs nest under it (best-effort; a failure just omits the ids).
|
|
299
|
-
* 2.
|
|
300
|
-
*
|
|
395
|
+
* 2. Record the session's usage on the co-agent prompt run, as client-direct calls do: `session.OnUsage` →
|
|
396
|
+
* {@link BridgeRealtimeUsageRecorder} → {@link AccumulatePromptRunUsage}, every 10 s and once more at finalize.
|
|
397
|
+
* 3. Wire `session.OnToolCall` → `ExecuteRelayedTool` → `session.SendToolResult`.
|
|
398
|
+
* 4. Guarantee finalize-once: wrap `session.Close()` and listen for an unexpected drop (`OnClose`), both
|
|
301
399
|
* routed through one idempotent finalizer. The bridge teardown calls `Close()`, so the run finalizes
|
|
302
|
-
* on graceful end; a dropped socket finalizes via `OnClose
|
|
400
|
+
* on graceful end; a dropped socket finalizes via `OnClose`, and a `Close()` after it waits for that
|
|
401
|
+
* finalize. The wrapped `Close()` closes the model session first (waiting up to
|
|
402
|
+
* {@link BRIDGE_SESSION_CLOSE_WAIT_MS}), so the usage it reports while closing is recorded; the
|
|
403
|
+
* finalizer then writes the session's last usage before it finalizes the runs, so the prompt run is
|
|
404
|
+
* priced from its final counts.
|
|
303
405
|
*
|
|
304
406
|
* @param session The live realtime session the bridge owns (from `model.StartSession`).
|
|
305
407
|
* @param input The same prep input used to build the session (carries AgentSessionID, TargetAgentID, …).
|
|
@@ -309,34 +411,126 @@ export class RealtimeClientSessionService {
|
|
|
309
411
|
* @returns A {@link BridgeRealtimeRuntime} the bridge holds for the session lifetime.
|
|
310
412
|
*/
|
|
311
413
|
async WireBridgeRealtimeSession(session, input, prep, contextUser, provider) {
|
|
414
|
+
this.reportBridgedAvatarStatus(session, input, prep);
|
|
312
415
|
const coAgent = prep.CoAgent;
|
|
313
416
|
const resolution = prep.Resolution;
|
|
314
417
|
if (!coAgent || !resolution) {
|
|
315
418
|
// Prep must have succeeded before wiring; degrade to a tool-error fallback rather than throw.
|
|
316
419
|
return this.wireBridgeFallbackRuntime(session);
|
|
317
420
|
}
|
|
421
|
+
// The delegation set the model may reach, narrowed to what THIS run-as user may run (the browser path
|
|
422
|
+
// applies the same filter; without it a bridged call would reach colleagues the caller cannot).
|
|
423
|
+
const allowedAgents = await FilterAllowedAgentsByCanRun(prep.EffectiveConfig?.realtime?.allowedAgents, contextUser);
|
|
424
|
+
// Spoken progress while delegated work runs — the same pacing/wording the generic session runner uses.
|
|
425
|
+
const narrator = new DelegationNarrator({
|
|
426
|
+
GetSession: () => session,
|
|
427
|
+
NarrationInstructionsTemplate: this.resolveNarrationInstructionsTemplate(),
|
|
428
|
+
NarrationPaceMs: GetNarrationPaceMs(prep.EffectiveConfig) ?? undefined,
|
|
429
|
+
});
|
|
430
|
+
let localToolHandler;
|
|
318
431
|
const promptID = this.resolveCoAgentSystemPrompt(coAgent).PromptID;
|
|
319
432
|
const obs = await this.createCoAgentObservabilityRun(coAgent, promptID, resolution.ModelID, resolution.VendorID, input.UserID || contextUser?.ID, input.AgentSessionID, contextUser, provider, input.ConversationID);
|
|
320
|
-
|
|
321
|
-
const finalize =
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
433
|
+
const usage = this.wireBridgeUsage(session, obs?.PromptRunID, contextUser, provider);
|
|
434
|
+
const finalize = this.createBridgeFinalizer(obs, usage, contextUser, provider);
|
|
435
|
+
this.wireBridgeToolCalls(session, {
|
|
436
|
+
Input: input,
|
|
437
|
+
Prep: prep,
|
|
438
|
+
ParentRunID: obs?.CoAgentRunID,
|
|
439
|
+
AllowedAgents: allowedAgents,
|
|
440
|
+
Narrator: narrator,
|
|
441
|
+
LocalToolHandler: () => localToolHandler,
|
|
442
|
+
ContextUser: contextUser,
|
|
443
|
+
Provider: provider,
|
|
444
|
+
});
|
|
445
|
+
this.finalizeOnBridgeClose(session, narrator, finalize, obs?.PromptRunID);
|
|
446
|
+
const runtime = {
|
|
447
|
+
CoAgentRunID: obs?.CoAgentRunID,
|
|
448
|
+
PromptRunID: obs?.PromptRunID,
|
|
449
|
+
Finalize: finalize,
|
|
450
|
+
CancelInFlightDelegations: () => {
|
|
451
|
+
narrator.Cancel(); // a stale "still working on it" line must not be spoken over the caller
|
|
452
|
+
return this.CancelInFlightDelegations(input.AgentSessionID);
|
|
453
|
+
},
|
|
454
|
+
CancelPendingNarration: () => narrator.Cancel(),
|
|
455
|
+
SetLocalToolHandler: (handler) => { localToolHandler = handler; },
|
|
456
|
+
WatchesMeetingVideo: GetWatchesMeetingVideo(prep.EffectiveConfig),
|
|
327
457
|
};
|
|
328
|
-
|
|
458
|
+
bridgeRuntimes.set(session, runtime);
|
|
459
|
+
return runtime;
|
|
460
|
+
}
|
|
461
|
+
/**
|
|
462
|
+
* Gives a bridged session the avatar status its driver had nothing to say about ({@link ResolveBridgedAvatarStatus}),
|
|
463
|
+
* with one log line: a prep that asked the driver for no avatar (a phone call, no face for the model's vendor, an
|
|
464
|
+
* unknown avatar, a Video/Output row that turns video off) leaves the driver nothing to report, and a driver that
|
|
465
|
+
* renders no avatar ignores a request and reports nothing. A host reads it like any driver's status (the LiveKit room
|
|
466
|
+
* puts it on the agent's bot, so the room shows its notice). A status the driver reported stands.
|
|
467
|
+
*/
|
|
468
|
+
reportBridgedAvatarStatus(session, input, prep) {
|
|
469
|
+
const model = prep.Resolution;
|
|
470
|
+
const status = ResolveBridgedAvatarStatus({
|
|
471
|
+
Resolution: prep.AvatarResolution,
|
|
472
|
+
RoomDelivery: input.AvatarDelivery === 'room',
|
|
473
|
+
ModelShowsAvatar: () => (model ? RealtimeModelShowsAvatar(model, AIEngine.Instance) : false),
|
|
474
|
+
DriverStatus: session.AvatarStatus,
|
|
475
|
+
});
|
|
476
|
+
if (!status || status === session.AvatarStatus) {
|
|
477
|
+
return;
|
|
478
|
+
}
|
|
479
|
+
session.AvatarStatus = status;
|
|
480
|
+
const asked = prep.AvatarResolution?.Reason ? 'the driver was asked for none' : 'the driver was asked for one and reported nothing';
|
|
481
|
+
LogStatus(`[RealtimeCoAgent] bridged session avatar: audio only (${status.Reason}); ${asked}.`);
|
|
482
|
+
}
|
|
483
|
+
/**
|
|
484
|
+
* Records a bridged session's usage on its co-agent prompt run. The recorder is the session's only `OnUsage`
|
|
485
|
+
* subscriber: drivers keep one usage handler. Without a prompt run there is nowhere to record, so nothing subscribes
|
|
486
|
+
* and one line is logged, as the client-direct relay logs a dropped delta.
|
|
487
|
+
*/
|
|
488
|
+
wireBridgeUsage(session, promptRunID, contextUser, provider) {
|
|
489
|
+
if (!promptRunID) {
|
|
490
|
+
LogStatus('WireBridgeRealtimeSession: the session has no co-agent prompt run, so its usage is not recorded.');
|
|
491
|
+
return undefined;
|
|
492
|
+
}
|
|
493
|
+
const recorder = new BridgeRealtimeUsageRecorder((write) => this.AccumulatePromptRunUsage(promptRunID, write.InputTokens, write.OutputTokens, contextUser, provider, write.Details), `prompt run ${promptRunID}`);
|
|
494
|
+
session.OnUsage((usage) => recorder.Add(usage));
|
|
495
|
+
return recorder;
|
|
496
|
+
}
|
|
497
|
+
/**
|
|
498
|
+
* The bridged session's finalizer. It closes the usage recorder first (no more updates; the unwritten usage is
|
|
499
|
+
* written), so the prompt run's cost, computed when finalize stamps `CompletedAt`, covers the whole session. On a
|
|
500
|
+
* graceful close it runs after the model session closed (see `finalizeOnBridgeClose`).
|
|
501
|
+
*
|
|
502
|
+
* It runs once, and every call returns the first call's promise: whichever path calls second (a `Close()` after a
|
|
503
|
+
* dropped connection's `OnClose` started the finalize) resolves only once the runs are finalized and priced. The
|
|
504
|
+
* first call decides the outcome (see {@link BridgeRealtimeRuntime.Finalize}).
|
|
505
|
+
*/
|
|
506
|
+
createBridgeFinalizer(obs, usage, contextUser, provider) {
|
|
507
|
+
let finalizing;
|
|
508
|
+
const finalizeRuns = async (success, errorMessage) => {
|
|
509
|
+
await usage?.Close();
|
|
510
|
+
await this.FinalizeCoAgentRun(obs?.CoAgentRunID ?? null, obs?.PromptRunID ?? null, contextUser, provider, success, obs?.CoAgentRunStepID ?? null, errorMessage);
|
|
511
|
+
};
|
|
512
|
+
return (success, errorMessage) => {
|
|
513
|
+
finalizing ??= finalizeRuns(success, errorMessage);
|
|
514
|
+
return finalizing;
|
|
515
|
+
};
|
|
516
|
+
}
|
|
517
|
+
/** Tool calls → the host's local handler or the shared delegation entry point, then the result back to the model. */
|
|
518
|
+
wireBridgeToolCalls(session, wiring) {
|
|
329
519
|
session.OnToolCall(async (call) => {
|
|
330
520
|
try {
|
|
331
|
-
const
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
521
|
+
const localHandler = wiring.LocalToolHandler();
|
|
522
|
+
const resultJson = localHandler?.Handles(call.ToolName)
|
|
523
|
+
? await localHandler.Execute(call)
|
|
524
|
+
: (await wiring.Narrator.Track(() => this.ExecuteRelayedTool({
|
|
525
|
+
AgentSessionID: wiring.Input.AgentSessionID,
|
|
526
|
+
ParentRunID: wiring.ParentRunID,
|
|
527
|
+
TargetAgentID: wiring.Input.TargetAgentID,
|
|
528
|
+
AllowedAgents: wiring.AllowedAgents,
|
|
529
|
+
DirectActions: wiring.Prep.EffectiveConfig?.realtime?.directActions,
|
|
530
|
+
OnProgress: (progress) => wiring.Narrator.HandleProgress(progress),
|
|
531
|
+
Call: call,
|
|
532
|
+
}, wiring.ContextUser, wiring.Provider))).ResultJson;
|
|
533
|
+
await session.SendToolResult(call.CallID, resultJson);
|
|
340
534
|
}
|
|
341
535
|
catch (error) {
|
|
342
536
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -344,15 +538,34 @@ export class RealtimeClientSessionService {
|
|
|
344
538
|
await session.SendToolResult(call.CallID, JSON.stringify({ success: false, error: message }));
|
|
345
539
|
}
|
|
346
540
|
});
|
|
347
|
-
|
|
348
|
-
|
|
541
|
+
}
|
|
542
|
+
/**
|
|
543
|
+
* Finalizes on graceful teardown (the bridge calls `Close()`) and on an unexpected drop. Both go through the one
|
|
544
|
+
* finalizer, which runs once: a double fire finalizes once, and a `Close()` after a drop waits for the finalize the
|
|
545
|
+
* drop started, so it resolves only once the runs are finalized. Both finalize as completed: a host whose bridge
|
|
546
|
+
* start failed calls {@link BridgeRealtimeRuntime.Finalize} with `false` and the start's error before it closes the
|
|
547
|
+
* session.
|
|
548
|
+
*
|
|
549
|
+
* `Close()` closes the model session before it finalizes: a session reports its last usage while it closes (a
|
|
550
|
+
* Gemini avatar session, the seconds of video not reported yet), and the finalizer stores only what was reported
|
|
551
|
+
* before it ran. The close is waited for up to {@link BRIDGE_SESSION_CLOSE_WAIT_MS}. The runs are finalized whether
|
|
552
|
+
* the close finished, failed or is still running; a failed close still rejects, after the finalize.
|
|
553
|
+
*
|
|
554
|
+
* @param promptRunID The session's co-agent prompt run, which names the session in a log line.
|
|
555
|
+
*/
|
|
556
|
+
finalizeOnBridgeClose(session, narrator, finalize, promptRunID) {
|
|
349
557
|
const originalClose = session.Close.bind(session);
|
|
558
|
+
const label = promptRunID ? `prompt run ${promptRunID}` : 'a session with no co-agent prompt run';
|
|
350
559
|
session.Close = async () => {
|
|
351
|
-
|
|
352
|
-
|
|
560
|
+
narrator.Cancel(); // nothing more is spoken on a session that is closing
|
|
561
|
+
try {
|
|
562
|
+
await closeBridgedSessionWithin(originalClose, BRIDGE_SESSION_CLOSE_WAIT_MS, label);
|
|
563
|
+
}
|
|
564
|
+
finally {
|
|
565
|
+
await finalize(true);
|
|
566
|
+
}
|
|
353
567
|
};
|
|
354
568
|
session.OnClose?.(() => { void finalize(true); });
|
|
355
|
-
return { CoAgentRunID: obs?.CoAgentRunID, PromptRunID: obs?.PromptRunID, Finalize: finalize };
|
|
356
569
|
}
|
|
357
570
|
/**
|
|
358
571
|
* Degenerate {@link BridgeRealtimeRuntime} for the rare case wiring is attempted without a resolved
|
|
@@ -363,7 +576,15 @@ export class RealtimeClientSessionService {
|
|
|
363
576
|
session.OnToolCall((call) => {
|
|
364
577
|
void session.SendToolResult(call.CallID, JSON.stringify({ success: false, error: 'Tool execution is unavailable — the co-agent did not resolve. Let the user know.' }));
|
|
365
578
|
});
|
|
366
|
-
|
|
579
|
+
const runtime = {
|
|
580
|
+
Finalize: async () => { },
|
|
581
|
+
CancelInFlightDelegations: () => 0,
|
|
582
|
+
CancelPendingNarration: () => { },
|
|
583
|
+
SetLocalToolHandler: () => { },
|
|
584
|
+
WatchesMeetingVideo: false,
|
|
585
|
+
};
|
|
586
|
+
bridgeRuntimes.set(session, runtime);
|
|
587
|
+
return runtime;
|
|
367
588
|
}
|
|
368
589
|
/**
|
|
369
590
|
* **The single source of truth for realtime session prep.** Builds the {@link RealtimeSessionParams}
|
|
@@ -397,13 +618,157 @@ export class RealtimeClientSessionService {
|
|
|
397
618
|
const targetAgent = this.resolveTargetAgent(input.TargetAgentID);
|
|
398
619
|
const appSettingsJson = await this.resolveAppRealtimeOverrides(input.ApplicationID, contextUser, provider);
|
|
399
620
|
const effectiveConfig = this.resolveEffectiveConfig(coAgent, input.ConfigOverridesJson, targetAgent, appSettingsJson);
|
|
621
|
+
this.logDeprecatedVideoKeys(effectiveConfig, coAgent, targetAgent);
|
|
400
622
|
const outcome = await this.resolveModelForSession(input, coAgent, effectiveConfig);
|
|
401
623
|
if (!outcome.Resolution) {
|
|
402
624
|
return { Success: false, ErrorMessage: outcome.ErrorMessage ?? this.noModelMessage() };
|
|
403
625
|
}
|
|
404
626
|
const resolution = outcome.Resolution;
|
|
405
|
-
|
|
406
|
-
|
|
627
|
+
let hostTools = input.HostTools;
|
|
628
|
+
if (input.ResolveHostTools) {
|
|
629
|
+
const dynamicHostTools = input.ResolveHostTools({
|
|
630
|
+
ModelID: resolution.ModelID,
|
|
631
|
+
ModelVendorID: resolution.ModelVendorID,
|
|
632
|
+
DriverClass: resolution.DriverClass,
|
|
633
|
+
});
|
|
634
|
+
if (dynamicHostTools !== undefined) {
|
|
635
|
+
hostTools = dynamicHostTools;
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
const effectiveInput = hostTools !== input.HostTools ? { ...input, HostTools: hostTools } : input;
|
|
639
|
+
// One answer for the whole session: it lowers channel exposure here and keeps the driver from
|
|
640
|
+
// turning on provider features that store session data (Gemini session resumption).
|
|
641
|
+
const zeroDataRetention = this.modelHasZeroDataRetention(resolution.ModelID, resolution.ModelVendorID);
|
|
642
|
+
// Channel scoping + client-tool tiers: narrows the declared tools to the scope's decision and
|
|
643
|
+
// folds the app tier into the capability manifest the prompt renders. The scoped input is what
|
|
644
|
+
// the prompt/tool builders see, so a vetoed channel is absent from the framing as well as the tools.
|
|
645
|
+
const scoped = await this.scopeSessionInput(effectiveInput, effectiveConfig, contextUser, provider, zeroDataRetention);
|
|
646
|
+
// A session nobody would see an avatar in (a phone call, an app without agent video) asks the driver for none
|
|
647
|
+
// and keeps the persona's voice.
|
|
648
|
+
const avatar = WithoutUnseenAvatar(this.ResolveSessionAvatar(scoped.Input, coAgent, effectiveConfig, resolution.ModelID, resolution.ModelVendorID), scoped.Input);
|
|
649
|
+
const sessionParams = {
|
|
650
|
+
...(await this.buildSessionParams(scoped.Input, coAgent, resolution.APIName, contextUser, provider, effectiveConfig, resolution.DriverClass, resolution.ModelID, resolution.ModelVendorID, avatar)),
|
|
651
|
+
ZeroDataRetention: zeroDataRetention,
|
|
652
|
+
};
|
|
653
|
+
return {
|
|
654
|
+
Success: true, CoAgent: coAgent, Resolution: resolution, EffectiveConfig: effectiveConfig, SessionParams: sessionParams,
|
|
655
|
+
ClientPolicy: scoped.ClientPolicy, AvatarResolution: avatar,
|
|
656
|
+
};
|
|
657
|
+
}
|
|
658
|
+
/**
|
|
659
|
+
* Applies channel scoping and the client-tool tiers to a prepare input.
|
|
660
|
+
*
|
|
661
|
+
* - **App tier into the prompt.** `Application.AgentSettings.ClientTools` is resolved to metadata
|
|
662
|
+
* and layered, through the unified {@link ResolveClientTools}, beneath the live surface's own
|
|
663
|
+
* manifest in the app-context snapshot — so the co-agent is told about tools the app declares
|
|
664
|
+
* even when the active surface did not republish them. (Only the HOST can run them; the browser
|
|
665
|
+
* reports "declared but not registered" for one it has no handler for.) The agent's static
|
|
666
|
+
* junction tools are deliberately NOT added to the voice prompt — voice prompts are
|
|
667
|
+
* token-sensitive and a tool with no browser handler is noise — but they ride the returned policy
|
|
668
|
+
* so the browser can describe them.
|
|
669
|
+
* - **Channel scoping.** When the browser reported candidates, they are scoped against the
|
|
670
|
+
* registry and the cascade's `channels` section, and the declared tools narrowed to match
|
|
671
|
+
* ({@link BuildSessionChannelPolicy}).
|
|
672
|
+
*
|
|
673
|
+
* @returns The (possibly adjusted) input, and the policy to hand back when candidates were reported.
|
|
674
|
+
*/
|
|
675
|
+
async scopeSessionInput(input, effectiveConfig, contextUser, provider, modelHasZeroDataRetention = false) {
|
|
676
|
+
const tiers = await this.resolveSessionClientToolTiers(input.ApplicationID, input.TargetAgentID, contextUser);
|
|
677
|
+
const appContext = this.withAppToolTier(input.AppContext, tiers.App, input.TargetAgentID);
|
|
678
|
+
const candidates = input.ChannelCandidates ?? [];
|
|
679
|
+
if (candidates.length === 0) {
|
|
680
|
+
return { Input: appContext === input.AppContext ? input : { ...input, AppContext: appContext } };
|
|
681
|
+
}
|
|
682
|
+
const outcome = BuildSessionChannelPolicy({
|
|
683
|
+
Candidates: candidates,
|
|
684
|
+
ChannelsConfig: effectiveConfig.realtime?.channels ?? null,
|
|
685
|
+
Registry: this.readChannelRegistry(provider),
|
|
686
|
+
ClientTools: input.ExtraTools,
|
|
687
|
+
ClientToolTiers: tiers,
|
|
688
|
+
ModelHasZeroDataRetention: modelHasZeroDataRetention,
|
|
689
|
+
});
|
|
690
|
+
return {
|
|
691
|
+
Input: { ...input, ExtraTools: outcome.ClientTools, AppContext: appContext },
|
|
692
|
+
ClientPolicy: outcome.Policy,
|
|
693
|
+
};
|
|
694
|
+
}
|
|
695
|
+
/**
|
|
696
|
+
* Whether the session model's effective catalog configuration (type < model < vendor < model-vendor)
|
|
697
|
+
* declares `Privacy.ZeroDataRetention: true`. Fails closed: a model or vendor row that cannot be
|
|
698
|
+
* resolved, or a catalog lookup that throws, reads as "not declared", so an agent that REQUIRES zero
|
|
699
|
+
* data retention loses exposure rather than gaining it. Overridable seam.
|
|
700
|
+
*
|
|
701
|
+
* @param modelID The resolved `MJ: AI Models` id.
|
|
702
|
+
* @param modelVendorID The resolved model-vendor row id (the cascade's most specific layer).
|
|
703
|
+
*/
|
|
704
|
+
modelHasZeroDataRetention(modelID, modelVendorID) {
|
|
705
|
+
if (!modelID) {
|
|
706
|
+
return false;
|
|
707
|
+
}
|
|
708
|
+
try {
|
|
709
|
+
return IsZeroDataRetention(AIEngine.Instance.GetEffectiveModelConfiguration(modelID, modelVendorID));
|
|
710
|
+
}
|
|
711
|
+
catch (error) {
|
|
712
|
+
LogError(`RealtimeClientSessionService.modelHasZeroDataRetention failed for model '${modelID}': ${error instanceof Error ? error.message : String(error)}`);
|
|
713
|
+
return false;
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
/**
|
|
717
|
+
* Layers the app's client tools beneath the surface's manifest in the app-context snapshot, through
|
|
718
|
+
* the unified resolver (`session > app`, first match by name wins). Returns the SAME snapshot object
|
|
719
|
+
* when nothing changes, so callers can detect "no change" by identity.
|
|
720
|
+
*/
|
|
721
|
+
withAppToolTier(appContext, appTools, targetAgentId) {
|
|
722
|
+
if (!appContext || !appTools || appTools.length === 0) {
|
|
723
|
+
return appContext;
|
|
724
|
+
}
|
|
725
|
+
const surfaceTools = appContext.Capabilities?.Tools;
|
|
726
|
+
const tools = ResolveClientTools({ agentId: targetAgentId, sessionTools: surfaceTools, appTools: [...appTools] });
|
|
727
|
+
if (surfaceTools && tools.length === surfaceTools.length) {
|
|
728
|
+
return appContext; // the surface already covered every app tool
|
|
729
|
+
}
|
|
730
|
+
return { ...appContext, Capabilities: { ...appContext.Capabilities, Tools: tools } };
|
|
731
|
+
}
|
|
732
|
+
/**
|
|
733
|
+
* The server's view of the channel registry (`MJ: AI Agent Channels`), read from the request
|
|
734
|
+
* provider's cached {@link AIEngineBase} rows (already configured by the prepare). Overridable seam.
|
|
735
|
+
* Tolerant: an unloaded cache yields an empty registry (every candidate then reads as having no
|
|
736
|
+
* row), never a throw.
|
|
737
|
+
*/
|
|
738
|
+
readChannelRegistry(provider) {
|
|
739
|
+
try {
|
|
740
|
+
const engine = AIEngineBase.GetProviderInstance(provider, AIEngineBase);
|
|
741
|
+
return (engine.AgentChannels ?? []).map(c => ({ Name: c.Name, IsActive: c.IsActive }));
|
|
742
|
+
}
|
|
743
|
+
catch (error) {
|
|
744
|
+
LogError(`RealtimeClientSessionService.readChannelRegistry failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
745
|
+
return [];
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
/**
|
|
749
|
+
* Resolves the client-tool tiers the browser is told about at mint: the APP tier
|
|
750
|
+
* (`Application.AgentSettings.ClientTools` resolved against the tool-definition catalog) and the
|
|
751
|
+
* STATIC tier (the target agent's `MJ: AI Agent Client Tools` junction). Overridable seam; tolerant —
|
|
752
|
+
* any failure yields empty tiers and never fails the mint.
|
|
753
|
+
*/
|
|
754
|
+
async resolveSessionClientToolTiers(applicationId, targetAgentId, contextUser) {
|
|
755
|
+
const tiers = {};
|
|
756
|
+
try {
|
|
757
|
+
const settings = await this.loadAppAgentSettings(applicationId, contextUser);
|
|
758
|
+
const definitions = AIEngine.Instance.ClientToolDefinitions ?? [];
|
|
759
|
+
const app = ResolveAppClientToolMetadata(settings?.ClientTools, definitions, ref => LogStatus(`RealtimeClientSessionService: the app's client tool '${ref.Name ?? ref.ClientToolDefinitionID}' matches no tool definition — skipping it.`));
|
|
760
|
+
if (app.length > 0) {
|
|
761
|
+
tiers.App = app;
|
|
762
|
+
}
|
|
763
|
+
const stat = targetAgentId ? AIEngine.Instance.GetClientToolsForAgent(targetAgentId).map(ClientToolMetadataFromDefinition) : [];
|
|
764
|
+
if (stat.length > 0) {
|
|
765
|
+
tiers.Static = stat;
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
catch (error) {
|
|
769
|
+
LogError(`RealtimeClientSessionService.resolveSessionClientToolTiers failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
770
|
+
}
|
|
771
|
+
return tiers;
|
|
407
772
|
}
|
|
408
773
|
/**
|
|
409
774
|
* Resolves the EFFECTIVE realtime configuration via the surface-agnostic precedence cascade:
|
|
@@ -422,6 +787,27 @@ export class RealtimeClientSessionService {
|
|
|
422
787
|
resolveEffectiveConfig(coAgent, overridesJson, targetAgent, appSettingsJson) {
|
|
423
788
|
return ResolveEffectiveRealtimeConfig(this.getAgentTypeDefaultConfiguration(coAgent), coAgent.TypeConfiguration ?? null, overridesJson ?? null, targetAgent?.TypeConfiguration ?? null, appSettingsJson ?? null);
|
|
424
789
|
}
|
|
790
|
+
/**
|
|
791
|
+
* Logs one line, once per co-agent and target pair per process, when the effective configuration sets
|
|
792
|
+
* `realtime.video.provider` or `realtime.video.providers`. Both are deprecated and nothing reads them, so an author
|
|
793
|
+
* who set them would otherwise get nothing and no word of it.
|
|
794
|
+
*
|
|
795
|
+
* @param config The effective configuration.
|
|
796
|
+
* @param coAgent The co-agent.
|
|
797
|
+
* @param targetAgent The voiced agent, when there is one.
|
|
798
|
+
*/
|
|
799
|
+
logDeprecatedVideoKeys(config, coAgent, targetAgent) {
|
|
800
|
+
const keys = FindDeprecatedRealtimeVideoKeys(config);
|
|
801
|
+
const pair = `${NormalizeUUID(coAgent.ID)}|${NormalizeUUID(targetAgent?.ID)}`;
|
|
802
|
+
if (keys.length === 0 || deprecatedVideoKeysLogged.has(pair)) {
|
|
803
|
+
return;
|
|
804
|
+
}
|
|
805
|
+
deprecatedVideoKeysLogged.add(pair);
|
|
806
|
+
const voiced = targetAgent && !UUIDsEqual(targetAgent.ID, coAgent.ID) ? `'${targetAgent.Name}' (co-agent '${coAgent.Name}')` : `'${coAgent.Name}'`;
|
|
807
|
+
const what = keys.length === 1 ? `${keys[0]}, which is deprecated and does nothing` : `${keys.join(' and ')}, which are deprecated and do nothing`;
|
|
808
|
+
LogStatus(`[RealtimeCoAgent] The realtime configuration for ${voiced} sets ${what}. Choose the model with ` +
|
|
809
|
+
"realtime.modelPreference, and set an avatar's settings on its persona's Video binding (VendorSettings.Avatar).");
|
|
810
|
+
}
|
|
425
811
|
/**
|
|
426
812
|
* Resolves the APP cascade layer JSON from `Application.AgentSettings`: the `Realtime` overrides
|
|
427
813
|
* (persona/disclosure/model) plus `RelevantAgents` mapped to the union-accumulated allowed-agent
|
|
@@ -438,6 +824,30 @@ export class RealtimeClientSessionService {
|
|
|
438
824
|
async resolveAppRealtimeOverrides(applicationId, contextUser,
|
|
439
825
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
440
826
|
provider) {
|
|
827
|
+
const settings = await this.loadAppAgentSettings(applicationId, contextUser);
|
|
828
|
+
if (!settings) {
|
|
829
|
+
return null;
|
|
830
|
+
}
|
|
831
|
+
const relevantAgents = (settings.RelevantAgents ?? []).map(r => {
|
|
832
|
+
const entry = { agentId: r.AgentID };
|
|
833
|
+
if (r.Label) {
|
|
834
|
+
entry.label = r.Label;
|
|
835
|
+
}
|
|
836
|
+
if (r.Disclosure === 'silent' || r.Disclosure === 'mention' || r.Disclosure === 'hand-voice') {
|
|
837
|
+
entry.disclosure = r.Disclosure;
|
|
838
|
+
}
|
|
839
|
+
return entry;
|
|
840
|
+
});
|
|
841
|
+
return BuildAppRealtimeOverridesJson(settings.Realtime ?? null, relevantAgents);
|
|
842
|
+
}
|
|
843
|
+
/**
|
|
844
|
+
* Reads and parses an application's `AgentSettings` (one tiny by-ID read, served from the provider's
|
|
845
|
+
* RunView cache on repeat). The RAW column is parsed with {@link ParseAgentSettings} rather than read
|
|
846
|
+
* through the CodeGen-generated `AgentSettingsObject` accessor, so a field added to the settings
|
|
847
|
+
* interface works the moment the code reading it ships, before CodeGen next refreshes the copy.
|
|
848
|
+
* Tolerant — a blank id, a failed read, a missing app, or an app with no usable settings returns `null`.
|
|
849
|
+
*/
|
|
850
|
+
async loadAppAgentSettings(applicationId, contextUser) {
|
|
441
851
|
const appId = applicationId?.trim();
|
|
442
852
|
if (!appId) {
|
|
443
853
|
return null;
|
|
@@ -453,24 +863,10 @@ export class RealtimeClientSessionService {
|
|
|
453
863
|
if (!result.Success || !result.Results || result.Results.length === 0) {
|
|
454
864
|
return null;
|
|
455
865
|
}
|
|
456
|
-
|
|
457
|
-
if (!settings) {
|
|
458
|
-
return null;
|
|
459
|
-
}
|
|
460
|
-
const relevantAgents = (settings.RelevantAgents ?? []).map(r => {
|
|
461
|
-
const entry = { agentId: r.AgentID };
|
|
462
|
-
if (r.Label) {
|
|
463
|
-
entry.label = r.Label;
|
|
464
|
-
}
|
|
465
|
-
if (r.Disclosure === 'silent' || r.Disclosure === 'mention' || r.Disclosure === 'hand-voice') {
|
|
466
|
-
entry.disclosure = r.Disclosure;
|
|
467
|
-
}
|
|
468
|
-
return entry;
|
|
469
|
-
});
|
|
470
|
-
return BuildAppRealtimeOverridesJson(settings.Realtime ?? null, relevantAgents);
|
|
866
|
+
return ParseAgentSettings(result.Results[0].AgentSettings);
|
|
471
867
|
}
|
|
472
868
|
catch (error) {
|
|
473
|
-
LogError(`RealtimeClientSessionService.
|
|
869
|
+
LogError(`RealtimeClientSessionService.loadAppAgentSettings failed for app '${appId}': ${error instanceof Error ? error.message : String(error)}`);
|
|
474
870
|
return null;
|
|
475
871
|
}
|
|
476
872
|
}
|
|
@@ -544,11 +940,30 @@ export class RealtimeClientSessionService {
|
|
|
544
940
|
run.UserID = userID;
|
|
545
941
|
}
|
|
546
942
|
if (await run.Save()) {
|
|
943
|
+
this.KeepCoAgentRunAlive(run.ID, provider, contextUser);
|
|
547
944
|
return run.ID;
|
|
548
945
|
}
|
|
549
946
|
LogError(`RealtimeClientSessionService.createCoAgentRun save failed: ${run.LatestResult?.CompleteMessage ?? 'unknown error'}`);
|
|
550
947
|
return null;
|
|
551
948
|
}
|
|
949
|
+
/**
|
|
950
|
+
* Keeps a voice session's co-agent run alive as far as the {@link AgentRunWatchdog} is concerned. The
|
|
951
|
+
* run spans the whole call, but no agent loop owns it, so nothing stamped its heartbeat: the watchdog
|
|
952
|
+
* force-failed every call that ran past ~5 minutes ("no liveness heartbeat … owning process presumed
|
|
953
|
+
* dead") while the call carried on. Called when the run is created and again on each persisted
|
|
954
|
+
* session heartbeat (`SessionManager`), so whichever server instance the session is talking to keeps
|
|
955
|
+
* it fresh; the watchdog drops it once it is finalized. Only a database provider can stamp heartbeats —
|
|
956
|
+
* any other provider is a no-op, as for every agent run.
|
|
957
|
+
*
|
|
958
|
+
* @param coAgentRunID The session's co-agent run id (from its `Config`), or nothing.
|
|
959
|
+
* @param provider The request-scoped metadata provider.
|
|
960
|
+
* @param contextUser The user the heartbeat writes run as.
|
|
961
|
+
*/
|
|
962
|
+
KeepCoAgentRunAlive(coAgentRunID, provider, contextUser) {
|
|
963
|
+
if (coAgentRunID && provider instanceof DatabaseProviderBase) {
|
|
964
|
+
AgentRunWatchdog.Instance.Track(coAgentRunID, provider, contextUser);
|
|
965
|
+
}
|
|
966
|
+
}
|
|
552
967
|
/**
|
|
553
968
|
* Creates the co-agent `AIPromptRun` row (Status `Running`) linked to the co-agent run via
|
|
554
969
|
* `AgentRunID` AND to the co-agent itself via `AgentID` — so the run shows up both on the
|
|
@@ -626,11 +1041,59 @@ export class RealtimeClientSessionService {
|
|
|
626
1041
|
* @param provider The request-scoped metadata provider.
|
|
627
1042
|
* @param success Whether the session ended successfully (controls Completed vs Failed).
|
|
628
1043
|
* @param coAgentRunStepID The co-agent run's single `MJ: AI Agent Run Steps` row id, or `null` to skip.
|
|
1044
|
+
* @param errorMessage Why the session failed (for example the bridge start's error), stamped as `ErrorMessage` on each
|
|
1045
|
+
* record it fails. Ignored when `success` is true. Without it the step gets a generic line and the runs none.
|
|
629
1046
|
*/
|
|
630
|
-
async FinalizeCoAgentRun(coAgentRunID, promptRunID, contextUser, provider, success = true, coAgentRunStepID = null) {
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
1047
|
+
async FinalizeCoAgentRun(coAgentRunID, promptRunID, contextUser, provider, success = true, coAgentRunStepID = null, errorMessage) {
|
|
1048
|
+
const failure = success ? undefined : errorMessage?.trim() || undefined;
|
|
1049
|
+
try {
|
|
1050
|
+
await this.finalizeAgentRun(coAgentRunID, contextUser, provider, success, failure);
|
|
1051
|
+
await this.finalizePromptRun(promptRunID, contextUser, provider, success, failure);
|
|
1052
|
+
await this.finalizeRunStep(coAgentRunStepID, contextUser, provider, success, failure);
|
|
1053
|
+
}
|
|
1054
|
+
finally {
|
|
1055
|
+
// Even when a finalize step throws, the run still owes its cost, and the watchdog must stop
|
|
1056
|
+
// treating it as alive — or a run stuck at Running would be kept fresh indefinitely.
|
|
1057
|
+
await this.rollUpCoAgentRunUsage(coAgentRunID, promptRunID, contextUser, provider);
|
|
1058
|
+
if (coAgentRunID) {
|
|
1059
|
+
AgentRunWatchdog.Instance.Untrack(coAgentRunID);
|
|
1060
|
+
}
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1063
|
+
/**
|
|
1064
|
+
* Copies the co-agent prompt run's tokens and cost onto the co-agent run. The realtime model's usage
|
|
1065
|
+
* accumulates on the prompt run ({@link AccumulatePromptRunUsage}), which prices itself; the run's
|
|
1066
|
+
* own `TotalCost` / `Total*TokensUsed` stayed 0, so everything that sums agent runs — the realtime
|
|
1067
|
+
* analytics dashboard's per-session cost among them — left out the voice model entirely and showed
|
|
1068
|
+
* only the delegated runs. Mirrors how an agent loop derives its run totals from its prompt runs.
|
|
1069
|
+
*
|
|
1070
|
+
* Applied whatever the run's status: a run the watchdog already failed, or one a shutdown cancelled,
|
|
1071
|
+
* still owes its cost. Runs after {@link finalizePromptRun}, which waits for in-flight usage writes,
|
|
1072
|
+
* so the copy sees the final counts. Tolerant: logs, never throws.
|
|
1073
|
+
*/
|
|
1074
|
+
async rollUpCoAgentRunUsage(coAgentRunID, promptRunID, contextUser, provider) {
|
|
1075
|
+
if (!coAgentRunID || !promptRunID) {
|
|
1076
|
+
return;
|
|
1077
|
+
}
|
|
1078
|
+
try {
|
|
1079
|
+
const promptRun = await provider.GetEntityObject('MJ: AI Prompt Runs', contextUser);
|
|
1080
|
+
const run = await provider.GetEntityObject('MJ: AI Agent Runs', contextUser);
|
|
1081
|
+
if (!(await promptRun.Load(promptRunID)) || !(await run.Load(coAgentRunID))) {
|
|
1082
|
+
return;
|
|
1083
|
+
}
|
|
1084
|
+
const promptTokens = promptRun.TokensPrompt ?? 0;
|
|
1085
|
+
const completionTokens = promptRun.TokensCompletion ?? 0;
|
|
1086
|
+
run.TotalPromptTokensUsed = promptTokens;
|
|
1087
|
+
run.TotalCompletionTokensUsed = completionTokens;
|
|
1088
|
+
run.TotalTokensUsed = promptRun.TokensUsed ?? promptTokens + completionTokens;
|
|
1089
|
+
run.TotalCost = promptRun.TotalCost ?? promptRun.Cost ?? 0;
|
|
1090
|
+
if (run.Dirty && !(await run.Save())) {
|
|
1091
|
+
LogError(`RealtimeClientSessionService.rollUpCoAgentRunUsage save failed: ${run.LatestResult?.CompleteMessage ?? 'unknown error'}`);
|
|
1092
|
+
}
|
|
1093
|
+
}
|
|
1094
|
+
catch (error) {
|
|
1095
|
+
LogError(`RealtimeClientSessionService.rollUpCoAgentRunUsage failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1096
|
+
}
|
|
634
1097
|
}
|
|
635
1098
|
/**
|
|
636
1099
|
* Finalizes the **co-agent observability run(s)** for an agent session that were left `Running` because
|
|
@@ -717,8 +1180,11 @@ export class RealtimeClientSessionService {
|
|
|
717
1180
|
}
|
|
718
1181
|
return { PromptRunID: promptRuns.Results[0]?.ID ?? null, StepID: stepID };
|
|
719
1182
|
}
|
|
720
|
-
/**
|
|
721
|
-
|
|
1183
|
+
/**
|
|
1184
|
+
* Loads + finalizes the co-agent `AIAgentRun` if still `Running`, with `failure` as its `ErrorMessage` when it fails.
|
|
1185
|
+
* Tolerant: logs, never throws.
|
|
1186
|
+
*/
|
|
1187
|
+
async finalizeAgentRun(coAgentRunID, contextUser, provider, success, failure) {
|
|
722
1188
|
if (!coAgentRunID) {
|
|
723
1189
|
return;
|
|
724
1190
|
}
|
|
@@ -729,16 +1195,20 @@ export class RealtimeClientSessionService {
|
|
|
729
1195
|
run.Status = success ? 'Completed' : 'Failed';
|
|
730
1196
|
run.CompletedAt = new Date();
|
|
731
1197
|
run.Success = success;
|
|
1198
|
+
if (failure) {
|
|
1199
|
+
run.ErrorMessage = failure;
|
|
1200
|
+
}
|
|
732
1201
|
if (!(await run.Save())) {
|
|
733
1202
|
LogError(`RealtimeClientSessionService.finalizeAgentRun save failed: ${run.LatestResult?.CompleteMessage ?? 'unknown error'}`);
|
|
734
1203
|
}
|
|
735
1204
|
}
|
|
736
1205
|
/**
|
|
737
1206
|
* Loads + finalizes the co-agent run's single system-prompt `MJ: AI Agent Run Steps` row if
|
|
738
|
-
* still `Running` (Status `Completed`/`Failed`, `CompletedAt`, `Success`
|
|
739
|
-
* missing/already-finalized step is a no-op; a
|
|
1207
|
+
* still `Running` (Status `Completed`/`Failed`, `CompletedAt`, `Success`; a failed step's `ErrorMessage` is
|
|
1208
|
+
* `failure`, or a generic line without one). Tolerant: a missing/already-finalized step is a no-op; a
|
|
1209
|
+
* load/save failure is logged, never thrown.
|
|
740
1210
|
*/
|
|
741
|
-
async finalizeRunStep(coAgentRunStepID, contextUser, provider, success) {
|
|
1211
|
+
async finalizeRunStep(coAgentRunStepID, contextUser, provider, success, failure) {
|
|
742
1212
|
if (!coAgentRunStepID) {
|
|
743
1213
|
return;
|
|
744
1214
|
}
|
|
@@ -751,7 +1221,7 @@ export class RealtimeClientSessionService {
|
|
|
751
1221
|
step.CompletedAt = new Date();
|
|
752
1222
|
step.Success = success;
|
|
753
1223
|
if (!success) {
|
|
754
|
-
step.ErrorMessage = 'The realtime session ended in an error state.';
|
|
1224
|
+
step.ErrorMessage = failure ?? 'The realtime session ended in an error state.';
|
|
755
1225
|
}
|
|
756
1226
|
if (!(await step.Save())) {
|
|
757
1227
|
LogError(`RealtimeClientSessionService.finalizeRunStep save failed: ${step.LatestResult?.CompleteMessage ?? 'unknown error'}`);
|
|
@@ -761,8 +1231,11 @@ export class RealtimeClientSessionService {
|
|
|
761
1231
|
LogError(`RealtimeClientSessionService.finalizeRunStep failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
762
1232
|
}
|
|
763
1233
|
}
|
|
764
|
-
/**
|
|
765
|
-
|
|
1234
|
+
/**
|
|
1235
|
+
* Loads + finalizes the co-agent `AIPromptRun` if still `Running`, with `failure` as its `ErrorMessage` when it fails.
|
|
1236
|
+
* Tolerant: logs, never throws.
|
|
1237
|
+
*/
|
|
1238
|
+
async finalizePromptRun(promptRunID, contextUser, provider, success, failure) {
|
|
766
1239
|
if (!promptRunID) {
|
|
767
1240
|
return;
|
|
768
1241
|
}
|
|
@@ -776,6 +1249,9 @@ export class RealtimeClientSessionService {
|
|
|
776
1249
|
run.Status = success ? 'Completed' : 'Failed';
|
|
777
1250
|
run.CompletedAt = new Date();
|
|
778
1251
|
run.Success = success;
|
|
1252
|
+
if (failure) {
|
|
1253
|
+
run.ErrorMessage = failure;
|
|
1254
|
+
}
|
|
779
1255
|
if (!(await run.Save())) {
|
|
780
1256
|
LogError(`RealtimeClientSessionService.finalizePromptRun save failed: ${run.LatestResult?.CompleteMessage ?? 'unknown error'}`);
|
|
781
1257
|
}
|
|
@@ -839,12 +1315,19 @@ export class RealtimeClientSessionService {
|
|
|
839
1315
|
* high-frequency usage checkpoint never overwrites freshly-appended transcript turns (and vice-versa).
|
|
840
1316
|
* Best-effort: load/save failures log and return `false`, never throw.
|
|
841
1317
|
*
|
|
1318
|
+
* The per-modality details add into the usage record the run keeps in `ModelSpecificResponseDetails`
|
|
1319
|
+
* (`RealtimeUsage.Input` / `.Output`, which pricing reads at finalize); every other key there is kept. The
|
|
1320
|
+
* stored output video seconds never exceed the run's elapsed time plus 30 seconds, and the stored output video
|
|
1321
|
+
* tokens never exceed 6,192 a second of that: usage is client-reported, and avatar video carries a price, per
|
|
1322
|
+
* minute or per token.
|
|
1323
|
+
*
|
|
842
1324
|
* @param promptRunID The co-agent observability prompt run.
|
|
843
1325
|
* @param inputDelta Input-token delta to add (caller clamps to >= 0).
|
|
844
1326
|
* @param outputDelta Output-token delta to add (caller clamps to >= 0).
|
|
1327
|
+
* @param details Per-modality usage to add (input and output blocks, avatar video seconds included).
|
|
845
1328
|
* @returns `true` when the accumulated usage was persisted.
|
|
846
1329
|
*/
|
|
847
|
-
async AccumulatePromptRunUsage(promptRunID, inputDelta, outputDelta, contextUser, provider) {
|
|
1330
|
+
async AccumulatePromptRunUsage(promptRunID, inputDelta, outputDelta, contextUser, provider, details) {
|
|
848
1331
|
return this.serializePromptRunWrite(promptRunID, async () => {
|
|
849
1332
|
try {
|
|
850
1333
|
const promptRun = await provider.GetEntityObject('MJ: AI Prompt Runs', contextUser);
|
|
@@ -855,6 +1338,9 @@ export class RealtimeClientSessionService {
|
|
|
855
1338
|
promptRun.TokensPrompt = (promptRun.TokensPrompt ?? 0) + inputDelta;
|
|
856
1339
|
promptRun.TokensCompletion = (promptRun.TokensCompletion ?? 0) + outputDelta;
|
|
857
1340
|
promptRun.TokensUsed = (promptRun.TokensPrompt ?? 0) + (promptRun.TokensCompletion ?? 0);
|
|
1341
|
+
if (details) {
|
|
1342
|
+
this.mergeUsageDetails(promptRun, details);
|
|
1343
|
+
}
|
|
858
1344
|
if (!(await promptRun.Save())) {
|
|
859
1345
|
LogError(`AccumulatePromptRunUsage: prompt run ${promptRunID} save failed: ${promptRun.LatestResult?.CompleteMessage ?? 'unknown error'}`);
|
|
860
1346
|
return false;
|
|
@@ -867,6 +1353,37 @@ export class RealtimeClientSessionService {
|
|
|
867
1353
|
}
|
|
868
1354
|
});
|
|
869
1355
|
}
|
|
1356
|
+
/**
|
|
1357
|
+
* Adds relayed per-modality usage into the run's usage record, with the output video seconds capped at the run's
|
|
1358
|
+
* elapsed time plus {@link VIDEO_SECONDS_GRACE}, and the output video tokens at that many seconds of
|
|
1359
|
+
* {@link VIDEO_TOKENS_PER_SECOND}. Details that are not a JSON object are left alone (logged).
|
|
1360
|
+
*/
|
|
1361
|
+
mergeUsageDetails(promptRun, details) {
|
|
1362
|
+
const runAt = promptRun.RunAt ? new Date(promptRun.RunAt).getTime() : Number.NaN;
|
|
1363
|
+
const elapsedSeconds = Math.max(0, (Date.now() - runAt) / 1000);
|
|
1364
|
+
const maxVideoSeconds = Number.isFinite(elapsedSeconds) ? elapsedSeconds + VIDEO_SECONDS_GRACE : undefined;
|
|
1365
|
+
const merged = MergeRealtimeUsageRecord(promptRun.ModelSpecificResponseDetails, details, {
|
|
1366
|
+
MaxOutputVideoSeconds: maxVideoSeconds,
|
|
1367
|
+
MaxOutputVideoTokens: maxVideoSeconds === undefined ? undefined : maxVideoSeconds * VIDEO_TOKENS_PER_SECOND,
|
|
1368
|
+
});
|
|
1369
|
+
if (!merged) {
|
|
1370
|
+
LogError(`AccumulatePromptRunUsage: prompt run ${promptRun.ID} has details that are not a JSON object — usage details dropped.`);
|
|
1371
|
+
return;
|
|
1372
|
+
}
|
|
1373
|
+
this.logClampedVideo(promptRun.ID, merged);
|
|
1374
|
+
promptRun.ModelSpecificResponseDetails = merged.Details;
|
|
1375
|
+
}
|
|
1376
|
+
/** One log line for each output video quantity the merge capped. */
|
|
1377
|
+
logClampedVideo(promptRunID, merged) {
|
|
1378
|
+
if (merged.ClampedVideoSeconds > 0) {
|
|
1379
|
+
LogStatus(`AccumulatePromptRunUsage: prompt run ${promptRunID} reported ${merged.ClampedVideoSeconds} s more avatar video than ` +
|
|
1380
|
+
`the run's elapsed time plus ${VIDEO_SECONDS_GRACE} s allows; the stored seconds are capped.`);
|
|
1381
|
+
}
|
|
1382
|
+
if (merged.ClampedVideoTokens > 0) {
|
|
1383
|
+
LogStatus(`AccumulatePromptRunUsage: prompt run ${promptRunID} reported ${merged.ClampedVideoTokens} more avatar video tokens than ` +
|
|
1384
|
+
`${VIDEO_TOKENS_PER_SECOND} a second of the run's elapsed time plus ${VIDEO_SECONDS_GRACE} s allows; the stored tokens are capped.`);
|
|
1385
|
+
}
|
|
1386
|
+
}
|
|
870
1387
|
/** Parses the prompt run's `Messages` JSON into a mutable chat-message array (tolerant: `[]` on empty/malformed). */
|
|
871
1388
|
parsePromptRunMessages(raw) {
|
|
872
1389
|
if (!raw || !raw.trim()) {
|
|
@@ -1030,12 +1547,19 @@ export class RealtimeClientSessionService {
|
|
|
1030
1547
|
* {@link resolveConfiguredModelPreference}. METADATA preferences degrade gracefully — an
|
|
1031
1548
|
* unsatisfiable preference logs and FALLS THROUGH to the default (mirroring the co-agent
|
|
1032
1549
|
* resolution chain's tolerant metadata steps), it never breaks calls.
|
|
1033
|
-
* - Without either: the
|
|
1034
|
-
* (
|
|
1550
|
+
* - Without either: the default walk ({@link resolveDefaultRealtimeModel}): for a co-agent with video on, in a session
|
|
1551
|
+
* that could show an avatar (a browser session, or a server-side one with {@link PrepareClientSessionInput.AvatarDelivery}
|
|
1552
|
+
* `'room'`), the first candidate that shows one ({@link ResolveAvatarRealtimeModel}); else the existing default via
|
|
1553
|
+
* {@link resolveRealtimeModel} (highest-PowerRank active Realtime model), with the generic
|
|
1554
|
+
* {@link noModelMessage} on failure. The two explicit choices above win over the video preference.
|
|
1555
|
+
*
|
|
1556
|
+
* On every path the chosen model's vendor follows whether the session will show an avatar (the video setting is on
|
|
1557
|
+
* and the session could show one): among the model's keyed vendors, one that renders avatars (Vertex AI for Gemini
|
|
1558
|
+
* 3.8 Live) only when it will, else one that renders none; see {@link resolveVendorAndInstantiate}.
|
|
1035
1559
|
*
|
|
1036
1560
|
* @param input The prepare-session input (carries the optional preferred model id).
|
|
1037
1561
|
* @param coAgent The resolved co-agent (threaded to the default-resolution seam).
|
|
1038
|
-
* @param effectiveConfig The resolved effective configuration (carries `modelPreference`).
|
|
1562
|
+
* @param effectiveConfig The resolved effective configuration (carries `modelPreference` and the video setting).
|
|
1039
1563
|
* @returns The resolution outcome (resolution or failure reason).
|
|
1040
1564
|
*/
|
|
1041
1565
|
async resolveModelForSession(input, coAgent, effectiveConfig) {
|
|
@@ -1046,16 +1570,62 @@ export class RealtimeClientSessionService {
|
|
|
1046
1570
|
// A resolver that fell back to the environment itself would answer before the seam, so a
|
|
1047
1571
|
// subclass that overrides the seam would lose to AI_VENDOR_API_KEY__<driver>.
|
|
1048
1572
|
const resolveRunKey = this.buildRunKeyResolver(input.APIKeys);
|
|
1573
|
+
const scope = input.CredentialScope ?? 'Any';
|
|
1574
|
+
const preferAvatar = effectiveConfig?.realtime?.video?.enabled === true && RealtimeSessionCanShowAvatar(input);
|
|
1049
1575
|
if (input.PreferredModelID) {
|
|
1050
|
-
return this.resolvePreferredRealtimeModel(input.PreferredModelID, resolveRunKey);
|
|
1576
|
+
return this.resolvePreferredRealtimeModel(input.PreferredModelID, resolveRunKey, scope, preferAvatar);
|
|
1051
1577
|
}
|
|
1052
|
-
const fromConfig = this.resolveConfiguredModelPreference(effectiveConfig, resolveRunKey);
|
|
1578
|
+
const fromConfig = this.resolveConfiguredModelPreference(effectiveConfig, resolveRunKey, scope, preferAvatar);
|
|
1053
1579
|
if (fromConfig) {
|
|
1054
1580
|
return { Resolution: fromConfig };
|
|
1055
1581
|
}
|
|
1056
|
-
const resolution = await this.
|
|
1582
|
+
const resolution = await this.resolveDefaultRealtimeModel(coAgent, preferAvatar, resolveRunKey, scope);
|
|
1057
1583
|
return resolution ? { Resolution: resolution } : { ErrorMessage: this.noModelMessage() };
|
|
1058
1584
|
}
|
|
1585
|
+
/**
|
|
1586
|
+
* The default walk. With `preferAvatar` (the co-agent's video setting is on and the session could show an avatar),
|
|
1587
|
+
* it takes the first candidate that shows one ({@link ResolveAvatarRealtimeModel}). When none does, or without it, it
|
|
1588
|
+
* takes {@link resolveRealtimeModel}'s choice; when one was preferred, one log line says the call stays audio only.
|
|
1589
|
+
*/
|
|
1590
|
+
async resolveDefaultRealtimeModel(coAgent, preferAvatar, resolve, credentialScope) {
|
|
1591
|
+
const withAvatar = preferAvatar ? this.ResolveAvatarRealtimeModel(coAgent, resolve, credentialScope) : null;
|
|
1592
|
+
if (withAvatar) {
|
|
1593
|
+
return withAvatar;
|
|
1594
|
+
}
|
|
1595
|
+
const resolution = await this.resolveRealtimeModel(coAgent, resolve, credentialScope, preferAvatar);
|
|
1596
|
+
if (preferAvatar && resolution) {
|
|
1597
|
+
LogStatus('[RealtimeCoAgent] Video is on, but no realtime model with a usable key shows an avatar (its Video/Output row ' +
|
|
1598
|
+
`and its endpoint): using '${resolution.ModelName ?? resolution.APIName}' on ${resolution.DriverClass ?? 'its driver'}, audio only.`);
|
|
1599
|
+
}
|
|
1600
|
+
return resolution;
|
|
1601
|
+
}
|
|
1602
|
+
/**
|
|
1603
|
+
* For a co-agent with video on: the first candidate that shows an avatar and supports client-direct sessions, in the
|
|
1604
|
+
* default walk's order (highest PowerRank first; a model's Active vendor rows by Priority, each with a usable key).
|
|
1605
|
+
* A candidate shows an avatar when its model's Video/Output row allows video or it has none, and its driver renders
|
|
1606
|
+
* avatars for the vendor's API name on its endpoint ({@link RealtimeModelShowsAvatar}). Every keyed vendor of a model
|
|
1607
|
+
* is tried, not only the first. **Overridable seam.**
|
|
1608
|
+
*
|
|
1609
|
+
* @param coAgent The co-agent (reserved for future per-agent model preference).
|
|
1610
|
+
* @param resolve The session's run-scoped key resolver (run keys only; see {@link resolveVendorAndInstantiate}).
|
|
1611
|
+
* @param credentialScope The session's credential scope; see {@link resolveVendorAndInstantiate}.
|
|
1612
|
+
* @returns The resolution, or `null` when no candidate shows an avatar.
|
|
1613
|
+
*/
|
|
1614
|
+
ResolveAvatarRealtimeModel(coAgent, resolve, credentialScope = 'Any') {
|
|
1615
|
+
const resolveKey = this.buildKeyChain(resolve, credentialScope);
|
|
1616
|
+
for (const model of this.selectRealtimeModelCandidates(coAgent)) {
|
|
1617
|
+
if (ReadRealtimeVideoOutputRow(model.ID, AIEngine.Instance) === 'unsupported') {
|
|
1618
|
+
continue;
|
|
1619
|
+
}
|
|
1620
|
+
for (const vendor of ListRealtimeVendorsForModel(model.ID)) {
|
|
1621
|
+
const resolution = this.instantiateRealtimeVendor(model, vendor, resolveKey);
|
|
1622
|
+
if (resolution?.Model.SupportsClientDirect && RealtimeModelShowsAvatar(resolution, AIEngine.Instance)) {
|
|
1623
|
+
return resolution;
|
|
1624
|
+
}
|
|
1625
|
+
}
|
|
1626
|
+
}
|
|
1627
|
+
return null;
|
|
1628
|
+
}
|
|
1059
1629
|
/**
|
|
1060
1630
|
* The session's run-scoped keys as a resolver: the key for a driver class when the run carries
|
|
1061
1631
|
* one, else `undefined` — no platform fallback. The fallback belongs to
|
|
@@ -1077,9 +1647,11 @@ export class RealtimeClientSessionService {
|
|
|
1077
1647
|
*
|
|
1078
1648
|
* @param effectiveConfig The resolved effective configuration.
|
|
1079
1649
|
* @param resolve The session's run-scoped key resolver (run keys only; see {@link resolveVendorAndInstantiate}).
|
|
1650
|
+
* @param credentialScope The session's credential scope; see {@link resolveVendorAndInstantiate}.
|
|
1651
|
+
* @param showsAvatar Whether the session will show an avatar; see {@link resolveVendorAndInstantiate}.
|
|
1080
1652
|
* @returns The resolution, or `null` when no preference is configured or it can't be satisfied.
|
|
1081
1653
|
*/
|
|
1082
|
-
resolveConfiguredModelPreference(effectiveConfig, resolve) {
|
|
1654
|
+
resolveConfiguredModelPreference(effectiveConfig, resolve, credentialScope = 'Any', showsAvatar = false) {
|
|
1083
1655
|
const preference = effectiveConfig?.realtime?.modelPreference;
|
|
1084
1656
|
if (!preference) {
|
|
1085
1657
|
return null;
|
|
@@ -1095,7 +1667,7 @@ export class RealtimeClientSessionService {
|
|
|
1095
1667
|
`'${model.Name}' but it is not an Active Realtime model — falling through to default resolution.`);
|
|
1096
1668
|
return null;
|
|
1097
1669
|
}
|
|
1098
|
-
const resolution = this.resolveVendorAndInstantiate(model, resolve);
|
|
1670
|
+
const resolution = this.resolveVendorAndInstantiate(model, resolve, credentialScope, showsAvatar);
|
|
1099
1671
|
if (!resolution) {
|
|
1100
1672
|
LogError(`RealtimeClientSessionService: configured realtime model preference '${model.Name}' has no usable ` +
|
|
1101
1673
|
'vendor DriverClass/API key — falling through to default resolution.');
|
|
@@ -1128,9 +1700,11 @@ export class RealtimeClientSessionService {
|
|
|
1128
1700
|
*
|
|
1129
1701
|
* @param preferredModelID The `MJ: AI Models.ID` the user chose.
|
|
1130
1702
|
* @param resolve The session's run-scoped key resolver (run keys only; see {@link resolveVendorAndInstantiate}).
|
|
1703
|
+
* @param credentialScope The session's credential scope; see {@link resolveVendorAndInstantiate}.
|
|
1704
|
+
* @param showsAvatar Whether the session will show an avatar; see {@link resolveVendorAndInstantiate}.
|
|
1131
1705
|
* @returns The resolution outcome (resolution or a specific failure reason).
|
|
1132
1706
|
*/
|
|
1133
|
-
resolvePreferredRealtimeModel(preferredModelID, resolve) {
|
|
1707
|
+
resolvePreferredRealtimeModel(preferredModelID, resolve, credentialScope = 'Any', showsAvatar = false) {
|
|
1134
1708
|
const model = this.findModelByID(preferredModelID);
|
|
1135
1709
|
if (!model) {
|
|
1136
1710
|
return { ErrorMessage: `The requested realtime model (id '${preferredModelID}') was not found in AI model metadata.` };
|
|
@@ -1141,7 +1715,7 @@ export class RealtimeClientSessionService {
|
|
|
1141
1715
|
if (!this.isRealtimeModel(model)) {
|
|
1142
1716
|
return { ErrorMessage: `The requested model '${model.Name}' is not a Realtime model (its type is '${model.AIModelType}').` };
|
|
1143
1717
|
}
|
|
1144
|
-
const resolution = this.resolveVendorAndInstantiate(model, resolve);
|
|
1718
|
+
const resolution = this.resolveVendorAndInstantiate(model, resolve, credentialScope, showsAvatar);
|
|
1145
1719
|
if (!resolution) {
|
|
1146
1720
|
return {
|
|
1147
1721
|
ErrorMessage: `The requested model '${model.Name}' has no active vendor with a usable DriverClass/API key ` +
|
|
@@ -1174,9 +1748,11 @@ export class RealtimeClientSessionService {
|
|
|
1174
1748
|
*
|
|
1175
1749
|
* @param coAgent The co-agent being voiced (reserved for future per-agent model preference).
|
|
1176
1750
|
* @param resolve The session's run-scoped key resolver (run keys only; see {@link resolveVendorAndInstantiate}).
|
|
1751
|
+
* @param credentialScope The session's credential scope; see {@link resolveVendorAndInstantiate}.
|
|
1752
|
+
* @param showsAvatar Whether the session will show an avatar; see {@link resolveVendorAndInstantiate}.
|
|
1177
1753
|
* @returns The resolved model + identifiers, or `null`.
|
|
1178
1754
|
*/
|
|
1179
|
-
async resolveRealtimeModel(coAgent, resolve) {
|
|
1755
|
+
async resolveRealtimeModel(coAgent, resolve, credentialScope = 'Any', showsAvatar = false) {
|
|
1180
1756
|
// Walk candidates in descending PowerRank, returning the FIRST that fully resolves to a usable
|
|
1181
1757
|
// client-direct driver (active vendor + API key + ClassFactory driver + SupportsClientDirect).
|
|
1182
1758
|
// Single-pick dead-ended whenever the highest-power model lacked a key or client-direct support
|
|
@@ -1184,7 +1760,7 @@ export class RealtimeClientSessionService {
|
|
|
1184
1760
|
// surfaced "No usable Realtime model" instead of falling through to a model that works.
|
|
1185
1761
|
const candidates = this.selectRealtimeModelCandidates(coAgent);
|
|
1186
1762
|
for (const model of candidates) {
|
|
1187
|
-
const resolution = this.resolveVendorAndInstantiate(model, resolve);
|
|
1763
|
+
const resolution = this.resolveVendorAndInstantiate(model, resolve, credentialScope, showsAvatar);
|
|
1188
1764
|
if (resolution && resolution.Model.SupportsClientDirect) {
|
|
1189
1765
|
return resolution;
|
|
1190
1766
|
}
|
|
@@ -1203,16 +1779,43 @@ export class RealtimeClientSessionService {
|
|
|
1203
1779
|
* @param resolve The session's run-scoped key resolver. Expected to answer with run keys only:
|
|
1204
1780
|
* one that falls back to the environment itself answers before {@link getAPIKeyForDriver} and
|
|
1205
1781
|
* so bypasses an override of it.
|
|
1782
|
+
* @param credentialScope A scope that rules out the `'Environment'` source (`'RuntimeOnly'`) drops the
|
|
1783
|
+
* {@link getAPIKeyForDriver} seam: the run's keys are the whole chain, so a vendor they do not key
|
|
1784
|
+
* is never selected on the platform's key.
|
|
1785
|
+
* @param showsAvatar Whether the session will show an avatar. The highest-Priority keyed vendor stands when it
|
|
1786
|
+
* renders avatars ({@link RealtimeModelShowsAvatar}) exactly when the session will show one; otherwise the first
|
|
1787
|
+
* other keyed vendor that does is taken, when there is one. So a vendor that renders avatars (Vertex AI for Gemini
|
|
1788
|
+
* 3.8 Live, with its own billing and limits) takes a session only when it will show an avatar, or when it is the
|
|
1789
|
+
* only keyed vendor; and a session that will show one goes to such a vendor whatever the Priority order.
|
|
1206
1790
|
* @returns The full resolution, or `null` when no vendor/key/driver can be satisfied.
|
|
1207
1791
|
*/
|
|
1208
|
-
resolveVendorAndInstantiate(model, resolve) {
|
|
1209
|
-
|
|
1210
|
-
// so a run-scoped credential wins without taking that seam away from anyone who replaced it.
|
|
1211
|
-
const resolveKey = (driverClass) => resolve?.(driverClass) ?? this.getAPIKeyForDriver(driverClass);
|
|
1792
|
+
resolveVendorAndInstantiate(model, resolve, credentialScope = 'Any', showsAvatar = false) {
|
|
1793
|
+
const resolveKey = this.buildKeyChain(resolve, credentialScope);
|
|
1212
1794
|
const vendor = this.selectRealtimeVendor(model.ID, resolveKey);
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1795
|
+
const first = vendor ? this.instantiateRealtimeVendor(model, vendor, resolveKey) : null;
|
|
1796
|
+
if (!vendor || !first) {
|
|
1797
|
+
return first;
|
|
1798
|
+
}
|
|
1799
|
+
return ChooseRealtimeVendorForAvatar(vendor, first, ListRealtimeVendorsForModel(model.ID), showsAvatar, {
|
|
1800
|
+
Instantiate: (other) => this.instantiateRealtimeVendor(model, other, resolveKey),
|
|
1801
|
+
RendersAvatar: (resolution) => RealtimeModelShowsAvatar(resolution, AIEngine.Instance),
|
|
1802
|
+
// A stand-in does client-direct sessions when the first one does.
|
|
1803
|
+
Accepts: (resolution) => resolution.Model.SupportsClientDirect || !first.Model.SupportsClientDirect,
|
|
1804
|
+
});
|
|
1805
|
+
}
|
|
1806
|
+
/**
|
|
1807
|
+
* The session's complete key chain for vendor selection and the mint (see {@link resolveVendorAndInstantiate}): the
|
|
1808
|
+
* run's keys first, then this service's own seam (which subclasses and tests override), so a run-scoped credential
|
|
1809
|
+
* wins without taking that seam away from anyone who replaced it. A scope that rules out the environment has no second
|
|
1810
|
+
* step: the seam's default is the platform key.
|
|
1811
|
+
*/
|
|
1812
|
+
buildKeyChain(resolve, credentialScope) {
|
|
1813
|
+
return CredentialScopeAllows(credentialScope, 'Environment')
|
|
1814
|
+
? (driverClass) => resolve?.(driverClass) ?? this.getAPIKeyForDriver(driverClass)
|
|
1815
|
+
: (driverClass) => resolve?.(driverClass);
|
|
1816
|
+
}
|
|
1817
|
+
/** Instantiates a vendor's realtime driver with its key from the chain; `null` when the key or the driver is missing. */
|
|
1818
|
+
instantiateRealtimeVendor(model, vendor, resolveKey) {
|
|
1216
1819
|
const apiKey = resolveKey(vendor.DriverClass);
|
|
1217
1820
|
if (!apiKey) {
|
|
1218
1821
|
return null;
|
|
@@ -1257,7 +1860,9 @@ export class RealtimeClientSessionService {
|
|
|
1257
1860
|
}
|
|
1258
1861
|
/**
|
|
1259
1862
|
* The active models of AIModelType `Realtime`, sorted highest-PowerRank first — the candidate
|
|
1260
|
-
* list {@link resolveRealtimeModel} walks until one yields a usable client-direct driver
|
|
1863
|
+
* list {@link resolveRealtimeModel} walks until one yields a usable client-direct driver, and
|
|
1864
|
+
* {@link ResolveAvatarRealtimeModel} walks first for a co-agent with video on. Equal PowerRanks
|
|
1865
|
+
* keep the engine's cached order (a stable sort).
|
|
1261
1866
|
* Returns ALL candidates (not just the top pick) so a keyless or non-client-direct top model
|
|
1262
1867
|
* falls through to the next usable one instead of dead-ending the whole resolution.
|
|
1263
1868
|
*
|
|
@@ -1308,20 +1913,24 @@ export class RealtimeClientSessionService {
|
|
|
1308
1913
|
* @param driverClass The resolved vendor's DriverClass — matches per-provider voice settings.
|
|
1309
1914
|
* @param modelID The resolved `MJ: AI Models` id — keys the model-catalog `ModelConfiguration` cascade.
|
|
1310
1915
|
* @param modelVendorID The resolved `MJ: AI Model Vendors` ROW id — the cascade's most-specific layer.
|
|
1916
|
+
* @param resolvedAvatar The session's avatar resolution, when the caller already made it; resolved here otherwise.
|
|
1311
1917
|
* @returns The assembled session params.
|
|
1312
1918
|
*/
|
|
1313
|
-
async buildSessionParams(input, coAgent, modelApiName, contextUser, provider, effectiveConfig, driverClass, modelID, modelVendorID) {
|
|
1314
|
-
const directTools = this.
|
|
1919
|
+
async buildSessionParams(input, coAgent, modelApiName, contextUser, provider, effectiveConfig, driverClass, modelID, modelVendorID, resolvedAvatar) {
|
|
1920
|
+
const directTools = this.projectDirectActions(input, effectiveConfig, driverClass);
|
|
1315
1921
|
const hasDirectTools = directTools.length > 0 || (input.ExtraTools != null && input.ExtraTools.length > 0);
|
|
1316
1922
|
const systemPrompt = await this.buildCompanionSystemPrompt(input, coAgent, contextUser, provider, effectiveConfig, hasDirectTools);
|
|
1317
1923
|
const memoryContext = await this.assembleMemoryContext(input, coAgent, contextUser, provider);
|
|
1318
1924
|
const combinedExtra = directTools.length > 0
|
|
1319
1925
|
? [...(input.ExtraTools ?? []), ...directTools]
|
|
1320
1926
|
: input.ExtraTools;
|
|
1321
|
-
const tools = this.buildStableToolSet(combinedExtra);
|
|
1927
|
+
const tools = this.appendHostTools(this.buildStableToolSet(combinedExtra), input.HostTools);
|
|
1322
1928
|
// Hoisted (rather than built inline at the return) so the mint log below can report the voice
|
|
1323
|
-
// that ACTUALLY reached the driver — see the `voice=` field. Same bag, built once.
|
|
1324
|
-
|
|
1929
|
+
// that ACTUALLY reached the driver — see the `voice=` field. Same bag, built once. A phone call, or
|
|
1930
|
+
// an app without agent video, carries no avatar request whoever resolved it (WithoutUnseenAvatar is
|
|
1931
|
+
// idempotent).
|
|
1932
|
+
const avatar = WithoutUnseenAvatar(resolvedAvatar ?? this.ResolveSessionAvatar(input, coAgent, effectiveConfig, modelID, modelVendorID), input);
|
|
1933
|
+
const configBag = this.withAvatarVoice(this.buildSessionConfigBag(input, effectiveConfig, driverClass, modelID, modelVendorID), avatar, input);
|
|
1325
1934
|
WarnOnUnmatchedProviderVoice(effectiveConfig, driverClass, 'RealtimeClientSessionService');
|
|
1326
1935
|
// One line per mint: confirms which tools + whether the channel-direct framing actually reach
|
|
1327
1936
|
// the model — settles "why does the co-agent delegate instead of calling browser_*" without
|
|
@@ -1330,6 +1939,7 @@ export class RealtimeClientSessionService {
|
|
|
1330
1939
|
// actually reached the driver legible — the whole failure mode in #3530 was that it was not.
|
|
1331
1940
|
console.log(`[RealtimeCoAgent] mint model=${modelApiName} driver=${driverClass ?? 'unknown'} ` +
|
|
1332
1941
|
`voice=${typeof configBag?.['voice'] === 'string' ? configBag['voice'] : 'none'} ` +
|
|
1942
|
+
`avatar=${avatar.Avatar ? avatar.Avatar.AvatarID : avatar.Reason ? `none(${avatar.Reason})` : 'none'} ` +
|
|
1333
1943
|
`tools=[${tools.map(t => t.Name).join(', ')}] ` +
|
|
1334
1944
|
`channelExceptionInPrompt=${systemPrompt.includes('interactive-surface')}`);
|
|
1335
1945
|
return {
|
|
@@ -1344,8 +1954,58 @@ export class RealtimeClientSessionService {
|
|
|
1344
1954
|
MaxSessionSeconds: input.MaxSessionSeconds,
|
|
1345
1955
|
UserID: contextUser?.ID,
|
|
1346
1956
|
HasToolFraming: true,
|
|
1957
|
+
...(avatar.Avatar ? { Avatar: input.AvatarDelivery ? { ...avatar.Avatar, Delivery: input.AvatarDelivery } : avatar.Avatar } : {}),
|
|
1347
1958
|
};
|
|
1348
1959
|
}
|
|
1960
|
+
/**
|
|
1961
|
+
* The live avatar this session asks for, if any: from the voiced agent's persona, or `realtime.video.avatarId`,
|
|
1962
|
+
* when the agent's video setting is on (see `ResolveRealtimeAvatar`). A call that picked an avatar (its runtime
|
|
1963
|
+
* override names `realtime.video.avatarId`) while that setting is off asks for none, with reason `video-off`. A model
|
|
1964
|
+
* whose `MJ: AI Model Modalities`
|
|
1965
|
+
* Video/Output row turns video off asks for none (reason `endpoint`, one log line); otherwise whether the session can
|
|
1966
|
+
* render it is the driver's call. A seam so tests can supply a resolution without the engine's persona metadata.
|
|
1967
|
+
*
|
|
1968
|
+
* @param input The prepare-session input (the voiced agent).
|
|
1969
|
+
* @param coAgent The co-agent.
|
|
1970
|
+
* @param effectiveConfig The effective configuration.
|
|
1971
|
+
* @param modelID The resolved model.
|
|
1972
|
+
* @param modelVendorID The resolved model-vendor row; its vendor scopes the persona bindings.
|
|
1973
|
+
*/
|
|
1974
|
+
ResolveSessionAvatar(input, coAgent, effectiveConfig, modelID, modelVendorID) {
|
|
1975
|
+
const videoOn = effectiveConfig?.realtime?.video?.enabled === true;
|
|
1976
|
+
const avatarPicked = !!ResolveEffectiveRealtimeConfig(null, null, input.ConfigOverridesJson).realtime?.video?.avatarId;
|
|
1977
|
+
if (!modelID || (!videoOn && !avatarPicked)) {
|
|
1978
|
+
return {};
|
|
1979
|
+
}
|
|
1980
|
+
const vendorID = modelVendorID ? AIEngine.Instance.ModelVendors.find((mv) => UUIDsEqual(mv.ID, modelVendorID))?.VendorID : undefined;
|
|
1981
|
+
const videoOutputRow = ReadRealtimeVideoOutputRow(modelID, AIEngine.Instance);
|
|
1982
|
+
if (videoOn && videoOutputRow === 'unsupported') {
|
|
1983
|
+
LogStatus(`[RealtimeCoAgent] No avatar asked for: the Video/Output modality row of '${this.findModelByID(modelID)?.Name ?? modelID}' ` +
|
|
1984
|
+
'turns video off (IsSupported false). The call is audio only. Reason: endpoint.');
|
|
1985
|
+
}
|
|
1986
|
+
return ResolveRealtimeAvatar({
|
|
1987
|
+
EffectiveConfig: effectiveConfig,
|
|
1988
|
+
TargetAgentID: input.TargetAgentID,
|
|
1989
|
+
CoAgentID: coAgent.ID,
|
|
1990
|
+
ModelID: modelID,
|
|
1991
|
+
VendorID: vendorID,
|
|
1992
|
+
VideoOutputRow: videoOutputRow,
|
|
1993
|
+
AvatarPicked: avatarPicked,
|
|
1994
|
+
}, AIEngine.Instance);
|
|
1995
|
+
}
|
|
1996
|
+
/**
|
|
1997
|
+
* The config bag with the avatar persona's voice, so the face and the voice match, unless a voice was picked in this
|
|
1998
|
+
* call (a runtime override's `realtime.voice.default.voice`), which wins. A session nobody would see the avatar in (a
|
|
1999
|
+
* phone call, an app without agent video) keeps the voice without the face, so the agent sounds the same as in a call
|
|
2000
|
+
* that shows it.
|
|
2001
|
+
*/
|
|
2002
|
+
withAvatarVoice(bag, avatar, input) {
|
|
2003
|
+
if (!avatar.Voice) {
|
|
2004
|
+
return bag;
|
|
2005
|
+
}
|
|
2006
|
+
const picked = ResolveEffectiveRealtimeConfig(null, null, input.ConfigOverridesJson).realtime?.voice?.default?.voice;
|
|
2007
|
+
return picked ? bag : { ...(bag ?? {}), voice: avatar.Voice };
|
|
2008
|
+
}
|
|
1349
2009
|
/**
|
|
1350
2010
|
* Builds the provider-pact `Config` bag for the session: the effective config's resolved voice
|
|
1351
2011
|
* settings (per {@link GetProviderVoiceSettings} — the persona's agnostic wire-level slots
|
|
@@ -1413,6 +2073,7 @@ export class RealtimeClientSessionService {
|
|
|
1413
2073
|
const colleagues = this.buildColleaguesFromConfig(effectiveConfig, input.TargetAgentID);
|
|
1414
2074
|
const framing = BuildRealtimeAgentFraming(targetName, this.buildInteractiveSurfaceFraming(input.ExtraTools), colleagues, hasDirectTools);
|
|
1415
2075
|
const meetingFraming = this.buildMeetingFraming(input);
|
|
2076
|
+
const hostFraming = input.HostFraming?.trim() ?? '';
|
|
1416
2077
|
const coAgentPrompt = this.getCoAgentSystemPromptText(coAgent);
|
|
1417
2078
|
const voiceManner = BuildVoiceMannerSection(effectiveConfig);
|
|
1418
2079
|
const targetIdentity = this.formatTargetIdentity(target);
|
|
@@ -1420,7 +2081,7 @@ export class RealtimeClientSessionService {
|
|
|
1420
2081
|
const priorTranscript = this.formatPriorTranscript(input.PriorTranscript);
|
|
1421
2082
|
const history = this.formatConversationHistory(input.ConversationMessages);
|
|
1422
2083
|
const memoryContext = await this.assembleMemoryContext(input, coAgent, contextUser, provider);
|
|
1423
|
-
return [framing, meetingFraming, coAgentPrompt, voiceManner, targetIdentity, appContextSection, priorTranscript, history, memoryContext]
|
|
2084
|
+
return [framing, meetingFraming, hostFraming, coAgentPrompt, voiceManner, targetIdentity, appContextSection, priorTranscript, history, memoryContext]
|
|
1424
2085
|
.filter(part => part && part.trim().length > 0)
|
|
1425
2086
|
.join('\n\n');
|
|
1426
2087
|
}
|
|
@@ -1542,7 +2203,9 @@ export class RealtimeClientSessionService {
|
|
|
1542
2203
|
`Knowledge Hub" → ContextTool with action 'NavigateToApp'; "open this record"; "switch to the X tab"). ` +
|
|
1543
2204
|
`Do NOT route in-app navigation/actions through '${INVOKE_TARGET_AGENT_TOOL_NAME}', and NEVER say you ` +
|
|
1544
2205
|
`need an active session id or a screen snapshot to do them — calling 'ContextTool' is all that's ` +
|
|
1545
|
-
`needed. Reserve '${INVOKE_TARGET_AGENT_TOOL_NAME}' for actual analysis / data work, not navigation
|
|
2206
|
+
`needed. Reserve '${INVOKE_TARGET_AGENT_TOOL_NAME}' for actual analysis / data work, not navigation. ` +
|
|
2207
|
+
`When a channel note lists interactive channels you can use, 'ContextTool' also addresses ONE of ` +
|
|
2208
|
+
`them: add "target": { "channel": "<channel name>" } and use that channel's action names.`;
|
|
1546
2209
|
}
|
|
1547
2210
|
return clause;
|
|
1548
2211
|
}
|
|
@@ -1746,6 +2409,21 @@ export class RealtimeClientSessionService {
|
|
|
1746
2409
|
}
|
|
1747
2410
|
return result;
|
|
1748
2411
|
}
|
|
2412
|
+
/** Appends the host-declared tools to a stable tool set, dropping any whose name is already taken. */
|
|
2413
|
+
appendHostTools(tools, hostTools) {
|
|
2414
|
+
if (!hostTools || hostTools.length === 0) {
|
|
2415
|
+
return tools;
|
|
2416
|
+
}
|
|
2417
|
+
const taken = new Set(tools.map((t) => t.Name.toLowerCase()));
|
|
2418
|
+
const merged = [...tools];
|
|
2419
|
+
for (const tool of hostTools) {
|
|
2420
|
+
if (!taken.has(tool.Name.toLowerCase())) {
|
|
2421
|
+
taken.add(tool.Name.toLowerCase());
|
|
2422
|
+
merged.push(tool);
|
|
2423
|
+
}
|
|
2424
|
+
}
|
|
2425
|
+
return merged;
|
|
2426
|
+
}
|
|
1749
2427
|
/**
|
|
1750
2428
|
* Builds the {@link RealtimeToolBroker} for a relayed tool call, wiring `DelegateToTarget` to a
|
|
1751
2429
|
* target-agent run and `ExecuteTool` to direct-action execution or a structured "not available" placeholder.
|
|
@@ -1934,11 +2612,13 @@ export class RealtimeClientSessionService {
|
|
|
1934
2612
|
/**
|
|
1935
2613
|
* Runs (or resumes) the target agent for a delegation. Threads the combined abort signal, parent
|
|
1936
2614
|
* run linkage, session id, and the `OnProgress` callback so the resolver can stream progress.
|
|
2615
|
+
* The browser session id becomes the run's `sessionID` (client-tool channel) and the app-context
|
|
2616
|
+
* snapshot rides `data.appContext` (prompt context and surface tools); both are optional.
|
|
1937
2617
|
* When {@link ExecuteRelayedToolInput.ResumeRunID} is set, resumes that paused run via
|
|
1938
2618
|
* `lastRunId` + `autoPopulateLastRunPayload` (the user's answer continues the same interactive
|
|
1939
2619
|
* run) instead of starting fresh.
|
|
1940
2620
|
*
|
|
1941
|
-
* @param input The relayed tool input (linkage, progress callback, optional resume id).
|
|
2621
|
+
* @param input The relayed tool input (linkage, progress callback, optional resume id, browser session, app context).
|
|
1942
2622
|
* @param request The broker's delegation request (call id + arguments + abort signal).
|
|
1943
2623
|
* @param target The resolved target agent.
|
|
1944
2624
|
* @param contextUser The calling user.
|
|
@@ -1961,6 +2641,8 @@ export class RealtimeClientSessionService {
|
|
|
1961
2641
|
cancellationToken: this.combineSignals(request.AbortSignal, input.AbortSignal),
|
|
1962
2642
|
parentRun: parentRun ?? undefined,
|
|
1963
2643
|
agentSessionID: input.AgentSessionID,
|
|
2644
|
+
sessionID: input.BrowserSessionID,
|
|
2645
|
+
data: input.AppContext ? { appContext: input.AppContext } : undefined,
|
|
1964
2646
|
onProgress: input.OnProgress,
|
|
1965
2647
|
lastRunId: input.ResumeRunID,
|
|
1966
2648
|
autoPopulateLastRunPayload: input.ResumeRunID ? true : undefined
|
|
@@ -2046,11 +2728,25 @@ export class RealtimeClientSessionService {
|
|
|
2046
2728
|
const agentActions = (AIEngine.Instance.AgentActions ?? []).filter(aa => UUIDsEqual(aa.AgentID, targetAgentID) && aa.Status === 'Active');
|
|
2047
2729
|
return (ActionEngineServer.Instance.Actions ?? []).filter(a => agentActions.some(aa => UUIDsEqual(aa.ActionID, a.ID)) && a.Status === 'Active');
|
|
2048
2730
|
}
|
|
2731
|
+
/**
|
|
2732
|
+
* The session's direct-action tools. When the input excludes direct actions, none, and the empty
|
|
2733
|
+
* projection is recorded so the session's relays run none.
|
|
2734
|
+
*/
|
|
2735
|
+
projectDirectActions(input, effectiveConfig, driverClass) {
|
|
2736
|
+
if (input.ExcludeDirectActions) {
|
|
2737
|
+
this.recordEmptyProjection(input.AgentSessionID);
|
|
2738
|
+
return [];
|
|
2739
|
+
}
|
|
2740
|
+
return this.BuildDirectActionTools(input.TargetAgentID, effectiveConfig, driverClass, input.AgentSessionID);
|
|
2741
|
+
}
|
|
2049
2742
|
/**
|
|
2050
2743
|
* Builds the projected direct action tool definitions for a target agent if supported by the driver.
|
|
2051
2744
|
* Gated by driver capability (`SupportsDynamicToolSet`) and explicit configuration opt-in
|
|
2052
2745
|
* (`Configuration.realtime.directActions.enabled = true`).
|
|
2053
2746
|
*
|
|
2747
|
+
* With `agentSessionID`, the projection is recorded for the session, even when it is empty: relayed
|
|
2748
|
+
* direct-action calls for the session may run only the projected actions.
|
|
2749
|
+
*
|
|
2054
2750
|
* @param targetAgentID The target agent ID being voiced.
|
|
2055
2751
|
* @param effectiveConfig The resolved effective configuration.
|
|
2056
2752
|
* @param driverClass The resolved vendor's DriverClass.
|
|
@@ -2058,10 +2754,12 @@ export class RealtimeClientSessionService {
|
|
|
2058
2754
|
*/
|
|
2059
2755
|
BuildDirectActionTools(targetAgentID, effectiveConfig, driverClass, agentSessionID) {
|
|
2060
2756
|
if (!targetAgentID || !this.driverSupportsDynamicToolSet(driverClass)) {
|
|
2757
|
+
this.recordEmptyProjection(agentSessionID);
|
|
2061
2758
|
return [];
|
|
2062
2759
|
}
|
|
2063
2760
|
const directConfig = GetDirectActionsConfig(effectiveConfig);
|
|
2064
2761
|
if (!directConfig || !directConfig.enabled) {
|
|
2762
|
+
this.recordEmptyProjection(agentSessionID);
|
|
2065
2763
|
return [];
|
|
2066
2764
|
}
|
|
2067
2765
|
const candidateActions = this.getTargetAgentActions(targetAgentID);
|
|
@@ -2080,6 +2778,12 @@ export class RealtimeClientSessionService {
|
|
|
2080
2778
|
}
|
|
2081
2779
|
return tools;
|
|
2082
2780
|
}
|
|
2781
|
+
/** Records that a session projected no direct actions, so its relays run none. */
|
|
2782
|
+
recordEmptyProjection(agentSessionID) {
|
|
2783
|
+
if (agentSessionID) {
|
|
2784
|
+
this.sessionWireActionMaps.Set(agentSessionID, new Map());
|
|
2785
|
+
}
|
|
2786
|
+
}
|
|
2083
2787
|
/** @deprecated Use {@link BuildDirectActionTools}. */
|
|
2084
2788
|
buildDirectActionTools(targetAgentID, effectiveConfig, driverClass, agentSessionID) {
|
|
2085
2789
|
return this.BuildDirectActionTools(targetAgentID, effectiveConfig, driverClass, agentSessionID);
|
|
@@ -2135,6 +2839,13 @@ export class RealtimeClientSessionService {
|
|
|
2135
2839
|
* configured timeout. If the tool is unrecognized, disallowed, or target resolution fails,
|
|
2136
2840
|
* returns a structured "not available" result.
|
|
2137
2841
|
*
|
|
2842
|
+
* When this service holds the session's start-time projection, only the projected actions may
|
|
2843
|
+
* run, whatever target the input names. When it holds none, the target agent's own metadata
|
|
2844
|
+
* decides: another process prepared the session, the entry was evicted or lost on a restart, or
|
|
2845
|
+
* the session was never started through this service (for example a row created through the
|
|
2846
|
+
* entity API). That fallback skips the driver-capability gate but is still limited to the
|
|
2847
|
+
* target's own direct-actions allowlist.
|
|
2848
|
+
*
|
|
2138
2849
|
* @param call The non-target tool call.
|
|
2139
2850
|
* @param input The optional relayed tool input context.
|
|
2140
2851
|
* @param contextUser The calling user context.
|
|
@@ -2156,11 +2867,9 @@ export class RealtimeClientSessionService {
|
|
|
2156
2867
|
Output: `Target agent is not available to execute '${call.ToolName}'.`
|
|
2157
2868
|
};
|
|
2158
2869
|
}
|
|
2159
|
-
const
|
|
2160
|
-
const candidateWireMap = this.BuildWireActionMap(
|
|
2161
|
-
const action = (
|
|
2162
|
-
?? candidateWireMap.get(call.ToolName)
|
|
2163
|
-
?? Array.from(candidateWireMap.entries()).find(([w]) => w.toLowerCase() === call.ToolName.trim().toLowerCase())?.[1];
|
|
2870
|
+
const projection = input?.AgentSessionID ? this.sessionWireActionMaps.Get(input.AgentSessionID) : undefined;
|
|
2871
|
+
const candidateWireMap = this.BuildWireActionMap(this.getTargetAgentActions(target.ID));
|
|
2872
|
+
const action = this.findWireAction(projection, call.ToolName) ?? this.findWireAction(candidateWireMap, call.ToolName);
|
|
2164
2873
|
if (!action) {
|
|
2165
2874
|
return {
|
|
2166
2875
|
CallID: call.CallID,
|
|
@@ -2171,7 +2880,7 @@ export class RealtimeClientSessionService {
|
|
|
2171
2880
|
const directConfig = input?.DirectActions
|
|
2172
2881
|
?? (input?.AgentSessionID ? this.sessionDirectConfigs.Get(input.AgentSessionID) : undefined)
|
|
2173
2882
|
?? GetDirectActionsConfig(this.resolveEffectiveConfig(target, undefined, target));
|
|
2174
|
-
if (!IsActionAllowedForDirectInvocation(action.Name, directConfig)) {
|
|
2883
|
+
if (!this.isInProjection(projection, action) || !IsActionAllowedForDirectInvocation(action.Name, directConfig)) {
|
|
2175
2884
|
return {
|
|
2176
2885
|
CallID: call.CallID,
|
|
2177
2886
|
Success: false,
|
|
@@ -2232,6 +2941,24 @@ export class RealtimeClientSessionService {
|
|
|
2232
2941
|
};
|
|
2233
2942
|
}
|
|
2234
2943
|
}
|
|
2944
|
+
/**
|
|
2945
|
+
* Finds the action a relayed tool name refers to: the exact wire name first, then a
|
|
2946
|
+
* case-insensitive match.
|
|
2947
|
+
*/
|
|
2948
|
+
findWireAction(wireMap, toolName) {
|
|
2949
|
+
if (!wireMap) {
|
|
2950
|
+
return undefined;
|
|
2951
|
+
}
|
|
2952
|
+
const wanted = toolName.trim().toLowerCase();
|
|
2953
|
+
return wireMap.get(toolName) ?? Array.from(wireMap.entries()).find(([wireName]) => wireName.toLowerCase() === wanted)?.[1];
|
|
2954
|
+
}
|
|
2955
|
+
/**
|
|
2956
|
+
* True when the action is in the session's start-time projection, or when this service holds no
|
|
2957
|
+
* projection for the session (see {@link executeNonTargetTool} for when that happens).
|
|
2958
|
+
*/
|
|
2959
|
+
isInProjection(projection, action) {
|
|
2960
|
+
return !projection || Array.from(projection.values()).some(projected => UUIDsEqual(projected.ID, action.ID));
|
|
2961
|
+
}
|
|
2235
2962
|
/**
|
|
2236
2963
|
* Parses JSON arguments string into a key-value record of parameters.
|
|
2237
2964
|
* Distinguishes "no arguments" (valid for zero-param action) from "unparseable JSON".
|