@memberjunction/ai-agents 5.40.1 → 5.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -0
- package/dist/AgentRunner.d.ts +5 -2
- package/dist/AgentRunner.d.ts.map +1 -1
- package/dist/AgentRunner.js +14 -4
- package/dist/AgentRunner.js.map +1 -1
- package/dist/MemoryWriteManager.d.ts +188 -0
- package/dist/MemoryWriteManager.d.ts.map +1 -0
- package/dist/MemoryWriteManager.js +299 -0
- package/dist/MemoryWriteManager.js.map +1 -0
- package/dist/agent-context-injector.d.ts +29 -0
- package/dist/agent-context-injector.d.ts.map +1 -1
- package/dist/agent-context-injector.js +90 -32
- package/dist/agent-context-injector.js.map +1 -1
- package/dist/agent-memory-context-builder.d.ts +100 -0
- package/dist/agent-memory-context-builder.d.ts.map +1 -0
- package/dist/agent-memory-context-builder.js +172 -0
- package/dist/agent-memory-context-builder.js.map +1 -0
- package/dist/agent-types/index.d.ts +1 -0
- package/dist/agent-types/index.d.ts.map +1 -1
- package/dist/agent-types/index.js +1 -0
- package/dist/agent-types/index.js.map +1 -1
- package/dist/agent-types/loop-agent-response-type.d.ts +12 -1
- package/dist/agent-types/loop-agent-response-type.d.ts.map +1 -1
- package/dist/agent-types/loop-agent-response-type.js.map +1 -1
- package/dist/agent-types/loop-agent-type.d.ts.map +1 -1
- package/dist/agent-types/loop-agent-type.js +4 -0
- package/dist/agent-types/loop-agent-type.js.map +1 -1
- package/dist/agent-types/realtime-agent-type.d.ts +146 -0
- package/dist/agent-types/realtime-agent-type.d.ts.map +1 -0
- package/dist/agent-types/realtime-agent-type.js +176 -0
- package/dist/agent-types/realtime-agent-type.js.map +1 -0
- package/dist/base-agent.d.ts +365 -24
- package/dist/base-agent.d.ts.map +1 -1
- package/dist/base-agent.js +995 -175
- package/dist/base-agent.js.map +1 -1
- package/dist/index.d.ts +11 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +15 -0
- package/dist/index.js.map +1 -1
- package/dist/memory-manager-agent.d.ts +55 -2
- package/dist/memory-manager-agent.d.ts.map +1 -1
- package/dist/memory-manager-agent.js +261 -62
- package/dist/memory-manager-agent.js.map +1 -1
- package/dist/realtime/meeting-controls-channel-server.d.ts +198 -0
- package/dist/realtime/meeting-controls-channel-server.d.ts.map +1 -0
- package/dist/realtime/meeting-controls-channel-server.js +319 -0
- package/dist/realtime/meeting-controls-channel-server.js.map +1 -0
- package/dist/realtime/meeting-controls-state.d.ts +191 -0
- package/dist/realtime/meeting-controls-state.d.ts.map +1 -0
- package/dist/realtime/meeting-controls-state.js +219 -0
- package/dist/realtime/meeting-controls-state.js.map +1 -0
- package/dist/realtime/realtime-channel-server-host.d.ts +166 -0
- package/dist/realtime/realtime-channel-server-host.d.ts.map +1 -0
- package/dist/realtime/realtime-channel-server-host.js +378 -0
- package/dist/realtime/realtime-channel-server-host.js.map +1 -0
- package/dist/realtime/realtime-client-session-service.d.ts +884 -0
- package/dist/realtime/realtime-client-session-service.d.ts.map +1 -0
- package/dist/realtime/realtime-client-session-service.js +1401 -0
- package/dist/realtime/realtime-client-session-service.js.map +1 -0
- package/dist/realtime/realtime-coagent-config.d.ts +202 -0
- package/dist/realtime/realtime-coagent-config.d.ts.map +1 -0
- package/dist/realtime/realtime-coagent-config.js +334 -0
- package/dist/realtime/realtime-coagent-config.js.map +1 -0
- package/dist/realtime/realtime-narration.d.ts +67 -0
- package/dist/realtime/realtime-narration.d.ts.map +1 -0
- package/dist/realtime/realtime-narration.js +127 -0
- package/dist/realtime/realtime-narration.js.map +1 -0
- package/dist/realtime/realtime-session-runner.d.ts +383 -0
- package/dist/realtime/realtime-session-runner.d.ts.map +1 -0
- package/dist/realtime/realtime-session-runner.js +532 -0
- package/dist/realtime/realtime-session-runner.js.map +1 -0
- package/dist/realtime/realtime-tool-broker.d.ts +279 -0
- package/dist/realtime/realtime-tool-broker.d.ts.map +1 -0
- package/dist/realtime/realtime-tool-broker.js +184 -0
- package/dist/realtime/realtime-tool-broker.js.map +1 -0
- package/dist/realtime/whiteboard-channel-server.d.ts +50 -0
- package/dist/realtime/whiteboard-channel-server.d.ts.map +1 -0
- package/dist/realtime/whiteboard-channel-server.js +85 -0
- package/dist/realtime/whiteboard-channel-server.js.map +1 -0
- package/package.json +17 -17
|
@@ -0,0 +1,884 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Server-agnostic preparer + tool relay for a CLIENT-DIRECT realtime session
|
|
3
|
+
* (the Realtime Co-Agent dual-topology design).
|
|
4
|
+
*
|
|
5
|
+
* In the client-direct topology the browser opens its OWN provider socket (e.g. WebRTC) using a
|
|
6
|
+
* server-minted ephemeral token, but the **server** still owns the system prompt and tool set and
|
|
7
|
+
* **executes** every tool call the browser relays back. This service is the server-side half of
|
|
8
|
+
* that contract. It does two things:
|
|
9
|
+
*
|
|
10
|
+
* 1. {@link RealtimeClientSessionService.PrepareClientSession} — resolves the Realtime model,
|
|
11
|
+
* assembles the companion system prompt (co-agent prompt + target identity + history + memory),
|
|
12
|
+
* builds the stable, target-independent tool set (always including `invoke-target-agent`), and
|
|
13
|
+
* asks the model to mint a {@link ClientRealtimeSessionConfig} (ephemeral token + provider
|
|
14
|
+
* session config) the browser applies verbatim.
|
|
15
|
+
* 2. {@link RealtimeClientSessionService.ExecuteRelayedTool} — executes a single tool call the
|
|
16
|
+
* browser relayed, routing it through the shared {@link RealtimeToolBroker} so the result is
|
|
17
|
+
* byte-for-byte identical to the server-bridged path. `invoke-target-agent` delegates to the
|
|
18
|
+
* target agent via {@link AgentRunner.RunAgent}; every other tool returns a structured
|
|
19
|
+
* "not available" result for now (action wiring is a later phase).
|
|
20
|
+
*
|
|
21
|
+
* **Why this duplicates BaseAgent.** The private helpers in `BaseAgent.executeRealtimeSession`
|
|
22
|
+
* (model resolution, companion-prompt assembly, target-agent resolution, delegation) are the
|
|
23
|
+
* server-bridged equivalents of the logic here, but they are `private` to `BaseAgent` and bound to
|
|
24
|
+
* an in-flight `AIAgentRun`/`StartSession` lifecycle. This service mirrors that logic for the
|
|
25
|
+
* client-direct topology, which has no server-side session loop. **A future refactor should extract
|
|
26
|
+
* a shared `RealtimeSessionPreparer`** that both `BaseAgent` and this service consume, eliminating
|
|
27
|
+
* the duplication. Until then, keep the two in sync intentionally.
|
|
28
|
+
*
|
|
29
|
+
* @module @memberjunction/ai-agents
|
|
30
|
+
* @author MemberJunction.com
|
|
31
|
+
*/
|
|
32
|
+
import { UserInfo, IMetadataProvider } from '@memberjunction/core';
|
|
33
|
+
import { BaseRealtimeModel, ChatMessage, ClientRealtimeSessionConfig, JSONObject, RealtimeSessionParams, RealtimeToolCall, RealtimeToolDefinition } from '@memberjunction/ai';
|
|
34
|
+
import { MJAIAgentEntityExtended, MJAIModelEntityExtended, MJAIAgentRunEntityExtended, AgentExecutionProgressCallback, ExecuteAgentResult } from '@memberjunction/ai-core-plus';
|
|
35
|
+
import { RealtimeToolBroker, DelegateToTargetRequest, DelegatedResult, DelegatedRunArtifact, ToolExecutionResult } from './realtime-tool-broker.js';
|
|
36
|
+
import { RealtimeCoAgentConfig } from './realtime-coagent-config.js';
|
|
37
|
+
/**
|
|
38
|
+
* Input for {@link RealtimeClientSessionService.PrepareClientSession}.
|
|
39
|
+
*
|
|
40
|
+
* The co-agent may be supplied either as a fully-loaded entity (`CoAgent`) or by id (`CoAgentID`),
|
|
41
|
+
* which is resolved from {@link AIEngine}'s cached agents. The target agent is always supplied by
|
|
42
|
+
* id — it is a runtime choice made when the voice session starts.
|
|
43
|
+
*/
|
|
44
|
+
export interface PrepareClientSessionInput {
|
|
45
|
+
/** The Realtime Co-Agent entity. Provide this OR {@link PrepareClientSessionInput.CoAgentID}. */
|
|
46
|
+
CoAgent?: MJAIAgentEntityExtended;
|
|
47
|
+
/** The Realtime Co-Agent id (resolved from cached metadata). Provide this OR {@link PrepareClientSessionInput.CoAgent}. */
|
|
48
|
+
CoAgentID?: string;
|
|
49
|
+
/** The top-level target agent the co-agent voices on behalf of (a runtime parameter). */
|
|
50
|
+
TargetAgentID: string;
|
|
51
|
+
/** The shared session id grouping this voice session's runs. */
|
|
52
|
+
AgentSessionID: string;
|
|
53
|
+
/** Optional conversation id the session is attached to — stamped on the co-agent observability run. */
|
|
54
|
+
ConversationID?: string;
|
|
55
|
+
/** Prior conversation history to seed the model's context. Optional. */
|
|
56
|
+
ConversationMessages?: ChatMessage[];
|
|
57
|
+
/**
|
|
58
|
+
* Pre-formatted, role-tagged transcript lines (`User: …` / `Assistant: …`, newline-separated)
|
|
59
|
+
* from the caller's PRIOR session leg(s) when this session RESUMES one (`lastSessionId`).
|
|
60
|
+
* The transport layer (the MJServer resolver) loads, ownership-checks, and caps this
|
|
61
|
+
* (~30 turns / ~8k chars, oldest dropped) before threading it here; the service only
|
|
62
|
+
* FRAMES it into the system prompt as a clearly-labeled prior-conversation section so the
|
|
63
|
+
* model remembers the previous leg. Optional — absent for fresh sessions, and any
|
|
64
|
+
* upstream load failure simply omits it (hydration never blocks a start).
|
|
65
|
+
*/
|
|
66
|
+
PriorTranscript?: string;
|
|
67
|
+
/** Optional user-scope id for memory/context retrieval (falls back to the context user). */
|
|
68
|
+
UserID?: string;
|
|
69
|
+
/** Optional company-scope id for memory/context retrieval. */
|
|
70
|
+
CompanyID?: string;
|
|
71
|
+
/** Optional provider-specific session config bag (voice, language, turn detection, etc.). */
|
|
72
|
+
Config?: JSONObject;
|
|
73
|
+
/** Optional extra, target-independent tools to expose in addition to `invoke-target-agent`. */
|
|
74
|
+
ExtraTools?: RealtimeToolDefinition[];
|
|
75
|
+
/**
|
|
76
|
+
* Optional EXPLICIT realtime model choice (`MJ: AI Models.ID`). When set, that exact model is
|
|
77
|
+
* used — it must be Active, of AIModelType `Realtime`, and have an active vendor whose
|
|
78
|
+
* `DriverClass` resolves an API key. If the preferred model cannot be satisfied the prepare
|
|
79
|
+
* FAILS with a clear reason (no silent fallback — the user explicitly chose). When omitted,
|
|
80
|
+
* the default highest-PowerRank resolution applies.
|
|
81
|
+
*/
|
|
82
|
+
PreferredModelID?: string;
|
|
83
|
+
/**
|
|
84
|
+
* Optional RUNTIME configuration-override layer (the most-specific layer of the effective
|
|
85
|
+
* configuration merge: type `DefaultConfiguration` ← agent `TypeConfiguration` ← this).
|
|
86
|
+
* **Pre-authorized by the transport layer** — the MJServer resolver gates it behind the
|
|
87
|
+
* `Realtime: Advanced Session Controls` authorization BEFORE threading it here; the service
|
|
88
|
+
* trusts the input. Malformed JSON is tolerated (it simply contributes nothing to the merge).
|
|
89
|
+
*/
|
|
90
|
+
ConfigOverridesJson?: string;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Result of {@link RealtimeClientSessionService.PrepareClientSession}.
|
|
94
|
+
*
|
|
95
|
+
* On success, {@link RealtimeClientSessionPrepResult.ClientConfig} is the server-minted config the
|
|
96
|
+
* browser applies, and {@link RealtimeClientSessionPrepResult.SessionParams} is the params the
|
|
97
|
+
* server used to mint it (handy for the resolver to echo/persist). On failure, `Success` is `false`
|
|
98
|
+
* and `ErrorMessage` explains why — this method never throws for an unresolvable model/key.
|
|
99
|
+
*/
|
|
100
|
+
export interface RealtimeClientSessionPrepResult {
|
|
101
|
+
/** Whether the client session config was minted successfully. */
|
|
102
|
+
Success: boolean;
|
|
103
|
+
/** The minted client-direct session config (token + provider session config). Present on success. */
|
|
104
|
+
ClientConfig?: ClientRealtimeSessionConfig;
|
|
105
|
+
/** The session params the server built (system prompt, model, tools). Present on success. */
|
|
106
|
+
SessionParams?: RealtimeSessionParams;
|
|
107
|
+
/**
|
|
108
|
+
* ID of the server-side co-agent observability `AIAgentRun` created for this session. Present
|
|
109
|
+
* when the run was created successfully; absent when run creation was skipped or failed
|
|
110
|
+
* (observability is best-effort and never fails the prepare). Delegated target-agent runs nest
|
|
111
|
+
* under this run via `ParentRunID`, and {@link RealtimeClientSessionService.FinalizeCoAgentRun}
|
|
112
|
+
* closes it when the session ends.
|
|
113
|
+
*/
|
|
114
|
+
CoAgentRunID?: string;
|
|
115
|
+
/**
|
|
116
|
+
* ID of the server-side co-agent `AIPromptRun` linked to {@link RealtimeClientSessionPrepResult.CoAgentRunID}.
|
|
117
|
+
* Present only when the co-agent's system prompt resolved (so a prompt run could be created).
|
|
118
|
+
*/
|
|
119
|
+
PromptRunID?: string;
|
|
120
|
+
/**
|
|
121
|
+
* ID of the single `MJ: AI Agent Run Steps` row created under {@link RealtimeClientSessionPrepResult.CoAgentRunID}
|
|
122
|
+
* for the realtime session's system prompt (StepType `Prompt`, TargetID = the system prompt,
|
|
123
|
+
* TargetLogID = {@link RealtimeClientSessionPrepResult.PromptRunID}). It makes the co-agent run's
|
|
124
|
+
* Timeline non-empty. Present only when the co-agent's system prompt resolved AND the step saved
|
|
125
|
+
* (step creation is best-effort, like the runs themselves). Finalized alongside the runs by
|
|
126
|
+
* {@link RealtimeClientSessionService.FinalizeCoAgentRun}.
|
|
127
|
+
*/
|
|
128
|
+
CoAgentRunStepID?: string;
|
|
129
|
+
/** A human-readable failure reason. Present on failure. */
|
|
130
|
+
ErrorMessage?: string;
|
|
131
|
+
/** The `MJ: AI Models` row id of the realtime model the session was minted with. Present on success. */
|
|
132
|
+
ModelID?: string;
|
|
133
|
+
/** The display name of the realtime model the session was minted with. Present on success. */
|
|
134
|
+
ModelName?: string;
|
|
135
|
+
/**
|
|
136
|
+
* The DB-driven progress-narration instruction template (the `Realtime Co-Agent - Progress
|
|
137
|
+
* Narration` prompt's `TemplateText`, containing a `{{ progressMessage }}` placeholder).
|
|
138
|
+
* `undefined` when that prompt is not present in metadata — clients fall back to their
|
|
139
|
+
* built-in narration instruction text.
|
|
140
|
+
*/
|
|
141
|
+
NarrationInstructionsTemplate?: string;
|
|
142
|
+
/**
|
|
143
|
+
* The RESOLVED effective realtime configuration for this session (type defaults ← agent
|
|
144
|
+
* config ← runtime overrides, deep-merged + normalized). Present on success — `{}` when no
|
|
145
|
+
* layer configured anything. Surfaced so the transport layer can echo it to the client
|
|
146
|
+
* (client drivers apply provider voice settings client-side in the client-direct topology).
|
|
147
|
+
*/
|
|
148
|
+
EffectiveConfig?: RealtimeCoAgentConfig;
|
|
149
|
+
/**
|
|
150
|
+
* The effective narration pace (`realtime.narration.paceMs`) — minimum gap in ms between
|
|
151
|
+
* spoken progress updates. `undefined` when not configured (clients/runners use their
|
|
152
|
+
* built-in default). In the CLIENT-DIRECT topology narration pacing is enforced client-side,
|
|
153
|
+
* so this is surfaced for the browser; the server-bridged runner consumes it directly via
|
|
154
|
+
* `RealtimeSessionRunnerDeps.NarrationPaceMs`.
|
|
155
|
+
*/
|
|
156
|
+
NarrationPaceMs?: number;
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* The resolved co-agent system prompt text plus the id of the prompt it came from, returned by
|
|
160
|
+
* {@link RealtimeClientSessionService.resolveCoAgentSystemPrompt}.
|
|
161
|
+
*/
|
|
162
|
+
export interface CoAgentSystemPromptResolution {
|
|
163
|
+
/** The co-agent's system prompt template text (empty string when none is configured). */
|
|
164
|
+
Text: string;
|
|
165
|
+
/** The `MJ: AI Prompts` row id, or `null` when the co-agent has no active prompt. */
|
|
166
|
+
PromptID: string | null;
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* Input for {@link RealtimeClientSessionService.ExecuteRelayedTool}.
|
|
170
|
+
*
|
|
171
|
+
* Carries the single tool call the browser relayed plus the linkage needed to run a delegated
|
|
172
|
+
* target-agent run under the same session.
|
|
173
|
+
*/
|
|
174
|
+
export interface ExecuteRelayedToolInput {
|
|
175
|
+
/** The shared session id grouping this voice session's runs. */
|
|
176
|
+
AgentSessionID: string;
|
|
177
|
+
/** The id of the (co-agent) run that owns this session, used as the delegated run's parent. Optional. */
|
|
178
|
+
ParentRunID?: string;
|
|
179
|
+
/** The top-level target agent id for `invoke-target-agent` delegation. */
|
|
180
|
+
TargetAgentID: string;
|
|
181
|
+
/** The tool call the browser relayed from the provider. */
|
|
182
|
+
Call: RealtimeToolCall;
|
|
183
|
+
/**
|
|
184
|
+
* Optional abort signal so a barge-in on the browser can cancel an in-flight delegated run.
|
|
185
|
+
* Threaded into the delegated agent run's `cancellationToken`.
|
|
186
|
+
*/
|
|
187
|
+
AbortSignal?: AbortSignal;
|
|
188
|
+
/**
|
|
189
|
+
* Optional progress callback invoked with each delegated-run progress event (mirrors the normal
|
|
190
|
+
* agent-run path's `onProgress`). The transport layer (the MJServer resolver) publishes these so
|
|
191
|
+
* the realtime model can narrate the target agent's progress while it runs. When omitted, the
|
|
192
|
+
* delegated run streams nothing and the model only receives the final tool result.
|
|
193
|
+
*/
|
|
194
|
+
OnProgress?: AgentExecutionProgressCallback;
|
|
195
|
+
/**
|
|
196
|
+
* Optional id of a previously-paused delegated run (Status `AwaitingFeedback`) to RESUME instead
|
|
197
|
+
* of starting a fresh run. When set, {@link delegateToTarget} passes it as `lastRunId` (with
|
|
198
|
+
* `autoPopulateLastRunPayload`) to {@link AgentRunner.RunAgent}, so the user's answer continues
|
|
199
|
+
* the SAME interactive run (e.g. confirming a Query Builder task graph).
|
|
200
|
+
*/
|
|
201
|
+
ResumeRunID?: string;
|
|
202
|
+
}
|
|
203
|
+
/**
|
|
204
|
+
* The resolved Realtime model plus its identifiers, returned by the model-resolution seam.
|
|
205
|
+
*/
|
|
206
|
+
export interface RealtimeModelResolution {
|
|
207
|
+
/** The instantiated realtime driver. */
|
|
208
|
+
Model: BaseRealtimeModel;
|
|
209
|
+
/** The `MJ: AI Models` row id. */
|
|
210
|
+
ModelID: string;
|
|
211
|
+
/** The chosen vendor id. */
|
|
212
|
+
VendorID: string;
|
|
213
|
+
/** The vendor API name passed to the provider as the model id. */
|
|
214
|
+
APIName: string;
|
|
215
|
+
/** The model's display name (`MJ: AI Models.Name`). Optional for back-compat with test seams. */
|
|
216
|
+
ModelName?: string;
|
|
217
|
+
/**
|
|
218
|
+
* The chosen vendor's `DriverClass` (e.g. `OpenAIRealtime`). Used to match the effective
|
|
219
|
+
* config's per-provider voice settings (`realtime.voice.providers`) onto the driver's open
|
|
220
|
+
* `Config` bag. Optional for back-compat with test seams.
|
|
221
|
+
*/
|
|
222
|
+
DriverClass?: string;
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* Outcome of resolving the realtime model for a session: either a usable {@link RealtimeModelResolution}
|
|
226
|
+
* or a specific, human-readable failure reason (used for explicit preferred-model failures, where the
|
|
227
|
+
* generic "no model" message would hide WHY the user's chosen model couldn't be used).
|
|
228
|
+
*/
|
|
229
|
+
export interface RealtimeModelResolutionOutcome {
|
|
230
|
+
/** The resolved model. Present on success. */
|
|
231
|
+
Resolution?: RealtimeModelResolution;
|
|
232
|
+
/** Why resolution failed. Present on failure. */
|
|
233
|
+
ErrorMessage?: string;
|
|
234
|
+
}
|
|
235
|
+
/**
|
|
236
|
+
* Server-agnostic service that prepares a client-direct realtime session and executes the tool
|
|
237
|
+
* calls the browser relays back. Constructed per-request (a normal injectable service — NOT a
|
|
238
|
+
* singleton) so the {@link UserInfo} and {@link IMetadataProvider} are always request-scoped.
|
|
239
|
+
*
|
|
240
|
+
* Every public method takes the `contextUser` and `provider` explicitly — this service never
|
|
241
|
+
* reaches for the global default provider, so it is safe in multi-provider/multi-tenant servers.
|
|
242
|
+
*/
|
|
243
|
+
export declare class RealtimeClientSessionService {
|
|
244
|
+
/**
|
|
245
|
+
* The seeded name of the `MJ: AI Prompts` row whose `TemplateText` carries the first-person
|
|
246
|
+
* progress-narration instructions (with a `{{ progressMessage }}` placeholder). Resolved at
|
|
247
|
+
* session prepare time so the browser narrates with DB-driven, product-tunable wording.
|
|
248
|
+
* Canonical value lives in `realtime-narration.ts` (shared with the server-bridged runner path).
|
|
249
|
+
*/
|
|
250
|
+
static readonly NarrationPromptName = "Realtime Co-Agent - Progress Narration";
|
|
251
|
+
/**
|
|
252
|
+
* DEPRECATED legacy name of the narration prompt, from before the co-agent's rename from
|
|
253
|
+
* "Voice Co-Agent" to "Realtime Co-Agent". Deployments that have not re-synced the prompt seed
|
|
254
|
+
* still carry this name, so {@link resolveNarrationInstructionsTemplate} falls back to it
|
|
255
|
+
* (with a deprecation log) when {@link RealtimeClientSessionService.NarrationPromptName} is absent.
|
|
256
|
+
*/
|
|
257
|
+
static readonly LegacyNarrationPromptName = "Voice Co-Agent - Progress Narration";
|
|
258
|
+
/**
|
|
259
|
+
* IN-FLIGHT DELEGATION REGISTRY — the server half of the client-direct CANCEL channel.
|
|
260
|
+
*
|
|
261
|
+
* Every relayed tool call registers an {@link AbortController} under
|
|
262
|
+
* `(agentSessionID, callID)` for the duration of {@link ExecuteRelayedTool}; the
|
|
263
|
+
* `CancelRealtimeSessionTool` mutation aborts entries via
|
|
264
|
+
* {@link CancelInFlightDelegations} so an explicit user cancel (the overlay's per-card ✕)
|
|
265
|
+
* kills the delegated target-agent run mid-flight. Entries are removed on completion
|
|
266
|
+
* (success, failure, or abort), so the registry only ever holds truly in-flight calls.
|
|
267
|
+
*
|
|
268
|
+
* Keys are normalized (trimmed, lowercased) so SQL Server's uppercase UUIDs and
|
|
269
|
+
* PostgreSQL's lowercase UUIDs address the same entry.
|
|
270
|
+
*
|
|
271
|
+
* NOTE: this registry is per-service-instance state (the resolver holds ONE shared service
|
|
272
|
+
* per server process), not per-request state — it deliberately spans requests so the cancel
|
|
273
|
+
* mutation can reach the execute mutation's in-flight controller.
|
|
274
|
+
*/
|
|
275
|
+
private readonly inFlightDelegations;
|
|
276
|
+
/**
|
|
277
|
+
* Per-`AIPromptRun` write serialization. Both the high-frequency usage checkpoint
|
|
278
|
+
* ({@link AccumulatePromptRunUsage}) and the per-turn message append ({@link AppendPromptRunMessage})
|
|
279
|
+
* do load-modify-save on the SAME run row. Run concurrently, the frequent usage save would rewrite the
|
|
280
|
+
* whole row — including the STALE `Messages` it loaded — and perpetually clobber freshly-appended turns
|
|
281
|
+
* back to an empty snapshot (the "transcript never persists" bug). Funnelling every write for a given
|
|
282
|
+
* run through a single promise chain makes each load happen AFTER the prior save committed, so no writer
|
|
283
|
+
* overwrites another's field. Keyed by promptRunID; the entry is dropped on {@link finalizePromptRun}.
|
|
284
|
+
*/
|
|
285
|
+
private readonly promptRunWriteChains;
|
|
286
|
+
/**
|
|
287
|
+
* Serializes `task` against all other writes to the same `AIPromptRun` (see {@link promptRunWriteChains}).
|
|
288
|
+
* Tasks run in call order; a failing task never breaks the chain for the next one. Returns the task's result.
|
|
289
|
+
*/
|
|
290
|
+
private serializePromptRunWrite;
|
|
291
|
+
/**
|
|
292
|
+
* Prepares a client-direct realtime session: resolves the model, assembles the companion
|
|
293
|
+
* system prompt + stable tool set, and mints the {@link ClientRealtimeSessionConfig}.
|
|
294
|
+
*
|
|
295
|
+
* Returns a failure result (never throws) when no Realtime model/key resolves or the provider
|
|
296
|
+
* cannot mint a client-direct session.
|
|
297
|
+
*
|
|
298
|
+
* @param input The co-agent/target/session inputs.
|
|
299
|
+
* @param contextUser The calling user (threaded to metadata + memory retrieval).
|
|
300
|
+
* @param provider The request-scoped metadata provider.
|
|
301
|
+
* @returns The prep result (Success + ClientConfig/SessionParams, or Success: false + ErrorMessage).
|
|
302
|
+
*/
|
|
303
|
+
PrepareClientSession(input: PrepareClientSessionInput, contextUser: UserInfo, provider: IMetadataProvider): Promise<RealtimeClientSessionPrepResult>;
|
|
304
|
+
/**
|
|
305
|
+
* Resolves the EFFECTIVE realtime configuration for a co-agent: the agent TYPE's
|
|
306
|
+
* `DefaultConfiguration` (base) ← the agent's `TypeConfiguration` ← the (pre-authorized)
|
|
307
|
+
* runtime overrides — deep-merged per key and normalized. Tolerant end-to-end: malformed
|
|
308
|
+
* layers contribute nothing and an unloaded metadata cache yields no type defaults.
|
|
309
|
+
*
|
|
310
|
+
* @param coAgent The resolved co-agent.
|
|
311
|
+
* @param overridesJson The pre-authorized runtime override layer, when present.
|
|
312
|
+
* @returns The normalized effective configuration (possibly empty, never `null`).
|
|
313
|
+
*/
|
|
314
|
+
protected resolveEffectiveConfig(coAgent: MJAIAgentEntityExtended, overridesJson?: string): RealtimeCoAgentConfig;
|
|
315
|
+
/**
|
|
316
|
+
* Reads the co-agent's TYPE-level `DefaultConfiguration` from {@link AIEngine}'s cached agent
|
|
317
|
+
* types. **Overridable seam**; tolerant — an absent type or unloaded cache returns `null`.
|
|
318
|
+
*/
|
|
319
|
+
protected getAgentTypeDefaultConfiguration(coAgent: MJAIAgentEntityExtended): string | null;
|
|
320
|
+
/**
|
|
321
|
+
* Creates the server-side co-agent observability runs for a voice session: an `AIAgentRun`
|
|
322
|
+
* (Status `Running`), and — when a co-agent system prompt resolved — a linked `AIPromptRun`
|
|
323
|
+
* (Status `Running`, `AgentRunID` = the co-agent run, `AgentID` = the co-agent) plus a single
|
|
324
|
+
* `MJ: AI Agent Run Steps` row (StepType `Prompt`) so the co-agent run's Timeline is non-empty.
|
|
325
|
+
* Delegated target-agent runs nest under the returned `CoAgentRunID` via `ParentRunID`.
|
|
326
|
+
*
|
|
327
|
+
* Best-effort: returns `null` (and logs) when the co-agent run cannot be saved, so callers can
|
|
328
|
+
* continue without observability rather than failing the whole prepare. A failed prompt-run or
|
|
329
|
+
* run-step save just omits that id.
|
|
330
|
+
*
|
|
331
|
+
* @param coAgent The resolved co-agent (its id stamps `AgentID` on both runs).
|
|
332
|
+
* @param promptID The co-agent system prompt id, or `null` to skip the prompt run + run step.
|
|
333
|
+
* @param modelID The resolved realtime model id (stamps the prompt run's `ModelID`).
|
|
334
|
+
* @param userID Optional owning user id for the agent run.
|
|
335
|
+
* @param agentSessionID The session id grouping this voice session's runs.
|
|
336
|
+
* @param contextUser The calling user.
|
|
337
|
+
* @param provider The request-scoped metadata provider.
|
|
338
|
+
* @returns The `{ CoAgentRunID, PromptRunID, CoAgentRunStepID }` ids, or `null` when the agent run failed.
|
|
339
|
+
*/
|
|
340
|
+
protected createCoAgentObservabilityRun(coAgent: MJAIAgentEntityExtended, promptID: string | null, modelID: string, vendorID: string, userID: string | undefined, agentSessionID: string, contextUser: UserInfo, provider: IMetadataProvider, conversationID?: string): Promise<{
|
|
341
|
+
CoAgentRunID: string;
|
|
342
|
+
PromptRunID?: string;
|
|
343
|
+
CoAgentRunStepID?: string;
|
|
344
|
+
} | null>;
|
|
345
|
+
/**
|
|
346
|
+
* Creates the co-agent `AIAgentRun` row (Status `Running`). Returns its id, or `null` (logging
|
|
347
|
+
* `CompleteMessage`) when the save fails.
|
|
348
|
+
*/
|
|
349
|
+
private createCoAgentRun;
|
|
350
|
+
/**
|
|
351
|
+
* Creates the co-agent `AIPromptRun` row (Status `Running`) linked to the co-agent run via
|
|
352
|
+
* `AgentRunID` AND to the co-agent itself via `AgentID` — so the run shows up both on the
|
|
353
|
+
* prompt's run history (`PromptID`) and in agent-scoped prompt-run views. Returns its id, or
|
|
354
|
+
* `null` when `promptID` is absent (skipped) or the save fails (logged).
|
|
355
|
+
*/
|
|
356
|
+
private createCoAgentPromptRun;
|
|
357
|
+
/**
|
|
358
|
+
* Creates the single `MJ: AI Agent Run Steps` row for the co-agent observability run — the
|
|
359
|
+
* realtime session has no iterative loop, so its Timeline carries exactly one step
|
|
360
|
+
* representing the session's system prompt (StepNumber 1, StepType `Prompt`, Status `Running`,
|
|
361
|
+
* `TargetID` = the system `AIPrompt`, `TargetLogID` = the linked `AIPromptRun` when one was
|
|
362
|
+
* created). Skipped (returns `null`) when no system prompt resolved. Best-effort: a save
|
|
363
|
+
* failure is logged and returns `null` — it never breaks the session.
|
|
364
|
+
*/
|
|
365
|
+
private createCoAgentRunStep;
|
|
366
|
+
/**
|
|
367
|
+
* Finalizes the server-side co-agent observability records when a voice session ends. Loads
|
|
368
|
+
* each (when its id is supplied) and, **only if it is still `Running`**, sets it to `Completed`
|
|
369
|
+
* (or `Failed` when `success` is false) with a `CompletedAt` + `Success` stamp. Idempotent and
|
|
370
|
+
* tolerant: a missing/already-finalized record is a no-op; a load/save failure is logged,
|
|
371
|
+
* never thrown.
|
|
372
|
+
*
|
|
373
|
+
* @param coAgentRunID The co-agent run id, or `null` to skip.
|
|
374
|
+
* @param promptRunID The co-agent prompt run id, or `null` to skip.
|
|
375
|
+
* @param contextUser The calling user.
|
|
376
|
+
* @param provider The request-scoped metadata provider.
|
|
377
|
+
* @param success Whether the session ended successfully (controls Completed vs Failed).
|
|
378
|
+
* @param coAgentRunStepID The co-agent run's single `MJ: AI Agent Run Steps` row id, or `null` to skip.
|
|
379
|
+
*/
|
|
380
|
+
FinalizeCoAgentRun(coAgentRunID: string | null, promptRunID: string | null, contextUser: UserInfo, provider: IMetadataProvider, success?: boolean, coAgentRunStepID?: string | null): Promise<void>;
|
|
381
|
+
/** Loads + finalizes the co-agent `AIAgentRun` if still `Running`. Tolerant: logs, never throws. */
|
|
382
|
+
private finalizeAgentRun;
|
|
383
|
+
/**
|
|
384
|
+
* Loads + finalizes the co-agent run's single system-prompt `MJ: AI Agent Run Steps` row if
|
|
385
|
+
* still `Running` (Status `Completed`/`Failed`, `CompletedAt`, `Success`). Tolerant: a
|
|
386
|
+
* missing/already-finalized step is a no-op; a load/save failure is logged, never thrown.
|
|
387
|
+
*/
|
|
388
|
+
private finalizeRunStep;
|
|
389
|
+
/** Loads + finalizes the co-agent `AIPromptRun` if still `Running`. Tolerant: logs, never throws. */
|
|
390
|
+
private finalizePromptRun;
|
|
391
|
+
/**
|
|
392
|
+
* Appends (or replaces) one transcript turn onto the co-agent's long-lived `AIPromptRun.Messages`,
|
|
393
|
+
* so the realtime co-agent's conversation is captured on its run exactly like every other MJ agent
|
|
394
|
+
* run — closing the observability gap where the run held only token totals, never the turns. The
|
|
395
|
+
* run viewer can then show what the co-agent heard and said. Mirrors {@link accumulatePromptRunUsage}'s
|
|
396
|
+
* load/append/save pattern; best-effort and tolerant (logs, never throws).
|
|
397
|
+
*
|
|
398
|
+
* `replacePrevious` swaps the last same-role message instead of appending — the streaming-correction
|
|
399
|
+
* case (an interim assistant turn finalized into its full text). The stored shape is the standard
|
|
400
|
+
* chat-message array (`[{ role, content }, …]`) the rest of MJ already reads from `Messages`.
|
|
401
|
+
*
|
|
402
|
+
* NOTE: load-append-save carries the same benign race as usage accumulation; realtime turns are
|
|
403
|
+
* sequential per session so collisions are rare. A dedicated child turn-row entity would remove the
|
|
404
|
+
* race (and the blob rewrite) entirely — a future increment. Tool-call turns (the browser_ and
|
|
405
|
+
* Whiteboard_ channel tools) are a separate increment that requires the client to relay them.
|
|
406
|
+
*
|
|
407
|
+
* @returns `true` when the turn was persisted onto the prompt run.
|
|
408
|
+
*/
|
|
409
|
+
AppendPromptRunMessage(promptRunID: string, role: 'user' | 'assistant' | 'system', content: string, replacePrevious: boolean, contextUser: UserInfo, provider: IMetadataProvider): Promise<boolean>;
|
|
410
|
+
/**
|
|
411
|
+
* Accumulates relayed usage DELTAS onto the co-agent `AIPromptRun`'s `TokensPrompt` / `TokensCompletion`
|
|
412
|
+
* (recomputing `TokensUsed`). Serialized against {@link AppendPromptRunMessage} on the same run so the
|
|
413
|
+
* high-frequency usage checkpoint never overwrites freshly-appended transcript turns (and vice-versa).
|
|
414
|
+
* Best-effort: load/save failures log and return `false`, never throw.
|
|
415
|
+
*
|
|
416
|
+
* @param promptRunID The co-agent observability prompt run.
|
|
417
|
+
* @param inputDelta Input-token delta to add (caller clamps to >= 0).
|
|
418
|
+
* @param outputDelta Output-token delta to add (caller clamps to >= 0).
|
|
419
|
+
* @returns `true` when the accumulated usage was persisted.
|
|
420
|
+
*/
|
|
421
|
+
AccumulatePromptRunUsage(promptRunID: string, inputDelta: number, outputDelta: number, contextUser: UserInfo, provider: IMetadataProvider): Promise<boolean>;
|
|
422
|
+
/** Parses the prompt run's `Messages` JSON into a mutable chat-message array (tolerant: `[]` on empty/malformed). */
|
|
423
|
+
private parsePromptRunMessages;
|
|
424
|
+
/**
|
|
425
|
+
* Executes a single tool call relayed from the browser and returns its serialized result.
|
|
426
|
+
*
|
|
427
|
+
* Builds a {@link RealtimeToolBroker} whose `DelegateToTarget` runs the target agent (threading
|
|
428
|
+
* the abort signal, parent run, and session id) and whose `ExecuteTool` returns a structured
|
|
429
|
+
* "not available" result for non-target tools (action wiring is a later phase). The broker
|
|
430
|
+
* routes the call and always resolves with structured JSON — failures become `tool_response`
|
|
431
|
+
* errors the model can narrate rather than thrown exceptions.
|
|
432
|
+
*
|
|
433
|
+
* @param input The relayed tool call plus delegation linkage.
|
|
434
|
+
* @param contextUser The calling user (threaded into the delegated agent run).
|
|
435
|
+
* @param provider The request-scoped metadata provider (threaded into the delegated agent run).
|
|
436
|
+
* @returns `{ ResultJson, Success, PausedRunID?, Artifacts? }` — the serialized tool result for
|
|
437
|
+
* the browser to relay back, the paused run id when the delegated target agent paused awaiting
|
|
438
|
+
* feedback (so the resolver can persist it and resume that run on the next answer), and the
|
|
439
|
+
* artifacts the delegated run produced (so the resolver can junction-link them into the
|
|
440
|
+
* session's conversation history — the same info is embedded in `ResultJson` for the client).
|
|
441
|
+
*/
|
|
442
|
+
ExecuteRelayedTool(input: ExecuteRelayedToolInput, contextUser: UserInfo, provider: IMetadataProvider): Promise<{
|
|
443
|
+
ResultJson: string;
|
|
444
|
+
Success: boolean;
|
|
445
|
+
PausedRunID?: string;
|
|
446
|
+
Artifacts?: DelegatedRunArtifact[];
|
|
447
|
+
}>;
|
|
448
|
+
/**
|
|
449
|
+
* Aborts in-flight relayed delegations for a session — the server half of the client-direct
|
|
450
|
+
* CANCEL channel (see the registry note on {@link inFlightDelegations}).
|
|
451
|
+
*
|
|
452
|
+
* @param agentSessionID The session whose in-flight delegations to abort.
|
|
453
|
+
* @param callID When supplied, only the delegation for this specific call is aborted; when
|
|
454
|
+
* omitted, EVERY in-flight delegation for the session is aborted.
|
|
455
|
+
* @returns The number of in-flight delegations aborted. **Tolerant by design**: an unknown
|
|
456
|
+
* session, an unknown call id, or a session with nothing in flight returns `0` — never throws
|
|
457
|
+
* (the call the user wanted dead may simply have finished already, which is a fine outcome).
|
|
458
|
+
*/
|
|
459
|
+
CancelInFlightDelegations(agentSessionID: string, callID?: string): number;
|
|
460
|
+
/** Normalized (trim + lowercase) registry key so UUID casing differences can't split entries. */
|
|
461
|
+
private registryKey;
|
|
462
|
+
/** Creates + registers the abort controller for one in-flight relayed call. */
|
|
463
|
+
private registerInFlightDelegation;
|
|
464
|
+
/**
|
|
465
|
+
* Removes one call's registry entry on completion — but only when the stored controller is
|
|
466
|
+
* STILL the one this execution registered (a cancel may already have removed it, and a
|
|
467
|
+
* same-callID retry may have replaced it).
|
|
468
|
+
*/
|
|
469
|
+
private unregisterInFlightDelegation;
|
|
470
|
+
/**
|
|
471
|
+
* Ensures {@link AIEngine} metadata is loaded before resolution. **Overridable seam** so tests
|
|
472
|
+
* can skip the DB-backed config load.
|
|
473
|
+
*
|
|
474
|
+
* @param contextUser The calling user.
|
|
475
|
+
* @param provider The request-scoped metadata provider.
|
|
476
|
+
*/
|
|
477
|
+
protected configureEngine(contextUser: UserInfo, provider: IMetadataProvider): Promise<void>;
|
|
478
|
+
/**
|
|
479
|
+
* Resolves the co-agent from either the supplied entity or its id (from cached metadata).
|
|
480
|
+
*
|
|
481
|
+
* @param input The prepare-session input.
|
|
482
|
+
* @returns The co-agent entity, or `null` when neither form resolves.
|
|
483
|
+
*/
|
|
484
|
+
protected resolveCoAgent(input: PrepareClientSessionInput): MJAIAgentEntityExtended | null;
|
|
485
|
+
/**
|
|
486
|
+
* Resolves the realtime model for a session, honoring an explicit user choice when present.
|
|
487
|
+
*
|
|
488
|
+
* - With {@link PrepareClientSessionInput.PreferredModelID}: resolve THAT model strictly via
|
|
489
|
+
* {@link resolvePreferredRealtimeModel} — failures return a specific reason and never fall
|
|
490
|
+
* back to another model (the user explicitly chose). (The transport layer has already
|
|
491
|
+
* authorization-gated a deviating explicit choice.)
|
|
492
|
+
* - Else, with an effective-config `realtime.modelPreference` (name or id): resolve it via
|
|
493
|
+
* {@link resolveConfiguredModelPreference}. METADATA preferences degrade gracefully — an
|
|
494
|
+
* unsatisfiable preference logs and FALLS THROUGH to the default (mirroring the co-agent
|
|
495
|
+
* resolution chain's tolerant metadata steps), it never breaks calls.
|
|
496
|
+
* - Without either: the existing default behavior via {@link resolveRealtimeModel}
|
|
497
|
+
* (highest-PowerRank active Realtime model), with the generic {@link noModelMessage} on failure.
|
|
498
|
+
*
|
|
499
|
+
* @param input The prepare-session input (carries the optional preferred model id).
|
|
500
|
+
* @param coAgent The resolved co-agent (threaded to the default-resolution seam).
|
|
501
|
+
* @param effectiveConfig The resolved effective configuration (carries `modelPreference`).
|
|
502
|
+
* @returns The resolution outcome (resolution or failure reason).
|
|
503
|
+
*/
|
|
504
|
+
protected resolveModelForSession(input: PrepareClientSessionInput, coAgent: MJAIAgentEntityExtended, effectiveConfig?: RealtimeCoAgentConfig): Promise<RealtimeModelResolutionOutcome>;
|
|
505
|
+
/**
|
|
506
|
+
* Resolves the effective config's `realtime.modelPreference` (an `MJ: AI Models` Name OR ID)
|
|
507
|
+
* into a usable realtime model. TOLERANT by design — this is a METADATA preference, so any
|
|
508
|
+
* failure (unknown model, inactive, wrong type, no vendor/key) logs a warning and returns
|
|
509
|
+
* `null`, falling through to the default highest-PowerRank resolution. Contrast with the
|
|
510
|
+
* explicit runtime choice ({@link resolvePreferredRealtimeModel}), which fails loud.
|
|
511
|
+
*
|
|
512
|
+
* @param effectiveConfig The resolved effective configuration.
|
|
513
|
+
* @returns The resolution, or `null` when no preference is configured or it can't be satisfied.
|
|
514
|
+
*/
|
|
515
|
+
protected resolveConfiguredModelPreference(effectiveConfig?: RealtimeCoAgentConfig): RealtimeModelResolution | null;
|
|
516
|
+
/**
|
|
517
|
+
* Looks up a model by ID (UUID-insensitive) or, failing that, by case/whitespace-insensitive
|
|
518
|
+
* Name in {@link AIEngine}'s cached models. **Overridable seam**; tolerant of an unloaded cache.
|
|
519
|
+
*
|
|
520
|
+
* @param preference The `MJ: AI Models` ID or Name.
|
|
521
|
+
* @returns The model entity, or `null`.
|
|
522
|
+
*/
|
|
523
|
+
protected findModelByIDOrName(preference: string): MJAIModelEntityExtended | null;
|
|
524
|
+
/**
|
|
525
|
+
* Strictly resolves an EXPLICITLY requested realtime model. Each precondition failure returns
|
|
526
|
+
* a clear, user-facing reason naming the model — there is NO fallback to another model, because
|
|
527
|
+
* the caller's user explicitly chose this one.
|
|
528
|
+
*
|
|
529
|
+
* @param preferredModelID The `MJ: AI Models.ID` the user chose.
|
|
530
|
+
* @returns The resolution outcome (resolution or a specific failure reason).
|
|
531
|
+
*/
|
|
532
|
+
protected resolvePreferredRealtimeModel(preferredModelID: string): RealtimeModelResolutionOutcome;
|
|
533
|
+
/**
|
|
534
|
+
* Looks up a model by id in {@link AIEngine}'s cached models. **Overridable seam** for tests.
|
|
535
|
+
*
|
|
536
|
+
* @param modelID The `MJ: AI Models.ID` to find.
|
|
537
|
+
* @returns The model entity, or `null` when not present.
|
|
538
|
+
*/
|
|
539
|
+
protected findModelByID(modelID: string): MJAIModelEntityExtended | null;
|
|
540
|
+
/** True when the model's denormalized `AIModelType` name is `Realtime` (case/whitespace-insensitive). */
|
|
541
|
+
private isRealtimeModel;
|
|
542
|
+
/**
|
|
543
|
+
* Resolves the Realtime model + vendor driver + API key, mirroring `BaseAgent`'s server-bridged
|
|
544
|
+
* resolution: highest-power active model of AIModelType `Realtime`; highest-priority active
|
|
545
|
+
* vendor whose `DriverClass` has a resolvable API key; instantiated via the `ClassFactory`.
|
|
546
|
+
*
|
|
547
|
+
* **Overridable seam.** Test subclasses override this to return a mock model so the service can
|
|
548
|
+
* be exercised without provider SDKs or DB metadata. Returns `null` (never throws) when any
|
|
549
|
+
* step can't be satisfied.
|
|
550
|
+
*
|
|
551
|
+
* @param coAgent The co-agent being voiced (reserved for future per-agent model preference).
|
|
552
|
+
* @returns The resolved model + identifiers, or `null`.
|
|
553
|
+
*/
|
|
554
|
+
protected resolveRealtimeModel(coAgent: MJAIAgentEntityExtended): Promise<RealtimeModelResolution | null>;
|
|
555
|
+
/**
|
|
556
|
+
* Shared tail of model resolution: picks the vendor (with a usable API key) for an
|
|
557
|
+
* already-chosen model entity and instantiates its realtime driver.
|
|
558
|
+
*
|
|
559
|
+
* @param model The chosen model entity.
|
|
560
|
+
* @returns The full resolution, or `null` when no vendor/key/driver can be satisfied.
|
|
561
|
+
*/
|
|
562
|
+
protected resolveVendorAndInstantiate(model: MJAIModelEntityExtended): RealtimeModelResolution | null;
|
|
563
|
+
/**
|
|
564
|
+
* Resolves the API key for a vendor driver class. **Overridable seam** (wraps the module-level
|
|
565
|
+
* {@link GetAIAPIKey}) so tests can simulate present/absent keys without environment setup.
|
|
566
|
+
*
|
|
567
|
+
* @param driverClass The vendor's `DriverClass`.
|
|
568
|
+
* @returns The API key, or a falsy value when none is configured.
|
|
569
|
+
*/
|
|
570
|
+
protected getAPIKeyForDriver(driverClass: string): string | undefined;
|
|
571
|
+
/**
|
|
572
|
+
* Instantiates the realtime driver for a vendor driver class via the ClassFactory.
|
|
573
|
+
* **Overridable seam** so tests can return a mock driver.
|
|
574
|
+
*
|
|
575
|
+
* @param driverClass The vendor's `DriverClass` (the ClassFactory key).
|
|
576
|
+
* @param apiKey The resolved API key (constructor argument).
|
|
577
|
+
* @returns The driver instance, or `null` when the factory cannot create one.
|
|
578
|
+
*/
|
|
579
|
+
protected createModelInstance(driverClass: string, apiKey: string): BaseRealtimeModel | null;
|
|
580
|
+
/**
|
|
581
|
+
* The active models of AIModelType `Realtime`, sorted highest-PowerRank first — the candidate
|
|
582
|
+
* list {@link resolveRealtimeModel} walks until one yields a usable client-direct driver.
|
|
583
|
+
* Returns ALL candidates (not just the top pick) so a keyless or non-client-direct top model
|
|
584
|
+
* falls through to the next usable one instead of dead-ending the whole resolution.
|
|
585
|
+
*
|
|
586
|
+
* @param coAgent The co-agent (reserved for future per-agent model preference).
|
|
587
|
+
* @returns The candidate models in resolution order (empty array when none are active).
|
|
588
|
+
*/
|
|
589
|
+
private selectRealtimeModelCandidates;
|
|
590
|
+
/**
|
|
591
|
+
* Selects the highest-priority active vendor for a model whose `DriverClass` has a resolvable
|
|
592
|
+
* API key. Mirrors `BaseAgent.selectRealtimeVendor`.
|
|
593
|
+
*
|
|
594
|
+
* @param modelID The chosen model's id.
|
|
595
|
+
* @returns The vendor driver/api identifiers, or `null` when none has a usable key.
|
|
596
|
+
*/
|
|
597
|
+
protected selectRealtimeVendor(modelID: string): {
|
|
598
|
+
VendorID: string;
|
|
599
|
+
DriverClass: string;
|
|
600
|
+
APIName: string;
|
|
601
|
+
} | null;
|
|
602
|
+
/**
|
|
603
|
+
* Resolves the DB-driven progress-narration instruction template: the Active `MJ: AI Prompts`
|
|
604
|
+
* row named {@link RealtimeClientSessionService.NarrationPromptName}, read from
|
|
605
|
+
* {@link AIEngine}'s cached prompts. When the current name is absent, falls back to the
|
|
606
|
+
* DEPRECATED {@link RealtimeClientSessionService.LegacyNarrationPromptName} (pre-rename seed)
|
|
607
|
+
* with a deprecation log. **Tolerant**: returns `null` (never throws) when neither prompt is
|
|
608
|
+
* present, the text is empty, or the engine cache is unavailable — clients fall back to their
|
|
609
|
+
* built-in narration instruction text.
|
|
610
|
+
*
|
|
611
|
+
* @returns The template text (containing a `{{ progressMessage }}` placeholder), or `null`.
|
|
612
|
+
*/
|
|
613
|
+
protected resolveNarrationInstructionsTemplate(): string | null;
|
|
614
|
+
/**
|
|
615
|
+
* Builds the {@link RealtimeSessionParams} for the client-direct session: the companion system
|
|
616
|
+
* prompt plus the stable, target-independent tool set.
|
|
617
|
+
*
|
|
618
|
+
* @param input The prepare-session input.
|
|
619
|
+
* @param coAgent The resolved co-agent.
|
|
620
|
+
* @param modelApiName The vendor API name of the resolved realtime model.
|
|
621
|
+
* @param contextUser The calling user.
|
|
622
|
+
* @param provider The request-scoped metadata provider.
|
|
623
|
+
* @param effectiveConfig The resolved effective configuration (voice persona + provider settings).
|
|
624
|
+
* @param driverClass The resolved vendor's DriverClass — matches per-provider voice settings.
|
|
625
|
+
* @returns The assembled session params.
|
|
626
|
+
*/
|
|
627
|
+
protected buildSessionParams(input: PrepareClientSessionInput, coAgent: MJAIAgentEntityExtended, modelApiName: string, contextUser: UserInfo, provider: IMetadataProvider, effectiveConfig?: RealtimeCoAgentConfig, driverClass?: string): Promise<RealtimeSessionParams>;
|
|
628
|
+
/**
|
|
629
|
+
* Builds the provider-pact `Config` bag for the session: the effective config's matching
|
|
630
|
+
* per-provider voice settings (`realtime.voice.providers.<provider>`) merged UNDER any
|
|
631
|
+
* caller-supplied {@link PrepareClientSessionInput.Config} (the runtime bag wins per key).
|
|
632
|
+
* The settings objects are OPAQUE driver pacts — each server driver consumes its own keys
|
|
633
|
+
* exactly as it consumes any other entry of the open config bag (OpenAI spreads it into
|
|
634
|
+
* `session.update`, AssemblyAI reads `voice`, Gemini merges it last). Returns the original
|
|
635
|
+
* `input.Config` (possibly `undefined`) when no provider settings match, preserving the
|
|
636
|
+
* pre-config behavior byte-for-byte.
|
|
637
|
+
*
|
|
638
|
+
* @param input The prepare-session input (carries the runtime config bag).
|
|
639
|
+
* @param effectiveConfig The resolved effective configuration.
|
|
640
|
+
* @param driverClass The resolved vendor's DriverClass.
|
|
641
|
+
* @returns The merged config bag, or `undefined` when nothing contributes.
|
|
642
|
+
*/
|
|
643
|
+
protected buildSessionConfigBag(input: PrepareClientSessionInput, effectiveConfig?: RealtimeCoAgentConfig, driverClass?: string): JSONObject | undefined;
|
|
644
|
+
/**
|
|
645
|
+
* Assembles the companion system prompt: the framing ("you are the voice for the target"), the
|
|
646
|
+
* co-agent's own system prompt text, the TARGET agent's identity/capabilities (Name +
|
|
647
|
+
* Description), the conversation history, and the same memory/context a loop agent assembles.
|
|
648
|
+
*
|
|
649
|
+
* When the effective configuration carries a voice persona (`realtime.voice.default`), a
|
|
650
|
+
* short "Voice & manner" section (tone / speaking style) is appended after the co-agent's
|
|
651
|
+
* own prompt so the model speaks in the configured manner.
|
|
652
|
+
*
|
|
653
|
+
* @param input The prepare-session input.
|
|
654
|
+
* @param coAgent The resolved co-agent.
|
|
655
|
+
* @param contextUser The calling user.
|
|
656
|
+
* @param provider The request-scoped metadata provider.
|
|
657
|
+
* @param effectiveConfig The resolved effective configuration (voice persona source).
|
|
658
|
+
* @returns The concatenated system prompt (never empty — the framing is always present).
|
|
659
|
+
*/
|
|
660
|
+
protected buildCompanionSystemPrompt(input: PrepareClientSessionInput, coAgent: MJAIAgentEntityExtended, contextUser: UserInfo, provider: IMetadataProvider, effectiveConfig?: RealtimeCoAgentConfig): Promise<string>;
|
|
661
|
+
/**
|
|
662
|
+
* Builds the "interactive-surface tools" exception clause appended to the co-agent framing when
|
|
663
|
+
* the client supplied channel tools (browser_*, Whiteboard_*, …) as ExtraTools. Without it the
|
|
664
|
+
* co-agent — told to route ALL work through invoke-target-agent — delegates browser/whiteboard
|
|
665
|
+
* requests to the target agent (which has no live channel of its own) instead of driving the
|
|
666
|
+
* surface itself, then hallucinates a "missing session id". The tools ARE already in its set
|
|
667
|
+
* ({@link buildStableToolSet} merges `[invokeTarget, ...extraTools]`); this clause tells the model
|
|
668
|
+
* to USE them directly. Returns empty for pure-voice sessions (no ExtraTools), keeping that
|
|
669
|
+
* framing untouched. Generic by design — it names browser_ and Whiteboard_ tools only as
|
|
670
|
+
* examples, so any future client channel is covered automatically.
|
|
671
|
+
*
|
|
672
|
+
* @param extraTools The client-supplied channel tools, when any.
|
|
673
|
+
* @returns The exception clause (leading space included), or '' when there are no extra tools.
|
|
674
|
+
*/
|
|
675
|
+
protected buildInteractiveSurfaceFraming(extraTools?: RealtimeToolDefinition[]): string;
|
|
676
|
+
/**
|
|
677
|
+
* Frames the prior-leg transcript (when a session resumes via `lastSessionId`) as a clearly
|
|
678
|
+
* labeled PRIOR-CONVERSATION section of the system prompt, so the model REMEMBERS the last
|
|
679
|
+
* live session rather than greeting the user cold. The transport layer supplies the
|
|
680
|
+
* already-capped, role-tagged lines (see {@link PrepareClientSessionInput.PriorTranscript});
|
|
681
|
+
* this method only adds the framing. Empty/whitespace input yields an empty section.
|
|
682
|
+
*
|
|
683
|
+
* @param priorTranscript The role-tagged transcript lines, or undefined.
|
|
684
|
+
* @returns The framed section, or empty string when there is nothing to frame.
|
|
685
|
+
*/
|
|
686
|
+
private formatPriorTranscript;
|
|
687
|
+
/**
|
|
688
|
+
* Resolves the target agent entity from cached metadata.
|
|
689
|
+
*
|
|
690
|
+
* @param targetAgentID The target agent id.
|
|
691
|
+
* @returns The target agent entity, or `null` when not found.
|
|
692
|
+
*/
|
|
693
|
+
protected resolveTargetAgent(targetAgentID: string): MJAIAgentEntityExtended | null;
|
|
694
|
+
/**
|
|
695
|
+
* Reads the co-agent's own system prompt text from its highest-priority active agent prompt,
|
|
696
|
+
* mirroring `BaseAgent.loadAgentConfiguration`'s child-prompt resolution.
|
|
697
|
+
*
|
|
698
|
+
* @param coAgent The resolved co-agent.
|
|
699
|
+
* @returns The co-agent's system prompt template text, or empty string when none is configured.
|
|
700
|
+
*/
|
|
701
|
+
protected getCoAgentSystemPromptText(coAgent: MJAIAgentEntityExtended): string;
|
|
702
|
+
/**
|
|
703
|
+
* Resolves the co-agent's highest-priority active system prompt, returning both its template
|
|
704
|
+
* text and its prompt id. The id is surfaced so {@link PrepareClientSession} can create a linked
|
|
705
|
+
* co-agent `AIPromptRun` for observability. Mirrors `BaseAgent.loadAgentConfiguration`'s
|
|
706
|
+
* child-prompt resolution.
|
|
707
|
+
*
|
|
708
|
+
* @param coAgent The resolved co-agent.
|
|
709
|
+
* @returns The prompt text + id, or `{ Text: '', PromptID: null }` when none is configured.
|
|
710
|
+
*/
|
|
711
|
+
protected resolveCoAgentSystemPrompt(coAgent: MJAIAgentEntityExtended): CoAgentSystemPromptResolution;
|
|
712
|
+
/**
|
|
713
|
+
* Formats the target agent's identity + capabilities block for the system prompt.
|
|
714
|
+
*
|
|
715
|
+
* @param target The target agent, or `null`.
|
|
716
|
+
* @returns The formatted block, or empty string when no target resolved.
|
|
717
|
+
*/
|
|
718
|
+
private formatTargetIdentity;
|
|
719
|
+
/**
|
|
720
|
+
* Formats prior conversation history as a plain-text block for the system prompt.
|
|
721
|
+
*
|
|
722
|
+
* @param messages The conversation messages, or undefined.
|
|
723
|
+
* @returns The formatted history block, or empty string when there is none.
|
|
724
|
+
*/
|
|
725
|
+
private formatConversationHistory;
|
|
726
|
+
/**
|
|
727
|
+
* Assembles the same memory/context block a loop agent injects, reusing
|
|
728
|
+
* {@link AgentMemoryContextBuilder} so there is no duplicated retrieval logic. The builder
|
|
729
|
+
* unshifts a system message onto a throwaway array, which we pull back out as plain text.
|
|
730
|
+
*
|
|
731
|
+
* @param input The prepare-session input.
|
|
732
|
+
* @param coAgent The resolved co-agent.
|
|
733
|
+
* @param contextUser The calling user.
|
|
734
|
+
* @returns The concatenated context text (empty string when nothing was injected).
|
|
735
|
+
*/
|
|
736
|
+
protected assembleMemoryContext(input: PrepareClientSessionInput, coAgent: MJAIAgentEntityExtended, contextUser: UserInfo): Promise<string>;
|
|
737
|
+
/**
|
|
738
|
+
* Builds the stable, target-independent tool set every voice session exposes: the single
|
|
739
|
+
* `invoke-target-agent` tool plus any caller-supplied extra tools. The target is a runtime
|
|
740
|
+
* argument *inside* the call, never a per-target tool — this keeps the provider contract
|
|
741
|
+
* identical across targets.
|
|
742
|
+
*
|
|
743
|
+
* @param extraTools Optional additional target-independent tools.
|
|
744
|
+
* @returns The tools to register at session start.
|
|
745
|
+
*/
|
|
746
|
+
protected buildStableToolSet(extraTools?: RealtimeToolDefinition[]): RealtimeToolDefinition[];
|
|
747
|
+
/**
|
|
748
|
+
* Builds the {@link RealtimeToolBroker} for a relayed tool call, wiring `DelegateToTarget` to a
|
|
749
|
+
* target-agent run and `ExecuteTool` to a structured "not available" placeholder.
|
|
750
|
+
*
|
|
751
|
+
* @param input The relayed tool input.
|
|
752
|
+
* @param contextUser The calling user.
|
|
753
|
+
* @param provider The request-scoped metadata provider.
|
|
754
|
+
* @returns The constructed broker.
|
|
755
|
+
*/
|
|
756
|
+
protected buildToolBroker(input: ExecuteRelayedToolInput, contextUser: UserInfo, provider: IMetadataProvider): RealtimeToolBroker;
|
|
757
|
+
/**
|
|
758
|
+
* Delegates an `invoke-target-agent` call to the target agent via {@link AgentRunner.RunAgent}.
|
|
759
|
+
*
|
|
760
|
+
* Threads the broker-owned abort signal (combined with any caller signal) into the child run's
|
|
761
|
+
* `cancellationToken`, links the child run to the co-agent run via `parentRunID`, and propagates
|
|
762
|
+
* `agentSessionID` so both runs group under the same session. Mirrors
|
|
763
|
+
* `BaseAgent.delegateRealtimeToTarget`.
|
|
764
|
+
*
|
|
765
|
+
* @param input The relayed tool input (target id + linkage).
|
|
766
|
+
* @param request The broker's delegation request (call id + arguments + abort signal).
|
|
767
|
+
* @param contextUser The calling user.
|
|
768
|
+
* @param provider The request-scoped metadata provider.
|
|
769
|
+
* @returns The delegated result for the model's tool_response.
|
|
770
|
+
*/
|
|
771
|
+
protected delegateToTarget(input: ExecuteRelayedToolInput, request: DelegateToTargetRequest, contextUser: UserInfo, provider: IMetadataProvider): Promise<DelegatedResult>;
|
|
772
|
+
/**
|
|
773
|
+
* Creates artifact(s) from a completed delegated run's payload — the voice-path equivalent of
|
|
774
|
+
* the chat path's artifact step in `AgentRunner.RunAgentInConversation`. Delegated voice runs
|
|
775
|
+
* execute via `AgentRunner.RunAgent` directly (no conversation detail), so without this step
|
|
776
|
+
* they would never produce artifacts at all.
|
|
777
|
+
*
|
|
778
|
+
* Eligibility guards (all must hold, mirroring the chat path's `processArtifacts`):
|
|
779
|
+
* - the run succeeded and did NOT pause awaiting feedback (a paused run has no deliverable yet);
|
|
780
|
+
* - the run returned a non-empty payload.
|
|
781
|
+
*
|
|
782
|
+
* The DB work is delegated to {@link processRunArtifacts} (an overridable seam), which reuses
|
|
783
|
+
* `AgentRunner.ProcessAgentArtifacts` — so ArtifactCreationMode, DefaultArtifactTypeID,
|
|
784
|
+
* name extraction, and duplicate-version dedup all behave exactly as in chat. **Best-effort:**
|
|
785
|
+
* any failure is logged and returns `undefined`; artifact surfacing never fails the delegation.
|
|
786
|
+
*
|
|
787
|
+
* @param result The delegated agent execution result.
|
|
788
|
+
* @param contextUser The calling user.
|
|
789
|
+
* @param provider The request-scoped metadata provider.
|
|
790
|
+
* @returns The produced artifact descriptor(s), or `undefined` when none were created.
|
|
791
|
+
*/
|
|
792
|
+
protected createDelegatedRunArtifacts(result: ExecuteAgentResult, contextUser: UserInfo, provider: IMetadataProvider): Promise<DelegatedRunArtifact[] | undefined>;
|
|
793
|
+
/**
|
|
794
|
+
* The DB-backed artifact-creation seam: runs `AgentRunner.ProcessAgentArtifacts` WITHOUT a
|
|
795
|
+
* conversation detail (the voice path has none — the artifact + version are created and the
|
|
796
|
+
* junction link is skipped), then loads the artifact header for its display name.
|
|
797
|
+
*
|
|
798
|
+
* Artifacts whose Visibility resolved to `System Only` (the agent's ArtifactCreationMode) are
|
|
799
|
+
* created but NOT surfaced to the overlay — matching how chat hides them from users.
|
|
800
|
+
*
|
|
801
|
+
* **Overridable seam** so tests can exercise {@link createDelegatedRunArtifacts}' eligibility
|
|
802
|
+
* guards without a DB.
|
|
803
|
+
*
|
|
804
|
+
* @param result The delegated agent execution result (payload + agentRun).
|
|
805
|
+
* @param contextUser The calling user.
|
|
806
|
+
* @param provider The request-scoped metadata provider.
|
|
807
|
+
* @returns The produced artifact descriptor(s), or `undefined`.
|
|
808
|
+
*/
|
|
809
|
+
protected processRunArtifacts(result: ExecuteAgentResult, contextUser: UserInfo, provider: IMetadataProvider): Promise<DelegatedRunArtifact[] | undefined>;
|
|
810
|
+
/**
|
|
811
|
+
* Runs (or resumes) the target agent for a delegation. Threads the combined abort signal, parent
|
|
812
|
+
* run linkage, session id, and the `OnProgress` callback so the resolver can stream progress.
|
|
813
|
+
* When {@link ExecuteRelayedToolInput.ResumeRunID} is set, resumes that paused run via
|
|
814
|
+
* `lastRunId` + `autoPopulateLastRunPayload` (the user's answer continues the same interactive
|
|
815
|
+
* run) instead of starting fresh.
|
|
816
|
+
*
|
|
817
|
+
* @param input The relayed tool input (linkage, progress callback, optional resume id).
|
|
818
|
+
* @param request The broker's delegation request (call id + arguments + abort signal).
|
|
819
|
+
* @param target The resolved target agent.
|
|
820
|
+
* @param contextUser The calling user.
|
|
821
|
+
* @param provider The request-scoped metadata provider.
|
|
822
|
+
* @returns The agent execution result.
|
|
823
|
+
*/
|
|
824
|
+
private runDelegatedAgent;
|
|
825
|
+
/**
|
|
826
|
+
* Maps an {@link ExecuteAgentResult} onto the broker's {@link DelegatedResult}, special-casing a
|
|
827
|
+
* run that paused awaiting feedback. An `AwaitingFeedback` run is a valid intermediate outcome,
|
|
828
|
+
* not an error: we return its clarifying QUESTION (the run's `Message`) as the tool Output —
|
|
829
|
+
* phrased so the realtime model relays it as a question to the user — set `Success: true`, and
|
|
830
|
+
* surface the paused run id so the resolver can resume that run on the user's next answer.
|
|
831
|
+
*
|
|
832
|
+
* @param callID The provider call id this result corresponds to.
|
|
833
|
+
* @param result The agent execution result.
|
|
834
|
+
* @param artifacts Artifacts the run produced (from {@link createDelegatedRunArtifacts}),
|
|
835
|
+
* threaded into the result so the broker serializes them for the call overlay.
|
|
836
|
+
* @returns The delegated result for the model's tool_response.
|
|
837
|
+
*/
|
|
838
|
+
private buildDelegatedResult;
|
|
839
|
+
/**
|
|
840
|
+
* Loads the co-agent run entity behind {@link ExecuteRelayedToolInput.ParentRunID} so the
|
|
841
|
+
* delegated run can link to it via `parentRun` (→ `ParentRunID`). Returns `null` when no id was
|
|
842
|
+
* supplied or the run cannot be loaded (delegation proceeds without parent linkage rather than
|
|
843
|
+
* failing the whole call).
|
|
844
|
+
*
|
|
845
|
+
* @param parentRunID The co-agent run id, or undefined.
|
|
846
|
+
* @param contextUser The calling user.
|
|
847
|
+
* @param provider The request-scoped metadata provider.
|
|
848
|
+
* @returns The loaded parent run entity, or `null`.
|
|
849
|
+
*/
|
|
850
|
+
protected loadParentRun(parentRunID: string | undefined, contextUser: UserInfo, provider: IMetadataProvider): Promise<MJAIAgentRunEntityExtended | null>;
|
|
851
|
+
/**
|
|
852
|
+
* Routes a non-target tool call. For now this returns a structured "not available" result —
|
|
853
|
+
* the richer client/UI/action routing is wired in a later phase. Documented minimal seam.
|
|
854
|
+
*
|
|
855
|
+
* @param call The non-target tool call.
|
|
856
|
+
* @returns A failed {@link ToolExecutionResult} the model can narrate.
|
|
857
|
+
*/
|
|
858
|
+
protected executeNonTargetTool(call: RealtimeToolCall): Promise<ToolExecutionResult>;
|
|
859
|
+
/**
|
|
860
|
+
* Parses the natural-language request text out of an `invoke-target-agent` call's arguments.
|
|
861
|
+
* Falls back to the raw argument string when it is not the expected `{ request: string }` JSON.
|
|
862
|
+
*
|
|
863
|
+
* @param argumentsJson The raw arguments string emitted by the model.
|
|
864
|
+
* @returns The request text to hand to the target agent.
|
|
865
|
+
*/
|
|
866
|
+
private parseDelegateRequestText;
|
|
867
|
+
/**
|
|
868
|
+
* Combines the broker's per-call abort signal with an optional caller-supplied signal so either
|
|
869
|
+
* source can cancel the delegated run. Returns the broker signal alone when no caller signal is
|
|
870
|
+
* present (the common case), avoiding an unnecessary controller.
|
|
871
|
+
*
|
|
872
|
+
* @param brokerSignal The broker-owned per-call abort signal (always present).
|
|
873
|
+
* @param callerSignal An optional caller signal (e.g. a request-scoped barge-in).
|
|
874
|
+
* @returns A single abort signal that fires when either source aborts.
|
|
875
|
+
*/
|
|
876
|
+
private combineSignals;
|
|
877
|
+
/**
|
|
878
|
+
* The clear, actionable message returned when no usable Realtime model can be resolved.
|
|
879
|
+
*
|
|
880
|
+
* @returns The failure message.
|
|
881
|
+
*/
|
|
882
|
+
private noModelMessage;
|
|
883
|
+
}
|
|
884
|
+
//# sourceMappingURL=realtime-client-session-service.d.ts.map
|