@memberjunction/ng-conversations 6.1.3 → 6.2.0-edge.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/README.md +1 -1
  2. package/dist/lib/components/conversation/conversation-chat-area.component.d.ts.map +1 -1
  3. package/dist/lib/components/conversation/conversation-chat-area.component.js +9 -26
  4. package/dist/lib/components/conversation/conversation-chat-area.component.js.map +1 -1
  5. package/dist/lib/components/realtime/channels/channel-onboarding-panel.component.d.ts +1 -1
  6. package/dist/lib/components/realtime/channels/channel-onboarding-panel.component.d.ts.map +1 -1
  7. package/dist/lib/components/realtime/channels/channel-onboarding-panel.component.js.map +1 -1
  8. package/dist/lib/components/realtime/channels/client-context-channel.d.ts +1 -1
  9. package/dist/lib/components/realtime/channels/client-context-channel.d.ts.map +1 -1
  10. package/dist/lib/components/realtime/channels/client-context-channel.js +1 -1
  11. package/dist/lib/components/realtime/channels/client-context-channel.js.map +1 -1
  12. package/dist/lib/components/realtime/channels/realtime-channel-pane.component.d.ts +1 -1
  13. package/dist/lib/components/realtime/channels/realtime-channel-pane.component.d.ts.map +1 -1
  14. package/dist/lib/components/realtime/channels/realtime-channel-pane.component.js +3 -0
  15. package/dist/lib/components/realtime/channels/realtime-channel-pane.component.js.map +1 -1
  16. package/dist/lib/components/realtime/media/media-channel.d.ts +2 -1
  17. package/dist/lib/components/realtime/media/media-channel.d.ts.map +1 -1
  18. package/dist/lib/components/realtime/media/media-channel.js +1 -1
  19. package/dist/lib/components/realtime/media/media-channel.js.map +1 -1
  20. package/dist/lib/components/realtime/realtime-activity-rail.component.d.ts +1 -1
  21. package/dist/lib/components/realtime/realtime-activity-rail.component.d.ts.map +1 -1
  22. package/dist/lib/components/realtime/realtime-activity-rail.component.js.map +1 -1
  23. package/dist/lib/components/realtime/realtime-agent-banner.component.d.ts +1 -1
  24. package/dist/lib/components/realtime/realtime-agent-banner.component.d.ts.map +1 -1
  25. package/dist/lib/components/realtime/realtime-agent-banner.component.js.map +1 -1
  26. package/dist/lib/components/realtime/realtime-composer.component.d.ts +3 -1
  27. package/dist/lib/components/realtime/realtime-composer.component.d.ts.map +1 -1
  28. package/dist/lib/components/realtime/realtime-composer.component.js +26 -8
  29. package/dist/lib/components/realtime/realtime-composer.component.js.map +1 -1
  30. package/dist/lib/components/realtime/realtime-delegation-card.component.d.ts +1 -1
  31. package/dist/lib/components/realtime/realtime-delegation-card.component.d.ts.map +1 -1
  32. package/dist/lib/components/realtime/realtime-delegation-card.component.js.map +1 -1
  33. package/dist/lib/components/realtime/realtime-session-overlay.component.d.ts +2 -2
  34. package/dist/lib/components/realtime/realtime-session-overlay.component.d.ts.map +1 -1
  35. package/dist/lib/components/realtime/realtime-session-overlay.component.js +1 -1
  36. package/dist/lib/components/realtime/realtime-session-overlay.component.js.map +1 -1
  37. package/dist/lib/components/realtime/realtime-session-state.d.ts +2 -3
  38. package/dist/lib/components/realtime/realtime-session-state.d.ts.map +1 -1
  39. package/dist/lib/components/realtime/realtime-session-state.js +3 -2
  40. package/dist/lib/components/realtime/realtime-session-state.js.map +1 -1
  41. package/dist/lib/components/realtime/realtime-session-thread.component.d.ts +17 -1
  42. package/dist/lib/components/realtime/realtime-session-thread.component.d.ts.map +1 -1
  43. package/dist/lib/components/realtime/realtime-session-thread.component.js +122 -45
  44. package/dist/lib/components/realtime/realtime-session-thread.component.js.map +1 -1
  45. package/dist/lib/components/realtime/realtime-session-timeline-card.component.d.ts +1 -9
  46. package/dist/lib/components/realtime/realtime-session-timeline-card.component.d.ts.map +1 -1
  47. package/dist/lib/components/realtime/realtime-session-timeline-card.component.js +13 -43
  48. package/dist/lib/components/realtime/realtime-session-timeline-card.component.js.map +1 -1
  49. package/dist/lib/components/realtime/realtime-surface-tabs.component.d.ts +2 -2
  50. package/dist/lib/components/realtime/realtime-surface-tabs.component.d.ts.map +1 -1
  51. package/dist/lib/components/realtime/realtime-surface-tabs.component.js.map +1 -1
  52. package/dist/lib/components/realtime/realtime-surface-tabs.model.d.ts +1 -1
  53. package/dist/lib/components/realtime/realtime-surface-tabs.model.d.ts.map +1 -1
  54. package/dist/lib/components/realtime/realtime-surface-tabs.model.js.map +1 -1
  55. package/dist/lib/components/realtime/remote-browser/remote-browser-channel.d.ts +1 -1
  56. package/dist/lib/components/realtime/remote-browser/remote-browser-channel.d.ts.map +1 -1
  57. package/dist/lib/components/realtime/remote-browser/remote-browser-channel.js +1 -1
  58. package/dist/lib/components/realtime/remote-browser/remote-browser-channel.js.map +1 -1
  59. package/dist/lib/components/realtime/whiteboard/whiteboard-channel.d.ts +1 -1
  60. package/dist/lib/components/realtime/whiteboard/whiteboard-channel.d.ts.map +1 -1
  61. package/dist/lib/components/realtime/whiteboard/whiteboard-channel.js +1 -1
  62. package/dist/lib/components/realtime/whiteboard/whiteboard-channel.js.map +1 -1
  63. package/dist/lib/conversations.module.d.ts.map +1 -1
  64. package/dist/lib/conversations.module.js.map +1 -1
  65. package/dist/lib/services/browser-realtime-media-host.d.ts +24 -0
  66. package/dist/lib/services/browser-realtime-media-host.d.ts.map +1 -0
  67. package/dist/lib/services/browser-realtime-media-host.js +95 -0
  68. package/dist/lib/services/browser-realtime-media-host.js.map +1 -0
  69. package/dist/lib/services/conversation-attachment.service.d.ts +24 -2
  70. package/dist/lib/services/conversation-attachment.service.d.ts.map +1 -1
  71. package/dist/lib/services/conversation-attachment.service.js +43 -4
  72. package/dist/lib/services/conversation-attachment.service.js.map +1 -1
  73. package/dist/lib/services/graphql-attachment-blob-store.d.ts +44 -0
  74. package/dist/lib/services/graphql-attachment-blob-store.d.ts.map +1 -0
  75. package/dist/lib/services/graphql-attachment-blob-store.js +113 -0
  76. package/dist/lib/services/graphql-attachment-blob-store.js.map +1 -0
  77. package/dist/lib/services/mention-autocomplete.service.d.ts +17 -141
  78. package/dist/lib/services/mention-autocomplete.service.d.ts.map +1 -1
  79. package/dist/lib/services/mention-autocomplete.service.js +15 -460
  80. package/dist/lib/services/mention-autocomplete.service.js.map +1 -1
  81. package/dist/lib/services/realtime-session-review.service.d.ts +1 -1
  82. package/dist/lib/services/realtime-session-review.service.d.ts.map +1 -1
  83. package/dist/lib/services/realtime-session-review.service.js +1 -1
  84. package/dist/lib/services/realtime-session-review.service.js.map +1 -1
  85. package/dist/lib/services/realtime-session.service.d.ts +16 -1150
  86. package/dist/lib/services/realtime-session.service.d.ts.map +1 -1
  87. package/dist/lib/services/realtime-session.service.js +28 -2430
  88. package/dist/lib/services/realtime-session.service.js.map +1 -1
  89. package/dist/lib/utils/realtime-session-timeline.d.ts +8 -79
  90. package/dist/lib/utils/realtime-session-timeline.d.ts.map +1 -1
  91. package/dist/lib/utils/realtime-session-timeline.js +8 -89
  92. package/dist/lib/utils/realtime-session-timeline.js.map +1 -1
  93. package/dist/public-api.d.ts +2 -2
  94. package/dist/public-api.d.ts.map +1 -1
  95. package/dist/public-api.js +6 -2
  96. package/dist/public-api.js.map +1 -1
  97. package/package.json +33 -32
  98. package/dist/lib/components/realtime/channels/base-realtime-channel-client.d.ts +0 -384
  99. package/dist/lib/components/realtime/channels/base-realtime-channel-client.d.ts.map +0 -1
  100. package/dist/lib/components/realtime/channels/base-realtime-channel-client.js +0 -231
  101. package/dist/lib/components/realtime/channels/base-realtime-channel-client.js.map +0 -1
  102. package/dist/lib/services/delegation-result-parser.d.ts +0 -50
  103. package/dist/lib/services/delegation-result-parser.d.ts.map +0 -1
  104. package/dist/lib/services/delegation-result-parser.js +0 -60
  105. package/dist/lib/services/delegation-result-parser.js.map +0 -1
  106. package/dist/lib/services/narration-template.d.ts +0 -42
  107. package/dist/lib/services/narration-template.d.ts.map +0 -1
  108. package/dist/lib/services/narration-template.js +0 -73
  109. package/dist/lib/services/narration-template.js.map +0 -1
  110. package/dist/lib/services/skill-picker-narrowing.d.ts +0 -13
  111. package/dist/lib/services/skill-picker-narrowing.d.ts.map +0 -1
  112. package/dist/lib/services/skill-picker-narrowing.js +0 -13
  113. package/dist/lib/services/skill-picker-narrowing.js.map +0 -1
@@ -1,2444 +1,42 @@
1
1
  import { Injectable } from '@angular/core';
2
- import { BehaviorSubject, Subject } from 'rxjs';
3
- import { Metadata } from '@memberjunction/core';
4
- import { UserInfoEngine } from '@memberjunction/core-entities';
5
- import { AIEngineBase } from '@memberjunction/ai-engine-base';
6
- import { MJGlobal } from '@memberjunction/global';
7
- import { DEFAULT_REALTIME_AUDIO_TRACKS } from '@memberjunction/ai';
8
- import { BaseRealtimeClient, LoadAssemblyAIRealtimeClient, LoadElevenLabsRealtimeClient, LoadGeminiRealtimeClient, LoadHuggingFaceRealtimeClient, LoadOpenAIRealtimeClient, LoadxAIRealtimeClient } from '@memberjunction/ai-realtime-client';
9
- import { BuildNarrationInstructions } from './narration-template';
10
- import { ParseDelegationResultJson, FormatToolName } from './delegation-result-parser';
11
- import { BaseRealtimeChannelClient } from '../components/realtime/channels/base-realtime-channel-client';
12
- import { RealtimeAudioRecorder } from './realtime-audio-recorder';
2
+ import { RealtimeSessionRuntime } from '@memberjunction/realtime-runtime';
3
+ import { LoadAssemblyAIRealtimeClient, LoadElevenLabsRealtimeClient, LoadGeminiRealtimeClient, LoadHuggingFaceRealtimeClient, LoadOpenAILiveClient, LoadOpenAIRealtimeClient, LoadxAIRealtimeClient, } from '@memberjunction/ai-realtime-client';
4
+ import { BrowserRealtimeMediaHost } from './browser-realtime-media-host';
13
5
  import * as i0 from "@angular/core";
14
- /**
15
- * `MJ: User Settings` key for the per-user "record this voice call" consent toggle. Stored as
16
- * the literal string `'true'`/`'false'` (read with `=== 'true'`), cross-device via
17
- * {@link UserInfoEngine}. The pre-call picker writes it; the session service reads it as the
18
- * default when the caller doesn't pass an explicit consent value.
19
- */
20
- export const REALTIME_RECORDING_CONSENT_KEY = 'mj.realtimeVoice.recordingConsent.v1';
21
- // Tree-shaking prevention: the OpenAI client is resolved dynamically through the
22
- // ClassFactory (by the server-reported Provider key), so this static call is what keeps
23
- // its @RegisterClass side effect from being eliminated by the bundler.
24
- // NOTE: the interactive-channel plugins (resolved dynamically from the `MJ: AI Agent
25
- // Channels` registry by ClientPluginClass key) get the same treatment, but their Load
26
- // calls live in `conversations.module.ts` — plugins carry Angular surface COMPONENTS,
27
- // and this service stays component-free (it must stay importable in plain-node tests).
6
+ // Tree-shaking prevention: client drivers are resolved dynamically through the ClassFactory (by
7
+ // the server-reported Provider key), so these static calls are what keep their @RegisterClass side
8
+ // effects from being eliminated by the bundler.
9
+ //
10
+ // NOTE: the interactive-channel plugins (resolved dynamically from the `MJ: AI Agent Channels`
11
+ // registry by ClientPluginClass key) get the same treatment, but their Load calls live in
12
+ // `conversations.module.ts` — plugins carry Angular surface COMPONENTS, and this service stays
13
+ // component-free.
28
14
  LoadOpenAIRealtimeClient();
15
+ LoadOpenAILiveClient();
29
16
  LoadGeminiRealtimeClient();
30
17
  LoadElevenLabsRealtimeClient();
31
18
  LoadAssemblyAIRealtimeClient();
32
19
  LoadxAIRealtimeClient();
33
20
  LoadHuggingFaceRealtimeClient();
34
21
  /**
35
- * Converts a {@link RealtimeTrackDescriptor} to its JSON form for the session-config bag.
36
- *
37
- * Every field's VALUE is already JSON-safe; the interface simply is not assignable to `JSONValue`
38
- * because it declares no index signature and `UsageBasis` is `readonly`. Written out field by field
39
- * rather than asserted, so adding a descriptor field is a compile error here instead of a field that
40
- * silently stops reaching the driver.
41
- */
42
- function trackDescriptorToJSON(track) {
43
- const json = { Modality: track.Modality, Direction: track.Direction };
44
- if (track.Encoding !== undefined) {
45
- json['Encoding'] = track.Encoding;
46
- }
47
- if (track.Rate !== undefined) {
48
- json['Rate'] = track.Rate;
49
- }
50
- if (track.UsageBasis !== undefined) {
51
- json['UsageBasis'] = [...track.UsageBasis];
52
- }
53
- if (track.RequiresConsent !== undefined) {
54
- json['RequiresConsent'] = track.RequiresConsent;
55
- }
56
- return json;
57
- }
58
- /**
59
- * Reads the `Direction:Modality` dedupe key off an already-JSON track entry, or `null` when the
60
- * entry is not a track-shaped object. Used for tracks the mint supplied, which arrive as raw JSON.
61
- */
62
- function trackKeyFromJSON(raw) {
63
- if (raw === null || typeof raw !== 'object' || Array.isArray(raw)) {
64
- return null;
65
- }
66
- const direction = raw['Direction'];
67
- const modality = raw['Modality'];
68
- if (typeof direction !== 'string' || typeof modality !== 'string') {
69
- return null;
70
- }
71
- return `${direction}:${modality}`;
72
- }
73
- /**
74
- * Drives a **client-direct** real-time voice session: the browser mints an ephemeral
75
- * token from the MJ server, then connects DIRECTLY to the realtime provider. Audio
76
- * frames never transit the MJ server (low latency); only tool calls and final
77
- * transcripts are relayed back to MJ over GraphQL.
78
- *
79
- * This service is PROVIDER-AGNOSTIC policy/orchestration. All provider wire concerns
80
- * (transport, event translation, the response state machine, narration-kind tagging,
81
- * playback tracking) live in a {@link BaseRealtimeClient} driver resolved through the
82
- * MJ ClassFactory by the server-reported `Provider` key (e.g. `'openai'` →
83
- * `OpenAIRealtimeClient`). Future providers (Gemini Live, …) snap in by registering a
84
- * new driver — this service does not change.
22
+ * Angular host for MemberJunction's realtime co-agent sessions.
85
23
  *
86
- * The Realtime Co-Agent (server-side) fronts the conversation's current agent — the server
87
- * bakes the companion instructions + tool set into `SessionConfigJson`, which the client
88
- * driver applies verbatim.
24
+ * All orchestration — mint, driver resolution, transcripts, tool relay, delegation narration,
25
+ * channel lifecycle, usage relay, teardown — lives in
26
+ * {@link RealtimeSessionRuntime} (`@memberjunction/realtime-runtime`), which is framework-agnostic
27
+ * and shared with every other host, including the React Native app. This class supplies the two
28
+ * things that are genuinely Angular/browser: the `@Injectable` shell that lets components inject
29
+ * it, and the browser's media host.
89
30
  *
90
- * Lifecycle: {@link StartRealtimeSession} → live duplex → {@link EndRealtimeSession}. A start is
91
- * two halves — MINT (the `StartRealtimeClientSession` mutation) and RUN (everything above) — and a
92
- * host that must mint through its own server surface enters at the second half via
93
- * {@link StartRealtimeSessionFromResult}; there is one implementation of the run half either way.
31
+ * Consumers are unaffected by that split — the injectable token, class name, and public surface
32
+ * are unchanged. Types that used to be declared in this file (`RealtimeCaption`,
33
+ * `RealtimeConnectionState`, `RealtimeSessionRunOptions`, …) now come from
34
+ * `@memberjunction/realtime-runtime`; import them from there directly, since MJ does not
35
+ * re-export across package boundaries.
94
36
  */
95
- export class RealtimeSessionService {
96
- // ── Reactive UI state ──────────────────────────────────────────────────────
97
- _connectionState$ = new BehaviorSubject('closed');
98
- _captions$ = new BehaviorSubject([]);
99
- _active$ = new BehaviorSubject(false);
100
- _delegationProgress$ = new Subject();
101
- _delegationResult$ = new Subject();
102
- _delegationNarration$ = new Subject();
103
- _thoughtNarration$ = new Subject();
104
- _agentName$ = new BehaviorSubject('Sage');
105
- _modelName$ = new BehaviorSubject(null);
106
- _minimized$ = new BehaviorSubject(false);
107
- _activeChannels$ = new BehaviorSubject([]);
108
- _channelFocus$ = new Subject();
109
- // ─── Generic session-lifecycle events (consumed by RealtimeSessionsAdapter to
110
- // bridge into @memberjunction/conversations-runtime's framework-agnostic
111
- // SessionsObserver). Why not derive from Active$ + agentSessionId? Because
112
- // Active$ flips true before mintSession resolves and sets agentSessionId —
113
- // a naive Active$ subscription would emit session-started with sessionId === null.
114
- // Emitting explicitly avoids the race entirely. ───
115
- _sessionStarted$ = new Subject();
116
- _sessionEnded$ = new Subject();
117
- _channelActivity$ = new Subject();
118
- /** Current connection / turn state. */
119
- ConnectionState$ = this._connectionState$.asObservable();
120
- /** Live captions for both sides of the conversation. */
121
- Captions$ = this._captions$.asObservable();
122
- /** True while a session is open (mic button active, overlay shown). */
123
- Active$ = this._active$.asObservable();
124
- /**
125
- * Progress updates from a delegated agent run (e.g. Sage) while the realtime model waits on it.
126
- * The future overlay subscribes to render a "working" card; the model also narrates these aloud.
127
- */
128
- DelegationProgress$ = this._delegationProgress$.asObservable();
129
- /** Terminal result of a delegation, so the overlay can complete the working card with real content. */
130
- DelegationResult$ = this._delegationResult$.asObservable();
131
- /**
132
- * EPHEMERAL spoken progress narrations (see {@link RealtimeDelegationNarration}). These are
133
- * deliberately kept OUT of {@link Captions$} and never relayed/persisted — the overlay
134
- * renders them as a transient "live note" near the active working card.
135
- */
136
- DelegationNarration$ = this._delegationNarration$.asObservable();
137
- /**
138
- * Model-authored thought / reasoning narrations (see {@link RealtimeThoughtNarration}). These are
139
- * reasoning summaries author-emitted during extended thinking, separate from spoken progress updates.
140
- */
141
- ThoughtNarration$ = this._thoughtNarration$.asObservable();
142
- /** Display name of the agent the active session fronts (set at session start). */
143
- AgentName$ = this._agentName$.asObservable();
144
- /**
145
- * Display name of the realtime MODEL the active session runs on (server-reported at session
146
- * start, e.g. "GPT Realtime 2"). `null` before a session starts / when the server didn't report
147
- * one. The overlay banner shows it subtly next to the agent identity.
148
- */
149
- ModelName$ = this._modelName$.asObservable();
150
- /**
151
- * True while the active call overlay is MINIMIZED to the host's floating "on call" pill
152
- * (e.g. after a dev link navigated away). The mic and session stay fully live — this is
153
- * pure presentation state, reset to `false` at session start and teardown.
154
- */
155
- Minimized$ = this._minimized$.asObservable();
156
- /**
157
- * The session's ACTIVE interactive-channel plugins, resolved from the `MJ: AI Agent
158
- * Channels` registry at session start (one instance per session, per channel). Emits
159
- * `[]` before a session starts and after teardown. The overlay subscribes to register
160
- * one surface tab per plugin — it never knows any concrete channel type.
161
- */
162
- ActiveChannels$ = this._activeChannels$.asObservable();
163
- /**
164
- * Channel requests to enter / leave the FOCUS layout (see
165
- * {@link RealtimeChannelFocusEvent}). Fired when a plugin calls its host context's
166
- * `SetFocusMode` — e.g. the whiteboard's "Focus board" toggle.
167
- */
168
- ChannelFocus$ = this._channelFocus$.asObservable();
169
- /**
170
- * Fired EXACTLY ONCE per session after both `agentSessionId` is set AND the
171
- * realtime client is connected. Carries the server-issued `sessionId` and the
172
- * `ChannelName` of each plugin resolved at session mint. Consumed by
173
- * `RealtimeSessionsAdapter` (in this package) to feed
174
- * `@memberjunction/conversations-runtime`'s `SessionsObserver`.
175
- *
176
- * **Why this exists separately from `Active$`:** `Active$` flips `true` BEFORE
177
- * `mintSession` resolves, so `agentSessionId` is still `null` at that moment.
178
- * Subscribers correlating `(Active$, agentSessionId)` would race; this event
179
- * removes the race.
180
- */
181
- SessionStarted$ = this._sessionStarted$.asObservable();
182
- /**
183
- * Fired EXACTLY ONCE per session as teardown begins, with the prior
184
- * `agentSessionId` (so subscribers can correlate against `SessionStarted$`'s
185
- * sessionId) and the client-distinguishable reason — `'explicit'` when the
186
- * user called `EndRealtimeSession`, `'error'` when teardown ran from a catch
187
- * block. Server-side close paths (janitor, shutdown) do NOT propagate here —
188
- * they happen out-of-process and have no client push channel today.
189
- */
190
- SessionEnded$ = this._sessionEnded$.asObservable();
191
- /**
192
- * Fires with the channel PLUGIN every time the agent ACTS on that channel (a tool call
193
- * was routed to its local executor — e.g. the agent drew on the whiteboard). The overlay
194
- * uses the FIRST emission per channel to auto-reveal + focus the channel's surface tab,
195
- * so the user discovers the surface the moment the agent starts using it. Finer-grained
196
- * than {@link SessionStarted$}/{@link SessionEnded$} (per tool call, not per session).
197
- */
198
- ChannelActivity$ = this._channelActivity$.asObservable();
199
- /** Synchronous access to the session's active interactive-channel plugins. */
200
- get ActiveChannels() {
201
- return this._activeChannels$.value;
202
- }
203
- /**
204
- * The `ChannelName`s the agent has used (acted on) at least once this session. The overlay
205
- * uses this to register a channel's surface tab only after it has come into play. A fresh
206
- * Set snapshot so callers can't mutate the service's tracking.
207
- */
208
- get UsedChannelNames() {
209
- return new Set(this.usedChannelNames);
210
- }
211
- /** Whether the agent has used (acted on) the named channel at least once this session. */
212
- HasChannelBeenUsed(channelName) {
213
- return this.usedChannelNames.has(channelName);
214
- }
215
- /** Synchronous access to the display name of the agent the active session fronts. */
216
- get CurrentAgentName() {
217
- return this._agentName$.value;
218
- }
219
- /**
220
- * ID of the active server-side agent session (`MJ: AI Agent Sessions`), or `null` when no
221
- * session is open / the session hasn't been minted yet. Powers the overlay's gear-gated
222
- * "Open session" dev link.
223
- */
224
- /** Conversation id the SERVER created for this session (null when the host supplied one). */
225
- createdConversationId = null;
226
- /** The session's conversation id (supplied or server-created). */
227
- sessionConversationId = null;
228
- /** First final user utterance of the live session (the naming seed). */
229
- firstUserTranscript = null;
230
- /** Buffer accumulating streaming user interim deltas into a single in-progress bubble. */
231
- pendingUserCaption = '';
232
- /** Whether an in-place interim user caption is currently placed in `_captions$`. */
233
- hasActiveInterimUserCaption = false;
234
- /**
235
- * When the active/last session CREATED its conversation (started without one), the new
236
- * conversation's id — the host uses it to refresh the cached list, conditionally select
237
- * it on close, and auto-name it. Null when the session joined an existing conversation.
238
- */
239
- get SessionCreatedConversationId() {
240
- return this.createdConversationId;
241
- }
242
- /** The first final user utterance of the session (naming seed); null before the user speaks. */
243
- get FirstUserTranscript() {
244
- return this.firstUserTranscript;
245
- }
246
- get CurrentAgentSessionId() {
247
- return this.agentSessionId;
248
- }
249
- /** Synchronous access to the minimized presentation state. */
250
- get IsMinimized() {
251
- return this._minimized$.value;
252
- }
253
- /**
254
- * Minimizes / restores the active call overlay (host renders the floating pill while
255
- * minimized). Presentation-only — the live audio session is untouched.
256
- */
257
- SetMinimized(minimized) {
258
- if (this._minimized$.value !== minimized) {
259
- this._minimized$.next(minimized);
260
- }
261
- }
262
- // ── Session internals ──────────────────────────────────────────────────────
263
- /** The provider-direct realtime client driving the live session (ClassFactory-resolved). */
264
- client = null;
265
- /** The mic capture stream — acquired here (permission UX) and handed to the client. */
266
- localStream = null;
267
- agentSessionId = null;
268
- /**
269
- * The application the active session runs in (sources the server-side app config cascade +
270
- * RelevantAgents → allowed-agent union, and the default-agent chain). `null` when no app context
271
- * was supplied. Set at {@link StartRealtimeSession}; sent to the mint mutation.
272
- */
273
- applicationId = null;
274
- /**
275
- * The live app-context snapshot (where the user is, what they see, the capability manifest),
276
- * pushed by the host (Explorer) at session start and on subsequent changes via
277
- * {@link UpdateAppContext}. The headless {@link import('../components/realtime/channels/client-context-channel').ClientContextChannel}
278
- * subscribes to {@link AppContext$} and streams deltas to the model via `SendContextNote`.
279
- */
280
- _appContext$ = new BehaviorSubject(null);
281
- /** Observable of the live app-context snapshot (see {@link _appContext$}). */
282
- AppContext$ = this._appContext$.asObservable();
283
- /**
284
- * Push an updated app-context snapshot mid-session (the continuous-streaming half of client-context
285
- * delivery). The host (Explorer) calls this when the user navigates / the active surface's state or
286
- * capability manifest changes; the ClientContextChannel turns the delta into a `SendContextNote`.
287
- * No-op semantics when no session is live — the channel simply re-reads on next start.
288
- *
289
- * @param snapshot The latest app-context snapshot (or null to clear).
290
- */
291
- UpdateAppContext(snapshot) {
292
- this._appContext$.next(snapshot);
293
- }
294
- /**
295
- * The DB-driven narration instruction template (server-resolved at session start, containing a
296
- * `{{ progressMessage }}` placeholder). `null` when the deployment hasn't synced the narration
297
- * prompt — {@link buildNarrationInstructions} then falls back to the built-in wording.
298
- */
299
- narrationTemplate = null;
300
- // ── Browser-side call recording ────────────────────────────────────────────
301
- /**
302
- * The active session's audio recorder (mic + agent mix), or `null` when the user didn't
303
- * consent or the browser can't record. Created after the client connects; stopped + uploaded
304
- * at teardown.
305
- */
306
- recorder = null;
307
- /** ISO timestamp of when recording started — sent to the server on session start. */
308
- recordingStartedAtIso = null;
309
- /** Interval that flushes ~15s crash-recovery shards to the server during a recording. */
310
- segmentTimer = null;
311
- /** 0-based index of the next recording shard to upload. */
312
- segmentIndex = 0;
313
- /** How often crash-recovery shards are flushed during a recording. */
314
- static SegmentFlushMs = 15000;
315
- // ── Server-side liveness ───────────────────────────────────────────────────
316
- /**
317
- * Interval that tells the server this session is still in use, or `null` when no session is
318
- * running. See {@link startLivenessPulse} for why the server cannot work this out itself.
319
- */
320
- livenessTimer = null;
321
- /**
322
- * How often the client asserts liveness. Comfortably under `SessionJanitor`'s
323
- * `closeThresholdMinutes` (15 by default) so several pulses must be missed in a row before a
324
- * live session is reaped, and well above `SessionManager`'s heartbeat write-coalescing window
325
- * so the DB sees at most a trickle of writes per session.
326
- */
327
- static LivenessPulseMs = 60000;
328
- /**
329
- * Recording-relative ms offset at which the IN-FLIGHT (not-yet-finalized) turn's audio
330
- * actually BEGAN — captured the moment that turn's audio/text starts flowing (its first
331
- * interim transcript), NOT inherited from the previous turn's end. `null` before the first
332
- * turn / between turns (until the next turn's audio starts). Sent as `utteranceStartMs` on
333
- * the turn's final transcript so per-turn timing lines up with the recording even when a
334
- * tool-call / silence gap sits between turns (the inherit-previous-end model mis-stamped
335
- * the post-gap turn at the pre-gap offset). See {@link markTurnAudioStart}.
336
- */
337
- currentTurnStartMs = null;
338
- /**
339
- * Wall-anchor of the SESSION clock (#3832): `performance.now()` at the moment the call went
340
- * live, or `null` before any call has. Read only through {@link nowTurnOffsetMs}.
341
- */
342
- sessionClockStartMs = null;
343
- /**
344
- * Per-turn guard for {@link markTurnAudioStart}: `true` once the in-flight turn's audio-start
345
- * offset has been captured, so mid-turn interim deltas don't overwrite it. Reset to `false`
346
- * at each finalization so the NEXT turn re-stamps from where ITS audio begins.
347
- */
348
- turnAudioStartCaptured = false;
349
- // ── Delegated-run progress streaming ───────────────────────────────────────
350
- /** First spoken update fires no earlier than this long after delegated work starts. */
351
- static FirstNarrationDelayMs = 5000;
352
- /** Minimum gap between SUBSEQUENT spoken updates (the 7–10s band; floods aggregate). */
353
- static NarrationIntervalMs = 8000;
354
- /** Retry delay when the fire moment finds the model busy / audio still playing. */
355
- static NarrationBusyRetryMs = 1500;
356
- /** Max progress messages aggregated into one spoken digest. */
357
- static MaxDigestMessages = 4;
358
- /** Max prior spoken narrations chained into the instructions (anti-repetition). */
359
- static MaxPriorNarrations = 3;
360
- /**
361
- * Aggregation buffer: distinct progress messages since the last spoken update (oldest
362
- * first, capped at {@link RealtimeSessionService.MaxDigestMessages}). A flood of small
363
- * updates becomes ONE digest; the buffer is discarded when the result lands first.
364
- */
365
- pendingNarrationMessages = [];
366
- /**
367
- * Tool calls currently executing on the server. Progress events ride PubSub and can
368
- * lag the (fast) mutation result — any progress for a call NOT in this set is stale
369
- * (already completed) and is dropped, so we never narrate "starting up" after the
370
- * answer was already spoken.
371
- */
372
- inFlightCallIds = new Set();
373
- /** Timer for the deferred narration; cancelled when the delegation result lands first. */
374
- narrationTimer = null;
375
- /**
376
- * Call ids the USER explicitly cancelled via {@link CancelDelegation} /
377
- * {@link CancelInFlightDelegations}. Their cards were already flipped to the
378
- * "Cancelled by user" failed result, so when the original tool mutation later resolves
379
- * with the aborted run's outcome, {@link emitDelegationResult} skips the duplicate card
380
- * emission (the model still receives the tool result). Cleared at teardown.
381
- */
382
- cancelledCallIds = new Set();
383
- // ── Usage telemetry relay (B7) ─────────────────────────────────────────────
384
- /** Debounce window for relaying accumulated usage deltas to the server. */
385
- static UsageFlushDebounceMs = 10000;
386
- /** Accumulated input-token delta since the last flush. */
387
- pendingUsageInput = 0;
388
- /** Accumulated output-token delta since the last flush. */
389
- pendingUsageOutput = 0;
390
- /** Pending debounced usage flush; also force-flushed at teardown. */
391
- usageFlushTimer = null;
392
- /** Active push-status subscription that feeds delegation progress; cleared on teardown. */
393
- delegationProgressSub = null;
394
- /** Timestamp (ms) of the last narration we triggered; 0 = never. */
395
- lastDelegationNarrationAt = 0;
396
- /** When the current delegation burst began (first in-flight call); anchors the 5s first update. */
397
- delegationBurstStartedAt = 0;
398
- /** Spoken updates so far in this burst (1-based numbering for the instructions). */
399
- narrationCount = 0;
400
- /** What the model actually SAID for prior updates this burst — chained in so it never repeats itself. */
401
- spokenNarrations = [];
402
- /** Tail message of the last digest, so an identical trailing progress event isn't re-buffered. */
403
- lastNarratedTail = '';
404
- /**
405
- * Registry of CLIENT-EXECUTED UI tool handlers, keyed by tool-name prefix (e.g.
406
- * `'Whiteboard_'`). Tool calls whose name matches a registered prefix run LOCALLY through the
407
- * handler (never relayed to the server); everything else takes the standard server-relay path.
408
- * Cleared at teardown.
409
- */
410
- clientToolHandlers = new Map();
411
- // ── Interactive channels (registry-resolved plugins) ───────────────────────
412
- /** Debounce window for persisting a channel's state of record after a change burst. */
413
- static ChannelSaveDebounceMs = 3000;
414
- /**
415
- * Pending DEBOUNCED channel-state saves, keyed by channel name. Each entry keeps the
416
- * LATEST serialized state plus the session id captured while the session was live —
417
- * the teardown flush runs as the live id is being torn down, so the capture guarantees
418
- * the final save still lands on the just-closed session.
419
- */
420
- pendingChannelSaves = new Map();
421
- /**
422
- * `ChannelName`s the agent has ACTED ON at least once this session (the channel's first
423
- * tool call routed to its local executor). The overlay reads this to decide which channel
424
- * surface tabs to register: a channel earns its tab only once it's been used (the
425
- * whiteboard is the sole exception — it tabs immediately, since a user may draw first).
426
- * Reset at session start via {@link resetState}.
427
- */
428
- usedChannelNames = new Set();
429
- _provider = null;
430
- /**
431
- * Metadata provider used for the GraphQL relay mutations. Falls back to the
432
- * global default when unset (single-provider apps see no change).
433
- */
434
- get Provider() {
435
- return this._provider ?? Metadata.Provider;
436
- }
437
- set Provider(value) {
438
- this._provider = value;
439
- }
440
- /** True when a session is currently open. */
441
- get IsActive() {
442
- return this._active$.value;
443
- }
444
- /**
445
- * Start a client-direct voice session fronting `targetAgentId`.
446
- *
447
- * @param targetAgentId The agent the Realtime Co-Agent voices on behalf of.
448
- * @param conversationId Optional existing conversation to bind + seed context from.
449
- * @param lastSessionId Optional prior session to chain to (resume / continuation).
450
- * @param agentName Optional display name of the target agent — resolved by the caller
451
- * (which knows the conversation's routing context) and surfaced on {@link AgentName$}
452
- * so ANY host (composer trigger, chat-area overlay) can render it without re-resolving.
453
- * @param preferredModelId Optional EXPLICIT realtime model choice (`MJ: AI Models.ID`). When
454
- * set, the server uses exactly that model and FAILS with a clear reason if it can't (no
455
- * silent fallback). Omit for the server's automatic (highest-PowerRank) selection.
456
- * @param clientTools Optional EXTRA client-executed UI tool declarations to expose to the
457
- * realtime model alongside the server's stable tool set and the interactive-channel
458
- * tools (which are aggregated automatically from the registry-resolved plugins — see
459
- * {@link ActiveChannels$}). The server only DECLARES these — execution stays in the
460
- * browser via handlers registered with {@link RegisterClientToolHandler}. This is an
461
- * extension point for hosts with bespoke (non-channel) UI tools; most callers omit it.
462
- * @param coAgentId Optional EXPLICIT co-agent choice (`MJ: AI Agents.ID` of an Active,
463
- * Realtime-type agent) — the highest-precedence step of the server's co-agent resolution
464
- * chain. When set, the server uses exactly that co-agent and FAILS with a clear reason if
465
- * it can't (no silent fallback). Omit to let server metadata drive the choice: the target
466
- * agent's `DefaultCoAgentID`, then the type-level `AIAgentCoAgent` default row, then the global Realtime Co-Agent.
467
- * @param configOverridesJson Optional JSON payload of SESSION CONFIG overrides (e.g.
468
- * `{"realtime":{"modelPreference":"<modelId>"}}`), forwarded verbatim on the mint
469
- * mutation. The server enforces the `Realtime: Advanced Session Controls`
470
- * authorization on any overrides — hosts only populate this from authorization-gated
471
- * pickers, and never synthesize overrides beyond what the user explicitly chose.
472
- * Omit/`null` for the server's defaults (today's behavior).
473
- * @param recordingConsent Optional EXPLICIT "record this call" consent for THIS session. When
474
- * `true`, the browser records a mic + agent-audio mix and uploads it at session end. When
475
- * omitted/`null`, the per-user persisted preference (`mj.realtimeVoice.recordingConsent.v1`
476
- * via {@link UserInfoEngine}) is read as the default. `false` never records.
477
- * @param mediaCollectionId Optional per-session media-kit override (`MJ: Collections.ID`). When set,
478
- * the server-side Media channel resolves THIS collection as the agent's media kit for the session,
479
- * taking precedence over the agent's `DefaultMediaCollectionID`. The server UUID-validates it
480
- * (malformed ⇒ ignored, the agent default applies). Omit/`null` to use the agent default kit.
481
- */
482
- async StartRealtimeSession(targetAgentId, conversationId, lastSessionId, agentName, preferredModelId, clientTools, coAgentId, configOverridesJson, recordingConsent, mediaCollectionId, applicationId, appContext) {
483
- if (this.IsActive) {
484
- return; // a session is already running — ignore duplicate starts
485
- }
486
- const consent = this.beginSessionStart({ agentName, recordingConsent, applicationId, appContext });
487
- // Captured BEFORE startChannels so the mint carries exactly the snapshot the prologue
488
- // resolved, whatever a channel plugin may push in the meantime.
489
- const effectiveAppContext = this._appContext$.value;
490
- let session;
491
- try {
492
- // Resolve + initialize the interactive-channel plugins FIRST: their client-executed
493
- // tool sets must be declared to the realtime model at session mint.
494
- const allClientTools = [...(clientTools ?? []), ...(await this.startChannels())];
495
- session = await this.mintSession(targetAgentId, conversationId, lastSessionId, preferredModelId, allClientTools, coAgentId, configOverridesJson, consent, this.recordingStartedAtIso, mediaCollectionId, this.applicationId, effectiveAppContext);
496
- }
497
- catch (error) {
498
- await this.failSessionStart(error);
499
- return;
500
- }
501
- await this.runMintedSession(session, conversationId ?? null, consent);
502
- }
503
- /**
504
- * Run a session the HOST has already minted itself — the second half of
505
- * {@link StartRealtimeSession}, without the `StartRealtimeClientSession` mutation.
506
- *
507
- * For hosts that must mint through their own server surface because they attach per-session
508
- * context the stock mutation cannot carry (e.g. an interview persona baked into the companion
509
- * prompt). They call their own mutation, shape the reply into a
510
- * {@link StartRealtimeClientSessionResult}, and hand it here: driver resolution, the ephemeral-token
511
- * connect, tool/transcript relays, recording, connection state and teardown are all identical to
512
- * the all-in-one path — there is exactly one implementation of the run half.
513
- *
514
- * NOTE: the interactive-channel plugins are NOT started on this path. Their tool sets must be
515
- * declared to the model AT MINT, which happened on the host's side — so a host that wants channels
516
- * owns that half too.
517
- *
518
- * @param result The minted session — the same ten fields the `StartRealtimeClientSession`
519
- * mutation returns. `EphemeralToken` and `Provider` are what actually open the call.
520
- * @param options Host-side inputs the result cannot carry; see {@link RealtimeSessionRunOptions}.
521
- * Every field defaults exactly as its {@link StartRealtimeSession} counterpart does.
522
- */
523
- async StartRealtimeSessionFromResult(result, options) {
524
- if (this.IsActive) {
525
- return; // a session is already running — ignore duplicate starts
526
- }
527
- const effectiveOptions = options ?? {};
528
- const consent = this.beginSessionStart(effectiveOptions);
529
- await this.runMintedSession(result, effectiveOptions.conversationId ?? null, consent);
530
- }
531
- /**
532
- * Start prologue shared by both entry points: bind the app layer, publish the agent name, reset
533
- * per-session state, and flip the session live (which is ALSO what makes the `IsActive` guard
534
- * suppress duplicate starts while the mint is still in flight — hence it runs before minting, not
535
- * after). Returns the resolved recording consent, which the mint half reports to the server and
536
- * the run half uses to decide whether to record.
537
- */
538
- beginSessionStart(options) {
539
- // App awareness (Move 1/3/4): the application the session runs in (sources the app config
540
- // cascade + RelevantAgents → allowed-agent union) and the live app-context snapshot injected
541
- // into the companion prompt at mint. Stored so the ClientContextChannel can stream subsequent
542
- // deltas. Absent ⇒ no app layer / no mint-time context (the pre-app behavior).
543
- this.applicationId = options.applicationId ?? null;
544
- // Prefer the explicit param, but fall back to the snapshot the host has ALREADY pushed via
545
- // UpdateAppContext (explorer-app streams the live snapshot continuously). The overlay's
546
- // [appContext] binding can still read null at the instant the mic is clicked — without this
547
- // fallback, StartRealtimeSession(null) would clobber a perfectly good snapshot and mint the
548
- // companion prompt with no app context (no NavigableApps / no tool schemas → the co-agent guesses
549
- // parameter names and navigation fails). Never overwrite a good value with null.
550
- this._appContext$.next(options.appContext ?? this._appContext$.value);
551
- if (options.agentName) {
552
- this._agentName$.next(options.agentName);
553
- }
554
- this.resetState();
555
- this._active$.next(true);
556
- this._connectionState$.next('connecting');
557
- // Resolve recording consent for this session: explicit value wins, else the per-user
558
- // persisted preference. Computed before mint so it can be reported to the server.
559
- const consent = options.recordingConsent ?? this.readPersistedRecordingConsent();
560
- this.recordingStartedAtIso = consent ? new Date().toISOString() : null;
561
- return consent;
562
- }
563
- /**
564
- * The RUN half of a session start, shared by both entry points: consume the minted result, open
565
- * the provider connection, and go live. `inputConversationId` is the conversation the START asked
566
- * for (null ⇒ "server, make me one") — the result alone can't distinguish the two.
567
- */
568
- async runMintedSession(session, inputConversationId, consent) {
569
- try {
570
- this.agentSessionId = session.AgentSessionId;
571
- // A null input conversationId means the SERVER created a fresh conversation for
572
- // this session — track it so the host can fold it into the cached list, select
573
- // it on close, and auto-name it (via the shared naming helper).
574
- this.createdConversationId = !inputConversationId && session.ConversationId ? session.ConversationId : null;
575
- this.sessionConversationId = session.ConversationId ?? inputConversationId ?? null;
576
- this.firstUserTranscript = null;
577
- this.narrationTemplate = session.NarrationInstructionsTemplate ?? null;
578
- this._modelName$.next(session.ModelName ?? null);
579
- // Resume continuity: rehydrate channel plugins from the PRIOR session's saved states
580
- // (e.g. the whiteboard) BEFORE any surface binds — tolerant, never blocks the start.
581
- this.applyPriorChannelStates(session.PriorChannelStatesJson);
582
- const client = this.createRealtimeClient(session.Provider);
583
- this.client = client;
584
- this.wireClientHandlers(client);
585
- this.localStream = await navigator.mediaDevices.getUserMedia({ audio: true });
586
- await client.Connect(this.buildClientConfig(session), this.localStream);
587
- // Notify active channels that the session client is connected and tracks are established
588
- for (const channel of this._activeChannels$.value) {
589
- try {
590
- channel.OnSessionStarted?.();
591
- }
592
- catch (err) {
593
- console.error(`[RealtimeSession] Error in channel '${channel.ChannelName}' OnSessionStarted:`, err);
594
- }
595
- }
596
- // Start browser-side recording (mic + agent mix) when consented. Best-effort: an
597
- // unsupported browser / missing remote stream degrades gracefully (mic-only or off)
598
- // and never blocks the call. The remote stream may still be null here (the WebRTC
599
- // ontrack can land slightly after Connect resolves) — the recorder mixes the mic now
600
- // and the agent audio rides through whenever its track is already attached.
601
- // The SESSION clock (#3832): anchored the moment the call goes live, whether or not a
602
- // recording exists. When the recorder runs, per-turn timings use ITS clock (offsets into a
603
- // seekable file); when it does not — every unconsented and every relay-captured session,
604
- // which is 100% of turns measured across two databases — this is the fallback that stops
605
- // `UtteranceStartMs`/`UtteranceEndMs` being categorically null. An offset into a session
606
- // with no audio is not seekable, but it is orderable and displayable ("3:42 into the
607
- // interview"), and it is stamped when the SPEECH happened rather than when the relay
608
- // mutation landed — which no server-side backfill can ever recover.
609
- this.sessionClockStartMs = performance.now();
610
- if (consent) {
611
- this.startRecording(client);
612
- }
613
- this.subscribeDelegationProgress();
614
- // State advances to 'listening' once the provider control channel opens
615
- // (driven by the client's OnStateChange events).
616
- // Surface a generic session-started event for the conversations runtime
617
- // SessionsObserver bridge. Emitting AFTER Connect() guarantees both that
618
- // agentSessionId is set (line ~468) AND the realtime client is connected,
619
- // so consumers can act on it without re-checking either condition.
620
- this._sessionStarted$.next({
621
- sessionId: this.agentSessionId,
622
- channelNames: this._activeChannels$.value.map(c => c.ChannelName),
623
- });
624
- // Same place, same reason: the session is connected and its id is known, which is exactly
625
- // the window in which the server needs to be told it is alive.
626
- this.startLivenessPulse();
627
- }
628
- catch (error) {
629
- await this.failSessionStart(error);
630
- }
631
- }
632
- /**
633
- * The single failure path for a session start (mint half or run half): report it, latch the
634
- * overlay into 'error', and unwind whatever the half-built session already opened.
635
- */
636
- async failSessionStart(error) {
637
- console.error('[RealtimeSession] Failed to start session:', error);
638
- this._connectionState$.next('error');
639
- await this.teardown(false);
640
- }
641
- /**
642
- * End the active session: stop the mic, tear down the provider connection, and close
643
- * the server-side agent session. Safe to call when no session is active.
644
- */
645
- async EndRealtimeSession() {
646
- if (!this.IsActive && !this.agentSessionId) {
647
- return;
648
- }
649
- await this.teardown(true);
650
- }
651
- /**
652
- * Inject a typed message into the live session as a user turn.
653
- *
654
- * Decomposed into two steps, each mirroring an existing voice path so the typed
655
- * turn behaves identically to a spoken one:
656
- * 1. {@link BaseRealtimeClient.SendText} injects the text as user input and triggers a
657
- * reply through the SAME collision-safe path tool results use — so it queues behind
658
- * any in-flight response (progress narration / prior turn) instead of colliding.
659
- * 2. Relay the turn through the same caption + transcript paths user speech uses
660
- * ({@link onUserTranscript}) so it shows in the live thread AND persists to MJ.
661
- *
662
- * No-op when no session is open / the control channel isn't ready, or when the text is empty.
663
- */
664
- SendText(text) {
665
- const trimmed = text?.trim() ?? '';
666
- if (trimmed.length === 0) {
667
- return;
668
- }
669
- const client = this.client;
670
- if (!client || !this.isSessionLive()) {
671
- return;
672
- }
673
- client.SendText(trimmed);
674
- // Relay as a user turn — same path spoken input uses (caption + persisted transcript).
675
- void this.onUserTranscript(trimmed);
676
- }
677
- /** Mute / unmute the local microphone track. Returns the new muted state. */
678
- ToggleMute() {
679
- const tracks = this.localStream?.getAudioTracks() ?? [];
680
- if (tracks.length === 0) {
681
- return false;
682
- }
683
- const muted = tracks[0].enabled; // currently enabled → becomes muted
684
- this.client?.SetMuted(muted);
685
- return muted;
686
- }
687
- // ── Client-executed UI tools ───────────────────────────────────────────────
688
- /**
689
- * Registers a handler for CLIENT-EXECUTED UI tools whose names start with `toolNamePrefix`
690
- * (e.g. `'Whiteboard_'` → all `Whiteboard_*` calls). Matching tool calls execute LOCALLY via
691
- * the handler — they are never relayed to the server — and the handler's result JSON is sent
692
- * back to the model as the `tool_response`. Re-registering the same prefix replaces the
693
- * handler. The registry is cleared at session teardown.
694
- */
695
- RegisterClientToolHandler(toolNamePrefix, handler) {
696
- this.clientToolHandlers.set(toolNamePrefix, handler);
697
- }
698
- /** Removes the handler registered for `toolNamePrefix` (no-op when none is registered). */
699
- UnregisterClientToolHandler(toolNamePrefix) {
700
- this.clientToolHandlers.delete(toolNamePrefix);
701
- }
702
- /**
703
- * Feeds a background context note into the live model (no spoken reply is requested) — the
704
- * perception channel interactive surfaces use (e.g. the whiteboard's coalesced scene deltas).
705
- * No-op when no session is live.
706
- */
707
- SendContextNote(text) {
708
- const trimmed = text?.trim() ?? '';
709
- if (trimmed.length === 0 || !this.client || !this.isSessionLive()) {
710
- return;
711
- }
712
- this.client.SendContextNote(trimmed);
713
- }
714
- /**
715
- * Asks the live model to SPEAK FIRST — before the human has said anything.
716
- *
717
- * Every other path into the model's voice reacts to something: the human spoke, or a channel
718
- * reported input. A host that needs the agent to open the conversation (an interviewer greeting
719
- * a candidate, a guide introducing a task) had no way to ask for that, so the session connected
720
- * and both sides waited for the other. The instructions are what to say, in the host's words —
721
- * the model still speaks in its own voice and persona.
722
- *
723
- * Returns whether the request was DELIVERED, which is the one way this deliberately differs from
724
- * {@link SendContextNote} beside it. A context note that is dropped costs the model a little
725
- * perception; an opening line that is dropped is a session that sits in silence, and the host
726
- * needs to be able to tell the two apart. `false` means no session was live (or the instructions
727
- * were empty) — usually a host that asked before the connection reached a speaking state, which
728
- * it can then retry.
729
- */
730
- RequestSpokenOpening(instructions) {
731
- const trimmed = instructions?.trim() ?? '';
732
- if (trimmed.length === 0 || !this.client || !this.isSessionLive()) {
733
- return false;
734
- }
735
- this.requestChannelSpokenResponse(trimmed);
736
- return true;
737
- }
738
- /**
739
- * The active client's current audio activity (per-direction RMS levels + spectrum
740
- * bins), or `null` when no session is live or the driver attached no audio meters.
741
- * Sampled by the overlay's animation-frame loop to drive the audio-reactive orb/EQ —
742
- * a cheap analyser read, never provider traffic.
743
- */
744
- GetAudioActivity() {
745
- return this.client?.GetAudioActivity() ?? null;
746
- }
747
- /**
748
- * The active {@link BaseRealtimeClient} driving the media plane, or null when not connected.
749
- */
750
- get Client() {
751
- return this.client;
752
- }
753
- /**
754
- * Relays a video frame to the underlying realtime client if active.
755
- */
756
- SendVideoFrame(base64Image, mimeType) {
757
- if (!this.client || !this.isSessionLive()) {
758
- return;
759
- }
760
- this.client.SendVideoFrame?.(base64Image, mimeType);
761
- }
762
- /**
763
- * Checks whether a media track is established on the active realtime client.
764
- */
765
- IsTrackEstablished(modality, direction) {
766
- return this.client?.IsTrackEstablished(modality, direction) ?? false;
767
- }
768
- // ── Browser-side call recording ────────────────────────────────────────────
769
- /**
770
- * Reads the per-user recording-consent preference from `MJ: User Settings` (via
771
- * {@link UserInfoEngine}'s synchronous cache). Defensive: any failure resolves to `false`
772
- * (don't record) so a settings hiccup can never opt a user into recording.
773
- */
774
- readPersistedRecordingConsent() {
775
- try {
776
- return UserInfoEngine.Instance.GetSetting(REALTIME_RECORDING_CONSENT_KEY) === 'true';
777
- }
778
- catch {
779
- return false;
780
- }
781
- }
782
- /**
783
- * Starts the browser-side recorder (mic + agent-audio mix). Best-effort — any failure is
784
- * contained so it never disturbs the live call; an unsupported browser simply records
785
- * nothing (the recorder disables itself).
786
- */
787
- startRecording(client) {
788
- try {
789
- if (!this.localStream) {
790
- return;
791
- }
792
- const remoteStream = client.GetRemoteMediaStream?.() ?? null;
793
- const recorder = new RealtimeAudioRecorder();
794
- recorder.Start(this.localStream, remoteStream);
795
- this.recorder = recorder.IsRecording ? recorder : null;
796
- // First turn's audio starts at ~0 (recording begins right as the call goes live). Seed it
797
- // here so the very first turn has a sane start even if its first interim is missed; later
798
- // turns re-stamp from where THEIR audio begins via markTurnAudioStart (handles tool gaps).
799
- // Seeded even though the session clock may already have stamped a start: the clocks have
800
- // different zeros, and a session-clock start carried into recorder-clock offsets would put
801
- // turn one's cue wherever the clocks happen to differ.
802
- this.currentTurnStartMs = recorder.IsRecording ? 0 : null;
803
- this.turnAudioStartCaptured = false;
804
- if (this.recorder) {
805
- // The agent's WebRTC audio track usually lands AFTER Connect() resolves, so `remoteStream`
806
- // above is typically null and we'd capture mic-only. Attach the agent stream whenever it
807
- // arrives (fires immediately if already present) so the recording includes the agent voice.
808
- client.OnRemoteMediaStream?.((stream) => this.recorder?.AttachRemoteStream(stream));
809
- this.startSegmentFlushing();
810
- }
811
- }
812
- catch (error) {
813
- console.warn('[RealtimeSession] Failed to start call recording:', error);
814
- this.recorder = null;
815
- }
816
- }
817
- /** Begins flushing ~15s crash-recovery shards to the server for the duration of the recording. */
818
- startSegmentFlushing() {
819
- this.segmentIndex = 0;
820
- this.segmentTimer = setInterval(() => { void this.flushRecordingSegment(); }, RealtimeSessionService.SegmentFlushMs);
821
- }
822
- /** Stops the periodic crash-recovery shard flush. */
823
- stopSegmentFlushing() {
824
- if (this.segmentTimer) {
825
- clearInterval(this.segmentTimer);
826
- this.segmentTimer = null;
827
- }
828
- }
829
- /**
830
- * Starts telling the server this session is still in use (#3533).
831
- *
832
- * **Why the server cannot work this out on its own.** In the client-direct topology the audio
833
- * goes browser → provider over WebRTC. The server sees the mint, a few channel actions in the
834
- * first seconds, and then nothing at all — so `SessionManager.RecordActivity` stops being
835
- * reached while the conversation is still going. `LastActiveAt` freezes ~45 seconds in, and
836
- * `SessionJanitor` — which cannot distinguish an active call from an abandoned one — force-closes
837
- * it at `closeThresholdMinutes`, mid-sentence, taking the user's surfaces with it. A session
838
- * whose channels are all client-side (whiteboard, media) goes quiet from the server's point of
839
- * view almost immediately.
840
- *
841
- * The browser is the only participant that knows the call is alive, so it is the one that has to
842
- * say so. Raising `closeThresholdMinutes` is not the fix — it just makes the janitor slower at
843
- * its real job (reaping rows orphaned by a crash) without making liveness observable.
844
- *
845
- * The pulse is best-effort by design: a failed beat is logged and skipped, never surfaced to the
846
- * user and never allowed to end the session. Losing one beat costs nothing because the threshold
847
- * is many beats wide; turning a transient network blip into a visible error would be a worse
848
- * failure than the one this fixes. Write amplification is bounded on the server side too, where
849
- * `SessionManager.Heartbeat` coalesces persisted writes.
850
- */
851
- startLivenessPulse() {
852
- this.stopLivenessPulse();
853
- this.livenessTimer = setInterval(() => { void this.pulseLiveness(); }, RealtimeSessionService.LivenessPulseMs);
854
- }
855
- /** Stops the liveness pulse. Idempotent — safe on a session that never started one. */
856
- stopLivenessPulse() {
857
- if (this.livenessTimer) {
858
- clearInterval(this.livenessTimer);
859
- this.livenessTimer = null;
860
- }
861
- }
862
- /**
863
- * One liveness beat. Reads the session id at fire time rather than closing over it, so a beat
864
- * that fires during teardown finds `null` and does nothing instead of resurrecting a closed row.
865
- */
866
- async pulseLiveness() {
867
- const agentSessionId = this.agentSessionId;
868
- if (!agentSessionId) {
869
- return;
870
- }
871
- const mutation = `
872
- mutation AgentSessionHeartbeat($agentSessionId: String!) {
873
- AgentSessionHeartbeat(agentSessionId: $agentSessionId)
874
- }
875
- `;
876
- try {
877
- await this.gql().ExecuteGQL(mutation, { agentSessionId });
878
- }
879
- catch (error) {
880
- // Best-effort: the next beat is 60s away and the janitor threshold is many beats wide.
881
- console.warn('[RealtimeSession] Liveness pulse failed (session continues):', error);
882
- }
883
- }
884
- /**
885
- * Uploads the chunks captured since the last flush as one crash-recovery shard (durability only;
886
- * the canonical file is still the full upload at teardown). Best-effort — never disturbs the call.
887
- */
888
- async flushRecordingSegment() {
889
- const recorder = this.recorder;
890
- const agentSessionId = this.agentSessionId;
891
- if (!recorder || !agentSessionId) {
892
- return;
893
- }
894
- const segment = recorder.SnapshotNewSegment();
895
- if (!segment || segment.size === 0) {
896
- return;
897
- }
898
- try {
899
- const audioBase64 = await this.blobToBase64(segment);
900
- if (!audioBase64) {
901
- return;
902
- }
903
- const index = this.segmentIndex++;
904
- const mutation = `
905
- mutation UploadRealtimeRecordingSegment($agentSessionId: String!, $segmentIndex: Int!, $audioBase64: String!, $mimeType: String!) {
906
- UploadRealtimeRecordingSegment(agentSessionId: $agentSessionId, segmentIndex: $segmentIndex, audioBase64: $audioBase64, mimeType: $mimeType)
907
- }
908
- `;
909
- // Shards are HEADER-LESS raw little-endian PCM16 (mime audio/L16 with the capture sample rate),
910
- // NOT individually-playable WAV — recovery is concatenate-in-order then WAV-wrap. The canonical
911
- // seekable WAV is the consolidated end-of-call upload below.
912
- const shardMime = `audio/L16;rate=${recorder.SampleRate}`;
913
- await this.gql().ExecuteGQL(mutation, { agentSessionId, segmentIndex: index, audioBase64, mimeType: shardMime });
914
- }
915
- catch (error) {
916
- console.warn('[RealtimeSession] Failed to flush recording shard:', error);
917
- }
918
- }
919
- /**
920
- * Stops the active recorder and uploads the captured audio via `UploadRealtimeRecording`.
921
- * Fully best-effort and wrapped in try/catch — recording upload must NEVER block teardown.
922
- * No-op when nothing was recorded or there's no session id to attach the file to.
923
- */
924
- async stopAndUploadRecording(agentSessionId) {
925
- this.stopSegmentFlushing();
926
- const recorder = this.recorder;
927
- this.recorder = null;
928
- this.currentTurnStartMs = null;
929
- this.turnAudioStartCaptured = false;
930
- if (!recorder) {
931
- return;
932
- }
933
- try {
934
- // Capture the recorder MIME (now 'audio/wav') BEFORE Stop() — the getter reads '' once stopped.
935
- const mimeType = recorder.MimeType;
936
- const blob = await recorder.Stop();
937
- // Read the real waveform peaks computed during capture (survives Stop via the recorder's snapshot).
938
- const peaks = recorder.GetPeaks();
939
- if (!blob || blob.size === 0 || !agentSessionId) {
940
- console.warn('[RealtimeSession] ⚠️ recording NOT uploaded — empty blob or no session id.');
941
- return;
942
- }
943
- const audioBase64 = await this.blobToBase64(blob);
944
- if (!audioBase64) {
945
- console.warn('[RealtimeSession] ⚠️ recording NOT uploaded — base64 encoding failed.');
946
- return;
947
- }
948
- await this.uploadRecording(agentSessionId, audioBase64, mimeType, peaks);
949
- }
950
- catch (error) {
951
- console.warn('[RealtimeSession] Failed to stop/upload call recording:', error);
952
- }
953
- }
954
- /**
955
- * Runs the `UploadRealtimeRecording` mutation; failures are logged, never thrown. Sends the
956
- * capture-time waveform `peaks` (max-abs per bucket, normalized 0..1) so the server can persist a
957
- * `peaks.json` sidecar for fast waveform rendering without re-decoding the audio.
958
- */
959
- async uploadRecording(agentSessionId, audioBase64, mimeType, peaks) {
960
- const mutation = `
961
- mutation UploadRealtimeRecording($agentSessionId: String!, $audioBase64: String!, $mimeType: String!, $consent: Boolean, $peaks: [Float!]) {
962
- UploadRealtimeRecording(agentSessionId: $agentSessionId, audioBase64: $audioBase64, mimeType: $mimeType, consent: $consent, peaks: $peaks) {
963
- Success
964
- FileID
965
- ErrorMessage
966
- }
967
- }
968
- `;
969
- const result = await this.gql().ExecuteGQL(mutation, {
970
- agentSessionId,
971
- audioBase64,
972
- mimeType,
973
- consent: true,
974
- peaks
975
- });
976
- const payload = result?.UploadRealtimeRecording;
977
- if (!payload?.Success) {
978
- console.warn(`[RealtimeSession] ❌ recording upload reported failure: ${payload?.ErrorMessage ?? 'unknown error'} (full result: ${JSON.stringify(result)})`);
979
- }
980
- }
981
- /**
982
- * Encodes a {@link Blob} as a base64 string (the data: prefix stripped) via {@link FileReader}.
983
- * Resolves `''` on any read failure so the upload path can bail cleanly.
984
- */
985
- blobToBase64(blob) {
986
- return new Promise((resolve) => {
987
- const reader = new FileReader();
988
- reader.onloadend = () => {
989
- const result = typeof reader.result === 'string' ? reader.result : '';
990
- const commaIndex = result.indexOf(',');
991
- resolve(commaIndex >= 0 ? result.slice(commaIndex + 1) : '');
992
- };
993
- reader.onerror = () => resolve('');
994
- reader.readAsDataURL(blob);
995
- });
996
- }
997
- // ── Interactive channels (registry-driven plugins) ─────────────────────────
998
- /**
999
- * Resolves, instantiates and initializes the session's interactive-channel plugins from
1000
- * the `MJ: AI Agent Channels` registry, publishes them on {@link ActiveChannels$}, and
1001
- * returns their aggregated client-executed tool declarations for the session mint.
1002
- * Tolerant by design: registry/resolution failures degrade to "no channels" — the voice
1003
- * session itself always proceeds.
1004
- */
1005
- async startChannels() {
1006
- const channels = await this.loadActiveChannels();
1007
- for (const plugin of channels) {
1008
- this.initializeChannel(plugin);
1009
- }
1010
- this._activeChannels$.next(channels);
1011
- return channels.flatMap(plugin => plugin.GetToolDefinitions());
1012
- }
1013
- /**
1014
- * Loads the ACTIVE channel definitions from the registry and resolves each row's
1015
- * `ClientPluginClass` through the MJ ClassFactory into a per-session plugin instance —
1016
- * the client-side mirror of how realtime-model drivers resolve from `BaseRealtimeModel`
1017
- * / `BaseRealtimeClient`. Rows whose plugin class isn't registered are skipped (logged),
1018
- * never fatal.
1019
- */
1020
- async loadActiveChannels() {
1021
- const rows = await this.fetchChannelDefinitions();
1022
- const channels = [];
1023
- for (const row of rows) {
1024
- const plugin = this.resolveChannelPlugin(row);
1025
- if (plugin) {
1026
- channels.push(plugin);
1027
- }
1028
- }
1029
- return channels;
1030
- }
1031
- /**
1032
- * Reads the ACTIVE `MJ: AI Agent Channels` rows from {@link AIEngineBase}'s cached
1033
- * `AgentChannels` (provider-scoped engine instance, lazy `Config` — no RunView
1034
- * round-trip; the engine's BaseEntity-event reactivity keeps the registry fresh).
1035
- * Failures are logged and degrade to an empty list — channel availability must
1036
- * never block the voice session.
1037
- */
1038
- async fetchChannelDefinitions() {
1039
- try {
1040
- const engine = AIEngineBase.GetProviderInstance(this.Provider, AIEngineBase);
1041
- await engine.Config(false, undefined, this.Provider);
1042
- return (engine.AgentChannels ?? [])
1043
- .filter(c => c.IsActive)
1044
- .map(c => ({ ID: c.ID, Name: c.Name, ClientPluginClass: c.ClientPluginClass }));
1045
- }
1046
- catch (error) {
1047
- console.warn('[RealtimeSession] Channel registry unavailable — starting with no channels:', error);
1048
- return [];
1049
- }
1050
- }
1051
- /**
1052
- * Resolves one registry row's `ClientPluginClass` via the ClassFactory (registration
1053
- * checked first, exactly like the realtime-client drivers) and instantiates a fresh
1054
- * per-session plugin. Returns `null` (logged) when no plugin is registered for the key
1055
- * — e.g. its Load function was never called or the package isn't included client-side.
1056
- */
1057
- resolveChannelPlugin(row) {
1058
- const key = row.ClientPluginClass?.trim();
1059
- if (!key) {
1060
- console.warn(`[RealtimeSession] Channel '${row.Name}' has no ClientPluginClass — skipping.`);
1061
- return null;
1062
- }
1063
- const registration = MJGlobal.Instance.ClassFactory.GetRegistration(BaseRealtimeChannelClient, key);
1064
- if (!registration) {
1065
- console.warn(`[RealtimeSession] No client plugin registered for channel '${row.Name}' (key '${key}') — skipping.`);
1066
- return null;
1067
- }
1068
- const plugin = MJGlobal.Instance.ClassFactory.CreateInstance(BaseRealtimeChannelClient, key);
1069
- if (!plugin) {
1070
- console.warn(`[RealtimeSession] Failed to instantiate client plugin for channel '${row.Name}' (key '${key}').`);
1071
- return null;
1072
- }
1073
- return plugin;
1074
- }
1075
- /**
1076
- * Wires one plugin into the session: hands it its host context and registers its
1077
- * prefix-routed local tool executor (so `<ToolNamePrefix>*` calls run in the browser
1078
- * through {@link BaseRealtimeChannelClient.ApplyAgentTool}, never the server relay).
1079
- */
1080
- initializeChannel(plugin) {
1081
- plugin.Initialize(this.buildChannelContext(plugin));
1082
- this.RegisterClientToolHandler(plugin.ToolNamePrefix, (toolName, argsJson) => {
1083
- // The agent is ACTING on this channel — surface-discovery signal for the overlay
1084
- // (first activity registers + auto-reveals + focuses the channel tab) before the
1085
- // tool applies. Record the channel as USED so the overlay tabs it (channels other
1086
- // than the whiteboard are tab-less until they're first used).
1087
- this.usedChannelNames.add(plugin.ChannelName);
1088
- this._channelActivity$.next(plugin);
1089
- return plugin.ApplyAgentTool(toolName, argsJson);
1090
- });
1091
- }
1092
- /** Builds the host-services context one channel plugin sees (its only line to the session). */
1093
- buildChannelContext(plugin) {
1094
- // Capture the service in a local so the AgentSessionID getter reads the SERVICE's live
1095
- // field (not the object literal's `this`) every time it's accessed.
1096
- const service = this;
1097
- return {
1098
- AgentName: this.CurrentAgentName,
1099
- // The live session's provider — threaded by channels into MJ-backed surfaces (e.g. the Media
1100
- // channel's mj-storage-media-player / CreateMediaAccessToken). `get` so it stays current.
1101
- get Provider() {
1102
- return service.Provider;
1103
- },
1104
- SendContextNote: (text) => this.SendContextNote(text),
1105
- RequestSpokenResponse: (instructions) => this.requestChannelSpokenResponse(instructions),
1106
- RequestSave: (stateJson) => this.scheduleChannelSave(plugin.ChannelName, stateJson),
1107
- SaveAsArtifact: (name, contentJson) => this.saveChannelArtifact(plugin.ChannelName, name, contentJson),
1108
- SetFocusMode: (on) => this._channelFocus$.next({ Channel: plugin, Focused: on }),
1109
- // Live session id + GraphQL escape hatch for SERVER-BACKED channels (e.g. Remote
1110
- // Browser). `get` so a channel always reads the CURRENT id — it's null at Initialize
1111
- // (the plugin is built before mintSession resolves) and set once the session is live.
1112
- get AgentSessionID() {
1113
- return service.agentSessionId;
1114
- },
1115
- ExecuteServerAction: (query, variables) => this.executeChannelServerAction(query, variables),
1116
- // App-context stream + client-tool execution for the headless ClientContextChannel. The host
1117
- // (Explorer) feeds both; absent on hosts that supply no app context / register no client tools.
1118
- AppContext$: this.AppContext$,
1119
- ExecuteClientTool: (name, params) => this.executeAppClientTool(name, params),
1120
- get Client() {
1121
- return service.client;
1122
- },
1123
- SendVideoFrame: (base64Image, mimeType) => this.SendVideoFrame(base64Image, mimeType),
1124
- IsTrackEstablished: (modality, direction) => this.IsTrackEstablished(modality, direction),
1125
- };
1126
- }
1127
- /**
1128
- * Host registry of surface CLIENT TOOLS (Name → handler), fed by the host (Explorer) from the
1129
- * active surface's `SetAgentClientTools`. The headless ClientContextChannel's `ContextTool` proxy
1130
- * executes against this via {@link executeAppClientTool}. Keys are lower-cased for case-insensitive
1131
- * model-supplied action names.
1132
- */
1133
- appClientToolHandlers = new Map();
1134
- /**
1135
- * Replaces the set of host-registered surface client tools the realtime ContextTool can execute.
1136
- * The host calls this at session start and whenever the active surface's tool set changes (the
1137
- * continuous-capability half of client-context delivery). Passing `[]` clears them.
1138
- *
1139
- * @param tools The current surface client tools (name + handler). Descriptions/schemas ride the
1140
- * app-context manifest separately; only the executable handler is needed here.
1141
- */
1142
- RegisterAppClientTools(tools) {
1143
- this.appClientToolHandlers.clear();
1144
- for (const tool of tools) {
1145
- if (tool?.Name && typeof tool.Handler === 'function') {
1146
- this.appClientToolHandlers.set(tool.Name.trim().toLowerCase(), tool.Handler);
1147
- }
1148
- }
1149
- }
1150
- /**
1151
- * Executes a host-registered surface client tool by name (the {@link RealtimeChannelContext.ExecuteClientTool}
1152
- * implementation). Tolerant: an unknown tool or a thrown handler resolves to a structured
1153
- * `Success: false` result the channel narrates — never throws.
1154
- *
1155
- * @param name The tool name (the model's `action`).
1156
- * @param params The tool parameters.
1157
- * @returns A structured result for the channel to serialize back to the model.
1158
- */
1159
- async executeAppClientTool(name, params) {
1160
- const handler = this.appClientToolHandlers.get((name ?? '').trim().toLowerCase());
1161
- if (!handler) {
1162
- const available = Array.from(this.appClientToolHandlers.keys()).join(', ');
1163
- return {
1164
- Success: false,
1165
- ErrorMessage: `No client tool named "${name}" is available on this surface. Available: ${available || '(none)'}.`
1166
- };
1167
- }
1168
- try {
1169
- const result = await handler(params ?? {});
1170
- return { Success: true, Result: result };
1171
- }
1172
- catch (error) {
1173
- return { Success: false, ErrorMessage: error instanceof Error ? error.message : String(error) };
1174
- }
1175
- }
1176
- /**
1177
- * Runs a channel-specific GraphQL operation through the live session's provider (the
1178
- * {@link RealtimeChannelContext.ExecuteServerAction} implementation). Best-effort: any
1179
- * transport/server error is logged and resolves to `null` so the calling channel can map
1180
- * the failure to a model-readable result string without `try/catch`.
1181
- */
1182
- async executeChannelServerAction(query, variables) {
1183
- try {
1184
- const result = await this.gql().ExecuteGQL(query, variables);
1185
- return result ?? null;
1186
- }
1187
- catch (error) {
1188
- console.error('[RealtimeSession] Channel server action failed:', error);
1189
- return null;
1190
- }
1191
- }
1192
- /**
1193
- * A channel asked the live model to SPEAK in reaction to channel input (e.g. a widget
1194
- * submission) — routed through the client's spoken-update channel. No-op when the
1195
- * session isn't live; empty instructions are dropped.
1196
- */
1197
- requestChannelSpokenResponse(instructions) {
1198
- const trimmed = instructions?.trim() ?? '';
1199
- if (trimmed.length === 0 || !this.client || !this.isSessionLive()) {
1200
- return;
1201
- }
1202
- this.client.RequestSpokenUpdate(trimmed);
1203
- }
1204
- /**
1205
- * Applies the PRIOR session's saved channel states (resume continuity): parses the
1206
- * server-supplied map and offers each entry to the matching active plugin via
1207
- * {@link BaseRealtimeChannelClient.RestoreState}. Fully tolerant — malformed payloads,
1208
- * unknown channels, and plugin rejections are logged and skipped; the session start is
1209
- * never affected.
1210
- */
1211
- applyPriorChannelStates(statesJson) {
1212
- if (!statesJson) {
1213
- return;
1214
- }
1215
- let states;
1216
- try {
1217
- const parsed = JSON.parse(statesJson);
1218
- if (parsed === null || typeof parsed !== 'object') {
1219
- return;
1220
- }
1221
- states = parsed;
1222
- }
1223
- catch {
1224
- console.warn('[RealtimeSession] PriorChannelStatesJson was malformed — starting channels fresh');
1225
- return;
1226
- }
1227
- for (const plugin of this._activeChannels$.value) {
1228
- const state = states[plugin.ChannelName];
1229
- if (typeof state === 'string' && state.length > 0) {
1230
- try {
1231
- const restored = plugin.RestoreState(state);
1232
- if (!restored) {
1233
- console.warn(`[RealtimeSession] Channel '${plugin.ChannelName}' declined its prior-session state — starting fresh`);
1234
- }
1235
- }
1236
- catch (error) {
1237
- console.warn(`[RealtimeSession] Channel '${plugin.ChannelName}' restore threw — starting fresh`, error);
1238
- }
1239
- }
1240
- }
1241
- }
1242
- /**
1243
- * Persists a channel's state as a first-class versioned artifact (`MJ: Artifacts`) via the
1244
- * `SaveSessionChannelArtifact` mutation — the channel-context capability behind e.g. the
1245
- * whiteboard's "Save to artifacts". Best-effort: returns the created Artifact ID, or null
1246
- * on any failure (logged, never thrown). Uses the live session id, falling back to the
1247
- * teardown-captured one so "save my board" works right after the call ends.
1248
- */
1249
- async saveChannelArtifact(channelName, name, contentJson) {
1250
- const sessionId = this.agentSessionId ?? this.lastKnownSessionIdForSaves();
1251
- if (!sessionId || !name.trim() || !contentJson) {
1252
- return null;
1253
- }
1254
- try {
1255
- const result = await this.gql().ExecuteGQL(`mutation SaveSessionChannelArtifact($agentSessionId: String!, $channelName: String!, $name: String!, $contentJson: String!) {
1256
- SaveSessionChannelArtifact(agentSessionId: $agentSessionId, channelName: $channelName, name: $name, contentJson: $contentJson) {
1257
- Success
1258
- ErrorMessage
1259
- ArtifactID
1260
- ArtifactVersionID
1261
- }
1262
- }`, { agentSessionId: sessionId, channelName, name: name.trim(), contentJson });
1263
- const payload = result?.SaveSessionChannelArtifact;
1264
- if (!payload?.Success) {
1265
- console.warn(`[RealtimeSession] Save-as-artifact failed for '${channelName}': ${payload?.ErrorMessage ?? 'unknown error'}`);
1266
- return null;
1267
- }
1268
- return payload.ArtifactID ?? null;
1269
- }
1270
- catch (error) {
1271
- console.warn(`[RealtimeSession] Save-as-artifact errored for '${channelName}':`, error);
1272
- return null;
1273
- }
1274
- }
1275
- /** Most recent session id captured by the save pipeline (post-teardown saves). */
1276
- lastKnownSessionIdForSaves() {
1277
- for (const pending of this.pendingChannelSaves.values()) {
1278
- if (pending.SessionID) {
1279
- return pending.SessionID;
1280
- }
1281
- }
1282
- return null;
1283
- }
1284
- /**
1285
- * Schedules the DEBOUNCED state-of-record save for a channel: each request replaces the
1286
- * pending payload (latest state wins) and re-arms the timer; the session id is captured
1287
- * while live so the teardown flush can persist onto the just-closed session.
1288
- */
1289
- scheduleChannelSave(channelName, stateJson) {
1290
- const pending = this.pendingChannelSaves.get(channelName);
1291
- if (pending) {
1292
- clearTimeout(pending.Timer);
1293
- }
1294
- this.pendingChannelSaves.set(channelName, {
1295
- Timer: setTimeout(() => this.flushChannelSave(channelName), RealtimeSessionService.ChannelSaveDebounceMs),
1296
- StateJson: stateJson,
1297
- SessionID: this.agentSessionId ?? pending?.SessionID ?? null
1298
- });
1299
- }
1300
- /** Fires one pending channel save (best-effort; {@link SaveChannelState} logs failures). */
1301
- flushChannelSave(channelName) {
1302
- const pending = this.pendingChannelSaves.get(channelName);
1303
- if (!pending) {
1304
- return;
1305
- }
1306
- this.pendingChannelSaves.delete(channelName);
1307
- clearTimeout(pending.Timer);
1308
- void this.SaveChannelState(channelName, pending.StateJson, pending.SessionID);
1309
- }
1310
- /** Final teardown flush: persist every channel's unsaved state immediately. */
1311
- flushAllChannelSaves() {
1312
- for (const channelName of [...this.pendingChannelSaves.keys()]) {
1313
- this.flushChannelSave(channelName);
1314
- }
1315
- }
1316
- /** Disposes all channel plugins (errors contained per plugin) and clears the live set. */
1317
- disposeChannels() {
1318
- for (const plugin of this._activeChannels$.value) {
1319
- try {
1320
- plugin.Dispose();
1321
- }
1322
- catch (error) {
1323
- console.error(`[RealtimeSession] Channel '${plugin.ChannelName}' Dispose failed:`, error);
1324
- }
1325
- }
1326
- if (this._activeChannels$.value.length > 0) {
1327
- this._activeChannels$.next([]);
1328
- }
1329
- this.usedChannelNames.clear();
1330
- }
1331
- // ── Realtime client resolution + wiring ────────────────────────────────────
1332
- /**
1333
- * Resolves the provider-direct realtime client for `provider` through the MJ
1334
- * ClassFactory — the client-side mirror of how server drivers are resolved from
1335
- * `BaseRealtimeModel`. Throws a clear error when no driver is registered for the
1336
- * provider (e.g. its Load function was never called).
1337
- */
1338
- createRealtimeClient(provider) {
1339
- const registration = MJGlobal.Instance.ClassFactory.GetRegistration(BaseRealtimeClient, provider);
1340
- if (!registration) {
1341
- throw new Error(`No realtime client registered for provider '${provider}'. ` +
1342
- `Ensure the provider's client driver package is imported and its Load function called.`);
1343
- }
1344
- const client = MJGlobal.Instance.ClassFactory.CreateInstance(BaseRealtimeClient, provider);
1345
- if (!client) {
1346
- throw new Error(`Failed to instantiate the realtime client for provider '${provider}'`);
1347
- }
1348
- return client;
1349
- }
1350
- /**
1351
- * Builds the client-direct session config the realtime client connects with.
1352
- * Aggregates tracks sourced by active channels into `requestedTracks` so the driver
1353
- * can negotiate them (e.g., establishing inbound video streaming for Whiteboard / RemoteBrowser).
1354
- */
1355
- buildClientConfig(session) {
1356
- const sessionConfig = this.parseSessionConfig(session.SessionConfigJson);
1357
- const channelTracks = this._activeChannels$.value.flatMap((c) => c.GetSourcedTracks());
1358
- if (channelTracks.length > 0) {
1359
- // `requestedTracks` crosses a JSON boundary — the driver reads it back out of the session
1360
- // config bag (`GeminiRealtimeClient.parseSessionConfig`). A `RealtimeTrackDescriptor` is NOT
1361
- // structurally a `JSONValue`: it has no index signature and `UsageBasis` is readonly, so the
1362
- // conversion is written out rather than asserted. Dedupe key and precedence are unchanged —
1363
- // audio floor first, then anything the mint supplied, then the channels' own tracks.
1364
- const existing = Array.isArray(sessionConfig['requestedTracks'])
1365
- ? sessionConfig['requestedTracks']
1366
- : [];
1367
- const trackMap = new Map();
1368
- for (const t of DEFAULT_REALTIME_AUDIO_TRACKS) {
1369
- trackMap.set(`${t.Direction}:${t.Modality}`, trackDescriptorToJSON(t));
1370
- }
1371
- for (const raw of existing) {
1372
- const key = trackKeyFromJSON(raw);
1373
- if (key) {
1374
- trackMap.set(key, raw);
1375
- }
1376
- }
1377
- for (const t of channelTracks) {
1378
- trackMap.set(`${t.Direction}:${t.Modality}`, trackDescriptorToJSON(t));
1379
- }
1380
- sessionConfig['requestedTracks'] = Array.from(trackMap.values());
1381
- }
1382
- return {
1383
- Provider: session.Provider,
1384
- Model: session.Model,
1385
- EphemeralToken: session.EphemeralToken,
1386
- ExpiresAt: session.ExpiresAt,
1387
- SessionConfig: sessionConfig
1388
- };
1389
- }
1390
- /**
1391
- * Parses the server-built session config JSON. On failure, logs and returns an empty
1392
- * object — the client treats an empty config as "nothing to apply", so the session
1393
- * still opens (mirroring the prior behavior of skipping the config update).
1394
- */
1395
- parseSessionConfig(sessionConfigJson) {
1396
- if (!sessionConfigJson) {
1397
- return {};
1398
- }
1399
- try {
1400
- return JSON.parse(sessionConfigJson);
1401
- }
1402
- catch (error) {
1403
- console.error('[RealtimeSession] Failed to parse/apply SessionConfigJson:', error);
1404
- return {};
1405
- }
1406
- }
1407
- /** Subscribes this service's policy handlers to the realtime client's events. */
1408
- wireClientHandlers(client) {
1409
- client.OnStateChange((state) => this.onClientStateChange(state));
1410
- client.OnTranscript((transcript) => {
1411
- void this.onClientTranscript(transcript);
1412
- });
1413
- client.OnToolCall((call) => {
1414
- void this.handleToolCall(call);
1415
- });
1416
- client.OnError((error) => {
1417
- console.error('[RealtimeSession] Provider error event:', JSON.stringify(error), error);
1418
- });
1419
- // Usage telemetry: accumulate the driver's per-response token DELTAS and relay them to
1420
- // the server (onto the co-agent AIPromptRun) debounced + once at teardown. Providers
1421
- // without usage events simply never emit — registering is always safe.
1422
- client.OnUsage((usage) => this.onUsageDelta(usage));
1423
- // TRUE BARGE-IN (user input cut off active model output — the driver already stopped
1424
- // the speech): the user took the floor, so any pending/queued progress narration is
1425
- // stale — cancel it; the next progress event re-schedules at the session-global pace.
1426
- // HOST POLICY (deliberate): barge-in does NOT abort in-flight delegated runs — the
1427
- // narration design EXPECTS the user to keep talking while delegated work runs, so
1428
- // killing the work on speech would cancel exactly the jobs the user asked for.
1429
- // Explicit cancellation is a separate, intentional act: the overlay's per-card ✕
1430
- // calls {@link CancelDelegation} (server cancel channel) instead.
1431
- client.OnInterruption(() => {
1432
- this.cancelPendingNarration();
1433
- });
1434
- }
1435
- /** Maps a client state event onto the UI connection state. */
1436
- onClientStateChange(state) {
1437
- const mapped = this.mapClientState(state);
1438
- if (mapped) {
1439
- this._connectionState$.next(mapped);
1440
- }
1441
- }
1442
- /**
1443
- * Translates {@link RealtimeClientState} into {@link RealtimeConnectionState}. `'connected'`
1444
- * is suppressed (the UI stays 'connecting' until the control channel opens → 'listening'),
1445
- * and `'closed'` never overwrites a terminal 'error' the service itself recorded.
1446
- */
1447
- mapClientState(state) {
1448
- switch (state) {
1449
- case 'connecting':
1450
- return 'connecting';
1451
- case 'connected':
1452
- return null;
1453
- case 'listening':
1454
- return 'listening';
1455
- case 'speaking':
1456
- return 'speaking';
1457
- case 'error':
1458
- return 'error';
1459
- case 'closed':
1460
- return this._connectionState$.value === 'error' ? null : 'closed';
1461
- }
1462
- }
1463
- /** True when the live control channel is usable (open and not torn down / failed). */
1464
- isSessionLive() {
1465
- const state = this._connectionState$.value;
1466
- return state === 'listening' || state === 'speaking' || state === 'thinking';
1467
- }
1468
- // ── Transcript policy ──────────────────────────────────────────────────────
1469
- /**
1470
- * Applies transcript policy to client transcript events. Interim deltas don't become
1471
- * captions/turns (the client already drives the speaking state) but DO mark this turn's
1472
- * audio-start offset against the recording (the first interim fires as the audio/text
1473
- * starts flowing — see {@link markTurnAudioStart}). Final NORMAL assistant turns become
1474
- * captions + persisted transcripts; final NARRATION turns are EPHEMERAL by product
1475
- * decision — emitted on {@link DelegationNarration$} only, never a caption, never
1476
- * relayed/persisted. User turns ride the caption + relay path.
1477
- */
1478
- async onClientTranscript(transcript) {
1479
- if (!transcript.IsFinal) {
1480
- // First interim of a NEW turn = that turn's audio is starting NOW. Stamp the
1481
- // recording-relative start here so a turn whose audio begins AFTER a tool-call /
1482
- // silence gap is timed where its audio really is — not inherited from the prior
1483
- // turn's end. Narration interims are ephemeral and excluded (Kind guard inside).
1484
- this.markTurnAudioStart(transcript.Kind);
1485
- if (transcript.Role === 'User') {
1486
- if (!this.hasActiveInterimUserCaption) {
1487
- if (transcript.Text.trim().length === 0) {
1488
- return;
1489
- }
1490
- this.hasActiveInterimUserCaption = true;
1491
- this.pendingUserCaption = transcript.Text;
1492
- this.appendCaption({ Role: 'User', Text: this.pendingUserCaption });
1493
- }
1494
- else {
1495
- this.pendingUserCaption += transcript.Text;
1496
- this.replaceLastCaption('User', this.pendingUserCaption);
1497
- }
1498
- }
1499
- return;
1500
- }
1501
- if (transcript.Role === 'Assistant') {
1502
- this.hasActiveInterimUserCaption = false;
1503
- this.pendingUserCaption = '';
1504
- if (transcript.Kind === 'narration') {
1505
- if (transcript.IsThought) {
1506
- this._thoughtNarration$.next({
1507
- CallID: 'thought-session',
1508
- Text: transcript.Text,
1509
- IsFinal: transcript.IsFinal ?? true,
1510
- });
1511
- }
1512
- else {
1513
- this._delegationNarration$.next({ Text: transcript.Text });
1514
- // Remember what was actually SAID so later updates build on it instead of repeating.
1515
- this.spokenNarrations.push(transcript.Text);
1516
- if (this.spokenNarrations.length > RealtimeSessionService.MaxPriorNarrations) {
1517
- this.spokenNarrations.shift();
1518
- }
1519
- }
1520
- }
1521
- else if (transcript.ReplacesPrevious) {
1522
- // CORRECTION (e.g. ElevenLabs post-barge-in re-finalization): this final
1523
- // SUPERSEDES the previous final assistant turn — replace the caption in place
1524
- // and tell the server to update the persisted turn instead of appending.
1525
- this.replaceLastCaption('Assistant', transcript.Text);
1526
- await this.relayTranscript('assistant', transcript.Text, true);
1527
- }
1528
- else {
1529
- this.appendCaption({ Role: 'Assistant', Text: transcript.Text });
1530
- await this.relayTranscript('assistant', transcript.Text);
1531
- }
1532
- }
1533
- else if (this.hasActiveInterimUserCaption) {
1534
- this.hasActiveInterimUserCaption = false;
1535
- this.pendingUserCaption = '';
1536
- if (transcript.Text.trim().length === 0) {
1537
- return;
1538
- }
1539
- this.replaceLastCaption('User', transcript.Text);
1540
- if (this.firstUserTranscript === null) {
1541
- this.firstUserTranscript = transcript.Text;
1542
- }
1543
- await this.relayTranscript('user', transcript.Text);
1544
- }
1545
- else if (transcript.ReplacesPrevious) {
1546
- // STREAMING user transcription: providers like Grok and OpenAI Live emit the growing utterance as repeated
1547
- // events (each the full text so far), flagging all but the first ReplacesPrevious. Update the
1548
- // in-place User caption + persisted turn instead of stacking a new bubble per increment — the
1549
- // same correction semantics the assistant branch uses. (Classic OpenAI Realtime sends one final → the else path.)
1550
- if (transcript.Text.trim().length === 0) {
1551
- return;
1552
- }
1553
- this.replaceLastCaption('User', transcript.Text);
1554
- await this.relayTranscript('user', transcript.Text, true);
1555
- }
1556
- else {
1557
- await this.onUserTranscript(transcript.Text);
1558
- }
1559
- }
1560
- /**
1561
- * Stamps the recording-relative offset at which the IN-FLIGHT turn's audio actually began,
1562
- * the moment that turn's audio/text first starts flowing (its FIRST interim transcript).
1563
- *
1564
- * This is the fix for transcript cues drifting out of sync with the audio when a tool-call /
1565
- * silence gap sits between turns: the old model inherited the next turn's start from the
1566
- * PREVIOUS turn's end (assumes contiguous turns), so a post-gap turn's cue pointed ~gap-length
1567
- * too early. Capturing the start where the audio truly begins keeps the cue aligned.
1568
- *
1569
- * Guards:
1570
- * - only when recording ({@link recorder} present),
1571
- * - only ONCE per turn ({@link turnAudioStartCaptured}) so mid-turn interim deltas don't move it,
1572
- * - NORMAL turns only — NARRATION interims are ephemeral and never persisted, so they must not
1573
- * claim the next real turn's start slot.
1574
- *
1575
- * Works for any role whose driver surfaces interim deltas (all drivers for the assistant; the
1576
- * relevant case here — the post-tool-gap assistant answer — and user-interim drivers like
1577
- * Gemini/AssemblyAI). For final-only user turns (OpenAI/xAI/ElevenLabs) no interim arrives, so
1578
- * {@link relayTranscript} falls back to the seeded/prior start — the gap case that drifts is the
1579
- * assistant answer, which always has interims.
1580
- */
1581
- markTurnAudioStart(kind) {
1582
- if (this.turnAudioStartCaptured || kind === 'narration') {
1583
- return;
1584
- }
1585
- const offset = this.nowTurnOffsetMs();
1586
- if (offset === null) {
1587
- return;
1588
- }
1589
- this.currentTurnStartMs = offset;
1590
- this.turnAudioStartCaptured = true;
1591
- }
1592
- /**
1593
- * The current per-turn offset in ms — the RECORDER's clock when one runs (an offset into a
1594
- * seekable file), else the SESSION clock (#3832: orderable and displayable, not seekable),
1595
- * else `null` before any call is live. One function so the two stamp sites cannot disagree
1596
- * about which clock a session is on.
1597
- */
1598
- nowTurnOffsetMs() {
1599
- if (this.recorder) {
1600
- return this.recorder.NowOffsetMs();
1601
- }
1602
- if (this.sessionClockStartMs !== null) {
1603
- return Math.max(0, Math.round(performance.now() - this.sessionClockStartMs));
1604
- }
1605
- return null;
1606
- }
1607
- /**
1608
- * Replaces the LAST caption of `role` in place (correction semantics); falls back to a
1609
- * plain append when no such caption exists yet (e.g. the superseded turn predates this
1610
- * client's caption window).
1611
- */
1612
- replaceLastCaption(role, text) {
1613
- const captions = this._captions$.value;
1614
- for (let i = captions.length - 1; i >= 0; i--) {
1615
- if (captions[i].Role === role) {
1616
- const next = [...captions];
1617
- next[i] = { Role: role, Text: text };
1618
- this._captions$.next(next);
1619
- return;
1620
- }
1621
- }
1622
- this.appendCaption({ Role: role, Text: text });
1623
- }
1624
- /** Finalizes the user turn: push a caption + relay the final transcript. */
1625
- async onUserTranscript(transcript) {
1626
- if (transcript.trim().length === 0) {
1627
- return;
1628
- }
1629
- if (this.firstUserTranscript === null) {
1630
- // First spoken user utterance — the naming seed for a session-created conversation.
1631
- this.firstUserTranscript = transcript;
1632
- }
1633
- this.appendCaption({ Role: 'User', Text: transcript });
1634
- await this.relayTranscript('user', transcript);
1635
- }
1636
- // ── Tool calling ───────────────────────────────────────────────────────────
1637
- /**
1638
- * Routes a provider tool call: names matching a registered client-tool prefix execute
1639
- * LOCALLY (UI tools — see {@link RegisterClientToolHandler}); everything else executes on
1640
- * the MJ server. Either way the result feeds back to the model via
1641
- * {@link BaseRealtimeClient.SendToolResult} so it speaks the outcome.
1642
- */
1643
- async handleToolCall(call) {
1644
- const clientHandler = this.findClientToolHandler(call.ToolName);
1645
- if (clientHandler) {
1646
- // Local UI tool: no server relay, no 'thinking' turn-state / narration burst, and intentionally
1647
- // NO thread card — these are fast, in-browser surface mutations (e.g. drawing on the whiteboard)
1648
- // whose visual effects are immediately visible on the dedicated canvas/surface.
1649
- const resultJson = await this.executeClientTool(clientHandler, call);
1650
- this.client?.SendToolResult(call.CallID, resultJson);
1651
- // Observability: record the channel tool call on the co-agent's run (run-only — NOT a chat
1652
- // turn). Without this the run shows speech but never the browser_/Whiteboard_ actions the
1653
- // co-agent took. Fire-and-forget; never disturbs the live surface mutation.
1654
- void this.relayToolTurn(call.ToolName, call.ArgumentsJson, resultJson);
1655
- return;
1656
- }
1657
- this._connectionState$.next('thinking');
1658
- if (this.inFlightCallIds.size === 0) {
1659
- // A fresh delegation burst: anchor the first-update delay and clear the digest
1660
- // buffer. Deliberately NOT reset: lastDelegationNarrationAt (the 8s spacing floor
1661
- // is SESSION-global — sequential tool calls seconds apart must not re-arm the
1662
- // faster first-update path, which read as "no debounce") and spokenNarrations
1663
- // (so the story never repeats across closely-spaced calls).
1664
- this.delegationBurstStartedAt = Date.now();
1665
- this.narrationCount = 0;
1666
- this.pendingNarrationMessages = [];
1667
- this.lastNarratedTail = '';
1668
- }
1669
- this.inFlightCallIds.add(call.CallID);
1670
- if (call.ToolName !== 'invoke-target-agent') {
1671
- // Direct action: emit synthetic progress immediately so the conversation thread
1672
- // and activity rail render an active "working" action card while the tool executes.
1673
- this._delegationProgress$.next({
1674
- CallID: call.CallID,
1675
- ToolName: call.ToolName,
1676
- Step: 'direct_action',
1677
- Message: `Executing ${FormatToolName(call.ToolName)}`
1678
- });
1679
- }
1680
- try {
1681
- const resultJson = await this.executeSessionTool(call.CallID, call.ToolName, call.ArgumentsJson);
1682
- this.emitDelegationResult(call.CallID, resultJson, call.ToolName);
1683
- this.client?.SendToolResult(call.CallID, resultJson);
1684
- }
1685
- catch (error) {
1686
- console.error('[RealtimeSession] Tool execution failed:', error);
1687
- // Feed the error back so the model can narrate it rather than going silent.
1688
- // success:false matters: ParseDelegationResultJson treats anything else as
1689
- // success, which would flip the overlay's working card to a SUCCESS card
1690
- // carrying the error text (matches the server broker's failure shape).
1691
- const errorJson = JSON.stringify({
1692
- success: false,
1693
- error: error instanceof Error ? error.message : String(error)
1694
- });
1695
- this.emitDelegationResult(call.CallID, errorJson, call.ToolName);
1696
- this.client?.SendToolResult(call.CallID, errorJson);
1697
- }
1698
- }
1699
- /** Finds the registered client-tool handler whose prefix matches `toolName`, or `null`. */
1700
- findClientToolHandler(toolName) {
1701
- for (const [prefix, handler] of this.clientToolHandlers) {
1702
- if (toolName.startsWith(prefix)) {
1703
- return handler;
1704
- }
1705
- }
1706
- return null;
1707
- }
1708
- /**
1709
- * Executes one client-tool call through its handler, wrapping any thrown error into a
1710
- * `{ success: false, error }` JSON payload so the model can narrate the failure instead of
1711
- * the call going silent.
1712
- */
1713
- async executeClientTool(handler, call) {
1714
- try {
1715
- return await handler(call.ToolName, call.ArgumentsJson);
1716
- }
1717
- catch (error) {
1718
- console.error('[RealtimeSession] Client tool execution failed:', error);
1719
- return JSON.stringify({
1720
- success: false,
1721
- error: error instanceof Error ? error.message : String(error)
1722
- });
1723
- }
1724
- }
1725
- /**
1726
- * Emits a delegation result so the overlay's "working" card flips to a result card with real
1727
- * content. Parses the broker's `{success, output, runId}` | `{success:false, error}` shape via
1728
- * {@link ParseDelegationResultJson}; if it isn't JSON, surfaces the raw string. The `runId`
1729
- * (the delegated `MJ: AI Agent Runs` record) rides along as {@link RealtimeDelegationResult.RunID}
1730
- * for the overlay's dev links, and any `artifacts` ride along as {@link RealtimeDelegationResult.Artifacts}
1731
- * for the surface panel's artifact tabs.
1732
- */
1733
- emitDelegationResult(callId, resultJson, toolName) {
1734
- // The result will be spoken next — a deferred interim update is now pointless
1735
- // (this is what keeps fast agents like Sage from narrating over their own answer),
1736
- // and any progress still in the PubSub pipe for this call is stale.
1737
- this.inFlightCallIds.delete(callId);
1738
- this.cancelPendingNarration();
1739
- if (this.cancelledCallIds.delete(callId)) {
1740
- // The user explicitly cancelled this call: its card already flipped to the
1741
- // "Cancelled by user" failed result, so the aborted run's late outcome must not
1742
- // overwrite it. (The tool result still flows back to the model via the caller.)
1743
- return;
1744
- }
1745
- const parsed = ParseDelegationResultJson(resultJson);
1746
- this._delegationResult$.next({
1747
- CallID: callId,
1748
- ToolName: toolName,
1749
- Success: parsed.Success,
1750
- Output: parsed.Output,
1751
- RunID: parsed.RunID,
1752
- Artifacts: parsed.Artifacts
1753
- });
1754
- }
1755
- // ── Explicit delegation cancellation (server cancel channel) ───────────────
1756
- /**
1757
- * Cancels ONE in-flight delegated tool call — the overlay's per-card ✕ affordance.
1758
- *
1759
- * EXPLICIT USER INTENT ONLY (deliberate host policy): true barge-in never aborts
1760
- * delegations — the narration design expects the user to talk while delegated work runs.
1761
- * Calls the `CancelRealtimeSessionTool` mutation (ownership-gated server-side); when the
1762
- * server reports it aborted the run, the card is flipped immediately to a FAILED
1763
- * "Cancelled by user" result and the eventual late result from the aborted run is
1764
- * suppressed (see {@link emitDelegationResult}).
1765
- *
1766
- * @returns `true` when the server aborted the in-flight run; `false` when there was
1767
- * nothing to cancel (the work finished first — its real result is already racing in)
1768
- * or the mutation failed (logged, never thrown).
1769
- */
1770
- async CancelDelegation(callId) {
1771
- if (!this.agentSessionId || !this.inFlightCallIds.has(callId)) {
1772
- return false;
1773
- }
1774
- const aborted = await this.cancelSessionTool(callId);
1775
- if (aborted <= 0) {
1776
- return false; // finished first / nothing in flight server-side — let the real result land
1777
- }
1778
- this.surfaceUserCancellation(callId);
1779
- return true;
1780
- }
1781
- /**
1782
- * Cancels EVERY in-flight delegated tool call for the active session (callId-less form of
1783
- * the `CancelRealtimeSessionTool` mutation). Exposed for host policies that need a
1784
- * sweep-cancel (e.g. an explicit "stop everything" affordance) — NOT wired to barge-in,
1785
- * by the same deliberate policy as {@link CancelDelegation}.
1786
- *
1787
- * @returns The number of in-flight runs the server aborted (0 when nothing was tracked
1788
- * in flight client-side, nothing was in flight server-side, or the mutation failed).
1789
- */
1790
- async CancelInFlightDelegations() {
1791
- if (!this.agentSessionId || this.inFlightCallIds.size === 0) {
1792
- return 0;
1793
- }
1794
- const aborted = await this.cancelSessionTool(null);
1795
- if (aborted <= 0) {
1796
- return 0;
1797
- }
1798
- for (const callId of [...this.inFlightCallIds]) {
1799
- this.surfaceUserCancellation(callId);
1800
- }
1801
- return aborted;
1802
- }
1803
- /** Flips a cancelled call's card to the failed "Cancelled by user" result and suppresses the late real result. */
1804
- surfaceUserCancellation(callId) {
1805
- this.inFlightCallIds.delete(callId);
1806
- this.cancelledCallIds.add(callId);
1807
- this.cancelPendingNarration();
1808
- this._delegationResult$.next({
1809
- CallID: callId,
1810
- Success: false,
1811
- Output: 'Cancelled by user'
1812
- });
1813
- }
1814
- /**
1815
- * Calls the `CancelRealtimeSessionTool` mutation and unwraps its structured
1816
- * `{ AbortedCount, Success, ErrorMessage }` result. Returns the aborted count —
1817
- * 0 on a structured failure or a thrown transport error (both logged, never thrown).
1818
- */
1819
- async cancelSessionTool(callId) {
1820
- try {
1821
- const mutation = `
1822
- mutation CancelRealtimeSessionTool($agentSessionId: String!, $callId: String) {
1823
- CancelRealtimeSessionTool(agentSessionId: $agentSessionId, callId: $callId) {
1824
- AbortedCount
1825
- Success
1826
- ErrorMessage
1827
- }
1828
- }
1829
- `;
1830
- const result = await this.gql().ExecuteGQL(mutation, { agentSessionId: this.agentSessionId, callId });
1831
- const payload = result?.CancelRealtimeSessionTool;
1832
- if (!payload?.Success) {
1833
- console.warn(`[RealtimeSession] Cancel reported failure: ${payload?.ErrorMessage ?? 'unknown error'}`);
1834
- return 0;
1835
- }
1836
- return typeof payload.AbortedCount === 'number' ? payload.AbortedCount : 0;
1837
- }
1838
- catch (error) {
1839
- console.error('[RealtimeSession] Failed to cancel in-flight delegation(s):', error);
1840
- return 0;
1841
- }
1842
- }
1843
- // ── Session minting (GraphQL) ──────────────────────────────────────────────
1844
- /** Calls the `StartRealtimeClientSession` mutation to obtain an ephemeral token + config. */
1845
- async mintSession(targetAgentId, conversationId, lastSessionId, preferredModelId, clientTools, coAgentId, configOverridesJson, recordingConsent, recordingStartedAt, mediaCollectionId, applicationId, appContext) {
1846
- const mutation = `
1847
- mutation StartRealtimeClientSession($targetAgentId: String!, $conversationId: String, $lastSessionId: String, $preferredModelId: String, $clientToolsJson: String, $coAgentId: String, $configOverridesJson: String, $recordingConsent: Boolean, $recordingStartedAt: String, $mediaCollectionId: String, $applicationId: String, $appContextJson: String) {
1848
- StartRealtimeClientSession(targetAgentId: $targetAgentId, conversationId: $conversationId, lastSessionId: $lastSessionId, preferredModelId: $preferredModelId, clientToolsJson: $clientToolsJson, coAgentId: $coAgentId, configOverridesJson: $configOverridesJson, recordingConsent: $recordingConsent, recordingStartedAt: $recordingStartedAt, mediaCollectionId: $mediaCollectionId, applicationId: $applicationId, appContextJson: $appContextJson) {
1849
- AgentSessionId
1850
- ConversationId
1851
- Provider
1852
- Model
1853
- EphemeralToken
1854
- ExpiresAt
1855
- SessionConfigJson
1856
- ModelName
1857
- NarrationInstructionsTemplate
1858
- PriorChannelStatesJson
1859
- }
1860
- }
1861
- `;
1862
- const variables = {
1863
- targetAgentId,
1864
- conversationId: conversationId ?? null,
1865
- lastSessionId: lastSessionId ?? null,
1866
- preferredModelId: preferredModelId ?? null,
1867
- clientToolsJson: clientTools && clientTools.length > 0 ? JSON.stringify(clientTools) : null,
1868
- coAgentId: coAgentId ?? null,
1869
- configOverridesJson: configOverridesJson ?? null,
1870
- recordingConsent: recordingConsent ?? false,
1871
- recordingStartedAt: recordingStartedAt ?? null,
1872
- mediaCollectionId: mediaCollectionId ?? null,
1873
- applicationId: applicationId ?? null,
1874
- appContextJson: appContext ? JSON.stringify(appContext) : null
1875
- };
1876
- const result = await this.gql().ExecuteGQL(mutation, variables);
1877
- const payload = result?.StartRealtimeClientSession;
1878
- if (!payload?.EphemeralToken) {
1879
- throw new Error('StartRealtimeClientSession returned no ephemeral token');
1880
- }
1881
- return payload;
1882
- }
1883
- /** Calls the `ExecuteRealtimeSessionTool` mutation; returns the ResultJson string. */
1884
- async executeSessionTool(callId, toolName, argsJson) {
1885
- if (!this.agentSessionId) {
1886
- throw new Error('No active agent session for tool execution');
1887
- }
1888
- const mutation = `
1889
- mutation ExecuteRealtimeSessionTool($agentSessionId: String!, $callId: String!, $toolName: String!, $argsJson: String!) {
1890
- ExecuteRealtimeSessionTool(agentSessionId: $agentSessionId, callId: $callId, toolName: $toolName, argsJson: $argsJson)
1891
- }
1892
- `;
1893
- const result = await this.gql().ExecuteGQL(mutation, {
1894
- agentSessionId: this.agentSessionId,
1895
- callId,
1896
- toolName,
1897
- argsJson
1898
- });
1899
- return result?.ExecuteRealtimeSessionTool ?? '{}';
1900
- }
1901
- /**
1902
- * Persists an interactive channel's state of record (e.g. the whiteboard's serialized scene)
1903
- * onto the session's `MJ: AI Agent Session Channels` row via `SaveSessionChannelState`.
1904
- *
1905
- * @param channelName The channel definition name (e.g. `'Whiteboard'`).
1906
- * @param stateJson The serialized channel state.
1907
- * @param agentSessionId Optional EXPLICIT session id. The debounced channel-save pipeline
1908
- * captures the id while the session is live and passes it here, so the final teardown
1909
- * flush still lands on the just-closed session. Falls back to the active session's id;
1910
- * returns `false` when neither is available.
1911
- * @returns Whether the server persisted the state. Failures are logged, never thrown — channel
1912
- * persistence is best-effort and must not disturb the live call.
1913
- */
1914
- async SaveChannelState(channelName, stateJson, agentSessionId) {
1915
- const sessionId = agentSessionId ?? this.agentSessionId;
1916
- if (!sessionId) {
1917
- return false;
1918
- }
1919
- try {
1920
- const mutation = `
1921
- mutation SaveSessionChannelState($agentSessionId: String!, $channelName: String!, $stateJson: String!) {
1922
- SaveSessionChannelState(agentSessionId: $agentSessionId, channelName: $channelName, stateJson: $stateJson)
1923
- }
1924
- `;
1925
- const result = await this.gql().ExecuteGQL(mutation, { agentSessionId: sessionId, channelName, stateJson });
1926
- return result?.SaveSessionChannelState ?? false;
1927
- }
1928
- catch (error) {
1929
- console.error('[RealtimeSession] Failed to save channel state:', error);
1930
- return false;
1931
- }
1932
- }
1933
- // ── Transcript relay (GraphQL) ─────────────────────────────────────────────
1934
- /**
1935
- * Relays a final transcript turn to MJ via `RelayRealtimeTranscript`.
1936
- *
1937
- * When the session is being recorded, per-turn timing rides along: `utteranceEndMs` is the
1938
- * recording-relative offset at finalization, and `utteranceStartMs` is the offset captured by
1939
- * {@link markTurnAudioStart} when THIS turn's audio actually began (its first interim) — NOT
1940
- * inherited from the previous turn's end. That distinction is the timing fix: when a tool-call
1941
- * / silence gap sits between turns, the post-gap turn's audio starts much later, so inheriting
1942
- * the prior turn's end stamped the cue ~gap-length too early. Both are omitted (left `null`)
1943
- * when the session isn't being recorded.
1944
- *
1945
- * A correction (`replacesPrevious`) doesn't open a new turn, so it carries no start and doesn't
1946
- * reset the per-turn start guard. After a normal finalization the guard is cleared so the NEXT
1947
- * turn re-stamps its start from where ITS audio begins.
1948
- *
1949
- * @param replacesPrevious CORRECTION semantics: the server updates the session's most
1950
- * recent persisted turn of this role IN PLACE instead of appending (e.g. ElevenLabs'
1951
- * post-barge-in `agent_response_correction`).
1952
- */
1953
- async relayTranscript(role, text, replacesPrevious = false) {
1954
- if (!this.agentSessionId) {
1955
- return;
1956
- }
1957
- // Per-turn timing against whichever clock the session is on (#3832): the recorder's when one
1958
- // runs, else the session clock. `utteranceStartMs` is where this turn's audio actually began
1959
- // (captured by markTurnAudioStart on the first interim); the `?? 0` fallback covers a turn
1960
- // whose interim was missed / a final-only first turn.
1961
- const utteranceEndMs = this.nowTurnOffsetMs();
1962
- const utteranceStartMs = utteranceEndMs !== null && !replacesPrevious ? (this.currentTurnStartMs ?? 0) : null;
1963
- if (utteranceEndMs !== null && !replacesPrevious) {
1964
- // This turn is finalized — arm the NEXT turn to re-stamp its start from its own first
1965
- // interim (handles a tool-call gap before the next turn). Stop inheriting this end as the
1966
- // next start. `null` means "not yet captured"; relay falls back to `?? 0` if no interim fires.
1967
- this.currentTurnStartMs = null;
1968
- this.turnAudioStartCaptured = false;
1969
- }
1970
- try {
1971
- const mutation = `
1972
- mutation RelayRealtimeTranscript($agentSessionId: String!, $role: String!, $text: String!, $replacesPrevious: Boolean, $utteranceStartMs: Int, $utteranceEndMs: Int) {
1973
- RelayRealtimeTranscript(agentSessionId: $agentSessionId, role: $role, text: $text, replacesPrevious: $replacesPrevious, utteranceStartMs: $utteranceStartMs, utteranceEndMs: $utteranceEndMs)
1974
- }
1975
- `;
1976
- await this.gql().ExecuteGQL(mutation, {
1977
- agentSessionId: this.agentSessionId,
1978
- role,
1979
- text,
1980
- replacesPrevious,
1981
- utteranceStartMs,
1982
- utteranceEndMs
1983
- });
1984
- }
1985
- catch (error) {
1986
- console.error('[RealtimeSession] Failed to relay transcript:', error);
1987
- }
1988
- }
1989
- /**
1990
- * Relays a co-agent CHANNEL tool-call turn (browser_ / Whiteboard_ etc.) to the session's run for
1991
- * observability via `RelayRealtimeToolTurn` — so the co-agent's AIPromptRun shows what it DID, not
1992
- * just what it said. Run-only by design: deliberately NOT a `ConversationDetail` turn, so the chat
1993
- * thread stays speech-only. Best-effort — a failed relay never disturbs the live call.
1994
- */
1995
- async relayToolTurn(toolName, argsJson, resultJson) {
1996
- if (!this.agentSessionId) {
1997
- return;
1998
- }
1999
- try {
2000
- const mutation = `
2001
- mutation RelayRealtimeToolTurn($agentSessionId: String!, $toolName: String!, $argsJson: String, $resultJson: String) {
2002
- RelayRealtimeToolTurn(agentSessionId: $agentSessionId, toolName: $toolName, argsJson: $argsJson, resultJson: $resultJson)
2003
- }
2004
- `;
2005
- await this.gql().ExecuteGQL(mutation, {
2006
- agentSessionId: this.agentSessionId,
2007
- toolName,
2008
- argsJson,
2009
- resultJson
2010
- });
2011
- }
2012
- catch (error) {
2013
- console.error('[RealtimeSession] Failed to relay tool turn:', error);
2014
- }
2015
- }
2016
- // ── Usage telemetry relay (B7) ─────────────────────────────────────────────
2017
- /**
2018
- * Accumulates one usage DELTA from the realtime client (per-response token counts —
2019
- * the `OnUsage` contract shape) and schedules the debounced relay. Negative / non-finite
2020
- * values are clamped to 0; an all-zero delta is dropped without arming the timer.
2021
- */
2022
- onUsageDelta(usage) {
2023
- const input = this.clampUsageDelta(usage.InputTokens);
2024
- const output = this.clampUsageDelta(usage.OutputTokens);
2025
- if (input === 0 && output === 0) {
2026
- return;
2027
- }
2028
- this.pendingUsageInput += input;
2029
- this.pendingUsageOutput += output;
2030
- if (!this.usageFlushTimer) {
2031
- this.usageFlushTimer = setTimeout(() => {
2032
- this.usageFlushTimer = null;
2033
- void this.flushPendingUsage();
2034
- }, RealtimeSessionService.UsageFlushDebounceMs);
2035
- }
2036
- }
2037
- /** Clamps a driver-reported token delta: undefined / negative / non-finite become 0. */
2038
- clampUsageDelta(value) {
2039
- return typeof value === 'number' && Number.isFinite(value) && value > 0 ? Math.floor(value) : 0;
2040
- }
2041
- /**
2042
- * Relays the accumulated usage deltas to the server via `RelayRealtimeUsage` (which
2043
- * accumulates them onto the co-agent `AIPromptRun`). Best-effort: a failed relay
2044
- * re-accumulates the captured deltas so the next debounce / teardown flush retries —
2045
- * usage telemetry must never disturb the live call.
2046
- *
2047
- * @param agentSessionId Optional EXPLICIT session id (the teardown flush runs while the
2048
- * live id is still set, but accepts it as a parameter for symmetry with channel saves).
2049
- */
2050
- async flushPendingUsage(agentSessionId) {
2051
- const sessionId = agentSessionId ?? this.agentSessionId;
2052
- const input = this.pendingUsageInput;
2053
- const output = this.pendingUsageOutput;
2054
- if (!sessionId || (input === 0 && output === 0)) {
2055
- return;
2056
- }
2057
- this.pendingUsageInput = 0;
2058
- this.pendingUsageOutput = 0;
2059
- try {
2060
- const mutation = `
2061
- mutation RelayRealtimeUsage($agentSessionId: String!, $inputTokens: Int!, $outputTokens: Int!) {
2062
- RelayRealtimeUsage(agentSessionId: $agentSessionId, inputTokens: $inputTokens, outputTokens: $outputTokens)
2063
- }
2064
- `;
2065
- await this.gql().ExecuteGQL(mutation, { agentSessionId: sessionId, inputTokens: input, outputTokens: output });
2066
- }
2067
- catch (error) {
2068
- console.error('[RealtimeSession] Failed to relay usage telemetry:', error);
2069
- // Re-accumulate so a later debounce / the teardown flush retries the same deltas.
2070
- this.pendingUsageInput += input;
2071
- this.pendingUsageOutput += output;
2072
- }
2073
- }
2074
- /** Cancels the pending debounced usage flush and zeroes the accumulators (teardown tail). */
2075
- resetUsageRelay() {
2076
- if (this.usageFlushTimer) {
2077
- clearTimeout(this.usageFlushTimer);
2078
- this.usageFlushTimer = null;
2079
- }
2080
- this.pendingUsageInput = 0;
2081
- this.pendingUsageOutput = 0;
2082
- }
2083
- // ── Delegated-run progress streaming ───────────────────────────────────────
2084
- /**
2085
- * Subscribes to the server's push-status topic (scoped by the GraphQL transport
2086
- * sessionId) to receive delegated-run progress for the active voice session.
2087
- * Each matching event is surfaced on {@link DelegationProgress$} and narrated.
2088
- */
2089
- subscribeDelegationProgress() {
2090
- if (this.delegationProgressSub) {
2091
- return; // already subscribed for this session
2092
- }
2093
- const transportSessionId = this.gql().sessionId;
2094
- this.lastDelegationNarrationAt = 0;
2095
- this.delegationProgressSub = this.gql()
2096
- .PushStatusUpdates(transportSessionId)
2097
- .subscribe({
2098
- next: (raw) => this.onDelegationStatusMessage(raw),
2099
- error: (err) => console.error('[RealtimeSession] Delegation progress stream error:', err)
2100
- });
2101
- }
2102
- /**
2103
- * Parses one push-status message and routes it: a Remote Browser screencast frame goes to the active
2104
- * Remote Browser channel's canvas; a delegation-progress event is dispatched + narrated. Other shapes
2105
- * (normal agent-run streams) are ignored. Screencast frames are checked FIRST and short-circuit, so the
2106
- * delegation path is untouched.
2107
- */
2108
- onDelegationStatusMessage(raw) {
2109
- const frame = this.parseScreencastFrame(raw);
2110
- if (frame) {
2111
- this.routeScreencastFrame(frame);
2112
- return;
2113
- }
2114
- const audio = this.parseAudioChunk(raw);
2115
- if (audio) {
2116
- this.routeAudioChunk(audio);
2117
- return;
2118
- }
2119
- const progress = this.parseProgress(raw);
2120
- if (progress) {
2121
- this.dispatchProgress(progress);
2122
- }
2123
- }
2124
- /**
2125
- * Parses a push-status message and returns it only when it's a Remote Browser screencast frame for the
2126
- * active session — otherwise `null` (ignored, so delegation progress falls through). Matched by
2127
- * `resolver` + `type`, then scoped to THIS session by `agentSessionID`.
2128
- */
2129
- parseScreencastFrame(raw) {
2130
- let payload;
2131
- try {
2132
- payload = JSON.parse(raw);
2133
- }
2134
- catch {
2135
- return null;
2136
- }
2137
- const matches = payload?.resolver === 'RemoteBrowserActionResolver' &&
2138
- payload?.type === 'RemoteBrowserScreencastFrame' &&
2139
- payload?.agentSessionID === this.agentSessionId &&
2140
- typeof payload?.dataBase64 === 'string';
2141
- return matches ? payload : null;
2142
- }
2143
- /**
2144
- * Forwards a screencast frame to the active Remote Browser channel plugin so it paints the frame on its
2145
- * surface canvas. The plugin is found among the session's active channels by its `ChannelName`; located
2146
- * via a structural guard so the service stays decoupled from the concrete channel class.
2147
- */
2148
- routeScreencastFrame(frame) {
2149
- for (const channel of this._activeChannels$.value) {
2150
- if (channel.ChannelName === 'Remote Browser' && this.hasOnScreencastFrame(channel)) {
2151
- // The URL rides along so the channel can notice a page change nobody on this side caused —
2152
- // under streaming the snapshot poll is stopped, and frames were pure pixels (#3496).
2153
- channel.OnScreencastFrame(frame.dataBase64, frame.currentUrl ?? null);
2154
- return;
2155
- }
2156
- }
2157
- }
2158
- /** Structural guard: true when the channel exposes an `OnScreencastFrame(dataBase64)` method. */
2159
- hasOnScreencastFrame(channel) {
2160
- return typeof channel.OnScreencastFrame === 'function';
2161
- }
2162
- /**
2163
- * Parses a push-status message and returns it only when it's a Remote Browser audio chunk for the active
2164
- * session — otherwise `null` (ignored). Matched by `resolver` + `type`, then scoped to THIS session by
2165
- * `agentSessionID`.
2166
- */
2167
- parseAudioChunk(raw) {
2168
- let payload;
2169
- try {
2170
- payload = JSON.parse(raw);
2171
- }
2172
- catch {
2173
- return null;
2174
- }
2175
- const matches = payload?.resolver === 'RemoteBrowserActionResolver' &&
2176
- payload?.type === 'RemoteBrowserAudioChunk' &&
2177
- payload?.agentSessionID === this.agentSessionId &&
2178
- typeof payload?.dataBase64 === 'string';
2179
- return matches ? payload : null;
2180
- }
2181
- /**
2182
- * Forwards an audio chunk to the active Remote Browser channel plugin so it plays the chunk through its
2183
- * client-side audio player. The plugin is found among the session's active channels by its `ChannelName`;
2184
- * located via a structural guard so the service stays decoupled from the concrete channel class.
2185
- */
2186
- routeAudioChunk(chunk) {
2187
- for (const channel of this._activeChannels$.value) {
2188
- if (channel.ChannelName === 'Remote Browser' && this.hasOnAudioChunk(channel)) {
2189
- channel.OnAudioChunk({
2190
- dataBase64: chunk.dataBase64,
2191
- codec: chunk.codec,
2192
- sampleRate: chunk.sampleRate,
2193
- channels: chunk.channels,
2194
- seq: chunk.seq,
2195
- });
2196
- return;
2197
- }
2198
- }
2199
- }
2200
- /** Structural guard: true when the channel exposes an `OnAudioChunk(chunk)` method. */
2201
- hasOnAudioChunk(channel) {
2202
- return typeof channel.OnAudioChunk === 'function';
2203
- }
2204
- /**
2205
- * Parses a push-status message and returns it only when it's a delegation
2206
- * progress event for the active voice session — otherwise `null` (ignored).
2207
- */
2208
- parseProgress(raw) {
2209
- let payload;
2210
- try {
2211
- payload = JSON.parse(raw);
2212
- }
2213
- catch {
2214
- return null; // non-JSON or unrelated frame
2215
- }
2216
- const matches = payload?.resolver === 'RealtimeClientSessionResolver' &&
2217
- payload?.type === 'RealtimeDelegationProgress' &&
2218
- payload?.agentSessionID === this.agentSessionId;
2219
- if (!matches) {
2220
- return null;
2221
- }
2222
- return {
2223
- CallID: payload.callID,
2224
- Step: payload.step,
2225
- Message: payload.message,
2226
- Percentage: payload.percentage
2227
- };
2228
- }
2229
- /** Emits the progress to the UI observable and feeds it to the realtime model. */
2230
- dispatchProgress(progress) {
2231
- // Drop stale progress: PubSub delivery can lag the mutation result, so events for a
2232
- // call that already completed (or was never seen) must not update cards or narrate.
2233
- if (!this.inFlightCallIds.has(progress.CallID)) {
2234
- return;
2235
- }
2236
- this._delegationProgress$.next(progress);
2237
- this.narrateProgress(progress);
2238
- }
2239
- /**
2240
- * Injects the progress into the model's context as a background note every time,
2241
- * then (throttled) asks the model to briefly voice a reassuring update so the
2242
- * background work doesn't sit in silence — without chattering or interrupting.
2243
- */
2244
- narrateProgress(progress) {
2245
- const client = this.client;
2246
- if (!client) {
2247
- return;
2248
- }
2249
- client.SendContextNote(`[delegated-agent progress] ${progress.Message}`);
2250
- // Floods of small updates AGGREGATE: each distinct message joins the digest buffer,
2251
- // and ONE spoken update fires per window (first at ~5s into the burst, then every
2252
- // ~8s). The buffer is discarded if the final result lands first.
2253
- this.bufferNarrationMessage(progress.Message);
2254
- if (this.pendingNarrationMessages.length > 0 && !this.narrationTimer) {
2255
- this.narrationTimer = setTimeout(() => this.fireDeferredNarration(), this.nextNarrationDelayMs());
2256
- }
2257
- }
2258
- /** Adds a progress message to the digest buffer (deduped, capped, oldest-first). */
2259
- bufferNarrationMessage(message) {
2260
- if (message === this.lastNarratedTail || this.pendingNarrationMessages.includes(message)) {
2261
- return;
2262
- }
2263
- this.pendingNarrationMessages.push(message);
2264
- if (this.pendingNarrationMessages.length > RealtimeSessionService.MaxDigestMessages) {
2265
- this.pendingNarrationMessages.shift();
2266
- }
2267
- }
2268
- /**
2269
- * ms until the next spoken update is allowed. Two constraints, BOTH enforced:
2270
- * - first update of a burst: no earlier than ~5s after the burst started;
2271
- * - ~8s since the last spoken update, SESSION-global — so sequential tool calls
2272
- * that reset the burst can never narrate faster than the interval.
2273
- */
2274
- nextNarrationDelayMs() {
2275
- const now = Date.now();
2276
- const firstAnchor = this.narrationCount === 0
2277
- ? this.delegationBurstStartedAt + RealtimeSessionService.FirstNarrationDelayMs
2278
- : 0;
2279
- const spacingFloor = this.lastDelegationNarrationAt > 0
2280
- ? this.lastDelegationNarrationAt + RealtimeSessionService.NarrationIntervalMs
2281
- : 0;
2282
- return Math.max(250, Math.max(firstAnchor, spacingFloor) - now);
2283
- }
2284
- /**
2285
- * Speaks the aggregated progress digest — unless the work already finished (buffer
2286
- * cancelled) or the model is busy / audio is still playing, in which case it retries
2287
- * shortly with the buffer intact (work is still running, so the update stays relevant).
2288
- */
2289
- fireDeferredNarration() {
2290
- this.narrationTimer = null;
2291
- const client = this.client;
2292
- if (this.pendingNarrationMessages.length === 0 || !client || this.inFlightCallIds.size === 0) {
2293
- this.pendingNarrationMessages = [];
2294
- return;
2295
- }
2296
- if (client.IsBusy || client.IsAudioPlaying) {
2297
- this.narrationTimer = setTimeout(() => this.fireDeferredNarration(), RealtimeSessionService.NarrationBusyRetryMs);
2298
- return;
2299
- }
2300
- const digest = this.pendingNarrationMessages.join(' → ');
2301
- this.lastNarratedTail = this.pendingNarrationMessages[this.pendingNarrationMessages.length - 1];
2302
- this.pendingNarrationMessages = [];
2303
- this.narrationCount++;
2304
- this.lastDelegationNarrationAt = Date.now();
2305
- client.RequestSpokenUpdate(this.buildNarrationInstructions(digest));
2306
- }
2307
- /** Cancels any deferred narration — the result is about to be spoken, so it's moot. */
2308
- cancelPendingNarration() {
2309
- if (this.narrationTimer) {
2310
- clearTimeout(this.narrationTimer);
2311
- this.narrationTimer = null;
2312
- }
2313
- this.pendingNarrationMessages = [];
2314
- }
2315
- /**
2316
- * Builds the one-off instructions for a short spoken update that conveys THIS specific
2317
- * progress message naturally — strictly first person, since the co-agent owns the work.
2318
- * The wording is DB-driven: the server-resolved `Realtime Co-Agent - Progress Narration`
2319
- * template (substituting `{{ progressMessage }}`) when present, otherwise the built-in
2320
- * fallback so deployments that haven't synced the prompt behave exactly as before.
2321
- * The client tags the resulting turn as narration, keeping it EPHEMERAL — surfaced on
2322
- * {@link DelegationNarration$} instead of becoming a caption / persisted ConversationDetail.
2323
- */
2324
- buildNarrationInstructions(digest) {
2325
- return BuildNarrationInstructions(this.narrationTemplate, digest, {
2326
- PriorNarrations: this.spokenNarrations.slice(-RealtimeSessionService.MaxPriorNarrations),
2327
- UpdateNumber: this.narrationCount
2328
- });
2329
- }
2330
- /** Tears down the delegation progress subscription and resets the narration throttle. */
2331
- teardownDelegationProgress() {
2332
- if (this.delegationProgressSub) {
2333
- this.delegationProgressSub.unsubscribe();
2334
- this.delegationProgressSub = null;
2335
- }
2336
- this.cancelPendingNarration();
2337
- this.inFlightCallIds.clear();
2338
- this.cancelledCallIds.clear();
2339
- this.lastDelegationNarrationAt = 0;
2340
- this.delegationBurstStartedAt = 0;
2341
- this.narrationCount = 0;
2342
- this.spokenNarrations = [];
2343
- this.lastNarratedTail = '';
2344
- }
2345
- // ── Teardown ───────────────────────────────────────────────────────────────
2346
- /**
2347
- * Tears down all client resources and (optionally) closes the server session.
2348
- * @param closeServerSession when true, calls `CloseAgentSession` on the server.
2349
- */
2350
- async teardown(closeServerSession) {
2351
- // First: stop asserting liveness. A pulse racing the close would re-stamp LastActiveAt on a
2352
- // session we are deliberately ending, leaving an Idle row the janitor then has to age out.
2353
- this.stopLivenessPulse();
2354
- this.teardownDelegationProgress();
2355
- // Channels first: flush any unsaved channel state WHILE the live session id is still
2356
- // set (the captured per-save id covers the race anyway), then dispose the plugins.
2357
- this.flushAllChannelSaves();
2358
- this.disposeChannels();
2359
- // Defensive: stop the mic even when Connect never ran (the client also stops the
2360
- // tracks it was handed — track.stop() is idempotent).
2361
- this.localStream?.getTracks().forEach(t => t.stop());
2362
- this.localStream = null;
2363
- if (this.client) {
2364
- await this.client.Disconnect();
2365
- this.client = null;
2366
- }
2367
- // Stop + upload the call recording WHILE the live session id is still set (the file is
2368
- // attached to it). Best-effort and never blocks teardown — stopAndUploadRecording swallows
2369
- // its own errors. No-op when nothing was recorded.
2370
- await this.stopAndUploadRecording(this.agentSessionId);
2371
- this.recordingStartedAtIso = null;
2372
- // Final usage flush WHILE the live session id is still set (the relay mutation also
2373
- // accepts a Closed session, so ordering vs. CloseAgentSession is belt-and-braces).
2374
- if (this.usageFlushTimer) {
2375
- clearTimeout(this.usageFlushTimer);
2376
- this.usageFlushTimer = null;
2377
- }
2378
- await this.flushPendingUsage(this.agentSessionId);
2379
- this.resetUsageRelay();
2380
- if (closeServerSession && this.agentSessionId) {
2381
- await this.closeServerSession(this.agentSessionId);
2382
- }
2383
- // Capture the session id BEFORE we null it so the lifecycle emit carries it.
2384
- // Skip emitting when there was no live session (defensive — teardown is safe
2385
- // to call without an active session).
2386
- const closedSessionId = this.agentSessionId;
2387
- this.agentSessionId = null;
2388
- this.narrationTemplate = null;
2389
- this.clientToolHandlers.clear();
2390
- this._modelName$.next(null);
2391
- this.SetMinimized(false);
2392
- this._active$.next(false);
2393
- if (this._connectionState$.value !== 'error') {
2394
- this._connectionState$.next('closed');
2395
- }
2396
- // Surface generic session-ended for the conversations runtime bridge.
2397
- // `closeServerSession=true` means the user explicitly called EndRealtimeSession;
2398
- // `false` means teardown ran from a catch block (start path error path).
2399
- if (closedSessionId) {
2400
- this._sessionEnded$.next({
2401
- sessionId: closedSessionId,
2402
- reason: closeServerSession ? 'explicit' : 'error',
2403
- });
2404
- }
2405
- }
2406
- /** Calls the `CloseAgentSession` mutation (provisioned in P4b). */
2407
- async closeServerSession(agentSessionId) {
2408
- try {
2409
- const mutation = `
2410
- mutation CloseAgentSession($agentSessionId: String!) {
2411
- CloseAgentSession(agentSessionId: $agentSessionId)
2412
- }
2413
- `;
2414
- await this.gql().ExecuteGQL(mutation, { agentSessionId });
2415
- }
2416
- catch (error) {
2417
- console.error('[RealtimeSession] Failed to close server session:', error);
2418
- }
2419
- }
2420
- // ── Helpers ────────────────────────────────────────────────────────────────
2421
- /** Pushes a caption onto the live list (immutable update for change detection). */
2422
- appendCaption(caption) {
2423
- this._captions$.next([...this._captions$.value, caption]);
2424
- }
2425
- /** Resets reactive + internal state at the start of a session. */
2426
- resetState() {
2427
- this._captions$.next([]);
2428
- this.pendingUserCaption = '';
2429
- this.hasActiveInterimUserCaption = false;
2430
- this.SetMinimized(false);
2431
- this.stopSegmentFlushing();
2432
- this.segmentIndex = 0;
2433
- this.recorder = null;
2434
- this.recordingStartedAtIso = null;
2435
- this.currentTurnStartMs = null;
2436
- this.turnAudioStartCaptured = false;
2437
- this.usedChannelNames.clear();
2438
- }
2439
- /** The GraphQL provider used for relay mutations. */
2440
- gql() {
2441
- return this.Provider;
37
+ export class RealtimeSessionService extends RealtimeSessionRuntime {
38
+ constructor() {
39
+ super(new BrowserRealtimeMediaHost());
2442
40
  }
2443
41
  static ɵfac = function RealtimeSessionService_Factory(__ngFactoryType__) { return new (__ngFactoryType__ || RealtimeSessionService)(); };
2444
42
  static ɵprov = /*@__PURE__*/ i0.ɵɵdefineInjectable({ token: RealtimeSessionService, factory: RealtimeSessionService.ɵfac, providedIn: 'root' });
@@ -2446,5 +44,5 @@ export class RealtimeSessionService {
2446
44
  (() => { (typeof ngDevMode === "undefined" || ngDevMode) && i0.ɵsetClassMetadata(RealtimeSessionService, [{
2447
45
  type: Injectable,
2448
46
  args: [{ providedIn: 'root' }]
2449
- }], null, null); })();
47
+ }], () => [], null); })();
2450
48
  //# sourceMappingURL=realtime-session.service.js.map