@memberjunction/realtime-runtime 6.2.0-edge.3 → 6.2.0-edge.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/README.md +75 -0
  2. package/dist/channels/base-realtime-channel-client.d.ts +535 -28
  3. package/dist/channels/base-realtime-channel-client.d.ts.map +1 -1
  4. package/dist/channels/base-realtime-channel-client.js +864 -10
  5. package/dist/channels/base-realtime-channel-client.js.map +1 -1
  6. package/dist/channels/channel-action-dispatcher.d.ts +70 -0
  7. package/dist/channels/channel-action-dispatcher.d.ts.map +1 -0
  8. package/dist/channels/channel-action-dispatcher.js +136 -0
  9. package/dist/channels/channel-action-dispatcher.js.map +1 -0
  10. package/dist/channels/channel-catalog-note.d.ts +48 -0
  11. package/dist/channels/channel-catalog-note.d.ts.map +1 -0
  12. package/dist/channels/channel-catalog-note.js +119 -0
  13. package/dist/channels/channel-catalog-note.js.map +1 -0
  14. package/dist/channels/channel-contract-types.d.ts +107 -0
  15. package/dist/channels/channel-contract-types.d.ts.map +1 -0
  16. package/dist/channels/channel-contract-types.js +12 -0
  17. package/dist/channels/channel-contract-types.js.map +1 -0
  18. package/dist/channels/channel-descriptor-synthesis.d.ts +56 -0
  19. package/dist/channels/channel-descriptor-synthesis.d.ts.map +1 -0
  20. package/dist/channels/channel-descriptor-synthesis.js +90 -0
  21. package/dist/channels/channel-descriptor-synthesis.js.map +1 -0
  22. package/dist/channels/channel-perception.d.ts +82 -0
  23. package/dist/channels/channel-perception.d.ts.map +1 -0
  24. package/dist/channels/channel-perception.js +120 -0
  25. package/dist/channels/channel-perception.js.map +1 -0
  26. package/dist/channels/channel-schema-format.d.ts +17 -0
  27. package/dist/channels/channel-schema-format.d.ts.map +1 -0
  28. package/dist/channels/channel-schema-format.js +45 -0
  29. package/dist/channels/channel-schema-format.js.map +1 -0
  30. package/dist/channels/channel-state-delta.d.ts +59 -0
  31. package/dist/channels/channel-state-delta.d.ts.map +1 -0
  32. package/dist/channels/channel-state-delta.js +116 -0
  33. package/dist/channels/channel-state-delta.js.map +1 -0
  34. package/dist/channels/channel-surface-placement.d.ts +25 -0
  35. package/dist/channels/channel-surface-placement.d.ts.map +1 -0
  36. package/dist/channels/channel-surface-placement.js +29 -0
  37. package/dist/channels/channel-surface-placement.js.map +1 -0
  38. package/dist/channels/channel-visual-pump.d.ts +126 -0
  39. package/dist/channels/channel-visual-pump.d.ts.map +1 -0
  40. package/dist/channels/channel-visual-pump.js +174 -0
  41. package/dist/channels/channel-visual-pump.js.map +1 -0
  42. package/dist/hosts/IRealtimeMediaHost.d.ts +61 -4
  43. package/dist/hosts/IRealtimeMediaHost.d.ts.map +1 -1
  44. package/dist/hosts/IRealtimeMediaHost.js +2 -1
  45. package/dist/hosts/IRealtimeMediaHost.js.map +1 -1
  46. package/dist/index.d.ts +21 -2
  47. package/dist/index.d.ts.map +1 -1
  48. package/dist/index.js +24 -0
  49. package/dist/index.js.map +1 -1
  50. package/dist/session/RealtimeSessionRuntime.d.ts +727 -74
  51. package/dist/session/RealtimeSessionRuntime.d.ts.map +1 -1
  52. package/dist/session/RealtimeSessionRuntime.js +1732 -242
  53. package/dist/session/RealtimeSessionRuntime.js.map +1 -1
  54. package/dist/session/RecordingShardQueue.d.ts +77 -0
  55. package/dist/session/RecordingShardQueue.d.ts.map +1 -0
  56. package/dist/session/RecordingShardQueue.js +172 -0
  57. package/dist/session/RecordingShardQueue.js.map +1 -0
  58. package/dist/session/app-client-tool-registry.d.ts +76 -0
  59. package/dist/session/app-client-tool-registry.d.ts.map +1 -0
  60. package/dist/session/app-client-tool-registry.js +100 -0
  61. package/dist/session/app-client-tool-registry.js.map +1 -0
  62. package/dist/session/avatar-notice.d.ts +43 -0
  63. package/dist/session/avatar-notice.d.ts.map +1 -0
  64. package/dist/session/avatar-notice.js +60 -0
  65. package/dist/session/avatar-notice.js.map +1 -0
  66. package/dist/session/call-end.d.ts +21 -0
  67. package/dist/session/call-end.d.ts.map +1 -0
  68. package/dist/session/call-end.js +19 -0
  69. package/dist/session/call-end.js.map +1 -0
  70. package/dist/session/channel-exposure-preferences.d.ts +70 -0
  71. package/dist/session/channel-exposure-preferences.d.ts.map +1 -0
  72. package/dist/session/channel-exposure-preferences.js +138 -0
  73. package/dist/session/channel-exposure-preferences.js.map +1 -0
  74. package/dist/session/channel-session-scope.d.ts +142 -0
  75. package/dist/session/channel-session-scope.d.ts.map +1 -0
  76. package/dist/session/channel-session-scope.js +128 -0
  77. package/dist/session/channel-session-scope.js.map +1 -0
  78. package/dist/session/client-session-deadline.d.ts +28 -0
  79. package/dist/session/client-session-deadline.d.ts.map +1 -0
  80. package/dist/session/client-session-deadline.js +50 -0
  81. package/dist/session/client-session-deadline.js.map +1 -0
  82. package/dist/session/realtime-captures.d.ts +236 -0
  83. package/dist/session/realtime-captures.d.ts.map +1 -0
  84. package/dist/session/realtime-captures.js +383 -0
  85. package/dist/session/realtime-captures.js.map +1 -0
  86. package/dist/session/realtime-microphone.d.ts +50 -0
  87. package/dist/session/realtime-microphone.d.ts.map +1 -0
  88. package/dist/session/realtime-microphone.js +51 -0
  89. package/dist/session/realtime-microphone.js.map +1 -0
  90. package/dist/session/session-event-hub.d.ts +72 -0
  91. package/dist/session/session-event-hub.d.ts.map +1 -0
  92. package/dist/session/session-event-hub.js +151 -0
  93. package/dist/session/session-event-hub.js.map +1 -0
  94. package/dist/session/session-launcher.d.ts +128 -0
  95. package/dist/session/session-launcher.d.ts.map +1 -0
  96. package/dist/session/session-launcher.js +160 -0
  97. package/dist/session/session-launcher.js.map +1 -0
  98. package/package.json +9 -9
@@ -1,13 +1,28 @@
1
1
  import { BehaviorSubject, Subject } from 'rxjs';
2
2
  import { Metadata } from '@memberjunction/core';
3
3
  import { UserInfoEngine } from '@memberjunction/core-entities';
4
- import { AIEngineBase } from '@memberjunction/ai-engine-base';
5
- import { MJGlobal } from '@memberjunction/global';
6
- import { DEFAULT_REALTIME_AUDIO_TRACKS } from '@memberjunction/ai';
7
- import { BaseRealtimeClient, LoadAssemblyAIRealtimeClient, LoadElevenLabsRealtimeClient, LoadGeminiRealtimeClient, LoadHuggingFaceRealtimeClient, LoadOpenAIRealtimeClient, LoadxAIRealtimeClient, REQUESTED_TRACKS_SESSION_KEY } from '@memberjunction/ai-realtime-client';
4
+ import { AddRealtimeUsageRecord, AIEngineBase, HasRealtimeUsage } from '@memberjunction/ai-engine-base';
5
+ import { GraphQLRealtimeSessionClient } from '@memberjunction/graphql-dataprovider';
6
+ import { MJGlobal, UUIDsEqual } from '@memberjunction/global';
7
+ import { DEFAULT_REALTIME_AUDIO_TRACKS, ParseRealtimeAvatarStatus, ParseRealtimeClientTransport } from '@memberjunction/ai';
8
+ import { CompareExposure, DeclaresNativeTools, IsIdentityVerifiedEventPayload, NormalizeChannelKey, ParseRealtimeSessionClientPolicy, ResolveClientTools, SelectNativeChannelTools, UserExposureReason } from '@memberjunction/ai-core-plus';
9
+ import { BaseRealtimeClient, LoadAssemblyAIRealtimeClient, LoadElevenLabsRealtimeClient, LoadGeminiEnterpriseRealtimeClient, LoadGeminiRealtimeClient, LoadHuggingFaceRealtimeClient, LoadOpenAIRealtimeClient, LoadxAIRealtimeClient, VideoSourceArbiter, REQUESTED_TRACKS_SESSION_KEY } from '@memberjunction/ai-realtime-client';
10
+ import { DefaultRealtimeSessionLauncher, HasClientCredential } from './session-launcher.js';
11
+ import { RequestsAgentVideo, ResolveAvatarNotice } from './avatar-notice.js';
12
+ import { ClientSessionDeadline } from './client-session-deadline.js';
13
+ import { RealtimeCaptures, REALTIME_CAPTURES_OFF, REALTIME_CAPTURE_OFFERS_NONE, } from './realtime-captures.js';
14
+ import { REALTIME_MICROPHONE_NONE, ReadMicrophoneState, SameMicrophoneState } from './realtime-microphone.js';
15
+ import { RealtimeSessionEventHub } from './session-event-hub.js';
8
16
  import { BuildNarrationInstructions } from '../narration/narration-template.js';
9
17
  import { ParseDelegationResultJson, FormatToolName } from './delegation-result-parser.js';
10
- import { BaseRealtimeChannelClient } from '../channels/base-realtime-channel-client.js';
18
+ import { BaseRealtimeChannelClient, ChannelVideoSourceID } from '../channels/base-realtime-channel-client.js';
19
+ import { DEFAULT_CHANNEL_SURFACE_PLACEMENT, ReadChannelSurfacePlacement } from '../channels/channel-surface-placement.js';
20
+ import { ChannelActionDispatcher } from '../channels/channel-action-dispatcher.js';
21
+ import { BuildChannelCatalogNote } from '../channels/channel-catalog-note.js';
22
+ import { AppClientToolRegistry, DEFAULT_APP_TOOL_OWNER } from './app-client-tool-registry.js';
23
+ import { UserSettingsExposurePreferences } from './channel-exposure-preferences.js';
24
+ import { BuildChannelCandidate, FindPreparedChannel, MergeToolMetadata, PreparedChannelsShowAgentVideo, ReconcileChannelsWithPolicy, ResolveLocalChannelScope, ToolsByChannelKey } from './channel-session-scope.js';
25
+ import { RecordingShardQueue } from './RecordingShardQueue.js';
11
26
  /**
12
27
  * `MJ: User Settings` key for the per-user "record this voice call" consent toggle. Stored as
13
28
  * the literal string `'true'`/`'false'` (read with `=== 'true'`), cross-device via
@@ -15,6 +30,15 @@ import { BaseRealtimeChannelClient } from '../channels/base-realtime-channel-cli
15
30
  * default when the caller doesn't pass an explicit consent value.
16
31
  */
17
32
  export const REALTIME_RECORDING_CONSENT_KEY = 'mj.realtimeVoice.recordingConsent.v1';
33
+ /**
34
+ * Relays usage onto the co-agent prompt run: token deltas, and the per-modality detail blocks as a
35
+ * JSON record (sent only when there are any; see `RelayRealtimeUsage` in MJServer).
36
+ */
37
+ const RELAY_REALTIME_USAGE_MUTATION = `
38
+ mutation RelayRealtimeUsage($agentSessionId: String!, $inputTokens: Int!, $outputTokens: Int!, $usageDetailsJson: String) {
39
+ RelayRealtimeUsage(agentSessionId: $agentSessionId, inputTokens: $inputTokens, outputTokens: $outputTokens, usageDetailsJson: $usageDetailsJson)
40
+ }
41
+ `;
18
42
  // Tree-shaking prevention: the OpenAI client is resolved dynamically through the
19
43
  // ClassFactory (by the server-reported Provider key), so this static call is what keeps
20
44
  // its @RegisterClass side effect from being eliminated by the bundler.
@@ -24,10 +48,37 @@ export const REALTIME_RECORDING_CONSENT_KEY = 'mj.realtimeVoice.recordingConsent
24
48
  // and this service stays component-free (it must stay importable in plain-node tests).
25
49
  LoadOpenAIRealtimeClient();
26
50
  LoadGeminiRealtimeClient();
51
+ LoadGeminiEnterpriseRealtimeClient();
27
52
  LoadElevenLabsRealtimeClient();
28
53
  LoadAssemblyAIRealtimeClient();
29
54
  LoadxAIRealtimeClient();
30
55
  LoadHuggingFaceRealtimeClient();
56
+ /**
57
+ * A registry row's surface placement. A row whose `UIConfig` is not valid JSON places its surface as a row without one
58
+ * does, so one bad row never costs the session its other channels.
59
+ */
60
+ function readRowSurfacePlacement(row) {
61
+ try {
62
+ return ReadChannelSurfacePlacement(row.UIConfigObject);
63
+ }
64
+ catch {
65
+ console.warn(`[RealtimeSession] Channel '${row.Name}' has a UIConfig that is not valid JSON; its surface starts on its tab.`);
66
+ return DEFAULT_CHANNEL_SURFACE_PLACEMENT;
67
+ }
68
+ }
69
+ /** The same, for a row read over GraphQL, whose `UIConfig` arrives as the column's JSON text. */
70
+ function readUIConfigTextSurfacePlacement(name, uiConfig) {
71
+ if (!uiConfig) {
72
+ return ReadChannelSurfacePlacement(null);
73
+ }
74
+ try {
75
+ return ReadChannelSurfacePlacement(JSON.parse(uiConfig));
76
+ }
77
+ catch {
78
+ console.warn(`[RealtimeSession] Channel '${name}' has a UIConfig that is not valid JSON; its surface starts on its tab.`);
79
+ return DEFAULT_CHANNEL_SURFACE_PLACEMENT;
80
+ }
81
+ }
31
82
  /**
32
83
  * Converts a {@link RealtimeTrackDescriptor} to its JSON form for the session-config bag.
33
84
  *
@@ -67,6 +118,37 @@ function trackKeyFromJSON(raw) {
67
118
  }
68
119
  return `${direction}:${modality}`;
69
120
  }
121
+ /** Whether a surface client tool handler's result is an object whose `Success` is `false`. */
122
+ function reportsFailure(result) {
123
+ return typeof result === 'object' && result !== null && 'Success' in result && result.Success === false;
124
+ }
125
+ /** The failed result's `ErrorMessage` when it is text that is not blank, else `'Tool failed'`. */
126
+ function failureMessage(result) {
127
+ const message = result.ErrorMessage;
128
+ return typeof message === 'string' && message.trim() ? message : 'Tool failed';
129
+ }
130
+ /**
131
+ * The `DOMException` name for each way the controller can fail to start the microphone, so a host that
132
+ * tells a denied microphone apart (`error.name === 'NotAllowedError'`) works the same with or without one.
133
+ */
134
+ const MICROPHONE_ERROR_NAMES = {
135
+ denied: 'NotAllowedError',
136
+ 'not-found': 'NotFoundError',
137
+ 'in-use': 'NotReadableError',
138
+ unsupported: 'NotSupportedError',
139
+ error: 'Error',
140
+ };
141
+ /** What the user is told when the call's policy refuses a capture. */
142
+ const CAPTURE_WORDING = {
143
+ camera: { NotInCall: 'The camera is not part of this call.', NotAllowed: 'This call cannot show the agent your camera' },
144
+ screen: { NotInCall: 'Screen sharing is not part of this call.', NotAllowed: 'This call cannot show the agent your screen' },
145
+ };
146
+ /** The error a failed microphone start reports, named as `getUserMedia` would have named it. */
147
+ function microphoneStartError(reason, message) {
148
+ const error = new Error(message);
149
+ error.name = MICROPHONE_ERROR_NAMES[reason];
150
+ return error;
151
+ }
70
152
  /**
71
153
  * Drives a **client-direct** real-time voice session: the browser mints an ephemeral
72
154
  * token from the MJ server, then connects DIRECTLY to the realtime provider. Audio
@@ -100,6 +182,7 @@ export class RealtimeSessionRuntime {
100
182
  this.mediaHost = mediaHost;
101
183
  // ── Reactive UI state ──────────────────────────────────────────────────────
102
184
  this._connectionState$ = new BehaviorSubject('closed');
185
+ this._resuming$ = new BehaviorSubject(false);
103
186
  this._captions$ = new BehaviorSubject([]);
104
187
  this._active$ = new BehaviorSubject(false);
105
188
  this._delegationProgress$ = new Subject();
@@ -109,6 +192,7 @@ export class RealtimeSessionRuntime {
109
192
  this._agentName$ = new BehaviorSubject('Sage');
110
193
  this._modelName$ = new BehaviorSubject(null);
111
194
  this._minimized$ = new BehaviorSubject(false);
195
+ this._savingRecording$ = new BehaviorSubject(false);
112
196
  this._activeChannels$ = new BehaviorSubject([]);
113
197
  this._channelFocus$ = new Subject();
114
198
  // ─── Generic session-lifecycle events (consumed by RealtimeSessionsAdapter to
@@ -122,6 +206,16 @@ export class RealtimeSessionRuntime {
122
206
  this._channelActivity$ = new Subject();
123
207
  /** Current connection / turn state. */
124
208
  this.ConnectionState$ = this._connectionState$.asObservable();
209
+ /**
210
+ * Whether the call is resuming: moving to a new connection with its provider mid-call. `true` from a `'connecting'` the
211
+ * driver reports while the call is live, until the call is live again (`'listening'`, `'speaking'` or `'thinking'`),
212
+ * fails (`'error'`) or ends (`'closed'`). A call's first connect is not a resume, so it is `false` until the call has
213
+ * been live, and outside a call. Gemini Live resumes when Google ends a connection (`goAway`) and after a dropped
214
+ * socket, and the agent's video can stop for a few seconds meanwhile: a host can show that as a pause rather than as
215
+ * the agent leaving. It changes before {@link ConnectionState$} does, so a resume's `'connecting'` arrives with it
216
+ * already `true`.
217
+ */
218
+ this.Resuming$ = this._resuming$.asObservable();
125
219
  /** Live captions for both sides of the conversation. */
126
220
  this.Captions$ = this._captions$.asObservable();
127
221
  /** True while a session is open (mic button active, overlay shown). */
@@ -158,6 +252,14 @@ export class RealtimeSessionRuntime {
158
252
  * pure presentation state, reset to `false` at session start and teardown.
159
253
  */
160
254
  this.Minimized$ = this._minimized$.asObservable();
255
+ /**
256
+ * True while the end-of-call recording is being saved: from the first statement of a teardown
257
+ * that holds a recorder until that teardown settles. False at construction and for unrecorded
258
+ * sessions. Exists for #5195: the page must stay open until the consolidated upload lands, or
259
+ * the recording is lost. The runtime is platform-free, so hosts own the reaction (a browser
260
+ * `beforeunload` guard, a "saving" indicator, etc.).
261
+ */
262
+ this.SavingRecording$ = this._savingRecording$.asObservable();
161
263
  /**
162
264
  * The session's ACTIVE interactive-channel plugins, resolved from the `MJ: AI Agent
163
265
  * Channels` registry at session start (one instance per session, per channel). Emits
@@ -165,6 +267,57 @@ export class RealtimeSessionRuntime {
165
267
  * one surface tab per plugin — it never knows any concrete channel type.
166
268
  */
167
269
  this.ActiveChannels$ = this._activeChannels$.asObservable();
270
+ this._videoSources$ = new BehaviorSubject([]);
271
+ /**
272
+ * The video sources the agent is perceiving or could perceive right now (a whiteboard, a remote browser, a
273
+ * shared screen), with whether each is on and whether its frames are reaching the model. Empty when no
274
+ * session is live or nothing has offered a frame source. This is what the "agent can see" control renders;
275
+ * toggle one with {@link SetVideoSourceEnabled}.
276
+ */
277
+ this.VideoSources$ = this._videoSources$.asObservable();
278
+ this._captures$ = new BehaviorSubject(REALTIME_CAPTURES_OFF);
279
+ /**
280
+ * The user's camera and screen share: off, starting, on (with the stream to show the user) or failed (with the
281
+ * reason). Both off outside a session. Start and stop them with {@link StartCamera}, {@link StartScreenShare},
282
+ * {@link StopCamera} and {@link StopScreenShare}; while one is on, it is also a source on {@link VideoSources$}.
283
+ */
284
+ this.Captures$ = this._captures$.asObservable();
285
+ this._captureOffers$ = new BehaviorSubject(REALTIME_CAPTURE_OFFERS_NONE);
286
+ /**
287
+ * Which captures the call offers, now and on every change. A capture is offered while the call is connected, the model
288
+ * takes inbound video, the host can open it (a camera controller; screen sharing) and the call's policy admits it (its
289
+ * channel is in the call and may show the agent pixels). A host shows its Camera and Share controls from this. Neither
290
+ * is offered outside a call.
291
+ */
292
+ this.CaptureOffers$ = this._captureOffers$.asObservable();
293
+ this._microphone$ = new BehaviorSubject(REALTIME_MICROPHONE_NONE);
294
+ /**
295
+ * The call's microphone, now and on every change: the one in use and the ones the user can move it to with
296
+ * {@link SwitchMicrophone}, as the host's camera-and-microphone controller lists them. Offered from the moment the call
297
+ * connects until it ends, and kept current as devices come and go. When the call loses its microphone (the one in use
298
+ * fails and so does the fallback to the default, or the last one goes away), its `Failure` says why: the agent hears
299
+ * nothing until the user picks a microphone, which opens it again. {@link REALTIME_MICROPHONE_NONE} outside a call and
300
+ * on a host without a controller, which opens the microphone itself (`IRealtimeMediaHost.AcquireMicrophone`).
301
+ */
302
+ this.Microphone$ = this._microphone$.asObservable();
303
+ this._agentVideo$ = new BehaviorSubject(null);
304
+ /**
305
+ * The agent's video (an avatar) while the model sends it: a live stream, or a player that owns the `<video>` element.
306
+ * `null` until the video arrives, for a model that sends none, and outside a call. The driver hands it over once it is
307
+ * live, and a newer one replaces it. A host shows it with `AttachVideoSource`; a channel that sinks outbound video
308
+ * follows it through its context, and counts as used once it arrives, so the host shows that channel's surface.
309
+ */
310
+ this.AgentVideo$ = this._agentVideo$.asObservable();
311
+ this._avatarNotice$ = new BehaviorSubject(null);
312
+ /**
313
+ * Why the call shows no avatar the agent asked for: set once per call, after it connects, when the model renders none,
314
+ * the persona has no face for it, the app shows no agent video (`host`) or the browser can't play it (`browser`); with
315
+ * no `Reason` when the mint gave one this version doesn't know, such as a newer server's. `null` when there is nothing
316
+ * to say (no avatar asked for, the avatar shows, or the mint reported no status) and after the call ends. A provider
317
+ * resume inside the call does not set it again. A host turns the reason into words once, for example "Audio only: this
318
+ * voice model can't show an avatar".
319
+ */
320
+ this.AvatarNotice$ = this._avatarNotice$.asObservable();
168
321
  /**
169
322
  * Channel requests to enter / leave the FOCUS layout (see
170
323
  * {@link RealtimeChannelFocusEvent}). Fired when a plugin calls its host context's
@@ -208,7 +361,11 @@ export class RealtimeSessionRuntime {
208
361
  */
209
362
  /** Conversation id the SERVER created for this session (null when the host supplied one). */
210
363
  this.createdConversationId = null;
211
- /** The session's conversation id (supplied or server-created). */
364
+ /**
365
+ * The session's conversation id (supplied or server-created). While a start's mint runs, the
366
+ * conversation the start asked for; null when it asked the server to create one. Null after
367
+ * teardown.
368
+ */
212
369
  this.sessionConversationId = null;
213
370
  /** First final user utterance of the live session (the naming seed). */
214
371
  this.firstUserTranscript = null;
@@ -219,8 +376,28 @@ export class RealtimeSessionRuntime {
219
376
  // ── Session internals ──────────────────────────────────────────────────────
220
377
  /** The provider-direct realtime client driving the live session (ClassFactory-resolved). */
221
378
  this.client = null;
222
- /** The mic capture stream — acquired here (permission UX) and handed to the client. */
379
+ /**
380
+ * The mic capture stream — acquired here (permission UX) and handed to the client. A microphone the controller opens
381
+ * again after the call lost one replaces it.
382
+ */
223
383
  this.localStream = null;
384
+ /**
385
+ * Whether the user muted the call's microphone, as {@link ToggleMute} last reported it. While the call has no microphone
386
+ * track there is nothing to mute, and a microphone opened again after the call lost one starts this way.
387
+ */
388
+ this.microphoneMuted = false;
389
+ /** The host's camera-and-microphone controller for this session, when the host offers one. */
390
+ this.localMedia = null;
391
+ /** Follows the controller's microphone, so a swapped-in track or a reopened microphone reaches the driver and the recorder. */
392
+ this.localMediaSubscription = null;
393
+ /**
394
+ * Whether the call offers its microphone for switching ({@link Microphone$}, {@link SwitchMicrophone}): from the moment
395
+ * the client is connected until the call ends. Not before: a driver still connecting may have bound the old track.
396
+ */
397
+ this.microphoneOffered = false;
398
+ /** The live session's camera and screen share; created once the client is connected. */
399
+ this.captures = null;
400
+ this.capturesSubscription = null;
224
401
  this.agentSessionId = null;
225
402
  /**
226
403
  * The application the active session runs in (sources the server-side app config cascade +
@@ -254,8 +431,15 @@ export class RealtimeSessionRuntime {
254
431
  this.recordingStartedAtIso = null;
255
432
  /** Interval that flushes ~15s crash-recovery shards to the server during a recording. */
256
433
  this.segmentTimer = null;
257
- /** 0-based index of the next recording shard to upload. */
258
- this.segmentIndex = 0;
434
+ /**
435
+ * Owns shard indexing and retry for the current recording: it assigns each shard its index and
436
+ * keeps failed shards to resend under that same index. A fresh queue per recording keeps a flush
437
+ * still in flight from a previous session off this session's indexes. Null when not recording.
438
+ * A recording that ends discards its queue, and any retained shard with it: the consolidated
439
+ * end-of-call upload supersedes them, and if that upload is lost too, server-side recovery
440
+ * (#5196) fills the gap with silence.
441
+ */
442
+ this.shardQueue = null;
259
443
  // ── Server-side liveness ───────────────────────────────────────────────────
260
444
  /**
261
445
  * Interval that tells the server this session is still in use, or `null` when no session is
@@ -310,6 +494,11 @@ export class RealtimeSessionRuntime {
310
494
  this.pendingUsageInput = 0;
311
495
  /** Accumulated output-token delta since the last flush. */
312
496
  this.pendingUsageOutput = 0;
497
+ /**
498
+ * Accumulated per-modality usage since the last flush (input and output detail blocks, avatar
499
+ * video seconds included), relayed in full; `null` when no update carried any.
500
+ */
501
+ this.pendingUsageDetails = null;
313
502
  /** Pending debounced usage flush; also force-flushed at teardown. */
314
503
  this.usageFlushTimer = null;
315
504
  /** Active push-status subscription that feeds delegation progress; cleared on teardown. */
@@ -352,7 +541,20 @@ export class RealtimeSessionRuntime {
352
541
  */
353
542
  this.teardownInFlight = null;
354
543
  /**
355
- * Why the last session start failed, or `null` when none has.
544
+ * The latest start's opening of the host's microphone, until {@link IRealtimeMediaHost.ReleaseMicrophone} hands it
545
+ * back; `null` when no opening is owed a release, so a session that ends before asking for the microphone releases
546
+ * nothing.
547
+ *
548
+ * The host gets one release per opening, after the opening returns. A release while the microphone is still
549
+ * opening undoes nothing: the React Native host puts iOS into the call audio category inside `AcquireMicrophone`,
550
+ * after the permission prompt. So {@link runTeardown} releases an opening that has returned and leaves one still
551
+ * opening to its start, which releases it once it returns ({@link unwindAbandonedStart}), unless a newer start has
552
+ * opened the microphone since; that start's own end releases it. Whoever releases clears this first, so no
553
+ * opening is released twice.
554
+ */
555
+ this.microphoneLease = null;
556
+ /**
557
+ * Why the latest session start failed, or `null` when it has not. Each start clears it ({@link beginSessionStart}).
356
558
  *
357
559
  * The runtime reports failure as `'error'` on {@link ConnectionState$}, which is enough to show
358
560
  * *that* something went wrong but not *what* — and the difference matters at exactly one point:
@@ -361,6 +563,11 @@ export class RealtimeSessionRuntime {
361
563
  * runtime's own try/catch, so the host never sees that rejection itself.
362
564
  */
363
565
  this.lastStartError = null;
566
+ /**
567
+ * The fatal error the call's client reported, or `null` when it has reported none. Each start clears it
568
+ * ({@link beginSessionStart}). See {@link LastSessionError}.
569
+ */
570
+ this.lastSessionError = null;
364
571
  /**
365
572
  * Pending DEBOUNCED channel-state saves, keyed by channel name. Each entry keeps the
366
573
  * LATEST serialized state plus the session id captured while the session was live —
@@ -377,13 +584,92 @@ export class RealtimeSessionRuntime {
377
584
  */
378
585
  this.usedChannelNames = new Set();
379
586
  this._provider = null;
587
+ this._launcher = new DefaultRealtimeSessionLauncher();
588
+ /**
589
+ * Loads channel code on demand. Called, with the `ClientPluginClass` keys of every channel about to be
590
+ * resolved (the registry's rows plus the host's declarations), after the registry is read and before any
591
+ * plugin is built — so a host can register a heavy channel's class only when the session could use it.
592
+ * It must resolve once the classes it knows are registered; a rejection is logged, never fatal.
593
+ */
594
+ this.ChannelClassLoader = null;
595
+ // ── Session events (server → this session's client) ────────────────────────
596
+ this._sessionEvents$ = new Subject();
597
+ /**
598
+ * Every event the server publishes to THIS session, for the session's life (identity verification
599
+ * today; apps add their own types). Hot, with no replay — subscribe before the session starts, once,
600
+ * and it keeps working across sessions. The runtime has already acted on the events it understands
601
+ * (see {@link handleIdentityVerified}) by the time one reaches here; this stream is for the host UI.
602
+ *
603
+ * Silent when the session's provider has no subscription transport (see {@link createSessionEventSource}).
604
+ */
605
+ this.SessionEvents$ = this._sessionEvents$.asObservable();
606
+ /** Keeps the live session's event stream open; null between sessions. */
607
+ this.sessionEventHub = null;
608
+ /** Per-type handlers the runtime runs (before publishing on {@link SessionEvents$}) for events it understands. */
609
+ this.sessionEventHandlers = new Map([
610
+ ['identity.verified', (event) => this.handleIdentityVerified(event)]
611
+ ]);
612
+ /**
613
+ * What the agent should SAY the moment the user's identity is verified, or `null` (the default) to
614
+ * say nothing — the agent is still told, silently, through a context note. A string is used as the
615
+ * instruction verbatim; a function receives the verified payload (e.g. to greet by name) and returns
616
+ * the instruction, or `null` to stay silent for this verification.
617
+ */
618
+ this.IdentityVerifiedSpokenResponse = null;
619
+ this.clientDeadline = new ClientSessionDeadline();
620
+ // ── Interactive channels (registry-driven + host-declared plugins) ─────────
621
+ //
622
+ // A session's channels come from two places — the `MJ: AI Agent Channels` registry and the channels
623
+ // the host declares — and pass through three stages:
624
+ //
625
+ // 1. PREPARE (before mint) construct the plugins, read their descriptors, report them as
626
+ // candidates, and work out the scope the browser can decide alone.
627
+ // 2. MINT the server scopes the candidates with the agent/app cascade and the
628
+ // registry's kill switch, and returns the resolved policy.
629
+ // 3. ACTIVATE (after mint) initialize exactly the policy's channels: `open-on-start`/`headless`
630
+ // ones mount now, `on-demand` ones wait to be opened through ContextTool.
631
+ //
632
+ // Nothing is initialized before the policy is known, so a channel the server vetoes never runs.
633
+ /** Whether the channel catalog note still has to be sent (see {@link flushChannelCatalogNote}). */
634
+ this.catalogNotePending = false;
635
+ /** Plugins resolved for the current start but not yet initialized. Consumed by {@link activateChannels}. */
636
+ this.preparedChannels = [];
637
+ /** The scope the browser resolves on its own — the fallback when no server policy comes back. */
638
+ this.localChannelScope = null;
639
+ /** `on-demand` channels that are in the session but not opened yet (initialized only when opened). */
640
+ this.advertisedChannels = [];
641
+ /** What the policy resolved for each in-session channel, keyed by normalized channel key. */
642
+ this.resolvedChannels = new Map();
643
+ /** The prior session's saved channel states, kept so an `on-demand` channel opened later is restored too. */
644
+ this.priorChannelStates = {};
645
+ /** The app/static client-tool tiers the server resolved for this session. */
646
+ this.sessionClientTools = null;
647
+ /** Validates and routes channel-addressed `ContextTool` calls. */
648
+ this.channelDispatcher = new ChannelActionDispatcher({
649
+ FindChannel: (key) => this.findDispatchableChannel(key),
650
+ ListChannelKeys: () => this.listAddressableChannelKeys(),
651
+ ActivateChannel: (plugin) => this.mountAdvertisedChannel(plugin),
652
+ });
653
+ /** Where the user's per-channel "how much can the agent see" choices are kept. Replace with {@link SetExposurePreferences}. */
654
+ this.exposurePreferences = new UserSettingsExposurePreferences(() => this.canPersistUserSettings());
655
+ /** The channel whose surface the user is looking at, kept so it applies to an arbiter created after it was set. */
656
+ this.focusedChannelKey = null;
657
+ this.stopWatchingVideoSources = null;
380
658
  /**
381
659
  * Host registry of surface CLIENT TOOLS (Name → handler), fed by the host (Explorer) from the
382
- * active surface's `SetAgentClientTools`. The headless ClientContextChannel's `ContextTool` proxy
383
- * executes against this via {@link executeAppClientTool}. Keys are lower-cased for case-insensitive
384
- * model-supplied action names.
660
+ * active surface's `SetAgentClientTools` and from its always-on globals. The headless
661
+ * ClientContextChannel's `ContextTool` proxy executes against this via {@link executeAppClientTool}.
662
+ * Owner-keyed — see {@link AppClientToolRegistry}.
385
663
  */
386
- this.appClientToolHandlers = new Map();
664
+ this.appToolRegistry = new AppClientToolRegistry();
665
+ }
666
+ /** Synchronous read of {@link SavingRecording$}. */
667
+ get IsSavingRecording() {
668
+ return this._savingRecording$.value;
669
+ }
670
+ /** Synchronous access to {@link AvatarNotice$}'s current value. */
671
+ get CurrentAvatarNotice() {
672
+ return this._avatarNotice$.value;
387
673
  }
388
674
  /** Synchronous access to the session's active interactive-channel plugins. */
389
675
  get ActiveChannels() {
@@ -480,10 +766,113 @@ export class RealtimeSessionRuntime {
480
766
  set Provider(value) {
481
767
  this._provider = value;
482
768
  }
769
+ /**
770
+ * How this runtime mints a session — the stock `StartRealtimeClientSession` mutation unless a host
771
+ * installs its own {@link IRealtimeSessionLauncher} (a guest-session exchange, a deployment-specific
772
+ * mutation). Everything after the mint — channel scoping, the driver, transcripts, teardown — is
773
+ * unchanged. Assign `null` to restore the default.
774
+ */
775
+ get Launcher() {
776
+ return this._launcher;
777
+ }
778
+ set Launcher(value) {
779
+ this._launcher = value ?? new DefaultRealtimeSessionLauncher();
780
+ }
781
+ /**
782
+ * The client's copy of the session's absolute deadline (`null` when none is known). A host that
783
+ * knows the session's cap at start sets it with {@link SetSessionDeadline}; verification extends it
784
+ * from the server's `identity.verified` payload. The SERVER enforces the real deadline — this exists
785
+ * so the host can end the call gracefully first and show a countdown. It only ever moves later by
786
+ * the server's word, so nothing a client does can extend the real one.
787
+ */
788
+ get SessionDeadline$() {
789
+ return this.clientDeadline.Deadline$;
790
+ }
791
+ /** Synchronous access to {@link SessionDeadline$}. */
792
+ get SessionDeadline() {
793
+ return this.clientDeadline.Value;
794
+ }
795
+ /** Sets the baseline deadline a host knows at session start (`null` clears). See {@link SessionDeadline$}. */
796
+ SetSessionDeadline(deadline) {
797
+ this.clientDeadline.Set(deadline);
798
+ }
799
+ /**
800
+ * Builds the transport that reads the session's events. The default rides the session's GraphQL
801
+ * provider; it returns `null` — "no session events here" — for a provider with no subscription
802
+ * support (a test double, a host on another transport), which the runtime tolerates. Override to
803
+ * supply another transport.
804
+ */
805
+ createSessionEventSource() {
806
+ const provider = this.Provider;
807
+ if (!provider || typeof provider.Subscribe !== 'function') {
808
+ return null;
809
+ }
810
+ return new GraphQLRealtimeSessionClient(provider);
811
+ }
812
+ /** Opens the live session's event stream. Best-effort: never disturbs the call. */
813
+ startSessionEvents(agentSessionId) {
814
+ this.stopSessionEvents();
815
+ try {
816
+ const source = this.createSessionEventSource();
817
+ if (!source) {
818
+ return;
819
+ }
820
+ this.sessionEventHub = new RealtimeSessionEventHub(source, (event) => this.routeSessionEvent(agentSessionId, event));
821
+ this.sessionEventHub.Start(agentSessionId);
822
+ }
823
+ catch (error) {
824
+ console.error('[RealtimeSession] Could not start the session event stream:', error);
825
+ }
826
+ }
827
+ stopSessionEvents() {
828
+ this.sessionEventHub?.Stop();
829
+ this.sessionEventHub = null;
830
+ }
831
+ /** Routes one event to its per-type handler, then publishes it. An event for another session is dropped. */
832
+ routeSessionEvent(expectedSessionId, event) {
833
+ if (event.AgentSessionID !== expectedSessionId || this.agentSessionId !== expectedSessionId) {
834
+ console.warn(`[RealtimeSession] Dropped a '${event.Type}' event addressed to another session.`);
835
+ return;
836
+ }
837
+ try {
838
+ this.sessionEventHandlers.get(event.Type)?.(event);
839
+ }
840
+ catch (error) {
841
+ console.error(`[RealtimeSession] Handling the '${event.Type}' session event failed:`, error);
842
+ }
843
+ this._sessionEvents$.next(event);
844
+ }
845
+ /**
846
+ * `identity.verified`: keep the client's deadline in step with the server's, tell the model — as a
847
+ * silent, structured note — who it is now talking to, and, when the host configured it, have it say
848
+ * something about that. The payload is validated again here (it crossed a wire); the name rides as
849
+ * JSON so user-entered text can never break out of the note's frame.
850
+ */
851
+ handleIdentityVerified(event) {
852
+ if (event.Type !== 'identity.verified' || !IsIdentityVerifiedEventPayload(event.Payload)) {
853
+ console.warn('[RealtimeSession] Ignored a malformed identity.verified event.');
854
+ return;
855
+ }
856
+ const payload = event.Payload;
857
+ this.clientDeadline.Extend(payload.MaxSessionDeadlineIso);
858
+ this.SendContextNote(`[identity] verified ${JSON.stringify({ email: payload.VerifiedEmail, name: payload.VerifiedName, method: payload.Method })} ` +
859
+ '(background context — the user has proven they control this email address; treat them as verified. Do not read this note aloud.)');
860
+ const spoken = typeof this.IdentityVerifiedSpokenResponse === 'function' ? this.IdentityVerifiedSpokenResponse(payload) : this.IdentityVerifiedSpokenResponse;
861
+ if (spoken && spoken.trim().length > 0 && this.client && this.isSessionLive()) {
862
+ this.requestChannelSpokenResponse(spoken.trim());
863
+ }
864
+ }
483
865
  /** True when a session is currently open. */
484
866
  get IsActive() {
485
867
  return this._active$.value;
486
868
  }
869
+ /**
870
+ * True when a session is open or starting for `conversationId`. Ids are compared without
871
+ * regard to case.
872
+ */
873
+ IsActiveFor(conversationId) {
874
+ return this.IsActive && conversationId != null && UUIDsEqual(this.sessionConversationId, conversationId);
875
+ }
487
876
  /**
488
877
  * Start a client-direct voice session fronting `targetAgentId`.
489
878
  *
@@ -521,31 +910,55 @@ export class RealtimeSessionRuntime {
521
910
  * the server-side Media channel resolves THIS collection as the agent's media kit for the session,
522
911
  * taking precedence over the agent's `DefaultMediaCollectionID`. The server UUID-validates it
523
912
  * (malformed ⇒ ignored, the agent default applies). Omit/`null` to use the agent default kit.
913
+ * @param applicationId Optional application the session runs in (sources the app config cascade,
914
+ * including `Application.AgentSettings.Realtime.Channels`).
915
+ * @param appContext Optional live app-context snapshot injected into the companion prompt at mint.
916
+ * @param options Optional per-start extras — chiefly the channels the HOST brings
917
+ * ({@link RealtimeSessionStartOptions.HostChannels}), the way a connect-only embed with no
918
+ * registry gets channels at all, and the fork the session starts on
919
+ * ({@link RealtimeSessionStartOptions.ConversationBranchID}; omitted is Main, where its transcript is written).
524
920
  */
525
- async StartRealtimeSession(targetAgentId, conversationId, lastSessionId, agentName, preferredModelId, clientTools, coAgentId, configOverridesJson, recordingConsent, mediaCollectionId, applicationId, appContext) {
921
+ async StartRealtimeSession(targetAgentId, conversationId, lastSessionId, agentName, preferredModelId, clientTools, coAgentId, configOverridesJson, recordingConsent, mediaCollectionId, applicationId, appContext, options) {
526
922
  if (this.IsActive) {
527
923
  return; // a session is already running — ignore duplicate starts
528
924
  }
529
925
  const consent = this.beginSessionStart({ agentName, recordingConsent, applicationId, appContext });
530
- // Captured BEFORE startChannels so the mint carries exactly the snapshot the prologue
926
+ this.sessionConversationId = conversationId ?? null;
927
+ // A teardown while the channels are prepared or the mint runs bumps this; see the checks below.
928
+ const generation = this.startGeneration;
929
+ // Captured BEFORE the channels are prepared so the mint carries exactly the snapshot the prologue
531
930
  // resolved, whatever a channel plugin may push in the meantime.
532
931
  const effectiveAppContext = this._appContext$.value;
533
932
  let session;
534
933
  try {
535
- // Resolve + initialize the interactive-channel plugins FIRST: their client-executed
536
- // tool sets must be declared to the realtime model at session mint.
537
- const allClientTools = [...(clientTools ?? []), ...(await this.startChannels())];
538
- session = await this.mintSession(targetAgentId, conversationId, lastSessionId, preferredModelId, allClientTools, coAgentId, configOverridesJson, consent, this.recordingStartedAtIso, mediaCollectionId, this.applicationId, effectiveAppContext);
934
+ // Resolve the interactive-channel plugins FIRST (constructed, not started): the tools of the
935
+ // channels mounted with the session must be declared to the realtime model at mint, and the
936
+ // server needs the candidates to scope them. Nothing is initialized until the policy is known.
937
+ const scope = await this.prepareChannelScope(options?.HostChannels);
938
+ const allClientTools = [...(clientTools ?? []), ...scope.NativeTools];
939
+ session = await this.mintSession(targetAgentId, conversationId, lastSessionId, preferredModelId, allClientTools, coAgentId, configOverridesJson, consent, this.recordingStartedAtIso, mediaCollectionId, this.applicationId, effectiveAppContext, scope.CandidatesJson, scope.ShowsAgentVideo, options?.ConversationBranchID ?? null);
539
940
  }
540
941
  catch (error) {
942
+ // An ended or replaced start only logs its failure: the error state and the teardown belong
943
+ // to the session that is current now, which a newer start may own.
944
+ if (this.startGeneration !== generation) {
945
+ console.error('[RealtimeSession] A start that was already ended failed:', error);
946
+ return;
947
+ }
541
948
  await this.failSessionStart(error);
542
949
  return;
543
950
  }
951
+ // The host ended the start while the mint ran. That teardown had no session id to close, so
952
+ // close the minted row here, and touch no shared state: a newer start may already own it.
953
+ if (this.startGeneration !== generation) {
954
+ await this.closeServerSession(session.AgentSessionId);
955
+ return;
956
+ }
544
957
  if (!this.hostCanUseProvider(session.Provider)) {
545
958
  await this.abortUnusableSession(session);
546
959
  return;
547
960
  }
548
- await this.runMintedSession(session, conversationId ?? null, consent);
961
+ await this.runMintedSession(session, conversationId ?? null, consent, options?.CameraCheck === true);
549
962
  }
550
963
  /**
551
964
  * Declares which provider keys this host can actually carry audio for.
@@ -581,7 +994,7 @@ export class RealtimeSessionRuntime {
581
994
  // published them on ActiveChannels$; skipping teardown would leave them undisposed, their tool
582
995
  // handlers registered, and their subscriptions live until the next start replaced them.
583
996
  this.agentSessionId = session.AgentSessionId ?? this.agentSessionId;
584
- this._connectionState$.next('error');
997
+ this.setConnectionState('error');
585
998
  await this.teardown(true);
586
999
  }
587
1000
  /**
@@ -600,7 +1013,8 @@ export class RealtimeSessionRuntime {
600
1013
  * owns that half too.
601
1014
  *
602
1015
  * @param result The minted session — the same ten fields the `StartRealtimeClientSession`
603
- * mutation returns. `EphemeralToken` and `Provider` are what actually open the call.
1016
+ * mutation returns. `EphemeralToken` and `Provider` are what actually open the call; a relay
1017
+ * session opens with its `Transport` and `RelayUrl` instead, so a host must pass those on.
604
1018
  * @param options Host-side inputs the result cannot carry; see {@link RealtimeSessionRunOptions}.
605
1019
  * Every field defaults exactly as its {@link StartRealtimeSession} counterpart does.
606
1020
  */
@@ -614,16 +1028,20 @@ export class RealtimeSessionRuntime {
614
1028
  return;
615
1029
  }
616
1030
  const consent = this.beginSessionStart(effectiveOptions);
617
- await this.runMintedSession(result, effectiveOptions.conversationId ?? null, consent);
1031
+ await this.runMintedSession(result, effectiveOptions.conversationId ?? null, consent, false);
618
1032
  }
619
1033
  /**
620
- * Start prologue shared by both entry points: bind the app layer, publish the agent name, reset
621
- * per-session state, and flip the session live (which is ALSO what makes the `IsActive` guard
622
- * suppress duplicate starts while the mint is still in flight — hence it runs before minting, not
623
- * after). Returns the resolved recording consent, which the mint half reports to the server and
624
- * the run half uses to decide whether to record.
1034
+ * Start prologue shared by both entry points: clear the last start's failure, bind the app layer,
1035
+ * publish the agent name, reset per-session state, and flip the session live (which is ALSO what
1036
+ * makes the `IsActive` guard suppress duplicate starts while the mint is still in flight — hence it
1037
+ * runs before minting, not after). Returns the resolved recording consent, which the mint half
1038
+ * reports to the server and the run half uses to decide whether to record.
625
1039
  */
626
1040
  beginSessionStart(options) {
1041
+ // Before this start publishes anything: hosts read LastStartError as this start's states arrive,
1042
+ // so from the first of them it must describe this start, not an earlier one.
1043
+ this.lastStartError = null;
1044
+ this.lastSessionError = null;
627
1045
  // App awareness (Move 1/3/4): the application the session runs in (sources the app config
628
1046
  // cascade + RelevantAgents → allowed-agent union) and the live app-context snapshot injected
629
1047
  // into the companion prompt at mint. Stored so the ClientContextChannel can stream subsequent
@@ -641,7 +1059,7 @@ export class RealtimeSessionRuntime {
641
1059
  }
642
1060
  this.resetState();
643
1061
  this._active$.next(true);
644
- this._connectionState$.next('connecting');
1062
+ this.setConnectionState('connecting');
645
1063
  // Resolve recording consent for this session: explicit value wins, else the per-user
646
1064
  // persisted preference. Computed before mint so it can be reported to the server.
647
1065
  const consent = options.recordingConsent ?? this.readPersistedRecordingConsent();
@@ -651,12 +1069,18 @@ export class RealtimeSessionRuntime {
651
1069
  /**
652
1070
  * The RUN half of a session start, shared by both entry points: consume the minted result, open
653
1071
  * the provider connection, and go live. `inputConversationId` is the conversation the START asked
654
- * for (null ⇒ "server, make me one") — the result alone can't distinguish the two.
1072
+ * for (null ⇒ "server, make me one") — the result alone can't distinguish the two. `cameraCheck` is
1073
+ * whether the host shows a camera check ({@link RealtimeSessionStartOptions.CameraCheck}).
655
1074
  */
656
- async runMintedSession(session, inputConversationId, consent) {
1075
+ async runMintedSession(session, inputConversationId, consent, cameraCheck) {
657
1076
  // Captured up front: every await below is a window in which the host can end the session.
658
1077
  const generation = this.startGeneration;
659
1078
  try {
1079
+ // Mount the channels the resolved policy puts in the session. Deliberately BEFORE the session
1080
+ // id is adopted: `ActiveChannels$` consumers (the sessions adapter) treat an emission that
1081
+ // arrives while no session id is set as the INITIAL set, whose opens they synthesize from
1082
+ // `SessionStarted$` — activating after the id is set would announce every channel twice.
1083
+ this.applySessionClientPolicy(session);
660
1084
  this.agentSessionId = session.AgentSessionId;
661
1085
  // A null input conversationId means the SERVER created a fresh conversation for
662
1086
  // this session — track it so the host can fold it into the cached list, select
@@ -671,20 +1095,39 @@ export class RealtimeSessionRuntime {
671
1095
  this.applyPriorChannelStates(session.PriorChannelStatesJson);
672
1096
  const client = this.createRealtimeClient(session.Provider);
673
1097
  this.client = client;
1098
+ this.watchVideoSources(client);
674
1099
  this.wireClientHandlers(client);
675
1100
  // Everything past here awaits on hardware and the network, during which the host may end the
676
- // session. Each await is followed by a staleness check so an abandoned start releases what it
677
- // just acquired instead of leaving a live microphone and a live call behind it.
678
- this.localStream = await this.mediaHost.AcquireMicrophone();
1101
+ // session. Each await is followed by a staleness check, before any failure is reported, so an
1102
+ // abandoned start releases what it just acquired instead of leaving a live microphone and a live
1103
+ // call behind it, and reports nothing to a host that has ended the call.
1104
+ const lease = { Opening: true };
1105
+ this.microphoneLease = lease;
1106
+ const microphone = await this.openMicrophone().finally(() => {
1107
+ lease.Opening = false;
1108
+ });
1109
+ this.localStream = microphone.Stream;
679
1110
  if (this.startGeneration !== generation) {
680
- await this.unwindAbandonedStart(session, client);
1111
+ await this.unwindAbandonedStart(session, client, lease);
681
1112
  return;
682
1113
  }
683
- await client.Connect(this.BuildClientConfig(session), this.localStream);
1114
+ if (microphone.Error) {
1115
+ throw microphone.Error;
1116
+ }
1117
+ const clientConfig = this.BuildClientConfig(session);
1118
+ const connectError = await this.connectClient(client, clientConfig, microphone.Stream);
684
1119
  if (this.startGeneration !== generation) {
685
- await this.unwindAbandonedStart(session, client);
1120
+ await this.unwindAbandonedStart(session, client, lease);
686
1121
  return;
687
1122
  }
1123
+ if (connectError) {
1124
+ throw connectError;
1125
+ }
1126
+ // Tracks are negotiated now, so a capture can tell whether the model takes video, and the call whether it shows
1127
+ // the avatar its agent asked for. The driver holds the microphone now, so the user may switch it.
1128
+ this.openCaptures(client, cameraCheck);
1129
+ this.offerMicrophone();
1130
+ this.publishAvatarNotice(session, clientConfig, client);
688
1131
  // Notify active channels that the session client is connected and tracks are established
689
1132
  for (const channel of this._activeChannels$.value) {
690
1133
  try {
@@ -694,6 +1137,10 @@ export class RealtimeSessionRuntime {
694
1137
  console.error(`[RealtimeSession] Error in channel '${channel.ChannelName}' OnSessionStarted:`, err);
695
1138
  }
696
1139
  }
1140
+ // Tell the model which channels exist and how to use them, from the channels' own descriptors —
1141
+ // as soon as the control channel is usable (which may be right now, or a moment after Connect).
1142
+ this.catalogNotePending = true;
1143
+ this.flushChannelCatalogNote();
697
1144
  // Start browser-side recording (mic + agent mix) when consented. Best-effort: an
698
1145
  // unsupported browser / missing remote stream degrades gracefully (mic-only or off)
699
1146
  // and never blocks the call. The remote stream may still be null here (the WebRTC
@@ -718,6 +1165,8 @@ export class RealtimeSessionRuntime {
718
1165
  // SessionsObserver bridge. Emitting AFTER Connect() guarantees both that
719
1166
  // agentSessionId is set (line ~468) AND the realtime client is connected,
720
1167
  // so consumers can act on it without re-checking either condition.
1168
+ // The session's own event stream (identity verification, app events) stays open for its life.
1169
+ this.startSessionEvents(this.agentSessionId);
721
1170
  this._sessionStarted$.next({
722
1171
  sessionId: this.agentSessionId,
723
1172
  channelNames: this._activeChannels$.value.map(c => c.ChannelName),
@@ -730,18 +1179,42 @@ export class RealtimeSessionRuntime {
730
1179
  await this.failSessionStart(error);
731
1180
  }
732
1181
  }
1182
+ /**
1183
+ * Tells the host, once per call, why it shows no avatar ({@link AvatarNotice$}): from the mint's status, whether this
1184
+ * session asked for the agent's video, and whether that track is live now that the call is connected.
1185
+ */
1186
+ publishAvatarNotice(session, config, client) {
1187
+ const notice = ResolveAvatarNotice(ParseRealtimeAvatarStatus(session.AvatarStatusJson), RequestsAgentVideo(config), client.IsTrackEstablished('video', 'outbound'));
1188
+ if (notice) {
1189
+ this._avatarNotice$.next(notice);
1190
+ }
1191
+ }
733
1192
  /**
734
1193
  * Releases everything a start acquired after the host had already ended the session.
735
1194
  *
736
1195
  * Reached only when {@link teardown} ran while this start was awaiting the microphone or the
737
- * provider connection. Teardown found nothing to release because nothing existed yet, so this
738
- * start owns the cleanup: stop the microphone, close the provider connection, and close the
739
- * server-side session row that the mint created.
1196
+ * provider connection, whether that wait then succeeded or failed: the host has ended the call,
1197
+ * so a failure there is not reported. Teardown could not release what did not exist yet, so this
1198
+ * start owns the cleanup: stop the microphone, hand it back to the host when teardown left that
1199
+ * to it ({@link microphoneLease}), close the provider connection, and close the server-side
1200
+ * session row that the mint created when no teardown has closed it or is closing it.
1201
+ *
1202
+ * @param lease This start's opening of the microphone.
740
1203
  */
741
- async unwindAbandonedStart(session, client) {
1204
+ async unwindAbandonedStart(session, client, lease) {
742
1205
  console.warn('[RealtimeSession] Session was ended while starting — releasing the partial session.');
1206
+ this.closeCaptures();
1207
+ this.closeLocalMedia();
743
1208
  this.localStream?.getTracks().forEach(t => t.stop());
744
1209
  this.localStream = null;
1210
+ this.microphoneMuted = false;
1211
+ // Teardown left the release to this start when the microphone was still opening. A cleared or
1212
+ // replaced lease means teardown already released it, or a newer start has opened the microphone
1213
+ // since and its own end releases it.
1214
+ if (this.microphoneLease === lease) {
1215
+ this.microphoneLease = null;
1216
+ await this.releaseMicrophone();
1217
+ }
745
1218
  try {
746
1219
  await client.Disconnect();
747
1220
  }
@@ -751,27 +1224,321 @@ export class RealtimeSessionRuntime {
751
1224
  // Only clear the shared slots when they still point at THIS attempt — a newer start may
752
1225
  // already have replaced them.
753
1226
  if (this.client === client) {
1227
+ this.unwatchVideoSources();
1228
+ this.clearAgentVideo();
754
1229
  this.client = null;
755
1230
  }
756
- // Close the server session only if teardown has NOT already done so. It nulls `agentSessionId`
757
- // after closing, so a still-matching id means this attempt still owns the row; a cleared one
758
- // means the teardown that invalidated this start already closed it, and closing again would
759
- // send a second `CloseAgentSession` for one session.
760
- if (session.AgentSessionId && this.agentSessionId === session.AgentSessionId) {
1231
+ // Close the server session only if no teardown has closed it or is closing it. Teardown nulls
1232
+ // `agentSessionId` after closing, so a cleared id means the teardown that invalidated this start
1233
+ // already closed it. A teardown still running closes it itself and then reports the end on
1234
+ // `SessionEnded$`: closing it here as well would send a second `CloseAgentSession` for one
1235
+ // session, and clearing the id would leave that teardown no session to report.
1236
+ if (session.AgentSessionId && this.agentSessionId === session.AgentSessionId && !this.teardownInFlight) {
761
1237
  this.agentSessionId = null;
762
1238
  await this.closeServerSession(session.AgentSessionId);
763
1239
  }
764
1240
  }
765
1241
  /**
766
- * Why the last session start failed, or `null` when the last start succeeded or none has run.
1242
+ * Opens the user's microphone: through the host's controller when it offers one, otherwise through
1243
+ * {@link IRealtimeMediaHost.AcquireMicrophone}. A failure is returned rather than thrown, so the caller can
1244
+ * first check whether the start was abandoned meanwhile (a teardown disposes the controller, which fails a
1245
+ * start still in flight) and unwind quietly instead of reporting an error nobody is waiting for.
1246
+ */
1247
+ async openMicrophone() {
1248
+ const controller = this.mediaHost.CreateLocalMediaController?.() ?? null;
1249
+ if (!controller) {
1250
+ try {
1251
+ return { Stream: await this.mediaHost.AcquireMicrophone(), Error: null };
1252
+ }
1253
+ catch (error) {
1254
+ return { Stream: null, Error: error instanceof Error ? error : new Error(String(error)) };
1255
+ }
1256
+ }
1257
+ this.localMedia = controller;
1258
+ const started = await controller.Start('microphone');
1259
+ if (started.Status === 'failed') {
1260
+ return { Stream: null, Error: microphoneStartError(started.Reason, started.Message) };
1261
+ }
1262
+ this.followMicrophone(controller, started.Stream);
1263
+ return { Stream: started.Stream, Error: null };
1264
+ }
1265
+ /**
1266
+ * Opens the provider connection. As with {@link openMicrophone}, a failure is returned rather than thrown, so the
1267
+ * caller can first check whether the start was abandoned meanwhile and unwind quietly instead of reporting an error
1268
+ * nobody is waiting for: a teardown disconnects the client, which with most drivers fails a connection still opening.
1269
+ */
1270
+ async connectClient(client, config, microphone) {
1271
+ try {
1272
+ await client.Connect(config, microphone);
1273
+ return null;
1274
+ }
1275
+ catch (error) {
1276
+ return error instanceof Error ? error : new Error(String(error));
1277
+ }
1278
+ }
1279
+ /**
1280
+ * Moves the driver and the recorder onto the call's microphone whenever the controller gives it a new track. A device
1281
+ * switch, or a lost device replaced by the default, swaps a track into the same stream, and the controller carries the
1282
+ * old track's mute over. A microphone opened again after the call lost one comes in a new stream: its track takes the
1283
+ * call's mute, and the stream becomes the one {@link ToggleMute} and the teardown act on. Mid-swap the stream holds no
1284
+ * track, so only a finished swap is followed. Every report also keeps {@link Microphone$} current once the call offers
1285
+ * it.
1286
+ */
1287
+ followMicrophone(controller, stream) {
1288
+ let followedStream = stream;
1289
+ let followedTrack = stream.getAudioTracks()[0] ?? null;
1290
+ this.localMediaSubscription = controller.State$.subscribe((media) => {
1291
+ const current = controller.GetStream('microphone') ?? followedStream;
1292
+ const track = current.getAudioTracks()[0] ?? null;
1293
+ if (track && track !== followedTrack) {
1294
+ if (current !== followedStream) {
1295
+ track.enabled = !this.microphoneMuted;
1296
+ this.localStream = current;
1297
+ followedStream = current;
1298
+ }
1299
+ followedTrack = track;
1300
+ this.recorder?.ReplaceMicrophone?.(current);
1301
+ this.client?.ReplaceMicrophone?.(current)?.catch((error) => {
1302
+ console.error('[RealtimeSession] The realtime driver could not move to the new microphone:', error);
1303
+ });
1304
+ }
1305
+ this.publishMicrophone(media);
1306
+ });
1307
+ }
1308
+ /** The client is connected: the call offers its microphone on {@link Microphone$}, and {@link SwitchMicrophone} moves it. */
1309
+ offerMicrophone() {
1310
+ if (!this.localMedia) {
1311
+ return;
1312
+ }
1313
+ this.microphoneOffered = true;
1314
+ this.publishMicrophone(this.localMedia.State);
1315
+ }
1316
+ /** Publishes the call's microphone from the controller's state while the call offers it, when it names a change. */
1317
+ publishMicrophone(media) {
1318
+ if (!this.microphoneOffered) {
1319
+ return;
1320
+ }
1321
+ const current = this._microphone$.value;
1322
+ const next = ReadMicrophoneState(media, current);
1323
+ if (!SameMicrophoneState(next, current)) {
1324
+ this._microphone$.next(next);
1325
+ }
1326
+ }
1327
+ /** Stops following the controller and disposes it, which releases its devices; the call no longer offers a microphone. */
1328
+ closeLocalMedia() {
1329
+ this.localMediaSubscription?.unsubscribe();
1330
+ this.localMediaSubscription = null;
1331
+ this.localMedia?.Dispose();
1332
+ this.localMedia = null;
1333
+ this.microphoneOffered = false;
1334
+ if (this._microphone$.value !== REALTIME_MICROPHONE_NONE) {
1335
+ this._microphone$.next(REALTIME_MICROPHONE_NONE);
1336
+ }
1337
+ }
1338
+ /**
1339
+ * Creates the session's camera and screen share and mirrors their state on {@link Captures$}. With `cameraCheck`, the
1340
+ * call's first camera start waits for {@link ConfirmCamera}.
1341
+ */
1342
+ openCaptures(client, cameraCheck) {
1343
+ this.closeCaptures();
1344
+ const captures = new RealtimeCaptures({
1345
+ Client: client,
1346
+ LocalMedia: this.localMedia,
1347
+ Host: this.mediaHost,
1348
+ CreateSampler: this.hostFrameSampler(),
1349
+ Admit: (kind) => this.admitCapture(kind),
1350
+ CameraCheck: cameraCheck,
1351
+ });
1352
+ this.captures = captures;
1353
+ this.capturesSubscription = captures.States$.subscribe((states) => this._captures$.next(states));
1354
+ this.refreshCaptureOffers();
1355
+ }
1356
+ /**
1357
+ * The host's frame sampler for the captures ({@link IRealtimeMediaHost.CreateFrameSampler}), or `undefined` when it offers
1358
+ * none and the captures use the DOM default.
1359
+ */
1360
+ hostFrameSampler() {
1361
+ const host = this.mediaHost;
1362
+ const create = host.CreateFrameSampler;
1363
+ return create ? (stream, options) => create.call(host, stream, options) : undefined;
1364
+ }
1365
+ /** Stops the camera and screen share, if any, and reports both off. */
1366
+ closeCaptures() {
1367
+ this.capturesSubscription?.unsubscribe();
1368
+ this.capturesSubscription = null;
1369
+ this.captures?.Dispose();
1370
+ this.captures = null;
1371
+ if (this._captures$.value !== REALTIME_CAPTURES_OFF) {
1372
+ this._captures$.next(REALTIME_CAPTURES_OFF);
1373
+ }
1374
+ this.refreshCaptureOffers();
1375
+ }
1376
+ /** Works out which captures the call offers ({@link CaptureOffers$}) and publishes a change. */
1377
+ refreshCaptureOffers() {
1378
+ const offers = this.captures && this.client?.SupportsInboundVideo ? { Camera: this.canOffer('camera'), Screen: this.canOffer('screen') } : REALTIME_CAPTURE_OFFERS_NONE;
1379
+ const current = this._captureOffers$.value;
1380
+ if (offers.Camera !== current.Camera || offers.Screen !== current.Screen) {
1381
+ this._captureOffers$.next(offers);
1382
+ }
1383
+ }
1384
+ /** Whether the host can open a capture and the call's policy admits it. */
1385
+ canOffer(kind) {
1386
+ const hostCanOpen = kind === 'camera' ? this.localMedia !== null : typeof this.mediaHost.RequestDisplayCapture === 'function';
1387
+ return hostCanOpen && this.admitCapture(kind).Admitted;
1388
+ }
1389
+ /** What a capture reports when there is no live session to show it to. */
1390
+ noSessionCapture() {
1391
+ return { Status: 'failed', Failure: 'no-session', Message: 'There is no call to share with.' };
1392
+ }
1393
+ /**
1394
+ * The session's policy for a capture. The channel that fronts it (its `CaptureKind`) must be in the session, open or
1395
+ * advertised, and the server's policy must let the agent see pixels through it (the agent's configuration and any
1396
+ * zero-data-retention rule). The user's own "agent can see" choice for the channel does not refuse a start: it decides
1397
+ * whether the frames reach the model.
1398
+ */
1399
+ admitCapture(kind) {
1400
+ const channel = this.findCaptureChannel(kind)?.Plugin;
1401
+ if (!channel) {
1402
+ return { Admitted: false, Message: CAPTURE_WORDING[kind].NotInCall };
1403
+ }
1404
+ const resolved = this.GetResolvedChannel(channel.ChannelName);
1405
+ const allowed = resolved?.Exposure ?? resolved?.MaxExposure ?? channel.GetDescriptor().MaxExposure;
1406
+ if (allowed !== 'pixels') {
1407
+ const reasons = (resolved?.ExposureLimits ?? []).map((limit) => limit.Reason);
1408
+ return { Admitted: false, Message: reasons.length > 0 ? `${CAPTURE_WORDING[kind].NotAllowed}: ${reasons.join('; ')}.` : `${CAPTURE_WORDING[kind].NotAllowed}.` };
1409
+ }
1410
+ return { Admitted: true, ChannelKey: channel.ChannelName, VisibleToAgent: channel.Exposure === 'pixels' };
1411
+ }
1412
+ /** The channel in the session, open or advertised, that fronts a capture. */
1413
+ findCaptureChannel(kind) {
1414
+ const open = this._activeChannels$.value.find((c) => c.CaptureKind === kind);
1415
+ if (open) {
1416
+ return { Plugin: open, IsOpen: true };
1417
+ }
1418
+ const advertised = this.advertisedChannels.find((p) => p.Plugin.CaptureKind === kind);
1419
+ return advertised ? { Plugin: advertised.Plugin, IsOpen: false } : null;
1420
+ }
1421
+ /**
1422
+ * Brings up the channel that fronts a capture once the capture is starting, so the host shows its surface: an advertised
1423
+ * channel is opened (and announces itself to the agent), an open one counts as used.
1424
+ */
1425
+ async revealCaptureChannel(kind) {
1426
+ const state = kind === 'camera' ? this.captures?.States.Camera : this.captures?.States.Screen;
1427
+ const channel = state?.Status === 'starting' ? this.findCaptureChannel(kind) : null;
1428
+ if (!channel) {
1429
+ return;
1430
+ }
1431
+ if (channel.IsOpen) {
1432
+ this.noteChannelActivity(channel.Plugin);
1433
+ return;
1434
+ }
1435
+ const opened = await this.OpenChannel(channel.Plugin.ChannelName);
1436
+ if (!opened.Success) {
1437
+ console.warn(`[RealtimeSession] Could not open channel '${channel.Plugin.ChannelName}' for the ${kind}: ${opened.ErrorMessage ?? opened.ErrorCode}`);
1438
+ }
1439
+ }
1440
+ /**
1441
+ * Starts the user's camera and shows it to the agent. Call it from the user's click: it may ask for camera
1442
+ * permission. Resolves with the camera's state; a failure (no session, a model that takes no video, a refused
1443
+ * permission) is a state with a message, never a throw. While the camera is on, it is a source on
1444
+ * {@link VideoSources$}, and it stops by itself if the device goes away.
1445
+ *
1446
+ * When the host shows a camera check ({@link RealtimeSessionStartOptions.CameraCheck}), the call's first start
1447
+ * resolves still starting, with `Checking` set and the stream for the host to preview. The agent sees the camera
1448
+ * once the host calls {@link ConfirmCamera}; {@link StopCamera} is the user's "not now".
1449
+ *
1450
+ * @param deviceId The camera to open; the system default when absent.
1451
+ */
1452
+ async StartCamera(deviceId) {
1453
+ if (!this.captures) {
1454
+ return this.noSessionCapture();
1455
+ }
1456
+ const started = this.captures.Start('camera', { DeviceID: deviceId });
1457
+ await this.revealCaptureChannel('camera');
1458
+ return started;
1459
+ }
1460
+ /** Stops the user's camera. Safe to call when it is off. */
1461
+ StopCamera() {
1462
+ this.captures?.Stop('camera');
1463
+ }
1464
+ /**
1465
+ * Moves the user's camera to another device while it is open (being checked or on): one of the camera's `Devices` on
1466
+ * {@link Captures$}. The stream stays the same. Resolves with the camera's state; nothing changes while the camera is
1467
+ * off.
1468
+ *
1469
+ * @param deviceId The camera to move to.
1470
+ */
1471
+ async SwitchCamera(deviceId) {
1472
+ return this.captures ? this.captures.SwitchCamera(deviceId) : this.noSessionCapture();
1473
+ }
1474
+ /**
1475
+ * Moves the call's microphone to another device: one of the `Devices` on {@link Microphone$}. The host's controller
1476
+ * swaps the new track into the same stream and carries the mute over, and the driver and the call's recording move to
1477
+ * it, so the agent keeps hearing the user. When the new microphone cannot open, the controller goes back to the one in
1478
+ * use. While the call has no microphone open (it lost the one in use and the default, or the last one went away), the
1479
+ * pick opens one again: the driver and the recording move to it, and it keeps the call's mute. Resolves with the
1480
+ * microphone's state once the switch is over. Nothing changes outside a call or on a host without a controller.
1481
+ *
1482
+ * @param deviceId The microphone to move to.
1483
+ */
1484
+ async SwitchMicrophone(deviceId) {
1485
+ const controller = this.localMedia;
1486
+ if (controller && this.microphoneOffered) {
1487
+ const status = controller.State.Microphone.Status;
1488
+ const live = status === 'on' || status === 'starting';
1489
+ await (live ? controller.SwitchDevice('microphone', deviceId) : controller.Start('microphone', deviceId));
1490
+ }
1491
+ return this._microphone$.value;
1492
+ }
1493
+ /**
1494
+ * The user checked the camera and turned it on: the agent sees it from now on, and later camera starts in this call
1495
+ * skip the check. Returns the camera's state; unless the camera is waiting for its check (see {@link StartCamera}),
1496
+ * nothing changes.
1497
+ */
1498
+ ConfirmCamera() {
1499
+ return this.captures ? this.captures.ConfirmCamera() : this.noSessionCapture();
1500
+ }
1501
+ /**
1502
+ * Asks the user for a screen, window or browser tab (or one panel of the page) and shows it to the agent. Call it
1503
+ * from the user's click: the browser's picker needs one. Resolves with the share's state; a failure (no session,
1504
+ * a model that takes no video, a closed picker, a host that cannot share) is a state with a message. The share
1505
+ * also stops when the user ends it from the browser's own bar.
1506
+ *
1507
+ * @param options What the picker offers first, or the panel to share and its name (`PanelLabel`), which the share's
1508
+ * state carries and the agent sees as "<name> (shared panel)".
1509
+ */
1510
+ async StartScreenShare(options) {
1511
+ if (!this.captures) {
1512
+ return this.noSessionCapture();
1513
+ }
1514
+ const started = this.captures.Start('screen', options);
1515
+ await this.revealCaptureChannel('screen');
1516
+ return started;
1517
+ }
1518
+ /** Stops the user's screen share. Safe to call when nothing is shared. */
1519
+ StopScreenShare() {
1520
+ this.captures?.Stop('screen');
1521
+ }
1522
+ /**
1523
+ * Why the latest session start failed, or `null` when it has not failed or none has run.
767
1524
  *
768
1525
  * Read it when {@link ConnectionState$} reports `'error'`, to tell a denied microphone apart from
769
- * a provider or backend failure and show copy the user can act on. Cleared at the start of every
770
- * session.
1526
+ * a provider or backend failure and show copy the user can act on. Each start clears it before it
1527
+ * reports `'connecting'`, so it never describes an earlier start.
771
1528
  */
772
1529
  get LastStartError() {
773
1530
  return this.lastStartError;
774
1531
  }
1532
+ /**
1533
+ * The fatal error the call's client reported, or `null` when it has reported none.
1534
+ *
1535
+ * Read it when {@link ConnectionState$} reports `'error'` on a call that was live, to tell a call that reached its time
1536
+ * limit (`Code` `'time-limit'`; {@link DescribeCallEnd} words it) from a dropped connection. Each start clears it before
1537
+ * it reports `'connecting'`.
1538
+ */
1539
+ get LastSessionError() {
1540
+ return this.lastSessionError;
1541
+ }
775
1542
  /**
776
1543
  * The single failure path for a session start (mint half or run half): report it, latch the
777
1544
  * overlay into 'error', and unwind whatever the half-built session already opened.
@@ -779,7 +1546,7 @@ export class RealtimeSessionRuntime {
779
1546
  async failSessionStart(error) {
780
1547
  console.error('[RealtimeSession] Failed to start session:', error);
781
1548
  this.lastStartError = error instanceof Error ? error : new Error(String(error));
782
- this._connectionState$.next('error');
1549
+ this.setConnectionState('error');
783
1550
  await this.teardown(false);
784
1551
  }
785
1552
  /**
@@ -818,14 +1585,19 @@ export class RealtimeSessionRuntime {
818
1585
  // Relay as a user turn — same path spoken input uses (caption + persisted transcript).
819
1586
  void this.onUserTranscript(trimmed);
820
1587
  }
821
- /** Mute / unmute the local microphone track. Returns the new muted state. */
1588
+ /**
1589
+ * Mute / unmute the local microphone track. Returns the new muted state. While the call has no microphone track (a
1590
+ * switch is putting a new one in, or the call lost its microphone) there is nothing to mute: nothing changes, and it
1591
+ * returns the call's mute, which the next microphone keeps. Outside a call it returns `false`.
1592
+ */
822
1593
  ToggleMute() {
823
1594
  const tracks = this.localStream?.getAudioTracks() ?? [];
824
1595
  if (tracks.length === 0) {
825
- return false;
1596
+ return this.microphoneMuted;
826
1597
  }
827
1598
  const muted = tracks[0].enabled; // currently enabled → becomes muted
828
1599
  this.client?.SetMuted(muted);
1600
+ this.microphoneMuted = muted;
829
1601
  return muted;
830
1602
  }
831
1603
  // ── Client-executed UI tools ───────────────────────────────────────────────
@@ -895,13 +1667,21 @@ export class RealtimeSessionRuntime {
895
1667
  return this.client;
896
1668
  }
897
1669
  /**
898
- * Relays a video frame to the underlying realtime client if active.
1670
+ * Relays a video frame straight to the underlying realtime client, if a session is live.
1671
+ *
1672
+ * It bypasses the session's {@link VideoSourceArbiter}: the frame overrides the source the arbiter picked, and the
1673
+ * model is not told what it is looking at. Channels no longer send through it (their context's `SendVideoFrame` goes
1674
+ * through the arbiter); a host that shows the model something registers a source there instead, as
1675
+ * `ChannelInboundVideoBridge` does.
1676
+ *
1677
+ * @returns `true` when the client accepted and sent the frame; `false` when no session is live, the
1678
+ * client cannot take video frames, or the client dropped the frame.
899
1679
  */
900
1680
  SendVideoFrame(base64Image, mimeType) {
901
1681
  if (!this.client || !this.isSessionLive()) {
902
- return;
1682
+ return false;
903
1683
  }
904
- this.client.SendVideoFrame?.(base64Image, mimeType);
1684
+ return this.client.SendVideoFrame?.(base64Image, mimeType) === true;
905
1685
  }
906
1686
  /**
907
1687
  * Checks whether a media track is established on the active realtime client.
@@ -949,9 +1729,11 @@ export class RealtimeSessionRuntime {
949
1729
  this.currentTurnStartMs = recorder.IsRecording ? 0 : null;
950
1730
  this.turnAudioStartCaptured = false;
951
1731
  if (this.recorder) {
952
- // The agent's WebRTC audio track usually lands AFTER Connect() resolves, so `remoteStream`
953
- // above is typically null and we'd capture mic-only. Attach the agent stream whenever it
954
- // arrives (fires immediately if already present) so the recording includes the agent voice.
1732
+ // WebRTC drivers (OpenAI): the agent's track usually lands AFTER Connect() resolves, so
1733
+ // `remoteStream` above is null here and this handler attaches it later. PCM-playback
1734
+ // drivers (Gemini, ElevenLabs, AssemblyAI, xAI, HuggingFace) publish at Connect, so
1735
+ // `remoteStream` is already set and the handler fires immediately with the same stream;
1736
+ // AttachRemoteStream is idempotent, so it is mixed only once.
955
1737
  client.OnRemoteMediaStream?.((stream) => this.recorder?.AttachRemoteStream(stream));
956
1738
  this.startSegmentFlushing();
957
1739
  }
@@ -963,7 +1745,7 @@ export class RealtimeSessionRuntime {
963
1745
  }
964
1746
  /** Begins flushing ~15s crash-recovery shards to the server for the duration of the recording. */
965
1747
  startSegmentFlushing() {
966
- this.segmentIndex = 0;
1748
+ this.shardQueue = new RecordingShardQueue();
967
1749
  this.segmentTimer = setInterval(() => { void this.flushRecordingSegment(); }, RealtimeSessionRuntime.segmentFlushMs);
968
1750
  }
969
1751
  /** Stops the periodic crash-recovery shard flush. */
@@ -1030,48 +1812,64 @@ export class RealtimeSessionRuntime {
1030
1812
  }
1031
1813
  /**
1032
1814
  * Uploads the chunks captured since the last flush as one crash-recovery shard (durability only;
1033
- * the canonical file is still the full upload at teardown). Best-effort — never disturbs the call.
1815
+ * the canonical file is still the full upload at teardown). Best-effort — never disturbs the call:
1816
+ * a failed shard is retained by the queue and resent under its own index on later flushes, up to a
1817
+ * bounded number of attempts (#5197).
1034
1818
  */
1035
1819
  async flushRecordingSegment() {
1036
1820
  const recorder = this.recorder;
1037
1821
  const agentSessionId = this.agentSessionId;
1038
- if (!recorder || !agentSessionId) {
1822
+ const queue = this.shardQueue;
1823
+ if (!recorder || !agentSessionId || !queue) {
1039
1824
  return;
1040
1825
  }
1041
- try {
1042
- const audioBase64 = await recorder.SnapshotNewSegmentBase64();
1043
- if (!audioBase64) {
1044
- return;
1045
- }
1046
- const index = this.segmentIndex++;
1047
- const mutation = `
1048
- mutation UploadRealtimeRecordingSegment($agentSessionId: String!, $segmentIndex: Int!, $audioBase64: String!, $mimeType: String!) {
1049
- UploadRealtimeRecordingSegment(agentSessionId: $agentSessionId, segmentIndex: $segmentIndex, audioBase64: $audioBase64, mimeType: $mimeType)
1050
- }
1051
- `;
1052
- // Shards are HEADER-LESS raw little-endian PCM16 (mime audio/L16 with the capture sample rate),
1053
- // NOT individually-playable WAV — recovery is concatenate-in-order then WAV-wrap. The canonical
1054
- // seekable WAV is the consolidated end-of-call upload below.
1055
- const shardMime = `audio/L16;rate=${recorder.SampleRate}`;
1056
- await this.gql().ExecuteGQL(mutation, { agentSessionId, segmentIndex: index, audioBase64, mimeType: shardMime });
1057
- }
1058
- catch (error) {
1059
- console.warn('[RealtimeSession] Failed to flush recording shard:', error);
1060
- }
1826
+ // Read once: a shard's sample rate is fixed at capture, and the recorder may be stopped by the
1827
+ // time a retry of it runs.
1828
+ const sampleRate = recorder.SampleRate;
1829
+ await queue.Flush(() => recorder.SnapshotNewSegmentBase64(), (segmentIndex, audioBase64) => this.uploadRecordingShard(agentSessionId, sampleRate, segmentIndex, audioBase64));
1061
1830
  }
1062
1831
  /**
1063
- * Stops the active recorder and uploads the captured audio via `UploadRealtimeRecording`.
1064
- * Fully best-effort and wrapped in try/catch — recording upload must NEVER block teardown.
1065
- * No-op when nothing was recorded or there's no session id to attach the file to.
1832
+ * Sends one shard and reports whether the server stored it. The resolver reports a failed
1833
+ * ownership load or storage write as `false`, not as an error — that is how #5197 lost a shard
1834
+ * without a single client-side log line — so the typed result must be checked, not just the absence
1835
+ * of a throw.
1066
1836
  */
1067
- async stopAndUploadRecording(agentSessionId) {
1837
+ async uploadRecordingShard(agentSessionId, sampleRate, segmentIndex, audioBase64) {
1838
+ const mutation = `
1839
+ mutation UploadRealtimeRecordingSegment($agentSessionId: String!, $segmentIndex: Int!, $audioBase64: String!, $mimeType: String!) {
1840
+ UploadRealtimeRecordingSegment(agentSessionId: $agentSessionId, segmentIndex: $segmentIndex, audioBase64: $audioBase64, mimeType: $mimeType)
1841
+ }
1842
+ `;
1843
+ // Shards are HEADER-LESS raw little-endian PCM16 (mime audio/L16 with the capture sample rate),
1844
+ // NOT individually-playable WAV — recovery is concatenate-in-order then WAV-wrap. The canonical
1845
+ // seekable WAV is the consolidated end-of-call upload below.
1846
+ const shardMime = `audio/L16;rate=${sampleRate}`;
1847
+ const result = (await this.gql().ExecuteGQL(mutation, {
1848
+ agentSessionId,
1849
+ segmentIndex,
1850
+ audioBase64,
1851
+ mimeType: shardMime,
1852
+ }));
1853
+ return result?.UploadRealtimeRecordingSegment === true;
1854
+ }
1855
+ /**
1856
+ * Stops the active recorder and encodes the captured audio. Split from the upload so teardown can
1857
+ * close the server session in between (#5195). Best-effort: returns `null` when nothing was
1858
+ * recorded or the stop failed (warned) — it must NEVER block teardown.
1859
+ */
1860
+ async stopRecording() {
1068
1861
  this.stopSegmentFlushing();
1862
+ // Close the queue with the recording: its retained shards (base64 audio) are released now, not at
1863
+ // the next session start, and a flush still in flight stops sending, so no resend lands after the
1864
+ // end-of-call upload below has deleted the shards.
1865
+ this.shardQueue?.Close();
1866
+ this.shardQueue = null;
1069
1867
  const recorder = this.recorder;
1070
1868
  this.recorder = null;
1071
1869
  this.currentTurnStartMs = null;
1072
1870
  this.turnAudioStartCaptured = false;
1073
1871
  if (!recorder) {
1074
- return;
1872
+ return null;
1075
1873
  }
1076
1874
  try {
1077
1875
  // Capture the recorder MIME (now 'audio/wav') BEFORE Stop() — the getter reads '' once stopped.
@@ -1079,18 +1877,39 @@ export class RealtimeSessionRuntime {
1079
1877
  const audioBase64 = await recorder.StopAndEncode();
1080
1878
  // Read the real waveform peaks computed during capture (survives the stop via the snapshot).
1081
1879
  const peaks = recorder.GetPeaks();
1082
- if (!audioBase64 || !agentSessionId) {
1083
- console.warn('[RealtimeSession] ⚠️ recording NOT uploaded — empty recording or no session id.');
1084
- return;
1880
+ if (!audioBase64) {
1881
+ console.warn('[RealtimeSession] ⚠️ recording NOT uploaded — empty recording.');
1882
+ return null;
1085
1883
  }
1086
- await this.uploadRecording(agentSessionId, audioBase64, mimeType, peaks);
1884
+ return { MimeType: mimeType, AudioBase64: audioBase64, Peaks: peaks };
1885
+ }
1886
+ catch (error) {
1887
+ console.warn('[RealtimeSession] Failed to stop call recording:', error);
1888
+ return null;
1889
+ }
1890
+ }
1891
+ /**
1892
+ * Uploads a recording returned by {@link stopRecording}. Never throws. No-op when there is
1893
+ * nothing to upload; warns when there is no session id to attach the file to.
1894
+ */
1895
+ async uploadStoppedRecording(agentSessionId, recording) {
1896
+ if (!recording) {
1897
+ return;
1898
+ }
1899
+ if (!agentSessionId) {
1900
+ console.warn('[RealtimeSession] ⚠️ recording NOT uploaded — no session id.');
1901
+ return;
1902
+ }
1903
+ try {
1904
+ await this.uploadRecording(agentSessionId, recording.AudioBase64, recording.MimeType, recording.Peaks);
1087
1905
  }
1088
1906
  catch (error) {
1089
- console.warn('[RealtimeSession] Failed to stop/upload call recording:', error);
1907
+ console.warn(`[RealtimeSession] Failed to upload call recording for session ${agentSessionId}:`, error);
1090
1908
  }
1091
1909
  }
1092
1910
  /**
1093
- * Runs the `UploadRealtimeRecording` mutation; failures are logged, never thrown. Sends the
1911
+ * Runs the `UploadRealtimeRecording` mutation. A `Success:false` payload is logged; a transport
1912
+ * rejection propagates to `uploadStoppedRecording`, which catches and logs it. Sends the
1094
1913
  * capture-time waveform `peaks` (max-abs per bucket, normalized 0..1) so the server can persist a
1095
1914
  * `peaks.json` sidecar for fast waveform rendering without re-decoding the audio.
1096
1915
  */
@@ -1116,46 +1935,377 @@ export class RealtimeSessionRuntime {
1116
1935
  console.warn(`[RealtimeSession] ❌ recording upload reported failure: ${payload?.ErrorMessage ?? 'unknown error'} (full result: ${JSON.stringify(result)})`);
1117
1936
  }
1118
1937
  }
1119
- // ── Interactive channels (registry-driven plugins) ─────────────────────────
1120
1938
  /**
1121
- * Resolves, instantiates and initializes the session's interactive-channel plugins from
1122
- * the `MJ: AI Agent Channels` registry, publishes them on {@link ActiveChannels$}, and
1123
- * returns their aggregated client-executed tool declarations for the session mint.
1124
- * Tolerant by design: registry/resolution failures degrade to "no channels" — the voice
1125
- * session itself always proceeds.
1939
+ * The resolved behavior of an in-session channel (display, exposure ceiling, config, what put it
1940
+ * in the session), or `null` when the channel is not in this session. Hosts and overlays read the
1941
+ * resolved display from here rather than from the channel's code default.
1942
+ *
1943
+ * @param channelName The channel's key (case-insensitive).
1944
+ */
1945
+ GetResolvedChannel(channelName) {
1946
+ return this.resolvedChannels.get(NormalizeChannelKey(channelName)) ?? null;
1947
+ }
1948
+ /**
1949
+ * Replaces the store the user's per-channel exposure choices are kept in. The default persists them in
1950
+ * the signed-in user's settings (`mj.realtime.visualPerception.v1`) and keeps them in memory for an
1951
+ * anonymous principal or a connect-only embed; a host that remembers them somewhere else supplies its own.
1952
+ */
1953
+ SetExposurePreferences(preferences) {
1954
+ this.exposurePreferences = preferences;
1955
+ }
1956
+ /** The store the user's per-channel exposure choices are currently kept in (so a host can wrap it). */
1957
+ get ExposurePreferences() {
1958
+ return this.exposurePreferences;
1959
+ }
1960
+ /** Whether user settings can be written: a signed-in user on a provider that has entity metadata. */
1961
+ canPersistUserSettings() {
1962
+ const provider = this.Provider;
1963
+ return Boolean(provider?.CurrentUser?.ID) && (provider?.Entities?.length ?? 0) > 0;
1964
+ }
1965
+ /**
1966
+ * How much of a channel the model may perceive right now: the channel's ceiling, lowered by the server's
1967
+ * policy and the user's choice. `null` when the channel is not in this session.
1968
+ *
1969
+ * @param channelName The channel's key (case-insensitive).
1970
+ */
1971
+ GetChannelExposure(channelName) {
1972
+ return this.findDispatchableChannel(channelName)?.Plugin.Exposure ?? null;
1973
+ }
1974
+ /**
1975
+ * The user's own choice of how much of a channel the agent may perceive. Takes effect immediately
1976
+ * (frames and notes the new level forbids stop, and the model is told), and is remembered per channel key
1977
+ * for next time. The level can only LOWER what the server's policy allows; choosing `'pixels'` for a channel
1978
+ * the agent's policy capped at `'state'` changes nothing.
1979
+ *
1980
+ * @param channelName The channel's key (case-insensitive).
1981
+ * @param level The user's choice; `undefined` clears it (back to what policy allows).
1982
+ * @returns `false` when the channel is not in this session (nothing is applied, but the choice is still remembered).
1126
1983
  */
1127
- async startChannels() {
1128
- const channels = await this.loadActiveChannels();
1129
- for (const plugin of channels) {
1130
- this.initializeChannel(plugin);
1984
+ SetUserChannelExposure(channelName, level) {
1985
+ this.exposurePreferences.Set(channelName, level);
1986
+ const channel = this.findDispatchableChannel(channelName);
1987
+ if (!channel) {
1988
+ return false;
1131
1989
  }
1132
- this._activeChannels$.next(channels);
1133
- return channels.flatMap(plugin => plugin.GetToolDefinitions());
1990
+ this.applyChannelExposure(channel.Plugin);
1991
+ return true;
1992
+ }
1993
+ /** Pushes a channel's current exposure inputs (server policy, user choice) into the channel. */
1994
+ applyChannelExposure(plugin) {
1995
+ const resolved = this.GetResolvedChannel(plugin.ChannelName);
1996
+ const user = this.exposurePreferences.Get(plugin.ChannelName);
1997
+ const reasons = (resolved?.ExposureLimits ?? []).map((limit) => limit.Reason);
1998
+ // The user's limit binds only when it is lower than what the server allows; otherwise it changes nothing
1999
+ // and the agent should not be told about it.
2000
+ const allowed = resolved?.Exposure ?? plugin.GetDescriptor().MaxExposure;
2001
+ if (user !== undefined && CompareExposure(user, allowed) < 0) {
2002
+ reasons.push(UserExposureReason(user));
2003
+ }
2004
+ plugin.ApplyExposure({ Policy: resolved?.Exposure, User: user, Reasons: reasons });
2005
+ // A capture this channel fronts follows it: the agent sees its frames only while the channel's exposure allows pixels.
2006
+ const kind = plugin.CaptureKind;
2007
+ if (kind) {
2008
+ this.captures?.SetVisibleToAgent(kind, plugin.Exposure === 'pixels');
2009
+ }
2010
+ this.syncChannelVideoSource(plugin);
1134
2011
  }
1135
2012
  /**
1136
- * Loads the ACTIVE channel definitions from the registry and resolves each row's
1137
- * `ClientPluginClass` through the MJ ClassFactory into a per-session plugin instance —
1138
- * the client-side mirror of how realtime-model drivers resolve from `BaseRealtimeModel`
1139
- * / `BaseRealtimeClient`. Rows whose plugin class isn't registered are skipped (logged),
1140
- * never fatal.
2013
+ * Sends a frame from a channel's context ({@link RealtimeChannelContext.SendVideoFrame}, deprecated) through the
2014
+ * session's {@link VideoSourceArbiter}, as the channel's own source: it reaches the model only while the arbiter picks
2015
+ * that source, and the arbiter tells the model when it switches. The source registers with the first frame, under the
2016
+ * id the channel's own frame bridge uses, switched on only while the channel's exposure allows pixels.
1141
2017
  */
1142
- async loadActiveChannels() {
2018
+ sendChannelVideoFrame(plugin, base64Image, mimeType) {
2019
+ const client = this.client;
2020
+ if (!client || !this.isSessionLive() || !client.IsTrackEstablished('video', 'inbound')) {
2021
+ return;
2022
+ }
2023
+ const arbiter = VideoSourceArbiter.ForSink(client);
2024
+ const descriptor = plugin.GetDescriptor();
2025
+ const sourceId = ChannelVideoSourceID(descriptor.Key, plugin.InstanceId);
2026
+ if (!arbiter.HasSource(sourceId)) {
2027
+ arbiter.RegisterSource({
2028
+ SourceID: sourceId,
2029
+ Label: descriptor.DisplayName,
2030
+ Kind: 'surface',
2031
+ ChannelKey: descriptor.Key,
2032
+ Enabled: plugin.Exposure === 'pixels',
2033
+ });
2034
+ }
2035
+ arbiter.PushFrame(sourceId, base64Image, mimeType);
2036
+ }
2037
+ /**
2038
+ * Keeps a channel's video source at the arbiter, when it has one, in step with the channel's exposure: on while the
2039
+ * agent may see its pixels, off otherwise. Quiet, because the channel tells the model about its own exposure change.
2040
+ */
2041
+ syncChannelVideoSource(plugin) {
2042
+ if (!this.client) {
2043
+ return;
2044
+ }
2045
+ const arbiter = VideoSourceArbiter.ForSink(this.client);
2046
+ const sourceId = ChannelVideoSourceID(plugin.GetDescriptor().Key, plugin.InstanceId);
2047
+ if (arbiter.HasSource(sourceId)) {
2048
+ arbiter.SetSourceEnabled(sourceId, plugin.Exposure === 'pixels', false);
2049
+ }
2050
+ }
2051
+ /** Follows the session client's video-source arbiter so {@link VideoSources$} reflects it. */
2052
+ watchVideoSources(client) {
2053
+ this.unwatchVideoSources();
2054
+ const arbiter = VideoSourceArbiter.ForSink(client);
2055
+ arbiter.SetFocusedChannel(this.focusedChannelKey);
2056
+ this.stopWatchingVideoSources = arbiter.OnChange(() => this._videoSources$.next(arbiter.GetSources()));
2057
+ this._videoSources$.next(arbiter.GetSources());
2058
+ }
2059
+ /**
2060
+ * Tells the session which channel's surface the user is looking at (`null` for none, e.g. the activity tab).
2061
+ * When the model can see only one video source and several are live, the one the user is looking at is the one
2062
+ * it sees (after an explicit pick, and after a camera or screen share the user started). Safe to call before the
2063
+ * session is live: it applies when the connection comes up.
2064
+ *
2065
+ * @param channelKey The focused channel's key, as it appears on {@link VideoSources$} entries' `ChannelKey`.
2066
+ */
2067
+ SetFocusedChannel(channelKey) {
2068
+ this.focusedChannelKey = channelKey;
2069
+ if (this.client) {
2070
+ VideoSourceArbiter.ForSink(this.client).SetFocusedChannel(channelKey);
2071
+ }
2072
+ }
2073
+ /** Stops following the arbiter and clears {@link VideoSources$}. Safe to call when nothing is watched. */
2074
+ unwatchVideoSources() {
2075
+ this.stopWatchingVideoSources?.();
2076
+ this.stopWatchingVideoSources = null;
2077
+ if (this._videoSources$.value.length > 0) {
2078
+ this._videoSources$.next([]);
2079
+ }
2080
+ }
2081
+ /**
2082
+ * Turns the agent's view of one video source on or off — what the "agent can see" control calls.
2083
+ *
2084
+ * A source that belongs to a channel goes through {@link SetUserChannelExposure}, so the choice is
2085
+ * remembered per channel and the channel itself tells the model. Any other source (a camera or screen
2086
+ * share that is not a channel) is switched at the arbiter, which tells the model.
2087
+ *
2088
+ * @param sourceId The source's id (from {@link VideoSources$}).
2089
+ * @param enabled Whether the agent may see it.
2090
+ * @returns `false` when there is no such source.
2091
+ */
2092
+ SetVideoSourceEnabled(sourceId, enabled) {
2093
+ const client = this.client;
2094
+ if (!client) {
2095
+ return false;
2096
+ }
2097
+ const arbiter = VideoSourceArbiter.ForSink(client);
2098
+ const source = arbiter.GetSources().find((s) => s.SourceID === sourceId);
2099
+ if (!source) {
2100
+ return false;
2101
+ }
2102
+ if (source.ChannelKey && this.findDispatchableChannel(source.ChannelKey)) {
2103
+ return this.SetUserChannelExposure(source.ChannelKey, enabled ? undefined : 'state');
2104
+ }
2105
+ return arbiter.SetSourceEnabled(sourceId, enabled);
2106
+ }
2107
+ /**
2108
+ * Picks the video source the agent sees: what the "agent can see" control calls when more sources are on than the
2109
+ * model takes. The pick beats every other rule (a camera or screen share the user started, the surface the user is
2110
+ * looking at) until it is cleared, or its source is turned off or leaves. The pick shows on {@link VideoSources$}
2111
+ * (`Picked`), and the model is told when what it sees changes.
2112
+ *
2113
+ * @param sourceId The source's id (from {@link VideoSources$}), or `null` to let the call decide again.
2114
+ * @returns `false` with no live session, or for a source that is not there or is turned off.
2115
+ */
2116
+ SelectVideoSource(sourceId) {
2117
+ return this.client ? VideoSourceArbiter.ForSink(this.client).SelectSource(sourceId) : false;
2118
+ }
2119
+ /**
2120
+ * The `on-demand` channels that are in the session but not open yet — what the agent can open
2121
+ * through `ContextTool`. A fresh array; empty before a session starts and after teardown.
2122
+ */
2123
+ get AdvertisedChannels() {
2124
+ return this.advertisedChannels.map((p) => p.Plugin);
2125
+ }
2126
+ /**
2127
+ * Resolves, initializes and mounts the session's channels in one step using the scope the browser
2128
+ * can decide alone (code defaults + host declarations), and returns the native tools to declare at
2129
+ * mint. This is the single-step composition of {@link prepareChannelScope} + {@link activateChannels}
2130
+ * that a session start spreads across the mint; it is kept as a unit because it is the cleanest seam
2131
+ * for exercising the plugin plumbing without a mint.
2132
+ */
2133
+ async startChannels(hostChannels) {
2134
+ const scope = await this.prepareChannelScope(hostChannels);
2135
+ this.activateChannels(this.localChannelScope?.Channels ?? []);
2136
+ return scope.NativeTools;
2137
+ }
2138
+ /**
2139
+ * Stage 1: constructs the channel plugins (registry rows and host declarations), reads their
2140
+ * descriptors, resolves the scope the browser can decide alone, and builds what the mint needs —
2141
+ * the native tools to declare and the candidates to report. Starts nothing.
2142
+ *
2143
+ * @param hostChannels Channels the host brings to this session.
2144
+ * @returns The native tools to declare at mint, the candidates as JSON (`null` when there are none,
2145
+ * so a channel-less session sends exactly the mint it always did), and whether the host could show the agent's
2146
+ * video ({@link PreparedChannelsShowAgentVideo}).
2147
+ */
2148
+ async prepareChannelScope(hostChannels) {
2149
+ this.discardUnmountedChannels();
2150
+ const prepared = await this.prepareChannels(hostChannels);
2151
+ this.preparedChannels = prepared;
2152
+ const candidates = prepared.map((p) => BuildChannelCandidate(p, p.Plugin.GetDescriptor()));
2153
+ const local = ResolveLocalChannelScope(candidates);
2154
+ this.localChannelScope = local;
2155
+ for (const excluded of local.Excluded) {
2156
+ console.warn(`[RealtimeSession] Channel '${excluded.Key}' is not in this session (${excluded.Reason}).`);
2157
+ }
2158
+ const nativeTools = SelectNativeChannelTools(local.Channels, ToolsByChannelKey(candidates));
2159
+ // `Registry` is the browser's own view and never goes over the wire: the server reads the registry itself.
2160
+ const wire = candidates.map(({ Registry: _registry, ...candidate }) => candidate);
2161
+ return { NativeTools: nativeTools, CandidatesJson: wire.length > 0 ? JSON.stringify(wire) : null, ShowsAgentVideo: PreparedChannelsShowAgentVideo(prepared) };
2162
+ }
2163
+ /**
2164
+ * Constructs a plugin for every active-or-inactive registry row and every host declaration. An
2165
+ * INACTIVE row is constructed too — only to learn its key so the kill switch can name it — and is
2166
+ * never initialized. A plugin whose descriptor cannot be read is skipped (logged), never fatal.
2167
+ */
2168
+ async prepareChannels(hostChannels) {
1143
2169
  const rows = await this.fetchChannelDefinitions();
1144
- const channels = [];
2170
+ await this.loadChannelClasses([...rows.map((r) => r.ClientPluginClass), ...(hostChannels ?? []).map((d) => d.ClientPluginClass)]);
2171
+ const prepared = [];
1145
2172
  for (const row of rows) {
1146
2173
  const plugin = this.resolveChannelPlugin(row);
1147
2174
  if (plugin) {
1148
- channels.push(plugin);
2175
+ plugin.ApplySurfacePlacement(row.SurfacePlacement);
2176
+ prepared.push({ Plugin: plugin, Key: plugin.ChannelName, Registry: row.IsActive ? 'active' : 'inactive' });
2177
+ }
2178
+ }
2179
+ for (const declaration of hostChannels ?? []) {
2180
+ const plugin = this.createHostChannelPlugin(declaration);
2181
+ if (!plugin) {
2182
+ continue;
2183
+ }
2184
+ const existing = FindPreparedChannel(prepared, plugin.ChannelName);
2185
+ if (existing) {
2186
+ // The host's instance replaces the registry's (it may carry the host's collaborators); the
2187
+ // registry row's state — notably the kill switch, and where its surface shows — still applies.
2188
+ plugin.ApplySurfacePlacement(existing.Plugin.SurfacePlacement);
2189
+ existing.Plugin = plugin;
2190
+ existing.HostDeclaration = declaration;
2191
+ }
2192
+ else {
2193
+ prepared.push({ Plugin: plugin, Key: plugin.ChannelName, Registry: 'none', HostDeclaration: declaration });
2194
+ }
2195
+ }
2196
+ return prepared.filter((p) => this.canDescribe(p));
2197
+ }
2198
+ /**
2199
+ * Gives a host that loads channel code on demand (a widget that keeps a heavy channel in its own download)
2200
+ * the chance to load it before the plugins are built. Never fatal: a loader that throws is logged and the
2201
+ * channel simply is not available, exactly as if its class had never been registered.
2202
+ *
2203
+ * @param keys The `ClientPluginClass` keys of every channel about to be resolved (blank/absent ones included).
2204
+ */
2205
+ async loadChannelClasses(keys) {
2206
+ const loader = this.ChannelClassLoader;
2207
+ if (!loader) {
2208
+ return;
2209
+ }
2210
+ const wanted = [...new Set(keys.map((k) => k?.trim() ?? '').filter((k) => k.length > 0))];
2211
+ if (wanted.length === 0) {
2212
+ return;
2213
+ }
2214
+ try {
2215
+ await loader(wanted);
2216
+ }
2217
+ catch (error) {
2218
+ console.error('[RealtimeSession] The channel class loader failed — channels that needed it are left out:', error);
2219
+ }
2220
+ }
2221
+ /** Whether a prepared plugin's descriptor can be read; logs and rejects one that throws. */
2222
+ canDescribe(prepared) {
2223
+ try {
2224
+ prepared.Plugin.GetDescriptor();
2225
+ return true;
2226
+ }
2227
+ catch (error) {
2228
+ console.error(`[RealtimeSession] Channel '${prepared.Key}' has an unreadable descriptor — leaving it out:`, error);
2229
+ return false;
2230
+ }
2231
+ }
2232
+ /** Builds a plugin from a host declaration; `null` (logged) when it names nothing buildable. */
2233
+ createHostChannelPlugin(declaration) {
2234
+ if (declaration.Create) {
2235
+ try {
2236
+ return declaration.Create();
2237
+ }
2238
+ catch (error) {
2239
+ console.error('[RealtimeSession] A host-declared channel factory threw — leaving the channel out:', error);
2240
+ return null;
2241
+ }
2242
+ }
2243
+ const key = declaration.ClientPluginClass?.trim();
2244
+ if (!key) {
2245
+ console.warn('[RealtimeSession] A host-declared channel names neither ClientPluginClass nor Create — ignoring it.');
2246
+ return null;
2247
+ }
2248
+ return this.resolveChannelPlugin({ Name: key, ClientPluginClass: key, IsActive: true });
2249
+ }
2250
+ /**
2251
+ * Stage 3: initializes the channels the resolved policy puts in the session. `open-on-start` and
2252
+ * `headless` channels are mounted and published on {@link ActiveChannels$}; `on-demand` channels
2253
+ * are held until opened. Prepared plugins the policy left out are dropped without ever having run.
2254
+ */
2255
+ activateChannels(channels) {
2256
+ const reconciled = ReconcileChannelsWithPolicy(this.preparedChannels, channels);
2257
+ for (const key of reconciled.Unknown) {
2258
+ console.warn(`[RealtimeSession] The session policy names channel '${key}' but this host has no plugin for it — ignoring it.`);
2259
+ }
2260
+ this.resolvedChannels.clear();
2261
+ this.advertisedChannels = [];
2262
+ const mounted = [];
2263
+ for (const { Prepared, Resolved } of reconciled.InSession) {
2264
+ this.resolvedChannels.set(NormalizeChannelKey(Resolved.Key), Resolved);
2265
+ // Exposure is applied BEFORE the channel initializes, so what it requests of the model (a video
2266
+ // track, in particular) already reflects what policy allows.
2267
+ this.applyChannelExposure(Prepared.Plugin);
2268
+ if (Resolved.DisplayPolicy === 'on-demand') {
2269
+ this.advertisedChannels.push(Prepared);
2270
+ }
2271
+ else {
2272
+ this.mountChannel(Prepared, Resolved);
2273
+ mounted.push(Prepared.Plugin);
1149
2274
  }
1150
2275
  }
1151
- return channels;
2276
+ this.preparedChannels = [];
2277
+ this._activeChannels$.next(mounted);
1152
2278
  }
1153
2279
  /**
1154
- * Reads the ACTIVE `MJ: AI Agent Channels` rows. With entity metadata: from {@link AIEngineBase}'s
1155
- * cached `AgentChannels` (provider-scoped engine instance, lazy `Config` — no RunView round-trip;
1156
- * the engine's BaseEntity-event reactivity keeps the registry fresh). On a connect-only provider:
1157
- * one `RunDynamicView` query ({@link fetchChannelDefinitionsOverGraphQL}). Failures are logged and
1158
- * degrade to an empty list — channel availability must never block the voice session.
2280
+ * Applies the mint's outcome to the channel set: the server's policy when it sent a usable one,
2281
+ * else the scope the browser resolved on its own. A policy can only SELECT among the plugins this
2282
+ * host prepared — it cannot add one.
2283
+ */
2284
+ applySessionClientPolicy(session) {
2285
+ const policy = ParseRealtimeSessionClientPolicy(session.ClientPolicyJson);
2286
+ if (session.ClientPolicyJson && !policy) {
2287
+ console.warn('[RealtimeSession] The mint returned a client policy this runtime cannot read — resolving channels locally.');
2288
+ }
2289
+ this.sessionClientTools = policy?.ClientTools ?? null;
2290
+ if (this.preparedChannels.length === 0) {
2291
+ this.resolvedChannels.clear();
2292
+ return; // nothing was prepared (a host that mints and runs the session itself) — no channel set to apply
2293
+ }
2294
+ this.activateChannels(policy?.Channels ?? this.localChannelScope?.Channels ?? []);
2295
+ }
2296
+ /** Drops plugins that were prepared for a start that never mounted them. They were never initialized, so there is nothing to release. */
2297
+ discardUnmountedChannels() {
2298
+ this.preparedChannels = [];
2299
+ this.advertisedChannels = [];
2300
+ this.localChannelScope = null;
2301
+ }
2302
+ /**
2303
+ * Loads the `MJ: AI Agent Channels` registry rows (ACTIVE AND INACTIVE — an inactive row is the master
2304
+ * kill switch, so the scope decision has to see it). With entity metadata: from {@link AIEngineBase}'s
2305
+ * cached `AgentChannels` (provider-scoped engine instance, lazy `Config` — no RunView round-trip; the
2306
+ * engine's BaseEntity-event reactivity keeps the registry fresh). On a connect-only provider: one
2307
+ * `RunDynamicView` query ({@link fetchChannelDefinitionsOverGraphQL}). Failures are logged and degrade to an
2308
+ * empty list — channel availability must never block the voice session.
1159
2309
  */
1160
2310
  async fetchChannelDefinitions() {
1161
2311
  // A connect-only provider (ConnectGraphQLClient — anonymous embeds) has no entity metadata,
@@ -1168,9 +2318,13 @@ export class RealtimeSessionRuntime {
1168
2318
  try {
1169
2319
  const engine = AIEngineBase.GetProviderInstance(this.Provider, AIEngineBase);
1170
2320
  await engine.Config(false, undefined, this.Provider);
1171
- return (engine.AgentChannels ?? [])
1172
- .filter(c => c.IsActive)
1173
- .map(c => ({ ID: c.ID, Name: c.Name, ClientPluginClass: c.ClientPluginClass }));
2321
+ return (engine.AgentChannels ?? []).map((c) => ({
2322
+ ID: c.ID,
2323
+ Name: c.Name,
2324
+ ClientPluginClass: c.ClientPluginClass,
2325
+ IsActive: c.IsActive,
2326
+ SurfacePlacement: readRowSurfacePlacement(c),
2327
+ }));
1174
2328
  }
1175
2329
  catch (error) {
1176
2330
  console.warn('[RealtimeSession] Channel registry unavailable — starting with no channels:', error);
@@ -1178,10 +2332,10 @@ export class RealtimeSessionRuntime {
1178
2332
  }
1179
2333
  }
1180
2334
  /**
1181
- * The connect-only path of {@link fetchChannelDefinitions}: the same ACTIVE `MJ: AI Agent Channels`
1182
- * rows, read with a dynamic view because a connect-only client has no entity metadata to build a
1183
- * typed RunView from. Same tolerance as the engine path — a failure is logged and means "no
1184
- * channels", never a blocked session.
2335
+ * The connect-only path of {@link fetchChannelDefinitions}: the same `MJ: AI Agent Channels` rows, active
2336
+ * and inactive (an inactive row is the kill switch), read with a dynamic view because a connect-only client
2337
+ * has no entity metadata to build a typed RunView from. Same tolerance as the engine path — a failure is
2338
+ * logged and means "no channels", never a blocked session.
1185
2339
  */
1186
2340
  async fetchChannelDefinitionsOverGraphQL() {
1187
2341
  const query = `query RealtimeChannelRegistry($input: RunDynamicViewInput!) {
@@ -1189,7 +2343,7 @@ export class RealtimeSessionRuntime {
1189
2343
  }`;
1190
2344
  try {
1191
2345
  const result = (await this.gql().ExecuteGQL(query, {
1192
- input: { EntityName: 'MJ: AI Agent Channels', ExtraFilter: 'IsActive = 1', Fields: ['ID', 'Name', 'ClientPluginClass', 'IsActive'] },
2346
+ input: { EntityName: 'MJ: AI Agent Channels', Fields: ['ID', 'Name', 'ClientPluginClass', 'IsActive', 'UIConfig'] },
1193
2347
  }));
1194
2348
  const view = result?.RunDynamicView;
1195
2349
  if (!view?.Success) {
@@ -1198,8 +2352,13 @@ export class RealtimeSessionRuntime {
1198
2352
  }
1199
2353
  return (view.Results ?? [])
1200
2354
  .map((r) => JSON.parse(r.Data))
1201
- .filter((row) => row.IsActive === true)
1202
- .map((row) => ({ ID: row.ID, Name: row.Name, ClientPluginClass: row.ClientPluginClass }));
2355
+ .map((row) => ({
2356
+ ID: row.ID,
2357
+ Name: row.Name,
2358
+ ClientPluginClass: row.ClientPluginClass,
2359
+ IsActive: row.IsActive === true,
2360
+ SurfacePlacement: readUIConfigTextSurfacePlacement(row.Name, row.UIConfig),
2361
+ }));
1203
2362
  }
1204
2363
  catch (error) {
1205
2364
  console.warn('[RealtimeSession] Channel registry unavailable — starting with no channels:', error instanceof Error ? error.message : String(error));
@@ -1211,16 +2370,23 @@ export class RealtimeSessionRuntime {
1211
2370
  * checked first, exactly like the realtime-client drivers) and instantiates a fresh
1212
2371
  * per-session plugin. Returns `null` (logged) when no plugin is registered for the key
1213
2372
  * — e.g. its Load function was never called or the package isn't included client-side.
2373
+ * An inactive row is resolved quietly: it only needs to be named, and a missing plugin for a
2374
+ * channel nobody wants is not worth a warning.
1214
2375
  */
1215
2376
  resolveChannelPlugin(row) {
1216
2377
  const key = row.ClientPluginClass?.trim();
2378
+ const quiet = !row.IsActive;
1217
2379
  if (!key) {
1218
- console.warn(`[RealtimeSession] Channel '${row.Name}' has no ClientPluginClass — skipping.`);
2380
+ if (!quiet) {
2381
+ console.warn(`[RealtimeSession] Channel '${row.Name}' has no ClientPluginClass — skipping.`);
2382
+ }
1219
2383
  return null;
1220
2384
  }
1221
2385
  const registration = MJGlobal.Instance.ClassFactory.GetRegistration(BaseRealtimeChannelClient, key);
1222
2386
  if (!registration) {
1223
- console.warn(`[RealtimeSession] No client plugin registered for channel '${row.Name}' (key '${key}') — skipping.`);
2387
+ if (!quiet) {
2388
+ console.warn(`[RealtimeSession] No client plugin registered for channel '${row.Name}' (key '${key}') — skipping.`);
2389
+ }
1224
2390
  return null;
1225
2391
  }
1226
2392
  const plugin = MJGlobal.Instance.ClassFactory.CreateInstance(BaseRealtimeChannelClient, key);
@@ -1231,24 +2397,157 @@ export class RealtimeSessionRuntime {
1231
2397
  return plugin;
1232
2398
  }
1233
2399
  /**
1234
- * Wires one plugin into the session: hands it its host context and registers its
1235
- * prefix-routed local tool executor (so `<ToolNamePrefix>*` calls run in the browser
1236
- * through {@link BaseRealtimeChannelClient.ApplyAgentTool}, never the server relay).
2400
+ * Wires one plugin into the session: hands it its host context (carrying its resolved config) and,
2401
+ * when it has native tools, registers its prefix-routed local tool executor (so `<ToolNamePrefix>*`
2402
+ * calls run in the browser through {@link BaseRealtimeChannelClient.ApplyAgentTool}, never the
2403
+ * server relay). A channel with no tool prefix has no native route — it is reached through
2404
+ * `ContextTool` — and registering an empty prefix would match EVERY tool name.
1237
2405
  */
1238
- initializeChannel(plugin) {
1239
- plugin.Initialize(this.buildChannelContext(plugin));
2406
+ mountChannel(prepared, resolved) {
2407
+ const plugin = prepared.Plugin;
2408
+ plugin.Initialize(this.buildChannelContext(plugin, resolved?.Config));
2409
+ if (plugin.ToolNamePrefix.length === 0) {
2410
+ return;
2411
+ }
1240
2412
  this.RegisterClientToolHandler(plugin.ToolNamePrefix, (toolName, argsJson) => {
1241
2413
  // The agent is ACTING on this channel — surface-discovery signal for the overlay
1242
2414
  // (first activity registers + auto-reveals + focuses the channel tab) before the
1243
2415
  // tool applies. Record the channel as USED so the overlay tabs it (channels other
1244
2416
  // than the whiteboard are tab-less until they're first used).
1245
- this.usedChannelNames.add(plugin.ChannelName);
1246
- this._channelActivity$.next(plugin);
2417
+ this.noteChannelActivity(plugin);
2418
+ // A native tool whose result would show the agent what exposure policy withholds is refused here, as it is on the
2419
+ // ContextTool route; the legacy tool-result shape carries the reason.
2420
+ const withheld = plugin.FindVerbForNativeTool(toolName);
2421
+ const refusal = withheld ? plugin.RefuseVerbForExposure(withheld) : null;
2422
+ if (refusal) {
2423
+ return JSON.stringify({ success: false, errorCode: 'exposure_restricted', error: refusal });
2424
+ }
1247
2425
  return plugin.ApplyAgentTool(toolName, argsJson);
1248
2426
  });
1249
2427
  }
2428
+ /** Records that the agent acted on a channel (the overlay's first-use reveal signal). */
2429
+ noteChannelActivity(plugin) {
2430
+ this.usedChannelNames.add(plugin.ChannelName);
2431
+ this._channelActivity$.next(plugin);
2432
+ }
2433
+ /**
2434
+ * Mounts an advertised `on-demand` channel when the agent opens it: initializes it, restores its
2435
+ * prior-session state, publishes it on {@link ActiveChannels$} (so the overlay can give it a tab)
2436
+ * and, if the call is already connected, tells it so. If initialization throws the channel goes
2437
+ * back to being advertised, so the agent can retry or move on.
2438
+ */
2439
+ async mountAdvertisedChannel(plugin) {
2440
+ const index = this.advertisedChannels.findIndex((p) => p.Plugin === plugin);
2441
+ if (index < 0) {
2442
+ return; // already mounted (or never advertised) — the dispatcher opens it regardless
2443
+ }
2444
+ const [prepared] = this.advertisedChannels.splice(index, 1);
2445
+ try {
2446
+ this.mountChannel(prepared, this.GetResolvedChannel(prepared.Key) ?? undefined);
2447
+ }
2448
+ catch (error) {
2449
+ this.advertisedChannels.splice(index, 0, prepared);
2450
+ throw error;
2451
+ }
2452
+ this.restorePriorChannelState(plugin);
2453
+ this._activeChannels$.next([...this._activeChannels$.value, plugin]);
2454
+ if (this.isSessionLive()) {
2455
+ this.notifyChannelSessionStarted(plugin);
2456
+ }
2457
+ }
2458
+ /** Tells a channel the session is connected, containing anything it throws. */
2459
+ notifyChannelSessionStarted(channel) {
2460
+ try {
2461
+ channel.OnSessionStarted?.();
2462
+ }
2463
+ catch (err) {
2464
+ console.error(`[RealtimeSession] Error in channel '${channel.ChannelName}' OnSessionStarted:`, err);
2465
+ }
2466
+ }
2467
+ /** Finds a session channel (open or merely advertised) for the dispatcher. */
2468
+ findDispatchableChannel(key) {
2469
+ const id = NormalizeChannelKey(key);
2470
+ const open = this._activeChannels$.value.find((c) => NormalizeChannelKey(c.ChannelName) === id);
2471
+ if (open) {
2472
+ return { Plugin: open, IsOpen: true };
2473
+ }
2474
+ const advertised = this.advertisedChannels.find((p) => NormalizeChannelKey(p.Key) === id);
2475
+ return advertised ? { Plugin: advertised.Plugin, IsOpen: false } : null;
2476
+ }
2477
+ /** The keys the agent can address, for "unknown channel" messages — channels with nothing to address are left out. */
2478
+ listAddressableChannelKeys() {
2479
+ const addressable = (plugin) => {
2480
+ const descriptor = plugin.GetDescriptor();
2481
+ return descriptor.Verbs.length > 0 || descriptor.Inputs !== undefined;
2482
+ };
2483
+ return [
2484
+ ...this._activeChannels$.value.filter(addressable).map((c) => c.ChannelName),
2485
+ ...this.advertisedChannels.filter((p) => addressable(p.Plugin)).map((p) => p.Key),
2486
+ ];
2487
+ }
2488
+ /**
2489
+ * Runs a channel-addressed `ContextTool` call (the {@link RealtimeChannelContext.DispatchContextAction}
2490
+ * implementation): validates it, opens an `on-demand` channel when asked, runs the verb, and on
2491
+ * success records the channel as used so the overlay reveals its tab. Never throws.
2492
+ */
2493
+ async dispatchContextAction(request) {
2494
+ const result = await this.channelDispatcher.Dispatch(request);
2495
+ if (result.Success) {
2496
+ const channel = this.findDispatchableChannel(request.Target.Channel);
2497
+ if (channel) {
2498
+ this.noteChannelActivity(channel.Plugin);
2499
+ }
2500
+ }
2501
+ return result;
2502
+ }
2503
+ /**
2504
+ * Opens (and seeds) a channel from the HOST — the same path the agent's `open` action takes: the channel
2505
+ * is mounted if it was only advertised, its `Inputs` schema validates `inputs`, and it announces itself to
2506
+ * the model. A channel that is already open is re-seeded. Never throws; a refusal is a structured result.
2507
+ *
2508
+ * @param channelKey The channel to open (case-insensitive).
2509
+ * @param inputs Seed inputs, validated against the channel's descriptor.
2510
+ */
2511
+ OpenChannel(channelKey, inputs = {}) {
2512
+ return this.dispatchContextAction({ Target: { Channel: channelKey }, Action: 'open', Params: inputs });
2513
+ }
2514
+ /**
2515
+ * Tells the model which channels exist and how to use them — rendered from the channels' own
2516
+ * descriptors, so a new channel needs no prompt change. Sent ONCE, the first moment the control
2517
+ * channel is usable — a context note sent before then is dropped, and `Connect` can resolve before the
2518
+ * provider reports `listening`, so this runs at start and again on each state change until it has gone.
2519
+ * A session whose only channels describe themselves through native tools sends nothing.
2520
+ */
2521
+ flushChannelCatalogNote() {
2522
+ if (!this.catalogNotePending || !this.isSessionLive()) {
2523
+ return;
2524
+ }
2525
+ this.catalogNotePending = false;
2526
+ const entries = [
2527
+ ...this._activeChannels$.value.map((plugin) => this.catalogEntry(plugin, true)),
2528
+ ...this.advertisedChannels.map((p) => this.catalogEntry(p.Plugin, false)),
2529
+ ];
2530
+ const note = BuildChannelCatalogNote(entries);
2531
+ if (note) {
2532
+ this.SendContextNote(note);
2533
+ }
2534
+ }
2535
+ /** One catalog entry: a channel's descriptor, whether it is open, and whether its tools were declared natively. */
2536
+ catalogEntry(plugin, isOpen) {
2537
+ const display = this.GetResolvedChannel(plugin.ChannelName)?.DisplayPolicy;
2538
+ const descriptor = plugin.GetDescriptor();
2539
+ const entry = {
2540
+ Descriptor: descriptor,
2541
+ IsOpen: isOpen,
2542
+ HasNativeTools: isOpen && display !== undefined && DeclaresNativeTools(display) && plugin.GetToolDefinitions().length > 0,
2543
+ };
2544
+ if (CompareExposure(plugin.Exposure, descriptor.MaxExposure) < 0) {
2545
+ entry.ExposureLimit = { Effective: plugin.Exposure, Ceiling: descriptor.MaxExposure, Reasons: [...plugin.ExposureReasons] };
2546
+ }
2547
+ return entry;
2548
+ }
1250
2549
  /** Builds the host-services context one channel plugin sees (its only line to the session). */
1251
- buildChannelContext(plugin) {
2550
+ buildChannelContext(plugin, channelConfig) {
1252
2551
  // Capture the service in a local so the AgentSessionID getter reads the SERVICE's live
1253
2552
  // field (not the object literal's `this`) every time it's accessed.
1254
2553
  const service = this;
@@ -1275,55 +2574,107 @@ export class RealtimeSessionRuntime {
1275
2574
  // (Explorer) feeds both; absent on hosts that supply no app context / register no client tools.
1276
2575
  AppContext$: this.AppContext$,
1277
2576
  ExecuteClientTool: (name, params) => this.executeAppClientTool(name, params),
2577
+ // The session's server events (identity verification, app events), for a channel that reacts to them.
2578
+ SessionEvents$: this.SessionEvents$,
2579
+ // Channel-addressed ContextTool calls: validated against the verb's schema, open `on-demand` channels.
2580
+ DispatchContextAction: (request) => this.dispatchContextAction(request),
2581
+ // This channel's resolved configuration (host defaults beneath agent/app config).
2582
+ ChannelConfig: channelConfig ?? {},
1278
2583
  get Client() {
1279
2584
  return service.client;
1280
2585
  },
1281
- SendVideoFrame: (base64Image, mimeType) => this.SendVideoFrame(base64Image, mimeType),
1282
- IsTrackEstablished: (modality, direction) => this.IsTrackEstablished(modality, direction)
2586
+ // Through the arbiter, as this channel's source (deprecated: channels use a ChannelInboundVideoBridge).
2587
+ SendVideoFrame: (base64Image, mimeType) => this.sendChannelVideoFrame(plugin, base64Image, mimeType),
2588
+ IsTrackEstablished: (modality, direction) => this.IsTrackEstablished(modality, direction),
2589
+ // The camera and screen share, for a channel that fronts one: their state, and the user's clicks on its surface.
2590
+ Captures$: this.Captures$,
2591
+ StartCapture: (kind) => (kind === 'camera' ? this.StartCamera() : this.StartScreenShare()),
2592
+ StopCapture: (kind) => (kind === 'camera' ? this.StopCamera() : this.StopScreenShare()),
2593
+ // The video sources and which of them the model is sent, so a capture's channel says the agent sees it only then.
2594
+ VideoSources$: this.VideoSources$,
2595
+ // The agent's video, the call's state and its resumes, for a channel that shows the agent.
2596
+ AgentVideo$: this.AgentVideo$,
2597
+ ConnectionState$: this.ConnectionState$,
2598
+ Resuming$: this.Resuming$
1283
2599
  };
1284
2600
  }
1285
2601
  /**
1286
- * Replaces the set of host-registered surface client tools the realtime ContextTool can execute.
1287
- * The host calls this at session start and whenever the active surface's tool set changes (the
1288
- * continuous-capability half of client-context delivery). Passing `[]` clears them.
2602
+ * Registers the tools ONE source of the host can run for the realtime `ContextTool`, replacing only
2603
+ * that source's previous set — other owners' tools are untouched. A host with two sources (Explorer's
2604
+ * always-available globals and the active surface's tools) registers each under its own owner key, so
2605
+ * refreshing the surface's tools can neither drop the globals nor leave the previous surface's tools
2606
+ * behind. Passing `[]` clears just that owner. When two owners register the same tool name, the
2607
+ * later-registered owner wins.
1289
2608
  *
1290
- * @param tools The current surface client tools (name + handler). Descriptions/schemas ride the
1291
- * app-context manifest separately; only the executable handler is needed here.
1292
- */
1293
- RegisterAppClientTools(tools) {
1294
- this.appClientToolHandlers.clear();
1295
- for (const tool of tools) {
1296
- if (tool?.Name && typeof tool.Handler === 'function') {
1297
- this.appClientToolHandlers.set(tool.Name.trim().toLowerCase(), tool.Handler);
1298
- }
1299
- }
2609
+ * Calling it with one argument (the original form) registers under {@link DEFAULT_APP_TOOL_OWNER}, so
2610
+ * a host with a single source behaves exactly as before.
2611
+ *
2612
+ * @param tools The owner's complete current set (name + handler; description/schema optional).
2613
+ * @param owner A stable key naming the source (e.g. `'explorer.surface'`). Defaults to a shared owner.
2614
+ */
2615
+ RegisterAppClientTools(tools, owner = DEFAULT_APP_TOOL_OWNER) {
2616
+ this.appToolRegistry.Register(owner, tools);
2617
+ }
2618
+ /** Removes one owner's tools (and its place in the collision order). No-op when it has none. */
2619
+ UnregisterAppClientTools(owner) {
2620
+ this.appToolRegistry.Unregister(owner);
2621
+ }
2622
+ /** Removes every owner's tools. */
2623
+ ClearAppClientTools() {
2624
+ this.appToolRegistry.Clear();
2625
+ }
2626
+ /**
2627
+ * The client tools in effect right now, resolved through the same unified resolver the server uses
2628
+ * (`override → session → app → static`, first match wins): the host's registered tools (described
2629
+ * by the manifest the host streams when it did not describe them itself) first, then the app and
2630
+ * agent tiers the server resolved at mint. Only the host-registered tools can RUN here; the
2631
+ * other tiers describe what the app declares.
2632
+ */
2633
+ resolveAppClientTools() {
2634
+ const manifest = this._appContext$.value?.Capabilities?.Tools ?? [];
2635
+ return ResolveClientTools({
2636
+ agentId: '',
2637
+ sessionTools: MergeToolMetadata(this.appToolRegistry.ToMetadata(), manifest),
2638
+ appTools: this.sessionClientTools?.App,
2639
+ staticTools: this.sessionClientTools?.Static,
2640
+ });
1300
2641
  }
1301
2642
  /**
1302
2643
  * Executes a host-registered surface client tool by name (the {@link RealtimeChannelContext.ExecuteClientTool}
1303
2644
  * implementation). Tolerant: an unknown tool or a thrown handler resolves to a structured
1304
- * `Success: false` result the channel narrates — never throws.
2645
+ * `Success: false` result the channel narrates — never throws. A tool the app declares but the host has
2646
+ * not registered a handler for says so, instead of reading like a typo. A handler that returns an object with
2647
+ * `Success: false` also gives `Success: false`, with the handler's `ErrorMessage` and its object as `Result`.
1305
2648
  *
1306
2649
  * @param name The tool name (the model's `action`).
1307
2650
  * @param params The tool parameters.
1308
2651
  * @returns A structured result for the channel to serialize back to the model.
1309
2652
  */
1310
2653
  async executeAppClientTool(name, params) {
1311
- const handler = this.appClientToolHandlers.get((name ?? '').trim().toLowerCase());
2654
+ const handler = this.appToolRegistry.Find(name)?.Handler;
1312
2655
  if (!handler) {
1313
- const available = Array.from(this.appClientToolHandlers.keys()).join(', ');
1314
- return {
1315
- Success: false,
1316
- ErrorMessage: `No client tool named "${name}" is available on this surface. Available: ${available || '(none)'}.`
1317
- };
2656
+ return { Success: false, ErrorMessage: this.describeMissingClientTool(name) };
1318
2657
  }
1319
2658
  try {
1320
2659
  const result = await handler(params ?? {});
1321
- return { Success: true, Result: result };
2660
+ return reportsFailure(result)
2661
+ ? { Success: false, ErrorMessage: failureMessage(result), Result: result }
2662
+ : { Success: true, Result: result };
1322
2663
  }
1323
2664
  catch (error) {
1324
2665
  return { Success: false, ErrorMessage: error instanceof Error ? error.message : String(error) };
1325
2666
  }
1326
2667
  }
2668
+ /** The model-readable reason a client tool could not run. */
2669
+ describeMissingClientTool(name) {
2670
+ const available = this.appToolRegistry.Names().join(', ');
2671
+ const wanted = (name ?? '').trim().toLowerCase();
2672
+ const declared = this.resolveAppClientTools().find((t) => t.Name.trim().toLowerCase() === wanted);
2673
+ if (declared) {
2674
+ return `The client tool "${name}" is declared for this app but this surface has not registered it, so it cannot run right now. Available: ${available || '(none)'}.`;
2675
+ }
2676
+ return `No client tool named "${name}" is available on this surface. Available: ${available || '(none)'}.`;
2677
+ }
1327
2678
  /**
1328
2679
  * Runs a channel-specific GraphQL operation through the live session's provider (the
1329
2680
  * {@link RealtimeChannelContext.ExecuteServerAction} implementation). Best-effort: any
@@ -1360,35 +2711,39 @@ export class RealtimeSessionRuntime {
1360
2711
  * never affected.
1361
2712
  */
1362
2713
  applyPriorChannelStates(statesJson) {
2714
+ this.priorChannelStates = {};
1363
2715
  if (!statesJson) {
1364
2716
  return;
1365
2717
  }
1366
- let states;
1367
2718
  try {
1368
2719
  const parsed = JSON.parse(statesJson);
1369
2720
  if (parsed === null || typeof parsed !== 'object') {
1370
2721
  return;
1371
2722
  }
1372
- states = parsed;
2723
+ this.priorChannelStates = parsed;
1373
2724
  }
1374
2725
  catch {
1375
2726
  console.warn('[RealtimeSession] PriorChannelStatesJson was malformed — starting channels fresh');
1376
2727
  return;
1377
2728
  }
1378
2729
  for (const plugin of this._activeChannels$.value) {
1379
- const state = states[plugin.ChannelName];
1380
- if (typeof state === 'string' && state.length > 0) {
1381
- try {
1382
- const restored = plugin.RestoreState(state);
1383
- if (!restored) {
1384
- console.warn(`[RealtimeSession] Channel '${plugin.ChannelName}' declined its prior-session state — starting fresh`);
1385
- }
1386
- }
1387
- catch (error) {
1388
- console.warn(`[RealtimeSession] Channel '${plugin.ChannelName}' restore threw — starting fresh`, error);
1389
- }
2730
+ this.restorePriorChannelState(plugin);
2731
+ }
2732
+ }
2733
+ /** Offers one plugin its prior-session state, if there is one. Rejections and throws are logged, never fatal. */
2734
+ restorePriorChannelState(plugin) {
2735
+ const state = this.priorChannelStates[plugin.ChannelName];
2736
+ if (typeof state !== 'string' || state.length === 0) {
2737
+ return;
2738
+ }
2739
+ try {
2740
+ if (!plugin.RestoreState(state)) {
2741
+ console.warn(`[RealtimeSession] Channel '${plugin.ChannelName}' declined its prior-session state — starting fresh`);
1390
2742
  }
1391
2743
  }
2744
+ catch (error) {
2745
+ console.warn(`[RealtimeSession] Channel '${plugin.ChannelName}' restore threw — starting fresh`, error);
2746
+ }
1392
2747
  }
1393
2748
  /**
1394
2749
  * Persists a channel's state as a first-class versioned artifact (`MJ: Artifacts`) via the
@@ -1478,6 +2833,13 @@ export class RealtimeSessionRuntime {
1478
2833
  this._activeChannels$.next([]);
1479
2834
  }
1480
2835
  this.usedChannelNames.clear();
2836
+ // Channels that were prepared or advertised but never mounted were never initialized, so there is
2837
+ // nothing to dispose — only the bookkeeping to forget.
2838
+ this.discardUnmountedChannels();
2839
+ this.catalogNotePending = false;
2840
+ this.resolvedChannels.clear();
2841
+ this.priorChannelStates = {};
2842
+ this.sessionClientTools = null;
1481
2843
  }
1482
2844
  // ── Realtime client resolution + wiring ────────────────────────────────────
1483
2845
  /**
@@ -1500,12 +2862,14 @@ export class RealtimeSessionRuntime {
1500
2862
  }
1501
2863
  /**
1502
2864
  * Builds the client-direct session config the realtime client connects with.
1503
- * Aggregates tracks sourced by active channels under {@link REQUESTED_TRACKS_SESSION_KEY} so the driver
1504
- * can negotiate them (e.g., establishing inbound video streaming for Whiteboard / RemoteBrowser).
2865
+ * Aggregates the tracks active channels source and sink under {@link REQUESTED_TRACKS_SESSION_KEY} so the
2866
+ * driver can negotiate them (e.g., inbound video for Whiteboard / RemoteBrowser, outbound video for a
2867
+ * channel that shows the agent's video). A track the model does not support resolves to `'unsupported'`.
2868
+ * The session's transport and relay URL go to the driver as minted (an unknown transport is left out: direct).
1505
2869
  */
1506
2870
  BuildClientConfig(session) {
1507
2871
  const sessionConfig = this.parseSessionConfig(session.SessionConfigJson);
1508
- const channelTracks = this._activeChannels$.value.flatMap((c) => c.GetSourcedTracks());
2872
+ const channelTracks = this._activeChannels$.value.flatMap((c) => [...c.GetSourcedTracks(), ...c.GetSunkTracks()]);
1509
2873
  if (channelTracks.length > 0) {
1510
2874
  // The requested tracks cross a JSON boundary — the driver reads them back out of the session
1511
2875
  // config bag (`GeminiRealtimeClient.parseSessionConfig`). A `RealtimeTrackDescriptor` is NOT
@@ -1530,11 +2894,14 @@ export class RealtimeSessionRuntime {
1530
2894
  }
1531
2895
  sessionConfig[REQUESTED_TRACKS_SESSION_KEY] = Array.from(trackMap.values());
1532
2896
  }
2897
+ const transport = ParseRealtimeClientTransport(session.Transport);
1533
2898
  return {
1534
2899
  Provider: session.Provider,
1535
2900
  Model: session.Model,
1536
2901
  EphemeralToken: session.EphemeralToken,
1537
2902
  ExpiresAt: session.ExpiresAt,
2903
+ ...(transport ? { Transport: transport } : {}),
2904
+ ...(session.RelayUrl ? { RelayUrl: session.RelayUrl } : {}),
1538
2905
  SessionConfig: sessionConfig
1539
2906
  };
1540
2907
  }
@@ -1570,6 +2937,9 @@ export class RealtimeSessionRuntime {
1570
2937
  });
1571
2938
  client.OnError((error) => {
1572
2939
  console.error('[RealtimeSession] Provider error event:', JSON.stringify(error), error);
2940
+ if (error.Fatal && this.client === client) {
2941
+ this.lastSessionError = error;
2942
+ }
1573
2943
  });
1574
2944
  // Usage telemetry: accumulate the driver's per-response token DELTAS and relay them to
1575
2945
  // the server (onto the co-agent AIPromptRun) debounced + once at teardown. Providers
@@ -1586,12 +2956,36 @@ export class RealtimeSessionRuntime {
1586
2956
  client.OnInterruption(() => {
1587
2957
  this.cancelPendingNarration();
1588
2958
  });
2959
+ client.OnRemoteVideo((video) => this.onAgentVideo(client, video));
2960
+ }
2961
+ /**
2962
+ * The model's video arrived: publish it on {@link AgentVideo$}, and mark each channel that shows it (one that sinks
2963
+ * outbound video) as used, so the host shows its surface. A client the session has already let go of is ignored, so a
2964
+ * late frame cannot bring the video back after the call.
2965
+ */
2966
+ onAgentVideo(client, video) {
2967
+ if (this.client !== client) {
2968
+ return;
2969
+ }
2970
+ this._agentVideo$.next(video);
2971
+ for (const channel of this._activeChannels$.value) {
2972
+ if (channel.ShowsAgentVideo) {
2973
+ this.noteChannelActivity(channel);
2974
+ }
2975
+ }
2976
+ }
2977
+ /** Clears {@link AgentVideo$} when the call's client goes. */
2978
+ clearAgentVideo() {
2979
+ if (this._agentVideo$.value !== null) {
2980
+ this._agentVideo$.next(null);
2981
+ }
1589
2982
  }
1590
2983
  /** Maps a client state event onto the UI connection state. */
1591
2984
  onClientStateChange(state) {
1592
2985
  const mapped = this.mapClientState(state);
1593
2986
  if (mapped) {
1594
- this._connectionState$.next(mapped);
2987
+ this.setConnectionState(mapped);
2988
+ this.flushChannelCatalogNote();
1595
2989
  }
1596
2990
  }
1597
2991
  /**
@@ -1620,6 +3014,17 @@ export class RealtimeSessionRuntime {
1620
3014
  const state = this._connectionState$.value;
1621
3015
  return state === 'listening' || state === 'speaking' || state === 'thinking';
1622
3016
  }
3017
+ /**
3018
+ * Moves {@link ConnectionState$} to `state`, and {@link Resuming$} first: a `'connecting'` while the call is live, or
3019
+ * already resuming, is a resume; any other state ends one. Every state change goes through here.
3020
+ */
3021
+ setConnectionState(state) {
3022
+ const resuming = state === 'connecting' && (this._resuming$.value || this.isSessionLive());
3023
+ if (resuming !== this._resuming$.value) {
3024
+ this._resuming$.next(resuming);
3025
+ }
3026
+ this._connectionState$.next(state);
3027
+ }
1623
3028
  // ── Transcript policy ──────────────────────────────────────────────────────
1624
3029
  /**
1625
3030
  * Applies transcript policy to client transcript events. Interim deltas don't become
@@ -1809,7 +3214,7 @@ export class RealtimeSessionRuntime {
1809
3214
  void this.relayToolTurn(call.ToolName, call.ArgumentsJson, resultJson);
1810
3215
  return;
1811
3216
  }
1812
- this._connectionState$.next('thinking');
3217
+ this.setConnectionState('thinking');
1813
3218
  if (this.inFlightCallIds.size === 0) {
1814
3219
  // A fresh delegation burst: anchor the first-update delay and clear the digest
1815
3220
  // buffer. Deliberately NOT reset: lastDelegationNarrationAt (the 8s spacing floor
@@ -1906,6 +3311,40 @@ export class RealtimeSessionRuntime {
1906
3311
  RunID: parsed.RunID,
1907
3312
  Artifacts: parsed.Artifacts
1908
3313
  });
3314
+ if (parsed.Success && parsed.Artifacts && parsed.Artifacts.length > 0) {
3315
+ void this.offerDelegationArtifacts(parsed.Artifacts);
3316
+ }
3317
+ }
3318
+ /**
3319
+ * Offers a delegated run's artifacts to the channels that host artifacts (an Interactive Component channel
3320
+ * shows a component artifact, or swaps in a newer version of one it already shows). Each channel is asked
3321
+ * first whether it wants them; an advertised, unopened channel that does is mounted before it is handed
3322
+ * them. One channel failing never stops the others, and nothing here can fail the delegation result,
3323
+ * which has already been emitted.
3324
+ */
3325
+ async offerDelegationArtifacts(artifacts) {
3326
+ const candidates = [
3327
+ ...this._activeChannels$.value.map((plugin) => ({ Plugin: plugin, IsOpen: true })),
3328
+ ...this.advertisedChannels.map((prepared) => ({ Plugin: prepared.Plugin, IsOpen: false }))
3329
+ ];
3330
+ for (const { Plugin: plugin, IsOpen: isOpen } of candidates) {
3331
+ try {
3332
+ const config = this.GetResolvedChannel(plugin.ChannelName)?.Config ?? {};
3333
+ if (!plugin.AcceptsDelegationArtifacts(artifacts, config)) {
3334
+ continue;
3335
+ }
3336
+ if (!isOpen) {
3337
+ await this.mountAdvertisedChannel(plugin);
3338
+ }
3339
+ await plugin.OnDelegationArtifacts(artifacts);
3340
+ if (!isOpen) {
3341
+ this.noteChannelActivity(plugin); // the channel was mounted for this: reveal its tab
3342
+ }
3343
+ }
3344
+ catch (error) {
3345
+ console.error(`[RealtimeSession] Channel '${plugin.ChannelName}' failed to take a delegated run's artifacts:`, error);
3346
+ }
3347
+ }
1909
3348
  }
1910
3349
  // ── Explicit delegation cancellation (server cancel channel) ───────────────
1911
3350
  /**
@@ -1996,44 +3435,34 @@ export class RealtimeSessionRuntime {
1996
3435
  }
1997
3436
  }
1998
3437
  // ── Session minting (GraphQL) ──────────────────────────────────────────────
1999
- /** Calls the `StartRealtimeClientSession` mutation to obtain an ephemeral token + config. */
2000
- async mintSession(targetAgentId, conversationId, lastSessionId, preferredModelId, clientTools, coAgentId, configOverridesJson, recordingConsent, recordingStartedAt, mediaCollectionId, applicationId, appContext) {
2001
- const mutation = `
2002
- mutation StartRealtimeClientSession($targetAgentId: String!, $conversationId: String, $lastSessionId: String, $preferredModelId: String, $clientToolsJson: String, $coAgentId: String, $configOverridesJson: String, $recordingConsent: Boolean, $recordingStartedAt: String, $mediaCollectionId: String, $applicationId: String, $appContextJson: String) {
2003
- StartRealtimeClientSession(targetAgentId: $targetAgentId, conversationId: $conversationId, lastSessionId: $lastSessionId, preferredModelId: $preferredModelId, clientToolsJson: $clientToolsJson, coAgentId: $coAgentId, configOverridesJson: $configOverridesJson, recordingConsent: $recordingConsent, recordingStartedAt: $recordingStartedAt, mediaCollectionId: $mediaCollectionId, applicationId: $applicationId, appContextJson: $appContextJson) {
2004
- AgentSessionId
2005
- ConversationId
2006
- Provider
2007
- Model
2008
- EphemeralToken
2009
- ExpiresAt
2010
- SessionConfigJson
2011
- ModelName
2012
- NarrationInstructionsTemplate
2013
- PriorChannelStatesJson
2014
- }
2015
- }
2016
- `;
2017
- const variables = {
2018
- targetAgentId,
2019
- conversationId: conversationId ?? null,
2020
- lastSessionId: lastSessionId ?? null,
2021
- preferredModelId: preferredModelId ?? null,
2022
- clientToolsJson: clientTools && clientTools.length > 0 ? JSON.stringify(clientTools) : null,
2023
- coAgentId: coAgentId ?? null,
2024
- configOverridesJson: configOverridesJson ?? null,
2025
- recordingConsent: recordingConsent ?? false,
2026
- recordingStartedAt: recordingStartedAt ?? null,
2027
- mediaCollectionId: mediaCollectionId ?? null,
2028
- applicationId: applicationId ?? null,
2029
- appContextJson: appContext ? JSON.stringify(appContext) : null
2030
- };
2031
- const result = await this.gql().ExecuteGQL(mutation, variables);
2032
- const payload = result?.StartRealtimeClientSession;
2033
- if (!payload?.EphemeralToken) {
2034
- throw new Error('StartRealtimeClientSession returned no ephemeral token');
2035
- }
2036
- return payload;
3438
+ /**
3439
+ * Mints a session through the installed {@link Launcher} — by default the stock
3440
+ * `StartRealtimeClientSession` mutation (see {@link DefaultRealtimeSessionLauncher}). `showsAgentVideo` is whether
3441
+ * one of the session's channels could show the agent's video; without one the server asks the model for no avatar.
3442
+ * `conversationBranchId` is the fork the session starts on; `null` is Main.
3443
+ */
3444
+ async mintSession(targetAgentId, conversationId, lastSessionId, preferredModelId, clientTools, coAgentId, configOverridesJson, recordingConsent, recordingStartedAt, mediaCollectionId, applicationId, appContext, channelCandidatesJson, showsAgentVideo, conversationBranchId) {
3445
+ const result = await this._launcher.Launch({
3446
+ TargetAgentId: targetAgentId,
3447
+ ConversationId: conversationId ?? null,
3448
+ LastSessionId: lastSessionId ?? null,
3449
+ PreferredModelId: preferredModelId ?? null,
3450
+ ClientTools: clientTools ?? [],
3451
+ CoAgentId: coAgentId ?? null,
3452
+ ConfigOverridesJson: configOverridesJson ?? null,
3453
+ RecordingConsent: recordingConsent ?? false,
3454
+ RecordingStartedAt: recordingStartedAt ?? null,
3455
+ MediaCollectionId: mediaCollectionId ?? null,
3456
+ ApplicationId: applicationId ?? null,
3457
+ AppContext: appContext ?? null,
3458
+ ChannelCandidatesJson: channelCandidatesJson ?? null,
3459
+ ConversationBranchId: conversationBranchId ?? null,
3460
+ ...(showsAgentVideo === undefined ? {} : { ShowsAgentVideo: showsAgentVideo })
3461
+ }, { Provider: this.Provider });
3462
+ if (!HasClientCredential(result)) {
3463
+ throw new Error('The session launcher returned no ephemeral token (or, for a relay session, no relay URL)');
3464
+ }
3465
+ return result;
2037
3466
  }
2038
3467
  /** Calls the `ExecuteRealtimeSessionTool` mutation; returns the ResultJson string. */
2039
3468
  async executeSessionTool(callId, toolName, argsJson) {
@@ -2172,16 +3601,28 @@ export class RealtimeSessionRuntime {
2172
3601
  /**
2173
3602
  * Accumulates one usage DELTA from the realtime client (per-response token counts —
2174
3603
  * the `OnUsage` contract shape) and schedules the debounced relay. Negative / non-finite
2175
- * values are clamped to 0; an all-zero delta is dropped without arming the timer.
3604
+ * values are clamped to 0; an all-zero delta is dropped without arming the timer. The
3605
+ * update's input and output detail blocks add into the pending details by the record's
3606
+ * rules (amounts add up, inbound video running totals keep the larger value), so an update
3607
+ * that carries only avatar video seconds still counts.
2176
3608
  */
2177
3609
  onUsageDelta(usage) {
2178
3610
  const input = this.clampUsageDelta(usage.InputTokens);
2179
3611
  const output = this.clampUsageDelta(usage.OutputTokens);
2180
- if (input === 0 && output === 0) {
3612
+ const details = AddRealtimeUsageRecord(null, { Input: usage.InputTokenDetails, Output: usage.OutputTokenDetails });
3613
+ const hasDetails = HasRealtimeUsage(details);
3614
+ if (input === 0 && output === 0 && !hasDetails) {
2181
3615
  return;
2182
3616
  }
2183
3617
  this.pendingUsageInput += input;
2184
3618
  this.pendingUsageOutput += output;
3619
+ if (hasDetails) {
3620
+ this.pendingUsageDetails = AddRealtimeUsageRecord(this.pendingUsageDetails, details);
3621
+ }
3622
+ this.armUsageFlush();
3623
+ }
3624
+ /** Schedules the debounced usage relay unless one is already pending. */
3625
+ armUsageFlush() {
2185
3626
  if (!this.usageFlushTimer) {
2186
3627
  this.usageFlushTimer = setTimeout(() => {
2187
3628
  this.usageFlushTimer = null;
@@ -2206,24 +3647,26 @@ export class RealtimeSessionRuntime {
2206
3647
  const sessionId = agentSessionId ?? this.agentSessionId;
2207
3648
  const input = this.pendingUsageInput;
2208
3649
  const output = this.pendingUsageOutput;
2209
- if (!sessionId || (input === 0 && output === 0)) {
3650
+ const details = this.pendingUsageDetails;
3651
+ if (!sessionId || (input === 0 && output === 0 && !details)) {
2210
3652
  return;
2211
3653
  }
2212
3654
  this.pendingUsageInput = 0;
2213
3655
  this.pendingUsageOutput = 0;
3656
+ this.pendingUsageDetails = null;
2214
3657
  try {
2215
- const mutation = `
2216
- mutation RelayRealtimeUsage($agentSessionId: String!, $inputTokens: Int!, $outputTokens: Int!) {
2217
- RelayRealtimeUsage(agentSessionId: $agentSessionId, inputTokens: $inputTokens, outputTokens: $outputTokens)
2218
- }
2219
- `;
2220
- await this.gql().ExecuteGQL(mutation, { agentSessionId: sessionId, inputTokens: input, outputTokens: output });
3658
+ // The details argument is sent only when there are details, so a token-only relay is unchanged.
3659
+ const variables = { agentSessionId: sessionId, inputTokens: input, outputTokens: output, ...(details ? { usageDetailsJson: JSON.stringify(details) } : {}) };
3660
+ await this.gql().ExecuteGQL(RELAY_REALTIME_USAGE_MUTATION, variables);
2221
3661
  }
2222
3662
  catch (error) {
2223
3663
  console.error('[RealtimeSession] Failed to relay usage telemetry:', error);
2224
3664
  // Re-accumulate so a later debounce / the teardown flush retries the same deltas.
2225
3665
  this.pendingUsageInput += input;
2226
3666
  this.pendingUsageOutput += output;
3667
+ if (details) {
3668
+ this.pendingUsageDetails = AddRealtimeUsageRecord(this.pendingUsageDetails, details);
3669
+ }
2227
3670
  }
2228
3671
  }
2229
3672
  /** Cancels the pending debounced usage flush and zeroes the accumulators (teardown tail). */
@@ -2234,6 +3677,7 @@ export class RealtimeSessionRuntime {
2234
3677
  }
2235
3678
  this.pendingUsageInput = 0;
2236
3679
  this.pendingUsageOutput = 0;
3680
+ this.pendingUsageDetails = null;
2237
3681
  }
2238
3682
  // ── Delegated-run progress streaming ───────────────────────────────────────
2239
3683
  /**
@@ -2513,42 +3957,66 @@ export class RealtimeSessionRuntime {
2513
3957
  return;
2514
3958
  }
2515
3959
  this.teardownInFlight = this.runTeardown(closeServerSession).finally(() => {
3960
+ if (this._savingRecording$.value) {
3961
+ this._savingRecording$.next(false);
3962
+ }
2516
3963
  this.teardownInFlight = null;
2517
3964
  });
2518
3965
  await this.teardownInFlight;
2519
3966
  }
2520
3967
  /** The body of {@link teardown}; never called concurrently with itself. */
2521
3968
  async runTeardown(closeServerSession) {
3969
+ // Arm BEFORE any await so a tab closed in the first milliseconds of teardown is still guarded.
3970
+ if (this.recorder) {
3971
+ this._savingRecording$.next(true);
3972
+ }
2522
3973
  // First: stop asserting liveness. A pulse racing the close would re-stamp LastActiveAt on a
2523
3974
  // session we are deliberately ending, leaving an Idle row the janitor then has to age out.
2524
3975
  this.stopLivenessPulse();
2525
3976
  this.teardownDelegationProgress();
3977
+ this.stopSessionEvents();
3978
+ this.clientDeadline.Clear();
2526
3979
  // Channels first: flush any unsaved channel state WHILE the live session id is still
2527
3980
  // set (the captured per-save id covers the race anyway), then dispose the plugins.
2528
3981
  this.flushAllChannelSaves();
2529
3982
  this.disposeChannels();
3983
+ // The camera and screen share go first: the camera runs on the controller closed next.
3984
+ this.closeCaptures();
2530
3985
  // Defensive: stop the mic even when Connect never ran (the client also stops the
2531
3986
  // tracks it was handed — track.stop() is idempotent).
3987
+ this.closeLocalMedia();
2532
3988
  this.localStream?.getTracks().forEach(t => t.stop());
2533
3989
  this.localStream = null;
2534
- // Hand the platform back whatever acquiring the microphone changed. Stopping the tracks is not
3990
+ this.microphoneMuted = false;
3991
+ // Hand the platform back whatever opening the microphone changed. Stopping the tracks is not
2535
3992
  // the same thing: iOS, for instance, is put into a record-and-play audio category for the call,
2536
3993
  // and leaving it there changes the route and volume behaviour of every sound the app makes
2537
- // afterwards. Best-effort by contract — a failure here must never block ending a call.
2538
- try {
2539
- await this.mediaHost.ReleaseMicrophone?.();
2540
- }
2541
- catch (error) {
2542
- console.error('[RealtimeSession] Media host failed to release the microphone:', error);
3994
+ // afterwards. A microphone still opening is left to its start, which releases it once the
3995
+ // opening returns ({@link microphoneLease}).
3996
+ const lease = this.microphoneLease;
3997
+ if (lease && !lease.Opening) {
3998
+ this.microphoneLease = null;
3999
+ await this.releaseMicrophone();
2543
4000
  }
2544
4001
  if (this.client) {
2545
- await this.client.Disconnect();
2546
- this.client = null;
4002
+ // A stranded Active session is worse than a noisy socket: a driver that throws on Disconnect
4003
+ // (socket already gone) must not stop the server session from being closed below.
4004
+ try {
4005
+ await this.client.Disconnect();
4006
+ }
4007
+ catch (error) {
4008
+ console.error(`[RealtimeSession] Driver failed to disconnect (session ${this.agentSessionId}); continuing teardown:`, error);
4009
+ }
4010
+ finally {
4011
+ this.unwatchVideoSources();
4012
+ this.clearAgentVideo();
4013
+ this.client = null;
4014
+ }
2547
4015
  }
2548
- // Stop + upload the call recording WHILE the live session id is still set (the file is
2549
- // attached to it). Best-effort and never blocks teardown — stopAndUploadRecording swallows
2550
- // its own errors. No-op when nothing was recorded.
2551
- await this.stopAndUploadRecording(this.agentSessionId);
4016
+ // Stop + encode the call recording now (the recorder must be detached before the session id is
4017
+ // cleared); the upload itself happens after the close below. Best-effort — stopRecording
4018
+ // swallows its own errors. Null when nothing was recorded.
4019
+ const recording = await this.stopRecording();
2552
4020
  this.recordingStartedAtIso = null;
2553
4021
  // Final usage flush WHILE the live session id is still set (the relay mutation also
2554
4022
  // accepts a Closed session, so ordering vs. CloseAgentSession is belt-and-braces).
@@ -2561,18 +4029,27 @@ export class RealtimeSessionRuntime {
2561
4029
  if (closeServerSession && this.agentSessionId) {
2562
4030
  await this.closeServerSession(this.agentSessionId);
2563
4031
  }
4032
+ // Upload AFTER the close. The WAV is tens of MB and takes seconds; a page unload aborts the
4033
+ // request, and nothing after an aborted await ever runs — so the small, critical close goes
4034
+ // first (#5195). The server accepts the upload on a Closed session (loadOwnedSession does not
4035
+ // check status), and uploadStoppedRecording never throws.
4036
+ await this.uploadStoppedRecording(this.agentSessionId, recording);
2564
4037
  // Capture the session id BEFORE we null it so the lifecycle emit carries it.
2565
4038
  // Skip emitting when there was no live session (defensive — teardown is safe
2566
4039
  // to call without an active session).
2567
4040
  const closedSessionId = this.agentSessionId;
2568
4041
  this.agentSessionId = null;
4042
+ this.sessionConversationId = null;
2569
4043
  this.narrationTemplate = null;
2570
4044
  this.clientToolHandlers.clear();
2571
4045
  this._modelName$.next(null);
4046
+ if (this._avatarNotice$.value !== null) {
4047
+ this._avatarNotice$.next(null);
4048
+ }
2572
4049
  this.SetMinimized(false);
2573
4050
  this._active$.next(false);
2574
4051
  if (this._connectionState$.value !== 'error') {
2575
- this._connectionState$.next('closed');
4052
+ this.setConnectionState('closed');
2576
4053
  }
2577
4054
  // Surface generic session-ended for the conversations runtime bridge.
2578
4055
  // `closeServerSession=true` means the user explicitly called EndRealtimeSession;
@@ -2584,6 +4061,18 @@ export class RealtimeSessionRuntime {
2584
4061
  });
2585
4062
  }
2586
4063
  }
4064
+ /**
4065
+ * Calls the host's {@link IRealtimeMediaHost.ReleaseMicrophone}. Best-effort by contract: a failure
4066
+ * is logged and never thrown, because it must never block ending a call.
4067
+ */
4068
+ async releaseMicrophone() {
4069
+ try {
4070
+ await this.mediaHost.ReleaseMicrophone?.();
4071
+ }
4072
+ catch (error) {
4073
+ console.error('[RealtimeSession] Media host failed to release the microphone:', error);
4074
+ }
4075
+ }
2587
4076
  /** Calls the `CloseAgentSession` mutation (provisioned in P4b). */
2588
4077
  async closeServerSession(agentSessionId) {
2589
4078
  try {
@@ -2610,12 +4099,13 @@ export class RealtimeSessionRuntime {
2610
4099
  this.hasActiveInterimUserCaption = false;
2611
4100
  this.SetMinimized(false);
2612
4101
  this.stopSegmentFlushing();
2613
- this.segmentIndex = 0;
4102
+ this.shardQueue = null;
2614
4103
  this.recorder = null;
2615
4104
  this.recordingStartedAtIso = null;
2616
4105
  this.currentTurnStartMs = null;
2617
4106
  this.turnAudioStartCaptured = false;
2618
4107
  this.usedChannelNames.clear();
4108
+ this.microphoneMuted = false;
2619
4109
  }
2620
4110
  /** The GraphQL provider used for relay mutations. */
2621
4111
  gql() {