@memberjunction/realtime-runtime 6.2.0-edge.2 → 6.2.0-edge.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/README.md +75 -0
  2. package/dist/channels/base-realtime-channel-client.d.ts +535 -28
  3. package/dist/channels/base-realtime-channel-client.d.ts.map +1 -1
  4. package/dist/channels/base-realtime-channel-client.js +864 -10
  5. package/dist/channels/base-realtime-channel-client.js.map +1 -1
  6. package/dist/channels/channel-action-dispatcher.d.ts +70 -0
  7. package/dist/channels/channel-action-dispatcher.d.ts.map +1 -0
  8. package/dist/channels/channel-action-dispatcher.js +136 -0
  9. package/dist/channels/channel-action-dispatcher.js.map +1 -0
  10. package/dist/channels/channel-catalog-note.d.ts +48 -0
  11. package/dist/channels/channel-catalog-note.d.ts.map +1 -0
  12. package/dist/channels/channel-catalog-note.js +119 -0
  13. package/dist/channels/channel-catalog-note.js.map +1 -0
  14. package/dist/channels/channel-contract-types.d.ts +107 -0
  15. package/dist/channels/channel-contract-types.d.ts.map +1 -0
  16. package/dist/channels/channel-contract-types.js +12 -0
  17. package/dist/channels/channel-contract-types.js.map +1 -0
  18. package/dist/channels/channel-descriptor-synthesis.d.ts +56 -0
  19. package/dist/channels/channel-descriptor-synthesis.d.ts.map +1 -0
  20. package/dist/channels/channel-descriptor-synthesis.js +90 -0
  21. package/dist/channels/channel-descriptor-synthesis.js.map +1 -0
  22. package/dist/channels/channel-perception.d.ts +82 -0
  23. package/dist/channels/channel-perception.d.ts.map +1 -0
  24. package/dist/channels/channel-perception.js +120 -0
  25. package/dist/channels/channel-perception.js.map +1 -0
  26. package/dist/channels/channel-schema-format.d.ts +17 -0
  27. package/dist/channels/channel-schema-format.d.ts.map +1 -0
  28. package/dist/channels/channel-schema-format.js +45 -0
  29. package/dist/channels/channel-schema-format.js.map +1 -0
  30. package/dist/channels/channel-state-delta.d.ts +59 -0
  31. package/dist/channels/channel-state-delta.d.ts.map +1 -0
  32. package/dist/channels/channel-state-delta.js +116 -0
  33. package/dist/channels/channel-state-delta.js.map +1 -0
  34. package/dist/channels/channel-surface-placement.d.ts +25 -0
  35. package/dist/channels/channel-surface-placement.d.ts.map +1 -0
  36. package/dist/channels/channel-surface-placement.js +29 -0
  37. package/dist/channels/channel-surface-placement.js.map +1 -0
  38. package/dist/channels/channel-visual-pump.d.ts +126 -0
  39. package/dist/channels/channel-visual-pump.d.ts.map +1 -0
  40. package/dist/channels/channel-visual-pump.js +174 -0
  41. package/dist/channels/channel-visual-pump.js.map +1 -0
  42. package/dist/hosts/IRealtimeMediaHost.d.ts +61 -4
  43. package/dist/hosts/IRealtimeMediaHost.d.ts.map +1 -1
  44. package/dist/hosts/IRealtimeMediaHost.js +2 -1
  45. package/dist/hosts/IRealtimeMediaHost.js.map +1 -1
  46. package/dist/index.d.ts +21 -2
  47. package/dist/index.d.ts.map +1 -1
  48. package/dist/index.js +24 -0
  49. package/dist/index.js.map +1 -1
  50. package/dist/session/RealtimeSessionRuntime.d.ts +730 -70
  51. package/dist/session/RealtimeSessionRuntime.d.ts.map +1 -1
  52. package/dist/session/RealtimeSessionRuntime.js +1761 -241
  53. package/dist/session/RealtimeSessionRuntime.js.map +1 -1
  54. package/dist/session/RecordingShardQueue.d.ts +77 -0
  55. package/dist/session/RecordingShardQueue.d.ts.map +1 -0
  56. package/dist/session/RecordingShardQueue.js +172 -0
  57. package/dist/session/RecordingShardQueue.js.map +1 -0
  58. package/dist/session/app-client-tool-registry.d.ts +76 -0
  59. package/dist/session/app-client-tool-registry.d.ts.map +1 -0
  60. package/dist/session/app-client-tool-registry.js +100 -0
  61. package/dist/session/app-client-tool-registry.js.map +1 -0
  62. package/dist/session/avatar-notice.d.ts +43 -0
  63. package/dist/session/avatar-notice.d.ts.map +1 -0
  64. package/dist/session/avatar-notice.js +60 -0
  65. package/dist/session/avatar-notice.js.map +1 -0
  66. package/dist/session/call-end.d.ts +21 -0
  67. package/dist/session/call-end.d.ts.map +1 -0
  68. package/dist/session/call-end.js +19 -0
  69. package/dist/session/call-end.js.map +1 -0
  70. package/dist/session/channel-exposure-preferences.d.ts +70 -0
  71. package/dist/session/channel-exposure-preferences.d.ts.map +1 -0
  72. package/dist/session/channel-exposure-preferences.js +138 -0
  73. package/dist/session/channel-exposure-preferences.js.map +1 -0
  74. package/dist/session/channel-session-scope.d.ts +142 -0
  75. package/dist/session/channel-session-scope.d.ts.map +1 -0
  76. package/dist/session/channel-session-scope.js +128 -0
  77. package/dist/session/channel-session-scope.js.map +1 -0
  78. package/dist/session/client-session-deadline.d.ts +28 -0
  79. package/dist/session/client-session-deadline.d.ts.map +1 -0
  80. package/dist/session/client-session-deadline.js +50 -0
  81. package/dist/session/client-session-deadline.js.map +1 -0
  82. package/dist/session/realtime-captures.d.ts +236 -0
  83. package/dist/session/realtime-captures.d.ts.map +1 -0
  84. package/dist/session/realtime-captures.js +383 -0
  85. package/dist/session/realtime-captures.js.map +1 -0
  86. package/dist/session/realtime-microphone.d.ts +50 -0
  87. package/dist/session/realtime-microphone.d.ts.map +1 -0
  88. package/dist/session/realtime-microphone.js +51 -0
  89. package/dist/session/realtime-microphone.js.map +1 -0
  90. package/dist/session/session-event-hub.d.ts +72 -0
  91. package/dist/session/session-event-hub.d.ts.map +1 -0
  92. package/dist/session/session-event-hub.js +151 -0
  93. package/dist/session/session-event-hub.js.map +1 -0
  94. package/dist/session/session-launcher.d.ts +128 -0
  95. package/dist/session/session-launcher.d.ts.map +1 -0
  96. package/dist/session/session-launcher.js +160 -0
  97. package/dist/session/session-launcher.js.map +1 -0
  98. package/package.json +9 -9
@@ -1,13 +1,28 @@
1
1
  import { BehaviorSubject, Subject } from 'rxjs';
2
2
  import { Metadata } from '@memberjunction/core';
3
3
  import { UserInfoEngine } from '@memberjunction/core-entities';
4
- import { AIEngineBase } from '@memberjunction/ai-engine-base';
5
- import { MJGlobal } from '@memberjunction/global';
6
- import { DEFAULT_REALTIME_AUDIO_TRACKS } from '@memberjunction/ai';
7
- import { BaseRealtimeClient, LoadAssemblyAIRealtimeClient, LoadElevenLabsRealtimeClient, LoadGeminiRealtimeClient, LoadHuggingFaceRealtimeClient, LoadOpenAIRealtimeClient, LoadxAIRealtimeClient } from '@memberjunction/ai-realtime-client';
4
+ import { AddRealtimeUsageRecord, AIEngineBase, HasRealtimeUsage } from '@memberjunction/ai-engine-base';
5
+ import { GraphQLRealtimeSessionClient } from '@memberjunction/graphql-dataprovider';
6
+ import { MJGlobal, UUIDsEqual } from '@memberjunction/global';
7
+ import { DEFAULT_REALTIME_AUDIO_TRACKS, ParseRealtimeAvatarStatus, ParseRealtimeClientTransport } from '@memberjunction/ai';
8
+ import { CompareExposure, DeclaresNativeTools, IsIdentityVerifiedEventPayload, NormalizeChannelKey, ParseRealtimeSessionClientPolicy, ResolveClientTools, SelectNativeChannelTools, UserExposureReason } from '@memberjunction/ai-core-plus';
9
+ import { BaseRealtimeClient, LoadAssemblyAIRealtimeClient, LoadElevenLabsRealtimeClient, LoadGeminiEnterpriseRealtimeClient, LoadGeminiRealtimeClient, LoadHuggingFaceRealtimeClient, LoadOpenAIRealtimeClient, LoadxAIRealtimeClient, VideoSourceArbiter, REQUESTED_TRACKS_SESSION_KEY } from '@memberjunction/ai-realtime-client';
10
+ import { DefaultRealtimeSessionLauncher, HasClientCredential } from './session-launcher.js';
11
+ import { RequestsAgentVideo, ResolveAvatarNotice } from './avatar-notice.js';
12
+ import { ClientSessionDeadline } from './client-session-deadline.js';
13
+ import { RealtimeCaptures, REALTIME_CAPTURES_OFF, REALTIME_CAPTURE_OFFERS_NONE, } from './realtime-captures.js';
14
+ import { REALTIME_MICROPHONE_NONE, ReadMicrophoneState, SameMicrophoneState } from './realtime-microphone.js';
15
+ import { RealtimeSessionEventHub } from './session-event-hub.js';
8
16
  import { BuildNarrationInstructions } from '../narration/narration-template.js';
9
17
  import { ParseDelegationResultJson, FormatToolName } from './delegation-result-parser.js';
10
- import { BaseRealtimeChannelClient } from '../channels/base-realtime-channel-client.js';
18
+ import { BaseRealtimeChannelClient, ChannelVideoSourceID } from '../channels/base-realtime-channel-client.js';
19
+ import { DEFAULT_CHANNEL_SURFACE_PLACEMENT, ReadChannelSurfacePlacement } from '../channels/channel-surface-placement.js';
20
+ import { ChannelActionDispatcher } from '../channels/channel-action-dispatcher.js';
21
+ import { BuildChannelCatalogNote } from '../channels/channel-catalog-note.js';
22
+ import { AppClientToolRegistry, DEFAULT_APP_TOOL_OWNER } from './app-client-tool-registry.js';
23
+ import { UserSettingsExposurePreferences } from './channel-exposure-preferences.js';
24
+ import { BuildChannelCandidate, FindPreparedChannel, MergeToolMetadata, PreparedChannelsShowAgentVideo, ReconcileChannelsWithPolicy, ResolveLocalChannelScope, ToolsByChannelKey } from './channel-session-scope.js';
25
+ import { RecordingShardQueue } from './RecordingShardQueue.js';
11
26
  /**
12
27
  * `MJ: User Settings` key for the per-user "record this voice call" consent toggle. Stored as
13
28
  * the literal string `'true'`/`'false'` (read with `=== 'true'`), cross-device via
@@ -15,6 +30,15 @@ import { BaseRealtimeChannelClient } from '../channels/base-realtime-channel-cli
15
30
  * default when the caller doesn't pass an explicit consent value.
16
31
  */
17
32
  export const REALTIME_RECORDING_CONSENT_KEY = 'mj.realtimeVoice.recordingConsent.v1';
33
+ /**
34
+ * Relays usage onto the co-agent prompt run: token deltas, and the per-modality detail blocks as a
35
+ * JSON record (sent only when there are any; see `RelayRealtimeUsage` in MJServer).
36
+ */
37
+ const RELAY_REALTIME_USAGE_MUTATION = `
38
+ mutation RelayRealtimeUsage($agentSessionId: String!, $inputTokens: Int!, $outputTokens: Int!, $usageDetailsJson: String) {
39
+ RelayRealtimeUsage(agentSessionId: $agentSessionId, inputTokens: $inputTokens, outputTokens: $outputTokens, usageDetailsJson: $usageDetailsJson)
40
+ }
41
+ `;
18
42
  // Tree-shaking prevention: the OpenAI client is resolved dynamically through the
19
43
  // ClassFactory (by the server-reported Provider key), so this static call is what keeps
20
44
  // its @RegisterClass side effect from being eliminated by the bundler.
@@ -24,10 +48,37 @@ export const REALTIME_RECORDING_CONSENT_KEY = 'mj.realtimeVoice.recordingConsent
24
48
  // and this service stays component-free (it must stay importable in plain-node tests).
25
49
  LoadOpenAIRealtimeClient();
26
50
  LoadGeminiRealtimeClient();
51
+ LoadGeminiEnterpriseRealtimeClient();
27
52
  LoadElevenLabsRealtimeClient();
28
53
  LoadAssemblyAIRealtimeClient();
29
54
  LoadxAIRealtimeClient();
30
55
  LoadHuggingFaceRealtimeClient();
56
+ /**
57
+ * A registry row's surface placement. A row whose `UIConfig` is not valid JSON places its surface as a row without one
58
+ * does, so one bad row never costs the session its other channels.
59
+ */
60
+ function readRowSurfacePlacement(row) {
61
+ try {
62
+ return ReadChannelSurfacePlacement(row.UIConfigObject);
63
+ }
64
+ catch {
65
+ console.warn(`[RealtimeSession] Channel '${row.Name}' has a UIConfig that is not valid JSON; its surface starts on its tab.`);
66
+ return DEFAULT_CHANNEL_SURFACE_PLACEMENT;
67
+ }
68
+ }
69
+ /** The same, for a row read over GraphQL, whose `UIConfig` arrives as the column's JSON text. */
70
+ function readUIConfigTextSurfacePlacement(name, uiConfig) {
71
+ if (!uiConfig) {
72
+ return ReadChannelSurfacePlacement(null);
73
+ }
74
+ try {
75
+ return ReadChannelSurfacePlacement(JSON.parse(uiConfig));
76
+ }
77
+ catch {
78
+ console.warn(`[RealtimeSession] Channel '${name}' has a UIConfig that is not valid JSON; its surface starts on its tab.`);
79
+ return DEFAULT_CHANNEL_SURFACE_PLACEMENT;
80
+ }
81
+ }
31
82
  /**
32
83
  * Converts a {@link RealtimeTrackDescriptor} to its JSON form for the session-config bag.
33
84
  *
@@ -67,6 +118,37 @@ function trackKeyFromJSON(raw) {
67
118
  }
68
119
  return `${direction}:${modality}`;
69
120
  }
121
+ /** Whether a surface client tool handler's result is an object whose `Success` is `false`. */
122
+ function reportsFailure(result) {
123
+ return typeof result === 'object' && result !== null && 'Success' in result && result.Success === false;
124
+ }
125
+ /** The failed result's `ErrorMessage` when it is text that is not blank, else `'Tool failed'`. */
126
+ function failureMessage(result) {
127
+ const message = result.ErrorMessage;
128
+ return typeof message === 'string' && message.trim() ? message : 'Tool failed';
129
+ }
130
+ /**
131
+ * The `DOMException` name for each way the controller can fail to start the microphone, so a host that
132
+ * tells a denied microphone apart (`error.name === 'NotAllowedError'`) works the same with or without one.
133
+ */
134
+ const MICROPHONE_ERROR_NAMES = {
135
+ denied: 'NotAllowedError',
136
+ 'not-found': 'NotFoundError',
137
+ 'in-use': 'NotReadableError',
138
+ unsupported: 'NotSupportedError',
139
+ error: 'Error',
140
+ };
141
+ /** What the user is told when the call's policy refuses a capture. */
142
+ const CAPTURE_WORDING = {
143
+ camera: { NotInCall: 'The camera is not part of this call.', NotAllowed: 'This call cannot show the agent your camera' },
144
+ screen: { NotInCall: 'Screen sharing is not part of this call.', NotAllowed: 'This call cannot show the agent your screen' },
145
+ };
146
+ /** The error a failed microphone start reports, named as `getUserMedia` would have named it. */
147
+ function microphoneStartError(reason, message) {
148
+ const error = new Error(message);
149
+ error.name = MICROPHONE_ERROR_NAMES[reason];
150
+ return error;
151
+ }
70
152
  /**
71
153
  * Drives a **client-direct** real-time voice session: the browser mints an ephemeral
72
154
  * token from the MJ server, then connects DIRECTLY to the realtime provider. Audio
@@ -100,6 +182,7 @@ export class RealtimeSessionRuntime {
100
182
  this.mediaHost = mediaHost;
101
183
  // ── Reactive UI state ──────────────────────────────────────────────────────
102
184
  this._connectionState$ = new BehaviorSubject('closed');
185
+ this._resuming$ = new BehaviorSubject(false);
103
186
  this._captions$ = new BehaviorSubject([]);
104
187
  this._active$ = new BehaviorSubject(false);
105
188
  this._delegationProgress$ = new Subject();
@@ -109,6 +192,7 @@ export class RealtimeSessionRuntime {
109
192
  this._agentName$ = new BehaviorSubject('Sage');
110
193
  this._modelName$ = new BehaviorSubject(null);
111
194
  this._minimized$ = new BehaviorSubject(false);
195
+ this._savingRecording$ = new BehaviorSubject(false);
112
196
  this._activeChannels$ = new BehaviorSubject([]);
113
197
  this._channelFocus$ = new Subject();
114
198
  // ─── Generic session-lifecycle events (consumed by RealtimeSessionsAdapter to
@@ -122,6 +206,16 @@ export class RealtimeSessionRuntime {
122
206
  this._channelActivity$ = new Subject();
123
207
  /** Current connection / turn state. */
124
208
  this.ConnectionState$ = this._connectionState$.asObservable();
209
+ /**
210
+ * Whether the call is resuming: moving to a new connection with its provider mid-call. `true` from a `'connecting'` the
211
+ * driver reports while the call is live, until the call is live again (`'listening'`, `'speaking'` or `'thinking'`),
212
+ * fails (`'error'`) or ends (`'closed'`). A call's first connect is not a resume, so it is `false` until the call has
213
+ * been live, and outside a call. Gemini Live resumes when Google ends a connection (`goAway`) and after a dropped
214
+ * socket, and the agent's video can stop for a few seconds meanwhile: a host can show that as a pause rather than as
215
+ * the agent leaving. It changes before {@link ConnectionState$} does, so a resume's `'connecting'` arrives with it
216
+ * already `true`.
217
+ */
218
+ this.Resuming$ = this._resuming$.asObservable();
125
219
  /** Live captions for both sides of the conversation. */
126
220
  this.Captions$ = this._captions$.asObservable();
127
221
  /** True while a session is open (mic button active, overlay shown). */
@@ -158,6 +252,14 @@ export class RealtimeSessionRuntime {
158
252
  * pure presentation state, reset to `false` at session start and teardown.
159
253
  */
160
254
  this.Minimized$ = this._minimized$.asObservable();
255
+ /**
256
+ * True while the end-of-call recording is being saved: from the first statement of a teardown
257
+ * that holds a recorder until that teardown settles. False at construction and for unrecorded
258
+ * sessions. Exists for #5195: the page must stay open until the consolidated upload lands, or
259
+ * the recording is lost. The runtime is platform-free, so hosts own the reaction (a browser
260
+ * `beforeunload` guard, a "saving" indicator, etc.).
261
+ */
262
+ this.SavingRecording$ = this._savingRecording$.asObservable();
161
263
  /**
162
264
  * The session's ACTIVE interactive-channel plugins, resolved from the `MJ: AI Agent
163
265
  * Channels` registry at session start (one instance per session, per channel). Emits
@@ -165,6 +267,57 @@ export class RealtimeSessionRuntime {
165
267
  * one surface tab per plugin — it never knows any concrete channel type.
166
268
  */
167
269
  this.ActiveChannels$ = this._activeChannels$.asObservable();
270
+ this._videoSources$ = new BehaviorSubject([]);
271
+ /**
272
+ * The video sources the agent is perceiving or could perceive right now (a whiteboard, a remote browser, a
273
+ * shared screen), with whether each is on and whether its frames are reaching the model. Empty when no
274
+ * session is live or nothing has offered a frame source. This is what the "agent can see" control renders;
275
+ * toggle one with {@link SetVideoSourceEnabled}.
276
+ */
277
+ this.VideoSources$ = this._videoSources$.asObservable();
278
+ this._captures$ = new BehaviorSubject(REALTIME_CAPTURES_OFF);
279
+ /**
280
+ * The user's camera and screen share: off, starting, on (with the stream to show the user) or failed (with the
281
+ * reason). Both off outside a session. Start and stop them with {@link StartCamera}, {@link StartScreenShare},
282
+ * {@link StopCamera} and {@link StopScreenShare}; while one is on, it is also a source on {@link VideoSources$}.
283
+ */
284
+ this.Captures$ = this._captures$.asObservable();
285
+ this._captureOffers$ = new BehaviorSubject(REALTIME_CAPTURE_OFFERS_NONE);
286
+ /**
287
+ * Which captures the call offers, now and on every change. A capture is offered while the call is connected, the model
288
+ * takes inbound video, the host can open it (a camera controller; screen sharing) and the call's policy admits it (its
289
+ * channel is in the call and may show the agent pixels). A host shows its Camera and Share controls from this. Neither
290
+ * is offered outside a call.
291
+ */
292
+ this.CaptureOffers$ = this._captureOffers$.asObservable();
293
+ this._microphone$ = new BehaviorSubject(REALTIME_MICROPHONE_NONE);
294
+ /**
295
+ * The call's microphone, now and on every change: the one in use and the ones the user can move it to with
296
+ * {@link SwitchMicrophone}, as the host's camera-and-microphone controller lists them. Offered from the moment the call
297
+ * connects until it ends, and kept current as devices come and go. When the call loses its microphone (the one in use
298
+ * fails and so does the fallback to the default, or the last one goes away), its `Failure` says why: the agent hears
299
+ * nothing until the user picks a microphone, which opens it again. {@link REALTIME_MICROPHONE_NONE} outside a call and
300
+ * on a host without a controller, which opens the microphone itself (`IRealtimeMediaHost.AcquireMicrophone`).
301
+ */
302
+ this.Microphone$ = this._microphone$.asObservable();
303
+ this._agentVideo$ = new BehaviorSubject(null);
304
+ /**
305
+ * The agent's video (an avatar) while the model sends it: a live stream, or a player that owns the `<video>` element.
306
+ * `null` until the video arrives, for a model that sends none, and outside a call. The driver hands it over once it is
307
+ * live, and a newer one replaces it. A host shows it with `AttachVideoSource`; a channel that sinks outbound video
308
+ * follows it through its context, and counts as used once it arrives, so the host shows that channel's surface.
309
+ */
310
+ this.AgentVideo$ = this._agentVideo$.asObservable();
311
+ this._avatarNotice$ = new BehaviorSubject(null);
312
+ /**
313
+ * Why the call shows no avatar the agent asked for: set once per call, after it connects, when the model renders none,
314
+ * the persona has no face for it, the app shows no agent video (`host`) or the browser can't play it (`browser`); with
315
+ * no `Reason` when the mint gave one this version doesn't know, such as a newer server's. `null` when there is nothing
316
+ * to say (no avatar asked for, the avatar shows, or the mint reported no status) and after the call ends. A provider
317
+ * resume inside the call does not set it again. A host turns the reason into words once, for example "Audio only: this
318
+ * voice model can't show an avatar".
319
+ */
320
+ this.AvatarNotice$ = this._avatarNotice$.asObservable();
168
321
  /**
169
322
  * Channel requests to enter / leave the FOCUS layout (see
170
323
  * {@link RealtimeChannelFocusEvent}). Fired when a plugin calls its host context's
@@ -208,7 +361,11 @@ export class RealtimeSessionRuntime {
208
361
  */
209
362
  /** Conversation id the SERVER created for this session (null when the host supplied one). */
210
363
  this.createdConversationId = null;
211
- /** The session's conversation id (supplied or server-created). */
364
+ /**
365
+ * The session's conversation id (supplied or server-created). While a start's mint runs, the
366
+ * conversation the start asked for; null when it asked the server to create one. Null after
367
+ * teardown.
368
+ */
212
369
  this.sessionConversationId = null;
213
370
  /** First final user utterance of the live session (the naming seed). */
214
371
  this.firstUserTranscript = null;
@@ -219,8 +376,28 @@ export class RealtimeSessionRuntime {
219
376
  // ── Session internals ──────────────────────────────────────────────────────
220
377
  /** The provider-direct realtime client driving the live session (ClassFactory-resolved). */
221
378
  this.client = null;
222
- /** The mic capture stream — acquired here (permission UX) and handed to the client. */
379
+ /**
380
+ * The mic capture stream — acquired here (permission UX) and handed to the client. A microphone the controller opens
381
+ * again after the call lost one replaces it.
382
+ */
223
383
  this.localStream = null;
384
+ /**
385
+ * Whether the user muted the call's microphone, as {@link ToggleMute} last reported it. While the call has no microphone
386
+ * track there is nothing to mute, and a microphone opened again after the call lost one starts this way.
387
+ */
388
+ this.microphoneMuted = false;
389
+ /** The host's camera-and-microphone controller for this session, when the host offers one. */
390
+ this.localMedia = null;
391
+ /** Follows the controller's microphone, so a swapped-in track or a reopened microphone reaches the driver and the recorder. */
392
+ this.localMediaSubscription = null;
393
+ /**
394
+ * Whether the call offers its microphone for switching ({@link Microphone$}, {@link SwitchMicrophone}): from the moment
395
+ * the client is connected until the call ends. Not before: a driver still connecting may have bound the old track.
396
+ */
397
+ this.microphoneOffered = false;
398
+ /** The live session's camera and screen share; created once the client is connected. */
399
+ this.captures = null;
400
+ this.capturesSubscription = null;
224
401
  this.agentSessionId = null;
225
402
  /**
226
403
  * The application the active session runs in (sources the server-side app config cascade +
@@ -254,8 +431,15 @@ export class RealtimeSessionRuntime {
254
431
  this.recordingStartedAtIso = null;
255
432
  /** Interval that flushes ~15s crash-recovery shards to the server during a recording. */
256
433
  this.segmentTimer = null;
257
- /** 0-based index of the next recording shard to upload. */
258
- this.segmentIndex = 0;
434
+ /**
435
+ * Owns shard indexing and retry for the current recording: it assigns each shard its index and
436
+ * keeps failed shards to resend under that same index. A fresh queue per recording keeps a flush
437
+ * still in flight from a previous session off this session's indexes. Null when not recording.
438
+ * A recording that ends discards its queue, and any retained shard with it: the consolidated
439
+ * end-of-call upload supersedes them, and if that upload is lost too, server-side recovery
440
+ * (#5196) fills the gap with silence.
441
+ */
442
+ this.shardQueue = null;
259
443
  // ── Server-side liveness ───────────────────────────────────────────────────
260
444
  /**
261
445
  * Interval that tells the server this session is still in use, or `null` when no session is
@@ -310,6 +494,11 @@ export class RealtimeSessionRuntime {
310
494
  this.pendingUsageInput = 0;
311
495
  /** Accumulated output-token delta since the last flush. */
312
496
  this.pendingUsageOutput = 0;
497
+ /**
498
+ * Accumulated per-modality usage since the last flush (input and output detail blocks, avatar
499
+ * video seconds included), relayed in full; `null` when no update carried any.
500
+ */
501
+ this.pendingUsageDetails = null;
313
502
  /** Pending debounced usage flush; also force-flushed at teardown. */
314
503
  this.usageFlushTimer = null;
315
504
  /** Active push-status subscription that feeds delegation progress; cleared on teardown. */
@@ -352,7 +541,20 @@ export class RealtimeSessionRuntime {
352
541
  */
353
542
  this.teardownInFlight = null;
354
543
  /**
355
- * Why the last session start failed, or `null` when none has.
544
+ * The latest start's opening of the host's microphone, until {@link IRealtimeMediaHost.ReleaseMicrophone} hands it
545
+ * back; `null` when no opening is owed a release, so a session that ends before asking for the microphone releases
546
+ * nothing.
547
+ *
548
+ * The host gets one release per opening, after the opening returns. A release while the microphone is still
549
+ * opening undoes nothing: the React Native host puts iOS into the call audio category inside `AcquireMicrophone`,
550
+ * after the permission prompt. So {@link runTeardown} releases an opening that has returned and leaves one still
551
+ * opening to its start, which releases it once it returns ({@link unwindAbandonedStart}), unless a newer start has
552
+ * opened the microphone since; that start's own end releases it. Whoever releases clears this first, so no
553
+ * opening is released twice.
554
+ */
555
+ this.microphoneLease = null;
556
+ /**
557
+ * Why the latest session start failed, or `null` when it has not. Each start clears it ({@link beginSessionStart}).
356
558
  *
357
559
  * The runtime reports failure as `'error'` on {@link ConnectionState$}, which is enough to show
358
560
  * *that* something went wrong but not *what* — and the difference matters at exactly one point:
@@ -361,6 +563,11 @@ export class RealtimeSessionRuntime {
361
563
  * runtime's own try/catch, so the host never sees that rejection itself.
362
564
  */
363
565
  this.lastStartError = null;
566
+ /**
567
+ * The fatal error the call's client reported, or `null` when it has reported none. Each start clears it
568
+ * ({@link beginSessionStart}). See {@link LastSessionError}.
569
+ */
570
+ this.lastSessionError = null;
364
571
  /**
365
572
  * Pending DEBOUNCED channel-state saves, keyed by channel name. Each entry keeps the
366
573
  * LATEST serialized state plus the session id captured while the session was live —
@@ -377,13 +584,92 @@ export class RealtimeSessionRuntime {
377
584
  */
378
585
  this.usedChannelNames = new Set();
379
586
  this._provider = null;
587
+ this._launcher = new DefaultRealtimeSessionLauncher();
588
+ /**
589
+ * Loads channel code on demand. Called, with the `ClientPluginClass` keys of every channel about to be
590
+ * resolved (the registry's rows plus the host's declarations), after the registry is read and before any
591
+ * plugin is built — so a host can register a heavy channel's class only when the session could use it.
592
+ * It must resolve once the classes it knows are registered; a rejection is logged, never fatal.
593
+ */
594
+ this.ChannelClassLoader = null;
595
+ // ── Session events (server → this session's client) ────────────────────────
596
+ this._sessionEvents$ = new Subject();
597
+ /**
598
+ * Every event the server publishes to THIS session, for the session's life (identity verification
599
+ * today; apps add their own types). Hot, with no replay — subscribe before the session starts, once,
600
+ * and it keeps working across sessions. The runtime has already acted on the events it understands
601
+ * (see {@link handleIdentityVerified}) by the time one reaches here; this stream is for the host UI.
602
+ *
603
+ * Silent when the session's provider has no subscription transport (see {@link createSessionEventSource}).
604
+ */
605
+ this.SessionEvents$ = this._sessionEvents$.asObservable();
606
+ /** Keeps the live session's event stream open; null between sessions. */
607
+ this.sessionEventHub = null;
608
+ /** Per-type handlers the runtime runs (before publishing on {@link SessionEvents$}) for events it understands. */
609
+ this.sessionEventHandlers = new Map([
610
+ ['identity.verified', (event) => this.handleIdentityVerified(event)]
611
+ ]);
612
+ /**
613
+ * What the agent should SAY the moment the user's identity is verified, or `null` (the default) to
614
+ * say nothing — the agent is still told, silently, through a context note. A string is used as the
615
+ * instruction verbatim; a function receives the verified payload (e.g. to greet by name) and returns
616
+ * the instruction, or `null` to stay silent for this verification.
617
+ */
618
+ this.IdentityVerifiedSpokenResponse = null;
619
+ this.clientDeadline = new ClientSessionDeadline();
620
+ // ── Interactive channels (registry-driven + host-declared plugins) ─────────
621
+ //
622
+ // A session's channels come from two places — the `MJ: AI Agent Channels` registry and the channels
623
+ // the host declares — and pass through three stages:
624
+ //
625
+ // 1. PREPARE (before mint) construct the plugins, read their descriptors, report them as
626
+ // candidates, and work out the scope the browser can decide alone.
627
+ // 2. MINT the server scopes the candidates with the agent/app cascade and the
628
+ // registry's kill switch, and returns the resolved policy.
629
+ // 3. ACTIVATE (after mint) initialize exactly the policy's channels: `open-on-start`/`headless`
630
+ // ones mount now, `on-demand` ones wait to be opened through ContextTool.
631
+ //
632
+ // Nothing is initialized before the policy is known, so a channel the server vetoes never runs.
633
+ /** Whether the channel catalog note still has to be sent (see {@link flushChannelCatalogNote}). */
634
+ this.catalogNotePending = false;
635
+ /** Plugins resolved for the current start but not yet initialized. Consumed by {@link activateChannels}. */
636
+ this.preparedChannels = [];
637
+ /** The scope the browser resolves on its own — the fallback when no server policy comes back. */
638
+ this.localChannelScope = null;
639
+ /** `on-demand` channels that are in the session but not opened yet (initialized only when opened). */
640
+ this.advertisedChannels = [];
641
+ /** What the policy resolved for each in-session channel, keyed by normalized channel key. */
642
+ this.resolvedChannels = new Map();
643
+ /** The prior session's saved channel states, kept so an `on-demand` channel opened later is restored too. */
644
+ this.priorChannelStates = {};
645
+ /** The app/static client-tool tiers the server resolved for this session. */
646
+ this.sessionClientTools = null;
647
+ /** Validates and routes channel-addressed `ContextTool` calls. */
648
+ this.channelDispatcher = new ChannelActionDispatcher({
649
+ FindChannel: (key) => this.findDispatchableChannel(key),
650
+ ListChannelKeys: () => this.listAddressableChannelKeys(),
651
+ ActivateChannel: (plugin) => this.mountAdvertisedChannel(plugin),
652
+ });
653
+ /** Where the user's per-channel "how much can the agent see" choices are kept. Replace with {@link SetExposurePreferences}. */
654
+ this.exposurePreferences = new UserSettingsExposurePreferences(() => this.canPersistUserSettings());
655
+ /** The channel whose surface the user is looking at, kept so it applies to an arbiter created after it was set. */
656
+ this.focusedChannelKey = null;
657
+ this.stopWatchingVideoSources = null;
380
658
  /**
381
659
  * Host registry of surface CLIENT TOOLS (Name → handler), fed by the host (Explorer) from the
382
- * active surface's `SetAgentClientTools`. The headless ClientContextChannel's `ContextTool` proxy
383
- * executes against this via {@link executeAppClientTool}. Keys are lower-cased for case-insensitive
384
- * model-supplied action names.
660
+ * active surface's `SetAgentClientTools` and from its always-on globals. The headless
661
+ * ClientContextChannel's `ContextTool` proxy executes against this via {@link executeAppClientTool}.
662
+ * Owner-keyed — see {@link AppClientToolRegistry}.
385
663
  */
386
- this.appClientToolHandlers = new Map();
664
+ this.appToolRegistry = new AppClientToolRegistry();
665
+ }
666
+ /** Synchronous read of {@link SavingRecording$}. */
667
+ get IsSavingRecording() {
668
+ return this._savingRecording$.value;
669
+ }
670
+ /** Synchronous access to {@link AvatarNotice$}'s current value. */
671
+ get CurrentAvatarNotice() {
672
+ return this._avatarNotice$.value;
387
673
  }
388
674
  /** Synchronous access to the session's active interactive-channel plugins. */
389
675
  get ActiveChannels() {
@@ -480,10 +766,113 @@ export class RealtimeSessionRuntime {
480
766
  set Provider(value) {
481
767
  this._provider = value;
482
768
  }
769
+ /**
770
+ * How this runtime mints a session — the stock `StartRealtimeClientSession` mutation unless a host
771
+ * installs its own {@link IRealtimeSessionLauncher} (a guest-session exchange, a deployment-specific
772
+ * mutation). Everything after the mint — channel scoping, the driver, transcripts, teardown — is
773
+ * unchanged. Assign `null` to restore the default.
774
+ */
775
+ get Launcher() {
776
+ return this._launcher;
777
+ }
778
+ set Launcher(value) {
779
+ this._launcher = value ?? new DefaultRealtimeSessionLauncher();
780
+ }
781
+ /**
782
+ * The client's copy of the session's absolute deadline (`null` when none is known). A host that
783
+ * knows the session's cap at start sets it with {@link SetSessionDeadline}; verification extends it
784
+ * from the server's `identity.verified` payload. The SERVER enforces the real deadline — this exists
785
+ * so the host can end the call gracefully first and show a countdown. It only ever moves later by
786
+ * the server's word, so nothing a client does can extend the real one.
787
+ */
788
+ get SessionDeadline$() {
789
+ return this.clientDeadline.Deadline$;
790
+ }
791
+ /** Synchronous access to {@link SessionDeadline$}. */
792
+ get SessionDeadline() {
793
+ return this.clientDeadline.Value;
794
+ }
795
+ /** Sets the baseline deadline a host knows at session start (`null` clears). See {@link SessionDeadline$}. */
796
+ SetSessionDeadline(deadline) {
797
+ this.clientDeadline.Set(deadline);
798
+ }
799
+ /**
800
+ * Builds the transport that reads the session's events. The default rides the session's GraphQL
801
+ * provider; it returns `null` — "no session events here" — for a provider with no subscription
802
+ * support (a test double, a host on another transport), which the runtime tolerates. Override to
803
+ * supply another transport.
804
+ */
805
+ createSessionEventSource() {
806
+ const provider = this.Provider;
807
+ if (!provider || typeof provider.Subscribe !== 'function') {
808
+ return null;
809
+ }
810
+ return new GraphQLRealtimeSessionClient(provider);
811
+ }
812
+ /** Opens the live session's event stream. Best-effort: never disturbs the call. */
813
+ startSessionEvents(agentSessionId) {
814
+ this.stopSessionEvents();
815
+ try {
816
+ const source = this.createSessionEventSource();
817
+ if (!source) {
818
+ return;
819
+ }
820
+ this.sessionEventHub = new RealtimeSessionEventHub(source, (event) => this.routeSessionEvent(agentSessionId, event));
821
+ this.sessionEventHub.Start(agentSessionId);
822
+ }
823
+ catch (error) {
824
+ console.error('[RealtimeSession] Could not start the session event stream:', error);
825
+ }
826
+ }
827
+ stopSessionEvents() {
828
+ this.sessionEventHub?.Stop();
829
+ this.sessionEventHub = null;
830
+ }
831
+ /** Routes one event to its per-type handler, then publishes it. An event for another session is dropped. */
832
+ routeSessionEvent(expectedSessionId, event) {
833
+ if (event.AgentSessionID !== expectedSessionId || this.agentSessionId !== expectedSessionId) {
834
+ console.warn(`[RealtimeSession] Dropped a '${event.Type}' event addressed to another session.`);
835
+ return;
836
+ }
837
+ try {
838
+ this.sessionEventHandlers.get(event.Type)?.(event);
839
+ }
840
+ catch (error) {
841
+ console.error(`[RealtimeSession] Handling the '${event.Type}' session event failed:`, error);
842
+ }
843
+ this._sessionEvents$.next(event);
844
+ }
845
+ /**
846
+ * `identity.verified`: keep the client's deadline in step with the server's, tell the model — as a
847
+ * silent, structured note — who it is now talking to, and, when the host configured it, have it say
848
+ * something about that. The payload is validated again here (it crossed a wire); the name rides as
849
+ * JSON so user-entered text can never break out of the note's frame.
850
+ */
851
+ handleIdentityVerified(event) {
852
+ if (event.Type !== 'identity.verified' || !IsIdentityVerifiedEventPayload(event.Payload)) {
853
+ console.warn('[RealtimeSession] Ignored a malformed identity.verified event.');
854
+ return;
855
+ }
856
+ const payload = event.Payload;
857
+ this.clientDeadline.Extend(payload.MaxSessionDeadlineIso);
858
+ this.SendContextNote(`[identity] verified ${JSON.stringify({ email: payload.VerifiedEmail, name: payload.VerifiedName, method: payload.Method })} ` +
859
+ '(background context — the user has proven they control this email address; treat them as verified. Do not read this note aloud.)');
860
+ const spoken = typeof this.IdentityVerifiedSpokenResponse === 'function' ? this.IdentityVerifiedSpokenResponse(payload) : this.IdentityVerifiedSpokenResponse;
861
+ if (spoken && spoken.trim().length > 0 && this.client && this.isSessionLive()) {
862
+ this.requestChannelSpokenResponse(spoken.trim());
863
+ }
864
+ }
483
865
  /** True when a session is currently open. */
484
866
  get IsActive() {
485
867
  return this._active$.value;
486
868
  }
869
+ /**
870
+ * True when a session is open or starting for `conversationId`. Ids are compared without
871
+ * regard to case.
872
+ */
873
+ IsActiveFor(conversationId) {
874
+ return this.IsActive && conversationId != null && UUIDsEqual(this.sessionConversationId, conversationId);
875
+ }
487
876
  /**
488
877
  * Start a client-direct voice session fronting `targetAgentId`.
489
878
  *
@@ -521,31 +910,55 @@ export class RealtimeSessionRuntime {
521
910
  * the server-side Media channel resolves THIS collection as the agent's media kit for the session,
522
911
  * taking precedence over the agent's `DefaultMediaCollectionID`. The server UUID-validates it
523
912
  * (malformed ⇒ ignored, the agent default applies). Omit/`null` to use the agent default kit.
913
+ * @param applicationId Optional application the session runs in (sources the app config cascade,
914
+ * including `Application.AgentSettings.Realtime.Channels`).
915
+ * @param appContext Optional live app-context snapshot injected into the companion prompt at mint.
916
+ * @param options Optional per-start extras — chiefly the channels the HOST brings
917
+ * ({@link RealtimeSessionStartOptions.HostChannels}), the way a connect-only embed with no
918
+ * registry gets channels at all, and the fork the session starts on
919
+ * ({@link RealtimeSessionStartOptions.ConversationBranchID}; omitted is Main, where its transcript is written).
524
920
  */
525
- async StartRealtimeSession(targetAgentId, conversationId, lastSessionId, agentName, preferredModelId, clientTools, coAgentId, configOverridesJson, recordingConsent, mediaCollectionId, applicationId, appContext) {
921
+ async StartRealtimeSession(targetAgentId, conversationId, lastSessionId, agentName, preferredModelId, clientTools, coAgentId, configOverridesJson, recordingConsent, mediaCollectionId, applicationId, appContext, options) {
526
922
  if (this.IsActive) {
527
923
  return; // a session is already running — ignore duplicate starts
528
924
  }
529
925
  const consent = this.beginSessionStart({ agentName, recordingConsent, applicationId, appContext });
530
- // Captured BEFORE startChannels so the mint carries exactly the snapshot the prologue
926
+ this.sessionConversationId = conversationId ?? null;
927
+ // A teardown while the channels are prepared or the mint runs bumps this; see the checks below.
928
+ const generation = this.startGeneration;
929
+ // Captured BEFORE the channels are prepared so the mint carries exactly the snapshot the prologue
531
930
  // resolved, whatever a channel plugin may push in the meantime.
532
931
  const effectiveAppContext = this._appContext$.value;
533
932
  let session;
534
933
  try {
535
- // Resolve + initialize the interactive-channel plugins FIRST: their client-executed
536
- // tool sets must be declared to the realtime model at session mint.
537
- const allClientTools = [...(clientTools ?? []), ...(await this.startChannels())];
538
- session = await this.mintSession(targetAgentId, conversationId, lastSessionId, preferredModelId, allClientTools, coAgentId, configOverridesJson, consent, this.recordingStartedAtIso, mediaCollectionId, this.applicationId, effectiveAppContext);
934
+ // Resolve the interactive-channel plugins FIRST (constructed, not started): the tools of the
935
+ // channels mounted with the session must be declared to the realtime model at mint, and the
936
+ // server needs the candidates to scope them. Nothing is initialized until the policy is known.
937
+ const scope = await this.prepareChannelScope(options?.HostChannels);
938
+ const allClientTools = [...(clientTools ?? []), ...scope.NativeTools];
939
+ session = await this.mintSession(targetAgentId, conversationId, lastSessionId, preferredModelId, allClientTools, coAgentId, configOverridesJson, consent, this.recordingStartedAtIso, mediaCollectionId, this.applicationId, effectiveAppContext, scope.CandidatesJson, scope.ShowsAgentVideo, options?.ConversationBranchID ?? null);
539
940
  }
540
941
  catch (error) {
942
+ // An ended or replaced start only logs its failure: the error state and the teardown belong
943
+ // to the session that is current now, which a newer start may own.
944
+ if (this.startGeneration !== generation) {
945
+ console.error('[RealtimeSession] A start that was already ended failed:', error);
946
+ return;
947
+ }
541
948
  await this.failSessionStart(error);
542
949
  return;
543
950
  }
951
+ // The host ended the start while the mint ran. That teardown had no session id to close, so
952
+ // close the minted row here, and touch no shared state: a newer start may already own it.
953
+ if (this.startGeneration !== generation) {
954
+ await this.closeServerSession(session.AgentSessionId);
955
+ return;
956
+ }
544
957
  if (!this.hostCanUseProvider(session.Provider)) {
545
958
  await this.abortUnusableSession(session);
546
959
  return;
547
960
  }
548
- await this.runMintedSession(session, conversationId ?? null, consent);
961
+ await this.runMintedSession(session, conversationId ?? null, consent, options?.CameraCheck === true);
549
962
  }
550
963
  /**
551
964
  * Declares which provider keys this host can actually carry audio for.
@@ -581,7 +994,7 @@ export class RealtimeSessionRuntime {
581
994
  // published them on ActiveChannels$; skipping teardown would leave them undisposed, their tool
582
995
  // handlers registered, and their subscriptions live until the next start replaced them.
583
996
  this.agentSessionId = session.AgentSessionId ?? this.agentSessionId;
584
- this._connectionState$.next('error');
997
+ this.setConnectionState('error');
585
998
  await this.teardown(true);
586
999
  }
587
1000
  /**
@@ -600,7 +1013,8 @@ export class RealtimeSessionRuntime {
600
1013
  * owns that half too.
601
1014
  *
602
1015
  * @param result The minted session — the same ten fields the `StartRealtimeClientSession`
603
- * mutation returns. `EphemeralToken` and `Provider` are what actually open the call.
1016
+ * mutation returns. `EphemeralToken` and `Provider` are what actually open the call; a relay
1017
+ * session opens with its `Transport` and `RelayUrl` instead, so a host must pass those on.
604
1018
  * @param options Host-side inputs the result cannot carry; see {@link RealtimeSessionRunOptions}.
605
1019
  * Every field defaults exactly as its {@link StartRealtimeSession} counterpart does.
606
1020
  */
@@ -614,16 +1028,20 @@ export class RealtimeSessionRuntime {
614
1028
  return;
615
1029
  }
616
1030
  const consent = this.beginSessionStart(effectiveOptions);
617
- await this.runMintedSession(result, effectiveOptions.conversationId ?? null, consent);
1031
+ await this.runMintedSession(result, effectiveOptions.conversationId ?? null, consent, false);
618
1032
  }
619
1033
  /**
620
- * Start prologue shared by both entry points: bind the app layer, publish the agent name, reset
621
- * per-session state, and flip the session live (which is ALSO what makes the `IsActive` guard
622
- * suppress duplicate starts while the mint is still in flight — hence it runs before minting, not
623
- * after). Returns the resolved recording consent, which the mint half reports to the server and
624
- * the run half uses to decide whether to record.
1034
+ * Start prologue shared by both entry points: clear the last start's failure, bind the app layer,
1035
+ * publish the agent name, reset per-session state, and flip the session live (which is ALSO what
1036
+ * makes the `IsActive` guard suppress duplicate starts while the mint is still in flight — hence it
1037
+ * runs before minting, not after). Returns the resolved recording consent, which the mint half
1038
+ * reports to the server and the run half uses to decide whether to record.
625
1039
  */
626
1040
  beginSessionStart(options) {
1041
+ // Before this start publishes anything: hosts read LastStartError as this start's states arrive,
1042
+ // so from the first of them it must describe this start, not an earlier one.
1043
+ this.lastStartError = null;
1044
+ this.lastSessionError = null;
627
1045
  // App awareness (Move 1/3/4): the application the session runs in (sources the app config
628
1046
  // cascade + RelevantAgents → allowed-agent union) and the live app-context snapshot injected
629
1047
  // into the companion prompt at mint. Stored so the ClientContextChannel can stream subsequent
@@ -641,7 +1059,7 @@ export class RealtimeSessionRuntime {
641
1059
  }
642
1060
  this.resetState();
643
1061
  this._active$.next(true);
644
- this._connectionState$.next('connecting');
1062
+ this.setConnectionState('connecting');
645
1063
  // Resolve recording consent for this session: explicit value wins, else the per-user
646
1064
  // persisted preference. Computed before mint so it can be reported to the server.
647
1065
  const consent = options.recordingConsent ?? this.readPersistedRecordingConsent();
@@ -651,12 +1069,18 @@ export class RealtimeSessionRuntime {
651
1069
  /**
652
1070
  * The RUN half of a session start, shared by both entry points: consume the minted result, open
653
1071
  * the provider connection, and go live. `inputConversationId` is the conversation the START asked
654
- * for (null ⇒ "server, make me one") — the result alone can't distinguish the two.
1072
+ * for (null ⇒ "server, make me one") — the result alone can't distinguish the two. `cameraCheck` is
1073
+ * whether the host shows a camera check ({@link RealtimeSessionStartOptions.CameraCheck}).
655
1074
  */
656
- async runMintedSession(session, inputConversationId, consent) {
1075
+ async runMintedSession(session, inputConversationId, consent, cameraCheck) {
657
1076
  // Captured up front: every await below is a window in which the host can end the session.
658
1077
  const generation = this.startGeneration;
659
1078
  try {
1079
+ // Mount the channels the resolved policy puts in the session. Deliberately BEFORE the session
1080
+ // id is adopted: `ActiveChannels$` consumers (the sessions adapter) treat an emission that
1081
+ // arrives while no session id is set as the INITIAL set, whose opens they synthesize from
1082
+ // `SessionStarted$` — activating after the id is set would announce every channel twice.
1083
+ this.applySessionClientPolicy(session);
660
1084
  this.agentSessionId = session.AgentSessionId;
661
1085
  // A null input conversationId means the SERVER created a fresh conversation for
662
1086
  // this session — track it so the host can fold it into the cached list, select
@@ -671,20 +1095,39 @@ export class RealtimeSessionRuntime {
671
1095
  this.applyPriorChannelStates(session.PriorChannelStatesJson);
672
1096
  const client = this.createRealtimeClient(session.Provider);
673
1097
  this.client = client;
1098
+ this.watchVideoSources(client);
674
1099
  this.wireClientHandlers(client);
675
1100
  // Everything past here awaits on hardware and the network, during which the host may end the
676
- // session. Each await is followed by a staleness check so an abandoned start releases what it
677
- // just acquired instead of leaving a live microphone and a live call behind it.
678
- this.localStream = await this.mediaHost.AcquireMicrophone();
1101
+ // session. Each await is followed by a staleness check, before any failure is reported, so an
1102
+ // abandoned start releases what it just acquired instead of leaving a live microphone and a live
1103
+ // call behind it, and reports nothing to a host that has ended the call.
1104
+ const lease = { Opening: true };
1105
+ this.microphoneLease = lease;
1106
+ const microphone = await this.openMicrophone().finally(() => {
1107
+ lease.Opening = false;
1108
+ });
1109
+ this.localStream = microphone.Stream;
679
1110
  if (this.startGeneration !== generation) {
680
- await this.unwindAbandonedStart(session, client);
1111
+ await this.unwindAbandonedStart(session, client, lease);
681
1112
  return;
682
1113
  }
683
- await client.Connect(this.BuildClientConfig(session), this.localStream);
1114
+ if (microphone.Error) {
1115
+ throw microphone.Error;
1116
+ }
1117
+ const clientConfig = this.BuildClientConfig(session);
1118
+ const connectError = await this.connectClient(client, clientConfig, microphone.Stream);
684
1119
  if (this.startGeneration !== generation) {
685
- await this.unwindAbandonedStart(session, client);
1120
+ await this.unwindAbandonedStart(session, client, lease);
686
1121
  return;
687
1122
  }
1123
+ if (connectError) {
1124
+ throw connectError;
1125
+ }
1126
+ // Tracks are negotiated now, so a capture can tell whether the model takes video, and the call whether it shows
1127
+ // the avatar its agent asked for. The driver holds the microphone now, so the user may switch it.
1128
+ this.openCaptures(client, cameraCheck);
1129
+ this.offerMicrophone();
1130
+ this.publishAvatarNotice(session, clientConfig, client);
688
1131
  // Notify active channels that the session client is connected and tracks are established
689
1132
  for (const channel of this._activeChannels$.value) {
690
1133
  try {
@@ -694,6 +1137,10 @@ export class RealtimeSessionRuntime {
694
1137
  console.error(`[RealtimeSession] Error in channel '${channel.ChannelName}' OnSessionStarted:`, err);
695
1138
  }
696
1139
  }
1140
+ // Tell the model which channels exist and how to use them, from the channels' own descriptors —
1141
+ // as soon as the control channel is usable (which may be right now, or a moment after Connect).
1142
+ this.catalogNotePending = true;
1143
+ this.flushChannelCatalogNote();
697
1144
  // Start browser-side recording (mic + agent mix) when consented. Best-effort: an
698
1145
  // unsupported browser / missing remote stream degrades gracefully (mic-only or off)
699
1146
  // and never blocks the call. The remote stream may still be null here (the WebRTC
@@ -718,6 +1165,8 @@ export class RealtimeSessionRuntime {
718
1165
  // SessionsObserver bridge. Emitting AFTER Connect() guarantees both that
719
1166
  // agentSessionId is set (line ~468) AND the realtime client is connected,
720
1167
  // so consumers can act on it without re-checking either condition.
1168
+ // The session's own event stream (identity verification, app events) stays open for its life.
1169
+ this.startSessionEvents(this.agentSessionId);
721
1170
  this._sessionStarted$.next({
722
1171
  sessionId: this.agentSessionId,
723
1172
  channelNames: this._activeChannels$.value.map(c => c.ChannelName),
@@ -730,18 +1179,42 @@ export class RealtimeSessionRuntime {
730
1179
  await this.failSessionStart(error);
731
1180
  }
732
1181
  }
1182
+ /**
1183
+ * Tells the host, once per call, why it shows no avatar ({@link AvatarNotice$}): from the mint's status, whether this
1184
+ * session asked for the agent's video, and whether that track is live now that the call is connected.
1185
+ */
1186
+ publishAvatarNotice(session, config, client) {
1187
+ const notice = ResolveAvatarNotice(ParseRealtimeAvatarStatus(session.AvatarStatusJson), RequestsAgentVideo(config), client.IsTrackEstablished('video', 'outbound'));
1188
+ if (notice) {
1189
+ this._avatarNotice$.next(notice);
1190
+ }
1191
+ }
733
1192
  /**
734
1193
  * Releases everything a start acquired after the host had already ended the session.
735
1194
  *
736
1195
  * Reached only when {@link teardown} ran while this start was awaiting the microphone or the
737
- * provider connection. Teardown found nothing to release because nothing existed yet, so this
738
- * start owns the cleanup: stop the microphone, close the provider connection, and close the
739
- * server-side session row that the mint created.
1196
+ * provider connection, whether that wait then succeeded or failed: the host has ended the call,
1197
+ * so a failure there is not reported. Teardown could not release what did not exist yet, so this
1198
+ * start owns the cleanup: stop the microphone, hand it back to the host when teardown left that
1199
+ * to it ({@link microphoneLease}), close the provider connection, and close the server-side
1200
+ * session row that the mint created when no teardown has closed it or is closing it.
1201
+ *
1202
+ * @param lease This start's opening of the microphone.
740
1203
  */
741
- async unwindAbandonedStart(session, client) {
1204
+ async unwindAbandonedStart(session, client, lease) {
742
1205
  console.warn('[RealtimeSession] Session was ended while starting — releasing the partial session.');
1206
+ this.closeCaptures();
1207
+ this.closeLocalMedia();
743
1208
  this.localStream?.getTracks().forEach(t => t.stop());
744
1209
  this.localStream = null;
1210
+ this.microphoneMuted = false;
1211
+ // Teardown left the release to this start when the microphone was still opening. A cleared or
1212
+ // replaced lease means teardown already released it, or a newer start has opened the microphone
1213
+ // since and its own end releases it.
1214
+ if (this.microphoneLease === lease) {
1215
+ this.microphoneLease = null;
1216
+ await this.releaseMicrophone();
1217
+ }
745
1218
  try {
746
1219
  await client.Disconnect();
747
1220
  }
@@ -751,27 +1224,321 @@ export class RealtimeSessionRuntime {
751
1224
  // Only clear the shared slots when they still point at THIS attempt — a newer start may
752
1225
  // already have replaced them.
753
1226
  if (this.client === client) {
1227
+ this.unwatchVideoSources();
1228
+ this.clearAgentVideo();
754
1229
  this.client = null;
755
1230
  }
756
- // Close the server session only if teardown has NOT already done so. It nulls `agentSessionId`
757
- // after closing, so a still-matching id means this attempt still owns the row; a cleared one
758
- // means the teardown that invalidated this start already closed it, and closing again would
759
- // send a second `CloseAgentSession` for one session.
760
- if (session.AgentSessionId && this.agentSessionId === session.AgentSessionId) {
1231
+ // Close the server session only if no teardown has closed it or is closing it. Teardown nulls
1232
+ // `agentSessionId` after closing, so a cleared id means the teardown that invalidated this start
1233
+ // already closed it. A teardown still running closes it itself and then reports the end on
1234
+ // `SessionEnded$`: closing it here as well would send a second `CloseAgentSession` for one
1235
+ // session, and clearing the id would leave that teardown no session to report.
1236
+ if (session.AgentSessionId && this.agentSessionId === session.AgentSessionId && !this.teardownInFlight) {
761
1237
  this.agentSessionId = null;
762
1238
  await this.closeServerSession(session.AgentSessionId);
763
1239
  }
764
1240
  }
765
1241
  /**
766
- * Why the last session start failed, or `null` when the last start succeeded or none has run.
1242
+ * Opens the user's microphone: through the host's controller when it offers one, otherwise through
1243
+ * {@link IRealtimeMediaHost.AcquireMicrophone}. A failure is returned rather than thrown, so the caller can
1244
+ * first check whether the start was abandoned meanwhile (a teardown disposes the controller, which fails a
1245
+ * start still in flight) and unwind quietly instead of reporting an error nobody is waiting for.
1246
+ */
1247
+ async openMicrophone() {
1248
+ const controller = this.mediaHost.CreateLocalMediaController?.() ?? null;
1249
+ if (!controller) {
1250
+ try {
1251
+ return { Stream: await this.mediaHost.AcquireMicrophone(), Error: null };
1252
+ }
1253
+ catch (error) {
1254
+ return { Stream: null, Error: error instanceof Error ? error : new Error(String(error)) };
1255
+ }
1256
+ }
1257
+ this.localMedia = controller;
1258
+ const started = await controller.Start('microphone');
1259
+ if (started.Status === 'failed') {
1260
+ return { Stream: null, Error: microphoneStartError(started.Reason, started.Message) };
1261
+ }
1262
+ this.followMicrophone(controller, started.Stream);
1263
+ return { Stream: started.Stream, Error: null };
1264
+ }
1265
+ /**
1266
+ * Opens the provider connection. As with {@link openMicrophone}, a failure is returned rather than thrown, so the
1267
+ * caller can first check whether the start was abandoned meanwhile and unwind quietly instead of reporting an error
1268
+ * nobody is waiting for: a teardown disconnects the client, which with most drivers fails a connection still opening.
1269
+ */
1270
+ async connectClient(client, config, microphone) {
1271
+ try {
1272
+ await client.Connect(config, microphone);
1273
+ return null;
1274
+ }
1275
+ catch (error) {
1276
+ return error instanceof Error ? error : new Error(String(error));
1277
+ }
1278
+ }
1279
+ /**
1280
+ * Moves the driver and the recorder onto the call's microphone whenever the controller gives it a new track. A device
1281
+ * switch, or a lost device replaced by the default, swaps a track into the same stream, and the controller carries the
1282
+ * old track's mute over. A microphone opened again after the call lost one comes in a new stream: its track takes the
1283
+ * call's mute, and the stream becomes the one {@link ToggleMute} and the teardown act on. Mid-swap the stream holds no
1284
+ * track, so only a finished swap is followed. Every report also keeps {@link Microphone$} current once the call offers
1285
+ * it.
1286
+ */
1287
+ followMicrophone(controller, stream) {
1288
+ let followedStream = stream;
1289
+ let followedTrack = stream.getAudioTracks()[0] ?? null;
1290
+ this.localMediaSubscription = controller.State$.subscribe((media) => {
1291
+ const current = controller.GetStream('microphone') ?? followedStream;
1292
+ const track = current.getAudioTracks()[0] ?? null;
1293
+ if (track && track !== followedTrack) {
1294
+ if (current !== followedStream) {
1295
+ track.enabled = !this.microphoneMuted;
1296
+ this.localStream = current;
1297
+ followedStream = current;
1298
+ }
1299
+ followedTrack = track;
1300
+ this.recorder?.ReplaceMicrophone?.(current);
1301
+ this.client?.ReplaceMicrophone?.(current)?.catch((error) => {
1302
+ console.error('[RealtimeSession] The realtime driver could not move to the new microphone:', error);
1303
+ });
1304
+ }
1305
+ this.publishMicrophone(media);
1306
+ });
1307
+ }
1308
+ /** The client is connected: the call offers its microphone on {@link Microphone$}, and {@link SwitchMicrophone} moves it. */
1309
+ offerMicrophone() {
1310
+ if (!this.localMedia) {
1311
+ return;
1312
+ }
1313
+ this.microphoneOffered = true;
1314
+ this.publishMicrophone(this.localMedia.State);
1315
+ }
1316
+ /** Publishes the call's microphone from the controller's state while the call offers it, when it names a change. */
1317
+ publishMicrophone(media) {
1318
+ if (!this.microphoneOffered) {
1319
+ return;
1320
+ }
1321
+ const current = this._microphone$.value;
1322
+ const next = ReadMicrophoneState(media, current);
1323
+ if (!SameMicrophoneState(next, current)) {
1324
+ this._microphone$.next(next);
1325
+ }
1326
+ }
1327
+ /** Stops following the controller and disposes it, which releases its devices; the call no longer offers a microphone. */
1328
+ closeLocalMedia() {
1329
+ this.localMediaSubscription?.unsubscribe();
1330
+ this.localMediaSubscription = null;
1331
+ this.localMedia?.Dispose();
1332
+ this.localMedia = null;
1333
+ this.microphoneOffered = false;
1334
+ if (this._microphone$.value !== REALTIME_MICROPHONE_NONE) {
1335
+ this._microphone$.next(REALTIME_MICROPHONE_NONE);
1336
+ }
1337
+ }
1338
+ /**
1339
+ * Creates the session's camera and screen share and mirrors their state on {@link Captures$}. With `cameraCheck`, the
1340
+ * call's first camera start waits for {@link ConfirmCamera}.
1341
+ */
1342
+ openCaptures(client, cameraCheck) {
1343
+ this.closeCaptures();
1344
+ const captures = new RealtimeCaptures({
1345
+ Client: client,
1346
+ LocalMedia: this.localMedia,
1347
+ Host: this.mediaHost,
1348
+ CreateSampler: this.hostFrameSampler(),
1349
+ Admit: (kind) => this.admitCapture(kind),
1350
+ CameraCheck: cameraCheck,
1351
+ });
1352
+ this.captures = captures;
1353
+ this.capturesSubscription = captures.States$.subscribe((states) => this._captures$.next(states));
1354
+ this.refreshCaptureOffers();
1355
+ }
1356
+ /**
1357
+ * The host's frame sampler for the captures ({@link IRealtimeMediaHost.CreateFrameSampler}), or `undefined` when it offers
1358
+ * none and the captures use the DOM default.
1359
+ */
1360
+ hostFrameSampler() {
1361
+ const host = this.mediaHost;
1362
+ const create = host.CreateFrameSampler;
1363
+ return create ? (stream, options) => create.call(host, stream, options) : undefined;
1364
+ }
1365
+ /** Stops the camera and screen share, if any, and reports both off. */
1366
+ closeCaptures() {
1367
+ this.capturesSubscription?.unsubscribe();
1368
+ this.capturesSubscription = null;
1369
+ this.captures?.Dispose();
1370
+ this.captures = null;
1371
+ if (this._captures$.value !== REALTIME_CAPTURES_OFF) {
1372
+ this._captures$.next(REALTIME_CAPTURES_OFF);
1373
+ }
1374
+ this.refreshCaptureOffers();
1375
+ }
1376
+ /** Works out which captures the call offers ({@link CaptureOffers$}) and publishes a change. */
1377
+ refreshCaptureOffers() {
1378
+ const offers = this.captures && this.client?.SupportsInboundVideo ? { Camera: this.canOffer('camera'), Screen: this.canOffer('screen') } : REALTIME_CAPTURE_OFFERS_NONE;
1379
+ const current = this._captureOffers$.value;
1380
+ if (offers.Camera !== current.Camera || offers.Screen !== current.Screen) {
1381
+ this._captureOffers$.next(offers);
1382
+ }
1383
+ }
1384
+ /** Whether the host can open a capture and the call's policy admits it. */
1385
+ canOffer(kind) {
1386
+ const hostCanOpen = kind === 'camera' ? this.localMedia !== null : typeof this.mediaHost.RequestDisplayCapture === 'function';
1387
+ return hostCanOpen && this.admitCapture(kind).Admitted;
1388
+ }
1389
+ /** What a capture reports when there is no live session to show it to. */
1390
+ noSessionCapture() {
1391
+ return { Status: 'failed', Failure: 'no-session', Message: 'There is no call to share with.' };
1392
+ }
1393
+ /**
1394
+ * The session's policy for a capture. The channel that fronts it (its `CaptureKind`) must be in the session, open or
1395
+ * advertised, and the server's policy must let the agent see pixels through it (the agent's configuration and any
1396
+ * zero-data-retention rule). The user's own "agent can see" choice for the channel does not refuse a start: it decides
1397
+ * whether the frames reach the model.
1398
+ */
1399
+ admitCapture(kind) {
1400
+ const channel = this.findCaptureChannel(kind)?.Plugin;
1401
+ if (!channel) {
1402
+ return { Admitted: false, Message: CAPTURE_WORDING[kind].NotInCall };
1403
+ }
1404
+ const resolved = this.GetResolvedChannel(channel.ChannelName);
1405
+ const allowed = resolved?.Exposure ?? resolved?.MaxExposure ?? channel.GetDescriptor().MaxExposure;
1406
+ if (allowed !== 'pixels') {
1407
+ const reasons = (resolved?.ExposureLimits ?? []).map((limit) => limit.Reason);
1408
+ return { Admitted: false, Message: reasons.length > 0 ? `${CAPTURE_WORDING[kind].NotAllowed}: ${reasons.join('; ')}.` : `${CAPTURE_WORDING[kind].NotAllowed}.` };
1409
+ }
1410
+ return { Admitted: true, ChannelKey: channel.ChannelName, VisibleToAgent: channel.Exposure === 'pixels' };
1411
+ }
1412
+ /** The channel in the session, open or advertised, that fronts a capture. */
1413
+ findCaptureChannel(kind) {
1414
+ const open = this._activeChannels$.value.find((c) => c.CaptureKind === kind);
1415
+ if (open) {
1416
+ return { Plugin: open, IsOpen: true };
1417
+ }
1418
+ const advertised = this.advertisedChannels.find((p) => p.Plugin.CaptureKind === kind);
1419
+ return advertised ? { Plugin: advertised.Plugin, IsOpen: false } : null;
1420
+ }
1421
+ /**
1422
+ * Brings up the channel that fronts a capture once the capture is starting, so the host shows its surface: an advertised
1423
+ * channel is opened (and announces itself to the agent), an open one counts as used.
1424
+ */
1425
+ async revealCaptureChannel(kind) {
1426
+ const state = kind === 'camera' ? this.captures?.States.Camera : this.captures?.States.Screen;
1427
+ const channel = state?.Status === 'starting' ? this.findCaptureChannel(kind) : null;
1428
+ if (!channel) {
1429
+ return;
1430
+ }
1431
+ if (channel.IsOpen) {
1432
+ this.noteChannelActivity(channel.Plugin);
1433
+ return;
1434
+ }
1435
+ const opened = await this.OpenChannel(channel.Plugin.ChannelName);
1436
+ if (!opened.Success) {
1437
+ console.warn(`[RealtimeSession] Could not open channel '${channel.Plugin.ChannelName}' for the ${kind}: ${opened.ErrorMessage ?? opened.ErrorCode}`);
1438
+ }
1439
+ }
1440
+ /**
1441
+ * Starts the user's camera and shows it to the agent. Call it from the user's click: it may ask for camera
1442
+ * permission. Resolves with the camera's state; a failure (no session, a model that takes no video, a refused
1443
+ * permission) is a state with a message, never a throw. While the camera is on, it is a source on
1444
+ * {@link VideoSources$}, and it stops by itself if the device goes away.
1445
+ *
1446
+ * When the host shows a camera check ({@link RealtimeSessionStartOptions.CameraCheck}), the call's first start
1447
+ * resolves still starting, with `Checking` set and the stream for the host to preview. The agent sees the camera
1448
+ * once the host calls {@link ConfirmCamera}; {@link StopCamera} is the user's "not now".
1449
+ *
1450
+ * @param deviceId The camera to open; the system default when absent.
1451
+ */
1452
+ async StartCamera(deviceId) {
1453
+ if (!this.captures) {
1454
+ return this.noSessionCapture();
1455
+ }
1456
+ const started = this.captures.Start('camera', { DeviceID: deviceId });
1457
+ await this.revealCaptureChannel('camera');
1458
+ return started;
1459
+ }
1460
+ /** Stops the user's camera. Safe to call when it is off. */
1461
+ StopCamera() {
1462
+ this.captures?.Stop('camera');
1463
+ }
1464
+ /**
1465
+ * Moves the user's camera to another device while it is open (being checked or on): one of the camera's `Devices` on
1466
+ * {@link Captures$}. The stream stays the same. Resolves with the camera's state; nothing changes while the camera is
1467
+ * off.
1468
+ *
1469
+ * @param deviceId The camera to move to.
1470
+ */
1471
+ async SwitchCamera(deviceId) {
1472
+ return this.captures ? this.captures.SwitchCamera(deviceId) : this.noSessionCapture();
1473
+ }
1474
+ /**
1475
+ * Moves the call's microphone to another device: one of the `Devices` on {@link Microphone$}. The host's controller
1476
+ * swaps the new track into the same stream and carries the mute over, and the driver and the call's recording move to
1477
+ * it, so the agent keeps hearing the user. When the new microphone cannot open, the controller goes back to the one in
1478
+ * use. While the call has no microphone open (it lost the one in use and the default, or the last one went away), the
1479
+ * pick opens one again: the driver and the recording move to it, and it keeps the call's mute. Resolves with the
1480
+ * microphone's state once the switch is over. Nothing changes outside a call or on a host without a controller.
1481
+ *
1482
+ * @param deviceId The microphone to move to.
1483
+ */
1484
+ async SwitchMicrophone(deviceId) {
1485
+ const controller = this.localMedia;
1486
+ if (controller && this.microphoneOffered) {
1487
+ const status = controller.State.Microphone.Status;
1488
+ const live = status === 'on' || status === 'starting';
1489
+ await (live ? controller.SwitchDevice('microphone', deviceId) : controller.Start('microphone', deviceId));
1490
+ }
1491
+ return this._microphone$.value;
1492
+ }
1493
+ /**
1494
+ * The user checked the camera and turned it on: the agent sees it from now on, and later camera starts in this call
1495
+ * skip the check. Returns the camera's state; unless the camera is waiting for its check (see {@link StartCamera}),
1496
+ * nothing changes.
1497
+ */
1498
+ ConfirmCamera() {
1499
+ return this.captures ? this.captures.ConfirmCamera() : this.noSessionCapture();
1500
+ }
1501
+ /**
1502
+ * Asks the user for a screen, window or browser tab (or one panel of the page) and shows it to the agent. Call it
1503
+ * from the user's click: the browser's picker needs one. Resolves with the share's state; a failure (no session,
1504
+ * a model that takes no video, a closed picker, a host that cannot share) is a state with a message. The share
1505
+ * also stops when the user ends it from the browser's own bar.
1506
+ *
1507
+ * @param options What the picker offers first, or the panel to share and its name (`PanelLabel`), which the share's
1508
+ * state carries and the agent sees as "<name> (shared panel)".
1509
+ */
1510
+ async StartScreenShare(options) {
1511
+ if (!this.captures) {
1512
+ return this.noSessionCapture();
1513
+ }
1514
+ const started = this.captures.Start('screen', options);
1515
+ await this.revealCaptureChannel('screen');
1516
+ return started;
1517
+ }
1518
+ /** Stops the user's screen share. Safe to call when nothing is shared. */
1519
+ StopScreenShare() {
1520
+ this.captures?.Stop('screen');
1521
+ }
1522
+ /**
1523
+ * Why the latest session start failed, or `null` when it has not failed or none has run.
767
1524
  *
768
1525
  * Read it when {@link ConnectionState$} reports `'error'`, to tell a denied microphone apart from
769
- * a provider or backend failure and show copy the user can act on. Cleared at the start of every
770
- * session.
1526
+ * a provider or backend failure and show copy the user can act on. Each start clears it before it
1527
+ * reports `'connecting'`, so it never describes an earlier start.
771
1528
  */
772
1529
  get LastStartError() {
773
1530
  return this.lastStartError;
774
1531
  }
1532
+ /**
1533
+ * The fatal error the call's client reported, or `null` when it has reported none.
1534
+ *
1535
+ * Read it when {@link ConnectionState$} reports `'error'` on a call that was live, to tell a call that reached its time
1536
+ * limit (`Code` `'time-limit'`; {@link DescribeCallEnd} words it) from a dropped connection. Each start clears it before
1537
+ * it reports `'connecting'`.
1538
+ */
1539
+ get LastSessionError() {
1540
+ return this.lastSessionError;
1541
+ }
775
1542
  /**
776
1543
  * The single failure path for a session start (mint half or run half): report it, latch the
777
1544
  * overlay into 'error', and unwind whatever the half-built session already opened.
@@ -779,7 +1546,7 @@ export class RealtimeSessionRuntime {
779
1546
  async failSessionStart(error) {
780
1547
  console.error('[RealtimeSession] Failed to start session:', error);
781
1548
  this.lastStartError = error instanceof Error ? error : new Error(String(error));
782
- this._connectionState$.next('error');
1549
+ this.setConnectionState('error');
783
1550
  await this.teardown(false);
784
1551
  }
785
1552
  /**
@@ -818,14 +1585,19 @@ export class RealtimeSessionRuntime {
818
1585
  // Relay as a user turn — same path spoken input uses (caption + persisted transcript).
819
1586
  void this.onUserTranscript(trimmed);
820
1587
  }
821
- /** Mute / unmute the local microphone track. Returns the new muted state. */
1588
+ /**
1589
+ * Mute / unmute the local microphone track. Returns the new muted state. While the call has no microphone track (a
1590
+ * switch is putting a new one in, or the call lost its microphone) there is nothing to mute: nothing changes, and it
1591
+ * returns the call's mute, which the next microphone keeps. Outside a call it returns `false`.
1592
+ */
822
1593
  ToggleMute() {
823
1594
  const tracks = this.localStream?.getAudioTracks() ?? [];
824
1595
  if (tracks.length === 0) {
825
- return false;
1596
+ return this.microphoneMuted;
826
1597
  }
827
1598
  const muted = tracks[0].enabled; // currently enabled → becomes muted
828
1599
  this.client?.SetMuted(muted);
1600
+ this.microphoneMuted = muted;
829
1601
  return muted;
830
1602
  }
831
1603
  // ── Client-executed UI tools ───────────────────────────────────────────────
@@ -895,13 +1667,21 @@ export class RealtimeSessionRuntime {
895
1667
  return this.client;
896
1668
  }
897
1669
  /**
898
- * Relays a video frame to the underlying realtime client if active.
1670
+ * Relays a video frame straight to the underlying realtime client, if a session is live.
1671
+ *
1672
+ * It bypasses the session's {@link VideoSourceArbiter}: the frame overrides the source the arbiter picked, and the
1673
+ * model is not told what it is looking at. Channels no longer send through it (their context's `SendVideoFrame` goes
1674
+ * through the arbiter); a host that shows the model something registers a source there instead, as
1675
+ * `ChannelInboundVideoBridge` does.
1676
+ *
1677
+ * @returns `true` when the client accepted and sent the frame; `false` when no session is live, the
1678
+ * client cannot take video frames, or the client dropped the frame.
899
1679
  */
900
1680
  SendVideoFrame(base64Image, mimeType) {
901
1681
  if (!this.client || !this.isSessionLive()) {
902
- return;
1682
+ return false;
903
1683
  }
904
- this.client.SendVideoFrame?.(base64Image, mimeType);
1684
+ return this.client.SendVideoFrame?.(base64Image, mimeType) === true;
905
1685
  }
906
1686
  /**
907
1687
  * Checks whether a media track is established on the active realtime client.
@@ -949,9 +1729,11 @@ export class RealtimeSessionRuntime {
949
1729
  this.currentTurnStartMs = recorder.IsRecording ? 0 : null;
950
1730
  this.turnAudioStartCaptured = false;
951
1731
  if (this.recorder) {
952
- // The agent's WebRTC audio track usually lands AFTER Connect() resolves, so `remoteStream`
953
- // above is typically null and we'd capture mic-only. Attach the agent stream whenever it
954
- // arrives (fires immediately if already present) so the recording includes the agent voice.
1732
+ // WebRTC drivers (OpenAI): the agent's track usually lands AFTER Connect() resolves, so
1733
+ // `remoteStream` above is null here and this handler attaches it later. PCM-playback
1734
+ // drivers (Gemini, ElevenLabs, AssemblyAI, xAI, HuggingFace) publish at Connect, so
1735
+ // `remoteStream` is already set and the handler fires immediately with the same stream;
1736
+ // AttachRemoteStream is idempotent, so it is mixed only once.
955
1737
  client.OnRemoteMediaStream?.((stream) => this.recorder?.AttachRemoteStream(stream));
956
1738
  this.startSegmentFlushing();
957
1739
  }
@@ -963,7 +1745,7 @@ export class RealtimeSessionRuntime {
963
1745
  }
964
1746
  /** Begins flushing ~15s crash-recovery shards to the server for the duration of the recording. */
965
1747
  startSegmentFlushing() {
966
- this.segmentIndex = 0;
1748
+ this.shardQueue = new RecordingShardQueue();
967
1749
  this.segmentTimer = setInterval(() => { void this.flushRecordingSegment(); }, RealtimeSessionRuntime.segmentFlushMs);
968
1750
  }
969
1751
  /** Stops the periodic crash-recovery shard flush. */
@@ -1030,48 +1812,64 @@ export class RealtimeSessionRuntime {
1030
1812
  }
1031
1813
  /**
1032
1814
  * Uploads the chunks captured since the last flush as one crash-recovery shard (durability only;
1033
- * the canonical file is still the full upload at teardown). Best-effort — never disturbs the call.
1815
+ * the canonical file is still the full upload at teardown). Best-effort — never disturbs the call:
1816
+ * a failed shard is retained by the queue and resent under its own index on later flushes, up to a
1817
+ * bounded number of attempts (#5197).
1034
1818
  */
1035
1819
  async flushRecordingSegment() {
1036
1820
  const recorder = this.recorder;
1037
1821
  const agentSessionId = this.agentSessionId;
1038
- if (!recorder || !agentSessionId) {
1822
+ const queue = this.shardQueue;
1823
+ if (!recorder || !agentSessionId || !queue) {
1039
1824
  return;
1040
1825
  }
1041
- try {
1042
- const audioBase64 = await recorder.SnapshotNewSegmentBase64();
1043
- if (!audioBase64) {
1044
- return;
1045
- }
1046
- const index = this.segmentIndex++;
1047
- const mutation = `
1048
- mutation UploadRealtimeRecordingSegment($agentSessionId: String!, $segmentIndex: Int!, $audioBase64: String!, $mimeType: String!) {
1049
- UploadRealtimeRecordingSegment(agentSessionId: $agentSessionId, segmentIndex: $segmentIndex, audioBase64: $audioBase64, mimeType: $mimeType)
1050
- }
1051
- `;
1052
- // Shards are HEADER-LESS raw little-endian PCM16 (mime audio/L16 with the capture sample rate),
1053
- // NOT individually-playable WAV — recovery is concatenate-in-order then WAV-wrap. The canonical
1054
- // seekable WAV is the consolidated end-of-call upload below.
1055
- const shardMime = `audio/L16;rate=${recorder.SampleRate}`;
1056
- await this.gql().ExecuteGQL(mutation, { agentSessionId, segmentIndex: index, audioBase64, mimeType: shardMime });
1057
- }
1058
- catch (error) {
1059
- console.warn('[RealtimeSession] Failed to flush recording shard:', error);
1060
- }
1826
+ // Read once: a shard's sample rate is fixed at capture, and the recorder may be stopped by the
1827
+ // time a retry of it runs.
1828
+ const sampleRate = recorder.SampleRate;
1829
+ await queue.Flush(() => recorder.SnapshotNewSegmentBase64(), (segmentIndex, audioBase64) => this.uploadRecordingShard(agentSessionId, sampleRate, segmentIndex, audioBase64));
1061
1830
  }
1062
1831
  /**
1063
- * Stops the active recorder and uploads the captured audio via `UploadRealtimeRecording`.
1064
- * Fully best-effort and wrapped in try/catch — recording upload must NEVER block teardown.
1065
- * No-op when nothing was recorded or there's no session id to attach the file to.
1832
+ * Sends one shard and reports whether the server stored it. The resolver reports a failed
1833
+ * ownership load or storage write as `false`, not as an error — that is how #5197 lost a shard
1834
+ * without a single client-side log line — so the typed result must be checked, not just the absence
1835
+ * of a throw.
1066
1836
  */
1067
- async stopAndUploadRecording(agentSessionId) {
1837
+ async uploadRecordingShard(agentSessionId, sampleRate, segmentIndex, audioBase64) {
1838
+ const mutation = `
1839
+ mutation UploadRealtimeRecordingSegment($agentSessionId: String!, $segmentIndex: Int!, $audioBase64: String!, $mimeType: String!) {
1840
+ UploadRealtimeRecordingSegment(agentSessionId: $agentSessionId, segmentIndex: $segmentIndex, audioBase64: $audioBase64, mimeType: $mimeType)
1841
+ }
1842
+ `;
1843
+ // Shards are HEADER-LESS raw little-endian PCM16 (mime audio/L16 with the capture sample rate),
1844
+ // NOT individually-playable WAV — recovery is concatenate-in-order then WAV-wrap. The canonical
1845
+ // seekable WAV is the consolidated end-of-call upload below.
1846
+ const shardMime = `audio/L16;rate=${sampleRate}`;
1847
+ const result = (await this.gql().ExecuteGQL(mutation, {
1848
+ agentSessionId,
1849
+ segmentIndex,
1850
+ audioBase64,
1851
+ mimeType: shardMime,
1852
+ }));
1853
+ return result?.UploadRealtimeRecordingSegment === true;
1854
+ }
1855
+ /**
1856
+ * Stops the active recorder and encodes the captured audio. Split from the upload so teardown can
1857
+ * close the server session in between (#5195). Best-effort: returns `null` when nothing was
1858
+ * recorded or the stop failed (warned) — it must NEVER block teardown.
1859
+ */
1860
+ async stopRecording() {
1068
1861
  this.stopSegmentFlushing();
1862
+ // Close the queue with the recording: its retained shards (base64 audio) are released now, not at
1863
+ // the next session start, and a flush still in flight stops sending, so no resend lands after the
1864
+ // end-of-call upload below has deleted the shards.
1865
+ this.shardQueue?.Close();
1866
+ this.shardQueue = null;
1069
1867
  const recorder = this.recorder;
1070
1868
  this.recorder = null;
1071
1869
  this.currentTurnStartMs = null;
1072
1870
  this.turnAudioStartCaptured = false;
1073
1871
  if (!recorder) {
1074
- return;
1872
+ return null;
1075
1873
  }
1076
1874
  try {
1077
1875
  // Capture the recorder MIME (now 'audio/wav') BEFORE Stop() — the getter reads '' once stopped.
@@ -1079,18 +1877,39 @@ export class RealtimeSessionRuntime {
1079
1877
  const audioBase64 = await recorder.StopAndEncode();
1080
1878
  // Read the real waveform peaks computed during capture (survives the stop via the snapshot).
1081
1879
  const peaks = recorder.GetPeaks();
1082
- if (!audioBase64 || !agentSessionId) {
1083
- console.warn('[RealtimeSession] ⚠️ recording NOT uploaded — empty recording or no session id.');
1084
- return;
1880
+ if (!audioBase64) {
1881
+ console.warn('[RealtimeSession] ⚠️ recording NOT uploaded — empty recording.');
1882
+ return null;
1085
1883
  }
1086
- await this.uploadRecording(agentSessionId, audioBase64, mimeType, peaks);
1884
+ return { MimeType: mimeType, AudioBase64: audioBase64, Peaks: peaks };
1885
+ }
1886
+ catch (error) {
1887
+ console.warn('[RealtimeSession] Failed to stop call recording:', error);
1888
+ return null;
1889
+ }
1890
+ }
1891
+ /**
1892
+ * Uploads a recording returned by {@link stopRecording}. Never throws. No-op when there is
1893
+ * nothing to upload; warns when there is no session id to attach the file to.
1894
+ */
1895
+ async uploadStoppedRecording(agentSessionId, recording) {
1896
+ if (!recording) {
1897
+ return;
1898
+ }
1899
+ if (!agentSessionId) {
1900
+ console.warn('[RealtimeSession] ⚠️ recording NOT uploaded — no session id.');
1901
+ return;
1902
+ }
1903
+ try {
1904
+ await this.uploadRecording(agentSessionId, recording.AudioBase64, recording.MimeType, recording.Peaks);
1087
1905
  }
1088
1906
  catch (error) {
1089
- console.warn('[RealtimeSession] Failed to stop/upload call recording:', error);
1907
+ console.warn(`[RealtimeSession] Failed to upload call recording for session ${agentSessionId}:`, error);
1090
1908
  }
1091
1909
  }
1092
1910
  /**
1093
- * Runs the `UploadRealtimeRecording` mutation; failures are logged, never thrown. Sends the
1911
+ * Runs the `UploadRealtimeRecording` mutation. A `Success:false` payload is logged; a transport
1912
+ * rejection propagates to `uploadStoppedRecording`, which catches and logs it. Sends the
1094
1913
  * capture-time waveform `peaks` (max-abs per bucket, normalized 0..1) so the server can persist a
1095
1914
  * `peaks.json` sidecar for fast waveform rendering without re-decoding the audio.
1096
1915
  */
@@ -1116,81 +1935,458 @@ export class RealtimeSessionRuntime {
1116
1935
  console.warn(`[RealtimeSession] ❌ recording upload reported failure: ${payload?.ErrorMessage ?? 'unknown error'} (full result: ${JSON.stringify(result)})`);
1117
1936
  }
1118
1937
  }
1119
- // ── Interactive channels (registry-driven plugins) ─────────────────────────
1120
1938
  /**
1121
- * Resolves, instantiates and initializes the session's interactive-channel plugins from
1122
- * the `MJ: AI Agent Channels` registry, publishes them on {@link ActiveChannels$}, and
1123
- * returns their aggregated client-executed tool declarations for the session mint.
1124
- * Tolerant by design: registry/resolution failures degrade to "no channels" — the voice
1125
- * session itself always proceeds.
1939
+ * The resolved behavior of an in-session channel (display, exposure ceiling, config, what put it
1940
+ * in the session), or `null` when the channel is not in this session. Hosts and overlays read the
1941
+ * resolved display from here rather than from the channel's code default.
1942
+ *
1943
+ * @param channelName The channel's key (case-insensitive).
1944
+ */
1945
+ GetResolvedChannel(channelName) {
1946
+ return this.resolvedChannels.get(NormalizeChannelKey(channelName)) ?? null;
1947
+ }
1948
+ /**
1949
+ * Replaces the store the user's per-channel exposure choices are kept in. The default persists them in
1950
+ * the signed-in user's settings (`mj.realtime.visualPerception.v1`) and keeps them in memory for an
1951
+ * anonymous principal or a connect-only embed; a host that remembers them somewhere else supplies its own.
1952
+ */
1953
+ SetExposurePreferences(preferences) {
1954
+ this.exposurePreferences = preferences;
1955
+ }
1956
+ /** The store the user's per-channel exposure choices are currently kept in (so a host can wrap it). */
1957
+ get ExposurePreferences() {
1958
+ return this.exposurePreferences;
1959
+ }
1960
+ /** Whether user settings can be written: a signed-in user on a provider that has entity metadata. */
1961
+ canPersistUserSettings() {
1962
+ const provider = this.Provider;
1963
+ return Boolean(provider?.CurrentUser?.ID) && (provider?.Entities?.length ?? 0) > 0;
1964
+ }
1965
+ /**
1966
+ * How much of a channel the model may perceive right now: the channel's ceiling, lowered by the server's
1967
+ * policy and the user's choice. `null` when the channel is not in this session.
1968
+ *
1969
+ * @param channelName The channel's key (case-insensitive).
1970
+ */
1971
+ GetChannelExposure(channelName) {
1972
+ return this.findDispatchableChannel(channelName)?.Plugin.Exposure ?? null;
1973
+ }
1974
+ /**
1975
+ * The user's own choice of how much of a channel the agent may perceive. Takes effect immediately
1976
+ * (frames and notes the new level forbids stop, and the model is told), and is remembered per channel key
1977
+ * for next time. The level can only LOWER what the server's policy allows; choosing `'pixels'` for a channel
1978
+ * the agent's policy capped at `'state'` changes nothing.
1979
+ *
1980
+ * @param channelName The channel's key (case-insensitive).
1981
+ * @param level The user's choice; `undefined` clears it (back to what policy allows).
1982
+ * @returns `false` when the channel is not in this session (nothing is applied, but the choice is still remembered).
1983
+ */
1984
+ SetUserChannelExposure(channelName, level) {
1985
+ this.exposurePreferences.Set(channelName, level);
1986
+ const channel = this.findDispatchableChannel(channelName);
1987
+ if (!channel) {
1988
+ return false;
1989
+ }
1990
+ this.applyChannelExposure(channel.Plugin);
1991
+ return true;
1992
+ }
1993
+ /** Pushes a channel's current exposure inputs (server policy, user choice) into the channel. */
1994
+ applyChannelExposure(plugin) {
1995
+ const resolved = this.GetResolvedChannel(plugin.ChannelName);
1996
+ const user = this.exposurePreferences.Get(plugin.ChannelName);
1997
+ const reasons = (resolved?.ExposureLimits ?? []).map((limit) => limit.Reason);
1998
+ // The user's limit binds only when it is lower than what the server allows; otherwise it changes nothing
1999
+ // and the agent should not be told about it.
2000
+ const allowed = resolved?.Exposure ?? plugin.GetDescriptor().MaxExposure;
2001
+ if (user !== undefined && CompareExposure(user, allowed) < 0) {
2002
+ reasons.push(UserExposureReason(user));
2003
+ }
2004
+ plugin.ApplyExposure({ Policy: resolved?.Exposure, User: user, Reasons: reasons });
2005
+ // A capture this channel fronts follows it: the agent sees its frames only while the channel's exposure allows pixels.
2006
+ const kind = plugin.CaptureKind;
2007
+ if (kind) {
2008
+ this.captures?.SetVisibleToAgent(kind, plugin.Exposure === 'pixels');
2009
+ }
2010
+ this.syncChannelVideoSource(plugin);
2011
+ }
2012
+ /**
2013
+ * Sends a frame from a channel's context ({@link RealtimeChannelContext.SendVideoFrame}, deprecated) through the
2014
+ * session's {@link VideoSourceArbiter}, as the channel's own source: it reaches the model only while the arbiter picks
2015
+ * that source, and the arbiter tells the model when it switches. The source registers with the first frame, under the
2016
+ * id the channel's own frame bridge uses, switched on only while the channel's exposure allows pixels.
2017
+ */
2018
+ sendChannelVideoFrame(plugin, base64Image, mimeType) {
2019
+ const client = this.client;
2020
+ if (!client || !this.isSessionLive() || !client.IsTrackEstablished('video', 'inbound')) {
2021
+ return;
2022
+ }
2023
+ const arbiter = VideoSourceArbiter.ForSink(client);
2024
+ const descriptor = plugin.GetDescriptor();
2025
+ const sourceId = ChannelVideoSourceID(descriptor.Key, plugin.InstanceId);
2026
+ if (!arbiter.HasSource(sourceId)) {
2027
+ arbiter.RegisterSource({
2028
+ SourceID: sourceId,
2029
+ Label: descriptor.DisplayName,
2030
+ Kind: 'surface',
2031
+ ChannelKey: descriptor.Key,
2032
+ Enabled: plugin.Exposure === 'pixels',
2033
+ });
2034
+ }
2035
+ arbiter.PushFrame(sourceId, base64Image, mimeType);
2036
+ }
2037
+ /**
2038
+ * Keeps a channel's video source at the arbiter, when it has one, in step with the channel's exposure: on while the
2039
+ * agent may see its pixels, off otherwise. Quiet, because the channel tells the model about its own exposure change.
2040
+ */
2041
+ syncChannelVideoSource(plugin) {
2042
+ if (!this.client) {
2043
+ return;
2044
+ }
2045
+ const arbiter = VideoSourceArbiter.ForSink(this.client);
2046
+ const sourceId = ChannelVideoSourceID(plugin.GetDescriptor().Key, plugin.InstanceId);
2047
+ if (arbiter.HasSource(sourceId)) {
2048
+ arbiter.SetSourceEnabled(sourceId, plugin.Exposure === 'pixels', false);
2049
+ }
2050
+ }
2051
+ /** Follows the session client's video-source arbiter so {@link VideoSources$} reflects it. */
2052
+ watchVideoSources(client) {
2053
+ this.unwatchVideoSources();
2054
+ const arbiter = VideoSourceArbiter.ForSink(client);
2055
+ arbiter.SetFocusedChannel(this.focusedChannelKey);
2056
+ this.stopWatchingVideoSources = arbiter.OnChange(() => this._videoSources$.next(arbiter.GetSources()));
2057
+ this._videoSources$.next(arbiter.GetSources());
2058
+ }
2059
+ /**
2060
+ * Tells the session which channel's surface the user is looking at (`null` for none, e.g. the activity tab).
2061
+ * When the model can see only one video source and several are live, the one the user is looking at is the one
2062
+ * it sees (after an explicit pick, and after a camera or screen share the user started). Safe to call before the
2063
+ * session is live: it applies when the connection comes up.
2064
+ *
2065
+ * @param channelKey The focused channel's key, as it appears on {@link VideoSources$} entries' `ChannelKey`.
2066
+ */
2067
+ SetFocusedChannel(channelKey) {
2068
+ this.focusedChannelKey = channelKey;
2069
+ if (this.client) {
2070
+ VideoSourceArbiter.ForSink(this.client).SetFocusedChannel(channelKey);
2071
+ }
2072
+ }
2073
+ /** Stops following the arbiter and clears {@link VideoSources$}. Safe to call when nothing is watched. */
2074
+ unwatchVideoSources() {
2075
+ this.stopWatchingVideoSources?.();
2076
+ this.stopWatchingVideoSources = null;
2077
+ if (this._videoSources$.value.length > 0) {
2078
+ this._videoSources$.next([]);
2079
+ }
2080
+ }
2081
+ /**
2082
+ * Turns the agent's view of one video source on or off — what the "agent can see" control calls.
2083
+ *
2084
+ * A source that belongs to a channel goes through {@link SetUserChannelExposure}, so the choice is
2085
+ * remembered per channel and the channel itself tells the model. Any other source (a camera or screen
2086
+ * share that is not a channel) is switched at the arbiter, which tells the model.
2087
+ *
2088
+ * @param sourceId The source's id (from {@link VideoSources$}).
2089
+ * @param enabled Whether the agent may see it.
2090
+ * @returns `false` when there is no such source.
1126
2091
  */
1127
- async startChannels() {
1128
- const channels = await this.loadActiveChannels();
1129
- for (const plugin of channels) {
1130
- this.initializeChannel(plugin);
2092
+ SetVideoSourceEnabled(sourceId, enabled) {
2093
+ const client = this.client;
2094
+ if (!client) {
2095
+ return false;
2096
+ }
2097
+ const arbiter = VideoSourceArbiter.ForSink(client);
2098
+ const source = arbiter.GetSources().find((s) => s.SourceID === sourceId);
2099
+ if (!source) {
2100
+ return false;
1131
2101
  }
1132
- this._activeChannels$.next(channels);
1133
- return channels.flatMap(plugin => plugin.GetToolDefinitions());
2102
+ if (source.ChannelKey && this.findDispatchableChannel(source.ChannelKey)) {
2103
+ return this.SetUserChannelExposure(source.ChannelKey, enabled ? undefined : 'state');
2104
+ }
2105
+ return arbiter.SetSourceEnabled(sourceId, enabled);
2106
+ }
2107
+ /**
2108
+ * Picks the video source the agent sees: what the "agent can see" control calls when more sources are on than the
2109
+ * model takes. The pick beats every other rule (a camera or screen share the user started, the surface the user is
2110
+ * looking at) until it is cleared, or its source is turned off or leaves. The pick shows on {@link VideoSources$}
2111
+ * (`Picked`), and the model is told when what it sees changes.
2112
+ *
2113
+ * @param sourceId The source's id (from {@link VideoSources$}), or `null` to let the call decide again.
2114
+ * @returns `false` with no live session, or for a source that is not there or is turned off.
2115
+ */
2116
+ SelectVideoSource(sourceId) {
2117
+ return this.client ? VideoSourceArbiter.ForSink(this.client).SelectSource(sourceId) : false;
2118
+ }
2119
+ /**
2120
+ * The `on-demand` channels that are in the session but not open yet — what the agent can open
2121
+ * through `ContextTool`. A fresh array; empty before a session starts and after teardown.
2122
+ */
2123
+ get AdvertisedChannels() {
2124
+ return this.advertisedChannels.map((p) => p.Plugin);
2125
+ }
2126
+ /**
2127
+ * Resolves, initializes and mounts the session's channels in one step using the scope the browser
2128
+ * can decide alone (code defaults + host declarations), and returns the native tools to declare at
2129
+ * mint. This is the single-step composition of {@link prepareChannelScope} + {@link activateChannels}
2130
+ * that a session start spreads across the mint; it is kept as a unit because it is the cleanest seam
2131
+ * for exercising the plugin plumbing without a mint.
2132
+ */
2133
+ async startChannels(hostChannels) {
2134
+ const scope = await this.prepareChannelScope(hostChannels);
2135
+ this.activateChannels(this.localChannelScope?.Channels ?? []);
2136
+ return scope.NativeTools;
2137
+ }
2138
+ /**
2139
+ * Stage 1: constructs the channel plugins (registry rows and host declarations), reads their
2140
+ * descriptors, resolves the scope the browser can decide alone, and builds what the mint needs —
2141
+ * the native tools to declare and the candidates to report. Starts nothing.
2142
+ *
2143
+ * @param hostChannels Channels the host brings to this session.
2144
+ * @returns The native tools to declare at mint, the candidates as JSON (`null` when there are none,
2145
+ * so a channel-less session sends exactly the mint it always did), and whether the host could show the agent's
2146
+ * video ({@link PreparedChannelsShowAgentVideo}).
2147
+ */
2148
+ async prepareChannelScope(hostChannels) {
2149
+ this.discardUnmountedChannels();
2150
+ const prepared = await this.prepareChannels(hostChannels);
2151
+ this.preparedChannels = prepared;
2152
+ const candidates = prepared.map((p) => BuildChannelCandidate(p, p.Plugin.GetDescriptor()));
2153
+ const local = ResolveLocalChannelScope(candidates);
2154
+ this.localChannelScope = local;
2155
+ for (const excluded of local.Excluded) {
2156
+ console.warn(`[RealtimeSession] Channel '${excluded.Key}' is not in this session (${excluded.Reason}).`);
2157
+ }
2158
+ const nativeTools = SelectNativeChannelTools(local.Channels, ToolsByChannelKey(candidates));
2159
+ // `Registry` is the browser's own view and never goes over the wire: the server reads the registry itself.
2160
+ const wire = candidates.map(({ Registry: _registry, ...candidate }) => candidate);
2161
+ return { NativeTools: nativeTools, CandidatesJson: wire.length > 0 ? JSON.stringify(wire) : null, ShowsAgentVideo: PreparedChannelsShowAgentVideo(prepared) };
1134
2162
  }
1135
2163
  /**
1136
- * Loads the ACTIVE channel definitions from the registry and resolves each row's
1137
- * `ClientPluginClass` through the MJ ClassFactory into a per-session plugin instance —
1138
- * the client-side mirror of how realtime-model drivers resolve from `BaseRealtimeModel`
1139
- * / `BaseRealtimeClient`. Rows whose plugin class isn't registered are skipped (logged),
1140
- * never fatal.
2164
+ * Constructs a plugin for every active-or-inactive registry row and every host declaration. An
2165
+ * INACTIVE row is constructed too — only to learn its key so the kill switch can name it — and is
2166
+ * never initialized. A plugin whose descriptor cannot be read is skipped (logged), never fatal.
1141
2167
  */
1142
- async loadActiveChannels() {
2168
+ async prepareChannels(hostChannels) {
1143
2169
  const rows = await this.fetchChannelDefinitions();
1144
- const channels = [];
2170
+ await this.loadChannelClasses([...rows.map((r) => r.ClientPluginClass), ...(hostChannels ?? []).map((d) => d.ClientPluginClass)]);
2171
+ const prepared = [];
1145
2172
  for (const row of rows) {
1146
2173
  const plugin = this.resolveChannelPlugin(row);
1147
2174
  if (plugin) {
1148
- channels.push(plugin);
2175
+ plugin.ApplySurfacePlacement(row.SurfacePlacement);
2176
+ prepared.push({ Plugin: plugin, Key: plugin.ChannelName, Registry: row.IsActive ? 'active' : 'inactive' });
1149
2177
  }
1150
2178
  }
1151
- return channels;
2179
+ for (const declaration of hostChannels ?? []) {
2180
+ const plugin = this.createHostChannelPlugin(declaration);
2181
+ if (!plugin) {
2182
+ continue;
2183
+ }
2184
+ const existing = FindPreparedChannel(prepared, plugin.ChannelName);
2185
+ if (existing) {
2186
+ // The host's instance replaces the registry's (it may carry the host's collaborators); the
2187
+ // registry row's state — notably the kill switch, and where its surface shows — still applies.
2188
+ plugin.ApplySurfacePlacement(existing.Plugin.SurfacePlacement);
2189
+ existing.Plugin = plugin;
2190
+ existing.HostDeclaration = declaration;
2191
+ }
2192
+ else {
2193
+ prepared.push({ Plugin: plugin, Key: plugin.ChannelName, Registry: 'none', HostDeclaration: declaration });
2194
+ }
2195
+ }
2196
+ return prepared.filter((p) => this.canDescribe(p));
1152
2197
  }
1153
2198
  /**
1154
- * Reads the ACTIVE `MJ: AI Agent Channels` rows from {@link AIEngineBase}'s cached
1155
- * `AgentChannels` (provider-scoped engine instance, lazy `Config` — no RunView
1156
- * round-trip; the engine's BaseEntity-event reactivity keeps the registry fresh).
1157
- * Failures are logged and degrade to an empty list — channel availability must
1158
- * never block the voice session.
2199
+ * Gives a host that loads channel code on demand (a widget that keeps a heavy channel in its own download)
2200
+ * the chance to load it before the plugins are built. Never fatal: a loader that throws is logged and the
2201
+ * channel simply is not available, exactly as if its class had never been registered.
2202
+ *
2203
+ * @param keys The `ClientPluginClass` keys of every channel about to be resolved (blank/absent ones included).
2204
+ */
2205
+ async loadChannelClasses(keys) {
2206
+ const loader = this.ChannelClassLoader;
2207
+ if (!loader) {
2208
+ return;
2209
+ }
2210
+ const wanted = [...new Set(keys.map((k) => k?.trim() ?? '').filter((k) => k.length > 0))];
2211
+ if (wanted.length === 0) {
2212
+ return;
2213
+ }
2214
+ try {
2215
+ await loader(wanted);
2216
+ }
2217
+ catch (error) {
2218
+ console.error('[RealtimeSession] The channel class loader failed — channels that needed it are left out:', error);
2219
+ }
2220
+ }
2221
+ /** Whether a prepared plugin's descriptor can be read; logs and rejects one that throws. */
2222
+ canDescribe(prepared) {
2223
+ try {
2224
+ prepared.Plugin.GetDescriptor();
2225
+ return true;
2226
+ }
2227
+ catch (error) {
2228
+ console.error(`[RealtimeSession] Channel '${prepared.Key}' has an unreadable descriptor — leaving it out:`, error);
2229
+ return false;
2230
+ }
2231
+ }
2232
+ /** Builds a plugin from a host declaration; `null` (logged) when it names nothing buildable. */
2233
+ createHostChannelPlugin(declaration) {
2234
+ if (declaration.Create) {
2235
+ try {
2236
+ return declaration.Create();
2237
+ }
2238
+ catch (error) {
2239
+ console.error('[RealtimeSession] A host-declared channel factory threw — leaving the channel out:', error);
2240
+ return null;
2241
+ }
2242
+ }
2243
+ const key = declaration.ClientPluginClass?.trim();
2244
+ if (!key) {
2245
+ console.warn('[RealtimeSession] A host-declared channel names neither ClientPluginClass nor Create — ignoring it.');
2246
+ return null;
2247
+ }
2248
+ return this.resolveChannelPlugin({ Name: key, ClientPluginClass: key, IsActive: true });
2249
+ }
2250
+ /**
2251
+ * Stage 3: initializes the channels the resolved policy puts in the session. `open-on-start` and
2252
+ * `headless` channels are mounted and published on {@link ActiveChannels$}; `on-demand` channels
2253
+ * are held until opened. Prepared plugins the policy left out are dropped without ever having run.
2254
+ */
2255
+ activateChannels(channels) {
2256
+ const reconciled = ReconcileChannelsWithPolicy(this.preparedChannels, channels);
2257
+ for (const key of reconciled.Unknown) {
2258
+ console.warn(`[RealtimeSession] The session policy names channel '${key}' but this host has no plugin for it — ignoring it.`);
2259
+ }
2260
+ this.resolvedChannels.clear();
2261
+ this.advertisedChannels = [];
2262
+ const mounted = [];
2263
+ for (const { Prepared, Resolved } of reconciled.InSession) {
2264
+ this.resolvedChannels.set(NormalizeChannelKey(Resolved.Key), Resolved);
2265
+ // Exposure is applied BEFORE the channel initializes, so what it requests of the model (a video
2266
+ // track, in particular) already reflects what policy allows.
2267
+ this.applyChannelExposure(Prepared.Plugin);
2268
+ if (Resolved.DisplayPolicy === 'on-demand') {
2269
+ this.advertisedChannels.push(Prepared);
2270
+ }
2271
+ else {
2272
+ this.mountChannel(Prepared, Resolved);
2273
+ mounted.push(Prepared.Plugin);
2274
+ }
2275
+ }
2276
+ this.preparedChannels = [];
2277
+ this._activeChannels$.next(mounted);
2278
+ }
2279
+ /**
2280
+ * Applies the mint's outcome to the channel set: the server's policy when it sent a usable one,
2281
+ * else the scope the browser resolved on its own. A policy can only SELECT among the plugins this
2282
+ * host prepared — it cannot add one.
2283
+ */
2284
+ applySessionClientPolicy(session) {
2285
+ const policy = ParseRealtimeSessionClientPolicy(session.ClientPolicyJson);
2286
+ if (session.ClientPolicyJson && !policy) {
2287
+ console.warn('[RealtimeSession] The mint returned a client policy this runtime cannot read — resolving channels locally.');
2288
+ }
2289
+ this.sessionClientTools = policy?.ClientTools ?? null;
2290
+ if (this.preparedChannels.length === 0) {
2291
+ this.resolvedChannels.clear();
2292
+ return; // nothing was prepared (a host that mints and runs the session itself) — no channel set to apply
2293
+ }
2294
+ this.activateChannels(policy?.Channels ?? this.localChannelScope?.Channels ?? []);
2295
+ }
2296
+ /** Drops plugins that were prepared for a start that never mounted them. They were never initialized, so there is nothing to release. */
2297
+ discardUnmountedChannels() {
2298
+ this.preparedChannels = [];
2299
+ this.advertisedChannels = [];
2300
+ this.localChannelScope = null;
2301
+ }
2302
+ /**
2303
+ * Loads the `MJ: AI Agent Channels` registry rows (ACTIVE AND INACTIVE — an inactive row is the master
2304
+ * kill switch, so the scope decision has to see it). With entity metadata: from {@link AIEngineBase}'s
2305
+ * cached `AgentChannels` (provider-scoped engine instance, lazy `Config` — no RunView round-trip; the
2306
+ * engine's BaseEntity-event reactivity keeps the registry fresh). On a connect-only provider: one
2307
+ * `RunDynamicView` query ({@link fetchChannelDefinitionsOverGraphQL}). Failures are logged and degrade to an
2308
+ * empty list — channel availability must never block the voice session.
1159
2309
  */
1160
2310
  async fetchChannelDefinitions() {
1161
2311
  // A connect-only provider (ConnectGraphQLClient — anonymous embeds) has no entity metadata,
1162
2312
  // so AIEngineBase cannot load; asking it would only fail with "Entity … not found in
1163
- // metadata". An embed brings its own channels, so "no registry channels" is the right answer.
2313
+ // metadata". The registry is still the authority, so read it over GraphQL instead — answering
2314
+ // "no channels" here cost an embed every channel tool (Whiteboard, Media) at mint.
1164
2315
  if ((this.Provider?.Entities?.length ?? 0) === 0) {
1165
- return [];
2316
+ return this.fetchChannelDefinitionsOverGraphQL();
1166
2317
  }
1167
2318
  try {
1168
2319
  const engine = AIEngineBase.GetProviderInstance(this.Provider, AIEngineBase);
1169
2320
  await engine.Config(false, undefined, this.Provider);
1170
- return (engine.AgentChannels ?? [])
1171
- .filter(c => c.IsActive)
1172
- .map(c => ({ ID: c.ID, Name: c.Name, ClientPluginClass: c.ClientPluginClass }));
2321
+ return (engine.AgentChannels ?? []).map((c) => ({
2322
+ ID: c.ID,
2323
+ Name: c.Name,
2324
+ ClientPluginClass: c.ClientPluginClass,
2325
+ IsActive: c.IsActive,
2326
+ SurfacePlacement: readRowSurfacePlacement(c),
2327
+ }));
1173
2328
  }
1174
2329
  catch (error) {
1175
2330
  console.warn('[RealtimeSession] Channel registry unavailable — starting with no channels:', error);
1176
2331
  return [];
1177
2332
  }
1178
2333
  }
2334
+ /**
2335
+ * The connect-only path of {@link fetchChannelDefinitions}: the same `MJ: AI Agent Channels` rows, active
2336
+ * and inactive (an inactive row is the kill switch), read with a dynamic view because a connect-only client
2337
+ * has no entity metadata to build a typed RunView from. Same tolerance as the engine path — a failure is
2338
+ * logged and means "no channels", never a blocked session.
2339
+ */
2340
+ async fetchChannelDefinitionsOverGraphQL() {
2341
+ const query = `query RealtimeChannelRegistry($input: RunDynamicViewInput!) {
2342
+ RunDynamicView(input: $input) { Success ErrorMessage Results { Data } }
2343
+ }`;
2344
+ try {
2345
+ const result = (await this.gql().ExecuteGQL(query, {
2346
+ input: { EntityName: 'MJ: AI Agent Channels', Fields: ['ID', 'Name', 'ClientPluginClass', 'IsActive', 'UIConfig'] },
2347
+ }));
2348
+ const view = result?.RunDynamicView;
2349
+ if (!view?.Success) {
2350
+ console.warn('[RealtimeSession] Channel registry unavailable — starting with no channels:', view?.ErrorMessage ?? 'no result');
2351
+ return [];
2352
+ }
2353
+ return (view.Results ?? [])
2354
+ .map((r) => JSON.parse(r.Data))
2355
+ .map((row) => ({
2356
+ ID: row.ID,
2357
+ Name: row.Name,
2358
+ ClientPluginClass: row.ClientPluginClass,
2359
+ IsActive: row.IsActive === true,
2360
+ SurfacePlacement: readUIConfigTextSurfacePlacement(row.Name, row.UIConfig),
2361
+ }));
2362
+ }
2363
+ catch (error) {
2364
+ console.warn('[RealtimeSession] Channel registry unavailable — starting with no channels:', error instanceof Error ? error.message : String(error));
2365
+ return [];
2366
+ }
2367
+ }
1179
2368
  /**
1180
2369
  * Resolves one registry row's `ClientPluginClass` via the ClassFactory (registration
1181
2370
  * checked first, exactly like the realtime-client drivers) and instantiates a fresh
1182
2371
  * per-session plugin. Returns `null` (logged) when no plugin is registered for the key
1183
2372
  * — e.g. its Load function was never called or the package isn't included client-side.
2373
+ * An inactive row is resolved quietly: it only needs to be named, and a missing plugin for a
2374
+ * channel nobody wants is not worth a warning.
1184
2375
  */
1185
2376
  resolveChannelPlugin(row) {
1186
2377
  const key = row.ClientPluginClass?.trim();
2378
+ const quiet = !row.IsActive;
1187
2379
  if (!key) {
1188
- console.warn(`[RealtimeSession] Channel '${row.Name}' has no ClientPluginClass — skipping.`);
2380
+ if (!quiet) {
2381
+ console.warn(`[RealtimeSession] Channel '${row.Name}' has no ClientPluginClass — skipping.`);
2382
+ }
1189
2383
  return null;
1190
2384
  }
1191
2385
  const registration = MJGlobal.Instance.ClassFactory.GetRegistration(BaseRealtimeChannelClient, key);
1192
2386
  if (!registration) {
1193
- console.warn(`[RealtimeSession] No client plugin registered for channel '${row.Name}' (key '${key}') — skipping.`);
2387
+ if (!quiet) {
2388
+ console.warn(`[RealtimeSession] No client plugin registered for channel '${row.Name}' (key '${key}') — skipping.`);
2389
+ }
1194
2390
  return null;
1195
2391
  }
1196
2392
  const plugin = MJGlobal.Instance.ClassFactory.CreateInstance(BaseRealtimeChannelClient, key);
@@ -1201,24 +2397,157 @@ export class RealtimeSessionRuntime {
1201
2397
  return plugin;
1202
2398
  }
1203
2399
  /**
1204
- * Wires one plugin into the session: hands it its host context and registers its
1205
- * prefix-routed local tool executor (so `<ToolNamePrefix>*` calls run in the browser
1206
- * through {@link BaseRealtimeChannelClient.ApplyAgentTool}, never the server relay).
2400
+ * Wires one plugin into the session: hands it its host context (carrying its resolved config) and,
2401
+ * when it has native tools, registers its prefix-routed local tool executor (so `<ToolNamePrefix>*`
2402
+ * calls run in the browser through {@link BaseRealtimeChannelClient.ApplyAgentTool}, never the
2403
+ * server relay). A channel with no tool prefix has no native route — it is reached through
2404
+ * `ContextTool` — and registering an empty prefix would match EVERY tool name.
1207
2405
  */
1208
- initializeChannel(plugin) {
1209
- plugin.Initialize(this.buildChannelContext(plugin));
2406
+ mountChannel(prepared, resolved) {
2407
+ const plugin = prepared.Plugin;
2408
+ plugin.Initialize(this.buildChannelContext(plugin, resolved?.Config));
2409
+ if (plugin.ToolNamePrefix.length === 0) {
2410
+ return;
2411
+ }
1210
2412
  this.RegisterClientToolHandler(plugin.ToolNamePrefix, (toolName, argsJson) => {
1211
2413
  // The agent is ACTING on this channel — surface-discovery signal for the overlay
1212
2414
  // (first activity registers + auto-reveals + focuses the channel tab) before the
1213
2415
  // tool applies. Record the channel as USED so the overlay tabs it (channels other
1214
2416
  // than the whiteboard are tab-less until they're first used).
1215
- this.usedChannelNames.add(plugin.ChannelName);
1216
- this._channelActivity$.next(plugin);
2417
+ this.noteChannelActivity(plugin);
2418
+ // A native tool whose result would show the agent what exposure policy withholds is refused here, as it is on the
2419
+ // ContextTool route; the legacy tool-result shape carries the reason.
2420
+ const withheld = plugin.FindVerbForNativeTool(toolName);
2421
+ const refusal = withheld ? plugin.RefuseVerbForExposure(withheld) : null;
2422
+ if (refusal) {
2423
+ return JSON.stringify({ success: false, errorCode: 'exposure_restricted', error: refusal });
2424
+ }
1217
2425
  return plugin.ApplyAgentTool(toolName, argsJson);
1218
2426
  });
1219
2427
  }
2428
+ /** Records that the agent acted on a channel (the overlay's first-use reveal signal). */
2429
+ noteChannelActivity(plugin) {
2430
+ this.usedChannelNames.add(plugin.ChannelName);
2431
+ this._channelActivity$.next(plugin);
2432
+ }
2433
+ /**
2434
+ * Mounts an advertised `on-demand` channel when the agent opens it: initializes it, restores its
2435
+ * prior-session state, publishes it on {@link ActiveChannels$} (so the overlay can give it a tab)
2436
+ * and, if the call is already connected, tells it so. If initialization throws the channel goes
2437
+ * back to being advertised, so the agent can retry or move on.
2438
+ */
2439
+ async mountAdvertisedChannel(plugin) {
2440
+ const index = this.advertisedChannels.findIndex((p) => p.Plugin === plugin);
2441
+ if (index < 0) {
2442
+ return; // already mounted (or never advertised) — the dispatcher opens it regardless
2443
+ }
2444
+ const [prepared] = this.advertisedChannels.splice(index, 1);
2445
+ try {
2446
+ this.mountChannel(prepared, this.GetResolvedChannel(prepared.Key) ?? undefined);
2447
+ }
2448
+ catch (error) {
2449
+ this.advertisedChannels.splice(index, 0, prepared);
2450
+ throw error;
2451
+ }
2452
+ this.restorePriorChannelState(plugin);
2453
+ this._activeChannels$.next([...this._activeChannels$.value, plugin]);
2454
+ if (this.isSessionLive()) {
2455
+ this.notifyChannelSessionStarted(plugin);
2456
+ }
2457
+ }
2458
+ /** Tells a channel the session is connected, containing anything it throws. */
2459
+ notifyChannelSessionStarted(channel) {
2460
+ try {
2461
+ channel.OnSessionStarted?.();
2462
+ }
2463
+ catch (err) {
2464
+ console.error(`[RealtimeSession] Error in channel '${channel.ChannelName}' OnSessionStarted:`, err);
2465
+ }
2466
+ }
2467
+ /** Finds a session channel (open or merely advertised) for the dispatcher. */
2468
+ findDispatchableChannel(key) {
2469
+ const id = NormalizeChannelKey(key);
2470
+ const open = this._activeChannels$.value.find((c) => NormalizeChannelKey(c.ChannelName) === id);
2471
+ if (open) {
2472
+ return { Plugin: open, IsOpen: true };
2473
+ }
2474
+ const advertised = this.advertisedChannels.find((p) => NormalizeChannelKey(p.Key) === id);
2475
+ return advertised ? { Plugin: advertised.Plugin, IsOpen: false } : null;
2476
+ }
2477
+ /** The keys the agent can address, for "unknown channel" messages — channels with nothing to address are left out. */
2478
+ listAddressableChannelKeys() {
2479
+ const addressable = (plugin) => {
2480
+ const descriptor = plugin.GetDescriptor();
2481
+ return descriptor.Verbs.length > 0 || descriptor.Inputs !== undefined;
2482
+ };
2483
+ return [
2484
+ ...this._activeChannels$.value.filter(addressable).map((c) => c.ChannelName),
2485
+ ...this.advertisedChannels.filter((p) => addressable(p.Plugin)).map((p) => p.Key),
2486
+ ];
2487
+ }
2488
+ /**
2489
+ * Runs a channel-addressed `ContextTool` call (the {@link RealtimeChannelContext.DispatchContextAction}
2490
+ * implementation): validates it, opens an `on-demand` channel when asked, runs the verb, and on
2491
+ * success records the channel as used so the overlay reveals its tab. Never throws.
2492
+ */
2493
+ async dispatchContextAction(request) {
2494
+ const result = await this.channelDispatcher.Dispatch(request);
2495
+ if (result.Success) {
2496
+ const channel = this.findDispatchableChannel(request.Target.Channel);
2497
+ if (channel) {
2498
+ this.noteChannelActivity(channel.Plugin);
2499
+ }
2500
+ }
2501
+ return result;
2502
+ }
2503
+ /**
2504
+ * Opens (and seeds) a channel from the HOST — the same path the agent's `open` action takes: the channel
2505
+ * is mounted if it was only advertised, its `Inputs` schema validates `inputs`, and it announces itself to
2506
+ * the model. A channel that is already open is re-seeded. Never throws; a refusal is a structured result.
2507
+ *
2508
+ * @param channelKey The channel to open (case-insensitive).
2509
+ * @param inputs Seed inputs, validated against the channel's descriptor.
2510
+ */
2511
+ OpenChannel(channelKey, inputs = {}) {
2512
+ return this.dispatchContextAction({ Target: { Channel: channelKey }, Action: 'open', Params: inputs });
2513
+ }
2514
+ /**
2515
+ * Tells the model which channels exist and how to use them — rendered from the channels' own
2516
+ * descriptors, so a new channel needs no prompt change. Sent ONCE, the first moment the control
2517
+ * channel is usable — a context note sent before then is dropped, and `Connect` can resolve before the
2518
+ * provider reports `listening`, so this runs at start and again on each state change until it has gone.
2519
+ * A session whose only channels describe themselves through native tools sends nothing.
2520
+ */
2521
+ flushChannelCatalogNote() {
2522
+ if (!this.catalogNotePending || !this.isSessionLive()) {
2523
+ return;
2524
+ }
2525
+ this.catalogNotePending = false;
2526
+ const entries = [
2527
+ ...this._activeChannels$.value.map((plugin) => this.catalogEntry(plugin, true)),
2528
+ ...this.advertisedChannels.map((p) => this.catalogEntry(p.Plugin, false)),
2529
+ ];
2530
+ const note = BuildChannelCatalogNote(entries);
2531
+ if (note) {
2532
+ this.SendContextNote(note);
2533
+ }
2534
+ }
2535
+ /** One catalog entry: a channel's descriptor, whether it is open, and whether its tools were declared natively. */
2536
+ catalogEntry(plugin, isOpen) {
2537
+ const display = this.GetResolvedChannel(plugin.ChannelName)?.DisplayPolicy;
2538
+ const descriptor = plugin.GetDescriptor();
2539
+ const entry = {
2540
+ Descriptor: descriptor,
2541
+ IsOpen: isOpen,
2542
+ HasNativeTools: isOpen && display !== undefined && DeclaresNativeTools(display) && plugin.GetToolDefinitions().length > 0,
2543
+ };
2544
+ if (CompareExposure(plugin.Exposure, descriptor.MaxExposure) < 0) {
2545
+ entry.ExposureLimit = { Effective: plugin.Exposure, Ceiling: descriptor.MaxExposure, Reasons: [...plugin.ExposureReasons] };
2546
+ }
2547
+ return entry;
2548
+ }
1220
2549
  /** Builds the host-services context one channel plugin sees (its only line to the session). */
1221
- buildChannelContext(plugin) {
2550
+ buildChannelContext(plugin, channelConfig) {
1222
2551
  // Capture the service in a local so the AgentSessionID getter reads the SERVICE's live
1223
2552
  // field (not the object literal's `this`) every time it's accessed.
1224
2553
  const service = this;
@@ -1245,55 +2574,107 @@ export class RealtimeSessionRuntime {
1245
2574
  // (Explorer) feeds both; absent on hosts that supply no app context / register no client tools.
1246
2575
  AppContext$: this.AppContext$,
1247
2576
  ExecuteClientTool: (name, params) => this.executeAppClientTool(name, params),
2577
+ // The session's server events (identity verification, app events), for a channel that reacts to them.
2578
+ SessionEvents$: this.SessionEvents$,
2579
+ // Channel-addressed ContextTool calls: validated against the verb's schema, open `on-demand` channels.
2580
+ DispatchContextAction: (request) => this.dispatchContextAction(request),
2581
+ // This channel's resolved configuration (host defaults beneath agent/app config).
2582
+ ChannelConfig: channelConfig ?? {},
1248
2583
  get Client() {
1249
2584
  return service.client;
1250
2585
  },
1251
- SendVideoFrame: (base64Image, mimeType) => this.SendVideoFrame(base64Image, mimeType),
1252
- IsTrackEstablished: (modality, direction) => this.IsTrackEstablished(modality, direction)
2586
+ // Through the arbiter, as this channel's source (deprecated: channels use a ChannelInboundVideoBridge).
2587
+ SendVideoFrame: (base64Image, mimeType) => this.sendChannelVideoFrame(plugin, base64Image, mimeType),
2588
+ IsTrackEstablished: (modality, direction) => this.IsTrackEstablished(modality, direction),
2589
+ // The camera and screen share, for a channel that fronts one: their state, and the user's clicks on its surface.
2590
+ Captures$: this.Captures$,
2591
+ StartCapture: (kind) => (kind === 'camera' ? this.StartCamera() : this.StartScreenShare()),
2592
+ StopCapture: (kind) => (kind === 'camera' ? this.StopCamera() : this.StopScreenShare()),
2593
+ // The video sources and which of them the model is sent, so a capture's channel says the agent sees it only then.
2594
+ VideoSources$: this.VideoSources$,
2595
+ // The agent's video, the call's state and its resumes, for a channel that shows the agent.
2596
+ AgentVideo$: this.AgentVideo$,
2597
+ ConnectionState$: this.ConnectionState$,
2598
+ Resuming$: this.Resuming$
1253
2599
  };
1254
2600
  }
1255
2601
  /**
1256
- * Replaces the set of host-registered surface client tools the realtime ContextTool can execute.
1257
- * The host calls this at session start and whenever the active surface's tool set changes (the
1258
- * continuous-capability half of client-context delivery). Passing `[]` clears them.
2602
+ * Registers the tools ONE source of the host can run for the realtime `ContextTool`, replacing only
2603
+ * that source's previous set — other owners' tools are untouched. A host with two sources (Explorer's
2604
+ * always-available globals and the active surface's tools) registers each under its own owner key, so
2605
+ * refreshing the surface's tools can neither drop the globals nor leave the previous surface's tools
2606
+ * behind. Passing `[]` clears just that owner. When two owners register the same tool name, the
2607
+ * later-registered owner wins.
1259
2608
  *
1260
- * @param tools The current surface client tools (name + handler). Descriptions/schemas ride the
1261
- * app-context manifest separately; only the executable handler is needed here.
1262
- */
1263
- RegisterAppClientTools(tools) {
1264
- this.appClientToolHandlers.clear();
1265
- for (const tool of tools) {
1266
- if (tool?.Name && typeof tool.Handler === 'function') {
1267
- this.appClientToolHandlers.set(tool.Name.trim().toLowerCase(), tool.Handler);
1268
- }
1269
- }
2609
+ * Calling it with one argument (the original form) registers under {@link DEFAULT_APP_TOOL_OWNER}, so
2610
+ * a host with a single source behaves exactly as before.
2611
+ *
2612
+ * @param tools The owner's complete current set (name + handler; description/schema optional).
2613
+ * @param owner A stable key naming the source (e.g. `'explorer.surface'`). Defaults to a shared owner.
2614
+ */
2615
+ RegisterAppClientTools(tools, owner = DEFAULT_APP_TOOL_OWNER) {
2616
+ this.appToolRegistry.Register(owner, tools);
2617
+ }
2618
+ /** Removes one owner's tools (and its place in the collision order). No-op when it has none. */
2619
+ UnregisterAppClientTools(owner) {
2620
+ this.appToolRegistry.Unregister(owner);
2621
+ }
2622
+ /** Removes every owner's tools. */
2623
+ ClearAppClientTools() {
2624
+ this.appToolRegistry.Clear();
2625
+ }
2626
+ /**
2627
+ * The client tools in effect right now, resolved through the same unified resolver the server uses
2628
+ * (`override → session → app → static`, first match wins): the host's registered tools (described
2629
+ * by the manifest the host streams when it did not describe them itself) first, then the app and
2630
+ * agent tiers the server resolved at mint. Only the host-registered tools can RUN here; the
2631
+ * other tiers describe what the app declares.
2632
+ */
2633
+ resolveAppClientTools() {
2634
+ const manifest = this._appContext$.value?.Capabilities?.Tools ?? [];
2635
+ return ResolveClientTools({
2636
+ agentId: '',
2637
+ sessionTools: MergeToolMetadata(this.appToolRegistry.ToMetadata(), manifest),
2638
+ appTools: this.sessionClientTools?.App,
2639
+ staticTools: this.sessionClientTools?.Static,
2640
+ });
1270
2641
  }
1271
2642
  /**
1272
2643
  * Executes a host-registered surface client tool by name (the {@link RealtimeChannelContext.ExecuteClientTool}
1273
2644
  * implementation). Tolerant: an unknown tool or a thrown handler resolves to a structured
1274
- * `Success: false` result the channel narrates — never throws.
2645
+ * `Success: false` result the channel narrates — never throws. A tool the app declares but the host has
2646
+ * not registered a handler for says so, instead of reading like a typo. A handler that returns an object with
2647
+ * `Success: false` also gives `Success: false`, with the handler's `ErrorMessage` and its object as `Result`.
1275
2648
  *
1276
2649
  * @param name The tool name (the model's `action`).
1277
2650
  * @param params The tool parameters.
1278
2651
  * @returns A structured result for the channel to serialize back to the model.
1279
2652
  */
1280
2653
  async executeAppClientTool(name, params) {
1281
- const handler = this.appClientToolHandlers.get((name ?? '').trim().toLowerCase());
2654
+ const handler = this.appToolRegistry.Find(name)?.Handler;
1282
2655
  if (!handler) {
1283
- const available = Array.from(this.appClientToolHandlers.keys()).join(', ');
1284
- return {
1285
- Success: false,
1286
- ErrorMessage: `No client tool named "${name}" is available on this surface. Available: ${available || '(none)'}.`
1287
- };
2656
+ return { Success: false, ErrorMessage: this.describeMissingClientTool(name) };
1288
2657
  }
1289
2658
  try {
1290
2659
  const result = await handler(params ?? {});
1291
- return { Success: true, Result: result };
2660
+ return reportsFailure(result)
2661
+ ? { Success: false, ErrorMessage: failureMessage(result), Result: result }
2662
+ : { Success: true, Result: result };
1292
2663
  }
1293
2664
  catch (error) {
1294
2665
  return { Success: false, ErrorMessage: error instanceof Error ? error.message : String(error) };
1295
2666
  }
1296
2667
  }
2668
+ /** The model-readable reason a client tool could not run. */
2669
+ describeMissingClientTool(name) {
2670
+ const available = this.appToolRegistry.Names().join(', ');
2671
+ const wanted = (name ?? '').trim().toLowerCase();
2672
+ const declared = this.resolveAppClientTools().find((t) => t.Name.trim().toLowerCase() === wanted);
2673
+ if (declared) {
2674
+ return `The client tool "${name}" is declared for this app but this surface has not registered it, so it cannot run right now. Available: ${available || '(none)'}.`;
2675
+ }
2676
+ return `No client tool named "${name}" is available on this surface. Available: ${available || '(none)'}.`;
2677
+ }
1297
2678
  /**
1298
2679
  * Runs a channel-specific GraphQL operation through the live session's provider (the
1299
2680
  * {@link RealtimeChannelContext.ExecuteServerAction} implementation). Best-effort: any
@@ -1330,35 +2711,39 @@ export class RealtimeSessionRuntime {
1330
2711
  * never affected.
1331
2712
  */
1332
2713
  applyPriorChannelStates(statesJson) {
2714
+ this.priorChannelStates = {};
1333
2715
  if (!statesJson) {
1334
2716
  return;
1335
2717
  }
1336
- let states;
1337
2718
  try {
1338
2719
  const parsed = JSON.parse(statesJson);
1339
2720
  if (parsed === null || typeof parsed !== 'object') {
1340
2721
  return;
1341
2722
  }
1342
- states = parsed;
2723
+ this.priorChannelStates = parsed;
1343
2724
  }
1344
2725
  catch {
1345
2726
  console.warn('[RealtimeSession] PriorChannelStatesJson was malformed — starting channels fresh');
1346
2727
  return;
1347
2728
  }
1348
2729
  for (const plugin of this._activeChannels$.value) {
1349
- const state = states[plugin.ChannelName];
1350
- if (typeof state === 'string' && state.length > 0) {
1351
- try {
1352
- const restored = plugin.RestoreState(state);
1353
- if (!restored) {
1354
- console.warn(`[RealtimeSession] Channel '${plugin.ChannelName}' declined its prior-session state — starting fresh`);
1355
- }
1356
- }
1357
- catch (error) {
1358
- console.warn(`[RealtimeSession] Channel '${plugin.ChannelName}' restore threw — starting fresh`, error);
1359
- }
2730
+ this.restorePriorChannelState(plugin);
2731
+ }
2732
+ }
2733
+ /** Offers one plugin its prior-session state, if there is one. Rejections and throws are logged, never fatal. */
2734
+ restorePriorChannelState(plugin) {
2735
+ const state = this.priorChannelStates[plugin.ChannelName];
2736
+ if (typeof state !== 'string' || state.length === 0) {
2737
+ return;
2738
+ }
2739
+ try {
2740
+ if (!plugin.RestoreState(state)) {
2741
+ console.warn(`[RealtimeSession] Channel '${plugin.ChannelName}' declined its prior-session state — starting fresh`);
1360
2742
  }
1361
2743
  }
2744
+ catch (error) {
2745
+ console.warn(`[RealtimeSession] Channel '${plugin.ChannelName}' restore threw — starting fresh`, error);
2746
+ }
1362
2747
  }
1363
2748
  /**
1364
2749
  * Persists a channel's state as a first-class versioned artifact (`MJ: Artifacts`) via the
@@ -1448,6 +2833,13 @@ export class RealtimeSessionRuntime {
1448
2833
  this._activeChannels$.next([]);
1449
2834
  }
1450
2835
  this.usedChannelNames.clear();
2836
+ // Channels that were prepared or advertised but never mounted were never initialized, so there is
2837
+ // nothing to dispose — only the bookkeeping to forget.
2838
+ this.discardUnmountedChannels();
2839
+ this.catalogNotePending = false;
2840
+ this.resolvedChannels.clear();
2841
+ this.priorChannelStates = {};
2842
+ this.sessionClientTools = null;
1451
2843
  }
1452
2844
  // ── Realtime client resolution + wiring ────────────────────────────────────
1453
2845
  /**
@@ -1470,20 +2862,22 @@ export class RealtimeSessionRuntime {
1470
2862
  }
1471
2863
  /**
1472
2864
  * Builds the client-direct session config the realtime client connects with.
1473
- * Aggregates tracks sourced by active channels into `requestedTracks` so the driver
1474
- * can negotiate them (e.g., establishing inbound video streaming for Whiteboard / RemoteBrowser).
2865
+ * Aggregates the tracks active channels source and sink under {@link REQUESTED_TRACKS_SESSION_KEY} so the
2866
+ * driver can negotiate them (e.g., inbound video for Whiteboard / RemoteBrowser, outbound video for a
2867
+ * channel that shows the agent's video). A track the model does not support resolves to `'unsupported'`.
2868
+ * The session's transport and relay URL go to the driver as minted (an unknown transport is left out: direct).
1475
2869
  */
1476
2870
  BuildClientConfig(session) {
1477
2871
  const sessionConfig = this.parseSessionConfig(session.SessionConfigJson);
1478
- const channelTracks = this._activeChannels$.value.flatMap((c) => c.GetSourcedTracks());
2872
+ const channelTracks = this._activeChannels$.value.flatMap((c) => [...c.GetSourcedTracks(), ...c.GetSunkTracks()]);
1479
2873
  if (channelTracks.length > 0) {
1480
- // `requestedTracks` crosses a JSON boundary — the driver reads it back out of the session
2874
+ // The requested tracks cross a JSON boundary — the driver reads them back out of the session
1481
2875
  // config bag (`GeminiRealtimeClient.parseSessionConfig`). A `RealtimeTrackDescriptor` is NOT
1482
2876
  // structurally a `JSONValue`: it has no index signature and `UsageBasis` is readonly, so the
1483
2877
  // conversion is written out rather than asserted. Dedupe key and precedence are unchanged —
1484
2878
  // audio floor first, then anything the mint supplied, then the channels' own tracks.
1485
- const existing = Array.isArray(sessionConfig['requestedTracks'])
1486
- ? sessionConfig['requestedTracks']
2879
+ const existing = Array.isArray(sessionConfig[REQUESTED_TRACKS_SESSION_KEY])
2880
+ ? sessionConfig[REQUESTED_TRACKS_SESSION_KEY]
1487
2881
  : [];
1488
2882
  const trackMap = new Map();
1489
2883
  for (const t of DEFAULT_REALTIME_AUDIO_TRACKS) {
@@ -1498,13 +2892,16 @@ export class RealtimeSessionRuntime {
1498
2892
  for (const t of channelTracks) {
1499
2893
  trackMap.set(`${t.Direction}:${t.Modality}`, trackDescriptorToJSON(t));
1500
2894
  }
1501
- sessionConfig['requestedTracks'] = Array.from(trackMap.values());
2895
+ sessionConfig[REQUESTED_TRACKS_SESSION_KEY] = Array.from(trackMap.values());
1502
2896
  }
2897
+ const transport = ParseRealtimeClientTransport(session.Transport);
1503
2898
  return {
1504
2899
  Provider: session.Provider,
1505
2900
  Model: session.Model,
1506
2901
  EphemeralToken: session.EphemeralToken,
1507
2902
  ExpiresAt: session.ExpiresAt,
2903
+ ...(transport ? { Transport: transport } : {}),
2904
+ ...(session.RelayUrl ? { RelayUrl: session.RelayUrl } : {}),
1508
2905
  SessionConfig: sessionConfig
1509
2906
  };
1510
2907
  }
@@ -1540,6 +2937,9 @@ export class RealtimeSessionRuntime {
1540
2937
  });
1541
2938
  client.OnError((error) => {
1542
2939
  console.error('[RealtimeSession] Provider error event:', JSON.stringify(error), error);
2940
+ if (error.Fatal && this.client === client) {
2941
+ this.lastSessionError = error;
2942
+ }
1543
2943
  });
1544
2944
  // Usage telemetry: accumulate the driver's per-response token DELTAS and relay them to
1545
2945
  // the server (onto the co-agent AIPromptRun) debounced + once at teardown. Providers
@@ -1556,12 +2956,36 @@ export class RealtimeSessionRuntime {
1556
2956
  client.OnInterruption(() => {
1557
2957
  this.cancelPendingNarration();
1558
2958
  });
2959
+ client.OnRemoteVideo((video) => this.onAgentVideo(client, video));
2960
+ }
2961
+ /**
2962
+ * The model's video arrived: publish it on {@link AgentVideo$}, and mark each channel that shows it (one that sinks
2963
+ * outbound video) as used, so the host shows its surface. A client the session has already let go of is ignored, so a
2964
+ * late frame cannot bring the video back after the call.
2965
+ */
2966
+ onAgentVideo(client, video) {
2967
+ if (this.client !== client) {
2968
+ return;
2969
+ }
2970
+ this._agentVideo$.next(video);
2971
+ for (const channel of this._activeChannels$.value) {
2972
+ if (channel.ShowsAgentVideo) {
2973
+ this.noteChannelActivity(channel);
2974
+ }
2975
+ }
2976
+ }
2977
+ /** Clears {@link AgentVideo$} when the call's client goes. */
2978
+ clearAgentVideo() {
2979
+ if (this._agentVideo$.value !== null) {
2980
+ this._agentVideo$.next(null);
2981
+ }
1559
2982
  }
1560
2983
  /** Maps a client state event onto the UI connection state. */
1561
2984
  onClientStateChange(state) {
1562
2985
  const mapped = this.mapClientState(state);
1563
2986
  if (mapped) {
1564
- this._connectionState$.next(mapped);
2987
+ this.setConnectionState(mapped);
2988
+ this.flushChannelCatalogNote();
1565
2989
  }
1566
2990
  }
1567
2991
  /**
@@ -1590,6 +3014,17 @@ export class RealtimeSessionRuntime {
1590
3014
  const state = this._connectionState$.value;
1591
3015
  return state === 'listening' || state === 'speaking' || state === 'thinking';
1592
3016
  }
3017
+ /**
3018
+ * Moves {@link ConnectionState$} to `state`, and {@link Resuming$} first: a `'connecting'` while the call is live, or
3019
+ * already resuming, is a resume; any other state ends one. Every state change goes through here.
3020
+ */
3021
+ setConnectionState(state) {
3022
+ const resuming = state === 'connecting' && (this._resuming$.value || this.isSessionLive());
3023
+ if (resuming !== this._resuming$.value) {
3024
+ this._resuming$.next(resuming);
3025
+ }
3026
+ this._connectionState$.next(state);
3027
+ }
1593
3028
  // ── Transcript policy ──────────────────────────────────────────────────────
1594
3029
  /**
1595
3030
  * Applies transcript policy to client transcript events. Interim deltas don't become
@@ -1779,7 +3214,7 @@ export class RealtimeSessionRuntime {
1779
3214
  void this.relayToolTurn(call.ToolName, call.ArgumentsJson, resultJson);
1780
3215
  return;
1781
3216
  }
1782
- this._connectionState$.next('thinking');
3217
+ this.setConnectionState('thinking');
1783
3218
  if (this.inFlightCallIds.size === 0) {
1784
3219
  // A fresh delegation burst: anchor the first-update delay and clear the digest
1785
3220
  // buffer. Deliberately NOT reset: lastDelegationNarrationAt (the 8s spacing floor
@@ -1876,6 +3311,40 @@ export class RealtimeSessionRuntime {
1876
3311
  RunID: parsed.RunID,
1877
3312
  Artifacts: parsed.Artifacts
1878
3313
  });
3314
+ if (parsed.Success && parsed.Artifacts && parsed.Artifacts.length > 0) {
3315
+ void this.offerDelegationArtifacts(parsed.Artifacts);
3316
+ }
3317
+ }
3318
+ /**
3319
+ * Offers a delegated run's artifacts to the channels that host artifacts (an Interactive Component channel
3320
+ * shows a component artifact, or swaps in a newer version of one it already shows). Each channel is asked
3321
+ * first whether it wants them; an advertised, unopened channel that does is mounted before it is handed
3322
+ * them. One channel failing never stops the others, and nothing here can fail the delegation result,
3323
+ * which has already been emitted.
3324
+ */
3325
+ async offerDelegationArtifacts(artifacts) {
3326
+ const candidates = [
3327
+ ...this._activeChannels$.value.map((plugin) => ({ Plugin: plugin, IsOpen: true })),
3328
+ ...this.advertisedChannels.map((prepared) => ({ Plugin: prepared.Plugin, IsOpen: false }))
3329
+ ];
3330
+ for (const { Plugin: plugin, IsOpen: isOpen } of candidates) {
3331
+ try {
3332
+ const config = this.GetResolvedChannel(plugin.ChannelName)?.Config ?? {};
3333
+ if (!plugin.AcceptsDelegationArtifacts(artifacts, config)) {
3334
+ continue;
3335
+ }
3336
+ if (!isOpen) {
3337
+ await this.mountAdvertisedChannel(plugin);
3338
+ }
3339
+ await plugin.OnDelegationArtifacts(artifacts);
3340
+ if (!isOpen) {
3341
+ this.noteChannelActivity(plugin); // the channel was mounted for this: reveal its tab
3342
+ }
3343
+ }
3344
+ catch (error) {
3345
+ console.error(`[RealtimeSession] Channel '${plugin.ChannelName}' failed to take a delegated run's artifacts:`, error);
3346
+ }
3347
+ }
1879
3348
  }
1880
3349
  // ── Explicit delegation cancellation (server cancel channel) ───────────────
1881
3350
  /**
@@ -1966,44 +3435,34 @@ export class RealtimeSessionRuntime {
1966
3435
  }
1967
3436
  }
1968
3437
  // ── Session minting (GraphQL) ──────────────────────────────────────────────
1969
- /** Calls the `StartRealtimeClientSession` mutation to obtain an ephemeral token + config. */
1970
- async mintSession(targetAgentId, conversationId, lastSessionId, preferredModelId, clientTools, coAgentId, configOverridesJson, recordingConsent, recordingStartedAt, mediaCollectionId, applicationId, appContext) {
1971
- const mutation = `
1972
- mutation StartRealtimeClientSession($targetAgentId: String!, $conversationId: String, $lastSessionId: String, $preferredModelId: String, $clientToolsJson: String, $coAgentId: String, $configOverridesJson: String, $recordingConsent: Boolean, $recordingStartedAt: String, $mediaCollectionId: String, $applicationId: String, $appContextJson: String) {
1973
- StartRealtimeClientSession(targetAgentId: $targetAgentId, conversationId: $conversationId, lastSessionId: $lastSessionId, preferredModelId: $preferredModelId, clientToolsJson: $clientToolsJson, coAgentId: $coAgentId, configOverridesJson: $configOverridesJson, recordingConsent: $recordingConsent, recordingStartedAt: $recordingStartedAt, mediaCollectionId: $mediaCollectionId, applicationId: $applicationId, appContextJson: $appContextJson) {
1974
- AgentSessionId
1975
- ConversationId
1976
- Provider
1977
- Model
1978
- EphemeralToken
1979
- ExpiresAt
1980
- SessionConfigJson
1981
- ModelName
1982
- NarrationInstructionsTemplate
1983
- PriorChannelStatesJson
1984
- }
1985
- }
1986
- `;
1987
- const variables = {
1988
- targetAgentId,
1989
- conversationId: conversationId ?? null,
1990
- lastSessionId: lastSessionId ?? null,
1991
- preferredModelId: preferredModelId ?? null,
1992
- clientToolsJson: clientTools && clientTools.length > 0 ? JSON.stringify(clientTools) : null,
1993
- coAgentId: coAgentId ?? null,
1994
- configOverridesJson: configOverridesJson ?? null,
1995
- recordingConsent: recordingConsent ?? false,
1996
- recordingStartedAt: recordingStartedAt ?? null,
1997
- mediaCollectionId: mediaCollectionId ?? null,
1998
- applicationId: applicationId ?? null,
1999
- appContextJson: appContext ? JSON.stringify(appContext) : null
2000
- };
2001
- const result = await this.gql().ExecuteGQL(mutation, variables);
2002
- const payload = result?.StartRealtimeClientSession;
2003
- if (!payload?.EphemeralToken) {
2004
- throw new Error('StartRealtimeClientSession returned no ephemeral token');
2005
- }
2006
- return payload;
3438
+ /**
3439
+ * Mints a session through the installed {@link Launcher} — by default the stock
3440
+ * `StartRealtimeClientSession` mutation (see {@link DefaultRealtimeSessionLauncher}). `showsAgentVideo` is whether
3441
+ * one of the session's channels could show the agent's video; without one the server asks the model for no avatar.
3442
+ * `conversationBranchId` is the fork the session starts on; `null` is Main.
3443
+ */
3444
+ async mintSession(targetAgentId, conversationId, lastSessionId, preferredModelId, clientTools, coAgentId, configOverridesJson, recordingConsent, recordingStartedAt, mediaCollectionId, applicationId, appContext, channelCandidatesJson, showsAgentVideo, conversationBranchId) {
3445
+ const result = await this._launcher.Launch({
3446
+ TargetAgentId: targetAgentId,
3447
+ ConversationId: conversationId ?? null,
3448
+ LastSessionId: lastSessionId ?? null,
3449
+ PreferredModelId: preferredModelId ?? null,
3450
+ ClientTools: clientTools ?? [],
3451
+ CoAgentId: coAgentId ?? null,
3452
+ ConfigOverridesJson: configOverridesJson ?? null,
3453
+ RecordingConsent: recordingConsent ?? false,
3454
+ RecordingStartedAt: recordingStartedAt ?? null,
3455
+ MediaCollectionId: mediaCollectionId ?? null,
3456
+ ApplicationId: applicationId ?? null,
3457
+ AppContext: appContext ?? null,
3458
+ ChannelCandidatesJson: channelCandidatesJson ?? null,
3459
+ ConversationBranchId: conversationBranchId ?? null,
3460
+ ...(showsAgentVideo === undefined ? {} : { ShowsAgentVideo: showsAgentVideo })
3461
+ }, { Provider: this.Provider });
3462
+ if (!HasClientCredential(result)) {
3463
+ throw new Error('The session launcher returned no ephemeral token (or, for a relay session, no relay URL)');
3464
+ }
3465
+ return result;
2007
3466
  }
2008
3467
  /** Calls the `ExecuteRealtimeSessionTool` mutation; returns the ResultJson string. */
2009
3468
  async executeSessionTool(callId, toolName, argsJson) {
@@ -2142,16 +3601,28 @@ export class RealtimeSessionRuntime {
2142
3601
  /**
2143
3602
  * Accumulates one usage DELTA from the realtime client (per-response token counts —
2144
3603
  * the `OnUsage` contract shape) and schedules the debounced relay. Negative / non-finite
2145
- * values are clamped to 0; an all-zero delta is dropped without arming the timer.
3604
+ * values are clamped to 0; an all-zero delta is dropped without arming the timer. The
3605
+ * update's input and output detail blocks add into the pending details by the record's
3606
+ * rules (amounts add up, inbound video running totals keep the larger value), so an update
3607
+ * that carries only avatar video seconds still counts.
2146
3608
  */
2147
3609
  onUsageDelta(usage) {
2148
3610
  const input = this.clampUsageDelta(usage.InputTokens);
2149
3611
  const output = this.clampUsageDelta(usage.OutputTokens);
2150
- if (input === 0 && output === 0) {
3612
+ const details = AddRealtimeUsageRecord(null, { Input: usage.InputTokenDetails, Output: usage.OutputTokenDetails });
3613
+ const hasDetails = HasRealtimeUsage(details);
3614
+ if (input === 0 && output === 0 && !hasDetails) {
2151
3615
  return;
2152
3616
  }
2153
3617
  this.pendingUsageInput += input;
2154
3618
  this.pendingUsageOutput += output;
3619
+ if (hasDetails) {
3620
+ this.pendingUsageDetails = AddRealtimeUsageRecord(this.pendingUsageDetails, details);
3621
+ }
3622
+ this.armUsageFlush();
3623
+ }
3624
+ /** Schedules the debounced usage relay unless one is already pending. */
3625
+ armUsageFlush() {
2155
3626
  if (!this.usageFlushTimer) {
2156
3627
  this.usageFlushTimer = setTimeout(() => {
2157
3628
  this.usageFlushTimer = null;
@@ -2176,24 +3647,26 @@ export class RealtimeSessionRuntime {
2176
3647
  const sessionId = agentSessionId ?? this.agentSessionId;
2177
3648
  const input = this.pendingUsageInput;
2178
3649
  const output = this.pendingUsageOutput;
2179
- if (!sessionId || (input === 0 && output === 0)) {
3650
+ const details = this.pendingUsageDetails;
3651
+ if (!sessionId || (input === 0 && output === 0 && !details)) {
2180
3652
  return;
2181
3653
  }
2182
3654
  this.pendingUsageInput = 0;
2183
3655
  this.pendingUsageOutput = 0;
3656
+ this.pendingUsageDetails = null;
2184
3657
  try {
2185
- const mutation = `
2186
- mutation RelayRealtimeUsage($agentSessionId: String!, $inputTokens: Int!, $outputTokens: Int!) {
2187
- RelayRealtimeUsage(agentSessionId: $agentSessionId, inputTokens: $inputTokens, outputTokens: $outputTokens)
2188
- }
2189
- `;
2190
- await this.gql().ExecuteGQL(mutation, { agentSessionId: sessionId, inputTokens: input, outputTokens: output });
3658
+ // The details argument is sent only when there are details, so a token-only relay is unchanged.
3659
+ const variables = { agentSessionId: sessionId, inputTokens: input, outputTokens: output, ...(details ? { usageDetailsJson: JSON.stringify(details) } : {}) };
3660
+ await this.gql().ExecuteGQL(RELAY_REALTIME_USAGE_MUTATION, variables);
2191
3661
  }
2192
3662
  catch (error) {
2193
3663
  console.error('[RealtimeSession] Failed to relay usage telemetry:', error);
2194
3664
  // Re-accumulate so a later debounce / the teardown flush retries the same deltas.
2195
3665
  this.pendingUsageInput += input;
2196
3666
  this.pendingUsageOutput += output;
3667
+ if (details) {
3668
+ this.pendingUsageDetails = AddRealtimeUsageRecord(this.pendingUsageDetails, details);
3669
+ }
2197
3670
  }
2198
3671
  }
2199
3672
  /** Cancels the pending debounced usage flush and zeroes the accumulators (teardown tail). */
@@ -2204,6 +3677,7 @@ export class RealtimeSessionRuntime {
2204
3677
  }
2205
3678
  this.pendingUsageInput = 0;
2206
3679
  this.pendingUsageOutput = 0;
3680
+ this.pendingUsageDetails = null;
2207
3681
  }
2208
3682
  // ── Delegated-run progress streaming ───────────────────────────────────────
2209
3683
  /**
@@ -2483,42 +3957,66 @@ export class RealtimeSessionRuntime {
2483
3957
  return;
2484
3958
  }
2485
3959
  this.teardownInFlight = this.runTeardown(closeServerSession).finally(() => {
3960
+ if (this._savingRecording$.value) {
3961
+ this._savingRecording$.next(false);
3962
+ }
2486
3963
  this.teardownInFlight = null;
2487
3964
  });
2488
3965
  await this.teardownInFlight;
2489
3966
  }
2490
3967
  /** The body of {@link teardown}; never called concurrently with itself. */
2491
3968
  async runTeardown(closeServerSession) {
3969
+ // Arm BEFORE any await so a tab closed in the first milliseconds of teardown is still guarded.
3970
+ if (this.recorder) {
3971
+ this._savingRecording$.next(true);
3972
+ }
2492
3973
  // First: stop asserting liveness. A pulse racing the close would re-stamp LastActiveAt on a
2493
3974
  // session we are deliberately ending, leaving an Idle row the janitor then has to age out.
2494
3975
  this.stopLivenessPulse();
2495
3976
  this.teardownDelegationProgress();
3977
+ this.stopSessionEvents();
3978
+ this.clientDeadline.Clear();
2496
3979
  // Channels first: flush any unsaved channel state WHILE the live session id is still
2497
3980
  // set (the captured per-save id covers the race anyway), then dispose the plugins.
2498
3981
  this.flushAllChannelSaves();
2499
3982
  this.disposeChannels();
3983
+ // The camera and screen share go first: the camera runs on the controller closed next.
3984
+ this.closeCaptures();
2500
3985
  // Defensive: stop the mic even when Connect never ran (the client also stops the
2501
3986
  // tracks it was handed — track.stop() is idempotent).
3987
+ this.closeLocalMedia();
2502
3988
  this.localStream?.getTracks().forEach(t => t.stop());
2503
3989
  this.localStream = null;
2504
- // Hand the platform back whatever acquiring the microphone changed. Stopping the tracks is not
3990
+ this.microphoneMuted = false;
3991
+ // Hand the platform back whatever opening the microphone changed. Stopping the tracks is not
2505
3992
  // the same thing: iOS, for instance, is put into a record-and-play audio category for the call,
2506
3993
  // and leaving it there changes the route and volume behaviour of every sound the app makes
2507
- // afterwards. Best-effort by contract — a failure here must never block ending a call.
2508
- try {
2509
- await this.mediaHost.ReleaseMicrophone?.();
2510
- }
2511
- catch (error) {
2512
- console.error('[RealtimeSession] Media host failed to release the microphone:', error);
3994
+ // afterwards. A microphone still opening is left to its start, which releases it once the
3995
+ // opening returns ({@link microphoneLease}).
3996
+ const lease = this.microphoneLease;
3997
+ if (lease && !lease.Opening) {
3998
+ this.microphoneLease = null;
3999
+ await this.releaseMicrophone();
2513
4000
  }
2514
4001
  if (this.client) {
2515
- await this.client.Disconnect();
2516
- this.client = null;
4002
+ // A stranded Active session is worse than a noisy socket: a driver that throws on Disconnect
4003
+ // (socket already gone) must not stop the server session from being closed below.
4004
+ try {
4005
+ await this.client.Disconnect();
4006
+ }
4007
+ catch (error) {
4008
+ console.error(`[RealtimeSession] Driver failed to disconnect (session ${this.agentSessionId}); continuing teardown:`, error);
4009
+ }
4010
+ finally {
4011
+ this.unwatchVideoSources();
4012
+ this.clearAgentVideo();
4013
+ this.client = null;
4014
+ }
2517
4015
  }
2518
- // Stop + upload the call recording WHILE the live session id is still set (the file is
2519
- // attached to it). Best-effort and never blocks teardown — stopAndUploadRecording swallows
2520
- // its own errors. No-op when nothing was recorded.
2521
- await this.stopAndUploadRecording(this.agentSessionId);
4016
+ // Stop + encode the call recording now (the recorder must be detached before the session id is
4017
+ // cleared); the upload itself happens after the close below. Best-effort — stopRecording
4018
+ // swallows its own errors. Null when nothing was recorded.
4019
+ const recording = await this.stopRecording();
2522
4020
  this.recordingStartedAtIso = null;
2523
4021
  // Final usage flush WHILE the live session id is still set (the relay mutation also
2524
4022
  // accepts a Closed session, so ordering vs. CloseAgentSession is belt-and-braces).
@@ -2531,18 +4029,27 @@ export class RealtimeSessionRuntime {
2531
4029
  if (closeServerSession && this.agentSessionId) {
2532
4030
  await this.closeServerSession(this.agentSessionId);
2533
4031
  }
4032
+ // Upload AFTER the close. The WAV is tens of MB and takes seconds; a page unload aborts the
4033
+ // request, and nothing after an aborted await ever runs — so the small, critical close goes
4034
+ // first (#5195). The server accepts the upload on a Closed session (loadOwnedSession does not
4035
+ // check status), and uploadStoppedRecording never throws.
4036
+ await this.uploadStoppedRecording(this.agentSessionId, recording);
2534
4037
  // Capture the session id BEFORE we null it so the lifecycle emit carries it.
2535
4038
  // Skip emitting when there was no live session (defensive — teardown is safe
2536
4039
  // to call without an active session).
2537
4040
  const closedSessionId = this.agentSessionId;
2538
4041
  this.agentSessionId = null;
4042
+ this.sessionConversationId = null;
2539
4043
  this.narrationTemplate = null;
2540
4044
  this.clientToolHandlers.clear();
2541
4045
  this._modelName$.next(null);
4046
+ if (this._avatarNotice$.value !== null) {
4047
+ this._avatarNotice$.next(null);
4048
+ }
2542
4049
  this.SetMinimized(false);
2543
4050
  this._active$.next(false);
2544
4051
  if (this._connectionState$.value !== 'error') {
2545
- this._connectionState$.next('closed');
4052
+ this.setConnectionState('closed');
2546
4053
  }
2547
4054
  // Surface generic session-ended for the conversations runtime bridge.
2548
4055
  // `closeServerSession=true` means the user explicitly called EndRealtimeSession;
@@ -2554,6 +4061,18 @@ export class RealtimeSessionRuntime {
2554
4061
  });
2555
4062
  }
2556
4063
  }
4064
+ /**
4065
+ * Calls the host's {@link IRealtimeMediaHost.ReleaseMicrophone}. Best-effort by contract: a failure
4066
+ * is logged and never thrown, because it must never block ending a call.
4067
+ */
4068
+ async releaseMicrophone() {
4069
+ try {
4070
+ await this.mediaHost.ReleaseMicrophone?.();
4071
+ }
4072
+ catch (error) {
4073
+ console.error('[RealtimeSession] Media host failed to release the microphone:', error);
4074
+ }
4075
+ }
2557
4076
  /** Calls the `CloseAgentSession` mutation (provisioned in P4b). */
2558
4077
  async closeServerSession(agentSessionId) {
2559
4078
  try {
@@ -2580,12 +4099,13 @@ export class RealtimeSessionRuntime {
2580
4099
  this.hasActiveInterimUserCaption = false;
2581
4100
  this.SetMinimized(false);
2582
4101
  this.stopSegmentFlushing();
2583
- this.segmentIndex = 0;
4102
+ this.shardQueue = null;
2584
4103
  this.recorder = null;
2585
4104
  this.recordingStartedAtIso = null;
2586
4105
  this.currentTurnStartMs = null;
2587
4106
  this.turnAudioStartCaptured = false;
2588
4107
  this.usedChannelNames.clear();
4108
+ this.microphoneMuted = false;
2589
4109
  }
2590
4110
  /** The GraphQL provider used for relay mutations. */
2591
4111
  gql() {