@memberjunction/realtime-runtime 6.2.0-edge.3 → 6.2.0-edge.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/README.md +75 -0
  2. package/dist/channels/base-realtime-channel-client.d.ts +535 -28
  3. package/dist/channels/base-realtime-channel-client.d.ts.map +1 -1
  4. package/dist/channels/base-realtime-channel-client.js +864 -10
  5. package/dist/channels/base-realtime-channel-client.js.map +1 -1
  6. package/dist/channels/channel-action-dispatcher.d.ts +70 -0
  7. package/dist/channels/channel-action-dispatcher.d.ts.map +1 -0
  8. package/dist/channels/channel-action-dispatcher.js +136 -0
  9. package/dist/channels/channel-action-dispatcher.js.map +1 -0
  10. package/dist/channels/channel-catalog-note.d.ts +48 -0
  11. package/dist/channels/channel-catalog-note.d.ts.map +1 -0
  12. package/dist/channels/channel-catalog-note.js +119 -0
  13. package/dist/channels/channel-catalog-note.js.map +1 -0
  14. package/dist/channels/channel-contract-types.d.ts +107 -0
  15. package/dist/channels/channel-contract-types.d.ts.map +1 -0
  16. package/dist/channels/channel-contract-types.js +12 -0
  17. package/dist/channels/channel-contract-types.js.map +1 -0
  18. package/dist/channels/channel-descriptor-synthesis.d.ts +56 -0
  19. package/dist/channels/channel-descriptor-synthesis.d.ts.map +1 -0
  20. package/dist/channels/channel-descriptor-synthesis.js +90 -0
  21. package/dist/channels/channel-descriptor-synthesis.js.map +1 -0
  22. package/dist/channels/channel-perception.d.ts +82 -0
  23. package/dist/channels/channel-perception.d.ts.map +1 -0
  24. package/dist/channels/channel-perception.js +120 -0
  25. package/dist/channels/channel-perception.js.map +1 -0
  26. package/dist/channels/channel-schema-format.d.ts +17 -0
  27. package/dist/channels/channel-schema-format.d.ts.map +1 -0
  28. package/dist/channels/channel-schema-format.js +45 -0
  29. package/dist/channels/channel-schema-format.js.map +1 -0
  30. package/dist/channels/channel-state-delta.d.ts +59 -0
  31. package/dist/channels/channel-state-delta.d.ts.map +1 -0
  32. package/dist/channels/channel-state-delta.js +116 -0
  33. package/dist/channels/channel-state-delta.js.map +1 -0
  34. package/dist/channels/channel-surface-placement.d.ts +25 -0
  35. package/dist/channels/channel-surface-placement.d.ts.map +1 -0
  36. package/dist/channels/channel-surface-placement.js +29 -0
  37. package/dist/channels/channel-surface-placement.js.map +1 -0
  38. package/dist/channels/channel-visual-pump.d.ts +126 -0
  39. package/dist/channels/channel-visual-pump.d.ts.map +1 -0
  40. package/dist/channels/channel-visual-pump.js +174 -0
  41. package/dist/channels/channel-visual-pump.js.map +1 -0
  42. package/dist/hosts/IRealtimeMediaHost.d.ts +86 -34
  43. package/dist/hosts/IRealtimeMediaHost.d.ts.map +1 -1
  44. package/dist/hosts/IRealtimeMediaHost.js +2 -1
  45. package/dist/hosts/IRealtimeMediaHost.js.map +1 -1
  46. package/dist/index.d.ts +26 -3
  47. package/dist/index.d.ts.map +1 -1
  48. package/dist/index.js +28 -1
  49. package/dist/index.js.map +1 -1
  50. package/dist/session/RealtimeSessionRuntime.d.ts +984 -111
  51. package/dist/session/RealtimeSessionRuntime.d.ts.map +1 -1
  52. package/dist/session/RealtimeSessionRuntime.js +2396 -381
  53. package/dist/session/RealtimeSessionRuntime.js.map +1 -1
  54. package/dist/session/RecordingShardDrain.d.ts +59 -0
  55. package/dist/session/RecordingShardDrain.d.ts.map +1 -0
  56. package/dist/session/RecordingShardDrain.js +135 -0
  57. package/dist/session/RecordingShardDrain.js.map +1 -0
  58. package/dist/session/RecordingShardQueue.d.ts +94 -0
  59. package/dist/session/RecordingShardQueue.d.ts.map +1 -0
  60. package/dist/session/RecordingShardQueue.js +176 -0
  61. package/dist/session/RecordingShardQueue.js.map +1 -0
  62. package/dist/session/RecordingShardStore.d.ts +57 -0
  63. package/dist/session/RecordingShardStore.d.ts.map +1 -0
  64. package/dist/session/RecordingShardStore.js +184 -0
  65. package/dist/session/RecordingShardStore.js.map +1 -0
  66. package/dist/session/RecordingShardWindow.d.ts +42 -0
  67. package/dist/session/RecordingShardWindow.d.ts.map +1 -0
  68. package/dist/session/RecordingShardWindow.js +88 -0
  69. package/dist/session/RecordingShardWindow.js.map +1 -0
  70. package/dist/session/RecordingTransport.d.ts +66 -0
  71. package/dist/session/RecordingTransport.d.ts.map +1 -0
  72. package/dist/session/RecordingTransport.js +61 -0
  73. package/dist/session/RecordingTransport.js.map +1 -0
  74. package/dist/session/app-client-tool-registry.d.ts +76 -0
  75. package/dist/session/app-client-tool-registry.d.ts.map +1 -0
  76. package/dist/session/app-client-tool-registry.js +100 -0
  77. package/dist/session/app-client-tool-registry.js.map +1 -0
  78. package/dist/session/avatar-notice.d.ts +43 -0
  79. package/dist/session/avatar-notice.d.ts.map +1 -0
  80. package/dist/session/avatar-notice.js +60 -0
  81. package/dist/session/avatar-notice.js.map +1 -0
  82. package/dist/session/call-end.d.ts +21 -0
  83. package/dist/session/call-end.d.ts.map +1 -0
  84. package/dist/session/call-end.js +19 -0
  85. package/dist/session/call-end.js.map +1 -0
  86. package/dist/session/channel-exposure-preferences.d.ts +70 -0
  87. package/dist/session/channel-exposure-preferences.d.ts.map +1 -0
  88. package/dist/session/channel-exposure-preferences.js +138 -0
  89. package/dist/session/channel-exposure-preferences.js.map +1 -0
  90. package/dist/session/channel-session-scope.d.ts +142 -0
  91. package/dist/session/channel-session-scope.d.ts.map +1 -0
  92. package/dist/session/channel-session-scope.js +128 -0
  93. package/dist/session/channel-session-scope.js.map +1 -0
  94. package/dist/session/client-session-deadline.d.ts +28 -0
  95. package/dist/session/client-session-deadline.d.ts.map +1 -0
  96. package/dist/session/client-session-deadline.js +50 -0
  97. package/dist/session/client-session-deadline.js.map +1 -0
  98. package/dist/session/realtime-captures.d.ts +236 -0
  99. package/dist/session/realtime-captures.d.ts.map +1 -0
  100. package/dist/session/realtime-captures.js +383 -0
  101. package/dist/session/realtime-captures.js.map +1 -0
  102. package/dist/session/realtime-microphone.d.ts +50 -0
  103. package/dist/session/realtime-microphone.d.ts.map +1 -0
  104. package/dist/session/realtime-microphone.js +51 -0
  105. package/dist/session/realtime-microphone.js.map +1 -0
  106. package/dist/session/session-event-hub.d.ts +72 -0
  107. package/dist/session/session-event-hub.d.ts.map +1 -0
  108. package/dist/session/session-event-hub.js +151 -0
  109. package/dist/session/session-event-hub.js.map +1 -0
  110. package/dist/session/session-launcher.d.ts +128 -0
  111. package/dist/session/session-launcher.d.ts.map +1 -0
  112. package/dist/session/session-launcher.js +160 -0
  113. package/dist/session/session-launcher.js.map +1 -0
  114. package/package.json +10 -9
@@ -1,13 +1,48 @@
1
1
  import { BehaviorSubject, Subject } from 'rxjs';
2
2
  import { Metadata } from '@memberjunction/core';
3
3
  import { UserInfoEngine } from '@memberjunction/core-entities';
4
- import { AIEngineBase } from '@memberjunction/ai-engine-base';
5
- import { MJGlobal } from '@memberjunction/global';
6
- import { DEFAULT_REALTIME_AUDIO_TRACKS } from '@memberjunction/ai';
7
- import { BaseRealtimeClient, LoadAssemblyAIRealtimeClient, LoadElevenLabsRealtimeClient, LoadGeminiRealtimeClient, LoadHuggingFaceRealtimeClient, LoadOpenAIRealtimeClient, LoadxAIRealtimeClient, REQUESTED_TRACKS_SESSION_KEY } from '@memberjunction/ai-realtime-client';
4
+ import { AddRealtimeUsageRecord, AIEngineBase, HasRealtimeUsage } from '@memberjunction/ai-engine-base';
5
+ import { GraphQLRealtimeSessionClient } from '@memberjunction/graphql-dataprovider';
6
+ import { MJGlobal, UUIDsEqual } from '@memberjunction/global';
7
+ import { DEFAULT_REALTIME_AUDIO_TRACKS, ParseRealtimeAvatarStatus, ParseRealtimeClientTransport } from '@memberjunction/ai';
8
+ import { CompareExposure, DeclaresNativeTools, IsIdentityVerifiedEventPayload, NormalizeChannelKey, ParseRealtimeSessionClientPolicy, RECORDING_TERMINAL_MAX_BYTES, ResolveClientTools, SelectNativeChannelTools, UserExposureReason } from '@memberjunction/ai-core-plus';
9
+ import { BaseRealtimeClient, LoadAssemblyAIRealtimeClient, LoadElevenLabsRealtimeClient, LoadGeminiEnterpriseRealtimeClient, LoadGeminiRealtimeClient, LoadHuggingFaceRealtimeClient, LoadOpenAIRealtimeClient, LoadxAIRealtimeClient, VideoSourceArbiter, REQUESTED_TRACKS_SESSION_KEY } from '@memberjunction/ai-realtime-client';
10
+ import { DefaultRealtimeSessionLauncher, HasClientCredential } from './session-launcher.js';
11
+ import { RequestsAgentVideo, ResolveAvatarNotice } from './avatar-notice.js';
12
+ import { ClientSessionDeadline } from './client-session-deadline.js';
13
+ import { RealtimeCaptures, REALTIME_CAPTURES_OFF, REALTIME_CAPTURE_OFFERS_NONE, } from './realtime-captures.js';
14
+ import { REALTIME_MICROPHONE_NONE, ReadMicrophoneState, SameMicrophoneState } from './realtime-microphone.js';
15
+ import { RealtimeSessionEventHub } from './session-event-hub.js';
8
16
  import { BuildNarrationInstructions } from '../narration/narration-template.js';
9
17
  import { ParseDelegationResultJson, FormatToolName } from './delegation-result-parser.js';
10
- import { BaseRealtimeChannelClient } from '../channels/base-realtime-channel-client.js';
18
+ import { BaseRealtimeChannelClient, ChannelVideoSourceID } from '../channels/base-realtime-channel-client.js';
19
+ import { DEFAULT_CHANNEL_SURFACE_PLACEMENT, ReadChannelSurfacePlacement } from '../channels/channel-surface-placement.js';
20
+ import { ChannelActionDispatcher } from '../channels/channel-action-dispatcher.js';
21
+ import { BuildChannelCatalogNote } from '../channels/channel-catalog-note.js';
22
+ import { AppClientToolRegistry, DEFAULT_APP_TOOL_OWNER } from './app-client-tool-registry.js';
23
+ import { UserSettingsExposurePreferences } from './channel-exposure-preferences.js';
24
+ import { BuildChannelCandidate, FindPreparedChannel, MergeToolMetadata, PreparedChannelsShowAgentVideo, ReconcileChannelsWithPolicy, ResolveLocalChannelScope, ToolsByChannelKey } from './channel-session-scope.js';
25
+ import { RecordingShardQueue } from './RecordingShardQueue.js';
26
+ import { RecordingShardWindow } from './RecordingShardWindow.js';
27
+ import { CreateRecordingShardStore } from './RecordingShardStore.js';
28
+ import { CreateRecordingTransportForProvider, DeleteStoredRecordingShard, DrainStoredRecordingShardsOnce } from './RecordingShardDrain.js';
29
+ /**
30
+ * How long teardown waits for the server to acknowledge the recording's terminal request, measured
31
+ * from after `CloseAgentSession`. The terminal is `keepalive`, so it completes even if the page goes
32
+ * away; the wait only keeps `SavingRecording$` (and so a host's "don't close yet" guard) up while the
33
+ * server has not confirmed. Bounded so a dead server can never hold the End of a call hostage —
34
+ * unacknowledged shards stay in the local store and the server janitor assembles what arrived.
35
+ */
36
+ export const RECORDING_TERMINAL_ACK_TIMEOUT_MS = 3000;
37
+ /**
38
+ * How long teardown waits for the recorder to stop and flush its encoder. `Stop()` runs BEFORE
39
+ * `CloseAgentSession`, so a wedged encoder must not be able to hold the End of a call (and the
40
+ * server session) open; past this the recording ends at the packets already encoded.
41
+ */
42
+ export const RECORDER_STOP_TIMEOUT_MS = 2000;
43
+ function describeRecordingFailure(failure) {
44
+ return failure instanceof Error ? failure.message : String(failure);
45
+ }
11
46
  /**
12
47
  * `MJ: User Settings` key for the per-user "record this voice call" consent toggle. Stored as
13
48
  * the literal string `'true'`/`'false'` (read with `=== 'true'`), cross-device via
@@ -15,6 +50,15 @@ import { BaseRealtimeChannelClient } from '../channels/base-realtime-channel-cli
15
50
  * default when the caller doesn't pass an explicit consent value.
16
51
  */
17
52
  export const REALTIME_RECORDING_CONSENT_KEY = 'mj.realtimeVoice.recordingConsent.v1';
53
+ /**
54
+ * Relays usage onto the co-agent prompt run: token deltas, and the per-modality detail blocks as a
55
+ * JSON record (sent only when there are any; see `RelayRealtimeUsage` in MJServer).
56
+ */
57
+ const RELAY_REALTIME_USAGE_MUTATION = `
58
+ mutation RelayRealtimeUsage($agentSessionId: String!, $inputTokens: Int!, $outputTokens: Int!, $usageDetailsJson: String) {
59
+ RelayRealtimeUsage(agentSessionId: $agentSessionId, inputTokens: $inputTokens, outputTokens: $outputTokens, usageDetailsJson: $usageDetailsJson)
60
+ }
61
+ `;
18
62
  // Tree-shaking prevention: the OpenAI client is resolved dynamically through the
19
63
  // ClassFactory (by the server-reported Provider key), so this static call is what keeps
20
64
  // its @RegisterClass side effect from being eliminated by the bundler.
@@ -24,10 +68,37 @@ export const REALTIME_RECORDING_CONSENT_KEY = 'mj.realtimeVoice.recordingConsent
24
68
  // and this service stays component-free (it must stay importable in plain-node tests).
25
69
  LoadOpenAIRealtimeClient();
26
70
  LoadGeminiRealtimeClient();
71
+ LoadGeminiEnterpriseRealtimeClient();
27
72
  LoadElevenLabsRealtimeClient();
28
73
  LoadAssemblyAIRealtimeClient();
29
74
  LoadxAIRealtimeClient();
30
75
  LoadHuggingFaceRealtimeClient();
76
+ /**
77
+ * A registry row's surface placement. A row whose `UIConfig` is not valid JSON places its surface as a row without one
78
+ * does, so one bad row never costs the session its other channels.
79
+ */
80
+ function readRowSurfacePlacement(row) {
81
+ try {
82
+ return ReadChannelSurfacePlacement(row.UIConfigObject);
83
+ }
84
+ catch {
85
+ console.warn(`[RealtimeSession] Channel '${row.Name}' has a UIConfig that is not valid JSON; its surface starts on its tab.`);
86
+ return DEFAULT_CHANNEL_SURFACE_PLACEMENT;
87
+ }
88
+ }
89
+ /** The same, for a row read over GraphQL, whose `UIConfig` arrives as the column's JSON text. */
90
+ function readUIConfigTextSurfacePlacement(name, uiConfig) {
91
+ if (!uiConfig) {
92
+ return ReadChannelSurfacePlacement(null);
93
+ }
94
+ try {
95
+ return ReadChannelSurfacePlacement(JSON.parse(uiConfig));
96
+ }
97
+ catch {
98
+ console.warn(`[RealtimeSession] Channel '${name}' has a UIConfig that is not valid JSON; its surface starts on its tab.`);
99
+ return DEFAULT_CHANNEL_SURFACE_PLACEMENT;
100
+ }
101
+ }
31
102
  /**
32
103
  * Converts a {@link RealtimeTrackDescriptor} to its JSON form for the session-config bag.
33
104
  *
@@ -67,6 +138,37 @@ function trackKeyFromJSON(raw) {
67
138
  }
68
139
  return `${direction}:${modality}`;
69
140
  }
141
+ /** Whether a surface client tool handler's result is an object whose `Success` is `false`. */
142
+ function reportsFailure(result) {
143
+ return typeof result === 'object' && result !== null && 'Success' in result && result.Success === false;
144
+ }
145
+ /** The failed result's `ErrorMessage` when it is text that is not blank, else `'Tool failed'`. */
146
+ function failureMessage(result) {
147
+ const message = result.ErrorMessage;
148
+ return typeof message === 'string' && message.trim() ? message : 'Tool failed';
149
+ }
150
+ /**
151
+ * The `DOMException` name for each way the controller can fail to start the microphone, so a host that
152
+ * tells a denied microphone apart (`error.name === 'NotAllowedError'`) works the same with or without one.
153
+ */
154
+ const MICROPHONE_ERROR_NAMES = {
155
+ denied: 'NotAllowedError',
156
+ 'not-found': 'NotFoundError',
157
+ 'in-use': 'NotReadableError',
158
+ unsupported: 'NotSupportedError',
159
+ error: 'Error',
160
+ };
161
+ /** What the user is told when the call's policy refuses a capture. */
162
+ const CAPTURE_WORDING = {
163
+ camera: { NotInCall: 'The camera is not part of this call.', NotAllowed: 'This call cannot show the agent your camera' },
164
+ screen: { NotInCall: 'Screen sharing is not part of this call.', NotAllowed: 'This call cannot show the agent your screen' },
165
+ };
166
+ /** The error a failed microphone start reports, named as `getUserMedia` would have named it. */
167
+ function microphoneStartError(reason, message) {
168
+ const error = new Error(message);
169
+ error.name = MICROPHONE_ERROR_NAMES[reason];
170
+ return error;
171
+ }
70
172
  /**
71
173
  * Drives a **client-direct** real-time voice session: the browser mints an ephemeral
72
174
  * token from the MJ server, then connects DIRECTLY to the realtime provider. Audio
@@ -92,14 +194,15 @@ function trackKeyFromJSON(raw) {
92
194
  export class RealtimeSessionRuntime {
93
195
  /**
94
196
  * @param mediaHost the platform's media capabilities. The runtime never touches `navigator`,
95
- * `Blob` or `FileReader` itself — microphone acquisition and audio recording
96
- * are the host's, because both are platform-specific product decisions
97
- * (permission UX, container format, where the bytes live).
197
+ * `Blob` or `FileReader` itself — microphone acquisition and audio capture/encoding
198
+ * are the host's, because both are platform-specific (permission UX, which
199
+ * encoder the platform offers).
98
200
  */
99
201
  constructor(mediaHost) {
100
202
  this.mediaHost = mediaHost;
101
203
  // ── Reactive UI state ──────────────────────────────────────────────────────
102
204
  this._connectionState$ = new BehaviorSubject('closed');
205
+ this._resuming$ = new BehaviorSubject(false);
103
206
  this._captions$ = new BehaviorSubject([]);
104
207
  this._active$ = new BehaviorSubject(false);
105
208
  this._delegationProgress$ = new Subject();
@@ -109,6 +212,7 @@ export class RealtimeSessionRuntime {
109
212
  this._agentName$ = new BehaviorSubject('Sage');
110
213
  this._modelName$ = new BehaviorSubject(null);
111
214
  this._minimized$ = new BehaviorSubject(false);
215
+ this._savingRecording$ = new BehaviorSubject(false);
112
216
  this._activeChannels$ = new BehaviorSubject([]);
113
217
  this._channelFocus$ = new Subject();
114
218
  // ─── Generic session-lifecycle events (consumed by RealtimeSessionsAdapter to
@@ -122,6 +226,16 @@ export class RealtimeSessionRuntime {
122
226
  this._channelActivity$ = new Subject();
123
227
  /** Current connection / turn state. */
124
228
  this.ConnectionState$ = this._connectionState$.asObservable();
229
+ /**
230
+ * Whether the call is resuming: moving to a new connection with its provider mid-call. `true` from a `'connecting'` the
231
+ * driver reports while the call is live, until the call is live again (`'listening'`, `'speaking'` or `'thinking'`),
232
+ * fails (`'error'`) or ends (`'closed'`). A call's first connect is not a resume, so it is `false` until the call has
233
+ * been live, and outside a call. Gemini Live resumes when Google ends a connection (`goAway`) and after a dropped
234
+ * socket, and the agent's video can stop for a few seconds meanwhile: a host can show that as a pause rather than as
235
+ * the agent leaving. It changes before {@link ConnectionState$} does, so a resume's `'connecting'` arrives with it
236
+ * already `true`.
237
+ */
238
+ this.Resuming$ = this._resuming$.asObservable();
125
239
  /** Live captions for both sides of the conversation. */
126
240
  this.Captions$ = this._captions$.asObservable();
127
241
  /** True while a session is open (mic button active, overlay shown). */
@@ -158,6 +272,15 @@ export class RealtimeSessionRuntime {
158
272
  * pure presentation state, reset to `false` at session start and teardown.
159
273
  */
160
274
  this.Minimized$ = this._minimized$.asObservable();
275
+ /**
276
+ * True while the end-of-call recording is being saved: from the first statement of a teardown
277
+ * that holds a recorder until that teardown settles — which includes waiting (at most
278
+ * {@link RECORDING_TERMINAL_ACK_TIMEOUT_MS}) for the server to acknowledge the recording's terminal
279
+ * request. False at construction and for unrecorded sessions. Exists for #5195; the runtime is
280
+ * platform-free, so hosts own the reaction (a "saving" indicator, a disabled navigation button,
281
+ * etc.).
282
+ */
283
+ this.SavingRecording$ = this._savingRecording$.asObservable();
161
284
  /**
162
285
  * The session's ACTIVE interactive-channel plugins, resolved from the `MJ: AI Agent
163
286
  * Channels` registry at session start (one instance per session, per channel). Emits
@@ -165,6 +288,57 @@ export class RealtimeSessionRuntime {
165
288
  * one surface tab per plugin — it never knows any concrete channel type.
166
289
  */
167
290
  this.ActiveChannels$ = this._activeChannels$.asObservable();
291
+ this._videoSources$ = new BehaviorSubject([]);
292
+ /**
293
+ * The video sources the agent is perceiving or could perceive right now (a whiteboard, a remote browser, a
294
+ * shared screen), with whether each is on and whether its frames are reaching the model. Empty when no
295
+ * session is live or nothing has offered a frame source. This is what the "agent can see" control renders;
296
+ * toggle one with {@link SetVideoSourceEnabled}.
297
+ */
298
+ this.VideoSources$ = this._videoSources$.asObservable();
299
+ this._captures$ = new BehaviorSubject(REALTIME_CAPTURES_OFF);
300
+ /**
301
+ * The user's camera and screen share: off, starting, on (with the stream to show the user) or failed (with the
302
+ * reason). Both off outside a session. Start and stop them with {@link StartCamera}, {@link StartScreenShare},
303
+ * {@link StopCamera} and {@link StopScreenShare}; while one is on, it is also a source on {@link VideoSources$}.
304
+ */
305
+ this.Captures$ = this._captures$.asObservable();
306
+ this._captureOffers$ = new BehaviorSubject(REALTIME_CAPTURE_OFFERS_NONE);
307
+ /**
308
+ * Which captures the call offers, now and on every change. A capture is offered while the call is connected, the model
309
+ * takes inbound video, the host can open it (a camera controller; screen sharing) and the call's policy admits it (its
310
+ * channel is in the call and may show the agent pixels). A host shows its Camera and Share controls from this. Neither
311
+ * is offered outside a call.
312
+ */
313
+ this.CaptureOffers$ = this._captureOffers$.asObservable();
314
+ this._microphone$ = new BehaviorSubject(REALTIME_MICROPHONE_NONE);
315
+ /**
316
+ * The call's microphone, now and on every change: the one in use and the ones the user can move it to with
317
+ * {@link SwitchMicrophone}, as the host's camera-and-microphone controller lists them. Offered from the moment the call
318
+ * connects until it ends, and kept current as devices come and go. When the call loses its microphone (the one in use
319
+ * fails and so does the fallback to the default, or the last one goes away), its `Failure` says why: the agent hears
320
+ * nothing until the user picks a microphone, which opens it again. {@link REALTIME_MICROPHONE_NONE} outside a call and
321
+ * on a host without a controller, which opens the microphone itself (`IRealtimeMediaHost.AcquireMicrophone`).
322
+ */
323
+ this.Microphone$ = this._microphone$.asObservable();
324
+ this._agentVideo$ = new BehaviorSubject(null);
325
+ /**
326
+ * The agent's video (an avatar) while the model sends it: a live stream, or a player that owns the `<video>` element.
327
+ * `null` until the video arrives, for a model that sends none, and outside a call. The driver hands it over once it is
328
+ * live, and a newer one replaces it. A host shows it with `AttachVideoSource`; a channel that sinks outbound video
329
+ * follows it through its context, and counts as used once it arrives, so the host shows that channel's surface.
330
+ */
331
+ this.AgentVideo$ = this._agentVideo$.asObservable();
332
+ this._avatarNotice$ = new BehaviorSubject(null);
333
+ /**
334
+ * Why the call shows no avatar the agent asked for: set once per call, after it connects, when the model renders none,
335
+ * the persona has no face for it, the app shows no agent video (`host`) or the browser can't play it (`browser`); with
336
+ * no `Reason` when the mint gave one this version doesn't know, such as a newer server's. `null` when there is nothing
337
+ * to say (no avatar asked for, the avatar shows, or the mint reported no status) and after the call ends. A provider
338
+ * resume inside the call does not set it again. A host turns the reason into words once, for example "Audio only: this
339
+ * voice model can't show an avatar".
340
+ */
341
+ this.AvatarNotice$ = this._avatarNotice$.asObservable();
168
342
  /**
169
343
  * Channel requests to enter / leave the FOCUS layout (see
170
344
  * {@link RealtimeChannelFocusEvent}). Fired when a plugin calls its host context's
@@ -208,7 +382,11 @@ export class RealtimeSessionRuntime {
208
382
  */
209
383
  /** Conversation id the SERVER created for this session (null when the host supplied one). */
210
384
  this.createdConversationId = null;
211
- /** The session's conversation id (supplied or server-created). */
385
+ /**
386
+ * The session's conversation id (supplied or server-created). While a start's mint runs, the
387
+ * conversation the start asked for; null when it asked the server to create one. Null after
388
+ * teardown.
389
+ */
212
390
  this.sessionConversationId = null;
213
391
  /** First final user utterance of the live session (the naming seed). */
214
392
  this.firstUserTranscript = null;
@@ -219,8 +397,28 @@ export class RealtimeSessionRuntime {
219
397
  // ── Session internals ──────────────────────────────────────────────────────
220
398
  /** The provider-direct realtime client driving the live session (ClassFactory-resolved). */
221
399
  this.client = null;
222
- /** The mic capture stream — acquired here (permission UX) and handed to the client. */
400
+ /**
401
+ * The mic capture stream — acquired here (permission UX) and handed to the client. A microphone the controller opens
402
+ * again after the call lost one replaces it.
403
+ */
223
404
  this.localStream = null;
405
+ /**
406
+ * Whether the user muted the call's microphone, as {@link ToggleMute} last reported it. While the call has no microphone
407
+ * track there is nothing to mute, and a microphone opened again after the call lost one starts this way.
408
+ */
409
+ this.microphoneMuted = false;
410
+ /** The host's camera-and-microphone controller for this session, when the host offers one. */
411
+ this.localMedia = null;
412
+ /** Follows the controller's microphone, so a swapped-in track or a reopened microphone reaches the driver and the recorder. */
413
+ this.localMediaSubscription = null;
414
+ /**
415
+ * Whether the call offers its microphone for switching ({@link Microphone$}, {@link SwitchMicrophone}): from the moment
416
+ * the client is connected until the call ends. Not before: a driver still connecting may have bound the old track.
417
+ */
418
+ this.microphoneOffered = false;
419
+ /** The live session's camera and screen share; created once the client is connected. */
420
+ this.captures = null;
421
+ this.capturesSubscription = null;
224
422
  this.agentSessionId = null;
225
423
  /**
226
424
  * The application the active session runs in (sources the server-side app config cascade +
@@ -246,16 +444,40 @@ export class RealtimeSessionRuntime {
246
444
  // ── Browser-side call recording ────────────────────────────────────────────
247
445
  /**
248
446
  * The active session's audio recorder (mic + agent mix), or `null` when the user didn't
249
- * consent or the browser can't record. Created after the client connects; stopped + uploaded
250
- * at teardown.
447
+ * consent or the browser can't record. Created after the client connects; its packets are
448
+ * shipped as 5 s shards during the call, and it is stopped at teardown.
251
449
  */
252
450
  this.recorder = null;
253
451
  /** ISO timestamp of when recording started — sent to the server on session start. */
254
452
  this.recordingStartedAtIso = null;
255
- /** Interval that flushes ~15s crash-recovery shards to the server during a recording. */
256
- this.segmentTimer = null;
257
- /** 0-based index of the next recording shard to upload. */
258
- this.segmentIndex = 0;
453
+ /** Interval that cuts and uploads a shard every {@link recordingShardFlushMs} during a recording. */
454
+ this.shardFlushTimer = null;
455
+ /**
456
+ * The current recording's shard pipeline (window → store → queue → transport), or null when not
457
+ * recording. A fresh pipeline per recording keeps a flush still in flight from a previous session
458
+ * off this session's indexes. Its queue is closed when the recording ends; anything it had not
459
+ * delivered stays in the local store for the next-load drain.
460
+ */
461
+ this.recordingShards = null;
462
+ /**
463
+ * Bytes of this page's recording `keepalive` requests still in flight. The browser's keepalive budget
464
+ * is per page, and closing a tab fires `visibilitychange` and `pagehide` back to back, so each dispatch
465
+ * budgets against what the previous one has not finished sending.
466
+ */
467
+ this.keepaliveBytesInFlight = 0;
468
+ /** `sessionId/index` of shards whose `keepalive` copy is still in flight; never sent twice at once. */
469
+ this.keepaliveShardsInFlight = new Set();
470
+ /** The durable local shard store, created on first use and shared by every recording of this runtime. */
471
+ this.shardStore = null;
472
+ /**
473
+ * True once a drain ran with a transport; the drain then does nothing until a recording ends and
474
+ * re-arms it ({@link rearmPendingShardDrain}).
475
+ */
476
+ this.pendingShardsDrained = false;
477
+ /** Bumped by every re-arm, so a drain already in flight cannot mark itself done over a re-arm. */
478
+ this.drainGeneration = 0;
479
+ /** The drain in flight, so overlapping callers share it instead of uploading the same shards twice. */
480
+ this.drainInFlight = null;
259
481
  // ── Server-side liveness ───────────────────────────────────────────────────
260
482
  /**
261
483
  * Interval that tells the server this session is still in use, or `null` when no session is
@@ -266,12 +488,33 @@ export class RealtimeSessionRuntime {
266
488
  * Recording-relative ms offset at which the IN-FLIGHT (not-yet-finalized) turn's audio
267
489
  * actually BEGAN — captured the moment that turn's audio/text starts flowing (its first
268
490
  * interim transcript), NOT inherited from the previous turn's end. `null` before the first
269
- * turn / between turns (until the next turn's audio starts). Sent as `utteranceStartMs` on
270
- * the turn's final transcript so per-turn timing lines up with the recording even when a
271
- * tool-call / silence gap sits between turns (the inherit-previous-end model mis-stamped
272
- * the post-gap turn at the pre-gap offset). See {@link markTurnAudioStart}.
491
+ * turn / between turns (until the next turn's audio starts), and it stays `null` for a turn
492
+ * whose start was never measured. Sent as `utteranceStartMs` on the turn's final transcript
493
+ * AS-IS — `null` is relayed as `null`, never replaced with 0 — so per-turn timing lines up
494
+ * with the recording even when a tool-call / silence gap sits between turns (the
495
+ * inherit-previous-end model mis-stamped the post-gap turn at the pre-gap offset). See
496
+ * {@link markTurnAudioStart}. A user speech onset signalled by the client does NOT land here;
497
+ * it has its own slot, {@link userSpeechStartMs}.
273
498
  */
274
499
  this.currentTurnStartMs = null;
500
+ /**
501
+ * Recording-relative ms offset of the first user speech onset the client signalled
502
+ * (`OnUserSpeechStart`) since the last final user transcript, or `null` when none was signalled.
503
+ * It has its own slot, apart from {@link currentTurnStartMs}, so a user onset never claims the
504
+ * assistant turn's start: OpenAI user transcripts arrive asynchronously, often after the
505
+ * assistant's interims (or its final) for the reply have already flowed. See
506
+ * {@link markUserSpeechStart}.
507
+ */
508
+ this.userSpeechStartMs = null;
509
+ /**
510
+ * `true` once an assistant turn has finalized after {@link userSpeechStartMs} was stamped BEFORE
511
+ * that turn began. The held onset may still be claimed by a late user transcript (OpenAI
512
+ * transcribes asynchronously), but the NEXT onset replaces it: an onset no transcript ever claimed
513
+ * (a VAD false positive, a failed transcription) must not become the start of the next real user
514
+ * turn. A barge-in onset (stamped during the assistant turn) is not superseded by that turn's
515
+ * final. An EMPTY user transcript releases the onset outright (see onClientTranscript).
516
+ */
517
+ this.userSpeechStartSuperseded = false;
275
518
  /**
276
519
  * Wall-anchor of the SESSION clock (#3832): `performance.now()` at the moment the call went
277
520
  * live, or `null` before any call has. Read only through {@link nowTurnOffsetMs}.
@@ -310,6 +553,11 @@ export class RealtimeSessionRuntime {
310
553
  this.pendingUsageInput = 0;
311
554
  /** Accumulated output-token delta since the last flush. */
312
555
  this.pendingUsageOutput = 0;
556
+ /**
557
+ * Accumulated per-modality usage since the last flush (input and output detail blocks, avatar
558
+ * video seconds included), relayed in full; `null` when no update carried any.
559
+ */
560
+ this.pendingUsageDetails = null;
313
561
  /** Pending debounced usage flush; also force-flushed at teardown. */
314
562
  this.usageFlushTimer = null;
315
563
  /** Active push-status subscription that feeds delegation progress; cleared on teardown. */
@@ -352,7 +600,20 @@ export class RealtimeSessionRuntime {
352
600
  */
353
601
  this.teardownInFlight = null;
354
602
  /**
355
- * Why the last session start failed, or `null` when none has.
603
+ * The latest start's opening of the host's microphone, until {@link IRealtimeMediaHost.ReleaseMicrophone} hands it
604
+ * back; `null` when no opening is owed a release, so a session that ends before asking for the microphone releases
605
+ * nothing.
606
+ *
607
+ * The host gets one release per opening, after the opening returns. A release while the microphone is still
608
+ * opening undoes nothing: the React Native host puts iOS into the call audio category inside `AcquireMicrophone`,
609
+ * after the permission prompt. So {@link runTeardown} releases an opening that has returned and leaves one still
610
+ * opening to its start, which releases it once it returns ({@link unwindAbandonedStart}), unless a newer start has
611
+ * opened the microphone since; that start's own end releases it. Whoever releases clears this first, so no
612
+ * opening is released twice.
613
+ */
614
+ this.microphoneLease = null;
615
+ /**
616
+ * Why the latest session start failed, or `null` when it has not. Each start clears it ({@link beginSessionStart}).
356
617
  *
357
618
  * The runtime reports failure as `'error'` on {@link ConnectionState$}, which is enough to show
358
619
  * *that* something went wrong but not *what* — and the difference matters at exactly one point:
@@ -361,6 +622,11 @@ export class RealtimeSessionRuntime {
361
622
  * runtime's own try/catch, so the host never sees that rejection itself.
362
623
  */
363
624
  this.lastStartError = null;
625
+ /**
626
+ * The fatal error the call's client reported, or `null` when it has reported none. Each start clears it
627
+ * ({@link beginSessionStart}). See {@link LastSessionError}.
628
+ */
629
+ this.lastSessionError = null;
364
630
  /**
365
631
  * Pending DEBOUNCED channel-state saves, keyed by channel name. Each entry keeps the
366
632
  * LATEST serialized state plus the session id captured while the session was live —
@@ -377,13 +643,102 @@ export class RealtimeSessionRuntime {
377
643
  */
378
644
  this.usedChannelNames = new Set();
379
645
  this._provider = null;
646
+ this._launcher = new DefaultRealtimeSessionLauncher();
647
+ /**
648
+ * Loads channel code on demand. Called, with the `ClientPluginClass` keys of every channel about to be
649
+ * resolved (the registry's rows plus the host's declarations), after the registry is read and before any
650
+ * plugin is built — so a host can register a heavy channel's class only when the session could use it.
651
+ * It must resolve once the classes it knows are registered; a rejection is logged, never fatal.
652
+ */
653
+ this.ChannelClassLoader = null;
654
+ // ── Session events (server → this session's client) ────────────────────────
655
+ this._sessionEvents$ = new Subject();
656
+ /**
657
+ * Every event the server publishes to THIS session, for the session's life (identity verification
658
+ * today; apps add their own types). Hot, with no replay — subscribe before the session starts, once,
659
+ * and it keeps working across sessions. The runtime has already acted on the events it understands
660
+ * (see {@link handleIdentityVerified}) by the time one reaches here; this stream is for the host UI.
661
+ *
662
+ * Silent when the session's provider has no subscription transport (see {@link createSessionEventSource}).
663
+ */
664
+ this.SessionEvents$ = this._sessionEvents$.asObservable();
665
+ /** Keeps the live session's event stream open; null between sessions. */
666
+ this.sessionEventHub = null;
667
+ /** Per-type handlers the runtime runs (before publishing on {@link SessionEvents$}) for events it understands. */
668
+ this.sessionEventHandlers = new Map([
669
+ ['identity.verified', (event) => this.handleIdentityVerified(event)]
670
+ ]);
671
+ /**
672
+ * What the agent should SAY the moment the user's identity is verified, or `null` (the default) to
673
+ * say nothing — the agent is still told, silently, through a context note. A string is used as the
674
+ * instruction verbatim; a function receives the verified payload (e.g. to greet by name) and returns
675
+ * the instruction, or `null` to stay silent for this verification.
676
+ */
677
+ this.IdentityVerifiedSpokenResponse = null;
678
+ this.clientDeadline = new ClientSessionDeadline();
679
+ // ── Interactive channels (registry-driven + host-declared plugins) ─────────
680
+ //
681
+ // A session's channels come from two places — the `MJ: AI Agent Channels` registry and the channels
682
+ // the host declares — and pass through three stages:
683
+ //
684
+ // 1. PREPARE (before mint) construct the plugins, read their descriptors, report them as
685
+ // candidates, and work out the scope the browser can decide alone.
686
+ // 2. MINT the server scopes the candidates with the agent/app cascade and the
687
+ // registry's kill switch, and returns the resolved policy.
688
+ // 3. ACTIVATE (after mint) initialize exactly the policy's channels: `open-on-start`/`headless`
689
+ // ones mount now, `on-demand` ones wait to be opened through ContextTool.
690
+ //
691
+ // Nothing is initialized before the policy is known, so a channel the server vetoes never runs.
692
+ /** Whether the channel catalog note still has to be sent (see {@link flushChannelCatalogNote}). */
693
+ this.catalogNotePending = false;
694
+ /** Plugins resolved for the current start but not yet initialized. Consumed by {@link activateChannels}. */
695
+ this.preparedChannels = [];
696
+ /** The scope the browser resolves on its own — the fallback when no server policy comes back. */
697
+ this.localChannelScope = null;
698
+ /** `on-demand` channels that are in the session but not opened yet (initialized only when opened). */
699
+ this.advertisedChannels = [];
700
+ /** What the policy resolved for each in-session channel, keyed by normalized channel key. */
701
+ this.resolvedChannels = new Map();
702
+ /** The prior session's saved channel states, kept so an `on-demand` channel opened later is restored too. */
703
+ this.priorChannelStates = {};
704
+ /** The app/static client-tool tiers the server resolved for this session. */
705
+ this.sessionClientTools = null;
706
+ /** Validates and routes channel-addressed `ContextTool` calls. */
707
+ this.channelDispatcher = new ChannelActionDispatcher({
708
+ FindChannel: (key) => this.findDispatchableChannel(key),
709
+ ListChannelKeys: () => this.listAddressableChannelKeys(),
710
+ ActivateChannel: (plugin) => this.mountAdvertisedChannel(plugin),
711
+ });
712
+ /** Where the user's per-channel "how much can the agent see" choices are kept. Replace with {@link SetExposurePreferences}. */
713
+ this.exposurePreferences = new UserSettingsExposurePreferences(() => this.canPersistUserSettings());
714
+ /** The channel whose surface the user is looking at, kept so it applies to an arbiter created after it was set. */
715
+ this.focusedChannelKey = null;
716
+ this.stopWatchingVideoSources = null;
380
717
  /**
381
718
  * Host registry of surface CLIENT TOOLS (Name → handler), fed by the host (Explorer) from the
382
- * active surface's `SetAgentClientTools`. The headless ClientContextChannel's `ContextTool` proxy
383
- * executes against this via {@link executeAppClientTool}. Keys are lower-cased for case-insensitive
384
- * model-supplied action names.
719
+ * active surface's `SetAgentClientTools` and from its always-on globals. The headless
720
+ * ClientContextChannel's `ContextTool` proxy executes against this via {@link executeAppClientTool}.
721
+ * Owner-keyed — see {@link AppClientToolRegistry}.
385
722
  */
386
- this.appClientToolHandlers = new Map();
723
+ this.appToolRegistry = new AppClientToolRegistry();
724
+ // Re-upload shards a previous, interrupted call left in the local store. Deferred a tick so a
725
+ // host can assign `Provider` right after construction; when it still has no transport the drain
726
+ // defers itself and session start tries again. Unref'd so it never holds a Node process open.
727
+ const drainTimer = setTimeout(() => {
728
+ void this.DrainPendingRecordingShards();
729
+ }, 0);
730
+ // Node returns a Timeout object with unref(); browsers return a number.
731
+ if (typeof drainTimer === 'object' && drainTimer !== null && 'unref' in drainTimer && typeof drainTimer.unref === 'function') {
732
+ drainTimer.unref();
733
+ }
734
+ }
735
+ /** Synchronous read of {@link SavingRecording$}. */
736
+ get IsSavingRecording() {
737
+ return this._savingRecording$.value;
738
+ }
739
+ /** Synchronous access to {@link AvatarNotice$}'s current value. */
740
+ get CurrentAvatarNotice() {
741
+ return this._avatarNotice$.value;
387
742
  }
388
743
  /** Synchronous access to the session's active interactive-channel plugins. */
389
744
  get ActiveChannels() {
@@ -444,8 +799,20 @@ export class RealtimeSessionRuntime {
444
799
  UpdateAppContext(snapshot) {
445
800
  this._appContext$.next(snapshot);
446
801
  }
447
- /** How often crash-recovery shards are flushed during a recording. */
448
- static { this.segmentFlushMs = 15000; }
802
+ /** How often packets are cut into shards: the 5 s shard window, so each tick seals about one window. */
803
+ static { this.recordingShardFlushMs = 5000; }
804
+ /**
805
+ * Bytes of `keepalive` request bodies one terminal may use, the tail included. Browsers share about
806
+ * 64 KiB of in-flight keepalive bodies per page; the brief reserves 48 KiB of it for the recording,
807
+ * leaving room for a host's own close beacon (Caliber sends one from `pagehide` too).
808
+ */
809
+ static { this.recordingKeepaliveBudgetBytes = RECORDING_TERMINAL_MAX_BYTES; }
810
+ /**
811
+ * Most runs one drain call makes (each run is itself bounded): it runs again when it was re-armed while
812
+ * running, or when a run deleted shards the server refused for good, which frees the page for the
813
+ * shards behind them.
814
+ */
815
+ static { this.maxDrainRunsPerCall = 3; }
449
816
  /**
450
817
  * How often the client asserts liveness. Comfortably under `SessionJanitor`'s
451
818
  * `closeThresholdMinutes` (15 by default) so several pulses must be missed in a row before a
@@ -480,10 +847,113 @@ export class RealtimeSessionRuntime {
480
847
  set Provider(value) {
481
848
  this._provider = value;
482
849
  }
850
+ /**
851
+ * How this runtime mints a session — the stock `StartRealtimeClientSession` mutation unless a host
852
+ * installs its own {@link IRealtimeSessionLauncher} (a guest-session exchange, a deployment-specific
853
+ * mutation). Everything after the mint — channel scoping, the driver, transcripts, teardown — is
854
+ * unchanged. Assign `null` to restore the default.
855
+ */
856
+ get Launcher() {
857
+ return this._launcher;
858
+ }
859
+ set Launcher(value) {
860
+ this._launcher = value ?? new DefaultRealtimeSessionLauncher();
861
+ }
862
+ /**
863
+ * The client's copy of the session's absolute deadline (`null` when none is known). A host that
864
+ * knows the session's cap at start sets it with {@link SetSessionDeadline}; verification extends it
865
+ * from the server's `identity.verified` payload. The SERVER enforces the real deadline — this exists
866
+ * so the host can end the call gracefully first and show a countdown. It only ever moves later by
867
+ * the server's word, so nothing a client does can extend the real one.
868
+ */
869
+ get SessionDeadline$() {
870
+ return this.clientDeadline.Deadline$;
871
+ }
872
+ /** Synchronous access to {@link SessionDeadline$}. */
873
+ get SessionDeadline() {
874
+ return this.clientDeadline.Value;
875
+ }
876
+ /** Sets the baseline deadline a host knows at session start (`null` clears). See {@link SessionDeadline$}. */
877
+ SetSessionDeadline(deadline) {
878
+ this.clientDeadline.Set(deadline);
879
+ }
880
+ /**
881
+ * Builds the transport that reads the session's events. The default rides the session's GraphQL
882
+ * provider; it returns `null` — "no session events here" — for a provider with no subscription
883
+ * support (a test double, a host on another transport), which the runtime tolerates. Override to
884
+ * supply another transport.
885
+ */
886
+ createSessionEventSource() {
887
+ const provider = this.Provider;
888
+ if (!provider || typeof provider.Subscribe !== 'function') {
889
+ return null;
890
+ }
891
+ return new GraphQLRealtimeSessionClient(provider);
892
+ }
893
+ /** Opens the live session's event stream. Best-effort: never disturbs the call. */
894
+ startSessionEvents(agentSessionId) {
895
+ this.stopSessionEvents();
896
+ try {
897
+ const source = this.createSessionEventSource();
898
+ if (!source) {
899
+ return;
900
+ }
901
+ this.sessionEventHub = new RealtimeSessionEventHub(source, (event) => this.routeSessionEvent(agentSessionId, event));
902
+ this.sessionEventHub.Start(agentSessionId);
903
+ }
904
+ catch (error) {
905
+ console.error('[RealtimeSession] Could not start the session event stream:', error);
906
+ }
907
+ }
908
+ stopSessionEvents() {
909
+ this.sessionEventHub?.Stop();
910
+ this.sessionEventHub = null;
911
+ }
912
+ /** Routes one event to its per-type handler, then publishes it. An event for another session is dropped. */
913
+ routeSessionEvent(expectedSessionId, event) {
914
+ if (event.AgentSessionID !== expectedSessionId || this.agentSessionId !== expectedSessionId) {
915
+ console.warn(`[RealtimeSession] Dropped a '${event.Type}' event addressed to another session.`);
916
+ return;
917
+ }
918
+ try {
919
+ this.sessionEventHandlers.get(event.Type)?.(event);
920
+ }
921
+ catch (error) {
922
+ console.error(`[RealtimeSession] Handling the '${event.Type}' session event failed:`, error);
923
+ }
924
+ this._sessionEvents$.next(event);
925
+ }
926
+ /**
927
+ * `identity.verified`: keep the client's deadline in step with the server's, tell the model — as a
928
+ * silent, structured note — who it is now talking to, and, when the host configured it, have it say
929
+ * something about that. The payload is validated again here (it crossed a wire); the name rides as
930
+ * JSON so user-entered text can never break out of the note's frame.
931
+ */
932
+ handleIdentityVerified(event) {
933
+ if (event.Type !== 'identity.verified' || !IsIdentityVerifiedEventPayload(event.Payload)) {
934
+ console.warn('[RealtimeSession] Ignored a malformed identity.verified event.');
935
+ return;
936
+ }
937
+ const payload = event.Payload;
938
+ this.clientDeadline.Extend(payload.MaxSessionDeadlineIso);
939
+ this.SendContextNote(`[identity] verified ${JSON.stringify({ email: payload.VerifiedEmail, name: payload.VerifiedName, method: payload.Method })} ` +
940
+ '(background context — the user has proven they control this email address; treat them as verified. Do not read this note aloud.)');
941
+ const spoken = typeof this.IdentityVerifiedSpokenResponse === 'function' ? this.IdentityVerifiedSpokenResponse(payload) : this.IdentityVerifiedSpokenResponse;
942
+ if (spoken && spoken.trim().length > 0 && this.client && this.isSessionLive()) {
943
+ this.requestChannelSpokenResponse(spoken.trim());
944
+ }
945
+ }
483
946
  /** True when a session is currently open. */
484
947
  get IsActive() {
485
948
  return this._active$.value;
486
949
  }
950
+ /**
951
+ * True when a session is open or starting for `conversationId`. Ids are compared without
952
+ * regard to case.
953
+ */
954
+ IsActiveFor(conversationId) {
955
+ return this.IsActive && conversationId != null && UUIDsEqual(this.sessionConversationId, conversationId);
956
+ }
487
957
  /**
488
958
  * Start a client-direct voice session fronting `targetAgentId`.
489
959
  *
@@ -514,38 +984,62 @@ export class RealtimeSessionRuntime {
514
984
  * pickers, and never synthesize overrides beyond what the user explicitly chose.
515
985
  * Omit/`null` for the server's defaults (today's behavior).
516
986
  * @param recordingConsent Optional EXPLICIT "record this call" consent for THIS session. When
517
- * `true`, the browser records a mic + agent-audio mix and uploads it at session end. When
987
+ * `true`, the browser records a mic + agent-audio mix and uploads it as 5 s Opus shards during the call. When
518
988
  * omitted/`null`, the per-user persisted preference (`mj.realtimeVoice.recordingConsent.v1`
519
989
  * via {@link UserInfoEngine}) is read as the default. `false` never records.
520
990
  * @param mediaCollectionId Optional per-session media-kit override (`MJ: Collections.ID`). When set,
521
991
  * the server-side Media channel resolves THIS collection as the agent's media kit for the session,
522
992
  * taking precedence over the agent's `DefaultMediaCollectionID`. The server UUID-validates it
523
993
  * (malformed ⇒ ignored, the agent default applies). Omit/`null` to use the agent default kit.
994
+ * @param applicationId Optional application the session runs in (sources the app config cascade,
995
+ * including `Application.AgentSettings.Realtime.Channels`).
996
+ * @param appContext Optional live app-context snapshot injected into the companion prompt at mint.
997
+ * @param options Optional per-start extras — chiefly the channels the HOST brings
998
+ * ({@link RealtimeSessionStartOptions.HostChannels}), the way a connect-only embed with no
999
+ * registry gets channels at all, and the fork the session starts on
1000
+ * ({@link RealtimeSessionStartOptions.ConversationBranchID}; omitted is Main, where its transcript is written).
524
1001
  */
525
- async StartRealtimeSession(targetAgentId, conversationId, lastSessionId, agentName, preferredModelId, clientTools, coAgentId, configOverridesJson, recordingConsent, mediaCollectionId, applicationId, appContext) {
1002
+ async StartRealtimeSession(targetAgentId, conversationId, lastSessionId, agentName, preferredModelId, clientTools, coAgentId, configOverridesJson, recordingConsent, mediaCollectionId, applicationId, appContext, options) {
526
1003
  if (this.IsActive) {
527
1004
  return; // a session is already running — ignore duplicate starts
528
1005
  }
529
1006
  const consent = this.beginSessionStart({ agentName, recordingConsent, applicationId, appContext });
530
- // Captured BEFORE startChannels so the mint carries exactly the snapshot the prologue
1007
+ this.sessionConversationId = conversationId ?? null;
1008
+ // A teardown while the channels are prepared or the mint runs bumps this; see the checks below.
1009
+ const generation = this.startGeneration;
1010
+ // Captured BEFORE the channels are prepared so the mint carries exactly the snapshot the prologue
531
1011
  // resolved, whatever a channel plugin may push in the meantime.
532
1012
  const effectiveAppContext = this._appContext$.value;
533
1013
  let session;
534
1014
  try {
535
- // Resolve + initialize the interactive-channel plugins FIRST: their client-executed
536
- // tool sets must be declared to the realtime model at session mint.
537
- const allClientTools = [...(clientTools ?? []), ...(await this.startChannels())];
538
- session = await this.mintSession(targetAgentId, conversationId, lastSessionId, preferredModelId, allClientTools, coAgentId, configOverridesJson, consent, this.recordingStartedAtIso, mediaCollectionId, this.applicationId, effectiveAppContext);
1015
+ // Resolve the interactive-channel plugins FIRST (constructed, not started): the tools of the
1016
+ // channels mounted with the session must be declared to the realtime model at mint, and the
1017
+ // server needs the candidates to scope them. Nothing is initialized until the policy is known.
1018
+ const scope = await this.prepareChannelScope(options?.HostChannels);
1019
+ const allClientTools = [...(clientTools ?? []), ...scope.NativeTools];
1020
+ session = await this.mintSession(targetAgentId, conversationId, lastSessionId, preferredModelId, allClientTools, coAgentId, configOverridesJson, consent, this.recordingStartedAtIso, mediaCollectionId, this.applicationId, effectiveAppContext, scope.CandidatesJson, scope.ShowsAgentVideo, options?.ConversationBranchID ?? null);
539
1021
  }
540
1022
  catch (error) {
1023
+ // An ended or replaced start only logs its failure: the error state and the teardown belong
1024
+ // to the session that is current now, which a newer start may own.
1025
+ if (this.startGeneration !== generation) {
1026
+ console.error('[RealtimeSession] A start that was already ended failed:', error);
1027
+ return;
1028
+ }
541
1029
  await this.failSessionStart(error);
542
1030
  return;
543
1031
  }
1032
+ // The host ended the start while the mint ran. That teardown had no session id to close, so
1033
+ // close the minted row here, and touch no shared state: a newer start may already own it.
1034
+ if (this.startGeneration !== generation) {
1035
+ await this.closeServerSession(session.AgentSessionId);
1036
+ return;
1037
+ }
544
1038
  if (!this.hostCanUseProvider(session.Provider)) {
545
1039
  await this.abortUnusableSession(session);
546
1040
  return;
547
1041
  }
548
- await this.runMintedSession(session, conversationId ?? null, consent);
1042
+ await this.runMintedSession(session, conversationId ?? null, consent, options?.CameraCheck === true);
549
1043
  }
550
1044
  /**
551
1045
  * Declares which provider keys this host can actually carry audio for.
@@ -581,7 +1075,7 @@ export class RealtimeSessionRuntime {
581
1075
  // published them on ActiveChannels$; skipping teardown would leave them undisposed, their tool
582
1076
  // handlers registered, and their subscriptions live until the next start replaced them.
583
1077
  this.agentSessionId = session.AgentSessionId ?? this.agentSessionId;
584
- this._connectionState$.next('error');
1078
+ this.setConnectionState('error');
585
1079
  await this.teardown(true);
586
1080
  }
587
1081
  /**
@@ -600,7 +1094,8 @@ export class RealtimeSessionRuntime {
600
1094
  * owns that half too.
601
1095
  *
602
1096
  * @param result The minted session — the same ten fields the `StartRealtimeClientSession`
603
- * mutation returns. `EphemeralToken` and `Provider` are what actually open the call.
1097
+ * mutation returns. `EphemeralToken` and `Provider` are what actually open the call; a relay
1098
+ * session opens with its `Transport` and `RelayUrl` instead, so a host must pass those on.
604
1099
  * @param options Host-side inputs the result cannot carry; see {@link RealtimeSessionRunOptions}.
605
1100
  * Every field defaults exactly as its {@link StartRealtimeSession} counterpart does.
606
1101
  */
@@ -614,16 +1109,20 @@ export class RealtimeSessionRuntime {
614
1109
  return;
615
1110
  }
616
1111
  const consent = this.beginSessionStart(effectiveOptions);
617
- await this.runMintedSession(result, effectiveOptions.conversationId ?? null, consent);
1112
+ await this.runMintedSession(result, effectiveOptions.conversationId ?? null, consent, false);
618
1113
  }
619
1114
  /**
620
- * Start prologue shared by both entry points: bind the app layer, publish the agent name, reset
621
- * per-session state, and flip the session live (which is ALSO what makes the `IsActive` guard
622
- * suppress duplicate starts while the mint is still in flight — hence it runs before minting, not
623
- * after). Returns the resolved recording consent, which the mint half reports to the server and
624
- * the run half uses to decide whether to record.
1115
+ * Start prologue shared by both entry points: clear the last start's failure, bind the app layer,
1116
+ * publish the agent name, reset per-session state, and flip the session live (which is ALSO what
1117
+ * makes the `IsActive` guard suppress duplicate starts while the mint is still in flight — hence it
1118
+ * runs before minting, not after). Returns the resolved recording consent, which the mint half
1119
+ * reports to the server and the run half uses to decide whether to record.
625
1120
  */
626
1121
  beginSessionStart(options) {
1122
+ // Before this start publishes anything: hosts read LastStartError as this start's states arrive,
1123
+ // so from the first of them it must describe this start, not an earlier one.
1124
+ this.lastStartError = null;
1125
+ this.lastSessionError = null;
627
1126
  // App awareness (Move 1/3/4): the application the session runs in (sources the app config
628
1127
  // cascade + RelevantAgents → allowed-agent union) and the live app-context snapshot injected
629
1128
  // into the companion prompt at mint. Stored so the ClientContextChannel can stream subsequent
@@ -641,7 +1140,10 @@ export class RealtimeSessionRuntime {
641
1140
  }
642
1141
  this.resetState();
643
1142
  this._active$.next(true);
644
- this._connectionState$.next('connecting');
1143
+ this.setConnectionState('connecting');
1144
+ // A drain the constructor deferred (no provider/transport yet) gets its second chance here, and
1145
+ // runs whether or not THIS session records: the pending shards belong to an earlier call.
1146
+ void this.DrainPendingRecordingShards();
645
1147
  // Resolve recording consent for this session: explicit value wins, else the per-user
646
1148
  // persisted preference. Computed before mint so it can be reported to the server.
647
1149
  const consent = options.recordingConsent ?? this.readPersistedRecordingConsent();
@@ -651,12 +1153,18 @@ export class RealtimeSessionRuntime {
651
1153
  /**
652
1154
  * The RUN half of a session start, shared by both entry points: consume the minted result, open
653
1155
  * the provider connection, and go live. `inputConversationId` is the conversation the START asked
654
- * for (null ⇒ "server, make me one") — the result alone can't distinguish the two.
1156
+ * for (null ⇒ "server, make me one") — the result alone can't distinguish the two. `cameraCheck` is
1157
+ * whether the host shows a camera check ({@link RealtimeSessionStartOptions.CameraCheck}).
655
1158
  */
656
- async runMintedSession(session, inputConversationId, consent) {
1159
+ async runMintedSession(session, inputConversationId, consent, cameraCheck) {
657
1160
  // Captured up front: every await below is a window in which the host can end the session.
658
1161
  const generation = this.startGeneration;
659
1162
  try {
1163
+ // Mount the channels the resolved policy puts in the session. Deliberately BEFORE the session
1164
+ // id is adopted: `ActiveChannels$` consumers (the sessions adapter) treat an emission that
1165
+ // arrives while no session id is set as the INITIAL set, whose opens they synthesize from
1166
+ // `SessionStarted$` — activating after the id is set would announce every channel twice.
1167
+ this.applySessionClientPolicy(session);
660
1168
  this.agentSessionId = session.AgentSessionId;
661
1169
  // A null input conversationId means the SERVER created a fresh conversation for
662
1170
  // this session — track it so the host can fold it into the cached list, select
@@ -671,20 +1179,39 @@ export class RealtimeSessionRuntime {
671
1179
  this.applyPriorChannelStates(session.PriorChannelStatesJson);
672
1180
  const client = this.createRealtimeClient(session.Provider);
673
1181
  this.client = client;
1182
+ this.watchVideoSources(client);
674
1183
  this.wireClientHandlers(client);
675
1184
  // Everything past here awaits on hardware and the network, during which the host may end the
676
- // session. Each await is followed by a staleness check so an abandoned start releases what it
677
- // just acquired instead of leaving a live microphone and a live call behind it.
678
- this.localStream = await this.mediaHost.AcquireMicrophone();
1185
+ // session. Each await is followed by a staleness check, before any failure is reported, so an
1186
+ // abandoned start releases what it just acquired instead of leaving a live microphone and a live
1187
+ // call behind it, and reports nothing to a host that has ended the call.
1188
+ const lease = { Opening: true };
1189
+ this.microphoneLease = lease;
1190
+ const microphone = await this.openMicrophone().finally(() => {
1191
+ lease.Opening = false;
1192
+ });
1193
+ this.localStream = microphone.Stream;
679
1194
  if (this.startGeneration !== generation) {
680
- await this.unwindAbandonedStart(session, client);
1195
+ await this.unwindAbandonedStart(session, client, lease);
681
1196
  return;
682
1197
  }
683
- await client.Connect(this.BuildClientConfig(session), this.localStream);
1198
+ if (microphone.Error) {
1199
+ throw microphone.Error;
1200
+ }
1201
+ const clientConfig = this.BuildClientConfig(session);
1202
+ const connectError = await this.connectClient(client, clientConfig, microphone.Stream);
684
1203
  if (this.startGeneration !== generation) {
685
- await this.unwindAbandonedStart(session, client);
1204
+ await this.unwindAbandonedStart(session, client, lease);
686
1205
  return;
687
1206
  }
1207
+ if (connectError) {
1208
+ throw connectError;
1209
+ }
1210
+ // Tracks are negotiated now, so a capture can tell whether the model takes video, and the call whether it shows
1211
+ // the avatar its agent asked for. The driver holds the microphone now, so the user may switch it.
1212
+ this.openCaptures(client, cameraCheck);
1213
+ this.offerMicrophone();
1214
+ this.publishAvatarNotice(session, clientConfig, client);
688
1215
  // Notify active channels that the session client is connected and tracks are established
689
1216
  for (const channel of this._activeChannels$.value) {
690
1217
  try {
@@ -694,6 +1221,10 @@ export class RealtimeSessionRuntime {
694
1221
  console.error(`[RealtimeSession] Error in channel '${channel.ChannelName}' OnSessionStarted:`, err);
695
1222
  }
696
1223
  }
1224
+ // Tell the model which channels exist and how to use them, from the channels' own descriptors —
1225
+ // as soon as the control channel is usable (which may be right now, or a moment after Connect).
1226
+ this.catalogNotePending = true;
1227
+ this.flushChannelCatalogNote();
697
1228
  // Start browser-side recording (mic + agent mix) when consented. Best-effort: an
698
1229
  // unsupported browser / missing remote stream degrades gracefully (mic-only or off)
699
1230
  // and never blocks the call. The remote stream may still be null here (the WebRTC
@@ -718,6 +1249,8 @@ export class RealtimeSessionRuntime {
718
1249
  // SessionsObserver bridge. Emitting AFTER Connect() guarantees both that
719
1250
  // agentSessionId is set (line ~468) AND the realtime client is connected,
720
1251
  // so consumers can act on it without re-checking either condition.
1252
+ // The session's own event stream (identity verification, app events) stays open for its life.
1253
+ this.startSessionEvents(this.agentSessionId);
721
1254
  this._sessionStarted$.next({
722
1255
  sessionId: this.agentSessionId,
723
1256
  channelNames: this._activeChannels$.value.map(c => c.ChannelName),
@@ -730,18 +1263,42 @@ export class RealtimeSessionRuntime {
730
1263
  await this.failSessionStart(error);
731
1264
  }
732
1265
  }
1266
+ /**
1267
+ * Tells the host, once per call, why it shows no avatar ({@link AvatarNotice$}): from the mint's status, whether this
1268
+ * session asked for the agent's video, and whether that track is live now that the call is connected.
1269
+ */
1270
+ publishAvatarNotice(session, config, client) {
1271
+ const notice = ResolveAvatarNotice(ParseRealtimeAvatarStatus(session.AvatarStatusJson), RequestsAgentVideo(config), client.IsTrackEstablished('video', 'outbound'));
1272
+ if (notice) {
1273
+ this._avatarNotice$.next(notice);
1274
+ }
1275
+ }
733
1276
  /**
734
1277
  * Releases everything a start acquired after the host had already ended the session.
735
1278
  *
736
1279
  * Reached only when {@link teardown} ran while this start was awaiting the microphone or the
737
- * provider connection. Teardown found nothing to release because nothing existed yet, so this
738
- * start owns the cleanup: stop the microphone, close the provider connection, and close the
739
- * server-side session row that the mint created.
1280
+ * provider connection, whether that wait then succeeded or failed: the host has ended the call,
1281
+ * so a failure there is not reported. Teardown could not release what did not exist yet, so this
1282
+ * start owns the cleanup: stop the microphone, hand it back to the host when teardown left that
1283
+ * to it ({@link microphoneLease}), close the provider connection, and close the server-side
1284
+ * session row that the mint created when no teardown has closed it or is closing it.
1285
+ *
1286
+ * @param lease This start's opening of the microphone.
740
1287
  */
741
- async unwindAbandonedStart(session, client) {
1288
+ async unwindAbandonedStart(session, client, lease) {
742
1289
  console.warn('[RealtimeSession] Session was ended while starting — releasing the partial session.');
1290
+ this.closeCaptures();
1291
+ this.closeLocalMedia();
743
1292
  this.localStream?.getTracks().forEach(t => t.stop());
744
1293
  this.localStream = null;
1294
+ this.microphoneMuted = false;
1295
+ // Teardown left the release to this start when the microphone was still opening. A cleared or
1296
+ // replaced lease means teardown already released it, or a newer start has opened the microphone
1297
+ // since and its own end releases it.
1298
+ if (this.microphoneLease === lease) {
1299
+ this.microphoneLease = null;
1300
+ await this.releaseMicrophone();
1301
+ }
745
1302
  try {
746
1303
  await client.Disconnect();
747
1304
  }
@@ -751,27 +1308,321 @@ export class RealtimeSessionRuntime {
751
1308
  // Only clear the shared slots when they still point at THIS attempt — a newer start may
752
1309
  // already have replaced them.
753
1310
  if (this.client === client) {
1311
+ this.unwatchVideoSources();
1312
+ this.clearAgentVideo();
754
1313
  this.client = null;
755
1314
  }
756
- // Close the server session only if teardown has NOT already done so. It nulls `agentSessionId`
757
- // after closing, so a still-matching id means this attempt still owns the row; a cleared one
758
- // means the teardown that invalidated this start already closed it, and closing again would
759
- // send a second `CloseAgentSession` for one session.
760
- if (session.AgentSessionId && this.agentSessionId === session.AgentSessionId) {
1315
+ // Close the server session only if no teardown has closed it or is closing it. Teardown nulls
1316
+ // `agentSessionId` after closing, so a cleared id means the teardown that invalidated this start
1317
+ // already closed it. A teardown still running closes it itself and then reports the end on
1318
+ // `SessionEnded$`: closing it here as well would send a second `CloseAgentSession` for one
1319
+ // session, and clearing the id would leave that teardown no session to report.
1320
+ if (session.AgentSessionId && this.agentSessionId === session.AgentSessionId && !this.teardownInFlight) {
761
1321
  this.agentSessionId = null;
762
1322
  await this.closeServerSession(session.AgentSessionId);
763
1323
  }
764
1324
  }
765
1325
  /**
766
- * Why the last session start failed, or `null` when the last start succeeded or none has run.
1326
+ * Opens the user's microphone: through the host's controller when it offers one, otherwise through
1327
+ * {@link IRealtimeMediaHost.AcquireMicrophone}. A failure is returned rather than thrown, so the caller can
1328
+ * first check whether the start was abandoned meanwhile (a teardown disposes the controller, which fails a
1329
+ * start still in flight) and unwind quietly instead of reporting an error nobody is waiting for.
1330
+ */
1331
+ async openMicrophone() {
1332
+ const controller = this.mediaHost.CreateLocalMediaController?.() ?? null;
1333
+ if (!controller) {
1334
+ try {
1335
+ return { Stream: await this.mediaHost.AcquireMicrophone(), Error: null };
1336
+ }
1337
+ catch (error) {
1338
+ return { Stream: null, Error: error instanceof Error ? error : new Error(String(error)) };
1339
+ }
1340
+ }
1341
+ this.localMedia = controller;
1342
+ const started = await controller.Start('microphone');
1343
+ if (started.Status === 'failed') {
1344
+ return { Stream: null, Error: microphoneStartError(started.Reason, started.Message) };
1345
+ }
1346
+ this.followMicrophone(controller, started.Stream);
1347
+ return { Stream: started.Stream, Error: null };
1348
+ }
1349
+ /**
1350
+ * Opens the provider connection. As with {@link openMicrophone}, a failure is returned rather than thrown, so the
1351
+ * caller can first check whether the start was abandoned meanwhile and unwind quietly instead of reporting an error
1352
+ * nobody is waiting for: a teardown disconnects the client, which with most drivers fails a connection still opening.
1353
+ */
1354
+ async connectClient(client, config, microphone) {
1355
+ try {
1356
+ await client.Connect(config, microphone);
1357
+ return null;
1358
+ }
1359
+ catch (error) {
1360
+ return error instanceof Error ? error : new Error(String(error));
1361
+ }
1362
+ }
1363
+ /**
1364
+ * Moves the driver and the recorder onto the call's microphone whenever the controller gives it a new track. A device
1365
+ * switch, or a lost device replaced by the default, swaps a track into the same stream, and the controller carries the
1366
+ * old track's mute over. A microphone opened again after the call lost one comes in a new stream: its track takes the
1367
+ * call's mute, and the stream becomes the one {@link ToggleMute} and the teardown act on. Mid-swap the stream holds no
1368
+ * track, so only a finished swap is followed. Every report also keeps {@link Microphone$} current once the call offers
1369
+ * it.
1370
+ */
1371
+ followMicrophone(controller, stream) {
1372
+ let followedStream = stream;
1373
+ let followedTrack = stream.getAudioTracks()[0] ?? null;
1374
+ this.localMediaSubscription = controller.State$.subscribe((media) => {
1375
+ const current = controller.GetStream('microphone') ?? followedStream;
1376
+ const track = current.getAudioTracks()[0] ?? null;
1377
+ if (track && track !== followedTrack) {
1378
+ if (current !== followedStream) {
1379
+ track.enabled = !this.microphoneMuted;
1380
+ this.localStream = current;
1381
+ followedStream = current;
1382
+ }
1383
+ followedTrack = track;
1384
+ this.recorder?.ReplaceMicrophone?.(current);
1385
+ this.client?.ReplaceMicrophone?.(current)?.catch((error) => {
1386
+ console.error('[RealtimeSession] The realtime driver could not move to the new microphone:', error);
1387
+ });
1388
+ }
1389
+ this.publishMicrophone(media);
1390
+ });
1391
+ }
1392
+ /** The client is connected: the call offers its microphone on {@link Microphone$}, and {@link SwitchMicrophone} moves it. */
1393
+ offerMicrophone() {
1394
+ if (!this.localMedia) {
1395
+ return;
1396
+ }
1397
+ this.microphoneOffered = true;
1398
+ this.publishMicrophone(this.localMedia.State);
1399
+ }
1400
+ /** Publishes the call's microphone from the controller's state while the call offers it, when it names a change. */
1401
+ publishMicrophone(media) {
1402
+ if (!this.microphoneOffered) {
1403
+ return;
1404
+ }
1405
+ const current = this._microphone$.value;
1406
+ const next = ReadMicrophoneState(media, current);
1407
+ if (!SameMicrophoneState(next, current)) {
1408
+ this._microphone$.next(next);
1409
+ }
1410
+ }
1411
+ /** Stops following the controller and disposes it, which releases its devices; the call no longer offers a microphone. */
1412
+ closeLocalMedia() {
1413
+ this.localMediaSubscription?.unsubscribe();
1414
+ this.localMediaSubscription = null;
1415
+ this.localMedia?.Dispose();
1416
+ this.localMedia = null;
1417
+ this.microphoneOffered = false;
1418
+ if (this._microphone$.value !== REALTIME_MICROPHONE_NONE) {
1419
+ this._microphone$.next(REALTIME_MICROPHONE_NONE);
1420
+ }
1421
+ }
1422
+ /**
1423
+ * Creates the session's camera and screen share and mirrors their state on {@link Captures$}. With `cameraCheck`, the
1424
+ * call's first camera start waits for {@link ConfirmCamera}.
1425
+ */
1426
+ openCaptures(client, cameraCheck) {
1427
+ this.closeCaptures();
1428
+ const captures = new RealtimeCaptures({
1429
+ Client: client,
1430
+ LocalMedia: this.localMedia,
1431
+ Host: this.mediaHost,
1432
+ CreateSampler: this.hostFrameSampler(),
1433
+ Admit: (kind) => this.admitCapture(kind),
1434
+ CameraCheck: cameraCheck,
1435
+ });
1436
+ this.captures = captures;
1437
+ this.capturesSubscription = captures.States$.subscribe((states) => this._captures$.next(states));
1438
+ this.refreshCaptureOffers();
1439
+ }
1440
+ /**
1441
+ * The host's frame sampler for the captures ({@link IRealtimeMediaHost.CreateFrameSampler}), or `undefined` when it offers
1442
+ * none and the captures use the DOM default.
1443
+ */
1444
+ hostFrameSampler() {
1445
+ const host = this.mediaHost;
1446
+ const create = host.CreateFrameSampler;
1447
+ return create ? (stream, options) => create.call(host, stream, options) : undefined;
1448
+ }
1449
+ /** Stops the camera and screen share, if any, and reports both off. */
1450
+ closeCaptures() {
1451
+ this.capturesSubscription?.unsubscribe();
1452
+ this.capturesSubscription = null;
1453
+ this.captures?.Dispose();
1454
+ this.captures = null;
1455
+ if (this._captures$.value !== REALTIME_CAPTURES_OFF) {
1456
+ this._captures$.next(REALTIME_CAPTURES_OFF);
1457
+ }
1458
+ this.refreshCaptureOffers();
1459
+ }
1460
+ /** Works out which captures the call offers ({@link CaptureOffers$}) and publishes a change. */
1461
+ refreshCaptureOffers() {
1462
+ const offers = this.captures && this.client?.SupportsInboundVideo ? { Camera: this.canOffer('camera'), Screen: this.canOffer('screen') } : REALTIME_CAPTURE_OFFERS_NONE;
1463
+ const current = this._captureOffers$.value;
1464
+ if (offers.Camera !== current.Camera || offers.Screen !== current.Screen) {
1465
+ this._captureOffers$.next(offers);
1466
+ }
1467
+ }
1468
+ /** Whether the host can open a capture and the call's policy admits it. */
1469
+ canOffer(kind) {
1470
+ const hostCanOpen = kind === 'camera' ? this.localMedia !== null : typeof this.mediaHost.RequestDisplayCapture === 'function';
1471
+ return hostCanOpen && this.admitCapture(kind).Admitted;
1472
+ }
1473
+ /** What a capture reports when there is no live session to show it to. */
1474
+ noSessionCapture() {
1475
+ return { Status: 'failed', Failure: 'no-session', Message: 'There is no call to share with.' };
1476
+ }
1477
+ /**
1478
+ * The session's policy for a capture. The channel that fronts it (its `CaptureKind`) must be in the session, open or
1479
+ * advertised, and the server's policy must let the agent see pixels through it (the agent's configuration and any
1480
+ * zero-data-retention rule). The user's own "agent can see" choice for the channel does not refuse a start: it decides
1481
+ * whether the frames reach the model.
1482
+ */
1483
+ admitCapture(kind) {
1484
+ const channel = this.findCaptureChannel(kind)?.Plugin;
1485
+ if (!channel) {
1486
+ return { Admitted: false, Message: CAPTURE_WORDING[kind].NotInCall };
1487
+ }
1488
+ const resolved = this.GetResolvedChannel(channel.ChannelName);
1489
+ const allowed = resolved?.Exposure ?? resolved?.MaxExposure ?? channel.GetDescriptor().MaxExposure;
1490
+ if (allowed !== 'pixels') {
1491
+ const reasons = (resolved?.ExposureLimits ?? []).map((limit) => limit.Reason);
1492
+ return { Admitted: false, Message: reasons.length > 0 ? `${CAPTURE_WORDING[kind].NotAllowed}: ${reasons.join('; ')}.` : `${CAPTURE_WORDING[kind].NotAllowed}.` };
1493
+ }
1494
+ return { Admitted: true, ChannelKey: channel.ChannelName, VisibleToAgent: channel.Exposure === 'pixels' };
1495
+ }
1496
+ /** The channel in the session, open or advertised, that fronts a capture. */
1497
+ findCaptureChannel(kind) {
1498
+ const open = this._activeChannels$.value.find((c) => c.CaptureKind === kind);
1499
+ if (open) {
1500
+ return { Plugin: open, IsOpen: true };
1501
+ }
1502
+ const advertised = this.advertisedChannels.find((p) => p.Plugin.CaptureKind === kind);
1503
+ return advertised ? { Plugin: advertised.Plugin, IsOpen: false } : null;
1504
+ }
1505
+ /**
1506
+ * Brings up the channel that fronts a capture once the capture is starting, so the host shows its surface: an advertised
1507
+ * channel is opened (and announces itself to the agent), an open one counts as used.
1508
+ */
1509
+ async revealCaptureChannel(kind) {
1510
+ const state = kind === 'camera' ? this.captures?.States.Camera : this.captures?.States.Screen;
1511
+ const channel = state?.Status === 'starting' ? this.findCaptureChannel(kind) : null;
1512
+ if (!channel) {
1513
+ return;
1514
+ }
1515
+ if (channel.IsOpen) {
1516
+ this.noteChannelActivity(channel.Plugin);
1517
+ return;
1518
+ }
1519
+ const opened = await this.OpenChannel(channel.Plugin.ChannelName);
1520
+ if (!opened.Success) {
1521
+ console.warn(`[RealtimeSession] Could not open channel '${channel.Plugin.ChannelName}' for the ${kind}: ${opened.ErrorMessage ?? opened.ErrorCode}`);
1522
+ }
1523
+ }
1524
+ /**
1525
+ * Starts the user's camera and shows it to the agent. Call it from the user's click: it may ask for camera
1526
+ * permission. Resolves with the camera's state; a failure (no session, a model that takes no video, a refused
1527
+ * permission) is a state with a message, never a throw. While the camera is on, it is a source on
1528
+ * {@link VideoSources$}, and it stops by itself if the device goes away.
1529
+ *
1530
+ * When the host shows a camera check ({@link RealtimeSessionStartOptions.CameraCheck}), the call's first start
1531
+ * resolves still starting, with `Checking` set and the stream for the host to preview. The agent sees the camera
1532
+ * once the host calls {@link ConfirmCamera}; {@link StopCamera} is the user's "not now".
1533
+ *
1534
+ * @param deviceId The camera to open; the system default when absent.
1535
+ */
1536
+ async StartCamera(deviceId) {
1537
+ if (!this.captures) {
1538
+ return this.noSessionCapture();
1539
+ }
1540
+ const started = this.captures.Start('camera', { DeviceID: deviceId });
1541
+ await this.revealCaptureChannel('camera');
1542
+ return started;
1543
+ }
1544
+ /** Stops the user's camera. Safe to call when it is off. */
1545
+ StopCamera() {
1546
+ this.captures?.Stop('camera');
1547
+ }
1548
+ /**
1549
+ * Moves the user's camera to another device while it is open (being checked or on): one of the camera's `Devices` on
1550
+ * {@link Captures$}. The stream stays the same. Resolves with the camera's state; nothing changes while the camera is
1551
+ * off.
1552
+ *
1553
+ * @param deviceId The camera to move to.
1554
+ */
1555
+ async SwitchCamera(deviceId) {
1556
+ return this.captures ? this.captures.SwitchCamera(deviceId) : this.noSessionCapture();
1557
+ }
1558
+ /**
1559
+ * Moves the call's microphone to another device: one of the `Devices` on {@link Microphone$}. The host's controller
1560
+ * swaps the new track into the same stream and carries the mute over, and the driver and the call's recording move to
1561
+ * it, so the agent keeps hearing the user. When the new microphone cannot open, the controller goes back to the one in
1562
+ * use. While the call has no microphone open (it lost the one in use and the default, or the last one went away), the
1563
+ * pick opens one again: the driver and the recording move to it, and it keeps the call's mute. Resolves with the
1564
+ * microphone's state once the switch is over. Nothing changes outside a call or on a host without a controller.
1565
+ *
1566
+ * @param deviceId The microphone to move to.
1567
+ */
1568
+ async SwitchMicrophone(deviceId) {
1569
+ const controller = this.localMedia;
1570
+ if (controller && this.microphoneOffered) {
1571
+ const status = controller.State.Microphone.Status;
1572
+ const live = status === 'on' || status === 'starting';
1573
+ await (live ? controller.SwitchDevice('microphone', deviceId) : controller.Start('microphone', deviceId));
1574
+ }
1575
+ return this._microphone$.value;
1576
+ }
1577
+ /**
1578
+ * The user checked the camera and turned it on: the agent sees it from now on, and later camera starts in this call
1579
+ * skip the check. Returns the camera's state; unless the camera is waiting for its check (see {@link StartCamera}),
1580
+ * nothing changes.
1581
+ */
1582
+ ConfirmCamera() {
1583
+ return this.captures ? this.captures.ConfirmCamera() : this.noSessionCapture();
1584
+ }
1585
+ /**
1586
+ * Asks the user for a screen, window or browser tab (or one panel of the page) and shows it to the agent. Call it
1587
+ * from the user's click: the browser's picker needs one. Resolves with the share's state; a failure (no session,
1588
+ * a model that takes no video, a closed picker, a host that cannot share) is a state with a message. The share
1589
+ * also stops when the user ends it from the browser's own bar.
1590
+ *
1591
+ * @param options What the picker offers first, or the panel to share and its name (`PanelLabel`), which the share's
1592
+ * state carries and the agent sees as "<name> (shared panel)".
1593
+ */
1594
+ async StartScreenShare(options) {
1595
+ if (!this.captures) {
1596
+ return this.noSessionCapture();
1597
+ }
1598
+ const started = this.captures.Start('screen', options);
1599
+ await this.revealCaptureChannel('screen');
1600
+ return started;
1601
+ }
1602
+ /** Stops the user's screen share. Safe to call when nothing is shared. */
1603
+ StopScreenShare() {
1604
+ this.captures?.Stop('screen');
1605
+ }
1606
+ /**
1607
+ * Why the latest session start failed, or `null` when it has not failed or none has run.
767
1608
  *
768
1609
  * Read it when {@link ConnectionState$} reports `'error'`, to tell a denied microphone apart from
769
- * a provider or backend failure and show copy the user can act on. Cleared at the start of every
770
- * session.
1610
+ * a provider or backend failure and show copy the user can act on. Each start clears it before it
1611
+ * reports `'connecting'`, so it never describes an earlier start.
771
1612
  */
772
1613
  get LastStartError() {
773
1614
  return this.lastStartError;
774
1615
  }
1616
+ /**
1617
+ * The fatal error the call's client reported, or `null` when it has reported none.
1618
+ *
1619
+ * Read it when {@link ConnectionState$} reports `'error'` on a call that was live, to tell a call that reached its time
1620
+ * limit (`Code` `'time-limit'`; {@link DescribeCallEnd} words it) from a dropped connection. Each start clears it before
1621
+ * it reports `'connecting'`.
1622
+ */
1623
+ get LastSessionError() {
1624
+ return this.lastSessionError;
1625
+ }
775
1626
  /**
776
1627
  * The single failure path for a session start (mint half or run half): report it, latch the
777
1628
  * overlay into 'error', and unwind whatever the half-built session already opened.
@@ -779,7 +1630,7 @@ export class RealtimeSessionRuntime {
779
1630
  async failSessionStart(error) {
780
1631
  console.error('[RealtimeSession] Failed to start session:', error);
781
1632
  this.lastStartError = error instanceof Error ? error : new Error(String(error));
782
- this._connectionState$.next('error');
1633
+ this.setConnectionState('error');
783
1634
  await this.teardown(false);
784
1635
  }
785
1636
  /**
@@ -816,16 +1667,21 @@ export class RealtimeSessionRuntime {
816
1667
  }
817
1668
  client.SendText(trimmed);
818
1669
  // Relay as a user turn — same path spoken input uses (caption + persisted transcript).
819
- void this.onUserTranscript(trimmed);
1670
+ void this.onUserTranscript(trimmed, 'typed');
820
1671
  }
821
- /** Mute / unmute the local microphone track. Returns the new muted state. */
1672
+ /**
1673
+ * Mute / unmute the local microphone track. Returns the new muted state. While the call has no microphone track (a
1674
+ * switch is putting a new one in, or the call lost its microphone) there is nothing to mute: nothing changes, and it
1675
+ * returns the call's mute, which the next microphone keeps. Outside a call it returns `false`.
1676
+ */
822
1677
  ToggleMute() {
823
1678
  const tracks = this.localStream?.getAudioTracks() ?? [];
824
1679
  if (tracks.length === 0) {
825
- return false;
1680
+ return this.microphoneMuted;
826
1681
  }
827
1682
  const muted = tracks[0].enabled; // currently enabled → becomes muted
828
1683
  this.client?.SetMuted(muted);
1684
+ this.microphoneMuted = muted;
829
1685
  return muted;
830
1686
  }
831
1687
  // ── Client-executed UI tools ───────────────────────────────────────────────
@@ -895,13 +1751,21 @@ export class RealtimeSessionRuntime {
895
1751
  return this.client;
896
1752
  }
897
1753
  /**
898
- * Relays a video frame to the underlying realtime client if active.
1754
+ * Relays a video frame straight to the underlying realtime client, if a session is live.
1755
+ *
1756
+ * It bypasses the session's {@link VideoSourceArbiter}: the frame overrides the source the arbiter picked, and the
1757
+ * model is not told what it is looking at. Channels no longer send through it (their context's `SendVideoFrame` goes
1758
+ * through the arbiter); a host that shows the model something registers a source there instead, as
1759
+ * `ChannelInboundVideoBridge` does.
1760
+ *
1761
+ * @returns `true` when the client accepted and sent the frame; `false` when no session is live, the
1762
+ * client cannot take video frames, or the client dropped the frame.
899
1763
  */
900
1764
  SendVideoFrame(base64Image, mimeType) {
901
1765
  if (!this.client || !this.isSessionLive()) {
902
- return;
1766
+ return false;
903
1767
  }
904
- this.client.SendVideoFrame?.(base64Image, mimeType);
1768
+ return this.client.SendVideoFrame?.(base64Image, mimeType) === true;
905
1769
  }
906
1770
  /**
907
1771
  * Checks whether a media track is established on the active realtime client.
@@ -948,12 +1812,16 @@ export class RealtimeSessionRuntime {
948
1812
  // turn one's cue wherever the clocks happen to differ.
949
1813
  this.currentTurnStartMs = recorder.IsRecording ? 0 : null;
950
1814
  this.turnAudioStartCaptured = false;
1815
+ this.userSpeechStartMs = null;
1816
+ this.userSpeechStartSuperseded = false;
951
1817
  if (this.recorder) {
952
- // The agent's WebRTC audio track usually lands AFTER Connect() resolves, so `remoteStream`
953
- // above is typically null and we'd capture mic-only. Attach the agent stream whenever it
954
- // arrives (fires immediately if already present) so the recording includes the agent voice.
1818
+ // WebRTC drivers (OpenAI): the agent's track usually lands AFTER Connect() resolves, so
1819
+ // `remoteStream` above is null here and this handler attaches it later. PCM-playback
1820
+ // drivers (Gemini, ElevenLabs, AssemblyAI, xAI, HuggingFace) publish at Connect, so
1821
+ // `remoteStream` is already set and the handler fires immediately with the same stream;
1822
+ // AttachRemoteStream is idempotent, so it is mixed only once.
955
1823
  client.OnRemoteMediaStream?.((stream) => this.recorder?.AttachRemoteStream(stream));
956
- this.startSegmentFlushing();
1824
+ this.startShardFlushing();
957
1825
  }
958
1826
  }
959
1827
  catch (error) {
@@ -961,201 +1829,928 @@ export class RealtimeSessionRuntime {
961
1829
  this.recorder = null;
962
1830
  }
963
1831
  }
964
- /** Begins flushing ~15s crash-recovery shards to the server for the duration of the recording. */
965
- startSegmentFlushing() {
966
- this.segmentIndex = 0;
967
- this.segmentTimer = setInterval(() => { void this.flushRecordingSegment(); }, RealtimeSessionRuntime.segmentFlushMs);
1832
+ // ── Recording shards (MJ#5294) ───────────────────────────────────────────────
1833
+ //
1834
+ // The recorder hands over Opus packets; the runtime cuts them into 5 s `MJOPUS1` shards, writes
1835
+ // each one to the local store BEFORE its upload starts, uploads it as raw bytes, and deletes it from
1836
+ // the store only on the server's ack. The call ends with a `keepalive` terminal request carrying the
1837
+ // final partial shard and the counts the server needs to know it has everything.
1838
+ /**
1839
+ * Builds the transport for shard uploads and terminal requests. The default posts to MJAPI's REST
1840
+ * recording routes, derived from the session's GraphQL provider configuration, with credentials
1841
+ * read per request so a refreshed token is used. Returns null — shards then wait in the local
1842
+ * store — when the provider has no URL (a test double, a provider that is not GraphQL, or one not
1843
+ * configured yet). Override to supply another transport.
1844
+ */
1845
+ createRecordingTransport() {
1846
+ return CreateRecordingTransportForProvider(this.Provider);
1847
+ }
1848
+ /**
1849
+ * Builds the durable local shard store. The default is IndexedDB where the host has it and a no-op
1850
+ * store elsewhere (Node, some React Native runtimes). Called once per runtime. Override to supply
1851
+ * another store.
1852
+ */
1853
+ createRecordingShardStore() {
1854
+ return CreateRecordingShardStore(globalThis.indexedDB);
1855
+ }
1856
+ /** The provider's signed-in user, or null (no provider yet, or an anonymous page). */
1857
+ signedInUserID() {
1858
+ return this.Provider?.CurrentUser?.ID ?? null;
1859
+ }
1860
+ /** The runtime's shard store, created on first use. */
1861
+ ensureShardStore() {
1862
+ this.shardStore ??= this.createRecordingShardStore();
1863
+ return this.shardStore;
1864
+ }
1865
+ /**
1866
+ * Re-uploads shards an earlier, interrupted call left in the local store (crash, closed tab, dead
1867
+ * network), each under its OWN session id and index, oldest first, at most
1868
+ * {@link MAX_DRAINED_SHARDS_PER_LOAD} per runtime. A shard is deleted only on the server's ack.
1869
+ *
1870
+ * Runs once per runtime instance, re-armed whenever a recording ends (which may leave shards behind
1871
+ * for the next session start on the same page). When there is no transport yet (no provider configuration, e.g.
1872
+ * before login) it returns 0 WITHOUT marking itself done, so a later call — session start — drains
1873
+ * instead of the shards being skipped for this page. Concurrent calls share one run. Never rejects.
1874
+ *
1875
+ * @returns the number of shards the server acknowledged in this call.
1876
+ */
1877
+ DrainPendingRecordingShards() {
1878
+ if (this.pendingShardsDrained) {
1879
+ return Promise.resolve(0);
1880
+ }
1881
+ this.drainInFlight ??= this.drainPendingShards().finally(() => {
1882
+ this.drainInFlight = null;
1883
+ });
1884
+ return this.drainInFlight;
1885
+ }
1886
+ /** Makes the next {@link DrainPendingRecordingShards} call run again, even one already in flight. */
1887
+ rearmPendingShardDrain() {
1888
+ this.pendingShardsDrained = false;
1889
+ this.drainGeneration++;
1890
+ }
1891
+ /**
1892
+ * The body of {@link DrainPendingRecordingShards}, up to {@link maxDrainRunsPerCall} runs. It runs
1893
+ * again after a re-arm while a run is in flight (a recovery after an outage gave up on more shards),
1894
+ * because the running pass listed the store before those shards were there; and after a run deleted
1895
+ * refused shards, because those filled the page that the shards behind them now get.
1896
+ */
1897
+ async drainPendingShards() {
1898
+ try {
1899
+ const transport = this.createRecordingTransport();
1900
+ if (!transport) {
1901
+ return 0;
1902
+ }
1903
+ let delivered = 0;
1904
+ for (let run = 1; run <= RealtimeSessionRuntime.maxDrainRunsPerCall; run++) {
1905
+ // A re-arm while this run is in flight must win over this run marking itself done.
1906
+ const generation = this.drainGeneration;
1907
+ const outcome = await this.drainStoredShardsOnce(transport);
1908
+ delivered += outcome.Delivered;
1909
+ // Done only when every listed shard was delivered or refused for good: a drain during an outage
1910
+ // must not stop the next session start or recovery from trying again.
1911
+ this.pendingShardsDrained = generation === this.drainGeneration && outcome.Delivered + outcome.Refused === outcome.Listed;
1912
+ if (generation === this.drainGeneration && outcome.Refused === 0) {
1913
+ break;
1914
+ }
1915
+ }
1916
+ return delivered;
1917
+ }
1918
+ catch (error) {
1919
+ console.warn(`[RealtimeSession] Draining stored recording shards failed: ${describeRecordingFailure(error)}; will retry at the next session start.`, error);
1920
+ return 0;
1921
+ }
1922
+ }
1923
+ /** One drain pass over the signed-in user's stored shards (see {@link DrainStoredRecordingShardsOnce}). */
1924
+ drainStoredShardsOnce(transport) {
1925
+ return DrainStoredRecordingShardsOnce(this.ensureShardStore(), transport, this.signedInUserID());
1926
+ }
1927
+ /** Builds this recording's shard pipeline and starts the 5 s flush. */
1928
+ startShardFlushing() {
1929
+ const sessionId = this.agentSessionId;
1930
+ if (!sessionId) {
1931
+ // Unreachable in practice (the session id is adopted before recording starts), but without
1932
+ // one there is nowhere to send shards, so say so rather than recording into the void.
1933
+ console.warn('[RealtimeSession] Recording started without an agent session id; shards will not be saved.');
1934
+ return;
1935
+ }
1936
+ const transport = this.createRecordingTransport();
1937
+ if (!transport) {
1938
+ console.warn(`[RealtimeSession] No recording transport for session ${sessionId} (provider has no REST configuration); shards are kept in the local store only.`);
1939
+ }
1940
+ // True once the queue gave up on a shard and until an upload succeeds again: that first success
1941
+ // re-sends what the queue dropped meanwhile (see recoverStoredShardsAfterOutage).
1942
+ let shardsGivenUp = false;
1943
+ this.recordingShards = {
1944
+ SessionID: sessionId,
1945
+ UserID: this.signedInUserID(),
1946
+ Window: new RecordingShardWindow(),
1947
+ Transport: transport,
1948
+ // Closes over this recording's session id and transport — never the runtime's current ones.
1949
+ Queue: new RecordingShardQueue(async (shard) => {
1950
+ if (!transport) {
1951
+ return false;
1952
+ }
1953
+ const acked = (await transport.UploadShard(sessionId, shard)) === 'Acked';
1954
+ if (acked) {
1955
+ await this.deleteStoredShard(sessionId, shard.Index);
1956
+ if (shardsGivenUp) {
1957
+ shardsGivenUp = false;
1958
+ this.recoverStoredShardsAfterOutage(sessionId);
1959
+ }
1960
+ }
1961
+ return acked;
1962
+ }, () => {
1963
+ shardsGivenUp = true;
1964
+ }),
1965
+ };
1966
+ this.shardFlushTimer = setInterval(() => { void this.flushRecordingShards(); }, RealtimeSessionRuntime.recordingShardFlushMs);
1967
+ }
1968
+ /**
1969
+ * Uploads succeed again after failing: re-sends, from the local store, the shards the queue gave up
1970
+ * on during the outage. The queue's attempts run out after about 10 s of a dead network (or an
1971
+ * expired token), and without this those shards would wait for a later page load — which a widget
1972
+ * guest never makes — leaving a permanent gap. Runs in the background, outside any page-lifecycle
1973
+ * handler, so the IndexedDB read is allowed. Never rejects.
1974
+ */
1975
+ recoverStoredShardsAfterOutage(sessionId) {
1976
+ console.log(`[RealtimeSession] Recording uploads for session ${sessionId} recovered; re-sending stored shards.`);
1977
+ this.rearmPendingShardDrain();
1978
+ void this.DrainPendingRecordingShards();
1979
+ }
1980
+ /** Stops the periodic shard flush. */
1981
+ stopShardFlushing() {
1982
+ if (this.shardFlushTimer) {
1983
+ clearInterval(this.shardFlushTimer);
1984
+ this.shardFlushTimer = null;
1985
+ }
1986
+ }
1987
+ /**
1988
+ * Starts telling the server this session is still in use (#3533).
1989
+ *
1990
+ * **Why the server cannot work this out on its own.** In the client-direct topology the audio
1991
+ * goes browser → provider over WebRTC. The server sees the mint, a few channel actions in the
1992
+ * first seconds, and then nothing at all — so `SessionManager.RecordActivity` stops being
1993
+ * reached while the conversation is still going. `LastActiveAt` freezes ~45 seconds in, and
1994
+ * `SessionJanitor` — which cannot distinguish an active call from an abandoned one — force-closes
1995
+ * it at `closeThresholdMinutes`, mid-sentence, taking the user's surfaces with it. A session
1996
+ * whose channels are all client-side (whiteboard, media) goes quiet from the server's point of
1997
+ * view almost immediately.
1998
+ *
1999
+ * The browser is the only participant that knows the call is alive, so it is the one that has to
2000
+ * say so. Raising `closeThresholdMinutes` is not the fix — it just makes the janitor slower at
2001
+ * its real job (reaping rows orphaned by a crash) without making liveness observable.
2002
+ *
2003
+ * The pulse is best-effort by design: a failed beat is logged and skipped, never surfaced to the
2004
+ * user and never allowed to end the session. Losing one beat costs nothing because the threshold
2005
+ * is many beats wide; turning a transient network blip into a visible error would be a worse
2006
+ * failure than the one this fixes. Write amplification is bounded on the server side too, where
2007
+ * `SessionManager.Heartbeat` coalesces persisted writes.
2008
+ */
2009
+ startLivenessPulse() {
2010
+ this.stopLivenessPulse();
2011
+ this.livenessTimer = setInterval(() => { void this.pulseLiveness(); }, RealtimeSessionRuntime.livenessPulseMs);
2012
+ }
2013
+ /** Stops the liveness pulse. Idempotent — safe on a session that never started one. */
2014
+ stopLivenessPulse() {
2015
+ if (this.livenessTimer) {
2016
+ clearInterval(this.livenessTimer);
2017
+ this.livenessTimer = null;
2018
+ }
2019
+ }
2020
+ /**
2021
+ * One liveness beat. Reads the session id at fire time rather than closing over it, so a beat
2022
+ * that fires during teardown finds `null` and does nothing instead of resurrecting a closed row.
2023
+ */
2024
+ async pulseLiveness() {
2025
+ const agentSessionId = this.agentSessionId;
2026
+ if (!agentSessionId) {
2027
+ return;
2028
+ }
2029
+ const mutation = `
2030
+ mutation AgentSessionHeartbeat($agentSessionId: String!) {
2031
+ AgentSessionHeartbeat(agentSessionId: $agentSessionId)
2032
+ }
2033
+ `;
2034
+ try {
2035
+ await this.gql().ExecuteGQL(mutation, { agentSessionId });
2036
+ }
2037
+ catch (error) {
2038
+ // Best-effort: the next beat is 60s away and the janitor threshold is many beats wide.
2039
+ console.warn('[RealtimeSession] Liveness pulse failed (session continues):', error);
2040
+ }
2041
+ }
2042
+ /**
2043
+ * One 5 s tick: cuts the packets encoded since the last tick into shards, stores each one, then
2044
+ * uploads them. Best-effort — never disturbs the call and never rejects. Overlapping ticks are
2045
+ * safe: `Append` and `SealTail` are synchronous, so indexes are assigned in order, and the queue
2046
+ * bounds how many flushes upload at once. A failed shard is retained by the queue and resent under
2047
+ * its own index on later ticks (#5197), and stays in the store until acknowledged.
2048
+ *
2049
+ * Every tick seals whatever it took, even under a full window (the window's 250-packet and byte caps
2050
+ * still split a larger batch). Capture starts slightly after this timer, so a tick almost never
2051
+ * holds exactly 250 packets; sealing only full windows would leave nearly 5 s of encoded audio in
2052
+ * memory at every moment, unstored and unsent, which a killed tab loses.
2053
+ */
2054
+ async flushRecordingShards() {
2055
+ const recorder = this.recorder;
2056
+ const pipeline = this.recordingShards;
2057
+ if (!recorder || !pipeline) {
2058
+ return;
2059
+ }
2060
+ let sealed;
2061
+ try {
2062
+ sealed = pipeline.Window.Append(recorder.TakeEncodedPackets());
2063
+ const tick = pipeline.Window.SealTail();
2064
+ if (tick) {
2065
+ sealed.push(tick);
2066
+ }
2067
+ }
2068
+ catch (error) {
2069
+ console.warn(`[RealtimeSession] Failed to take encoded recording packets for session ${pipeline.SessionID}; this tick's audio is lost:`, error);
2070
+ return;
2071
+ }
2072
+ await this.storeShards(pipeline, sealed);
2073
+ await pipeline.Queue.Flush(sealed);
2074
+ }
2075
+ /** Writes shards to the local store, in order, before their upload starts. Never rejects. */
2076
+ async storeShards(pipeline, shards) {
2077
+ const store = this.ensureShardStore();
2078
+ for (const shard of shards) {
2079
+ try {
2080
+ await store.Put({ SessionID: pipeline.SessionID, UserID: pipeline.UserID, Index: shard.Index, Bytes: shard.Bytes, SavedAt: Date.now() });
2081
+ }
2082
+ catch (error) {
2083
+ // The store contract is never-throw; an override that does must still not stop the upload.
2084
+ console.warn(`[RealtimeSession] Failed to store recording shard ${pipeline.SessionID}/${shard.Index}; uploading it anyway:`, error);
2085
+ }
2086
+ }
2087
+ }
2088
+ /** Removes an acknowledged shard from the local store. Never rejects. */
2089
+ deleteStoredShard(sessionId, index) {
2090
+ return DeleteStoredRecordingShard(this.ensureShardStore(), sessionId, index);
2091
+ }
2092
+ /**
2093
+ * Tells the server, right now, how much of the recording exists — for a host's `pagehide` and
2094
+ * `visibilitychange → hidden` handlers, where the page may be gone a moment later. Synchronous:
2095
+ * it seals the pending partial shard and fires the `keepalive` terminal with it, then re-sends
2096
+ * with `keepalive` whatever the server has not acknowledged yet (see
2097
+ * {@link dispatchRecordingTerminal}), and returns without waiting. Never stops the recorder: after
2098
+ * a `hidden` the call may well continue, and later audio simply starts the next shard index (the
2099
+ * tail is sealed, so no index is ever re-sent with different bytes). Also works while End is still
2100
+ * stopping the recorder, which is exactly when a host that does not await End navigates away.
2101
+ * No-op when nothing is recording, there is no session id or no transport, or nothing was
2102
+ * captured. Never throws.
2103
+ *
2104
+ * Deliberately opens NO IndexedDB transaction inside the handler: a `pagehide` handler gets a few
2105
+ * milliseconds, and a store write started there could not complete during unload anyway. The
2106
+ * shards it sealed are written to the store from a `setTimeout(0)` task after the handler returns:
2107
+ * on a page that is merely hidden (the call goes on) that keeps the durable copy until the server
2108
+ * acknowledges it; on a page that is unloading the task just never runs.
2109
+ */
2110
+ SendRecordingTerminal(options) {
2111
+ const recorder = this.recorder;
2112
+ const pipeline = this.recordingShards;
2113
+ if (!recorder || !pipeline || !pipeline.Transport) {
2114
+ return;
2115
+ }
2116
+ try {
2117
+ const sealed = pipeline.Window.Append(recorder.TakeEncodedPackets());
2118
+ const tail = pipeline.Window.SealTail();
2119
+ if (pipeline.Window.LastSealedIndex < 0) {
2120
+ return;
2121
+ }
2122
+ const final = options?.Final ?? true;
2123
+ if (sealed.length === 0 && !tail && pipeline.FinalTerminal?.LastSealedIndex === pipeline.Window.LastSealedIndex) {
2124
+ return;
2125
+ }
2126
+ void this.dispatchRecordingTerminal(pipeline, sealed, tail, true, final);
2127
+ }
2128
+ catch (error) {
2129
+ console.warn(`[RealtimeSession] Failed to send the recording terminal for session ${pipeline.SessionID}:`, error);
2130
+ }
2131
+ }
2132
+ /**
2133
+ * Fires the terminal request, then re-sends with `keepalive` every shard the server has not
2134
+ * acknowledged, then stores the shards sealed for this terminal. Shared by End and the page-lifecycle
2135
+ * handlers. Everything that needs the network is dispatched synchronously, before any storage work,
2136
+ * because the page may close at any moment from here on: an ordinary `fetch` is aborted on unload,
2137
+ * and a `keepalive` one is not.
2138
+ *
2139
+ * - The terminal goes first so it always gets the keepalive budget. The server waits for the shards
2140
+ * the terminal says exist, and assembles as soon as the last missing one arrives.
2141
+ * - The re-sent shards are the queue's unacknowledged ones (on the wire, waiting in a batch, or
2142
+ * retained after a failure) plus `sealedNow`, oldest first, within
2143
+ * {@link recordingKeepaliveBudgetBytes} together with the tail. A `sealedNow` shard that does not
2144
+ * fit, or whose keepalive upload fails, goes through the ordinary queue instead (it may still
2145
+ * land if the page stays). The queue keeps its own shards either way; a duplicate is acked as one.
2146
+ * - `sealedNow` and the tail are stored after the dispatch, from a `setTimeout(0)` task when called
2147
+ * from a page-lifecycle handler (`deferStore`). A shard's stored copy is deleted on its ack, after
2148
+ * the store write, so a fast ack can never leave an orphan row.
2149
+ *
2150
+ * Resolves true when the terminal was acknowledged. Never rejects.
2151
+ */
2152
+ dispatchRecordingTerminal(pipeline, sealedNow, tail, deferStore, final) {
2153
+ const sessionId = pipeline.SessionID;
2154
+ const terminalAcked = this.sendRecordingTerminal(pipeline, tail, final);
2155
+ const toStore = tail ? [...sealedNow, tail] : sealedNow;
2156
+ let stored = Promise.resolve();
2157
+ if (toStore.length > 0) {
2158
+ stored = deferStore ? this.storeShardsLater(pipeline, toStore) : this.storeShards(pipeline, toStore);
2159
+ }
2160
+ this.resendUnacknowledgedWithKeepalive(pipeline, sealedNow, stored);
2161
+ const ackedPromise = terminalAcked.then(async (acked) => {
2162
+ await stored;
2163
+ if (!tail) {
2164
+ return acked;
2165
+ }
2166
+ if (acked) {
2167
+ await this.deleteStoredShard(sessionId, tail.Index);
2168
+ }
2169
+ else {
2170
+ // Retried as an ordinary shard; a Flush on a closed queue (the recording ended) is a logged no-op.
2171
+ await pipeline.Queue.Flush([tail]);
2172
+ }
2173
+ return acked;
2174
+ });
2175
+ if (final) {
2176
+ pipeline.FinalTerminal = { LastSealedIndex: pipeline.Window.LastSealedIndex, Acked: terminalAcked };
2177
+ }
2178
+ return ackedPromise;
2179
+ }
2180
+ /** Counts `bytes` against the page's keepalive budget until `request` settles. */
2181
+ trackKeepalive(bytes, request) {
2182
+ this.keepaliveBytesInFlight += bytes;
2183
+ return request.finally(() => {
2184
+ this.keepaliveBytesInFlight -= bytes;
2185
+ });
2186
+ }
2187
+ /**
2188
+ * The `keepalive` re-send part of {@link dispatchRecordingTerminal}, budgeted against everything this
2189
+ * page still has in flight (the terminal just sent included). A shard whose `keepalive` copy is already
2190
+ * on the wire is not sent again. Never throws.
2191
+ */
2192
+ resendUnacknowledgedWithKeepalive(pipeline, sealedNow, stored) {
2193
+ const transport = pipeline.Transport;
2194
+ const sessionId = pipeline.SessionID;
2195
+ const fresh = new Set(sealedNow.map((shard) => shard.Index));
2196
+ const toQueue = (shard) => stored.then(() => pipeline.Queue.Flush([shard]));
2197
+ const byIndex = new Map();
2198
+ for (const shard of [...pipeline.Queue.Unacknowledged(), ...sealedNow]) {
2199
+ byIndex.set(shard.Index, shard);
2200
+ }
2201
+ for (const shard of [...byIndex.values()].sort((a, b) => a.Index - b.Index)) {
2202
+ const key = `${sessionId}/${shard.Index}`;
2203
+ if (this.keepaliveShardsInFlight.has(key)) {
2204
+ continue;
2205
+ }
2206
+ const budget = RealtimeSessionRuntime.recordingKeepaliveBudgetBytes - this.keepaliveBytesInFlight;
2207
+ if (!transport || shard.Bytes.length > budget) {
2208
+ if (fresh.has(shard.Index)) {
2209
+ void toQueue(shard);
2210
+ }
2211
+ continue;
2212
+ }
2213
+ this.keepaliveShardsInFlight.add(key);
2214
+ void this.trackKeepalive(shard.Bytes.length, transport.UploadShard(sessionId, shard, { Keepalive: true }).then((outcome) => outcome === 'Acked'))
2215
+ .finally(() => this.keepaliveShardsInFlight.delete(key))
2216
+ .catch((error) => {
2217
+ console.warn(`[RealtimeSession] Keepalive upload of recording shard ${sessionId}/${shard.Index} failed: ${describeRecordingFailure(error)}`, error);
2218
+ return false;
2219
+ })
2220
+ .then(async (acked) => {
2221
+ await stored;
2222
+ if (acked) {
2223
+ await this.deleteStoredShard(sessionId, shard.Index);
2224
+ }
2225
+ else if (fresh.has(shard.Index)) {
2226
+ await pipeline.Queue.Flush([shard]);
2227
+ }
2228
+ });
2229
+ }
2230
+ }
2231
+ /**
2232
+ * Stores shards from a `setTimeout(0)` task rather than now — the path for shards sealed inside a
2233
+ * page-lifecycle handler, which must not open an IndexedDB transaction itself. Resolves once they
2234
+ * are stored (never, if the page unloads first). Never rejects.
2235
+ */
2236
+ storeShardsLater(pipeline, shards) {
2237
+ return new Promise((resolve) => {
2238
+ setTimeout(() => {
2239
+ void this.storeShards(pipeline, shards).then(resolve);
2240
+ }, 0);
2241
+ });
2242
+ }
2243
+ /**
2244
+ * Sends the terminal request for a pipeline whose tail is already sealed. Resolves true on a server
2245
+ * ack and logs the outcome with the session id. Never rejects.
2246
+ */
2247
+ async sendRecordingTerminal(pipeline, tail, final) {
2248
+ const lastShardIndex = pipeline.Window.LastSealedIndex;
2249
+ const capturedSamples = pipeline.Window.SealedSamples;
2250
+ const context = `session ${pipeline.SessionID} (last shard ${lastShardIndex}, ${capturedSamples} samples)`;
2251
+ if (!pipeline.Transport) {
2252
+ console.warn(`[RealtimeSession] Recording terminal NOT sent for ${context}: no recording transport; stored shards are drained on a later load.`);
2253
+ return false;
2254
+ }
2255
+ try {
2256
+ const acked = await this.trackKeepalive(tail?.Bytes.length ?? 0, pipeline.Transport.SendTerminal(pipeline.SessionID, {
2257
+ Tail: tail?.Bytes ?? null,
2258
+ LastShardIndex: lastShardIndex,
2259
+ CapturedSamples: capturedSamples,
2260
+ Final: final,
2261
+ }));
2262
+ if (!acked) {
2263
+ console.warn(`[RealtimeSession] Recording terminal for ${context} was not acknowledged; the server janitor assembles what arrived.`);
2264
+ }
2265
+ return acked;
2266
+ }
2267
+ catch (error) {
2268
+ console.warn(`[RealtimeSession] Recording terminal for ${context} failed: ${describeRecordingFailure(error)}`, error);
2269
+ return false;
2270
+ }
2271
+ }
2272
+ /**
2273
+ * Ends the recording at teardown and STARTS its terminal request: stops the recorder (flushing the
2274
+ * encoder), cuts the remaining packets into shards with the tail trimmed to the recorder's captured
2275
+ * sample count, and dispatches the `keepalive` terminal plus `keepalive` re-sends of everything not
2276
+ * yet acknowledged, before storing anything ({@link dispatchRecordingTerminal}). Teardown awaits the
2277
+ * returned terminal only after `CloseAgentSession` ({@link awaitRecordingTerminal}), so the small,
2278
+ * critical close never waits behind the recording (#5195).
2279
+ *
2280
+ * The recorder and pipeline stay visible to {@link SendRecordingTerminal} until the recorder has
2281
+ * stopped: a `pagehide` in that window (a host that navigates away without awaiting End) still
2282
+ * sends what was captured, and End then continues at the next index.
2283
+ *
2284
+ * Returns null when nothing was recording or nothing was captured (no terminal is sent then).
2285
+ * Best-effort: never throws, never blocks teardown on the network.
2286
+ */
2287
+ async finishRecording() {
2288
+ this.stopShardFlushing();
2289
+ const recorder = this.recorder;
2290
+ const pipeline = this.recordingShards;
2291
+ this.currentTurnStartMs = null;
2292
+ this.turnAudioStartCaptured = false;
2293
+ this.userSpeechStartMs = null;
2294
+ this.userSpeechStartSuperseded = false;
2295
+ if (!recorder || !pipeline) {
2296
+ this.recorder = null;
2297
+ this.recordingShards = null;
2298
+ return null;
2299
+ }
2300
+ try {
2301
+ const capturedSamples = await this.stopRecorder(recorder, pipeline.SessionID);
2302
+ this.recorder = null;
2303
+ this.recordingShards = null;
2304
+ const sealed = pipeline.Window.Append(recorder.TakeEncodedPackets());
2305
+ // Without a count from the recorder the tail keeps every packet it holds (SealTail's default).
2306
+ const tail = pipeline.Window.SealTail(capturedSamples === null ? undefined : Math.max(0, capturedSamples - pipeline.Window.SealedSamples));
2307
+ if (pipeline.Window.LastSealedIndex < 0) {
2308
+ pipeline.Queue.Close();
2309
+ return null;
2310
+ }
2311
+ if (sealed.length === 0 && !tail && pipeline.FinalTerminal?.LastSealedIndex === pipeline.Window.LastSealedIndex) {
2312
+ return { Pipeline: pipeline, Acked: pipeline.FinalTerminal.Acked };
2313
+ }
2314
+ return { Pipeline: pipeline, Acked: this.dispatchRecordingTerminal(pipeline, sealed, tail, false, true) };
2315
+ }
2316
+ catch (error) {
2317
+ console.warn(`[RealtimeSession] Failed to finish the call recording for session ${pipeline.SessionID}; stored shards are drained at the next session start or load:`, error);
2318
+ this.recorder = null;
2319
+ this.recordingShards = null;
2320
+ pipeline.Queue.Close();
2321
+ this.rearmPendingShardDrain();
2322
+ return null;
2323
+ }
2324
+ }
2325
+ /**
2326
+ * Stops the recorder, bounded by {@link RECORDER_STOP_TIMEOUT_MS}. Resolves the captured sample
2327
+ * count, or null when the recorder threw or did not finish in time (logged) — the caller then ends
2328
+ * the recording at the packets already encoded. Never rejects.
2329
+ */
2330
+ async stopRecorder(recorder, sessionId) {
2331
+ let timer;
2332
+ const timedOut = new Promise((resolve) => {
2333
+ timer = setTimeout(() => resolve('timeout'), RECORDER_STOP_TIMEOUT_MS);
2334
+ });
2335
+ try {
2336
+ const outcome = await Promise.race([recorder.Stop(), timedOut]);
2337
+ if (outcome === 'timeout') {
2338
+ console.warn(`[RealtimeSession] Recorder did not stop within ${RECORDER_STOP_TIMEOUT_MS}ms for session ${sessionId}; ending the recording at the packets already encoded.`);
2339
+ return null;
2340
+ }
2341
+ return outcome;
2342
+ }
2343
+ catch (error) {
2344
+ console.warn(`[RealtimeSession] Recorder failed to stop for session ${sessionId}; ending the recording at the packets already encoded:`, error);
2345
+ return null;
2346
+ }
2347
+ finally {
2348
+ clearTimeout(timer);
2349
+ }
2350
+ }
2351
+ /**
2352
+ * Waits for the terminal {@link finishRecording} started, bounded by
2353
+ * {@link RECORDING_TERMINAL_ACK_TIMEOUT_MS}, logs the outcome, then closes the recording's queue
2354
+ * (anything it had not delivered stays in the local store for the next-load drain). Never throws.
2355
+ */
2356
+ async awaitRecordingTerminal(pending) {
2357
+ const sessionId = pending.Pipeline.SessionID;
2358
+ let timer;
2359
+ const timedOut = new Promise((resolve) => {
2360
+ timer = setTimeout(() => resolve('timeout'), RECORDING_TERMINAL_ACK_TIMEOUT_MS);
2361
+ });
2362
+ try {
2363
+ const outcome = await Promise.race([pending.Acked.then((ok) => (ok ? 'ack' : 'failed')), timedOut]);
2364
+ if (outcome === 'ack') {
2365
+ console.log(`[RealtimeSession] Recording terminal acknowledged for session ${sessionId}.`);
2366
+ }
2367
+ else if (outcome === 'timeout') {
2368
+ console.warn(`[RealtimeSession] Recording terminal for session ${sessionId} not acknowledged within ${RECORDING_TERMINAL_ACK_TIMEOUT_MS}ms; ending anyway (the keepalive request may still land).`);
2369
+ }
2370
+ // 'failed' was already logged with its reason by sendRecordingTerminal.
2371
+ }
2372
+ finally {
2373
+ clearTimeout(timer);
2374
+ pending.Pipeline.Queue.Close();
2375
+ // The recording may have left shards in the store (an unacknowledged tail, windows the queue
2376
+ // still held or had on the wire at Close), so re-arm the drain for the next session start on
2377
+ // this page. Re-armed here, after the wait, so it never races the recording's own terminal;
2378
+ // and unconditionally, because whether a fire-and-forget upload landed is not known here and
2379
+ // a drain of an empty store costs one read.
2380
+ this.rearmPendingShardDrain();
2381
+ }
2382
+ }
2383
+ /**
2384
+ * The resolved behavior of an in-session channel (display, exposure ceiling, config, what put it
2385
+ * in the session), or `null` when the channel is not in this session. Hosts and overlays read the
2386
+ * resolved display from here rather than from the channel's code default.
2387
+ *
2388
+ * @param channelName The channel's key (case-insensitive).
2389
+ */
2390
+ GetResolvedChannel(channelName) {
2391
+ return this.resolvedChannels.get(NormalizeChannelKey(channelName)) ?? null;
2392
+ }
2393
+ /**
2394
+ * Replaces the store the user's per-channel exposure choices are kept in. The default persists them in
2395
+ * the signed-in user's settings (`mj.realtime.visualPerception.v1`) and keeps them in memory for an
2396
+ * anonymous principal or a connect-only embed; a host that remembers them somewhere else supplies its own.
2397
+ */
2398
+ SetExposurePreferences(preferences) {
2399
+ this.exposurePreferences = preferences;
2400
+ }
2401
+ /** The store the user's per-channel exposure choices are currently kept in (so a host can wrap it). */
2402
+ get ExposurePreferences() {
2403
+ return this.exposurePreferences;
2404
+ }
2405
+ /** Whether user settings can be written: a signed-in user on a provider that has entity metadata. */
2406
+ canPersistUserSettings() {
2407
+ const provider = this.Provider;
2408
+ return Boolean(provider?.CurrentUser?.ID) && (provider?.Entities?.length ?? 0) > 0;
2409
+ }
2410
+ /**
2411
+ * How much of a channel the model may perceive right now: the channel's ceiling, lowered by the server's
2412
+ * policy and the user's choice. `null` when the channel is not in this session.
2413
+ *
2414
+ * @param channelName The channel's key (case-insensitive).
2415
+ */
2416
+ GetChannelExposure(channelName) {
2417
+ return this.findDispatchableChannel(channelName)?.Plugin.Exposure ?? null;
2418
+ }
2419
+ /**
2420
+ * The user's own choice of how much of a channel the agent may perceive. Takes effect immediately
2421
+ * (frames and notes the new level forbids stop, and the model is told), and is remembered per channel key
2422
+ * for next time. The level can only LOWER what the server's policy allows; choosing `'pixels'` for a channel
2423
+ * the agent's policy capped at `'state'` changes nothing.
2424
+ *
2425
+ * @param channelName The channel's key (case-insensitive).
2426
+ * @param level The user's choice; `undefined` clears it (back to what policy allows).
2427
+ * @returns `false` when the channel is not in this session (nothing is applied, but the choice is still remembered).
2428
+ */
2429
+ SetUserChannelExposure(channelName, level) {
2430
+ this.exposurePreferences.Set(channelName, level);
2431
+ const channel = this.findDispatchableChannel(channelName);
2432
+ if (!channel) {
2433
+ return false;
2434
+ }
2435
+ this.applyChannelExposure(channel.Plugin);
2436
+ return true;
2437
+ }
2438
+ /** Pushes a channel's current exposure inputs (server policy, user choice) into the channel. */
2439
+ applyChannelExposure(plugin) {
2440
+ const resolved = this.GetResolvedChannel(plugin.ChannelName);
2441
+ const user = this.exposurePreferences.Get(plugin.ChannelName);
2442
+ const reasons = (resolved?.ExposureLimits ?? []).map((limit) => limit.Reason);
2443
+ // The user's limit binds only when it is lower than what the server allows; otherwise it changes nothing
2444
+ // and the agent should not be told about it.
2445
+ const allowed = resolved?.Exposure ?? plugin.GetDescriptor().MaxExposure;
2446
+ if (user !== undefined && CompareExposure(user, allowed) < 0) {
2447
+ reasons.push(UserExposureReason(user));
2448
+ }
2449
+ plugin.ApplyExposure({ Policy: resolved?.Exposure, User: user, Reasons: reasons });
2450
+ // A capture this channel fronts follows it: the agent sees its frames only while the channel's exposure allows pixels.
2451
+ const kind = plugin.CaptureKind;
2452
+ if (kind) {
2453
+ this.captures?.SetVisibleToAgent(kind, plugin.Exposure === 'pixels');
2454
+ }
2455
+ this.syncChannelVideoSource(plugin);
2456
+ }
2457
+ /**
2458
+ * Sends a frame from a channel's context ({@link RealtimeChannelContext.SendVideoFrame}, deprecated) through the
2459
+ * session's {@link VideoSourceArbiter}, as the channel's own source: it reaches the model only while the arbiter picks
2460
+ * that source, and the arbiter tells the model when it switches. The source registers with the first frame, under the
2461
+ * id the channel's own frame bridge uses, switched on only while the channel's exposure allows pixels.
2462
+ */
2463
+ sendChannelVideoFrame(plugin, base64Image, mimeType) {
2464
+ const client = this.client;
2465
+ if (!client || !this.isSessionLive() || !client.IsTrackEstablished('video', 'inbound')) {
2466
+ return;
2467
+ }
2468
+ const arbiter = VideoSourceArbiter.ForSink(client);
2469
+ const descriptor = plugin.GetDescriptor();
2470
+ const sourceId = ChannelVideoSourceID(descriptor.Key, plugin.InstanceId);
2471
+ if (!arbiter.HasSource(sourceId)) {
2472
+ arbiter.RegisterSource({
2473
+ SourceID: sourceId,
2474
+ Label: descriptor.DisplayName,
2475
+ Kind: 'surface',
2476
+ ChannelKey: descriptor.Key,
2477
+ Enabled: plugin.Exposure === 'pixels',
2478
+ });
2479
+ }
2480
+ arbiter.PushFrame(sourceId, base64Image, mimeType);
2481
+ }
2482
+ /**
2483
+ * Keeps a channel's video source at the arbiter, when it has one, in step with the channel's exposure: on while the
2484
+ * agent may see its pixels, off otherwise. Quiet, because the channel tells the model about its own exposure change.
2485
+ */
2486
+ syncChannelVideoSource(plugin) {
2487
+ if (!this.client) {
2488
+ return;
2489
+ }
2490
+ const arbiter = VideoSourceArbiter.ForSink(this.client);
2491
+ const sourceId = ChannelVideoSourceID(plugin.GetDescriptor().Key, plugin.InstanceId);
2492
+ if (arbiter.HasSource(sourceId)) {
2493
+ arbiter.SetSourceEnabled(sourceId, plugin.Exposure === 'pixels', false);
2494
+ }
2495
+ }
2496
+ /** Follows the session client's video-source arbiter so {@link VideoSources$} reflects it. */
2497
+ watchVideoSources(client) {
2498
+ this.unwatchVideoSources();
2499
+ const arbiter = VideoSourceArbiter.ForSink(client);
2500
+ arbiter.SetFocusedChannel(this.focusedChannelKey);
2501
+ this.stopWatchingVideoSources = arbiter.OnChange(() => this._videoSources$.next(arbiter.GetSources()));
2502
+ this._videoSources$.next(arbiter.GetSources());
2503
+ }
2504
+ /**
2505
+ * Tells the session which channel's surface the user is looking at (`null` for none, e.g. the activity tab).
2506
+ * When the model can see only one video source and several are live, the one the user is looking at is the one
2507
+ * it sees (after an explicit pick, and after a camera or screen share the user started). Safe to call before the
2508
+ * session is live: it applies when the connection comes up.
2509
+ *
2510
+ * @param channelKey The focused channel's key, as it appears on {@link VideoSources$} entries' `ChannelKey`.
2511
+ */
2512
+ SetFocusedChannel(channelKey) {
2513
+ this.focusedChannelKey = channelKey;
2514
+ if (this.client) {
2515
+ VideoSourceArbiter.ForSink(this.client).SetFocusedChannel(channelKey);
2516
+ }
2517
+ }
2518
+ /** Stops following the arbiter and clears {@link VideoSources$}. Safe to call when nothing is watched. */
2519
+ unwatchVideoSources() {
2520
+ this.stopWatchingVideoSources?.();
2521
+ this.stopWatchingVideoSources = null;
2522
+ if (this._videoSources$.value.length > 0) {
2523
+ this._videoSources$.next([]);
2524
+ }
2525
+ }
2526
+ /**
2527
+ * Turns the agent's view of one video source on or off — what the "agent can see" control calls.
2528
+ *
2529
+ * A source that belongs to a channel goes through {@link SetUserChannelExposure}, so the choice is
2530
+ * remembered per channel and the channel itself tells the model. Any other source (a camera or screen
2531
+ * share that is not a channel) is switched at the arbiter, which tells the model.
2532
+ *
2533
+ * @param sourceId The source's id (from {@link VideoSources$}).
2534
+ * @param enabled Whether the agent may see it.
2535
+ * @returns `false` when there is no such source.
2536
+ */
2537
+ SetVideoSourceEnabled(sourceId, enabled) {
2538
+ const client = this.client;
2539
+ if (!client) {
2540
+ return false;
2541
+ }
2542
+ const arbiter = VideoSourceArbiter.ForSink(client);
2543
+ const source = arbiter.GetSources().find((s) => s.SourceID === sourceId);
2544
+ if (!source) {
2545
+ return false;
2546
+ }
2547
+ if (source.ChannelKey && this.findDispatchableChannel(source.ChannelKey)) {
2548
+ return this.SetUserChannelExposure(source.ChannelKey, enabled ? undefined : 'state');
2549
+ }
2550
+ return arbiter.SetSourceEnabled(sourceId, enabled);
2551
+ }
2552
+ /**
2553
+ * Picks the video source the agent sees: what the "agent can see" control calls when more sources are on than the
2554
+ * model takes. The pick beats every other rule (a camera or screen share the user started, the surface the user is
2555
+ * looking at) until it is cleared, or its source is turned off or leaves. The pick shows on {@link VideoSources$}
2556
+ * (`Picked`), and the model is told when what it sees changes.
2557
+ *
2558
+ * @param sourceId The source's id (from {@link VideoSources$}), or `null` to let the call decide again.
2559
+ * @returns `false` with no live session, or for a source that is not there or is turned off.
2560
+ */
2561
+ SelectVideoSource(sourceId) {
2562
+ return this.client ? VideoSourceArbiter.ForSink(this.client).SelectSource(sourceId) : false;
2563
+ }
2564
+ /**
2565
+ * The `on-demand` channels that are in the session but not open yet — what the agent can open
2566
+ * through `ContextTool`. A fresh array; empty before a session starts and after teardown.
2567
+ */
2568
+ get AdvertisedChannels() {
2569
+ return this.advertisedChannels.map((p) => p.Plugin);
968
2570
  }
969
- /** Stops the periodic crash-recovery shard flush. */
970
- stopSegmentFlushing() {
971
- if (this.segmentTimer) {
972
- clearInterval(this.segmentTimer);
973
- this.segmentTimer = null;
974
- }
2571
+ /**
2572
+ * Resolves, initializes and mounts the session's channels in one step using the scope the browser
2573
+ * can decide alone (code defaults + host declarations), and returns the native tools to declare at
2574
+ * mint. This is the single-step composition of {@link prepareChannelScope} + {@link activateChannels}
2575
+ * that a session start spreads across the mint; it is kept as a unit because it is the cleanest seam
2576
+ * for exercising the plugin plumbing without a mint.
2577
+ */
2578
+ async startChannels(hostChannels) {
2579
+ const scope = await this.prepareChannelScope(hostChannels);
2580
+ this.activateChannels(this.localChannelScope?.Channels ?? []);
2581
+ return scope.NativeTools;
975
2582
  }
976
2583
  /**
977
- * Starts telling the server this session is still in use (#3533).
978
- *
979
- * **Why the server cannot work this out on its own.** In the client-direct topology the audio
980
- * goes browser → provider over WebRTC. The server sees the mint, a few channel actions in the
981
- * first seconds, and then nothing at all — so `SessionManager.RecordActivity` stops being
982
- * reached while the conversation is still going. `LastActiveAt` freezes ~45 seconds in, and
983
- * `SessionJanitor` — which cannot distinguish an active call from an abandoned one — force-closes
984
- * it at `closeThresholdMinutes`, mid-sentence, taking the user's surfaces with it. A session
985
- * whose channels are all client-side (whiteboard, media) goes quiet from the server's point of
986
- * view almost immediately.
2584
+ * Stage 1: constructs the channel plugins (registry rows and host declarations), reads their
2585
+ * descriptors, resolves the scope the browser can decide alone, and builds what the mint needs —
2586
+ * the native tools to declare and the candidates to report. Starts nothing.
987
2587
  *
988
- * The browser is the only participant that knows the call is alive, so it is the one that has to
989
- * say so. Raising `closeThresholdMinutes` is not the fix — it just makes the janitor slower at
990
- * its real job (reaping rows orphaned by a crash) without making liveness observable.
991
- *
992
- * The pulse is best-effort by design: a failed beat is logged and skipped, never surfaced to the
993
- * user and never allowed to end the session. Losing one beat costs nothing because the threshold
994
- * is many beats wide; turning a transient network blip into a visible error would be a worse
995
- * failure than the one this fixes. Write amplification is bounded on the server side too, where
996
- * `SessionManager.Heartbeat` coalesces persisted writes.
2588
+ * @param hostChannels Channels the host brings to this session.
2589
+ * @returns The native tools to declare at mint, the candidates as JSON (`null` when there are none,
2590
+ * so a channel-less session sends exactly the mint it always did), and whether the host could show the agent's
2591
+ * video ({@link PreparedChannelsShowAgentVideo}).
997
2592
  */
998
- startLivenessPulse() {
999
- this.stopLivenessPulse();
1000
- this.livenessTimer = setInterval(() => { void this.pulseLiveness(); }, RealtimeSessionRuntime.livenessPulseMs);
1001
- }
1002
- /** Stops the liveness pulse. Idempotent — safe on a session that never started one. */
1003
- stopLivenessPulse() {
1004
- if (this.livenessTimer) {
1005
- clearInterval(this.livenessTimer);
1006
- this.livenessTimer = null;
1007
- }
2593
+ async prepareChannelScope(hostChannels) {
2594
+ this.discardUnmountedChannels();
2595
+ const prepared = await this.prepareChannels(hostChannels);
2596
+ this.preparedChannels = prepared;
2597
+ const candidates = prepared.map((p) => BuildChannelCandidate(p, p.Plugin.GetDescriptor()));
2598
+ const local = ResolveLocalChannelScope(candidates);
2599
+ this.localChannelScope = local;
2600
+ for (const excluded of local.Excluded) {
2601
+ console.warn(`[RealtimeSession] Channel '${excluded.Key}' is not in this session (${excluded.Reason}).`);
2602
+ }
2603
+ const nativeTools = SelectNativeChannelTools(local.Channels, ToolsByChannelKey(candidates));
2604
+ // `Registry` is the browser's own view and never goes over the wire: the server reads the registry itself.
2605
+ const wire = candidates.map(({ Registry: _registry, ...candidate }) => candidate);
2606
+ return { NativeTools: nativeTools, CandidatesJson: wire.length > 0 ? JSON.stringify(wire) : null, ShowsAgentVideo: PreparedChannelsShowAgentVideo(prepared) };
1008
2607
  }
1009
2608
  /**
1010
- * One liveness beat. Reads the session id at fire time rather than closing over it, so a beat
1011
- * that fires during teardown finds `null` and does nothing instead of resurrecting a closed row.
2609
+ * Constructs a plugin for every active-or-inactive registry row and every host declaration. An
2610
+ * INACTIVE row is constructed too — only to learn its key so the kill switch can name it — and is
2611
+ * never initialized. A plugin whose descriptor cannot be read is skipped (logged), never fatal.
1012
2612
  */
1013
- async pulseLiveness() {
1014
- const agentSessionId = this.agentSessionId;
1015
- if (!agentSessionId) {
1016
- return;
1017
- }
1018
- const mutation = `
1019
- mutation AgentSessionHeartbeat($agentSessionId: String!) {
1020
- AgentSessionHeartbeat(agentSessionId: $agentSessionId)
1021
- }
1022
- `;
1023
- try {
1024
- await this.gql().ExecuteGQL(mutation, { agentSessionId });
2613
+ async prepareChannels(hostChannels) {
2614
+ const rows = await this.fetchChannelDefinitions();
2615
+ await this.loadChannelClasses([...rows.map((r) => r.ClientPluginClass), ...(hostChannels ?? []).map((d) => d.ClientPluginClass)]);
2616
+ const prepared = [];
2617
+ for (const row of rows) {
2618
+ const plugin = this.resolveChannelPlugin(row);
2619
+ if (plugin) {
2620
+ plugin.ApplySurfacePlacement(row.SurfacePlacement);
2621
+ prepared.push({ Plugin: plugin, Key: plugin.ChannelName, Registry: row.IsActive ? 'active' : 'inactive' });
2622
+ }
1025
2623
  }
1026
- catch (error) {
1027
- // Best-effort: the next beat is 60s away and the janitor threshold is many beats wide.
1028
- console.warn('[RealtimeSession] Liveness pulse failed (session continues):', error);
2624
+ for (const declaration of hostChannels ?? []) {
2625
+ const plugin = this.createHostChannelPlugin(declaration);
2626
+ if (!plugin) {
2627
+ continue;
2628
+ }
2629
+ const existing = FindPreparedChannel(prepared, plugin.ChannelName);
2630
+ if (existing) {
2631
+ // The host's instance replaces the registry's (it may carry the host's collaborators); the
2632
+ // registry row's state — notably the kill switch, and where its surface shows — still applies.
2633
+ plugin.ApplySurfacePlacement(existing.Plugin.SurfacePlacement);
2634
+ existing.Plugin = plugin;
2635
+ existing.HostDeclaration = declaration;
2636
+ }
2637
+ else {
2638
+ prepared.push({ Plugin: plugin, Key: plugin.ChannelName, Registry: 'none', HostDeclaration: declaration });
2639
+ }
1029
2640
  }
2641
+ return prepared.filter((p) => this.canDescribe(p));
1030
2642
  }
1031
2643
  /**
1032
- * Uploads the chunks captured since the last flush as one crash-recovery shard (durability only;
1033
- * the canonical file is still the full upload at teardown). Best-effort — never disturbs the call.
2644
+ * Gives a host that loads channel code on demand (a widget that keeps a heavy channel in its own download)
2645
+ * the chance to load it before the plugins are built. Never fatal: a loader that throws is logged and the
2646
+ * channel simply is not available, exactly as if its class had never been registered.
2647
+ *
2648
+ * @param keys The `ClientPluginClass` keys of every channel about to be resolved (blank/absent ones included).
1034
2649
  */
1035
- async flushRecordingSegment() {
1036
- const recorder = this.recorder;
1037
- const agentSessionId = this.agentSessionId;
1038
- if (!recorder || !agentSessionId) {
2650
+ async loadChannelClasses(keys) {
2651
+ const loader = this.ChannelClassLoader;
2652
+ if (!loader) {
1039
2653
  return;
1040
2654
  }
1041
- try {
1042
- const audioBase64 = await recorder.SnapshotNewSegmentBase64();
1043
- if (!audioBase64) {
1044
- return;
1045
- }
1046
- const index = this.segmentIndex++;
1047
- const mutation = `
1048
- mutation UploadRealtimeRecordingSegment($agentSessionId: String!, $segmentIndex: Int!, $audioBase64: String!, $mimeType: String!) {
1049
- UploadRealtimeRecordingSegment(agentSessionId: $agentSessionId, segmentIndex: $segmentIndex, audioBase64: $audioBase64, mimeType: $mimeType)
2655
+ const wanted = [...new Set(keys.map((k) => k?.trim() ?? '').filter((k) => k.length > 0))];
2656
+ if (wanted.length === 0) {
2657
+ return;
1050
2658
  }
1051
- `;
1052
- // Shards are HEADER-LESS raw little-endian PCM16 (mime audio/L16 with the capture sample rate),
1053
- // NOT individually-playable WAV — recovery is concatenate-in-order then WAV-wrap. The canonical
1054
- // seekable WAV is the consolidated end-of-call upload below.
1055
- const shardMime = `audio/L16;rate=${recorder.SampleRate}`;
1056
- await this.gql().ExecuteGQL(mutation, { agentSessionId, segmentIndex: index, audioBase64, mimeType: shardMime });
2659
+ try {
2660
+ await loader(wanted);
1057
2661
  }
1058
2662
  catch (error) {
1059
- console.warn('[RealtimeSession] Failed to flush recording shard:', error);
2663
+ console.error('[RealtimeSession] The channel class loader failed — channels that needed it are left out:', error);
1060
2664
  }
1061
2665
  }
1062
- /**
1063
- * Stops the active recorder and uploads the captured audio via `UploadRealtimeRecording`.
1064
- * Fully best-effort and wrapped in try/catch — recording upload must NEVER block teardown.
1065
- * No-op when nothing was recorded or there's no session id to attach the file to.
1066
- */
1067
- async stopAndUploadRecording(agentSessionId) {
1068
- this.stopSegmentFlushing();
1069
- const recorder = this.recorder;
1070
- this.recorder = null;
1071
- this.currentTurnStartMs = null;
1072
- this.turnAudioStartCaptured = false;
1073
- if (!recorder) {
1074
- return;
1075
- }
2666
+ /** Whether a prepared plugin's descriptor can be read; logs and rejects one that throws. */
2667
+ canDescribe(prepared) {
1076
2668
  try {
1077
- // Capture the recorder MIME (now 'audio/wav') BEFORE Stop() — the getter reads '' once stopped.
1078
- const mimeType = recorder.MimeType;
1079
- const audioBase64 = await recorder.StopAndEncode();
1080
- // Read the real waveform peaks computed during capture (survives the stop via the snapshot).
1081
- const peaks = recorder.GetPeaks();
1082
- if (!audioBase64 || !agentSessionId) {
1083
- console.warn('[RealtimeSession] ⚠️ recording NOT uploaded — empty recording or no session id.');
1084
- return;
1085
- }
1086
- await this.uploadRecording(agentSessionId, audioBase64, mimeType, peaks);
2669
+ prepared.Plugin.GetDescriptor();
2670
+ return true;
1087
2671
  }
1088
2672
  catch (error) {
1089
- console.warn('[RealtimeSession] Failed to stop/upload call recording:', error);
2673
+ console.error(`[RealtimeSession] Channel '${prepared.Key}' has an unreadable descriptor — leaving it out:`, error);
2674
+ return false;
1090
2675
  }
1091
2676
  }
1092
- /**
1093
- * Runs the `UploadRealtimeRecording` mutation; failures are logged, never thrown. Sends the
1094
- * capture-time waveform `peaks` (max-abs per bucket, normalized 0..1) so the server can persist a
1095
- * `peaks.json` sidecar for fast waveform rendering without re-decoding the audio.
1096
- */
1097
- async uploadRecording(agentSessionId, audioBase64, mimeType, peaks) {
1098
- const mutation = `
1099
- mutation UploadRealtimeRecording($agentSessionId: String!, $audioBase64: String!, $mimeType: String!, $consent: Boolean, $peaks: [Float!]) {
1100
- UploadRealtimeRecording(agentSessionId: $agentSessionId, audioBase64: $audioBase64, mimeType: $mimeType, consent: $consent, peaks: $peaks) {
1101
- Success
1102
- FileID
1103
- ErrorMessage
2677
+ /** Builds a plugin from a host declaration; `null` (logged) when it names nothing buildable. */
2678
+ createHostChannelPlugin(declaration) {
2679
+ if (declaration.Create) {
2680
+ try {
2681
+ return declaration.Create();
2682
+ }
2683
+ catch (error) {
2684
+ console.error('[RealtimeSession] A host-declared channel factory threw — leaving the channel out:', error);
2685
+ return null;
2686
+ }
1104
2687
  }
1105
- }
1106
- `;
1107
- const result = await this.gql().ExecuteGQL(mutation, {
1108
- agentSessionId,
1109
- audioBase64,
1110
- mimeType,
1111
- consent: true,
1112
- peaks
1113
- });
1114
- const payload = result?.UploadRealtimeRecording;
1115
- if (!payload?.Success) {
1116
- console.warn(`[RealtimeSession] ❌ recording upload reported failure: ${payload?.ErrorMessage ?? 'unknown error'} (full result: ${JSON.stringify(result)})`);
2688
+ const key = declaration.ClientPluginClass?.trim();
2689
+ if (!key) {
2690
+ console.warn('[RealtimeSession] A host-declared channel names neither ClientPluginClass nor Create — ignoring it.');
2691
+ return null;
1117
2692
  }
2693
+ return this.resolveChannelPlugin({ Name: key, ClientPluginClass: key, IsActive: true });
1118
2694
  }
1119
- // ── Interactive channels (registry-driven plugins) ─────────────────────────
1120
2695
  /**
1121
- * Resolves, instantiates and initializes the session's interactive-channel plugins from
1122
- * the `MJ: AI Agent Channels` registry, publishes them on {@link ActiveChannels$}, and
1123
- * returns their aggregated client-executed tool declarations for the session mint.
1124
- * Tolerant by design: registry/resolution failures degrade to "no channels" — the voice
1125
- * session itself always proceeds.
2696
+ * Stage 3: initializes the channels the resolved policy puts in the session. `open-on-start` and
2697
+ * `headless` channels are mounted and published on {@link ActiveChannels$}; `on-demand` channels
2698
+ * are held until opened. Prepared plugins the policy left out are dropped without ever having run.
1126
2699
  */
1127
- async startChannels() {
1128
- const channels = await this.loadActiveChannels();
1129
- for (const plugin of channels) {
1130
- this.initializeChannel(plugin);
2700
+ activateChannels(channels) {
2701
+ const reconciled = ReconcileChannelsWithPolicy(this.preparedChannels, channels);
2702
+ for (const key of reconciled.Unknown) {
2703
+ console.warn(`[RealtimeSession] The session policy names channel '${key}' but this host has no plugin for it — ignoring it.`);
2704
+ }
2705
+ this.resolvedChannels.clear();
2706
+ this.advertisedChannels = [];
2707
+ const mounted = [];
2708
+ for (const { Prepared, Resolved } of reconciled.InSession) {
2709
+ this.resolvedChannels.set(NormalizeChannelKey(Resolved.Key), Resolved);
2710
+ // Exposure is applied BEFORE the channel initializes, so what it requests of the model (a video
2711
+ // track, in particular) already reflects what policy allows.
2712
+ this.applyChannelExposure(Prepared.Plugin);
2713
+ if (Resolved.DisplayPolicy === 'on-demand') {
2714
+ this.advertisedChannels.push(Prepared);
2715
+ }
2716
+ else {
2717
+ this.mountChannel(Prepared, Resolved);
2718
+ mounted.push(Prepared.Plugin);
2719
+ }
1131
2720
  }
1132
- this._activeChannels$.next(channels);
1133
- return channels.flatMap(plugin => plugin.GetToolDefinitions());
2721
+ this.preparedChannels = [];
2722
+ this._activeChannels$.next(mounted);
1134
2723
  }
1135
2724
  /**
1136
- * Loads the ACTIVE channel definitions from the registry and resolves each row's
1137
- * `ClientPluginClass` through the MJ ClassFactory into a per-session plugin instance —
1138
- * the client-side mirror of how realtime-model drivers resolve from `BaseRealtimeModel`
1139
- * / `BaseRealtimeClient`. Rows whose plugin class isn't registered are skipped (logged),
1140
- * never fatal.
2725
+ * Applies the mint's outcome to the channel set: the server's policy when it sent a usable one,
2726
+ * else the scope the browser resolved on its own. A policy can only SELECT among the plugins this
2727
+ * host prepared — it cannot add one.
1141
2728
  */
1142
- async loadActiveChannels() {
1143
- const rows = await this.fetchChannelDefinitions();
1144
- const channels = [];
1145
- for (const row of rows) {
1146
- const plugin = this.resolveChannelPlugin(row);
1147
- if (plugin) {
1148
- channels.push(plugin);
1149
- }
2729
+ applySessionClientPolicy(session) {
2730
+ const policy = ParseRealtimeSessionClientPolicy(session.ClientPolicyJson);
2731
+ if (session.ClientPolicyJson && !policy) {
2732
+ console.warn('[RealtimeSession] The mint returned a client policy this runtime cannot read — resolving channels locally.');
1150
2733
  }
1151
- return channels;
2734
+ this.sessionClientTools = policy?.ClientTools ?? null;
2735
+ if (this.preparedChannels.length === 0) {
2736
+ this.resolvedChannels.clear();
2737
+ return; // nothing was prepared (a host that mints and runs the session itself) — no channel set to apply
2738
+ }
2739
+ this.activateChannels(policy?.Channels ?? this.localChannelScope?.Channels ?? []);
2740
+ }
2741
+ /** Drops plugins that were prepared for a start that never mounted them. They were never initialized, so there is nothing to release. */
2742
+ discardUnmountedChannels() {
2743
+ this.preparedChannels = [];
2744
+ this.advertisedChannels = [];
2745
+ this.localChannelScope = null;
1152
2746
  }
1153
2747
  /**
1154
- * Reads the ACTIVE `MJ: AI Agent Channels` rows. With entity metadata: from {@link AIEngineBase}'s
1155
- * cached `AgentChannels` (provider-scoped engine instance, lazy `Config` — no RunView round-trip;
1156
- * the engine's BaseEntity-event reactivity keeps the registry fresh). On a connect-only provider:
1157
- * one `RunDynamicView` query ({@link fetchChannelDefinitionsOverGraphQL}). Failures are logged and
1158
- * degrade to an empty list — channel availability must never block the voice session.
2748
+ * Loads the `MJ: AI Agent Channels` registry rows (ACTIVE AND INACTIVE — an inactive row is the master
2749
+ * kill switch, so the scope decision has to see it). With entity metadata: from {@link AIEngineBase}'s
2750
+ * cached `AgentChannels` (provider-scoped engine instance, lazy `Config` — no RunView round-trip; the
2751
+ * engine's BaseEntity-event reactivity keeps the registry fresh). On a connect-only provider: one
2752
+ * `RunDynamicView` query ({@link fetchChannelDefinitionsOverGraphQL}). Failures are logged and degrade to an
2753
+ * empty list — channel availability must never block the voice session.
1159
2754
  */
1160
2755
  async fetchChannelDefinitions() {
1161
2756
  // A connect-only provider (ConnectGraphQLClient — anonymous embeds) has no entity metadata,
@@ -1168,9 +2763,13 @@ export class RealtimeSessionRuntime {
1168
2763
  try {
1169
2764
  const engine = AIEngineBase.GetProviderInstance(this.Provider, AIEngineBase);
1170
2765
  await engine.Config(false, undefined, this.Provider);
1171
- return (engine.AgentChannels ?? [])
1172
- .filter(c => c.IsActive)
1173
- .map(c => ({ ID: c.ID, Name: c.Name, ClientPluginClass: c.ClientPluginClass }));
2766
+ return (engine.AgentChannels ?? []).map((c) => ({
2767
+ ID: c.ID,
2768
+ Name: c.Name,
2769
+ ClientPluginClass: c.ClientPluginClass,
2770
+ IsActive: c.IsActive,
2771
+ SurfacePlacement: readRowSurfacePlacement(c),
2772
+ }));
1174
2773
  }
1175
2774
  catch (error) {
1176
2775
  console.warn('[RealtimeSession] Channel registry unavailable — starting with no channels:', error);
@@ -1178,10 +2777,10 @@ export class RealtimeSessionRuntime {
1178
2777
  }
1179
2778
  }
1180
2779
  /**
1181
- * The connect-only path of {@link fetchChannelDefinitions}: the same ACTIVE `MJ: AI Agent Channels`
1182
- * rows, read with a dynamic view because a connect-only client has no entity metadata to build a
1183
- * typed RunView from. Same tolerance as the engine path — a failure is logged and means "no
1184
- * channels", never a blocked session.
2780
+ * The connect-only path of {@link fetchChannelDefinitions}: the same `MJ: AI Agent Channels` rows, active
2781
+ * and inactive (an inactive row is the kill switch), read with a dynamic view because a connect-only client
2782
+ * has no entity metadata to build a typed RunView from. Same tolerance as the engine path — a failure is
2783
+ * logged and means "no channels", never a blocked session.
1185
2784
  */
1186
2785
  async fetchChannelDefinitionsOverGraphQL() {
1187
2786
  const query = `query RealtimeChannelRegistry($input: RunDynamicViewInput!) {
@@ -1189,7 +2788,7 @@ export class RealtimeSessionRuntime {
1189
2788
  }`;
1190
2789
  try {
1191
2790
  const result = (await this.gql().ExecuteGQL(query, {
1192
- input: { EntityName: 'MJ: AI Agent Channels', ExtraFilter: 'IsActive = 1', Fields: ['ID', 'Name', 'ClientPluginClass', 'IsActive'] },
2791
+ input: { EntityName: 'MJ: AI Agent Channels', Fields: ['ID', 'Name', 'ClientPluginClass', 'IsActive', 'UIConfig'] },
1193
2792
  }));
1194
2793
  const view = result?.RunDynamicView;
1195
2794
  if (!view?.Success) {
@@ -1198,8 +2797,13 @@ export class RealtimeSessionRuntime {
1198
2797
  }
1199
2798
  return (view.Results ?? [])
1200
2799
  .map((r) => JSON.parse(r.Data))
1201
- .filter((row) => row.IsActive === true)
1202
- .map((row) => ({ ID: row.ID, Name: row.Name, ClientPluginClass: row.ClientPluginClass }));
2800
+ .map((row) => ({
2801
+ ID: row.ID,
2802
+ Name: row.Name,
2803
+ ClientPluginClass: row.ClientPluginClass,
2804
+ IsActive: row.IsActive === true,
2805
+ SurfacePlacement: readUIConfigTextSurfacePlacement(row.Name, row.UIConfig),
2806
+ }));
1203
2807
  }
1204
2808
  catch (error) {
1205
2809
  console.warn('[RealtimeSession] Channel registry unavailable — starting with no channels:', error instanceof Error ? error.message : String(error));
@@ -1211,16 +2815,23 @@ export class RealtimeSessionRuntime {
1211
2815
  * checked first, exactly like the realtime-client drivers) and instantiates a fresh
1212
2816
  * per-session plugin. Returns `null` (logged) when no plugin is registered for the key
1213
2817
  * — e.g. its Load function was never called or the package isn't included client-side.
2818
+ * An inactive row is resolved quietly: it only needs to be named, and a missing plugin for a
2819
+ * channel nobody wants is not worth a warning.
1214
2820
  */
1215
2821
  resolveChannelPlugin(row) {
1216
2822
  const key = row.ClientPluginClass?.trim();
2823
+ const quiet = !row.IsActive;
1217
2824
  if (!key) {
1218
- console.warn(`[RealtimeSession] Channel '${row.Name}' has no ClientPluginClass — skipping.`);
2825
+ if (!quiet) {
2826
+ console.warn(`[RealtimeSession] Channel '${row.Name}' has no ClientPluginClass — skipping.`);
2827
+ }
1219
2828
  return null;
1220
2829
  }
1221
2830
  const registration = MJGlobal.Instance.ClassFactory.GetRegistration(BaseRealtimeChannelClient, key);
1222
2831
  if (!registration) {
1223
- console.warn(`[RealtimeSession] No client plugin registered for channel '${row.Name}' (key '${key}') — skipping.`);
2832
+ if (!quiet) {
2833
+ console.warn(`[RealtimeSession] No client plugin registered for channel '${row.Name}' (key '${key}') — skipping.`);
2834
+ }
1224
2835
  return null;
1225
2836
  }
1226
2837
  const plugin = MJGlobal.Instance.ClassFactory.CreateInstance(BaseRealtimeChannelClient, key);
@@ -1231,24 +2842,157 @@ export class RealtimeSessionRuntime {
1231
2842
  return plugin;
1232
2843
  }
1233
2844
  /**
1234
- * Wires one plugin into the session: hands it its host context and registers its
1235
- * prefix-routed local tool executor (so `<ToolNamePrefix>*` calls run in the browser
1236
- * through {@link BaseRealtimeChannelClient.ApplyAgentTool}, never the server relay).
2845
+ * Wires one plugin into the session: hands it its host context (carrying its resolved config) and,
2846
+ * when it has native tools, registers its prefix-routed local tool executor (so `<ToolNamePrefix>*`
2847
+ * calls run in the browser through {@link BaseRealtimeChannelClient.ApplyAgentTool}, never the
2848
+ * server relay). A channel with no tool prefix has no native route — it is reached through
2849
+ * `ContextTool` — and registering an empty prefix would match EVERY tool name.
1237
2850
  */
1238
- initializeChannel(plugin) {
1239
- plugin.Initialize(this.buildChannelContext(plugin));
2851
+ mountChannel(prepared, resolved) {
2852
+ const plugin = prepared.Plugin;
2853
+ plugin.Initialize(this.buildChannelContext(plugin, resolved?.Config));
2854
+ if (plugin.ToolNamePrefix.length === 0) {
2855
+ return;
2856
+ }
1240
2857
  this.RegisterClientToolHandler(plugin.ToolNamePrefix, (toolName, argsJson) => {
1241
2858
  // The agent is ACTING on this channel — surface-discovery signal for the overlay
1242
2859
  // (first activity registers + auto-reveals + focuses the channel tab) before the
1243
2860
  // tool applies. Record the channel as USED so the overlay tabs it (channels other
1244
2861
  // than the whiteboard are tab-less until they're first used).
1245
- this.usedChannelNames.add(plugin.ChannelName);
1246
- this._channelActivity$.next(plugin);
2862
+ this.noteChannelActivity(plugin);
2863
+ // A native tool whose result would show the agent what exposure policy withholds is refused here, as it is on the
2864
+ // ContextTool route; the legacy tool-result shape carries the reason.
2865
+ const withheld = plugin.FindVerbForNativeTool(toolName);
2866
+ const refusal = withheld ? plugin.RefuseVerbForExposure(withheld) : null;
2867
+ if (refusal) {
2868
+ return JSON.stringify({ success: false, errorCode: 'exposure_restricted', error: refusal });
2869
+ }
1247
2870
  return plugin.ApplyAgentTool(toolName, argsJson);
1248
2871
  });
1249
2872
  }
2873
+ /** Records that the agent acted on a channel (the overlay's first-use reveal signal). */
2874
+ noteChannelActivity(plugin) {
2875
+ this.usedChannelNames.add(plugin.ChannelName);
2876
+ this._channelActivity$.next(plugin);
2877
+ }
2878
+ /**
2879
+ * Mounts an advertised `on-demand` channel when the agent opens it: initializes it, restores its
2880
+ * prior-session state, publishes it on {@link ActiveChannels$} (so the overlay can give it a tab)
2881
+ * and, if the call is already connected, tells it so. If initialization throws the channel goes
2882
+ * back to being advertised, so the agent can retry or move on.
2883
+ */
2884
+ async mountAdvertisedChannel(plugin) {
2885
+ const index = this.advertisedChannels.findIndex((p) => p.Plugin === plugin);
2886
+ if (index < 0) {
2887
+ return; // already mounted (or never advertised) — the dispatcher opens it regardless
2888
+ }
2889
+ const [prepared] = this.advertisedChannels.splice(index, 1);
2890
+ try {
2891
+ this.mountChannel(prepared, this.GetResolvedChannel(prepared.Key) ?? undefined);
2892
+ }
2893
+ catch (error) {
2894
+ this.advertisedChannels.splice(index, 0, prepared);
2895
+ throw error;
2896
+ }
2897
+ this.restorePriorChannelState(plugin);
2898
+ this._activeChannels$.next([...this._activeChannels$.value, plugin]);
2899
+ if (this.isSessionLive()) {
2900
+ this.notifyChannelSessionStarted(plugin);
2901
+ }
2902
+ }
2903
+ /** Tells a channel the session is connected, containing anything it throws. */
2904
+ notifyChannelSessionStarted(channel) {
2905
+ try {
2906
+ channel.OnSessionStarted?.();
2907
+ }
2908
+ catch (err) {
2909
+ console.error(`[RealtimeSession] Error in channel '${channel.ChannelName}' OnSessionStarted:`, err);
2910
+ }
2911
+ }
2912
+ /** Finds a session channel (open or merely advertised) for the dispatcher. */
2913
+ findDispatchableChannel(key) {
2914
+ const id = NormalizeChannelKey(key);
2915
+ const open = this._activeChannels$.value.find((c) => NormalizeChannelKey(c.ChannelName) === id);
2916
+ if (open) {
2917
+ return { Plugin: open, IsOpen: true };
2918
+ }
2919
+ const advertised = this.advertisedChannels.find((p) => NormalizeChannelKey(p.Key) === id);
2920
+ return advertised ? { Plugin: advertised.Plugin, IsOpen: false } : null;
2921
+ }
2922
+ /** The keys the agent can address, for "unknown channel" messages — channels with nothing to address are left out. */
2923
+ listAddressableChannelKeys() {
2924
+ const addressable = (plugin) => {
2925
+ const descriptor = plugin.GetDescriptor();
2926
+ return descriptor.Verbs.length > 0 || descriptor.Inputs !== undefined;
2927
+ };
2928
+ return [
2929
+ ...this._activeChannels$.value.filter(addressable).map((c) => c.ChannelName),
2930
+ ...this.advertisedChannels.filter((p) => addressable(p.Plugin)).map((p) => p.Key),
2931
+ ];
2932
+ }
2933
+ /**
2934
+ * Runs a channel-addressed `ContextTool` call (the {@link RealtimeChannelContext.DispatchContextAction}
2935
+ * implementation): validates it, opens an `on-demand` channel when asked, runs the verb, and on
2936
+ * success records the channel as used so the overlay reveals its tab. Never throws.
2937
+ */
2938
+ async dispatchContextAction(request) {
2939
+ const result = await this.channelDispatcher.Dispatch(request);
2940
+ if (result.Success) {
2941
+ const channel = this.findDispatchableChannel(request.Target.Channel);
2942
+ if (channel) {
2943
+ this.noteChannelActivity(channel.Plugin);
2944
+ }
2945
+ }
2946
+ return result;
2947
+ }
2948
+ /**
2949
+ * Opens (and seeds) a channel from the HOST — the same path the agent's `open` action takes: the channel
2950
+ * is mounted if it was only advertised, its `Inputs` schema validates `inputs`, and it announces itself to
2951
+ * the model. A channel that is already open is re-seeded. Never throws; a refusal is a structured result.
2952
+ *
2953
+ * @param channelKey The channel to open (case-insensitive).
2954
+ * @param inputs Seed inputs, validated against the channel's descriptor.
2955
+ */
2956
+ OpenChannel(channelKey, inputs = {}) {
2957
+ return this.dispatchContextAction({ Target: { Channel: channelKey }, Action: 'open', Params: inputs });
2958
+ }
2959
+ /**
2960
+ * Tells the model which channels exist and how to use them — rendered from the channels' own
2961
+ * descriptors, so a new channel needs no prompt change. Sent ONCE, the first moment the control
2962
+ * channel is usable — a context note sent before then is dropped, and `Connect` can resolve before the
2963
+ * provider reports `listening`, so this runs at start and again on each state change until it has gone.
2964
+ * A session whose only channels describe themselves through native tools sends nothing.
2965
+ */
2966
+ flushChannelCatalogNote() {
2967
+ if (!this.catalogNotePending || !this.isSessionLive()) {
2968
+ return;
2969
+ }
2970
+ this.catalogNotePending = false;
2971
+ const entries = [
2972
+ ...this._activeChannels$.value.map((plugin) => this.catalogEntry(plugin, true)),
2973
+ ...this.advertisedChannels.map((p) => this.catalogEntry(p.Plugin, false)),
2974
+ ];
2975
+ const note = BuildChannelCatalogNote(entries);
2976
+ if (note) {
2977
+ this.SendContextNote(note);
2978
+ }
2979
+ }
2980
+ /** One catalog entry: a channel's descriptor, whether it is open, and whether its tools were declared natively. */
2981
+ catalogEntry(plugin, isOpen) {
2982
+ const display = this.GetResolvedChannel(plugin.ChannelName)?.DisplayPolicy;
2983
+ const descriptor = plugin.GetDescriptor();
2984
+ const entry = {
2985
+ Descriptor: descriptor,
2986
+ IsOpen: isOpen,
2987
+ HasNativeTools: isOpen && display !== undefined && DeclaresNativeTools(display) && plugin.GetToolDefinitions().length > 0,
2988
+ };
2989
+ if (CompareExposure(plugin.Exposure, descriptor.MaxExposure) < 0) {
2990
+ entry.ExposureLimit = { Effective: plugin.Exposure, Ceiling: descriptor.MaxExposure, Reasons: [...plugin.ExposureReasons] };
2991
+ }
2992
+ return entry;
2993
+ }
1250
2994
  /** Builds the host-services context one channel plugin sees (its only line to the session). */
1251
- buildChannelContext(plugin) {
2995
+ buildChannelContext(plugin, channelConfig) {
1252
2996
  // Capture the service in a local so the AgentSessionID getter reads the SERVICE's live
1253
2997
  // field (not the object literal's `this`) every time it's accessed.
1254
2998
  const service = this;
@@ -1275,55 +3019,107 @@ export class RealtimeSessionRuntime {
1275
3019
  // (Explorer) feeds both; absent on hosts that supply no app context / register no client tools.
1276
3020
  AppContext$: this.AppContext$,
1277
3021
  ExecuteClientTool: (name, params) => this.executeAppClientTool(name, params),
3022
+ // The session's server events (identity verification, app events), for a channel that reacts to them.
3023
+ SessionEvents$: this.SessionEvents$,
3024
+ // Channel-addressed ContextTool calls: validated against the verb's schema, open `on-demand` channels.
3025
+ DispatchContextAction: (request) => this.dispatchContextAction(request),
3026
+ // This channel's resolved configuration (host defaults beneath agent/app config).
3027
+ ChannelConfig: channelConfig ?? {},
1278
3028
  get Client() {
1279
3029
  return service.client;
1280
3030
  },
1281
- SendVideoFrame: (base64Image, mimeType) => this.SendVideoFrame(base64Image, mimeType),
1282
- IsTrackEstablished: (modality, direction) => this.IsTrackEstablished(modality, direction)
3031
+ // Through the arbiter, as this channel's source (deprecated: channels use a ChannelInboundVideoBridge).
3032
+ SendVideoFrame: (base64Image, mimeType) => this.sendChannelVideoFrame(plugin, base64Image, mimeType),
3033
+ IsTrackEstablished: (modality, direction) => this.IsTrackEstablished(modality, direction),
3034
+ // The camera and screen share, for a channel that fronts one: their state, and the user's clicks on its surface.
3035
+ Captures$: this.Captures$,
3036
+ StartCapture: (kind) => (kind === 'camera' ? this.StartCamera() : this.StartScreenShare()),
3037
+ StopCapture: (kind) => (kind === 'camera' ? this.StopCamera() : this.StopScreenShare()),
3038
+ // The video sources and which of them the model is sent, so a capture's channel says the agent sees it only then.
3039
+ VideoSources$: this.VideoSources$,
3040
+ // The agent's video, the call's state and its resumes, for a channel that shows the agent.
3041
+ AgentVideo$: this.AgentVideo$,
3042
+ ConnectionState$: this.ConnectionState$,
3043
+ Resuming$: this.Resuming$
1283
3044
  };
1284
3045
  }
1285
3046
  /**
1286
- * Replaces the set of host-registered surface client tools the realtime ContextTool can execute.
1287
- * The host calls this at session start and whenever the active surface's tool set changes (the
1288
- * continuous-capability half of client-context delivery). Passing `[]` clears them.
3047
+ * Registers the tools ONE source of the host can run for the realtime `ContextTool`, replacing only
3048
+ * that source's previous set — other owners' tools are untouched. A host with two sources (Explorer's
3049
+ * always-available globals and the active surface's tools) registers each under its own owner key, so
3050
+ * refreshing the surface's tools can neither drop the globals nor leave the previous surface's tools
3051
+ * behind. Passing `[]` clears just that owner. When two owners register the same tool name, the
3052
+ * later-registered owner wins.
1289
3053
  *
1290
- * @param tools The current surface client tools (name + handler). Descriptions/schemas ride the
1291
- * app-context manifest separately; only the executable handler is needed here.
3054
+ * Calling it with one argument (the original form) registers under {@link DEFAULT_APP_TOOL_OWNER}, so
3055
+ * a host with a single source behaves exactly as before.
3056
+ *
3057
+ * @param tools The owner's complete current set (name + handler; description/schema optional).
3058
+ * @param owner A stable key naming the source (e.g. `'explorer.surface'`). Defaults to a shared owner.
1292
3059
  */
1293
- RegisterAppClientTools(tools) {
1294
- this.appClientToolHandlers.clear();
1295
- for (const tool of tools) {
1296
- if (tool?.Name && typeof tool.Handler === 'function') {
1297
- this.appClientToolHandlers.set(tool.Name.trim().toLowerCase(), tool.Handler);
1298
- }
1299
- }
3060
+ RegisterAppClientTools(tools, owner = DEFAULT_APP_TOOL_OWNER) {
3061
+ this.appToolRegistry.Register(owner, tools);
3062
+ }
3063
+ /** Removes one owner's tools (and its place in the collision order). No-op when it has none. */
3064
+ UnregisterAppClientTools(owner) {
3065
+ this.appToolRegistry.Unregister(owner);
3066
+ }
3067
+ /** Removes every owner's tools. */
3068
+ ClearAppClientTools() {
3069
+ this.appToolRegistry.Clear();
3070
+ }
3071
+ /**
3072
+ * The client tools in effect right now, resolved through the same unified resolver the server uses
3073
+ * (`override → session → app → static`, first match wins): the host's registered tools (described
3074
+ * by the manifest the host streams when it did not describe them itself) first, then the app and
3075
+ * agent tiers the server resolved at mint. Only the host-registered tools can RUN here; the
3076
+ * other tiers describe what the app declares.
3077
+ */
3078
+ resolveAppClientTools() {
3079
+ const manifest = this._appContext$.value?.Capabilities?.Tools ?? [];
3080
+ return ResolveClientTools({
3081
+ agentId: '',
3082
+ sessionTools: MergeToolMetadata(this.appToolRegistry.ToMetadata(), manifest),
3083
+ appTools: this.sessionClientTools?.App,
3084
+ staticTools: this.sessionClientTools?.Static,
3085
+ });
1300
3086
  }
1301
3087
  /**
1302
3088
  * Executes a host-registered surface client tool by name (the {@link RealtimeChannelContext.ExecuteClientTool}
1303
3089
  * implementation). Tolerant: an unknown tool or a thrown handler resolves to a structured
1304
- * `Success: false` result the channel narrates — never throws.
3090
+ * `Success: false` result the channel narrates — never throws. A tool the app declares but the host has
3091
+ * not registered a handler for says so, instead of reading like a typo. A handler that returns an object with
3092
+ * `Success: false` also gives `Success: false`, with the handler's `ErrorMessage` and its object as `Result`.
1305
3093
  *
1306
3094
  * @param name The tool name (the model's `action`).
1307
3095
  * @param params The tool parameters.
1308
3096
  * @returns A structured result for the channel to serialize back to the model.
1309
3097
  */
1310
3098
  async executeAppClientTool(name, params) {
1311
- const handler = this.appClientToolHandlers.get((name ?? '').trim().toLowerCase());
3099
+ const handler = this.appToolRegistry.Find(name)?.Handler;
1312
3100
  if (!handler) {
1313
- const available = Array.from(this.appClientToolHandlers.keys()).join(', ');
1314
- return {
1315
- Success: false,
1316
- ErrorMessage: `No client tool named "${name}" is available on this surface. Available: ${available || '(none)'}.`
1317
- };
3101
+ return { Success: false, ErrorMessage: this.describeMissingClientTool(name) };
1318
3102
  }
1319
3103
  try {
1320
3104
  const result = await handler(params ?? {});
1321
- return { Success: true, Result: result };
3105
+ return reportsFailure(result)
3106
+ ? { Success: false, ErrorMessage: failureMessage(result), Result: result }
3107
+ : { Success: true, Result: result };
1322
3108
  }
1323
3109
  catch (error) {
1324
3110
  return { Success: false, ErrorMessage: error instanceof Error ? error.message : String(error) };
1325
3111
  }
1326
3112
  }
3113
+ /** The model-readable reason a client tool could not run. */
3114
+ describeMissingClientTool(name) {
3115
+ const available = this.appToolRegistry.Names().join(', ');
3116
+ const wanted = (name ?? '').trim().toLowerCase();
3117
+ const declared = this.resolveAppClientTools().find((t) => t.Name.trim().toLowerCase() === wanted);
3118
+ if (declared) {
3119
+ return `The client tool "${name}" is declared for this app but this surface has not registered it, so it cannot run right now. Available: ${available || '(none)'}.`;
3120
+ }
3121
+ return `No client tool named "${name}" is available on this surface. Available: ${available || '(none)'}.`;
3122
+ }
1327
3123
  /**
1328
3124
  * Runs a channel-specific GraphQL operation through the live session's provider (the
1329
3125
  * {@link RealtimeChannelContext.ExecuteServerAction} implementation). Best-effort: any
@@ -1360,35 +3156,39 @@ export class RealtimeSessionRuntime {
1360
3156
  * never affected.
1361
3157
  */
1362
3158
  applyPriorChannelStates(statesJson) {
3159
+ this.priorChannelStates = {};
1363
3160
  if (!statesJson) {
1364
3161
  return;
1365
3162
  }
1366
- let states;
1367
3163
  try {
1368
3164
  const parsed = JSON.parse(statesJson);
1369
3165
  if (parsed === null || typeof parsed !== 'object') {
1370
3166
  return;
1371
3167
  }
1372
- states = parsed;
3168
+ this.priorChannelStates = parsed;
1373
3169
  }
1374
3170
  catch {
1375
3171
  console.warn('[RealtimeSession] PriorChannelStatesJson was malformed — starting channels fresh');
1376
3172
  return;
1377
3173
  }
1378
3174
  for (const plugin of this._activeChannels$.value) {
1379
- const state = states[plugin.ChannelName];
1380
- if (typeof state === 'string' && state.length > 0) {
1381
- try {
1382
- const restored = plugin.RestoreState(state);
1383
- if (!restored) {
1384
- console.warn(`[RealtimeSession] Channel '${plugin.ChannelName}' declined its prior-session state — starting fresh`);
1385
- }
1386
- }
1387
- catch (error) {
1388
- console.warn(`[RealtimeSession] Channel '${plugin.ChannelName}' restore threw — starting fresh`, error);
1389
- }
3175
+ this.restorePriorChannelState(plugin);
3176
+ }
3177
+ }
3178
+ /** Offers one plugin its prior-session state, if there is one. Rejections and throws are logged, never fatal. */
3179
+ restorePriorChannelState(plugin) {
3180
+ const state = this.priorChannelStates[plugin.ChannelName];
3181
+ if (typeof state !== 'string' || state.length === 0) {
3182
+ return;
3183
+ }
3184
+ try {
3185
+ if (!plugin.RestoreState(state)) {
3186
+ console.warn(`[RealtimeSession] Channel '${plugin.ChannelName}' declined its prior-session state — starting fresh`);
1390
3187
  }
1391
3188
  }
3189
+ catch (error) {
3190
+ console.warn(`[RealtimeSession] Channel '${plugin.ChannelName}' restore threw — starting fresh`, error);
3191
+ }
1392
3192
  }
1393
3193
  /**
1394
3194
  * Persists a channel's state as a first-class versioned artifact (`MJ: Artifacts`) via the
@@ -1478,6 +3278,13 @@ export class RealtimeSessionRuntime {
1478
3278
  this._activeChannels$.next([]);
1479
3279
  }
1480
3280
  this.usedChannelNames.clear();
3281
+ // Channels that were prepared or advertised but never mounted were never initialized, so there is
3282
+ // nothing to dispose — only the bookkeeping to forget.
3283
+ this.discardUnmountedChannels();
3284
+ this.catalogNotePending = false;
3285
+ this.resolvedChannels.clear();
3286
+ this.priorChannelStates = {};
3287
+ this.sessionClientTools = null;
1481
3288
  }
1482
3289
  // ── Realtime client resolution + wiring ────────────────────────────────────
1483
3290
  /**
@@ -1500,12 +3307,14 @@ export class RealtimeSessionRuntime {
1500
3307
  }
1501
3308
  /**
1502
3309
  * Builds the client-direct session config the realtime client connects with.
1503
- * Aggregates tracks sourced by active channels under {@link REQUESTED_TRACKS_SESSION_KEY} so the driver
1504
- * can negotiate them (e.g., establishing inbound video streaming for Whiteboard / RemoteBrowser).
3310
+ * Aggregates the tracks active channels source and sink under {@link REQUESTED_TRACKS_SESSION_KEY} so the
3311
+ * driver can negotiate them (e.g., inbound video for Whiteboard / RemoteBrowser, outbound video for a
3312
+ * channel that shows the agent's video). A track the model does not support resolves to `'unsupported'`.
3313
+ * The session's transport and relay URL go to the driver as minted (an unknown transport is left out: direct).
1505
3314
  */
1506
3315
  BuildClientConfig(session) {
1507
3316
  const sessionConfig = this.parseSessionConfig(session.SessionConfigJson);
1508
- const channelTracks = this._activeChannels$.value.flatMap((c) => c.GetSourcedTracks());
3317
+ const channelTracks = this._activeChannels$.value.flatMap((c) => [...c.GetSourcedTracks(), ...c.GetSunkTracks()]);
1509
3318
  if (channelTracks.length > 0) {
1510
3319
  // The requested tracks cross a JSON boundary — the driver reads them back out of the session
1511
3320
  // config bag (`GeminiRealtimeClient.parseSessionConfig`). A `RealtimeTrackDescriptor` is NOT
@@ -1530,11 +3339,14 @@ export class RealtimeSessionRuntime {
1530
3339
  }
1531
3340
  sessionConfig[REQUESTED_TRACKS_SESSION_KEY] = Array.from(trackMap.values());
1532
3341
  }
3342
+ const transport = ParseRealtimeClientTransport(session.Transport);
1533
3343
  return {
1534
3344
  Provider: session.Provider,
1535
3345
  Model: session.Model,
1536
3346
  EphemeralToken: session.EphemeralToken,
1537
3347
  ExpiresAt: session.ExpiresAt,
3348
+ ...(transport ? { Transport: transport } : {}),
3349
+ ...(session.RelayUrl ? { RelayUrl: session.RelayUrl } : {}),
1538
3350
  SessionConfig: sessionConfig
1539
3351
  };
1540
3352
  }
@@ -1570,6 +3382,9 @@ export class RealtimeSessionRuntime {
1570
3382
  });
1571
3383
  client.OnError((error) => {
1572
3384
  console.error('[RealtimeSession] Provider error event:', JSON.stringify(error), error);
3385
+ if (error.Fatal && this.client === client) {
3386
+ this.lastSessionError = error;
3387
+ }
1573
3388
  });
1574
3389
  // Usage telemetry: accumulate the driver's per-response token DELTAS and relay them to
1575
3390
  // the server (onto the co-agent AIPromptRun) debounced + once at teardown. Providers
@@ -1586,12 +3401,41 @@ export class RealtimeSessionRuntime {
1586
3401
  client.OnInterruption(() => {
1587
3402
  this.cancelPendingNarration();
1588
3403
  });
3404
+ // The provider VAD saw the user start speaking. Final-only user-transcript drivers (OpenAI,
3405
+ // HuggingFace, xAI) emit no user interim, so this is their only real user-turn start. It goes
3406
+ // to its own slot, not the shared turn slot: the user transcript arrives asynchronously, so the
3407
+ // assistant's reply may already be streaming and must keep its own start.
3408
+ client.OnUserSpeechStart(() => this.markUserSpeechStart());
3409
+ client.OnRemoteVideo((video) => this.onAgentVideo(client, video));
3410
+ }
3411
+ /**
3412
+ * The model's video arrived: publish it on {@link AgentVideo$}, and mark each channel that shows it (one that sinks
3413
+ * outbound video) as used, so the host shows its surface. A client the session has already let go of is ignored, so a
3414
+ * late frame cannot bring the video back after the call.
3415
+ */
3416
+ onAgentVideo(client, video) {
3417
+ if (this.client !== client) {
3418
+ return;
3419
+ }
3420
+ this._agentVideo$.next(video);
3421
+ for (const channel of this._activeChannels$.value) {
3422
+ if (channel.ShowsAgentVideo) {
3423
+ this.noteChannelActivity(channel);
3424
+ }
3425
+ }
3426
+ }
3427
+ /** Clears {@link AgentVideo$} when the call's client goes. */
3428
+ clearAgentVideo() {
3429
+ if (this._agentVideo$.value !== null) {
3430
+ this._agentVideo$.next(null);
3431
+ }
1589
3432
  }
1590
3433
  /** Maps a client state event onto the UI connection state. */
1591
3434
  onClientStateChange(state) {
1592
3435
  const mapped = this.mapClientState(state);
1593
3436
  if (mapped) {
1594
- this._connectionState$.next(mapped);
3437
+ this.setConnectionState(mapped);
3438
+ this.flushChannelCatalogNote();
1595
3439
  }
1596
3440
  }
1597
3441
  /**
@@ -1620,6 +3464,17 @@ export class RealtimeSessionRuntime {
1620
3464
  const state = this._connectionState$.value;
1621
3465
  return state === 'listening' || state === 'speaking' || state === 'thinking';
1622
3466
  }
3467
+ /**
3468
+ * Moves {@link ConnectionState$} to `state`, and {@link Resuming$} first: a `'connecting'` while the call is live, or
3469
+ * already resuming, is a resume; any other state ends one. Every state change goes through here.
3470
+ */
3471
+ setConnectionState(state) {
3472
+ const resuming = state === 'connecting' && (this._resuming$.value || this.isSessionLive());
3473
+ if (resuming !== this._resuming$.value) {
3474
+ this._resuming$.next(resuming);
3475
+ }
3476
+ this._connectionState$.next(state);
3477
+ }
1623
3478
  // ── Transcript policy ──────────────────────────────────────────────────────
1624
3479
  /**
1625
3480
  * Applies transcript policy to client transcript events. Interim deltas don't become
@@ -1653,6 +3508,11 @@ export class RealtimeSessionRuntime {
1653
3508
  }
1654
3509
  return;
1655
3510
  }
3511
+ if (transcript.Role === 'User' && transcript.Text.trim().length === 0) {
3512
+ // An empty final user transcript (a VAD false positive, an empty transcription) never reaches
3513
+ // relayTranscript, so release its onset here: it must not become the next real turn's start.
3514
+ this.releaseUserSpeechStart();
3515
+ }
1656
3516
  if (transcript.Role === 'Assistant') {
1657
3517
  this.hasActiveInterimUserCaption = false;
1658
3518
  this.pendingUserCaption = '';
@@ -1727,11 +3587,10 @@ export class RealtimeSessionRuntime {
1727
3587
  * - NORMAL turns only — NARRATION interims are ephemeral and never persisted, so they must not
1728
3588
  * claim the next real turn's start slot.
1729
3589
  *
1730
- * Works for any role whose driver surfaces interim deltas (all drivers for the assistant; the
1731
- * relevant case here — the post-tool-gap assistant answer — and user-interim drivers like
1732
- * Gemini/AssemblyAI). For final-only user turns (OpenAI/xAI/ElevenLabs) no interim arrives, so
1733
- * {@link relayTranscript} falls back to the seeded/prior start — the gap case that drifts is the
1734
- * assistant answer, which always has interims.
3590
+ * Stamped on a turn's first interim transcript (all drivers for the assistant — the post-tool-gap
3591
+ * assistant answer — and user-interim drivers like Gemini/AssemblyAI). Final-only user turns
3592
+ * (OpenAI-protocol drivers) take their start from {@link markUserSpeechStart}'s own slot
3593
+ * instead; one with no speech-start signal either (ElevenLabs) relays `null` as-is.
1735
3594
  */
1736
3595
  markTurnAudioStart(kind) {
1737
3596
  if (this.turnAudioStartCaptured || kind === 'narration') {
@@ -1744,15 +3603,39 @@ export class RealtimeSessionRuntime {
1744
3603
  this.currentTurnStartMs = offset;
1745
3604
  this.turnAudioStartCaptured = true;
1746
3605
  }
3606
+ /**
3607
+ * Stamps {@link userSpeechStartMs} when the client signals that the user began to speak. Keeps
3608
+ * the FIRST onset since the last final user transcript: the VAD fires again after a
3609
+ * mid-sentence pause, and that second onset is not where the utterance began. An onset that an
3610
+ * assistant turn which began after it has since superseded ({@link userSpeechStartSuperseded}) is
3611
+ * replaced instead.
3612
+ */
3613
+ markUserSpeechStart() {
3614
+ if (this.userSpeechStartMs !== null && !this.userSpeechStartSuperseded) {
3615
+ return;
3616
+ }
3617
+ const offset = this.nowTurnOffsetMs();
3618
+ if (offset === null) {
3619
+ return;
3620
+ }
3621
+ this.userSpeechStartMs = offset;
3622
+ this.userSpeechStartSuperseded = false;
3623
+ }
3624
+ /** Ends the held user onset: a final user transcript (empty or not) closed the utterance it began. */
3625
+ releaseUserSpeechStart() {
3626
+ this.userSpeechStartMs = null;
3627
+ this.userSpeechStartSuperseded = false;
3628
+ }
1747
3629
  /**
1748
3630
  * The current per-turn offset in ms — the RECORDER's clock when one runs (an offset into a
1749
3631
  * seekable file), else the SESSION clock (#3832: orderable and displayable, not seekable),
1750
3632
  * else `null` before any call is live. One function so the two stamp sites cannot disagree
1751
- * about which clock a session is on.
3633
+ * about which clock a session is on. Always whole milliseconds: the offsets travel as GraphQL
3634
+ * `Int`, which refuses a fraction, and a 48 kHz sample clock rarely lands on a whole millisecond.
1752
3635
  */
1753
3636
  nowTurnOffsetMs() {
1754
3637
  if (this.recorder) {
1755
- return this.recorder.NowOffsetMs();
3638
+ return Math.round(this.recorder.NowOffsetMs());
1756
3639
  }
1757
3640
  if (this.sessionClockStartMs !== null) {
1758
3641
  return Math.max(0, Math.round(performance.now() - this.sessionClockStartMs));
@@ -1776,8 +3659,13 @@ export class RealtimeSessionRuntime {
1776
3659
  }
1777
3660
  this.appendCaption({ Role: role, Text: text });
1778
3661
  }
1779
- /** Finalizes the user turn: push a caption + relay the final transcript. */
1780
- async onUserTranscript(transcript) {
3662
+ /**
3663
+ * Finalizes the user turn: push a caption + relay the final transcript.
3664
+ *
3665
+ * @param source `'typed'` for a {@link SendText} turn: it has no speech onset, so it neither takes
3666
+ * nor clears the held one (a spoken transcript may still be pending for it).
3667
+ */
3668
+ async onUserTranscript(transcript, source = 'speech') {
1781
3669
  if (transcript.trim().length === 0) {
1782
3670
  return;
1783
3671
  }
@@ -1786,7 +3674,7 @@ export class RealtimeSessionRuntime {
1786
3674
  this.firstUserTranscript = transcript;
1787
3675
  }
1788
3676
  this.appendCaption({ Role: 'User', Text: transcript });
1789
- await this.relayTranscript('user', transcript);
3677
+ await this.relayTranscript('user', transcript, false, source);
1790
3678
  }
1791
3679
  // ── Tool calling ───────────────────────────────────────────────────────────
1792
3680
  /**
@@ -1809,7 +3697,7 @@ export class RealtimeSessionRuntime {
1809
3697
  void this.relayToolTurn(call.ToolName, call.ArgumentsJson, resultJson);
1810
3698
  return;
1811
3699
  }
1812
- this._connectionState$.next('thinking');
3700
+ this.setConnectionState('thinking');
1813
3701
  if (this.inFlightCallIds.size === 0) {
1814
3702
  // A fresh delegation burst: anchor the first-update delay and clear the digest
1815
3703
  // buffer. Deliberately NOT reset: lastDelegationNarrationAt (the 8s spacing floor
@@ -1906,6 +3794,40 @@ export class RealtimeSessionRuntime {
1906
3794
  RunID: parsed.RunID,
1907
3795
  Artifacts: parsed.Artifacts
1908
3796
  });
3797
+ if (parsed.Success && parsed.Artifacts && parsed.Artifacts.length > 0) {
3798
+ void this.offerDelegationArtifacts(parsed.Artifacts);
3799
+ }
3800
+ }
3801
+ /**
3802
+ * Offers a delegated run's artifacts to the channels that host artifacts (an Interactive Component channel
3803
+ * shows a component artifact, or swaps in a newer version of one it already shows). Each channel is asked
3804
+ * first whether it wants them; an advertised, unopened channel that does is mounted before it is handed
3805
+ * them. One channel failing never stops the others, and nothing here can fail the delegation result,
3806
+ * which has already been emitted.
3807
+ */
3808
+ async offerDelegationArtifacts(artifacts) {
3809
+ const candidates = [
3810
+ ...this._activeChannels$.value.map((plugin) => ({ Plugin: plugin, IsOpen: true })),
3811
+ ...this.advertisedChannels.map((prepared) => ({ Plugin: prepared.Plugin, IsOpen: false }))
3812
+ ];
3813
+ for (const { Plugin: plugin, IsOpen: isOpen } of candidates) {
3814
+ try {
3815
+ const config = this.GetResolvedChannel(plugin.ChannelName)?.Config ?? {};
3816
+ if (!plugin.AcceptsDelegationArtifacts(artifacts, config)) {
3817
+ continue;
3818
+ }
3819
+ if (!isOpen) {
3820
+ await this.mountAdvertisedChannel(plugin);
3821
+ }
3822
+ await plugin.OnDelegationArtifacts(artifacts);
3823
+ if (!isOpen) {
3824
+ this.noteChannelActivity(plugin); // the channel was mounted for this: reveal its tab
3825
+ }
3826
+ }
3827
+ catch (error) {
3828
+ console.error(`[RealtimeSession] Channel '${plugin.ChannelName}' failed to take a delegated run's artifacts:`, error);
3829
+ }
3830
+ }
1909
3831
  }
1910
3832
  // ── Explicit delegation cancellation (server cancel channel) ───────────────
1911
3833
  /**
@@ -1996,44 +3918,34 @@ export class RealtimeSessionRuntime {
1996
3918
  }
1997
3919
  }
1998
3920
  // ── Session minting (GraphQL) ──────────────────────────────────────────────
1999
- /** Calls the `StartRealtimeClientSession` mutation to obtain an ephemeral token + config. */
2000
- async mintSession(targetAgentId, conversationId, lastSessionId, preferredModelId, clientTools, coAgentId, configOverridesJson, recordingConsent, recordingStartedAt, mediaCollectionId, applicationId, appContext) {
2001
- const mutation = `
2002
- mutation StartRealtimeClientSession($targetAgentId: String!, $conversationId: String, $lastSessionId: String, $preferredModelId: String, $clientToolsJson: String, $coAgentId: String, $configOverridesJson: String, $recordingConsent: Boolean, $recordingStartedAt: String, $mediaCollectionId: String, $applicationId: String, $appContextJson: String) {
2003
- StartRealtimeClientSession(targetAgentId: $targetAgentId, conversationId: $conversationId, lastSessionId: $lastSessionId, preferredModelId: $preferredModelId, clientToolsJson: $clientToolsJson, coAgentId: $coAgentId, configOverridesJson: $configOverridesJson, recordingConsent: $recordingConsent, recordingStartedAt: $recordingStartedAt, mediaCollectionId: $mediaCollectionId, applicationId: $applicationId, appContextJson: $appContextJson) {
2004
- AgentSessionId
2005
- ConversationId
2006
- Provider
2007
- Model
2008
- EphemeralToken
2009
- ExpiresAt
2010
- SessionConfigJson
2011
- ModelName
2012
- NarrationInstructionsTemplate
2013
- PriorChannelStatesJson
2014
- }
2015
- }
2016
- `;
2017
- const variables = {
2018
- targetAgentId,
2019
- conversationId: conversationId ?? null,
2020
- lastSessionId: lastSessionId ?? null,
2021
- preferredModelId: preferredModelId ?? null,
2022
- clientToolsJson: clientTools && clientTools.length > 0 ? JSON.stringify(clientTools) : null,
2023
- coAgentId: coAgentId ?? null,
2024
- configOverridesJson: configOverridesJson ?? null,
2025
- recordingConsent: recordingConsent ?? false,
2026
- recordingStartedAt: recordingStartedAt ?? null,
2027
- mediaCollectionId: mediaCollectionId ?? null,
2028
- applicationId: applicationId ?? null,
2029
- appContextJson: appContext ? JSON.stringify(appContext) : null
2030
- };
2031
- const result = await this.gql().ExecuteGQL(mutation, variables);
2032
- const payload = result?.StartRealtimeClientSession;
2033
- if (!payload?.EphemeralToken) {
2034
- throw new Error('StartRealtimeClientSession returned no ephemeral token');
2035
- }
2036
- return payload;
3921
+ /**
3922
+ * Mints a session through the installed {@link Launcher} — by default the stock
3923
+ * `StartRealtimeClientSession` mutation (see {@link DefaultRealtimeSessionLauncher}). `showsAgentVideo` is whether
3924
+ * one of the session's channels could show the agent's video; without one the server asks the model for no avatar.
3925
+ * `conversationBranchId` is the fork the session starts on; `null` is Main.
3926
+ */
3927
+ async mintSession(targetAgentId, conversationId, lastSessionId, preferredModelId, clientTools, coAgentId, configOverridesJson, recordingConsent, recordingStartedAt, mediaCollectionId, applicationId, appContext, channelCandidatesJson, showsAgentVideo, conversationBranchId) {
3928
+ const result = await this._launcher.Launch({
3929
+ TargetAgentId: targetAgentId,
3930
+ ConversationId: conversationId ?? null,
3931
+ LastSessionId: lastSessionId ?? null,
3932
+ PreferredModelId: preferredModelId ?? null,
3933
+ ClientTools: clientTools ?? [],
3934
+ CoAgentId: coAgentId ?? null,
3935
+ ConfigOverridesJson: configOverridesJson ?? null,
3936
+ RecordingConsent: recordingConsent ?? false,
3937
+ RecordingStartedAt: recordingStartedAt ?? null,
3938
+ MediaCollectionId: mediaCollectionId ?? null,
3939
+ ApplicationId: applicationId ?? null,
3940
+ AppContext: appContext ?? null,
3941
+ ChannelCandidatesJson: channelCandidatesJson ?? null,
3942
+ ConversationBranchId: conversationBranchId ?? null,
3943
+ ...(showsAgentVideo === undefined ? {} : { ShowsAgentVideo: showsAgentVideo })
3944
+ }, { Provider: this.Provider });
3945
+ if (!HasClientCredential(result)) {
3946
+ throw new Error('The session launcher returned no ephemeral token (or, for a relay session, no relay URL)');
3947
+ }
3948
+ return result;
2037
3949
  }
2038
3950
  /** Calls the `ExecuteRealtimeSessionTool` mutation; returns the ResultJson string. */
2039
3951
  async executeSessionTool(callId, toolName, argsJson) {
@@ -2090,37 +4002,71 @@ export class RealtimeSessionRuntime {
2090
4002
  * Relays a final transcript turn to MJ via `RelayRealtimeTranscript`.
2091
4003
  *
2092
4004
  * When the session is being recorded, per-turn timing rides along: `utteranceEndMs` is the
2093
- * recording-relative offset at finalization, and `utteranceStartMs` is the offset captured by
2094
- * {@link markTurnAudioStart} when THIS turn's audio actually began (its first interim) — NOT
2095
- * inherited from the previous turn's end. That distinction is the timing fix: when a tool-call
2096
- * / silence gap sits between turns, the post-gap turn's audio starts much later, so inheriting
2097
- * the prior turn's end stamped the cue ~gap-length too early. Both are omitted (left `null`)
2098
- * when the session isn't being recorded.
4005
+ * recording-relative offset at finalization, and `utteranceStartMs` is where THIS turn's audio
4006
+ * actually began — NOT inherited from the previous turn's end. For a user turn whose onset the
4007
+ * client signalled, that is {@link userSpeechStartMs}; otherwise it is the offset captured by
4008
+ * {@link markTurnAudioStart} on the turn's first interim. That distinction is the timing fix:
4009
+ * when a tool-call / silence gap sits between turns, the post-gap turn's audio starts much
4010
+ * later, so inheriting the prior turn's end stamped the cue ~gap-length too early. Both are
4011
+ * omitted (left `null`) when the session isn't being recorded. `utteranceStartMs` is also
4012
+ * `null` when no start was measured for this turn; it is never replaced with a placeholder 0.
2099
4013
  *
2100
4014
  * A correction (`replacesPrevious`) doesn't open a new turn, so it carries no start and doesn't
2101
4015
  * reset the per-turn start guard. After a normal finalization the guard is cleared so the NEXT
2102
- * turn re-stamps its start from where ITS audio begins.
4016
+ * turn re-stamps its start from where ITS audio begins — except for a user turn that used its
4017
+ * own speech-start slot, which leaves the shared slot alone (it may hold the in-flight
4018
+ * assistant turn's start). Every final user transcript, corrections included, clears the
4019
+ * speech-start slot.
2103
4020
  *
2104
4021
  * @param replacesPrevious CORRECTION semantics: the server updates the session's most
2105
4022
  * recent persisted turn of this role IN PLACE instead of appending (e.g. ElevenLabs'
2106
4023
  * post-barge-in `agent_response_correction`).
4024
+ * @param source `'typed'` for a user turn sent with {@link SendText}: it does not read or clear
4025
+ * {@link userSpeechStartMs}, which belongs to the user's speech.
2107
4026
  */
2108
- async relayTranscript(role, text, replacesPrevious = false) {
4027
+ async relayTranscript(role, text, replacesPrevious = false, source = 'speech') {
2109
4028
  if (!this.agentSessionId) {
2110
4029
  return;
2111
4030
  }
4031
+ // Every final user transcript closes the signalled onset — a correction too, since it ends the
4032
+ // same utterance — so a later onset (an xAI mid-sentence resume) can't leak into the next turn.
4033
+ let userSpeechStartMs = null;
4034
+ if (role === 'user' && source === 'speech') {
4035
+ userSpeechStartMs = this.userSpeechStartMs;
4036
+ this.releaseUserSpeechStart();
4037
+ }
2112
4038
  // Per-turn timing against whichever clock the session is on (#3832): the recorder's when one
2113
- // runs, else the session clock. `utteranceStartMs` is where this turn's audio actually began
2114
- // (captured by markTurnAudioStart on the first interim); the `?? 0` fallback covers a turn
2115
- // whose interim was missed / a final-only first turn.
4039
+ // runs, else the session clock. `utteranceStartMs` is where this turn's audio actually began.
4040
+ // It is null when no start was measured for this turn (a final-only driver with no speech-start
4041
+ // signal), so the server stores NULL rather than a placeholder 0.
2116
4042
  const utteranceEndMs = this.nowTurnOffsetMs();
2117
- const utteranceStartMs = utteranceEndMs !== null && !replacesPrevious ? (this.currentTurnStartMs ?? 0) : null;
2118
- if (utteranceEndMs !== null && !replacesPrevious) {
4043
+ const opensTurn = utteranceEndMs !== null && !replacesPrevious;
4044
+ let utteranceStartMs = null;
4045
+ if (opensTurn && userSpeechStartMs !== null) {
4046
+ // A user turn with a signalled onset. Leave the shared slot alone: the user transcript
4047
+ // arrives asynchronously, so it may already hold the in-flight assistant reply's start.
4048
+ utteranceStartMs = userSpeechStartMs;
4049
+ if (!this.turnAudioStartCaptured) {
4050
+ // No turn has captured a start, so the slot holds only the recording-start seed (0). This
4051
+ // user turn was the first turn; drop the seed so a later unmeasured turn relays null, not 0.
4052
+ this.currentTurnStartMs = null;
4053
+ }
4054
+ }
4055
+ else if (opensTurn) {
4056
+ utteranceStartMs = this.currentTurnStartMs;
2119
4057
  // This turn is finalized — arm the NEXT turn to re-stamp its start from its own first
2120
4058
  // interim (handles a tool-call gap before the next turn). Stop inheriting this end as the
2121
- // next start. `null` means "not yet captured"; relay falls back to `?? 0` if no interim fires.
4059
+ // next start. `null` means "not yet captured" and is relayed as-is if nothing stamps it.
2122
4060
  this.currentTurnStartMs = null;
2123
4061
  this.turnAudioStartCaptured = false;
4062
+ if (role === 'assistant' && this.userSpeechStartMs !== null && (utteranceStartMs === null || this.userSpeechStartMs < utteranceStartMs)) {
4063
+ // The floor moved on past an onset that began BEFORE this assistant turn: a late user
4064
+ // transcript may still claim it, but the next onset replaces it (see userSpeechStartSuperseded).
4065
+ // An onset that began DURING this turn is a barge-in; it belongs to the user speech still
4066
+ // pending, so it stays the first onset. With no measured assistant start we cannot tell, and
4067
+ // treat the onset as earlier.
4068
+ this.userSpeechStartSuperseded = true;
4069
+ }
2124
4070
  }
2125
4071
  try {
2126
4072
  const mutation = `
@@ -2172,16 +4118,28 @@ export class RealtimeSessionRuntime {
2172
4118
  /**
2173
4119
  * Accumulates one usage DELTA from the realtime client (per-response token counts —
2174
4120
  * the `OnUsage` contract shape) and schedules the debounced relay. Negative / non-finite
2175
- * values are clamped to 0; an all-zero delta is dropped without arming the timer.
4121
+ * values are clamped to 0; an all-zero delta is dropped without arming the timer. The
4122
+ * update's input and output detail blocks add into the pending details by the record's
4123
+ * rules (amounts add up, inbound video running totals keep the larger value), so an update
4124
+ * that carries only avatar video seconds still counts.
2176
4125
  */
2177
4126
  onUsageDelta(usage) {
2178
4127
  const input = this.clampUsageDelta(usage.InputTokens);
2179
4128
  const output = this.clampUsageDelta(usage.OutputTokens);
2180
- if (input === 0 && output === 0) {
4129
+ const details = AddRealtimeUsageRecord(null, { Input: usage.InputTokenDetails, Output: usage.OutputTokenDetails });
4130
+ const hasDetails = HasRealtimeUsage(details);
4131
+ if (input === 0 && output === 0 && !hasDetails) {
2181
4132
  return;
2182
4133
  }
2183
4134
  this.pendingUsageInput += input;
2184
4135
  this.pendingUsageOutput += output;
4136
+ if (hasDetails) {
4137
+ this.pendingUsageDetails = AddRealtimeUsageRecord(this.pendingUsageDetails, details);
4138
+ }
4139
+ this.armUsageFlush();
4140
+ }
4141
+ /** Schedules the debounced usage relay unless one is already pending. */
4142
+ armUsageFlush() {
2185
4143
  if (!this.usageFlushTimer) {
2186
4144
  this.usageFlushTimer = setTimeout(() => {
2187
4145
  this.usageFlushTimer = null;
@@ -2206,24 +4164,26 @@ export class RealtimeSessionRuntime {
2206
4164
  const sessionId = agentSessionId ?? this.agentSessionId;
2207
4165
  const input = this.pendingUsageInput;
2208
4166
  const output = this.pendingUsageOutput;
2209
- if (!sessionId || (input === 0 && output === 0)) {
4167
+ const details = this.pendingUsageDetails;
4168
+ if (!sessionId || (input === 0 && output === 0 && !details)) {
2210
4169
  return;
2211
4170
  }
2212
4171
  this.pendingUsageInput = 0;
2213
4172
  this.pendingUsageOutput = 0;
4173
+ this.pendingUsageDetails = null;
2214
4174
  try {
2215
- const mutation = `
2216
- mutation RelayRealtimeUsage($agentSessionId: String!, $inputTokens: Int!, $outputTokens: Int!) {
2217
- RelayRealtimeUsage(agentSessionId: $agentSessionId, inputTokens: $inputTokens, outputTokens: $outputTokens)
2218
- }
2219
- `;
2220
- await this.gql().ExecuteGQL(mutation, { agentSessionId: sessionId, inputTokens: input, outputTokens: output });
4175
+ // The details argument is sent only when there are details, so a token-only relay is unchanged.
4176
+ const variables = { agentSessionId: sessionId, inputTokens: input, outputTokens: output, ...(details ? { usageDetailsJson: JSON.stringify(details) } : {}) };
4177
+ await this.gql().ExecuteGQL(RELAY_REALTIME_USAGE_MUTATION, variables);
2221
4178
  }
2222
4179
  catch (error) {
2223
4180
  console.error('[RealtimeSession] Failed to relay usage telemetry:', error);
2224
4181
  // Re-accumulate so a later debounce / the teardown flush retries the same deltas.
2225
4182
  this.pendingUsageInput += input;
2226
4183
  this.pendingUsageOutput += output;
4184
+ if (details) {
4185
+ this.pendingUsageDetails = AddRealtimeUsageRecord(this.pendingUsageDetails, details);
4186
+ }
2227
4187
  }
2228
4188
  }
2229
4189
  /** Cancels the pending debounced usage flush and zeroes the accumulators (teardown tail). */
@@ -2234,6 +4194,7 @@ export class RealtimeSessionRuntime {
2234
4194
  }
2235
4195
  this.pendingUsageInput = 0;
2236
4196
  this.pendingUsageOutput = 0;
4197
+ this.pendingUsageDetails = null;
2237
4198
  }
2238
4199
  // ── Delegated-run progress streaming ───────────────────────────────────────
2239
4200
  /**
@@ -2513,43 +4474,70 @@ export class RealtimeSessionRuntime {
2513
4474
  return;
2514
4475
  }
2515
4476
  this.teardownInFlight = this.runTeardown(closeServerSession).finally(() => {
4477
+ if (this._savingRecording$.value) {
4478
+ this._savingRecording$.next(false);
4479
+ }
2516
4480
  this.teardownInFlight = null;
2517
4481
  });
2518
4482
  await this.teardownInFlight;
2519
4483
  }
2520
4484
  /** The body of {@link teardown}; never called concurrently with itself. */
2521
4485
  async runTeardown(closeServerSession) {
4486
+ // Arm BEFORE any await so a tab closed in the first milliseconds of teardown is still guarded.
4487
+ if (this.recorder) {
4488
+ this._savingRecording$.next(true);
4489
+ }
2522
4490
  // First: stop asserting liveness. A pulse racing the close would re-stamp LastActiveAt on a
2523
4491
  // session we are deliberately ending, leaving an Idle row the janitor then has to age out.
2524
4492
  this.stopLivenessPulse();
2525
4493
  this.teardownDelegationProgress();
4494
+ this.stopSessionEvents();
4495
+ this.clientDeadline.Clear();
2526
4496
  // Channels first: flush any unsaved channel state WHILE the live session id is still
2527
4497
  // set (the captured per-save id covers the race anyway), then dispose the plugins.
2528
4498
  this.flushAllChannelSaves();
2529
4499
  this.disposeChannels();
4500
+ // End the recording and START its keepalive terminal request now: stop the recorder (which flushes
4501
+ // the encoder) before anything else that awaits. A tab closed right after End gets only the pagehide
4502
+ // terminal, which carries packets already encoded; releasing the mic and disconnecting the driver can
4503
+ // take long enough for the page to die first, and the audio still inside the encoder would be lost.
4504
+ // It also goes out before the usage flush and the close, so it is on the wire if the page dies during
4505
+ // either. Best-effort — finishRecording never throws. Null when nothing was recorded.
4506
+ const recordingTerminal = await this.finishRecording();
4507
+ this.recordingStartedAtIso = null;
4508
+ // The camera and screen share go first: the camera runs on the controller closed next.
4509
+ this.closeCaptures();
2530
4510
  // Defensive: stop the mic even when Connect never ran (the client also stops the
2531
4511
  // tracks it was handed — track.stop() is idempotent).
4512
+ this.closeLocalMedia();
2532
4513
  this.localStream?.getTracks().forEach(t => t.stop());
2533
4514
  this.localStream = null;
2534
- // Hand the platform back whatever acquiring the microphone changed. Stopping the tracks is not
4515
+ this.microphoneMuted = false;
4516
+ // Hand the platform back whatever opening the microphone changed. Stopping the tracks is not
2535
4517
  // the same thing: iOS, for instance, is put into a record-and-play audio category for the call,
2536
4518
  // and leaving it there changes the route and volume behaviour of every sound the app makes
2537
- // afterwards. Best-effort by contract — a failure here must never block ending a call.
2538
- try {
2539
- await this.mediaHost.ReleaseMicrophone?.();
2540
- }
2541
- catch (error) {
2542
- console.error('[RealtimeSession] Media host failed to release the microphone:', error);
4519
+ // afterwards. A microphone still opening is left to its start, which releases it once the
4520
+ // opening returns ({@link microphoneLease}).
4521
+ const lease = this.microphoneLease;
4522
+ if (lease && !lease.Opening) {
4523
+ this.microphoneLease = null;
4524
+ await this.releaseMicrophone();
2543
4525
  }
2544
4526
  if (this.client) {
2545
- await this.client.Disconnect();
2546
- this.client = null;
4527
+ // A stranded Active session is worse than a noisy socket: a driver that throws on Disconnect
4528
+ // (socket already gone) must not stop the server session from being closed below.
4529
+ try {
4530
+ await this.client.Disconnect();
4531
+ }
4532
+ catch (error) {
4533
+ console.error(`[RealtimeSession] Driver failed to disconnect (session ${this.agentSessionId}); continuing teardown:`, error);
4534
+ }
4535
+ finally {
4536
+ this.unwatchVideoSources();
4537
+ this.clearAgentVideo();
4538
+ this.client = null;
4539
+ }
2547
4540
  }
2548
- // Stop + upload the call recording WHILE the live session id is still set (the file is
2549
- // attached to it). Best-effort and never blocks teardown — stopAndUploadRecording swallows
2550
- // its own errors. No-op when nothing was recorded.
2551
- await this.stopAndUploadRecording(this.agentSessionId);
2552
- this.recordingStartedAtIso = null;
2553
4541
  // Final usage flush WHILE the live session id is still set (the relay mutation also
2554
4542
  // accepts a Closed session, so ordering vs. CloseAgentSession is belt-and-braces).
2555
4543
  if (this.usageFlushTimer) {
@@ -2561,18 +4549,28 @@ export class RealtimeSessionRuntime {
2561
4549
  if (closeServerSession && this.agentSessionId) {
2562
4550
  await this.closeServerSession(this.agentSessionId);
2563
4551
  }
4552
+ // Wait for the terminal's ack only AFTER the close: nothing after an aborted await ever runs, so
4553
+ // the small, critical close must never queue behind the recording (#5195). Bounded by
4554
+ // RECORDING_TERMINAL_ACK_TIMEOUT_MS; SavingRecording$ stays up until it settles.
4555
+ if (recordingTerminal) {
4556
+ await this.awaitRecordingTerminal(recordingTerminal);
4557
+ }
2564
4558
  // Capture the session id BEFORE we null it so the lifecycle emit carries it.
2565
4559
  // Skip emitting when there was no live session (defensive — teardown is safe
2566
4560
  // to call without an active session).
2567
4561
  const closedSessionId = this.agentSessionId;
2568
4562
  this.agentSessionId = null;
4563
+ this.sessionConversationId = null;
2569
4564
  this.narrationTemplate = null;
2570
4565
  this.clientToolHandlers.clear();
2571
4566
  this._modelName$.next(null);
4567
+ if (this._avatarNotice$.value !== null) {
4568
+ this._avatarNotice$.next(null);
4569
+ }
2572
4570
  this.SetMinimized(false);
2573
4571
  this._active$.next(false);
2574
4572
  if (this._connectionState$.value !== 'error') {
2575
- this._connectionState$.next('closed');
4573
+ this.setConnectionState('closed');
2576
4574
  }
2577
4575
  // Surface generic session-ended for the conversations runtime bridge.
2578
4576
  // `closeServerSession=true` means the user explicitly called EndRealtimeSession;
@@ -2584,6 +4582,18 @@ export class RealtimeSessionRuntime {
2584
4582
  });
2585
4583
  }
2586
4584
  }
4585
+ /**
4586
+ * Calls the host's {@link IRealtimeMediaHost.ReleaseMicrophone}. Best-effort by contract: a failure
4587
+ * is logged and never thrown, because it must never block ending a call.
4588
+ */
4589
+ async releaseMicrophone() {
4590
+ try {
4591
+ await this.mediaHost.ReleaseMicrophone?.();
4592
+ }
4593
+ catch (error) {
4594
+ console.error('[RealtimeSession] Media host failed to release the microphone:', error);
4595
+ }
4596
+ }
2587
4597
  /** Calls the `CloseAgentSession` mutation (provisioned in P4b). */
2588
4598
  async closeServerSession(agentSessionId) {
2589
4599
  try {
@@ -2609,13 +4619,18 @@ export class RealtimeSessionRuntime {
2609
4619
  this.pendingUserCaption = '';
2610
4620
  this.hasActiveInterimUserCaption = false;
2611
4621
  this.SetMinimized(false);
2612
- this.stopSegmentFlushing();
2613
- this.segmentIndex = 0;
4622
+ this.stopShardFlushing();
4623
+ // Defensive: teardown already ended any previous recording; a stray pipeline must not keep sending.
4624
+ this.recordingShards?.Queue.Close();
4625
+ this.recordingShards = null;
2614
4626
  this.recorder = null;
2615
4627
  this.recordingStartedAtIso = null;
2616
4628
  this.currentTurnStartMs = null;
2617
4629
  this.turnAudioStartCaptured = false;
4630
+ this.userSpeechStartMs = null;
4631
+ this.userSpeechStartSuperseded = false;
2618
4632
  this.usedChannelNames.clear();
4633
+ this.microphoneMuted = false;
2619
4634
  }
2620
4635
  /** The GraphQL provider used for relay mutations. */
2621
4636
  gql() {