@memberjunction/realtime-runtime 6.2.0-edge.3 → 6.2.0-edge.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +75 -0
- package/dist/channels/base-realtime-channel-client.d.ts +535 -28
- package/dist/channels/base-realtime-channel-client.d.ts.map +1 -1
- package/dist/channels/base-realtime-channel-client.js +864 -10
- package/dist/channels/base-realtime-channel-client.js.map +1 -1
- package/dist/channels/channel-action-dispatcher.d.ts +70 -0
- package/dist/channels/channel-action-dispatcher.d.ts.map +1 -0
- package/dist/channels/channel-action-dispatcher.js +136 -0
- package/dist/channels/channel-action-dispatcher.js.map +1 -0
- package/dist/channels/channel-catalog-note.d.ts +48 -0
- package/dist/channels/channel-catalog-note.d.ts.map +1 -0
- package/dist/channels/channel-catalog-note.js +119 -0
- package/dist/channels/channel-catalog-note.js.map +1 -0
- package/dist/channels/channel-contract-types.d.ts +107 -0
- package/dist/channels/channel-contract-types.d.ts.map +1 -0
- package/dist/channels/channel-contract-types.js +12 -0
- package/dist/channels/channel-contract-types.js.map +1 -0
- package/dist/channels/channel-descriptor-synthesis.d.ts +56 -0
- package/dist/channels/channel-descriptor-synthesis.d.ts.map +1 -0
- package/dist/channels/channel-descriptor-synthesis.js +90 -0
- package/dist/channels/channel-descriptor-synthesis.js.map +1 -0
- package/dist/channels/channel-perception.d.ts +82 -0
- package/dist/channels/channel-perception.d.ts.map +1 -0
- package/dist/channels/channel-perception.js +120 -0
- package/dist/channels/channel-perception.js.map +1 -0
- package/dist/channels/channel-schema-format.d.ts +17 -0
- package/dist/channels/channel-schema-format.d.ts.map +1 -0
- package/dist/channels/channel-schema-format.js +45 -0
- package/dist/channels/channel-schema-format.js.map +1 -0
- package/dist/channels/channel-state-delta.d.ts +59 -0
- package/dist/channels/channel-state-delta.d.ts.map +1 -0
- package/dist/channels/channel-state-delta.js +116 -0
- package/dist/channels/channel-state-delta.js.map +1 -0
- package/dist/channels/channel-surface-placement.d.ts +25 -0
- package/dist/channels/channel-surface-placement.d.ts.map +1 -0
- package/dist/channels/channel-surface-placement.js +29 -0
- package/dist/channels/channel-surface-placement.js.map +1 -0
- package/dist/channels/channel-visual-pump.d.ts +126 -0
- package/dist/channels/channel-visual-pump.d.ts.map +1 -0
- package/dist/channels/channel-visual-pump.js +174 -0
- package/dist/channels/channel-visual-pump.js.map +1 -0
- package/dist/hosts/IRealtimeMediaHost.d.ts +61 -4
- package/dist/hosts/IRealtimeMediaHost.d.ts.map +1 -1
- package/dist/hosts/IRealtimeMediaHost.js +2 -1
- package/dist/hosts/IRealtimeMediaHost.js.map +1 -1
- package/dist/index.d.ts +21 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +24 -0
- package/dist/index.js.map +1 -1
- package/dist/session/RealtimeSessionRuntime.d.ts +727 -74
- package/dist/session/RealtimeSessionRuntime.d.ts.map +1 -1
- package/dist/session/RealtimeSessionRuntime.js +1732 -242
- package/dist/session/RealtimeSessionRuntime.js.map +1 -1
- package/dist/session/RecordingShardQueue.d.ts +77 -0
- package/dist/session/RecordingShardQueue.d.ts.map +1 -0
- package/dist/session/RecordingShardQueue.js +172 -0
- package/dist/session/RecordingShardQueue.js.map +1 -0
- package/dist/session/app-client-tool-registry.d.ts +76 -0
- package/dist/session/app-client-tool-registry.d.ts.map +1 -0
- package/dist/session/app-client-tool-registry.js +100 -0
- package/dist/session/app-client-tool-registry.js.map +1 -0
- package/dist/session/avatar-notice.d.ts +43 -0
- package/dist/session/avatar-notice.d.ts.map +1 -0
- package/dist/session/avatar-notice.js +60 -0
- package/dist/session/avatar-notice.js.map +1 -0
- package/dist/session/call-end.d.ts +21 -0
- package/dist/session/call-end.d.ts.map +1 -0
- package/dist/session/call-end.js +19 -0
- package/dist/session/call-end.js.map +1 -0
- package/dist/session/channel-exposure-preferences.d.ts +70 -0
- package/dist/session/channel-exposure-preferences.d.ts.map +1 -0
- package/dist/session/channel-exposure-preferences.js +138 -0
- package/dist/session/channel-exposure-preferences.js.map +1 -0
- package/dist/session/channel-session-scope.d.ts +142 -0
- package/dist/session/channel-session-scope.d.ts.map +1 -0
- package/dist/session/channel-session-scope.js +128 -0
- package/dist/session/channel-session-scope.js.map +1 -0
- package/dist/session/client-session-deadline.d.ts +28 -0
- package/dist/session/client-session-deadline.d.ts.map +1 -0
- package/dist/session/client-session-deadline.js +50 -0
- package/dist/session/client-session-deadline.js.map +1 -0
- package/dist/session/realtime-captures.d.ts +236 -0
- package/dist/session/realtime-captures.d.ts.map +1 -0
- package/dist/session/realtime-captures.js +383 -0
- package/dist/session/realtime-captures.js.map +1 -0
- package/dist/session/realtime-microphone.d.ts +50 -0
- package/dist/session/realtime-microphone.d.ts.map +1 -0
- package/dist/session/realtime-microphone.js +51 -0
- package/dist/session/realtime-microphone.js.map +1 -0
- package/dist/session/session-event-hub.d.ts +72 -0
- package/dist/session/session-event-hub.d.ts.map +1 -0
- package/dist/session/session-event-hub.js +151 -0
- package/dist/session/session-event-hub.js.map +1 -0
- package/dist/session/session-launcher.d.ts +128 -0
- package/dist/session/session-launcher.d.ts.map +1 -0
- package/dist/session/session-launcher.js +160 -0
- package/dist/session/session-launcher.js.map +1 -0
- package/package.json +9 -9
|
@@ -1,13 +1,28 @@
|
|
|
1
1
|
import { BehaviorSubject, Subject } from 'rxjs';
|
|
2
2
|
import { Metadata } from '@memberjunction/core';
|
|
3
3
|
import { UserInfoEngine } from '@memberjunction/core-entities';
|
|
4
|
-
import { AIEngineBase } from '@memberjunction/ai-engine-base';
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
4
|
+
import { AddRealtimeUsageRecord, AIEngineBase, HasRealtimeUsage } from '@memberjunction/ai-engine-base';
|
|
5
|
+
import { GraphQLRealtimeSessionClient } from '@memberjunction/graphql-dataprovider';
|
|
6
|
+
import { MJGlobal, UUIDsEqual } from '@memberjunction/global';
|
|
7
|
+
import { DEFAULT_REALTIME_AUDIO_TRACKS, ParseRealtimeAvatarStatus, ParseRealtimeClientTransport } from '@memberjunction/ai';
|
|
8
|
+
import { CompareExposure, DeclaresNativeTools, IsIdentityVerifiedEventPayload, NormalizeChannelKey, ParseRealtimeSessionClientPolicy, ResolveClientTools, SelectNativeChannelTools, UserExposureReason } from '@memberjunction/ai-core-plus';
|
|
9
|
+
import { BaseRealtimeClient, LoadAssemblyAIRealtimeClient, LoadElevenLabsRealtimeClient, LoadGeminiEnterpriseRealtimeClient, LoadGeminiRealtimeClient, LoadHuggingFaceRealtimeClient, LoadOpenAIRealtimeClient, LoadxAIRealtimeClient, VideoSourceArbiter, REQUESTED_TRACKS_SESSION_KEY } from '@memberjunction/ai-realtime-client';
|
|
10
|
+
import { DefaultRealtimeSessionLauncher, HasClientCredential } from './session-launcher.js';
|
|
11
|
+
import { RequestsAgentVideo, ResolveAvatarNotice } from './avatar-notice.js';
|
|
12
|
+
import { ClientSessionDeadline } from './client-session-deadline.js';
|
|
13
|
+
import { RealtimeCaptures, REALTIME_CAPTURES_OFF, REALTIME_CAPTURE_OFFERS_NONE, } from './realtime-captures.js';
|
|
14
|
+
import { REALTIME_MICROPHONE_NONE, ReadMicrophoneState, SameMicrophoneState } from './realtime-microphone.js';
|
|
15
|
+
import { RealtimeSessionEventHub } from './session-event-hub.js';
|
|
8
16
|
import { BuildNarrationInstructions } from '../narration/narration-template.js';
|
|
9
17
|
import { ParseDelegationResultJson, FormatToolName } from './delegation-result-parser.js';
|
|
10
|
-
import { BaseRealtimeChannelClient } from '../channels/base-realtime-channel-client.js';
|
|
18
|
+
import { BaseRealtimeChannelClient, ChannelVideoSourceID } from '../channels/base-realtime-channel-client.js';
|
|
19
|
+
import { DEFAULT_CHANNEL_SURFACE_PLACEMENT, ReadChannelSurfacePlacement } from '../channels/channel-surface-placement.js';
|
|
20
|
+
import { ChannelActionDispatcher } from '../channels/channel-action-dispatcher.js';
|
|
21
|
+
import { BuildChannelCatalogNote } from '../channels/channel-catalog-note.js';
|
|
22
|
+
import { AppClientToolRegistry, DEFAULT_APP_TOOL_OWNER } from './app-client-tool-registry.js';
|
|
23
|
+
import { UserSettingsExposurePreferences } from './channel-exposure-preferences.js';
|
|
24
|
+
import { BuildChannelCandidate, FindPreparedChannel, MergeToolMetadata, PreparedChannelsShowAgentVideo, ReconcileChannelsWithPolicy, ResolveLocalChannelScope, ToolsByChannelKey } from './channel-session-scope.js';
|
|
25
|
+
import { RecordingShardQueue } from './RecordingShardQueue.js';
|
|
11
26
|
/**
|
|
12
27
|
* `MJ: User Settings` key for the per-user "record this voice call" consent toggle. Stored as
|
|
13
28
|
* the literal string `'true'`/`'false'` (read with `=== 'true'`), cross-device via
|
|
@@ -15,6 +30,15 @@ import { BaseRealtimeChannelClient } from '../channels/base-realtime-channel-cli
|
|
|
15
30
|
* default when the caller doesn't pass an explicit consent value.
|
|
16
31
|
*/
|
|
17
32
|
export const REALTIME_RECORDING_CONSENT_KEY = 'mj.realtimeVoice.recordingConsent.v1';
|
|
33
|
+
/**
|
|
34
|
+
* Relays usage onto the co-agent prompt run: token deltas, and the per-modality detail blocks as a
|
|
35
|
+
* JSON record (sent only when there are any; see `RelayRealtimeUsage` in MJServer).
|
|
36
|
+
*/
|
|
37
|
+
const RELAY_REALTIME_USAGE_MUTATION = `
|
|
38
|
+
mutation RelayRealtimeUsage($agentSessionId: String!, $inputTokens: Int!, $outputTokens: Int!, $usageDetailsJson: String) {
|
|
39
|
+
RelayRealtimeUsage(agentSessionId: $agentSessionId, inputTokens: $inputTokens, outputTokens: $outputTokens, usageDetailsJson: $usageDetailsJson)
|
|
40
|
+
}
|
|
41
|
+
`;
|
|
18
42
|
// Tree-shaking prevention: the OpenAI client is resolved dynamically through the
|
|
19
43
|
// ClassFactory (by the server-reported Provider key), so this static call is what keeps
|
|
20
44
|
// its @RegisterClass side effect from being eliminated by the bundler.
|
|
@@ -24,10 +48,37 @@ export const REALTIME_RECORDING_CONSENT_KEY = 'mj.realtimeVoice.recordingConsent
|
|
|
24
48
|
// and this service stays component-free (it must stay importable in plain-node tests).
|
|
25
49
|
LoadOpenAIRealtimeClient();
|
|
26
50
|
LoadGeminiRealtimeClient();
|
|
51
|
+
LoadGeminiEnterpriseRealtimeClient();
|
|
27
52
|
LoadElevenLabsRealtimeClient();
|
|
28
53
|
LoadAssemblyAIRealtimeClient();
|
|
29
54
|
LoadxAIRealtimeClient();
|
|
30
55
|
LoadHuggingFaceRealtimeClient();
|
|
56
|
+
/**
|
|
57
|
+
* A registry row's surface placement. A row whose `UIConfig` is not valid JSON places its surface as a row without one
|
|
58
|
+
* does, so one bad row never costs the session its other channels.
|
|
59
|
+
*/
|
|
60
|
+
function readRowSurfacePlacement(row) {
|
|
61
|
+
try {
|
|
62
|
+
return ReadChannelSurfacePlacement(row.UIConfigObject);
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
console.warn(`[RealtimeSession] Channel '${row.Name}' has a UIConfig that is not valid JSON; its surface starts on its tab.`);
|
|
66
|
+
return DEFAULT_CHANNEL_SURFACE_PLACEMENT;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
/** The same, for a row read over GraphQL, whose `UIConfig` arrives as the column's JSON text. */
|
|
70
|
+
function readUIConfigTextSurfacePlacement(name, uiConfig) {
|
|
71
|
+
if (!uiConfig) {
|
|
72
|
+
return ReadChannelSurfacePlacement(null);
|
|
73
|
+
}
|
|
74
|
+
try {
|
|
75
|
+
return ReadChannelSurfacePlacement(JSON.parse(uiConfig));
|
|
76
|
+
}
|
|
77
|
+
catch {
|
|
78
|
+
console.warn(`[RealtimeSession] Channel '${name}' has a UIConfig that is not valid JSON; its surface starts on its tab.`);
|
|
79
|
+
return DEFAULT_CHANNEL_SURFACE_PLACEMENT;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
31
82
|
/**
|
|
32
83
|
* Converts a {@link RealtimeTrackDescriptor} to its JSON form for the session-config bag.
|
|
33
84
|
*
|
|
@@ -67,6 +118,37 @@ function trackKeyFromJSON(raw) {
|
|
|
67
118
|
}
|
|
68
119
|
return `${direction}:${modality}`;
|
|
69
120
|
}
|
|
121
|
+
/** Whether a surface client tool handler's result is an object whose `Success` is `false`. */
|
|
122
|
+
function reportsFailure(result) {
|
|
123
|
+
return typeof result === 'object' && result !== null && 'Success' in result && result.Success === false;
|
|
124
|
+
}
|
|
125
|
+
/** The failed result's `ErrorMessage` when it is text that is not blank, else `'Tool failed'`. */
|
|
126
|
+
function failureMessage(result) {
|
|
127
|
+
const message = result.ErrorMessage;
|
|
128
|
+
return typeof message === 'string' && message.trim() ? message : 'Tool failed';
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* The `DOMException` name for each way the controller can fail to start the microphone, so a host that
|
|
132
|
+
* tells a denied microphone apart (`error.name === 'NotAllowedError'`) works the same with or without one.
|
|
133
|
+
*/
|
|
134
|
+
const MICROPHONE_ERROR_NAMES = {
|
|
135
|
+
denied: 'NotAllowedError',
|
|
136
|
+
'not-found': 'NotFoundError',
|
|
137
|
+
'in-use': 'NotReadableError',
|
|
138
|
+
unsupported: 'NotSupportedError',
|
|
139
|
+
error: 'Error',
|
|
140
|
+
};
|
|
141
|
+
/** What the user is told when the call's policy refuses a capture. */
|
|
142
|
+
const CAPTURE_WORDING = {
|
|
143
|
+
camera: { NotInCall: 'The camera is not part of this call.', NotAllowed: 'This call cannot show the agent your camera' },
|
|
144
|
+
screen: { NotInCall: 'Screen sharing is not part of this call.', NotAllowed: 'This call cannot show the agent your screen' },
|
|
145
|
+
};
|
|
146
|
+
/** The error a failed microphone start reports, named as `getUserMedia` would have named it. */
|
|
147
|
+
function microphoneStartError(reason, message) {
|
|
148
|
+
const error = new Error(message);
|
|
149
|
+
error.name = MICROPHONE_ERROR_NAMES[reason];
|
|
150
|
+
return error;
|
|
151
|
+
}
|
|
70
152
|
/**
|
|
71
153
|
* Drives a **client-direct** real-time voice session: the browser mints an ephemeral
|
|
72
154
|
* token from the MJ server, then connects DIRECTLY to the realtime provider. Audio
|
|
@@ -100,6 +182,7 @@ export class RealtimeSessionRuntime {
|
|
|
100
182
|
this.mediaHost = mediaHost;
|
|
101
183
|
// ── Reactive UI state ──────────────────────────────────────────────────────
|
|
102
184
|
this._connectionState$ = new BehaviorSubject('closed');
|
|
185
|
+
this._resuming$ = new BehaviorSubject(false);
|
|
103
186
|
this._captions$ = new BehaviorSubject([]);
|
|
104
187
|
this._active$ = new BehaviorSubject(false);
|
|
105
188
|
this._delegationProgress$ = new Subject();
|
|
@@ -109,6 +192,7 @@ export class RealtimeSessionRuntime {
|
|
|
109
192
|
this._agentName$ = new BehaviorSubject('Sage');
|
|
110
193
|
this._modelName$ = new BehaviorSubject(null);
|
|
111
194
|
this._minimized$ = new BehaviorSubject(false);
|
|
195
|
+
this._savingRecording$ = new BehaviorSubject(false);
|
|
112
196
|
this._activeChannels$ = new BehaviorSubject([]);
|
|
113
197
|
this._channelFocus$ = new Subject();
|
|
114
198
|
// ─── Generic session-lifecycle events (consumed by RealtimeSessionsAdapter to
|
|
@@ -122,6 +206,16 @@ export class RealtimeSessionRuntime {
|
|
|
122
206
|
this._channelActivity$ = new Subject();
|
|
123
207
|
/** Current connection / turn state. */
|
|
124
208
|
this.ConnectionState$ = this._connectionState$.asObservable();
|
|
209
|
+
/**
|
|
210
|
+
* Whether the call is resuming: moving to a new connection with its provider mid-call. `true` from a `'connecting'` the
|
|
211
|
+
* driver reports while the call is live, until the call is live again (`'listening'`, `'speaking'` or `'thinking'`),
|
|
212
|
+
* fails (`'error'`) or ends (`'closed'`). A call's first connect is not a resume, so it is `false` until the call has
|
|
213
|
+
* been live, and outside a call. Gemini Live resumes when Google ends a connection (`goAway`) and after a dropped
|
|
214
|
+
* socket, and the agent's video can stop for a few seconds meanwhile: a host can show that as a pause rather than as
|
|
215
|
+
* the agent leaving. It changes before {@link ConnectionState$} does, so a resume's `'connecting'` arrives with it
|
|
216
|
+
* already `true`.
|
|
217
|
+
*/
|
|
218
|
+
this.Resuming$ = this._resuming$.asObservable();
|
|
125
219
|
/** Live captions for both sides of the conversation. */
|
|
126
220
|
this.Captions$ = this._captions$.asObservable();
|
|
127
221
|
/** True while a session is open (mic button active, overlay shown). */
|
|
@@ -158,6 +252,14 @@ export class RealtimeSessionRuntime {
|
|
|
158
252
|
* pure presentation state, reset to `false` at session start and teardown.
|
|
159
253
|
*/
|
|
160
254
|
this.Minimized$ = this._minimized$.asObservable();
|
|
255
|
+
/**
|
|
256
|
+
* True while the end-of-call recording is being saved: from the first statement of a teardown
|
|
257
|
+
* that holds a recorder until that teardown settles. False at construction and for unrecorded
|
|
258
|
+
* sessions. Exists for #5195: the page must stay open until the consolidated upload lands, or
|
|
259
|
+
* the recording is lost. The runtime is platform-free, so hosts own the reaction (a browser
|
|
260
|
+
* `beforeunload` guard, a "saving" indicator, etc.).
|
|
261
|
+
*/
|
|
262
|
+
this.SavingRecording$ = this._savingRecording$.asObservable();
|
|
161
263
|
/**
|
|
162
264
|
* The session's ACTIVE interactive-channel plugins, resolved from the `MJ: AI Agent
|
|
163
265
|
* Channels` registry at session start (one instance per session, per channel). Emits
|
|
@@ -165,6 +267,57 @@ export class RealtimeSessionRuntime {
|
|
|
165
267
|
* one surface tab per plugin — it never knows any concrete channel type.
|
|
166
268
|
*/
|
|
167
269
|
this.ActiveChannels$ = this._activeChannels$.asObservable();
|
|
270
|
+
this._videoSources$ = new BehaviorSubject([]);
|
|
271
|
+
/**
|
|
272
|
+
* The video sources the agent is perceiving or could perceive right now (a whiteboard, a remote browser, a
|
|
273
|
+
* shared screen), with whether each is on and whether its frames are reaching the model. Empty when no
|
|
274
|
+
* session is live or nothing has offered a frame source. This is what the "agent can see" control renders;
|
|
275
|
+
* toggle one with {@link SetVideoSourceEnabled}.
|
|
276
|
+
*/
|
|
277
|
+
this.VideoSources$ = this._videoSources$.asObservable();
|
|
278
|
+
this._captures$ = new BehaviorSubject(REALTIME_CAPTURES_OFF);
|
|
279
|
+
/**
|
|
280
|
+
* The user's camera and screen share: off, starting, on (with the stream to show the user) or failed (with the
|
|
281
|
+
* reason). Both off outside a session. Start and stop them with {@link StartCamera}, {@link StartScreenShare},
|
|
282
|
+
* {@link StopCamera} and {@link StopScreenShare}; while one is on, it is also a source on {@link VideoSources$}.
|
|
283
|
+
*/
|
|
284
|
+
this.Captures$ = this._captures$.asObservable();
|
|
285
|
+
this._captureOffers$ = new BehaviorSubject(REALTIME_CAPTURE_OFFERS_NONE);
|
|
286
|
+
/**
|
|
287
|
+
* Which captures the call offers, now and on every change. A capture is offered while the call is connected, the model
|
|
288
|
+
* takes inbound video, the host can open it (a camera controller; screen sharing) and the call's policy admits it (its
|
|
289
|
+
* channel is in the call and may show the agent pixels). A host shows its Camera and Share controls from this. Neither
|
|
290
|
+
* is offered outside a call.
|
|
291
|
+
*/
|
|
292
|
+
this.CaptureOffers$ = this._captureOffers$.asObservable();
|
|
293
|
+
this._microphone$ = new BehaviorSubject(REALTIME_MICROPHONE_NONE);
|
|
294
|
+
/**
|
|
295
|
+
* The call's microphone, now and on every change: the one in use and the ones the user can move it to with
|
|
296
|
+
* {@link SwitchMicrophone}, as the host's camera-and-microphone controller lists them. Offered from the moment the call
|
|
297
|
+
* connects until it ends, and kept current as devices come and go. When the call loses its microphone (the one in use
|
|
298
|
+
* fails and so does the fallback to the default, or the last one goes away), its `Failure` says why: the agent hears
|
|
299
|
+
* nothing until the user picks a microphone, which opens it again. {@link REALTIME_MICROPHONE_NONE} outside a call and
|
|
300
|
+
* on a host without a controller, which opens the microphone itself (`IRealtimeMediaHost.AcquireMicrophone`).
|
|
301
|
+
*/
|
|
302
|
+
this.Microphone$ = this._microphone$.asObservable();
|
|
303
|
+
this._agentVideo$ = new BehaviorSubject(null);
|
|
304
|
+
/**
|
|
305
|
+
* The agent's video (an avatar) while the model sends it: a live stream, or a player that owns the `<video>` element.
|
|
306
|
+
* `null` until the video arrives, for a model that sends none, and outside a call. The driver hands it over once it is
|
|
307
|
+
* live, and a newer one replaces it. A host shows it with `AttachVideoSource`; a channel that sinks outbound video
|
|
308
|
+
* follows it through its context, and counts as used once it arrives, so the host shows that channel's surface.
|
|
309
|
+
*/
|
|
310
|
+
this.AgentVideo$ = this._agentVideo$.asObservable();
|
|
311
|
+
this._avatarNotice$ = new BehaviorSubject(null);
|
|
312
|
+
/**
|
|
313
|
+
* Why the call shows no avatar the agent asked for: set once per call, after it connects, when the model renders none,
|
|
314
|
+
* the persona has no face for it, the app shows no agent video (`host`) or the browser can't play it (`browser`); with
|
|
315
|
+
* no `Reason` when the mint gave one this version doesn't know, such as a newer server's. `null` when there is nothing
|
|
316
|
+
* to say (no avatar asked for, the avatar shows, or the mint reported no status) and after the call ends. A provider
|
|
317
|
+
* resume inside the call does not set it again. A host turns the reason into words once, for example "Audio only: this
|
|
318
|
+
* voice model can't show an avatar".
|
|
319
|
+
*/
|
|
320
|
+
this.AvatarNotice$ = this._avatarNotice$.asObservable();
|
|
168
321
|
/**
|
|
169
322
|
* Channel requests to enter / leave the FOCUS layout (see
|
|
170
323
|
* {@link RealtimeChannelFocusEvent}). Fired when a plugin calls its host context's
|
|
@@ -208,7 +361,11 @@ export class RealtimeSessionRuntime {
|
|
|
208
361
|
*/
|
|
209
362
|
/** Conversation id the SERVER created for this session (null when the host supplied one). */
|
|
210
363
|
this.createdConversationId = null;
|
|
211
|
-
/**
|
|
364
|
+
/**
|
|
365
|
+
* The session's conversation id (supplied or server-created). While a start's mint runs, the
|
|
366
|
+
* conversation the start asked for; null when it asked the server to create one. Null after
|
|
367
|
+
* teardown.
|
|
368
|
+
*/
|
|
212
369
|
this.sessionConversationId = null;
|
|
213
370
|
/** First final user utterance of the live session (the naming seed). */
|
|
214
371
|
this.firstUserTranscript = null;
|
|
@@ -219,8 +376,28 @@ export class RealtimeSessionRuntime {
|
|
|
219
376
|
// ── Session internals ──────────────────────────────────────────────────────
|
|
220
377
|
/** The provider-direct realtime client driving the live session (ClassFactory-resolved). */
|
|
221
378
|
this.client = null;
|
|
222
|
-
/**
|
|
379
|
+
/**
|
|
380
|
+
* The mic capture stream — acquired here (permission UX) and handed to the client. A microphone the controller opens
|
|
381
|
+
* again after the call lost one replaces it.
|
|
382
|
+
*/
|
|
223
383
|
this.localStream = null;
|
|
384
|
+
/**
|
|
385
|
+
* Whether the user muted the call's microphone, as {@link ToggleMute} last reported it. While the call has no microphone
|
|
386
|
+
* track there is nothing to mute, and a microphone opened again after the call lost one starts this way.
|
|
387
|
+
*/
|
|
388
|
+
this.microphoneMuted = false;
|
|
389
|
+
/** The host's camera-and-microphone controller for this session, when the host offers one. */
|
|
390
|
+
this.localMedia = null;
|
|
391
|
+
/** Follows the controller's microphone, so a swapped-in track or a reopened microphone reaches the driver and the recorder. */
|
|
392
|
+
this.localMediaSubscription = null;
|
|
393
|
+
/**
|
|
394
|
+
* Whether the call offers its microphone for switching ({@link Microphone$}, {@link SwitchMicrophone}): from the moment
|
|
395
|
+
* the client is connected until the call ends. Not before: a driver still connecting may have bound the old track.
|
|
396
|
+
*/
|
|
397
|
+
this.microphoneOffered = false;
|
|
398
|
+
/** The live session's camera and screen share; created once the client is connected. */
|
|
399
|
+
this.captures = null;
|
|
400
|
+
this.capturesSubscription = null;
|
|
224
401
|
this.agentSessionId = null;
|
|
225
402
|
/**
|
|
226
403
|
* The application the active session runs in (sources the server-side app config cascade +
|
|
@@ -254,8 +431,15 @@ export class RealtimeSessionRuntime {
|
|
|
254
431
|
this.recordingStartedAtIso = null;
|
|
255
432
|
/** Interval that flushes ~15s crash-recovery shards to the server during a recording. */
|
|
256
433
|
this.segmentTimer = null;
|
|
257
|
-
/**
|
|
258
|
-
|
|
434
|
+
/**
|
|
435
|
+
* Owns shard indexing and retry for the current recording: it assigns each shard its index and
|
|
436
|
+
* keeps failed shards to resend under that same index. A fresh queue per recording keeps a flush
|
|
437
|
+
* still in flight from a previous session off this session's indexes. Null when not recording.
|
|
438
|
+
* A recording that ends discards its queue, and any retained shard with it: the consolidated
|
|
439
|
+
* end-of-call upload supersedes them, and if that upload is lost too, server-side recovery
|
|
440
|
+
* (#5196) fills the gap with silence.
|
|
441
|
+
*/
|
|
442
|
+
this.shardQueue = null;
|
|
259
443
|
// ── Server-side liveness ───────────────────────────────────────────────────
|
|
260
444
|
/**
|
|
261
445
|
* Interval that tells the server this session is still in use, or `null` when no session is
|
|
@@ -310,6 +494,11 @@ export class RealtimeSessionRuntime {
|
|
|
310
494
|
this.pendingUsageInput = 0;
|
|
311
495
|
/** Accumulated output-token delta since the last flush. */
|
|
312
496
|
this.pendingUsageOutput = 0;
|
|
497
|
+
/**
|
|
498
|
+
* Accumulated per-modality usage since the last flush (input and output detail blocks, avatar
|
|
499
|
+
* video seconds included), relayed in full; `null` when no update carried any.
|
|
500
|
+
*/
|
|
501
|
+
this.pendingUsageDetails = null;
|
|
313
502
|
/** Pending debounced usage flush; also force-flushed at teardown. */
|
|
314
503
|
this.usageFlushTimer = null;
|
|
315
504
|
/** Active push-status subscription that feeds delegation progress; cleared on teardown. */
|
|
@@ -352,7 +541,20 @@ export class RealtimeSessionRuntime {
|
|
|
352
541
|
*/
|
|
353
542
|
this.teardownInFlight = null;
|
|
354
543
|
/**
|
|
355
|
-
*
|
|
544
|
+
* The latest start's opening of the host's microphone, until {@link IRealtimeMediaHost.ReleaseMicrophone} hands it
|
|
545
|
+
* back; `null` when no opening is owed a release, so a session that ends before asking for the microphone releases
|
|
546
|
+
* nothing.
|
|
547
|
+
*
|
|
548
|
+
* The host gets one release per opening, after the opening returns. A release while the microphone is still
|
|
549
|
+
* opening undoes nothing: the React Native host puts iOS into the call audio category inside `AcquireMicrophone`,
|
|
550
|
+
* after the permission prompt. So {@link runTeardown} releases an opening that has returned and leaves one still
|
|
551
|
+
* opening to its start, which releases it once it returns ({@link unwindAbandonedStart}), unless a newer start has
|
|
552
|
+
* opened the microphone since; that start's own end releases it. Whoever releases clears this first, so no
|
|
553
|
+
* opening is released twice.
|
|
554
|
+
*/
|
|
555
|
+
this.microphoneLease = null;
|
|
556
|
+
/**
|
|
557
|
+
* Why the latest session start failed, or `null` when it has not. Each start clears it ({@link beginSessionStart}).
|
|
356
558
|
*
|
|
357
559
|
* The runtime reports failure as `'error'` on {@link ConnectionState$}, which is enough to show
|
|
358
560
|
* *that* something went wrong but not *what* — and the difference matters at exactly one point:
|
|
@@ -361,6 +563,11 @@ export class RealtimeSessionRuntime {
|
|
|
361
563
|
* runtime's own try/catch, so the host never sees that rejection itself.
|
|
362
564
|
*/
|
|
363
565
|
this.lastStartError = null;
|
|
566
|
+
/**
|
|
567
|
+
* The fatal error the call's client reported, or `null` when it has reported none. Each start clears it
|
|
568
|
+
* ({@link beginSessionStart}). See {@link LastSessionError}.
|
|
569
|
+
*/
|
|
570
|
+
this.lastSessionError = null;
|
|
364
571
|
/**
|
|
365
572
|
* Pending DEBOUNCED channel-state saves, keyed by channel name. Each entry keeps the
|
|
366
573
|
* LATEST serialized state plus the session id captured while the session was live —
|
|
@@ -377,13 +584,92 @@ export class RealtimeSessionRuntime {
|
|
|
377
584
|
*/
|
|
378
585
|
this.usedChannelNames = new Set();
|
|
379
586
|
this._provider = null;
|
|
587
|
+
this._launcher = new DefaultRealtimeSessionLauncher();
|
|
588
|
+
/**
|
|
589
|
+
* Loads channel code on demand. Called, with the `ClientPluginClass` keys of every channel about to be
|
|
590
|
+
* resolved (the registry's rows plus the host's declarations), after the registry is read and before any
|
|
591
|
+
* plugin is built — so a host can register a heavy channel's class only when the session could use it.
|
|
592
|
+
* It must resolve once the classes it knows are registered; a rejection is logged, never fatal.
|
|
593
|
+
*/
|
|
594
|
+
this.ChannelClassLoader = null;
|
|
595
|
+
// ── Session events (server → this session's client) ────────────────────────
|
|
596
|
+
this._sessionEvents$ = new Subject();
|
|
597
|
+
/**
|
|
598
|
+
* Every event the server publishes to THIS session, for the session's life (identity verification
|
|
599
|
+
* today; apps add their own types). Hot, with no replay — subscribe before the session starts, once,
|
|
600
|
+
* and it keeps working across sessions. The runtime has already acted on the events it understands
|
|
601
|
+
* (see {@link handleIdentityVerified}) by the time one reaches here; this stream is for the host UI.
|
|
602
|
+
*
|
|
603
|
+
* Silent when the session's provider has no subscription transport (see {@link createSessionEventSource}).
|
|
604
|
+
*/
|
|
605
|
+
this.SessionEvents$ = this._sessionEvents$.asObservable();
|
|
606
|
+
/** Keeps the live session's event stream open; null between sessions. */
|
|
607
|
+
this.sessionEventHub = null;
|
|
608
|
+
/** Per-type handlers the runtime runs (before publishing on {@link SessionEvents$}) for events it understands. */
|
|
609
|
+
this.sessionEventHandlers = new Map([
|
|
610
|
+
['identity.verified', (event) => this.handleIdentityVerified(event)]
|
|
611
|
+
]);
|
|
612
|
+
/**
|
|
613
|
+
* What the agent should SAY the moment the user's identity is verified, or `null` (the default) to
|
|
614
|
+
* say nothing — the agent is still told, silently, through a context note. A string is used as the
|
|
615
|
+
* instruction verbatim; a function receives the verified payload (e.g. to greet by name) and returns
|
|
616
|
+
* the instruction, or `null` to stay silent for this verification.
|
|
617
|
+
*/
|
|
618
|
+
this.IdentityVerifiedSpokenResponse = null;
|
|
619
|
+
this.clientDeadline = new ClientSessionDeadline();
|
|
620
|
+
// ── Interactive channels (registry-driven + host-declared plugins) ─────────
|
|
621
|
+
//
|
|
622
|
+
// A session's channels come from two places — the `MJ: AI Agent Channels` registry and the channels
|
|
623
|
+
// the host declares — and pass through three stages:
|
|
624
|
+
//
|
|
625
|
+
// 1. PREPARE (before mint) construct the plugins, read their descriptors, report them as
|
|
626
|
+
// candidates, and work out the scope the browser can decide alone.
|
|
627
|
+
// 2. MINT the server scopes the candidates with the agent/app cascade and the
|
|
628
|
+
// registry's kill switch, and returns the resolved policy.
|
|
629
|
+
// 3. ACTIVATE (after mint) initialize exactly the policy's channels: `open-on-start`/`headless`
|
|
630
|
+
// ones mount now, `on-demand` ones wait to be opened through ContextTool.
|
|
631
|
+
//
|
|
632
|
+
// Nothing is initialized before the policy is known, so a channel the server vetoes never runs.
|
|
633
|
+
/** Whether the channel catalog note still has to be sent (see {@link flushChannelCatalogNote}). */
|
|
634
|
+
this.catalogNotePending = false;
|
|
635
|
+
/** Plugins resolved for the current start but not yet initialized. Consumed by {@link activateChannels}. */
|
|
636
|
+
this.preparedChannels = [];
|
|
637
|
+
/** The scope the browser resolves on its own — the fallback when no server policy comes back. */
|
|
638
|
+
this.localChannelScope = null;
|
|
639
|
+
/** `on-demand` channels that are in the session but not opened yet (initialized only when opened). */
|
|
640
|
+
this.advertisedChannels = [];
|
|
641
|
+
/** What the policy resolved for each in-session channel, keyed by normalized channel key. */
|
|
642
|
+
this.resolvedChannels = new Map();
|
|
643
|
+
/** The prior session's saved channel states, kept so an `on-demand` channel opened later is restored too. */
|
|
644
|
+
this.priorChannelStates = {};
|
|
645
|
+
/** The app/static client-tool tiers the server resolved for this session. */
|
|
646
|
+
this.sessionClientTools = null;
|
|
647
|
+
/** Validates and routes channel-addressed `ContextTool` calls. */
|
|
648
|
+
this.channelDispatcher = new ChannelActionDispatcher({
|
|
649
|
+
FindChannel: (key) => this.findDispatchableChannel(key),
|
|
650
|
+
ListChannelKeys: () => this.listAddressableChannelKeys(),
|
|
651
|
+
ActivateChannel: (plugin) => this.mountAdvertisedChannel(plugin),
|
|
652
|
+
});
|
|
653
|
+
/** Where the user's per-channel "how much can the agent see" choices are kept. Replace with {@link SetExposurePreferences}. */
|
|
654
|
+
this.exposurePreferences = new UserSettingsExposurePreferences(() => this.canPersistUserSettings());
|
|
655
|
+
/** The channel whose surface the user is looking at, kept so it applies to an arbiter created after it was set. */
|
|
656
|
+
this.focusedChannelKey = null;
|
|
657
|
+
this.stopWatchingVideoSources = null;
|
|
380
658
|
/**
|
|
381
659
|
* Host registry of surface CLIENT TOOLS (Name → handler), fed by the host (Explorer) from the
|
|
382
|
-
* active surface's `SetAgentClientTools
|
|
383
|
-
* executes against this via {@link executeAppClientTool}.
|
|
384
|
-
*
|
|
660
|
+
* active surface's `SetAgentClientTools` and from its always-on globals. The headless
|
|
661
|
+
* ClientContextChannel's `ContextTool` proxy executes against this via {@link executeAppClientTool}.
|
|
662
|
+
* Owner-keyed — see {@link AppClientToolRegistry}.
|
|
385
663
|
*/
|
|
386
|
-
this.
|
|
664
|
+
this.appToolRegistry = new AppClientToolRegistry();
|
|
665
|
+
}
|
|
666
|
+
/** Synchronous read of {@link SavingRecording$}. */
|
|
667
|
+
get IsSavingRecording() {
|
|
668
|
+
return this._savingRecording$.value;
|
|
669
|
+
}
|
|
670
|
+
/** Synchronous access to {@link AvatarNotice$}'s current value. */
|
|
671
|
+
get CurrentAvatarNotice() {
|
|
672
|
+
return this._avatarNotice$.value;
|
|
387
673
|
}
|
|
388
674
|
/** Synchronous access to the session's active interactive-channel plugins. */
|
|
389
675
|
get ActiveChannels() {
|
|
@@ -480,10 +766,113 @@ export class RealtimeSessionRuntime {
|
|
|
480
766
|
set Provider(value) {
|
|
481
767
|
this._provider = value;
|
|
482
768
|
}
|
|
769
|
+
/**
|
|
770
|
+
* How this runtime mints a session — the stock `StartRealtimeClientSession` mutation unless a host
|
|
771
|
+
* installs its own {@link IRealtimeSessionLauncher} (a guest-session exchange, a deployment-specific
|
|
772
|
+
* mutation). Everything after the mint — channel scoping, the driver, transcripts, teardown — is
|
|
773
|
+
* unchanged. Assign `null` to restore the default.
|
|
774
|
+
*/
|
|
775
|
+
get Launcher() {
|
|
776
|
+
return this._launcher;
|
|
777
|
+
}
|
|
778
|
+
set Launcher(value) {
|
|
779
|
+
this._launcher = value ?? new DefaultRealtimeSessionLauncher();
|
|
780
|
+
}
|
|
781
|
+
/**
|
|
782
|
+
* The client's copy of the session's absolute deadline (`null` when none is known). A host that
|
|
783
|
+
* knows the session's cap at start sets it with {@link SetSessionDeadline}; verification extends it
|
|
784
|
+
* from the server's `identity.verified` payload. The SERVER enforces the real deadline — this exists
|
|
785
|
+
* so the host can end the call gracefully first and show a countdown. It only ever moves later by
|
|
786
|
+
* the server's word, so nothing a client does can extend the real one.
|
|
787
|
+
*/
|
|
788
|
+
get SessionDeadline$() {
|
|
789
|
+
return this.clientDeadline.Deadline$;
|
|
790
|
+
}
|
|
791
|
+
/** Synchronous access to {@link SessionDeadline$}. */
|
|
792
|
+
get SessionDeadline() {
|
|
793
|
+
return this.clientDeadline.Value;
|
|
794
|
+
}
|
|
795
|
+
/** Sets the baseline deadline a host knows at session start (`null` clears). See {@link SessionDeadline$}. */
|
|
796
|
+
SetSessionDeadline(deadline) {
|
|
797
|
+
this.clientDeadline.Set(deadline);
|
|
798
|
+
}
|
|
799
|
+
/**
|
|
800
|
+
* Builds the transport that reads the session's events. The default rides the session's GraphQL
|
|
801
|
+
* provider; it returns `null` — "no session events here" — for a provider with no subscription
|
|
802
|
+
* support (a test double, a host on another transport), which the runtime tolerates. Override to
|
|
803
|
+
* supply another transport.
|
|
804
|
+
*/
|
|
805
|
+
createSessionEventSource() {
|
|
806
|
+
const provider = this.Provider;
|
|
807
|
+
if (!provider || typeof provider.Subscribe !== 'function') {
|
|
808
|
+
return null;
|
|
809
|
+
}
|
|
810
|
+
return new GraphQLRealtimeSessionClient(provider);
|
|
811
|
+
}
|
|
812
|
+
/** Opens the live session's event stream. Best-effort: never disturbs the call. */
|
|
813
|
+
startSessionEvents(agentSessionId) {
|
|
814
|
+
this.stopSessionEvents();
|
|
815
|
+
try {
|
|
816
|
+
const source = this.createSessionEventSource();
|
|
817
|
+
if (!source) {
|
|
818
|
+
return;
|
|
819
|
+
}
|
|
820
|
+
this.sessionEventHub = new RealtimeSessionEventHub(source, (event) => this.routeSessionEvent(agentSessionId, event));
|
|
821
|
+
this.sessionEventHub.Start(agentSessionId);
|
|
822
|
+
}
|
|
823
|
+
catch (error) {
|
|
824
|
+
console.error('[RealtimeSession] Could not start the session event stream:', error);
|
|
825
|
+
}
|
|
826
|
+
}
|
|
827
|
+
stopSessionEvents() {
|
|
828
|
+
this.sessionEventHub?.Stop();
|
|
829
|
+
this.sessionEventHub = null;
|
|
830
|
+
}
|
|
831
|
+
/** Routes one event to its per-type handler, then publishes it. An event for another session is dropped. */
|
|
832
|
+
routeSessionEvent(expectedSessionId, event) {
|
|
833
|
+
if (event.AgentSessionID !== expectedSessionId || this.agentSessionId !== expectedSessionId) {
|
|
834
|
+
console.warn(`[RealtimeSession] Dropped a '${event.Type}' event addressed to another session.`);
|
|
835
|
+
return;
|
|
836
|
+
}
|
|
837
|
+
try {
|
|
838
|
+
this.sessionEventHandlers.get(event.Type)?.(event);
|
|
839
|
+
}
|
|
840
|
+
catch (error) {
|
|
841
|
+
console.error(`[RealtimeSession] Handling the '${event.Type}' session event failed:`, error);
|
|
842
|
+
}
|
|
843
|
+
this._sessionEvents$.next(event);
|
|
844
|
+
}
|
|
845
|
+
/**
|
|
846
|
+
* `identity.verified`: keep the client's deadline in step with the server's, tell the model — as a
|
|
847
|
+
* silent, structured note — who it is now talking to, and, when the host configured it, have it say
|
|
848
|
+
* something about that. The payload is validated again here (it crossed a wire); the name rides as
|
|
849
|
+
* JSON so user-entered text can never break out of the note's frame.
|
|
850
|
+
*/
|
|
851
|
+
handleIdentityVerified(event) {
|
|
852
|
+
if (event.Type !== 'identity.verified' || !IsIdentityVerifiedEventPayload(event.Payload)) {
|
|
853
|
+
console.warn('[RealtimeSession] Ignored a malformed identity.verified event.');
|
|
854
|
+
return;
|
|
855
|
+
}
|
|
856
|
+
const payload = event.Payload;
|
|
857
|
+
this.clientDeadline.Extend(payload.MaxSessionDeadlineIso);
|
|
858
|
+
this.SendContextNote(`[identity] verified ${JSON.stringify({ email: payload.VerifiedEmail, name: payload.VerifiedName, method: payload.Method })} ` +
|
|
859
|
+
'(background context — the user has proven they control this email address; treat them as verified. Do not read this note aloud.)');
|
|
860
|
+
const spoken = typeof this.IdentityVerifiedSpokenResponse === 'function' ? this.IdentityVerifiedSpokenResponse(payload) : this.IdentityVerifiedSpokenResponse;
|
|
861
|
+
if (spoken && spoken.trim().length > 0 && this.client && this.isSessionLive()) {
|
|
862
|
+
this.requestChannelSpokenResponse(spoken.trim());
|
|
863
|
+
}
|
|
864
|
+
}
|
|
483
865
|
/** True when a session is currently open. */
|
|
484
866
|
get IsActive() {
|
|
485
867
|
return this._active$.value;
|
|
486
868
|
}
|
|
869
|
+
/**
|
|
870
|
+
* True when a session is open or starting for `conversationId`. Ids are compared without
|
|
871
|
+
* regard to case.
|
|
872
|
+
*/
|
|
873
|
+
IsActiveFor(conversationId) {
|
|
874
|
+
return this.IsActive && conversationId != null && UUIDsEqual(this.sessionConversationId, conversationId);
|
|
875
|
+
}
|
|
487
876
|
/**
|
|
488
877
|
* Start a client-direct voice session fronting `targetAgentId`.
|
|
489
878
|
*
|
|
@@ -521,31 +910,55 @@ export class RealtimeSessionRuntime {
|
|
|
521
910
|
* the server-side Media channel resolves THIS collection as the agent's media kit for the session,
|
|
522
911
|
* taking precedence over the agent's `DefaultMediaCollectionID`. The server UUID-validates it
|
|
523
912
|
* (malformed ⇒ ignored, the agent default applies). Omit/`null` to use the agent default kit.
|
|
913
|
+
* @param applicationId Optional application the session runs in (sources the app config cascade,
|
|
914
|
+
* including `Application.AgentSettings.Realtime.Channels`).
|
|
915
|
+
* @param appContext Optional live app-context snapshot injected into the companion prompt at mint.
|
|
916
|
+
* @param options Optional per-start extras — chiefly the channels the HOST brings
|
|
917
|
+
* ({@link RealtimeSessionStartOptions.HostChannels}), the way a connect-only embed with no
|
|
918
|
+
* registry gets channels at all, and the fork the session starts on
|
|
919
|
+
* ({@link RealtimeSessionStartOptions.ConversationBranchID}; omitted is Main, where its transcript is written).
|
|
524
920
|
*/
|
|
525
|
-
async StartRealtimeSession(targetAgentId, conversationId, lastSessionId, agentName, preferredModelId, clientTools, coAgentId, configOverridesJson, recordingConsent, mediaCollectionId, applicationId, appContext) {
|
|
921
|
+
async StartRealtimeSession(targetAgentId, conversationId, lastSessionId, agentName, preferredModelId, clientTools, coAgentId, configOverridesJson, recordingConsent, mediaCollectionId, applicationId, appContext, options) {
|
|
526
922
|
if (this.IsActive) {
|
|
527
923
|
return; // a session is already running — ignore duplicate starts
|
|
528
924
|
}
|
|
529
925
|
const consent = this.beginSessionStart({ agentName, recordingConsent, applicationId, appContext });
|
|
530
|
-
|
|
926
|
+
this.sessionConversationId = conversationId ?? null;
|
|
927
|
+
// A teardown while the channels are prepared or the mint runs bumps this; see the checks below.
|
|
928
|
+
const generation = this.startGeneration;
|
|
929
|
+
// Captured BEFORE the channels are prepared so the mint carries exactly the snapshot the prologue
|
|
531
930
|
// resolved, whatever a channel plugin may push in the meantime.
|
|
532
931
|
const effectiveAppContext = this._appContext$.value;
|
|
533
932
|
let session;
|
|
534
933
|
try {
|
|
535
|
-
// Resolve
|
|
536
|
-
//
|
|
537
|
-
|
|
538
|
-
|
|
934
|
+
// Resolve the interactive-channel plugins FIRST (constructed, not started): the tools of the
|
|
935
|
+
// channels mounted with the session must be declared to the realtime model at mint, and the
|
|
936
|
+
// server needs the candidates to scope them. Nothing is initialized until the policy is known.
|
|
937
|
+
const scope = await this.prepareChannelScope(options?.HostChannels);
|
|
938
|
+
const allClientTools = [...(clientTools ?? []), ...scope.NativeTools];
|
|
939
|
+
session = await this.mintSession(targetAgentId, conversationId, lastSessionId, preferredModelId, allClientTools, coAgentId, configOverridesJson, consent, this.recordingStartedAtIso, mediaCollectionId, this.applicationId, effectiveAppContext, scope.CandidatesJson, scope.ShowsAgentVideo, options?.ConversationBranchID ?? null);
|
|
539
940
|
}
|
|
540
941
|
catch (error) {
|
|
942
|
+
// An ended or replaced start only logs its failure: the error state and the teardown belong
|
|
943
|
+
// to the session that is current now, which a newer start may own.
|
|
944
|
+
if (this.startGeneration !== generation) {
|
|
945
|
+
console.error('[RealtimeSession] A start that was already ended failed:', error);
|
|
946
|
+
return;
|
|
947
|
+
}
|
|
541
948
|
await this.failSessionStart(error);
|
|
542
949
|
return;
|
|
543
950
|
}
|
|
951
|
+
// The host ended the start while the mint ran. That teardown had no session id to close, so
|
|
952
|
+
// close the minted row here, and touch no shared state: a newer start may already own it.
|
|
953
|
+
if (this.startGeneration !== generation) {
|
|
954
|
+
await this.closeServerSession(session.AgentSessionId);
|
|
955
|
+
return;
|
|
956
|
+
}
|
|
544
957
|
if (!this.hostCanUseProvider(session.Provider)) {
|
|
545
958
|
await this.abortUnusableSession(session);
|
|
546
959
|
return;
|
|
547
960
|
}
|
|
548
|
-
await this.runMintedSession(session, conversationId ?? null, consent);
|
|
961
|
+
await this.runMintedSession(session, conversationId ?? null, consent, options?.CameraCheck === true);
|
|
549
962
|
}
|
|
550
963
|
/**
|
|
551
964
|
* Declares which provider keys this host can actually carry audio for.
|
|
@@ -581,7 +994,7 @@ export class RealtimeSessionRuntime {
|
|
|
581
994
|
// published them on ActiveChannels$; skipping teardown would leave them undisposed, their tool
|
|
582
995
|
// handlers registered, and their subscriptions live until the next start replaced them.
|
|
583
996
|
this.agentSessionId = session.AgentSessionId ?? this.agentSessionId;
|
|
584
|
-
this.
|
|
997
|
+
this.setConnectionState('error');
|
|
585
998
|
await this.teardown(true);
|
|
586
999
|
}
|
|
587
1000
|
/**
|
|
@@ -600,7 +1013,8 @@ export class RealtimeSessionRuntime {
|
|
|
600
1013
|
* owns that half too.
|
|
601
1014
|
*
|
|
602
1015
|
* @param result The minted session — the same ten fields the `StartRealtimeClientSession`
|
|
603
|
-
* mutation returns. `EphemeralToken` and `Provider` are what actually open the call
|
|
1016
|
+
* mutation returns. `EphemeralToken` and `Provider` are what actually open the call; a relay
|
|
1017
|
+
* session opens with its `Transport` and `RelayUrl` instead, so a host must pass those on.
|
|
604
1018
|
* @param options Host-side inputs the result cannot carry; see {@link RealtimeSessionRunOptions}.
|
|
605
1019
|
* Every field defaults exactly as its {@link StartRealtimeSession} counterpart does.
|
|
606
1020
|
*/
|
|
@@ -614,16 +1028,20 @@ export class RealtimeSessionRuntime {
|
|
|
614
1028
|
return;
|
|
615
1029
|
}
|
|
616
1030
|
const consent = this.beginSessionStart(effectiveOptions);
|
|
617
|
-
await this.runMintedSession(result, effectiveOptions.conversationId ?? null, consent);
|
|
1031
|
+
await this.runMintedSession(result, effectiveOptions.conversationId ?? null, consent, false);
|
|
618
1032
|
}
|
|
619
1033
|
/**
|
|
620
|
-
* Start prologue shared by both entry points:
|
|
621
|
-
* per-session state, and flip the session live (which is ALSO what
|
|
622
|
-
* suppress duplicate starts while the mint is still in flight — hence it
|
|
623
|
-
* after). Returns the resolved recording consent, which the mint half
|
|
624
|
-
* the run half uses to decide whether to record.
|
|
1034
|
+
* Start prologue shared by both entry points: clear the last start's failure, bind the app layer,
|
|
1035
|
+
* publish the agent name, reset per-session state, and flip the session live (which is ALSO what
|
|
1036
|
+
* makes the `IsActive` guard suppress duplicate starts while the mint is still in flight — hence it
|
|
1037
|
+
* runs before minting, not after). Returns the resolved recording consent, which the mint half
|
|
1038
|
+
* reports to the server and the run half uses to decide whether to record.
|
|
625
1039
|
*/
|
|
626
1040
|
beginSessionStart(options) {
|
|
1041
|
+
// Before this start publishes anything: hosts read LastStartError as this start's states arrive,
|
|
1042
|
+
// so from the first of them it must describe this start, not an earlier one.
|
|
1043
|
+
this.lastStartError = null;
|
|
1044
|
+
this.lastSessionError = null;
|
|
627
1045
|
// App awareness (Move 1/3/4): the application the session runs in (sources the app config
|
|
628
1046
|
// cascade + RelevantAgents → allowed-agent union) and the live app-context snapshot injected
|
|
629
1047
|
// into the companion prompt at mint. Stored so the ClientContextChannel can stream subsequent
|
|
@@ -641,7 +1059,7 @@ export class RealtimeSessionRuntime {
|
|
|
641
1059
|
}
|
|
642
1060
|
this.resetState();
|
|
643
1061
|
this._active$.next(true);
|
|
644
|
-
this.
|
|
1062
|
+
this.setConnectionState('connecting');
|
|
645
1063
|
// Resolve recording consent for this session: explicit value wins, else the per-user
|
|
646
1064
|
// persisted preference. Computed before mint so it can be reported to the server.
|
|
647
1065
|
const consent = options.recordingConsent ?? this.readPersistedRecordingConsent();
|
|
@@ -651,12 +1069,18 @@ export class RealtimeSessionRuntime {
|
|
|
651
1069
|
/**
|
|
652
1070
|
* The RUN half of a session start, shared by both entry points: consume the minted result, open
|
|
653
1071
|
* the provider connection, and go live. `inputConversationId` is the conversation the START asked
|
|
654
|
-
* for (null ⇒ "server, make me one") — the result alone can't distinguish the two.
|
|
1072
|
+
* for (null ⇒ "server, make me one") — the result alone can't distinguish the two. `cameraCheck` is
|
|
1073
|
+
* whether the host shows a camera check ({@link RealtimeSessionStartOptions.CameraCheck}).
|
|
655
1074
|
*/
|
|
656
|
-
async runMintedSession(session, inputConversationId, consent) {
|
|
1075
|
+
async runMintedSession(session, inputConversationId, consent, cameraCheck) {
|
|
657
1076
|
// Captured up front: every await below is a window in which the host can end the session.
|
|
658
1077
|
const generation = this.startGeneration;
|
|
659
1078
|
try {
|
|
1079
|
+
// Mount the channels the resolved policy puts in the session. Deliberately BEFORE the session
|
|
1080
|
+
// id is adopted: `ActiveChannels$` consumers (the sessions adapter) treat an emission that
|
|
1081
|
+
// arrives while no session id is set as the INITIAL set, whose opens they synthesize from
|
|
1082
|
+
// `SessionStarted$` — activating after the id is set would announce every channel twice.
|
|
1083
|
+
this.applySessionClientPolicy(session);
|
|
660
1084
|
this.agentSessionId = session.AgentSessionId;
|
|
661
1085
|
// A null input conversationId means the SERVER created a fresh conversation for
|
|
662
1086
|
// this session — track it so the host can fold it into the cached list, select
|
|
@@ -671,20 +1095,39 @@ export class RealtimeSessionRuntime {
|
|
|
671
1095
|
this.applyPriorChannelStates(session.PriorChannelStatesJson);
|
|
672
1096
|
const client = this.createRealtimeClient(session.Provider);
|
|
673
1097
|
this.client = client;
|
|
1098
|
+
this.watchVideoSources(client);
|
|
674
1099
|
this.wireClientHandlers(client);
|
|
675
1100
|
// Everything past here awaits on hardware and the network, during which the host may end the
|
|
676
|
-
// session. Each await is followed by a staleness check
|
|
677
|
-
// just acquired instead of leaving a live microphone and a live
|
|
678
|
-
|
|
1101
|
+
// session. Each await is followed by a staleness check, before any failure is reported, so an
|
|
1102
|
+
// abandoned start releases what it just acquired instead of leaving a live microphone and a live
|
|
1103
|
+
// call behind it, and reports nothing to a host that has ended the call.
|
|
1104
|
+
const lease = { Opening: true };
|
|
1105
|
+
this.microphoneLease = lease;
|
|
1106
|
+
const microphone = await this.openMicrophone().finally(() => {
|
|
1107
|
+
lease.Opening = false;
|
|
1108
|
+
});
|
|
1109
|
+
this.localStream = microphone.Stream;
|
|
679
1110
|
if (this.startGeneration !== generation) {
|
|
680
|
-
await this.unwindAbandonedStart(session, client);
|
|
1111
|
+
await this.unwindAbandonedStart(session, client, lease);
|
|
681
1112
|
return;
|
|
682
1113
|
}
|
|
683
|
-
|
|
1114
|
+
if (microphone.Error) {
|
|
1115
|
+
throw microphone.Error;
|
|
1116
|
+
}
|
|
1117
|
+
const clientConfig = this.BuildClientConfig(session);
|
|
1118
|
+
const connectError = await this.connectClient(client, clientConfig, microphone.Stream);
|
|
684
1119
|
if (this.startGeneration !== generation) {
|
|
685
|
-
await this.unwindAbandonedStart(session, client);
|
|
1120
|
+
await this.unwindAbandonedStart(session, client, lease);
|
|
686
1121
|
return;
|
|
687
1122
|
}
|
|
1123
|
+
if (connectError) {
|
|
1124
|
+
throw connectError;
|
|
1125
|
+
}
|
|
1126
|
+
// Tracks are negotiated now, so a capture can tell whether the model takes video, and the call whether it shows
|
|
1127
|
+
// the avatar its agent asked for. The driver holds the microphone now, so the user may switch it.
|
|
1128
|
+
this.openCaptures(client, cameraCheck);
|
|
1129
|
+
this.offerMicrophone();
|
|
1130
|
+
this.publishAvatarNotice(session, clientConfig, client);
|
|
688
1131
|
// Notify active channels that the session client is connected and tracks are established
|
|
689
1132
|
for (const channel of this._activeChannels$.value) {
|
|
690
1133
|
try {
|
|
@@ -694,6 +1137,10 @@ export class RealtimeSessionRuntime {
|
|
|
694
1137
|
console.error(`[RealtimeSession] Error in channel '${channel.ChannelName}' OnSessionStarted:`, err);
|
|
695
1138
|
}
|
|
696
1139
|
}
|
|
1140
|
+
// Tell the model which channels exist and how to use them, from the channels' own descriptors —
|
|
1141
|
+
// as soon as the control channel is usable (which may be right now, or a moment after Connect).
|
|
1142
|
+
this.catalogNotePending = true;
|
|
1143
|
+
this.flushChannelCatalogNote();
|
|
697
1144
|
// Start browser-side recording (mic + agent mix) when consented. Best-effort: an
|
|
698
1145
|
// unsupported browser / missing remote stream degrades gracefully (mic-only or off)
|
|
699
1146
|
// and never blocks the call. The remote stream may still be null here (the WebRTC
|
|
@@ -718,6 +1165,8 @@ export class RealtimeSessionRuntime {
|
|
|
718
1165
|
// SessionsObserver bridge. Emitting AFTER Connect() guarantees both that
|
|
719
1166
|
// agentSessionId is set (line ~468) AND the realtime client is connected,
|
|
720
1167
|
// so consumers can act on it without re-checking either condition.
|
|
1168
|
+
// The session's own event stream (identity verification, app events) stays open for its life.
|
|
1169
|
+
this.startSessionEvents(this.agentSessionId);
|
|
721
1170
|
this._sessionStarted$.next({
|
|
722
1171
|
sessionId: this.agentSessionId,
|
|
723
1172
|
channelNames: this._activeChannels$.value.map(c => c.ChannelName),
|
|
@@ -730,18 +1179,42 @@ export class RealtimeSessionRuntime {
|
|
|
730
1179
|
await this.failSessionStart(error);
|
|
731
1180
|
}
|
|
732
1181
|
}
|
|
1182
|
+
/**
|
|
1183
|
+
* Tells the host, once per call, why it shows no avatar ({@link AvatarNotice$}): from the mint's status, whether this
|
|
1184
|
+
* session asked for the agent's video, and whether that track is live now that the call is connected.
|
|
1185
|
+
*/
|
|
1186
|
+
publishAvatarNotice(session, config, client) {
|
|
1187
|
+
const notice = ResolveAvatarNotice(ParseRealtimeAvatarStatus(session.AvatarStatusJson), RequestsAgentVideo(config), client.IsTrackEstablished('video', 'outbound'));
|
|
1188
|
+
if (notice) {
|
|
1189
|
+
this._avatarNotice$.next(notice);
|
|
1190
|
+
}
|
|
1191
|
+
}
|
|
733
1192
|
/**
|
|
734
1193
|
* Releases everything a start acquired after the host had already ended the session.
|
|
735
1194
|
*
|
|
736
1195
|
* Reached only when {@link teardown} ran while this start was awaiting the microphone or the
|
|
737
|
-
* provider connection
|
|
738
|
-
*
|
|
739
|
-
*
|
|
1196
|
+
* provider connection, whether that wait then succeeded or failed: the host has ended the call,
|
|
1197
|
+
* so a failure there is not reported. Teardown could not release what did not exist yet, so this
|
|
1198
|
+
* start owns the cleanup: stop the microphone, hand it back to the host when teardown left that
|
|
1199
|
+
* to it ({@link microphoneLease}), close the provider connection, and close the server-side
|
|
1200
|
+
* session row that the mint created when no teardown has closed it or is closing it.
|
|
1201
|
+
*
|
|
1202
|
+
* @param lease This start's opening of the microphone.
|
|
740
1203
|
*/
|
|
741
|
-
async unwindAbandonedStart(session, client) {
|
|
1204
|
+
async unwindAbandonedStart(session, client, lease) {
|
|
742
1205
|
console.warn('[RealtimeSession] Session was ended while starting — releasing the partial session.');
|
|
1206
|
+
this.closeCaptures();
|
|
1207
|
+
this.closeLocalMedia();
|
|
743
1208
|
this.localStream?.getTracks().forEach(t => t.stop());
|
|
744
1209
|
this.localStream = null;
|
|
1210
|
+
this.microphoneMuted = false;
|
|
1211
|
+
// Teardown left the release to this start when the microphone was still opening. A cleared or
|
|
1212
|
+
// replaced lease means teardown already released it, or a newer start has opened the microphone
|
|
1213
|
+
// since and its own end releases it.
|
|
1214
|
+
if (this.microphoneLease === lease) {
|
|
1215
|
+
this.microphoneLease = null;
|
|
1216
|
+
await this.releaseMicrophone();
|
|
1217
|
+
}
|
|
745
1218
|
try {
|
|
746
1219
|
await client.Disconnect();
|
|
747
1220
|
}
|
|
@@ -751,27 +1224,321 @@ export class RealtimeSessionRuntime {
|
|
|
751
1224
|
// Only clear the shared slots when they still point at THIS attempt — a newer start may
|
|
752
1225
|
// already have replaced them.
|
|
753
1226
|
if (this.client === client) {
|
|
1227
|
+
this.unwatchVideoSources();
|
|
1228
|
+
this.clearAgentVideo();
|
|
754
1229
|
this.client = null;
|
|
755
1230
|
}
|
|
756
|
-
// Close the server session only if teardown has
|
|
757
|
-
// after closing, so a
|
|
758
|
-
//
|
|
759
|
-
// send a second `CloseAgentSession` for one
|
|
760
|
-
|
|
1231
|
+
// Close the server session only if no teardown has closed it or is closing it. Teardown nulls
|
|
1232
|
+
// `agentSessionId` after closing, so a cleared id means the teardown that invalidated this start
|
|
1233
|
+
// already closed it. A teardown still running closes it itself and then reports the end on
|
|
1234
|
+
// `SessionEnded$`: closing it here as well would send a second `CloseAgentSession` for one
|
|
1235
|
+
// session, and clearing the id would leave that teardown no session to report.
|
|
1236
|
+
if (session.AgentSessionId && this.agentSessionId === session.AgentSessionId && !this.teardownInFlight) {
|
|
761
1237
|
this.agentSessionId = null;
|
|
762
1238
|
await this.closeServerSession(session.AgentSessionId);
|
|
763
1239
|
}
|
|
764
1240
|
}
|
|
765
1241
|
/**
|
|
766
|
-
*
|
|
1242
|
+
* Opens the user's microphone: through the host's controller when it offers one, otherwise through
|
|
1243
|
+
* {@link IRealtimeMediaHost.AcquireMicrophone}. A failure is returned rather than thrown, so the caller can
|
|
1244
|
+
* first check whether the start was abandoned meanwhile (a teardown disposes the controller, which fails a
|
|
1245
|
+
* start still in flight) and unwind quietly instead of reporting an error nobody is waiting for.
|
|
1246
|
+
*/
|
|
1247
|
+
async openMicrophone() {
|
|
1248
|
+
const controller = this.mediaHost.CreateLocalMediaController?.() ?? null;
|
|
1249
|
+
if (!controller) {
|
|
1250
|
+
try {
|
|
1251
|
+
return { Stream: await this.mediaHost.AcquireMicrophone(), Error: null };
|
|
1252
|
+
}
|
|
1253
|
+
catch (error) {
|
|
1254
|
+
return { Stream: null, Error: error instanceof Error ? error : new Error(String(error)) };
|
|
1255
|
+
}
|
|
1256
|
+
}
|
|
1257
|
+
this.localMedia = controller;
|
|
1258
|
+
const started = await controller.Start('microphone');
|
|
1259
|
+
if (started.Status === 'failed') {
|
|
1260
|
+
return { Stream: null, Error: microphoneStartError(started.Reason, started.Message) };
|
|
1261
|
+
}
|
|
1262
|
+
this.followMicrophone(controller, started.Stream);
|
|
1263
|
+
return { Stream: started.Stream, Error: null };
|
|
1264
|
+
}
|
|
1265
|
+
/**
|
|
1266
|
+
* Opens the provider connection. As with {@link openMicrophone}, a failure is returned rather than thrown, so the
|
|
1267
|
+
* caller can first check whether the start was abandoned meanwhile and unwind quietly instead of reporting an error
|
|
1268
|
+
* nobody is waiting for: a teardown disconnects the client, which with most drivers fails a connection still opening.
|
|
1269
|
+
*/
|
|
1270
|
+
async connectClient(client, config, microphone) {
|
|
1271
|
+
try {
|
|
1272
|
+
await client.Connect(config, microphone);
|
|
1273
|
+
return null;
|
|
1274
|
+
}
|
|
1275
|
+
catch (error) {
|
|
1276
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
1277
|
+
}
|
|
1278
|
+
}
|
|
1279
|
+
/**
|
|
1280
|
+
* Moves the driver and the recorder onto the call's microphone whenever the controller gives it a new track. A device
|
|
1281
|
+
* switch, or a lost device replaced by the default, swaps a track into the same stream, and the controller carries the
|
|
1282
|
+
* old track's mute over. A microphone opened again after the call lost one comes in a new stream: its track takes the
|
|
1283
|
+
* call's mute, and the stream becomes the one {@link ToggleMute} and the teardown act on. Mid-swap the stream holds no
|
|
1284
|
+
* track, so only a finished swap is followed. Every report also keeps {@link Microphone$} current once the call offers
|
|
1285
|
+
* it.
|
|
1286
|
+
*/
|
|
1287
|
+
followMicrophone(controller, stream) {
|
|
1288
|
+
let followedStream = stream;
|
|
1289
|
+
let followedTrack = stream.getAudioTracks()[0] ?? null;
|
|
1290
|
+
this.localMediaSubscription = controller.State$.subscribe((media) => {
|
|
1291
|
+
const current = controller.GetStream('microphone') ?? followedStream;
|
|
1292
|
+
const track = current.getAudioTracks()[0] ?? null;
|
|
1293
|
+
if (track && track !== followedTrack) {
|
|
1294
|
+
if (current !== followedStream) {
|
|
1295
|
+
track.enabled = !this.microphoneMuted;
|
|
1296
|
+
this.localStream = current;
|
|
1297
|
+
followedStream = current;
|
|
1298
|
+
}
|
|
1299
|
+
followedTrack = track;
|
|
1300
|
+
this.recorder?.ReplaceMicrophone?.(current);
|
|
1301
|
+
this.client?.ReplaceMicrophone?.(current)?.catch((error) => {
|
|
1302
|
+
console.error('[RealtimeSession] The realtime driver could not move to the new microphone:', error);
|
|
1303
|
+
});
|
|
1304
|
+
}
|
|
1305
|
+
this.publishMicrophone(media);
|
|
1306
|
+
});
|
|
1307
|
+
}
|
|
1308
|
+
/** The client is connected: the call offers its microphone on {@link Microphone$}, and {@link SwitchMicrophone} moves it. */
|
|
1309
|
+
offerMicrophone() {
|
|
1310
|
+
if (!this.localMedia) {
|
|
1311
|
+
return;
|
|
1312
|
+
}
|
|
1313
|
+
this.microphoneOffered = true;
|
|
1314
|
+
this.publishMicrophone(this.localMedia.State);
|
|
1315
|
+
}
|
|
1316
|
+
/** Publishes the call's microphone from the controller's state while the call offers it, when it names a change. */
|
|
1317
|
+
publishMicrophone(media) {
|
|
1318
|
+
if (!this.microphoneOffered) {
|
|
1319
|
+
return;
|
|
1320
|
+
}
|
|
1321
|
+
const current = this._microphone$.value;
|
|
1322
|
+
const next = ReadMicrophoneState(media, current);
|
|
1323
|
+
if (!SameMicrophoneState(next, current)) {
|
|
1324
|
+
this._microphone$.next(next);
|
|
1325
|
+
}
|
|
1326
|
+
}
|
|
1327
|
+
/** Stops following the controller and disposes it, which releases its devices; the call no longer offers a microphone. */
|
|
1328
|
+
closeLocalMedia() {
|
|
1329
|
+
this.localMediaSubscription?.unsubscribe();
|
|
1330
|
+
this.localMediaSubscription = null;
|
|
1331
|
+
this.localMedia?.Dispose();
|
|
1332
|
+
this.localMedia = null;
|
|
1333
|
+
this.microphoneOffered = false;
|
|
1334
|
+
if (this._microphone$.value !== REALTIME_MICROPHONE_NONE) {
|
|
1335
|
+
this._microphone$.next(REALTIME_MICROPHONE_NONE);
|
|
1336
|
+
}
|
|
1337
|
+
}
|
|
1338
|
+
/**
|
|
1339
|
+
* Creates the session's camera and screen share and mirrors their state on {@link Captures$}. With `cameraCheck`, the
|
|
1340
|
+
* call's first camera start waits for {@link ConfirmCamera}.
|
|
1341
|
+
*/
|
|
1342
|
+
openCaptures(client, cameraCheck) {
|
|
1343
|
+
this.closeCaptures();
|
|
1344
|
+
const captures = new RealtimeCaptures({
|
|
1345
|
+
Client: client,
|
|
1346
|
+
LocalMedia: this.localMedia,
|
|
1347
|
+
Host: this.mediaHost,
|
|
1348
|
+
CreateSampler: this.hostFrameSampler(),
|
|
1349
|
+
Admit: (kind) => this.admitCapture(kind),
|
|
1350
|
+
CameraCheck: cameraCheck,
|
|
1351
|
+
});
|
|
1352
|
+
this.captures = captures;
|
|
1353
|
+
this.capturesSubscription = captures.States$.subscribe((states) => this._captures$.next(states));
|
|
1354
|
+
this.refreshCaptureOffers();
|
|
1355
|
+
}
|
|
1356
|
+
/**
|
|
1357
|
+
* The host's frame sampler for the captures ({@link IRealtimeMediaHost.CreateFrameSampler}), or `undefined` when it offers
|
|
1358
|
+
* none and the captures use the DOM default.
|
|
1359
|
+
*/
|
|
1360
|
+
hostFrameSampler() {
|
|
1361
|
+
const host = this.mediaHost;
|
|
1362
|
+
const create = host.CreateFrameSampler;
|
|
1363
|
+
return create ? (stream, options) => create.call(host, stream, options) : undefined;
|
|
1364
|
+
}
|
|
1365
|
+
/** Stops the camera and screen share, if any, and reports both off. */
|
|
1366
|
+
closeCaptures() {
|
|
1367
|
+
this.capturesSubscription?.unsubscribe();
|
|
1368
|
+
this.capturesSubscription = null;
|
|
1369
|
+
this.captures?.Dispose();
|
|
1370
|
+
this.captures = null;
|
|
1371
|
+
if (this._captures$.value !== REALTIME_CAPTURES_OFF) {
|
|
1372
|
+
this._captures$.next(REALTIME_CAPTURES_OFF);
|
|
1373
|
+
}
|
|
1374
|
+
this.refreshCaptureOffers();
|
|
1375
|
+
}
|
|
1376
|
+
/** Works out which captures the call offers ({@link CaptureOffers$}) and publishes a change. */
|
|
1377
|
+
refreshCaptureOffers() {
|
|
1378
|
+
const offers = this.captures && this.client?.SupportsInboundVideo ? { Camera: this.canOffer('camera'), Screen: this.canOffer('screen') } : REALTIME_CAPTURE_OFFERS_NONE;
|
|
1379
|
+
const current = this._captureOffers$.value;
|
|
1380
|
+
if (offers.Camera !== current.Camera || offers.Screen !== current.Screen) {
|
|
1381
|
+
this._captureOffers$.next(offers);
|
|
1382
|
+
}
|
|
1383
|
+
}
|
|
1384
|
+
/** Whether the host can open a capture and the call's policy admits it. */
|
|
1385
|
+
canOffer(kind) {
|
|
1386
|
+
const hostCanOpen = kind === 'camera' ? this.localMedia !== null : typeof this.mediaHost.RequestDisplayCapture === 'function';
|
|
1387
|
+
return hostCanOpen && this.admitCapture(kind).Admitted;
|
|
1388
|
+
}
|
|
1389
|
+
/** What a capture reports when there is no live session to show it to. */
|
|
1390
|
+
noSessionCapture() {
|
|
1391
|
+
return { Status: 'failed', Failure: 'no-session', Message: 'There is no call to share with.' };
|
|
1392
|
+
}
|
|
1393
|
+
/**
|
|
1394
|
+
* The session's policy for a capture. The channel that fronts it (its `CaptureKind`) must be in the session, open or
|
|
1395
|
+
* advertised, and the server's policy must let the agent see pixels through it (the agent's configuration and any
|
|
1396
|
+
* zero-data-retention rule). The user's own "agent can see" choice for the channel does not refuse a start: it decides
|
|
1397
|
+
* whether the frames reach the model.
|
|
1398
|
+
*/
|
|
1399
|
+
admitCapture(kind) {
|
|
1400
|
+
const channel = this.findCaptureChannel(kind)?.Plugin;
|
|
1401
|
+
if (!channel) {
|
|
1402
|
+
return { Admitted: false, Message: CAPTURE_WORDING[kind].NotInCall };
|
|
1403
|
+
}
|
|
1404
|
+
const resolved = this.GetResolvedChannel(channel.ChannelName);
|
|
1405
|
+
const allowed = resolved?.Exposure ?? resolved?.MaxExposure ?? channel.GetDescriptor().MaxExposure;
|
|
1406
|
+
if (allowed !== 'pixels') {
|
|
1407
|
+
const reasons = (resolved?.ExposureLimits ?? []).map((limit) => limit.Reason);
|
|
1408
|
+
return { Admitted: false, Message: reasons.length > 0 ? `${CAPTURE_WORDING[kind].NotAllowed}: ${reasons.join('; ')}.` : `${CAPTURE_WORDING[kind].NotAllowed}.` };
|
|
1409
|
+
}
|
|
1410
|
+
return { Admitted: true, ChannelKey: channel.ChannelName, VisibleToAgent: channel.Exposure === 'pixels' };
|
|
1411
|
+
}
|
|
1412
|
+
/** The channel in the session, open or advertised, that fronts a capture. */
|
|
1413
|
+
findCaptureChannel(kind) {
|
|
1414
|
+
const open = this._activeChannels$.value.find((c) => c.CaptureKind === kind);
|
|
1415
|
+
if (open) {
|
|
1416
|
+
return { Plugin: open, IsOpen: true };
|
|
1417
|
+
}
|
|
1418
|
+
const advertised = this.advertisedChannels.find((p) => p.Plugin.CaptureKind === kind);
|
|
1419
|
+
return advertised ? { Plugin: advertised.Plugin, IsOpen: false } : null;
|
|
1420
|
+
}
|
|
1421
|
+
/**
|
|
1422
|
+
* Brings up the channel that fronts a capture once the capture is starting, so the host shows its surface: an advertised
|
|
1423
|
+
* channel is opened (and announces itself to the agent), an open one counts as used.
|
|
1424
|
+
*/
|
|
1425
|
+
async revealCaptureChannel(kind) {
|
|
1426
|
+
const state = kind === 'camera' ? this.captures?.States.Camera : this.captures?.States.Screen;
|
|
1427
|
+
const channel = state?.Status === 'starting' ? this.findCaptureChannel(kind) : null;
|
|
1428
|
+
if (!channel) {
|
|
1429
|
+
return;
|
|
1430
|
+
}
|
|
1431
|
+
if (channel.IsOpen) {
|
|
1432
|
+
this.noteChannelActivity(channel.Plugin);
|
|
1433
|
+
return;
|
|
1434
|
+
}
|
|
1435
|
+
const opened = await this.OpenChannel(channel.Plugin.ChannelName);
|
|
1436
|
+
if (!opened.Success) {
|
|
1437
|
+
console.warn(`[RealtimeSession] Could not open channel '${channel.Plugin.ChannelName}' for the ${kind}: ${opened.ErrorMessage ?? opened.ErrorCode}`);
|
|
1438
|
+
}
|
|
1439
|
+
}
|
|
1440
|
+
/**
|
|
1441
|
+
* Starts the user's camera and shows it to the agent. Call it from the user's click: it may ask for camera
|
|
1442
|
+
* permission. Resolves with the camera's state; a failure (no session, a model that takes no video, a refused
|
|
1443
|
+
* permission) is a state with a message, never a throw. While the camera is on, it is a source on
|
|
1444
|
+
* {@link VideoSources$}, and it stops by itself if the device goes away.
|
|
1445
|
+
*
|
|
1446
|
+
* When the host shows a camera check ({@link RealtimeSessionStartOptions.CameraCheck}), the call's first start
|
|
1447
|
+
* resolves still starting, with `Checking` set and the stream for the host to preview. The agent sees the camera
|
|
1448
|
+
* once the host calls {@link ConfirmCamera}; {@link StopCamera} is the user's "not now".
|
|
1449
|
+
*
|
|
1450
|
+
* @param deviceId The camera to open; the system default when absent.
|
|
1451
|
+
*/
|
|
1452
|
+
async StartCamera(deviceId) {
|
|
1453
|
+
if (!this.captures) {
|
|
1454
|
+
return this.noSessionCapture();
|
|
1455
|
+
}
|
|
1456
|
+
const started = this.captures.Start('camera', { DeviceID: deviceId });
|
|
1457
|
+
await this.revealCaptureChannel('camera');
|
|
1458
|
+
return started;
|
|
1459
|
+
}
|
|
1460
|
+
/** Stops the user's camera. Safe to call when it is off. */
|
|
1461
|
+
StopCamera() {
|
|
1462
|
+
this.captures?.Stop('camera');
|
|
1463
|
+
}
|
|
1464
|
+
/**
|
|
1465
|
+
* Moves the user's camera to another device while it is open (being checked or on): one of the camera's `Devices` on
|
|
1466
|
+
* {@link Captures$}. The stream stays the same. Resolves with the camera's state; nothing changes while the camera is
|
|
1467
|
+
* off.
|
|
1468
|
+
*
|
|
1469
|
+
* @param deviceId The camera to move to.
|
|
1470
|
+
*/
|
|
1471
|
+
async SwitchCamera(deviceId) {
|
|
1472
|
+
return this.captures ? this.captures.SwitchCamera(deviceId) : this.noSessionCapture();
|
|
1473
|
+
}
|
|
1474
|
+
/**
|
|
1475
|
+
* Moves the call's microphone to another device: one of the `Devices` on {@link Microphone$}. The host's controller
|
|
1476
|
+
* swaps the new track into the same stream and carries the mute over, and the driver and the call's recording move to
|
|
1477
|
+
* it, so the agent keeps hearing the user. When the new microphone cannot open, the controller goes back to the one in
|
|
1478
|
+
* use. While the call has no microphone open (it lost the one in use and the default, or the last one went away), the
|
|
1479
|
+
* pick opens one again: the driver and the recording move to it, and it keeps the call's mute. Resolves with the
|
|
1480
|
+
* microphone's state once the switch is over. Nothing changes outside a call or on a host without a controller.
|
|
1481
|
+
*
|
|
1482
|
+
* @param deviceId The microphone to move to.
|
|
1483
|
+
*/
|
|
1484
|
+
async SwitchMicrophone(deviceId) {
|
|
1485
|
+
const controller = this.localMedia;
|
|
1486
|
+
if (controller && this.microphoneOffered) {
|
|
1487
|
+
const status = controller.State.Microphone.Status;
|
|
1488
|
+
const live = status === 'on' || status === 'starting';
|
|
1489
|
+
await (live ? controller.SwitchDevice('microphone', deviceId) : controller.Start('microphone', deviceId));
|
|
1490
|
+
}
|
|
1491
|
+
return this._microphone$.value;
|
|
1492
|
+
}
|
|
1493
|
+
/**
|
|
1494
|
+
* The user checked the camera and turned it on: the agent sees it from now on, and later camera starts in this call
|
|
1495
|
+
* skip the check. Returns the camera's state; unless the camera is waiting for its check (see {@link StartCamera}),
|
|
1496
|
+
* nothing changes.
|
|
1497
|
+
*/
|
|
1498
|
+
ConfirmCamera() {
|
|
1499
|
+
return this.captures ? this.captures.ConfirmCamera() : this.noSessionCapture();
|
|
1500
|
+
}
|
|
1501
|
+
/**
|
|
1502
|
+
* Asks the user for a screen, window or browser tab (or one panel of the page) and shows it to the agent. Call it
|
|
1503
|
+
* from the user's click: the browser's picker needs one. Resolves with the share's state; a failure (no session,
|
|
1504
|
+
* a model that takes no video, a closed picker, a host that cannot share) is a state with a message. The share
|
|
1505
|
+
* also stops when the user ends it from the browser's own bar.
|
|
1506
|
+
*
|
|
1507
|
+
* @param options What the picker offers first, or the panel to share and its name (`PanelLabel`), which the share's
|
|
1508
|
+
* state carries and the agent sees as "<name> (shared panel)".
|
|
1509
|
+
*/
|
|
1510
|
+
async StartScreenShare(options) {
|
|
1511
|
+
if (!this.captures) {
|
|
1512
|
+
return this.noSessionCapture();
|
|
1513
|
+
}
|
|
1514
|
+
const started = this.captures.Start('screen', options);
|
|
1515
|
+
await this.revealCaptureChannel('screen');
|
|
1516
|
+
return started;
|
|
1517
|
+
}
|
|
1518
|
+
/** Stops the user's screen share. Safe to call when nothing is shared. */
|
|
1519
|
+
StopScreenShare() {
|
|
1520
|
+
this.captures?.Stop('screen');
|
|
1521
|
+
}
|
|
1522
|
+
/**
|
|
1523
|
+
* Why the latest session start failed, or `null` when it has not failed or none has run.
|
|
767
1524
|
*
|
|
768
1525
|
* Read it when {@link ConnectionState$} reports `'error'`, to tell a denied microphone apart from
|
|
769
|
-
* a provider or backend failure and show copy the user can act on.
|
|
770
|
-
*
|
|
1526
|
+
* a provider or backend failure and show copy the user can act on. Each start clears it before it
|
|
1527
|
+
* reports `'connecting'`, so it never describes an earlier start.
|
|
771
1528
|
*/
|
|
772
1529
|
get LastStartError() {
|
|
773
1530
|
return this.lastStartError;
|
|
774
1531
|
}
|
|
1532
|
+
/**
|
|
1533
|
+
* The fatal error the call's client reported, or `null` when it has reported none.
|
|
1534
|
+
*
|
|
1535
|
+
* Read it when {@link ConnectionState$} reports `'error'` on a call that was live, to tell a call that reached its time
|
|
1536
|
+
* limit (`Code` `'time-limit'`; {@link DescribeCallEnd} words it) from a dropped connection. Each start clears it before
|
|
1537
|
+
* it reports `'connecting'`.
|
|
1538
|
+
*/
|
|
1539
|
+
get LastSessionError() {
|
|
1540
|
+
return this.lastSessionError;
|
|
1541
|
+
}
|
|
775
1542
|
/**
|
|
776
1543
|
* The single failure path for a session start (mint half or run half): report it, latch the
|
|
777
1544
|
* overlay into 'error', and unwind whatever the half-built session already opened.
|
|
@@ -779,7 +1546,7 @@ export class RealtimeSessionRuntime {
|
|
|
779
1546
|
async failSessionStart(error) {
|
|
780
1547
|
console.error('[RealtimeSession] Failed to start session:', error);
|
|
781
1548
|
this.lastStartError = error instanceof Error ? error : new Error(String(error));
|
|
782
|
-
this.
|
|
1549
|
+
this.setConnectionState('error');
|
|
783
1550
|
await this.teardown(false);
|
|
784
1551
|
}
|
|
785
1552
|
/**
|
|
@@ -818,14 +1585,19 @@ export class RealtimeSessionRuntime {
|
|
|
818
1585
|
// Relay as a user turn — same path spoken input uses (caption + persisted transcript).
|
|
819
1586
|
void this.onUserTranscript(trimmed);
|
|
820
1587
|
}
|
|
821
|
-
/**
|
|
1588
|
+
/**
|
|
1589
|
+
* Mute / unmute the local microphone track. Returns the new muted state. While the call has no microphone track (a
|
|
1590
|
+
* switch is putting a new one in, or the call lost its microphone) there is nothing to mute: nothing changes, and it
|
|
1591
|
+
* returns the call's mute, which the next microphone keeps. Outside a call it returns `false`.
|
|
1592
|
+
*/
|
|
822
1593
|
ToggleMute() {
|
|
823
1594
|
const tracks = this.localStream?.getAudioTracks() ?? [];
|
|
824
1595
|
if (tracks.length === 0) {
|
|
825
|
-
return
|
|
1596
|
+
return this.microphoneMuted;
|
|
826
1597
|
}
|
|
827
1598
|
const muted = tracks[0].enabled; // currently enabled → becomes muted
|
|
828
1599
|
this.client?.SetMuted(muted);
|
|
1600
|
+
this.microphoneMuted = muted;
|
|
829
1601
|
return muted;
|
|
830
1602
|
}
|
|
831
1603
|
// ── Client-executed UI tools ───────────────────────────────────────────────
|
|
@@ -895,13 +1667,21 @@ export class RealtimeSessionRuntime {
|
|
|
895
1667
|
return this.client;
|
|
896
1668
|
}
|
|
897
1669
|
/**
|
|
898
|
-
* Relays a video frame to the underlying realtime client if
|
|
1670
|
+
* Relays a video frame straight to the underlying realtime client, if a session is live.
|
|
1671
|
+
*
|
|
1672
|
+
* It bypasses the session's {@link VideoSourceArbiter}: the frame overrides the source the arbiter picked, and the
|
|
1673
|
+
* model is not told what it is looking at. Channels no longer send through it (their context's `SendVideoFrame` goes
|
|
1674
|
+
* through the arbiter); a host that shows the model something registers a source there instead, as
|
|
1675
|
+
* `ChannelInboundVideoBridge` does.
|
|
1676
|
+
*
|
|
1677
|
+
* @returns `true` when the client accepted and sent the frame; `false` when no session is live, the
|
|
1678
|
+
* client cannot take video frames, or the client dropped the frame.
|
|
899
1679
|
*/
|
|
900
1680
|
SendVideoFrame(base64Image, mimeType) {
|
|
901
1681
|
if (!this.client || !this.isSessionLive()) {
|
|
902
|
-
return;
|
|
1682
|
+
return false;
|
|
903
1683
|
}
|
|
904
|
-
this.client.SendVideoFrame?.(base64Image, mimeType);
|
|
1684
|
+
return this.client.SendVideoFrame?.(base64Image, mimeType) === true;
|
|
905
1685
|
}
|
|
906
1686
|
/**
|
|
907
1687
|
* Checks whether a media track is established on the active realtime client.
|
|
@@ -949,9 +1729,11 @@ export class RealtimeSessionRuntime {
|
|
|
949
1729
|
this.currentTurnStartMs = recorder.IsRecording ? 0 : null;
|
|
950
1730
|
this.turnAudioStartCaptured = false;
|
|
951
1731
|
if (this.recorder) {
|
|
952
|
-
//
|
|
953
|
-
// above is
|
|
954
|
-
//
|
|
1732
|
+
// WebRTC drivers (OpenAI): the agent's track usually lands AFTER Connect() resolves, so
|
|
1733
|
+
// `remoteStream` above is null here and this handler attaches it later. PCM-playback
|
|
1734
|
+
// drivers (Gemini, ElevenLabs, AssemblyAI, xAI, HuggingFace) publish at Connect, so
|
|
1735
|
+
// `remoteStream` is already set and the handler fires immediately with the same stream;
|
|
1736
|
+
// AttachRemoteStream is idempotent, so it is mixed only once.
|
|
955
1737
|
client.OnRemoteMediaStream?.((stream) => this.recorder?.AttachRemoteStream(stream));
|
|
956
1738
|
this.startSegmentFlushing();
|
|
957
1739
|
}
|
|
@@ -963,7 +1745,7 @@ export class RealtimeSessionRuntime {
|
|
|
963
1745
|
}
|
|
964
1746
|
/** Begins flushing ~15s crash-recovery shards to the server for the duration of the recording. */
|
|
965
1747
|
startSegmentFlushing() {
|
|
966
|
-
this.
|
|
1748
|
+
this.shardQueue = new RecordingShardQueue();
|
|
967
1749
|
this.segmentTimer = setInterval(() => { void this.flushRecordingSegment(); }, RealtimeSessionRuntime.segmentFlushMs);
|
|
968
1750
|
}
|
|
969
1751
|
/** Stops the periodic crash-recovery shard flush. */
|
|
@@ -1030,48 +1812,64 @@ export class RealtimeSessionRuntime {
|
|
|
1030
1812
|
}
|
|
1031
1813
|
/**
|
|
1032
1814
|
* Uploads the chunks captured since the last flush as one crash-recovery shard (durability only;
|
|
1033
|
-
* the canonical file is still the full upload at teardown). Best-effort — never disturbs the call
|
|
1815
|
+
* the canonical file is still the full upload at teardown). Best-effort — never disturbs the call:
|
|
1816
|
+
* a failed shard is retained by the queue and resent under its own index on later flushes, up to a
|
|
1817
|
+
* bounded number of attempts (#5197).
|
|
1034
1818
|
*/
|
|
1035
1819
|
async flushRecordingSegment() {
|
|
1036
1820
|
const recorder = this.recorder;
|
|
1037
1821
|
const agentSessionId = this.agentSessionId;
|
|
1038
|
-
|
|
1822
|
+
const queue = this.shardQueue;
|
|
1823
|
+
if (!recorder || !agentSessionId || !queue) {
|
|
1039
1824
|
return;
|
|
1040
1825
|
}
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
}
|
|
1046
|
-
const index = this.segmentIndex++;
|
|
1047
|
-
const mutation = `
|
|
1048
|
-
mutation UploadRealtimeRecordingSegment($agentSessionId: String!, $segmentIndex: Int!, $audioBase64: String!, $mimeType: String!) {
|
|
1049
|
-
UploadRealtimeRecordingSegment(agentSessionId: $agentSessionId, segmentIndex: $segmentIndex, audioBase64: $audioBase64, mimeType: $mimeType)
|
|
1050
|
-
}
|
|
1051
|
-
`;
|
|
1052
|
-
// Shards are HEADER-LESS raw little-endian PCM16 (mime audio/L16 with the capture sample rate),
|
|
1053
|
-
// NOT individually-playable WAV — recovery is concatenate-in-order then WAV-wrap. The canonical
|
|
1054
|
-
// seekable WAV is the consolidated end-of-call upload below.
|
|
1055
|
-
const shardMime = `audio/L16;rate=${recorder.SampleRate}`;
|
|
1056
|
-
await this.gql().ExecuteGQL(mutation, { agentSessionId, segmentIndex: index, audioBase64, mimeType: shardMime });
|
|
1057
|
-
}
|
|
1058
|
-
catch (error) {
|
|
1059
|
-
console.warn('[RealtimeSession] Failed to flush recording shard:', error);
|
|
1060
|
-
}
|
|
1826
|
+
// Read once: a shard's sample rate is fixed at capture, and the recorder may be stopped by the
|
|
1827
|
+
// time a retry of it runs.
|
|
1828
|
+
const sampleRate = recorder.SampleRate;
|
|
1829
|
+
await queue.Flush(() => recorder.SnapshotNewSegmentBase64(), (segmentIndex, audioBase64) => this.uploadRecordingShard(agentSessionId, sampleRate, segmentIndex, audioBase64));
|
|
1061
1830
|
}
|
|
1062
1831
|
/**
|
|
1063
|
-
*
|
|
1064
|
-
*
|
|
1065
|
-
*
|
|
1832
|
+
* Sends one shard and reports whether the server stored it. The resolver reports a failed
|
|
1833
|
+
* ownership load or storage write as `false`, not as an error — that is how #5197 lost a shard
|
|
1834
|
+
* without a single client-side log line — so the typed result must be checked, not just the absence
|
|
1835
|
+
* of a throw.
|
|
1066
1836
|
*/
|
|
1067
|
-
async
|
|
1837
|
+
async uploadRecordingShard(agentSessionId, sampleRate, segmentIndex, audioBase64) {
|
|
1838
|
+
const mutation = `
|
|
1839
|
+
mutation UploadRealtimeRecordingSegment($agentSessionId: String!, $segmentIndex: Int!, $audioBase64: String!, $mimeType: String!) {
|
|
1840
|
+
UploadRealtimeRecordingSegment(agentSessionId: $agentSessionId, segmentIndex: $segmentIndex, audioBase64: $audioBase64, mimeType: $mimeType)
|
|
1841
|
+
}
|
|
1842
|
+
`;
|
|
1843
|
+
// Shards are HEADER-LESS raw little-endian PCM16 (mime audio/L16 with the capture sample rate),
|
|
1844
|
+
// NOT individually-playable WAV — recovery is concatenate-in-order then WAV-wrap. The canonical
|
|
1845
|
+
// seekable WAV is the consolidated end-of-call upload below.
|
|
1846
|
+
const shardMime = `audio/L16;rate=${sampleRate}`;
|
|
1847
|
+
const result = (await this.gql().ExecuteGQL(mutation, {
|
|
1848
|
+
agentSessionId,
|
|
1849
|
+
segmentIndex,
|
|
1850
|
+
audioBase64,
|
|
1851
|
+
mimeType: shardMime,
|
|
1852
|
+
}));
|
|
1853
|
+
return result?.UploadRealtimeRecordingSegment === true;
|
|
1854
|
+
}
|
|
1855
|
+
/**
|
|
1856
|
+
* Stops the active recorder and encodes the captured audio. Split from the upload so teardown can
|
|
1857
|
+
* close the server session in between (#5195). Best-effort: returns `null` when nothing was
|
|
1858
|
+
* recorded or the stop failed (warned) — it must NEVER block teardown.
|
|
1859
|
+
*/
|
|
1860
|
+
async stopRecording() {
|
|
1068
1861
|
this.stopSegmentFlushing();
|
|
1862
|
+
// Close the queue with the recording: its retained shards (base64 audio) are released now, not at
|
|
1863
|
+
// the next session start, and a flush still in flight stops sending, so no resend lands after the
|
|
1864
|
+
// end-of-call upload below has deleted the shards.
|
|
1865
|
+
this.shardQueue?.Close();
|
|
1866
|
+
this.shardQueue = null;
|
|
1069
1867
|
const recorder = this.recorder;
|
|
1070
1868
|
this.recorder = null;
|
|
1071
1869
|
this.currentTurnStartMs = null;
|
|
1072
1870
|
this.turnAudioStartCaptured = false;
|
|
1073
1871
|
if (!recorder) {
|
|
1074
|
-
return;
|
|
1872
|
+
return null;
|
|
1075
1873
|
}
|
|
1076
1874
|
try {
|
|
1077
1875
|
// Capture the recorder MIME (now 'audio/wav') BEFORE Stop() — the getter reads '' once stopped.
|
|
@@ -1079,18 +1877,39 @@ export class RealtimeSessionRuntime {
|
|
|
1079
1877
|
const audioBase64 = await recorder.StopAndEncode();
|
|
1080
1878
|
// Read the real waveform peaks computed during capture (survives the stop via the snapshot).
|
|
1081
1879
|
const peaks = recorder.GetPeaks();
|
|
1082
|
-
if (!audioBase64
|
|
1083
|
-
console.warn('[RealtimeSession] ⚠️ recording NOT uploaded — empty recording
|
|
1084
|
-
return;
|
|
1880
|
+
if (!audioBase64) {
|
|
1881
|
+
console.warn('[RealtimeSession] ⚠️ recording NOT uploaded — empty recording.');
|
|
1882
|
+
return null;
|
|
1085
1883
|
}
|
|
1086
|
-
|
|
1884
|
+
return { MimeType: mimeType, AudioBase64: audioBase64, Peaks: peaks };
|
|
1885
|
+
}
|
|
1886
|
+
catch (error) {
|
|
1887
|
+
console.warn('[RealtimeSession] Failed to stop call recording:', error);
|
|
1888
|
+
return null;
|
|
1889
|
+
}
|
|
1890
|
+
}
|
|
1891
|
+
/**
|
|
1892
|
+
* Uploads a recording returned by {@link stopRecording}. Never throws. No-op when there is
|
|
1893
|
+
* nothing to upload; warns when there is no session id to attach the file to.
|
|
1894
|
+
*/
|
|
1895
|
+
async uploadStoppedRecording(agentSessionId, recording) {
|
|
1896
|
+
if (!recording) {
|
|
1897
|
+
return;
|
|
1898
|
+
}
|
|
1899
|
+
if (!agentSessionId) {
|
|
1900
|
+
console.warn('[RealtimeSession] ⚠️ recording NOT uploaded — no session id.');
|
|
1901
|
+
return;
|
|
1902
|
+
}
|
|
1903
|
+
try {
|
|
1904
|
+
await this.uploadRecording(agentSessionId, recording.AudioBase64, recording.MimeType, recording.Peaks);
|
|
1087
1905
|
}
|
|
1088
1906
|
catch (error) {
|
|
1089
|
-
console.warn(
|
|
1907
|
+
console.warn(`[RealtimeSession] Failed to upload call recording for session ${agentSessionId}:`, error);
|
|
1090
1908
|
}
|
|
1091
1909
|
}
|
|
1092
1910
|
/**
|
|
1093
|
-
* Runs the `UploadRealtimeRecording` mutation
|
|
1911
|
+
* Runs the `UploadRealtimeRecording` mutation. A `Success:false` payload is logged; a transport
|
|
1912
|
+
* rejection propagates to `uploadStoppedRecording`, which catches and logs it. Sends the
|
|
1094
1913
|
* capture-time waveform `peaks` (max-abs per bucket, normalized 0..1) so the server can persist a
|
|
1095
1914
|
* `peaks.json` sidecar for fast waveform rendering without re-decoding the audio.
|
|
1096
1915
|
*/
|
|
@@ -1116,46 +1935,377 @@ export class RealtimeSessionRuntime {
|
|
|
1116
1935
|
console.warn(`[RealtimeSession] ❌ recording upload reported failure: ${payload?.ErrorMessage ?? 'unknown error'} (full result: ${JSON.stringify(result)})`);
|
|
1117
1936
|
}
|
|
1118
1937
|
}
|
|
1119
|
-
// ── Interactive channels (registry-driven plugins) ─────────────────────────
|
|
1120
1938
|
/**
|
|
1121
|
-
*
|
|
1122
|
-
* the `
|
|
1123
|
-
*
|
|
1124
|
-
*
|
|
1125
|
-
*
|
|
1939
|
+
* The resolved behavior of an in-session channel (display, exposure ceiling, config, what put it
|
|
1940
|
+
* in the session), or `null` when the channel is not in this session. Hosts and overlays read the
|
|
1941
|
+
* resolved display from here rather than from the channel's code default.
|
|
1942
|
+
*
|
|
1943
|
+
* @param channelName The channel's key (case-insensitive).
|
|
1944
|
+
*/
|
|
1945
|
+
GetResolvedChannel(channelName) {
|
|
1946
|
+
return this.resolvedChannels.get(NormalizeChannelKey(channelName)) ?? null;
|
|
1947
|
+
}
|
|
1948
|
+
/**
|
|
1949
|
+
* Replaces the store the user's per-channel exposure choices are kept in. The default persists them in
|
|
1950
|
+
* the signed-in user's settings (`mj.realtime.visualPerception.v1`) and keeps them in memory for an
|
|
1951
|
+
* anonymous principal or a connect-only embed; a host that remembers them somewhere else supplies its own.
|
|
1952
|
+
*/
|
|
1953
|
+
SetExposurePreferences(preferences) {
|
|
1954
|
+
this.exposurePreferences = preferences;
|
|
1955
|
+
}
|
|
1956
|
+
/** The store the user's per-channel exposure choices are currently kept in (so a host can wrap it). */
|
|
1957
|
+
get ExposurePreferences() {
|
|
1958
|
+
return this.exposurePreferences;
|
|
1959
|
+
}
|
|
1960
|
+
/** Whether user settings can be written: a signed-in user on a provider that has entity metadata. */
|
|
1961
|
+
canPersistUserSettings() {
|
|
1962
|
+
const provider = this.Provider;
|
|
1963
|
+
return Boolean(provider?.CurrentUser?.ID) && (provider?.Entities?.length ?? 0) > 0;
|
|
1964
|
+
}
|
|
1965
|
+
/**
|
|
1966
|
+
* How much of a channel the model may perceive right now: the channel's ceiling, lowered by the server's
|
|
1967
|
+
* policy and the user's choice. `null` when the channel is not in this session.
|
|
1968
|
+
*
|
|
1969
|
+
* @param channelName The channel's key (case-insensitive).
|
|
1970
|
+
*/
|
|
1971
|
+
GetChannelExposure(channelName) {
|
|
1972
|
+
return this.findDispatchableChannel(channelName)?.Plugin.Exposure ?? null;
|
|
1973
|
+
}
|
|
1974
|
+
/**
|
|
1975
|
+
* The user's own choice of how much of a channel the agent may perceive. Takes effect immediately
|
|
1976
|
+
* (frames and notes the new level forbids stop, and the model is told), and is remembered per channel key
|
|
1977
|
+
* for next time. The level can only LOWER what the server's policy allows; choosing `'pixels'` for a channel
|
|
1978
|
+
* the agent's policy capped at `'state'` changes nothing.
|
|
1979
|
+
*
|
|
1980
|
+
* @param channelName The channel's key (case-insensitive).
|
|
1981
|
+
* @param level The user's choice; `undefined` clears it (back to what policy allows).
|
|
1982
|
+
* @returns `false` when the channel is not in this session (nothing is applied, but the choice is still remembered).
|
|
1126
1983
|
*/
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1984
|
+
SetUserChannelExposure(channelName, level) {
|
|
1985
|
+
this.exposurePreferences.Set(channelName, level);
|
|
1986
|
+
const channel = this.findDispatchableChannel(channelName);
|
|
1987
|
+
if (!channel) {
|
|
1988
|
+
return false;
|
|
1131
1989
|
}
|
|
1132
|
-
this.
|
|
1133
|
-
return
|
|
1990
|
+
this.applyChannelExposure(channel.Plugin);
|
|
1991
|
+
return true;
|
|
1992
|
+
}
|
|
1993
|
+
/** Pushes a channel's current exposure inputs (server policy, user choice) into the channel. */
|
|
1994
|
+
applyChannelExposure(plugin) {
|
|
1995
|
+
const resolved = this.GetResolvedChannel(plugin.ChannelName);
|
|
1996
|
+
const user = this.exposurePreferences.Get(plugin.ChannelName);
|
|
1997
|
+
const reasons = (resolved?.ExposureLimits ?? []).map((limit) => limit.Reason);
|
|
1998
|
+
// The user's limit binds only when it is lower than what the server allows; otherwise it changes nothing
|
|
1999
|
+
// and the agent should not be told about it.
|
|
2000
|
+
const allowed = resolved?.Exposure ?? plugin.GetDescriptor().MaxExposure;
|
|
2001
|
+
if (user !== undefined && CompareExposure(user, allowed) < 0) {
|
|
2002
|
+
reasons.push(UserExposureReason(user));
|
|
2003
|
+
}
|
|
2004
|
+
plugin.ApplyExposure({ Policy: resolved?.Exposure, User: user, Reasons: reasons });
|
|
2005
|
+
// A capture this channel fronts follows it: the agent sees its frames only while the channel's exposure allows pixels.
|
|
2006
|
+
const kind = plugin.CaptureKind;
|
|
2007
|
+
if (kind) {
|
|
2008
|
+
this.captures?.SetVisibleToAgent(kind, plugin.Exposure === 'pixels');
|
|
2009
|
+
}
|
|
2010
|
+
this.syncChannelVideoSource(plugin);
|
|
1134
2011
|
}
|
|
1135
2012
|
/**
|
|
1136
|
-
*
|
|
1137
|
-
*
|
|
1138
|
-
* the
|
|
1139
|
-
*
|
|
1140
|
-
* never fatal.
|
|
2013
|
+
* Sends a frame from a channel's context ({@link RealtimeChannelContext.SendVideoFrame}, deprecated) through the
|
|
2014
|
+
* session's {@link VideoSourceArbiter}, as the channel's own source: it reaches the model only while the arbiter picks
|
|
2015
|
+
* that source, and the arbiter tells the model when it switches. The source registers with the first frame, under the
|
|
2016
|
+
* id the channel's own frame bridge uses, switched on only while the channel's exposure allows pixels.
|
|
1141
2017
|
*/
|
|
1142
|
-
|
|
2018
|
+
sendChannelVideoFrame(plugin, base64Image, mimeType) {
|
|
2019
|
+
const client = this.client;
|
|
2020
|
+
if (!client || !this.isSessionLive() || !client.IsTrackEstablished('video', 'inbound')) {
|
|
2021
|
+
return;
|
|
2022
|
+
}
|
|
2023
|
+
const arbiter = VideoSourceArbiter.ForSink(client);
|
|
2024
|
+
const descriptor = plugin.GetDescriptor();
|
|
2025
|
+
const sourceId = ChannelVideoSourceID(descriptor.Key, plugin.InstanceId);
|
|
2026
|
+
if (!arbiter.HasSource(sourceId)) {
|
|
2027
|
+
arbiter.RegisterSource({
|
|
2028
|
+
SourceID: sourceId,
|
|
2029
|
+
Label: descriptor.DisplayName,
|
|
2030
|
+
Kind: 'surface',
|
|
2031
|
+
ChannelKey: descriptor.Key,
|
|
2032
|
+
Enabled: plugin.Exposure === 'pixels',
|
|
2033
|
+
});
|
|
2034
|
+
}
|
|
2035
|
+
arbiter.PushFrame(sourceId, base64Image, mimeType);
|
|
2036
|
+
}
|
|
2037
|
+
/**
|
|
2038
|
+
* Keeps a channel's video source at the arbiter, when it has one, in step with the channel's exposure: on while the
|
|
2039
|
+
* agent may see its pixels, off otherwise. Quiet, because the channel tells the model about its own exposure change.
|
|
2040
|
+
*/
|
|
2041
|
+
syncChannelVideoSource(plugin) {
|
|
2042
|
+
if (!this.client) {
|
|
2043
|
+
return;
|
|
2044
|
+
}
|
|
2045
|
+
const arbiter = VideoSourceArbiter.ForSink(this.client);
|
|
2046
|
+
const sourceId = ChannelVideoSourceID(plugin.GetDescriptor().Key, plugin.InstanceId);
|
|
2047
|
+
if (arbiter.HasSource(sourceId)) {
|
|
2048
|
+
arbiter.SetSourceEnabled(sourceId, plugin.Exposure === 'pixels', false);
|
|
2049
|
+
}
|
|
2050
|
+
}
|
|
2051
|
+
/** Follows the session client's video-source arbiter so {@link VideoSources$} reflects it. */
|
|
2052
|
+
watchVideoSources(client) {
|
|
2053
|
+
this.unwatchVideoSources();
|
|
2054
|
+
const arbiter = VideoSourceArbiter.ForSink(client);
|
|
2055
|
+
arbiter.SetFocusedChannel(this.focusedChannelKey);
|
|
2056
|
+
this.stopWatchingVideoSources = arbiter.OnChange(() => this._videoSources$.next(arbiter.GetSources()));
|
|
2057
|
+
this._videoSources$.next(arbiter.GetSources());
|
|
2058
|
+
}
|
|
2059
|
+
/**
|
|
2060
|
+
* Tells the session which channel's surface the user is looking at (`null` for none, e.g. the activity tab).
|
|
2061
|
+
* When the model can see only one video source and several are live, the one the user is looking at is the one
|
|
2062
|
+
* it sees (after an explicit pick, and after a camera or screen share the user started). Safe to call before the
|
|
2063
|
+
* session is live: it applies when the connection comes up.
|
|
2064
|
+
*
|
|
2065
|
+
* @param channelKey The focused channel's key, as it appears on {@link VideoSources$} entries' `ChannelKey`.
|
|
2066
|
+
*/
|
|
2067
|
+
SetFocusedChannel(channelKey) {
|
|
2068
|
+
this.focusedChannelKey = channelKey;
|
|
2069
|
+
if (this.client) {
|
|
2070
|
+
VideoSourceArbiter.ForSink(this.client).SetFocusedChannel(channelKey);
|
|
2071
|
+
}
|
|
2072
|
+
}
|
|
2073
|
+
/** Stops following the arbiter and clears {@link VideoSources$}. Safe to call when nothing is watched. */
|
|
2074
|
+
unwatchVideoSources() {
|
|
2075
|
+
this.stopWatchingVideoSources?.();
|
|
2076
|
+
this.stopWatchingVideoSources = null;
|
|
2077
|
+
if (this._videoSources$.value.length > 0) {
|
|
2078
|
+
this._videoSources$.next([]);
|
|
2079
|
+
}
|
|
2080
|
+
}
|
|
2081
|
+
/**
|
|
2082
|
+
* Turns the agent's view of one video source on or off — what the "agent can see" control calls.
|
|
2083
|
+
*
|
|
2084
|
+
* A source that belongs to a channel goes through {@link SetUserChannelExposure}, so the choice is
|
|
2085
|
+
* remembered per channel and the channel itself tells the model. Any other source (a camera or screen
|
|
2086
|
+
* share that is not a channel) is switched at the arbiter, which tells the model.
|
|
2087
|
+
*
|
|
2088
|
+
* @param sourceId The source's id (from {@link VideoSources$}).
|
|
2089
|
+
* @param enabled Whether the agent may see it.
|
|
2090
|
+
* @returns `false` when there is no such source.
|
|
2091
|
+
*/
|
|
2092
|
+
SetVideoSourceEnabled(sourceId, enabled) {
|
|
2093
|
+
const client = this.client;
|
|
2094
|
+
if (!client) {
|
|
2095
|
+
return false;
|
|
2096
|
+
}
|
|
2097
|
+
const arbiter = VideoSourceArbiter.ForSink(client);
|
|
2098
|
+
const source = arbiter.GetSources().find((s) => s.SourceID === sourceId);
|
|
2099
|
+
if (!source) {
|
|
2100
|
+
return false;
|
|
2101
|
+
}
|
|
2102
|
+
if (source.ChannelKey && this.findDispatchableChannel(source.ChannelKey)) {
|
|
2103
|
+
return this.SetUserChannelExposure(source.ChannelKey, enabled ? undefined : 'state');
|
|
2104
|
+
}
|
|
2105
|
+
return arbiter.SetSourceEnabled(sourceId, enabled);
|
|
2106
|
+
}
|
|
2107
|
+
/**
|
|
2108
|
+
* Picks the video source the agent sees: what the "agent can see" control calls when more sources are on than the
|
|
2109
|
+
* model takes. The pick beats every other rule (a camera or screen share the user started, the surface the user is
|
|
2110
|
+
* looking at) until it is cleared, or its source is turned off or leaves. The pick shows on {@link VideoSources$}
|
|
2111
|
+
* (`Picked`), and the model is told when what it sees changes.
|
|
2112
|
+
*
|
|
2113
|
+
* @param sourceId The source's id (from {@link VideoSources$}), or `null` to let the call decide again.
|
|
2114
|
+
* @returns `false` with no live session, or for a source that is not there or is turned off.
|
|
2115
|
+
*/
|
|
2116
|
+
SelectVideoSource(sourceId) {
|
|
2117
|
+
return this.client ? VideoSourceArbiter.ForSink(this.client).SelectSource(sourceId) : false;
|
|
2118
|
+
}
|
|
2119
|
+
/**
|
|
2120
|
+
* The `on-demand` channels that are in the session but not open yet — what the agent can open
|
|
2121
|
+
* through `ContextTool`. A fresh array; empty before a session starts and after teardown.
|
|
2122
|
+
*/
|
|
2123
|
+
get AdvertisedChannels() {
|
|
2124
|
+
return this.advertisedChannels.map((p) => p.Plugin);
|
|
2125
|
+
}
|
|
2126
|
+
/**
|
|
2127
|
+
* Resolves, initializes and mounts the session's channels in one step using the scope the browser
|
|
2128
|
+
* can decide alone (code defaults + host declarations), and returns the native tools to declare at
|
|
2129
|
+
* mint. This is the single-step composition of {@link prepareChannelScope} + {@link activateChannels}
|
|
2130
|
+
* that a session start spreads across the mint; it is kept as a unit because it is the cleanest seam
|
|
2131
|
+
* for exercising the plugin plumbing without a mint.
|
|
2132
|
+
*/
|
|
2133
|
+
async startChannels(hostChannels) {
|
|
2134
|
+
const scope = await this.prepareChannelScope(hostChannels);
|
|
2135
|
+
this.activateChannels(this.localChannelScope?.Channels ?? []);
|
|
2136
|
+
return scope.NativeTools;
|
|
2137
|
+
}
|
|
2138
|
+
/**
|
|
2139
|
+
* Stage 1: constructs the channel plugins (registry rows and host declarations), reads their
|
|
2140
|
+
* descriptors, resolves the scope the browser can decide alone, and builds what the mint needs —
|
|
2141
|
+
* the native tools to declare and the candidates to report. Starts nothing.
|
|
2142
|
+
*
|
|
2143
|
+
* @param hostChannels Channels the host brings to this session.
|
|
2144
|
+
* @returns The native tools to declare at mint, the candidates as JSON (`null` when there are none,
|
|
2145
|
+
* so a channel-less session sends exactly the mint it always did), and whether the host could show the agent's
|
|
2146
|
+
* video ({@link PreparedChannelsShowAgentVideo}).
|
|
2147
|
+
*/
|
|
2148
|
+
async prepareChannelScope(hostChannels) {
|
|
2149
|
+
this.discardUnmountedChannels();
|
|
2150
|
+
const prepared = await this.prepareChannels(hostChannels);
|
|
2151
|
+
this.preparedChannels = prepared;
|
|
2152
|
+
const candidates = prepared.map((p) => BuildChannelCandidate(p, p.Plugin.GetDescriptor()));
|
|
2153
|
+
const local = ResolveLocalChannelScope(candidates);
|
|
2154
|
+
this.localChannelScope = local;
|
|
2155
|
+
for (const excluded of local.Excluded) {
|
|
2156
|
+
console.warn(`[RealtimeSession] Channel '${excluded.Key}' is not in this session (${excluded.Reason}).`);
|
|
2157
|
+
}
|
|
2158
|
+
const nativeTools = SelectNativeChannelTools(local.Channels, ToolsByChannelKey(candidates));
|
|
2159
|
+
// `Registry` is the browser's own view and never goes over the wire: the server reads the registry itself.
|
|
2160
|
+
const wire = candidates.map(({ Registry: _registry, ...candidate }) => candidate);
|
|
2161
|
+
return { NativeTools: nativeTools, CandidatesJson: wire.length > 0 ? JSON.stringify(wire) : null, ShowsAgentVideo: PreparedChannelsShowAgentVideo(prepared) };
|
|
2162
|
+
}
|
|
2163
|
+
/**
|
|
2164
|
+
* Constructs a plugin for every active-or-inactive registry row and every host declaration. An
|
|
2165
|
+
* INACTIVE row is constructed too — only to learn its key so the kill switch can name it — and is
|
|
2166
|
+
* never initialized. A plugin whose descriptor cannot be read is skipped (logged), never fatal.
|
|
2167
|
+
*/
|
|
2168
|
+
async prepareChannels(hostChannels) {
|
|
1143
2169
|
const rows = await this.fetchChannelDefinitions();
|
|
1144
|
-
|
|
2170
|
+
await this.loadChannelClasses([...rows.map((r) => r.ClientPluginClass), ...(hostChannels ?? []).map((d) => d.ClientPluginClass)]);
|
|
2171
|
+
const prepared = [];
|
|
1145
2172
|
for (const row of rows) {
|
|
1146
2173
|
const plugin = this.resolveChannelPlugin(row);
|
|
1147
2174
|
if (plugin) {
|
|
1148
|
-
|
|
2175
|
+
plugin.ApplySurfacePlacement(row.SurfacePlacement);
|
|
2176
|
+
prepared.push({ Plugin: plugin, Key: plugin.ChannelName, Registry: row.IsActive ? 'active' : 'inactive' });
|
|
2177
|
+
}
|
|
2178
|
+
}
|
|
2179
|
+
for (const declaration of hostChannels ?? []) {
|
|
2180
|
+
const plugin = this.createHostChannelPlugin(declaration);
|
|
2181
|
+
if (!plugin) {
|
|
2182
|
+
continue;
|
|
2183
|
+
}
|
|
2184
|
+
const existing = FindPreparedChannel(prepared, plugin.ChannelName);
|
|
2185
|
+
if (existing) {
|
|
2186
|
+
// The host's instance replaces the registry's (it may carry the host's collaborators); the
|
|
2187
|
+
// registry row's state — notably the kill switch, and where its surface shows — still applies.
|
|
2188
|
+
plugin.ApplySurfacePlacement(existing.Plugin.SurfacePlacement);
|
|
2189
|
+
existing.Plugin = plugin;
|
|
2190
|
+
existing.HostDeclaration = declaration;
|
|
2191
|
+
}
|
|
2192
|
+
else {
|
|
2193
|
+
prepared.push({ Plugin: plugin, Key: plugin.ChannelName, Registry: 'none', HostDeclaration: declaration });
|
|
2194
|
+
}
|
|
2195
|
+
}
|
|
2196
|
+
return prepared.filter((p) => this.canDescribe(p));
|
|
2197
|
+
}
|
|
2198
|
+
/**
|
|
2199
|
+
* Gives a host that loads channel code on demand (a widget that keeps a heavy channel in its own download)
|
|
2200
|
+
* the chance to load it before the plugins are built. Never fatal: a loader that throws is logged and the
|
|
2201
|
+
* channel simply is not available, exactly as if its class had never been registered.
|
|
2202
|
+
*
|
|
2203
|
+
* @param keys The `ClientPluginClass` keys of every channel about to be resolved (blank/absent ones included).
|
|
2204
|
+
*/
|
|
2205
|
+
async loadChannelClasses(keys) {
|
|
2206
|
+
const loader = this.ChannelClassLoader;
|
|
2207
|
+
if (!loader) {
|
|
2208
|
+
return;
|
|
2209
|
+
}
|
|
2210
|
+
const wanted = [...new Set(keys.map((k) => k?.trim() ?? '').filter((k) => k.length > 0))];
|
|
2211
|
+
if (wanted.length === 0) {
|
|
2212
|
+
return;
|
|
2213
|
+
}
|
|
2214
|
+
try {
|
|
2215
|
+
await loader(wanted);
|
|
2216
|
+
}
|
|
2217
|
+
catch (error) {
|
|
2218
|
+
console.error('[RealtimeSession] The channel class loader failed — channels that needed it are left out:', error);
|
|
2219
|
+
}
|
|
2220
|
+
}
|
|
2221
|
+
/** Whether a prepared plugin's descriptor can be read; logs and rejects one that throws. */
|
|
2222
|
+
canDescribe(prepared) {
|
|
2223
|
+
try {
|
|
2224
|
+
prepared.Plugin.GetDescriptor();
|
|
2225
|
+
return true;
|
|
2226
|
+
}
|
|
2227
|
+
catch (error) {
|
|
2228
|
+
console.error(`[RealtimeSession] Channel '${prepared.Key}' has an unreadable descriptor — leaving it out:`, error);
|
|
2229
|
+
return false;
|
|
2230
|
+
}
|
|
2231
|
+
}
|
|
2232
|
+
/** Builds a plugin from a host declaration; `null` (logged) when it names nothing buildable. */
|
|
2233
|
+
createHostChannelPlugin(declaration) {
|
|
2234
|
+
if (declaration.Create) {
|
|
2235
|
+
try {
|
|
2236
|
+
return declaration.Create();
|
|
2237
|
+
}
|
|
2238
|
+
catch (error) {
|
|
2239
|
+
console.error('[RealtimeSession] A host-declared channel factory threw — leaving the channel out:', error);
|
|
2240
|
+
return null;
|
|
2241
|
+
}
|
|
2242
|
+
}
|
|
2243
|
+
const key = declaration.ClientPluginClass?.trim();
|
|
2244
|
+
if (!key) {
|
|
2245
|
+
console.warn('[RealtimeSession] A host-declared channel names neither ClientPluginClass nor Create — ignoring it.');
|
|
2246
|
+
return null;
|
|
2247
|
+
}
|
|
2248
|
+
return this.resolveChannelPlugin({ Name: key, ClientPluginClass: key, IsActive: true });
|
|
2249
|
+
}
|
|
2250
|
+
/**
|
|
2251
|
+
* Stage 3: initializes the channels the resolved policy puts in the session. `open-on-start` and
|
|
2252
|
+
* `headless` channels are mounted and published on {@link ActiveChannels$}; `on-demand` channels
|
|
2253
|
+
* are held until opened. Prepared plugins the policy left out are dropped without ever having run.
|
|
2254
|
+
*/
|
|
2255
|
+
activateChannels(channels) {
|
|
2256
|
+
const reconciled = ReconcileChannelsWithPolicy(this.preparedChannels, channels);
|
|
2257
|
+
for (const key of reconciled.Unknown) {
|
|
2258
|
+
console.warn(`[RealtimeSession] The session policy names channel '${key}' but this host has no plugin for it — ignoring it.`);
|
|
2259
|
+
}
|
|
2260
|
+
this.resolvedChannels.clear();
|
|
2261
|
+
this.advertisedChannels = [];
|
|
2262
|
+
const mounted = [];
|
|
2263
|
+
for (const { Prepared, Resolved } of reconciled.InSession) {
|
|
2264
|
+
this.resolvedChannels.set(NormalizeChannelKey(Resolved.Key), Resolved);
|
|
2265
|
+
// Exposure is applied BEFORE the channel initializes, so what it requests of the model (a video
|
|
2266
|
+
// track, in particular) already reflects what policy allows.
|
|
2267
|
+
this.applyChannelExposure(Prepared.Plugin);
|
|
2268
|
+
if (Resolved.DisplayPolicy === 'on-demand') {
|
|
2269
|
+
this.advertisedChannels.push(Prepared);
|
|
2270
|
+
}
|
|
2271
|
+
else {
|
|
2272
|
+
this.mountChannel(Prepared, Resolved);
|
|
2273
|
+
mounted.push(Prepared.Plugin);
|
|
1149
2274
|
}
|
|
1150
2275
|
}
|
|
1151
|
-
|
|
2276
|
+
this.preparedChannels = [];
|
|
2277
|
+
this._activeChannels$.next(mounted);
|
|
1152
2278
|
}
|
|
1153
2279
|
/**
|
|
1154
|
-
*
|
|
1155
|
-
*
|
|
1156
|
-
*
|
|
1157
|
-
|
|
1158
|
-
|
|
2280
|
+
* Applies the mint's outcome to the channel set: the server's policy when it sent a usable one,
|
|
2281
|
+
* else the scope the browser resolved on its own. A policy can only SELECT among the plugins this
|
|
2282
|
+
* host prepared — it cannot add one.
|
|
2283
|
+
*/
|
|
2284
|
+
applySessionClientPolicy(session) {
|
|
2285
|
+
const policy = ParseRealtimeSessionClientPolicy(session.ClientPolicyJson);
|
|
2286
|
+
if (session.ClientPolicyJson && !policy) {
|
|
2287
|
+
console.warn('[RealtimeSession] The mint returned a client policy this runtime cannot read — resolving channels locally.');
|
|
2288
|
+
}
|
|
2289
|
+
this.sessionClientTools = policy?.ClientTools ?? null;
|
|
2290
|
+
if (this.preparedChannels.length === 0) {
|
|
2291
|
+
this.resolvedChannels.clear();
|
|
2292
|
+
return; // nothing was prepared (a host that mints and runs the session itself) — no channel set to apply
|
|
2293
|
+
}
|
|
2294
|
+
this.activateChannels(policy?.Channels ?? this.localChannelScope?.Channels ?? []);
|
|
2295
|
+
}
|
|
2296
|
+
/** Drops plugins that were prepared for a start that never mounted them. They were never initialized, so there is nothing to release. */
|
|
2297
|
+
discardUnmountedChannels() {
|
|
2298
|
+
this.preparedChannels = [];
|
|
2299
|
+
this.advertisedChannels = [];
|
|
2300
|
+
this.localChannelScope = null;
|
|
2301
|
+
}
|
|
2302
|
+
/**
|
|
2303
|
+
* Loads the `MJ: AI Agent Channels` registry rows (ACTIVE AND INACTIVE — an inactive row is the master
|
|
2304
|
+
* kill switch, so the scope decision has to see it). With entity metadata: from {@link AIEngineBase}'s
|
|
2305
|
+
* cached `AgentChannels` (provider-scoped engine instance, lazy `Config` — no RunView round-trip; the
|
|
2306
|
+
* engine's BaseEntity-event reactivity keeps the registry fresh). On a connect-only provider: one
|
|
2307
|
+
* `RunDynamicView` query ({@link fetchChannelDefinitionsOverGraphQL}). Failures are logged and degrade to an
|
|
2308
|
+
* empty list — channel availability must never block the voice session.
|
|
1159
2309
|
*/
|
|
1160
2310
|
async fetchChannelDefinitions() {
|
|
1161
2311
|
// A connect-only provider (ConnectGraphQLClient — anonymous embeds) has no entity metadata,
|
|
@@ -1168,9 +2318,13 @@ export class RealtimeSessionRuntime {
|
|
|
1168
2318
|
try {
|
|
1169
2319
|
const engine = AIEngineBase.GetProviderInstance(this.Provider, AIEngineBase);
|
|
1170
2320
|
await engine.Config(false, undefined, this.Provider);
|
|
1171
|
-
return (engine.AgentChannels ?? [])
|
|
1172
|
-
|
|
1173
|
-
|
|
2321
|
+
return (engine.AgentChannels ?? []).map((c) => ({
|
|
2322
|
+
ID: c.ID,
|
|
2323
|
+
Name: c.Name,
|
|
2324
|
+
ClientPluginClass: c.ClientPluginClass,
|
|
2325
|
+
IsActive: c.IsActive,
|
|
2326
|
+
SurfacePlacement: readRowSurfacePlacement(c),
|
|
2327
|
+
}));
|
|
1174
2328
|
}
|
|
1175
2329
|
catch (error) {
|
|
1176
2330
|
console.warn('[RealtimeSession] Channel registry unavailable — starting with no channels:', error);
|
|
@@ -1178,10 +2332,10 @@ export class RealtimeSessionRuntime {
|
|
|
1178
2332
|
}
|
|
1179
2333
|
}
|
|
1180
2334
|
/**
|
|
1181
|
-
* The connect-only path of {@link fetchChannelDefinitions}: the same
|
|
1182
|
-
*
|
|
1183
|
-
* typed RunView from. Same tolerance as the engine path — a failure is
|
|
1184
|
-
* channels", never a blocked session.
|
|
2335
|
+
* The connect-only path of {@link fetchChannelDefinitions}: the same `MJ: AI Agent Channels` rows, active
|
|
2336
|
+
* and inactive (an inactive row is the kill switch), read with a dynamic view because a connect-only client
|
|
2337
|
+
* has no entity metadata to build a typed RunView from. Same tolerance as the engine path — a failure is
|
|
2338
|
+
* logged and means "no channels", never a blocked session.
|
|
1185
2339
|
*/
|
|
1186
2340
|
async fetchChannelDefinitionsOverGraphQL() {
|
|
1187
2341
|
const query = `query RealtimeChannelRegistry($input: RunDynamicViewInput!) {
|
|
@@ -1189,7 +2343,7 @@ export class RealtimeSessionRuntime {
|
|
|
1189
2343
|
}`;
|
|
1190
2344
|
try {
|
|
1191
2345
|
const result = (await this.gql().ExecuteGQL(query, {
|
|
1192
|
-
input: { EntityName: 'MJ: AI Agent Channels',
|
|
2346
|
+
input: { EntityName: 'MJ: AI Agent Channels', Fields: ['ID', 'Name', 'ClientPluginClass', 'IsActive', 'UIConfig'] },
|
|
1193
2347
|
}));
|
|
1194
2348
|
const view = result?.RunDynamicView;
|
|
1195
2349
|
if (!view?.Success) {
|
|
@@ -1198,8 +2352,13 @@ export class RealtimeSessionRuntime {
|
|
|
1198
2352
|
}
|
|
1199
2353
|
return (view.Results ?? [])
|
|
1200
2354
|
.map((r) => JSON.parse(r.Data))
|
|
1201
|
-
.
|
|
1202
|
-
|
|
2355
|
+
.map((row) => ({
|
|
2356
|
+
ID: row.ID,
|
|
2357
|
+
Name: row.Name,
|
|
2358
|
+
ClientPluginClass: row.ClientPluginClass,
|
|
2359
|
+
IsActive: row.IsActive === true,
|
|
2360
|
+
SurfacePlacement: readUIConfigTextSurfacePlacement(row.Name, row.UIConfig),
|
|
2361
|
+
}));
|
|
1203
2362
|
}
|
|
1204
2363
|
catch (error) {
|
|
1205
2364
|
console.warn('[RealtimeSession] Channel registry unavailable — starting with no channels:', error instanceof Error ? error.message : String(error));
|
|
@@ -1211,16 +2370,23 @@ export class RealtimeSessionRuntime {
|
|
|
1211
2370
|
* checked first, exactly like the realtime-client drivers) and instantiates a fresh
|
|
1212
2371
|
* per-session plugin. Returns `null` (logged) when no plugin is registered for the key
|
|
1213
2372
|
* — e.g. its Load function was never called or the package isn't included client-side.
|
|
2373
|
+
* An inactive row is resolved quietly: it only needs to be named, and a missing plugin for a
|
|
2374
|
+
* channel nobody wants is not worth a warning.
|
|
1214
2375
|
*/
|
|
1215
2376
|
resolveChannelPlugin(row) {
|
|
1216
2377
|
const key = row.ClientPluginClass?.trim();
|
|
2378
|
+
const quiet = !row.IsActive;
|
|
1217
2379
|
if (!key) {
|
|
1218
|
-
|
|
2380
|
+
if (!quiet) {
|
|
2381
|
+
console.warn(`[RealtimeSession] Channel '${row.Name}' has no ClientPluginClass — skipping.`);
|
|
2382
|
+
}
|
|
1219
2383
|
return null;
|
|
1220
2384
|
}
|
|
1221
2385
|
const registration = MJGlobal.Instance.ClassFactory.GetRegistration(BaseRealtimeChannelClient, key);
|
|
1222
2386
|
if (!registration) {
|
|
1223
|
-
|
|
2387
|
+
if (!quiet) {
|
|
2388
|
+
console.warn(`[RealtimeSession] No client plugin registered for channel '${row.Name}' (key '${key}') — skipping.`);
|
|
2389
|
+
}
|
|
1224
2390
|
return null;
|
|
1225
2391
|
}
|
|
1226
2392
|
const plugin = MJGlobal.Instance.ClassFactory.CreateInstance(BaseRealtimeChannelClient, key);
|
|
@@ -1231,24 +2397,157 @@ export class RealtimeSessionRuntime {
|
|
|
1231
2397
|
return plugin;
|
|
1232
2398
|
}
|
|
1233
2399
|
/**
|
|
1234
|
-
* Wires one plugin into the session: hands it its host context
|
|
1235
|
-
* prefix-routed local tool executor (so `<ToolNamePrefix>*`
|
|
1236
|
-
* through {@link BaseRealtimeChannelClient.ApplyAgentTool}, never the
|
|
2400
|
+
* Wires one plugin into the session: hands it its host context (carrying its resolved config) and,
|
|
2401
|
+
* when it has native tools, registers its prefix-routed local tool executor (so `<ToolNamePrefix>*`
|
|
2402
|
+
* calls run in the browser through {@link BaseRealtimeChannelClient.ApplyAgentTool}, never the
|
|
2403
|
+
* server relay). A channel with no tool prefix has no native route — it is reached through
|
|
2404
|
+
* `ContextTool` — and registering an empty prefix would match EVERY tool name.
|
|
1237
2405
|
*/
|
|
1238
|
-
|
|
1239
|
-
plugin.
|
|
2406
|
+
mountChannel(prepared, resolved) {
|
|
2407
|
+
const plugin = prepared.Plugin;
|
|
2408
|
+
plugin.Initialize(this.buildChannelContext(plugin, resolved?.Config));
|
|
2409
|
+
if (plugin.ToolNamePrefix.length === 0) {
|
|
2410
|
+
return;
|
|
2411
|
+
}
|
|
1240
2412
|
this.RegisterClientToolHandler(plugin.ToolNamePrefix, (toolName, argsJson) => {
|
|
1241
2413
|
// The agent is ACTING on this channel — surface-discovery signal for the overlay
|
|
1242
2414
|
// (first activity registers + auto-reveals + focuses the channel tab) before the
|
|
1243
2415
|
// tool applies. Record the channel as USED so the overlay tabs it (channels other
|
|
1244
2416
|
// than the whiteboard are tab-less until they're first used).
|
|
1245
|
-
this.
|
|
1246
|
-
|
|
2417
|
+
this.noteChannelActivity(plugin);
|
|
2418
|
+
// A native tool whose result would show the agent what exposure policy withholds is refused here, as it is on the
|
|
2419
|
+
// ContextTool route; the legacy tool-result shape carries the reason.
|
|
2420
|
+
const withheld = plugin.FindVerbForNativeTool(toolName);
|
|
2421
|
+
const refusal = withheld ? plugin.RefuseVerbForExposure(withheld) : null;
|
|
2422
|
+
if (refusal) {
|
|
2423
|
+
return JSON.stringify({ success: false, errorCode: 'exposure_restricted', error: refusal });
|
|
2424
|
+
}
|
|
1247
2425
|
return plugin.ApplyAgentTool(toolName, argsJson);
|
|
1248
2426
|
});
|
|
1249
2427
|
}
|
|
2428
|
+
/** Records that the agent acted on a channel (the overlay's first-use reveal signal). */
|
|
2429
|
+
noteChannelActivity(plugin) {
|
|
2430
|
+
this.usedChannelNames.add(plugin.ChannelName);
|
|
2431
|
+
this._channelActivity$.next(plugin);
|
|
2432
|
+
}
|
|
2433
|
+
/**
|
|
2434
|
+
* Mounts an advertised `on-demand` channel when the agent opens it: initializes it, restores its
|
|
2435
|
+
* prior-session state, publishes it on {@link ActiveChannels$} (so the overlay can give it a tab)
|
|
2436
|
+
* and, if the call is already connected, tells it so. If initialization throws the channel goes
|
|
2437
|
+
* back to being advertised, so the agent can retry or move on.
|
|
2438
|
+
*/
|
|
2439
|
+
async mountAdvertisedChannel(plugin) {
|
|
2440
|
+
const index = this.advertisedChannels.findIndex((p) => p.Plugin === plugin);
|
|
2441
|
+
if (index < 0) {
|
|
2442
|
+
return; // already mounted (or never advertised) — the dispatcher opens it regardless
|
|
2443
|
+
}
|
|
2444
|
+
const [prepared] = this.advertisedChannels.splice(index, 1);
|
|
2445
|
+
try {
|
|
2446
|
+
this.mountChannel(prepared, this.GetResolvedChannel(prepared.Key) ?? undefined);
|
|
2447
|
+
}
|
|
2448
|
+
catch (error) {
|
|
2449
|
+
this.advertisedChannels.splice(index, 0, prepared);
|
|
2450
|
+
throw error;
|
|
2451
|
+
}
|
|
2452
|
+
this.restorePriorChannelState(plugin);
|
|
2453
|
+
this._activeChannels$.next([...this._activeChannels$.value, plugin]);
|
|
2454
|
+
if (this.isSessionLive()) {
|
|
2455
|
+
this.notifyChannelSessionStarted(plugin);
|
|
2456
|
+
}
|
|
2457
|
+
}
|
|
2458
|
+
/** Tells a channel the session is connected, containing anything it throws. */
|
|
2459
|
+
notifyChannelSessionStarted(channel) {
|
|
2460
|
+
try {
|
|
2461
|
+
channel.OnSessionStarted?.();
|
|
2462
|
+
}
|
|
2463
|
+
catch (err) {
|
|
2464
|
+
console.error(`[RealtimeSession] Error in channel '${channel.ChannelName}' OnSessionStarted:`, err);
|
|
2465
|
+
}
|
|
2466
|
+
}
|
|
2467
|
+
/** Finds a session channel (open or merely advertised) for the dispatcher. */
|
|
2468
|
+
findDispatchableChannel(key) {
|
|
2469
|
+
const id = NormalizeChannelKey(key);
|
|
2470
|
+
const open = this._activeChannels$.value.find((c) => NormalizeChannelKey(c.ChannelName) === id);
|
|
2471
|
+
if (open) {
|
|
2472
|
+
return { Plugin: open, IsOpen: true };
|
|
2473
|
+
}
|
|
2474
|
+
const advertised = this.advertisedChannels.find((p) => NormalizeChannelKey(p.Key) === id);
|
|
2475
|
+
return advertised ? { Plugin: advertised.Plugin, IsOpen: false } : null;
|
|
2476
|
+
}
|
|
2477
|
+
/** The keys the agent can address, for "unknown channel" messages — channels with nothing to address are left out. */
|
|
2478
|
+
listAddressableChannelKeys() {
|
|
2479
|
+
const addressable = (plugin) => {
|
|
2480
|
+
const descriptor = plugin.GetDescriptor();
|
|
2481
|
+
return descriptor.Verbs.length > 0 || descriptor.Inputs !== undefined;
|
|
2482
|
+
};
|
|
2483
|
+
return [
|
|
2484
|
+
...this._activeChannels$.value.filter(addressable).map((c) => c.ChannelName),
|
|
2485
|
+
...this.advertisedChannels.filter((p) => addressable(p.Plugin)).map((p) => p.Key),
|
|
2486
|
+
];
|
|
2487
|
+
}
|
|
2488
|
+
/**
|
|
2489
|
+
* Runs a channel-addressed `ContextTool` call (the {@link RealtimeChannelContext.DispatchContextAction}
|
|
2490
|
+
* implementation): validates it, opens an `on-demand` channel when asked, runs the verb, and on
|
|
2491
|
+
* success records the channel as used so the overlay reveals its tab. Never throws.
|
|
2492
|
+
*/
|
|
2493
|
+
async dispatchContextAction(request) {
|
|
2494
|
+
const result = await this.channelDispatcher.Dispatch(request);
|
|
2495
|
+
if (result.Success) {
|
|
2496
|
+
const channel = this.findDispatchableChannel(request.Target.Channel);
|
|
2497
|
+
if (channel) {
|
|
2498
|
+
this.noteChannelActivity(channel.Plugin);
|
|
2499
|
+
}
|
|
2500
|
+
}
|
|
2501
|
+
return result;
|
|
2502
|
+
}
|
|
2503
|
+
/**
|
|
2504
|
+
* Opens (and seeds) a channel from the HOST — the same path the agent's `open` action takes: the channel
|
|
2505
|
+
* is mounted if it was only advertised, its `Inputs` schema validates `inputs`, and it announces itself to
|
|
2506
|
+
* the model. A channel that is already open is re-seeded. Never throws; a refusal is a structured result.
|
|
2507
|
+
*
|
|
2508
|
+
* @param channelKey The channel to open (case-insensitive).
|
|
2509
|
+
* @param inputs Seed inputs, validated against the channel's descriptor.
|
|
2510
|
+
*/
|
|
2511
|
+
OpenChannel(channelKey, inputs = {}) {
|
|
2512
|
+
return this.dispatchContextAction({ Target: { Channel: channelKey }, Action: 'open', Params: inputs });
|
|
2513
|
+
}
|
|
2514
|
+
/**
|
|
2515
|
+
* Tells the model which channels exist and how to use them — rendered from the channels' own
|
|
2516
|
+
* descriptors, so a new channel needs no prompt change. Sent ONCE, the first moment the control
|
|
2517
|
+
* channel is usable — a context note sent before then is dropped, and `Connect` can resolve before the
|
|
2518
|
+
* provider reports `listening`, so this runs at start and again on each state change until it has gone.
|
|
2519
|
+
* A session whose only channels describe themselves through native tools sends nothing.
|
|
2520
|
+
*/
|
|
2521
|
+
flushChannelCatalogNote() {
|
|
2522
|
+
if (!this.catalogNotePending || !this.isSessionLive()) {
|
|
2523
|
+
return;
|
|
2524
|
+
}
|
|
2525
|
+
this.catalogNotePending = false;
|
|
2526
|
+
const entries = [
|
|
2527
|
+
...this._activeChannels$.value.map((plugin) => this.catalogEntry(plugin, true)),
|
|
2528
|
+
...this.advertisedChannels.map((p) => this.catalogEntry(p.Plugin, false)),
|
|
2529
|
+
];
|
|
2530
|
+
const note = BuildChannelCatalogNote(entries);
|
|
2531
|
+
if (note) {
|
|
2532
|
+
this.SendContextNote(note);
|
|
2533
|
+
}
|
|
2534
|
+
}
|
|
2535
|
+
/** One catalog entry: a channel's descriptor, whether it is open, and whether its tools were declared natively. */
|
|
2536
|
+
catalogEntry(plugin, isOpen) {
|
|
2537
|
+
const display = this.GetResolvedChannel(plugin.ChannelName)?.DisplayPolicy;
|
|
2538
|
+
const descriptor = plugin.GetDescriptor();
|
|
2539
|
+
const entry = {
|
|
2540
|
+
Descriptor: descriptor,
|
|
2541
|
+
IsOpen: isOpen,
|
|
2542
|
+
HasNativeTools: isOpen && display !== undefined && DeclaresNativeTools(display) && plugin.GetToolDefinitions().length > 0,
|
|
2543
|
+
};
|
|
2544
|
+
if (CompareExposure(plugin.Exposure, descriptor.MaxExposure) < 0) {
|
|
2545
|
+
entry.ExposureLimit = { Effective: plugin.Exposure, Ceiling: descriptor.MaxExposure, Reasons: [...plugin.ExposureReasons] };
|
|
2546
|
+
}
|
|
2547
|
+
return entry;
|
|
2548
|
+
}
|
|
1250
2549
|
/** Builds the host-services context one channel plugin sees (its only line to the session). */
|
|
1251
|
-
buildChannelContext(plugin) {
|
|
2550
|
+
buildChannelContext(plugin, channelConfig) {
|
|
1252
2551
|
// Capture the service in a local so the AgentSessionID getter reads the SERVICE's live
|
|
1253
2552
|
// field (not the object literal's `this`) every time it's accessed.
|
|
1254
2553
|
const service = this;
|
|
@@ -1275,55 +2574,107 @@ export class RealtimeSessionRuntime {
|
|
|
1275
2574
|
// (Explorer) feeds both; absent on hosts that supply no app context / register no client tools.
|
|
1276
2575
|
AppContext$: this.AppContext$,
|
|
1277
2576
|
ExecuteClientTool: (name, params) => this.executeAppClientTool(name, params),
|
|
2577
|
+
// The session's server events (identity verification, app events), for a channel that reacts to them.
|
|
2578
|
+
SessionEvents$: this.SessionEvents$,
|
|
2579
|
+
// Channel-addressed ContextTool calls: validated against the verb's schema, open `on-demand` channels.
|
|
2580
|
+
DispatchContextAction: (request) => this.dispatchContextAction(request),
|
|
2581
|
+
// This channel's resolved configuration (host defaults beneath agent/app config).
|
|
2582
|
+
ChannelConfig: channelConfig ?? {},
|
|
1278
2583
|
get Client() {
|
|
1279
2584
|
return service.client;
|
|
1280
2585
|
},
|
|
1281
|
-
|
|
1282
|
-
|
|
2586
|
+
// Through the arbiter, as this channel's source (deprecated: channels use a ChannelInboundVideoBridge).
|
|
2587
|
+
SendVideoFrame: (base64Image, mimeType) => this.sendChannelVideoFrame(plugin, base64Image, mimeType),
|
|
2588
|
+
IsTrackEstablished: (modality, direction) => this.IsTrackEstablished(modality, direction),
|
|
2589
|
+
// The camera and screen share, for a channel that fronts one: their state, and the user's clicks on its surface.
|
|
2590
|
+
Captures$: this.Captures$,
|
|
2591
|
+
StartCapture: (kind) => (kind === 'camera' ? this.StartCamera() : this.StartScreenShare()),
|
|
2592
|
+
StopCapture: (kind) => (kind === 'camera' ? this.StopCamera() : this.StopScreenShare()),
|
|
2593
|
+
// The video sources and which of them the model is sent, so a capture's channel says the agent sees it only then.
|
|
2594
|
+
VideoSources$: this.VideoSources$,
|
|
2595
|
+
// The agent's video, the call's state and its resumes, for a channel that shows the agent.
|
|
2596
|
+
AgentVideo$: this.AgentVideo$,
|
|
2597
|
+
ConnectionState$: this.ConnectionState$,
|
|
2598
|
+
Resuming$: this.Resuming$
|
|
1283
2599
|
};
|
|
1284
2600
|
}
|
|
1285
2601
|
/**
|
|
1286
|
-
*
|
|
1287
|
-
*
|
|
1288
|
-
*
|
|
2602
|
+
* Registers the tools ONE source of the host can run for the realtime `ContextTool`, replacing only
|
|
2603
|
+
* that source's previous set — other owners' tools are untouched. A host with two sources (Explorer's
|
|
2604
|
+
* always-available globals and the active surface's tools) registers each under its own owner key, so
|
|
2605
|
+
* refreshing the surface's tools can neither drop the globals nor leave the previous surface's tools
|
|
2606
|
+
* behind. Passing `[]` clears just that owner. When two owners register the same tool name, the
|
|
2607
|
+
* later-registered owner wins.
|
|
1289
2608
|
*
|
|
1290
|
-
*
|
|
1291
|
-
*
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
2609
|
+
* Calling it with one argument (the original form) registers under {@link DEFAULT_APP_TOOL_OWNER}, so
|
|
2610
|
+
* a host with a single source behaves exactly as before.
|
|
2611
|
+
*
|
|
2612
|
+
* @param tools The owner's complete current set (name + handler; description/schema optional).
|
|
2613
|
+
* @param owner A stable key naming the source (e.g. `'explorer.surface'`). Defaults to a shared owner.
|
|
2614
|
+
*/
|
|
2615
|
+
RegisterAppClientTools(tools, owner = DEFAULT_APP_TOOL_OWNER) {
|
|
2616
|
+
this.appToolRegistry.Register(owner, tools);
|
|
2617
|
+
}
|
|
2618
|
+
/** Removes one owner's tools (and its place in the collision order). No-op when it has none. */
|
|
2619
|
+
UnregisterAppClientTools(owner) {
|
|
2620
|
+
this.appToolRegistry.Unregister(owner);
|
|
2621
|
+
}
|
|
2622
|
+
/** Removes every owner's tools. */
|
|
2623
|
+
ClearAppClientTools() {
|
|
2624
|
+
this.appToolRegistry.Clear();
|
|
2625
|
+
}
|
|
2626
|
+
/**
|
|
2627
|
+
* The client tools in effect right now, resolved through the same unified resolver the server uses
|
|
2628
|
+
* (`override → session → app → static`, first match wins): the host's registered tools (described
|
|
2629
|
+
* by the manifest the host streams when it did not describe them itself) first, then the app and
|
|
2630
|
+
* agent tiers the server resolved at mint. Only the host-registered tools can RUN here; the
|
|
2631
|
+
* other tiers describe what the app declares.
|
|
2632
|
+
*/
|
|
2633
|
+
resolveAppClientTools() {
|
|
2634
|
+
const manifest = this._appContext$.value?.Capabilities?.Tools ?? [];
|
|
2635
|
+
return ResolveClientTools({
|
|
2636
|
+
agentId: '',
|
|
2637
|
+
sessionTools: MergeToolMetadata(this.appToolRegistry.ToMetadata(), manifest),
|
|
2638
|
+
appTools: this.sessionClientTools?.App,
|
|
2639
|
+
staticTools: this.sessionClientTools?.Static,
|
|
2640
|
+
});
|
|
1300
2641
|
}
|
|
1301
2642
|
/**
|
|
1302
2643
|
* Executes a host-registered surface client tool by name (the {@link RealtimeChannelContext.ExecuteClientTool}
|
|
1303
2644
|
* implementation). Tolerant: an unknown tool or a thrown handler resolves to a structured
|
|
1304
|
-
* `Success: false` result the channel narrates — never throws.
|
|
2645
|
+
* `Success: false` result the channel narrates — never throws. A tool the app declares but the host has
|
|
2646
|
+
* not registered a handler for says so, instead of reading like a typo. A handler that returns an object with
|
|
2647
|
+
* `Success: false` also gives `Success: false`, with the handler's `ErrorMessage` and its object as `Result`.
|
|
1305
2648
|
*
|
|
1306
2649
|
* @param name The tool name (the model's `action`).
|
|
1307
2650
|
* @param params The tool parameters.
|
|
1308
2651
|
* @returns A structured result for the channel to serialize back to the model.
|
|
1309
2652
|
*/
|
|
1310
2653
|
async executeAppClientTool(name, params) {
|
|
1311
|
-
const handler = this.
|
|
2654
|
+
const handler = this.appToolRegistry.Find(name)?.Handler;
|
|
1312
2655
|
if (!handler) {
|
|
1313
|
-
|
|
1314
|
-
return {
|
|
1315
|
-
Success: false,
|
|
1316
|
-
ErrorMessage: `No client tool named "${name}" is available on this surface. Available: ${available || '(none)'}.`
|
|
1317
|
-
};
|
|
2656
|
+
return { Success: false, ErrorMessage: this.describeMissingClientTool(name) };
|
|
1318
2657
|
}
|
|
1319
2658
|
try {
|
|
1320
2659
|
const result = await handler(params ?? {});
|
|
1321
|
-
return
|
|
2660
|
+
return reportsFailure(result)
|
|
2661
|
+
? { Success: false, ErrorMessage: failureMessage(result), Result: result }
|
|
2662
|
+
: { Success: true, Result: result };
|
|
1322
2663
|
}
|
|
1323
2664
|
catch (error) {
|
|
1324
2665
|
return { Success: false, ErrorMessage: error instanceof Error ? error.message : String(error) };
|
|
1325
2666
|
}
|
|
1326
2667
|
}
|
|
2668
|
+
/** The model-readable reason a client tool could not run. */
|
|
2669
|
+
describeMissingClientTool(name) {
|
|
2670
|
+
const available = this.appToolRegistry.Names().join(', ');
|
|
2671
|
+
const wanted = (name ?? '').trim().toLowerCase();
|
|
2672
|
+
const declared = this.resolveAppClientTools().find((t) => t.Name.trim().toLowerCase() === wanted);
|
|
2673
|
+
if (declared) {
|
|
2674
|
+
return `The client tool "${name}" is declared for this app but this surface has not registered it, so it cannot run right now. Available: ${available || '(none)'}.`;
|
|
2675
|
+
}
|
|
2676
|
+
return `No client tool named "${name}" is available on this surface. Available: ${available || '(none)'}.`;
|
|
2677
|
+
}
|
|
1327
2678
|
/**
|
|
1328
2679
|
* Runs a channel-specific GraphQL operation through the live session's provider (the
|
|
1329
2680
|
* {@link RealtimeChannelContext.ExecuteServerAction} implementation). Best-effort: any
|
|
@@ -1360,35 +2711,39 @@ export class RealtimeSessionRuntime {
|
|
|
1360
2711
|
* never affected.
|
|
1361
2712
|
*/
|
|
1362
2713
|
applyPriorChannelStates(statesJson) {
|
|
2714
|
+
this.priorChannelStates = {};
|
|
1363
2715
|
if (!statesJson) {
|
|
1364
2716
|
return;
|
|
1365
2717
|
}
|
|
1366
|
-
let states;
|
|
1367
2718
|
try {
|
|
1368
2719
|
const parsed = JSON.parse(statesJson);
|
|
1369
2720
|
if (parsed === null || typeof parsed !== 'object') {
|
|
1370
2721
|
return;
|
|
1371
2722
|
}
|
|
1372
|
-
|
|
2723
|
+
this.priorChannelStates = parsed;
|
|
1373
2724
|
}
|
|
1374
2725
|
catch {
|
|
1375
2726
|
console.warn('[RealtimeSession] PriorChannelStatesJson was malformed — starting channels fresh');
|
|
1376
2727
|
return;
|
|
1377
2728
|
}
|
|
1378
2729
|
for (const plugin of this._activeChannels$.value) {
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
2730
|
+
this.restorePriorChannelState(plugin);
|
|
2731
|
+
}
|
|
2732
|
+
}
|
|
2733
|
+
/** Offers one plugin its prior-session state, if there is one. Rejections and throws are logged, never fatal. */
|
|
2734
|
+
restorePriorChannelState(plugin) {
|
|
2735
|
+
const state = this.priorChannelStates[plugin.ChannelName];
|
|
2736
|
+
if (typeof state !== 'string' || state.length === 0) {
|
|
2737
|
+
return;
|
|
2738
|
+
}
|
|
2739
|
+
try {
|
|
2740
|
+
if (!plugin.RestoreState(state)) {
|
|
2741
|
+
console.warn(`[RealtimeSession] Channel '${plugin.ChannelName}' declined its prior-session state — starting fresh`);
|
|
1390
2742
|
}
|
|
1391
2743
|
}
|
|
2744
|
+
catch (error) {
|
|
2745
|
+
console.warn(`[RealtimeSession] Channel '${plugin.ChannelName}' restore threw — starting fresh`, error);
|
|
2746
|
+
}
|
|
1392
2747
|
}
|
|
1393
2748
|
/**
|
|
1394
2749
|
* Persists a channel's state as a first-class versioned artifact (`MJ: Artifacts`) via the
|
|
@@ -1478,6 +2833,13 @@ export class RealtimeSessionRuntime {
|
|
|
1478
2833
|
this._activeChannels$.next([]);
|
|
1479
2834
|
}
|
|
1480
2835
|
this.usedChannelNames.clear();
|
|
2836
|
+
// Channels that were prepared or advertised but never mounted were never initialized, so there is
|
|
2837
|
+
// nothing to dispose — only the bookkeeping to forget.
|
|
2838
|
+
this.discardUnmountedChannels();
|
|
2839
|
+
this.catalogNotePending = false;
|
|
2840
|
+
this.resolvedChannels.clear();
|
|
2841
|
+
this.priorChannelStates = {};
|
|
2842
|
+
this.sessionClientTools = null;
|
|
1481
2843
|
}
|
|
1482
2844
|
// ── Realtime client resolution + wiring ────────────────────────────────────
|
|
1483
2845
|
/**
|
|
@@ -1500,12 +2862,14 @@ export class RealtimeSessionRuntime {
|
|
|
1500
2862
|
}
|
|
1501
2863
|
/**
|
|
1502
2864
|
* Builds the client-direct session config the realtime client connects with.
|
|
1503
|
-
* Aggregates tracks
|
|
1504
|
-
* can negotiate them (e.g.,
|
|
2865
|
+
* Aggregates the tracks active channels source and sink under {@link REQUESTED_TRACKS_SESSION_KEY} so the
|
|
2866
|
+
* driver can negotiate them (e.g., inbound video for Whiteboard / RemoteBrowser, outbound video for a
|
|
2867
|
+
* channel that shows the agent's video). A track the model does not support resolves to `'unsupported'`.
|
|
2868
|
+
* The session's transport and relay URL go to the driver as minted (an unknown transport is left out: direct).
|
|
1505
2869
|
*/
|
|
1506
2870
|
BuildClientConfig(session) {
|
|
1507
2871
|
const sessionConfig = this.parseSessionConfig(session.SessionConfigJson);
|
|
1508
|
-
const channelTracks = this._activeChannels$.value.flatMap((c) => c.GetSourcedTracks());
|
|
2872
|
+
const channelTracks = this._activeChannels$.value.flatMap((c) => [...c.GetSourcedTracks(), ...c.GetSunkTracks()]);
|
|
1509
2873
|
if (channelTracks.length > 0) {
|
|
1510
2874
|
// The requested tracks cross a JSON boundary — the driver reads them back out of the session
|
|
1511
2875
|
// config bag (`GeminiRealtimeClient.parseSessionConfig`). A `RealtimeTrackDescriptor` is NOT
|
|
@@ -1530,11 +2894,14 @@ export class RealtimeSessionRuntime {
|
|
|
1530
2894
|
}
|
|
1531
2895
|
sessionConfig[REQUESTED_TRACKS_SESSION_KEY] = Array.from(trackMap.values());
|
|
1532
2896
|
}
|
|
2897
|
+
const transport = ParseRealtimeClientTransport(session.Transport);
|
|
1533
2898
|
return {
|
|
1534
2899
|
Provider: session.Provider,
|
|
1535
2900
|
Model: session.Model,
|
|
1536
2901
|
EphemeralToken: session.EphemeralToken,
|
|
1537
2902
|
ExpiresAt: session.ExpiresAt,
|
|
2903
|
+
...(transport ? { Transport: transport } : {}),
|
|
2904
|
+
...(session.RelayUrl ? { RelayUrl: session.RelayUrl } : {}),
|
|
1538
2905
|
SessionConfig: sessionConfig
|
|
1539
2906
|
};
|
|
1540
2907
|
}
|
|
@@ -1570,6 +2937,9 @@ export class RealtimeSessionRuntime {
|
|
|
1570
2937
|
});
|
|
1571
2938
|
client.OnError((error) => {
|
|
1572
2939
|
console.error('[RealtimeSession] Provider error event:', JSON.stringify(error), error);
|
|
2940
|
+
if (error.Fatal && this.client === client) {
|
|
2941
|
+
this.lastSessionError = error;
|
|
2942
|
+
}
|
|
1573
2943
|
});
|
|
1574
2944
|
// Usage telemetry: accumulate the driver's per-response token DELTAS and relay them to
|
|
1575
2945
|
// the server (onto the co-agent AIPromptRun) debounced + once at teardown. Providers
|
|
@@ -1586,12 +2956,36 @@ export class RealtimeSessionRuntime {
|
|
|
1586
2956
|
client.OnInterruption(() => {
|
|
1587
2957
|
this.cancelPendingNarration();
|
|
1588
2958
|
});
|
|
2959
|
+
client.OnRemoteVideo((video) => this.onAgentVideo(client, video));
|
|
2960
|
+
}
|
|
2961
|
+
/**
|
|
2962
|
+
* The model's video arrived: publish it on {@link AgentVideo$}, and mark each channel that shows it (one that sinks
|
|
2963
|
+
* outbound video) as used, so the host shows its surface. A client the session has already let go of is ignored, so a
|
|
2964
|
+
* late frame cannot bring the video back after the call.
|
|
2965
|
+
*/
|
|
2966
|
+
onAgentVideo(client, video) {
|
|
2967
|
+
if (this.client !== client) {
|
|
2968
|
+
return;
|
|
2969
|
+
}
|
|
2970
|
+
this._agentVideo$.next(video);
|
|
2971
|
+
for (const channel of this._activeChannels$.value) {
|
|
2972
|
+
if (channel.ShowsAgentVideo) {
|
|
2973
|
+
this.noteChannelActivity(channel);
|
|
2974
|
+
}
|
|
2975
|
+
}
|
|
2976
|
+
}
|
|
2977
|
+
/** Clears {@link AgentVideo$} when the call's client goes. */
|
|
2978
|
+
clearAgentVideo() {
|
|
2979
|
+
if (this._agentVideo$.value !== null) {
|
|
2980
|
+
this._agentVideo$.next(null);
|
|
2981
|
+
}
|
|
1589
2982
|
}
|
|
1590
2983
|
/** Maps a client state event onto the UI connection state. */
|
|
1591
2984
|
onClientStateChange(state) {
|
|
1592
2985
|
const mapped = this.mapClientState(state);
|
|
1593
2986
|
if (mapped) {
|
|
1594
|
-
this.
|
|
2987
|
+
this.setConnectionState(mapped);
|
|
2988
|
+
this.flushChannelCatalogNote();
|
|
1595
2989
|
}
|
|
1596
2990
|
}
|
|
1597
2991
|
/**
|
|
@@ -1620,6 +3014,17 @@ export class RealtimeSessionRuntime {
|
|
|
1620
3014
|
const state = this._connectionState$.value;
|
|
1621
3015
|
return state === 'listening' || state === 'speaking' || state === 'thinking';
|
|
1622
3016
|
}
|
|
3017
|
+
/**
|
|
3018
|
+
* Moves {@link ConnectionState$} to `state`, and {@link Resuming$} first: a `'connecting'` while the call is live, or
|
|
3019
|
+
* already resuming, is a resume; any other state ends one. Every state change goes through here.
|
|
3020
|
+
*/
|
|
3021
|
+
setConnectionState(state) {
|
|
3022
|
+
const resuming = state === 'connecting' && (this._resuming$.value || this.isSessionLive());
|
|
3023
|
+
if (resuming !== this._resuming$.value) {
|
|
3024
|
+
this._resuming$.next(resuming);
|
|
3025
|
+
}
|
|
3026
|
+
this._connectionState$.next(state);
|
|
3027
|
+
}
|
|
1623
3028
|
// ── Transcript policy ──────────────────────────────────────────────────────
|
|
1624
3029
|
/**
|
|
1625
3030
|
* Applies transcript policy to client transcript events. Interim deltas don't become
|
|
@@ -1809,7 +3214,7 @@ export class RealtimeSessionRuntime {
|
|
|
1809
3214
|
void this.relayToolTurn(call.ToolName, call.ArgumentsJson, resultJson);
|
|
1810
3215
|
return;
|
|
1811
3216
|
}
|
|
1812
|
-
this.
|
|
3217
|
+
this.setConnectionState('thinking');
|
|
1813
3218
|
if (this.inFlightCallIds.size === 0) {
|
|
1814
3219
|
// A fresh delegation burst: anchor the first-update delay and clear the digest
|
|
1815
3220
|
// buffer. Deliberately NOT reset: lastDelegationNarrationAt (the 8s spacing floor
|
|
@@ -1906,6 +3311,40 @@ export class RealtimeSessionRuntime {
|
|
|
1906
3311
|
RunID: parsed.RunID,
|
|
1907
3312
|
Artifacts: parsed.Artifacts
|
|
1908
3313
|
});
|
|
3314
|
+
if (parsed.Success && parsed.Artifacts && parsed.Artifacts.length > 0) {
|
|
3315
|
+
void this.offerDelegationArtifacts(parsed.Artifacts);
|
|
3316
|
+
}
|
|
3317
|
+
}
|
|
3318
|
+
/**
|
|
3319
|
+
* Offers a delegated run's artifacts to the channels that host artifacts (an Interactive Component channel
|
|
3320
|
+
* shows a component artifact, or swaps in a newer version of one it already shows). Each channel is asked
|
|
3321
|
+
* first whether it wants them; an advertised, unopened channel that does is mounted before it is handed
|
|
3322
|
+
* them. One channel failing never stops the others, and nothing here can fail the delegation result,
|
|
3323
|
+
* which has already been emitted.
|
|
3324
|
+
*/
|
|
3325
|
+
async offerDelegationArtifacts(artifacts) {
|
|
3326
|
+
const candidates = [
|
|
3327
|
+
...this._activeChannels$.value.map((plugin) => ({ Plugin: plugin, IsOpen: true })),
|
|
3328
|
+
...this.advertisedChannels.map((prepared) => ({ Plugin: prepared.Plugin, IsOpen: false }))
|
|
3329
|
+
];
|
|
3330
|
+
for (const { Plugin: plugin, IsOpen: isOpen } of candidates) {
|
|
3331
|
+
try {
|
|
3332
|
+
const config = this.GetResolvedChannel(plugin.ChannelName)?.Config ?? {};
|
|
3333
|
+
if (!plugin.AcceptsDelegationArtifacts(artifacts, config)) {
|
|
3334
|
+
continue;
|
|
3335
|
+
}
|
|
3336
|
+
if (!isOpen) {
|
|
3337
|
+
await this.mountAdvertisedChannel(plugin);
|
|
3338
|
+
}
|
|
3339
|
+
await plugin.OnDelegationArtifacts(artifacts);
|
|
3340
|
+
if (!isOpen) {
|
|
3341
|
+
this.noteChannelActivity(plugin); // the channel was mounted for this: reveal its tab
|
|
3342
|
+
}
|
|
3343
|
+
}
|
|
3344
|
+
catch (error) {
|
|
3345
|
+
console.error(`[RealtimeSession] Channel '${plugin.ChannelName}' failed to take a delegated run's artifacts:`, error);
|
|
3346
|
+
}
|
|
3347
|
+
}
|
|
1909
3348
|
}
|
|
1910
3349
|
// ── Explicit delegation cancellation (server cancel channel) ───────────────
|
|
1911
3350
|
/**
|
|
@@ -1996,44 +3435,34 @@ export class RealtimeSessionRuntime {
|
|
|
1996
3435
|
}
|
|
1997
3436
|
}
|
|
1998
3437
|
// ── Session minting (GraphQL) ──────────────────────────────────────────────
|
|
1999
|
-
/**
|
|
2000
|
-
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
mediaCollectionId: mediaCollectionId ?? null,
|
|
2028
|
-
applicationId: applicationId ?? null,
|
|
2029
|
-
appContextJson: appContext ? JSON.stringify(appContext) : null
|
|
2030
|
-
};
|
|
2031
|
-
const result = await this.gql().ExecuteGQL(mutation, variables);
|
|
2032
|
-
const payload = result?.StartRealtimeClientSession;
|
|
2033
|
-
if (!payload?.EphemeralToken) {
|
|
2034
|
-
throw new Error('StartRealtimeClientSession returned no ephemeral token');
|
|
2035
|
-
}
|
|
2036
|
-
return payload;
|
|
3438
|
+
/**
|
|
3439
|
+
* Mints a session through the installed {@link Launcher} — by default the stock
|
|
3440
|
+
* `StartRealtimeClientSession` mutation (see {@link DefaultRealtimeSessionLauncher}). `showsAgentVideo` is whether
|
|
3441
|
+
* one of the session's channels could show the agent's video; without one the server asks the model for no avatar.
|
|
3442
|
+
* `conversationBranchId` is the fork the session starts on; `null` is Main.
|
|
3443
|
+
*/
|
|
3444
|
+
async mintSession(targetAgentId, conversationId, lastSessionId, preferredModelId, clientTools, coAgentId, configOverridesJson, recordingConsent, recordingStartedAt, mediaCollectionId, applicationId, appContext, channelCandidatesJson, showsAgentVideo, conversationBranchId) {
|
|
3445
|
+
const result = await this._launcher.Launch({
|
|
3446
|
+
TargetAgentId: targetAgentId,
|
|
3447
|
+
ConversationId: conversationId ?? null,
|
|
3448
|
+
LastSessionId: lastSessionId ?? null,
|
|
3449
|
+
PreferredModelId: preferredModelId ?? null,
|
|
3450
|
+
ClientTools: clientTools ?? [],
|
|
3451
|
+
CoAgentId: coAgentId ?? null,
|
|
3452
|
+
ConfigOverridesJson: configOverridesJson ?? null,
|
|
3453
|
+
RecordingConsent: recordingConsent ?? false,
|
|
3454
|
+
RecordingStartedAt: recordingStartedAt ?? null,
|
|
3455
|
+
MediaCollectionId: mediaCollectionId ?? null,
|
|
3456
|
+
ApplicationId: applicationId ?? null,
|
|
3457
|
+
AppContext: appContext ?? null,
|
|
3458
|
+
ChannelCandidatesJson: channelCandidatesJson ?? null,
|
|
3459
|
+
ConversationBranchId: conversationBranchId ?? null,
|
|
3460
|
+
...(showsAgentVideo === undefined ? {} : { ShowsAgentVideo: showsAgentVideo })
|
|
3461
|
+
}, { Provider: this.Provider });
|
|
3462
|
+
if (!HasClientCredential(result)) {
|
|
3463
|
+
throw new Error('The session launcher returned no ephemeral token (or, for a relay session, no relay URL)');
|
|
3464
|
+
}
|
|
3465
|
+
return result;
|
|
2037
3466
|
}
|
|
2038
3467
|
/** Calls the `ExecuteRealtimeSessionTool` mutation; returns the ResultJson string. */
|
|
2039
3468
|
async executeSessionTool(callId, toolName, argsJson) {
|
|
@@ -2172,16 +3601,28 @@ export class RealtimeSessionRuntime {
|
|
|
2172
3601
|
/**
|
|
2173
3602
|
* Accumulates one usage DELTA from the realtime client (per-response token counts —
|
|
2174
3603
|
* the `OnUsage` contract shape) and schedules the debounced relay. Negative / non-finite
|
|
2175
|
-
* values are clamped to 0; an all-zero delta is dropped without arming the timer.
|
|
3604
|
+
* values are clamped to 0; an all-zero delta is dropped without arming the timer. The
|
|
3605
|
+
* update's input and output detail blocks add into the pending details by the record's
|
|
3606
|
+
* rules (amounts add up, inbound video running totals keep the larger value), so an update
|
|
3607
|
+
* that carries only avatar video seconds still counts.
|
|
2176
3608
|
*/
|
|
2177
3609
|
onUsageDelta(usage) {
|
|
2178
3610
|
const input = this.clampUsageDelta(usage.InputTokens);
|
|
2179
3611
|
const output = this.clampUsageDelta(usage.OutputTokens);
|
|
2180
|
-
|
|
3612
|
+
const details = AddRealtimeUsageRecord(null, { Input: usage.InputTokenDetails, Output: usage.OutputTokenDetails });
|
|
3613
|
+
const hasDetails = HasRealtimeUsage(details);
|
|
3614
|
+
if (input === 0 && output === 0 && !hasDetails) {
|
|
2181
3615
|
return;
|
|
2182
3616
|
}
|
|
2183
3617
|
this.pendingUsageInput += input;
|
|
2184
3618
|
this.pendingUsageOutput += output;
|
|
3619
|
+
if (hasDetails) {
|
|
3620
|
+
this.pendingUsageDetails = AddRealtimeUsageRecord(this.pendingUsageDetails, details);
|
|
3621
|
+
}
|
|
3622
|
+
this.armUsageFlush();
|
|
3623
|
+
}
|
|
3624
|
+
/** Schedules the debounced usage relay unless one is already pending. */
|
|
3625
|
+
armUsageFlush() {
|
|
2185
3626
|
if (!this.usageFlushTimer) {
|
|
2186
3627
|
this.usageFlushTimer = setTimeout(() => {
|
|
2187
3628
|
this.usageFlushTimer = null;
|
|
@@ -2206,24 +3647,26 @@ export class RealtimeSessionRuntime {
|
|
|
2206
3647
|
const sessionId = agentSessionId ?? this.agentSessionId;
|
|
2207
3648
|
const input = this.pendingUsageInput;
|
|
2208
3649
|
const output = this.pendingUsageOutput;
|
|
2209
|
-
|
|
3650
|
+
const details = this.pendingUsageDetails;
|
|
3651
|
+
if (!sessionId || (input === 0 && output === 0 && !details)) {
|
|
2210
3652
|
return;
|
|
2211
3653
|
}
|
|
2212
3654
|
this.pendingUsageInput = 0;
|
|
2213
3655
|
this.pendingUsageOutput = 0;
|
|
3656
|
+
this.pendingUsageDetails = null;
|
|
2214
3657
|
try {
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
}
|
|
2219
|
-
`;
|
|
2220
|
-
await this.gql().ExecuteGQL(mutation, { agentSessionId: sessionId, inputTokens: input, outputTokens: output });
|
|
3658
|
+
// The details argument is sent only when there are details, so a token-only relay is unchanged.
|
|
3659
|
+
const variables = { agentSessionId: sessionId, inputTokens: input, outputTokens: output, ...(details ? { usageDetailsJson: JSON.stringify(details) } : {}) };
|
|
3660
|
+
await this.gql().ExecuteGQL(RELAY_REALTIME_USAGE_MUTATION, variables);
|
|
2221
3661
|
}
|
|
2222
3662
|
catch (error) {
|
|
2223
3663
|
console.error('[RealtimeSession] Failed to relay usage telemetry:', error);
|
|
2224
3664
|
// Re-accumulate so a later debounce / the teardown flush retries the same deltas.
|
|
2225
3665
|
this.pendingUsageInput += input;
|
|
2226
3666
|
this.pendingUsageOutput += output;
|
|
3667
|
+
if (details) {
|
|
3668
|
+
this.pendingUsageDetails = AddRealtimeUsageRecord(this.pendingUsageDetails, details);
|
|
3669
|
+
}
|
|
2227
3670
|
}
|
|
2228
3671
|
}
|
|
2229
3672
|
/** Cancels the pending debounced usage flush and zeroes the accumulators (teardown tail). */
|
|
@@ -2234,6 +3677,7 @@ export class RealtimeSessionRuntime {
|
|
|
2234
3677
|
}
|
|
2235
3678
|
this.pendingUsageInput = 0;
|
|
2236
3679
|
this.pendingUsageOutput = 0;
|
|
3680
|
+
this.pendingUsageDetails = null;
|
|
2237
3681
|
}
|
|
2238
3682
|
// ── Delegated-run progress streaming ───────────────────────────────────────
|
|
2239
3683
|
/**
|
|
@@ -2513,42 +3957,66 @@ export class RealtimeSessionRuntime {
|
|
|
2513
3957
|
return;
|
|
2514
3958
|
}
|
|
2515
3959
|
this.teardownInFlight = this.runTeardown(closeServerSession).finally(() => {
|
|
3960
|
+
if (this._savingRecording$.value) {
|
|
3961
|
+
this._savingRecording$.next(false);
|
|
3962
|
+
}
|
|
2516
3963
|
this.teardownInFlight = null;
|
|
2517
3964
|
});
|
|
2518
3965
|
await this.teardownInFlight;
|
|
2519
3966
|
}
|
|
2520
3967
|
/** The body of {@link teardown}; never called concurrently with itself. */
|
|
2521
3968
|
async runTeardown(closeServerSession) {
|
|
3969
|
+
// Arm BEFORE any await so a tab closed in the first milliseconds of teardown is still guarded.
|
|
3970
|
+
if (this.recorder) {
|
|
3971
|
+
this._savingRecording$.next(true);
|
|
3972
|
+
}
|
|
2522
3973
|
// First: stop asserting liveness. A pulse racing the close would re-stamp LastActiveAt on a
|
|
2523
3974
|
// session we are deliberately ending, leaving an Idle row the janitor then has to age out.
|
|
2524
3975
|
this.stopLivenessPulse();
|
|
2525
3976
|
this.teardownDelegationProgress();
|
|
3977
|
+
this.stopSessionEvents();
|
|
3978
|
+
this.clientDeadline.Clear();
|
|
2526
3979
|
// Channels first: flush any unsaved channel state WHILE the live session id is still
|
|
2527
3980
|
// set (the captured per-save id covers the race anyway), then dispose the plugins.
|
|
2528
3981
|
this.flushAllChannelSaves();
|
|
2529
3982
|
this.disposeChannels();
|
|
3983
|
+
// The camera and screen share go first: the camera runs on the controller closed next.
|
|
3984
|
+
this.closeCaptures();
|
|
2530
3985
|
// Defensive: stop the mic even when Connect never ran (the client also stops the
|
|
2531
3986
|
// tracks it was handed — track.stop() is idempotent).
|
|
3987
|
+
this.closeLocalMedia();
|
|
2532
3988
|
this.localStream?.getTracks().forEach(t => t.stop());
|
|
2533
3989
|
this.localStream = null;
|
|
2534
|
-
|
|
3990
|
+
this.microphoneMuted = false;
|
|
3991
|
+
// Hand the platform back whatever opening the microphone changed. Stopping the tracks is not
|
|
2535
3992
|
// the same thing: iOS, for instance, is put into a record-and-play audio category for the call,
|
|
2536
3993
|
// and leaving it there changes the route and volume behaviour of every sound the app makes
|
|
2537
|
-
// afterwards.
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
|
|
2541
|
-
|
|
2542
|
-
|
|
3994
|
+
// afterwards. A microphone still opening is left to its start, which releases it once the
|
|
3995
|
+
// opening returns ({@link microphoneLease}).
|
|
3996
|
+
const lease = this.microphoneLease;
|
|
3997
|
+
if (lease && !lease.Opening) {
|
|
3998
|
+
this.microphoneLease = null;
|
|
3999
|
+
await this.releaseMicrophone();
|
|
2543
4000
|
}
|
|
2544
4001
|
if (this.client) {
|
|
2545
|
-
|
|
2546
|
-
|
|
4002
|
+
// A stranded Active session is worse than a noisy socket: a driver that throws on Disconnect
|
|
4003
|
+
// (socket already gone) must not stop the server session from being closed below.
|
|
4004
|
+
try {
|
|
4005
|
+
await this.client.Disconnect();
|
|
4006
|
+
}
|
|
4007
|
+
catch (error) {
|
|
4008
|
+
console.error(`[RealtimeSession] Driver failed to disconnect (session ${this.agentSessionId}); continuing teardown:`, error);
|
|
4009
|
+
}
|
|
4010
|
+
finally {
|
|
4011
|
+
this.unwatchVideoSources();
|
|
4012
|
+
this.clearAgentVideo();
|
|
4013
|
+
this.client = null;
|
|
4014
|
+
}
|
|
2547
4015
|
}
|
|
2548
|
-
// Stop +
|
|
2549
|
-
//
|
|
2550
|
-
// its own errors.
|
|
2551
|
-
await this.
|
|
4016
|
+
// Stop + encode the call recording now (the recorder must be detached before the session id is
|
|
4017
|
+
// cleared); the upload itself happens after the close below. Best-effort — stopRecording
|
|
4018
|
+
// swallows its own errors. Null when nothing was recorded.
|
|
4019
|
+
const recording = await this.stopRecording();
|
|
2552
4020
|
this.recordingStartedAtIso = null;
|
|
2553
4021
|
// Final usage flush WHILE the live session id is still set (the relay mutation also
|
|
2554
4022
|
// accepts a Closed session, so ordering vs. CloseAgentSession is belt-and-braces).
|
|
@@ -2561,18 +4029,27 @@ export class RealtimeSessionRuntime {
|
|
|
2561
4029
|
if (closeServerSession && this.agentSessionId) {
|
|
2562
4030
|
await this.closeServerSession(this.agentSessionId);
|
|
2563
4031
|
}
|
|
4032
|
+
// Upload AFTER the close. The WAV is tens of MB and takes seconds; a page unload aborts the
|
|
4033
|
+
// request, and nothing after an aborted await ever runs — so the small, critical close goes
|
|
4034
|
+
// first (#5195). The server accepts the upload on a Closed session (loadOwnedSession does not
|
|
4035
|
+
// check status), and uploadStoppedRecording never throws.
|
|
4036
|
+
await this.uploadStoppedRecording(this.agentSessionId, recording);
|
|
2564
4037
|
// Capture the session id BEFORE we null it so the lifecycle emit carries it.
|
|
2565
4038
|
// Skip emitting when there was no live session (defensive — teardown is safe
|
|
2566
4039
|
// to call without an active session).
|
|
2567
4040
|
const closedSessionId = this.agentSessionId;
|
|
2568
4041
|
this.agentSessionId = null;
|
|
4042
|
+
this.sessionConversationId = null;
|
|
2569
4043
|
this.narrationTemplate = null;
|
|
2570
4044
|
this.clientToolHandlers.clear();
|
|
2571
4045
|
this._modelName$.next(null);
|
|
4046
|
+
if (this._avatarNotice$.value !== null) {
|
|
4047
|
+
this._avatarNotice$.next(null);
|
|
4048
|
+
}
|
|
2572
4049
|
this.SetMinimized(false);
|
|
2573
4050
|
this._active$.next(false);
|
|
2574
4051
|
if (this._connectionState$.value !== 'error') {
|
|
2575
|
-
this.
|
|
4052
|
+
this.setConnectionState('closed');
|
|
2576
4053
|
}
|
|
2577
4054
|
// Surface generic session-ended for the conversations runtime bridge.
|
|
2578
4055
|
// `closeServerSession=true` means the user explicitly called EndRealtimeSession;
|
|
@@ -2584,6 +4061,18 @@ export class RealtimeSessionRuntime {
|
|
|
2584
4061
|
});
|
|
2585
4062
|
}
|
|
2586
4063
|
}
|
|
4064
|
+
/**
|
|
4065
|
+
* Calls the host's {@link IRealtimeMediaHost.ReleaseMicrophone}. Best-effort by contract: a failure
|
|
4066
|
+
* is logged and never thrown, because it must never block ending a call.
|
|
4067
|
+
*/
|
|
4068
|
+
async releaseMicrophone() {
|
|
4069
|
+
try {
|
|
4070
|
+
await this.mediaHost.ReleaseMicrophone?.();
|
|
4071
|
+
}
|
|
4072
|
+
catch (error) {
|
|
4073
|
+
console.error('[RealtimeSession] Media host failed to release the microphone:', error);
|
|
4074
|
+
}
|
|
4075
|
+
}
|
|
2587
4076
|
/** Calls the `CloseAgentSession` mutation (provisioned in P4b). */
|
|
2588
4077
|
async closeServerSession(agentSessionId) {
|
|
2589
4078
|
try {
|
|
@@ -2610,12 +4099,13 @@ export class RealtimeSessionRuntime {
|
|
|
2610
4099
|
this.hasActiveInterimUserCaption = false;
|
|
2611
4100
|
this.SetMinimized(false);
|
|
2612
4101
|
this.stopSegmentFlushing();
|
|
2613
|
-
this.
|
|
4102
|
+
this.shardQueue = null;
|
|
2614
4103
|
this.recorder = null;
|
|
2615
4104
|
this.recordingStartedAtIso = null;
|
|
2616
4105
|
this.currentTurnStartMs = null;
|
|
2617
4106
|
this.turnAudioStartCaptured = false;
|
|
2618
4107
|
this.usedChannelNames.clear();
|
|
4108
|
+
this.microphoneMuted = false;
|
|
2619
4109
|
}
|
|
2620
4110
|
/** The GraphQL provider used for relay mutations. */
|
|
2621
4111
|
gql() {
|