@memberjunction/realtime-runtime 6.2.0-edge.3 → 6.2.0-edge.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +75 -0
- package/dist/channels/base-realtime-channel-client.d.ts +535 -28
- package/dist/channels/base-realtime-channel-client.d.ts.map +1 -1
- package/dist/channels/base-realtime-channel-client.js +864 -10
- package/dist/channels/base-realtime-channel-client.js.map +1 -1
- package/dist/channels/channel-action-dispatcher.d.ts +70 -0
- package/dist/channels/channel-action-dispatcher.d.ts.map +1 -0
- package/dist/channels/channel-action-dispatcher.js +136 -0
- package/dist/channels/channel-action-dispatcher.js.map +1 -0
- package/dist/channels/channel-catalog-note.d.ts +48 -0
- package/dist/channels/channel-catalog-note.d.ts.map +1 -0
- package/dist/channels/channel-catalog-note.js +119 -0
- package/dist/channels/channel-catalog-note.js.map +1 -0
- package/dist/channels/channel-contract-types.d.ts +107 -0
- package/dist/channels/channel-contract-types.d.ts.map +1 -0
- package/dist/channels/channel-contract-types.js +12 -0
- package/dist/channels/channel-contract-types.js.map +1 -0
- package/dist/channels/channel-descriptor-synthesis.d.ts +56 -0
- package/dist/channels/channel-descriptor-synthesis.d.ts.map +1 -0
- package/dist/channels/channel-descriptor-synthesis.js +90 -0
- package/dist/channels/channel-descriptor-synthesis.js.map +1 -0
- package/dist/channels/channel-perception.d.ts +82 -0
- package/dist/channels/channel-perception.d.ts.map +1 -0
- package/dist/channels/channel-perception.js +120 -0
- package/dist/channels/channel-perception.js.map +1 -0
- package/dist/channels/channel-schema-format.d.ts +17 -0
- package/dist/channels/channel-schema-format.d.ts.map +1 -0
- package/dist/channels/channel-schema-format.js +45 -0
- package/dist/channels/channel-schema-format.js.map +1 -0
- package/dist/channels/channel-state-delta.d.ts +59 -0
- package/dist/channels/channel-state-delta.d.ts.map +1 -0
- package/dist/channels/channel-state-delta.js +116 -0
- package/dist/channels/channel-state-delta.js.map +1 -0
- package/dist/channels/channel-surface-placement.d.ts +25 -0
- package/dist/channels/channel-surface-placement.d.ts.map +1 -0
- package/dist/channels/channel-surface-placement.js +29 -0
- package/dist/channels/channel-surface-placement.js.map +1 -0
- package/dist/channels/channel-visual-pump.d.ts +126 -0
- package/dist/channels/channel-visual-pump.d.ts.map +1 -0
- package/dist/channels/channel-visual-pump.js +174 -0
- package/dist/channels/channel-visual-pump.js.map +1 -0
- package/dist/hosts/IRealtimeMediaHost.d.ts +86 -34
- package/dist/hosts/IRealtimeMediaHost.d.ts.map +1 -1
- package/dist/hosts/IRealtimeMediaHost.js +2 -1
- package/dist/hosts/IRealtimeMediaHost.js.map +1 -1
- package/dist/index.d.ts +26 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +28 -1
- package/dist/index.js.map +1 -1
- package/dist/session/RealtimeSessionRuntime.d.ts +984 -111
- package/dist/session/RealtimeSessionRuntime.d.ts.map +1 -1
- package/dist/session/RealtimeSessionRuntime.js +2396 -381
- package/dist/session/RealtimeSessionRuntime.js.map +1 -1
- package/dist/session/RecordingShardDrain.d.ts +59 -0
- package/dist/session/RecordingShardDrain.d.ts.map +1 -0
- package/dist/session/RecordingShardDrain.js +135 -0
- package/dist/session/RecordingShardDrain.js.map +1 -0
- package/dist/session/RecordingShardQueue.d.ts +94 -0
- package/dist/session/RecordingShardQueue.d.ts.map +1 -0
- package/dist/session/RecordingShardQueue.js +176 -0
- package/dist/session/RecordingShardQueue.js.map +1 -0
- package/dist/session/RecordingShardStore.d.ts +57 -0
- package/dist/session/RecordingShardStore.d.ts.map +1 -0
- package/dist/session/RecordingShardStore.js +184 -0
- package/dist/session/RecordingShardStore.js.map +1 -0
- package/dist/session/RecordingShardWindow.d.ts +42 -0
- package/dist/session/RecordingShardWindow.d.ts.map +1 -0
- package/dist/session/RecordingShardWindow.js +88 -0
- package/dist/session/RecordingShardWindow.js.map +1 -0
- package/dist/session/RecordingTransport.d.ts +66 -0
- package/dist/session/RecordingTransport.d.ts.map +1 -0
- package/dist/session/RecordingTransport.js +61 -0
- package/dist/session/RecordingTransport.js.map +1 -0
- package/dist/session/app-client-tool-registry.d.ts +76 -0
- package/dist/session/app-client-tool-registry.d.ts.map +1 -0
- package/dist/session/app-client-tool-registry.js +100 -0
- package/dist/session/app-client-tool-registry.js.map +1 -0
- package/dist/session/avatar-notice.d.ts +43 -0
- package/dist/session/avatar-notice.d.ts.map +1 -0
- package/dist/session/avatar-notice.js +60 -0
- package/dist/session/avatar-notice.js.map +1 -0
- package/dist/session/call-end.d.ts +21 -0
- package/dist/session/call-end.d.ts.map +1 -0
- package/dist/session/call-end.js +19 -0
- package/dist/session/call-end.js.map +1 -0
- package/dist/session/channel-exposure-preferences.d.ts +70 -0
- package/dist/session/channel-exposure-preferences.d.ts.map +1 -0
- package/dist/session/channel-exposure-preferences.js +138 -0
- package/dist/session/channel-exposure-preferences.js.map +1 -0
- package/dist/session/channel-session-scope.d.ts +142 -0
- package/dist/session/channel-session-scope.d.ts.map +1 -0
- package/dist/session/channel-session-scope.js +128 -0
- package/dist/session/channel-session-scope.js.map +1 -0
- package/dist/session/client-session-deadline.d.ts +28 -0
- package/dist/session/client-session-deadline.d.ts.map +1 -0
- package/dist/session/client-session-deadline.js +50 -0
- package/dist/session/client-session-deadline.js.map +1 -0
- package/dist/session/realtime-captures.d.ts +236 -0
- package/dist/session/realtime-captures.d.ts.map +1 -0
- package/dist/session/realtime-captures.js +383 -0
- package/dist/session/realtime-captures.js.map +1 -0
- package/dist/session/realtime-microphone.d.ts +50 -0
- package/dist/session/realtime-microphone.d.ts.map +1 -0
- package/dist/session/realtime-microphone.js +51 -0
- package/dist/session/realtime-microphone.js.map +1 -0
- package/dist/session/session-event-hub.d.ts +72 -0
- package/dist/session/session-event-hub.d.ts.map +1 -0
- package/dist/session/session-event-hub.js +151 -0
- package/dist/session/session-event-hub.js.map +1 -0
- package/dist/session/session-launcher.d.ts +128 -0
- package/dist/session/session-launcher.d.ts.map +1 -0
- package/dist/session/session-launcher.js +160 -0
- package/dist/session/session-launcher.js.map +1 -0
- package/package.json +10 -9
|
@@ -1,13 +1,48 @@
|
|
|
1
1
|
import { BehaviorSubject, Subject } from 'rxjs';
|
|
2
2
|
import { Metadata } from '@memberjunction/core';
|
|
3
3
|
import { UserInfoEngine } from '@memberjunction/core-entities';
|
|
4
|
-
import { AIEngineBase } from '@memberjunction/ai-engine-base';
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
4
|
+
import { AddRealtimeUsageRecord, AIEngineBase, HasRealtimeUsage } from '@memberjunction/ai-engine-base';
|
|
5
|
+
import { GraphQLRealtimeSessionClient } from '@memberjunction/graphql-dataprovider';
|
|
6
|
+
import { MJGlobal, UUIDsEqual } from '@memberjunction/global';
|
|
7
|
+
import { DEFAULT_REALTIME_AUDIO_TRACKS, ParseRealtimeAvatarStatus, ParseRealtimeClientTransport } from '@memberjunction/ai';
|
|
8
|
+
import { CompareExposure, DeclaresNativeTools, IsIdentityVerifiedEventPayload, NormalizeChannelKey, ParseRealtimeSessionClientPolicy, RECORDING_TERMINAL_MAX_BYTES, ResolveClientTools, SelectNativeChannelTools, UserExposureReason } from '@memberjunction/ai-core-plus';
|
|
9
|
+
import { BaseRealtimeClient, LoadAssemblyAIRealtimeClient, LoadElevenLabsRealtimeClient, LoadGeminiEnterpriseRealtimeClient, LoadGeminiRealtimeClient, LoadHuggingFaceRealtimeClient, LoadOpenAIRealtimeClient, LoadxAIRealtimeClient, VideoSourceArbiter, REQUESTED_TRACKS_SESSION_KEY } from '@memberjunction/ai-realtime-client';
|
|
10
|
+
import { DefaultRealtimeSessionLauncher, HasClientCredential } from './session-launcher.js';
|
|
11
|
+
import { RequestsAgentVideo, ResolveAvatarNotice } from './avatar-notice.js';
|
|
12
|
+
import { ClientSessionDeadline } from './client-session-deadline.js';
|
|
13
|
+
import { RealtimeCaptures, REALTIME_CAPTURES_OFF, REALTIME_CAPTURE_OFFERS_NONE, } from './realtime-captures.js';
|
|
14
|
+
import { REALTIME_MICROPHONE_NONE, ReadMicrophoneState, SameMicrophoneState } from './realtime-microphone.js';
|
|
15
|
+
import { RealtimeSessionEventHub } from './session-event-hub.js';
|
|
8
16
|
import { BuildNarrationInstructions } from '../narration/narration-template.js';
|
|
9
17
|
import { ParseDelegationResultJson, FormatToolName } from './delegation-result-parser.js';
|
|
10
|
-
import { BaseRealtimeChannelClient } from '../channels/base-realtime-channel-client.js';
|
|
18
|
+
import { BaseRealtimeChannelClient, ChannelVideoSourceID } from '../channels/base-realtime-channel-client.js';
|
|
19
|
+
import { DEFAULT_CHANNEL_SURFACE_PLACEMENT, ReadChannelSurfacePlacement } from '../channels/channel-surface-placement.js';
|
|
20
|
+
import { ChannelActionDispatcher } from '../channels/channel-action-dispatcher.js';
|
|
21
|
+
import { BuildChannelCatalogNote } from '../channels/channel-catalog-note.js';
|
|
22
|
+
import { AppClientToolRegistry, DEFAULT_APP_TOOL_OWNER } from './app-client-tool-registry.js';
|
|
23
|
+
import { UserSettingsExposurePreferences } from './channel-exposure-preferences.js';
|
|
24
|
+
import { BuildChannelCandidate, FindPreparedChannel, MergeToolMetadata, PreparedChannelsShowAgentVideo, ReconcileChannelsWithPolicy, ResolveLocalChannelScope, ToolsByChannelKey } from './channel-session-scope.js';
|
|
25
|
+
import { RecordingShardQueue } from './RecordingShardQueue.js';
|
|
26
|
+
import { RecordingShardWindow } from './RecordingShardWindow.js';
|
|
27
|
+
import { CreateRecordingShardStore } from './RecordingShardStore.js';
|
|
28
|
+
import { CreateRecordingTransportForProvider, DeleteStoredRecordingShard, DrainStoredRecordingShardsOnce } from './RecordingShardDrain.js';
|
|
29
|
+
/**
|
|
30
|
+
* How long teardown waits for the server to acknowledge the recording's terminal request, measured
|
|
31
|
+
* from after `CloseAgentSession`. The terminal is `keepalive`, so it completes even if the page goes
|
|
32
|
+
* away; the wait only keeps `SavingRecording$` (and so a host's "don't close yet" guard) up while the
|
|
33
|
+
* server has not confirmed. Bounded so a dead server can never hold the End of a call hostage —
|
|
34
|
+
* unacknowledged shards stay in the local store and the server janitor assembles what arrived.
|
|
35
|
+
*/
|
|
36
|
+
export const RECORDING_TERMINAL_ACK_TIMEOUT_MS = 3000;
|
|
37
|
+
/**
|
|
38
|
+
* How long teardown waits for the recorder to stop and flush its encoder. `Stop()` runs BEFORE
|
|
39
|
+
* `CloseAgentSession`, so a wedged encoder must not be able to hold the End of a call (and the
|
|
40
|
+
* server session) open; past this the recording ends at the packets already encoded.
|
|
41
|
+
*/
|
|
42
|
+
export const RECORDER_STOP_TIMEOUT_MS = 2000;
|
|
43
|
+
function describeRecordingFailure(failure) {
|
|
44
|
+
return failure instanceof Error ? failure.message : String(failure);
|
|
45
|
+
}
|
|
11
46
|
/**
|
|
12
47
|
* `MJ: User Settings` key for the per-user "record this voice call" consent toggle. Stored as
|
|
13
48
|
* the literal string `'true'`/`'false'` (read with `=== 'true'`), cross-device via
|
|
@@ -15,6 +50,15 @@ import { BaseRealtimeChannelClient } from '../channels/base-realtime-channel-cli
|
|
|
15
50
|
* default when the caller doesn't pass an explicit consent value.
|
|
16
51
|
*/
|
|
17
52
|
export const REALTIME_RECORDING_CONSENT_KEY = 'mj.realtimeVoice.recordingConsent.v1';
|
|
53
|
+
/**
|
|
54
|
+
* Relays usage onto the co-agent prompt run: token deltas, and the per-modality detail blocks as a
|
|
55
|
+
* JSON record (sent only when there are any; see `RelayRealtimeUsage` in MJServer).
|
|
56
|
+
*/
|
|
57
|
+
const RELAY_REALTIME_USAGE_MUTATION = `
|
|
58
|
+
mutation RelayRealtimeUsage($agentSessionId: String!, $inputTokens: Int!, $outputTokens: Int!, $usageDetailsJson: String) {
|
|
59
|
+
RelayRealtimeUsage(agentSessionId: $agentSessionId, inputTokens: $inputTokens, outputTokens: $outputTokens, usageDetailsJson: $usageDetailsJson)
|
|
60
|
+
}
|
|
61
|
+
`;
|
|
18
62
|
// Tree-shaking prevention: the OpenAI client is resolved dynamically through the
|
|
19
63
|
// ClassFactory (by the server-reported Provider key), so this static call is what keeps
|
|
20
64
|
// its @RegisterClass side effect from being eliminated by the bundler.
|
|
@@ -24,10 +68,37 @@ export const REALTIME_RECORDING_CONSENT_KEY = 'mj.realtimeVoice.recordingConsent
|
|
|
24
68
|
// and this service stays component-free (it must stay importable in plain-node tests).
|
|
25
69
|
LoadOpenAIRealtimeClient();
|
|
26
70
|
LoadGeminiRealtimeClient();
|
|
71
|
+
LoadGeminiEnterpriseRealtimeClient();
|
|
27
72
|
LoadElevenLabsRealtimeClient();
|
|
28
73
|
LoadAssemblyAIRealtimeClient();
|
|
29
74
|
LoadxAIRealtimeClient();
|
|
30
75
|
LoadHuggingFaceRealtimeClient();
|
|
76
|
+
/**
|
|
77
|
+
* A registry row's surface placement. A row whose `UIConfig` is not valid JSON places its surface as a row without one
|
|
78
|
+
* does, so one bad row never costs the session its other channels.
|
|
79
|
+
*/
|
|
80
|
+
function readRowSurfacePlacement(row) {
|
|
81
|
+
try {
|
|
82
|
+
return ReadChannelSurfacePlacement(row.UIConfigObject);
|
|
83
|
+
}
|
|
84
|
+
catch {
|
|
85
|
+
console.warn(`[RealtimeSession] Channel '${row.Name}' has a UIConfig that is not valid JSON; its surface starts on its tab.`);
|
|
86
|
+
return DEFAULT_CHANNEL_SURFACE_PLACEMENT;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
/** The same, for a row read over GraphQL, whose `UIConfig` arrives as the column's JSON text. */
|
|
90
|
+
function readUIConfigTextSurfacePlacement(name, uiConfig) {
|
|
91
|
+
if (!uiConfig) {
|
|
92
|
+
return ReadChannelSurfacePlacement(null);
|
|
93
|
+
}
|
|
94
|
+
try {
|
|
95
|
+
return ReadChannelSurfacePlacement(JSON.parse(uiConfig));
|
|
96
|
+
}
|
|
97
|
+
catch {
|
|
98
|
+
console.warn(`[RealtimeSession] Channel '${name}' has a UIConfig that is not valid JSON; its surface starts on its tab.`);
|
|
99
|
+
return DEFAULT_CHANNEL_SURFACE_PLACEMENT;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
31
102
|
/**
|
|
32
103
|
* Converts a {@link RealtimeTrackDescriptor} to its JSON form for the session-config bag.
|
|
33
104
|
*
|
|
@@ -67,6 +138,37 @@ function trackKeyFromJSON(raw) {
|
|
|
67
138
|
}
|
|
68
139
|
return `${direction}:${modality}`;
|
|
69
140
|
}
|
|
141
|
+
/** Whether a surface client tool handler's result is an object whose `Success` is `false`. */
|
|
142
|
+
function reportsFailure(result) {
|
|
143
|
+
return typeof result === 'object' && result !== null && 'Success' in result && result.Success === false;
|
|
144
|
+
}
|
|
145
|
+
/** The failed result's `ErrorMessage` when it is text that is not blank, else `'Tool failed'`. */
|
|
146
|
+
function failureMessage(result) {
|
|
147
|
+
const message = result.ErrorMessage;
|
|
148
|
+
return typeof message === 'string' && message.trim() ? message : 'Tool failed';
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* The `DOMException` name for each way the controller can fail to start the microphone, so a host that
|
|
152
|
+
* tells a denied microphone apart (`error.name === 'NotAllowedError'`) works the same with or without one.
|
|
153
|
+
*/
|
|
154
|
+
const MICROPHONE_ERROR_NAMES = {
|
|
155
|
+
denied: 'NotAllowedError',
|
|
156
|
+
'not-found': 'NotFoundError',
|
|
157
|
+
'in-use': 'NotReadableError',
|
|
158
|
+
unsupported: 'NotSupportedError',
|
|
159
|
+
error: 'Error',
|
|
160
|
+
};
|
|
161
|
+
/** What the user is told when the call's policy refuses a capture. */
|
|
162
|
+
const CAPTURE_WORDING = {
|
|
163
|
+
camera: { NotInCall: 'The camera is not part of this call.', NotAllowed: 'This call cannot show the agent your camera' },
|
|
164
|
+
screen: { NotInCall: 'Screen sharing is not part of this call.', NotAllowed: 'This call cannot show the agent your screen' },
|
|
165
|
+
};
|
|
166
|
+
/** The error a failed microphone start reports, named as `getUserMedia` would have named it. */
|
|
167
|
+
function microphoneStartError(reason, message) {
|
|
168
|
+
const error = new Error(message);
|
|
169
|
+
error.name = MICROPHONE_ERROR_NAMES[reason];
|
|
170
|
+
return error;
|
|
171
|
+
}
|
|
70
172
|
/**
|
|
71
173
|
* Drives a **client-direct** real-time voice session: the browser mints an ephemeral
|
|
72
174
|
* token from the MJ server, then connects DIRECTLY to the realtime provider. Audio
|
|
@@ -92,14 +194,15 @@ function trackKeyFromJSON(raw) {
|
|
|
92
194
|
export class RealtimeSessionRuntime {
|
|
93
195
|
/**
|
|
94
196
|
* @param mediaHost the platform's media capabilities. The runtime never touches `navigator`,
|
|
95
|
-
* `Blob` or `FileReader` itself — microphone acquisition and audio
|
|
96
|
-
* are the host's, because both are platform-specific
|
|
97
|
-
*
|
|
197
|
+
* `Blob` or `FileReader` itself — microphone acquisition and audio capture/encoding
|
|
198
|
+
* are the host's, because both are platform-specific (permission UX, which
|
|
199
|
+
* encoder the platform offers).
|
|
98
200
|
*/
|
|
99
201
|
constructor(mediaHost) {
|
|
100
202
|
this.mediaHost = mediaHost;
|
|
101
203
|
// ── Reactive UI state ──────────────────────────────────────────────────────
|
|
102
204
|
this._connectionState$ = new BehaviorSubject('closed');
|
|
205
|
+
this._resuming$ = new BehaviorSubject(false);
|
|
103
206
|
this._captions$ = new BehaviorSubject([]);
|
|
104
207
|
this._active$ = new BehaviorSubject(false);
|
|
105
208
|
this._delegationProgress$ = new Subject();
|
|
@@ -109,6 +212,7 @@ export class RealtimeSessionRuntime {
|
|
|
109
212
|
this._agentName$ = new BehaviorSubject('Sage');
|
|
110
213
|
this._modelName$ = new BehaviorSubject(null);
|
|
111
214
|
this._minimized$ = new BehaviorSubject(false);
|
|
215
|
+
this._savingRecording$ = new BehaviorSubject(false);
|
|
112
216
|
this._activeChannels$ = new BehaviorSubject([]);
|
|
113
217
|
this._channelFocus$ = new Subject();
|
|
114
218
|
// ─── Generic session-lifecycle events (consumed by RealtimeSessionsAdapter to
|
|
@@ -122,6 +226,16 @@ export class RealtimeSessionRuntime {
|
|
|
122
226
|
this._channelActivity$ = new Subject();
|
|
123
227
|
/** Current connection / turn state. */
|
|
124
228
|
this.ConnectionState$ = this._connectionState$.asObservable();
|
|
229
|
+
/**
|
|
230
|
+
* Whether the call is resuming: moving to a new connection with its provider mid-call. `true` from a `'connecting'` the
|
|
231
|
+
* driver reports while the call is live, until the call is live again (`'listening'`, `'speaking'` or `'thinking'`),
|
|
232
|
+
* fails (`'error'`) or ends (`'closed'`). A call's first connect is not a resume, so it is `false` until the call has
|
|
233
|
+
* been live, and outside a call. Gemini Live resumes when Google ends a connection (`goAway`) and after a dropped
|
|
234
|
+
* socket, and the agent's video can stop for a few seconds meanwhile: a host can show that as a pause rather than as
|
|
235
|
+
* the agent leaving. It changes before {@link ConnectionState$} does, so a resume's `'connecting'` arrives with it
|
|
236
|
+
* already `true`.
|
|
237
|
+
*/
|
|
238
|
+
this.Resuming$ = this._resuming$.asObservable();
|
|
125
239
|
/** Live captions for both sides of the conversation. */
|
|
126
240
|
this.Captions$ = this._captions$.asObservable();
|
|
127
241
|
/** True while a session is open (mic button active, overlay shown). */
|
|
@@ -158,6 +272,15 @@ export class RealtimeSessionRuntime {
|
|
|
158
272
|
* pure presentation state, reset to `false` at session start and teardown.
|
|
159
273
|
*/
|
|
160
274
|
this.Minimized$ = this._minimized$.asObservable();
|
|
275
|
+
/**
|
|
276
|
+
* True while the end-of-call recording is being saved: from the first statement of a teardown
|
|
277
|
+
* that holds a recorder until that teardown settles — which includes waiting (at most
|
|
278
|
+
* {@link RECORDING_TERMINAL_ACK_TIMEOUT_MS}) for the server to acknowledge the recording's terminal
|
|
279
|
+
* request. False at construction and for unrecorded sessions. Exists for #5195; the runtime is
|
|
280
|
+
* platform-free, so hosts own the reaction (a "saving" indicator, a disabled navigation button,
|
|
281
|
+
* etc.).
|
|
282
|
+
*/
|
|
283
|
+
this.SavingRecording$ = this._savingRecording$.asObservable();
|
|
161
284
|
/**
|
|
162
285
|
* The session's ACTIVE interactive-channel plugins, resolved from the `MJ: AI Agent
|
|
163
286
|
* Channels` registry at session start (one instance per session, per channel). Emits
|
|
@@ -165,6 +288,57 @@ export class RealtimeSessionRuntime {
|
|
|
165
288
|
* one surface tab per plugin — it never knows any concrete channel type.
|
|
166
289
|
*/
|
|
167
290
|
this.ActiveChannels$ = this._activeChannels$.asObservable();
|
|
291
|
+
this._videoSources$ = new BehaviorSubject([]);
|
|
292
|
+
/**
|
|
293
|
+
* The video sources the agent is perceiving or could perceive right now (a whiteboard, a remote browser, a
|
|
294
|
+
* shared screen), with whether each is on and whether its frames are reaching the model. Empty when no
|
|
295
|
+
* session is live or nothing has offered a frame source. This is what the "agent can see" control renders;
|
|
296
|
+
* toggle one with {@link SetVideoSourceEnabled}.
|
|
297
|
+
*/
|
|
298
|
+
this.VideoSources$ = this._videoSources$.asObservable();
|
|
299
|
+
this._captures$ = new BehaviorSubject(REALTIME_CAPTURES_OFF);
|
|
300
|
+
/**
|
|
301
|
+
* The user's camera and screen share: off, starting, on (with the stream to show the user) or failed (with the
|
|
302
|
+
* reason). Both off outside a session. Start and stop them with {@link StartCamera}, {@link StartScreenShare},
|
|
303
|
+
* {@link StopCamera} and {@link StopScreenShare}; while one is on, it is also a source on {@link VideoSources$}.
|
|
304
|
+
*/
|
|
305
|
+
this.Captures$ = this._captures$.asObservable();
|
|
306
|
+
this._captureOffers$ = new BehaviorSubject(REALTIME_CAPTURE_OFFERS_NONE);
|
|
307
|
+
/**
|
|
308
|
+
* Which captures the call offers, now and on every change. A capture is offered while the call is connected, the model
|
|
309
|
+
* takes inbound video, the host can open it (a camera controller; screen sharing) and the call's policy admits it (its
|
|
310
|
+
* channel is in the call and may show the agent pixels). A host shows its Camera and Share controls from this. Neither
|
|
311
|
+
* is offered outside a call.
|
|
312
|
+
*/
|
|
313
|
+
this.CaptureOffers$ = this._captureOffers$.asObservable();
|
|
314
|
+
this._microphone$ = new BehaviorSubject(REALTIME_MICROPHONE_NONE);
|
|
315
|
+
/**
|
|
316
|
+
* The call's microphone, now and on every change: the one in use and the ones the user can move it to with
|
|
317
|
+
* {@link SwitchMicrophone}, as the host's camera-and-microphone controller lists them. Offered from the moment the call
|
|
318
|
+
* connects until it ends, and kept current as devices come and go. When the call loses its microphone (the one in use
|
|
319
|
+
* fails and so does the fallback to the default, or the last one goes away), its `Failure` says why: the agent hears
|
|
320
|
+
* nothing until the user picks a microphone, which opens it again. {@link REALTIME_MICROPHONE_NONE} outside a call and
|
|
321
|
+
* on a host without a controller, which opens the microphone itself (`IRealtimeMediaHost.AcquireMicrophone`).
|
|
322
|
+
*/
|
|
323
|
+
this.Microphone$ = this._microphone$.asObservable();
|
|
324
|
+
this._agentVideo$ = new BehaviorSubject(null);
|
|
325
|
+
/**
|
|
326
|
+
* The agent's video (an avatar) while the model sends it: a live stream, or a player that owns the `<video>` element.
|
|
327
|
+
* `null` until the video arrives, for a model that sends none, and outside a call. The driver hands it over once it is
|
|
328
|
+
* live, and a newer one replaces it. A host shows it with `AttachVideoSource`; a channel that sinks outbound video
|
|
329
|
+
* follows it through its context, and counts as used once it arrives, so the host shows that channel's surface.
|
|
330
|
+
*/
|
|
331
|
+
this.AgentVideo$ = this._agentVideo$.asObservable();
|
|
332
|
+
this._avatarNotice$ = new BehaviorSubject(null);
|
|
333
|
+
/**
|
|
334
|
+
* Why the call shows no avatar the agent asked for: set once per call, after it connects, when the model renders none,
|
|
335
|
+
* the persona has no face for it, the app shows no agent video (`host`) or the browser can't play it (`browser`); with
|
|
336
|
+
* no `Reason` when the mint gave one this version doesn't know, such as a newer server's. `null` when there is nothing
|
|
337
|
+
* to say (no avatar asked for, the avatar shows, or the mint reported no status) and after the call ends. A provider
|
|
338
|
+
* resume inside the call does not set it again. A host turns the reason into words once, for example "Audio only: this
|
|
339
|
+
* voice model can't show an avatar".
|
|
340
|
+
*/
|
|
341
|
+
this.AvatarNotice$ = this._avatarNotice$.asObservable();
|
|
168
342
|
/**
|
|
169
343
|
* Channel requests to enter / leave the FOCUS layout (see
|
|
170
344
|
* {@link RealtimeChannelFocusEvent}). Fired when a plugin calls its host context's
|
|
@@ -208,7 +382,11 @@ export class RealtimeSessionRuntime {
|
|
|
208
382
|
*/
|
|
209
383
|
/** Conversation id the SERVER created for this session (null when the host supplied one). */
|
|
210
384
|
this.createdConversationId = null;
|
|
211
|
-
/**
|
|
385
|
+
/**
|
|
386
|
+
* The session's conversation id (supplied or server-created). While a start's mint runs, the
|
|
387
|
+
* conversation the start asked for; null when it asked the server to create one. Null after
|
|
388
|
+
* teardown.
|
|
389
|
+
*/
|
|
212
390
|
this.sessionConversationId = null;
|
|
213
391
|
/** First final user utterance of the live session (the naming seed). */
|
|
214
392
|
this.firstUserTranscript = null;
|
|
@@ -219,8 +397,28 @@ export class RealtimeSessionRuntime {
|
|
|
219
397
|
// ── Session internals ──────────────────────────────────────────────────────
|
|
220
398
|
/** The provider-direct realtime client driving the live session (ClassFactory-resolved). */
|
|
221
399
|
this.client = null;
|
|
222
|
-
/**
|
|
400
|
+
/**
|
|
401
|
+
* The mic capture stream — acquired here (permission UX) and handed to the client. A microphone the controller opens
|
|
402
|
+
* again after the call lost one replaces it.
|
|
403
|
+
*/
|
|
223
404
|
this.localStream = null;
|
|
405
|
+
/**
|
|
406
|
+
* Whether the user muted the call's microphone, as {@link ToggleMute} last reported it. While the call has no microphone
|
|
407
|
+
* track there is nothing to mute, and a microphone opened again after the call lost one starts this way.
|
|
408
|
+
*/
|
|
409
|
+
this.microphoneMuted = false;
|
|
410
|
+
/** The host's camera-and-microphone controller for this session, when the host offers one. */
|
|
411
|
+
this.localMedia = null;
|
|
412
|
+
/** Follows the controller's microphone, so a swapped-in track or a reopened microphone reaches the driver and the recorder. */
|
|
413
|
+
this.localMediaSubscription = null;
|
|
414
|
+
/**
|
|
415
|
+
* Whether the call offers its microphone for switching ({@link Microphone$}, {@link SwitchMicrophone}): from the moment
|
|
416
|
+
* the client is connected until the call ends. Not before: a driver still connecting may have bound the old track.
|
|
417
|
+
*/
|
|
418
|
+
this.microphoneOffered = false;
|
|
419
|
+
/** The live session's camera and screen share; created once the client is connected. */
|
|
420
|
+
this.captures = null;
|
|
421
|
+
this.capturesSubscription = null;
|
|
224
422
|
this.agentSessionId = null;
|
|
225
423
|
/**
|
|
226
424
|
* The application the active session runs in (sources the server-side app config cascade +
|
|
@@ -246,16 +444,40 @@ export class RealtimeSessionRuntime {
|
|
|
246
444
|
// ── Browser-side call recording ────────────────────────────────────────────
|
|
247
445
|
/**
|
|
248
446
|
* The active session's audio recorder (mic + agent mix), or `null` when the user didn't
|
|
249
|
-
* consent or the browser can't record. Created after the client connects;
|
|
250
|
-
* at teardown.
|
|
447
|
+
* consent or the browser can't record. Created after the client connects; its packets are
|
|
448
|
+
* shipped as 5 s shards during the call, and it is stopped at teardown.
|
|
251
449
|
*/
|
|
252
450
|
this.recorder = null;
|
|
253
451
|
/** ISO timestamp of when recording started — sent to the server on session start. */
|
|
254
452
|
this.recordingStartedAtIso = null;
|
|
255
|
-
/** Interval that
|
|
256
|
-
this.
|
|
257
|
-
/**
|
|
258
|
-
|
|
453
|
+
/** Interval that cuts and uploads a shard every {@link recordingShardFlushMs} during a recording. */
|
|
454
|
+
this.shardFlushTimer = null;
|
|
455
|
+
/**
|
|
456
|
+
* The current recording's shard pipeline (window → store → queue → transport), or null when not
|
|
457
|
+
* recording. A fresh pipeline per recording keeps a flush still in flight from a previous session
|
|
458
|
+
* off this session's indexes. Its queue is closed when the recording ends; anything it had not
|
|
459
|
+
* delivered stays in the local store for the next-load drain.
|
|
460
|
+
*/
|
|
461
|
+
this.recordingShards = null;
|
|
462
|
+
/**
|
|
463
|
+
* Bytes of this page's recording `keepalive` requests still in flight. The browser's keepalive budget
|
|
464
|
+
* is per page, and closing a tab fires `visibilitychange` and `pagehide` back to back, so each dispatch
|
|
465
|
+
* budgets against what the previous one has not finished sending.
|
|
466
|
+
*/
|
|
467
|
+
this.keepaliveBytesInFlight = 0;
|
|
468
|
+
/** `sessionId/index` of shards whose `keepalive` copy is still in flight; never sent twice at once. */
|
|
469
|
+
this.keepaliveShardsInFlight = new Set();
|
|
470
|
+
/** The durable local shard store, created on first use and shared by every recording of this runtime. */
|
|
471
|
+
this.shardStore = null;
|
|
472
|
+
/**
|
|
473
|
+
* True once a drain ran with a transport; the drain then does nothing until a recording ends and
|
|
474
|
+
* re-arms it ({@link rearmPendingShardDrain}).
|
|
475
|
+
*/
|
|
476
|
+
this.pendingShardsDrained = false;
|
|
477
|
+
/** Bumped by every re-arm, so a drain already in flight cannot mark itself done over a re-arm. */
|
|
478
|
+
this.drainGeneration = 0;
|
|
479
|
+
/** The drain in flight, so overlapping callers share it instead of uploading the same shards twice. */
|
|
480
|
+
this.drainInFlight = null;
|
|
259
481
|
// ── Server-side liveness ───────────────────────────────────────────────────
|
|
260
482
|
/**
|
|
261
483
|
* Interval that tells the server this session is still in use, or `null` when no session is
|
|
@@ -266,12 +488,33 @@ export class RealtimeSessionRuntime {
|
|
|
266
488
|
* Recording-relative ms offset at which the IN-FLIGHT (not-yet-finalized) turn's audio
|
|
267
489
|
* actually BEGAN — captured the moment that turn's audio/text starts flowing (its first
|
|
268
490
|
* interim transcript), NOT inherited from the previous turn's end. `null` before the first
|
|
269
|
-
* turn / between turns (until the next turn's audio starts)
|
|
270
|
-
*
|
|
271
|
-
*
|
|
272
|
-
* the
|
|
491
|
+
* turn / between turns (until the next turn's audio starts), and it stays `null` for a turn
|
|
492
|
+
* whose start was never measured. Sent as `utteranceStartMs` on the turn's final transcript
|
|
493
|
+
* AS-IS — `null` is relayed as `null`, never replaced with 0 — so per-turn timing lines up
|
|
494
|
+
* with the recording even when a tool-call / silence gap sits between turns (the
|
|
495
|
+
* inherit-previous-end model mis-stamped the post-gap turn at the pre-gap offset). See
|
|
496
|
+
* {@link markTurnAudioStart}. A user speech onset signalled by the client does NOT land here;
|
|
497
|
+
* it has its own slot, {@link userSpeechStartMs}.
|
|
273
498
|
*/
|
|
274
499
|
this.currentTurnStartMs = null;
|
|
500
|
+
/**
|
|
501
|
+
* Recording-relative ms offset of the first user speech onset the client signalled
|
|
502
|
+
* (`OnUserSpeechStart`) since the last final user transcript, or `null` when none was signalled.
|
|
503
|
+
* It has its own slot, apart from {@link currentTurnStartMs}, so a user onset never claims the
|
|
504
|
+
* assistant turn's start: OpenAI user transcripts arrive asynchronously, often after the
|
|
505
|
+
* assistant's interims (or its final) for the reply have already flowed. See
|
|
506
|
+
* {@link markUserSpeechStart}.
|
|
507
|
+
*/
|
|
508
|
+
this.userSpeechStartMs = null;
|
|
509
|
+
/**
|
|
510
|
+
* `true` once an assistant turn has finalized after {@link userSpeechStartMs} was stamped BEFORE
|
|
511
|
+
* that turn began. The held onset may still be claimed by a late user transcript (OpenAI
|
|
512
|
+
* transcribes asynchronously), but the NEXT onset replaces it: an onset no transcript ever claimed
|
|
513
|
+
* (a VAD false positive, a failed transcription) must not become the start of the next real user
|
|
514
|
+
* turn. A barge-in onset (stamped during the assistant turn) is not superseded by that turn's
|
|
515
|
+
* final. An EMPTY user transcript releases the onset outright (see onClientTranscript).
|
|
516
|
+
*/
|
|
517
|
+
this.userSpeechStartSuperseded = false;
|
|
275
518
|
/**
|
|
276
519
|
* Wall-anchor of the SESSION clock (#3832): `performance.now()` at the moment the call went
|
|
277
520
|
* live, or `null` before any call has. Read only through {@link nowTurnOffsetMs}.
|
|
@@ -310,6 +553,11 @@ export class RealtimeSessionRuntime {
|
|
|
310
553
|
this.pendingUsageInput = 0;
|
|
311
554
|
/** Accumulated output-token delta since the last flush. */
|
|
312
555
|
this.pendingUsageOutput = 0;
|
|
556
|
+
/**
|
|
557
|
+
* Accumulated per-modality usage since the last flush (input and output detail blocks, avatar
|
|
558
|
+
* video seconds included), relayed in full; `null` when no update carried any.
|
|
559
|
+
*/
|
|
560
|
+
this.pendingUsageDetails = null;
|
|
313
561
|
/** Pending debounced usage flush; also force-flushed at teardown. */
|
|
314
562
|
this.usageFlushTimer = null;
|
|
315
563
|
/** Active push-status subscription that feeds delegation progress; cleared on teardown. */
|
|
@@ -352,7 +600,20 @@ export class RealtimeSessionRuntime {
|
|
|
352
600
|
*/
|
|
353
601
|
this.teardownInFlight = null;
|
|
354
602
|
/**
|
|
355
|
-
*
|
|
603
|
+
* The latest start's opening of the host's microphone, until {@link IRealtimeMediaHost.ReleaseMicrophone} hands it
|
|
604
|
+
* back; `null` when no opening is owed a release, so a session that ends before asking for the microphone releases
|
|
605
|
+
* nothing.
|
|
606
|
+
*
|
|
607
|
+
* The host gets one release per opening, after the opening returns. A release while the microphone is still
|
|
608
|
+
* opening undoes nothing: the React Native host puts iOS into the call audio category inside `AcquireMicrophone`,
|
|
609
|
+
* after the permission prompt. So {@link runTeardown} releases an opening that has returned and leaves one still
|
|
610
|
+
* opening to its start, which releases it once it returns ({@link unwindAbandonedStart}), unless a newer start has
|
|
611
|
+
* opened the microphone since; that start's own end releases it. Whoever releases clears this first, so no
|
|
612
|
+
* opening is released twice.
|
|
613
|
+
*/
|
|
614
|
+
this.microphoneLease = null;
|
|
615
|
+
/**
|
|
616
|
+
* Why the latest session start failed, or `null` when it has not. Each start clears it ({@link beginSessionStart}).
|
|
356
617
|
*
|
|
357
618
|
* The runtime reports failure as `'error'` on {@link ConnectionState$}, which is enough to show
|
|
358
619
|
* *that* something went wrong but not *what* — and the difference matters at exactly one point:
|
|
@@ -361,6 +622,11 @@ export class RealtimeSessionRuntime {
|
|
|
361
622
|
* runtime's own try/catch, so the host never sees that rejection itself.
|
|
362
623
|
*/
|
|
363
624
|
this.lastStartError = null;
|
|
625
|
+
/**
|
|
626
|
+
* The fatal error the call's client reported, or `null` when it has reported none. Each start clears it
|
|
627
|
+
* ({@link beginSessionStart}). See {@link LastSessionError}.
|
|
628
|
+
*/
|
|
629
|
+
this.lastSessionError = null;
|
|
364
630
|
/**
|
|
365
631
|
* Pending DEBOUNCED channel-state saves, keyed by channel name. Each entry keeps the
|
|
366
632
|
* LATEST serialized state plus the session id captured while the session was live —
|
|
@@ -377,13 +643,102 @@ export class RealtimeSessionRuntime {
|
|
|
377
643
|
*/
|
|
378
644
|
this.usedChannelNames = new Set();
|
|
379
645
|
this._provider = null;
|
|
646
|
+
this._launcher = new DefaultRealtimeSessionLauncher();
|
|
647
|
+
/**
|
|
648
|
+
* Loads channel code on demand. Called, with the `ClientPluginClass` keys of every channel about to be
|
|
649
|
+
* resolved (the registry's rows plus the host's declarations), after the registry is read and before any
|
|
650
|
+
* plugin is built — so a host can register a heavy channel's class only when the session could use it.
|
|
651
|
+
* It must resolve once the classes it knows are registered; a rejection is logged, never fatal.
|
|
652
|
+
*/
|
|
653
|
+
this.ChannelClassLoader = null;
|
|
654
|
+
// ── Session events (server → this session's client) ────────────────────────
|
|
655
|
+
this._sessionEvents$ = new Subject();
|
|
656
|
+
/**
|
|
657
|
+
* Every event the server publishes to THIS session, for the session's life (identity verification
|
|
658
|
+
* today; apps add their own types). Hot, with no replay — subscribe before the session starts, once,
|
|
659
|
+
* and it keeps working across sessions. The runtime has already acted on the events it understands
|
|
660
|
+
* (see {@link handleIdentityVerified}) by the time one reaches here; this stream is for the host UI.
|
|
661
|
+
*
|
|
662
|
+
* Silent when the session's provider has no subscription transport (see {@link createSessionEventSource}).
|
|
663
|
+
*/
|
|
664
|
+
this.SessionEvents$ = this._sessionEvents$.asObservable();
|
|
665
|
+
/** Keeps the live session's event stream open; null between sessions. */
|
|
666
|
+
this.sessionEventHub = null;
|
|
667
|
+
/** Per-type handlers the runtime runs (before publishing on {@link SessionEvents$}) for events it understands. */
|
|
668
|
+
this.sessionEventHandlers = new Map([
|
|
669
|
+
['identity.verified', (event) => this.handleIdentityVerified(event)]
|
|
670
|
+
]);
|
|
671
|
+
/**
|
|
672
|
+
* What the agent should SAY the moment the user's identity is verified, or `null` (the default) to
|
|
673
|
+
* say nothing — the agent is still told, silently, through a context note. A string is used as the
|
|
674
|
+
* instruction verbatim; a function receives the verified payload (e.g. to greet by name) and returns
|
|
675
|
+
* the instruction, or `null` to stay silent for this verification.
|
|
676
|
+
*/
|
|
677
|
+
this.IdentityVerifiedSpokenResponse = null;
|
|
678
|
+
this.clientDeadline = new ClientSessionDeadline();
|
|
679
|
+
// ── Interactive channels (registry-driven + host-declared plugins) ─────────
|
|
680
|
+
//
|
|
681
|
+
// A session's channels come from two places — the `MJ: AI Agent Channels` registry and the channels
|
|
682
|
+
// the host declares — and pass through three stages:
|
|
683
|
+
//
|
|
684
|
+
// 1. PREPARE (before mint) construct the plugins, read their descriptors, report them as
|
|
685
|
+
// candidates, and work out the scope the browser can decide alone.
|
|
686
|
+
// 2. MINT the server scopes the candidates with the agent/app cascade and the
|
|
687
|
+
// registry's kill switch, and returns the resolved policy.
|
|
688
|
+
// 3. ACTIVATE (after mint) initialize exactly the policy's channels: `open-on-start`/`headless`
|
|
689
|
+
// ones mount now, `on-demand` ones wait to be opened through ContextTool.
|
|
690
|
+
//
|
|
691
|
+
// Nothing is initialized before the policy is known, so a channel the server vetoes never runs.
|
|
692
|
+
/** Whether the channel catalog note still has to be sent (see {@link flushChannelCatalogNote}). */
|
|
693
|
+
this.catalogNotePending = false;
|
|
694
|
+
/** Plugins resolved for the current start but not yet initialized. Consumed by {@link activateChannels}. */
|
|
695
|
+
this.preparedChannels = [];
|
|
696
|
+
/** The scope the browser resolves on its own — the fallback when no server policy comes back. */
|
|
697
|
+
this.localChannelScope = null;
|
|
698
|
+
/** `on-demand` channels that are in the session but not opened yet (initialized only when opened). */
|
|
699
|
+
this.advertisedChannels = [];
|
|
700
|
+
/** What the policy resolved for each in-session channel, keyed by normalized channel key. */
|
|
701
|
+
this.resolvedChannels = new Map();
|
|
702
|
+
/** The prior session's saved channel states, kept so an `on-demand` channel opened later is restored too. */
|
|
703
|
+
this.priorChannelStates = {};
|
|
704
|
+
/** The app/static client-tool tiers the server resolved for this session. */
|
|
705
|
+
this.sessionClientTools = null;
|
|
706
|
+
/** Validates and routes channel-addressed `ContextTool` calls. */
|
|
707
|
+
this.channelDispatcher = new ChannelActionDispatcher({
|
|
708
|
+
FindChannel: (key) => this.findDispatchableChannel(key),
|
|
709
|
+
ListChannelKeys: () => this.listAddressableChannelKeys(),
|
|
710
|
+
ActivateChannel: (plugin) => this.mountAdvertisedChannel(plugin),
|
|
711
|
+
});
|
|
712
|
+
/** Where the user's per-channel "how much can the agent see" choices are kept. Replace with {@link SetExposurePreferences}. */
|
|
713
|
+
this.exposurePreferences = new UserSettingsExposurePreferences(() => this.canPersistUserSettings());
|
|
714
|
+
/** The channel whose surface the user is looking at, kept so it applies to an arbiter created after it was set. */
|
|
715
|
+
this.focusedChannelKey = null;
|
|
716
|
+
this.stopWatchingVideoSources = null;
|
|
380
717
|
/**
|
|
381
718
|
* Host registry of surface CLIENT TOOLS (Name → handler), fed by the host (Explorer) from the
|
|
382
|
-
* active surface's `SetAgentClientTools
|
|
383
|
-
* executes against this via {@link executeAppClientTool}.
|
|
384
|
-
*
|
|
719
|
+
* active surface's `SetAgentClientTools` and from its always-on globals. The headless
|
|
720
|
+
* ClientContextChannel's `ContextTool` proxy executes against this via {@link executeAppClientTool}.
|
|
721
|
+
* Owner-keyed — see {@link AppClientToolRegistry}.
|
|
385
722
|
*/
|
|
386
|
-
this.
|
|
723
|
+
this.appToolRegistry = new AppClientToolRegistry();
|
|
724
|
+
// Re-upload shards a previous, interrupted call left in the local store. Deferred a tick so a
|
|
725
|
+
// host can assign `Provider` right after construction; when it still has no transport the drain
|
|
726
|
+
// defers itself and session start tries again. Unref'd so it never holds a Node process open.
|
|
727
|
+
const drainTimer = setTimeout(() => {
|
|
728
|
+
void this.DrainPendingRecordingShards();
|
|
729
|
+
}, 0);
|
|
730
|
+
// Node returns a Timeout object with unref(); browsers return a number.
|
|
731
|
+
if (typeof drainTimer === 'object' && drainTimer !== null && 'unref' in drainTimer && typeof drainTimer.unref === 'function') {
|
|
732
|
+
drainTimer.unref();
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
/** Synchronous read of {@link SavingRecording$}. */
|
|
736
|
+
get IsSavingRecording() {
|
|
737
|
+
return this._savingRecording$.value;
|
|
738
|
+
}
|
|
739
|
+
/** Synchronous access to {@link AvatarNotice$}'s current value. */
|
|
740
|
+
get CurrentAvatarNotice() {
|
|
741
|
+
return this._avatarNotice$.value;
|
|
387
742
|
}
|
|
388
743
|
/** Synchronous access to the session's active interactive-channel plugins. */
|
|
389
744
|
get ActiveChannels() {
|
|
@@ -444,8 +799,20 @@ export class RealtimeSessionRuntime {
|
|
|
444
799
|
UpdateAppContext(snapshot) {
|
|
445
800
|
this._appContext$.next(snapshot);
|
|
446
801
|
}
|
|
447
|
-
/** How often
|
|
448
|
-
static { this.
|
|
802
|
+
/** How often packets are cut into shards: the 5 s shard window, so each tick seals about one window. */
|
|
803
|
+
static { this.recordingShardFlushMs = 5000; }
|
|
804
|
+
/**
|
|
805
|
+
* Bytes of `keepalive` request bodies one terminal may use, the tail included. Browsers share about
|
|
806
|
+
* 64 KiB of in-flight keepalive bodies per page; the brief reserves 48 KiB of it for the recording,
|
|
807
|
+
* leaving room for a host's own close beacon (Caliber sends one from `pagehide` too).
|
|
808
|
+
*/
|
|
809
|
+
static { this.recordingKeepaliveBudgetBytes = RECORDING_TERMINAL_MAX_BYTES; }
|
|
810
|
+
/**
|
|
811
|
+
* Most runs one drain call makes (each run is itself bounded): it runs again when it was re-armed while
|
|
812
|
+
* running, or when a run deleted shards the server refused for good, which frees the page for the
|
|
813
|
+
* shards behind them.
|
|
814
|
+
*/
|
|
815
|
+
static { this.maxDrainRunsPerCall = 3; }
|
|
449
816
|
/**
|
|
450
817
|
* How often the client asserts liveness. Comfortably under `SessionJanitor`'s
|
|
451
818
|
* `closeThresholdMinutes` (15 by default) so several pulses must be missed in a row before a
|
|
@@ -480,10 +847,113 @@ export class RealtimeSessionRuntime {
|
|
|
480
847
|
set Provider(value) {
|
|
481
848
|
this._provider = value;
|
|
482
849
|
}
|
|
850
|
+
/**
|
|
851
|
+
* How this runtime mints a session — the stock `StartRealtimeClientSession` mutation unless a host
|
|
852
|
+
* installs its own {@link IRealtimeSessionLauncher} (a guest-session exchange, a deployment-specific
|
|
853
|
+
* mutation). Everything after the mint — channel scoping, the driver, transcripts, teardown — is
|
|
854
|
+
* unchanged. Assign `null` to restore the default.
|
|
855
|
+
*/
|
|
856
|
+
get Launcher() {
|
|
857
|
+
return this._launcher;
|
|
858
|
+
}
|
|
859
|
+
set Launcher(value) {
|
|
860
|
+
this._launcher = value ?? new DefaultRealtimeSessionLauncher();
|
|
861
|
+
}
|
|
862
|
+
/**
|
|
863
|
+
* The client's copy of the session's absolute deadline (`null` when none is known). A host that
|
|
864
|
+
* knows the session's cap at start sets it with {@link SetSessionDeadline}; verification extends it
|
|
865
|
+
* from the server's `identity.verified` payload. The SERVER enforces the real deadline — this exists
|
|
866
|
+
* so the host can end the call gracefully first and show a countdown. It only ever moves later by
|
|
867
|
+
* the server's word, so nothing a client does can extend the real one.
|
|
868
|
+
*/
|
|
869
|
+
get SessionDeadline$() {
|
|
870
|
+
return this.clientDeadline.Deadline$;
|
|
871
|
+
}
|
|
872
|
+
/** Synchronous access to {@link SessionDeadline$}. */
|
|
873
|
+
get SessionDeadline() {
|
|
874
|
+
return this.clientDeadline.Value;
|
|
875
|
+
}
|
|
876
|
+
/** Sets the baseline deadline a host knows at session start (`null` clears). See {@link SessionDeadline$}. */
|
|
877
|
+
SetSessionDeadline(deadline) {
|
|
878
|
+
this.clientDeadline.Set(deadline);
|
|
879
|
+
}
|
|
880
|
+
/**
|
|
881
|
+
* Builds the transport that reads the session's events. The default rides the session's GraphQL
|
|
882
|
+
* provider; it returns `null` — "no session events here" — for a provider with no subscription
|
|
883
|
+
* support (a test double, a host on another transport), which the runtime tolerates. Override to
|
|
884
|
+
* supply another transport.
|
|
885
|
+
*/
|
|
886
|
+
createSessionEventSource() {
|
|
887
|
+
const provider = this.Provider;
|
|
888
|
+
if (!provider || typeof provider.Subscribe !== 'function') {
|
|
889
|
+
return null;
|
|
890
|
+
}
|
|
891
|
+
return new GraphQLRealtimeSessionClient(provider);
|
|
892
|
+
}
|
|
893
|
+
/** Opens the live session's event stream. Best-effort: never disturbs the call. */
|
|
894
|
+
startSessionEvents(agentSessionId) {
|
|
895
|
+
this.stopSessionEvents();
|
|
896
|
+
try {
|
|
897
|
+
const source = this.createSessionEventSource();
|
|
898
|
+
if (!source) {
|
|
899
|
+
return;
|
|
900
|
+
}
|
|
901
|
+
this.sessionEventHub = new RealtimeSessionEventHub(source, (event) => this.routeSessionEvent(agentSessionId, event));
|
|
902
|
+
this.sessionEventHub.Start(agentSessionId);
|
|
903
|
+
}
|
|
904
|
+
catch (error) {
|
|
905
|
+
console.error('[RealtimeSession] Could not start the session event stream:', error);
|
|
906
|
+
}
|
|
907
|
+
}
|
|
908
|
+
stopSessionEvents() {
|
|
909
|
+
this.sessionEventHub?.Stop();
|
|
910
|
+
this.sessionEventHub = null;
|
|
911
|
+
}
|
|
912
|
+
/** Routes one event to its per-type handler, then publishes it. An event for another session is dropped. */
|
|
913
|
+
routeSessionEvent(expectedSessionId, event) {
|
|
914
|
+
if (event.AgentSessionID !== expectedSessionId || this.agentSessionId !== expectedSessionId) {
|
|
915
|
+
console.warn(`[RealtimeSession] Dropped a '${event.Type}' event addressed to another session.`);
|
|
916
|
+
return;
|
|
917
|
+
}
|
|
918
|
+
try {
|
|
919
|
+
this.sessionEventHandlers.get(event.Type)?.(event);
|
|
920
|
+
}
|
|
921
|
+
catch (error) {
|
|
922
|
+
console.error(`[RealtimeSession] Handling the '${event.Type}' session event failed:`, error);
|
|
923
|
+
}
|
|
924
|
+
this._sessionEvents$.next(event);
|
|
925
|
+
}
|
|
926
|
+
/**
|
|
927
|
+
* `identity.verified`: keep the client's deadline in step with the server's, tell the model — as a
|
|
928
|
+
* silent, structured note — who it is now talking to, and, when the host configured it, have it say
|
|
929
|
+
* something about that. The payload is validated again here (it crossed a wire); the name rides as
|
|
930
|
+
* JSON so user-entered text can never break out of the note's frame.
|
|
931
|
+
*/
|
|
932
|
+
handleIdentityVerified(event) {
|
|
933
|
+
if (event.Type !== 'identity.verified' || !IsIdentityVerifiedEventPayload(event.Payload)) {
|
|
934
|
+
console.warn('[RealtimeSession] Ignored a malformed identity.verified event.');
|
|
935
|
+
return;
|
|
936
|
+
}
|
|
937
|
+
const payload = event.Payload;
|
|
938
|
+
this.clientDeadline.Extend(payload.MaxSessionDeadlineIso);
|
|
939
|
+
this.SendContextNote(`[identity] verified ${JSON.stringify({ email: payload.VerifiedEmail, name: payload.VerifiedName, method: payload.Method })} ` +
|
|
940
|
+
'(background context — the user has proven they control this email address; treat them as verified. Do not read this note aloud.)');
|
|
941
|
+
const spoken = typeof this.IdentityVerifiedSpokenResponse === 'function' ? this.IdentityVerifiedSpokenResponse(payload) : this.IdentityVerifiedSpokenResponse;
|
|
942
|
+
if (spoken && spoken.trim().length > 0 && this.client && this.isSessionLive()) {
|
|
943
|
+
this.requestChannelSpokenResponse(spoken.trim());
|
|
944
|
+
}
|
|
945
|
+
}
|
|
483
946
|
/** True when a session is currently open. */
|
|
484
947
|
get IsActive() {
|
|
485
948
|
return this._active$.value;
|
|
486
949
|
}
|
|
950
|
+
/**
|
|
951
|
+
* True when a session is open or starting for `conversationId`. Ids are compared without
|
|
952
|
+
* regard to case.
|
|
953
|
+
*/
|
|
954
|
+
IsActiveFor(conversationId) {
|
|
955
|
+
return this.IsActive && conversationId != null && UUIDsEqual(this.sessionConversationId, conversationId);
|
|
956
|
+
}
|
|
487
957
|
/**
|
|
488
958
|
* Start a client-direct voice session fronting `targetAgentId`.
|
|
489
959
|
*
|
|
@@ -514,38 +984,62 @@ export class RealtimeSessionRuntime {
|
|
|
514
984
|
* pickers, and never synthesize overrides beyond what the user explicitly chose.
|
|
515
985
|
* Omit/`null` for the server's defaults (today's behavior).
|
|
516
986
|
* @param recordingConsent Optional EXPLICIT "record this call" consent for THIS session. When
|
|
517
|
-
* `true`, the browser records a mic + agent-audio mix and uploads it
|
|
987
|
+
* `true`, the browser records a mic + agent-audio mix and uploads it as 5 s Opus shards during the call. When
|
|
518
988
|
* omitted/`null`, the per-user persisted preference (`mj.realtimeVoice.recordingConsent.v1`
|
|
519
989
|
* via {@link UserInfoEngine}) is read as the default. `false` never records.
|
|
520
990
|
* @param mediaCollectionId Optional per-session media-kit override (`MJ: Collections.ID`). When set,
|
|
521
991
|
* the server-side Media channel resolves THIS collection as the agent's media kit for the session,
|
|
522
992
|
* taking precedence over the agent's `DefaultMediaCollectionID`. The server UUID-validates it
|
|
523
993
|
* (malformed ⇒ ignored, the agent default applies). Omit/`null` to use the agent default kit.
|
|
994
|
+
* @param applicationId Optional application the session runs in (sources the app config cascade,
|
|
995
|
+
* including `Application.AgentSettings.Realtime.Channels`).
|
|
996
|
+
* @param appContext Optional live app-context snapshot injected into the companion prompt at mint.
|
|
997
|
+
* @param options Optional per-start extras — chiefly the channels the HOST brings
|
|
998
|
+
* ({@link RealtimeSessionStartOptions.HostChannels}), the way a connect-only embed with no
|
|
999
|
+
* registry gets channels at all, and the fork the session starts on
|
|
1000
|
+
* ({@link RealtimeSessionStartOptions.ConversationBranchID}; omitted is Main, where its transcript is written).
|
|
524
1001
|
*/
|
|
525
|
-
async StartRealtimeSession(targetAgentId, conversationId, lastSessionId, agentName, preferredModelId, clientTools, coAgentId, configOverridesJson, recordingConsent, mediaCollectionId, applicationId, appContext) {
|
|
1002
|
+
async StartRealtimeSession(targetAgentId, conversationId, lastSessionId, agentName, preferredModelId, clientTools, coAgentId, configOverridesJson, recordingConsent, mediaCollectionId, applicationId, appContext, options) {
|
|
526
1003
|
if (this.IsActive) {
|
|
527
1004
|
return; // a session is already running — ignore duplicate starts
|
|
528
1005
|
}
|
|
529
1006
|
const consent = this.beginSessionStart({ agentName, recordingConsent, applicationId, appContext });
|
|
530
|
-
|
|
1007
|
+
this.sessionConversationId = conversationId ?? null;
|
|
1008
|
+
// A teardown while the channels are prepared or the mint runs bumps this; see the checks below.
|
|
1009
|
+
const generation = this.startGeneration;
|
|
1010
|
+
// Captured BEFORE the channels are prepared so the mint carries exactly the snapshot the prologue
|
|
531
1011
|
// resolved, whatever a channel plugin may push in the meantime.
|
|
532
1012
|
const effectiveAppContext = this._appContext$.value;
|
|
533
1013
|
let session;
|
|
534
1014
|
try {
|
|
535
|
-
// Resolve
|
|
536
|
-
//
|
|
537
|
-
|
|
538
|
-
|
|
1015
|
+
// Resolve the interactive-channel plugins FIRST (constructed, not started): the tools of the
|
|
1016
|
+
// channels mounted with the session must be declared to the realtime model at mint, and the
|
|
1017
|
+
// server needs the candidates to scope them. Nothing is initialized until the policy is known.
|
|
1018
|
+
const scope = await this.prepareChannelScope(options?.HostChannels);
|
|
1019
|
+
const allClientTools = [...(clientTools ?? []), ...scope.NativeTools];
|
|
1020
|
+
session = await this.mintSession(targetAgentId, conversationId, lastSessionId, preferredModelId, allClientTools, coAgentId, configOverridesJson, consent, this.recordingStartedAtIso, mediaCollectionId, this.applicationId, effectiveAppContext, scope.CandidatesJson, scope.ShowsAgentVideo, options?.ConversationBranchID ?? null);
|
|
539
1021
|
}
|
|
540
1022
|
catch (error) {
|
|
1023
|
+
// An ended or replaced start only logs its failure: the error state and the teardown belong
|
|
1024
|
+
// to the session that is current now, which a newer start may own.
|
|
1025
|
+
if (this.startGeneration !== generation) {
|
|
1026
|
+
console.error('[RealtimeSession] A start that was already ended failed:', error);
|
|
1027
|
+
return;
|
|
1028
|
+
}
|
|
541
1029
|
await this.failSessionStart(error);
|
|
542
1030
|
return;
|
|
543
1031
|
}
|
|
1032
|
+
// The host ended the start while the mint ran. That teardown had no session id to close, so
|
|
1033
|
+
// close the minted row here, and touch no shared state: a newer start may already own it.
|
|
1034
|
+
if (this.startGeneration !== generation) {
|
|
1035
|
+
await this.closeServerSession(session.AgentSessionId);
|
|
1036
|
+
return;
|
|
1037
|
+
}
|
|
544
1038
|
if (!this.hostCanUseProvider(session.Provider)) {
|
|
545
1039
|
await this.abortUnusableSession(session);
|
|
546
1040
|
return;
|
|
547
1041
|
}
|
|
548
|
-
await this.runMintedSession(session, conversationId ?? null, consent);
|
|
1042
|
+
await this.runMintedSession(session, conversationId ?? null, consent, options?.CameraCheck === true);
|
|
549
1043
|
}
|
|
550
1044
|
/**
|
|
551
1045
|
* Declares which provider keys this host can actually carry audio for.
|
|
@@ -581,7 +1075,7 @@ export class RealtimeSessionRuntime {
|
|
|
581
1075
|
// published them on ActiveChannels$; skipping teardown would leave them undisposed, their tool
|
|
582
1076
|
// handlers registered, and their subscriptions live until the next start replaced them.
|
|
583
1077
|
this.agentSessionId = session.AgentSessionId ?? this.agentSessionId;
|
|
584
|
-
this.
|
|
1078
|
+
this.setConnectionState('error');
|
|
585
1079
|
await this.teardown(true);
|
|
586
1080
|
}
|
|
587
1081
|
/**
|
|
@@ -600,7 +1094,8 @@ export class RealtimeSessionRuntime {
|
|
|
600
1094
|
* owns that half too.
|
|
601
1095
|
*
|
|
602
1096
|
* @param result The minted session — the same ten fields the `StartRealtimeClientSession`
|
|
603
|
-
* mutation returns. `EphemeralToken` and `Provider` are what actually open the call
|
|
1097
|
+
* mutation returns. `EphemeralToken` and `Provider` are what actually open the call; a relay
|
|
1098
|
+
* session opens with its `Transport` and `RelayUrl` instead, so a host must pass those on.
|
|
604
1099
|
* @param options Host-side inputs the result cannot carry; see {@link RealtimeSessionRunOptions}.
|
|
605
1100
|
* Every field defaults exactly as its {@link StartRealtimeSession} counterpart does.
|
|
606
1101
|
*/
|
|
@@ -614,16 +1109,20 @@ export class RealtimeSessionRuntime {
|
|
|
614
1109
|
return;
|
|
615
1110
|
}
|
|
616
1111
|
const consent = this.beginSessionStart(effectiveOptions);
|
|
617
|
-
await this.runMintedSession(result, effectiveOptions.conversationId ?? null, consent);
|
|
1112
|
+
await this.runMintedSession(result, effectiveOptions.conversationId ?? null, consent, false);
|
|
618
1113
|
}
|
|
619
1114
|
/**
|
|
620
|
-
* Start prologue shared by both entry points:
|
|
621
|
-
* per-session state, and flip the session live (which is ALSO what
|
|
622
|
-
* suppress duplicate starts while the mint is still in flight — hence it
|
|
623
|
-
* after). Returns the resolved recording consent, which the mint half
|
|
624
|
-
* the run half uses to decide whether to record.
|
|
1115
|
+
* Start prologue shared by both entry points: clear the last start's failure, bind the app layer,
|
|
1116
|
+
* publish the agent name, reset per-session state, and flip the session live (which is ALSO what
|
|
1117
|
+
* makes the `IsActive` guard suppress duplicate starts while the mint is still in flight — hence it
|
|
1118
|
+
* runs before minting, not after). Returns the resolved recording consent, which the mint half
|
|
1119
|
+
* reports to the server and the run half uses to decide whether to record.
|
|
625
1120
|
*/
|
|
626
1121
|
beginSessionStart(options) {
|
|
1122
|
+
// Before this start publishes anything: hosts read LastStartError as this start's states arrive,
|
|
1123
|
+
// so from the first of them it must describe this start, not an earlier one.
|
|
1124
|
+
this.lastStartError = null;
|
|
1125
|
+
this.lastSessionError = null;
|
|
627
1126
|
// App awareness (Move 1/3/4): the application the session runs in (sources the app config
|
|
628
1127
|
// cascade + RelevantAgents → allowed-agent union) and the live app-context snapshot injected
|
|
629
1128
|
// into the companion prompt at mint. Stored so the ClientContextChannel can stream subsequent
|
|
@@ -641,7 +1140,10 @@ export class RealtimeSessionRuntime {
|
|
|
641
1140
|
}
|
|
642
1141
|
this.resetState();
|
|
643
1142
|
this._active$.next(true);
|
|
644
|
-
this.
|
|
1143
|
+
this.setConnectionState('connecting');
|
|
1144
|
+
// A drain the constructor deferred (no provider/transport yet) gets its second chance here, and
|
|
1145
|
+
// runs whether or not THIS session records: the pending shards belong to an earlier call.
|
|
1146
|
+
void this.DrainPendingRecordingShards();
|
|
645
1147
|
// Resolve recording consent for this session: explicit value wins, else the per-user
|
|
646
1148
|
// persisted preference. Computed before mint so it can be reported to the server.
|
|
647
1149
|
const consent = options.recordingConsent ?? this.readPersistedRecordingConsent();
|
|
@@ -651,12 +1153,18 @@ export class RealtimeSessionRuntime {
|
|
|
651
1153
|
/**
|
|
652
1154
|
* The RUN half of a session start, shared by both entry points: consume the minted result, open
|
|
653
1155
|
* the provider connection, and go live. `inputConversationId` is the conversation the START asked
|
|
654
|
-
* for (null ⇒ "server, make me one") — the result alone can't distinguish the two.
|
|
1156
|
+
* for (null ⇒ "server, make me one") — the result alone can't distinguish the two. `cameraCheck` is
|
|
1157
|
+
* whether the host shows a camera check ({@link RealtimeSessionStartOptions.CameraCheck}).
|
|
655
1158
|
*/
|
|
656
|
-
async runMintedSession(session, inputConversationId, consent) {
|
|
1159
|
+
async runMintedSession(session, inputConversationId, consent, cameraCheck) {
|
|
657
1160
|
// Captured up front: every await below is a window in which the host can end the session.
|
|
658
1161
|
const generation = this.startGeneration;
|
|
659
1162
|
try {
|
|
1163
|
+
// Mount the channels the resolved policy puts in the session. Deliberately BEFORE the session
|
|
1164
|
+
// id is adopted: `ActiveChannels$` consumers (the sessions adapter) treat an emission that
|
|
1165
|
+
// arrives while no session id is set as the INITIAL set, whose opens they synthesize from
|
|
1166
|
+
// `SessionStarted$` — activating after the id is set would announce every channel twice.
|
|
1167
|
+
this.applySessionClientPolicy(session);
|
|
660
1168
|
this.agentSessionId = session.AgentSessionId;
|
|
661
1169
|
// A null input conversationId means the SERVER created a fresh conversation for
|
|
662
1170
|
// this session — track it so the host can fold it into the cached list, select
|
|
@@ -671,20 +1179,39 @@ export class RealtimeSessionRuntime {
|
|
|
671
1179
|
this.applyPriorChannelStates(session.PriorChannelStatesJson);
|
|
672
1180
|
const client = this.createRealtimeClient(session.Provider);
|
|
673
1181
|
this.client = client;
|
|
1182
|
+
this.watchVideoSources(client);
|
|
674
1183
|
this.wireClientHandlers(client);
|
|
675
1184
|
// Everything past here awaits on hardware and the network, during which the host may end the
|
|
676
|
-
// session. Each await is followed by a staleness check
|
|
677
|
-
// just acquired instead of leaving a live microphone and a live
|
|
678
|
-
|
|
1185
|
+
// session. Each await is followed by a staleness check, before any failure is reported, so an
|
|
1186
|
+
// abandoned start releases what it just acquired instead of leaving a live microphone and a live
|
|
1187
|
+
// call behind it, and reports nothing to a host that has ended the call.
|
|
1188
|
+
const lease = { Opening: true };
|
|
1189
|
+
this.microphoneLease = lease;
|
|
1190
|
+
const microphone = await this.openMicrophone().finally(() => {
|
|
1191
|
+
lease.Opening = false;
|
|
1192
|
+
});
|
|
1193
|
+
this.localStream = microphone.Stream;
|
|
679
1194
|
if (this.startGeneration !== generation) {
|
|
680
|
-
await this.unwindAbandonedStart(session, client);
|
|
1195
|
+
await this.unwindAbandonedStart(session, client, lease);
|
|
681
1196
|
return;
|
|
682
1197
|
}
|
|
683
|
-
|
|
1198
|
+
if (microphone.Error) {
|
|
1199
|
+
throw microphone.Error;
|
|
1200
|
+
}
|
|
1201
|
+
const clientConfig = this.BuildClientConfig(session);
|
|
1202
|
+
const connectError = await this.connectClient(client, clientConfig, microphone.Stream);
|
|
684
1203
|
if (this.startGeneration !== generation) {
|
|
685
|
-
await this.unwindAbandonedStart(session, client);
|
|
1204
|
+
await this.unwindAbandonedStart(session, client, lease);
|
|
686
1205
|
return;
|
|
687
1206
|
}
|
|
1207
|
+
if (connectError) {
|
|
1208
|
+
throw connectError;
|
|
1209
|
+
}
|
|
1210
|
+
// Tracks are negotiated now, so a capture can tell whether the model takes video, and the call whether it shows
|
|
1211
|
+
// the avatar its agent asked for. The driver holds the microphone now, so the user may switch it.
|
|
1212
|
+
this.openCaptures(client, cameraCheck);
|
|
1213
|
+
this.offerMicrophone();
|
|
1214
|
+
this.publishAvatarNotice(session, clientConfig, client);
|
|
688
1215
|
// Notify active channels that the session client is connected and tracks are established
|
|
689
1216
|
for (const channel of this._activeChannels$.value) {
|
|
690
1217
|
try {
|
|
@@ -694,6 +1221,10 @@ export class RealtimeSessionRuntime {
|
|
|
694
1221
|
console.error(`[RealtimeSession] Error in channel '${channel.ChannelName}' OnSessionStarted:`, err);
|
|
695
1222
|
}
|
|
696
1223
|
}
|
|
1224
|
+
// Tell the model which channels exist and how to use them, from the channels' own descriptors —
|
|
1225
|
+
// as soon as the control channel is usable (which may be right now, or a moment after Connect).
|
|
1226
|
+
this.catalogNotePending = true;
|
|
1227
|
+
this.flushChannelCatalogNote();
|
|
697
1228
|
// Start browser-side recording (mic + agent mix) when consented. Best-effort: an
|
|
698
1229
|
// unsupported browser / missing remote stream degrades gracefully (mic-only or off)
|
|
699
1230
|
// and never blocks the call. The remote stream may still be null here (the WebRTC
|
|
@@ -718,6 +1249,8 @@ export class RealtimeSessionRuntime {
|
|
|
718
1249
|
// SessionsObserver bridge. Emitting AFTER Connect() guarantees both that
|
|
719
1250
|
// agentSessionId is set (line ~468) AND the realtime client is connected,
|
|
720
1251
|
// so consumers can act on it without re-checking either condition.
|
|
1252
|
+
// The session's own event stream (identity verification, app events) stays open for its life.
|
|
1253
|
+
this.startSessionEvents(this.agentSessionId);
|
|
721
1254
|
this._sessionStarted$.next({
|
|
722
1255
|
sessionId: this.agentSessionId,
|
|
723
1256
|
channelNames: this._activeChannels$.value.map(c => c.ChannelName),
|
|
@@ -730,18 +1263,42 @@ export class RealtimeSessionRuntime {
|
|
|
730
1263
|
await this.failSessionStart(error);
|
|
731
1264
|
}
|
|
732
1265
|
}
|
|
1266
|
+
/**
|
|
1267
|
+
* Tells the host, once per call, why it shows no avatar ({@link AvatarNotice$}): from the mint's status, whether this
|
|
1268
|
+
* session asked for the agent's video, and whether that track is live now that the call is connected.
|
|
1269
|
+
*/
|
|
1270
|
+
publishAvatarNotice(session, config, client) {
|
|
1271
|
+
const notice = ResolveAvatarNotice(ParseRealtimeAvatarStatus(session.AvatarStatusJson), RequestsAgentVideo(config), client.IsTrackEstablished('video', 'outbound'));
|
|
1272
|
+
if (notice) {
|
|
1273
|
+
this._avatarNotice$.next(notice);
|
|
1274
|
+
}
|
|
1275
|
+
}
|
|
733
1276
|
/**
|
|
734
1277
|
* Releases everything a start acquired after the host had already ended the session.
|
|
735
1278
|
*
|
|
736
1279
|
* Reached only when {@link teardown} ran while this start was awaiting the microphone or the
|
|
737
|
-
* provider connection
|
|
738
|
-
*
|
|
739
|
-
*
|
|
1280
|
+
* provider connection, whether that wait then succeeded or failed: the host has ended the call,
|
|
1281
|
+
* so a failure there is not reported. Teardown could not release what did not exist yet, so this
|
|
1282
|
+
* start owns the cleanup: stop the microphone, hand it back to the host when teardown left that
|
|
1283
|
+
* to it ({@link microphoneLease}), close the provider connection, and close the server-side
|
|
1284
|
+
* session row that the mint created when no teardown has closed it or is closing it.
|
|
1285
|
+
*
|
|
1286
|
+
* @param lease This start's opening of the microphone.
|
|
740
1287
|
*/
|
|
741
|
-
async unwindAbandonedStart(session, client) {
|
|
1288
|
+
async unwindAbandonedStart(session, client, lease) {
|
|
742
1289
|
console.warn('[RealtimeSession] Session was ended while starting — releasing the partial session.');
|
|
1290
|
+
this.closeCaptures();
|
|
1291
|
+
this.closeLocalMedia();
|
|
743
1292
|
this.localStream?.getTracks().forEach(t => t.stop());
|
|
744
1293
|
this.localStream = null;
|
|
1294
|
+
this.microphoneMuted = false;
|
|
1295
|
+
// Teardown left the release to this start when the microphone was still opening. A cleared or
|
|
1296
|
+
// replaced lease means teardown already released it, or a newer start has opened the microphone
|
|
1297
|
+
// since and its own end releases it.
|
|
1298
|
+
if (this.microphoneLease === lease) {
|
|
1299
|
+
this.microphoneLease = null;
|
|
1300
|
+
await this.releaseMicrophone();
|
|
1301
|
+
}
|
|
745
1302
|
try {
|
|
746
1303
|
await client.Disconnect();
|
|
747
1304
|
}
|
|
@@ -751,27 +1308,321 @@ export class RealtimeSessionRuntime {
|
|
|
751
1308
|
// Only clear the shared slots when they still point at THIS attempt — a newer start may
|
|
752
1309
|
// already have replaced them.
|
|
753
1310
|
if (this.client === client) {
|
|
1311
|
+
this.unwatchVideoSources();
|
|
1312
|
+
this.clearAgentVideo();
|
|
754
1313
|
this.client = null;
|
|
755
1314
|
}
|
|
756
|
-
// Close the server session only if teardown has
|
|
757
|
-
// after closing, so a
|
|
758
|
-
//
|
|
759
|
-
// send a second `CloseAgentSession` for one
|
|
760
|
-
|
|
1315
|
+
// Close the server session only if no teardown has closed it or is closing it. Teardown nulls
|
|
1316
|
+
// `agentSessionId` after closing, so a cleared id means the teardown that invalidated this start
|
|
1317
|
+
// already closed it. A teardown still running closes it itself and then reports the end on
|
|
1318
|
+
// `SessionEnded$`: closing it here as well would send a second `CloseAgentSession` for one
|
|
1319
|
+
// session, and clearing the id would leave that teardown no session to report.
|
|
1320
|
+
if (session.AgentSessionId && this.agentSessionId === session.AgentSessionId && !this.teardownInFlight) {
|
|
761
1321
|
this.agentSessionId = null;
|
|
762
1322
|
await this.closeServerSession(session.AgentSessionId);
|
|
763
1323
|
}
|
|
764
1324
|
}
|
|
765
1325
|
/**
|
|
766
|
-
*
|
|
1326
|
+
* Opens the user's microphone: through the host's controller when it offers one, otherwise through
|
|
1327
|
+
* {@link IRealtimeMediaHost.AcquireMicrophone}. A failure is returned rather than thrown, so the caller can
|
|
1328
|
+
* first check whether the start was abandoned meanwhile (a teardown disposes the controller, which fails a
|
|
1329
|
+
* start still in flight) and unwind quietly instead of reporting an error nobody is waiting for.
|
|
1330
|
+
*/
|
|
1331
|
+
async openMicrophone() {
|
|
1332
|
+
const controller = this.mediaHost.CreateLocalMediaController?.() ?? null;
|
|
1333
|
+
if (!controller) {
|
|
1334
|
+
try {
|
|
1335
|
+
return { Stream: await this.mediaHost.AcquireMicrophone(), Error: null };
|
|
1336
|
+
}
|
|
1337
|
+
catch (error) {
|
|
1338
|
+
return { Stream: null, Error: error instanceof Error ? error : new Error(String(error)) };
|
|
1339
|
+
}
|
|
1340
|
+
}
|
|
1341
|
+
this.localMedia = controller;
|
|
1342
|
+
const started = await controller.Start('microphone');
|
|
1343
|
+
if (started.Status === 'failed') {
|
|
1344
|
+
return { Stream: null, Error: microphoneStartError(started.Reason, started.Message) };
|
|
1345
|
+
}
|
|
1346
|
+
this.followMicrophone(controller, started.Stream);
|
|
1347
|
+
return { Stream: started.Stream, Error: null };
|
|
1348
|
+
}
|
|
1349
|
+
/**
|
|
1350
|
+
* Opens the provider connection. As with {@link openMicrophone}, a failure is returned rather than thrown, so the
|
|
1351
|
+
* caller can first check whether the start was abandoned meanwhile and unwind quietly instead of reporting an error
|
|
1352
|
+
* nobody is waiting for: a teardown disconnects the client, which with most drivers fails a connection still opening.
|
|
1353
|
+
*/
|
|
1354
|
+
async connectClient(client, config, microphone) {
|
|
1355
|
+
try {
|
|
1356
|
+
await client.Connect(config, microphone);
|
|
1357
|
+
return null;
|
|
1358
|
+
}
|
|
1359
|
+
catch (error) {
|
|
1360
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
1361
|
+
}
|
|
1362
|
+
}
|
|
1363
|
+
/**
|
|
1364
|
+
* Moves the driver and the recorder onto the call's microphone whenever the controller gives it a new track. A device
|
|
1365
|
+
* switch, or a lost device replaced by the default, swaps a track into the same stream, and the controller carries the
|
|
1366
|
+
* old track's mute over. A microphone opened again after the call lost one comes in a new stream: its track takes the
|
|
1367
|
+
* call's mute, and the stream becomes the one {@link ToggleMute} and the teardown act on. Mid-swap the stream holds no
|
|
1368
|
+
* track, so only a finished swap is followed. Every report also keeps {@link Microphone$} current once the call offers
|
|
1369
|
+
* it.
|
|
1370
|
+
*/
|
|
1371
|
+
followMicrophone(controller, stream) {
|
|
1372
|
+
let followedStream = stream;
|
|
1373
|
+
let followedTrack = stream.getAudioTracks()[0] ?? null;
|
|
1374
|
+
this.localMediaSubscription = controller.State$.subscribe((media) => {
|
|
1375
|
+
const current = controller.GetStream('microphone') ?? followedStream;
|
|
1376
|
+
const track = current.getAudioTracks()[0] ?? null;
|
|
1377
|
+
if (track && track !== followedTrack) {
|
|
1378
|
+
if (current !== followedStream) {
|
|
1379
|
+
track.enabled = !this.microphoneMuted;
|
|
1380
|
+
this.localStream = current;
|
|
1381
|
+
followedStream = current;
|
|
1382
|
+
}
|
|
1383
|
+
followedTrack = track;
|
|
1384
|
+
this.recorder?.ReplaceMicrophone?.(current);
|
|
1385
|
+
this.client?.ReplaceMicrophone?.(current)?.catch((error) => {
|
|
1386
|
+
console.error('[RealtimeSession] The realtime driver could not move to the new microphone:', error);
|
|
1387
|
+
});
|
|
1388
|
+
}
|
|
1389
|
+
this.publishMicrophone(media);
|
|
1390
|
+
});
|
|
1391
|
+
}
|
|
1392
|
+
/** The client is connected: the call offers its microphone on {@link Microphone$}, and {@link SwitchMicrophone} moves it. */
|
|
1393
|
+
offerMicrophone() {
|
|
1394
|
+
if (!this.localMedia) {
|
|
1395
|
+
return;
|
|
1396
|
+
}
|
|
1397
|
+
this.microphoneOffered = true;
|
|
1398
|
+
this.publishMicrophone(this.localMedia.State);
|
|
1399
|
+
}
|
|
1400
|
+
/** Publishes the call's microphone from the controller's state while the call offers it, when it names a change. */
|
|
1401
|
+
publishMicrophone(media) {
|
|
1402
|
+
if (!this.microphoneOffered) {
|
|
1403
|
+
return;
|
|
1404
|
+
}
|
|
1405
|
+
const current = this._microphone$.value;
|
|
1406
|
+
const next = ReadMicrophoneState(media, current);
|
|
1407
|
+
if (!SameMicrophoneState(next, current)) {
|
|
1408
|
+
this._microphone$.next(next);
|
|
1409
|
+
}
|
|
1410
|
+
}
|
|
1411
|
+
/** Stops following the controller and disposes it, which releases its devices; the call no longer offers a microphone. */
|
|
1412
|
+
closeLocalMedia() {
|
|
1413
|
+
this.localMediaSubscription?.unsubscribe();
|
|
1414
|
+
this.localMediaSubscription = null;
|
|
1415
|
+
this.localMedia?.Dispose();
|
|
1416
|
+
this.localMedia = null;
|
|
1417
|
+
this.microphoneOffered = false;
|
|
1418
|
+
if (this._microphone$.value !== REALTIME_MICROPHONE_NONE) {
|
|
1419
|
+
this._microphone$.next(REALTIME_MICROPHONE_NONE);
|
|
1420
|
+
}
|
|
1421
|
+
}
|
|
1422
|
+
/**
|
|
1423
|
+
* Creates the session's camera and screen share and mirrors their state on {@link Captures$}. With `cameraCheck`, the
|
|
1424
|
+
* call's first camera start waits for {@link ConfirmCamera}.
|
|
1425
|
+
*/
|
|
1426
|
+
openCaptures(client, cameraCheck) {
|
|
1427
|
+
this.closeCaptures();
|
|
1428
|
+
const captures = new RealtimeCaptures({
|
|
1429
|
+
Client: client,
|
|
1430
|
+
LocalMedia: this.localMedia,
|
|
1431
|
+
Host: this.mediaHost,
|
|
1432
|
+
CreateSampler: this.hostFrameSampler(),
|
|
1433
|
+
Admit: (kind) => this.admitCapture(kind),
|
|
1434
|
+
CameraCheck: cameraCheck,
|
|
1435
|
+
});
|
|
1436
|
+
this.captures = captures;
|
|
1437
|
+
this.capturesSubscription = captures.States$.subscribe((states) => this._captures$.next(states));
|
|
1438
|
+
this.refreshCaptureOffers();
|
|
1439
|
+
}
|
|
1440
|
+
/**
|
|
1441
|
+
* The host's frame sampler for the captures ({@link IRealtimeMediaHost.CreateFrameSampler}), or `undefined` when it offers
|
|
1442
|
+
* none and the captures use the DOM default.
|
|
1443
|
+
*/
|
|
1444
|
+
hostFrameSampler() {
|
|
1445
|
+
const host = this.mediaHost;
|
|
1446
|
+
const create = host.CreateFrameSampler;
|
|
1447
|
+
return create ? (stream, options) => create.call(host, stream, options) : undefined;
|
|
1448
|
+
}
|
|
1449
|
+
/** Stops the camera and screen share, if any, and reports both off. */
|
|
1450
|
+
closeCaptures() {
|
|
1451
|
+
this.capturesSubscription?.unsubscribe();
|
|
1452
|
+
this.capturesSubscription = null;
|
|
1453
|
+
this.captures?.Dispose();
|
|
1454
|
+
this.captures = null;
|
|
1455
|
+
if (this._captures$.value !== REALTIME_CAPTURES_OFF) {
|
|
1456
|
+
this._captures$.next(REALTIME_CAPTURES_OFF);
|
|
1457
|
+
}
|
|
1458
|
+
this.refreshCaptureOffers();
|
|
1459
|
+
}
|
|
1460
|
+
/** Works out which captures the call offers ({@link CaptureOffers$}) and publishes a change. */
|
|
1461
|
+
refreshCaptureOffers() {
|
|
1462
|
+
const offers = this.captures && this.client?.SupportsInboundVideo ? { Camera: this.canOffer('camera'), Screen: this.canOffer('screen') } : REALTIME_CAPTURE_OFFERS_NONE;
|
|
1463
|
+
const current = this._captureOffers$.value;
|
|
1464
|
+
if (offers.Camera !== current.Camera || offers.Screen !== current.Screen) {
|
|
1465
|
+
this._captureOffers$.next(offers);
|
|
1466
|
+
}
|
|
1467
|
+
}
|
|
1468
|
+
/** Whether the host can open a capture and the call's policy admits it. */
|
|
1469
|
+
canOffer(kind) {
|
|
1470
|
+
const hostCanOpen = kind === 'camera' ? this.localMedia !== null : typeof this.mediaHost.RequestDisplayCapture === 'function';
|
|
1471
|
+
return hostCanOpen && this.admitCapture(kind).Admitted;
|
|
1472
|
+
}
|
|
1473
|
+
/** What a capture reports when there is no live session to show it to. */
|
|
1474
|
+
noSessionCapture() {
|
|
1475
|
+
return { Status: 'failed', Failure: 'no-session', Message: 'There is no call to share with.' };
|
|
1476
|
+
}
|
|
1477
|
+
/**
|
|
1478
|
+
* The session's policy for a capture. The channel that fronts it (its `CaptureKind`) must be in the session, open or
|
|
1479
|
+
* advertised, and the server's policy must let the agent see pixels through it (the agent's configuration and any
|
|
1480
|
+
* zero-data-retention rule). The user's own "agent can see" choice for the channel does not refuse a start: it decides
|
|
1481
|
+
* whether the frames reach the model.
|
|
1482
|
+
*/
|
|
1483
|
+
admitCapture(kind) {
|
|
1484
|
+
const channel = this.findCaptureChannel(kind)?.Plugin;
|
|
1485
|
+
if (!channel) {
|
|
1486
|
+
return { Admitted: false, Message: CAPTURE_WORDING[kind].NotInCall };
|
|
1487
|
+
}
|
|
1488
|
+
const resolved = this.GetResolvedChannel(channel.ChannelName);
|
|
1489
|
+
const allowed = resolved?.Exposure ?? resolved?.MaxExposure ?? channel.GetDescriptor().MaxExposure;
|
|
1490
|
+
if (allowed !== 'pixels') {
|
|
1491
|
+
const reasons = (resolved?.ExposureLimits ?? []).map((limit) => limit.Reason);
|
|
1492
|
+
return { Admitted: false, Message: reasons.length > 0 ? `${CAPTURE_WORDING[kind].NotAllowed}: ${reasons.join('; ')}.` : `${CAPTURE_WORDING[kind].NotAllowed}.` };
|
|
1493
|
+
}
|
|
1494
|
+
return { Admitted: true, ChannelKey: channel.ChannelName, VisibleToAgent: channel.Exposure === 'pixels' };
|
|
1495
|
+
}
|
|
1496
|
+
/** The channel in the session, open or advertised, that fronts a capture. */
|
|
1497
|
+
findCaptureChannel(kind) {
|
|
1498
|
+
const open = this._activeChannels$.value.find((c) => c.CaptureKind === kind);
|
|
1499
|
+
if (open) {
|
|
1500
|
+
return { Plugin: open, IsOpen: true };
|
|
1501
|
+
}
|
|
1502
|
+
const advertised = this.advertisedChannels.find((p) => p.Plugin.CaptureKind === kind);
|
|
1503
|
+
return advertised ? { Plugin: advertised.Plugin, IsOpen: false } : null;
|
|
1504
|
+
}
|
|
1505
|
+
/**
|
|
1506
|
+
* Brings up the channel that fronts a capture once the capture is starting, so the host shows its surface: an advertised
|
|
1507
|
+
* channel is opened (and announces itself to the agent), an open one counts as used.
|
|
1508
|
+
*/
|
|
1509
|
+
async revealCaptureChannel(kind) {
|
|
1510
|
+
const state = kind === 'camera' ? this.captures?.States.Camera : this.captures?.States.Screen;
|
|
1511
|
+
const channel = state?.Status === 'starting' ? this.findCaptureChannel(kind) : null;
|
|
1512
|
+
if (!channel) {
|
|
1513
|
+
return;
|
|
1514
|
+
}
|
|
1515
|
+
if (channel.IsOpen) {
|
|
1516
|
+
this.noteChannelActivity(channel.Plugin);
|
|
1517
|
+
return;
|
|
1518
|
+
}
|
|
1519
|
+
const opened = await this.OpenChannel(channel.Plugin.ChannelName);
|
|
1520
|
+
if (!opened.Success) {
|
|
1521
|
+
console.warn(`[RealtimeSession] Could not open channel '${channel.Plugin.ChannelName}' for the ${kind}: ${opened.ErrorMessage ?? opened.ErrorCode}`);
|
|
1522
|
+
}
|
|
1523
|
+
}
|
|
1524
|
+
/**
|
|
1525
|
+
* Starts the user's camera and shows it to the agent. Call it from the user's click: it may ask for camera
|
|
1526
|
+
* permission. Resolves with the camera's state; a failure (no session, a model that takes no video, a refused
|
|
1527
|
+
* permission) is a state with a message, never a throw. While the camera is on, it is a source on
|
|
1528
|
+
* {@link VideoSources$}, and it stops by itself if the device goes away.
|
|
1529
|
+
*
|
|
1530
|
+
* When the host shows a camera check ({@link RealtimeSessionStartOptions.CameraCheck}), the call's first start
|
|
1531
|
+
* resolves still starting, with `Checking` set and the stream for the host to preview. The agent sees the camera
|
|
1532
|
+
* once the host calls {@link ConfirmCamera}; {@link StopCamera} is the user's "not now".
|
|
1533
|
+
*
|
|
1534
|
+
* @param deviceId The camera to open; the system default when absent.
|
|
1535
|
+
*/
|
|
1536
|
+
async StartCamera(deviceId) {
|
|
1537
|
+
if (!this.captures) {
|
|
1538
|
+
return this.noSessionCapture();
|
|
1539
|
+
}
|
|
1540
|
+
const started = this.captures.Start('camera', { DeviceID: deviceId });
|
|
1541
|
+
await this.revealCaptureChannel('camera');
|
|
1542
|
+
return started;
|
|
1543
|
+
}
|
|
1544
|
+
/** Stops the user's camera. Safe to call when it is off. */
|
|
1545
|
+
StopCamera() {
|
|
1546
|
+
this.captures?.Stop('camera');
|
|
1547
|
+
}
|
|
1548
|
+
/**
|
|
1549
|
+
* Moves the user's camera to another device while it is open (being checked or on): one of the camera's `Devices` on
|
|
1550
|
+
* {@link Captures$}. The stream stays the same. Resolves with the camera's state; nothing changes while the camera is
|
|
1551
|
+
* off.
|
|
1552
|
+
*
|
|
1553
|
+
* @param deviceId The camera to move to.
|
|
1554
|
+
*/
|
|
1555
|
+
async SwitchCamera(deviceId) {
|
|
1556
|
+
return this.captures ? this.captures.SwitchCamera(deviceId) : this.noSessionCapture();
|
|
1557
|
+
}
|
|
1558
|
+
/**
|
|
1559
|
+
* Moves the call's microphone to another device: one of the `Devices` on {@link Microphone$}. The host's controller
|
|
1560
|
+
* swaps the new track into the same stream and carries the mute over, and the driver and the call's recording move to
|
|
1561
|
+
* it, so the agent keeps hearing the user. When the new microphone cannot open, the controller goes back to the one in
|
|
1562
|
+
* use. While the call has no microphone open (it lost the one in use and the default, or the last one went away), the
|
|
1563
|
+
* pick opens one again: the driver and the recording move to it, and it keeps the call's mute. Resolves with the
|
|
1564
|
+
* microphone's state once the switch is over. Nothing changes outside a call or on a host without a controller.
|
|
1565
|
+
*
|
|
1566
|
+
* @param deviceId The microphone to move to.
|
|
1567
|
+
*/
|
|
1568
|
+
async SwitchMicrophone(deviceId) {
|
|
1569
|
+
const controller = this.localMedia;
|
|
1570
|
+
if (controller && this.microphoneOffered) {
|
|
1571
|
+
const status = controller.State.Microphone.Status;
|
|
1572
|
+
const live = status === 'on' || status === 'starting';
|
|
1573
|
+
await (live ? controller.SwitchDevice('microphone', deviceId) : controller.Start('microphone', deviceId));
|
|
1574
|
+
}
|
|
1575
|
+
return this._microphone$.value;
|
|
1576
|
+
}
|
|
1577
|
+
/**
|
|
1578
|
+
* The user checked the camera and turned it on: the agent sees it from now on, and later camera starts in this call
|
|
1579
|
+
* skip the check. Returns the camera's state; unless the camera is waiting for its check (see {@link StartCamera}),
|
|
1580
|
+
* nothing changes.
|
|
1581
|
+
*/
|
|
1582
|
+
ConfirmCamera() {
|
|
1583
|
+
return this.captures ? this.captures.ConfirmCamera() : this.noSessionCapture();
|
|
1584
|
+
}
|
|
1585
|
+
/**
|
|
1586
|
+
* Asks the user for a screen, window or browser tab (or one panel of the page) and shows it to the agent. Call it
|
|
1587
|
+
* from the user's click: the browser's picker needs one. Resolves with the share's state; a failure (no session,
|
|
1588
|
+
* a model that takes no video, a closed picker, a host that cannot share) is a state with a message. The share
|
|
1589
|
+
* also stops when the user ends it from the browser's own bar.
|
|
1590
|
+
*
|
|
1591
|
+
* @param options What the picker offers first, or the panel to share and its name (`PanelLabel`), which the share's
|
|
1592
|
+
* state carries and the agent sees as "<name> (shared panel)".
|
|
1593
|
+
*/
|
|
1594
|
+
async StartScreenShare(options) {
|
|
1595
|
+
if (!this.captures) {
|
|
1596
|
+
return this.noSessionCapture();
|
|
1597
|
+
}
|
|
1598
|
+
const started = this.captures.Start('screen', options);
|
|
1599
|
+
await this.revealCaptureChannel('screen');
|
|
1600
|
+
return started;
|
|
1601
|
+
}
|
|
1602
|
+
/** Stops the user's screen share. Safe to call when nothing is shared. */
|
|
1603
|
+
StopScreenShare() {
|
|
1604
|
+
this.captures?.Stop('screen');
|
|
1605
|
+
}
|
|
1606
|
+
/**
|
|
1607
|
+
* Why the latest session start failed, or `null` when it has not failed or none has run.
|
|
767
1608
|
*
|
|
768
1609
|
* Read it when {@link ConnectionState$} reports `'error'`, to tell a denied microphone apart from
|
|
769
|
-
* a provider or backend failure and show copy the user can act on.
|
|
770
|
-
*
|
|
1610
|
+
* a provider or backend failure and show copy the user can act on. Each start clears it before it
|
|
1611
|
+
* reports `'connecting'`, so it never describes an earlier start.
|
|
771
1612
|
*/
|
|
772
1613
|
get LastStartError() {
|
|
773
1614
|
return this.lastStartError;
|
|
774
1615
|
}
|
|
1616
|
+
/**
|
|
1617
|
+
* The fatal error the call's client reported, or `null` when it has reported none.
|
|
1618
|
+
*
|
|
1619
|
+
* Read it when {@link ConnectionState$} reports `'error'` on a call that was live, to tell a call that reached its time
|
|
1620
|
+
* limit (`Code` `'time-limit'`; {@link DescribeCallEnd} words it) from a dropped connection. Each start clears it before
|
|
1621
|
+
* it reports `'connecting'`.
|
|
1622
|
+
*/
|
|
1623
|
+
get LastSessionError() {
|
|
1624
|
+
return this.lastSessionError;
|
|
1625
|
+
}
|
|
775
1626
|
/**
|
|
776
1627
|
* The single failure path for a session start (mint half or run half): report it, latch the
|
|
777
1628
|
* overlay into 'error', and unwind whatever the half-built session already opened.
|
|
@@ -779,7 +1630,7 @@ export class RealtimeSessionRuntime {
|
|
|
779
1630
|
async failSessionStart(error) {
|
|
780
1631
|
console.error('[RealtimeSession] Failed to start session:', error);
|
|
781
1632
|
this.lastStartError = error instanceof Error ? error : new Error(String(error));
|
|
782
|
-
this.
|
|
1633
|
+
this.setConnectionState('error');
|
|
783
1634
|
await this.teardown(false);
|
|
784
1635
|
}
|
|
785
1636
|
/**
|
|
@@ -816,16 +1667,21 @@ export class RealtimeSessionRuntime {
|
|
|
816
1667
|
}
|
|
817
1668
|
client.SendText(trimmed);
|
|
818
1669
|
// Relay as a user turn — same path spoken input uses (caption + persisted transcript).
|
|
819
|
-
void this.onUserTranscript(trimmed);
|
|
1670
|
+
void this.onUserTranscript(trimmed, 'typed');
|
|
820
1671
|
}
|
|
821
|
-
/**
|
|
1672
|
+
/**
|
|
1673
|
+
* Mute / unmute the local microphone track. Returns the new muted state. While the call has no microphone track (a
|
|
1674
|
+
* switch is putting a new one in, or the call lost its microphone) there is nothing to mute: nothing changes, and it
|
|
1675
|
+
* returns the call's mute, which the next microphone keeps. Outside a call it returns `false`.
|
|
1676
|
+
*/
|
|
822
1677
|
ToggleMute() {
|
|
823
1678
|
const tracks = this.localStream?.getAudioTracks() ?? [];
|
|
824
1679
|
if (tracks.length === 0) {
|
|
825
|
-
return
|
|
1680
|
+
return this.microphoneMuted;
|
|
826
1681
|
}
|
|
827
1682
|
const muted = tracks[0].enabled; // currently enabled → becomes muted
|
|
828
1683
|
this.client?.SetMuted(muted);
|
|
1684
|
+
this.microphoneMuted = muted;
|
|
829
1685
|
return muted;
|
|
830
1686
|
}
|
|
831
1687
|
// ── Client-executed UI tools ───────────────────────────────────────────────
|
|
@@ -895,13 +1751,21 @@ export class RealtimeSessionRuntime {
|
|
|
895
1751
|
return this.client;
|
|
896
1752
|
}
|
|
897
1753
|
/**
|
|
898
|
-
* Relays a video frame to the underlying realtime client if
|
|
1754
|
+
* Relays a video frame straight to the underlying realtime client, if a session is live.
|
|
1755
|
+
*
|
|
1756
|
+
* It bypasses the session's {@link VideoSourceArbiter}: the frame overrides the source the arbiter picked, and the
|
|
1757
|
+
* model is not told what it is looking at. Channels no longer send through it (their context's `SendVideoFrame` goes
|
|
1758
|
+
* through the arbiter); a host that shows the model something registers a source there instead, as
|
|
1759
|
+
* `ChannelInboundVideoBridge` does.
|
|
1760
|
+
*
|
|
1761
|
+
* @returns `true` when the client accepted and sent the frame; `false` when no session is live, the
|
|
1762
|
+
* client cannot take video frames, or the client dropped the frame.
|
|
899
1763
|
*/
|
|
900
1764
|
SendVideoFrame(base64Image, mimeType) {
|
|
901
1765
|
if (!this.client || !this.isSessionLive()) {
|
|
902
|
-
return;
|
|
1766
|
+
return false;
|
|
903
1767
|
}
|
|
904
|
-
this.client.SendVideoFrame?.(base64Image, mimeType);
|
|
1768
|
+
return this.client.SendVideoFrame?.(base64Image, mimeType) === true;
|
|
905
1769
|
}
|
|
906
1770
|
/**
|
|
907
1771
|
* Checks whether a media track is established on the active realtime client.
|
|
@@ -948,12 +1812,16 @@ export class RealtimeSessionRuntime {
|
|
|
948
1812
|
// turn one's cue wherever the clocks happen to differ.
|
|
949
1813
|
this.currentTurnStartMs = recorder.IsRecording ? 0 : null;
|
|
950
1814
|
this.turnAudioStartCaptured = false;
|
|
1815
|
+
this.userSpeechStartMs = null;
|
|
1816
|
+
this.userSpeechStartSuperseded = false;
|
|
951
1817
|
if (this.recorder) {
|
|
952
|
-
//
|
|
953
|
-
// above is
|
|
954
|
-
//
|
|
1818
|
+
// WebRTC drivers (OpenAI): the agent's track usually lands AFTER Connect() resolves, so
|
|
1819
|
+
// `remoteStream` above is null here and this handler attaches it later. PCM-playback
|
|
1820
|
+
// drivers (Gemini, ElevenLabs, AssemblyAI, xAI, HuggingFace) publish at Connect, so
|
|
1821
|
+
// `remoteStream` is already set and the handler fires immediately with the same stream;
|
|
1822
|
+
// AttachRemoteStream is idempotent, so it is mixed only once.
|
|
955
1823
|
client.OnRemoteMediaStream?.((stream) => this.recorder?.AttachRemoteStream(stream));
|
|
956
|
-
this.
|
|
1824
|
+
this.startShardFlushing();
|
|
957
1825
|
}
|
|
958
1826
|
}
|
|
959
1827
|
catch (error) {
|
|
@@ -961,201 +1829,928 @@ export class RealtimeSessionRuntime {
|
|
|
961
1829
|
this.recorder = null;
|
|
962
1830
|
}
|
|
963
1831
|
}
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
1832
|
+
// ── Recording shards (MJ#5294) ───────────────────────────────────────────────
|
|
1833
|
+
//
|
|
1834
|
+
// The recorder hands over Opus packets; the runtime cuts them into 5 s `MJOPUS1` shards, writes
|
|
1835
|
+
// each one to the local store BEFORE its upload starts, uploads it as raw bytes, and deletes it from
|
|
1836
|
+
// the store only on the server's ack. The call ends with a `keepalive` terminal request carrying the
|
|
1837
|
+
// final partial shard and the counts the server needs to know it has everything.
|
|
1838
|
+
/**
|
|
1839
|
+
* Builds the transport for shard uploads and terminal requests. The default posts to MJAPI's REST
|
|
1840
|
+
* recording routes, derived from the session's GraphQL provider configuration, with credentials
|
|
1841
|
+
* read per request so a refreshed token is used. Returns null — shards then wait in the local
|
|
1842
|
+
* store — when the provider has no URL (a test double, a provider that is not GraphQL, or one not
|
|
1843
|
+
* configured yet). Override to supply another transport.
|
|
1844
|
+
*/
|
|
1845
|
+
createRecordingTransport() {
|
|
1846
|
+
return CreateRecordingTransportForProvider(this.Provider);
|
|
1847
|
+
}
|
|
1848
|
+
/**
|
|
1849
|
+
* Builds the durable local shard store. The default is IndexedDB where the host has it and a no-op
|
|
1850
|
+
* store elsewhere (Node, some React Native runtimes). Called once per runtime. Override to supply
|
|
1851
|
+
* another store.
|
|
1852
|
+
*/
|
|
1853
|
+
createRecordingShardStore() {
|
|
1854
|
+
return CreateRecordingShardStore(globalThis.indexedDB);
|
|
1855
|
+
}
|
|
1856
|
+
/** The provider's signed-in user, or null (no provider yet, or an anonymous page). */
|
|
1857
|
+
signedInUserID() {
|
|
1858
|
+
return this.Provider?.CurrentUser?.ID ?? null;
|
|
1859
|
+
}
|
|
1860
|
+
/** The runtime's shard store, created on first use. */
|
|
1861
|
+
ensureShardStore() {
|
|
1862
|
+
this.shardStore ??= this.createRecordingShardStore();
|
|
1863
|
+
return this.shardStore;
|
|
1864
|
+
}
|
|
1865
|
+
/**
|
|
1866
|
+
* Re-uploads shards an earlier, interrupted call left in the local store (crash, closed tab, dead
|
|
1867
|
+
* network), each under its OWN session id and index, oldest first, at most
|
|
1868
|
+
* {@link MAX_DRAINED_SHARDS_PER_LOAD} per runtime. A shard is deleted only on the server's ack.
|
|
1869
|
+
*
|
|
1870
|
+
* Runs once per runtime instance, re-armed whenever a recording ends (which may leave shards behind
|
|
1871
|
+
* for the next session start on the same page). When there is no transport yet (no provider configuration, e.g.
|
|
1872
|
+
* before login) it returns 0 WITHOUT marking itself done, so a later call — session start — drains
|
|
1873
|
+
* instead of the shards being skipped for this page. Concurrent calls share one run. Never rejects.
|
|
1874
|
+
*
|
|
1875
|
+
* @returns the number of shards the server acknowledged in this call.
|
|
1876
|
+
*/
|
|
1877
|
+
DrainPendingRecordingShards() {
|
|
1878
|
+
if (this.pendingShardsDrained) {
|
|
1879
|
+
return Promise.resolve(0);
|
|
1880
|
+
}
|
|
1881
|
+
this.drainInFlight ??= this.drainPendingShards().finally(() => {
|
|
1882
|
+
this.drainInFlight = null;
|
|
1883
|
+
});
|
|
1884
|
+
return this.drainInFlight;
|
|
1885
|
+
}
|
|
1886
|
+
/** Makes the next {@link DrainPendingRecordingShards} call run again, even one already in flight. */
|
|
1887
|
+
rearmPendingShardDrain() {
|
|
1888
|
+
this.pendingShardsDrained = false;
|
|
1889
|
+
this.drainGeneration++;
|
|
1890
|
+
}
|
|
1891
|
+
/**
|
|
1892
|
+
* The body of {@link DrainPendingRecordingShards}, up to {@link maxDrainRunsPerCall} runs. It runs
|
|
1893
|
+
* again after a re-arm while a run is in flight (a recovery after an outage gave up on more shards),
|
|
1894
|
+
* because the running pass listed the store before those shards were there; and after a run deleted
|
|
1895
|
+
* refused shards, because those filled the page that the shards behind them now get.
|
|
1896
|
+
*/
|
|
1897
|
+
async drainPendingShards() {
|
|
1898
|
+
try {
|
|
1899
|
+
const transport = this.createRecordingTransport();
|
|
1900
|
+
if (!transport) {
|
|
1901
|
+
return 0;
|
|
1902
|
+
}
|
|
1903
|
+
let delivered = 0;
|
|
1904
|
+
for (let run = 1; run <= RealtimeSessionRuntime.maxDrainRunsPerCall; run++) {
|
|
1905
|
+
// A re-arm while this run is in flight must win over this run marking itself done.
|
|
1906
|
+
const generation = this.drainGeneration;
|
|
1907
|
+
const outcome = await this.drainStoredShardsOnce(transport);
|
|
1908
|
+
delivered += outcome.Delivered;
|
|
1909
|
+
// Done only when every listed shard was delivered or refused for good: a drain during an outage
|
|
1910
|
+
// must not stop the next session start or recovery from trying again.
|
|
1911
|
+
this.pendingShardsDrained = generation === this.drainGeneration && outcome.Delivered + outcome.Refused === outcome.Listed;
|
|
1912
|
+
if (generation === this.drainGeneration && outcome.Refused === 0) {
|
|
1913
|
+
break;
|
|
1914
|
+
}
|
|
1915
|
+
}
|
|
1916
|
+
return delivered;
|
|
1917
|
+
}
|
|
1918
|
+
catch (error) {
|
|
1919
|
+
console.warn(`[RealtimeSession] Draining stored recording shards failed: ${describeRecordingFailure(error)}; will retry at the next session start.`, error);
|
|
1920
|
+
return 0;
|
|
1921
|
+
}
|
|
1922
|
+
}
|
|
1923
|
+
/** One drain pass over the signed-in user's stored shards (see {@link DrainStoredRecordingShardsOnce}). */
|
|
1924
|
+
drainStoredShardsOnce(transport) {
|
|
1925
|
+
return DrainStoredRecordingShardsOnce(this.ensureShardStore(), transport, this.signedInUserID());
|
|
1926
|
+
}
|
|
1927
|
+
/** Builds this recording's shard pipeline and starts the 5 s flush. */
|
|
1928
|
+
startShardFlushing() {
|
|
1929
|
+
const sessionId = this.agentSessionId;
|
|
1930
|
+
if (!sessionId) {
|
|
1931
|
+
// Unreachable in practice (the session id is adopted before recording starts), but without
|
|
1932
|
+
// one there is nowhere to send shards, so say so rather than recording into the void.
|
|
1933
|
+
console.warn('[RealtimeSession] Recording started without an agent session id; shards will not be saved.');
|
|
1934
|
+
return;
|
|
1935
|
+
}
|
|
1936
|
+
const transport = this.createRecordingTransport();
|
|
1937
|
+
if (!transport) {
|
|
1938
|
+
console.warn(`[RealtimeSession] No recording transport for session ${sessionId} (provider has no REST configuration); shards are kept in the local store only.`);
|
|
1939
|
+
}
|
|
1940
|
+
// True once the queue gave up on a shard and until an upload succeeds again: that first success
|
|
1941
|
+
// re-sends what the queue dropped meanwhile (see recoverStoredShardsAfterOutage).
|
|
1942
|
+
let shardsGivenUp = false;
|
|
1943
|
+
this.recordingShards = {
|
|
1944
|
+
SessionID: sessionId,
|
|
1945
|
+
UserID: this.signedInUserID(),
|
|
1946
|
+
Window: new RecordingShardWindow(),
|
|
1947
|
+
Transport: transport,
|
|
1948
|
+
// Closes over this recording's session id and transport — never the runtime's current ones.
|
|
1949
|
+
Queue: new RecordingShardQueue(async (shard) => {
|
|
1950
|
+
if (!transport) {
|
|
1951
|
+
return false;
|
|
1952
|
+
}
|
|
1953
|
+
const acked = (await transport.UploadShard(sessionId, shard)) === 'Acked';
|
|
1954
|
+
if (acked) {
|
|
1955
|
+
await this.deleteStoredShard(sessionId, shard.Index);
|
|
1956
|
+
if (shardsGivenUp) {
|
|
1957
|
+
shardsGivenUp = false;
|
|
1958
|
+
this.recoverStoredShardsAfterOutage(sessionId);
|
|
1959
|
+
}
|
|
1960
|
+
}
|
|
1961
|
+
return acked;
|
|
1962
|
+
}, () => {
|
|
1963
|
+
shardsGivenUp = true;
|
|
1964
|
+
}),
|
|
1965
|
+
};
|
|
1966
|
+
this.shardFlushTimer = setInterval(() => { void this.flushRecordingShards(); }, RealtimeSessionRuntime.recordingShardFlushMs);
|
|
1967
|
+
}
|
|
1968
|
+
/**
|
|
1969
|
+
* Uploads succeed again after failing: re-sends, from the local store, the shards the queue gave up
|
|
1970
|
+
* on during the outage. The queue's attempts run out after about 10 s of a dead network (or an
|
|
1971
|
+
* expired token), and without this those shards would wait for a later page load — which a widget
|
|
1972
|
+
* guest never makes — leaving a permanent gap. Runs in the background, outside any page-lifecycle
|
|
1973
|
+
* handler, so the IndexedDB read is allowed. Never rejects.
|
|
1974
|
+
*/
|
|
1975
|
+
recoverStoredShardsAfterOutage(sessionId) {
|
|
1976
|
+
console.log(`[RealtimeSession] Recording uploads for session ${sessionId} recovered; re-sending stored shards.`);
|
|
1977
|
+
this.rearmPendingShardDrain();
|
|
1978
|
+
void this.DrainPendingRecordingShards();
|
|
1979
|
+
}
|
|
1980
|
+
/** Stops the periodic shard flush. */
|
|
1981
|
+
stopShardFlushing() {
|
|
1982
|
+
if (this.shardFlushTimer) {
|
|
1983
|
+
clearInterval(this.shardFlushTimer);
|
|
1984
|
+
this.shardFlushTimer = null;
|
|
1985
|
+
}
|
|
1986
|
+
}
|
|
1987
|
+
/**
|
|
1988
|
+
* Starts telling the server this session is still in use (#3533).
|
|
1989
|
+
*
|
|
1990
|
+
* **Why the server cannot work this out on its own.** In the client-direct topology the audio
|
|
1991
|
+
* goes browser → provider over WebRTC. The server sees the mint, a few channel actions in the
|
|
1992
|
+
* first seconds, and then nothing at all — so `SessionManager.RecordActivity` stops being
|
|
1993
|
+
* reached while the conversation is still going. `LastActiveAt` freezes ~45 seconds in, and
|
|
1994
|
+
* `SessionJanitor` — which cannot distinguish an active call from an abandoned one — force-closes
|
|
1995
|
+
* it at `closeThresholdMinutes`, mid-sentence, taking the user's surfaces with it. A session
|
|
1996
|
+
* whose channels are all client-side (whiteboard, media) goes quiet from the server's point of
|
|
1997
|
+
* view almost immediately.
|
|
1998
|
+
*
|
|
1999
|
+
* The browser is the only participant that knows the call is alive, so it is the one that has to
|
|
2000
|
+
* say so. Raising `closeThresholdMinutes` is not the fix — it just makes the janitor slower at
|
|
2001
|
+
* its real job (reaping rows orphaned by a crash) without making liveness observable.
|
|
2002
|
+
*
|
|
2003
|
+
* The pulse is best-effort by design: a failed beat is logged and skipped, never surfaced to the
|
|
2004
|
+
* user and never allowed to end the session. Losing one beat costs nothing because the threshold
|
|
2005
|
+
* is many beats wide; turning a transient network blip into a visible error would be a worse
|
|
2006
|
+
* failure than the one this fixes. Write amplification is bounded on the server side too, where
|
|
2007
|
+
* `SessionManager.Heartbeat` coalesces persisted writes.
|
|
2008
|
+
*/
|
|
2009
|
+
startLivenessPulse() {
|
|
2010
|
+
this.stopLivenessPulse();
|
|
2011
|
+
this.livenessTimer = setInterval(() => { void this.pulseLiveness(); }, RealtimeSessionRuntime.livenessPulseMs);
|
|
2012
|
+
}
|
|
2013
|
+
/** Stops the liveness pulse. Idempotent — safe on a session that never started one. */
|
|
2014
|
+
stopLivenessPulse() {
|
|
2015
|
+
if (this.livenessTimer) {
|
|
2016
|
+
clearInterval(this.livenessTimer);
|
|
2017
|
+
this.livenessTimer = null;
|
|
2018
|
+
}
|
|
2019
|
+
}
|
|
2020
|
+
/**
|
|
2021
|
+
* One liveness beat. Reads the session id at fire time rather than closing over it, so a beat
|
|
2022
|
+
* that fires during teardown finds `null` and does nothing instead of resurrecting a closed row.
|
|
2023
|
+
*/
|
|
2024
|
+
async pulseLiveness() {
|
|
2025
|
+
const agentSessionId = this.agentSessionId;
|
|
2026
|
+
if (!agentSessionId) {
|
|
2027
|
+
return;
|
|
2028
|
+
}
|
|
2029
|
+
const mutation = `
|
|
2030
|
+
mutation AgentSessionHeartbeat($agentSessionId: String!) {
|
|
2031
|
+
AgentSessionHeartbeat(agentSessionId: $agentSessionId)
|
|
2032
|
+
}
|
|
2033
|
+
`;
|
|
2034
|
+
try {
|
|
2035
|
+
await this.gql().ExecuteGQL(mutation, { agentSessionId });
|
|
2036
|
+
}
|
|
2037
|
+
catch (error) {
|
|
2038
|
+
// Best-effort: the next beat is 60s away and the janitor threshold is many beats wide.
|
|
2039
|
+
console.warn('[RealtimeSession] Liveness pulse failed (session continues):', error);
|
|
2040
|
+
}
|
|
2041
|
+
}
|
|
2042
|
+
/**
|
|
2043
|
+
* One 5 s tick: cuts the packets encoded since the last tick into shards, stores each one, then
|
|
2044
|
+
* uploads them. Best-effort — never disturbs the call and never rejects. Overlapping ticks are
|
|
2045
|
+
* safe: `Append` and `SealTail` are synchronous, so indexes are assigned in order, and the queue
|
|
2046
|
+
* bounds how many flushes upload at once. A failed shard is retained by the queue and resent under
|
|
2047
|
+
* its own index on later ticks (#5197), and stays in the store until acknowledged.
|
|
2048
|
+
*
|
|
2049
|
+
* Every tick seals whatever it took, even under a full window (the window's 250-packet and byte caps
|
|
2050
|
+
* still split a larger batch). Capture starts slightly after this timer, so a tick almost never
|
|
2051
|
+
* holds exactly 250 packets; sealing only full windows would leave nearly 5 s of encoded audio in
|
|
2052
|
+
* memory at every moment, unstored and unsent, which a killed tab loses.
|
|
2053
|
+
*/
|
|
2054
|
+
async flushRecordingShards() {
|
|
2055
|
+
const recorder = this.recorder;
|
|
2056
|
+
const pipeline = this.recordingShards;
|
|
2057
|
+
if (!recorder || !pipeline) {
|
|
2058
|
+
return;
|
|
2059
|
+
}
|
|
2060
|
+
let sealed;
|
|
2061
|
+
try {
|
|
2062
|
+
sealed = pipeline.Window.Append(recorder.TakeEncodedPackets());
|
|
2063
|
+
const tick = pipeline.Window.SealTail();
|
|
2064
|
+
if (tick) {
|
|
2065
|
+
sealed.push(tick);
|
|
2066
|
+
}
|
|
2067
|
+
}
|
|
2068
|
+
catch (error) {
|
|
2069
|
+
console.warn(`[RealtimeSession] Failed to take encoded recording packets for session ${pipeline.SessionID}; this tick's audio is lost:`, error);
|
|
2070
|
+
return;
|
|
2071
|
+
}
|
|
2072
|
+
await this.storeShards(pipeline, sealed);
|
|
2073
|
+
await pipeline.Queue.Flush(sealed);
|
|
2074
|
+
}
|
|
2075
|
+
/** Writes shards to the local store, in order, before their upload starts. Never rejects. */
|
|
2076
|
+
async storeShards(pipeline, shards) {
|
|
2077
|
+
const store = this.ensureShardStore();
|
|
2078
|
+
for (const shard of shards) {
|
|
2079
|
+
try {
|
|
2080
|
+
await store.Put({ SessionID: pipeline.SessionID, UserID: pipeline.UserID, Index: shard.Index, Bytes: shard.Bytes, SavedAt: Date.now() });
|
|
2081
|
+
}
|
|
2082
|
+
catch (error) {
|
|
2083
|
+
// The store contract is never-throw; an override that does must still not stop the upload.
|
|
2084
|
+
console.warn(`[RealtimeSession] Failed to store recording shard ${pipeline.SessionID}/${shard.Index}; uploading it anyway:`, error);
|
|
2085
|
+
}
|
|
2086
|
+
}
|
|
2087
|
+
}
|
|
2088
|
+
/** Removes an acknowledged shard from the local store. Never rejects. */
|
|
2089
|
+
deleteStoredShard(sessionId, index) {
|
|
2090
|
+
return DeleteStoredRecordingShard(this.ensureShardStore(), sessionId, index);
|
|
2091
|
+
}
|
|
2092
|
+
/**
|
|
2093
|
+
* Tells the server, right now, how much of the recording exists — for a host's `pagehide` and
|
|
2094
|
+
* `visibilitychange → hidden` handlers, where the page may be gone a moment later. Synchronous:
|
|
2095
|
+
* it seals the pending partial shard and fires the `keepalive` terminal with it, then re-sends
|
|
2096
|
+
* with `keepalive` whatever the server has not acknowledged yet (see
|
|
2097
|
+
* {@link dispatchRecordingTerminal}), and returns without waiting. Never stops the recorder: after
|
|
2098
|
+
* a `hidden` the call may well continue, and later audio simply starts the next shard index (the
|
|
2099
|
+
* tail is sealed, so no index is ever re-sent with different bytes). Also works while End is still
|
|
2100
|
+
* stopping the recorder, which is exactly when a host that does not await End navigates away.
|
|
2101
|
+
* No-op when nothing is recording, there is no session id or no transport, or nothing was
|
|
2102
|
+
* captured. Never throws.
|
|
2103
|
+
*
|
|
2104
|
+
* Deliberately opens NO IndexedDB transaction inside the handler: a `pagehide` handler gets a few
|
|
2105
|
+
* milliseconds, and a store write started there could not complete during unload anyway. The
|
|
2106
|
+
* shards it sealed are written to the store from a `setTimeout(0)` task after the handler returns:
|
|
2107
|
+
* on a page that is merely hidden (the call goes on) that keeps the durable copy until the server
|
|
2108
|
+
* acknowledges it; on a page that is unloading the task just never runs.
|
|
2109
|
+
*/
|
|
2110
|
+
SendRecordingTerminal(options) {
|
|
2111
|
+
const recorder = this.recorder;
|
|
2112
|
+
const pipeline = this.recordingShards;
|
|
2113
|
+
if (!recorder || !pipeline || !pipeline.Transport) {
|
|
2114
|
+
return;
|
|
2115
|
+
}
|
|
2116
|
+
try {
|
|
2117
|
+
const sealed = pipeline.Window.Append(recorder.TakeEncodedPackets());
|
|
2118
|
+
const tail = pipeline.Window.SealTail();
|
|
2119
|
+
if (pipeline.Window.LastSealedIndex < 0) {
|
|
2120
|
+
return;
|
|
2121
|
+
}
|
|
2122
|
+
const final = options?.Final ?? true;
|
|
2123
|
+
if (sealed.length === 0 && !tail && pipeline.FinalTerminal?.LastSealedIndex === pipeline.Window.LastSealedIndex) {
|
|
2124
|
+
return;
|
|
2125
|
+
}
|
|
2126
|
+
void this.dispatchRecordingTerminal(pipeline, sealed, tail, true, final);
|
|
2127
|
+
}
|
|
2128
|
+
catch (error) {
|
|
2129
|
+
console.warn(`[RealtimeSession] Failed to send the recording terminal for session ${pipeline.SessionID}:`, error);
|
|
2130
|
+
}
|
|
2131
|
+
}
|
|
2132
|
+
/**
|
|
2133
|
+
* Fires the terminal request, then re-sends with `keepalive` every shard the server has not
|
|
2134
|
+
* acknowledged, then stores the shards sealed for this terminal. Shared by End and the page-lifecycle
|
|
2135
|
+
* handlers. Everything that needs the network is dispatched synchronously, before any storage work,
|
|
2136
|
+
* because the page may close at any moment from here on: an ordinary `fetch` is aborted on unload,
|
|
2137
|
+
* and a `keepalive` one is not.
|
|
2138
|
+
*
|
|
2139
|
+
* - The terminal goes first so it always gets the keepalive budget. The server waits for the shards
|
|
2140
|
+
* the terminal says exist, and assembles as soon as the last missing one arrives.
|
|
2141
|
+
* - The re-sent shards are the queue's unacknowledged ones (on the wire, waiting in a batch, or
|
|
2142
|
+
* retained after a failure) plus `sealedNow`, oldest first, within
|
|
2143
|
+
* {@link recordingKeepaliveBudgetBytes} together with the tail. A `sealedNow` shard that does not
|
|
2144
|
+
* fit, or whose keepalive upload fails, goes through the ordinary queue instead (it may still
|
|
2145
|
+
* land if the page stays). The queue keeps its own shards either way; a duplicate is acked as one.
|
|
2146
|
+
* - `sealedNow` and the tail are stored after the dispatch, from a `setTimeout(0)` task when called
|
|
2147
|
+
* from a page-lifecycle handler (`deferStore`). A shard's stored copy is deleted on its ack, after
|
|
2148
|
+
* the store write, so a fast ack can never leave an orphan row.
|
|
2149
|
+
*
|
|
2150
|
+
* Resolves true when the terminal was acknowledged. Never rejects.
|
|
2151
|
+
*/
|
|
2152
|
+
dispatchRecordingTerminal(pipeline, sealedNow, tail, deferStore, final) {
|
|
2153
|
+
const sessionId = pipeline.SessionID;
|
|
2154
|
+
const terminalAcked = this.sendRecordingTerminal(pipeline, tail, final);
|
|
2155
|
+
const toStore = tail ? [...sealedNow, tail] : sealedNow;
|
|
2156
|
+
let stored = Promise.resolve();
|
|
2157
|
+
if (toStore.length > 0) {
|
|
2158
|
+
stored = deferStore ? this.storeShardsLater(pipeline, toStore) : this.storeShards(pipeline, toStore);
|
|
2159
|
+
}
|
|
2160
|
+
this.resendUnacknowledgedWithKeepalive(pipeline, sealedNow, stored);
|
|
2161
|
+
const ackedPromise = terminalAcked.then(async (acked) => {
|
|
2162
|
+
await stored;
|
|
2163
|
+
if (!tail) {
|
|
2164
|
+
return acked;
|
|
2165
|
+
}
|
|
2166
|
+
if (acked) {
|
|
2167
|
+
await this.deleteStoredShard(sessionId, tail.Index);
|
|
2168
|
+
}
|
|
2169
|
+
else {
|
|
2170
|
+
// Retried as an ordinary shard; a Flush on a closed queue (the recording ended) is a logged no-op.
|
|
2171
|
+
await pipeline.Queue.Flush([tail]);
|
|
2172
|
+
}
|
|
2173
|
+
return acked;
|
|
2174
|
+
});
|
|
2175
|
+
if (final) {
|
|
2176
|
+
pipeline.FinalTerminal = { LastSealedIndex: pipeline.Window.LastSealedIndex, Acked: terminalAcked };
|
|
2177
|
+
}
|
|
2178
|
+
return ackedPromise;
|
|
2179
|
+
}
|
|
2180
|
+
/** Counts `bytes` against the page's keepalive budget until `request` settles. */
|
|
2181
|
+
trackKeepalive(bytes, request) {
|
|
2182
|
+
this.keepaliveBytesInFlight += bytes;
|
|
2183
|
+
return request.finally(() => {
|
|
2184
|
+
this.keepaliveBytesInFlight -= bytes;
|
|
2185
|
+
});
|
|
2186
|
+
}
|
|
2187
|
+
/**
|
|
2188
|
+
* The `keepalive` re-send part of {@link dispatchRecordingTerminal}, budgeted against everything this
|
|
2189
|
+
* page still has in flight (the terminal just sent included). A shard whose `keepalive` copy is already
|
|
2190
|
+
* on the wire is not sent again. Never throws.
|
|
2191
|
+
*/
|
|
2192
|
+
resendUnacknowledgedWithKeepalive(pipeline, sealedNow, stored) {
|
|
2193
|
+
const transport = pipeline.Transport;
|
|
2194
|
+
const sessionId = pipeline.SessionID;
|
|
2195
|
+
const fresh = new Set(sealedNow.map((shard) => shard.Index));
|
|
2196
|
+
const toQueue = (shard) => stored.then(() => pipeline.Queue.Flush([shard]));
|
|
2197
|
+
const byIndex = new Map();
|
|
2198
|
+
for (const shard of [...pipeline.Queue.Unacknowledged(), ...sealedNow]) {
|
|
2199
|
+
byIndex.set(shard.Index, shard);
|
|
2200
|
+
}
|
|
2201
|
+
for (const shard of [...byIndex.values()].sort((a, b) => a.Index - b.Index)) {
|
|
2202
|
+
const key = `${sessionId}/${shard.Index}`;
|
|
2203
|
+
if (this.keepaliveShardsInFlight.has(key)) {
|
|
2204
|
+
continue;
|
|
2205
|
+
}
|
|
2206
|
+
const budget = RealtimeSessionRuntime.recordingKeepaliveBudgetBytes - this.keepaliveBytesInFlight;
|
|
2207
|
+
if (!transport || shard.Bytes.length > budget) {
|
|
2208
|
+
if (fresh.has(shard.Index)) {
|
|
2209
|
+
void toQueue(shard);
|
|
2210
|
+
}
|
|
2211
|
+
continue;
|
|
2212
|
+
}
|
|
2213
|
+
this.keepaliveShardsInFlight.add(key);
|
|
2214
|
+
void this.trackKeepalive(shard.Bytes.length, transport.UploadShard(sessionId, shard, { Keepalive: true }).then((outcome) => outcome === 'Acked'))
|
|
2215
|
+
.finally(() => this.keepaliveShardsInFlight.delete(key))
|
|
2216
|
+
.catch((error) => {
|
|
2217
|
+
console.warn(`[RealtimeSession] Keepalive upload of recording shard ${sessionId}/${shard.Index} failed: ${describeRecordingFailure(error)}`, error);
|
|
2218
|
+
return false;
|
|
2219
|
+
})
|
|
2220
|
+
.then(async (acked) => {
|
|
2221
|
+
await stored;
|
|
2222
|
+
if (acked) {
|
|
2223
|
+
await this.deleteStoredShard(sessionId, shard.Index);
|
|
2224
|
+
}
|
|
2225
|
+
else if (fresh.has(shard.Index)) {
|
|
2226
|
+
await pipeline.Queue.Flush([shard]);
|
|
2227
|
+
}
|
|
2228
|
+
});
|
|
2229
|
+
}
|
|
2230
|
+
}
|
|
2231
|
+
/**
|
|
2232
|
+
* Stores shards from a `setTimeout(0)` task rather than now — the path for shards sealed inside a
|
|
2233
|
+
* page-lifecycle handler, which must not open an IndexedDB transaction itself. Resolves once they
|
|
2234
|
+
* are stored (never, if the page unloads first). Never rejects.
|
|
2235
|
+
*/
|
|
2236
|
+
storeShardsLater(pipeline, shards) {
|
|
2237
|
+
return new Promise((resolve) => {
|
|
2238
|
+
setTimeout(() => {
|
|
2239
|
+
void this.storeShards(pipeline, shards).then(resolve);
|
|
2240
|
+
}, 0);
|
|
2241
|
+
});
|
|
2242
|
+
}
|
|
2243
|
+
/**
|
|
2244
|
+
* Sends the terminal request for a pipeline whose tail is already sealed. Resolves true on a server
|
|
2245
|
+
* ack and logs the outcome with the session id. Never rejects.
|
|
2246
|
+
*/
|
|
2247
|
+
async sendRecordingTerminal(pipeline, tail, final) {
|
|
2248
|
+
const lastShardIndex = pipeline.Window.LastSealedIndex;
|
|
2249
|
+
const capturedSamples = pipeline.Window.SealedSamples;
|
|
2250
|
+
const context = `session ${pipeline.SessionID} (last shard ${lastShardIndex}, ${capturedSamples} samples)`;
|
|
2251
|
+
if (!pipeline.Transport) {
|
|
2252
|
+
console.warn(`[RealtimeSession] Recording terminal NOT sent for ${context}: no recording transport; stored shards are drained on a later load.`);
|
|
2253
|
+
return false;
|
|
2254
|
+
}
|
|
2255
|
+
try {
|
|
2256
|
+
const acked = await this.trackKeepalive(tail?.Bytes.length ?? 0, pipeline.Transport.SendTerminal(pipeline.SessionID, {
|
|
2257
|
+
Tail: tail?.Bytes ?? null,
|
|
2258
|
+
LastShardIndex: lastShardIndex,
|
|
2259
|
+
CapturedSamples: capturedSamples,
|
|
2260
|
+
Final: final,
|
|
2261
|
+
}));
|
|
2262
|
+
if (!acked) {
|
|
2263
|
+
console.warn(`[RealtimeSession] Recording terminal for ${context} was not acknowledged; the server janitor assembles what arrived.`);
|
|
2264
|
+
}
|
|
2265
|
+
return acked;
|
|
2266
|
+
}
|
|
2267
|
+
catch (error) {
|
|
2268
|
+
console.warn(`[RealtimeSession] Recording terminal for ${context} failed: ${describeRecordingFailure(error)}`, error);
|
|
2269
|
+
return false;
|
|
2270
|
+
}
|
|
2271
|
+
}
|
|
2272
|
+
/**
|
|
2273
|
+
* Ends the recording at teardown and STARTS its terminal request: stops the recorder (flushing the
|
|
2274
|
+
* encoder), cuts the remaining packets into shards with the tail trimmed to the recorder's captured
|
|
2275
|
+
* sample count, and dispatches the `keepalive` terminal plus `keepalive` re-sends of everything not
|
|
2276
|
+
* yet acknowledged, before storing anything ({@link dispatchRecordingTerminal}). Teardown awaits the
|
|
2277
|
+
* returned terminal only after `CloseAgentSession` ({@link awaitRecordingTerminal}), so the small,
|
|
2278
|
+
* critical close never waits behind the recording (#5195).
|
|
2279
|
+
*
|
|
2280
|
+
* The recorder and pipeline stay visible to {@link SendRecordingTerminal} until the recorder has
|
|
2281
|
+
* stopped: a `pagehide` in that window (a host that navigates away without awaiting End) still
|
|
2282
|
+
* sends what was captured, and End then continues at the next index.
|
|
2283
|
+
*
|
|
2284
|
+
* Returns null when nothing was recording or nothing was captured (no terminal is sent then).
|
|
2285
|
+
* Best-effort: never throws, never blocks teardown on the network.
|
|
2286
|
+
*/
|
|
2287
|
+
async finishRecording() {
|
|
2288
|
+
this.stopShardFlushing();
|
|
2289
|
+
const recorder = this.recorder;
|
|
2290
|
+
const pipeline = this.recordingShards;
|
|
2291
|
+
this.currentTurnStartMs = null;
|
|
2292
|
+
this.turnAudioStartCaptured = false;
|
|
2293
|
+
this.userSpeechStartMs = null;
|
|
2294
|
+
this.userSpeechStartSuperseded = false;
|
|
2295
|
+
if (!recorder || !pipeline) {
|
|
2296
|
+
this.recorder = null;
|
|
2297
|
+
this.recordingShards = null;
|
|
2298
|
+
return null;
|
|
2299
|
+
}
|
|
2300
|
+
try {
|
|
2301
|
+
const capturedSamples = await this.stopRecorder(recorder, pipeline.SessionID);
|
|
2302
|
+
this.recorder = null;
|
|
2303
|
+
this.recordingShards = null;
|
|
2304
|
+
const sealed = pipeline.Window.Append(recorder.TakeEncodedPackets());
|
|
2305
|
+
// Without a count from the recorder the tail keeps every packet it holds (SealTail's default).
|
|
2306
|
+
const tail = pipeline.Window.SealTail(capturedSamples === null ? undefined : Math.max(0, capturedSamples - pipeline.Window.SealedSamples));
|
|
2307
|
+
if (pipeline.Window.LastSealedIndex < 0) {
|
|
2308
|
+
pipeline.Queue.Close();
|
|
2309
|
+
return null;
|
|
2310
|
+
}
|
|
2311
|
+
if (sealed.length === 0 && !tail && pipeline.FinalTerminal?.LastSealedIndex === pipeline.Window.LastSealedIndex) {
|
|
2312
|
+
return { Pipeline: pipeline, Acked: pipeline.FinalTerminal.Acked };
|
|
2313
|
+
}
|
|
2314
|
+
return { Pipeline: pipeline, Acked: this.dispatchRecordingTerminal(pipeline, sealed, tail, false, true) };
|
|
2315
|
+
}
|
|
2316
|
+
catch (error) {
|
|
2317
|
+
console.warn(`[RealtimeSession] Failed to finish the call recording for session ${pipeline.SessionID}; stored shards are drained at the next session start or load:`, error);
|
|
2318
|
+
this.recorder = null;
|
|
2319
|
+
this.recordingShards = null;
|
|
2320
|
+
pipeline.Queue.Close();
|
|
2321
|
+
this.rearmPendingShardDrain();
|
|
2322
|
+
return null;
|
|
2323
|
+
}
|
|
2324
|
+
}
|
|
2325
|
+
/**
|
|
2326
|
+
* Stops the recorder, bounded by {@link RECORDER_STOP_TIMEOUT_MS}. Resolves the captured sample
|
|
2327
|
+
* count, or null when the recorder threw or did not finish in time (logged) — the caller then ends
|
|
2328
|
+
* the recording at the packets already encoded. Never rejects.
|
|
2329
|
+
*/
|
|
2330
|
+
async stopRecorder(recorder, sessionId) {
|
|
2331
|
+
let timer;
|
|
2332
|
+
const timedOut = new Promise((resolve) => {
|
|
2333
|
+
timer = setTimeout(() => resolve('timeout'), RECORDER_STOP_TIMEOUT_MS);
|
|
2334
|
+
});
|
|
2335
|
+
try {
|
|
2336
|
+
const outcome = await Promise.race([recorder.Stop(), timedOut]);
|
|
2337
|
+
if (outcome === 'timeout') {
|
|
2338
|
+
console.warn(`[RealtimeSession] Recorder did not stop within ${RECORDER_STOP_TIMEOUT_MS}ms for session ${sessionId}; ending the recording at the packets already encoded.`);
|
|
2339
|
+
return null;
|
|
2340
|
+
}
|
|
2341
|
+
return outcome;
|
|
2342
|
+
}
|
|
2343
|
+
catch (error) {
|
|
2344
|
+
console.warn(`[RealtimeSession] Recorder failed to stop for session ${sessionId}; ending the recording at the packets already encoded:`, error);
|
|
2345
|
+
return null;
|
|
2346
|
+
}
|
|
2347
|
+
finally {
|
|
2348
|
+
clearTimeout(timer);
|
|
2349
|
+
}
|
|
2350
|
+
}
|
|
2351
|
+
/**
|
|
2352
|
+
* Waits for the terminal {@link finishRecording} started, bounded by
|
|
2353
|
+
* {@link RECORDING_TERMINAL_ACK_TIMEOUT_MS}, logs the outcome, then closes the recording's queue
|
|
2354
|
+
* (anything it had not delivered stays in the local store for the next-load drain). Never throws.
|
|
2355
|
+
*/
|
|
2356
|
+
async awaitRecordingTerminal(pending) {
|
|
2357
|
+
const sessionId = pending.Pipeline.SessionID;
|
|
2358
|
+
let timer;
|
|
2359
|
+
const timedOut = new Promise((resolve) => {
|
|
2360
|
+
timer = setTimeout(() => resolve('timeout'), RECORDING_TERMINAL_ACK_TIMEOUT_MS);
|
|
2361
|
+
});
|
|
2362
|
+
try {
|
|
2363
|
+
const outcome = await Promise.race([pending.Acked.then((ok) => (ok ? 'ack' : 'failed')), timedOut]);
|
|
2364
|
+
if (outcome === 'ack') {
|
|
2365
|
+
console.log(`[RealtimeSession] Recording terminal acknowledged for session ${sessionId}.`);
|
|
2366
|
+
}
|
|
2367
|
+
else if (outcome === 'timeout') {
|
|
2368
|
+
console.warn(`[RealtimeSession] Recording terminal for session ${sessionId} not acknowledged within ${RECORDING_TERMINAL_ACK_TIMEOUT_MS}ms; ending anyway (the keepalive request may still land).`);
|
|
2369
|
+
}
|
|
2370
|
+
// 'failed' was already logged with its reason by sendRecordingTerminal.
|
|
2371
|
+
}
|
|
2372
|
+
finally {
|
|
2373
|
+
clearTimeout(timer);
|
|
2374
|
+
pending.Pipeline.Queue.Close();
|
|
2375
|
+
// The recording may have left shards in the store (an unacknowledged tail, windows the queue
|
|
2376
|
+
// still held or had on the wire at Close), so re-arm the drain for the next session start on
|
|
2377
|
+
// this page. Re-armed here, after the wait, so it never races the recording's own terminal;
|
|
2378
|
+
// and unconditionally, because whether a fire-and-forget upload landed is not known here and
|
|
2379
|
+
// a drain of an empty store costs one read.
|
|
2380
|
+
this.rearmPendingShardDrain();
|
|
2381
|
+
}
|
|
2382
|
+
}
|
|
2383
|
+
/**
|
|
2384
|
+
* The resolved behavior of an in-session channel (display, exposure ceiling, config, what put it
|
|
2385
|
+
* in the session), or `null` when the channel is not in this session. Hosts and overlays read the
|
|
2386
|
+
* resolved display from here rather than from the channel's code default.
|
|
2387
|
+
*
|
|
2388
|
+
* @param channelName The channel's key (case-insensitive).
|
|
2389
|
+
*/
|
|
2390
|
+
GetResolvedChannel(channelName) {
|
|
2391
|
+
return this.resolvedChannels.get(NormalizeChannelKey(channelName)) ?? null;
|
|
2392
|
+
}
|
|
2393
|
+
/**
|
|
2394
|
+
* Replaces the store the user's per-channel exposure choices are kept in. The default persists them in
|
|
2395
|
+
* the signed-in user's settings (`mj.realtime.visualPerception.v1`) and keeps them in memory for an
|
|
2396
|
+
* anonymous principal or a connect-only embed; a host that remembers them somewhere else supplies its own.
|
|
2397
|
+
*/
|
|
2398
|
+
SetExposurePreferences(preferences) {
|
|
2399
|
+
this.exposurePreferences = preferences;
|
|
2400
|
+
}
|
|
2401
|
+
/** The store the user's per-channel exposure choices are currently kept in (so a host can wrap it). */
|
|
2402
|
+
get ExposurePreferences() {
|
|
2403
|
+
return this.exposurePreferences;
|
|
2404
|
+
}
|
|
2405
|
+
/** Whether user settings can be written: a signed-in user on a provider that has entity metadata. */
|
|
2406
|
+
canPersistUserSettings() {
|
|
2407
|
+
const provider = this.Provider;
|
|
2408
|
+
return Boolean(provider?.CurrentUser?.ID) && (provider?.Entities?.length ?? 0) > 0;
|
|
2409
|
+
}
|
|
2410
|
+
/**
|
|
2411
|
+
* How much of a channel the model may perceive right now: the channel's ceiling, lowered by the server's
|
|
2412
|
+
* policy and the user's choice. `null` when the channel is not in this session.
|
|
2413
|
+
*
|
|
2414
|
+
* @param channelName The channel's key (case-insensitive).
|
|
2415
|
+
*/
|
|
2416
|
+
GetChannelExposure(channelName) {
|
|
2417
|
+
return this.findDispatchableChannel(channelName)?.Plugin.Exposure ?? null;
|
|
2418
|
+
}
|
|
2419
|
+
/**
|
|
2420
|
+
* The user's own choice of how much of a channel the agent may perceive. Takes effect immediately
|
|
2421
|
+
* (frames and notes the new level forbids stop, and the model is told), and is remembered per channel key
|
|
2422
|
+
* for next time. The level can only LOWER what the server's policy allows; choosing `'pixels'` for a channel
|
|
2423
|
+
* the agent's policy capped at `'state'` changes nothing.
|
|
2424
|
+
*
|
|
2425
|
+
* @param channelName The channel's key (case-insensitive).
|
|
2426
|
+
* @param level The user's choice; `undefined` clears it (back to what policy allows).
|
|
2427
|
+
* @returns `false` when the channel is not in this session (nothing is applied, but the choice is still remembered).
|
|
2428
|
+
*/
|
|
2429
|
+
SetUserChannelExposure(channelName, level) {
|
|
2430
|
+
this.exposurePreferences.Set(channelName, level);
|
|
2431
|
+
const channel = this.findDispatchableChannel(channelName);
|
|
2432
|
+
if (!channel) {
|
|
2433
|
+
return false;
|
|
2434
|
+
}
|
|
2435
|
+
this.applyChannelExposure(channel.Plugin);
|
|
2436
|
+
return true;
|
|
2437
|
+
}
|
|
2438
|
+
/** Pushes a channel's current exposure inputs (server policy, user choice) into the channel. */
|
|
2439
|
+
applyChannelExposure(plugin) {
|
|
2440
|
+
const resolved = this.GetResolvedChannel(plugin.ChannelName);
|
|
2441
|
+
const user = this.exposurePreferences.Get(plugin.ChannelName);
|
|
2442
|
+
const reasons = (resolved?.ExposureLimits ?? []).map((limit) => limit.Reason);
|
|
2443
|
+
// The user's limit binds only when it is lower than what the server allows; otherwise it changes nothing
|
|
2444
|
+
// and the agent should not be told about it.
|
|
2445
|
+
const allowed = resolved?.Exposure ?? plugin.GetDescriptor().MaxExposure;
|
|
2446
|
+
if (user !== undefined && CompareExposure(user, allowed) < 0) {
|
|
2447
|
+
reasons.push(UserExposureReason(user));
|
|
2448
|
+
}
|
|
2449
|
+
plugin.ApplyExposure({ Policy: resolved?.Exposure, User: user, Reasons: reasons });
|
|
2450
|
+
// A capture this channel fronts follows it: the agent sees its frames only while the channel's exposure allows pixels.
|
|
2451
|
+
const kind = plugin.CaptureKind;
|
|
2452
|
+
if (kind) {
|
|
2453
|
+
this.captures?.SetVisibleToAgent(kind, plugin.Exposure === 'pixels');
|
|
2454
|
+
}
|
|
2455
|
+
this.syncChannelVideoSource(plugin);
|
|
2456
|
+
}
|
|
2457
|
+
/**
|
|
2458
|
+
* Sends a frame from a channel's context ({@link RealtimeChannelContext.SendVideoFrame}, deprecated) through the
|
|
2459
|
+
* session's {@link VideoSourceArbiter}, as the channel's own source: it reaches the model only while the arbiter picks
|
|
2460
|
+
* that source, and the arbiter tells the model when it switches. The source registers with the first frame, under the
|
|
2461
|
+
* id the channel's own frame bridge uses, switched on only while the channel's exposure allows pixels.
|
|
2462
|
+
*/
|
|
2463
|
+
sendChannelVideoFrame(plugin, base64Image, mimeType) {
|
|
2464
|
+
const client = this.client;
|
|
2465
|
+
if (!client || !this.isSessionLive() || !client.IsTrackEstablished('video', 'inbound')) {
|
|
2466
|
+
return;
|
|
2467
|
+
}
|
|
2468
|
+
const arbiter = VideoSourceArbiter.ForSink(client);
|
|
2469
|
+
const descriptor = plugin.GetDescriptor();
|
|
2470
|
+
const sourceId = ChannelVideoSourceID(descriptor.Key, plugin.InstanceId);
|
|
2471
|
+
if (!arbiter.HasSource(sourceId)) {
|
|
2472
|
+
arbiter.RegisterSource({
|
|
2473
|
+
SourceID: sourceId,
|
|
2474
|
+
Label: descriptor.DisplayName,
|
|
2475
|
+
Kind: 'surface',
|
|
2476
|
+
ChannelKey: descriptor.Key,
|
|
2477
|
+
Enabled: plugin.Exposure === 'pixels',
|
|
2478
|
+
});
|
|
2479
|
+
}
|
|
2480
|
+
arbiter.PushFrame(sourceId, base64Image, mimeType);
|
|
2481
|
+
}
|
|
2482
|
+
/**
|
|
2483
|
+
* Keeps a channel's video source at the arbiter, when it has one, in step with the channel's exposure: on while the
|
|
2484
|
+
* agent may see its pixels, off otherwise. Quiet, because the channel tells the model about its own exposure change.
|
|
2485
|
+
*/
|
|
2486
|
+
syncChannelVideoSource(plugin) {
|
|
2487
|
+
if (!this.client) {
|
|
2488
|
+
return;
|
|
2489
|
+
}
|
|
2490
|
+
const arbiter = VideoSourceArbiter.ForSink(this.client);
|
|
2491
|
+
const sourceId = ChannelVideoSourceID(plugin.GetDescriptor().Key, plugin.InstanceId);
|
|
2492
|
+
if (arbiter.HasSource(sourceId)) {
|
|
2493
|
+
arbiter.SetSourceEnabled(sourceId, plugin.Exposure === 'pixels', false);
|
|
2494
|
+
}
|
|
2495
|
+
}
|
|
2496
|
+
/** Follows the session client's video-source arbiter so {@link VideoSources$} reflects it. */
|
|
2497
|
+
watchVideoSources(client) {
|
|
2498
|
+
this.unwatchVideoSources();
|
|
2499
|
+
const arbiter = VideoSourceArbiter.ForSink(client);
|
|
2500
|
+
arbiter.SetFocusedChannel(this.focusedChannelKey);
|
|
2501
|
+
this.stopWatchingVideoSources = arbiter.OnChange(() => this._videoSources$.next(arbiter.GetSources()));
|
|
2502
|
+
this._videoSources$.next(arbiter.GetSources());
|
|
2503
|
+
}
|
|
2504
|
+
/**
|
|
2505
|
+
* Tells the session which channel's surface the user is looking at (`null` for none, e.g. the activity tab).
|
|
2506
|
+
* When the model can see only one video source and several are live, the one the user is looking at is the one
|
|
2507
|
+
* it sees (after an explicit pick, and after a camera or screen share the user started). Safe to call before the
|
|
2508
|
+
* session is live: it applies when the connection comes up.
|
|
2509
|
+
*
|
|
2510
|
+
* @param channelKey The focused channel's key, as it appears on {@link VideoSources$} entries' `ChannelKey`.
|
|
2511
|
+
*/
|
|
2512
|
+
SetFocusedChannel(channelKey) {
|
|
2513
|
+
this.focusedChannelKey = channelKey;
|
|
2514
|
+
if (this.client) {
|
|
2515
|
+
VideoSourceArbiter.ForSink(this.client).SetFocusedChannel(channelKey);
|
|
2516
|
+
}
|
|
2517
|
+
}
|
|
2518
|
+
/** Stops following the arbiter and clears {@link VideoSources$}. Safe to call when nothing is watched. */
|
|
2519
|
+
unwatchVideoSources() {
|
|
2520
|
+
this.stopWatchingVideoSources?.();
|
|
2521
|
+
this.stopWatchingVideoSources = null;
|
|
2522
|
+
if (this._videoSources$.value.length > 0) {
|
|
2523
|
+
this._videoSources$.next([]);
|
|
2524
|
+
}
|
|
2525
|
+
}
|
|
2526
|
+
/**
|
|
2527
|
+
* Turns the agent's view of one video source on or off — what the "agent can see" control calls.
|
|
2528
|
+
*
|
|
2529
|
+
* A source that belongs to a channel goes through {@link SetUserChannelExposure}, so the choice is
|
|
2530
|
+
* remembered per channel and the channel itself tells the model. Any other source (a camera or screen
|
|
2531
|
+
* share that is not a channel) is switched at the arbiter, which tells the model.
|
|
2532
|
+
*
|
|
2533
|
+
* @param sourceId The source's id (from {@link VideoSources$}).
|
|
2534
|
+
* @param enabled Whether the agent may see it.
|
|
2535
|
+
* @returns `false` when there is no such source.
|
|
2536
|
+
*/
|
|
2537
|
+
SetVideoSourceEnabled(sourceId, enabled) {
|
|
2538
|
+
const client = this.client;
|
|
2539
|
+
if (!client) {
|
|
2540
|
+
return false;
|
|
2541
|
+
}
|
|
2542
|
+
const arbiter = VideoSourceArbiter.ForSink(client);
|
|
2543
|
+
const source = arbiter.GetSources().find((s) => s.SourceID === sourceId);
|
|
2544
|
+
if (!source) {
|
|
2545
|
+
return false;
|
|
2546
|
+
}
|
|
2547
|
+
if (source.ChannelKey && this.findDispatchableChannel(source.ChannelKey)) {
|
|
2548
|
+
return this.SetUserChannelExposure(source.ChannelKey, enabled ? undefined : 'state');
|
|
2549
|
+
}
|
|
2550
|
+
return arbiter.SetSourceEnabled(sourceId, enabled);
|
|
2551
|
+
}
|
|
2552
|
+
/**
|
|
2553
|
+
* Picks the video source the agent sees: what the "agent can see" control calls when more sources are on than the
|
|
2554
|
+
* model takes. The pick beats every other rule (a camera or screen share the user started, the surface the user is
|
|
2555
|
+
* looking at) until it is cleared, or its source is turned off or leaves. The pick shows on {@link VideoSources$}
|
|
2556
|
+
* (`Picked`), and the model is told when what it sees changes.
|
|
2557
|
+
*
|
|
2558
|
+
* @param sourceId The source's id (from {@link VideoSources$}), or `null` to let the call decide again.
|
|
2559
|
+
* @returns `false` with no live session, or for a source that is not there or is turned off.
|
|
2560
|
+
*/
|
|
2561
|
+
SelectVideoSource(sourceId) {
|
|
2562
|
+
return this.client ? VideoSourceArbiter.ForSink(this.client).SelectSource(sourceId) : false;
|
|
2563
|
+
}
|
|
2564
|
+
/**
|
|
2565
|
+
* The `on-demand` channels that are in the session but not open yet — what the agent can open
|
|
2566
|
+
* through `ContextTool`. A fresh array; empty before a session starts and after teardown.
|
|
2567
|
+
*/
|
|
2568
|
+
get AdvertisedChannels() {
|
|
2569
|
+
return this.advertisedChannels.map((p) => p.Plugin);
|
|
968
2570
|
}
|
|
969
|
-
/**
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
2571
|
+
/**
|
|
2572
|
+
* Resolves, initializes and mounts the session's channels in one step using the scope the browser
|
|
2573
|
+
* can decide alone (code defaults + host declarations), and returns the native tools to declare at
|
|
2574
|
+
* mint. This is the single-step composition of {@link prepareChannelScope} + {@link activateChannels}
|
|
2575
|
+
* that a session start spreads across the mint; it is kept as a unit because it is the cleanest seam
|
|
2576
|
+
* for exercising the plugin plumbing without a mint.
|
|
2577
|
+
*/
|
|
2578
|
+
async startChannels(hostChannels) {
|
|
2579
|
+
const scope = await this.prepareChannelScope(hostChannels);
|
|
2580
|
+
this.activateChannels(this.localChannelScope?.Channels ?? []);
|
|
2581
|
+
return scope.NativeTools;
|
|
975
2582
|
}
|
|
976
2583
|
/**
|
|
977
|
-
*
|
|
978
|
-
*
|
|
979
|
-
*
|
|
980
|
-
* goes browser → provider over WebRTC. The server sees the mint, a few channel actions in the
|
|
981
|
-
* first seconds, and then nothing at all — so `SessionManager.RecordActivity` stops being
|
|
982
|
-
* reached while the conversation is still going. `LastActiveAt` freezes ~45 seconds in, and
|
|
983
|
-
* `SessionJanitor` — which cannot distinguish an active call from an abandoned one — force-closes
|
|
984
|
-
* it at `closeThresholdMinutes`, mid-sentence, taking the user's surfaces with it. A session
|
|
985
|
-
* whose channels are all client-side (whiteboard, media) goes quiet from the server's point of
|
|
986
|
-
* view almost immediately.
|
|
2584
|
+
* Stage 1: constructs the channel plugins (registry rows and host declarations), reads their
|
|
2585
|
+
* descriptors, resolves the scope the browser can decide alone, and builds what the mint needs —
|
|
2586
|
+
* the native tools to declare and the candidates to report. Starts nothing.
|
|
987
2587
|
*
|
|
988
|
-
*
|
|
989
|
-
*
|
|
990
|
-
*
|
|
991
|
-
*
|
|
992
|
-
* The pulse is best-effort by design: a failed beat is logged and skipped, never surfaced to the
|
|
993
|
-
* user and never allowed to end the session. Losing one beat costs nothing because the threshold
|
|
994
|
-
* is many beats wide; turning a transient network blip into a visible error would be a worse
|
|
995
|
-
* failure than the one this fixes. Write amplification is bounded on the server side too, where
|
|
996
|
-
* `SessionManager.Heartbeat` coalesces persisted writes.
|
|
2588
|
+
* @param hostChannels Channels the host brings to this session.
|
|
2589
|
+
* @returns The native tools to declare at mint, the candidates as JSON (`null` when there are none,
|
|
2590
|
+
* so a channel-less session sends exactly the mint it always did), and whether the host could show the agent's
|
|
2591
|
+
* video ({@link PreparedChannelsShowAgentVideo}).
|
|
997
2592
|
*/
|
|
998
|
-
|
|
999
|
-
this.
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
}
|
|
2593
|
+
async prepareChannelScope(hostChannels) {
|
|
2594
|
+
this.discardUnmountedChannels();
|
|
2595
|
+
const prepared = await this.prepareChannels(hostChannels);
|
|
2596
|
+
this.preparedChannels = prepared;
|
|
2597
|
+
const candidates = prepared.map((p) => BuildChannelCandidate(p, p.Plugin.GetDescriptor()));
|
|
2598
|
+
const local = ResolveLocalChannelScope(candidates);
|
|
2599
|
+
this.localChannelScope = local;
|
|
2600
|
+
for (const excluded of local.Excluded) {
|
|
2601
|
+
console.warn(`[RealtimeSession] Channel '${excluded.Key}' is not in this session (${excluded.Reason}).`);
|
|
2602
|
+
}
|
|
2603
|
+
const nativeTools = SelectNativeChannelTools(local.Channels, ToolsByChannelKey(candidates));
|
|
2604
|
+
// `Registry` is the browser's own view and never goes over the wire: the server reads the registry itself.
|
|
2605
|
+
const wire = candidates.map(({ Registry: _registry, ...candidate }) => candidate);
|
|
2606
|
+
return { NativeTools: nativeTools, CandidatesJson: wire.length > 0 ? JSON.stringify(wire) : null, ShowsAgentVideo: PreparedChannelsShowAgentVideo(prepared) };
|
|
1008
2607
|
}
|
|
1009
2608
|
/**
|
|
1010
|
-
*
|
|
1011
|
-
*
|
|
2609
|
+
* Constructs a plugin for every active-or-inactive registry row and every host declaration. An
|
|
2610
|
+
* INACTIVE row is constructed too — only to learn its key so the kill switch can name it — and is
|
|
2611
|
+
* never initialized. A plugin whose descriptor cannot be read is skipped (logged), never fatal.
|
|
1012
2612
|
*/
|
|
1013
|
-
async
|
|
1014
|
-
const
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
try {
|
|
1024
|
-
await this.gql().ExecuteGQL(mutation, { agentSessionId });
|
|
2613
|
+
async prepareChannels(hostChannels) {
|
|
2614
|
+
const rows = await this.fetchChannelDefinitions();
|
|
2615
|
+
await this.loadChannelClasses([...rows.map((r) => r.ClientPluginClass), ...(hostChannels ?? []).map((d) => d.ClientPluginClass)]);
|
|
2616
|
+
const prepared = [];
|
|
2617
|
+
for (const row of rows) {
|
|
2618
|
+
const plugin = this.resolveChannelPlugin(row);
|
|
2619
|
+
if (plugin) {
|
|
2620
|
+
plugin.ApplySurfacePlacement(row.SurfacePlacement);
|
|
2621
|
+
prepared.push({ Plugin: plugin, Key: plugin.ChannelName, Registry: row.IsActive ? 'active' : 'inactive' });
|
|
2622
|
+
}
|
|
1025
2623
|
}
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
2624
|
+
for (const declaration of hostChannels ?? []) {
|
|
2625
|
+
const plugin = this.createHostChannelPlugin(declaration);
|
|
2626
|
+
if (!plugin) {
|
|
2627
|
+
continue;
|
|
2628
|
+
}
|
|
2629
|
+
const existing = FindPreparedChannel(prepared, plugin.ChannelName);
|
|
2630
|
+
if (existing) {
|
|
2631
|
+
// The host's instance replaces the registry's (it may carry the host's collaborators); the
|
|
2632
|
+
// registry row's state — notably the kill switch, and where its surface shows — still applies.
|
|
2633
|
+
plugin.ApplySurfacePlacement(existing.Plugin.SurfacePlacement);
|
|
2634
|
+
existing.Plugin = plugin;
|
|
2635
|
+
existing.HostDeclaration = declaration;
|
|
2636
|
+
}
|
|
2637
|
+
else {
|
|
2638
|
+
prepared.push({ Plugin: plugin, Key: plugin.ChannelName, Registry: 'none', HostDeclaration: declaration });
|
|
2639
|
+
}
|
|
1029
2640
|
}
|
|
2641
|
+
return prepared.filter((p) => this.canDescribe(p));
|
|
1030
2642
|
}
|
|
1031
2643
|
/**
|
|
1032
|
-
*
|
|
1033
|
-
* the
|
|
2644
|
+
* Gives a host that loads channel code on demand (a widget that keeps a heavy channel in its own download)
|
|
2645
|
+
* the chance to load it before the plugins are built. Never fatal: a loader that throws is logged and the
|
|
2646
|
+
* channel simply is not available, exactly as if its class had never been registered.
|
|
2647
|
+
*
|
|
2648
|
+
* @param keys The `ClientPluginClass` keys of every channel about to be resolved (blank/absent ones included).
|
|
1034
2649
|
*/
|
|
1035
|
-
async
|
|
1036
|
-
const
|
|
1037
|
-
|
|
1038
|
-
if (!recorder || !agentSessionId) {
|
|
2650
|
+
async loadChannelClasses(keys) {
|
|
2651
|
+
const loader = this.ChannelClassLoader;
|
|
2652
|
+
if (!loader) {
|
|
1039
2653
|
return;
|
|
1040
2654
|
}
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
return;
|
|
1045
|
-
}
|
|
1046
|
-
const index = this.segmentIndex++;
|
|
1047
|
-
const mutation = `
|
|
1048
|
-
mutation UploadRealtimeRecordingSegment($agentSessionId: String!, $segmentIndex: Int!, $audioBase64: String!, $mimeType: String!) {
|
|
1049
|
-
UploadRealtimeRecordingSegment(agentSessionId: $agentSessionId, segmentIndex: $segmentIndex, audioBase64: $audioBase64, mimeType: $mimeType)
|
|
2655
|
+
const wanted = [...new Set(keys.map((k) => k?.trim() ?? '').filter((k) => k.length > 0))];
|
|
2656
|
+
if (wanted.length === 0) {
|
|
2657
|
+
return;
|
|
1050
2658
|
}
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
// NOT individually-playable WAV — recovery is concatenate-in-order then WAV-wrap. The canonical
|
|
1054
|
-
// seekable WAV is the consolidated end-of-call upload below.
|
|
1055
|
-
const shardMime = `audio/L16;rate=${recorder.SampleRate}`;
|
|
1056
|
-
await this.gql().ExecuteGQL(mutation, { agentSessionId, segmentIndex: index, audioBase64, mimeType: shardMime });
|
|
2659
|
+
try {
|
|
2660
|
+
await loader(wanted);
|
|
1057
2661
|
}
|
|
1058
2662
|
catch (error) {
|
|
1059
|
-
console.
|
|
2663
|
+
console.error('[RealtimeSession] The channel class loader failed — channels that needed it are left out:', error);
|
|
1060
2664
|
}
|
|
1061
2665
|
}
|
|
1062
|
-
/**
|
|
1063
|
-
|
|
1064
|
-
* Fully best-effort and wrapped in try/catch — recording upload must NEVER block teardown.
|
|
1065
|
-
* No-op when nothing was recorded or there's no session id to attach the file to.
|
|
1066
|
-
*/
|
|
1067
|
-
async stopAndUploadRecording(agentSessionId) {
|
|
1068
|
-
this.stopSegmentFlushing();
|
|
1069
|
-
const recorder = this.recorder;
|
|
1070
|
-
this.recorder = null;
|
|
1071
|
-
this.currentTurnStartMs = null;
|
|
1072
|
-
this.turnAudioStartCaptured = false;
|
|
1073
|
-
if (!recorder) {
|
|
1074
|
-
return;
|
|
1075
|
-
}
|
|
2666
|
+
/** Whether a prepared plugin's descriptor can be read; logs and rejects one that throws. */
|
|
2667
|
+
canDescribe(prepared) {
|
|
1076
2668
|
try {
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
const audioBase64 = await recorder.StopAndEncode();
|
|
1080
|
-
// Read the real waveform peaks computed during capture (survives the stop via the snapshot).
|
|
1081
|
-
const peaks = recorder.GetPeaks();
|
|
1082
|
-
if (!audioBase64 || !agentSessionId) {
|
|
1083
|
-
console.warn('[RealtimeSession] ⚠️ recording NOT uploaded — empty recording or no session id.');
|
|
1084
|
-
return;
|
|
1085
|
-
}
|
|
1086
|
-
await this.uploadRecording(agentSessionId, audioBase64, mimeType, peaks);
|
|
2669
|
+
prepared.Plugin.GetDescriptor();
|
|
2670
|
+
return true;
|
|
1087
2671
|
}
|
|
1088
2672
|
catch (error) {
|
|
1089
|
-
console.
|
|
2673
|
+
console.error(`[RealtimeSession] Channel '${prepared.Key}' has an unreadable descriptor — leaving it out:`, error);
|
|
2674
|
+
return false;
|
|
1090
2675
|
}
|
|
1091
2676
|
}
|
|
1092
|
-
/**
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
FileID
|
|
1103
|
-
ErrorMessage
|
|
2677
|
+
/** Builds a plugin from a host declaration; `null` (logged) when it names nothing buildable. */
|
|
2678
|
+
createHostChannelPlugin(declaration) {
|
|
2679
|
+
if (declaration.Create) {
|
|
2680
|
+
try {
|
|
2681
|
+
return declaration.Create();
|
|
2682
|
+
}
|
|
2683
|
+
catch (error) {
|
|
2684
|
+
console.error('[RealtimeSession] A host-declared channel factory threw — leaving the channel out:', error);
|
|
2685
|
+
return null;
|
|
2686
|
+
}
|
|
1104
2687
|
}
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
audioBase64,
|
|
1110
|
-
mimeType,
|
|
1111
|
-
consent: true,
|
|
1112
|
-
peaks
|
|
1113
|
-
});
|
|
1114
|
-
const payload = result?.UploadRealtimeRecording;
|
|
1115
|
-
if (!payload?.Success) {
|
|
1116
|
-
console.warn(`[RealtimeSession] ❌ recording upload reported failure: ${payload?.ErrorMessage ?? 'unknown error'} (full result: ${JSON.stringify(result)})`);
|
|
2688
|
+
const key = declaration.ClientPluginClass?.trim();
|
|
2689
|
+
if (!key) {
|
|
2690
|
+
console.warn('[RealtimeSession] A host-declared channel names neither ClientPluginClass nor Create — ignoring it.');
|
|
2691
|
+
return null;
|
|
1117
2692
|
}
|
|
2693
|
+
return this.resolveChannelPlugin({ Name: key, ClientPluginClass: key, IsActive: true });
|
|
1118
2694
|
}
|
|
1119
|
-
// ── Interactive channels (registry-driven plugins) ─────────────────────────
|
|
1120
2695
|
/**
|
|
1121
|
-
*
|
|
1122
|
-
*
|
|
1123
|
-
*
|
|
1124
|
-
* Tolerant by design: registry/resolution failures degrade to "no channels" — the voice
|
|
1125
|
-
* session itself always proceeds.
|
|
2696
|
+
* Stage 3: initializes the channels the resolved policy puts in the session. `open-on-start` and
|
|
2697
|
+
* `headless` channels are mounted and published on {@link ActiveChannels$}; `on-demand` channels
|
|
2698
|
+
* are held until opened. Prepared plugins the policy left out are dropped without ever having run.
|
|
1126
2699
|
*/
|
|
1127
|
-
|
|
1128
|
-
const
|
|
1129
|
-
for (const
|
|
1130
|
-
|
|
2700
|
+
activateChannels(channels) {
|
|
2701
|
+
const reconciled = ReconcileChannelsWithPolicy(this.preparedChannels, channels);
|
|
2702
|
+
for (const key of reconciled.Unknown) {
|
|
2703
|
+
console.warn(`[RealtimeSession] The session policy names channel '${key}' but this host has no plugin for it — ignoring it.`);
|
|
2704
|
+
}
|
|
2705
|
+
this.resolvedChannels.clear();
|
|
2706
|
+
this.advertisedChannels = [];
|
|
2707
|
+
const mounted = [];
|
|
2708
|
+
for (const { Prepared, Resolved } of reconciled.InSession) {
|
|
2709
|
+
this.resolvedChannels.set(NormalizeChannelKey(Resolved.Key), Resolved);
|
|
2710
|
+
// Exposure is applied BEFORE the channel initializes, so what it requests of the model (a video
|
|
2711
|
+
// track, in particular) already reflects what policy allows.
|
|
2712
|
+
this.applyChannelExposure(Prepared.Plugin);
|
|
2713
|
+
if (Resolved.DisplayPolicy === 'on-demand') {
|
|
2714
|
+
this.advertisedChannels.push(Prepared);
|
|
2715
|
+
}
|
|
2716
|
+
else {
|
|
2717
|
+
this.mountChannel(Prepared, Resolved);
|
|
2718
|
+
mounted.push(Prepared.Plugin);
|
|
2719
|
+
}
|
|
1131
2720
|
}
|
|
1132
|
-
this.
|
|
1133
|
-
|
|
2721
|
+
this.preparedChannels = [];
|
|
2722
|
+
this._activeChannels$.next(mounted);
|
|
1134
2723
|
}
|
|
1135
2724
|
/**
|
|
1136
|
-
*
|
|
1137
|
-
*
|
|
1138
|
-
*
|
|
1139
|
-
* / `BaseRealtimeClient`. Rows whose plugin class isn't registered are skipped (logged),
|
|
1140
|
-
* never fatal.
|
|
2725
|
+
* Applies the mint's outcome to the channel set: the server's policy when it sent a usable one,
|
|
2726
|
+
* else the scope the browser resolved on its own. A policy can only SELECT among the plugins this
|
|
2727
|
+
* host prepared — it cannot add one.
|
|
1141
2728
|
*/
|
|
1142
|
-
|
|
1143
|
-
const
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
const plugin = this.resolveChannelPlugin(row);
|
|
1147
|
-
if (plugin) {
|
|
1148
|
-
channels.push(plugin);
|
|
1149
|
-
}
|
|
2729
|
+
applySessionClientPolicy(session) {
|
|
2730
|
+
const policy = ParseRealtimeSessionClientPolicy(session.ClientPolicyJson);
|
|
2731
|
+
if (session.ClientPolicyJson && !policy) {
|
|
2732
|
+
console.warn('[RealtimeSession] The mint returned a client policy this runtime cannot read — resolving channels locally.');
|
|
1150
2733
|
}
|
|
1151
|
-
|
|
2734
|
+
this.sessionClientTools = policy?.ClientTools ?? null;
|
|
2735
|
+
if (this.preparedChannels.length === 0) {
|
|
2736
|
+
this.resolvedChannels.clear();
|
|
2737
|
+
return; // nothing was prepared (a host that mints and runs the session itself) — no channel set to apply
|
|
2738
|
+
}
|
|
2739
|
+
this.activateChannels(policy?.Channels ?? this.localChannelScope?.Channels ?? []);
|
|
2740
|
+
}
|
|
2741
|
+
/** Drops plugins that were prepared for a start that never mounted them. They were never initialized, so there is nothing to release. */
|
|
2742
|
+
discardUnmountedChannels() {
|
|
2743
|
+
this.preparedChannels = [];
|
|
2744
|
+
this.advertisedChannels = [];
|
|
2745
|
+
this.localChannelScope = null;
|
|
1152
2746
|
}
|
|
1153
2747
|
/**
|
|
1154
|
-
*
|
|
1155
|
-
*
|
|
1156
|
-
*
|
|
1157
|
-
*
|
|
1158
|
-
*
|
|
2748
|
+
* Loads the `MJ: AI Agent Channels` registry rows (ACTIVE AND INACTIVE — an inactive row is the master
|
|
2749
|
+
* kill switch, so the scope decision has to see it). With entity metadata: from {@link AIEngineBase}'s
|
|
2750
|
+
* cached `AgentChannels` (provider-scoped engine instance, lazy `Config` — no RunView round-trip; the
|
|
2751
|
+
* engine's BaseEntity-event reactivity keeps the registry fresh). On a connect-only provider: one
|
|
2752
|
+
* `RunDynamicView` query ({@link fetchChannelDefinitionsOverGraphQL}). Failures are logged and degrade to an
|
|
2753
|
+
* empty list — channel availability must never block the voice session.
|
|
1159
2754
|
*/
|
|
1160
2755
|
async fetchChannelDefinitions() {
|
|
1161
2756
|
// A connect-only provider (ConnectGraphQLClient — anonymous embeds) has no entity metadata,
|
|
@@ -1168,9 +2763,13 @@ export class RealtimeSessionRuntime {
|
|
|
1168
2763
|
try {
|
|
1169
2764
|
const engine = AIEngineBase.GetProviderInstance(this.Provider, AIEngineBase);
|
|
1170
2765
|
await engine.Config(false, undefined, this.Provider);
|
|
1171
|
-
return (engine.AgentChannels ?? [])
|
|
1172
|
-
|
|
1173
|
-
|
|
2766
|
+
return (engine.AgentChannels ?? []).map((c) => ({
|
|
2767
|
+
ID: c.ID,
|
|
2768
|
+
Name: c.Name,
|
|
2769
|
+
ClientPluginClass: c.ClientPluginClass,
|
|
2770
|
+
IsActive: c.IsActive,
|
|
2771
|
+
SurfacePlacement: readRowSurfacePlacement(c),
|
|
2772
|
+
}));
|
|
1174
2773
|
}
|
|
1175
2774
|
catch (error) {
|
|
1176
2775
|
console.warn('[RealtimeSession] Channel registry unavailable — starting with no channels:', error);
|
|
@@ -1178,10 +2777,10 @@ export class RealtimeSessionRuntime {
|
|
|
1178
2777
|
}
|
|
1179
2778
|
}
|
|
1180
2779
|
/**
|
|
1181
|
-
* The connect-only path of {@link fetchChannelDefinitions}: the same
|
|
1182
|
-
*
|
|
1183
|
-
* typed RunView from. Same tolerance as the engine path — a failure is
|
|
1184
|
-
* channels", never a blocked session.
|
|
2780
|
+
* The connect-only path of {@link fetchChannelDefinitions}: the same `MJ: AI Agent Channels` rows, active
|
|
2781
|
+
* and inactive (an inactive row is the kill switch), read with a dynamic view because a connect-only client
|
|
2782
|
+
* has no entity metadata to build a typed RunView from. Same tolerance as the engine path — a failure is
|
|
2783
|
+
* logged and means "no channels", never a blocked session.
|
|
1185
2784
|
*/
|
|
1186
2785
|
async fetchChannelDefinitionsOverGraphQL() {
|
|
1187
2786
|
const query = `query RealtimeChannelRegistry($input: RunDynamicViewInput!) {
|
|
@@ -1189,7 +2788,7 @@ export class RealtimeSessionRuntime {
|
|
|
1189
2788
|
}`;
|
|
1190
2789
|
try {
|
|
1191
2790
|
const result = (await this.gql().ExecuteGQL(query, {
|
|
1192
|
-
input: { EntityName: 'MJ: AI Agent Channels',
|
|
2791
|
+
input: { EntityName: 'MJ: AI Agent Channels', Fields: ['ID', 'Name', 'ClientPluginClass', 'IsActive', 'UIConfig'] },
|
|
1193
2792
|
}));
|
|
1194
2793
|
const view = result?.RunDynamicView;
|
|
1195
2794
|
if (!view?.Success) {
|
|
@@ -1198,8 +2797,13 @@ export class RealtimeSessionRuntime {
|
|
|
1198
2797
|
}
|
|
1199
2798
|
return (view.Results ?? [])
|
|
1200
2799
|
.map((r) => JSON.parse(r.Data))
|
|
1201
|
-
.
|
|
1202
|
-
|
|
2800
|
+
.map((row) => ({
|
|
2801
|
+
ID: row.ID,
|
|
2802
|
+
Name: row.Name,
|
|
2803
|
+
ClientPluginClass: row.ClientPluginClass,
|
|
2804
|
+
IsActive: row.IsActive === true,
|
|
2805
|
+
SurfacePlacement: readUIConfigTextSurfacePlacement(row.Name, row.UIConfig),
|
|
2806
|
+
}));
|
|
1203
2807
|
}
|
|
1204
2808
|
catch (error) {
|
|
1205
2809
|
console.warn('[RealtimeSession] Channel registry unavailable — starting with no channels:', error instanceof Error ? error.message : String(error));
|
|
@@ -1211,16 +2815,23 @@ export class RealtimeSessionRuntime {
|
|
|
1211
2815
|
* checked first, exactly like the realtime-client drivers) and instantiates a fresh
|
|
1212
2816
|
* per-session plugin. Returns `null` (logged) when no plugin is registered for the key
|
|
1213
2817
|
* — e.g. its Load function was never called or the package isn't included client-side.
|
|
2818
|
+
* An inactive row is resolved quietly: it only needs to be named, and a missing plugin for a
|
|
2819
|
+
* channel nobody wants is not worth a warning.
|
|
1214
2820
|
*/
|
|
1215
2821
|
resolveChannelPlugin(row) {
|
|
1216
2822
|
const key = row.ClientPluginClass?.trim();
|
|
2823
|
+
const quiet = !row.IsActive;
|
|
1217
2824
|
if (!key) {
|
|
1218
|
-
|
|
2825
|
+
if (!quiet) {
|
|
2826
|
+
console.warn(`[RealtimeSession] Channel '${row.Name}' has no ClientPluginClass — skipping.`);
|
|
2827
|
+
}
|
|
1219
2828
|
return null;
|
|
1220
2829
|
}
|
|
1221
2830
|
const registration = MJGlobal.Instance.ClassFactory.GetRegistration(BaseRealtimeChannelClient, key);
|
|
1222
2831
|
if (!registration) {
|
|
1223
|
-
|
|
2832
|
+
if (!quiet) {
|
|
2833
|
+
console.warn(`[RealtimeSession] No client plugin registered for channel '${row.Name}' (key '${key}') — skipping.`);
|
|
2834
|
+
}
|
|
1224
2835
|
return null;
|
|
1225
2836
|
}
|
|
1226
2837
|
const plugin = MJGlobal.Instance.ClassFactory.CreateInstance(BaseRealtimeChannelClient, key);
|
|
@@ -1231,24 +2842,157 @@ export class RealtimeSessionRuntime {
|
|
|
1231
2842
|
return plugin;
|
|
1232
2843
|
}
|
|
1233
2844
|
/**
|
|
1234
|
-
* Wires one plugin into the session: hands it its host context
|
|
1235
|
-
* prefix-routed local tool executor (so `<ToolNamePrefix>*`
|
|
1236
|
-
* through {@link BaseRealtimeChannelClient.ApplyAgentTool}, never the
|
|
2845
|
+
* Wires one plugin into the session: hands it its host context (carrying its resolved config) and,
|
|
2846
|
+
* when it has native tools, registers its prefix-routed local tool executor (so `<ToolNamePrefix>*`
|
|
2847
|
+
* calls run in the browser through {@link BaseRealtimeChannelClient.ApplyAgentTool}, never the
|
|
2848
|
+
* server relay). A channel with no tool prefix has no native route — it is reached through
|
|
2849
|
+
* `ContextTool` — and registering an empty prefix would match EVERY tool name.
|
|
1237
2850
|
*/
|
|
1238
|
-
|
|
1239
|
-
plugin.
|
|
2851
|
+
mountChannel(prepared, resolved) {
|
|
2852
|
+
const plugin = prepared.Plugin;
|
|
2853
|
+
plugin.Initialize(this.buildChannelContext(plugin, resolved?.Config));
|
|
2854
|
+
if (plugin.ToolNamePrefix.length === 0) {
|
|
2855
|
+
return;
|
|
2856
|
+
}
|
|
1240
2857
|
this.RegisterClientToolHandler(plugin.ToolNamePrefix, (toolName, argsJson) => {
|
|
1241
2858
|
// The agent is ACTING on this channel — surface-discovery signal for the overlay
|
|
1242
2859
|
// (first activity registers + auto-reveals + focuses the channel tab) before the
|
|
1243
2860
|
// tool applies. Record the channel as USED so the overlay tabs it (channels other
|
|
1244
2861
|
// than the whiteboard are tab-less until they're first used).
|
|
1245
|
-
this.
|
|
1246
|
-
|
|
2862
|
+
this.noteChannelActivity(plugin);
|
|
2863
|
+
// A native tool whose result would show the agent what exposure policy withholds is refused here, as it is on the
|
|
2864
|
+
// ContextTool route; the legacy tool-result shape carries the reason.
|
|
2865
|
+
const withheld = plugin.FindVerbForNativeTool(toolName);
|
|
2866
|
+
const refusal = withheld ? plugin.RefuseVerbForExposure(withheld) : null;
|
|
2867
|
+
if (refusal) {
|
|
2868
|
+
return JSON.stringify({ success: false, errorCode: 'exposure_restricted', error: refusal });
|
|
2869
|
+
}
|
|
1247
2870
|
return plugin.ApplyAgentTool(toolName, argsJson);
|
|
1248
2871
|
});
|
|
1249
2872
|
}
|
|
2873
|
+
/** Records that the agent acted on a channel (the overlay's first-use reveal signal). */
|
|
2874
|
+
noteChannelActivity(plugin) {
|
|
2875
|
+
this.usedChannelNames.add(plugin.ChannelName);
|
|
2876
|
+
this._channelActivity$.next(plugin);
|
|
2877
|
+
}
|
|
2878
|
+
/**
|
|
2879
|
+
* Mounts an advertised `on-demand` channel when the agent opens it: initializes it, restores its
|
|
2880
|
+
* prior-session state, publishes it on {@link ActiveChannels$} (so the overlay can give it a tab)
|
|
2881
|
+
* and, if the call is already connected, tells it so. If initialization throws the channel goes
|
|
2882
|
+
* back to being advertised, so the agent can retry or move on.
|
|
2883
|
+
*/
|
|
2884
|
+
async mountAdvertisedChannel(plugin) {
|
|
2885
|
+
const index = this.advertisedChannels.findIndex((p) => p.Plugin === plugin);
|
|
2886
|
+
if (index < 0) {
|
|
2887
|
+
return; // already mounted (or never advertised) — the dispatcher opens it regardless
|
|
2888
|
+
}
|
|
2889
|
+
const [prepared] = this.advertisedChannels.splice(index, 1);
|
|
2890
|
+
try {
|
|
2891
|
+
this.mountChannel(prepared, this.GetResolvedChannel(prepared.Key) ?? undefined);
|
|
2892
|
+
}
|
|
2893
|
+
catch (error) {
|
|
2894
|
+
this.advertisedChannels.splice(index, 0, prepared);
|
|
2895
|
+
throw error;
|
|
2896
|
+
}
|
|
2897
|
+
this.restorePriorChannelState(plugin);
|
|
2898
|
+
this._activeChannels$.next([...this._activeChannels$.value, plugin]);
|
|
2899
|
+
if (this.isSessionLive()) {
|
|
2900
|
+
this.notifyChannelSessionStarted(plugin);
|
|
2901
|
+
}
|
|
2902
|
+
}
|
|
2903
|
+
/** Tells a channel the session is connected, containing anything it throws. */
|
|
2904
|
+
notifyChannelSessionStarted(channel) {
|
|
2905
|
+
try {
|
|
2906
|
+
channel.OnSessionStarted?.();
|
|
2907
|
+
}
|
|
2908
|
+
catch (err) {
|
|
2909
|
+
console.error(`[RealtimeSession] Error in channel '${channel.ChannelName}' OnSessionStarted:`, err);
|
|
2910
|
+
}
|
|
2911
|
+
}
|
|
2912
|
+
/** Finds a session channel (open or merely advertised) for the dispatcher. */
|
|
2913
|
+
findDispatchableChannel(key) {
|
|
2914
|
+
const id = NormalizeChannelKey(key);
|
|
2915
|
+
const open = this._activeChannels$.value.find((c) => NormalizeChannelKey(c.ChannelName) === id);
|
|
2916
|
+
if (open) {
|
|
2917
|
+
return { Plugin: open, IsOpen: true };
|
|
2918
|
+
}
|
|
2919
|
+
const advertised = this.advertisedChannels.find((p) => NormalizeChannelKey(p.Key) === id);
|
|
2920
|
+
return advertised ? { Plugin: advertised.Plugin, IsOpen: false } : null;
|
|
2921
|
+
}
|
|
2922
|
+
/** The keys the agent can address, for "unknown channel" messages — channels with nothing to address are left out. */
|
|
2923
|
+
listAddressableChannelKeys() {
|
|
2924
|
+
const addressable = (plugin) => {
|
|
2925
|
+
const descriptor = plugin.GetDescriptor();
|
|
2926
|
+
return descriptor.Verbs.length > 0 || descriptor.Inputs !== undefined;
|
|
2927
|
+
};
|
|
2928
|
+
return [
|
|
2929
|
+
...this._activeChannels$.value.filter(addressable).map((c) => c.ChannelName),
|
|
2930
|
+
...this.advertisedChannels.filter((p) => addressable(p.Plugin)).map((p) => p.Key),
|
|
2931
|
+
];
|
|
2932
|
+
}
|
|
2933
|
+
/**
|
|
2934
|
+
* Runs a channel-addressed `ContextTool` call (the {@link RealtimeChannelContext.DispatchContextAction}
|
|
2935
|
+
* implementation): validates it, opens an `on-demand` channel when asked, runs the verb, and on
|
|
2936
|
+
* success records the channel as used so the overlay reveals its tab. Never throws.
|
|
2937
|
+
*/
|
|
2938
|
+
async dispatchContextAction(request) {
|
|
2939
|
+
const result = await this.channelDispatcher.Dispatch(request);
|
|
2940
|
+
if (result.Success) {
|
|
2941
|
+
const channel = this.findDispatchableChannel(request.Target.Channel);
|
|
2942
|
+
if (channel) {
|
|
2943
|
+
this.noteChannelActivity(channel.Plugin);
|
|
2944
|
+
}
|
|
2945
|
+
}
|
|
2946
|
+
return result;
|
|
2947
|
+
}
|
|
2948
|
+
/**
|
|
2949
|
+
* Opens (and seeds) a channel from the HOST — the same path the agent's `open` action takes: the channel
|
|
2950
|
+
* is mounted if it was only advertised, its `Inputs` schema validates `inputs`, and it announces itself to
|
|
2951
|
+
* the model. A channel that is already open is re-seeded. Never throws; a refusal is a structured result.
|
|
2952
|
+
*
|
|
2953
|
+
* @param channelKey The channel to open (case-insensitive).
|
|
2954
|
+
* @param inputs Seed inputs, validated against the channel's descriptor.
|
|
2955
|
+
*/
|
|
2956
|
+
OpenChannel(channelKey, inputs = {}) {
|
|
2957
|
+
return this.dispatchContextAction({ Target: { Channel: channelKey }, Action: 'open', Params: inputs });
|
|
2958
|
+
}
|
|
2959
|
+
/**
|
|
2960
|
+
* Tells the model which channels exist and how to use them — rendered from the channels' own
|
|
2961
|
+
* descriptors, so a new channel needs no prompt change. Sent ONCE, the first moment the control
|
|
2962
|
+
* channel is usable — a context note sent before then is dropped, and `Connect` can resolve before the
|
|
2963
|
+
* provider reports `listening`, so this runs at start and again on each state change until it has gone.
|
|
2964
|
+
* A session whose only channels describe themselves through native tools sends nothing.
|
|
2965
|
+
*/
|
|
2966
|
+
flushChannelCatalogNote() {
|
|
2967
|
+
if (!this.catalogNotePending || !this.isSessionLive()) {
|
|
2968
|
+
return;
|
|
2969
|
+
}
|
|
2970
|
+
this.catalogNotePending = false;
|
|
2971
|
+
const entries = [
|
|
2972
|
+
...this._activeChannels$.value.map((plugin) => this.catalogEntry(plugin, true)),
|
|
2973
|
+
...this.advertisedChannels.map((p) => this.catalogEntry(p.Plugin, false)),
|
|
2974
|
+
];
|
|
2975
|
+
const note = BuildChannelCatalogNote(entries);
|
|
2976
|
+
if (note) {
|
|
2977
|
+
this.SendContextNote(note);
|
|
2978
|
+
}
|
|
2979
|
+
}
|
|
2980
|
+
/** One catalog entry: a channel's descriptor, whether it is open, and whether its tools were declared natively. */
|
|
2981
|
+
catalogEntry(plugin, isOpen) {
|
|
2982
|
+
const display = this.GetResolvedChannel(plugin.ChannelName)?.DisplayPolicy;
|
|
2983
|
+
const descriptor = plugin.GetDescriptor();
|
|
2984
|
+
const entry = {
|
|
2985
|
+
Descriptor: descriptor,
|
|
2986
|
+
IsOpen: isOpen,
|
|
2987
|
+
HasNativeTools: isOpen && display !== undefined && DeclaresNativeTools(display) && plugin.GetToolDefinitions().length > 0,
|
|
2988
|
+
};
|
|
2989
|
+
if (CompareExposure(plugin.Exposure, descriptor.MaxExposure) < 0) {
|
|
2990
|
+
entry.ExposureLimit = { Effective: plugin.Exposure, Ceiling: descriptor.MaxExposure, Reasons: [...plugin.ExposureReasons] };
|
|
2991
|
+
}
|
|
2992
|
+
return entry;
|
|
2993
|
+
}
|
|
1250
2994
|
/** Builds the host-services context one channel plugin sees (its only line to the session). */
|
|
1251
|
-
buildChannelContext(plugin) {
|
|
2995
|
+
buildChannelContext(plugin, channelConfig) {
|
|
1252
2996
|
// Capture the service in a local so the AgentSessionID getter reads the SERVICE's live
|
|
1253
2997
|
// field (not the object literal's `this`) every time it's accessed.
|
|
1254
2998
|
const service = this;
|
|
@@ -1275,55 +3019,107 @@ export class RealtimeSessionRuntime {
|
|
|
1275
3019
|
// (Explorer) feeds both; absent on hosts that supply no app context / register no client tools.
|
|
1276
3020
|
AppContext$: this.AppContext$,
|
|
1277
3021
|
ExecuteClientTool: (name, params) => this.executeAppClientTool(name, params),
|
|
3022
|
+
// The session's server events (identity verification, app events), for a channel that reacts to them.
|
|
3023
|
+
SessionEvents$: this.SessionEvents$,
|
|
3024
|
+
// Channel-addressed ContextTool calls: validated against the verb's schema, open `on-demand` channels.
|
|
3025
|
+
DispatchContextAction: (request) => this.dispatchContextAction(request),
|
|
3026
|
+
// This channel's resolved configuration (host defaults beneath agent/app config).
|
|
3027
|
+
ChannelConfig: channelConfig ?? {},
|
|
1278
3028
|
get Client() {
|
|
1279
3029
|
return service.client;
|
|
1280
3030
|
},
|
|
1281
|
-
|
|
1282
|
-
|
|
3031
|
+
// Through the arbiter, as this channel's source (deprecated: channels use a ChannelInboundVideoBridge).
|
|
3032
|
+
SendVideoFrame: (base64Image, mimeType) => this.sendChannelVideoFrame(plugin, base64Image, mimeType),
|
|
3033
|
+
IsTrackEstablished: (modality, direction) => this.IsTrackEstablished(modality, direction),
|
|
3034
|
+
// The camera and screen share, for a channel that fronts one: their state, and the user's clicks on its surface.
|
|
3035
|
+
Captures$: this.Captures$,
|
|
3036
|
+
StartCapture: (kind) => (kind === 'camera' ? this.StartCamera() : this.StartScreenShare()),
|
|
3037
|
+
StopCapture: (kind) => (kind === 'camera' ? this.StopCamera() : this.StopScreenShare()),
|
|
3038
|
+
// The video sources and which of them the model is sent, so a capture's channel says the agent sees it only then.
|
|
3039
|
+
VideoSources$: this.VideoSources$,
|
|
3040
|
+
// The agent's video, the call's state and its resumes, for a channel that shows the agent.
|
|
3041
|
+
AgentVideo$: this.AgentVideo$,
|
|
3042
|
+
ConnectionState$: this.ConnectionState$,
|
|
3043
|
+
Resuming$: this.Resuming$
|
|
1283
3044
|
};
|
|
1284
3045
|
}
|
|
1285
3046
|
/**
|
|
1286
|
-
*
|
|
1287
|
-
*
|
|
1288
|
-
*
|
|
3047
|
+
* Registers the tools ONE source of the host can run for the realtime `ContextTool`, replacing only
|
|
3048
|
+
* that source's previous set — other owners' tools are untouched. A host with two sources (Explorer's
|
|
3049
|
+
* always-available globals and the active surface's tools) registers each under its own owner key, so
|
|
3050
|
+
* refreshing the surface's tools can neither drop the globals nor leave the previous surface's tools
|
|
3051
|
+
* behind. Passing `[]` clears just that owner. When two owners register the same tool name, the
|
|
3052
|
+
* later-registered owner wins.
|
|
1289
3053
|
*
|
|
1290
|
-
*
|
|
1291
|
-
*
|
|
3054
|
+
* Calling it with one argument (the original form) registers under {@link DEFAULT_APP_TOOL_OWNER}, so
|
|
3055
|
+
* a host with a single source behaves exactly as before.
|
|
3056
|
+
*
|
|
3057
|
+
* @param tools The owner's complete current set (name + handler; description/schema optional).
|
|
3058
|
+
* @param owner A stable key naming the source (e.g. `'explorer.surface'`). Defaults to a shared owner.
|
|
1292
3059
|
*/
|
|
1293
|
-
RegisterAppClientTools(tools) {
|
|
1294
|
-
this.
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
3060
|
+
RegisterAppClientTools(tools, owner = DEFAULT_APP_TOOL_OWNER) {
|
|
3061
|
+
this.appToolRegistry.Register(owner, tools);
|
|
3062
|
+
}
|
|
3063
|
+
/** Removes one owner's tools (and its place in the collision order). No-op when it has none. */
|
|
3064
|
+
UnregisterAppClientTools(owner) {
|
|
3065
|
+
this.appToolRegistry.Unregister(owner);
|
|
3066
|
+
}
|
|
3067
|
+
/** Removes every owner's tools. */
|
|
3068
|
+
ClearAppClientTools() {
|
|
3069
|
+
this.appToolRegistry.Clear();
|
|
3070
|
+
}
|
|
3071
|
+
/**
|
|
3072
|
+
* The client tools in effect right now, resolved through the same unified resolver the server uses
|
|
3073
|
+
* (`override → session → app → static`, first match wins): the host's registered tools (described
|
|
3074
|
+
* by the manifest the host streams when it did not describe them itself) first, then the app and
|
|
3075
|
+
* agent tiers the server resolved at mint. Only the host-registered tools can RUN here; the
|
|
3076
|
+
* other tiers describe what the app declares.
|
|
3077
|
+
*/
|
|
3078
|
+
resolveAppClientTools() {
|
|
3079
|
+
const manifest = this._appContext$.value?.Capabilities?.Tools ?? [];
|
|
3080
|
+
return ResolveClientTools({
|
|
3081
|
+
agentId: '',
|
|
3082
|
+
sessionTools: MergeToolMetadata(this.appToolRegistry.ToMetadata(), manifest),
|
|
3083
|
+
appTools: this.sessionClientTools?.App,
|
|
3084
|
+
staticTools: this.sessionClientTools?.Static,
|
|
3085
|
+
});
|
|
1300
3086
|
}
|
|
1301
3087
|
/**
|
|
1302
3088
|
* Executes a host-registered surface client tool by name (the {@link RealtimeChannelContext.ExecuteClientTool}
|
|
1303
3089
|
* implementation). Tolerant: an unknown tool or a thrown handler resolves to a structured
|
|
1304
|
-
* `Success: false` result the channel narrates — never throws.
|
|
3090
|
+
* `Success: false` result the channel narrates — never throws. A tool the app declares but the host has
|
|
3091
|
+
* not registered a handler for says so, instead of reading like a typo. A handler that returns an object with
|
|
3092
|
+
* `Success: false` also gives `Success: false`, with the handler's `ErrorMessage` and its object as `Result`.
|
|
1305
3093
|
*
|
|
1306
3094
|
* @param name The tool name (the model's `action`).
|
|
1307
3095
|
* @param params The tool parameters.
|
|
1308
3096
|
* @returns A structured result for the channel to serialize back to the model.
|
|
1309
3097
|
*/
|
|
1310
3098
|
async executeAppClientTool(name, params) {
|
|
1311
|
-
const handler = this.
|
|
3099
|
+
const handler = this.appToolRegistry.Find(name)?.Handler;
|
|
1312
3100
|
if (!handler) {
|
|
1313
|
-
|
|
1314
|
-
return {
|
|
1315
|
-
Success: false,
|
|
1316
|
-
ErrorMessage: `No client tool named "${name}" is available on this surface. Available: ${available || '(none)'}.`
|
|
1317
|
-
};
|
|
3101
|
+
return { Success: false, ErrorMessage: this.describeMissingClientTool(name) };
|
|
1318
3102
|
}
|
|
1319
3103
|
try {
|
|
1320
3104
|
const result = await handler(params ?? {});
|
|
1321
|
-
return
|
|
3105
|
+
return reportsFailure(result)
|
|
3106
|
+
? { Success: false, ErrorMessage: failureMessage(result), Result: result }
|
|
3107
|
+
: { Success: true, Result: result };
|
|
1322
3108
|
}
|
|
1323
3109
|
catch (error) {
|
|
1324
3110
|
return { Success: false, ErrorMessage: error instanceof Error ? error.message : String(error) };
|
|
1325
3111
|
}
|
|
1326
3112
|
}
|
|
3113
|
+
/** The model-readable reason a client tool could not run. */
|
|
3114
|
+
describeMissingClientTool(name) {
|
|
3115
|
+
const available = this.appToolRegistry.Names().join(', ');
|
|
3116
|
+
const wanted = (name ?? '').trim().toLowerCase();
|
|
3117
|
+
const declared = this.resolveAppClientTools().find((t) => t.Name.trim().toLowerCase() === wanted);
|
|
3118
|
+
if (declared) {
|
|
3119
|
+
return `The client tool "${name}" is declared for this app but this surface has not registered it, so it cannot run right now. Available: ${available || '(none)'}.`;
|
|
3120
|
+
}
|
|
3121
|
+
return `No client tool named "${name}" is available on this surface. Available: ${available || '(none)'}.`;
|
|
3122
|
+
}
|
|
1327
3123
|
/**
|
|
1328
3124
|
* Runs a channel-specific GraphQL operation through the live session's provider (the
|
|
1329
3125
|
* {@link RealtimeChannelContext.ExecuteServerAction} implementation). Best-effort: any
|
|
@@ -1360,35 +3156,39 @@ export class RealtimeSessionRuntime {
|
|
|
1360
3156
|
* never affected.
|
|
1361
3157
|
*/
|
|
1362
3158
|
applyPriorChannelStates(statesJson) {
|
|
3159
|
+
this.priorChannelStates = {};
|
|
1363
3160
|
if (!statesJson) {
|
|
1364
3161
|
return;
|
|
1365
3162
|
}
|
|
1366
|
-
let states;
|
|
1367
3163
|
try {
|
|
1368
3164
|
const parsed = JSON.parse(statesJson);
|
|
1369
3165
|
if (parsed === null || typeof parsed !== 'object') {
|
|
1370
3166
|
return;
|
|
1371
3167
|
}
|
|
1372
|
-
|
|
3168
|
+
this.priorChannelStates = parsed;
|
|
1373
3169
|
}
|
|
1374
3170
|
catch {
|
|
1375
3171
|
console.warn('[RealtimeSession] PriorChannelStatesJson was malformed — starting channels fresh');
|
|
1376
3172
|
return;
|
|
1377
3173
|
}
|
|
1378
3174
|
for (const plugin of this._activeChannels$.value) {
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
3175
|
+
this.restorePriorChannelState(plugin);
|
|
3176
|
+
}
|
|
3177
|
+
}
|
|
3178
|
+
/** Offers one plugin its prior-session state, if there is one. Rejections and throws are logged, never fatal. */
|
|
3179
|
+
restorePriorChannelState(plugin) {
|
|
3180
|
+
const state = this.priorChannelStates[plugin.ChannelName];
|
|
3181
|
+
if (typeof state !== 'string' || state.length === 0) {
|
|
3182
|
+
return;
|
|
3183
|
+
}
|
|
3184
|
+
try {
|
|
3185
|
+
if (!plugin.RestoreState(state)) {
|
|
3186
|
+
console.warn(`[RealtimeSession] Channel '${plugin.ChannelName}' declined its prior-session state — starting fresh`);
|
|
1390
3187
|
}
|
|
1391
3188
|
}
|
|
3189
|
+
catch (error) {
|
|
3190
|
+
console.warn(`[RealtimeSession] Channel '${plugin.ChannelName}' restore threw — starting fresh`, error);
|
|
3191
|
+
}
|
|
1392
3192
|
}
|
|
1393
3193
|
/**
|
|
1394
3194
|
* Persists a channel's state as a first-class versioned artifact (`MJ: Artifacts`) via the
|
|
@@ -1478,6 +3278,13 @@ export class RealtimeSessionRuntime {
|
|
|
1478
3278
|
this._activeChannels$.next([]);
|
|
1479
3279
|
}
|
|
1480
3280
|
this.usedChannelNames.clear();
|
|
3281
|
+
// Channels that were prepared or advertised but never mounted were never initialized, so there is
|
|
3282
|
+
// nothing to dispose — only the bookkeeping to forget.
|
|
3283
|
+
this.discardUnmountedChannels();
|
|
3284
|
+
this.catalogNotePending = false;
|
|
3285
|
+
this.resolvedChannels.clear();
|
|
3286
|
+
this.priorChannelStates = {};
|
|
3287
|
+
this.sessionClientTools = null;
|
|
1481
3288
|
}
|
|
1482
3289
|
// ── Realtime client resolution + wiring ────────────────────────────────────
|
|
1483
3290
|
/**
|
|
@@ -1500,12 +3307,14 @@ export class RealtimeSessionRuntime {
|
|
|
1500
3307
|
}
|
|
1501
3308
|
/**
|
|
1502
3309
|
* Builds the client-direct session config the realtime client connects with.
|
|
1503
|
-
* Aggregates tracks
|
|
1504
|
-
* can negotiate them (e.g.,
|
|
3310
|
+
* Aggregates the tracks active channels source and sink under {@link REQUESTED_TRACKS_SESSION_KEY} so the
|
|
3311
|
+
* driver can negotiate them (e.g., inbound video for Whiteboard / RemoteBrowser, outbound video for a
|
|
3312
|
+
* channel that shows the agent's video). A track the model does not support resolves to `'unsupported'`.
|
|
3313
|
+
* The session's transport and relay URL go to the driver as minted (an unknown transport is left out: direct).
|
|
1505
3314
|
*/
|
|
1506
3315
|
BuildClientConfig(session) {
|
|
1507
3316
|
const sessionConfig = this.parseSessionConfig(session.SessionConfigJson);
|
|
1508
|
-
const channelTracks = this._activeChannels$.value.flatMap((c) => c.GetSourcedTracks());
|
|
3317
|
+
const channelTracks = this._activeChannels$.value.flatMap((c) => [...c.GetSourcedTracks(), ...c.GetSunkTracks()]);
|
|
1509
3318
|
if (channelTracks.length > 0) {
|
|
1510
3319
|
// The requested tracks cross a JSON boundary — the driver reads them back out of the session
|
|
1511
3320
|
// config bag (`GeminiRealtimeClient.parseSessionConfig`). A `RealtimeTrackDescriptor` is NOT
|
|
@@ -1530,11 +3339,14 @@ export class RealtimeSessionRuntime {
|
|
|
1530
3339
|
}
|
|
1531
3340
|
sessionConfig[REQUESTED_TRACKS_SESSION_KEY] = Array.from(trackMap.values());
|
|
1532
3341
|
}
|
|
3342
|
+
const transport = ParseRealtimeClientTransport(session.Transport);
|
|
1533
3343
|
return {
|
|
1534
3344
|
Provider: session.Provider,
|
|
1535
3345
|
Model: session.Model,
|
|
1536
3346
|
EphemeralToken: session.EphemeralToken,
|
|
1537
3347
|
ExpiresAt: session.ExpiresAt,
|
|
3348
|
+
...(transport ? { Transport: transport } : {}),
|
|
3349
|
+
...(session.RelayUrl ? { RelayUrl: session.RelayUrl } : {}),
|
|
1538
3350
|
SessionConfig: sessionConfig
|
|
1539
3351
|
};
|
|
1540
3352
|
}
|
|
@@ -1570,6 +3382,9 @@ export class RealtimeSessionRuntime {
|
|
|
1570
3382
|
});
|
|
1571
3383
|
client.OnError((error) => {
|
|
1572
3384
|
console.error('[RealtimeSession] Provider error event:', JSON.stringify(error), error);
|
|
3385
|
+
if (error.Fatal && this.client === client) {
|
|
3386
|
+
this.lastSessionError = error;
|
|
3387
|
+
}
|
|
1573
3388
|
});
|
|
1574
3389
|
// Usage telemetry: accumulate the driver's per-response token DELTAS and relay them to
|
|
1575
3390
|
// the server (onto the co-agent AIPromptRun) debounced + once at teardown. Providers
|
|
@@ -1586,12 +3401,41 @@ export class RealtimeSessionRuntime {
|
|
|
1586
3401
|
client.OnInterruption(() => {
|
|
1587
3402
|
this.cancelPendingNarration();
|
|
1588
3403
|
});
|
|
3404
|
+
// The provider VAD saw the user start speaking. Final-only user-transcript drivers (OpenAI,
|
|
3405
|
+
// HuggingFace, xAI) emit no user interim, so this is their only real user-turn start. It goes
|
|
3406
|
+
// to its own slot, not the shared turn slot: the user transcript arrives asynchronously, so the
|
|
3407
|
+
// assistant's reply may already be streaming and must keep its own start.
|
|
3408
|
+
client.OnUserSpeechStart(() => this.markUserSpeechStart());
|
|
3409
|
+
client.OnRemoteVideo((video) => this.onAgentVideo(client, video));
|
|
3410
|
+
}
|
|
3411
|
+
/**
|
|
3412
|
+
* The model's video arrived: publish it on {@link AgentVideo$}, and mark each channel that shows it (one that sinks
|
|
3413
|
+
* outbound video) as used, so the host shows its surface. A client the session has already let go of is ignored, so a
|
|
3414
|
+
* late frame cannot bring the video back after the call.
|
|
3415
|
+
*/
|
|
3416
|
+
onAgentVideo(client, video) {
|
|
3417
|
+
if (this.client !== client) {
|
|
3418
|
+
return;
|
|
3419
|
+
}
|
|
3420
|
+
this._agentVideo$.next(video);
|
|
3421
|
+
for (const channel of this._activeChannels$.value) {
|
|
3422
|
+
if (channel.ShowsAgentVideo) {
|
|
3423
|
+
this.noteChannelActivity(channel);
|
|
3424
|
+
}
|
|
3425
|
+
}
|
|
3426
|
+
}
|
|
3427
|
+
/** Clears {@link AgentVideo$} when the call's client goes. */
|
|
3428
|
+
clearAgentVideo() {
|
|
3429
|
+
if (this._agentVideo$.value !== null) {
|
|
3430
|
+
this._agentVideo$.next(null);
|
|
3431
|
+
}
|
|
1589
3432
|
}
|
|
1590
3433
|
/** Maps a client state event onto the UI connection state. */
|
|
1591
3434
|
onClientStateChange(state) {
|
|
1592
3435
|
const mapped = this.mapClientState(state);
|
|
1593
3436
|
if (mapped) {
|
|
1594
|
-
this.
|
|
3437
|
+
this.setConnectionState(mapped);
|
|
3438
|
+
this.flushChannelCatalogNote();
|
|
1595
3439
|
}
|
|
1596
3440
|
}
|
|
1597
3441
|
/**
|
|
@@ -1620,6 +3464,17 @@ export class RealtimeSessionRuntime {
|
|
|
1620
3464
|
const state = this._connectionState$.value;
|
|
1621
3465
|
return state === 'listening' || state === 'speaking' || state === 'thinking';
|
|
1622
3466
|
}
|
|
3467
|
+
/**
|
|
3468
|
+
* Moves {@link ConnectionState$} to `state`, and {@link Resuming$} first: a `'connecting'` while the call is live, or
|
|
3469
|
+
* already resuming, is a resume; any other state ends one. Every state change goes through here.
|
|
3470
|
+
*/
|
|
3471
|
+
setConnectionState(state) {
|
|
3472
|
+
const resuming = state === 'connecting' && (this._resuming$.value || this.isSessionLive());
|
|
3473
|
+
if (resuming !== this._resuming$.value) {
|
|
3474
|
+
this._resuming$.next(resuming);
|
|
3475
|
+
}
|
|
3476
|
+
this._connectionState$.next(state);
|
|
3477
|
+
}
|
|
1623
3478
|
// ── Transcript policy ──────────────────────────────────────────────────────
|
|
1624
3479
|
/**
|
|
1625
3480
|
* Applies transcript policy to client transcript events. Interim deltas don't become
|
|
@@ -1653,6 +3508,11 @@ export class RealtimeSessionRuntime {
|
|
|
1653
3508
|
}
|
|
1654
3509
|
return;
|
|
1655
3510
|
}
|
|
3511
|
+
if (transcript.Role === 'User' && transcript.Text.trim().length === 0) {
|
|
3512
|
+
// An empty final user transcript (a VAD false positive, an empty transcription) never reaches
|
|
3513
|
+
// relayTranscript, so release its onset here: it must not become the next real turn's start.
|
|
3514
|
+
this.releaseUserSpeechStart();
|
|
3515
|
+
}
|
|
1656
3516
|
if (transcript.Role === 'Assistant') {
|
|
1657
3517
|
this.hasActiveInterimUserCaption = false;
|
|
1658
3518
|
this.pendingUserCaption = '';
|
|
@@ -1727,11 +3587,10 @@ export class RealtimeSessionRuntime {
|
|
|
1727
3587
|
* - NORMAL turns only — NARRATION interims are ephemeral and never persisted, so they must not
|
|
1728
3588
|
* claim the next real turn's start slot.
|
|
1729
3589
|
*
|
|
1730
|
-
*
|
|
1731
|
-
*
|
|
1732
|
-
*
|
|
1733
|
-
*
|
|
1734
|
-
* assistant answer, which always has interims.
|
|
3590
|
+
* Stamped on a turn's first interim transcript (all drivers for the assistant — the post-tool-gap
|
|
3591
|
+
* assistant answer — and user-interim drivers like Gemini/AssemblyAI). Final-only user turns
|
|
3592
|
+
* (OpenAI-protocol drivers) take their start from {@link markUserSpeechStart}'s own slot
|
|
3593
|
+
* instead; one with no speech-start signal either (ElevenLabs) relays `null` as-is.
|
|
1735
3594
|
*/
|
|
1736
3595
|
markTurnAudioStart(kind) {
|
|
1737
3596
|
if (this.turnAudioStartCaptured || kind === 'narration') {
|
|
@@ -1744,15 +3603,39 @@ export class RealtimeSessionRuntime {
|
|
|
1744
3603
|
this.currentTurnStartMs = offset;
|
|
1745
3604
|
this.turnAudioStartCaptured = true;
|
|
1746
3605
|
}
|
|
3606
|
+
/**
|
|
3607
|
+
* Stamps {@link userSpeechStartMs} when the client signals that the user began to speak. Keeps
|
|
3608
|
+
* the FIRST onset since the last final user transcript: the VAD fires again after a
|
|
3609
|
+
* mid-sentence pause, and that second onset is not where the utterance began. An onset that an
|
|
3610
|
+
* assistant turn which began after it has since superseded ({@link userSpeechStartSuperseded}) is
|
|
3611
|
+
* replaced instead.
|
|
3612
|
+
*/
|
|
3613
|
+
markUserSpeechStart() {
|
|
3614
|
+
if (this.userSpeechStartMs !== null && !this.userSpeechStartSuperseded) {
|
|
3615
|
+
return;
|
|
3616
|
+
}
|
|
3617
|
+
const offset = this.nowTurnOffsetMs();
|
|
3618
|
+
if (offset === null) {
|
|
3619
|
+
return;
|
|
3620
|
+
}
|
|
3621
|
+
this.userSpeechStartMs = offset;
|
|
3622
|
+
this.userSpeechStartSuperseded = false;
|
|
3623
|
+
}
|
|
3624
|
+
/** Ends the held user onset: a final user transcript (empty or not) closed the utterance it began. */
|
|
3625
|
+
releaseUserSpeechStart() {
|
|
3626
|
+
this.userSpeechStartMs = null;
|
|
3627
|
+
this.userSpeechStartSuperseded = false;
|
|
3628
|
+
}
|
|
1747
3629
|
/**
|
|
1748
3630
|
* The current per-turn offset in ms — the RECORDER's clock when one runs (an offset into a
|
|
1749
3631
|
* seekable file), else the SESSION clock (#3832: orderable and displayable, not seekable),
|
|
1750
3632
|
* else `null` before any call is live. One function so the two stamp sites cannot disagree
|
|
1751
|
-
* about which clock a session is on.
|
|
3633
|
+
* about which clock a session is on. Always whole milliseconds: the offsets travel as GraphQL
|
|
3634
|
+
* `Int`, which refuses a fraction, and a 48 kHz sample clock rarely lands on a whole millisecond.
|
|
1752
3635
|
*/
|
|
1753
3636
|
nowTurnOffsetMs() {
|
|
1754
3637
|
if (this.recorder) {
|
|
1755
|
-
return this.recorder.NowOffsetMs();
|
|
3638
|
+
return Math.round(this.recorder.NowOffsetMs());
|
|
1756
3639
|
}
|
|
1757
3640
|
if (this.sessionClockStartMs !== null) {
|
|
1758
3641
|
return Math.max(0, Math.round(performance.now() - this.sessionClockStartMs));
|
|
@@ -1776,8 +3659,13 @@ export class RealtimeSessionRuntime {
|
|
|
1776
3659
|
}
|
|
1777
3660
|
this.appendCaption({ Role: role, Text: text });
|
|
1778
3661
|
}
|
|
1779
|
-
/**
|
|
1780
|
-
|
|
3662
|
+
/**
|
|
3663
|
+
* Finalizes the user turn: push a caption + relay the final transcript.
|
|
3664
|
+
*
|
|
3665
|
+
* @param source `'typed'` for a {@link SendText} turn: it has no speech onset, so it neither takes
|
|
3666
|
+
* nor clears the held one (a spoken transcript may still be pending for it).
|
|
3667
|
+
*/
|
|
3668
|
+
async onUserTranscript(transcript, source = 'speech') {
|
|
1781
3669
|
if (transcript.trim().length === 0) {
|
|
1782
3670
|
return;
|
|
1783
3671
|
}
|
|
@@ -1786,7 +3674,7 @@ export class RealtimeSessionRuntime {
|
|
|
1786
3674
|
this.firstUserTranscript = transcript;
|
|
1787
3675
|
}
|
|
1788
3676
|
this.appendCaption({ Role: 'User', Text: transcript });
|
|
1789
|
-
await this.relayTranscript('user', transcript);
|
|
3677
|
+
await this.relayTranscript('user', transcript, false, source);
|
|
1790
3678
|
}
|
|
1791
3679
|
// ── Tool calling ───────────────────────────────────────────────────────────
|
|
1792
3680
|
/**
|
|
@@ -1809,7 +3697,7 @@ export class RealtimeSessionRuntime {
|
|
|
1809
3697
|
void this.relayToolTurn(call.ToolName, call.ArgumentsJson, resultJson);
|
|
1810
3698
|
return;
|
|
1811
3699
|
}
|
|
1812
|
-
this.
|
|
3700
|
+
this.setConnectionState('thinking');
|
|
1813
3701
|
if (this.inFlightCallIds.size === 0) {
|
|
1814
3702
|
// A fresh delegation burst: anchor the first-update delay and clear the digest
|
|
1815
3703
|
// buffer. Deliberately NOT reset: lastDelegationNarrationAt (the 8s spacing floor
|
|
@@ -1906,6 +3794,40 @@ export class RealtimeSessionRuntime {
|
|
|
1906
3794
|
RunID: parsed.RunID,
|
|
1907
3795
|
Artifacts: parsed.Artifacts
|
|
1908
3796
|
});
|
|
3797
|
+
if (parsed.Success && parsed.Artifacts && parsed.Artifacts.length > 0) {
|
|
3798
|
+
void this.offerDelegationArtifacts(parsed.Artifacts);
|
|
3799
|
+
}
|
|
3800
|
+
}
|
|
3801
|
+
/**
|
|
3802
|
+
* Offers a delegated run's artifacts to the channels that host artifacts (an Interactive Component channel
|
|
3803
|
+
* shows a component artifact, or swaps in a newer version of one it already shows). Each channel is asked
|
|
3804
|
+
* first whether it wants them; an advertised, unopened channel that does is mounted before it is handed
|
|
3805
|
+
* them. One channel failing never stops the others, and nothing here can fail the delegation result,
|
|
3806
|
+
* which has already been emitted.
|
|
3807
|
+
*/
|
|
3808
|
+
async offerDelegationArtifacts(artifacts) {
|
|
3809
|
+
const candidates = [
|
|
3810
|
+
...this._activeChannels$.value.map((plugin) => ({ Plugin: plugin, IsOpen: true })),
|
|
3811
|
+
...this.advertisedChannels.map((prepared) => ({ Plugin: prepared.Plugin, IsOpen: false }))
|
|
3812
|
+
];
|
|
3813
|
+
for (const { Plugin: plugin, IsOpen: isOpen } of candidates) {
|
|
3814
|
+
try {
|
|
3815
|
+
const config = this.GetResolvedChannel(plugin.ChannelName)?.Config ?? {};
|
|
3816
|
+
if (!plugin.AcceptsDelegationArtifacts(artifacts, config)) {
|
|
3817
|
+
continue;
|
|
3818
|
+
}
|
|
3819
|
+
if (!isOpen) {
|
|
3820
|
+
await this.mountAdvertisedChannel(plugin);
|
|
3821
|
+
}
|
|
3822
|
+
await plugin.OnDelegationArtifacts(artifacts);
|
|
3823
|
+
if (!isOpen) {
|
|
3824
|
+
this.noteChannelActivity(plugin); // the channel was mounted for this: reveal its tab
|
|
3825
|
+
}
|
|
3826
|
+
}
|
|
3827
|
+
catch (error) {
|
|
3828
|
+
console.error(`[RealtimeSession] Channel '${plugin.ChannelName}' failed to take a delegated run's artifacts:`, error);
|
|
3829
|
+
}
|
|
3830
|
+
}
|
|
1909
3831
|
}
|
|
1910
3832
|
// ── Explicit delegation cancellation (server cancel channel) ───────────────
|
|
1911
3833
|
/**
|
|
@@ -1996,44 +3918,34 @@ export class RealtimeSessionRuntime {
|
|
|
1996
3918
|
}
|
|
1997
3919
|
}
|
|
1998
3920
|
// ── Session minting (GraphQL) ──────────────────────────────────────────────
|
|
1999
|
-
/**
|
|
2000
|
-
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
mediaCollectionId: mediaCollectionId ?? null,
|
|
2028
|
-
applicationId: applicationId ?? null,
|
|
2029
|
-
appContextJson: appContext ? JSON.stringify(appContext) : null
|
|
2030
|
-
};
|
|
2031
|
-
const result = await this.gql().ExecuteGQL(mutation, variables);
|
|
2032
|
-
const payload = result?.StartRealtimeClientSession;
|
|
2033
|
-
if (!payload?.EphemeralToken) {
|
|
2034
|
-
throw new Error('StartRealtimeClientSession returned no ephemeral token');
|
|
2035
|
-
}
|
|
2036
|
-
return payload;
|
|
3921
|
+
/**
|
|
3922
|
+
* Mints a session through the installed {@link Launcher} — by default the stock
|
|
3923
|
+
* `StartRealtimeClientSession` mutation (see {@link DefaultRealtimeSessionLauncher}). `showsAgentVideo` is whether
|
|
3924
|
+
* one of the session's channels could show the agent's video; without one the server asks the model for no avatar.
|
|
3925
|
+
* `conversationBranchId` is the fork the session starts on; `null` is Main.
|
|
3926
|
+
*/
|
|
3927
|
+
async mintSession(targetAgentId, conversationId, lastSessionId, preferredModelId, clientTools, coAgentId, configOverridesJson, recordingConsent, recordingStartedAt, mediaCollectionId, applicationId, appContext, channelCandidatesJson, showsAgentVideo, conversationBranchId) {
|
|
3928
|
+
const result = await this._launcher.Launch({
|
|
3929
|
+
TargetAgentId: targetAgentId,
|
|
3930
|
+
ConversationId: conversationId ?? null,
|
|
3931
|
+
LastSessionId: lastSessionId ?? null,
|
|
3932
|
+
PreferredModelId: preferredModelId ?? null,
|
|
3933
|
+
ClientTools: clientTools ?? [],
|
|
3934
|
+
CoAgentId: coAgentId ?? null,
|
|
3935
|
+
ConfigOverridesJson: configOverridesJson ?? null,
|
|
3936
|
+
RecordingConsent: recordingConsent ?? false,
|
|
3937
|
+
RecordingStartedAt: recordingStartedAt ?? null,
|
|
3938
|
+
MediaCollectionId: mediaCollectionId ?? null,
|
|
3939
|
+
ApplicationId: applicationId ?? null,
|
|
3940
|
+
AppContext: appContext ?? null,
|
|
3941
|
+
ChannelCandidatesJson: channelCandidatesJson ?? null,
|
|
3942
|
+
ConversationBranchId: conversationBranchId ?? null,
|
|
3943
|
+
...(showsAgentVideo === undefined ? {} : { ShowsAgentVideo: showsAgentVideo })
|
|
3944
|
+
}, { Provider: this.Provider });
|
|
3945
|
+
if (!HasClientCredential(result)) {
|
|
3946
|
+
throw new Error('The session launcher returned no ephemeral token (or, for a relay session, no relay URL)');
|
|
3947
|
+
}
|
|
3948
|
+
return result;
|
|
2037
3949
|
}
|
|
2038
3950
|
/** Calls the `ExecuteRealtimeSessionTool` mutation; returns the ResultJson string. */
|
|
2039
3951
|
async executeSessionTool(callId, toolName, argsJson) {
|
|
@@ -2090,37 +4002,71 @@ export class RealtimeSessionRuntime {
|
|
|
2090
4002
|
* Relays a final transcript turn to MJ via `RelayRealtimeTranscript`.
|
|
2091
4003
|
*
|
|
2092
4004
|
* When the session is being recorded, per-turn timing rides along: `utteranceEndMs` is the
|
|
2093
|
-
* recording-relative offset at finalization, and `utteranceStartMs` is
|
|
2094
|
-
*
|
|
2095
|
-
*
|
|
2096
|
-
*
|
|
2097
|
-
*
|
|
2098
|
-
*
|
|
4005
|
+
* recording-relative offset at finalization, and `utteranceStartMs` is where THIS turn's audio
|
|
4006
|
+
* actually began — NOT inherited from the previous turn's end. For a user turn whose onset the
|
|
4007
|
+
* client signalled, that is {@link userSpeechStartMs}; otherwise it is the offset captured by
|
|
4008
|
+
* {@link markTurnAudioStart} on the turn's first interim. That distinction is the timing fix:
|
|
4009
|
+
* when a tool-call / silence gap sits between turns, the post-gap turn's audio starts much
|
|
4010
|
+
* later, so inheriting the prior turn's end stamped the cue ~gap-length too early. Both are
|
|
4011
|
+
* omitted (left `null`) when the session isn't being recorded. `utteranceStartMs` is also
|
|
4012
|
+
* `null` when no start was measured for this turn; it is never replaced with a placeholder 0.
|
|
2099
4013
|
*
|
|
2100
4014
|
* A correction (`replacesPrevious`) doesn't open a new turn, so it carries no start and doesn't
|
|
2101
4015
|
* reset the per-turn start guard. After a normal finalization the guard is cleared so the NEXT
|
|
2102
|
-
* turn re-stamps its start from where ITS audio begins
|
|
4016
|
+
* turn re-stamps its start from where ITS audio begins — except for a user turn that used its
|
|
4017
|
+
* own speech-start slot, which leaves the shared slot alone (it may hold the in-flight
|
|
4018
|
+
* assistant turn's start). Every final user transcript, corrections included, clears the
|
|
4019
|
+
* speech-start slot.
|
|
2103
4020
|
*
|
|
2104
4021
|
* @param replacesPrevious CORRECTION semantics: the server updates the session's most
|
|
2105
4022
|
* recent persisted turn of this role IN PLACE instead of appending (e.g. ElevenLabs'
|
|
2106
4023
|
* post-barge-in `agent_response_correction`).
|
|
4024
|
+
* @param source `'typed'` for a user turn sent with {@link SendText}: it does not read or clear
|
|
4025
|
+
* {@link userSpeechStartMs}, which belongs to the user's speech.
|
|
2107
4026
|
*/
|
|
2108
|
-
async relayTranscript(role, text, replacesPrevious = false) {
|
|
4027
|
+
async relayTranscript(role, text, replacesPrevious = false, source = 'speech') {
|
|
2109
4028
|
if (!this.agentSessionId) {
|
|
2110
4029
|
return;
|
|
2111
4030
|
}
|
|
4031
|
+
// Every final user transcript closes the signalled onset — a correction too, since it ends the
|
|
4032
|
+
// same utterance — so a later onset (an xAI mid-sentence resume) can't leak into the next turn.
|
|
4033
|
+
let userSpeechStartMs = null;
|
|
4034
|
+
if (role === 'user' && source === 'speech') {
|
|
4035
|
+
userSpeechStartMs = this.userSpeechStartMs;
|
|
4036
|
+
this.releaseUserSpeechStart();
|
|
4037
|
+
}
|
|
2112
4038
|
// Per-turn timing against whichever clock the session is on (#3832): the recorder's when one
|
|
2113
|
-
// runs, else the session clock. `utteranceStartMs` is where this turn's audio actually began
|
|
2114
|
-
//
|
|
2115
|
-
//
|
|
4039
|
+
// runs, else the session clock. `utteranceStartMs` is where this turn's audio actually began.
|
|
4040
|
+
// It is null when no start was measured for this turn (a final-only driver with no speech-start
|
|
4041
|
+
// signal), so the server stores NULL rather than a placeholder 0.
|
|
2116
4042
|
const utteranceEndMs = this.nowTurnOffsetMs();
|
|
2117
|
-
const
|
|
2118
|
-
|
|
4043
|
+
const opensTurn = utteranceEndMs !== null && !replacesPrevious;
|
|
4044
|
+
let utteranceStartMs = null;
|
|
4045
|
+
if (opensTurn && userSpeechStartMs !== null) {
|
|
4046
|
+
// A user turn with a signalled onset. Leave the shared slot alone: the user transcript
|
|
4047
|
+
// arrives asynchronously, so it may already hold the in-flight assistant reply's start.
|
|
4048
|
+
utteranceStartMs = userSpeechStartMs;
|
|
4049
|
+
if (!this.turnAudioStartCaptured) {
|
|
4050
|
+
// No turn has captured a start, so the slot holds only the recording-start seed (0). This
|
|
4051
|
+
// user turn was the first turn; drop the seed so a later unmeasured turn relays null, not 0.
|
|
4052
|
+
this.currentTurnStartMs = null;
|
|
4053
|
+
}
|
|
4054
|
+
}
|
|
4055
|
+
else if (opensTurn) {
|
|
4056
|
+
utteranceStartMs = this.currentTurnStartMs;
|
|
2119
4057
|
// This turn is finalized — arm the NEXT turn to re-stamp its start from its own first
|
|
2120
4058
|
// interim (handles a tool-call gap before the next turn). Stop inheriting this end as the
|
|
2121
|
-
// next start. `null` means "not yet captured"
|
|
4059
|
+
// next start. `null` means "not yet captured" and is relayed as-is if nothing stamps it.
|
|
2122
4060
|
this.currentTurnStartMs = null;
|
|
2123
4061
|
this.turnAudioStartCaptured = false;
|
|
4062
|
+
if (role === 'assistant' && this.userSpeechStartMs !== null && (utteranceStartMs === null || this.userSpeechStartMs < utteranceStartMs)) {
|
|
4063
|
+
// The floor moved on past an onset that began BEFORE this assistant turn: a late user
|
|
4064
|
+
// transcript may still claim it, but the next onset replaces it (see userSpeechStartSuperseded).
|
|
4065
|
+
// An onset that began DURING this turn is a barge-in; it belongs to the user speech still
|
|
4066
|
+
// pending, so it stays the first onset. With no measured assistant start we cannot tell, and
|
|
4067
|
+
// treat the onset as earlier.
|
|
4068
|
+
this.userSpeechStartSuperseded = true;
|
|
4069
|
+
}
|
|
2124
4070
|
}
|
|
2125
4071
|
try {
|
|
2126
4072
|
const mutation = `
|
|
@@ -2172,16 +4118,28 @@ export class RealtimeSessionRuntime {
|
|
|
2172
4118
|
/**
|
|
2173
4119
|
* Accumulates one usage DELTA from the realtime client (per-response token counts —
|
|
2174
4120
|
* the `OnUsage` contract shape) and schedules the debounced relay. Negative / non-finite
|
|
2175
|
-
* values are clamped to 0; an all-zero delta is dropped without arming the timer.
|
|
4121
|
+
* values are clamped to 0; an all-zero delta is dropped without arming the timer. The
|
|
4122
|
+
* update's input and output detail blocks add into the pending details by the record's
|
|
4123
|
+
* rules (amounts add up, inbound video running totals keep the larger value), so an update
|
|
4124
|
+
* that carries only avatar video seconds still counts.
|
|
2176
4125
|
*/
|
|
2177
4126
|
onUsageDelta(usage) {
|
|
2178
4127
|
const input = this.clampUsageDelta(usage.InputTokens);
|
|
2179
4128
|
const output = this.clampUsageDelta(usage.OutputTokens);
|
|
2180
|
-
|
|
4129
|
+
const details = AddRealtimeUsageRecord(null, { Input: usage.InputTokenDetails, Output: usage.OutputTokenDetails });
|
|
4130
|
+
const hasDetails = HasRealtimeUsage(details);
|
|
4131
|
+
if (input === 0 && output === 0 && !hasDetails) {
|
|
2181
4132
|
return;
|
|
2182
4133
|
}
|
|
2183
4134
|
this.pendingUsageInput += input;
|
|
2184
4135
|
this.pendingUsageOutput += output;
|
|
4136
|
+
if (hasDetails) {
|
|
4137
|
+
this.pendingUsageDetails = AddRealtimeUsageRecord(this.pendingUsageDetails, details);
|
|
4138
|
+
}
|
|
4139
|
+
this.armUsageFlush();
|
|
4140
|
+
}
|
|
4141
|
+
/** Schedules the debounced usage relay unless one is already pending. */
|
|
4142
|
+
armUsageFlush() {
|
|
2185
4143
|
if (!this.usageFlushTimer) {
|
|
2186
4144
|
this.usageFlushTimer = setTimeout(() => {
|
|
2187
4145
|
this.usageFlushTimer = null;
|
|
@@ -2206,24 +4164,26 @@ export class RealtimeSessionRuntime {
|
|
|
2206
4164
|
const sessionId = agentSessionId ?? this.agentSessionId;
|
|
2207
4165
|
const input = this.pendingUsageInput;
|
|
2208
4166
|
const output = this.pendingUsageOutput;
|
|
2209
|
-
|
|
4167
|
+
const details = this.pendingUsageDetails;
|
|
4168
|
+
if (!sessionId || (input === 0 && output === 0 && !details)) {
|
|
2210
4169
|
return;
|
|
2211
4170
|
}
|
|
2212
4171
|
this.pendingUsageInput = 0;
|
|
2213
4172
|
this.pendingUsageOutput = 0;
|
|
4173
|
+
this.pendingUsageDetails = null;
|
|
2214
4174
|
try {
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
}
|
|
2219
|
-
`;
|
|
2220
|
-
await this.gql().ExecuteGQL(mutation, { agentSessionId: sessionId, inputTokens: input, outputTokens: output });
|
|
4175
|
+
// The details argument is sent only when there are details, so a token-only relay is unchanged.
|
|
4176
|
+
const variables = { agentSessionId: sessionId, inputTokens: input, outputTokens: output, ...(details ? { usageDetailsJson: JSON.stringify(details) } : {}) };
|
|
4177
|
+
await this.gql().ExecuteGQL(RELAY_REALTIME_USAGE_MUTATION, variables);
|
|
2221
4178
|
}
|
|
2222
4179
|
catch (error) {
|
|
2223
4180
|
console.error('[RealtimeSession] Failed to relay usage telemetry:', error);
|
|
2224
4181
|
// Re-accumulate so a later debounce / the teardown flush retries the same deltas.
|
|
2225
4182
|
this.pendingUsageInput += input;
|
|
2226
4183
|
this.pendingUsageOutput += output;
|
|
4184
|
+
if (details) {
|
|
4185
|
+
this.pendingUsageDetails = AddRealtimeUsageRecord(this.pendingUsageDetails, details);
|
|
4186
|
+
}
|
|
2227
4187
|
}
|
|
2228
4188
|
}
|
|
2229
4189
|
/** Cancels the pending debounced usage flush and zeroes the accumulators (teardown tail). */
|
|
@@ -2234,6 +4194,7 @@ export class RealtimeSessionRuntime {
|
|
|
2234
4194
|
}
|
|
2235
4195
|
this.pendingUsageInput = 0;
|
|
2236
4196
|
this.pendingUsageOutput = 0;
|
|
4197
|
+
this.pendingUsageDetails = null;
|
|
2237
4198
|
}
|
|
2238
4199
|
// ── Delegated-run progress streaming ───────────────────────────────────────
|
|
2239
4200
|
/**
|
|
@@ -2513,43 +4474,70 @@ export class RealtimeSessionRuntime {
|
|
|
2513
4474
|
return;
|
|
2514
4475
|
}
|
|
2515
4476
|
this.teardownInFlight = this.runTeardown(closeServerSession).finally(() => {
|
|
4477
|
+
if (this._savingRecording$.value) {
|
|
4478
|
+
this._savingRecording$.next(false);
|
|
4479
|
+
}
|
|
2516
4480
|
this.teardownInFlight = null;
|
|
2517
4481
|
});
|
|
2518
4482
|
await this.teardownInFlight;
|
|
2519
4483
|
}
|
|
2520
4484
|
/** The body of {@link teardown}; never called concurrently with itself. */
|
|
2521
4485
|
async runTeardown(closeServerSession) {
|
|
4486
|
+
// Arm BEFORE any await so a tab closed in the first milliseconds of teardown is still guarded.
|
|
4487
|
+
if (this.recorder) {
|
|
4488
|
+
this._savingRecording$.next(true);
|
|
4489
|
+
}
|
|
2522
4490
|
// First: stop asserting liveness. A pulse racing the close would re-stamp LastActiveAt on a
|
|
2523
4491
|
// session we are deliberately ending, leaving an Idle row the janitor then has to age out.
|
|
2524
4492
|
this.stopLivenessPulse();
|
|
2525
4493
|
this.teardownDelegationProgress();
|
|
4494
|
+
this.stopSessionEvents();
|
|
4495
|
+
this.clientDeadline.Clear();
|
|
2526
4496
|
// Channels first: flush any unsaved channel state WHILE the live session id is still
|
|
2527
4497
|
// set (the captured per-save id covers the race anyway), then dispose the plugins.
|
|
2528
4498
|
this.flushAllChannelSaves();
|
|
2529
4499
|
this.disposeChannels();
|
|
4500
|
+
// End the recording and START its keepalive terminal request now: stop the recorder (which flushes
|
|
4501
|
+
// the encoder) before anything else that awaits. A tab closed right after End gets only the pagehide
|
|
4502
|
+
// terminal, which carries packets already encoded; releasing the mic and disconnecting the driver can
|
|
4503
|
+
// take long enough for the page to die first, and the audio still inside the encoder would be lost.
|
|
4504
|
+
// It also goes out before the usage flush and the close, so it is on the wire if the page dies during
|
|
4505
|
+
// either. Best-effort — finishRecording never throws. Null when nothing was recorded.
|
|
4506
|
+
const recordingTerminal = await this.finishRecording();
|
|
4507
|
+
this.recordingStartedAtIso = null;
|
|
4508
|
+
// The camera and screen share go first: the camera runs on the controller closed next.
|
|
4509
|
+
this.closeCaptures();
|
|
2530
4510
|
// Defensive: stop the mic even when Connect never ran (the client also stops the
|
|
2531
4511
|
// tracks it was handed — track.stop() is idempotent).
|
|
4512
|
+
this.closeLocalMedia();
|
|
2532
4513
|
this.localStream?.getTracks().forEach(t => t.stop());
|
|
2533
4514
|
this.localStream = null;
|
|
2534
|
-
|
|
4515
|
+
this.microphoneMuted = false;
|
|
4516
|
+
// Hand the platform back whatever opening the microphone changed. Stopping the tracks is not
|
|
2535
4517
|
// the same thing: iOS, for instance, is put into a record-and-play audio category for the call,
|
|
2536
4518
|
// and leaving it there changes the route and volume behaviour of every sound the app makes
|
|
2537
|
-
// afterwards.
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
|
|
2541
|
-
|
|
2542
|
-
|
|
4519
|
+
// afterwards. A microphone still opening is left to its start, which releases it once the
|
|
4520
|
+
// opening returns ({@link microphoneLease}).
|
|
4521
|
+
const lease = this.microphoneLease;
|
|
4522
|
+
if (lease && !lease.Opening) {
|
|
4523
|
+
this.microphoneLease = null;
|
|
4524
|
+
await this.releaseMicrophone();
|
|
2543
4525
|
}
|
|
2544
4526
|
if (this.client) {
|
|
2545
|
-
|
|
2546
|
-
|
|
4527
|
+
// A stranded Active session is worse than a noisy socket: a driver that throws on Disconnect
|
|
4528
|
+
// (socket already gone) must not stop the server session from being closed below.
|
|
4529
|
+
try {
|
|
4530
|
+
await this.client.Disconnect();
|
|
4531
|
+
}
|
|
4532
|
+
catch (error) {
|
|
4533
|
+
console.error(`[RealtimeSession] Driver failed to disconnect (session ${this.agentSessionId}); continuing teardown:`, error);
|
|
4534
|
+
}
|
|
4535
|
+
finally {
|
|
4536
|
+
this.unwatchVideoSources();
|
|
4537
|
+
this.clearAgentVideo();
|
|
4538
|
+
this.client = null;
|
|
4539
|
+
}
|
|
2547
4540
|
}
|
|
2548
|
-
// Stop + upload the call recording WHILE the live session id is still set (the file is
|
|
2549
|
-
// attached to it). Best-effort and never blocks teardown — stopAndUploadRecording swallows
|
|
2550
|
-
// its own errors. No-op when nothing was recorded.
|
|
2551
|
-
await this.stopAndUploadRecording(this.agentSessionId);
|
|
2552
|
-
this.recordingStartedAtIso = null;
|
|
2553
4541
|
// Final usage flush WHILE the live session id is still set (the relay mutation also
|
|
2554
4542
|
// accepts a Closed session, so ordering vs. CloseAgentSession is belt-and-braces).
|
|
2555
4543
|
if (this.usageFlushTimer) {
|
|
@@ -2561,18 +4549,28 @@ export class RealtimeSessionRuntime {
|
|
|
2561
4549
|
if (closeServerSession && this.agentSessionId) {
|
|
2562
4550
|
await this.closeServerSession(this.agentSessionId);
|
|
2563
4551
|
}
|
|
4552
|
+
// Wait for the terminal's ack only AFTER the close: nothing after an aborted await ever runs, so
|
|
4553
|
+
// the small, critical close must never queue behind the recording (#5195). Bounded by
|
|
4554
|
+
// RECORDING_TERMINAL_ACK_TIMEOUT_MS; SavingRecording$ stays up until it settles.
|
|
4555
|
+
if (recordingTerminal) {
|
|
4556
|
+
await this.awaitRecordingTerminal(recordingTerminal);
|
|
4557
|
+
}
|
|
2564
4558
|
// Capture the session id BEFORE we null it so the lifecycle emit carries it.
|
|
2565
4559
|
// Skip emitting when there was no live session (defensive — teardown is safe
|
|
2566
4560
|
// to call without an active session).
|
|
2567
4561
|
const closedSessionId = this.agentSessionId;
|
|
2568
4562
|
this.agentSessionId = null;
|
|
4563
|
+
this.sessionConversationId = null;
|
|
2569
4564
|
this.narrationTemplate = null;
|
|
2570
4565
|
this.clientToolHandlers.clear();
|
|
2571
4566
|
this._modelName$.next(null);
|
|
4567
|
+
if (this._avatarNotice$.value !== null) {
|
|
4568
|
+
this._avatarNotice$.next(null);
|
|
4569
|
+
}
|
|
2572
4570
|
this.SetMinimized(false);
|
|
2573
4571
|
this._active$.next(false);
|
|
2574
4572
|
if (this._connectionState$.value !== 'error') {
|
|
2575
|
-
this.
|
|
4573
|
+
this.setConnectionState('closed');
|
|
2576
4574
|
}
|
|
2577
4575
|
// Surface generic session-ended for the conversations runtime bridge.
|
|
2578
4576
|
// `closeServerSession=true` means the user explicitly called EndRealtimeSession;
|
|
@@ -2584,6 +4582,18 @@ export class RealtimeSessionRuntime {
|
|
|
2584
4582
|
});
|
|
2585
4583
|
}
|
|
2586
4584
|
}
|
|
4585
|
+
/**
|
|
4586
|
+
* Calls the host's {@link IRealtimeMediaHost.ReleaseMicrophone}. Best-effort by contract: a failure
|
|
4587
|
+
* is logged and never thrown, because it must never block ending a call.
|
|
4588
|
+
*/
|
|
4589
|
+
async releaseMicrophone() {
|
|
4590
|
+
try {
|
|
4591
|
+
await this.mediaHost.ReleaseMicrophone?.();
|
|
4592
|
+
}
|
|
4593
|
+
catch (error) {
|
|
4594
|
+
console.error('[RealtimeSession] Media host failed to release the microphone:', error);
|
|
4595
|
+
}
|
|
4596
|
+
}
|
|
2587
4597
|
/** Calls the `CloseAgentSession` mutation (provisioned in P4b). */
|
|
2588
4598
|
async closeServerSession(agentSessionId) {
|
|
2589
4599
|
try {
|
|
@@ -2609,13 +4619,18 @@ export class RealtimeSessionRuntime {
|
|
|
2609
4619
|
this.pendingUserCaption = '';
|
|
2610
4620
|
this.hasActiveInterimUserCaption = false;
|
|
2611
4621
|
this.SetMinimized(false);
|
|
2612
|
-
this.
|
|
2613
|
-
|
|
4622
|
+
this.stopShardFlushing();
|
|
4623
|
+
// Defensive: teardown already ended any previous recording; a stray pipeline must not keep sending.
|
|
4624
|
+
this.recordingShards?.Queue.Close();
|
|
4625
|
+
this.recordingShards = null;
|
|
2614
4626
|
this.recorder = null;
|
|
2615
4627
|
this.recordingStartedAtIso = null;
|
|
2616
4628
|
this.currentTurnStartMs = null;
|
|
2617
4629
|
this.turnAudioStartCaptured = false;
|
|
4630
|
+
this.userSpeechStartMs = null;
|
|
4631
|
+
this.userSpeechStartSuperseded = false;
|
|
2618
4632
|
this.usedChannelNames.clear();
|
|
4633
|
+
this.microphoneMuted = false;
|
|
2619
4634
|
}
|
|
2620
4635
|
/** The GraphQL provider used for relay mutations. */
|
|
2621
4636
|
gql() {
|