@crestapps/ai-chat-ui 2.0.0-preview.154 → 2.0.0-preview.162

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,2236 @@
1
+ /*
2
+ ** NOTE: This file is generated by Gulp and should not be edited directly!
3
+ ** Any changes made directly to this file will be overwritten next time its asset group is processed by Gulp.
4
+ */
5
+
6
+ /*
7
+ * Shared realtime (speech-to-speech) audio controller for interaction-style chat surfaces.
8
+ *
9
+ * Encapsulates the browser side of a realtime voice conversation: PCM16 mic capture and streaming,
10
+ * PCM playback with volume, the half-duplex echo guard, push-to-talk, the per-device audio settings
11
+ * popover, and switching the input between text and audio-only when a realtime deployment is selected.
12
+ *
13
+ * It is host-agnostic: the host (an MVC inline script or the Blazor chat-interaction Vue app) supplies the
14
+ * SignalR connection, a sendStart callback (so each surface can address its own hub method/identifier),
15
+ * element selectors, and a few display hooks. Reused so realtime lives in one place, not one copy per host.
16
+ *
17
+ * Usage:
18
+ * var controller = window.CoreAIRealtime.attach({
19
+ * connection: conn,
20
+ * ensureConnected: function () { return startPromise; }, // () => Promise<any>
21
+ * sendStart: function (subject, voice, language, silenceMs, vadThreshold) { ... },
22
+ * voiceName: 'alloy',
23
+ * capableDeployments: ['gpt-realtime'],
24
+ * realtimeEnabled: true,
25
+ * selectors: { realtimeButton, input, sendButton, micButton, conversationButton, deploymentSelect },
26
+ * onActivate: function () { ... }, // realtime session started (host: show transcripts)
27
+ * onDeactivate: function () { ... }, // realtime session stopped
28
+ * onEnterRealtimeMode: function () { ... } // switched to audio-only (host: stop any STT recording)
29
+ * });
30
+ * // then route audio to it: controller.receivePcm(bytes) // from ReceiveAudioChunk 'audio/pcm'
31
+ */
32
+ (function (window, document) {
33
+ 'use strict';
34
+
35
+ var REALTIME_SAMPLE_RATE = 24000;
36
+
37
+ // --- Microphone gate --------------------------------------------------------------------------------------
38
+ //
39
+ // The outbound mic is silenced unless the user is genuinely speaking, so the model never receives silence,
40
+ // room noise, or the assistant's echo — which is what stopped it answering itself and stopped the input
41
+ // transcriber hallucinating phantom "Thank you" turns.
42
+ //
43
+ // - push-to-talk: open only while the key is held;
44
+ // - barge-in on: open whenever the user speaks — even over the assistant (interrupts);
45
+ // - barge-in off: open only when the user speaks and the assistant is silent (wait your turn).
46
+ //
47
+ // There are no user-facing modes or thresholds: everything the gate needs it measures for itself. It tracks
48
+ // the room's noise floor, and — the part that makes open speakers work — it learns how loud the assistant's
49
+ // own voice comes back into the microphone after echo cancellation (the echo return level). While the
50
+ // assistant is audible, only a voice clearly louder than that expected echo counts as the user interrupting;
51
+ // with a headset the expected echo is the room floor and interruptions are cheap, with loud speakers next to
52
+ // the microphone it is high and the gate simply waits its turn. Echo cancellation is still the first line of
53
+ // defence; this is what keeps the model from hearing itself when cancellation is not enough.
54
+ //
55
+ // It runs in an AudioWorklet rather than on a requestAnimationFrame loop. rAF is paused for hidden documents,
56
+ // so the old gate froze in whatever state it held the moment the user switched tabs: stuck closed meant they
57
+ // were never heard again. The audio thread keeps running regardless of visibility, sees every sample instead
58
+ // of a 10 ms window sampled 60 times a second, and can hold a look-ahead delay so the gate is already open
59
+ // when the first consonant arrives.
60
+ //
61
+ // Exported as window.CoreAIRealtime.createMicGate so both chat clients share one implementation.
62
+ var REALTIME_GATE_LOOKAHEAD_MS = 80;
63
+
64
+ /*
65
+ * The gate's decision, as a pure function of the current state and one block of measurements.
66
+ *
67
+ * It lives on its own for two reasons. It has to run inside an AudioWorklet, which cannot close over module
68
+ * scope — so it is stringified into the worklet below rather than duplicated there. And it is the part worth
69
+ * testing exhaustively: every one of this gate's historical failures (a quiet microphone never opening it, a
70
+ * loud room's echo opening it, one sentence chopped into several turns) is a rule in here, and each is far
71
+ * easier to pin down with numbers than with audio.
72
+ *
73
+ * `state` is mutated and returned. Callers create it with createGateState().
74
+ *
75
+ * input: { micDb, assistantDb, nowMs, mode }
76
+ * mode: { pushToTalk, pttActive, bargeIn, speechHangoverMs }
77
+ */
78
+ function coreAiGateDecide(state, input) {
79
+ var micDb = input.micDb;
80
+ var assistantDb = typeof input.assistantDb === 'number' ? input.assistantDb : -100;
81
+ var nowMs = input.nowMs;
82
+ var mode = input.mode || {};
83
+
84
+ // Elapsed time since the previous decision, so the time constants below mean the same thing whether the
85
+ // gate runs per 128-sample block on the audio thread or on a 20 ms timer.
86
+ var dtMs = state.lastMs === null ? 0 : Math.max(0, Math.min(50, nowMs - state.lastMs));
87
+ state.lastMs = nowMs;
88
+
89
+ // "The assistant is speaking" must survive the gaps between its words. Without this hangover the
90
+ // half-duplex gate re-opened in every inter-word pause and fed the echo tail straight back to the model.
91
+ if (assistantDb > -50) {
92
+ state.assistantUntil = nowMs + 400;
93
+ }
94
+ var assistantSpeaking = nowMs < state.assistantUntil;
95
+
96
+ // Peak-held assistant level. The echo of a word reaches the microphone 50-200 ms after the word was played,
97
+ // so the expected echo has to be judged against what the assistant was doing a moment ago, not against the
98
+ // instantaneous level (which is silent between its words). Decays at 20 dB/s.
99
+ if (assistantDb > state.assistantPeakDb) {
100
+ state.assistantPeakDb = assistantDb;
101
+ } else {
102
+ state.assistantPeakDb = Math.max(-100, state.assistantPeakDb - 0.02 * dtMs);
103
+ }
104
+
105
+ // Adaptive noise floor. It falls to a quieter room quickly, and rises only while the gate is shut and the
106
+ // assistant is silent — that is, only from audio we have judged is neither speech nor echo.
107
+ //
108
+ // Letting it rise during speech makes the gate close on the talker: the estimate climbs towards their own
109
+ // voice, the level stops clearing floor + margin, and the sentence is cut part-way through.
110
+ //
111
+ // Cold start. Digital silence is not a measurement: a capture pipeline delivers a few empty blocks before
112
+ // the microphone is live (and noise suppression emits exact zeros in a quiet headset), and seeding on
113
+ // those anchored the floor at the clamp and made the room itself look like speech. The first real block
114
+ // is not trustworthy either — it may well be the user already talking, and a floor seeded at their own
115
+ // level meant the gate never opened for them at all. So the seed is the first real block capped at a
116
+ // typical quiet-room level; the fast downward adaptation below then finds the real floor within a few
117
+ // hundred milliseconds. (Seeding from the quietest block instead was tried and rejected: block-to-block
118
+ // level varies by several dB even on steady room noise, so a floor set at the minimum let ordinary noise
119
+ // peaks open the gate.)
120
+ if (micDb <= -95) {
121
+ if (state.floorDb === null) {
122
+ state.open = false;
123
+ return state;
124
+ }
125
+ } else if (state.floorDb === null) {
126
+ state.floorDb = Math.min(micDb, -50);
127
+ }
128
+ if (micDb < state.floorDb) {
129
+ state.floorDb += (micDb - state.floorDb) * Math.min(1, 0.0075 * dtMs);
130
+ } else if (!assistantSpeaking) {
131
+ // Rising. While the gate is shut this is responsive (~2 s). While it is open the audio is presumed to
132
+ // be speech and the floor moves slowly (~10 s), because learning from the talker's own voice is what
133
+ // closed the gate part-way through their sentence — but not so slowly that a steady sound could hold
134
+ // the gate open indefinitely. Past 10 s the assumption is abandoned altogether: nobody speaks
135
+ // continuously for that long, so it is ambient noise and the floor tracks it normally.
136
+ var sustained = state.openSince !== null && nowMs - state.openSince > 10000;
137
+ var risingPerMs = state.open && !sustained ? 0.0001 : 0.00056;
138
+ state.floorDb += (micDb - state.floorDb) * Math.min(1, risingPerMs * dtMs);
139
+ }
140
+ if (state.floorDb < -75) {
141
+ state.floorDb = -75;
142
+ }
143
+
144
+ // Echo return level: how loud the assistant comes back into the microphone, relative to its own level,
145
+ // after echo cancellation has done what it can. Learned only while the assistant is audible and the gate
146
+ // is shut, i.e. from audio we have judged is not the user. It warms up fast for the first second so the
147
+ // very first reply is protected, then follows slowly so a user interrupting cannot be mistaken for echo
148
+ // in the time it takes to confirm them (below). It falls slowly too, so turning the volume down is
149
+ // noticed within a couple of seconds.
150
+ if (assistantSpeaking && !state.open && assistantDb > -50 && !mode.pushToTalk) {
151
+ var erl = micDb - state.assistantPeakDb;
152
+ if (state.echoReturnDb === null) {
153
+ state.echoReturnDb = erl;
154
+ } else {
155
+ var warm = state.echoLearnedMs < 1000;
156
+ var ratePerMs = erl > state.echoReturnDb ? warm ? 0.02 : 0.0012 : 0.0006;
157
+ state.echoReturnDb += (erl - state.echoReturnDb) * Math.min(1, ratePerMs * dtMs);
158
+ }
159
+ state.echoLearnedMs += dtMs;
160
+ }
161
+
162
+ // Where the assistant's echo is expected to sit right now, or far below anything if it has never been
163
+ // heard (a headset).
164
+ var expectedEchoDb = state.echoReturnDb === null ? -100 : state.assistantPeakDb + state.echoReturnDb;
165
+ var openThreshold, holdThreshold;
166
+ if (assistantSpeaking) {
167
+ // Interrupting: the voice must clearly beat both the room and the expected echo, and be real speech
168
+ // in absolute terms — echo residual can sit well above a very quiet room's floor without being
169
+ // anywhere near a voice.
170
+ openThreshold = Math.max(state.floorDb + 12, expectedEchoDb + 8, -45);
171
+ holdThreshold = Math.max(state.floorDb + 3, expectedEchoDb + 3);
172
+ } else {
173
+ openThreshold = state.floorDb + 9;
174
+ holdThreshold = state.floorDb + 3;
175
+ }
176
+ var hangoverMs = mode.speechHangoverMs || 2000;
177
+ if (micDb > openThreshold) {
178
+ if (assistantSpeaking && !state.open) {
179
+ // An interruption has to be sustained for a moment before it counts. A single loud block over the
180
+ // assistant is far more often a click, a cough or an echo peak than the user; the look-ahead delay
181
+ // keeps the opening of a genuine interruption from being lost while we wait.
182
+ if (state.aboveSince === null) {
183
+ state.aboveSince = nowMs;
184
+ }
185
+ if (nowMs - state.aboveSince >= 250) {
186
+ state.speakUntil = nowMs + hangoverMs;
187
+ }
188
+ } else {
189
+ // The hangover must outlast the provider's end-of-turn detection. The gate emits digital silence
190
+ // when it closes, so whichever of the two expires first is what actually ends the user's turn —
191
+ // and a gate that closes first cuts people off mid-sentence and answers half a question.
192
+ state.speakUntil = nowMs + hangoverMs;
193
+ }
194
+ } else {
195
+ state.aboveSince = null;
196
+ if (state.open && micDb > holdThreshold) {
197
+ // Hysteresis: once open, stay open until the level really settles back towards the floor, so one
198
+ // sentence is not chopped into several provider turns.
199
+ state.speakUntil = Math.max(state.speakUntil, nowMs + 250);
200
+ }
201
+ }
202
+ var userActive = nowMs < state.speakUntil;
203
+ if (!assistantSpeaking && state.wasAssistantSpeaking) {
204
+ state.listenGraceUntil = nowMs + 1500;
205
+ }
206
+ state.wasAssistantSpeaking = assistantSpeaking;
207
+ state.assistantSpeaking = assistantSpeaking;
208
+ if (mode.pushToTalk) {
209
+ state.open = !!mode.pttActive;
210
+ } else if (mode.bargeIn) {
211
+ state.open = userActive;
212
+ } else {
213
+ state.open = !assistantSpeaking && (userActive || nowMs < state.listenGraceUntil);
214
+ }
215
+ if (state.open && state.openSince === null) {
216
+ state.openSince = nowMs;
217
+ } else if (!state.open) {
218
+ state.openSince = null;
219
+ }
220
+ return state;
221
+ }
222
+ function createGateState() {
223
+ return {
224
+ // Null until the first measurement, so the floor starts from the room rather than from a guess.
225
+ floorDb: null,
226
+ lastMs: null,
227
+ speakUntil: 0,
228
+ assistantUntil: 0,
229
+ assistantPeakDb: -100,
230
+ // Null until the assistant has been heard while the gate was shut (never, with a headset).
231
+ echoReturnDb: null,
232
+ echoLearnedMs: 0,
233
+ aboveSince: null,
234
+ listenGraceUntil: 0,
235
+ wasAssistantSpeaking: false,
236
+ assistantSpeaking: false,
237
+ open: false,
238
+ openSince: null
239
+ };
240
+ }
241
+
242
+ // Root-mean-square of a block, in dBFS. RMS (not peak) tracks perceived loudness, and dB is the only scale on
243
+ // which a fixed "N above the noise floor" rule behaves the same in a quiet room and a loud one.
244
+ function coreAiRmsDb(block) {
245
+ if (!block || !block.length) {
246
+ return -100;
247
+ }
248
+ var sum = 0;
249
+ for (var i = 0; i < block.length; i++) {
250
+ sum += block[i] * block[i];
251
+ }
252
+ var rms = Math.sqrt(sum / block.length);
253
+ return rms > 1e-7 ? 20 * Math.log10(rms) : -100;
254
+ }
255
+
256
+ // The gate decision itself, as an AudioWorkletProcessor. Registered from a Blob URL so it needs no separate
257
+ // asset file (and no host-specific path configuration). The decision and level functions are stringified in
258
+ // rather than rewritten, so the worklet and the fallback below can never disagree.
259
+ var REALTIME_GATE_WORKLET_SOURCE = ['var decide = ' + coreAiGateDecide.toString() + ';', 'var newState = ' + createGateState.toString() + ';', 'var rmsDb = ' + coreAiRmsDb.toString() + ';', 'class CoreAiMicGateProcessor extends AudioWorkletProcessor {', ' constructor(options) {', ' super();', ' var o = (options && options.processorOptions) || {};',
260
+ // The gate reads the live signal but emits it delayed, so by the time a word's first sample leaves here
261
+ // the gate has already seen it and opened. Without this the initial consonant of every utterance was
262
+ // clipped, which is exactly what degrades transcription and the model's understanding.
263
+ ' this.delay = new Float32Array(Math.max(1, Math.round(sampleRate * (o.lookaheadMs || 80) / 1000)));', ' this.delayPos = 0;', ' this.gain = 0;', ' this.mode = o.mode || {};', ' this.state = newState();', ' this.lastPost = 0;', ' this.port.onmessage = function (e) { this.mode = e.data || {}; }.bind(this);', ' }', ' process(inputs, outputs) {', ' var mic = (inputs[0] && inputs[0][0]) || null;', ' var assistant = (inputs[1] && inputs[1][0]) || null;', ' var out = (outputs[0] && outputs[0][0]) || null;', ' if (!out) { return true; }', ' var micDb = rmsDb(mic);', ' decide(this.state, { micDb: micDb, assistantDb: rmsDb(assistant), nowMs: currentTime * 1000, mode: this.mode });', ' var target = this.state.open ? 1 : 0;',
264
+ // Ramp over ~10 ms rather than switching instantly, so opening and closing never clicks.
265
+ ' var step = 1 / (sampleRate * 0.01);', ' var delay = this.delay;', ' var pos = this.delayPos;', ' var len = delay.length;', ' for (var i = 0; i < out.length; i++) {', ' var delayed = delay[pos];', ' delay[pos] = mic ? mic[i] : 0;', ' pos = (pos + 1) % len;', ' if (this.gain < target) { this.gain = Math.min(target, this.gain + step); }', ' else if (this.gain > target) { this.gain = Math.max(target, this.gain - step); }', ' out[i] = delayed * this.gain;', ' }', ' this.delayPos = pos;', ' if (currentTime * 1000 - this.lastPost > 100) {', ' this.lastPost = currentTime * 1000;', ' this.port.postMessage({ micDb: micDb, floorDb: this.state.floorDb, echoReturnDb: this.state.echoReturnDb, open: this.state.open, assistantSpeaking: this.state.assistantSpeaking });', ' }', ' return true;', ' }', '}', 'registerProcessor("coreai-mic-gate", CoreAiMicGateProcessor);'].join('\n');
266
+
267
+ /*
268
+ * Builds a microphone gate for a captured stream.
269
+ *
270
+ * createMicGate(stream, mode) => Promise<{
271
+ * track, // the gated track to send (or the raw mic when Web Audio is missing)
272
+ * setMode(mode), // { pushToTalk, pttActive, bargeIn, speechHangoverMs }
273
+ * attachAssistantStream(stream),// so the gate knows when the assistant is audible
274
+ * getLevel(), // last measurement, for a mic meter
275
+ * stop()
276
+ * }>
277
+ */
278
+ function createMicGate(rawStream, mode) {
279
+ var rawTrack = rawStream && rawStream.getAudioTracks && rawStream.getAudioTracks()[0] || null;
280
+ var state = {
281
+ ctx: null,
282
+ node: null,
283
+ micSource: null,
284
+ remoteSource: null,
285
+ dest: null,
286
+ timer: 0,
287
+ level: null,
288
+ fallback: null,
289
+ track: rawTrack,
290
+ stopped: false
291
+ };
292
+ var gate = {
293
+ track: rawTrack,
294
+ // True when the gate is running on the audio thread. The fallback still gates, but on a timer that a
295
+ // hidden tab throttles, so hosts and tests can tell the two apart.
296
+ usingWorklet: false,
297
+ getLevel: function getLevel() {
298
+ return state.level;
299
+ },
300
+ setMode: function setMode(next) {
301
+ mode = next || mode;
302
+ if (state.node && state.node.port) {
303
+ try {
304
+ state.node.port.postMessage(mode);
305
+ } catch (e) {}
306
+ }
307
+ if (state.fallback) {
308
+ state.fallback.mode = mode;
309
+ }
310
+ },
311
+ attachAssistantStream: function attachAssistantStream(remoteStream) {
312
+ attachAssistant(state, remoteStream);
313
+ },
314
+ stop: function stop() {
315
+ state.stopped = true;
316
+ if (state.timer) {
317
+ try {
318
+ window.clearInterval(state.timer);
319
+ } catch (e) {}
320
+ state.timer = 0;
321
+ }
322
+ if (state.node) {
323
+ try {
324
+ state.node.port.onmessage = null;
325
+ state.node.disconnect();
326
+ } catch (e) {}
327
+ state.node = null;
328
+ }
329
+ if (state.micSource) {
330
+ try {
331
+ state.micSource.disconnect();
332
+ } catch (e) {}
333
+ state.micSource = null;
334
+ }
335
+ if (state.remoteSource) {
336
+ try {
337
+ state.remoteSource.disconnect();
338
+ } catch (e) {}
339
+ state.remoteSource = null;
340
+ }
341
+ if (state.dest) {
342
+ try {
343
+ state.dest.disconnect();
344
+ } catch (e) {}
345
+ state.dest = null;
346
+ }
347
+ if (state.ctx) {
348
+ try {
349
+ state.ctx.close();
350
+ } catch (e) {}
351
+ state.ctx = null;
352
+ }
353
+ state.fallback = null;
354
+ }
355
+ };
356
+ var AudioCtor = window.AudioContext || window.webkitAudioContext;
357
+ if (!AudioCtor || !rawTrack) {
358
+ return Promise.resolve(gate);
359
+ }
360
+ try {
361
+ state.ctx = new AudioCtor();
362
+ // A context created under a strict autoplay policy starts suspended, and a suspended gate emits
363
+ // nothing at all — the model would simply never hear the user.
364
+ if (state.ctx.state === 'suspended' && typeof state.ctx.resume === 'function') {
365
+ state.ctx.resume()["catch"](function () {});
366
+ }
367
+ } catch (err) {
368
+ state.ctx = null;
369
+ return Promise.resolve(gate);
370
+ }
371
+ var blobUrl = null;
372
+ if (state.ctx.audioWorklet && typeof state.ctx.audioWorklet.addModule === 'function') {
373
+ try {
374
+ blobUrl = URL.createObjectURL(new Blob([REALTIME_GATE_WORKLET_SOURCE], {
375
+ type: 'application/javascript'
376
+ }));
377
+ } catch (err) {
378
+ blobUrl = null;
379
+ }
380
+ }
381
+ if (!blobUrl) {
382
+ gate.track = buildFallbackGate(state, rawStream, rawTrack, mode);
383
+ return Promise.resolve(gate);
384
+ }
385
+ return state.ctx.audioWorklet.addModule(blobUrl).then(function () {
386
+ try {
387
+ URL.revokeObjectURL(blobUrl);
388
+ } catch (e) {}
389
+ if (state.stopped) {
390
+ return gate;
391
+ }
392
+ var node = new AudioWorkletNode(state.ctx, 'coreai-mic-gate', {
393
+ numberOfInputs: 2,
394
+ numberOfOutputs: 1,
395
+ outputChannelCount: [1],
396
+ processorOptions: {
397
+ lookaheadMs: REALTIME_GATE_LOOKAHEAD_MS,
398
+ mode: mode
399
+ }
400
+ });
401
+ state.node = node;
402
+ node.port.onmessage = function (e) {
403
+ state.level = e.data || null;
404
+ // Also published on the module for support diagnostics (a host can read it from the console
405
+ // without holding a reference to the gate).
406
+ window.CoreAIRealtime.lastGateLevel = state.level;
407
+ };
408
+ state.micSource = state.ctx.createMediaStreamSource(rawStream);
409
+ state.micSource.connect(node, 0, 0);
410
+ state.dest = state.ctx.createMediaStreamDestination();
411
+ node.connect(state.dest);
412
+ if (state.pendingRemoteStream) {
413
+ attachAssistant(state, state.pendingRemoteStream);
414
+ }
415
+ gate.track = state.dest.stream.getAudioTracks()[0] || rawTrack;
416
+ gate.usingWorklet = true;
417
+ return gate;
418
+ })["catch"](function (err) {
419
+ try {
420
+ URL.revokeObjectURL(blobUrl);
421
+ } catch (e) {}
422
+ if (window.console && console.warn) {
423
+ console.warn('The realtime microphone gate worklet could not be loaded; using the simpler gate.', err);
424
+ }
425
+ gate.track = buildFallbackGate(state, rawStream, rawTrack, mode);
426
+ return gate;
427
+ });
428
+ }
429
+ function attachAssistant(state, remoteStream) {
430
+ state.pendingRemoteStream = remoteStream || null;
431
+ if (!state.ctx || !remoteStream || state.stopped) {
432
+ return;
433
+ }
434
+ try {
435
+ if (state.remoteSource) {
436
+ try {
437
+ state.remoteSource.disconnect();
438
+ } catch (e) {}
439
+ }
440
+ var src = state.ctx.createMediaStreamSource(remoteStream);
441
+ state.remoteSource = src;
442
+ if (state.node) {
443
+ src.connect(state.node, 0, 1);
444
+ return;
445
+ }
446
+ if (state.fallback) {
447
+ var analyser = state.ctx.createAnalyser();
448
+ analyser.fftSize = 2048;
449
+ src.connect(analyser);
450
+ state.fallback.remoteAnalyser = analyser;
451
+ state.fallback.remoteData = new Float32Array(analyser.fftSize);
452
+ }
453
+ } catch (e) {
454
+ // Assistant-level detection unavailable; the gate still opens on user speech.
455
+ }
456
+ }
457
+
458
+ // Fallback for browsers without AudioWorklet: the same decision on a timer instead of the audio thread. A
459
+ // timer is throttled in background tabs — worse than the worklet, better than nothing — and there is no
460
+ // look-ahead here, so utterance onsets are still clipped.
461
+ function buildFallbackGate(state, rawStream, rawTrack, mode) {
462
+ try {
463
+ var ctx = state.ctx;
464
+ var source = ctx.createMediaStreamSource(rawStream);
465
+ state.micSource = source;
466
+ var analyser = ctx.createAnalyser();
467
+ analyser.fftSize = 2048;
468
+ source.connect(analyser);
469
+ var gain = ctx.createGain();
470
+ gain.gain.value = 0;
471
+ source.connect(gain);
472
+ var dest = ctx.createMediaStreamDestination();
473
+ state.dest = dest;
474
+ gain.connect(dest);
475
+ var f = {
476
+ mode: mode,
477
+ remoteAnalyser: null,
478
+ remoteData: null
479
+ };
480
+ state.fallback = f;
481
+ if (state.pendingRemoteStream) {
482
+ attachAssistant(state, state.pendingRemoteStream);
483
+ }
484
+ var data = new Float32Array(analyser.fftSize);
485
+
486
+ // The same decision the worklet runs, on a timer instead of the audio thread. The floor adapts faster
487
+ // here only because this loop runs ~50x/s rather than ~375x/s; everything else is shared.
488
+ var gateState = createGateState();
489
+ state.timer = window.setInterval(function () {
490
+ analyser.getFloatTimeDomainData(data);
491
+ var micDb = coreAiRmsDb(data);
492
+ var assistantDb = -100;
493
+ if (f.remoteAnalyser && f.remoteData) {
494
+ f.remoteAnalyser.getFloatTimeDomainData(f.remoteData);
495
+ assistantDb = coreAiRmsDb(f.remoteData);
496
+ }
497
+ var nowMs = window.performance && performance.now ? performance.now() : Date.now();
498
+ coreAiGateDecide(gateState, {
499
+ micDb: micDb,
500
+ assistantDb: assistantDb,
501
+ nowMs: nowMs,
502
+ mode: f.mode || {}
503
+ });
504
+ state.level = {
505
+ micDb: micDb,
506
+ floorDb: gateState.floorDb,
507
+ open: gateState.open,
508
+ assistantSpeaking: gateState.assistantSpeaking
509
+ };
510
+ var target = gateState.open ? 1 : 0;
511
+ try {
512
+ gain.gain.setTargetAtTime(target, ctx.currentTime, 0.015);
513
+ } catch (e) {
514
+ gain.gain.value = target;
515
+ }
516
+ }, 20);
517
+ return dest.stream.getAudioTracks()[0] || rawTrack;
518
+ } catch (err) {
519
+ // Web Audio unavailable; send the raw mic (ungated).
520
+ return rawTrack;
521
+ }
522
+ }
523
+
524
+ /*
525
+ * Routes an audio element to the device the browser's echo canceller couples to, or to an explicitly chosen
526
+ * one. Preference order:
527
+ * 1. The caller's chosen device, when they picked one.
528
+ * 2. The "communications" sink (Chromium). Rendering here doesn't just pick a device — it puts the browser's
529
+ * audio pipeline in communications mode, which is what actually couples playback with the microphone's
530
+ * echo canceller, and what keeps full duplex stable across turns.
531
+ * 3. The concrete device the "default" alias points at, matched by groupId.
532
+ * If neither alias exists — Firefox lists only concrete outputs — nothing is set and the browser's own default
533
+ * stands. Guessing there routed the assistant to whatever was enumerated first (often an HDMI monitor), so it
534
+ * appeared silent.
535
+ */
536
+ function routeOutputToPreferredDevice(el, chosenDeviceId) {
537
+ if (!el || typeof el.setSinkId !== 'function') {
538
+ return;
539
+ }
540
+ if (chosenDeviceId) {
541
+ try {
542
+ el.setSinkId(chosenDeviceId)["catch"](function () {});
543
+ } catch (e) {}
544
+ return;
545
+ }
546
+ if (!navigator.mediaDevices || !navigator.mediaDevices.enumerateDevices) {
547
+ return;
548
+ }
549
+ navigator.mediaDevices.enumerateDevices().then(function (devices) {
550
+ var outputs = devices.filter(function (d) {
551
+ return d.kind === 'audiooutput';
552
+ });
553
+ if (!outputs.length) {
554
+ return;
555
+ }
556
+ var target = '';
557
+ for (var i = 0; i < outputs.length; i++) {
558
+ if (outputs[i].deviceId === 'communications') {
559
+ target = 'communications';
560
+ break;
561
+ }
562
+ }
563
+ if (!target) {
564
+ var defaultEntry = null;
565
+ for (var d = 0; d < outputs.length; d++) {
566
+ if (outputs[d].deviceId === 'default') {
567
+ defaultEntry = outputs[d];
568
+ break;
569
+ }
570
+ }
571
+ if (defaultEntry && defaultEntry.groupId) {
572
+ for (var j = 0; j < outputs.length; j++) {
573
+ if (outputs[j].deviceId !== 'default' && outputs[j].deviceId !== 'communications' && outputs[j].groupId === defaultEntry.groupId) {
574
+ target = outputs[j].deviceId;
575
+ break;
576
+ }
577
+ }
578
+ }
579
+ }
580
+ if (target) {
581
+ try {
582
+ el.setSinkId(target)["catch"](function () {});
583
+ } catch (e) {}
584
+ }
585
+ })["catch"](function () {});
586
+ }
587
+
588
+ // Autoplay policies (Firefox's "Block audio and video", locked-down profiles) can reject playback even after a
589
+ // click, and a rejected play() is silent — the assistant simply never becomes audible. Ask explicitly and
590
+ // report the failure instead of leaving the user with a mute conversation.
591
+ function ensurePlayback(el, onBlocked) {
592
+ if (!el || typeof el.play !== 'function') {
593
+ return;
594
+ }
595
+ var played = el.play();
596
+ if (played && typeof played["catch"] === 'function') {
597
+ played["catch"](function (err) {
598
+ if (typeof onBlocked === 'function') {
599
+ onBlocked(err);
600
+ }
601
+ if (window.console && console.warn) {
602
+ console.warn('The browser blocked realtime audio playback. Interact with the page to enable audio.', err);
603
+ }
604
+ });
605
+ }
606
+ }
607
+ function attach(opts) {
608
+ opts = opts || {};
609
+ var connection = opts.connection;
610
+ var sel = opts.selectors || {};
611
+ var realtimeVoiceName = opts.voiceName || '';
612
+ var realtimeCapableDeployments = (opts.capableDeployments || []).map(function (n) {
613
+ return (n || '').toLowerCase();
614
+ });
615
+ var isRealtimeMode = !!opts.realtimeEnabled;
616
+ var noop = function noop() {};
617
+ var onActivate = opts.onActivate || noop;
618
+ var onDeactivate = opts.onDeactivate || noop;
619
+ var onEnterRealtimeMode = opts.onEnterRealtimeMode || noop;
620
+ // The server tells us a user utterance was captured before it has been transcribed, so the host can show a
621
+ // placeholder in the right position; onUserTurnDropped removes it when the utterance produced nothing.
622
+ var onUserTurnPending = opts.onUserTurnPending || noop;
623
+ var onUserTurnDropped = opts.onUserTurnDropped || noop;
624
+ // Hosts are localizable elsewhere, so let them override the handful of strings this module renders.
625
+ // Anything not supplied falls back to the English default.
626
+ var strings = opts.strings || {};
627
+ function localize(key, fallback) {
628
+ return typeof strings[key] === 'string' ? strings[key] : fallback;
629
+ }
630
+ var ensureConnected = opts.ensureConnected || function () {
631
+ return Promise.resolve(true);
632
+ };
633
+ var sendStart = opts.sendStart || noop;
634
+ // Optional: resolve the assistant voice live at session start (e.g. from a settings picker) so it
635
+ // reflects the current selection rather than the value captured when the controller was attached.
636
+ var getVoiceName = opts.getVoiceName || null;
637
+ // Optional server-relay WebRTC transport: when the server advertises it and the callback is supplied, the
638
+ // controller uses WebRTC (real acoustic echo cancellation) instead of the PCM-over-SignalR path.
639
+ var sendStartWebRtc = opts.sendStartWebRtc || null;
640
+ var webRtcEnabled = opts.webRtcEnabled === true && typeof window.RTCPeerConnection === 'function' && !!sendStartWebRtc;
641
+ // A host-supplied override; normally the servers are fetched from the hub right before the peer is created
642
+ // (see resolveIceServers) so configured TURN relays — and their short-lived credentials — are actually used.
643
+ var webRtcIceServers = Array.isArray(opts.webRtcIceServers) && opts.webRtcIceServers.length ? opts.webRtcIceServers : null;
644
+ var DEFAULT_ICE_SERVERS = [{
645
+ urls: 'stun:stun.l.google.com:19302'
646
+ }];
647
+ // Optional: notified on every state transition so the host can render "Connecting…", "Listening", etc.
648
+ var onStateChange = opts.onStateChange || noop;
649
+ var realtimeState = 'idle';
650
+ function q(name) {
651
+ var s = sel[name];
652
+ return s ? document.querySelector(s) : null;
653
+ }
654
+ var isRealtimeActive = false;
655
+ // Server-driven session lifecycle (see bindRealtimeLifecycleHandlers): the server tells us when the
656
+ // provider session is live, when the user interrupted, and when the session ended for any reason.
657
+ var realtimeLifecycleBound = false,
658
+ realtimeSessionReady = false,
659
+ realtimeEndedNotice = null,
660
+ realtimeCompatibilityNoticeEl = null,
661
+ realtimeStatusEl = null;
662
+ var realtimeStream = null,
663
+ realtimeAudioCtx = null,
664
+ realtimeSubject = null,
665
+ realtimeProcessor = null,
666
+ realtimeMicSource = null,
667
+ realtimeZeroGain = null,
668
+ realtimeGain = null,
669
+ realtimePlayHead = 0,
670
+ realtimeSources = [];
671
+ // WebRTC transport state.
672
+ var realtimeIsWebRtc = false,
673
+ realtimePc = null,
674
+ realtimeRemoteAudioEl = null,
675
+ realtimeWebRtcHandlersBound = false,
676
+ realtimeRemoteDescriptionSet = false,
677
+ realtimePendingIce = [];
678
+ // WebRTC connect-time fallback state: we decide WebRTC-vs-WebSocket only at connect time (never mid-session).
679
+ var realtimeWebRtcConnected = false,
680
+ realtimeWebRtcConnectTimer = null,
681
+ realtimeFellBack = false;
682
+ // How long to wait for the WebRTC peer to connect before dropping to the WebSocket transport. Once ICE
683
+ // gathering has finished without producing a relay candidate there is nothing left to wait for on a
684
+ // network that blocks UDP, so the wait is cut short rather than run to the full timeout.
685
+ var REALTIME_WEBRTC_CONNECT_TIMEOUT_MS = 8000;
686
+ var REALTIME_WEBRTC_NO_RELAY_TIMEOUT_MS = 5000;
687
+ // Fixed receive-side cushion for assistant audio (see pc.ontrack). The server paces the stream on a
688
+ // 20 ms clock, so this only has to cover network jitter, not provider burstiness.
689
+ var REALTIME_WEBRTC_JITTER_BUFFER_TARGET_MS = 150;
690
+ // Remembering a failed attempt for the rest of the browser session matters: without it, a user on a
691
+ // network where WebRTC cannot work pays the full connection timeout — and a second microphone prompt —
692
+ // at the start of every single conversation.
693
+ var REALTIME_WEBRTC_BLOCKED_KEY = 'coreai.realtime.webrtcBlocked';
694
+ var realtimeSawRelayCandidate = false;
695
+ // WebRTC half-duplex echo guard: the shared microphone gate (see createMicGate) watches both the mic and
696
+ // the assistant's remote audio and silences the outbound track unless the user is genuinely speaking, so
697
+ // the model cannot hear itself even when AEC underperforms against loud open-room speakers. It also
698
+ // enforces push-to-talk on the WebRTC mic track.
699
+ var realtimeWebRtcMicTrack = null,
700
+ realtimeGate = null,
701
+ realtimeGatePendingRemoteStream = null;
702
+
703
+ // Per-device audio preferences (interruptions, push-to-talk, volume, devices, language). They live in
704
+ // localStorage because they depend on the listener's hardware, not on the interaction.
705
+ var realtimeBargeIn = true,
706
+ realtimePushToTalk = false,
707
+ realtimePttActive = false,
708
+ realtimePttBound = false,
709
+ realtimePttKeyDown = null,
710
+ realtimePttKeyUp = null,
711
+ realtimePttButton = null,
712
+ realtimePttUiEl = null,
713
+ realtimeVolume = 1,
714
+ realtimeMicDeviceId = '',
715
+ realtimeOutputDeviceId = '',
716
+ realtimeLanguage = '',
717
+ realtimeAudioSettingsBuilt = false;
718
+
719
+ // The half-duplex echo tail on the WebSocket transport: how long the microphone stays muted after the
720
+ // scheduled end of the assistant's playback, covering the room's reverberation.
721
+ var REALTIME_WS_ECHO_HANGOVER_SEC = 0.5;
722
+
723
+ // Version 3 dropped the audio-setup presets, the voice-gate modes and the turn-detection sliders — all of
724
+ // that is measured or decided automatically now — and, the part that matters to anyone who used version 2,
725
+ // restores interruptions. The removed device-label guess had switched them off for every microphone whose
726
+ // name merely looked like a room microphone, and the earlier repair deliberately left that alone, so users
727
+ // found interruptions "off by default" with no idea why.
728
+ var REALTIME_PREFS_VERSION = 3;
729
+ function loadRealtimeAudioPrefs() {
730
+ var prefs = {
731
+ bargeIn: true,
732
+ pushToTalk: false,
733
+ volume: 1,
734
+ micDeviceId: '',
735
+ outputDeviceId: '',
736
+ language: '',
737
+ version: 0
738
+ };
739
+ try {
740
+ var raw = window.localStorage.getItem('coreai.realtime.audioPrefs');
741
+ if (raw) {
742
+ var parsed = JSON.parse(raw);
743
+ if (typeof parsed.bargeIn === 'boolean') {
744
+ prefs.bargeIn = parsed.bargeIn;
745
+ }
746
+ if (typeof parsed.pushToTalk === 'boolean') {
747
+ prefs.pushToTalk = parsed.pushToTalk;
748
+ }
749
+ if (typeof parsed.volume === 'number' && isFinite(parsed.volume)) {
750
+ prefs.volume = Math.min(1, Math.max(0, parsed.volume));
751
+ }
752
+ if (typeof parsed.micDeviceId === 'string') {
753
+ prefs.micDeviceId = parsed.micDeviceId;
754
+ }
755
+ if (typeof parsed.outputDeviceId === 'string') {
756
+ prefs.outputDeviceId = parsed.outputDeviceId;
757
+ }
758
+ if (typeof parsed.language === 'string') {
759
+ prefs.language = parsed.language;
760
+ }
761
+ if (typeof parsed.version === 'number') {
762
+ prefs.version = parsed.version;
763
+ }
764
+ }
765
+ } catch (err) {/* storage unavailable or blocked */}
766
+ return repairRealtimeAudioPrefs(prefs);
767
+ }
768
+ function repairRealtimeAudioPrefs(prefs) {
769
+ if (prefs.version === REALTIME_PREFS_VERSION) {
770
+ return prefs;
771
+ }
772
+ prefs.version = REALTIME_PREFS_VERSION;
773
+ prefs.bargeIn = true;
774
+ prefs.pushToTalk = false;
775
+ saveRealtimeAudioPrefs(prefs);
776
+ return prefs;
777
+ }
778
+ function saveRealtimeAudioPrefs(prefs) {
779
+ try {
780
+ window.localStorage.setItem('coreai.realtime.audioPrefs', JSON.stringify(prefs));
781
+ } catch (err) {}
782
+ }
783
+ function applyRealtimeAudioPrefs(prefs) {
784
+ realtimeBargeIn = prefs.bargeIn !== false;
785
+ realtimePushToTalk = !!prefs.pushToTalk;
786
+ realtimeVolume = typeof prefs.volume === 'number' ? prefs.volume : 1;
787
+ realtimeMicDeviceId = prefs.micDeviceId || '';
788
+ realtimeOutputDeviceId = prefs.outputDeviceId || '';
789
+ realtimeLanguage = prefs.language || '';
790
+ // The gate runs on the audio thread and cannot see these variables; push the change to it.
791
+ syncRealtimeGateMode();
792
+ if (realtimeGain) {
793
+ realtimeGain.gain.value = realtimeVolume;
794
+ }
795
+ // On WebRTC the assistant plays through the hidden <audio> element, so the volume slider must drive its
796
+ // volume directly. Lowering it also cuts the acoustic echo into the mic, which helps AEC with barge-in on.
797
+ if (realtimeRemoteAudioEl) {
798
+ realtimeRemoteAudioEl.volume = Math.max(0, Math.min(1, realtimeVolume));
799
+ }
800
+ }
801
+
802
+ // The push-to-talk handlers below run in the capture phase and swallow Space, which would otherwise eat
803
+ // every space character the user types elsewhere on the page while a voice session is active.
804
+ function isEditableTarget(target) {
805
+ if (!target || !target.tagName) {
806
+ return false;
807
+ }
808
+ var tag = target.tagName.toLowerCase();
809
+ return tag === 'input' || tag === 'textarea' || tag === 'select' || target.isContentEditable === true;
810
+ }
811
+
812
+ // Push-to-talk: hold Space to open the mic. Bound only while a realtime session is active so it never
813
+ // interferes with normal typing. Capture phase + preventDefault so Space never scrolls the page or
814
+ // activates a focused button (Space-up on a focused button fires a click that would toggle the session).
815
+ function attachRealtimePushToTalk() {
816
+ if (realtimePttBound) {
817
+ return;
818
+ }
819
+ realtimePttBound = true;
820
+ realtimePttActive = false;
821
+ realtimePttKeyDown = function realtimePttKeyDown(e) {
822
+ if (realtimePushToTalk && !isEditableTarget(e.target) && (e.code === 'Space' || e.key === ' ')) {
823
+ e.preventDefault();
824
+ if (!e.repeat) {
825
+ setRealtimePttActive(true);
826
+ }
827
+ }
828
+ };
829
+ realtimePttKeyUp = function realtimePttKeyUp(e) {
830
+ if (realtimePushToTalk && !isEditableTarget(e.target) && (e.code === 'Space' || e.key === ' ')) {
831
+ e.preventDefault();
832
+ setRealtimePttActive(false);
833
+ }
834
+ };
835
+ document.addEventListener('keydown', realtimePttKeyDown, true);
836
+ document.addEventListener('keyup', realtimePttKeyUp, true);
837
+ }
838
+ function detachRealtimePushToTalk() {
839
+ if (!realtimePttBound) {
840
+ return;
841
+ }
842
+ realtimePttBound = false;
843
+ realtimePttActive = false;
844
+ try {
845
+ document.removeEventListener('keydown', realtimePttKeyDown, true);
846
+ } catch (err) {}
847
+ try {
848
+ document.removeEventListener('keyup', realtimePttKeyUp, true);
849
+ } catch (err) {}
850
+ }
851
+ function setRealtimePttActive(active) {
852
+ realtimePttActive = active;
853
+ // The gate lives on the audio thread and cannot read this variable.
854
+ syncRealtimeGateMode();
855
+ if (realtimePttButton) {
856
+ realtimePttButton.classList.toggle('btn-danger', active);
857
+ realtimePttButton.classList.toggle('btn-outline-primary', !active);
858
+ realtimePttButton.innerHTML = active ? '<i class="bi bi-mic-fill me-1"></i> Listening…' : '<i class="bi bi-mic me-1"></i> Hold to talk';
859
+ }
860
+ }
861
+
862
+ // Shows a "hold to talk" control + hint while a push-to-talk realtime session is active.
863
+ function buildRealtimePttUi() {
864
+ var realtimeBtn = q('realtimeButton');
865
+ if (!realtimeBtn || realtimePttUiEl) {
866
+ return;
867
+ }
868
+ var coarse = window.matchMedia && window.matchMedia('(pointer: coarse)').matches;
869
+ var bar = document.createElement('div');
870
+ bar.className = 'text-center mt-2';
871
+ var btn = document.createElement('button');
872
+ btn.type = 'button';
873
+ btn.className = 'btn btn-outline-primary btn-sm';
874
+ btn.innerHTML = '<i class="bi bi-mic me-1"></i> Hold to talk';
875
+ var hint = document.createElement('div');
876
+ hint.className = 'form-text mt-1 mb-0';
877
+ hint.innerHTML = coarse ? 'Push-to-talk is on — press and hold the button to speak.' : 'Push-to-talk is on — hold <kbd>Space</kbd> or this button to speak.';
878
+ bar.appendChild(btn);
879
+ bar.appendChild(hint);
880
+ var row = realtimeBtn.parentElement;
881
+ if (row && row.parentElement) {
882
+ row.parentElement.appendChild(bar);
883
+ } else {
884
+ realtimeBtn.insertAdjacentElement('afterend', bar);
885
+ }
886
+ realtimePttButton = btn;
887
+ realtimePttUiEl = bar;
888
+ btn.addEventListener('pointerdown', function (e) {
889
+ e.preventDefault();
890
+ setRealtimePttActive(true);
891
+ });
892
+ btn.addEventListener('pointerup', function () {
893
+ setRealtimePttActive(false);
894
+ });
895
+ btn.addEventListener('pointerleave', function () {
896
+ setRealtimePttActive(false);
897
+ });
898
+ btn.addEventListener('pointercancel', function () {
899
+ setRealtimePttActive(false);
900
+ });
901
+ setRealtimePttActive(false);
902
+ }
903
+ function removeRealtimePttUi() {
904
+ realtimePttButton = null;
905
+ if (realtimePttUiEl) {
906
+ try {
907
+ realtimePttUiEl.remove();
908
+ } catch (err) {}
909
+ realtimePttUiEl = null;
910
+ }
911
+ }
912
+
913
+ // Builds a gear-triggered popover next to the realtime button with the per-device audio settings.
914
+ //
915
+ // It is deliberately short: a microphone, a speaker, the volume, the language, and two switches. Everything
916
+ // that used to be a knob here — echo margins, gate modes, turn-detection timing, audio-setup presets, an
917
+ // echo self-test — is measured or decided automatically now. A user should be able to press "Start
918
+ // speaking" and talk, in any room, without first understanding acoustics.
919
+ function setupRealtimeAudioSettings() {
920
+ var realtimeBtn = q('realtimeButton');
921
+ if (!realtimeBtn || realtimeAudioSettingsBuilt) {
922
+ return;
923
+ }
924
+ realtimeAudioSettingsBuilt = true;
925
+ var prefs = loadRealtimeAudioPrefs();
926
+ applyRealtimeAudioPrefs(prefs);
927
+ var wrap = document.createElement('span');
928
+ wrap.id = 'realtime-audio-settings';
929
+ wrap.style.position = 'relative';
930
+ wrap.style.display = 'inline-flex';
931
+ var gear = document.createElement('button');
932
+ gear.type = 'button';
933
+ gear.className = 'btn btn-outline-secondary';
934
+ gear.title = localize('settingsTitle', 'Voice settings');
935
+ gear.setAttribute('aria-label', localize('settingsTitle', 'Voice settings'));
936
+ gear.innerHTML = '<i class="bi bi-gear"></i>';
937
+ var langs = [['en', 'English'], ['es', 'Spanish'], ['fr', 'French'], ['de', 'German'], ['it', 'Italian'], ['pt', 'Portuguese'], ['nl', 'Dutch'], ['zh', 'Chinese'], ['ja', 'Japanese'], ['ko', 'Korean'], ['ar', 'Arabic'], ['hi', 'Hindi'], ['ru', 'Russian']];
938
+ var langOptions = '<option value=""' + (prefs.language === '' ? ' selected' : '') + '>' + localize('languageAuto', 'Automatic') + '</option>' + langs.map(function (l) {
939
+ return '<option value="' + l[0] + '"' + (prefs.language === l[0] ? ' selected' : '') + '>' + l[1] + '</option>';
940
+ }).join('');
941
+ var panel = document.createElement('div');
942
+ panel.className = 'card shadow-sm';
943
+ panel.style.cssText = 'position:absolute;bottom:calc(100% + 6px);right:0;z-index:1080;width:290px;max-height:70vh;overflow:auto;display:none;';
944
+ panel.innerHTML = '<div class="card-body p-3">' + '<div class="fw-semibold mb-2" style="font-size:0.85rem;">' + localize('settingsTitle', 'Voice settings') + '</div>' + '<label class="form-label mb-1 d-block" style="font-size:0.8rem;">' + localize('microphone', 'Microphone') + '</label>' + '<select class="form-select form-select-sm js-mic mb-2"><option value="">' + localize('defaultMicrophone', 'Default microphone') + '</option></select>' + '<label class="form-label mb-1 d-block" style="font-size:0.8rem;">' + localize('speaker', 'Speaker') + '</label>' + '<select class="form-select form-select-sm js-out mb-1"><option value="">' + localize('automatic', 'Automatic') + '</option></select>' + '<button type="button" class="btn btn-outline-secondary btn-sm mb-2 js-pick-out" style="display:none;">' + localize('chooseSpeaker', 'Choose speaker…') + '</button>' + '<div class="form-text mb-2 js-out-help">' + localize('speakerHelp', 'Automatic uses the device your system uses for calls. If you cannot hear the assistant, pick the speakers you are actually listening to.') + '</div>' + '<label class="form-label mb-1 d-block" style="font-size:0.8rem;">' + localize('assistantVolume', 'Assistant volume') + ': <strong class="js-vol-val">' + Math.round(prefs.volume * 100) + '</strong>%</label>' + '<input type="range" class="form-range js-vol mb-2" min="0" max="100" step="5" value="' + Math.round(prefs.volume * 100) + '">' + '<label class="form-label mb-1 d-block" style="font-size:0.8rem;">' + localize('language', 'Language') + '</label>' + '<select class="form-select form-select-sm js-lang mb-2">' + langOptions + '</select>' + '<div class="form-check form-switch mb-1">' + '<input class="form-check-input js-barge" type="checkbox" id="realtime-setting-barge"' + (prefs.bargeIn ? ' checked' : '') + '>' + '<label class="form-check-label" for="realtime-setting-barge">' + localize('allowInterruptions', 'Allow interruptions') + '</label>' + '</div>' + '<div class="form-text mb-2">' + localize('allowInterruptionsHelp', 'Talk over the assistant to interrupt it. Turn this off if the assistant keeps hearing itself through your speakers.') + '</div>' + '<div class="form-check form-switch mb-1">' + '<input class="form-check-input js-ptt" type="checkbox" id="realtime-setting-ptt"' + (prefs.pushToTalk ? ' checked' : '') + '>' + '<label class="form-check-label" for="realtime-setting-ptt">' + localize('pushToTalk', 'Push-to-talk') + '</label>' + '</div>' + '<div class="form-text">' + localize('pushToTalkHelp', 'Hold <kbd>Space</kbd> (or the button) to talk. For very noisy places.') + '</div>' + '</div>';
945
+ wrap.appendChild(gear);
946
+ wrap.appendChild(panel);
947
+ realtimeBtn.insertAdjacentElement('afterend', wrap);
948
+ var bargeInput = panel.querySelector('.js-barge');
949
+ var pttInput = panel.querySelector('.js-ptt');
950
+ var volInput = panel.querySelector('.js-vol');
951
+ var volVal = panel.querySelector('.js-vol-val');
952
+ var micSelect = panel.querySelector('.js-mic');
953
+ var outSelect = panel.querySelector('.js-out');
954
+ var outPickButton = panel.querySelector('.js-pick-out');
955
+ var langSelect = panel.querySelector('.js-lang');
956
+ function persist() {
957
+ var next = {
958
+ bargeIn: bargeInput.checked,
959
+ pushToTalk: pttInput.checked,
960
+ volume: (parseInt(volInput.value, 10) || 0) / 100,
961
+ micDeviceId: micSelect.value || '',
962
+ outputDeviceId: outSelect.value || '',
963
+ language: langSelect.value || '',
964
+ version: REALTIME_PREFS_VERSION
965
+ };
966
+ applyRealtimeAudioPrefs(next);
967
+ saveRealtimeAudioPrefs(next);
968
+ pushRealtimeSettingsToServer();
969
+ }
970
+ function populateDevices() {
971
+ if (!navigator.mediaDevices || !navigator.mediaDevices.enumerateDevices) {
972
+ return;
973
+ }
974
+ navigator.mediaDevices.enumerateDevices().then(function (devices) {
975
+ var options = ['<option value="">' + localize('defaultMicrophone', 'Default microphone') + '</option>'];
976
+ devices.filter(function (d) {
977
+ return d.kind === 'audioinput';
978
+ }).forEach(function (d, i) {
979
+ var label = d.label || localize('microphone', 'Microphone') + ' ' + (i + 1);
980
+ options.push('<option value="' + d.deviceId + '"' + (d.deviceId === realtimeMicDeviceId ? ' selected' : '') + '>' + label.replace(/</g, '') + '</option>');
981
+ });
982
+ micSelect.innerHTML = options.join('');
983
+
984
+ // Output devices are only enumerable on browsers that expose them (Chromium); elsewhere the
985
+ // list stays on "Automatic" and the picker button below is the way in.
986
+ var outs = devices.filter(function (d) {
987
+ return d.kind === 'audiooutput' && d.deviceId && d.deviceId !== 'communications' && d.deviceId !== 'default';
988
+ });
989
+ if (outs.length) {
990
+ var outOptions = ['<option value="">' + localize('automatic', 'Automatic') + '</option>'];
991
+ outs.forEach(function (d, i) {
992
+ var label = d.label || localize('speaker', 'Speaker') + ' ' + (i + 1);
993
+ outOptions.push('<option value="' + d.deviceId + '"' + (d.deviceId === realtimeOutputDeviceId ? ' selected' : '') + '>' + label.replace(/</g, '') + '</option>');
994
+ });
995
+ outSelect.innerHTML = outOptions.join('');
996
+ }
997
+ })["catch"](function () {});
998
+ }
999
+ populateDevices();
1000
+
1001
+ // Unplugging the headset mid-conversation silently changes which devices exist. Refresh the lists, and
1002
+ // end the session if the microphone it was actually using has gone — continuing on a different device
1003
+ // without saying so is worse than stopping and letting the user restart.
1004
+ if (navigator.mediaDevices && typeof navigator.mediaDevices.addEventListener === 'function') {
1005
+ navigator.mediaDevices.addEventListener('devicechange', function () {
1006
+ populateDevices();
1007
+ if (!isRealtimeActive || !realtimeMicDeviceId) {
1008
+ return;
1009
+ }
1010
+ navigator.mediaDevices.enumerateDevices().then(function (devices) {
1011
+ var stillThere = devices.some(function (d) {
1012
+ return d.kind === 'audioinput' && d.deviceId === realtimeMicDeviceId;
1013
+ });
1014
+ if (!stillThere) {
1015
+ endRealtimeSessionFromServer('device_lost');
1016
+ }
1017
+ })["catch"](function () {});
1018
+ });
1019
+ }
1020
+ function constrainPanel() {
1021
+ // Keep the popover on screen horizontally. It is anchored to the button's right edge, which is
1022
+ // correct where the button sits on the right of a wide row — but in a narrow layout, or with the
1023
+ // button near the left edge, a 290px panel would otherwise hang off the side of the window.
1024
+ panel.style.right = '0px';
1025
+ panel.style.left = 'auto';
1026
+ var panelRect = panel.getBoundingClientRect();
1027
+ if (panelRect.left < 8) {
1028
+ panel.style.right = panelRect.left - 8 + 'px';
1029
+ }
1030
+
1031
+ // Cap its height to the space above the button up to the nearest scroll/fixed ancestor, so it
1032
+ // scrolls instead of spilling out the top.
1033
+ var rect = gear.getBoundingClientRect();
1034
+ var boundaryTop = 8;
1035
+ var node = panel.parentElement ? panel.parentElement.parentElement : null;
1036
+ while (node && node !== document.body && node !== document.documentElement) {
1037
+ var s = window.getComputedStyle(node);
1038
+ if (s.overflowY === 'auto' || s.overflowY === 'scroll' || s.overflowY === 'hidden' || s.position === 'fixed') {
1039
+ boundaryTop = Math.max(boundaryTop, node.getBoundingClientRect().top);
1040
+ break;
1041
+ }
1042
+ node = node.parentElement;
1043
+ }
1044
+ panel.style.maxHeight = Math.max(160, rect.top - boundaryTop - 10) + 'px';
1045
+ }
1046
+ gear.addEventListener('click', function (e) {
1047
+ e.stopPropagation();
1048
+ var showing = panel.style.display === 'none';
1049
+ if (showing) {
1050
+ panel.style.display = 'block';
1051
+ constrainPanel();
1052
+ populateDevices();
1053
+ }
1054
+ panel.style.display = showing ? 'block' : 'none';
1055
+ });
1056
+ panel.addEventListener('click', function (e) {
1057
+ e.stopPropagation();
1058
+ });
1059
+ document.addEventListener('click', function () {
1060
+ panel.style.display = 'none';
1061
+ });
1062
+ bargeInput.addEventListener('change', persist);
1063
+ pttInput.addEventListener('change', persist);
1064
+ volInput.addEventListener('input', function () {
1065
+ volVal.textContent = volInput.value;
1066
+ persist();
1067
+ });
1068
+ micSelect.addEventListener('change', persist);
1069
+ langSelect.addEventListener('change', persist);
1070
+ outSelect.addEventListener('change', function () {
1071
+ persist();
1072
+ // Apply live: the whole point of the picker is that a user who cannot hear the assistant can fix
1073
+ // it without ending the conversation.
1074
+ applyRealtimeOutputDevice();
1075
+ });
1076
+
1077
+ // Firefox exposes no output device list until the user picks one through this prompt.
1078
+ if (navigator.mediaDevices && typeof navigator.mediaDevices.selectAudioOutput === 'function') {
1079
+ outPickButton.style.display = '';
1080
+ outPickButton.addEventListener('click', function () {
1081
+ navigator.mediaDevices.selectAudioOutput().then(function (device) {
1082
+ if (!device || !device.deviceId) {
1083
+ return;
1084
+ }
1085
+ var option = document.createElement('option');
1086
+ option.value = device.deviceId;
1087
+ option.textContent = device.label || localize('selectedSpeaker', 'Selected speaker');
1088
+ outSelect.appendChild(option);
1089
+ outSelect.value = device.deviceId;
1090
+ persist();
1091
+ applyRealtimeOutputDevice();
1092
+ })["catch"](function () {});
1093
+ });
1094
+ }
1095
+ }
1096
+
1097
+ // The host supplies the button's markup as a data attribute. It is sanitized when DOMPurify is present;
1098
+ // when it is not, the label falls back to text rather than injecting unsanitized markup — the module is
1099
+ // shared, and not every host that loads it also loads DOMPurify.
1100
+ function setRealtimeButtonHtml(button, html) {
1101
+ if (!html) {
1102
+ return;
1103
+ }
1104
+ if (window.DOMPurify && typeof window.DOMPurify.sanitize === 'function') {
1105
+ button.replaceChildren(window.DOMPurify.sanitize(html, {
1106
+ RETURN_DOM_FRAGMENT: true
1107
+ }));
1108
+ return;
1109
+ }
1110
+
1111
+ // No DOMPurify: never parse the untrusted markup as HTML. Strip any tags with a plain-string pass and
1112
+ // collapse whitespace, then render the remaining label as text only.
1113
+ var text = html.replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ').trim();
1114
+ button.replaceChildren(document.createTextNode(text));
1115
+ }
1116
+ function updateRealtimeButton() {
1117
+ var realtimeBtn = q('realtimeButton');
1118
+ if (!realtimeBtn) {
1119
+ return;
1120
+ }
1121
+ if (isRealtimeActive) {
1122
+ realtimeBtn.classList.add('active', 'btn-primary');
1123
+ realtimeBtn.classList.remove('btn-dark', 'btn-outline-dark', 'btn-outline-secondary');
1124
+ realtimeBtn.title = realtimeBtn.getAttribute('data-end-title') || 'End Conversation';
1125
+ setRealtimeButtonHtml(realtimeBtn, realtimeBtn.getAttribute('data-end-html'));
1126
+ } else {
1127
+ realtimeBtn.classList.remove('active', 'btn-primary', 'btn-dark', 'btn-outline-secondary');
1128
+ realtimeBtn.classList.add('btn-outline-dark');
1129
+ realtimeBtn.blur();
1130
+ realtimeBtn.title = realtimeBtn.getAttribute('data-start-title') || 'Start speaking';
1131
+ setRealtimeButtonHtml(realtimeBtn, realtimeBtn.getAttribute('data-start-html'));
1132
+ }
1133
+ }
1134
+
1135
+ // Switches the input between text and audio-only (realtime). Called at load and whenever the selected
1136
+ // deployment changes to/from a realtime model.
1137
+ function applyRealtimeMode(enable) {
1138
+ isRealtimeMode = enable;
1139
+ var realtimeBtn = q('realtimeButton');
1140
+ var micBtn = q('micButton');
1141
+ var conversationBtn = q('conversationButton');
1142
+ var inputEl = q('input');
1143
+ var sendBtn = q('sendButton');
1144
+ var audioSettings = document.getElementById('realtime-audio-settings');
1145
+ if (enable) {
1146
+ onEnterRealtimeMode();
1147
+ if (inputEl) {
1148
+ inputEl.hidden = true;
1149
+ }
1150
+ if (sendBtn) {
1151
+ sendBtn.hidden = true;
1152
+ }
1153
+ if (micBtn) {
1154
+ micBtn.hidden = true;
1155
+ }
1156
+ if (conversationBtn) {
1157
+ conversationBtn.hidden = true;
1158
+ }
1159
+ if (realtimeBtn) {
1160
+ realtimeBtn.hidden = false;
1161
+ }
1162
+ if (audioSettings) {
1163
+ audioSettings.hidden = false;
1164
+ }
1165
+ } else {
1166
+ if (isRealtimeActive) {
1167
+ stopRealtimeConversation();
1168
+ }
1169
+ if (realtimeBtn) {
1170
+ realtimeBtn.hidden = true;
1171
+ }
1172
+ if (audioSettings) {
1173
+ audioSettings.hidden = true;
1174
+ }
1175
+ removeCompatibilityModeNotice();
1176
+ clearRealtimeStatus();
1177
+ if (inputEl) {
1178
+ inputEl.hidden = false;
1179
+ }
1180
+ if (sendBtn) {
1181
+ sendBtn.hidden = false;
1182
+ }
1183
+ if (micBtn) {
1184
+ micBtn.hidden = false;
1185
+ }
1186
+ if (conversationBtn) {
1187
+ conversationBtn.hidden = false;
1188
+ }
1189
+ }
1190
+ updateRealtimeButton();
1191
+ }
1192
+
1193
+ // Barge-in and the voice-activity knobs are enforced by three layers at once: this client's microphone
1194
+ // gate, the server's input pump, and the provider's own turn detection. Changing only this one leaves the
1195
+ // three disagreeing — the classic symptom being barge-in switched off mid-conversation while the assistant
1196
+ // still interrupts itself — so a change during a live session is sent on to the other two.
1197
+ function pushRealtimeSettingsToServer() {
1198
+ if (!isRealtimeActive || !connection || typeof connection.send !== 'function') {
1199
+ return;
1200
+ }
1201
+ try {
1202
+ connection.send('UpdateRealtimeSettings', realtimeBargeIn, null, null);
1203
+ } catch (err) {}
1204
+ }
1205
+
1206
+ // Reports the session's state to the host so it can show something more honest than a button that flips to
1207
+ // "End Conversation" before anything has connected.
1208
+ // What each state says to the user. Only the states worth narrating appear here; the rest clear the line.
1209
+ var REALTIME_STATE_TEXT = {
1210
+ 'requesting-mic': localize('requestingMic', 'Waiting for microphone access…'),
1211
+ 'connecting': localize('connecting', 'Connecting…'),
1212
+ 'listening': localize('listening', 'Listening'),
1213
+ 'playback-blocked': localize('playbackBlocked', 'Audio is blocked by the browser — click the page to enable it.')
1214
+ };
1215
+ function setRealtimeState(state, detail) {
1216
+ if (realtimeState === state) {
1217
+ return;
1218
+ }
1219
+ realtimeState = state;
1220
+ var text = REALTIME_STATE_TEXT[state];
1221
+ if (text) {
1222
+ showRealtimeStatus(text, false);
1223
+ } else if (state === 'idle') {
1224
+ clearRealtimeStatus();
1225
+ }
1226
+ try {
1227
+ onStateChange({
1228
+ state: state,
1229
+ transport: realtimeIsWebRtc ? 'webrtc' : 'websocket',
1230
+ detail: detail || null
1231
+ });
1232
+ } catch (err) {}
1233
+ }
1234
+
1235
+ // The server owns the session's lifecycle; the browser cannot infer it from the audio it happens to
1236
+ // receive. Without these handlers a session that ended server-side (provider socket closed, session cap,
1237
+ // error, authorization failure) leaves the mic open and the button stuck on "End Conversation" while audio
1238
+ // is streamed into nothing.
1239
+ function bindRealtimeLifecycleHandlers() {
1240
+ if (realtimeLifecycleBound || !connection) {
1241
+ return;
1242
+ }
1243
+ realtimeLifecycleBound = true;
1244
+ connection.on('ReceiveRealtimeEvent', function (identifier, type, payload) {
1245
+ if (!isRealtimeActive) {
1246
+ return;
1247
+ }
1248
+ if (type === 'session_ready') {
1249
+ realtimeSessionReady = true;
1250
+ setRealtimeState('listening');
1251
+ } else if (type === 'session_ended') {
1252
+ endRealtimeSessionFromServer(payload || 'completed');
1253
+ } else if (type === 'speech_started') {
1254
+ // Barge-in: stop the reply that is already scheduled for playback. On WebRTC the server flushes
1255
+ // the peer itself, so this only matters for the WebSocket transport — but calling it there too
1256
+ // is harmless and keeps the two transports behaving the same.
1257
+ if (realtimeBargeIn) {
1258
+ flushRealtimePlayback();
1259
+ }
1260
+ } else if (type === 'playback_flush') {
1261
+ flushRealtimePlayback();
1262
+ } else if (type === 'user_turn_pending') {
1263
+ onUserTurnPending(payload);
1264
+ } else if (type === 'user_turn_dropped') {
1265
+ onUserTurnDropped(payload);
1266
+ }
1267
+ });
1268
+
1269
+ // An error while a voice session is running is terminal for that session: the server has already
1270
+ // stopped, so keep the mic open no longer.
1271
+ connection.on('ReceiveError', function () {
1272
+ if (!isRealtimeActive) {
1273
+ return;
1274
+ }
1275
+
1276
+ // An error while the WebRTC peer is still being set up is a reason to try the other transport, not
1277
+ // to give up: the WebSocket path may well work. Treating every error as terminal is what turned a
1278
+ // recoverable server-side handshake failure into a session that silently did nothing at all.
1279
+ if (realtimeIsWebRtc && !realtimeWebRtcConnected && !realtimeFellBack) {
1280
+ fallbackToWebSocket('server reported an error before the peer connected');
1281
+ return;
1282
+ }
1283
+
1284
+ // Once the session is live, an error is terminal — the server has already stopped.
1285
+ endRealtimeSessionFromServer('error');
1286
+ });
1287
+ if (typeof connection.onclose === 'function') {
1288
+ connection.onclose(function () {
1289
+ if (isRealtimeActive) {
1290
+ endRealtimeSessionFromServer('disconnected');
1291
+ }
1292
+ });
1293
+ }
1294
+ if (typeof connection.onreconnecting === 'function') {
1295
+ connection.onreconnecting(function () {
1296
+ if (isRealtimeActive) {
1297
+ endRealtimeSessionFromServer('disconnected');
1298
+ }
1299
+ });
1300
+ }
1301
+ }
1302
+ function endRealtimeSessionFromServer(reason) {
1303
+ realtimeEndedNotice = reason;
1304
+ stopRealtimeConversation();
1305
+ setRealtimeState('ended', reason);
1306
+ if (window.console && console.info) {
1307
+ console.info('The realtime voice session ended (' + reason + ').');
1308
+ }
1309
+
1310
+ // An idle close is deliberate and recoverable, so say so and offer to pick the conversation back up
1311
+ // rather than leaving the user staring at a button that silently stopped working.
1312
+ if (reason === 'idle') {
1313
+ showRealtimeStatus(localize('endedIdle', 'Voice paused after a period of silence.'), true);
1314
+ } else if (reason === 'disconnected') {
1315
+ showRealtimeStatus(localize('endedDisconnected', 'Voice session ended — the connection dropped.'), true);
1316
+ } else if (reason === 'error') {
1317
+ showRealtimeStatus(localize('endedError', 'Voice session ended unexpectedly.'), true);
1318
+ } else if (reason === 'device_lost') {
1319
+ showRealtimeStatus(localize('endedDeviceLost', 'Voice session ended — the microphone was disconnected.'), true);
1320
+ }
1321
+ }
1322
+
1323
+ // A single status line under the realtime button, optionally with a Resume button.
1324
+ function showRealtimeStatus(text, offerResume) {
1325
+ var realtimeBtn = q('realtimeButton');
1326
+ if (!realtimeBtn) {
1327
+ return;
1328
+ }
1329
+ if (!realtimeStatusEl) {
1330
+ realtimeStatusEl = document.createElement('div');
1331
+ realtimeStatusEl.className = 'form-text mt-2 mb-0 text-center';
1332
+ realtimeStatusEl.setAttribute('role', 'status');
1333
+ realtimeStatusEl.setAttribute('aria-live', 'polite');
1334
+ var row = realtimeBtn.parentElement;
1335
+ if (row && row.parentElement) {
1336
+ row.parentElement.appendChild(realtimeStatusEl);
1337
+ } else {
1338
+ realtimeBtn.insertAdjacentElement('afterend', realtimeStatusEl);
1339
+ }
1340
+ }
1341
+ realtimeStatusEl.replaceChildren();
1342
+ realtimeStatusEl.appendChild(document.createTextNode(text));
1343
+ if (offerResume) {
1344
+ var resume = document.createElement('button');
1345
+ resume.type = 'button';
1346
+ resume.className = 'btn btn-link btn-sm p-0 ms-2 align-baseline';
1347
+ resume.textContent = localize('resume', 'Resume');
1348
+ resume.addEventListener('click', function () {
1349
+ clearRealtimeStatus();
1350
+ startRealtimeConversation();
1351
+ });
1352
+ realtimeStatusEl.appendChild(resume);
1353
+ }
1354
+ }
1355
+ function clearRealtimeStatus() {
1356
+ if (!realtimeStatusEl) {
1357
+ return;
1358
+ }
1359
+ try {
1360
+ realtimeStatusEl.remove();
1361
+ } catch (err) {}
1362
+ realtimeStatusEl = null;
1363
+ }
1364
+
1365
+ // Resolves the ICE servers from the server immediately before creating the peer. Fetching them per session
1366
+ // (rather than embedding them at page render time) is what makes ephemeral TURN credentials usable: they
1367
+ // expire, and a page open for an hour would otherwise hand the browser a dead credential. Falls back to a
1368
+ // public STUN server so a failed call degrades to today's behavior rather than breaking the session.
1369
+ function resolveIceServers() {
1370
+ if (webRtcIceServers) {
1371
+ return Promise.resolve(webRtcIceServers);
1372
+ }
1373
+ if (!connection || typeof connection.invoke !== 'function') {
1374
+ return Promise.resolve(DEFAULT_ICE_SERVERS);
1375
+ }
1376
+ return connection.invoke('GetRealtimeIceServers').then(function (servers) {
1377
+ return Array.isArray(servers) && servers.length ? servers : DEFAULT_ICE_SERVERS;
1378
+ })["catch"](function (err) {
1379
+ if (window.console && console.warn) {
1380
+ console.warn('Could not resolve the realtime ICE servers; using the default STUN server.', err);
1381
+ }
1382
+ return DEFAULT_ICE_SERVERS;
1383
+ });
1384
+ }
1385
+ function isWebRtcKnownBlocked() {
1386
+ try {
1387
+ return window.sessionStorage.getItem(REALTIME_WEBRTC_BLOCKED_KEY) === '1';
1388
+ } catch (err) {
1389
+ return false;
1390
+ }
1391
+ }
1392
+ function rememberWebRtcBlocked() {
1393
+ try {
1394
+ window.sessionStorage.setItem(REALTIME_WEBRTC_BLOCKED_KEY, '1');
1395
+ } catch (err) {}
1396
+ }
1397
+
1398
+ // A one-line, dismissible note under the realtime button. Shown once per browser session when voice falls
1399
+ // back to the WebSocket transport, because the difference is audible (weaker echo handling in an open
1400
+ // room) and users otherwise have no way to know why.
1401
+ function showCompatibilityModeNotice() {
1402
+ if (realtimeCompatibilityNoticeEl) {
1403
+ return;
1404
+ }
1405
+ var realtimeBtn = q('realtimeButton');
1406
+ if (!realtimeBtn) {
1407
+ return;
1408
+ }
1409
+ var note = document.createElement('div');
1410
+ note.className = 'form-text mt-2 mb-0 text-center';
1411
+ note.setAttribute('role', 'status');
1412
+ note.textContent = localize('compatibilityMode', 'Using compatibility audio mode. Echo cancellation may be weaker — headphones are recommended.');
1413
+ var row = realtimeBtn.parentElement;
1414
+ if (row && row.parentElement) {
1415
+ row.parentElement.appendChild(note);
1416
+ } else {
1417
+ realtimeBtn.insertAdjacentElement('afterend', note);
1418
+ }
1419
+ realtimeCompatibilityNoticeEl = note;
1420
+ }
1421
+ function removeCompatibilityModeNotice() {
1422
+ if (!realtimeCompatibilityNoticeEl) {
1423
+ return;
1424
+ }
1425
+ try {
1426
+ realtimeCompatibilityNoticeEl.remove();
1427
+ } catch (err) {}
1428
+ realtimeCompatibilityNoticeEl = null;
1429
+ }
1430
+ function startRealtimeConversation() {
1431
+ if (!isRealtimeMode || isRealtimeActive || !connection) {
1432
+ return;
1433
+ }
1434
+ applyRealtimeAudioPrefs(loadRealtimeAudioPrefs());
1435
+ realtimeFellBack = false;
1436
+ realtimeSessionReady = false;
1437
+ realtimeEndedNotice = null;
1438
+ clearRealtimeStatus();
1439
+ bindRealtimeLifecycleHandlers();
1440
+ setRealtimeState('requesting-mic');
1441
+
1442
+ // Prefer the WebRTC transport when the server advertises it: the browser's echo canceller references
1443
+ // the assistant's media track, so the model can ignore its own voice with the mic open (open rooms).
1444
+ // If the peer cannot connect (blocked UDP, no TURN, unsupported), we fall back to WebSocket at connect
1445
+ // time — see fallbackToWebSocket. The decision is made once, before the session starts.
1446
+ if (webRtcEnabled && !isWebRtcKnownBlocked()) {
1447
+ startRealtimeWebRtcConversation();
1448
+ return;
1449
+ }
1450
+ startRealtimeWebSocketConversation();
1451
+ }
1452
+ function startRealtimeWebSocketConversation() {
1453
+ if (webRtcEnabled) {
1454
+ showCompatibilityModeNotice();
1455
+ }
1456
+ attachRealtimePushToTalk();
1457
+ // Drop focus from the just-clicked button so Space acts as push-to-talk, not a re-click.
1458
+ var focusedBtn = q('realtimeButton');
1459
+ if (focusedBtn) {
1460
+ try {
1461
+ focusedBtn.blur();
1462
+ } catch (err) {}
1463
+ }
1464
+ var audioConstraints = {
1465
+ channelCount: 1,
1466
+ echoCancellation: {
1467
+ ideal: true
1468
+ },
1469
+ noiseSuppression: true,
1470
+ autoGainControl: true,
1471
+ // Ask for the strongest available acoustic echo cancellation so the model can ignore its own
1472
+ // voice in an open room (loud speakers + open mic). These are Chromium hints requested as
1473
+ // "ideal", so any browser that doesn't support them simply ignores them — never a hard failure.
1474
+ // echoCancellationType 'system' prefers the OS/hardware AEC over the browser's software AEC.
1475
+ echoCancellationType: {
1476
+ ideal: 'system'
1477
+ },
1478
+ voiceIsolation: {
1479
+ ideal: true
1480
+ }
1481
+ };
1482
+ if (realtimeMicDeviceId) {
1483
+ audioConstraints.deviceId = {
1484
+ exact: realtimeMicDeviceId
1485
+ };
1486
+ }
1487
+ navigator.mediaDevices.getUserMedia({
1488
+ audio: audioConstraints
1489
+ }).then(function (stream) {
1490
+ ensureConnected().then(function () {
1491
+ realtimeStream = stream;
1492
+ isRealtimeActive = true;
1493
+ onActivate();
1494
+ updateRealtimeButton();
1495
+ setRealtimeState('connecting');
1496
+ var AudioContextCtor = window.AudioContext || window.webkitAudioContext;
1497
+ realtimeAudioCtx = new AudioContextCtor({
1498
+ sampleRate: REALTIME_SAMPLE_RATE
1499
+ });
1500
+ // A context created under a strict autoplay policy starts suspended, and a suspended
1501
+ // context plays nothing at all — silently. Ask for it explicitly.
1502
+ if (realtimeAudioCtx.state === 'suspended' && typeof realtimeAudioCtx.resume === 'function') {
1503
+ realtimeAudioCtx.resume()["catch"](function (err) {
1504
+ setRealtimeState('playback-blocked');
1505
+ if (window.console && console.warn) {
1506
+ console.warn('The browser blocked realtime audio playback. Interact with the page to enable audio.', err);
1507
+ }
1508
+ });
1509
+ }
1510
+ realtimePlayHead = 0;
1511
+ realtimeSources = [];
1512
+ realtimeGain = realtimeAudioCtx.createGain();
1513
+ realtimeGain.gain.value = realtimeVolume != null ? realtimeVolume : 1;
1514
+ realtimeGain.connect(realtimeAudioCtx.destination);
1515
+ realtimeSubject = new window.signalR.Subject();
1516
+ var ctxAtStart = realtimeAudioCtx;
1517
+ var processor = realtimeAudioCtx.createScriptProcessor(4096, 1, 1);
1518
+ realtimeProcessor = processor;
1519
+
1520
+ // The gate watches the assistant's playback to know when it is audible; on this
1521
+ // transport the assistant is a Web Audio graph, so tap the output gain into a stream.
1522
+ var monitorDest = realtimeAudioCtx.createMediaStreamDestination();
1523
+ realtimeGain.connect(monitorDest);
1524
+
1525
+ // Send the gated microphone rather than the raw one, exactly as the WebRTC transport
1526
+ // does, so this fallback is not the one transport where the model hears its own echo.
1527
+ // Until the gate's worklet is ready the processor has no input and streams silence.
1528
+ setupMicGate(stream).then(function (micTrack) {
1529
+ if (!isRealtimeActive || realtimeAudioCtx !== ctxAtStart || !micTrack) {
1530
+ return;
1531
+ }
1532
+ var gatedStream = new MediaStream([micTrack]);
1533
+ var source = realtimeAudioCtx.createMediaStreamSource(gatedStream);
1534
+ realtimeMicSource = source;
1535
+ source.connect(processor);
1536
+ if (realtimeGate) {
1537
+ realtimeGate.attachAssistantStream(monitorDest.stream);
1538
+ }
1539
+ });
1540
+ processor.onaudioprocess = function (event) {
1541
+ var input = event.inputBuffer.getChannelData(0);
1542
+ // Always send a frame (silence when muted) so the server keeps a continuous audio
1543
+ // stream and its voice-activity detector promptly notices the pause and responds.
1544
+ // Muted cases: push-to-talk not held, or the echo guard while the assistant plays back.
1545
+ var muted;
1546
+ if (realtimePushToTalk) {
1547
+ muted = !realtimePttActive;
1548
+ } else {
1549
+ muted = !realtimeBargeIn && realtimeAudioCtx && realtimeAudioCtx.currentTime < realtimePlayHead + REALTIME_WS_ECHO_HANGOVER_SEC;
1550
+ }
1551
+ var pcm = new Int16Array(input.length);
1552
+ if (!muted) {
1553
+ for (var i = 0; i < input.length; i++) {
1554
+ var s = Math.max(-1, Math.min(1, input[i]));
1555
+ pcm[i] = s < 0 ? s * 0x8000 : s * 0x7FFF;
1556
+ }
1557
+ }
1558
+ var bytes = new Uint8Array(pcm.buffer);
1559
+ var binary = '';
1560
+ for (var b = 0; b < bytes.length; b++) {
1561
+ binary += String.fromCharCode(bytes[b]);
1562
+ }
1563
+ try {
1564
+ realtimeSubject.next(btoa(binary));
1565
+ } catch (err) {/* completed */}
1566
+ };
1567
+
1568
+ // A zero-gain node keeps the processor alive without echoing the mic to the speakers.
1569
+ var zeroGain = realtimeAudioCtx.createGain();
1570
+ zeroGain.gain.value = 0;
1571
+ realtimeZeroGain = zeroGain;
1572
+ processor.connect(zeroGain);
1573
+ zeroGain.connect(realtimeAudioCtx.destination);
1574
+
1575
+ // "Auto" means auto-detect: send nothing. Sending the browser's locale instead pinned transcription
1576
+ // and the reply language to it, so a bilingual user with an English browser speaking Spanish
1577
+ // got English-forced (garbage) transcripts and an English-locked assistant.
1578
+ var language = realtimeLanguage || null;
1579
+ var voice = getVoiceName && getVoiceName() || realtimeVoiceName || '';
1580
+ sendStart(realtimeSubject, voice, language, null, null, realtimeBargeIn);
1581
+ if (realtimePushToTalk) {
1582
+ buildRealtimePttUi();
1583
+ }
1584
+ })["catch"](function (err) {
1585
+ stream.getTracks().forEach(function (track) {
1586
+ track.stop();
1587
+ });
1588
+ isRealtimeActive = false;
1589
+ onDeactivate();
1590
+ updateRealtimeButton();
1591
+ console.error('The realtime conversation could not start because the chat connection is not available.', err);
1592
+ });
1593
+ })["catch"](function (err) {
1594
+ console.error('Microphone access denied:', err);
1595
+ isRealtimeActive = false;
1596
+ onDeactivate();
1597
+ updateRealtimeButton();
1598
+ });
1599
+ }
1600
+
1601
+ // --- WebRTC (server-relay) transport ---
1602
+
1603
+ function startRealtimeWebRtcConversation() {
1604
+ // Keep the WebRTC capture constraints minimal. WebRTC's own echo canceller (AEC3) already references
1605
+ // the remote assistant track we render into the hidden <audio> element, which is the entire reason we
1606
+ // use WebRTC for open-room audio. The stronger hints we request on the WebSocket path
1607
+ // (echoCancellationType:'system' + voiceIsolation) are counterproductive here: layered on top of the
1608
+ // peer-connection AEC loop they gate the mic down to near-silence (~1-3% of full scale), so the model
1609
+ // never detects speech. Ask only for standard echo cancellation and let AEC3 do its job.
1610
+ var micConstraints = {
1611
+ // Match what ChatGPT's voice mode requests: a plain echoCancellation:true and nothing else exotic.
1612
+ // The browser still applies its default noise suppression and auto gain; we only pass those through
1613
+ // so the user's toggles can turn them off.
1614
+ echoCancellation: true,
1615
+ noiseSuppression: true,
1616
+ autoGainControl: true
1617
+ };
1618
+ if (realtimeMicDeviceId) {
1619
+ micConstraints.deviceId = {
1620
+ exact: realtimeMicDeviceId
1621
+ };
1622
+ }
1623
+ navigator.mediaDevices.getUserMedia({
1624
+ audio: micConstraints
1625
+ }).then(function (stream) {
1626
+ ensureConnected().then(function () {
1627
+ return resolveIceServers();
1628
+ }).then(function (iceServers) {
1629
+ realtimeStream = stream;
1630
+ isRealtimeActive = true;
1631
+ realtimeIsWebRtc = true;
1632
+ onActivate();
1633
+ updateRealtimeButton();
1634
+ setRealtimeState('connecting');
1635
+ realtimeRemoteDescriptionSet = false;
1636
+ realtimePendingIce = [];
1637
+ realtimeWebRtcConnected = false;
1638
+ var pc = new RTCPeerConnection({
1639
+ iceServers: iceServers
1640
+ });
1641
+ realtimePc = pc;
1642
+
1643
+ // Push-to-talk and the barge-in/echo guard both gate the outbound mic track. Wire the
1644
+ // same push-to-talk controls the WebSocket path uses so they work on WebRTC too.
1645
+ // Device labels only become readable after permission is granted, so this is the first
1646
+ // chance to notice a headset (safe for full duplex) or a standalone mic in front of
1647
+ // speakers (not). Only ever a suggestion: an explicit choice is never overridden.
1648
+
1649
+ attachRealtimePushToTalk();
1650
+ var focusedBtn = q('realtimeButton');
1651
+ if (focusedBtn) {
1652
+ try {
1653
+ focusedBtn.blur();
1654
+ } catch (err) {}
1655
+ }
1656
+ if (realtimePushToTalk) {
1657
+ buildRealtimePttUi();
1658
+ }
1659
+
1660
+ // Play the assistant's remote track through a hidden <audio> element. This gives the
1661
+ // browser's echo canceller a reference to remove the assistant's voice from the mic, and
1662
+ // is what the echo-guard monitor listens to for half-duplex mic muting.
1663
+ var audioEl = document.createElement('audio');
1664
+ audioEl.autoplay = true;
1665
+ audioEl.style.display = 'none';
1666
+ audioEl.volume = Math.max(0, Math.min(1, realtimeVolume != null ? realtimeVolume : 1));
1667
+ document.body.appendChild(audioEl);
1668
+ realtimeRemoteAudioEl = audioEl;
1669
+ // Route playback so the browser couples it with the mic's echo canceller as a proper
1670
+ // AEC reference (see routeRealtimeOutputToDefaultDevice for the preference order).
1671
+ routeRealtimeOutputToDefaultDevice(audioEl);
1672
+ pc.ontrack = function (e) {
1673
+ // Ask the browser to hold a fixed cushion of received audio before playing it,
1674
+ // rather than the adaptive minimum it would otherwise pick. Network jitter then
1675
+ // lands inside the cushion instead of making the jitter buffer stretch or speed
1676
+ // speech up to re-adapt; the audio itself is played as received. Ignored by
1677
+ // browsers without the property.
1678
+ try {
1679
+ if (e.receiver && 'jitterBufferTarget' in e.receiver) {
1680
+ e.receiver.jitterBufferTarget = REALTIME_WEBRTC_JITTER_BUFFER_TARGET_MS;
1681
+ }
1682
+ } catch (err) {}
1683
+ if (e.streams && e.streams[0]) {
1684
+ audioEl.srcObject = e.streams[0];
1685
+ ensureRealtimePlayback(audioEl);
1686
+ // Let the gate watch the assistant's level so it can raise the speech threshold
1687
+ // while the assistant talks (barge-in on) and stay closed until it finishes
1688
+ // (barge-in off).
1689
+ attachWebRtcAssistantAnalyser(e.streams[0]);
1690
+ }
1691
+ };
1692
+ realtimeSawRelayCandidate = false;
1693
+ pc.onicecandidate = function (e) {
1694
+ if (e.candidate) {
1695
+ if (e.candidate.candidate && e.candidate.candidate.indexOf(' relay ') !== -1) {
1696
+ realtimeSawRelayCandidate = true;
1697
+ }
1698
+ try {
1699
+ connection.send('AddRealtimeIceCandidate', e.candidate.candidate, e.candidate.sdpMid || '', e.candidate.sdpMLineIndex || 0);
1700
+ } catch (err) {}
1701
+ return;
1702
+ }
1703
+
1704
+ // Gathering finished. With no relay candidate there is no path left for a network
1705
+ // that blocks UDP, so stop waiting the full timeout for something that cannot come.
1706
+ if (!realtimeSawRelayCandidate && !realtimeWebRtcConnected && realtimeWebRtcConnectTimer) {
1707
+ clearTimeout(realtimeWebRtcConnectTimer);
1708
+ realtimeWebRtcConnectTimer = setTimeout(function () {
1709
+ if (!realtimeWebRtcConnected && isRealtimeActive) {
1710
+ fallbackToWebSocket('no relay candidate and no direct connection');
1711
+ }
1712
+ }, REALTIME_WEBRTC_NO_RELAY_TIMEOUT_MS);
1713
+ }
1714
+ };
1715
+ pc.onconnectionstatechange = function () {
1716
+ if (pc.connectionState === 'connected') {
1717
+ markWebRtcConnected();
1718
+ } else if (pc.connectionState === 'failed') {
1719
+ // Before the peer connects, a failure means WebRTC is unusable here — fall back.
1720
+ // After it connects, a drop ends the session (we never migrate mid-session).
1721
+ if (!realtimeWebRtcConnected) {
1722
+ fallbackToWebSocket('connection failed');
1723
+ } else if (isRealtimeActive) {
1724
+ stopRealtimeConversation();
1725
+ }
1726
+ } else if (pc.connectionState === 'closed' && isRealtimeActive && realtimeWebRtcConnected) {
1727
+ stopRealtimeConversation();
1728
+ }
1729
+ };
1730
+ pc.oniceconnectionstatechange = function () {
1731
+ var s = pc.iceConnectionState;
1732
+ if (s === 'connected' || s === 'completed') {
1733
+ markWebRtcConnected();
1734
+ } else if (s === 'failed' && !realtimeWebRtcConnected) {
1735
+ fallbackToWebSocket('ICE failed');
1736
+ }
1737
+ };
1738
+ bindWebRtcSignalingHandlers();
1739
+
1740
+ // Fall back to WebSocket if the peer does not connect within the timeout (blocked UDP,
1741
+ // missing TURN, etc.). Cleared as soon as the peer connects (markWebRtcConnected).
1742
+ realtimeWebRtcConnectTimer = setTimeout(function () {
1743
+ if (!realtimeWebRtcConnected && isRealtimeActive) {
1744
+ fallbackToWebSocket('connection timed out');
1745
+ }
1746
+ }, REALTIME_WEBRTC_CONNECT_TIMEOUT_MS);
1747
+
1748
+ // Send a gated version of the mic: silent unless the user is actually speaking. The gate
1749
+ // loads an AudioWorklet, so the track is only available once that resolves — hence the
1750
+ // offer is created here rather than synchronously above.
1751
+ setupMicGate(stream).then(function (micTrackToSend) {
1752
+ if (realtimePc !== pc) {
1753
+ return null;
1754
+ }
1755
+ if (micTrackToSend) {
1756
+ pc.addTrack(micTrackToSend, stream);
1757
+ }
1758
+ return pc.createOffer();
1759
+ }).then(function (offer) {
1760
+ if (!offer || realtimePc !== pc) {
1761
+ return null;
1762
+ }
1763
+ return pc.setLocalDescription(offer).then(function () {
1764
+ return offer;
1765
+ });
1766
+ }).then(function (offer) {
1767
+ if (!offer) {
1768
+ return;
1769
+ }
1770
+ // "Auto" means auto-detect: send nothing (see the WebSocket path above).
1771
+ var language = realtimeLanguage || null;
1772
+ var voice = getVoiceName && getVoiceName() || realtimeVoiceName || '';
1773
+ sendStartWebRtc(offer.sdp, voice, language, null, null, realtimeBargeIn);
1774
+ })["catch"](function (err) {
1775
+ console.error('Failed to create the WebRTC offer; falling back to WebSocket.', err);
1776
+ fallbackToWebSocket('offer failed');
1777
+ });
1778
+ })["catch"](function (err) {
1779
+ stream.getTracks().forEach(function (t) {
1780
+ t.stop();
1781
+ });
1782
+ isRealtimeActive = false;
1783
+ realtimeIsWebRtc = false;
1784
+ onDeactivate();
1785
+ updateRealtimeButton();
1786
+ console.error('The realtime WebRTC conversation could not start because the chat connection is not available.', err);
1787
+ });
1788
+ })["catch"](function (err) {
1789
+ console.error('Microphone access denied:', err);
1790
+ isRealtimeActive = false;
1791
+ realtimeIsWebRtc = false;
1792
+ onDeactivate();
1793
+ updateRealtimeButton();
1794
+ });
1795
+ }
1796
+ function bindWebRtcSignalingHandlers() {
1797
+ if (realtimeWebRtcHandlersBound) {
1798
+ return;
1799
+ }
1800
+ realtimeWebRtcHandlersBound = true;
1801
+ connection.on('ReceiveRealtimeAnswer', function (sdp) {
1802
+ if (!realtimePc) {
1803
+ return;
1804
+ }
1805
+ realtimePc.setRemoteDescription({
1806
+ type: 'answer',
1807
+ sdp: sdp
1808
+ }).then(function () {
1809
+ realtimeRemoteDescriptionSet = true;
1810
+ // Flush any ICE candidates that arrived before the answer was applied.
1811
+ var pending = realtimePendingIce;
1812
+ realtimePendingIce = [];
1813
+ pending.forEach(function (init) {
1814
+ try {
1815
+ realtimePc.addIceCandidate(init)["catch"](function () {});
1816
+ } catch (err) {}
1817
+ });
1818
+ })["catch"](function (err) {
1819
+ console.error('Failed to apply the WebRTC answer.', err);
1820
+ });
1821
+ });
1822
+ connection.on('ReceiveRealtimeIceCandidate', function (candidate, sdpMid, sdpMLineIndex) {
1823
+ if (!realtimePc || !candidate) {
1824
+ return;
1825
+ }
1826
+ var init = {
1827
+ candidate: candidate,
1828
+ sdpMid: sdpMid,
1829
+ sdpMLineIndex: sdpMLineIndex
1830
+ };
1831
+ // A candidate can only be added after the remote description (answer) is set; buffer until then.
1832
+ if (realtimeRemoteDescriptionSet) {
1833
+ try {
1834
+ realtimePc.addIceCandidate(init)["catch"](function () {});
1835
+ } catch (err) {}
1836
+ } else {
1837
+ realtimePendingIce.push(init);
1838
+ }
1839
+ });
1840
+ }
1841
+ function stopRealtimeWebRtc() {
1842
+ stopWebRtcEchoGuard();
1843
+ realtimeWebRtcMicTrack = null;
1844
+ if (realtimePc) {
1845
+ try {
1846
+ realtimePc.close();
1847
+ } catch (err) {}
1848
+ realtimePc = null;
1849
+ }
1850
+ if (realtimeRemoteAudioEl) {
1851
+ try {
1852
+ realtimeRemoteAudioEl.srcObject = null;
1853
+ realtimeRemoteAudioEl.remove();
1854
+ } catch (err) {}
1855
+ realtimeRemoteAudioEl = null;
1856
+ }
1857
+ realtimeIsWebRtc = false;
1858
+ }
1859
+ function markWebRtcConnected() {
1860
+ if (realtimeWebRtcConnected) {
1861
+ return;
1862
+ }
1863
+ realtimeWebRtcConnected = true;
1864
+ if (realtimeWebRtcConnectTimer) {
1865
+ clearTimeout(realtimeWebRtcConnectTimer);
1866
+ realtimeWebRtcConnectTimer = null;
1867
+ }
1868
+ }
1869
+
1870
+ // --- Microphone gate (thin wrapper over the shared factory) ----------------------------------------
1871
+
1872
+ // How long the gate stays open after the user's level drops. The gate is a safety net around echo, not a
1873
+ // turn detector: the provider's turn detection sees the real speech and should be the one deciding a
1874
+ // turn is over, so this must comfortably outlast the pause it is willing to wait through. The gate emits
1875
+ // digital silence when it closes, and whichever of the two expires first is what actually ends the turn.
1876
+ var REALTIME_GATE_HANGOVER_MS = 2000;
1877
+ function realtimeGateModeMessage() {
1878
+ return {
1879
+ pushToTalk: !!realtimePushToTalk,
1880
+ pttActive: !!realtimePttActive,
1881
+ bargeIn: !!realtimeBargeIn,
1882
+ speechHangoverMs: REALTIME_GATE_HANGOVER_MS
1883
+ };
1884
+ }
1885
+
1886
+ // The gate runs on the audio thread and cannot read this closure's variables, so every settings or
1887
+ // push-to-talk change has to be pushed to it explicitly.
1888
+ function syncRealtimeGateMode() {
1889
+ if (realtimeGate) {
1890
+ realtimeGate.setMode(realtimeGateModeMessage());
1891
+ }
1892
+ }
1893
+
1894
+ // Builds the gate and resolves with the track to send (over the peer, or into the WebSocket capture graph).
1895
+ function setupMicGate(rawStream) {
1896
+ stopWebRtcEchoGuard();
1897
+ var rawTrack = rawStream && rawStream.getAudioTracks()[0] || null;
1898
+ realtimeWebRtcMicTrack = rawTrack;
1899
+ return createMicGate(rawStream, realtimeGateModeMessage()).then(function (gate) {
1900
+ realtimeGate = gate;
1901
+ if (realtimeGatePendingRemoteStream) {
1902
+ gate.attachAssistantStream(realtimeGatePendingRemoteStream);
1903
+ }
1904
+ return gate.track || rawTrack;
1905
+ });
1906
+ }
1907
+
1908
+ // Feeds the assistant's remote stream into the gate so it knows when the assistant is audible — for the
1909
+ // raised threshold with barge-in on, and for half-duplex with barge-in off. The track can arrive before
1910
+ // the gate's worklet has finished loading, so remember it either way.
1911
+ function attachWebRtcAssistantAnalyser(remoteStream) {
1912
+ realtimeGatePendingRemoteStream = remoteStream || null;
1913
+ if (realtimeGate && remoteStream) {
1914
+ realtimeGate.attachAssistantStream(remoteStream);
1915
+ }
1916
+ }
1917
+ function stopWebRtcEchoGuard() {
1918
+ if (realtimeGate) {
1919
+ try {
1920
+ realtimeGate.stop();
1921
+ } catch (e) {}
1922
+ realtimeGate = null;
1923
+ }
1924
+ realtimeGatePendingRemoteStream = null;
1925
+ }
1926
+
1927
+ // Route the assistant playback element so the browser's echo canceller couples to it as an AEC reference,
1928
+ // the way ChatGPT's voice mode does. Preference order:
1929
+ // 1. The "communications" sink (Chrome/Edge). Rendering here doesn't just pick a device — it puts the
1930
+ // browser's audio pipeline in COMMUNICATIONS mode, which is what actually couples playback with the
1931
+ // mic's echo canceller. This is what ChatGPT uses and what keeps full-duplex stable across turns.
1932
+ // 2. The concrete device the "default" alias points at, matched by groupId.
1933
+ // If neither alias exists — Firefox exposes only concrete outputs — we deliberately do nothing and let the
1934
+ // browser use its own default. Picking "the first concrete output" there routed the assistant to whatever
1935
+ // device happened to be enumerated first (often an HDMI monitor), so the assistant appeared silent.
1936
+ // Applies the user's chosen speaker to the playing element, live.
1937
+ function applyRealtimeOutputDevice() {
1938
+ routeOutputToPreferredDevice(realtimeRemoteAudioEl, realtimeOutputDeviceId);
1939
+ }
1940
+ function routeRealtimeOutputToDefaultDevice(el) {
1941
+ routeOutputToPreferredDevice(el, realtimeOutputDeviceId);
1942
+ }
1943
+ function ensureRealtimePlayback(el) {
1944
+ ensurePlayback(el, function () {
1945
+ setRealtimeState('playback-blocked');
1946
+ });
1947
+ }
1948
+
1949
+ // Connect-time only: tear down the failed WebRTC attempt and restart on the known-good WebSocket path.
1950
+ // Never called once a session is established (see the connection-state handlers), so we never migrate audio
1951
+ // mid-conversation — we only choose the transport before the model starts responding.
1952
+ function fallbackToWebSocket(reason) {
1953
+ if (realtimeFellBack) {
1954
+ return;
1955
+ }
1956
+ realtimeFellBack = true;
1957
+ if (realtimeWebRtcConnectTimer) {
1958
+ clearTimeout(realtimeWebRtcConnectTimer);
1959
+ realtimeWebRtcConnectTimer = null;
1960
+ }
1961
+ if (window.console && console.warn) {
1962
+ console.warn('Realtime WebRTC transport unavailable (' + reason + '); falling back to the WebSocket transport.');
1963
+ }
1964
+ rememberWebRtcBlocked();
1965
+ showCompatibilityModeNotice();
1966
+ try {
1967
+ if (realtimeStream) {
1968
+ realtimeStream.getTracks().forEach(function (t) {
1969
+ t.stop();
1970
+ });
1971
+ }
1972
+ } catch (err) {}
1973
+ realtimeStream = null;
1974
+ stopRealtimeWebRtc();
1975
+ realtimeIsWebRtc = false;
1976
+ // Reset the active flag so the WebSocket start proceeds cleanly; it re-activates immediately.
1977
+ isRealtimeActive = false;
1978
+ startRealtimeWebSocketConversation();
1979
+ }
1980
+ function stopRealtimeConversation() {
1981
+ if (!isRealtimeActive) {
1982
+ return;
1983
+ }
1984
+ if (realtimeWebRtcConnectTimer) {
1985
+ clearTimeout(realtimeWebRtcConnectTimer);
1986
+ realtimeWebRtcConnectTimer = null;
1987
+ }
1988
+ isRealtimeActive = false;
1989
+ realtimeSessionReady = false;
1990
+ onDeactivate();
1991
+ updateRealtimeButton();
1992
+ detachRealtimePushToTalk();
1993
+ removeRealtimePttUi();
1994
+ if (!realtimeEndedNotice) {
1995
+ setRealtimeState('idle');
1996
+ }
1997
+ if (realtimeIsWebRtc) {
1998
+ stopRealtimeWebRtc();
1999
+ try {
2000
+ if (realtimeStream) {
2001
+ realtimeStream.getTracks().forEach(function (t) {
2002
+ t.stop();
2003
+ });
2004
+ }
2005
+ } catch (err) {}
2006
+ realtimeStream = null;
2007
+ return;
2008
+ }
2009
+ try {
2010
+ if (realtimeSubject) {
2011
+ realtimeSubject.complete();
2012
+ }
2013
+ } catch (err) {/* already completed */}
2014
+ realtimeSubject = null;
2015
+ stopWebRtcEchoGuard();
2016
+ try {
2017
+ if (realtimeProcessor) {
2018
+ realtimeProcessor.disconnect();
2019
+ realtimeProcessor.onaudioprocess = null;
2020
+ }
2021
+ } catch (err) {}
2022
+ try {
2023
+ if (realtimeMicSource) {
2024
+ realtimeMicSource.disconnect();
2025
+ }
2026
+ } catch (err) {}
2027
+ try {
2028
+ if (realtimeZeroGain) {
2029
+ realtimeZeroGain.disconnect();
2030
+ }
2031
+ } catch (err) {}
2032
+ try {
2033
+ if (realtimeGain) {
2034
+ realtimeGain.disconnect();
2035
+ }
2036
+ } catch (err) {}
2037
+ try {
2038
+ if (realtimeStream) {
2039
+ realtimeStream.getTracks().forEach(function (t) {
2040
+ t.stop();
2041
+ });
2042
+ }
2043
+ } catch (err) {}
2044
+ realtimeProcessor = null;
2045
+ realtimeMicSource = null;
2046
+ realtimeZeroGain = null;
2047
+ realtimeGain = null;
2048
+ realtimeStream = null;
2049
+ flushRealtimePlayback();
2050
+ try {
2051
+ if (realtimeAudioCtx) {
2052
+ realtimeAudioCtx.close();
2053
+ }
2054
+ } catch (err) {}
2055
+ realtimeAudioCtx = null;
2056
+ }
2057
+ function toggleRealtime() {
2058
+ if (isRealtimeActive) {
2059
+ stopRealtimeConversation();
2060
+ return;
2061
+ }
2062
+ startRealtimeConversation();
2063
+ }
2064
+
2065
+ // Resolves with the receive-side statistics of the assistant's audio track (WebRTC transport only), in
2066
+ // the browser's own words: packets lost, jitter, how much the jitter buffer is holding, and how often it
2067
+ // had to conceal or time-stretch audio. Null on the WebSocket transport.
2068
+ function getTransportStats() {
2069
+ if (!realtimePc || typeof realtimePc.getStats !== 'function') {
2070
+ return Promise.resolve(null);
2071
+ }
2072
+ return realtimePc.getStats().then(function (report) {
2073
+ // Where playback actually goes (the "communications" sink puts Chrome's pipeline in communications
2074
+ // mode; empty means the browser default) and the receiver's requested cushion, so a report of
2075
+ // "sounds different in this browser" can be tied to the output path.
2076
+ var result = {
2077
+ transport: 'webrtc',
2078
+ inbound: null,
2079
+ codec: null,
2080
+ sinkId: realtimeRemoteAudioEl && typeof realtimeRemoteAudioEl.sinkId === 'string' ? realtimeRemoteAudioEl.sinkId : null,
2081
+ jitterBufferTargetMs: function () {
2082
+ try {
2083
+ var r = realtimePc.getReceivers().filter(function (x) {
2084
+ return x.track && x.track.kind === 'audio';
2085
+ })[0];
2086
+ return r && 'jitterBufferTarget' in r ? r.jitterBufferTarget : null;
2087
+ } catch (e) {
2088
+ return null;
2089
+ }
2090
+ }()
2091
+ };
2092
+ report.forEach(function (stat) {
2093
+ if (stat.type === 'inbound-rtp' && stat.kind === 'audio') {
2094
+ result.inbound = {
2095
+ packetsReceived: stat.packetsReceived,
2096
+ packetsLost: stat.packetsLost,
2097
+ packetsDiscarded: stat.packetsDiscarded,
2098
+ jitterMs: typeof stat.jitter === 'number' ? Math.round(stat.jitter * 1000) : null,
2099
+ jitterBufferDelayMs: typeof stat.jitterBufferDelay === 'number' && stat.jitterBufferEmittedCount ? Math.round(stat.jitterBufferDelay / stat.jitterBufferEmittedCount * 1000) : null,
2100
+ totalSamplesReceived: stat.totalSamplesReceived,
2101
+ concealedSamples: stat.concealedSamples,
2102
+ concealmentEvents: stat.concealmentEvents,
2103
+ silentConcealedSamples: stat.silentConcealedSamples,
2104
+ insertedSamplesForDeceleration: stat.insertedSamplesForDeceleration,
2105
+ removedSamplesForAcceleration: stat.removedSamplesForAcceleration,
2106
+ audioLevel: stat.audioLevel
2107
+ };
2108
+ if (stat.codecId) {
2109
+ var codec = report.get(stat.codecId);
2110
+ if (codec) {
2111
+ result.codec = {
2112
+ mimeType: codec.mimeType,
2113
+ clockRate: codec.clockRate,
2114
+ channels: codec.channels,
2115
+ sdpFmtpLine: codec.sdpFmtpLine
2116
+ };
2117
+ }
2118
+ }
2119
+ }
2120
+ });
2121
+ return result;
2122
+ })["catch"](function () {
2123
+ return null;
2124
+ });
2125
+ }
2126
+ function playRealtimePcm(bytes) {
2127
+ if (!realtimeAudioCtx || !bytes || bytes.length < 2) {
2128
+ return;
2129
+ }
2130
+ var ctx = realtimeAudioCtx;
2131
+ var sampleCount = Math.floor(bytes.length / 2);
2132
+ var pcm = new Int16Array(bytes.buffer, bytes.byteOffset, sampleCount);
2133
+ var f32 = new Float32Array(sampleCount);
2134
+ for (var i = 0; i < sampleCount; i++) {
2135
+ f32[i] = pcm[i] / 0x8000;
2136
+ }
2137
+ var buffer = ctx.createBuffer(1, sampleCount, ctx.sampleRate);
2138
+ buffer.copyToChannel(f32, 0);
2139
+ var src = ctx.createBufferSource();
2140
+ src.buffer = buffer;
2141
+ src.connect(realtimeGain || ctx.destination);
2142
+ var now = ctx.currentTime;
2143
+ if (realtimePlayHead < now) {
2144
+ realtimePlayHead = now;
2145
+ }
2146
+ src.start(realtimePlayHead);
2147
+ realtimePlayHead += buffer.duration;
2148
+ realtimeSources.push(src);
2149
+ src.onended = function () {
2150
+ realtimeSources = realtimeSources.filter(function (s) {
2151
+ return s !== src;
2152
+ });
2153
+ };
2154
+ }
2155
+ function flushRealtimePlayback() {
2156
+ realtimeSources.forEach(function (s) {
2157
+ try {
2158
+ s.stop();
2159
+ } catch (err) {}
2160
+ });
2161
+ realtimeSources = [];
2162
+ if (realtimeAudioCtx) {
2163
+ realtimePlayHead = realtimeAudioCtx.currentTime;
2164
+ }
2165
+ }
2166
+ function isRealtimeDeployment(deploymentName) {
2167
+ if (!deploymentName) {
2168
+ return false;
2169
+ }
2170
+ return realtimeCapableDeployments.indexOf(deploymentName.toLowerCase()) !== -1;
2171
+ }
2172
+
2173
+ // --- Wire up ---
2174
+ var realtimeBtn = q('realtimeButton');
2175
+ if (realtimeBtn) {
2176
+ realtimeBtn.addEventListener('click', function (e) {
2177
+ // Ignore keyboard-synthesized clicks (Space/Enter) during an active realtime session, so a
2178
+ // push-to-talk Space press can never toggle the session off.
2179
+ if (e && e.detail === 0 && isRealtimeActive) {
2180
+ return;
2181
+ }
2182
+ toggleRealtime();
2183
+ });
2184
+ setupRealtimeAudioSettings();
2185
+ }
2186
+ var deploymentSelect = q('deploymentSelect');
2187
+ if (deploymentSelect) {
2188
+ deploymentSelect.addEventListener('change', function () {
2189
+ applyRealtimeMode(isRealtimeDeployment(deploymentSelect.value));
2190
+ });
2191
+ }
2192
+
2193
+ // Apply the initial input mode: audio-only when the interaction already uses a realtime deployment.
2194
+ applyRealtimeMode(isRealtimeMode || deploymentSelect && isRealtimeDeployment(deploymentSelect.value));
2195
+ var controller = {
2196
+ toggle: toggleRealtime,
2197
+ start: startRealtimeConversation,
2198
+ stop: stopRealtimeConversation,
2199
+ applyMode: applyRealtimeMode,
2200
+ isRealtimeDeployment: isRealtimeDeployment,
2201
+ receivePcm: playRealtimePcm,
2202
+ isActive: function isActive() {
2203
+ return isRealtimeActive;
2204
+ },
2205
+ getState: function getState() {
2206
+ return realtimeState;
2207
+ },
2208
+ // The gate's most recent measurement (level, tracked noise floor, open/closed, assistant audible), for
2209
+ // hosts that want to render a microphone meter or explain why the mic is muted.
2210
+ getGateLevel: function getGateLevel() {
2211
+ return realtimeGate ? realtimeGate.getLevel() : null;
2212
+ },
2213
+ // The browser's own receive-side statistics for the assistant's audio on the WebRTC transport
2214
+ // (packets lost, jitter, jitter-buffer delay, concealment), for diagnosing choppy playback.
2215
+ getTransportStats: getTransportStats
2216
+ };
2217
+
2218
+ // The most recently attached controller, for diagnostics from the console or a test.
2219
+ window.CoreAIRealtime.activeController = controller;
2220
+ return controller;
2221
+ }
2222
+ window.CoreAIRealtime = {
2223
+ attach: attach,
2224
+ createMicGate: createMicGate,
2225
+ // Exposed for tests: the gate's rules are where every one of its historical failures lived, and they are
2226
+ // far easier to pin down with numbers than with audio.
2227
+ decideGate: coreAiGateDecide,
2228
+ createGateState: createGateState,
2229
+ rmsDb: coreAiRmsDb,
2230
+ // Host-agnostic playback helpers, so a host that drives realtime itself still shares one implementation
2231
+ // of these rather than growing its own.
2232
+ routeOutputToPreferredDevice: routeOutputToPreferredDevice,
2233
+ ensurePlayback: ensurePlayback
2234
+ };
2235
+ })(window, document);
2236
+ //# sourceMappingURL=realtime-audio.js.map