@crestapps/ai-chat-ui 2.0.0-preview.215 → 2.0.0-preview.223

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/dist/ai-chat-widget.js +762 -964
  2. package/dist/ai-chat-widget.js.map +1 -1
  3. package/dist/ai-chat-widget.min.js +1 -1
  4. package/dist/ai-chat-widget.min.js.map +1 -1
  5. package/dist/ai-chat.css +0 -2
  6. package/dist/ai-chat.js +2380 -3698
  7. package/dist/ai-chat.js.map +1 -1
  8. package/dist/ai-chat.min.css +1 -1
  9. package/dist/ai-chat.min.js +78 -1
  10. package/dist/ai-chat.min.js.map +1 -1
  11. package/dist/chat-image-carousel.css +0 -2
  12. package/dist/chat-image-carousel.js +306 -482
  13. package/dist/chat-image-carousel.js.map +1 -1
  14. package/dist/chat-image-carousel.min.css +1 -1
  15. package/dist/chat-image-carousel.min.js +1 -1
  16. package/dist/chat-image-carousel.min.js.map +1 -1
  17. package/dist/chat-interaction-settings.js +221 -293
  18. package/dist/chat-interaction-settings.js.map +1 -1
  19. package/dist/chat-interaction-settings.min.js +1 -1
  20. package/dist/chat-interaction-settings.min.js.map +1 -1
  21. package/dist/chat-interaction.js +2330 -3557
  22. package/dist/chat-interaction.js.map +1 -1
  23. package/dist/chat-interaction.min.js +62 -1
  24. package/dist/chat-interaction.min.js.map +1 -1
  25. package/dist/chat-markers.js +193 -485
  26. package/dist/chat-markers.js.map +1 -1
  27. package/dist/chat-markers.min.js +2 -1
  28. package/dist/chat-markers.min.js.map +1 -1
  29. package/dist/chat-widget.css +0 -2
  30. package/dist/chat-widget.min.css +1 -1
  31. package/dist/document-drop-zone.css +0 -2
  32. package/dist/document-drop-zone.js +127 -153
  33. package/dist/document-drop-zone.js.map +1 -1
  34. package/dist/document-drop-zone.min.css +1 -1
  35. package/dist/document-drop-zone.min.js +1 -1
  36. package/dist/document-drop-zone.min.js.map +1 -1
  37. package/dist/realtime-audio.js +1639 -2504
  38. package/dist/realtime-audio.js.map +1 -1
  39. package/dist/realtime-audio.min.js +3 -1
  40. package/dist/realtime-audio.min.js.map +1 -1
  41. package/dist/technical-name-generator.js +58 -80
  42. package/dist/technical-name-generator.js.map +1 -1
  43. package/dist/technical-name-generator.min.js +1 -1
  44. package/dist/technical-name-generator.min.js.map +1 -1
  45. package/package.json +1 -5
  46. package/dist/ai-chat.css.map +0 -1
  47. package/dist/chat-image-carousel.css.map +0 -1
  48. package/dist/chat-widget.css.map +0 -1
  49. package/dist/document-drop-zone.css.map +0 -1
@@ -1,2507 +1,1642 @@
1
1
  /*
2
- ** NOTE: This file is generated by Gulp and should not be edited directly!
3
- ** Any changes made directly to this file will be overwritten next time its asset group is processed by Gulp.
2
+ ** NOTE: This file is generated by the assets build (npm run build) and should not be edited directly!
3
+ ** Edit its sources under Assets/ and rebuild; see Assets.json for the files it is built from.
4
4
  */
5
-
6
- /*
7
- * Shared realtime (speech-to-speech) audio controller for interaction-style chat surfaces.
8
- *
9
- * Encapsulates the browser side of a realtime voice conversation: PCM16 mic capture and streaming,
10
- * PCM playback with volume, the half-duplex echo guard, push-to-talk, the per-device audio settings
11
- * popover, and switching the input between text and audio-only when a realtime deployment is selected.
12
- *
13
- * It is host-agnostic: the host (an MVC inline script or the Blazor chat-interaction Vue app) supplies the
14
- * SignalR connection, a sendStart callback (so each surface can address its own hub method/identifier),
15
- * element selectors, and a few display hooks. Reused so realtime lives in one place, not one copy per host.
16
- *
17
- * Usage:
18
- * var controller = window.CoreAIRealtime.attach({
19
- * connection: conn,
20
- * ensureConnected: function () { return startPromise; }, // () => Promise<any>
21
- * sendStart: function (subject, voice, language, silenceMs, vadThreshold) { ... },
22
- * voiceName: 'alloy',
23
- * capableDeployments: ['gpt-realtime'],
24
- * realtimeEnabled: true,
25
- * selectors: { realtimeButton, input, sendButton, micButton, conversationButton, deploymentSelect },
26
- * onActivate: function () { ... }, // realtime session started (host: show transcripts)
27
- * onDeactivate: function () { ... }, // realtime session stopped
28
- * onEnterRealtimeMode: function () { ... } // switched to audio-only (host: stop any STT recording)
29
- * });
30
- * // then route audio to it: controller.receivePcm(bytes) // from ReceiveAudioChunk 'audio/pcm'
31
- */
32
- (function (window, document) {
33
- 'use strict';
34
-
35
- var REALTIME_SAMPLE_RATE = 24000;
36
-
37
- // --- Microphone gate --------------------------------------------------------------------------------------
38
- //
39
- // The outbound mic is silenced unless the user is genuinely speaking, so the model never receives silence,
40
- // room noise, or the assistant's echo — which is what stopped it answering itself and stopped the input
41
- // transcriber hallucinating phantom "Thank you" turns.
42
- //
43
- // - push-to-talk: open only while the key is held;
44
- // - barge-in on: open whenever the user speaks — even over the assistant (interrupts);
45
- // - barge-in off: open only when the user speaks and the assistant is silent (wait your turn).
46
- //
47
- // There are no user-facing modes or thresholds: everything the gate needs it measures for itself. It tracks
48
- // the room's noise floor, and — the part that makes open speakers work — it learns how loud the assistant's
49
- // own voice comes back into the microphone after echo cancellation (the echo return level). While the
50
- // assistant is audible, only a voice clearly louder than that expected echo counts as the user interrupting;
51
- // with a headset the expected echo is the room floor and interruptions are cheap, with loud speakers next to
52
- // the microphone it is high and the gate simply waits its turn. Echo cancellation is still the first line of
53
- // defence; this is what keeps the model from hearing itself when cancellation is not enough.
54
- //
55
- // It runs in an AudioWorklet rather than on a requestAnimationFrame loop. rAF is paused for hidden documents,
56
- // so the old gate froze in whatever state it held the moment the user switched tabs: stuck closed meant they
57
- // were never heard again. The audio thread keeps running regardless of visibility, sees every sample instead
58
- // of a 10 ms window sampled 60 times a second, and can hold a look-ahead delay so the gate is already open
59
- // when the first consonant arrives.
60
- //
61
- // Exported as window.CoreAIRealtime.createMicGate so both chat clients share one implementation.
62
- var REALTIME_GATE_LOOKAHEAD_MS = 80;
63
-
64
- /*
65
- * The gate's decision, as a pure function of the current state and one block of measurements.
66
- *
67
- * It lives on its own for two reasons. It has to run inside an AudioWorklet, which cannot close over module
68
- * scope — so it is stringified into the worklet below rather than duplicated there. And it is the part worth
69
- * testing exhaustively: every one of this gate's historical failures (a quiet microphone never opening it, a
70
- * loud room's echo opening it, one sentence chopped into several turns) is a rule in here, and each is far
71
- * easier to pin down with numbers than with audio.
72
- *
73
- * `state` is mutated and returned. Callers create it with createGateState().
74
- *
75
- * input: { micDb, assistantDb, nowMs, mode }
76
- * mode: { pushToTalk, pttActive, bargeIn, speechHangoverMs }
77
- */
78
- function coreAiGateDecide(state, input) {
79
- var micDb = input.micDb;
80
- var assistantDb = typeof input.assistantDb === 'number' ? input.assistantDb : -100;
81
- var nowMs = input.nowMs;
82
- var mode = input.mode || {};
83
-
84
- // Elapsed time since the previous decision, so the time constants below mean the same thing whether the
85
- // gate runs per 128-sample block on the audio thread or on a 20 ms timer.
86
- var dtMs = state.lastMs === null ? 0 : Math.max(0, Math.min(50, nowMs - state.lastMs));
87
- state.lastMs = nowMs;
88
-
89
- // "The assistant is speaking" must survive the gaps between its words. Without this hangover the
90
- // half-duplex gate re-opened in every inter-word pause and fed the echo tail straight back to the model.
91
- if (assistantDb > -50) {
92
- state.assistantUntil = nowMs + 400;
93
- }
94
- var assistantSpeaking = nowMs < state.assistantUntil;
95
-
96
- // Peak-held assistant level. The echo of a word reaches the microphone 50-200 ms after the word was played,
97
- // so the expected echo has to be judged against what the assistant was doing a moment ago, not against the
98
- // instantaneous level (which is silent between its words). Decays at 20 dB/s.
99
- if (assistantDb > state.assistantPeakDb) {
100
- state.assistantPeakDb = assistantDb;
101
- } else {
102
- state.assistantPeakDb = Math.max(-100, state.assistantPeakDb - 0.02 * dtMs);
103
- }
104
-
105
- // Adaptive noise floor. It falls to a quieter room quickly, and rises only while the gate is shut and the
106
- // assistant is silent — that is, only from audio we have judged is neither speech nor echo.
107
- //
108
- // Letting it rise during speech makes the gate close on the talker: the estimate climbs towards their own
109
- // voice, the level stops clearing floor + margin, and the sentence is cut part-way through.
110
- //
111
- // Cold start. Digital silence is not a measurement: a capture pipeline delivers a few empty blocks before
112
- // the microphone is live (and noise suppression emits exact zeros in a quiet headset), and seeding on
113
- // those anchored the floor at the clamp and made the room itself look like speech. The first real block
114
- // is not trustworthy either — it may well be the user already talking, and a floor seeded at their own
115
- // level meant the gate never opened for them at all. So the seed is the first real block capped at a
116
- // typical quiet-room level; the fast downward adaptation below then finds the real floor within a few
117
- // hundred milliseconds. (Seeding from the quietest block instead was tried and rejected: block-to-block
118
- // level varies by several dB even on steady room noise, so a floor set at the minimum let ordinary noise
119
- // peaks open the gate.)
120
- if (micDb <= -95) {
121
- if (state.floorDb === null) {
122
- state.open = false;
123
- return state;
124
- }
125
- } else if (state.floorDb === null) {
126
- state.floorDb = Math.min(micDb, -50);
127
- }
128
- if (micDb < state.floorDb) {
129
- state.floorDb += (micDb - state.floorDb) * Math.min(1, 0.0075 * dtMs);
130
- } else if (!assistantSpeaking) {
131
- // Rising. While the gate is shut this is responsive (~2 s). While it is open the audio is presumed to
132
- // be speech and the floor moves slowly (~10 s), because learning from the talker's own voice is what
133
- // closed the gate part-way through their sentence — but not so slowly that a steady sound could hold
134
- // the gate open indefinitely. Past 10 s the assumption is abandoned altogether: nobody speaks
135
- // continuously for that long, so it is ambient noise and the floor tracks it normally.
136
- var sustained = state.openSince !== null && nowMs - state.openSince > 10000;
137
- var risingPerMs = state.open && !sustained ? 0.0001 : 0.00056;
138
- state.floorDb += (micDb - state.floorDb) * Math.min(1, risingPerMs * dtMs);
139
- }
140
- if (state.floorDb < -75) {
141
- state.floorDb = -75;
142
- }
143
-
144
- // Echo return level: how loud the assistant comes back into the microphone, relative to its own level,
145
- // after echo cancellation has done what it can. Learned only while the assistant is audible and the gate
146
- // is shut, i.e. from audio we have judged is not the user. It warms up fast for the first second so the
147
- // very first reply is protected, then follows slowly so a user interrupting cannot be mistaken for echo
148
- // in the time it takes to confirm them (below). It falls slowly too, so turning the volume down is
149
- // noticed within a couple of seconds.
150
- if (assistantSpeaking && !state.open && assistantDb > -50 && !mode.pushToTalk) {
151
- var erl = micDb - state.assistantPeakDb;
152
- if (state.echoReturnDb === null) {
153
- state.echoReturnDb = erl;
154
- } else {
155
- var warm = state.echoLearnedMs < 1000;
156
- var ratePerMs = erl > state.echoReturnDb ? warm ? 0.02 : 0.0012 : 0.0006;
157
- state.echoReturnDb += (erl - state.echoReturnDb) * Math.min(1, ratePerMs * dtMs);
158
- }
159
- state.echoLearnedMs += dtMs;
160
- }
161
-
162
- // Where the assistant's echo is expected to sit right now, or far below anything if it has never been
163
- // heard (a headset).
164
- var expectedEchoDb = state.echoReturnDb === null ? -100 : state.assistantPeakDb + state.echoReturnDb;
165
- var openThreshold, holdThreshold;
166
- if (assistantSpeaking) {
167
- // Interrupting: the voice must clearly beat both the room and the expected echo, and be real speech
168
- // in absolute terms — echo residual can sit well above a very quiet room's floor without being
169
- // anywhere near a voice.
170
- openThreshold = Math.max(state.floorDb + 12, expectedEchoDb + 8, -45);
171
- holdThreshold = Math.max(state.floorDb + 3, expectedEchoDb + 3);
172
- } else {
173
- openThreshold = state.floorDb + 9;
174
- holdThreshold = state.floorDb + 3;
175
- }
176
- var hangoverMs = mode.speechHangoverMs || 2000;
177
- if (micDb > openThreshold) {
178
- if (assistantSpeaking && !state.open) {
179
- // An interruption has to be sustained for a moment before it counts. A single loud block over the
180
- // assistant is far more often a click, a cough or an echo peak than the user; the look-ahead delay
181
- // keeps the opening of a genuine interruption from being lost while we wait.
182
- if (state.aboveSince === null) {
183
- state.aboveSince = nowMs;
184
- }
185
- if (nowMs - state.aboveSince >= 250) {
186
- state.speakUntil = nowMs + hangoverMs;
187
- }
188
- } else {
189
- // The hangover must outlast the provider's end-of-turn detection. The gate emits digital silence
190
- // when it closes, so whichever of the two expires first is what actually ends the user's turn —
191
- // and a gate that closes first cuts people off mid-sentence and answers half a question.
192
- state.speakUntil = nowMs + hangoverMs;
193
- }
194
- } else {
195
- state.aboveSince = null;
196
- if (state.open && micDb > holdThreshold) {
197
- // Hysteresis: once open, stay open until the level really settles back towards the floor, so one
198
- // sentence is not chopped into several provider turns.
199
- state.speakUntil = Math.max(state.speakUntil, nowMs + 250);
200
- }
201
- }
202
- var userActive = nowMs < state.speakUntil;
203
- if (!assistantSpeaking && state.wasAssistantSpeaking) {
204
- state.listenGraceUntil = nowMs + 1500;
205
- }
206
- state.wasAssistantSpeaking = assistantSpeaking;
207
- state.assistantSpeaking = assistantSpeaking;
208
- if (mode.pushToTalk) {
209
- state.open = !!mode.pttActive;
210
- } else if (mode.bargeIn) {
211
- state.open = userActive;
212
- } else {
213
- state.open = !assistantSpeaking && (userActive || nowMs < state.listenGraceUntil);
214
- }
215
- if (state.open && state.openSince === null) {
216
- state.openSince = nowMs;
217
- } else if (!state.open) {
218
- state.openSince = null;
219
- }
220
- return state;
221
- }
222
- function createGateState() {
223
- return {
224
- // Null until the first measurement, so the floor starts from the room rather than from a guess.
225
- floorDb: null,
226
- lastMs: null,
227
- speakUntil: 0,
228
- assistantUntil: 0,
229
- assistantPeakDb: -100,
230
- // Null until the assistant has been heard while the gate was shut (never, with a headset).
231
- echoReturnDb: null,
232
- echoLearnedMs: 0,
233
- aboveSince: null,
234
- listenGraceUntil: 0,
235
- wasAssistantSpeaking: false,
236
- assistantSpeaking: false,
237
- open: false,
238
- openSince: null
239
- };
240
- }
241
-
242
- // Root-mean-square of a block, in dBFS. RMS (not peak) tracks perceived loudness, and dB is the only scale on
243
- // which a fixed "N above the noise floor" rule behaves the same in a quiet room and a loud one.
244
- function coreAiRmsDb(block) {
245
- if (!block || !block.length) {
246
- return -100;
247
- }
248
- var sum = 0;
249
- for (var i = 0; i < block.length; i++) {
250
- sum += block[i] * block[i];
251
- }
252
- var rms = Math.sqrt(sum / block.length);
253
- return rms > 1e-7 ? 20 * Math.log10(rms) : -100;
254
- }
255
-
256
- // The gate decision itself, as an AudioWorkletProcessor. Registered from a Blob URL so it needs no separate
257
- // asset file (and no host-specific path configuration). The decision and level functions are stringified in
258
- // rather than rewritten, so the worklet and the fallback below can never disagree.
259
- var REALTIME_GATE_WORKLET_SOURCE = ['var decide = ' + coreAiGateDecide.toString() + ';', 'var newState = ' + createGateState.toString() + ';', 'var rmsDb = ' + coreAiRmsDb.toString() + ';', 'class CoreAiMicGateProcessor extends AudioWorkletProcessor {', ' constructor(options) {', ' super();', ' var o = (options && options.processorOptions) || {};',
260
- // The gate reads the live signal but emits it delayed, so by the time a word's first sample leaves here
261
- // the gate has already seen it and opened. Without this the initial consonant of every utterance was
262
- // clipped, which is exactly what degrades transcription and the model's understanding.
263
- ' this.delay = new Float32Array(Math.max(1, Math.round(sampleRate * (o.lookaheadMs || 80) / 1000)));', ' this.delayPos = 0;', ' this.gain = 0;', ' this.mode = o.mode || {};', ' this.state = newState();', ' this.lastPost = 0;', ' this.port.onmessage = function (e) { this.mode = e.data || {}; }.bind(this);', ' }', ' process(inputs, outputs) {', ' var mic = (inputs[0] && inputs[0][0]) || null;', ' var assistant = (inputs[1] && inputs[1][0]) || null;', ' var out = (outputs[0] && outputs[0][0]) || null;', ' if (!out) { return true; }', ' var micDb = rmsDb(mic);', ' decide(this.state, { micDb: micDb, assistantDb: rmsDb(assistant), nowMs: currentTime * 1000, mode: this.mode });', ' var target = this.state.open ? 1 : 0;',
264
- // Ramp over ~10 ms rather than switching instantly, so opening and closing never clicks.
265
- ' var step = 1 / (sampleRate * 0.01);', ' var delay = this.delay;', ' var pos = this.delayPos;', ' var len = delay.length;', ' for (var i = 0; i < out.length; i++) {', ' var delayed = delay[pos];', ' delay[pos] = mic ? mic[i] : 0;', ' pos = (pos + 1) % len;', ' if (this.gain < target) { this.gain = Math.min(target, this.gain + step); }', ' else if (this.gain > target) { this.gain = Math.max(target, this.gain - step); }', ' out[i] = delayed * this.gain;', ' }', ' this.delayPos = pos;', ' if (currentTime * 1000 - this.lastPost > 100) {', ' this.lastPost = currentTime * 1000;', ' this.port.postMessage({ micDb: micDb, floorDb: this.state.floorDb, echoReturnDb: this.state.echoReturnDb, open: this.state.open, assistantSpeaking: this.state.assistantSpeaking });', ' }', ' return true;', ' }', '}', 'registerProcessor("coreai-mic-gate", CoreAiMicGateProcessor);'].join('\n');
266
-
267
- // How many samples make up one frame sent to the server. Unchanged from the ScriptProcessorNode this
268
- // replaces, so the server sees exactly the cadence it always has.
269
- var REALTIME_CAPTURE_FRAME_SAMPLES = 4096;
270
-
271
- // Microphone capture as an AudioWorkletProcessor. ScriptProcessorNode, which this replaces, has been
272
- // deprecated for years and ran its callback on the main thread, where a long task shows up as a gap in the
273
- // captured audio. This buffers on the audio thread and posts whole frames instead. It deliberately keeps
274
- // emitting frames when nothing is connected to its input: the server's voice-activity detector needs a
275
- // continuous stream to notice a pause, and the gate's own worklet takes a moment to come up.
276
- var REALTIME_CAPTURE_WORKLET_SOURCE = ['class CoreAiCaptureProcessor extends AudioWorkletProcessor {', ' constructor(options) {', ' super();', ' var o = (options && options.processorOptions) || {};', ' this.size = Math.max(128, o.frameSamples || 4096);', ' this.buf = new Float32Array(this.size);', ' this.pos = 0;', ' }', ' process(inputs) {', ' var input = (inputs[0] && inputs[0][0]) || null;',
277
- // A render quantum is 128 frames; with no input connected there are no channels to read, so emit the
278
- // same length of silence rather than stalling the stream.
279
- ' var n = input ? input.length : 128;', ' for (var i = 0; i < n; i++) {', ' this.buf[this.pos++] = input ? input[i] : 0;', ' if (this.pos === this.size) {',
280
- // postMessage structured-clones the buffer, so the processor keeps filling its own copy.
281
- ' this.port.postMessage(this.buf);', ' this.pos = 0;', ' }', ' }', ' return true;', ' }', '}', 'registerProcessor("coreai-pcm-capture", CoreAiCaptureProcessor);'].join('\n');
282
-
283
- /*
284
- * Resolves with the node that turns captured microphone audio into fixed-size frames, handing each one to
285
- * onFrame as a Float32Array. Prefers an AudioWorkletNode and falls back to a ScriptProcessorNode only where
286
- * AudioWorklet is unavailable, so an older browser keeps working (and keeps its deprecation warning).
287
- */
288
- function createRealtimeCaptureNode(ctx, onFrame) {
289
- function scriptProcessorNode() {
290
- var node = ctx.createScriptProcessor(REALTIME_CAPTURE_FRAME_SAMPLES, 1, 1);
291
- node.onaudioprocess = function (event) {
292
- onFrame(event.inputBuffer.getChannelData(0));
293
- };
294
- return node;
295
- }
296
- var blobUrl = null;
297
- if (ctx.audioWorklet && typeof ctx.audioWorklet.addModule === 'function' && typeof AudioWorkletNode === 'function') {
298
- try {
299
- blobUrl = URL.createObjectURL(new Blob([REALTIME_CAPTURE_WORKLET_SOURCE], {
300
- type: 'application/javascript'
301
- }));
302
- } catch (err) {
303
- blobUrl = null;
304
- }
305
- }
306
- if (!blobUrl) {
307
- return Promise.resolve(scriptProcessorNode());
308
- }
309
- return ctx.audioWorklet.addModule(blobUrl).then(function () {
310
- try {
311
- URL.revokeObjectURL(blobUrl);
312
- } catch (e) {}
313
- var node = new AudioWorkletNode(ctx, 'coreai-pcm-capture', {
314
- numberOfInputs: 1,
315
- numberOfOutputs: 1,
316
- outputChannelCount: [1],
317
- processorOptions: {
318
- frameSamples: REALTIME_CAPTURE_FRAME_SAMPLES
319
- }
320
- });
321
- node.port.onmessage = function (e) {
322
- if (e.data) {
323
- onFrame(e.data);
324
- }
325
- };
326
- return node;
327
- })["catch"](function (err) {
328
- try {
329
- URL.revokeObjectURL(blobUrl);
330
- } catch (e) {}
331
- if (window.console && console.warn) {
332
- console.warn('The realtime capture worklet could not be loaded; falling back to a ScriptProcessorNode.', err);
333
- }
334
- return scriptProcessorNode();
335
- });
336
- }
337
-
338
- /*
339
- * Builds a microphone gate for a captured stream.
340
- *
341
- * createMicGate(stream, mode) => Promise<{
342
- * track, // the gated track to send (or the raw mic when Web Audio is missing)
343
- * setMode(mode), // { pushToTalk, pttActive, bargeIn, speechHangoverMs }
344
- * attachAssistantStream(stream),// so the gate knows when the assistant is audible
345
- * getLevel(), // last measurement, for a mic meter
346
- * stop()
347
- * }>
348
- */
349
- function createMicGate(rawStream, mode) {
350
- var rawTrack = rawStream && rawStream.getAudioTracks && rawStream.getAudioTracks()[0] || null;
351
- var state = {
352
- ctx: null,
353
- node: null,
354
- micSource: null,
355
- remoteSource: null,
356
- dest: null,
357
- timer: 0,
358
- level: null,
359
- fallback: null,
360
- track: rawTrack,
361
- stopped: false
362
- };
363
- var gate = {
364
- track: rawTrack,
365
- // True when the gate is running on the audio thread. The fallback still gates, but on a timer that a
366
- // hidden tab throttles, so hosts and tests can tell the two apart.
367
- usingWorklet: false,
368
- getLevel: function getLevel() {
369
- return state.level;
370
- },
371
- setMode: function setMode(next) {
372
- mode = next || mode;
373
- if (state.node && state.node.port) {
374
- try {
375
- state.node.port.postMessage(mode);
376
- } catch (e) {}
377
- }
378
- if (state.fallback) {
379
- state.fallback.mode = mode;
380
- }
381
- },
382
- attachAssistantStream: function attachAssistantStream(remoteStream) {
383
- attachAssistant(state, remoteStream);
384
- },
385
- stop: function stop() {
386
- state.stopped = true;
387
- if (state.timer) {
388
- try {
389
- window.clearInterval(state.timer);
390
- } catch (e) {}
391
- state.timer = 0;
392
- }
393
- if (state.node) {
394
- try {
395
- state.node.port.onmessage = null;
396
- state.node.disconnect();
397
- } catch (e) {}
398
- state.node = null;
399
- }
400
- if (state.micSource) {
401
- try {
402
- state.micSource.disconnect();
403
- } catch (e) {}
404
- state.micSource = null;
405
- }
406
- if (state.remoteSource) {
407
- try {
408
- state.remoteSource.disconnect();
409
- } catch (e) {}
410
- state.remoteSource = null;
411
- }
412
- if (state.dest) {
413
- try {
414
- state.dest.disconnect();
415
- } catch (e) {}
416
- state.dest = null;
417
- }
418
- if (state.ctx) {
419
- try {
420
- state.ctx.close();
421
- } catch (e) {}
422
- state.ctx = null;
423
- }
424
- state.fallback = null;
425
- }
426
- };
427
- var AudioCtor = window.AudioContext || window.webkitAudioContext;
428
- if (!AudioCtor || !rawTrack) {
429
- return Promise.resolve(gate);
430
- }
431
- try {
432
- state.ctx = new AudioCtor();
433
- // A context created under a strict autoplay policy starts suspended, and a suspended gate emits
434
- // nothing at all — the model would simply never hear the user.
435
- if (state.ctx.state === 'suspended' && typeof state.ctx.resume === 'function') {
436
- state.ctx.resume()["catch"](function () {});
437
- }
438
- } catch (err) {
439
- state.ctx = null;
440
- return Promise.resolve(gate);
441
- }
442
- var blobUrl = null;
443
- if (state.ctx.audioWorklet && typeof state.ctx.audioWorklet.addModule === 'function') {
444
- try {
445
- blobUrl = URL.createObjectURL(new Blob([REALTIME_GATE_WORKLET_SOURCE], {
446
- type: 'application/javascript'
447
- }));
448
- } catch (err) {
449
- blobUrl = null;
450
- }
451
- }
452
- if (!blobUrl) {
453
- gate.track = buildFallbackGate(state, rawStream, rawTrack, mode);
454
- return Promise.resolve(gate);
455
- }
456
- return state.ctx.audioWorklet.addModule(blobUrl).then(function () {
457
- try {
458
- URL.revokeObjectURL(blobUrl);
459
- } catch (e) {}
460
- if (state.stopped) {
461
- return gate;
462
- }
463
- var node = new AudioWorkletNode(state.ctx, 'coreai-mic-gate', {
464
- numberOfInputs: 2,
465
- numberOfOutputs: 1,
466
- outputChannelCount: [1],
467
- processorOptions: {
468
- lookaheadMs: REALTIME_GATE_LOOKAHEAD_MS,
469
- mode: mode
470
- }
471
- });
472
- state.node = node;
473
- node.port.onmessage = function (e) {
474
- state.level = e.data || null;
475
- // Also published on the module for support diagnostics (a host can read it from the console
476
- // without holding a reference to the gate).
477
- window.CoreAIRealtime.lastGateLevel = state.level;
478
- };
479
- state.micSource = state.ctx.createMediaStreamSource(rawStream);
480
- state.micSource.connect(node, 0, 0);
481
- state.dest = state.ctx.createMediaStreamDestination();
482
- node.connect(state.dest);
483
- if (state.pendingRemoteStream) {
484
- attachAssistant(state, state.pendingRemoteStream);
485
- }
486
- gate.track = state.dest.stream.getAudioTracks()[0] || rawTrack;
487
- gate.usingWorklet = true;
488
- return gate;
489
- })["catch"](function (err) {
490
- try {
491
- URL.revokeObjectURL(blobUrl);
492
- } catch (e) {}
493
- if (window.console && console.warn) {
494
- console.warn('The realtime microphone gate worklet could not be loaded; using the simpler gate.', err);
495
- }
496
- gate.track = buildFallbackGate(state, rawStream, rawTrack, mode);
497
- return gate;
498
- });
499
- }
500
- function attachAssistant(state, remoteStream) {
501
- state.pendingRemoteStream = remoteStream || null;
502
- if (!state.ctx || !remoteStream || state.stopped) {
503
- return;
504
- }
505
- try {
506
- if (state.remoteSource) {
507
- try {
508
- state.remoteSource.disconnect();
509
- } catch (e) {}
510
- }
511
- var src = state.ctx.createMediaStreamSource(remoteStream);
512
- state.remoteSource = src;
513
- if (state.node) {
514
- src.connect(state.node, 0, 1);
515
- return;
516
- }
517
- if (state.fallback) {
518
- var analyser = state.ctx.createAnalyser();
519
- analyser.fftSize = 2048;
520
- src.connect(analyser);
521
- state.fallback.remoteAnalyser = analyser;
522
- state.fallback.remoteData = new Float32Array(analyser.fftSize);
523
- }
524
- } catch (e) {
525
- // Assistant-level detection unavailable; the gate still opens on user speech.
526
- }
527
- }
528
-
529
- // Fallback for browsers without AudioWorklet: the same decision on a timer instead of the audio thread. A
530
- // timer is throttled in background tabs — worse than the worklet, better than nothing — and there is no
531
- // look-ahead here, so utterance onsets are still clipped.
532
- function buildFallbackGate(state, rawStream, rawTrack, mode) {
533
- try {
534
- var ctx = state.ctx;
535
- var source = ctx.createMediaStreamSource(rawStream);
536
- state.micSource = source;
537
- var analyser = ctx.createAnalyser();
538
- analyser.fftSize = 2048;
539
- source.connect(analyser);
540
- var gain = ctx.createGain();
541
- gain.gain.value = 0;
542
- source.connect(gain);
543
- var dest = ctx.createMediaStreamDestination();
544
- state.dest = dest;
545
- gain.connect(dest);
546
- var f = {
547
- mode: mode,
548
- remoteAnalyser: null,
549
- remoteData: null
550
- };
551
- state.fallback = f;
552
- if (state.pendingRemoteStream) {
553
- attachAssistant(state, state.pendingRemoteStream);
554
- }
555
- var data = new Float32Array(analyser.fftSize);
556
-
557
- // The same decision the worklet runs, on a timer instead of the audio thread. The floor adapts faster
558
- // here only because this loop runs ~50x/s rather than ~375x/s; everything else is shared.
559
- var gateState = createGateState();
560
- state.timer = window.setInterval(function () {
561
- analyser.getFloatTimeDomainData(data);
562
- var micDb = coreAiRmsDb(data);
563
- var assistantDb = -100;
564
- if (f.remoteAnalyser && f.remoteData) {
565
- f.remoteAnalyser.getFloatTimeDomainData(f.remoteData);
566
- assistantDb = coreAiRmsDb(f.remoteData);
567
- }
568
- var nowMs = window.performance && performance.now ? performance.now() : Date.now();
569
- coreAiGateDecide(gateState, {
570
- micDb: micDb,
571
- assistantDb: assistantDb,
572
- nowMs: nowMs,
573
- mode: f.mode || {}
574
- });
575
- state.level = {
576
- micDb: micDb,
577
- floorDb: gateState.floorDb,
578
- open: gateState.open,
579
- assistantSpeaking: gateState.assistantSpeaking
580
- };
581
- var target = gateState.open ? 1 : 0;
582
- try {
583
- gain.gain.setTargetAtTime(target, ctx.currentTime, 0.015);
584
- } catch (e) {
585
- gain.gain.value = target;
586
- }
587
- }, 20);
588
- return dest.stream.getAudioTracks()[0] || rawTrack;
589
- } catch (err) {
590
- // Web Audio unavailable; send the raw mic (ungated).
591
- return rawTrack;
592
- }
593
- }
594
-
595
- /*
596
- * Routes an audio element to the device the browser's echo canceller couples to, or to an explicitly chosen
597
- * one. Preference order:
598
- * 1. The caller's chosen device, when they picked one.
599
- * 2. The "communications" sink (Chromium). Rendering here doesn't just pick a device — it puts the browser's
600
- * audio pipeline in communications mode, which is what actually couples playback with the microphone's
601
- * echo canceller, and what keeps full duplex stable across turns.
602
- * 3. The concrete device the "default" alias points at, matched by groupId.
603
- * If neither alias exists — Firefox lists only concrete outputs — nothing is set and the browser's own default
604
- * stands. Guessing there routed the assistant to whatever was enumerated first (often an HDMI monitor), so it
605
- * appeared silent.
606
- */
607
- function routeOutputToPreferredDevice(el, chosenDeviceId) {
608
- if (!el || typeof el.setSinkId !== 'function') {
609
- return;
610
- }
611
- if (chosenDeviceId) {
612
- try {
613
- el.setSinkId(chosenDeviceId)["catch"](function () {});
614
- } catch (e) {}
615
- return;
616
- }
617
- if (!navigator.mediaDevices || !navigator.mediaDevices.enumerateDevices) {
618
- return;
619
- }
620
- navigator.mediaDevices.enumerateDevices().then(function (devices) {
621
- var outputs = devices.filter(function (d) {
622
- return d.kind === 'audiooutput';
623
- });
624
- if (!outputs.length) {
625
- return;
626
- }
627
- var target = '';
628
- for (var i = 0; i < outputs.length; i++) {
629
- if (outputs[i].deviceId === 'communications') {
630
- target = 'communications';
631
- break;
632
- }
633
- }
634
- if (!target) {
635
- var defaultEntry = null;
636
- for (var d = 0; d < outputs.length; d++) {
637
- if (outputs[d].deviceId === 'default') {
638
- defaultEntry = outputs[d];
639
- break;
640
- }
641
- }
642
- if (defaultEntry && defaultEntry.groupId) {
643
- for (var j = 0; j < outputs.length; j++) {
644
- if (outputs[j].deviceId !== 'default' && outputs[j].deviceId !== 'communications' && outputs[j].groupId === defaultEntry.groupId) {
645
- target = outputs[j].deviceId;
646
- break;
647
- }
648
- }
649
- }
650
- }
651
- if (target) {
652
- try {
653
- el.setSinkId(target)["catch"](function () {});
654
- } catch (e) {}
655
- }
656
- })["catch"](function () {});
657
- }
658
-
659
- // Autoplay policies (Firefox's "Block audio and video", locked-down profiles) can reject playback even after a
660
- // click, and a rejected play() is silent — the assistant simply never becomes audible. Ask explicitly and
661
- // report the failure instead of leaving the user with a mute conversation.
662
- function ensurePlayback(el, onBlocked) {
663
- if (!el || typeof el.play !== 'function') {
664
- return;
665
- }
666
- var played = el.play();
667
- if (played && typeof played["catch"] === 'function') {
668
- played["catch"](function (err) {
669
- if (typeof onBlocked === 'function') {
670
- onBlocked(err);
671
- }
672
- if (window.console && console.warn) {
673
- console.warn('The browser blocked realtime audio playback. Interact with the page to enable audio.', err);
674
- }
675
- });
676
- }
677
- }
678
- function attach(opts) {
679
- opts = opts || {};
680
- var connection = opts.connection;
681
- var sel = opts.selectors || {};
682
- var realtimeVoiceName = opts.voiceName || '';
683
- var realtimeCapableDeployments = (opts.capableDeployments || []).map(function (n) {
684
- return (n || '').toLowerCase();
685
- });
686
- var isRealtimeMode = !!opts.realtimeEnabled;
687
- var noop = function noop() {};
688
- var onActivate = opts.onActivate || noop;
689
- var onDeactivate = opts.onDeactivate || noop;
690
- // Reports whether a session is live, every time the button is repainted, so a host that shows or hides
691
- // controls around the session reads the same flag the button does. onActivate/onDeactivate are events
692
- // and can be missed -- the transport fallback deactivates WebRTC and activates the WebSocket without
693
- // pairing them -- which left a host tracking the session itself able to drift out of step with the
694
- // button, showing "Start speaking" over a message box that was still hidden.
695
- var onSessionStateChanged = opts.onSessionStateChanged || noop;
696
- var onEnterRealtimeMode = opts.onEnterRealtimeMode || noop;
697
- // The server tells us a user utterance was captured before it has been transcribed, so the host can show a
698
- // placeholder in the right position; onUserTurnDropped removes it when the utterance produced nothing.
699
- var onUserTurnPending = opts.onUserTurnPending || noop;
700
- var onUserTurnDropped = opts.onUserTurnDropped || noop;
701
- // Hosts are localizable elsewhere, so let them override the handful of strings this module renders.
702
- // Anything not supplied falls back to the English default.
703
- var strings = opts.strings || {};
704
- function localize(key, fallback) {
705
- return typeof strings[key] === 'string' ? strings[key] : fallback;
706
- }
707
- var ensureConnected = opts.ensureConnected || function () {
708
- return Promise.resolve(true);
709
- };
710
- var sendStart = opts.sendStart || noop;
711
- // Optional: resolve the assistant voice live at session start (e.g. from a settings picker) so it
712
- // reflects the current selection rather than the value captured when the controller was attached.
713
- var getVoiceName = opts.getVoiceName || null;
714
- // Optional server-relay WebRTC transport: when the server advertises it and the callback is supplied, the
715
- // controller uses WebRTC (real acoustic echo cancellation) instead of the PCM-over-SignalR path.
716
- var sendStartWebRtc = opts.sendStartWebRtc || null;
717
- var webRtcEnabled = opts.webRtcEnabled === true && typeof window.RTCPeerConnection === 'function' && !!sendStartWebRtc;
718
- // A host-supplied override; normally the servers are fetched from the hub right before the peer is created
719
- // (see resolveIceServers) so configured TURN relays — and their short-lived credentials — are actually used.
720
- var webRtcIceServers = Array.isArray(opts.webRtcIceServers) && opts.webRtcIceServers.length ? opts.webRtcIceServers : null;
721
- var DEFAULT_ICE_SERVERS = [{
722
- urls: 'stun:stun.l.google.com:19302'
723
- }];
724
- // Optional: notified on every state transition so the host can render "Connecting…", "Listening", etc.
725
- var onStateChange = opts.onStateChange || noop;
726
- var realtimeState = 'idle';
727
- function q(name) {
728
- var s = sel[name];
729
- return s ? document.querySelector(s) : null;
730
- }
731
- var isRealtimeActive = false;
732
- // Server-driven session lifecycle (see bindRealtimeLifecycleHandlers): the server tells us when the
733
- // provider session is live, when the user interrupted, and when the session ended for any reason.
734
- var realtimeLifecycleBound = false,
735
- realtimeSessionReady = false,
736
- realtimeEndedNotice = null,
737
- realtimeCompatibilityNoticeEl = null,
738
- realtimeStatusEl = null;
739
- var realtimeStream = null,
740
- realtimeAudioCtx = null,
741
- realtimeSubject = null,
742
- realtimeProcessor = null,
743
- realtimeMicSource = null,
744
- realtimeZeroGain = null,
745
- realtimeGain = null,
746
- realtimePlayHead = 0,
747
- realtimeSources = [];
748
- // WebRTC transport state.
749
- var realtimeIsWebRtc = false,
750
- realtimePc = null,
751
- realtimeRemoteAudioEl = null,
752
- realtimeWebRtcHandlersBound = false,
753
- realtimeRemoteDescriptionSet = false,
754
- realtimePendingIce = [];
755
- // WebRTC connect-time fallback state: we decide WebRTC-vs-WebSocket only at connect time (never mid-session).
756
- var realtimeWebRtcConnected = false,
757
- realtimeWebRtcConnectTimer = null,
758
- realtimeFellBack = false;
759
- // How long to wait for the WebRTC peer to connect before dropping to the WebSocket transport. Once ICE
760
- // gathering has finished without producing a relay candidate there is nothing left to wait for on a
761
- // network that blocks UDP, so the wait is cut short rather than run to the full timeout.
762
- var REALTIME_WEBRTC_CONNECT_TIMEOUT_MS = 8000;
763
- var REALTIME_WEBRTC_NO_RELAY_TIMEOUT_MS = 5000;
764
- // Fixed receive-side cushion for assistant audio (see pc.ontrack). The server paces the stream on a
765
- // 20 ms clock, so this only has to cover network jitter, not provider burstiness.
766
- var REALTIME_WEBRTC_JITTER_BUFFER_TARGET_MS = 150;
767
- // Remembering a failed attempt for the rest of the browser session matters: without it, a user on a
768
- // network where WebRTC cannot work pays the full connection timeout — and a second microphone prompt —
769
- // at the start of every single conversation.
770
- var REALTIME_WEBRTC_BLOCKED_KEY = 'coreai.realtime.webrtcBlocked';
771
- // The ICE servers resolved for the conversation currently starting. Scoped to one attempt, not to the page:
772
- // TURN credentials are short-lived, so every conversation resolves its own set — but within an attempt the
773
- // transport decision and the peer must see the same servers, and one hub round-trip is enough.
774
- var realtimeAttemptIceServers = null;
775
- // Identifies the conversation currently starting, so an ICE resolution that lands after the user gave up
776
- // (or started again) cannot open a session for an attempt that no longer exists.
777
- var realtimeAttemptToken = 0;
778
- var realtimeSawRelayCandidate = false;
779
- // WebRTC half-duplex echo guard: the shared microphone gate (see createMicGate) watches both the mic and
780
- // the assistant's remote audio and silences the outbound track unless the user is genuinely speaking, so
781
- // the model cannot hear itself even when AEC underperforms against loud open-room speakers. It also
782
- // enforces push-to-talk on the WebRTC mic track.
783
- var realtimeWebRtcMicTrack = null,
784
- realtimeGate = null,
785
- realtimeGatePendingRemoteStream = null;
786
-
787
- // Per-device audio preferences (interruptions, push-to-talk, volume, devices, language). They live in
788
- // localStorage because they depend on the listener's hardware, not on the interaction.
789
- var realtimeBargeIn = true,
790
- realtimePushToTalk = false,
791
- realtimePttActive = false,
792
- realtimePttBound = false,
793
- realtimePttKeyDown = null,
794
- realtimePttKeyUp = null,
795
- realtimePttButton = null,
796
- realtimePttUiEl = null,
797
- realtimeVolume = 1,
798
- realtimeMicDeviceId = '',
799
- realtimeOutputDeviceId = '',
800
- realtimeLanguage = '',
801
- // The listener's saved speed (null for the normal pace), and what each session reports about its own.
802
- realtimeSpeechSpeed = null,
803
- realtimeSessionSpeechSpeed = 1,
804
- realtimeSpeechSpeedAdjustable = false,
805
- realtimeAudioSettingsBuilt = false,
806
- // The settings popover this instance built. A page can host two chat clients at once -- its own chat
807
- // and the admin widget -- and both popovers carry the same element id, so looking one up by id would
808
- // hand the second instance the first one's popover.
809
- realtimeAudioSettingsElement = null;
810
-
811
- // The half-duplex echo tail on the WebSocket transport: how long the microphone stays muted after the
812
- // scheduled end of the assistant's playback, covering the room's reverberation.
813
- var REALTIME_WS_ECHO_HANGOVER_SEC = 0.5;
814
-
815
- // Version 3 dropped the audio-setup presets, the voice-gate modes and the turn-detection sliders — all of
816
- // that is measured or decided automatically now — and, the part that matters to anyone who used version 2,
817
- // restores interruptions. The removed device-label guess had switched them off for every microphone whose
818
- // name merely looked like a room microphone, and the earlier repair deliberately left that alone, so users
819
- // found interruptions "off by default" with no idea why.
820
- var REALTIME_PREFS_VERSION = 3;
821
-
822
- // Mirrors RealtimeSpeechSpeedRange on the server.
823
- var REALTIME_SPEECH_SPEED_MIN = 0.75,
824
- REALTIME_SPEECH_SPEED_MAX = 1.5,
825
- REALTIME_SPEECH_SPEED_DEFAULT = 1;
826
- function clampRealtimeSpeechSpeed(speed) {
827
- return Math.round(Math.min(REALTIME_SPEECH_SPEED_MAX, Math.max(REALTIME_SPEECH_SPEED_MIN, speed)) * 100) / 100;
828
- }
829
- function loadRealtimeAudioPrefs() {
830
- var prefs = {
831
- bargeIn: true,
832
- pushToTalk: false,
833
- volume: 1,
834
- micDeviceId: '',
835
- outputDeviceId: '',
836
- language: '',
837
- speechSpeed: null,
838
- version: 0
839
- };
840
- try {
841
- var raw = window.localStorage.getItem('coreai.realtime.audioPrefs');
842
- if (raw) {
843
- var parsed = JSON.parse(raw);
844
- if (typeof parsed.bargeIn === 'boolean') {
845
- prefs.bargeIn = parsed.bargeIn;
846
- }
847
- if (typeof parsed.pushToTalk === 'boolean') {
848
- prefs.pushToTalk = parsed.pushToTalk;
849
- }
850
- if (typeof parsed.volume === 'number' && isFinite(parsed.volume)) {
851
- prefs.volume = Math.min(1, Math.max(0, parsed.volume));
852
- }
853
- if (typeof parsed.micDeviceId === 'string') {
854
- prefs.micDeviceId = parsed.micDeviceId;
855
- }
856
- if (typeof parsed.outputDeviceId === 'string') {
857
- prefs.outputDeviceId = parsed.outputDeviceId;
858
- }
859
- if (typeof parsed.language === 'string') {
860
- prefs.language = parsed.language;
861
- }
862
- if (typeof parsed.speechSpeed === 'number' && isFinite(parsed.speechSpeed)) {
863
- prefs.speechSpeed = clampRealtimeSpeechSpeed(parsed.speechSpeed);
864
- }
865
- if (typeof parsed.version === 'number') {
866
- prefs.version = parsed.version;
867
- }
868
- }
869
- } catch (err) {/* storage unavailable or blocked */}
870
- return repairRealtimeAudioPrefs(prefs);
871
- }
872
- function repairRealtimeAudioPrefs(prefs) {
873
- if (prefs.version === REALTIME_PREFS_VERSION) {
874
- return prefs;
875
- }
876
- prefs.version = REALTIME_PREFS_VERSION;
877
- prefs.bargeIn = true;
878
- prefs.pushToTalk = false;
879
- saveRealtimeAudioPrefs(prefs);
880
- return prefs;
881
- }
882
- function saveRealtimeAudioPrefs(prefs) {
883
- try {
884
- window.localStorage.setItem('coreai.realtime.audioPrefs', JSON.stringify(prefs));
885
- } catch (err) {}
886
- }
887
- function applyRealtimeAudioPrefs(prefs) {
888
- realtimeBargeIn = prefs.bargeIn !== false;
889
- realtimePushToTalk = !!prefs.pushToTalk;
890
- realtimeVolume = typeof prefs.volume === 'number' ? prefs.volume : 1;
891
- realtimeMicDeviceId = prefs.micDeviceId || '';
892
- realtimeOutputDeviceId = prefs.outputDeviceId || '';
893
- realtimeLanguage = prefs.language || '';
894
- realtimeSpeechSpeed = typeof prefs.speechSpeed === 'number' ? prefs.speechSpeed : null;
895
- // The gate runs on the audio thread and cannot see these variables; push the change to it.
896
- syncRealtimeGateMode();
897
- if (realtimeGain) {
898
- realtimeGain.gain.value = realtimeVolume;
899
- }
900
- // On WebRTC the assistant plays through the hidden <audio> element, so the volume slider must drive its
901
- // volume directly. Lowering it also cuts the acoustic echo into the mic, which helps AEC with barge-in on.
902
- if (realtimeRemoteAudioEl) {
903
- realtimeRemoteAudioEl.volume = Math.max(0, Math.min(1, realtimeVolume));
904
- }
905
- }
906
-
907
- // The push-to-talk handlers below run in the capture phase and swallow Space, which would otherwise eat
908
- // every space character the user types elsewhere on the page while a voice session is active.
909
- function isEditableTarget(target) {
910
- if (!target || !target.tagName) {
911
- return false;
912
- }
913
- var tag = target.tagName.toLowerCase();
914
- return tag === 'input' || tag === 'textarea' || tag === 'select' || target.isContentEditable === true;
915
- }
916
-
917
- // Push-to-talk: hold Space to open the mic. Bound only while a realtime session is active so it never
918
- // interferes with normal typing. Capture phase + preventDefault so Space never scrolls the page or
919
- // activates a focused button (Space-up on a focused button fires a click that would toggle the session).
920
- function attachRealtimePushToTalk() {
921
- if (realtimePttBound) {
922
- return;
923
- }
924
- realtimePttBound = true;
925
- realtimePttActive = false;
926
- realtimePttKeyDown = function realtimePttKeyDown(e) {
927
- if (realtimePushToTalk && !isEditableTarget(e.target) && (e.code === 'Space' || e.key === ' ')) {
928
- e.preventDefault();
929
- if (!e.repeat) {
930
- setRealtimePttActive(true);
931
- }
932
- }
933
- };
934
- realtimePttKeyUp = function realtimePttKeyUp(e) {
935
- if (realtimePushToTalk && !isEditableTarget(e.target) && (e.code === 'Space' || e.key === ' ')) {
936
- e.preventDefault();
937
- setRealtimePttActive(false);
938
- }
939
- };
940
- document.addEventListener('keydown', realtimePttKeyDown, true);
941
- document.addEventListener('keyup', realtimePttKeyUp, true);
942
- }
943
- function detachRealtimePushToTalk() {
944
- if (!realtimePttBound) {
945
- return;
946
- }
947
- realtimePttBound = false;
948
- realtimePttActive = false;
949
- try {
950
- document.removeEventListener('keydown', realtimePttKeyDown, true);
951
- } catch (err) {}
952
- try {
953
- document.removeEventListener('keyup', realtimePttKeyUp, true);
954
- } catch (err) {}
955
- }
956
- function setRealtimePttActive(active) {
957
- realtimePttActive = active;
958
- // The gate lives on the audio thread and cannot read this variable.
959
- syncRealtimeGateMode();
960
- if (realtimePttButton) {
961
- realtimePttButton.classList.toggle('btn-danger', active);
962
- realtimePttButton.classList.toggle('btn-outline-primary', !active);
963
- realtimePttButton.innerHTML = active ? '<i class="fa-solid fa-microphone me-1"></i> Listening…' : '<i class="fa-solid fa-microphone me-1"></i> Hold to talk';
964
- }
965
- }
966
-
967
- // Shows a "hold to talk" control + hint while a push-to-talk realtime session is active.
968
- function buildRealtimePttUi() {
969
- var realtimeBtn = q('realtimeButton');
970
- if (!realtimeBtn || realtimePttUiEl) {
971
- return;
972
- }
973
- var coarse = window.matchMedia && window.matchMedia('(pointer: coarse)').matches;
974
- var bar = document.createElement('div');
975
- bar.className = 'text-center mt-2';
976
- var btn = document.createElement('button');
977
- btn.type = 'button';
978
- btn.className = 'btn btn-outline-primary btn-sm';
979
- btn.innerHTML = '<i class="fa-solid fa-microphone me-1"></i> Hold to talk';
980
- var hint = document.createElement('div');
981
- hint.className = 'form-text mt-1 mb-0';
982
- hint.innerHTML = coarse ? 'Push-to-talk is on — press and hold the button to speak.' : 'Push-to-talk is on — hold <kbd>Space</kbd> or this button to speak.';
983
- bar.appendChild(btn);
984
- bar.appendChild(hint);
985
- var row = realtimeBtn.parentElement;
986
- if (row && row.parentElement) {
987
- row.parentElement.appendChild(bar);
988
- } else {
989
- realtimeBtn.insertAdjacentElement('afterend', bar);
990
- }
991
- realtimePttButton = btn;
992
- realtimePttUiEl = bar;
993
- btn.addEventListener('pointerdown', function (e) {
994
- e.preventDefault();
995
- setRealtimePttActive(true);
996
- });
997
- btn.addEventListener('pointerup', function () {
998
- setRealtimePttActive(false);
999
- });
1000
- btn.addEventListener('pointerleave', function () {
1001
- setRealtimePttActive(false);
1002
- });
1003
- btn.addEventListener('pointercancel', function () {
1004
- setRealtimePttActive(false);
1005
- });
1006
- setRealtimePttActive(false);
1007
- }
1008
- function removeRealtimePttUi() {
1009
- realtimePttButton = null;
1010
- if (realtimePttUiEl) {
1011
- try {
1012
- realtimePttUiEl.remove();
1013
- } catch (err) {}
1014
- realtimePttUiEl = null;
1015
- }
1016
- }
1017
-
1018
- // Builds a gear-triggered popover next to the realtime button with the per-device audio settings.
1019
- //
1020
- // It is deliberately short: a microphone, a speaker, the volume, the language, and two switches. Everything
1021
- // that used to be a knob here — echo margins, gate modes, turn-detection timing, audio-setup presets, an
1022
- // echo self-test — is measured or decided automatically now. A user should be able to press "Start
1023
- // speaking" and talk, in any room, without first understanding acoustics.
1024
- function setupRealtimeAudioSettings() {
1025
- var realtimeBtn = q('realtimeButton');
1026
- if (!realtimeBtn || realtimeAudioSettingsBuilt) {
1027
- return;
1028
- }
1029
- realtimeAudioSettingsBuilt = true;
1030
- var prefs = loadRealtimeAudioPrefs();
1031
- applyRealtimeAudioPrefs(prefs);
1032
- var wrap = document.createElement('span');
1033
- wrap.id = 'realtime-audio-settings';
1034
- wrap.style.position = 'relative';
1035
- wrap.style.display = 'inline-flex';
1036
- var gear = document.createElement('button');
1037
- gear.type = 'button';
1038
- gear.className = 'btn btn-outline-secondary';
1039
- gear.title = localize('settingsTitle', 'Voice settings');
1040
- gear.setAttribute('aria-label', localize('settingsTitle', 'Voice settings'));
1041
- gear.innerHTML = '<i class="fa-solid fa-headset"></i>';
1042
- var langs = [['en', 'English'], ['es', 'Spanish'], ['fr', 'French'], ['de', 'German'], ['it', 'Italian'], ['pt', 'Portuguese'], ['nl', 'Dutch'], ['zh', 'Chinese'], ['ja', 'Japanese'], ['ko', 'Korean'], ['ar', 'Arabic'], ['hi', 'Hindi'], ['ru', 'Russian']];
1043
- var langOptions = '<option value=""' + (prefs.language === '' ? ' selected' : '') + '>' + localize('languageAuto', 'Automatic') + '</option>' + langs.map(function (l) {
1044
- return '<option value="' + l[0] + '"' + (prefs.language === l[0] ? ' selected' : '') + '>' + l[1] + '</option>';
1045
- }).join('');
1046
- var panel = document.createElement('div');
1047
- panel.className = 'card shadow-sm';
1048
- panel.style.cssText = 'position:absolute;bottom:calc(100% + 6px);right:0;z-index:1080;width:290px;max-height:70vh;overflow:auto;display:none;';
1049
- panel.innerHTML = '<div class="card-body p-3">' + '<div class="fw-semibold mb-2" style="font-size:0.85rem;">' + localize('settingsTitle', 'Voice settings') + '</div>' + '<label class="form-label mb-1 d-block" style="font-size:0.8rem;">' + localize('microphone', 'Microphone') + '</label>' + '<select class="form-select form-select-sm js-mic mb-2"><option value="">' + localize('defaultMicrophone', 'Default microphone') + '</option></select>' + '<label class="form-label mb-1 d-block" style="font-size:0.8rem;">' + localize('speaker', 'Speaker') + '</label>' + '<select class="form-select form-select-sm js-out mb-1"><option value="">' + localize('automatic', 'Automatic') + '</option></select>' + '<button type="button" class="btn btn-outline-secondary btn-sm mb-2 js-pick-out" style="display:none;">' + localize('chooseSpeaker', 'Choose speaker…') + '</button>' + '<div class="form-text mb-2 js-out-help">' + localize('speakerHelp', 'Automatic uses the device your system uses for calls. If you cannot hear the assistant, pick the speakers you are actually listening to.') + '</div>' + '<label class="form-label mb-1 d-block" style="font-size:0.8rem;">' + localize('assistantVolume', 'Assistant volume') + ': <strong class="js-vol-val">' + Math.round(prefs.volume * 100) + '</strong>%</label>' + '<input type="range" class="form-range js-vol mb-2" min="0" max="100" step="5" value="' + Math.round(prefs.volume * 100) + '">' +
1050
- // Hidden until the session reports that its speed can change.
1051
- '<div class="js-speed-row" hidden>' + '<label class="form-label mb-1 d-flex align-items-baseline" style="font-size:0.8rem;">' + '<span>' + localize('speechSpeed', 'Speaking speed') + ': <strong class="js-speed-val">1.00</strong>×</span>' + '<button type="button" class="btn btn-link btn-sm p-0 ms-auto js-speed-reset" style="font-size:0.8rem;" hidden>' + localize('speechSpeedReset', 'Reset') + '</button>' + '</label>' + '<input type="range" class="form-range js-speed mb-0" min="' + REALTIME_SPEECH_SPEED_MIN + '" max="' + REALTIME_SPEECH_SPEED_MAX + '" step="0.05" value="' + REALTIME_SPEECH_SPEED_DEFAULT + '">' + '<div class="form-text mt-0 mb-2">' + localize('speechSpeedHelp', 'Applies from the assistant\'s next reply.') + '</div>' + '</div>' + '<label class="form-label mb-1 d-block" style="font-size:0.8rem;">' + localize('language', 'Language') + '</label>' + '<select class="form-select form-select-sm js-lang mb-2">' + langOptions + '</select>' + '<div class="form-check form-switch mb-1">' + '<input class="form-check-input js-barge" type="checkbox" id="realtime-setting-barge"' + (prefs.bargeIn ? ' checked' : '') + '>' + '<label class="form-check-label" for="realtime-setting-barge">' + localize('allowInterruptions', 'Allow interruptions') + '</label>' + '</div>' + '<div class="form-text mb-2">' + localize('allowInterruptionsHelp', 'Talk over the assistant to interrupt it. Turn this off if the assistant keeps hearing itself through your speakers.') + '</div>' + '<div class="form-check form-switch mb-1">' + '<input class="form-check-input js-ptt" type="checkbox" id="realtime-setting-ptt"' + (prefs.pushToTalk ? ' checked' : '') + '>' + '<label class="form-check-label" for="realtime-setting-ptt">' + localize('pushToTalk', 'Push-to-talk') + '</label>' + '</div>' + '<div class="form-text">' + localize('pushToTalkHelp', 'Hold <kbd>Space</kbd> (or the button) to talk. For very noisy places.') + '</div>' + '</div>';
1052
- wrap.appendChild(gear);
1053
- wrap.appendChild(panel);
1054
- realtimeBtn.insertAdjacentElement('afterend', wrap);
1055
- realtimeAudioSettingsElement = wrap;
1056
- var bargeInput = panel.querySelector('.js-barge');
1057
- var pttInput = panel.querySelector('.js-ptt');
1058
- var volInput = panel.querySelector('.js-vol');
1059
- var volVal = panel.querySelector('.js-vol-val');
1060
- var micSelect = panel.querySelector('.js-mic');
1061
- var outSelect = panel.querySelector('.js-out');
1062
- var outPickButton = panel.querySelector('.js-pick-out');
1063
- var langSelect = panel.querySelector('.js-lang');
1064
- var speedInput = panel.querySelector('.js-speed');
1065
- var speedVal = panel.querySelector('.js-speed-val');
1066
- var speedReset = panel.querySelector('.js-speed-reset');
1067
- function readPanelPrefs() {
1068
- return {
1069
- bargeIn: bargeInput.checked,
1070
- pushToTalk: pttInput.checked,
1071
- volume: (parseInt(volInput.value, 10) || 0) / 100,
1072
- micDeviceId: micSelect.value || '',
1073
- outputDeviceId: outSelect.value || '',
1074
- language: langSelect.value || '',
1075
- speechSpeed: realtimeSpeechSpeed,
1076
- version: REALTIME_PREFS_VERSION
1077
- };
1078
- }
1079
- function persist() {
1080
- var next = readPanelPrefs();
1081
- applyRealtimeAudioPrefs(next);
1082
- saveRealtimeAudioPrefs(next);
1083
- pushRealtimeSettingsToServer();
1084
- }
1085
-
1086
- // Kept out of persist(), which resends turn detection on every volume tick.
1087
- function persistSpeechSpeed(speed) {
1088
- realtimeSpeechSpeed = speed;
1089
- saveRealtimeAudioPrefs(readPanelPrefs());
1090
- syncRealtimeSpeechSpeedControl();
1091
- pushRealtimeSpeechSpeedToServer();
1092
- }
1093
- function populateDevices() {
1094
- if (!navigator.mediaDevices || !navigator.mediaDevices.enumerateDevices) {
1095
- return;
1096
- }
1097
- navigator.mediaDevices.enumerateDevices().then(function (devices) {
1098
- var options = ['<option value="">' + localize('defaultMicrophone', 'Default microphone') + '</option>'];
1099
- devices.filter(function (d) {
1100
- return d.kind === 'audioinput';
1101
- }).forEach(function (d, i) {
1102
- var label = d.label || localize('microphone', 'Microphone') + ' ' + (i + 1);
1103
- options.push('<option value="' + d.deviceId + '"' + (d.deviceId === realtimeMicDeviceId ? ' selected' : '') + '>' + label.replace(/</g, '') + '</option>');
1104
- });
1105
- micSelect.innerHTML = options.join('');
1106
-
1107
- // Output devices are only enumerable on browsers that expose them (Chromium); elsewhere the
1108
- // list stays on "Automatic" and the picker button below is the way in.
1109
- var outs = devices.filter(function (d) {
1110
- return d.kind === 'audiooutput' && d.deviceId && d.deviceId !== 'communications' && d.deviceId !== 'default';
1111
- });
1112
- if (outs.length) {
1113
- var outOptions = ['<option value="">' + localize('automatic', 'Automatic') + '</option>'];
1114
- outs.forEach(function (d, i) {
1115
- var label = d.label || localize('speaker', 'Speaker') + ' ' + (i + 1);
1116
- outOptions.push('<option value="' + d.deviceId + '"' + (d.deviceId === realtimeOutputDeviceId ? ' selected' : '') + '>' + label.replace(/</g, '') + '</option>');
1117
- });
1118
- outSelect.innerHTML = outOptions.join('');
1119
- }
1120
- })["catch"](function () {});
1121
- }
1122
- populateDevices();
1123
-
1124
- // Unplugging the headset mid-conversation silently changes which devices exist. Refresh the lists, and
1125
- // end the session if the microphone it was actually using has gone — continuing on a different device
1126
- // without saying so is worse than stopping and letting the user restart.
1127
- if (navigator.mediaDevices && typeof navigator.mediaDevices.addEventListener === 'function') {
1128
- navigator.mediaDevices.addEventListener('devicechange', function () {
1129
- populateDevices();
1130
- if (!isRealtimeActive || !realtimeMicDeviceId) {
1131
- return;
1132
- }
1133
- navigator.mediaDevices.enumerateDevices().then(function (devices) {
1134
- var stillThere = devices.some(function (d) {
1135
- return d.kind === 'audioinput' && d.deviceId === realtimeMicDeviceId;
1136
- });
1137
- if (!stillThere) {
1138
- endRealtimeSessionFromServer('device_lost');
1139
- }
1140
- })["catch"](function () {});
1141
- });
1142
- }
1143
- function constrainPanel() {
1144
- // Keep the popover on screen horizontally. It is anchored to the button's right edge, which is
1145
- // correct where the button sits on the right of a wide row — but in a narrow layout, or with the
1146
- // button near the left edge, a 290px panel would otherwise hang off the side of the window.
1147
- panel.style.right = '0px';
1148
- panel.style.left = 'auto';
1149
- var panelRect = panel.getBoundingClientRect();
1150
- if (panelRect.left < 8) {
1151
- panel.style.right = panelRect.left - 8 + 'px';
1152
- }
1153
-
1154
- // Cap its height to the space above the button up to the nearest scroll/fixed ancestor, so it
1155
- // scrolls instead of spilling out the top.
1156
- var rect = gear.getBoundingClientRect();
1157
- var boundaryTop = 8;
1158
- var node = panel.parentElement ? panel.parentElement.parentElement : null;
1159
- while (node && node !== document.body && node !== document.documentElement) {
1160
- var s = window.getComputedStyle(node);
1161
- if (s.overflowY === 'auto' || s.overflowY === 'scroll' || s.overflowY === 'hidden' || s.position === 'fixed') {
1162
- boundaryTop = Math.max(boundaryTop, node.getBoundingClientRect().top);
1163
- break;
1164
- }
1165
- node = node.parentElement;
1166
- }
1167
- panel.style.maxHeight = Math.max(160, rect.top - boundaryTop - 10) + 'px';
1168
- }
1169
- gear.addEventListener('click', function (e) {
1170
- e.stopPropagation();
1171
- var showing = panel.style.display === 'none';
1172
- if (showing) {
1173
- panel.style.display = 'block';
1174
- constrainPanel();
1175
- populateDevices();
1176
- }
1177
- panel.style.display = showing ? 'block' : 'none';
1178
- });
1179
- panel.addEventListener('click', function (e) {
1180
- e.stopPropagation();
1181
- });
1182
- document.addEventListener('click', function () {
1183
- panel.style.display = 'none';
1184
- });
1185
- bargeInput.addEventListener('change', persist);
1186
- pttInput.addEventListener('change', persist);
1187
- volInput.addEventListener('input', function () {
1188
- volVal.textContent = volInput.value;
1189
- persist();
1190
- });
1191
- // The label follows the drag; the speed is sent once, on release.
1192
- speedInput.addEventListener('input', function () {
1193
- speedVal.textContent = formatRealtimeSpeechSpeed(parseFloat(speedInput.value));
1194
- });
1195
- speedInput.addEventListener('change', function () {
1196
- persistSpeechSpeed(clampRealtimeSpeechSpeed(parseFloat(speedInput.value)));
1197
- });
1198
- // Back to the model's normal pace.
1199
- speedReset.addEventListener('click', function () {
1200
- persistSpeechSpeed(null);
1201
- });
1202
- micSelect.addEventListener('change', persist);
1203
- langSelect.addEventListener('change', persist);
1204
- outSelect.addEventListener('change', function () {
1205
- persist();
1206
- // Apply live: the whole point of the picker is that a user who cannot hear the assistant can fix
1207
- // it without ending the conversation.
1208
- applyRealtimeOutputDevice();
1209
- });
1210
-
1211
- // Firefox exposes no output device list until the user picks one through this prompt.
1212
- if (navigator.mediaDevices && typeof navigator.mediaDevices.selectAudioOutput === 'function') {
1213
- outPickButton.style.display = '';
1214
- outPickButton.addEventListener('click', function () {
1215
- navigator.mediaDevices.selectAudioOutput().then(function (device) {
1216
- if (!device || !device.deviceId) {
1217
- return;
1218
- }
1219
- var option = document.createElement('option');
1220
- option.value = device.deviceId;
1221
- option.textContent = device.label || localize('selectedSpeaker', 'Selected speaker');
1222
- outSelect.appendChild(option);
1223
- outSelect.value = device.deviceId;
1224
- persist();
1225
- applyRealtimeOutputDevice();
1226
- })["catch"](function () {});
1227
- });
1228
- }
1229
- }
1230
-
1231
- // The host supplies the button's markup as a data attribute. It is sanitized when DOMPurify is present;
1232
- // when it is not, the label falls back to text rather than injecting unsanitized markup — the module is
1233
- // shared, and not every host that loads it also loads DOMPurify.
1234
- function setRealtimeButtonHtml(button, html) {
1235
- if (!html) {
1236
- return;
1237
- }
1238
- if (window.DOMPurify && typeof window.DOMPurify.sanitize === 'function') {
1239
- button.replaceChildren(window.DOMPurify.sanitize(html, {
1240
- RETURN_DOM_FRAGMENT: true
1241
- }));
1242
- return;
1243
- }
1244
-
1245
- // No DOMPurify: never parse the untrusted markup as HTML. Strip any tags with a plain-string pass and
1246
- // collapse whitespace, then render the remaining label as text only.
1247
- var text = html.replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ').trim();
1248
- button.replaceChildren(document.createTextNode(text));
1249
- }
1250
- function updateRealtimeButton() {
1251
- // Reported before the early return: a host still has to be told the session ended even on a surface
1252
- // that never rendered the button.
1253
- onSessionStateChanged(isRealtimeActive);
1254
- var realtimeBtn = q('realtimeButton');
1255
- if (!realtimeBtn) {
1256
- return;
1257
- }
1258
- if (isRealtimeActive) {
1259
- realtimeBtn.classList.add('active', 'btn-primary');
1260
- realtimeBtn.classList.remove('btn-dark', 'btn-outline-dark', 'btn-outline-secondary');
1261
- realtimeBtn.title = realtimeBtn.getAttribute('data-end-title') || 'End Conversation';
1262
- setRealtimeButtonHtml(realtimeBtn, realtimeBtn.getAttribute('data-end-html'));
1263
- } else {
1264
- realtimeBtn.classList.remove('active', 'btn-primary', 'btn-dark', 'btn-outline-secondary');
1265
- realtimeBtn.classList.add('btn-outline-dark');
1266
- realtimeBtn.blur();
1267
- realtimeBtn.title = realtimeBtn.getAttribute('data-start-title') || 'Start speaking';
1268
- setRealtimeButtonHtml(realtimeBtn, realtimeBtn.getAttribute('data-start-html'));
1269
- }
1270
- }
1271
-
1272
- // Switches the input between text and audio-only (realtime). Called at load and whenever the selected
1273
- // deployment changes to/from a realtime model.
1274
- function applyRealtimeMode(enable) {
1275
- isRealtimeMode = enable;
1276
- var realtimeBtn = q('realtimeButton');
1277
- var micBtn = q('micButton');
1278
- var conversationBtn = q('conversationButton');
1279
- var inputEl = q('input');
1280
- var sendBtn = q('sendButton');
1281
- var audioSettings = realtimeAudioSettingsElement;
1282
- if (enable) {
1283
- onEnterRealtimeMode();
1284
- if (inputEl) {
1285
- inputEl.hidden = true;
1286
- }
1287
- if (sendBtn) {
1288
- sendBtn.hidden = true;
1289
- }
1290
- if (micBtn) {
1291
- micBtn.hidden = true;
1292
- }
1293
- if (conversationBtn) {
1294
- conversationBtn.hidden = true;
1295
- }
1296
- if (realtimeBtn) {
1297
- realtimeBtn.hidden = false;
1298
- }
1299
- if (audioSettings) {
1300
- audioSettings.hidden = false;
1301
- }
1302
- } else {
1303
- if (isRealtimeActive) {
1304
- stopRealtimeConversation();
1305
- }
1306
- if (realtimeBtn) {
1307
- realtimeBtn.hidden = true;
1308
- }
1309
- if (audioSettings) {
1310
- audioSettings.hidden = true;
1311
- }
1312
- removeCompatibilityModeNotice();
1313
- clearRealtimeStatus();
1314
- if (inputEl) {
1315
- inputEl.hidden = false;
1316
- }
1317
- if (sendBtn) {
1318
- sendBtn.hidden = false;
1319
- }
1320
- if (micBtn) {
1321
- micBtn.hidden = false;
1322
- }
1323
- if (conversationBtn) {
1324
- conversationBtn.hidden = false;
1325
- }
1326
- }
1327
- updateRealtimeButton();
1328
- }
1329
-
1330
- // Barge-in and the voice-activity knobs are enforced by three layers at once: this client's microphone
1331
- // gate, the server's input pump, and the provider's own turn detection. Changing only this one leaves the
1332
- // three disagreeing — the classic symptom being barge-in switched off mid-conversation while the assistant
1333
- // still interrupts itself — so a change during a live session is sent on to the other two.
1334
- function pushRealtimeSettingsToServer() {
1335
- if (!isRealtimeActive || !connection || typeof connection.send !== 'function') {
1336
- return;
1337
- }
1338
- try {
1339
- connection.send('UpdateRealtimeSettings', realtimeBargeIn, null, null);
1340
- } catch (err) {}
1341
- }
1342
- function formatRealtimeSpeechSpeed(speed) {
1343
- return (isFinite(speed) ? speed : REALTIME_SPEECH_SPEED_DEFAULT).toFixed(2);
1344
- }
1345
-
1346
- // The listener's saved speed wins over the session's own.
1347
- function effectiveRealtimeSpeechSpeed() {
1348
- return realtimeSpeechSpeed !== null ? realtimeSpeechSpeed : realtimeSessionSpeechSpeed;
1349
- }
1350
- function syncRealtimeSpeechSpeedControl() {
1351
- if (!realtimeAudioSettingsElement) {
1352
- return;
1353
- }
1354
- var row = realtimeAudioSettingsElement.querySelector('.js-speed-row');
1355
- var input = realtimeAudioSettingsElement.querySelector('.js-speed');
1356
- var value = realtimeAudioSettingsElement.querySelector('.js-speed-val');
1357
- var reset = realtimeAudioSettingsElement.querySelector('.js-speed-reset');
1358
- if (!row || !input || !value || !reset) {
1359
- return;
1360
- }
1361
- var speed = effectiveRealtimeSpeechSpeed();
1362
- row.hidden = !realtimeSpeechSpeedAdjustable;
1363
- input.value = String(speed);
1364
- value.textContent = formatRealtimeSpeechSpeed(speed);
1365
- reset.hidden = realtimeSpeechSpeed === null;
1366
- }
1367
- function pushRealtimeSpeechSpeedToServer() {
1368
- if (!isRealtimeActive || !realtimeSpeechSpeedAdjustable || !connection || typeof connection.send !== 'function') {
1369
- return;
1370
- }
1371
- try {
1372
- connection.send('UpdateRealtimeSpeechSpeed', effectiveRealtimeSpeechSpeed());
1373
- } catch (err) {}
1374
- }
1375
-
1376
- // An empty payload means this session's speed can't change; otherwise a saved speed is applied before anyone speaks.
1377
- function applySessionSpeechSpeed(payload) {
1378
- var speed = parseFloat(payload);
1379
- realtimeSpeechSpeedAdjustable = isFinite(speed);
1380
- realtimeSessionSpeechSpeed = realtimeSpeechSpeedAdjustable ? speed : REALTIME_SPEECH_SPEED_DEFAULT;
1381
- syncRealtimeSpeechSpeedControl();
1382
- if (realtimeSpeechSpeed !== null && realtimeSpeechSpeed !== realtimeSessionSpeechSpeed) {
1383
- pushRealtimeSpeechSpeedToServer();
1384
- }
1385
- }
1386
-
1387
- // Reports the session's state to the host so it can show something more honest than a button that flips to
1388
- // "End Conversation" before anything has connected.
1389
- // What each state says to the user. Only the states worth narrating appear here; the rest clear the line.
1390
- var REALTIME_STATE_TEXT = {
1391
- 'requesting-mic': localize('requestingMic', 'Waiting for microphone access…'),
1392
- 'connecting': localize('connecting', 'Connecting…'),
1393
- 'listening': localize('listening', 'Listening'),
1394
- 'playback-blocked': localize('playbackBlocked', 'Audio is blocked by the browser — click the page to enable it.')
1395
- };
1396
- function setRealtimeState(state, detail) {
1397
- if (realtimeState === state) {
1398
- return;
1399
- }
1400
- realtimeState = state;
1401
- var text = REALTIME_STATE_TEXT[state];
1402
- if (text) {
1403
- showRealtimeStatus(text, false);
1404
- } else if (state === 'idle') {
1405
- clearRealtimeStatus();
1406
- }
1407
- try {
1408
- onStateChange({
1409
- state: state,
1410
- transport: realtimeIsWebRtc ? 'webrtc' : 'websocket',
1411
- detail: detail || null
1412
- });
1413
- } catch (err) {}
1414
- }
1415
-
1416
- // The server owns the session's lifecycle; the browser cannot infer it from the audio it happens to
1417
- // receive. Without these handlers a session that ended server-side (provider socket closed, session cap,
1418
- // error, authorization failure) leaves the mic open and the button stuck on "End Conversation" while audio
1419
- // is streamed into nothing.
1420
- function bindRealtimeLifecycleHandlers() {
1421
- if (realtimeLifecycleBound || !connection) {
1422
- return;
1423
- }
1424
- realtimeLifecycleBound = true;
1425
- connection.on('ReceiveRealtimeEvent', function (identifier, type, payload) {
1426
- if (!isRealtimeActive) {
1427
- return;
1428
- }
1429
- if (type === 'speech_speed') {
1430
- applySessionSpeechSpeed(payload);
1431
- } else if (type === 'session_ready') {
1432
- realtimeSessionReady = true;
1433
- setRealtimeState('listening');
1434
- } else if (type === 'session_ended') {
1435
- endRealtimeSessionFromServer(payload || 'completed');
1436
- } else if (type === 'speech_started') {
1437
- // Barge-in: stop the reply that is already scheduled for playback. On WebRTC the server flushes
1438
- // the peer itself, so this only matters for the WebSocket transport — but calling it there too
1439
- // is harmless and keeps the two transports behaving the same.
1440
- if (realtimeBargeIn) {
1441
- flushRealtimePlayback();
1442
- }
1443
- } else if (type === 'playback_flush') {
1444
- flushRealtimePlayback();
1445
- } else if (type === 'user_turn_pending') {
1446
- onUserTurnPending(payload);
1447
- } else if (type === 'user_turn_dropped') {
1448
- onUserTurnDropped(payload);
1449
- }
1450
- });
1451
-
1452
- // An error while a voice session is running is terminal for that session: the server has already
1453
- // stopped, so keep the mic open no longer.
1454
- connection.on('ReceiveError', function () {
1455
- if (!isRealtimeActive) {
1456
- return;
1457
- }
1458
-
1459
- // An error while the WebRTC peer is still being set up is a reason to try the other transport, not
1460
- // to give up: the WebSocket path may well work. Treating every error as terminal is what turned a
1461
- // recoverable server-side handshake failure into a session that silently did nothing at all.
1462
- if (realtimeIsWebRtc && !realtimeWebRtcConnected && !realtimeFellBack) {
1463
- fallbackToWebSocket('server reported an error before the peer connected');
1464
- return;
1465
- }
1466
-
1467
- // Once the session is live, an error is terminal — the server has already stopped.
1468
- endRealtimeSessionFromServer('error');
1469
- });
1470
- if (typeof connection.onclose === 'function') {
1471
- connection.onclose(function () {
1472
- if (isRealtimeActive) {
1473
- endRealtimeSessionFromServer('disconnected');
1474
- }
1475
- });
1476
- }
1477
- if (typeof connection.onreconnecting === 'function') {
1478
- connection.onreconnecting(function () {
1479
- if (isRealtimeActive) {
1480
- endRealtimeSessionFromServer('disconnected');
1481
- }
1482
- });
1483
- }
1484
- }
1485
- function endRealtimeSessionFromServer(reason) {
1486
- realtimeEndedNotice = reason;
1487
- stopRealtimeConversation();
1488
- setRealtimeState('ended', reason);
1489
- if (window.console && console.info) {
1490
- console.info('The realtime voice session ended (' + reason + ').');
1491
- }
1492
-
1493
- // An idle or duration close is deliberate and recoverable, so say so and offer to pick the conversation
1494
- // back up rather than leaving the user staring at a button that silently stopped working.
1495
- if (reason === 'idle') {
1496
- showRealtimeStatus(localize('endedIdle', 'Voice paused after a period of silence.'), true);
1497
- } else if (reason === 'max_duration') {
1498
- showRealtimeStatus(localize('endedMaxDuration', 'Voice paused — this session reached its time limit. You can start another.'), true);
1499
- } else if (reason === 'disconnected') {
1500
- showRealtimeStatus(localize('endedDisconnected', 'Voice session ended — the connection dropped.'), true);
1501
- } else if (reason === 'error') {
1502
- showRealtimeStatus(localize('endedError', 'Voice session ended unexpectedly.'), true);
1503
- } else if (reason === 'device_lost') {
1504
- showRealtimeStatus(localize('endedDeviceLost', 'Voice session ended — the microphone was disconnected.'), true);
1505
- }
1506
- }
1507
-
1508
- // A single status line under the realtime button, optionally with a Resume button.
1509
- function showRealtimeStatus(text, offerResume) {
1510
- var realtimeBtn = q('realtimeButton');
1511
- if (!realtimeBtn) {
1512
- return;
1513
- }
1514
- if (!realtimeStatusEl) {
1515
- realtimeStatusEl = document.createElement('div');
1516
- realtimeStatusEl.className = 'form-text mt-2 mb-0 text-center';
1517
- realtimeStatusEl.setAttribute('role', 'status');
1518
- realtimeStatusEl.setAttribute('aria-live', 'polite');
1519
- var row = realtimeBtn.parentElement;
1520
- if (row && row.parentElement) {
1521
- row.parentElement.appendChild(realtimeStatusEl);
1522
- } else {
1523
- realtimeBtn.insertAdjacentElement('afterend', realtimeStatusEl);
1524
- }
1525
- }
1526
- realtimeStatusEl.replaceChildren();
1527
- realtimeStatusEl.appendChild(document.createTextNode(text));
1528
- if (offerResume) {
1529
- var resume = document.createElement('button');
1530
- resume.type = 'button';
1531
- resume.className = 'btn btn-link btn-sm p-0 ms-2 align-baseline';
1532
- resume.textContent = localize('resume', 'Resume');
1533
- resume.addEventListener('click', function () {
1534
- clearRealtimeStatus();
1535
- startRealtimeConversation();
1536
- });
1537
- realtimeStatusEl.appendChild(resume);
1538
- }
1539
- }
1540
- function clearRealtimeStatus() {
1541
- if (!realtimeStatusEl) {
1542
- return;
1543
- }
1544
- try {
1545
- realtimeStatusEl.remove();
1546
- } catch (err) {}
1547
- realtimeStatusEl = null;
1548
- }
1549
-
1550
- // Resolves the ICE servers from the server immediately before creating the peer. Fetching them per session
1551
- // (rather than embedding them at page render time) is what makes ephemeral TURN credentials usable: they
1552
- // expire, and a page open for an hour would otherwise hand the browser a dead credential. Falls back to a
1553
- // public STUN server so a failed call degrades to today's behavior rather than breaking the session.
1554
- function resolveIceServers() {
1555
- if (webRtcIceServers) {
1556
- return Promise.resolve(webRtcIceServers);
1557
- }
1558
- if (realtimeAttemptIceServers) {
1559
- return Promise.resolve(realtimeAttemptIceServers);
1560
- }
1561
- if (!connection || typeof connection.invoke !== 'function') {
1562
- return Promise.resolve(DEFAULT_ICE_SERVERS);
1563
- }
1564
- return connection.invoke('GetRealtimeIceServers').then(function (servers) {
1565
- realtimeAttemptIceServers = Array.isArray(servers) && servers.length ? servers : DEFAULT_ICE_SERVERS;
1566
- return realtimeAttemptIceServers;
1567
- })["catch"](function (err) {
1568
- if (window.console && console.warn) {
1569
- console.warn('Could not resolve the realtime ICE servers; using the default STUN server.', err);
1570
- }
1571
- realtimeAttemptIceServers = DEFAULT_ICE_SERVERS;
1572
- return realtimeAttemptIceServers;
1573
- });
1574
- }
1575
-
1576
- // Identifies an ICE configuration by its server URLs, deliberately ignoring usernames and credentials:
1577
- // ephemeral TURN credentials (Cloudflare, coturn HMAC) are reissued for every session, so including them
1578
- // would change the fingerprint on each attempt and make the remembered failure worthless. Adding, removing
1579
- // or repointing a STUN/TURN server does change it — which is the case that must invalidate the memory.
1580
- function iceServersFingerprint(servers) {
1581
- var urls = [];
1582
- try {
1583
- (servers || []).forEach(function (server) {
1584
- var list = server && server.urls;
1585
- if (typeof list === 'string') {
1586
- urls.push(list);
1587
- } else if (Array.isArray(list)) {
1588
- list.forEach(function (url) {
1589
- if (url) {
1590
- urls.push(url);
1591
- }
1592
- });
1593
- }
1594
- });
1595
- } catch (err) {
1596
- return 'none';
1597
- }
1598
- return urls.length ? urls.sort().join('|') : 'none';
1599
- }
1600
-
1601
- // A remembered failure only applies to the ICE configuration that produced it. Anything else strands a
1602
- // deployment that has since been given a TURN server on the WebSocket transport until every open tab is
1603
- // closed, with no way for the user to tell why. Values written by older builds ('1') match no fingerprint,
1604
- // so upgrading clears the flag on its own.
1605
- function isWebRtcKnownBlocked(servers) {
1606
- try {
1607
- return window.sessionStorage.getItem(REALTIME_WEBRTC_BLOCKED_KEY) === iceServersFingerprint(servers);
1608
- } catch (err) {
1609
- return false;
1610
- }
1611
- }
1612
- function rememberWebRtcBlocked() {
1613
- try {
1614
- window.sessionStorage.setItem(REALTIME_WEBRTC_BLOCKED_KEY, iceServersFingerprint(realtimeAttemptIceServers || webRtcIceServers));
1615
- } catch (err) {}
1616
- }
1617
-
1618
- // A one-line, dismissible note under the realtime button. Shown once per browser session when voice falls
1619
- // back to the WebSocket transport, because the difference is audible (weaker echo handling in an open
1620
- // room) and users otherwise have no way to know why.
1621
- function showCompatibilityModeNotice() {
1622
- if (realtimeCompatibilityNoticeEl) {
1623
- return;
1624
- }
1625
- var realtimeBtn = q('realtimeButton');
1626
- if (!realtimeBtn) {
1627
- return;
1628
- }
1629
- var note = document.createElement('div');
1630
- note.className = 'form-text mt-2 mb-0 text-center';
1631
- note.setAttribute('role', 'status');
1632
- note.textContent = localize('compatibilityMode', 'Using compatibility audio mode. Echo cancellation may be weaker — headphones are recommended.');
1633
- var row = realtimeBtn.parentElement;
1634
- if (row && row.parentElement) {
1635
- row.parentElement.appendChild(note);
1636
- } else {
1637
- realtimeBtn.insertAdjacentElement('afterend', note);
1638
- }
1639
- realtimeCompatibilityNoticeEl = note;
1640
- }
1641
- function removeCompatibilityModeNotice() {
1642
- if (!realtimeCompatibilityNoticeEl) {
1643
- return;
1644
- }
1645
- try {
1646
- realtimeCompatibilityNoticeEl.remove();
1647
- } catch (err) {}
1648
- realtimeCompatibilityNoticeEl = null;
1649
- }
1650
- function startRealtimeConversation() {
1651
- if (!isRealtimeMode || isRealtimeActive || !connection) {
1652
- return;
1653
- }
1654
- applyRealtimeAudioPrefs(loadRealtimeAudioPrefs());
1655
- realtimeAttemptIceServers = null;
1656
- var attempt = ++realtimeAttemptToken;
1657
- realtimeFellBack = false;
1658
- realtimeSessionReady = false;
1659
- // Hidden until this session reports its own speed.
1660
- realtimeSpeechSpeedAdjustable = false;
1661
- syncRealtimeSpeechSpeedControl();
1662
- realtimeEndedNotice = null;
1663
- clearRealtimeStatus();
1664
- bindRealtimeLifecycleHandlers();
1665
- setRealtimeState('requesting-mic');
1666
-
1667
- // Starting on WebSocket without even attempting WebRTC is a deployment fact worth stating once. The
1668
- // console is otherwise indistinguishable from a healthy WebRTC session, so "no warning" gets read as
1669
- // "WebRTC is working" when it can equally mean the server never offered the transport at all.
1670
- if (!webRtcEnabled) {
1671
- logWebSocketTransportReason('the server did not advertise the WebRTC transport');
1672
- startRealtimeWebSocketConversation();
1673
- return;
1674
- }
1675
-
1676
- // Prefer the WebRTC transport when the server advertises it: the browser's echo canceller references
1677
- // the assistant's media track, so the model can ignore its own voice with the mic open (open rooms).
1678
- // If the peer cannot connect (blocked UDP, no TURN, unsupported), we fall back to WebSocket at connect
1679
- // time — see fallbackToWebSocket. The decision is made once, before the session starts.
1680
- //
1681
- // Resolving the ICE servers before choosing a transport costs no extra round-trip — the peer reuses
1682
- // realtimeAttemptIceServers — and it is what lets a remembered failure be scoped to the configuration
1683
- // that actually caused it.
1684
- resolveIceServers().then(function (servers) {
1685
- if (attempt !== realtimeAttemptToken || !isRealtimeMode || isRealtimeActive) {
1686
- return;
1687
- }
1688
- if (isWebRtcKnownBlocked(servers)) {
1689
- logWebSocketTransportReason('a WebRTC attempt already failed earlier in this browser session with the same ICE configuration');
1690
- startRealtimeWebSocketConversation();
1691
- return;
1692
- }
1693
- startRealtimeWebRtcConversation();
1694
- });
1695
- }
1696
- function startRealtimeWebSocketConversation() {
1697
- if (webRtcEnabled) {
1698
- showCompatibilityModeNotice();
1699
- }
1700
- attachRealtimePushToTalk();
1701
- // Drop focus from the just-clicked button so Space acts as push-to-talk, not a re-click.
1702
- var focusedBtn = q('realtimeButton');
1703
- if (focusedBtn) {
1704
- try {
1705
- focusedBtn.blur();
1706
- } catch (err) {}
1707
- }
1708
- var audioConstraints = {
1709
- channelCount: 1,
1710
- echoCancellation: {
1711
- ideal: true
1712
- },
1713
- noiseSuppression: true,
1714
- autoGainControl: true,
1715
- // Ask for the strongest available acoustic echo cancellation so the model can ignore its own
1716
- // voice in an open room (loud speakers + open mic). These are Chromium hints requested as
1717
- // "ideal", so any browser that doesn't support them simply ignores them — never a hard failure.
1718
- // echoCancellationType 'system' prefers the OS/hardware AEC over the browser's software AEC.
1719
- echoCancellationType: {
1720
- ideal: 'system'
1721
- },
1722
- voiceIsolation: {
1723
- ideal: true
1724
- }
1725
- };
1726
- if (realtimeMicDeviceId) {
1727
- audioConstraints.deviceId = {
1728
- exact: realtimeMicDeviceId
1729
- };
1730
- }
1731
- navigator.mediaDevices.getUserMedia({
1732
- audio: audioConstraints
1733
- }).then(function (stream) {
1734
- ensureConnected().then(function () {
1735
- realtimeStream = stream;
1736
- isRealtimeActive = true;
1737
- onActivate();
1738
- updateRealtimeButton();
1739
- setRealtimeState('connecting');
1740
- var AudioContextCtor = window.AudioContext || window.webkitAudioContext;
1741
- realtimeAudioCtx = new AudioContextCtor({
1742
- sampleRate: REALTIME_SAMPLE_RATE
1743
- });
1744
- // A context created under a strict autoplay policy starts suspended, and a suspended
1745
- // context plays nothing at all — silently. Ask for it explicitly.
1746
- if (realtimeAudioCtx.state === 'suspended' && typeof realtimeAudioCtx.resume === 'function') {
1747
- realtimeAudioCtx.resume()["catch"](function (err) {
1748
- setRealtimeState('playback-blocked');
1749
- if (window.console && console.warn) {
1750
- console.warn('The browser blocked realtime audio playback. Interact with the page to enable audio.', err);
1751
- }
1752
- });
1753
- }
1754
- realtimePlayHead = 0;
1755
- realtimeSources = [];
1756
- realtimeGain = realtimeAudioCtx.createGain();
1757
- realtimeGain.gain.value = realtimeVolume != null ? realtimeVolume : 1;
1758
- realtimeGain.connect(realtimeAudioCtx.destination);
1759
- realtimeSubject = new window.signalR.Subject();
1760
- var ctxAtStart = realtimeAudioCtx;
1761
-
1762
- // A zero-gain node keeps the capture node alive without echoing the mic to the speakers.
1763
- var zeroGain = realtimeAudioCtx.createGain();
1764
- zeroGain.gain.value = 0;
1765
- realtimeZeroGain = zeroGain;
1766
- zeroGain.connect(realtimeAudioCtx.destination);
1767
-
1768
- // The gate watches the assistant's playback to know when it is audible; on this
1769
- // transport the assistant is a Web Audio graph, so tap the output gain into a stream.
1770
- var monitorDest = realtimeAudioCtx.createMediaStreamDestination();
1771
- realtimeGain.connect(monitorDest);
1772
-
1773
- // Always send a frame (silence when muted) so the server keeps a continuous audio
1774
- // stream and its voice-activity detector promptly notices the pause and responds.
1775
- // Muted cases: push-to-talk not held, or the echo guard while the assistant plays back.
1776
- // The decision stays on the main thread because that is where the push-to-talk and
1777
- // playback state lives; the audio thread only frames the samples.
1778
- var sendCapturedFrame = function sendCapturedFrame(input) {
1779
- var muted;
1780
- if (realtimePushToTalk) {
1781
- muted = !realtimePttActive;
1782
- } else {
1783
- muted = !realtimeBargeIn && realtimeAudioCtx && realtimeAudioCtx.currentTime < realtimePlayHead + REALTIME_WS_ECHO_HANGOVER_SEC;
1784
- }
1785
- var pcm = new Int16Array(input.length);
1786
- if (!muted) {
1787
- for (var i = 0; i < input.length; i++) {
1788
- var s = Math.max(-1, Math.min(1, input[i]));
1789
- pcm[i] = s < 0 ? s * 0x8000 : s * 0x7FFF;
1790
- }
1791
- }
1792
- var bytes = new Uint8Array(pcm.buffer);
1793
- var binary = '';
1794
- for (var b = 0; b < bytes.length; b++) {
1795
- binary += String.fromCharCode(bytes[b]);
1796
- }
1797
- try {
1798
- realtimeSubject.next(btoa(binary));
1799
- } catch (err) {/* completed */}
1800
- };
1801
-
1802
- // The capture node loads a worklet, so it arrives a tick later than the rest of the
1803
- // graph; it streams silence until the gated microphone is attached below.
1804
- createRealtimeCaptureNode(realtimeAudioCtx, sendCapturedFrame).then(function (captureNode) {
1805
- if (!isRealtimeActive || realtimeAudioCtx !== ctxAtStart) {
1806
- try {
1807
- captureNode.disconnect();
1808
- } catch (err) {}
1809
- return;
1810
- }
1811
- realtimeProcessor = captureNode;
1812
- captureNode.connect(zeroGain);
1813
-
1814
- // Send the gated microphone rather than the raw one, exactly as the WebRTC transport
1815
- // does, so this fallback is not the one transport where the model hears its own echo.
1816
- setupMicGate(stream).then(function (micTrack) {
1817
- if (!isRealtimeActive || realtimeAudioCtx !== ctxAtStart || !micTrack) {
1818
- return;
1819
- }
1820
- var gatedStream = new MediaStream([micTrack]);
1821
- var source = realtimeAudioCtx.createMediaStreamSource(gatedStream);
1822
- realtimeMicSource = source;
1823
- source.connect(captureNode);
1824
- if (realtimeGate) {
1825
- realtimeGate.attachAssistantStream(monitorDest.stream);
1826
- }
1827
- });
1828
- });
1829
-
1830
- // "Auto" means auto-detect: send nothing. Sending the browser's locale instead pinned transcription
1831
- // and the reply language to it, so a bilingual user with an English browser speaking Spanish
1832
- // got English-forced (garbage) transcripts and an English-locked assistant.
1833
- var language = realtimeLanguage || null;
1834
- var voice = getVoiceName && getVoiceName() || realtimeVoiceName || '';
1835
- sendStart(realtimeSubject, voice, language, null, null, realtimeBargeIn);
1836
- if (realtimePushToTalk) {
1837
- buildRealtimePttUi();
1838
- }
1839
- })["catch"](function (err) {
1840
- stream.getTracks().forEach(function (track) {
1841
- track.stop();
1842
- });
1843
- isRealtimeActive = false;
1844
- onDeactivate();
1845
- updateRealtimeButton();
1846
- console.error('The realtime conversation could not start because the chat connection is not available.', err);
1847
- });
1848
- })["catch"](function (err) {
1849
- console.error('Microphone access denied:', err);
1850
- isRealtimeActive = false;
1851
- onDeactivate();
1852
- updateRealtimeButton();
1853
- });
1854
- }
1855
-
1856
- // --- WebRTC (server-relay) transport ---
1857
-
1858
- function startRealtimeWebRtcConversation() {
1859
- // Keep the WebRTC capture constraints minimal. WebRTC's own echo canceller (AEC3) already references
1860
- // the remote assistant track we render into the hidden <audio> element, which is the entire reason we
1861
- // use WebRTC for open-room audio. The stronger hints we request on the WebSocket path
1862
- // (echoCancellationType:'system' + voiceIsolation) are counterproductive here: layered on top of the
1863
- // peer-connection AEC loop they gate the mic down to near-silence (~1-3% of full scale), so the model
1864
- // never detects speech. Ask only for standard echo cancellation and let AEC3 do its job.
1865
- var micConstraints = {
1866
- // Match what ChatGPT's voice mode requests: a plain echoCancellation:true and nothing else exotic.
1867
- // The browser still applies its default noise suppression and auto gain; we only pass those through
1868
- // so the user's toggles can turn them off.
1869
- echoCancellation: true,
1870
- noiseSuppression: true,
1871
- autoGainControl: true
1872
- };
1873
- if (realtimeMicDeviceId) {
1874
- micConstraints.deviceId = {
1875
- exact: realtimeMicDeviceId
1876
- };
1877
- }
1878
- navigator.mediaDevices.getUserMedia({
1879
- audio: micConstraints
1880
- }).then(function (stream) {
1881
- ensureConnected().then(function () {
1882
- return resolveIceServers();
1883
- }).then(function (iceServers) {
1884
- realtimeStream = stream;
1885
- isRealtimeActive = true;
1886
- realtimeIsWebRtc = true;
1887
- onActivate();
1888
- updateRealtimeButton();
1889
- setRealtimeState('connecting');
1890
- realtimeRemoteDescriptionSet = false;
1891
- realtimePendingIce = [];
1892
- realtimeWebRtcConnected = false;
1893
- var pc = new RTCPeerConnection({
1894
- iceServers: iceServers
1895
- });
1896
- realtimePc = pc;
1897
-
1898
- // Push-to-talk and the barge-in/echo guard both gate the outbound mic track. Wire the
1899
- // same push-to-talk controls the WebSocket path uses so they work on WebRTC too.
1900
- // Device labels only become readable after permission is granted, so this is the first
1901
- // chance to notice a headset (safe for full duplex) or a standalone mic in front of
1902
- // speakers (not). Only ever a suggestion: an explicit choice is never overridden.
1903
-
1904
- attachRealtimePushToTalk();
1905
- var focusedBtn = q('realtimeButton');
1906
- if (focusedBtn) {
1907
- try {
1908
- focusedBtn.blur();
1909
- } catch (err) {}
1910
- }
1911
- if (realtimePushToTalk) {
1912
- buildRealtimePttUi();
1913
- }
1914
-
1915
- // Play the assistant's remote track through a hidden <audio> element. This gives the
1916
- // browser's echo canceller a reference to remove the assistant's voice from the mic, and
1917
- // is what the echo-guard monitor listens to for half-duplex mic muting.
1918
- var audioEl = document.createElement('audio');
1919
- audioEl.autoplay = true;
1920
- audioEl.style.display = 'none';
1921
- audioEl.volume = Math.max(0, Math.min(1, realtimeVolume != null ? realtimeVolume : 1));
1922
- document.body.appendChild(audioEl);
1923
- realtimeRemoteAudioEl = audioEl;
1924
- // Route playback so the browser couples it with the mic's echo canceller as a proper
1925
- // AEC reference (see routeRealtimeOutputToDefaultDevice for the preference order).
1926
- routeRealtimeOutputToDefaultDevice(audioEl);
1927
- pc.ontrack = function (e) {
1928
- // Ask the browser to hold a fixed cushion of received audio before playing it,
1929
- // rather than the adaptive minimum it would otherwise pick. Network jitter then
1930
- // lands inside the cushion instead of making the jitter buffer stretch or speed
1931
- // speech up to re-adapt; the audio itself is played as received. Ignored by
1932
- // browsers without the property.
1933
- try {
1934
- if (e.receiver && 'jitterBufferTarget' in e.receiver) {
1935
- e.receiver.jitterBufferTarget = REALTIME_WEBRTC_JITTER_BUFFER_TARGET_MS;
1936
- }
1937
- } catch (err) {}
1938
- if (e.streams && e.streams[0]) {
1939
- audioEl.srcObject = e.streams[0];
1940
- ensureRealtimePlayback(audioEl);
1941
- // Let the gate watch the assistant's level so it can raise the speech threshold
1942
- // while the assistant talks (barge-in on) and stay closed until it finishes
1943
- // (barge-in off).
1944
- attachWebRtcAssistantAnalyser(e.streams[0]);
1945
- }
1946
- };
1947
- realtimeSawRelayCandidate = false;
1948
- pc.onicecandidate = function (e) {
1949
- if (e.candidate) {
1950
- if (e.candidate.candidate && e.candidate.candidate.indexOf(' relay ') !== -1) {
1951
- realtimeSawRelayCandidate = true;
1952
- }
1953
- try {
1954
- connection.send('AddRealtimeIceCandidate', e.candidate.candidate, e.candidate.sdpMid || '', e.candidate.sdpMLineIndex || 0);
1955
- } catch (err) {}
1956
- return;
1957
- }
1958
-
1959
- // Gathering finished. With no relay candidate there is no path left for a network
1960
- // that blocks UDP, so stop waiting the full timeout for something that cannot come.
1961
- if (!realtimeSawRelayCandidate && !realtimeWebRtcConnected && realtimeWebRtcConnectTimer) {
1962
- clearTimeout(realtimeWebRtcConnectTimer);
1963
- realtimeWebRtcConnectTimer = setTimeout(function () {
1964
- if (!realtimeWebRtcConnected && isRealtimeActive) {
1965
- fallbackToWebSocket('no relay candidate and no direct connection');
1966
- }
1967
- }, REALTIME_WEBRTC_NO_RELAY_TIMEOUT_MS);
1968
- }
1969
- };
1970
- pc.onconnectionstatechange = function () {
1971
- if (pc.connectionState === 'connected') {
1972
- markWebRtcConnected();
1973
- } else if (pc.connectionState === 'failed') {
1974
- // Before the peer connects, a failure means WebRTC is unusable here — fall back.
1975
- // After it connects, a drop ends the session (we never migrate mid-session).
1976
- if (!realtimeWebRtcConnected) {
1977
- fallbackToWebSocket('connection failed');
1978
- } else if (isRealtimeActive) {
1979
- stopRealtimeConversation();
1980
- }
1981
- } else if (pc.connectionState === 'closed' && isRealtimeActive && realtimeWebRtcConnected) {
1982
- stopRealtimeConversation();
1983
- }
1984
- };
1985
- pc.oniceconnectionstatechange = function () {
1986
- var s = pc.iceConnectionState;
1987
- if (s === 'connected' || s === 'completed') {
1988
- markWebRtcConnected();
1989
- } else if (s === 'failed' && !realtimeWebRtcConnected) {
1990
- fallbackToWebSocket('ICE failed');
1991
- }
1992
- };
1993
- bindWebRtcSignalingHandlers();
1994
-
1995
- // Fall back to WebSocket if the peer does not connect within the timeout (blocked UDP,
1996
- // missing TURN, etc.). Cleared as soon as the peer connects (markWebRtcConnected).
1997
- realtimeWebRtcConnectTimer = setTimeout(function () {
1998
- if (!realtimeWebRtcConnected && isRealtimeActive) {
1999
- fallbackToWebSocket('connection timed out');
2000
- }
2001
- }, REALTIME_WEBRTC_CONNECT_TIMEOUT_MS);
2002
-
2003
- // Send a gated version of the mic: silent unless the user is actually speaking. The gate
2004
- // loads an AudioWorklet, so the track is only available once that resolves — hence the
2005
- // offer is created here rather than synchronously above.
2006
- setupMicGate(stream).then(function (micTrackToSend) {
2007
- if (realtimePc !== pc) {
2008
- return null;
2009
- }
2010
- if (micTrackToSend) {
2011
- pc.addTrack(micTrackToSend, stream);
2012
- }
2013
- return pc.createOffer();
2014
- }).then(function (offer) {
2015
- if (!offer || realtimePc !== pc) {
2016
- return null;
2017
- }
2018
- return pc.setLocalDescription(offer).then(function () {
2019
- return offer;
2020
- });
2021
- }).then(function (offer) {
2022
- if (!offer) {
2023
- return;
2024
- }
2025
- // "Auto" means auto-detect: send nothing (see the WebSocket path above).
2026
- var language = realtimeLanguage || null;
2027
- var voice = getVoiceName && getVoiceName() || realtimeVoiceName || '';
2028
- sendStartWebRtc(offer.sdp, voice, language, null, null, realtimeBargeIn);
2029
- })["catch"](function (err) {
2030
- console.error('Failed to create the WebRTC offer; falling back to WebSocket.', err);
2031
- fallbackToWebSocket('offer failed');
2032
- });
2033
- })["catch"](function (err) {
2034
- stream.getTracks().forEach(function (t) {
2035
- t.stop();
2036
- });
2037
- isRealtimeActive = false;
2038
- realtimeIsWebRtc = false;
2039
- onDeactivate();
2040
- updateRealtimeButton();
2041
- console.error('The realtime WebRTC conversation could not start because the chat connection is not available.', err);
2042
- });
2043
- })["catch"](function (err) {
2044
- console.error('Microphone access denied:', err);
2045
- isRealtimeActive = false;
2046
- realtimeIsWebRtc = false;
2047
- onDeactivate();
2048
- updateRealtimeButton();
2049
- });
2050
- }
2051
- function bindWebRtcSignalingHandlers() {
2052
- if (realtimeWebRtcHandlersBound) {
2053
- return;
2054
- }
2055
- realtimeWebRtcHandlersBound = true;
2056
- connection.on('ReceiveRealtimeAnswer', function (sdp) {
2057
- if (!realtimePc) {
2058
- return;
2059
- }
2060
- realtimePc.setRemoteDescription({
2061
- type: 'answer',
2062
- sdp: sdp
2063
- }).then(function () {
2064
- realtimeRemoteDescriptionSet = true;
2065
- // Flush any ICE candidates that arrived before the answer was applied.
2066
- var pending = realtimePendingIce;
2067
- realtimePendingIce = [];
2068
- pending.forEach(function (init) {
2069
- try {
2070
- realtimePc.addIceCandidate(init)["catch"](function () {});
2071
- } catch (err) {}
2072
- });
2073
- })["catch"](function (err) {
2074
- console.error('Failed to apply the WebRTC answer.', err);
2075
- });
2076
- });
2077
- connection.on('ReceiveRealtimeIceCandidate', function (candidate, sdpMid, sdpMLineIndex) {
2078
- if (!realtimePc || !candidate) {
2079
- return;
2080
- }
2081
- var init = {
2082
- candidate: candidate,
2083
- sdpMid: sdpMid,
2084
- sdpMLineIndex: sdpMLineIndex
2085
- };
2086
- // A candidate can only be added after the remote description (answer) is set; buffer until then.
2087
- if (realtimeRemoteDescriptionSet) {
2088
- try {
2089
- realtimePc.addIceCandidate(init)["catch"](function () {});
2090
- } catch (err) {}
2091
- } else {
2092
- realtimePendingIce.push(init);
2093
- }
2094
- });
2095
- }
2096
- function stopRealtimeWebRtc() {
2097
- stopWebRtcEchoGuard();
2098
- realtimeWebRtcMicTrack = null;
2099
- if (realtimePc) {
2100
- try {
2101
- realtimePc.close();
2102
- } catch (err) {}
2103
- realtimePc = null;
2104
- }
2105
- if (realtimeRemoteAudioEl) {
2106
- try {
2107
- realtimeRemoteAudioEl.srcObject = null;
2108
- realtimeRemoteAudioEl.remove();
2109
- } catch (err) {}
2110
- realtimeRemoteAudioEl = null;
2111
- }
2112
- realtimeIsWebRtc = false;
2113
- }
2114
- function markWebRtcConnected() {
2115
- if (realtimeWebRtcConnected) {
2116
- return;
2117
- }
2118
- realtimeWebRtcConnected = true;
2119
- if (realtimeWebRtcConnectTimer) {
2120
- clearTimeout(realtimeWebRtcConnectTimer);
2121
- realtimeWebRtcConnectTimer = null;
2122
- }
2123
- }
2124
-
2125
- // --- Microphone gate (thin wrapper over the shared factory) ----------------------------------------
2126
-
2127
- // How long the gate stays open after the user's level drops. The gate is a safety net around echo, not a
2128
- // turn detector: the provider's turn detection sees the real speech and should be the one deciding a
2129
- // turn is over, so this must comfortably outlast the pause it is willing to wait through. The gate emits
2130
- // digital silence when it closes, and whichever of the two expires first is what actually ends the turn.
2131
- var REALTIME_GATE_HANGOVER_MS = 2000;
2132
- function realtimeGateModeMessage() {
2133
- return {
2134
- pushToTalk: !!realtimePushToTalk,
2135
- pttActive: !!realtimePttActive,
2136
- bargeIn: !!realtimeBargeIn,
2137
- speechHangoverMs: REALTIME_GATE_HANGOVER_MS
2138
- };
2139
- }
2140
-
2141
- // The gate runs on the audio thread and cannot read this closure's variables, so every settings or
2142
- // push-to-talk change has to be pushed to it explicitly.
2143
- function syncRealtimeGateMode() {
2144
- if (realtimeGate) {
2145
- realtimeGate.setMode(realtimeGateModeMessage());
2146
- }
2147
- }
2148
-
2149
- // Builds the gate and resolves with the track to send (over the peer, or into the WebSocket capture graph).
2150
- function setupMicGate(rawStream) {
2151
- stopWebRtcEchoGuard();
2152
- var rawTrack = rawStream && rawStream.getAudioTracks()[0] || null;
2153
- realtimeWebRtcMicTrack = rawTrack;
2154
- return createMicGate(rawStream, realtimeGateModeMessage()).then(function (gate) {
2155
- realtimeGate = gate;
2156
- if (realtimeGatePendingRemoteStream) {
2157
- gate.attachAssistantStream(realtimeGatePendingRemoteStream);
2158
- }
2159
- return gate.track || rawTrack;
2160
- });
2161
- }
2162
-
2163
- // Feeds the assistant's remote stream into the gate so it knows when the assistant is audible — for the
2164
- // raised threshold with barge-in on, and for half-duplex with barge-in off. The track can arrive before
2165
- // the gate's worklet has finished loading, so remember it either way.
2166
- function attachWebRtcAssistantAnalyser(remoteStream) {
2167
- realtimeGatePendingRemoteStream = remoteStream || null;
2168
- if (realtimeGate && remoteStream) {
2169
- realtimeGate.attachAssistantStream(remoteStream);
2170
- }
2171
- }
2172
- function stopWebRtcEchoGuard() {
2173
- if (realtimeGate) {
2174
- try {
2175
- realtimeGate.stop();
2176
- } catch (e) {}
2177
- realtimeGate = null;
2178
- }
2179
- realtimeGatePendingRemoteStream = null;
2180
- }
2181
-
2182
- // Route the assistant playback element so the browser's echo canceller couples to it as an AEC reference,
2183
- // the way ChatGPT's voice mode does. Preference order:
2184
- // 1. The "communications" sink (Chrome/Edge). Rendering here doesn't just pick a device — it puts the
2185
- // browser's audio pipeline in COMMUNICATIONS mode, which is what actually couples playback with the
2186
- // mic's echo canceller. This is what ChatGPT uses and what keeps full-duplex stable across turns.
2187
- // 2. The concrete device the "default" alias points at, matched by groupId.
2188
- // If neither alias exists — Firefox exposes only concrete outputs — we deliberately do nothing and let the
2189
- // browser use its own default. Picking "the first concrete output" there routed the assistant to whatever
2190
- // device happened to be enumerated first (often an HDMI monitor), so the assistant appeared silent.
2191
- // Applies the user's chosen speaker to the playing element, live.
2192
- function applyRealtimeOutputDevice() {
2193
- routeOutputToPreferredDevice(realtimeRemoteAudioEl, realtimeOutputDeviceId);
2194
- }
2195
- function routeRealtimeOutputToDefaultDevice(el) {
2196
- routeOutputToPreferredDevice(el, realtimeOutputDeviceId);
2197
- }
2198
- function ensureRealtimePlayback(el) {
2199
- ensurePlayback(el, function () {
2200
- setRealtimeState('playback-blocked');
2201
- });
2202
- }
2203
-
2204
- // Says why a realtime session is running on the WebSocket transport when it never attempted WebRTC.
2205
- // The connect-time fallback reports its own reason instead (see fallbackToWebSocket).
2206
- function logWebSocketTransportReason(reason) {
2207
- if (window.console && console.warn) {
2208
- console.warn('Realtime is using the WebSocket transport (' + reason + '); acoustic echo cancellation is weaker than on WebRTC.');
2209
- }
2210
- }
2211
-
2212
- // Connect-time only: tear down the failed WebRTC attempt and restart on the known-good WebSocket path.
2213
- // Never called once a session is established (see the connection-state handlers), so we never migrate audio
2214
- // mid-conversation — we only choose the transport before the model starts responding.
2215
- function fallbackToWebSocket(reason) {
2216
- if (realtimeFellBack) {
2217
- return;
2218
- }
2219
- realtimeFellBack = true;
2220
- if (realtimeWebRtcConnectTimer) {
2221
- clearTimeout(realtimeWebRtcConnectTimer);
2222
- realtimeWebRtcConnectTimer = null;
2223
- }
2224
- if (window.console && console.warn) {
2225
- console.warn('Realtime WebRTC transport unavailable (' + reason + '); falling back to the WebSocket transport.');
2226
- }
2227
- rememberWebRtcBlocked();
2228
- showCompatibilityModeNotice();
2229
- try {
2230
- if (realtimeStream) {
2231
- realtimeStream.getTracks().forEach(function (t) {
2232
- t.stop();
2233
- });
2234
- }
2235
- } catch (err) {}
2236
- realtimeStream = null;
2237
- stopRealtimeWebRtc();
2238
- realtimeIsWebRtc = false;
2239
- // Reset the active flag so the WebSocket start proceeds cleanly; it re-activates immediately.
2240
- isRealtimeActive = false;
2241
- startRealtimeWebSocketConversation();
2242
- }
2243
- function stopRealtimeConversation() {
2244
- if (!isRealtimeActive) {
2245
- return;
2246
- }
2247
- if (realtimeWebRtcConnectTimer) {
2248
- clearTimeout(realtimeWebRtcConnectTimer);
2249
- realtimeWebRtcConnectTimer = null;
2250
- }
2251
- isRealtimeActive = false;
2252
- realtimeSessionReady = false;
2253
- onDeactivate();
2254
- updateRealtimeButton();
2255
- detachRealtimePushToTalk();
2256
- removeRealtimePttUi();
2257
- if (!realtimeEndedNotice) {
2258
- setRealtimeState('idle');
2259
- }
2260
- if (realtimeIsWebRtc) {
2261
- stopRealtimeWebRtc();
2262
- try {
2263
- if (realtimeStream) {
2264
- realtimeStream.getTracks().forEach(function (t) {
2265
- t.stop();
2266
- });
2267
- }
2268
- } catch (err) {}
2269
- realtimeStream = null;
2270
- return;
2271
- }
2272
- try {
2273
- if (realtimeSubject) {
2274
- realtimeSubject.complete();
2275
- }
2276
- } catch (err) {/* already completed */}
2277
- realtimeSubject = null;
2278
- stopWebRtcEchoGuard();
2279
-
2280
- // Detach both shapes the capture node can take: the worklet delivers frames over its port, the
2281
- // ScriptProcessorNode fallback over onaudioprocess. Leaving either attached keeps pushing frames at
2282
- // a subject that is already completed.
2283
- try {
2284
- if (realtimeProcessor) {
2285
- realtimeProcessor.disconnect();
2286
- realtimeProcessor.onaudioprocess = null;
2287
- if (realtimeProcessor.port) {
2288
- realtimeProcessor.port.onmessage = null;
2289
- realtimeProcessor.port.close();
2290
- }
2291
- }
2292
- } catch (err) {}
2293
- try {
2294
- if (realtimeMicSource) {
2295
- realtimeMicSource.disconnect();
2296
- }
2297
- } catch (err) {}
2298
- try {
2299
- if (realtimeZeroGain) {
2300
- realtimeZeroGain.disconnect();
2301
- }
2302
- } catch (err) {}
2303
- try {
2304
- if (realtimeGain) {
2305
- realtimeGain.disconnect();
2306
- }
2307
- } catch (err) {}
2308
- try {
2309
- if (realtimeStream) {
2310
- realtimeStream.getTracks().forEach(function (t) {
2311
- t.stop();
2312
- });
2313
- }
2314
- } catch (err) {}
2315
- realtimeProcessor = null;
2316
- realtimeMicSource = null;
2317
- realtimeZeroGain = null;
2318
- realtimeGain = null;
2319
- realtimeStream = null;
2320
- flushRealtimePlayback();
2321
- try {
2322
- if (realtimeAudioCtx) {
2323
- realtimeAudioCtx.close();
2324
- }
2325
- } catch (err) {}
2326
- realtimeAudioCtx = null;
2327
- }
2328
- function toggleRealtime() {
2329
- if (isRealtimeActive) {
2330
- stopRealtimeConversation();
2331
- return;
2332
- }
2333
- startRealtimeConversation();
2334
- }
2335
-
2336
- // Resolves with the receive-side statistics of the assistant's audio track (WebRTC transport only), in
2337
- // the browser's own words: packets lost, jitter, how much the jitter buffer is holding, and how often it
2338
- // had to conceal or time-stretch audio. Null on the WebSocket transport.
2339
- function getTransportStats() {
2340
- if (!realtimePc || typeof realtimePc.getStats !== 'function') {
2341
- return Promise.resolve(null);
2342
- }
2343
- return realtimePc.getStats().then(function (report) {
2344
- // Where playback actually goes (the "communications" sink puts Chrome's pipeline in communications
2345
- // mode; empty means the browser default) and the receiver's requested cushion, so a report of
2346
- // "sounds different in this browser" can be tied to the output path.
2347
- var result = {
2348
- transport: 'webrtc',
2349
- inbound: null,
2350
- codec: null,
2351
- sinkId: realtimeRemoteAudioEl && typeof realtimeRemoteAudioEl.sinkId === 'string' ? realtimeRemoteAudioEl.sinkId : null,
2352
- jitterBufferTargetMs: function () {
2353
- try {
2354
- var r = realtimePc.getReceivers().filter(function (x) {
2355
- return x.track && x.track.kind === 'audio';
2356
- })[0];
2357
- return r && 'jitterBufferTarget' in r ? r.jitterBufferTarget : null;
2358
- } catch (e) {
2359
- return null;
2360
- }
2361
- }()
2362
- };
2363
- report.forEach(function (stat) {
2364
- if (stat.type === 'inbound-rtp' && stat.kind === 'audio') {
2365
- result.inbound = {
2366
- packetsReceived: stat.packetsReceived,
2367
- packetsLost: stat.packetsLost,
2368
- packetsDiscarded: stat.packetsDiscarded,
2369
- jitterMs: typeof stat.jitter === 'number' ? Math.round(stat.jitter * 1000) : null,
2370
- jitterBufferDelayMs: typeof stat.jitterBufferDelay === 'number' && stat.jitterBufferEmittedCount ? Math.round(stat.jitterBufferDelay / stat.jitterBufferEmittedCount * 1000) : null,
2371
- totalSamplesReceived: stat.totalSamplesReceived,
2372
- concealedSamples: stat.concealedSamples,
2373
- concealmentEvents: stat.concealmentEvents,
2374
- silentConcealedSamples: stat.silentConcealedSamples,
2375
- insertedSamplesForDeceleration: stat.insertedSamplesForDeceleration,
2376
- removedSamplesForAcceleration: stat.removedSamplesForAcceleration,
2377
- audioLevel: stat.audioLevel
2378
- };
2379
- if (stat.codecId) {
2380
- var codec = report.get(stat.codecId);
2381
- if (codec) {
2382
- result.codec = {
2383
- mimeType: codec.mimeType,
2384
- clockRate: codec.clockRate,
2385
- channels: codec.channels,
2386
- sdpFmtpLine: codec.sdpFmtpLine
2387
- };
2388
- }
2389
- }
2390
- }
2391
- });
2392
- return result;
2393
- })["catch"](function () {
2394
- return null;
2395
- });
2396
- }
2397
- function playRealtimePcm(bytes) {
2398
- if (!realtimeAudioCtx || !bytes || bytes.length < 2) {
2399
- return;
2400
- }
2401
- var ctx = realtimeAudioCtx;
2402
- var sampleCount = Math.floor(bytes.length / 2);
2403
- var pcm = new Int16Array(bytes.buffer, bytes.byteOffset, sampleCount);
2404
- var f32 = new Float32Array(sampleCount);
2405
- for (var i = 0; i < sampleCount; i++) {
2406
- f32[i] = pcm[i] / 0x8000;
2407
- }
2408
- var buffer = ctx.createBuffer(1, sampleCount, ctx.sampleRate);
2409
- buffer.copyToChannel(f32, 0);
2410
- var src = ctx.createBufferSource();
2411
- src.buffer = buffer;
2412
- src.connect(realtimeGain || ctx.destination);
2413
- var now = ctx.currentTime;
2414
- if (realtimePlayHead < now) {
2415
- realtimePlayHead = now;
2416
- }
2417
- src.start(realtimePlayHead);
2418
- realtimePlayHead += buffer.duration;
2419
- realtimeSources.push(src);
2420
- src.onended = function () {
2421
- realtimeSources = realtimeSources.filter(function (s) {
2422
- return s !== src;
2423
- });
2424
- };
2425
- }
2426
- function flushRealtimePlayback() {
2427
- realtimeSources.forEach(function (s) {
2428
- try {
2429
- s.stop();
2430
- } catch (err) {}
2431
- });
2432
- realtimeSources = [];
2433
- if (realtimeAudioCtx) {
2434
- realtimePlayHead = realtimeAudioCtx.currentTime;
2435
- }
2436
- }
2437
- function isRealtimeDeployment(deploymentName) {
2438
- if (!deploymentName) {
2439
- return false;
2440
- }
2441
- return realtimeCapableDeployments.indexOf(deploymentName.toLowerCase()) !== -1;
2442
- }
2443
-
2444
- // --- Wire up ---
2445
- var realtimeBtn = q('realtimeButton');
2446
- if (realtimeBtn) {
2447
- realtimeBtn.addEventListener('click', function (e) {
2448
- // Ignore keyboard-synthesized clicks (Space/Enter) during an active realtime session, so a
2449
- // push-to-talk Space press can never toggle the session off.
2450
- if (e && e.detail === 0 && isRealtimeActive) {
2451
- return;
2452
- }
2453
- toggleRealtime();
2454
- });
2455
- setupRealtimeAudioSettings();
2456
- }
2457
- var deploymentSelect = q('deploymentSelect');
2458
- if (deploymentSelect) {
2459
- deploymentSelect.addEventListener('change', function () {
2460
- applyRealtimeMode(isRealtimeDeployment(deploymentSelect.value));
2461
- });
2462
- }
2463
-
2464
- // Apply the initial input mode: audio-only when the interaction already uses a realtime deployment.
2465
- applyRealtimeMode(isRealtimeMode || deploymentSelect && isRealtimeDeployment(deploymentSelect.value));
2466
- var controller = {
2467
- toggle: toggleRealtime,
2468
- start: startRealtimeConversation,
2469
- stop: stopRealtimeConversation,
2470
- applyMode: applyRealtimeMode,
2471
- isRealtimeDeployment: isRealtimeDeployment,
2472
- receivePcm: playRealtimePcm,
2473
- isActive: function isActive() {
2474
- return isRealtimeActive;
2475
- },
2476
- getState: function getState() {
2477
- return realtimeState;
2478
- },
2479
- // The gate's most recent measurement (level, tracked noise floor, open/closed, assistant audible), for
2480
- // hosts that want to render a microphone meter or explain why the mic is muted.
2481
- getGateLevel: function getGateLevel() {
2482
- return realtimeGate ? realtimeGate.getLevel() : null;
2483
- },
2484
- // The browser's own receive-side statistics for the assistant's audio on the WebRTC transport
2485
- // (packets lost, jitter, jitter-buffer delay, concealment), for diagnosing choppy playback.
2486
- getTransportStats: getTransportStats
2487
- };
2488
-
2489
- // The most recently attached controller, for diagnostics from the console or a test.
2490
- window.CoreAIRealtime.activeController = controller;
2491
- return controller;
2492
- }
2493
- window.CoreAIRealtime = {
2494
- attach: attach,
2495
- createMicGate: createMicGate,
2496
- // Exposed for tests: the gate's rules are where every one of its historical failures lived, and they are
2497
- // far easier to pin down with numbers than with audio.
2498
- decideGate: coreAiGateDecide,
2499
- createGateState: createGateState,
2500
- rmsDb: coreAiRmsDb,
2501
- // Host-agnostic playback helpers, so a host that drives realtime itself still shares one implementation
2502
- // of these rather than growing its own.
2503
- routeOutputToPreferredDevice: routeOutputToPreferredDevice,
2504
- ensurePlayback: ensurePlayback
2505
- };
2506
- })(window, document);
5
+ (function() {
6
+ //#region src/Resources/CrestApps.AI.Resources/Assets/js/realtime-audio.js
7
+ (function(window, document) {
8
+ "use strict";
9
+ var REALTIME_SAMPLE_RATE = 24e3;
10
+ var REALTIME_GATE_LOOKAHEAD_MS = 80;
11
+ function coreAiGateDecide(state, input) {
12
+ var micDb = input.micDb;
13
+ var assistantDb = typeof input.assistantDb === "number" ? input.assistantDb : -100;
14
+ var nowMs = input.nowMs;
15
+ var mode = input.mode || {};
16
+ var dtMs = state.lastMs === null ? 0 : Math.max(0, Math.min(50, nowMs - state.lastMs));
17
+ state.lastMs = nowMs;
18
+ if (assistantDb > -50) state.assistantUntil = nowMs + 400;
19
+ var assistantSpeaking = nowMs < state.assistantUntil;
20
+ if (assistantDb > state.assistantPeakDb) state.assistantPeakDb = assistantDb;
21
+ else state.assistantPeakDb = Math.max(-100, state.assistantPeakDb - .02 * dtMs);
22
+ if (micDb <= -95) {
23
+ if (state.floorDb === null) {
24
+ state.open = false;
25
+ return state;
26
+ }
27
+ } else if (state.floorDb === null) state.floorDb = Math.min(micDb, -50);
28
+ if (micDb < state.floorDb) state.floorDb += (micDb - state.floorDb) * Math.min(1, .0075 * dtMs);
29
+ else if (!assistantSpeaking) {
30
+ var sustained = state.openSince !== null && nowMs - state.openSince > 1e4;
31
+ var risingPerMs = state.open && !sustained ? 1e-4 : 56e-5;
32
+ state.floorDb += (micDb - state.floorDb) * Math.min(1, risingPerMs * dtMs);
33
+ }
34
+ if (state.floorDb < -75) state.floorDb = -75;
35
+ if (assistantSpeaking && !state.open && assistantDb > -50 && !mode.pushToTalk) {
36
+ var erl = micDb - state.assistantPeakDb;
37
+ if (state.echoReturnDb === null) state.echoReturnDb = erl;
38
+ else {
39
+ var warm = state.echoLearnedMs < 1e3;
40
+ var ratePerMs = erl > state.echoReturnDb ? warm ? .02 : .0012 : 6e-4;
41
+ state.echoReturnDb += (erl - state.echoReturnDb) * Math.min(1, ratePerMs * dtMs);
42
+ }
43
+ state.echoLearnedMs += dtMs;
44
+ }
45
+ var expectedEchoDb = state.echoReturnDb === null ? -100 : state.assistantPeakDb + state.echoReturnDb;
46
+ var openThreshold, holdThreshold;
47
+ if (assistantSpeaking) {
48
+ openThreshold = Math.max(state.floorDb + 12, expectedEchoDb + 8, -45);
49
+ holdThreshold = Math.max(state.floorDb + 3, expectedEchoDb + 3);
50
+ } else {
51
+ openThreshold = state.floorDb + 9;
52
+ holdThreshold = state.floorDb + 3;
53
+ }
54
+ var hangoverMs = mode.speechHangoverMs || 2e3;
55
+ if (micDb > openThreshold) {
56
+ if (assistantSpeaking && !state.open) {
57
+ if (state.aboveSince === null) state.aboveSince = nowMs;
58
+ if (nowMs - state.aboveSince >= 250) state.speakUntil = nowMs + hangoverMs;
59
+ } else state.speakUntil = nowMs + hangoverMs;
60
+ } else {
61
+ state.aboveSince = null;
62
+ if (state.open && micDb > holdThreshold) state.speakUntil = Math.max(state.speakUntil, nowMs + 250);
63
+ }
64
+ var userActive = nowMs < state.speakUntil;
65
+ if (!assistantSpeaking && state.wasAssistantSpeaking) state.listenGraceUntil = nowMs + 1500;
66
+ state.wasAssistantSpeaking = assistantSpeaking;
67
+ state.assistantSpeaking = assistantSpeaking;
68
+ if (mode.pushToTalk) state.open = !!mode.pttActive;
69
+ else if (mode.bargeIn) state.open = userActive;
70
+ else state.open = !assistantSpeaking && (userActive || nowMs < state.listenGraceUntil);
71
+ if (state.open && state.openSince === null) state.openSince = nowMs;
72
+ else if (!state.open) state.openSince = null;
73
+ return state;
74
+ }
75
+ function createGateState() {
76
+ return {
77
+ floorDb: null,
78
+ lastMs: null,
79
+ speakUntil: 0,
80
+ assistantUntil: 0,
81
+ assistantPeakDb: -100,
82
+ echoReturnDb: null,
83
+ echoLearnedMs: 0,
84
+ aboveSince: null,
85
+ listenGraceUntil: 0,
86
+ wasAssistantSpeaking: false,
87
+ assistantSpeaking: false,
88
+ open: false,
89
+ openSince: null
90
+ };
91
+ }
92
+ function coreAiRmsDb(block) {
93
+ if (!block || !block.length) return -100;
94
+ var sum = 0;
95
+ for (var i = 0; i < block.length; i++) sum += block[i] * block[i];
96
+ var rms = Math.sqrt(sum / block.length);
97
+ return rms > 1e-7 ? 20 * Math.log10(rms) : -100;
98
+ }
99
+ var REALTIME_GATE_WORKLET_SOURCE = [
100
+ "var decide = " + coreAiGateDecide.toString() + ";",
101
+ "var newState = " + createGateState.toString() + ";",
102
+ "var rmsDb = " + coreAiRmsDb.toString() + ";",
103
+ "class CoreAiMicGateProcessor extends AudioWorkletProcessor {",
104
+ " constructor(options) {",
105
+ " super();",
106
+ " var o = (options && options.processorOptions) || {};",
107
+ " this.delay = new Float32Array(Math.max(1, Math.round(sampleRate * (o.lookaheadMs || 80) / 1000)));",
108
+ " this.delayPos = 0;",
109
+ " this.gain = 0;",
110
+ " this.mode = o.mode || {};",
111
+ " this.state = newState();",
112
+ " this.lastPost = 0;",
113
+ " this.port.onmessage = function (e) { this.mode = e.data || {}; }.bind(this);",
114
+ " }",
115
+ " process(inputs, outputs) {",
116
+ " var mic = (inputs[0] && inputs[0][0]) || null;",
117
+ " var assistant = (inputs[1] && inputs[1][0]) || null;",
118
+ " var out = (outputs[0] && outputs[0][0]) || null;",
119
+ " if (!out) { return true; }",
120
+ " var micDb = rmsDb(mic);",
121
+ " decide(this.state, { micDb: micDb, assistantDb: rmsDb(assistant), nowMs: currentTime * 1000, mode: this.mode });",
122
+ " var target = this.state.open ? 1 : 0;",
123
+ " var step = 1 / (sampleRate * 0.01);",
124
+ " var delay = this.delay;",
125
+ " var pos = this.delayPos;",
126
+ " var len = delay.length;",
127
+ " for (var i = 0; i < out.length; i++) {",
128
+ " var delayed = delay[pos];",
129
+ " delay[pos] = mic ? mic[i] : 0;",
130
+ " pos = (pos + 1) % len;",
131
+ " if (this.gain < target) { this.gain = Math.min(target, this.gain + step); }",
132
+ " else if (this.gain > target) { this.gain = Math.max(target, this.gain - step); }",
133
+ " out[i] = delayed * this.gain;",
134
+ " }",
135
+ " this.delayPos = pos;",
136
+ " if (currentTime * 1000 - this.lastPost > 100) {",
137
+ " this.lastPost = currentTime * 1000;",
138
+ " this.port.postMessage({ micDb: micDb, floorDb: this.state.floorDb, echoReturnDb: this.state.echoReturnDb, open: this.state.open, assistantSpeaking: this.state.assistantSpeaking });",
139
+ " }",
140
+ " return true;",
141
+ " }",
142
+ "}",
143
+ "registerProcessor(\"coreai-mic-gate\", CoreAiMicGateProcessor);"
144
+ ].join("\n");
145
+ var REALTIME_CAPTURE_FRAME_SAMPLES = 4096;
146
+ var REALTIME_CAPTURE_WORKLET_SOURCE = [
147
+ "class CoreAiCaptureProcessor extends AudioWorkletProcessor {",
148
+ " constructor(options) {",
149
+ " super();",
150
+ " var o = (options && options.processorOptions) || {};",
151
+ " this.size = Math.max(128, o.frameSamples || 4096);",
152
+ " this.buf = new Float32Array(this.size);",
153
+ " this.pos = 0;",
154
+ " }",
155
+ " process(inputs) {",
156
+ " var input = (inputs[0] && inputs[0][0]) || null;",
157
+ " var n = input ? input.length : 128;",
158
+ " for (var i = 0; i < n; i++) {",
159
+ " this.buf[this.pos++] = input ? input[i] : 0;",
160
+ " if (this.pos === this.size) {",
161
+ " this.port.postMessage(this.buf);",
162
+ " this.pos = 0;",
163
+ " }",
164
+ " }",
165
+ " return true;",
166
+ " }",
167
+ "}",
168
+ "registerProcessor(\"coreai-pcm-capture\", CoreAiCaptureProcessor);"
169
+ ].join("\n");
170
+ function createRealtimeCaptureNode(ctx, onFrame) {
171
+ function scriptProcessorNode() {
172
+ var node = ctx.createScriptProcessor(REALTIME_CAPTURE_FRAME_SAMPLES, 1, 1);
173
+ node.onaudioprocess = function(event) {
174
+ onFrame(event.inputBuffer.getChannelData(0));
175
+ };
176
+ return node;
177
+ }
178
+ var blobUrl = null;
179
+ if (ctx.audioWorklet && typeof ctx.audioWorklet.addModule === "function" && typeof AudioWorkletNode === "function") try {
180
+ blobUrl = URL.createObjectURL(new Blob([REALTIME_CAPTURE_WORKLET_SOURCE], { type: "application/javascript" }));
181
+ } catch (err) {
182
+ blobUrl = null;
183
+ }
184
+ if (!blobUrl) return Promise.resolve(scriptProcessorNode());
185
+ return ctx.audioWorklet.addModule(blobUrl).then(function() {
186
+ try {
187
+ URL.revokeObjectURL(blobUrl);
188
+ } catch (e) {}
189
+ var node = new AudioWorkletNode(ctx, "coreai-pcm-capture", {
190
+ numberOfInputs: 1,
191
+ numberOfOutputs: 1,
192
+ outputChannelCount: [1],
193
+ processorOptions: { frameSamples: REALTIME_CAPTURE_FRAME_SAMPLES }
194
+ });
195
+ node.port.onmessage = function(e) {
196
+ if (e.data) onFrame(e.data);
197
+ };
198
+ return node;
199
+ }).catch(function(err) {
200
+ try {
201
+ URL.revokeObjectURL(blobUrl);
202
+ } catch (e) {}
203
+ if (window.console && console.warn) console.warn("The realtime capture worklet could not be loaded; falling back to a ScriptProcessorNode.", err);
204
+ return scriptProcessorNode();
205
+ });
206
+ }
207
+ function createMicGate(rawStream, mode) {
208
+ var rawTrack = rawStream && rawStream.getAudioTracks && rawStream.getAudioTracks()[0] || null;
209
+ var state = {
210
+ ctx: null,
211
+ node: null,
212
+ micSource: null,
213
+ remoteSource: null,
214
+ dest: null,
215
+ timer: 0,
216
+ level: null,
217
+ fallback: null,
218
+ track: rawTrack,
219
+ stopped: false
220
+ };
221
+ var gate = {
222
+ track: rawTrack,
223
+ usingWorklet: false,
224
+ getLevel: function() {
225
+ return state.level;
226
+ },
227
+ setMode: function(next) {
228
+ mode = next || mode;
229
+ if (state.node && state.node.port) try {
230
+ state.node.port.postMessage(mode);
231
+ } catch (e) {}
232
+ if (state.fallback) state.fallback.mode = mode;
233
+ },
234
+ attachAssistantStream: function(remoteStream) {
235
+ attachAssistant(state, remoteStream);
236
+ },
237
+ stop: function() {
238
+ state.stopped = true;
239
+ if (state.timer) {
240
+ try {
241
+ window.clearInterval(state.timer);
242
+ } catch (e) {}
243
+ state.timer = 0;
244
+ }
245
+ if (state.node) {
246
+ try {
247
+ state.node.port.onmessage = null;
248
+ state.node.disconnect();
249
+ } catch (e) {}
250
+ state.node = null;
251
+ }
252
+ if (state.micSource) {
253
+ try {
254
+ state.micSource.disconnect();
255
+ } catch (e) {}
256
+ state.micSource = null;
257
+ }
258
+ if (state.remoteSource) {
259
+ try {
260
+ state.remoteSource.disconnect();
261
+ } catch (e) {}
262
+ state.remoteSource = null;
263
+ }
264
+ if (state.dest) {
265
+ try {
266
+ state.dest.disconnect();
267
+ } catch (e) {}
268
+ state.dest = null;
269
+ }
270
+ if (state.ctx) {
271
+ try {
272
+ state.ctx.close();
273
+ } catch (e) {}
274
+ state.ctx = null;
275
+ }
276
+ state.fallback = null;
277
+ }
278
+ };
279
+ var AudioCtor = window.AudioContext || window.webkitAudioContext;
280
+ if (!AudioCtor || !rawTrack) return Promise.resolve(gate);
281
+ try {
282
+ state.ctx = new AudioCtor();
283
+ if (state.ctx.state === "suspended" && typeof state.ctx.resume === "function") state.ctx.resume().catch(function() {});
284
+ } catch (err) {
285
+ state.ctx = null;
286
+ return Promise.resolve(gate);
287
+ }
288
+ var blobUrl = null;
289
+ if (state.ctx.audioWorklet && typeof state.ctx.audioWorklet.addModule === "function") try {
290
+ blobUrl = URL.createObjectURL(new Blob([REALTIME_GATE_WORKLET_SOURCE], { type: "application/javascript" }));
291
+ } catch (err) {
292
+ blobUrl = null;
293
+ }
294
+ if (!blobUrl) {
295
+ gate.track = buildFallbackGate(state, rawStream, rawTrack, mode);
296
+ return Promise.resolve(gate);
297
+ }
298
+ return state.ctx.audioWorklet.addModule(blobUrl).then(function() {
299
+ try {
300
+ URL.revokeObjectURL(blobUrl);
301
+ } catch (e) {}
302
+ if (state.stopped) return gate;
303
+ var node = new AudioWorkletNode(state.ctx, "coreai-mic-gate", {
304
+ numberOfInputs: 2,
305
+ numberOfOutputs: 1,
306
+ outputChannelCount: [1],
307
+ processorOptions: {
308
+ lookaheadMs: REALTIME_GATE_LOOKAHEAD_MS,
309
+ mode
310
+ }
311
+ });
312
+ state.node = node;
313
+ node.port.onmessage = function(e) {
314
+ state.level = e.data || null;
315
+ window.CoreAIRealtime.lastGateLevel = state.level;
316
+ };
317
+ state.micSource = state.ctx.createMediaStreamSource(rawStream);
318
+ state.micSource.connect(node, 0, 0);
319
+ state.dest = state.ctx.createMediaStreamDestination();
320
+ node.connect(state.dest);
321
+ if (state.pendingRemoteStream) attachAssistant(state, state.pendingRemoteStream);
322
+ gate.track = state.dest.stream.getAudioTracks()[0] || rawTrack;
323
+ gate.usingWorklet = true;
324
+ return gate;
325
+ }).catch(function(err) {
326
+ try {
327
+ URL.revokeObjectURL(blobUrl);
328
+ } catch (e) {}
329
+ if (window.console && console.warn) console.warn("The realtime microphone gate worklet could not be loaded; using the simpler gate.", err);
330
+ gate.track = buildFallbackGate(state, rawStream, rawTrack, mode);
331
+ return gate;
332
+ });
333
+ }
334
+ function attachAssistant(state, remoteStream) {
335
+ state.pendingRemoteStream = remoteStream || null;
336
+ if (!state.ctx || !remoteStream || state.stopped) return;
337
+ try {
338
+ if (state.remoteSource) try {
339
+ state.remoteSource.disconnect();
340
+ } catch (e) {}
341
+ var src = state.ctx.createMediaStreamSource(remoteStream);
342
+ state.remoteSource = src;
343
+ if (state.node) {
344
+ src.connect(state.node, 0, 1);
345
+ return;
346
+ }
347
+ if (state.fallback) {
348
+ var analyser = state.ctx.createAnalyser();
349
+ analyser.fftSize = 2048;
350
+ src.connect(analyser);
351
+ state.fallback.remoteAnalyser = analyser;
352
+ state.fallback.remoteData = new Float32Array(analyser.fftSize);
353
+ }
354
+ } catch (e) {}
355
+ }
356
+ function buildFallbackGate(state, rawStream, rawTrack, mode) {
357
+ try {
358
+ var ctx = state.ctx;
359
+ var source = ctx.createMediaStreamSource(rawStream);
360
+ state.micSource = source;
361
+ var analyser = ctx.createAnalyser();
362
+ analyser.fftSize = 2048;
363
+ source.connect(analyser);
364
+ var gain = ctx.createGain();
365
+ gain.gain.value = 0;
366
+ source.connect(gain);
367
+ var dest = ctx.createMediaStreamDestination();
368
+ state.dest = dest;
369
+ gain.connect(dest);
370
+ var f = {
371
+ mode,
372
+ remoteAnalyser: null,
373
+ remoteData: null
374
+ };
375
+ state.fallback = f;
376
+ if (state.pendingRemoteStream) attachAssistant(state, state.pendingRemoteStream);
377
+ var data = new Float32Array(analyser.fftSize);
378
+ var gateState = createGateState();
379
+ state.timer = window.setInterval(function() {
380
+ analyser.getFloatTimeDomainData(data);
381
+ var micDb = coreAiRmsDb(data);
382
+ var assistantDb = -100;
383
+ if (f.remoteAnalyser && f.remoteData) {
384
+ f.remoteAnalyser.getFloatTimeDomainData(f.remoteData);
385
+ assistantDb = coreAiRmsDb(f.remoteData);
386
+ }
387
+ var nowMs = window.performance && performance.now ? performance.now() : Date.now();
388
+ coreAiGateDecide(gateState, {
389
+ micDb,
390
+ assistantDb,
391
+ nowMs,
392
+ mode: f.mode || {}
393
+ });
394
+ state.level = {
395
+ micDb,
396
+ floorDb: gateState.floorDb,
397
+ open: gateState.open,
398
+ assistantSpeaking: gateState.assistantSpeaking
399
+ };
400
+ var target = gateState.open ? 1 : 0;
401
+ try {
402
+ gain.gain.setTargetAtTime(target, ctx.currentTime, .015);
403
+ } catch (e) {
404
+ gain.gain.value = target;
405
+ }
406
+ }, 20);
407
+ return dest.stream.getAudioTracks()[0] || rawTrack;
408
+ } catch (err) {
409
+ return rawTrack;
410
+ }
411
+ }
412
+ function routeOutputToPreferredDevice(el, chosenDeviceId) {
413
+ if (!el || typeof el.setSinkId !== "function") return;
414
+ if (chosenDeviceId) {
415
+ try {
416
+ el.setSinkId(chosenDeviceId).catch(function() {});
417
+ } catch (e) {}
418
+ return;
419
+ }
420
+ if (!navigator.mediaDevices || !navigator.mediaDevices.enumerateDevices) return;
421
+ navigator.mediaDevices.enumerateDevices().then(function(devices) {
422
+ var outputs = devices.filter(function(d) {
423
+ return d.kind === "audiooutput";
424
+ });
425
+ if (!outputs.length) return;
426
+ var target = "";
427
+ for (var i = 0; i < outputs.length; i++) if (outputs[i].deviceId === "communications") {
428
+ target = "communications";
429
+ break;
430
+ }
431
+ if (!target) {
432
+ var defaultEntry = null;
433
+ for (var d = 0; d < outputs.length; d++) if (outputs[d].deviceId === "default") {
434
+ defaultEntry = outputs[d];
435
+ break;
436
+ }
437
+ if (defaultEntry && defaultEntry.groupId) {
438
+ for (var j = 0; j < outputs.length; j++) if (outputs[j].deviceId !== "default" && outputs[j].deviceId !== "communications" && outputs[j].groupId === defaultEntry.groupId) {
439
+ target = outputs[j].deviceId;
440
+ break;
441
+ }
442
+ }
443
+ }
444
+ if (target) try {
445
+ el.setSinkId(target).catch(function() {});
446
+ } catch (e) {}
447
+ }).catch(function() {});
448
+ }
449
+ function ensurePlayback(el, onBlocked) {
450
+ if (!el || typeof el.play !== "function") return;
451
+ var played = el.play();
452
+ if (played && typeof played.catch === "function") played.catch(function(err) {
453
+ if (typeof onBlocked === "function") onBlocked(err);
454
+ if (window.console && console.warn) console.warn("The browser blocked realtime audio playback. Interact with the page to enable audio.", err);
455
+ });
456
+ }
457
+ function attach(opts) {
458
+ opts = opts || {};
459
+ var connection = opts.connection;
460
+ var sel = opts.selectors || {};
461
+ var realtimeVoiceName = opts.voiceName || "";
462
+ var realtimeCapableDeployments = (opts.capableDeployments || []).map(function(n) {
463
+ return (n || "").toLowerCase();
464
+ });
465
+ var isRealtimeMode = !!opts.realtimeEnabled;
466
+ var noop = function() {};
467
+ var onActivate = opts.onActivate || noop;
468
+ var onDeactivate = opts.onDeactivate || noop;
469
+ var onSessionStateChanged = opts.onSessionStateChanged || noop;
470
+ var onEnterRealtimeMode = opts.onEnterRealtimeMode || noop;
471
+ var onUserTurnPending = opts.onUserTurnPending || noop;
472
+ var onUserTurnDropped = opts.onUserTurnDropped || noop;
473
+ var strings = opts.strings || {};
474
+ function localize(key, fallback) {
475
+ return typeof strings[key] === "string" ? strings[key] : fallback;
476
+ }
477
+ var ensureConnected = opts.ensureConnected || function() {
478
+ return Promise.resolve(true);
479
+ };
480
+ var sendStart = opts.sendStart || noop;
481
+ var getVoiceName = opts.getVoiceName || null;
482
+ var sendStartWebRtc = opts.sendStartWebRtc || null;
483
+ var webRtcEnabled = opts.webRtcEnabled === true && typeof window.RTCPeerConnection === "function" && !!sendStartWebRtc;
484
+ var webRtcIceServers = Array.isArray(opts.webRtcIceServers) && opts.webRtcIceServers.length ? opts.webRtcIceServers : null;
485
+ var DEFAULT_ICE_SERVERS = [{ urls: "stun:stun.l.google.com:19302" }];
486
+ var onStateChange = opts.onStateChange || noop;
487
+ var realtimeState = "idle";
488
+ function q(name) {
489
+ var s = sel[name];
490
+ return s ? document.querySelector(s) : null;
491
+ }
492
+ var isRealtimeActive = false, realtimeLifecycleBound = false, realtimeEndedNotice = null, realtimeCompatibilityNoticeEl = null, realtimeStatusEl = null;
493
+ var realtimeStream = null, realtimeAudioCtx = null, realtimeSubject = null, realtimeProcessor = null, realtimeMicSource = null, realtimeZeroGain = null, realtimeGain = null, realtimePlayHead = 0, realtimeSources = [];
494
+ var realtimeIsWebRtc = false, realtimePc = null, realtimeRemoteAudioEl = null, realtimeWebRtcHandlersBound = false, realtimeRemoteDescriptionSet = false, realtimePendingIce = [];
495
+ var realtimeWebRtcConnected = false, realtimeWebRtcConnectTimer = null, realtimeFellBack = false;
496
+ var REALTIME_WEBRTC_CONNECT_TIMEOUT_MS = 8e3;
497
+ var REALTIME_WEBRTC_NO_RELAY_TIMEOUT_MS = 5e3;
498
+ var REALTIME_WEBRTC_JITTER_BUFFER_TARGET_MS = 150;
499
+ var REALTIME_WEBRTC_BLOCKED_KEY = "coreai.realtime.webrtcBlocked";
500
+ var realtimeAttemptIceServers = null;
501
+ var realtimeAttemptToken = 0;
502
+ var realtimeSawRelayCandidate = false, realtimeGate = null, realtimeGatePendingRemoteStream = null;
503
+ var realtimeBargeIn = true, realtimePushToTalk = false, realtimePttActive = false, realtimePttBound = false, realtimePttKeyDown = null, realtimePttKeyUp = null, realtimePttButton = null, realtimePttUiEl = null, realtimeVolume = 1, realtimeMicDeviceId = "", realtimeOutputDeviceId = "", realtimeLanguage = "", realtimeSpeechSpeed = null, realtimeSessionSpeechSpeed = 1, realtimeSpeechSpeedAdjustable = false, realtimeAudioSettingsBuilt = false, realtimeAudioSettingsElement = null;
504
+ var REALTIME_WS_ECHO_HANGOVER_SEC = .5;
505
+ var REALTIME_PREFS_VERSION = 3;
506
+ var REALTIME_SPEECH_SPEED_MIN = .75, REALTIME_SPEECH_SPEED_MAX = 1.5, REALTIME_SPEECH_SPEED_DEFAULT = 1;
507
+ function clampRealtimeSpeechSpeed(speed) {
508
+ return Math.round(Math.min(REALTIME_SPEECH_SPEED_MAX, Math.max(REALTIME_SPEECH_SPEED_MIN, speed)) * 100) / 100;
509
+ }
510
+ function loadRealtimeAudioPrefs() {
511
+ var prefs = {
512
+ bargeIn: true,
513
+ pushToTalk: false,
514
+ volume: 1,
515
+ micDeviceId: "",
516
+ outputDeviceId: "",
517
+ language: "",
518
+ speechSpeed: null,
519
+ version: 0
520
+ };
521
+ try {
522
+ var raw = window.localStorage.getItem("coreai.realtime.audioPrefs");
523
+ if (raw) {
524
+ var parsed = JSON.parse(raw);
525
+ if (typeof parsed.bargeIn === "boolean") prefs.bargeIn = parsed.bargeIn;
526
+ if (typeof parsed.pushToTalk === "boolean") prefs.pushToTalk = parsed.pushToTalk;
527
+ if (typeof parsed.volume === "number" && isFinite(parsed.volume)) prefs.volume = Math.min(1, Math.max(0, parsed.volume));
528
+ if (typeof parsed.micDeviceId === "string") prefs.micDeviceId = parsed.micDeviceId;
529
+ if (typeof parsed.outputDeviceId === "string") prefs.outputDeviceId = parsed.outputDeviceId;
530
+ if (typeof parsed.language === "string") prefs.language = parsed.language;
531
+ if (typeof parsed.speechSpeed === "number" && isFinite(parsed.speechSpeed)) prefs.speechSpeed = clampRealtimeSpeechSpeed(parsed.speechSpeed);
532
+ if (typeof parsed.version === "number") prefs.version = parsed.version;
533
+ }
534
+ } catch (err) {}
535
+ return repairRealtimeAudioPrefs(prefs);
536
+ }
537
+ function repairRealtimeAudioPrefs(prefs) {
538
+ if (prefs.version === REALTIME_PREFS_VERSION) return prefs;
539
+ prefs.version = REALTIME_PREFS_VERSION;
540
+ prefs.bargeIn = true;
541
+ prefs.pushToTalk = false;
542
+ saveRealtimeAudioPrefs(prefs);
543
+ return prefs;
544
+ }
545
+ function saveRealtimeAudioPrefs(prefs) {
546
+ try {
547
+ window.localStorage.setItem("coreai.realtime.audioPrefs", JSON.stringify(prefs));
548
+ } catch (err) {}
549
+ }
550
+ function applyRealtimeAudioPrefs(prefs) {
551
+ realtimeBargeIn = prefs.bargeIn !== false;
552
+ realtimePushToTalk = !!prefs.pushToTalk;
553
+ realtimeVolume = typeof prefs.volume === "number" ? prefs.volume : 1;
554
+ realtimeMicDeviceId = prefs.micDeviceId || "";
555
+ realtimeOutputDeviceId = prefs.outputDeviceId || "";
556
+ realtimeLanguage = prefs.language || "";
557
+ realtimeSpeechSpeed = typeof prefs.speechSpeed === "number" ? prefs.speechSpeed : null;
558
+ syncRealtimeGateMode();
559
+ if (realtimeGain) realtimeGain.gain.value = realtimeVolume;
560
+ if (realtimeRemoteAudioEl) realtimeRemoteAudioEl.volume = Math.max(0, Math.min(1, realtimeVolume));
561
+ }
562
+ function isEditableTarget(target) {
563
+ if (!target || !target.tagName) return false;
564
+ var tag = target.tagName.toLowerCase();
565
+ return tag === "input" || tag === "textarea" || tag === "select" || target.isContentEditable === true;
566
+ }
567
+ function attachRealtimePushToTalk() {
568
+ if (realtimePttBound) return;
569
+ realtimePttBound = true;
570
+ realtimePttActive = false;
571
+ realtimePttKeyDown = function(e) {
572
+ if (realtimePushToTalk && !isEditableTarget(e.target) && (e.code === "Space" || e.key === " ")) {
573
+ e.preventDefault();
574
+ if (!e.repeat) setRealtimePttActive(true);
575
+ }
576
+ };
577
+ realtimePttKeyUp = function(e) {
578
+ if (realtimePushToTalk && !isEditableTarget(e.target) && (e.code === "Space" || e.key === " ")) {
579
+ e.preventDefault();
580
+ setRealtimePttActive(false);
581
+ }
582
+ };
583
+ document.addEventListener("keydown", realtimePttKeyDown, true);
584
+ document.addEventListener("keyup", realtimePttKeyUp, true);
585
+ }
586
+ function detachRealtimePushToTalk() {
587
+ if (!realtimePttBound) return;
588
+ realtimePttBound = false;
589
+ realtimePttActive = false;
590
+ try {
591
+ document.removeEventListener("keydown", realtimePttKeyDown, true);
592
+ } catch (err) {}
593
+ try {
594
+ document.removeEventListener("keyup", realtimePttKeyUp, true);
595
+ } catch (err) {}
596
+ }
597
+ function setRealtimePttActive(active) {
598
+ realtimePttActive = active;
599
+ syncRealtimeGateMode();
600
+ if (realtimePttButton) {
601
+ realtimePttButton.classList.toggle("btn-danger", active);
602
+ realtimePttButton.classList.toggle("btn-outline-primary", !active);
603
+ realtimePttButton.innerHTML = active ? "<i class=\"fa-solid fa-microphone me-1\"></i> Listening…" : "<i class=\"fa-solid fa-microphone me-1\"></i> Hold to talk";
604
+ }
605
+ }
606
+ function buildRealtimePttUi() {
607
+ var realtimeBtn = q("realtimeButton");
608
+ if (!realtimeBtn || realtimePttUiEl) return;
609
+ var coarse = window.matchMedia && window.matchMedia("(pointer: coarse)").matches;
610
+ var bar = document.createElement("div");
611
+ bar.className = "text-center mt-2";
612
+ var btn = document.createElement("button");
613
+ btn.type = "button";
614
+ btn.className = "btn btn-outline-primary btn-sm";
615
+ btn.innerHTML = "<i class=\"fa-solid fa-microphone me-1\"></i> Hold to talk";
616
+ var hint = document.createElement("div");
617
+ hint.className = "form-text mt-1 mb-0";
618
+ hint.innerHTML = coarse ? "Push-to-talk is on — press and hold the button to speak." : "Push-to-talk is on — hold <kbd>Space</kbd> or this button to speak.";
619
+ bar.appendChild(btn);
620
+ bar.appendChild(hint);
621
+ var row = realtimeBtn.parentElement;
622
+ if (row && row.parentElement) row.parentElement.appendChild(bar);
623
+ else realtimeBtn.insertAdjacentElement("afterend", bar);
624
+ realtimePttButton = btn;
625
+ realtimePttUiEl = bar;
626
+ btn.addEventListener("pointerdown", function(e) {
627
+ e.preventDefault();
628
+ setRealtimePttActive(true);
629
+ });
630
+ btn.addEventListener("pointerup", function() {
631
+ setRealtimePttActive(false);
632
+ });
633
+ btn.addEventListener("pointerleave", function() {
634
+ setRealtimePttActive(false);
635
+ });
636
+ btn.addEventListener("pointercancel", function() {
637
+ setRealtimePttActive(false);
638
+ });
639
+ setRealtimePttActive(false);
640
+ }
641
+ function removeRealtimePttUi() {
642
+ realtimePttButton = null;
643
+ if (realtimePttUiEl) {
644
+ try {
645
+ realtimePttUiEl.remove();
646
+ } catch (err) {}
647
+ realtimePttUiEl = null;
648
+ }
649
+ }
650
+ function setupRealtimeAudioSettings() {
651
+ var realtimeBtn = q("realtimeButton");
652
+ if (!realtimeBtn || realtimeAudioSettingsBuilt) return;
653
+ realtimeAudioSettingsBuilt = true;
654
+ var prefs = loadRealtimeAudioPrefs();
655
+ applyRealtimeAudioPrefs(prefs);
656
+ var wrap = document.createElement("span");
657
+ wrap.id = "realtime-audio-settings";
658
+ wrap.style.position = "relative";
659
+ wrap.style.display = "inline-flex";
660
+ var gear = document.createElement("button");
661
+ gear.type = "button";
662
+ gear.className = "btn btn-outline-secondary";
663
+ gear.title = localize("settingsTitle", "Voice settings");
664
+ gear.setAttribute("aria-label", localize("settingsTitle", "Voice settings"));
665
+ gear.innerHTML = "<i class=\"fa-solid fa-headset\"></i>";
666
+ var langOptions = "<option value=\"\"" + (prefs.language === "" ? " selected" : "") + ">" + localize("languageAuto", "Automatic") + "</option>" + [
667
+ ["en", "English"],
668
+ ["es", "Spanish"],
669
+ ["fr", "French"],
670
+ ["de", "German"],
671
+ ["it", "Italian"],
672
+ ["pt", "Portuguese"],
673
+ ["nl", "Dutch"],
674
+ ["zh", "Chinese"],
675
+ ["ja", "Japanese"],
676
+ ["ko", "Korean"],
677
+ ["ar", "Arabic"],
678
+ ["hi", "Hindi"],
679
+ ["ru", "Russian"]
680
+ ].map(function(l) {
681
+ return "<option value=\"" + l[0] + "\"" + (prefs.language === l[0] ? " selected" : "") + ">" + l[1] + "</option>";
682
+ }).join("");
683
+ var panel = document.createElement("div");
684
+ panel.className = "card shadow-sm";
685
+ panel.style.cssText = "position:absolute;bottom:calc(100% + 6px);right:0;z-index:1080;width:290px;max-height:70vh;overflow:auto;display:none;";
686
+ panel.innerHTML = "<div class=\"card-body p-3\"><div class=\"fw-semibold mb-2\" style=\"font-size:0.85rem;\">" + localize("settingsTitle", "Voice settings") + "</div><label class=\"form-label mb-1 d-block\" style=\"font-size:0.8rem;\">" + localize("microphone", "Microphone") + "</label><select class=\"form-select form-select-sm js-mic mb-2\"><option value=\"\">" + localize("defaultMicrophone", "Default microphone") + "</option></select><label class=\"form-label mb-1 d-block\" style=\"font-size:0.8rem;\">" + localize("speaker", "Speaker") + "</label><select class=\"form-select form-select-sm js-out mb-1\"><option value=\"\">" + localize("automatic", "Automatic") + "</option></select><button type=\"button\" class=\"btn btn-outline-secondary btn-sm mb-2 js-pick-out\" style=\"display:none;\">" + localize("chooseSpeaker", "Choose speaker…") + "</button><div class=\"form-text mb-2 js-out-help\">" + localize("speakerHelp", "Automatic uses the device your system uses for calls. If you cannot hear the assistant, pick the speakers you are actually listening to.") + "</div><label class=\"form-label mb-1 d-block\" style=\"font-size:0.8rem;\">" + localize("assistantVolume", "Assistant volume") + ": <strong class=\"js-vol-val\">" + Math.round(prefs.volume * 100) + "</strong>%</label><input type=\"range\" class=\"form-range js-vol mb-2\" min=\"0\" max=\"100\" step=\"5\" value=\"" + Math.round(prefs.volume * 100) + "\"><div class=\"js-speed-row\" hidden><label class=\"form-label mb-1 d-flex align-items-baseline\" style=\"font-size:0.8rem;\"><span>" + localize("speechSpeed", "Speaking speed") + ": <strong class=\"js-speed-val\">1.00</strong>×</span><button type=\"button\" class=\"btn btn-link btn-sm p-0 ms-auto js-speed-reset\" style=\"font-size:0.8rem;\" hidden>" + localize("speechSpeedReset", "Reset") + "</button></label><input type=\"range\" class=\"form-range js-speed mb-0\" min=\"" + REALTIME_SPEECH_SPEED_MIN + "\" max=\"" + REALTIME_SPEECH_SPEED_MAX + "\" step=\"0.05\" value=\"" + REALTIME_SPEECH_SPEED_DEFAULT + "\"><div class=\"form-text mt-0 mb-2\">" + localize("speechSpeedHelp", "Applies from the assistant's next reply.") + "</div></div><label class=\"form-label mb-1 d-block\" style=\"font-size:0.8rem;\">" + localize("language", "Language") + "</label><select class=\"form-select form-select-sm js-lang mb-2\">" + langOptions + "</select><div class=\"form-check form-switch mb-1\"><input class=\"form-check-input js-barge\" type=\"checkbox\" id=\"realtime-setting-barge\"" + (prefs.bargeIn ? " checked" : "") + "><label class=\"form-check-label\" for=\"realtime-setting-barge\">" + localize("allowInterruptions", "Allow interruptions") + "</label></div><div class=\"form-text mb-2\">" + localize("allowInterruptionsHelp", "Talk over the assistant to interrupt it. Turn this off if the assistant keeps hearing itself through your speakers.") + "</div><div class=\"form-check form-switch mb-1\"><input class=\"form-check-input js-ptt\" type=\"checkbox\" id=\"realtime-setting-ptt\"" + (prefs.pushToTalk ? " checked" : "") + "><label class=\"form-check-label\" for=\"realtime-setting-ptt\">" + localize("pushToTalk", "Push-to-talk") + "</label></div><div class=\"form-text\">" + localize("pushToTalkHelp", "Hold <kbd>Space</kbd> (or the button) to talk. For very noisy places.") + "</div></div>";
687
+ wrap.appendChild(gear);
688
+ wrap.appendChild(panel);
689
+ realtimeBtn.insertAdjacentElement("afterend", wrap);
690
+ realtimeAudioSettingsElement = wrap;
691
+ var bargeInput = panel.querySelector(".js-barge");
692
+ var pttInput = panel.querySelector(".js-ptt");
693
+ var volInput = panel.querySelector(".js-vol");
694
+ var volVal = panel.querySelector(".js-vol-val");
695
+ var micSelect = panel.querySelector(".js-mic");
696
+ var outSelect = panel.querySelector(".js-out");
697
+ var outPickButton = panel.querySelector(".js-pick-out");
698
+ var langSelect = panel.querySelector(".js-lang");
699
+ var speedInput = panel.querySelector(".js-speed");
700
+ var speedVal = panel.querySelector(".js-speed-val");
701
+ var speedReset = panel.querySelector(".js-speed-reset");
702
+ function readPanelPrefs() {
703
+ return {
704
+ bargeIn: bargeInput.checked,
705
+ pushToTalk: pttInput.checked,
706
+ volume: (parseInt(volInput.value, 10) || 0) / 100,
707
+ micDeviceId: micSelect.value || "",
708
+ outputDeviceId: outSelect.value || "",
709
+ language: langSelect.value || "",
710
+ speechSpeed: realtimeSpeechSpeed,
711
+ version: REALTIME_PREFS_VERSION
712
+ };
713
+ }
714
+ function persist() {
715
+ var next = readPanelPrefs();
716
+ applyRealtimeAudioPrefs(next);
717
+ saveRealtimeAudioPrefs(next);
718
+ pushRealtimeSettingsToServer();
719
+ }
720
+ function persistSpeechSpeed(speed) {
721
+ realtimeSpeechSpeed = speed;
722
+ saveRealtimeAudioPrefs(readPanelPrefs());
723
+ syncRealtimeSpeechSpeedControl();
724
+ pushRealtimeSpeechSpeedToServer();
725
+ }
726
+ function populateDevices() {
727
+ if (!navigator.mediaDevices || !navigator.mediaDevices.enumerateDevices) return;
728
+ navigator.mediaDevices.enumerateDevices().then(function(devices) {
729
+ var options = ["<option value=\"\">" + localize("defaultMicrophone", "Default microphone") + "</option>"];
730
+ devices.filter(function(d) {
731
+ return d.kind === "audioinput";
732
+ }).forEach(function(d, i) {
733
+ var label = d.label || localize("microphone", "Microphone") + " " + (i + 1);
734
+ options.push("<option value=\"" + d.deviceId + "\"" + (d.deviceId === realtimeMicDeviceId ? " selected" : "") + ">" + label.replace(/</g, "") + "</option>");
735
+ });
736
+ micSelect.innerHTML = options.join("");
737
+ var outs = devices.filter(function(d) {
738
+ return d.kind === "audiooutput" && d.deviceId && d.deviceId !== "communications" && d.deviceId !== "default";
739
+ });
740
+ if (outs.length) {
741
+ var outOptions = ["<option value=\"\">" + localize("automatic", "Automatic") + "</option>"];
742
+ outs.forEach(function(d, i) {
743
+ var label = d.label || localize("speaker", "Speaker") + " " + (i + 1);
744
+ outOptions.push("<option value=\"" + d.deviceId + "\"" + (d.deviceId === realtimeOutputDeviceId ? " selected" : "") + ">" + label.replace(/</g, "") + "</option>");
745
+ });
746
+ outSelect.innerHTML = outOptions.join("");
747
+ }
748
+ }).catch(function() {});
749
+ }
750
+ populateDevices();
751
+ if (navigator.mediaDevices && typeof navigator.mediaDevices.addEventListener === "function") navigator.mediaDevices.addEventListener("devicechange", function() {
752
+ populateDevices();
753
+ if (!isRealtimeActive || !realtimeMicDeviceId) return;
754
+ navigator.mediaDevices.enumerateDevices().then(function(devices) {
755
+ if (!devices.some(function(d) {
756
+ return d.kind === "audioinput" && d.deviceId === realtimeMicDeviceId;
757
+ })) endRealtimeSessionFromServer("device_lost");
758
+ }).catch(function() {});
759
+ });
760
+ function constrainPanel() {
761
+ panel.style.right = "0px";
762
+ panel.style.left = "auto";
763
+ var panelRect = panel.getBoundingClientRect();
764
+ if (panelRect.left < 8) panel.style.right = panelRect.left - 8 + "px";
765
+ var rect = gear.getBoundingClientRect();
766
+ var boundaryTop = 8;
767
+ var node = panel.parentElement ? panel.parentElement.parentElement : null;
768
+ while (node && node !== document.body && node !== document.documentElement) {
769
+ var s = window.getComputedStyle(node);
770
+ if (s.overflowY === "auto" || s.overflowY === "scroll" || s.overflowY === "hidden" || s.position === "fixed") {
771
+ boundaryTop = Math.max(boundaryTop, node.getBoundingClientRect().top);
772
+ break;
773
+ }
774
+ node = node.parentElement;
775
+ }
776
+ panel.style.maxHeight = Math.max(160, rect.top - boundaryTop - 10) + "px";
777
+ }
778
+ gear.addEventListener("click", function(e) {
779
+ e.stopPropagation();
780
+ var showing = panel.style.display === "none";
781
+ if (showing) {
782
+ panel.style.display = "block";
783
+ constrainPanel();
784
+ populateDevices();
785
+ }
786
+ panel.style.display = showing ? "block" : "none";
787
+ });
788
+ panel.addEventListener("click", function(e) {
789
+ e.stopPropagation();
790
+ });
791
+ document.addEventListener("click", function() {
792
+ panel.style.display = "none";
793
+ });
794
+ bargeInput.addEventListener("change", persist);
795
+ pttInput.addEventListener("change", persist);
796
+ volInput.addEventListener("input", function() {
797
+ volVal.textContent = volInput.value;
798
+ persist();
799
+ });
800
+ speedInput.addEventListener("input", function() {
801
+ speedVal.textContent = formatRealtimeSpeechSpeed(parseFloat(speedInput.value));
802
+ });
803
+ speedInput.addEventListener("change", function() {
804
+ persistSpeechSpeed(clampRealtimeSpeechSpeed(parseFloat(speedInput.value)));
805
+ });
806
+ speedReset.addEventListener("click", function() {
807
+ persistSpeechSpeed(null);
808
+ });
809
+ micSelect.addEventListener("change", persist);
810
+ langSelect.addEventListener("change", persist);
811
+ outSelect.addEventListener("change", function() {
812
+ persist();
813
+ applyRealtimeOutputDevice();
814
+ });
815
+ if (navigator.mediaDevices && typeof navigator.mediaDevices.selectAudioOutput === "function") {
816
+ outPickButton.style.display = "";
817
+ outPickButton.addEventListener("click", function() {
818
+ navigator.mediaDevices.selectAudioOutput().then(function(device) {
819
+ if (!device || !device.deviceId) return;
820
+ var option = document.createElement("option");
821
+ option.value = device.deviceId;
822
+ option.textContent = device.label || localize("selectedSpeaker", "Selected speaker");
823
+ outSelect.appendChild(option);
824
+ outSelect.value = device.deviceId;
825
+ persist();
826
+ applyRealtimeOutputDevice();
827
+ }).catch(function() {});
828
+ });
829
+ }
830
+ }
831
+ function setRealtimeButtonHtml(button, html) {
832
+ if (!html) return;
833
+ if (window.DOMPurify && typeof window.DOMPurify.sanitize === "function") {
834
+ button.replaceChildren(window.DOMPurify.sanitize(html, { RETURN_DOM_FRAGMENT: true }));
835
+ return;
836
+ }
837
+ var text = html.replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim();
838
+ button.replaceChildren(document.createTextNode(text));
839
+ }
840
+ function updateRealtimeButton() {
841
+ onSessionStateChanged(isRealtimeActive);
842
+ var realtimeBtn = q("realtimeButton");
843
+ if (!realtimeBtn) return;
844
+ if (isRealtimeActive) {
845
+ realtimeBtn.classList.add("active", "btn-primary");
846
+ realtimeBtn.classList.remove("btn-dark", "btn-outline-dark", "btn-outline-secondary");
847
+ realtimeBtn.title = realtimeBtn.getAttribute("data-end-title") || "End Conversation";
848
+ setRealtimeButtonHtml(realtimeBtn, realtimeBtn.getAttribute("data-end-html"));
849
+ } else {
850
+ realtimeBtn.classList.remove("active", "btn-primary", "btn-dark", "btn-outline-secondary");
851
+ realtimeBtn.classList.add("btn-outline-dark");
852
+ realtimeBtn.blur();
853
+ realtimeBtn.title = realtimeBtn.getAttribute("data-start-title") || "Start speaking";
854
+ setRealtimeButtonHtml(realtimeBtn, realtimeBtn.getAttribute("data-start-html"));
855
+ }
856
+ }
857
+ function applyRealtimeMode(enable) {
858
+ isRealtimeMode = enable;
859
+ var realtimeBtn = q("realtimeButton");
860
+ var micBtn = q("micButton");
861
+ var conversationBtn = q("conversationButton");
862
+ var inputEl = q("input");
863
+ var sendBtn = q("sendButton");
864
+ var audioSettings = realtimeAudioSettingsElement;
865
+ if (enable) {
866
+ onEnterRealtimeMode();
867
+ if (inputEl) inputEl.hidden = true;
868
+ if (sendBtn) sendBtn.hidden = true;
869
+ if (micBtn) micBtn.hidden = true;
870
+ if (conversationBtn) conversationBtn.hidden = true;
871
+ if (realtimeBtn) realtimeBtn.hidden = false;
872
+ if (audioSettings) audioSettings.hidden = false;
873
+ } else {
874
+ if (isRealtimeActive) stopRealtimeConversation();
875
+ if (realtimeBtn) realtimeBtn.hidden = true;
876
+ if (audioSettings) audioSettings.hidden = true;
877
+ removeCompatibilityModeNotice();
878
+ clearRealtimeStatus();
879
+ if (inputEl) inputEl.hidden = false;
880
+ if (sendBtn) sendBtn.hidden = false;
881
+ if (micBtn) micBtn.hidden = false;
882
+ if (conversationBtn) conversationBtn.hidden = false;
883
+ }
884
+ updateRealtimeButton();
885
+ }
886
+ function pushRealtimeSettingsToServer() {
887
+ if (!isRealtimeActive || !connection || typeof connection.send !== "function") return;
888
+ try {
889
+ connection.send("UpdateRealtimeSettings", realtimeBargeIn, null, null);
890
+ } catch (err) {}
891
+ }
892
+ function formatRealtimeSpeechSpeed(speed) {
893
+ return (isFinite(speed) ? speed : REALTIME_SPEECH_SPEED_DEFAULT).toFixed(2);
894
+ }
895
+ function effectiveRealtimeSpeechSpeed() {
896
+ return realtimeSpeechSpeed !== null ? realtimeSpeechSpeed : realtimeSessionSpeechSpeed;
897
+ }
898
+ function syncRealtimeSpeechSpeedControl() {
899
+ if (!realtimeAudioSettingsElement) return;
900
+ var row = realtimeAudioSettingsElement.querySelector(".js-speed-row");
901
+ var input = realtimeAudioSettingsElement.querySelector(".js-speed");
902
+ var value = realtimeAudioSettingsElement.querySelector(".js-speed-val");
903
+ var reset = realtimeAudioSettingsElement.querySelector(".js-speed-reset");
904
+ if (!row || !input || !value || !reset) return;
905
+ var speed = effectiveRealtimeSpeechSpeed();
906
+ row.hidden = !realtimeSpeechSpeedAdjustable;
907
+ input.value = String(speed);
908
+ value.textContent = formatRealtimeSpeechSpeed(speed);
909
+ reset.hidden = realtimeSpeechSpeed === null;
910
+ }
911
+ function pushRealtimeSpeechSpeedToServer() {
912
+ if (!isRealtimeActive || !realtimeSpeechSpeedAdjustable || !connection || typeof connection.send !== "function") return;
913
+ try {
914
+ connection.send("UpdateRealtimeSpeechSpeed", effectiveRealtimeSpeechSpeed());
915
+ } catch (err) {}
916
+ }
917
+ function applySessionSpeechSpeed(payload) {
918
+ var speed = parseFloat(payload);
919
+ realtimeSpeechSpeedAdjustable = isFinite(speed);
920
+ realtimeSessionSpeechSpeed = realtimeSpeechSpeedAdjustable ? speed : REALTIME_SPEECH_SPEED_DEFAULT;
921
+ syncRealtimeSpeechSpeedControl();
922
+ if (realtimeSpeechSpeed !== null && realtimeSpeechSpeed !== realtimeSessionSpeechSpeed) pushRealtimeSpeechSpeedToServer();
923
+ }
924
+ var REALTIME_STATE_TEXT = {
925
+ "requesting-mic": localize("requestingMic", "Waiting for microphone access…"),
926
+ "connecting": localize("connecting", "Connecting…"),
927
+ "listening": localize("listening", "Listening"),
928
+ "playback-blocked": localize("playbackBlocked", "Audio is blocked by the browser — click the page to enable it.")
929
+ };
930
+ function setRealtimeState(state, detail) {
931
+ if (realtimeState === state) return;
932
+ realtimeState = state;
933
+ var text = REALTIME_STATE_TEXT[state];
934
+ if (text) showRealtimeStatus(text, false);
935
+ else if (state === "idle") clearRealtimeStatus();
936
+ try {
937
+ onStateChange({
938
+ state,
939
+ transport: realtimeIsWebRtc ? "webrtc" : "websocket",
940
+ detail: detail || null
941
+ });
942
+ } catch (err) {}
943
+ }
944
+ function bindRealtimeLifecycleHandlers() {
945
+ if (realtimeLifecycleBound || !connection) return;
946
+ realtimeLifecycleBound = true;
947
+ connection.on("ReceiveRealtimeEvent", function(identifier, type, payload) {
948
+ if (!isRealtimeActive) return;
949
+ if (type === "speech_speed") applySessionSpeechSpeed(payload);
950
+ else if (type === "session_ready") setRealtimeState("listening");
951
+ else if (type === "session_ended") endRealtimeSessionFromServer(payload || "completed");
952
+ else if (type === "speech_started") {
953
+ if (realtimeBargeIn) flushRealtimePlayback();
954
+ } else if (type === "playback_flush") flushRealtimePlayback();
955
+ else if (type === "user_turn_pending") onUserTurnPending(payload);
956
+ else if (type === "user_turn_dropped") onUserTurnDropped(payload);
957
+ });
958
+ connection.on("ReceiveError", function() {
959
+ if (!isRealtimeActive) return;
960
+ if (realtimeIsWebRtc && !realtimeWebRtcConnected && !realtimeFellBack) {
961
+ fallbackToWebSocket("server reported an error before the peer connected");
962
+ return;
963
+ }
964
+ endRealtimeSessionFromServer("error");
965
+ });
966
+ if (typeof connection.onclose === "function") connection.onclose(function() {
967
+ if (isRealtimeActive) endRealtimeSessionFromServer("disconnected");
968
+ });
969
+ if (typeof connection.onreconnecting === "function") connection.onreconnecting(function() {
970
+ if (isRealtimeActive) endRealtimeSessionFromServer("disconnected");
971
+ });
972
+ }
973
+ function endRealtimeSessionFromServer(reason) {
974
+ realtimeEndedNotice = reason;
975
+ stopRealtimeConversation();
976
+ setRealtimeState("ended", reason);
977
+ if (window.console && console.info) console.info("The realtime voice session ended (" + reason + ").");
978
+ if (reason === "idle") showRealtimeStatus(localize("endedIdle", "Voice paused after a period of silence."), true);
979
+ else if (reason === "max_duration") showRealtimeStatus(localize("endedMaxDuration", "Voice paused — this session reached its time limit. You can start another."), true);
980
+ else if (reason === "disconnected") showRealtimeStatus(localize("endedDisconnected", "Voice session ended — the connection dropped."), true);
981
+ else if (reason === "error") showRealtimeStatus(localize("endedError", "Voice session ended unexpectedly."), true);
982
+ else if (reason === "device_lost") showRealtimeStatus(localize("endedDeviceLost", "Voice session ended — the microphone was disconnected."), true);
983
+ }
984
+ function showRealtimeStatus(text, offerResume) {
985
+ var realtimeBtn = q("realtimeButton");
986
+ if (!realtimeBtn) return;
987
+ if (!realtimeStatusEl) {
988
+ realtimeStatusEl = document.createElement("div");
989
+ realtimeStatusEl.className = "form-text mt-2 mb-0 text-center";
990
+ realtimeStatusEl.setAttribute("role", "status");
991
+ realtimeStatusEl.setAttribute("aria-live", "polite");
992
+ var row = realtimeBtn.parentElement;
993
+ if (row && row.parentElement) row.parentElement.appendChild(realtimeStatusEl);
994
+ else realtimeBtn.insertAdjacentElement("afterend", realtimeStatusEl);
995
+ }
996
+ realtimeStatusEl.replaceChildren();
997
+ realtimeStatusEl.appendChild(document.createTextNode(text));
998
+ if (offerResume) {
999
+ var resume = document.createElement("button");
1000
+ resume.type = "button";
1001
+ resume.className = "btn btn-link btn-sm p-0 ms-2 align-baseline";
1002
+ resume.textContent = localize("resume", "Resume");
1003
+ resume.addEventListener("click", function() {
1004
+ clearRealtimeStatus();
1005
+ startRealtimeConversation();
1006
+ });
1007
+ realtimeStatusEl.appendChild(resume);
1008
+ }
1009
+ }
1010
+ function clearRealtimeStatus() {
1011
+ if (!realtimeStatusEl) return;
1012
+ try {
1013
+ realtimeStatusEl.remove();
1014
+ } catch (err) {}
1015
+ realtimeStatusEl = null;
1016
+ }
1017
+ function resolveIceServers() {
1018
+ if (webRtcIceServers) return Promise.resolve(webRtcIceServers);
1019
+ if (realtimeAttemptIceServers) return Promise.resolve(realtimeAttemptIceServers);
1020
+ if (!connection || typeof connection.invoke !== "function") return Promise.resolve(DEFAULT_ICE_SERVERS);
1021
+ return connection.invoke("GetRealtimeIceServers").then(function(servers) {
1022
+ realtimeAttemptIceServers = Array.isArray(servers) && servers.length ? servers : DEFAULT_ICE_SERVERS;
1023
+ return realtimeAttemptIceServers;
1024
+ }).catch(function(err) {
1025
+ if (window.console && console.warn) console.warn("Could not resolve the realtime ICE servers; using the default STUN server.", err);
1026
+ realtimeAttemptIceServers = DEFAULT_ICE_SERVERS;
1027
+ return realtimeAttemptIceServers;
1028
+ });
1029
+ }
1030
+ function iceServersFingerprint(servers) {
1031
+ var urls = [];
1032
+ try {
1033
+ (servers || []).forEach(function(server) {
1034
+ var list = server && server.urls;
1035
+ if (typeof list === "string") urls.push(list);
1036
+ else if (Array.isArray(list)) list.forEach(function(url) {
1037
+ if (url) urls.push(url);
1038
+ });
1039
+ });
1040
+ } catch (err) {
1041
+ return "none";
1042
+ }
1043
+ return urls.length ? urls.sort().join("|") : "none";
1044
+ }
1045
+ function isWebRtcKnownBlocked(servers) {
1046
+ try {
1047
+ return window.sessionStorage.getItem(REALTIME_WEBRTC_BLOCKED_KEY) === iceServersFingerprint(servers);
1048
+ } catch (err) {
1049
+ return false;
1050
+ }
1051
+ }
1052
+ function rememberWebRtcBlocked() {
1053
+ try {
1054
+ window.sessionStorage.setItem(REALTIME_WEBRTC_BLOCKED_KEY, iceServersFingerprint(realtimeAttemptIceServers || webRtcIceServers));
1055
+ } catch (err) {}
1056
+ }
1057
+ function showCompatibilityModeNotice() {
1058
+ if (realtimeCompatibilityNoticeEl) return;
1059
+ var realtimeBtn = q("realtimeButton");
1060
+ if (!realtimeBtn) return;
1061
+ var note = document.createElement("div");
1062
+ note.className = "form-text mt-2 mb-0 text-center";
1063
+ note.setAttribute("role", "status");
1064
+ note.textContent = localize("compatibilityMode", "Using compatibility audio mode. Echo cancellation may be weaker — headphones are recommended.");
1065
+ var row = realtimeBtn.parentElement;
1066
+ if (row && row.parentElement) row.parentElement.appendChild(note);
1067
+ else realtimeBtn.insertAdjacentElement("afterend", note);
1068
+ realtimeCompatibilityNoticeEl = note;
1069
+ }
1070
+ function removeCompatibilityModeNotice() {
1071
+ if (!realtimeCompatibilityNoticeEl) return;
1072
+ try {
1073
+ realtimeCompatibilityNoticeEl.remove();
1074
+ } catch (err) {}
1075
+ realtimeCompatibilityNoticeEl = null;
1076
+ }
1077
+ function startRealtimeConversation() {
1078
+ if (!isRealtimeMode || isRealtimeActive || !connection) return;
1079
+ applyRealtimeAudioPrefs(loadRealtimeAudioPrefs());
1080
+ realtimeAttemptIceServers = null;
1081
+ var attempt = ++realtimeAttemptToken;
1082
+ realtimeFellBack = false;
1083
+ realtimeSpeechSpeedAdjustable = false;
1084
+ syncRealtimeSpeechSpeedControl();
1085
+ realtimeEndedNotice = null;
1086
+ clearRealtimeStatus();
1087
+ bindRealtimeLifecycleHandlers();
1088
+ setRealtimeState("requesting-mic");
1089
+ if (!webRtcEnabled) {
1090
+ logWebSocketTransportReason("the server did not advertise the WebRTC transport");
1091
+ startRealtimeWebSocketConversation();
1092
+ return;
1093
+ }
1094
+ resolveIceServers().then(function(servers) {
1095
+ if (attempt !== realtimeAttemptToken || !isRealtimeMode || isRealtimeActive) return;
1096
+ if (isWebRtcKnownBlocked(servers)) {
1097
+ logWebSocketTransportReason("a WebRTC attempt already failed earlier in this browser session with the same ICE configuration");
1098
+ startRealtimeWebSocketConversation();
1099
+ return;
1100
+ }
1101
+ startRealtimeWebRtcConversation();
1102
+ });
1103
+ }
1104
+ function startRealtimeWebSocketConversation() {
1105
+ if (webRtcEnabled) showCompatibilityModeNotice();
1106
+ attachRealtimePushToTalk();
1107
+ var focusedBtn = q("realtimeButton");
1108
+ if (focusedBtn) try {
1109
+ focusedBtn.blur();
1110
+ } catch (err) {}
1111
+ var audioConstraints = {
1112
+ channelCount: 1,
1113
+ echoCancellation: { ideal: true },
1114
+ noiseSuppression: true,
1115
+ autoGainControl: true,
1116
+ echoCancellationType: { ideal: "system" },
1117
+ voiceIsolation: { ideal: true }
1118
+ };
1119
+ if (realtimeMicDeviceId) audioConstraints.deviceId = { exact: realtimeMicDeviceId };
1120
+ navigator.mediaDevices.getUserMedia({ audio: audioConstraints }).then(function(stream) {
1121
+ ensureConnected().then(function() {
1122
+ realtimeStream = stream;
1123
+ isRealtimeActive = true;
1124
+ onActivate();
1125
+ updateRealtimeButton();
1126
+ setRealtimeState("connecting");
1127
+ realtimeAudioCtx = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: REALTIME_SAMPLE_RATE });
1128
+ if (realtimeAudioCtx.state === "suspended" && typeof realtimeAudioCtx.resume === "function") realtimeAudioCtx.resume().catch(function(err) {
1129
+ setRealtimeState("playback-blocked");
1130
+ if (window.console && console.warn) console.warn("The browser blocked realtime audio playback. Interact with the page to enable audio.", err);
1131
+ });
1132
+ realtimePlayHead = 0;
1133
+ realtimeSources = [];
1134
+ realtimeGain = realtimeAudioCtx.createGain();
1135
+ realtimeGain.gain.value = realtimeVolume != null ? realtimeVolume : 1;
1136
+ realtimeGain.connect(realtimeAudioCtx.destination);
1137
+ realtimeSubject = new window.signalR.Subject();
1138
+ var ctxAtStart = realtimeAudioCtx;
1139
+ var zeroGain = realtimeAudioCtx.createGain();
1140
+ zeroGain.gain.value = 0;
1141
+ realtimeZeroGain = zeroGain;
1142
+ zeroGain.connect(realtimeAudioCtx.destination);
1143
+ var monitorDest = realtimeAudioCtx.createMediaStreamDestination();
1144
+ realtimeGain.connect(monitorDest);
1145
+ var sendCapturedFrame = function(input) {
1146
+ var muted;
1147
+ if (realtimePushToTalk) muted = !realtimePttActive;
1148
+ else muted = !realtimeBargeIn && realtimeAudioCtx && realtimeAudioCtx.currentTime < realtimePlayHead + REALTIME_WS_ECHO_HANGOVER_SEC;
1149
+ var pcm = new Int16Array(input.length);
1150
+ if (!muted) for (var i = 0; i < input.length; i++) {
1151
+ var s = Math.max(-1, Math.min(1, input[i]));
1152
+ pcm[i] = s < 0 ? s * 32768 : s * 32767;
1153
+ }
1154
+ var bytes = new Uint8Array(pcm.buffer);
1155
+ var binary = "";
1156
+ for (var b = 0; b < bytes.length; b++) binary += String.fromCharCode(bytes[b]);
1157
+ try {
1158
+ realtimeSubject.next(btoa(binary));
1159
+ } catch (err) {}
1160
+ };
1161
+ createRealtimeCaptureNode(realtimeAudioCtx, sendCapturedFrame).then(function(captureNode) {
1162
+ if (!isRealtimeActive || realtimeAudioCtx !== ctxAtStart) {
1163
+ try {
1164
+ captureNode.disconnect();
1165
+ } catch (err) {}
1166
+ return;
1167
+ }
1168
+ realtimeProcessor = captureNode;
1169
+ captureNode.connect(zeroGain);
1170
+ setupMicGate(stream).then(function(micTrack) {
1171
+ if (!isRealtimeActive || realtimeAudioCtx !== ctxAtStart || !micTrack) return;
1172
+ var gatedStream = new MediaStream([micTrack]);
1173
+ var source = realtimeAudioCtx.createMediaStreamSource(gatedStream);
1174
+ realtimeMicSource = source;
1175
+ source.connect(captureNode);
1176
+ if (realtimeGate) realtimeGate.attachAssistantStream(monitorDest.stream);
1177
+ });
1178
+ });
1179
+ var language = realtimeLanguage || null;
1180
+ var voice = getVoiceName && getVoiceName() || realtimeVoiceName || "";
1181
+ sendStart(realtimeSubject, voice, language, null, null, realtimeBargeIn);
1182
+ if (realtimePushToTalk) buildRealtimePttUi();
1183
+ }).catch(function(err) {
1184
+ stream.getTracks().forEach(function(track) {
1185
+ track.stop();
1186
+ });
1187
+ isRealtimeActive = false;
1188
+ onDeactivate();
1189
+ updateRealtimeButton();
1190
+ console.error("The realtime conversation could not start because the chat connection is not available.", err);
1191
+ });
1192
+ }).catch(function(err) {
1193
+ console.error("Microphone access denied:", err);
1194
+ isRealtimeActive = false;
1195
+ onDeactivate();
1196
+ updateRealtimeButton();
1197
+ });
1198
+ }
1199
+ function startRealtimeWebRtcConversation() {
1200
+ var micConstraints = {
1201
+ echoCancellation: true,
1202
+ noiseSuppression: true,
1203
+ autoGainControl: true
1204
+ };
1205
+ if (realtimeMicDeviceId) micConstraints.deviceId = { exact: realtimeMicDeviceId };
1206
+ navigator.mediaDevices.getUserMedia({ audio: micConstraints }).then(function(stream) {
1207
+ ensureConnected().then(function() {
1208
+ return resolveIceServers();
1209
+ }).then(function(iceServers) {
1210
+ realtimeStream = stream;
1211
+ isRealtimeActive = true;
1212
+ realtimeIsWebRtc = true;
1213
+ onActivate();
1214
+ updateRealtimeButton();
1215
+ setRealtimeState("connecting");
1216
+ realtimeRemoteDescriptionSet = false;
1217
+ realtimePendingIce = [];
1218
+ realtimeWebRtcConnected = false;
1219
+ var pc = new RTCPeerConnection({ iceServers });
1220
+ realtimePc = pc;
1221
+ attachRealtimePushToTalk();
1222
+ var focusedBtn = q("realtimeButton");
1223
+ if (focusedBtn) try {
1224
+ focusedBtn.blur();
1225
+ } catch (err) {}
1226
+ if (realtimePushToTalk) buildRealtimePttUi();
1227
+ var audioEl = document.createElement("audio");
1228
+ audioEl.autoplay = true;
1229
+ audioEl.style.display = "none";
1230
+ audioEl.volume = Math.max(0, Math.min(1, realtimeVolume != null ? realtimeVolume : 1));
1231
+ document.body.appendChild(audioEl);
1232
+ realtimeRemoteAudioEl = audioEl;
1233
+ routeRealtimeOutputToDefaultDevice(audioEl);
1234
+ pc.ontrack = function(e) {
1235
+ try {
1236
+ if (e.receiver && "jitterBufferTarget" in e.receiver) e.receiver.jitterBufferTarget = REALTIME_WEBRTC_JITTER_BUFFER_TARGET_MS;
1237
+ } catch (err) {}
1238
+ if (e.streams && e.streams[0]) {
1239
+ audioEl.srcObject = e.streams[0];
1240
+ ensureRealtimePlayback(audioEl);
1241
+ attachWebRtcAssistantAnalyser(e.streams[0]);
1242
+ }
1243
+ };
1244
+ realtimeSawRelayCandidate = false;
1245
+ pc.onicecandidate = function(e) {
1246
+ if (e.candidate) {
1247
+ if (e.candidate.candidate && e.candidate.candidate.indexOf(" relay ") !== -1) realtimeSawRelayCandidate = true;
1248
+ try {
1249
+ connection.send("AddRealtimeIceCandidate", e.candidate.candidate, e.candidate.sdpMid || "", e.candidate.sdpMLineIndex || 0);
1250
+ } catch (err) {}
1251
+ return;
1252
+ }
1253
+ if (!realtimeSawRelayCandidate && !realtimeWebRtcConnected && realtimeWebRtcConnectTimer) {
1254
+ clearTimeout(realtimeWebRtcConnectTimer);
1255
+ realtimeWebRtcConnectTimer = setTimeout(function() {
1256
+ if (!realtimeWebRtcConnected && isRealtimeActive) fallbackToWebSocket("no relay candidate and no direct connection");
1257
+ }, REALTIME_WEBRTC_NO_RELAY_TIMEOUT_MS);
1258
+ }
1259
+ };
1260
+ pc.onconnectionstatechange = function() {
1261
+ if (pc.connectionState === "connected") markWebRtcConnected();
1262
+ else if (pc.connectionState === "failed") {
1263
+ if (!realtimeWebRtcConnected) fallbackToWebSocket("connection failed");
1264
+ else if (isRealtimeActive) stopRealtimeConversation();
1265
+ } else if (pc.connectionState === "closed" && isRealtimeActive && realtimeWebRtcConnected) stopRealtimeConversation();
1266
+ };
1267
+ pc.oniceconnectionstatechange = function() {
1268
+ var s = pc.iceConnectionState;
1269
+ if (s === "connected" || s === "completed") markWebRtcConnected();
1270
+ else if (s === "failed" && !realtimeWebRtcConnected) fallbackToWebSocket("ICE failed");
1271
+ };
1272
+ bindWebRtcSignalingHandlers();
1273
+ realtimeWebRtcConnectTimer = setTimeout(function() {
1274
+ if (!realtimeWebRtcConnected && isRealtimeActive) fallbackToWebSocket("connection timed out");
1275
+ }, REALTIME_WEBRTC_CONNECT_TIMEOUT_MS);
1276
+ setupMicGate(stream).then(function(micTrackToSend) {
1277
+ if (realtimePc !== pc) return null;
1278
+ if (micTrackToSend) pc.addTrack(micTrackToSend, stream);
1279
+ return pc.createOffer();
1280
+ }).then(function(offer) {
1281
+ if (!offer || realtimePc !== pc) return null;
1282
+ return pc.setLocalDescription(offer).then(function() {
1283
+ return offer;
1284
+ });
1285
+ }).then(function(offer) {
1286
+ if (!offer) return;
1287
+ var language = realtimeLanguage || null;
1288
+ var voice = getVoiceName && getVoiceName() || realtimeVoiceName || "";
1289
+ sendStartWebRtc(offer.sdp, voice, language, null, null, realtimeBargeIn);
1290
+ }).catch(function(err) {
1291
+ console.error("Failed to create the WebRTC offer; falling back to WebSocket.", err);
1292
+ fallbackToWebSocket("offer failed");
1293
+ });
1294
+ }).catch(function(err) {
1295
+ stream.getTracks().forEach(function(t) {
1296
+ t.stop();
1297
+ });
1298
+ isRealtimeActive = false;
1299
+ realtimeIsWebRtc = false;
1300
+ onDeactivate();
1301
+ updateRealtimeButton();
1302
+ console.error("The realtime WebRTC conversation could not start because the chat connection is not available.", err);
1303
+ });
1304
+ }).catch(function(err) {
1305
+ console.error("Microphone access denied:", err);
1306
+ isRealtimeActive = false;
1307
+ realtimeIsWebRtc = false;
1308
+ onDeactivate();
1309
+ updateRealtimeButton();
1310
+ });
1311
+ }
1312
+ function bindWebRtcSignalingHandlers() {
1313
+ if (realtimeWebRtcHandlersBound) return;
1314
+ realtimeWebRtcHandlersBound = true;
1315
+ connection.on("ReceiveRealtimeAnswer", function(sdp) {
1316
+ if (!realtimePc) return;
1317
+ realtimePc.setRemoteDescription({
1318
+ type: "answer",
1319
+ sdp
1320
+ }).then(function() {
1321
+ realtimeRemoteDescriptionSet = true;
1322
+ var pending = realtimePendingIce;
1323
+ realtimePendingIce = [];
1324
+ pending.forEach(function(init) {
1325
+ try {
1326
+ realtimePc.addIceCandidate(init).catch(function() {});
1327
+ } catch (err) {}
1328
+ });
1329
+ }).catch(function(err) {
1330
+ console.error("Failed to apply the WebRTC answer.", err);
1331
+ });
1332
+ });
1333
+ connection.on("ReceiveRealtimeIceCandidate", function(candidate, sdpMid, sdpMLineIndex) {
1334
+ if (!realtimePc || !candidate) return;
1335
+ var init = {
1336
+ candidate,
1337
+ sdpMid,
1338
+ sdpMLineIndex
1339
+ };
1340
+ if (realtimeRemoteDescriptionSet) try {
1341
+ realtimePc.addIceCandidate(init).catch(function() {});
1342
+ } catch (err) {}
1343
+ else realtimePendingIce.push(init);
1344
+ });
1345
+ }
1346
+ function stopRealtimeWebRtc() {
1347
+ stopWebRtcEchoGuard();
1348
+ if (realtimePc) {
1349
+ try {
1350
+ realtimePc.close();
1351
+ } catch (err) {}
1352
+ realtimePc = null;
1353
+ }
1354
+ if (realtimeRemoteAudioEl) {
1355
+ try {
1356
+ realtimeRemoteAudioEl.srcObject = null;
1357
+ realtimeRemoteAudioEl.remove();
1358
+ } catch (err) {}
1359
+ realtimeRemoteAudioEl = null;
1360
+ }
1361
+ realtimeIsWebRtc = false;
1362
+ }
1363
+ function markWebRtcConnected() {
1364
+ if (realtimeWebRtcConnected) return;
1365
+ realtimeWebRtcConnected = true;
1366
+ if (realtimeWebRtcConnectTimer) {
1367
+ clearTimeout(realtimeWebRtcConnectTimer);
1368
+ realtimeWebRtcConnectTimer = null;
1369
+ }
1370
+ }
1371
+ var REALTIME_GATE_HANGOVER_MS = 2e3;
1372
+ function realtimeGateModeMessage() {
1373
+ return {
1374
+ pushToTalk: !!realtimePushToTalk,
1375
+ pttActive: !!realtimePttActive,
1376
+ bargeIn: !!realtimeBargeIn,
1377
+ speechHangoverMs: REALTIME_GATE_HANGOVER_MS
1378
+ };
1379
+ }
1380
+ function syncRealtimeGateMode() {
1381
+ if (realtimeGate) realtimeGate.setMode(realtimeGateModeMessage());
1382
+ }
1383
+ function setupMicGate(rawStream) {
1384
+ stopWebRtcEchoGuard();
1385
+ var rawTrack = rawStream && rawStream.getAudioTracks()[0] || null;
1386
+ return createMicGate(rawStream, realtimeGateModeMessage()).then(function(gate) {
1387
+ realtimeGate = gate;
1388
+ if (realtimeGatePendingRemoteStream) gate.attachAssistantStream(realtimeGatePendingRemoteStream);
1389
+ return gate.track || rawTrack;
1390
+ });
1391
+ }
1392
+ function attachWebRtcAssistantAnalyser(remoteStream) {
1393
+ realtimeGatePendingRemoteStream = remoteStream || null;
1394
+ if (realtimeGate && remoteStream) realtimeGate.attachAssistantStream(remoteStream);
1395
+ }
1396
+ function stopWebRtcEchoGuard() {
1397
+ if (realtimeGate) {
1398
+ try {
1399
+ realtimeGate.stop();
1400
+ } catch (e) {}
1401
+ realtimeGate = null;
1402
+ }
1403
+ realtimeGatePendingRemoteStream = null;
1404
+ }
1405
+ function applyRealtimeOutputDevice() {
1406
+ routeOutputToPreferredDevice(realtimeRemoteAudioEl, realtimeOutputDeviceId);
1407
+ }
1408
+ function routeRealtimeOutputToDefaultDevice(el) {
1409
+ routeOutputToPreferredDevice(el, realtimeOutputDeviceId);
1410
+ }
1411
+ function ensureRealtimePlayback(el) {
1412
+ ensurePlayback(el, function() {
1413
+ setRealtimeState("playback-blocked");
1414
+ });
1415
+ }
1416
+ function logWebSocketTransportReason(reason) {
1417
+ if (window.console && console.warn) console.warn("Realtime is using the WebSocket transport (" + reason + "); acoustic echo cancellation is weaker than on WebRTC.");
1418
+ }
1419
+ function fallbackToWebSocket(reason) {
1420
+ if (realtimeFellBack) return;
1421
+ realtimeFellBack = true;
1422
+ if (realtimeWebRtcConnectTimer) {
1423
+ clearTimeout(realtimeWebRtcConnectTimer);
1424
+ realtimeWebRtcConnectTimer = null;
1425
+ }
1426
+ if (window.console && console.warn) console.warn("Realtime WebRTC transport unavailable (" + reason + "); falling back to the WebSocket transport.");
1427
+ rememberWebRtcBlocked();
1428
+ showCompatibilityModeNotice();
1429
+ try {
1430
+ if (realtimeStream) realtimeStream.getTracks().forEach(function(t) {
1431
+ t.stop();
1432
+ });
1433
+ } catch (err) {}
1434
+ realtimeStream = null;
1435
+ stopRealtimeWebRtc();
1436
+ realtimeIsWebRtc = false;
1437
+ isRealtimeActive = false;
1438
+ startRealtimeWebSocketConversation();
1439
+ }
1440
+ function stopRealtimeConversation() {
1441
+ if (!isRealtimeActive) return;
1442
+ if (realtimeWebRtcConnectTimer) {
1443
+ clearTimeout(realtimeWebRtcConnectTimer);
1444
+ realtimeWebRtcConnectTimer = null;
1445
+ }
1446
+ isRealtimeActive = false;
1447
+ onDeactivate();
1448
+ updateRealtimeButton();
1449
+ detachRealtimePushToTalk();
1450
+ removeRealtimePttUi();
1451
+ if (!realtimeEndedNotice) setRealtimeState("idle");
1452
+ if (realtimeIsWebRtc) {
1453
+ stopRealtimeWebRtc();
1454
+ try {
1455
+ if (realtimeStream) realtimeStream.getTracks().forEach(function(t) {
1456
+ t.stop();
1457
+ });
1458
+ } catch (err) {}
1459
+ realtimeStream = null;
1460
+ return;
1461
+ }
1462
+ try {
1463
+ if (realtimeSubject) realtimeSubject.complete();
1464
+ } catch (err) {}
1465
+ realtimeSubject = null;
1466
+ stopWebRtcEchoGuard();
1467
+ try {
1468
+ if (realtimeProcessor) {
1469
+ realtimeProcessor.disconnect();
1470
+ realtimeProcessor.onaudioprocess = null;
1471
+ if (realtimeProcessor.port) {
1472
+ realtimeProcessor.port.onmessage = null;
1473
+ realtimeProcessor.port.close();
1474
+ }
1475
+ }
1476
+ } catch (err) {}
1477
+ try {
1478
+ if (realtimeMicSource) realtimeMicSource.disconnect();
1479
+ } catch (err) {}
1480
+ try {
1481
+ if (realtimeZeroGain) realtimeZeroGain.disconnect();
1482
+ } catch (err) {}
1483
+ try {
1484
+ if (realtimeGain) realtimeGain.disconnect();
1485
+ } catch (err) {}
1486
+ try {
1487
+ if (realtimeStream) realtimeStream.getTracks().forEach(function(t) {
1488
+ t.stop();
1489
+ });
1490
+ } catch (err) {}
1491
+ realtimeProcessor = null;
1492
+ realtimeMicSource = null;
1493
+ realtimeZeroGain = null;
1494
+ realtimeGain = null;
1495
+ realtimeStream = null;
1496
+ flushRealtimePlayback();
1497
+ try {
1498
+ if (realtimeAudioCtx) realtimeAudioCtx.close();
1499
+ } catch (err) {}
1500
+ realtimeAudioCtx = null;
1501
+ }
1502
+ function toggleRealtime() {
1503
+ if (isRealtimeActive) {
1504
+ stopRealtimeConversation();
1505
+ return;
1506
+ }
1507
+ startRealtimeConversation();
1508
+ }
1509
+ function getTransportStats() {
1510
+ if (!realtimePc || typeof realtimePc.getStats !== "function") return Promise.resolve(null);
1511
+ return realtimePc.getStats().then(function(report) {
1512
+ var result = {
1513
+ transport: "webrtc",
1514
+ inbound: null,
1515
+ codec: null,
1516
+ sinkId: realtimeRemoteAudioEl && typeof realtimeRemoteAudioEl.sinkId === "string" ? realtimeRemoteAudioEl.sinkId : null,
1517
+ jitterBufferTargetMs: (function() {
1518
+ try {
1519
+ var r = realtimePc.getReceivers().filter(function(x) {
1520
+ return x.track && x.track.kind === "audio";
1521
+ })[0];
1522
+ return r && "jitterBufferTarget" in r ? r.jitterBufferTarget : null;
1523
+ } catch (e) {
1524
+ return null;
1525
+ }
1526
+ })()
1527
+ };
1528
+ report.forEach(function(stat) {
1529
+ if (stat.type === "inbound-rtp" && stat.kind === "audio") {
1530
+ result.inbound = {
1531
+ packetsReceived: stat.packetsReceived,
1532
+ packetsLost: stat.packetsLost,
1533
+ packetsDiscarded: stat.packetsDiscarded,
1534
+ jitterMs: typeof stat.jitter === "number" ? Math.round(stat.jitter * 1e3) : null,
1535
+ jitterBufferDelayMs: typeof stat.jitterBufferDelay === "number" && stat.jitterBufferEmittedCount ? Math.round(stat.jitterBufferDelay / stat.jitterBufferEmittedCount * 1e3) : null,
1536
+ totalSamplesReceived: stat.totalSamplesReceived,
1537
+ concealedSamples: stat.concealedSamples,
1538
+ concealmentEvents: stat.concealmentEvents,
1539
+ silentConcealedSamples: stat.silentConcealedSamples,
1540
+ insertedSamplesForDeceleration: stat.insertedSamplesForDeceleration,
1541
+ removedSamplesForAcceleration: stat.removedSamplesForAcceleration,
1542
+ audioLevel: stat.audioLevel
1543
+ };
1544
+ if (stat.codecId) {
1545
+ var codec = report.get(stat.codecId);
1546
+ if (codec) result.codec = {
1547
+ mimeType: codec.mimeType,
1548
+ clockRate: codec.clockRate,
1549
+ channels: codec.channels,
1550
+ sdpFmtpLine: codec.sdpFmtpLine
1551
+ };
1552
+ }
1553
+ }
1554
+ });
1555
+ return result;
1556
+ }).catch(function() {
1557
+ return null;
1558
+ });
1559
+ }
1560
+ function playRealtimePcm(bytes) {
1561
+ if (!realtimeAudioCtx || !bytes || bytes.length < 2) return;
1562
+ var ctx = realtimeAudioCtx;
1563
+ var sampleCount = Math.floor(bytes.length / 2);
1564
+ var pcm = new Int16Array(bytes.buffer, bytes.byteOffset, sampleCount);
1565
+ var f32 = new Float32Array(sampleCount);
1566
+ for (var i = 0; i < sampleCount; i++) f32[i] = pcm[i] / 32768;
1567
+ var buffer = ctx.createBuffer(1, sampleCount, ctx.sampleRate);
1568
+ buffer.copyToChannel(f32, 0);
1569
+ var src = ctx.createBufferSource();
1570
+ src.buffer = buffer;
1571
+ src.connect(realtimeGain || ctx.destination);
1572
+ var now = ctx.currentTime;
1573
+ if (realtimePlayHead < now) realtimePlayHead = now;
1574
+ src.start(realtimePlayHead);
1575
+ realtimePlayHead += buffer.duration;
1576
+ realtimeSources.push(src);
1577
+ src.onended = function() {
1578
+ realtimeSources = realtimeSources.filter(function(s) {
1579
+ return s !== src;
1580
+ });
1581
+ };
1582
+ }
1583
+ function flushRealtimePlayback() {
1584
+ realtimeSources.forEach(function(s) {
1585
+ try {
1586
+ s.stop();
1587
+ } catch (err) {}
1588
+ });
1589
+ realtimeSources = [];
1590
+ if (realtimeAudioCtx) realtimePlayHead = realtimeAudioCtx.currentTime;
1591
+ }
1592
+ function isRealtimeDeployment(deploymentName) {
1593
+ if (!deploymentName) return false;
1594
+ return realtimeCapableDeployments.indexOf(deploymentName.toLowerCase()) !== -1;
1595
+ }
1596
+ var realtimeBtn = q("realtimeButton");
1597
+ if (realtimeBtn) {
1598
+ realtimeBtn.addEventListener("click", function(e) {
1599
+ if (e && e.detail === 0 && isRealtimeActive) return;
1600
+ toggleRealtime();
1601
+ });
1602
+ setupRealtimeAudioSettings();
1603
+ }
1604
+ var deploymentSelect = q("deploymentSelect");
1605
+ if (deploymentSelect) deploymentSelect.addEventListener("change", function() {
1606
+ applyRealtimeMode(isRealtimeDeployment(deploymentSelect.value));
1607
+ });
1608
+ applyRealtimeMode(isRealtimeMode || deploymentSelect && isRealtimeDeployment(deploymentSelect.value));
1609
+ var controller = {
1610
+ toggle: toggleRealtime,
1611
+ start: startRealtimeConversation,
1612
+ stop: stopRealtimeConversation,
1613
+ applyMode: applyRealtimeMode,
1614
+ isRealtimeDeployment,
1615
+ receivePcm: playRealtimePcm,
1616
+ isActive: function() {
1617
+ return isRealtimeActive;
1618
+ },
1619
+ getState: function() {
1620
+ return realtimeState;
1621
+ },
1622
+ getGateLevel: function() {
1623
+ return realtimeGate ? realtimeGate.getLevel() : null;
1624
+ },
1625
+ getTransportStats
1626
+ };
1627
+ window.CoreAIRealtime.activeController = controller;
1628
+ return controller;
1629
+ }
1630
+ window.CoreAIRealtime = {
1631
+ attach,
1632
+ createMicGate,
1633
+ decideGate: coreAiGateDecide,
1634
+ createGateState,
1635
+ rmsDb: coreAiRmsDb,
1636
+ routeOutputToPreferredDevice,
1637
+ ensurePlayback
1638
+ };
1639
+ })(window, document);
1640
+ //#endregion
1641
+ })();
2507
1642
  //# sourceMappingURL=realtime-audio.js.map