@crestapps/ai-chat-ui 2.0.0-preview.215 → 2.0.0-preview.223
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai-chat-widget.js +762 -964
- package/dist/ai-chat-widget.js.map +1 -1
- package/dist/ai-chat-widget.min.js +1 -1
- package/dist/ai-chat-widget.min.js.map +1 -1
- package/dist/ai-chat.css +0 -2
- package/dist/ai-chat.js +2380 -3698
- package/dist/ai-chat.js.map +1 -1
- package/dist/ai-chat.min.css +1 -1
- package/dist/ai-chat.min.js +78 -1
- package/dist/ai-chat.min.js.map +1 -1
- package/dist/chat-image-carousel.css +0 -2
- package/dist/chat-image-carousel.js +306 -482
- package/dist/chat-image-carousel.js.map +1 -1
- package/dist/chat-image-carousel.min.css +1 -1
- package/dist/chat-image-carousel.min.js +1 -1
- package/dist/chat-image-carousel.min.js.map +1 -1
- package/dist/chat-interaction-settings.js +221 -293
- package/dist/chat-interaction-settings.js.map +1 -1
- package/dist/chat-interaction-settings.min.js +1 -1
- package/dist/chat-interaction-settings.min.js.map +1 -1
- package/dist/chat-interaction.js +2330 -3557
- package/dist/chat-interaction.js.map +1 -1
- package/dist/chat-interaction.min.js +62 -1
- package/dist/chat-interaction.min.js.map +1 -1
- package/dist/chat-markers.js +193 -485
- package/dist/chat-markers.js.map +1 -1
- package/dist/chat-markers.min.js +2 -1
- package/dist/chat-markers.min.js.map +1 -1
- package/dist/chat-widget.css +0 -2
- package/dist/chat-widget.min.css +1 -1
- package/dist/document-drop-zone.css +0 -2
- package/dist/document-drop-zone.js +127 -153
- package/dist/document-drop-zone.js.map +1 -1
- package/dist/document-drop-zone.min.css +1 -1
- package/dist/document-drop-zone.min.js +1 -1
- package/dist/document-drop-zone.min.js.map +1 -1
- package/dist/realtime-audio.js +1639 -2504
- package/dist/realtime-audio.js.map +1 -1
- package/dist/realtime-audio.min.js +3 -1
- package/dist/realtime-audio.min.js.map +1 -1
- package/dist/technical-name-generator.js +58 -80
- package/dist/technical-name-generator.js.map +1 -1
- package/dist/technical-name-generator.min.js +1 -1
- package/dist/technical-name-generator.min.js.map +1 -1
- package/package.json +1 -5
- package/dist/ai-chat.css.map +0 -1
- package/dist/chat-image-carousel.css.map +0 -1
- package/dist/chat-widget.css.map +0 -1
- package/dist/document-drop-zone.css.map +0 -1
package/dist/realtime-audio.js
CHANGED
|
@@ -1,2507 +1,1642 @@
|
|
|
1
1
|
/*
|
|
2
|
-
** NOTE: This file is generated by
|
|
3
|
-
**
|
|
2
|
+
** NOTE: This file is generated by the assets build (npm run build) and should not be edited directly!
|
|
3
|
+
** Edit its sources under Assets/ and rebuild; see Assets.json for the files it is built from.
|
|
4
4
|
*/
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
(
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
if (!realtimeCompatibilityNoticeEl) {
|
|
1643
|
-
return;
|
|
1644
|
-
}
|
|
1645
|
-
try {
|
|
1646
|
-
realtimeCompatibilityNoticeEl.remove();
|
|
1647
|
-
} catch (err) {}
|
|
1648
|
-
realtimeCompatibilityNoticeEl = null;
|
|
1649
|
-
}
|
|
1650
|
-
function startRealtimeConversation() {
|
|
1651
|
-
if (!isRealtimeMode || isRealtimeActive || !connection) {
|
|
1652
|
-
return;
|
|
1653
|
-
}
|
|
1654
|
-
applyRealtimeAudioPrefs(loadRealtimeAudioPrefs());
|
|
1655
|
-
realtimeAttemptIceServers = null;
|
|
1656
|
-
var attempt = ++realtimeAttemptToken;
|
|
1657
|
-
realtimeFellBack = false;
|
|
1658
|
-
realtimeSessionReady = false;
|
|
1659
|
-
// Hidden until this session reports its own speed.
|
|
1660
|
-
realtimeSpeechSpeedAdjustable = false;
|
|
1661
|
-
syncRealtimeSpeechSpeedControl();
|
|
1662
|
-
realtimeEndedNotice = null;
|
|
1663
|
-
clearRealtimeStatus();
|
|
1664
|
-
bindRealtimeLifecycleHandlers();
|
|
1665
|
-
setRealtimeState('requesting-mic');
|
|
1666
|
-
|
|
1667
|
-
// Starting on WebSocket without even attempting WebRTC is a deployment fact worth stating once. The
|
|
1668
|
-
// console is otherwise indistinguishable from a healthy WebRTC session, so "no warning" gets read as
|
|
1669
|
-
// "WebRTC is working" when it can equally mean the server never offered the transport at all.
|
|
1670
|
-
if (!webRtcEnabled) {
|
|
1671
|
-
logWebSocketTransportReason('the server did not advertise the WebRTC transport');
|
|
1672
|
-
startRealtimeWebSocketConversation();
|
|
1673
|
-
return;
|
|
1674
|
-
}
|
|
1675
|
-
|
|
1676
|
-
// Prefer the WebRTC transport when the server advertises it: the browser's echo canceller references
|
|
1677
|
-
// the assistant's media track, so the model can ignore its own voice with the mic open (open rooms).
|
|
1678
|
-
// If the peer cannot connect (blocked UDP, no TURN, unsupported), we fall back to WebSocket at connect
|
|
1679
|
-
// time — see fallbackToWebSocket. The decision is made once, before the session starts.
|
|
1680
|
-
//
|
|
1681
|
-
// Resolving the ICE servers before choosing a transport costs no extra round-trip — the peer reuses
|
|
1682
|
-
// realtimeAttemptIceServers — and it is what lets a remembered failure be scoped to the configuration
|
|
1683
|
-
// that actually caused it.
|
|
1684
|
-
resolveIceServers().then(function (servers) {
|
|
1685
|
-
if (attempt !== realtimeAttemptToken || !isRealtimeMode || isRealtimeActive) {
|
|
1686
|
-
return;
|
|
1687
|
-
}
|
|
1688
|
-
if (isWebRtcKnownBlocked(servers)) {
|
|
1689
|
-
logWebSocketTransportReason('a WebRTC attempt already failed earlier in this browser session with the same ICE configuration');
|
|
1690
|
-
startRealtimeWebSocketConversation();
|
|
1691
|
-
return;
|
|
1692
|
-
}
|
|
1693
|
-
startRealtimeWebRtcConversation();
|
|
1694
|
-
});
|
|
1695
|
-
}
|
|
1696
|
-
function startRealtimeWebSocketConversation() {
|
|
1697
|
-
if (webRtcEnabled) {
|
|
1698
|
-
showCompatibilityModeNotice();
|
|
1699
|
-
}
|
|
1700
|
-
attachRealtimePushToTalk();
|
|
1701
|
-
// Drop focus from the just-clicked button so Space acts as push-to-talk, not a re-click.
|
|
1702
|
-
var focusedBtn = q('realtimeButton');
|
|
1703
|
-
if (focusedBtn) {
|
|
1704
|
-
try {
|
|
1705
|
-
focusedBtn.blur();
|
|
1706
|
-
} catch (err) {}
|
|
1707
|
-
}
|
|
1708
|
-
var audioConstraints = {
|
|
1709
|
-
channelCount: 1,
|
|
1710
|
-
echoCancellation: {
|
|
1711
|
-
ideal: true
|
|
1712
|
-
},
|
|
1713
|
-
noiseSuppression: true,
|
|
1714
|
-
autoGainControl: true,
|
|
1715
|
-
// Ask for the strongest available acoustic echo cancellation so the model can ignore its own
|
|
1716
|
-
// voice in an open room (loud speakers + open mic). These are Chromium hints requested as
|
|
1717
|
-
// "ideal", so any browser that doesn't support them simply ignores them — never a hard failure.
|
|
1718
|
-
// echoCancellationType 'system' prefers the OS/hardware AEC over the browser's software AEC.
|
|
1719
|
-
echoCancellationType: {
|
|
1720
|
-
ideal: 'system'
|
|
1721
|
-
},
|
|
1722
|
-
voiceIsolation: {
|
|
1723
|
-
ideal: true
|
|
1724
|
-
}
|
|
1725
|
-
};
|
|
1726
|
-
if (realtimeMicDeviceId) {
|
|
1727
|
-
audioConstraints.deviceId = {
|
|
1728
|
-
exact: realtimeMicDeviceId
|
|
1729
|
-
};
|
|
1730
|
-
}
|
|
1731
|
-
navigator.mediaDevices.getUserMedia({
|
|
1732
|
-
audio: audioConstraints
|
|
1733
|
-
}).then(function (stream) {
|
|
1734
|
-
ensureConnected().then(function () {
|
|
1735
|
-
realtimeStream = stream;
|
|
1736
|
-
isRealtimeActive = true;
|
|
1737
|
-
onActivate();
|
|
1738
|
-
updateRealtimeButton();
|
|
1739
|
-
setRealtimeState('connecting');
|
|
1740
|
-
var AudioContextCtor = window.AudioContext || window.webkitAudioContext;
|
|
1741
|
-
realtimeAudioCtx = new AudioContextCtor({
|
|
1742
|
-
sampleRate: REALTIME_SAMPLE_RATE
|
|
1743
|
-
});
|
|
1744
|
-
// A context created under a strict autoplay policy starts suspended, and a suspended
|
|
1745
|
-
// context plays nothing at all — silently. Ask for it explicitly.
|
|
1746
|
-
if (realtimeAudioCtx.state === 'suspended' && typeof realtimeAudioCtx.resume === 'function') {
|
|
1747
|
-
realtimeAudioCtx.resume()["catch"](function (err) {
|
|
1748
|
-
setRealtimeState('playback-blocked');
|
|
1749
|
-
if (window.console && console.warn) {
|
|
1750
|
-
console.warn('The browser blocked realtime audio playback. Interact with the page to enable audio.', err);
|
|
1751
|
-
}
|
|
1752
|
-
});
|
|
1753
|
-
}
|
|
1754
|
-
realtimePlayHead = 0;
|
|
1755
|
-
realtimeSources = [];
|
|
1756
|
-
realtimeGain = realtimeAudioCtx.createGain();
|
|
1757
|
-
realtimeGain.gain.value = realtimeVolume != null ? realtimeVolume : 1;
|
|
1758
|
-
realtimeGain.connect(realtimeAudioCtx.destination);
|
|
1759
|
-
realtimeSubject = new window.signalR.Subject();
|
|
1760
|
-
var ctxAtStart = realtimeAudioCtx;
|
|
1761
|
-
|
|
1762
|
-
// A zero-gain node keeps the capture node alive without echoing the mic to the speakers.
|
|
1763
|
-
var zeroGain = realtimeAudioCtx.createGain();
|
|
1764
|
-
zeroGain.gain.value = 0;
|
|
1765
|
-
realtimeZeroGain = zeroGain;
|
|
1766
|
-
zeroGain.connect(realtimeAudioCtx.destination);
|
|
1767
|
-
|
|
1768
|
-
// The gate watches the assistant's playback to know when it is audible; on this
|
|
1769
|
-
// transport the assistant is a Web Audio graph, so tap the output gain into a stream.
|
|
1770
|
-
var monitorDest = realtimeAudioCtx.createMediaStreamDestination();
|
|
1771
|
-
realtimeGain.connect(monitorDest);
|
|
1772
|
-
|
|
1773
|
-
// Always send a frame (silence when muted) so the server keeps a continuous audio
|
|
1774
|
-
// stream and its voice-activity detector promptly notices the pause and responds.
|
|
1775
|
-
// Muted cases: push-to-talk not held, or the echo guard while the assistant plays back.
|
|
1776
|
-
// The decision stays on the main thread because that is where the push-to-talk and
|
|
1777
|
-
// playback state lives; the audio thread only frames the samples.
|
|
1778
|
-
var sendCapturedFrame = function sendCapturedFrame(input) {
|
|
1779
|
-
var muted;
|
|
1780
|
-
if (realtimePushToTalk) {
|
|
1781
|
-
muted = !realtimePttActive;
|
|
1782
|
-
} else {
|
|
1783
|
-
muted = !realtimeBargeIn && realtimeAudioCtx && realtimeAudioCtx.currentTime < realtimePlayHead + REALTIME_WS_ECHO_HANGOVER_SEC;
|
|
1784
|
-
}
|
|
1785
|
-
var pcm = new Int16Array(input.length);
|
|
1786
|
-
if (!muted) {
|
|
1787
|
-
for (var i = 0; i < input.length; i++) {
|
|
1788
|
-
var s = Math.max(-1, Math.min(1, input[i]));
|
|
1789
|
-
pcm[i] = s < 0 ? s * 0x8000 : s * 0x7FFF;
|
|
1790
|
-
}
|
|
1791
|
-
}
|
|
1792
|
-
var bytes = new Uint8Array(pcm.buffer);
|
|
1793
|
-
var binary = '';
|
|
1794
|
-
for (var b = 0; b < bytes.length; b++) {
|
|
1795
|
-
binary += String.fromCharCode(bytes[b]);
|
|
1796
|
-
}
|
|
1797
|
-
try {
|
|
1798
|
-
realtimeSubject.next(btoa(binary));
|
|
1799
|
-
} catch (err) {/* completed */}
|
|
1800
|
-
};
|
|
1801
|
-
|
|
1802
|
-
// The capture node loads a worklet, so it arrives a tick later than the rest of the
|
|
1803
|
-
// graph; it streams silence until the gated microphone is attached below.
|
|
1804
|
-
createRealtimeCaptureNode(realtimeAudioCtx, sendCapturedFrame).then(function (captureNode) {
|
|
1805
|
-
if (!isRealtimeActive || realtimeAudioCtx !== ctxAtStart) {
|
|
1806
|
-
try {
|
|
1807
|
-
captureNode.disconnect();
|
|
1808
|
-
} catch (err) {}
|
|
1809
|
-
return;
|
|
1810
|
-
}
|
|
1811
|
-
realtimeProcessor = captureNode;
|
|
1812
|
-
captureNode.connect(zeroGain);
|
|
1813
|
-
|
|
1814
|
-
// Send the gated microphone rather than the raw one, exactly as the WebRTC transport
|
|
1815
|
-
// does, so this fallback is not the one transport where the model hears its own echo.
|
|
1816
|
-
setupMicGate(stream).then(function (micTrack) {
|
|
1817
|
-
if (!isRealtimeActive || realtimeAudioCtx !== ctxAtStart || !micTrack) {
|
|
1818
|
-
return;
|
|
1819
|
-
}
|
|
1820
|
-
var gatedStream = new MediaStream([micTrack]);
|
|
1821
|
-
var source = realtimeAudioCtx.createMediaStreamSource(gatedStream);
|
|
1822
|
-
realtimeMicSource = source;
|
|
1823
|
-
source.connect(captureNode);
|
|
1824
|
-
if (realtimeGate) {
|
|
1825
|
-
realtimeGate.attachAssistantStream(monitorDest.stream);
|
|
1826
|
-
}
|
|
1827
|
-
});
|
|
1828
|
-
});
|
|
1829
|
-
|
|
1830
|
-
// "Auto" means auto-detect: send nothing. Sending the browser's locale instead pinned transcription
|
|
1831
|
-
// and the reply language to it, so a bilingual user with an English browser speaking Spanish
|
|
1832
|
-
// got English-forced (garbage) transcripts and an English-locked assistant.
|
|
1833
|
-
var language = realtimeLanguage || null;
|
|
1834
|
-
var voice = getVoiceName && getVoiceName() || realtimeVoiceName || '';
|
|
1835
|
-
sendStart(realtimeSubject, voice, language, null, null, realtimeBargeIn);
|
|
1836
|
-
if (realtimePushToTalk) {
|
|
1837
|
-
buildRealtimePttUi();
|
|
1838
|
-
}
|
|
1839
|
-
})["catch"](function (err) {
|
|
1840
|
-
stream.getTracks().forEach(function (track) {
|
|
1841
|
-
track.stop();
|
|
1842
|
-
});
|
|
1843
|
-
isRealtimeActive = false;
|
|
1844
|
-
onDeactivate();
|
|
1845
|
-
updateRealtimeButton();
|
|
1846
|
-
console.error('The realtime conversation could not start because the chat connection is not available.', err);
|
|
1847
|
-
});
|
|
1848
|
-
})["catch"](function (err) {
|
|
1849
|
-
console.error('Microphone access denied:', err);
|
|
1850
|
-
isRealtimeActive = false;
|
|
1851
|
-
onDeactivate();
|
|
1852
|
-
updateRealtimeButton();
|
|
1853
|
-
});
|
|
1854
|
-
}
|
|
1855
|
-
|
|
1856
|
-
// --- WebRTC (server-relay) transport ---
|
|
1857
|
-
|
|
1858
|
-
function startRealtimeWebRtcConversation() {
|
|
1859
|
-
// Keep the WebRTC capture constraints minimal. WebRTC's own echo canceller (AEC3) already references
|
|
1860
|
-
// the remote assistant track we render into the hidden <audio> element, which is the entire reason we
|
|
1861
|
-
// use WebRTC for open-room audio. The stronger hints we request on the WebSocket path
|
|
1862
|
-
// (echoCancellationType:'system' + voiceIsolation) are counterproductive here: layered on top of the
|
|
1863
|
-
// peer-connection AEC loop they gate the mic down to near-silence (~1-3% of full scale), so the model
|
|
1864
|
-
// never detects speech. Ask only for standard echo cancellation and let AEC3 do its job.
|
|
1865
|
-
var micConstraints = {
|
|
1866
|
-
// Match what ChatGPT's voice mode requests: a plain echoCancellation:true and nothing else exotic.
|
|
1867
|
-
// The browser still applies its default noise suppression and auto gain; we only pass those through
|
|
1868
|
-
// so the user's toggles can turn them off.
|
|
1869
|
-
echoCancellation: true,
|
|
1870
|
-
noiseSuppression: true,
|
|
1871
|
-
autoGainControl: true
|
|
1872
|
-
};
|
|
1873
|
-
if (realtimeMicDeviceId) {
|
|
1874
|
-
micConstraints.deviceId = {
|
|
1875
|
-
exact: realtimeMicDeviceId
|
|
1876
|
-
};
|
|
1877
|
-
}
|
|
1878
|
-
navigator.mediaDevices.getUserMedia({
|
|
1879
|
-
audio: micConstraints
|
|
1880
|
-
}).then(function (stream) {
|
|
1881
|
-
ensureConnected().then(function () {
|
|
1882
|
-
return resolveIceServers();
|
|
1883
|
-
}).then(function (iceServers) {
|
|
1884
|
-
realtimeStream = stream;
|
|
1885
|
-
isRealtimeActive = true;
|
|
1886
|
-
realtimeIsWebRtc = true;
|
|
1887
|
-
onActivate();
|
|
1888
|
-
updateRealtimeButton();
|
|
1889
|
-
setRealtimeState('connecting');
|
|
1890
|
-
realtimeRemoteDescriptionSet = false;
|
|
1891
|
-
realtimePendingIce = [];
|
|
1892
|
-
realtimeWebRtcConnected = false;
|
|
1893
|
-
var pc = new RTCPeerConnection({
|
|
1894
|
-
iceServers: iceServers
|
|
1895
|
-
});
|
|
1896
|
-
realtimePc = pc;
|
|
1897
|
-
|
|
1898
|
-
// Push-to-talk and the barge-in/echo guard both gate the outbound mic track. Wire the
|
|
1899
|
-
// same push-to-talk controls the WebSocket path uses so they work on WebRTC too.
|
|
1900
|
-
// Device labels only become readable after permission is granted, so this is the first
|
|
1901
|
-
// chance to notice a headset (safe for full duplex) or a standalone mic in front of
|
|
1902
|
-
// speakers (not). Only ever a suggestion: an explicit choice is never overridden.
|
|
1903
|
-
|
|
1904
|
-
attachRealtimePushToTalk();
|
|
1905
|
-
var focusedBtn = q('realtimeButton');
|
|
1906
|
-
if (focusedBtn) {
|
|
1907
|
-
try {
|
|
1908
|
-
focusedBtn.blur();
|
|
1909
|
-
} catch (err) {}
|
|
1910
|
-
}
|
|
1911
|
-
if (realtimePushToTalk) {
|
|
1912
|
-
buildRealtimePttUi();
|
|
1913
|
-
}
|
|
1914
|
-
|
|
1915
|
-
// Play the assistant's remote track through a hidden <audio> element. This gives the
|
|
1916
|
-
// browser's echo canceller a reference to remove the assistant's voice from the mic, and
|
|
1917
|
-
// is what the echo-guard monitor listens to for half-duplex mic muting.
|
|
1918
|
-
var audioEl = document.createElement('audio');
|
|
1919
|
-
audioEl.autoplay = true;
|
|
1920
|
-
audioEl.style.display = 'none';
|
|
1921
|
-
audioEl.volume = Math.max(0, Math.min(1, realtimeVolume != null ? realtimeVolume : 1));
|
|
1922
|
-
document.body.appendChild(audioEl);
|
|
1923
|
-
realtimeRemoteAudioEl = audioEl;
|
|
1924
|
-
// Route playback so the browser couples it with the mic's echo canceller as a proper
|
|
1925
|
-
// AEC reference (see routeRealtimeOutputToDefaultDevice for the preference order).
|
|
1926
|
-
routeRealtimeOutputToDefaultDevice(audioEl);
|
|
1927
|
-
pc.ontrack = function (e) {
|
|
1928
|
-
// Ask the browser to hold a fixed cushion of received audio before playing it,
|
|
1929
|
-
// rather than the adaptive minimum it would otherwise pick. Network jitter then
|
|
1930
|
-
// lands inside the cushion instead of making the jitter buffer stretch or speed
|
|
1931
|
-
// speech up to re-adapt; the audio itself is played as received. Ignored by
|
|
1932
|
-
// browsers without the property.
|
|
1933
|
-
try {
|
|
1934
|
-
if (e.receiver && 'jitterBufferTarget' in e.receiver) {
|
|
1935
|
-
e.receiver.jitterBufferTarget = REALTIME_WEBRTC_JITTER_BUFFER_TARGET_MS;
|
|
1936
|
-
}
|
|
1937
|
-
} catch (err) {}
|
|
1938
|
-
if (e.streams && e.streams[0]) {
|
|
1939
|
-
audioEl.srcObject = e.streams[0];
|
|
1940
|
-
ensureRealtimePlayback(audioEl);
|
|
1941
|
-
// Let the gate watch the assistant's level so it can raise the speech threshold
|
|
1942
|
-
// while the assistant talks (barge-in on) and stay closed until it finishes
|
|
1943
|
-
// (barge-in off).
|
|
1944
|
-
attachWebRtcAssistantAnalyser(e.streams[0]);
|
|
1945
|
-
}
|
|
1946
|
-
};
|
|
1947
|
-
realtimeSawRelayCandidate = false;
|
|
1948
|
-
pc.onicecandidate = function (e) {
|
|
1949
|
-
if (e.candidate) {
|
|
1950
|
-
if (e.candidate.candidate && e.candidate.candidate.indexOf(' relay ') !== -1) {
|
|
1951
|
-
realtimeSawRelayCandidate = true;
|
|
1952
|
-
}
|
|
1953
|
-
try {
|
|
1954
|
-
connection.send('AddRealtimeIceCandidate', e.candidate.candidate, e.candidate.sdpMid || '', e.candidate.sdpMLineIndex || 0);
|
|
1955
|
-
} catch (err) {}
|
|
1956
|
-
return;
|
|
1957
|
-
}
|
|
1958
|
-
|
|
1959
|
-
// Gathering finished. With no relay candidate there is no path left for a network
|
|
1960
|
-
// that blocks UDP, so stop waiting the full timeout for something that cannot come.
|
|
1961
|
-
if (!realtimeSawRelayCandidate && !realtimeWebRtcConnected && realtimeWebRtcConnectTimer) {
|
|
1962
|
-
clearTimeout(realtimeWebRtcConnectTimer);
|
|
1963
|
-
realtimeWebRtcConnectTimer = setTimeout(function () {
|
|
1964
|
-
if (!realtimeWebRtcConnected && isRealtimeActive) {
|
|
1965
|
-
fallbackToWebSocket('no relay candidate and no direct connection');
|
|
1966
|
-
}
|
|
1967
|
-
}, REALTIME_WEBRTC_NO_RELAY_TIMEOUT_MS);
|
|
1968
|
-
}
|
|
1969
|
-
};
|
|
1970
|
-
pc.onconnectionstatechange = function () {
|
|
1971
|
-
if (pc.connectionState === 'connected') {
|
|
1972
|
-
markWebRtcConnected();
|
|
1973
|
-
} else if (pc.connectionState === 'failed') {
|
|
1974
|
-
// Before the peer connects, a failure means WebRTC is unusable here — fall back.
|
|
1975
|
-
// After it connects, a drop ends the session (we never migrate mid-session).
|
|
1976
|
-
if (!realtimeWebRtcConnected) {
|
|
1977
|
-
fallbackToWebSocket('connection failed');
|
|
1978
|
-
} else if (isRealtimeActive) {
|
|
1979
|
-
stopRealtimeConversation();
|
|
1980
|
-
}
|
|
1981
|
-
} else if (pc.connectionState === 'closed' && isRealtimeActive && realtimeWebRtcConnected) {
|
|
1982
|
-
stopRealtimeConversation();
|
|
1983
|
-
}
|
|
1984
|
-
};
|
|
1985
|
-
pc.oniceconnectionstatechange = function () {
|
|
1986
|
-
var s = pc.iceConnectionState;
|
|
1987
|
-
if (s === 'connected' || s === 'completed') {
|
|
1988
|
-
markWebRtcConnected();
|
|
1989
|
-
} else if (s === 'failed' && !realtimeWebRtcConnected) {
|
|
1990
|
-
fallbackToWebSocket('ICE failed');
|
|
1991
|
-
}
|
|
1992
|
-
};
|
|
1993
|
-
bindWebRtcSignalingHandlers();
|
|
1994
|
-
|
|
1995
|
-
// Fall back to WebSocket if the peer does not connect within the timeout (blocked UDP,
|
|
1996
|
-
// missing TURN, etc.). Cleared as soon as the peer connects (markWebRtcConnected).
|
|
1997
|
-
realtimeWebRtcConnectTimer = setTimeout(function () {
|
|
1998
|
-
if (!realtimeWebRtcConnected && isRealtimeActive) {
|
|
1999
|
-
fallbackToWebSocket('connection timed out');
|
|
2000
|
-
}
|
|
2001
|
-
}, REALTIME_WEBRTC_CONNECT_TIMEOUT_MS);
|
|
2002
|
-
|
|
2003
|
-
// Send a gated version of the mic: silent unless the user is actually speaking. The gate
|
|
2004
|
-
// loads an AudioWorklet, so the track is only available once that resolves — hence the
|
|
2005
|
-
// offer is created here rather than synchronously above.
|
|
2006
|
-
setupMicGate(stream).then(function (micTrackToSend) {
|
|
2007
|
-
if (realtimePc !== pc) {
|
|
2008
|
-
return null;
|
|
2009
|
-
}
|
|
2010
|
-
if (micTrackToSend) {
|
|
2011
|
-
pc.addTrack(micTrackToSend, stream);
|
|
2012
|
-
}
|
|
2013
|
-
return pc.createOffer();
|
|
2014
|
-
}).then(function (offer) {
|
|
2015
|
-
if (!offer || realtimePc !== pc) {
|
|
2016
|
-
return null;
|
|
2017
|
-
}
|
|
2018
|
-
return pc.setLocalDescription(offer).then(function () {
|
|
2019
|
-
return offer;
|
|
2020
|
-
});
|
|
2021
|
-
}).then(function (offer) {
|
|
2022
|
-
if (!offer) {
|
|
2023
|
-
return;
|
|
2024
|
-
}
|
|
2025
|
-
// "Auto" means auto-detect: send nothing (see the WebSocket path above).
|
|
2026
|
-
var language = realtimeLanguage || null;
|
|
2027
|
-
var voice = getVoiceName && getVoiceName() || realtimeVoiceName || '';
|
|
2028
|
-
sendStartWebRtc(offer.sdp, voice, language, null, null, realtimeBargeIn);
|
|
2029
|
-
})["catch"](function (err) {
|
|
2030
|
-
console.error('Failed to create the WebRTC offer; falling back to WebSocket.', err);
|
|
2031
|
-
fallbackToWebSocket('offer failed');
|
|
2032
|
-
});
|
|
2033
|
-
})["catch"](function (err) {
|
|
2034
|
-
stream.getTracks().forEach(function (t) {
|
|
2035
|
-
t.stop();
|
|
2036
|
-
});
|
|
2037
|
-
isRealtimeActive = false;
|
|
2038
|
-
realtimeIsWebRtc = false;
|
|
2039
|
-
onDeactivate();
|
|
2040
|
-
updateRealtimeButton();
|
|
2041
|
-
console.error('The realtime WebRTC conversation could not start because the chat connection is not available.', err);
|
|
2042
|
-
});
|
|
2043
|
-
})["catch"](function (err) {
|
|
2044
|
-
console.error('Microphone access denied:', err);
|
|
2045
|
-
isRealtimeActive = false;
|
|
2046
|
-
realtimeIsWebRtc = false;
|
|
2047
|
-
onDeactivate();
|
|
2048
|
-
updateRealtimeButton();
|
|
2049
|
-
});
|
|
2050
|
-
}
|
|
2051
|
-
function bindWebRtcSignalingHandlers() {
|
|
2052
|
-
if (realtimeWebRtcHandlersBound) {
|
|
2053
|
-
return;
|
|
2054
|
-
}
|
|
2055
|
-
realtimeWebRtcHandlersBound = true;
|
|
2056
|
-
connection.on('ReceiveRealtimeAnswer', function (sdp) {
|
|
2057
|
-
if (!realtimePc) {
|
|
2058
|
-
return;
|
|
2059
|
-
}
|
|
2060
|
-
realtimePc.setRemoteDescription({
|
|
2061
|
-
type: 'answer',
|
|
2062
|
-
sdp: sdp
|
|
2063
|
-
}).then(function () {
|
|
2064
|
-
realtimeRemoteDescriptionSet = true;
|
|
2065
|
-
// Flush any ICE candidates that arrived before the answer was applied.
|
|
2066
|
-
var pending = realtimePendingIce;
|
|
2067
|
-
realtimePendingIce = [];
|
|
2068
|
-
pending.forEach(function (init) {
|
|
2069
|
-
try {
|
|
2070
|
-
realtimePc.addIceCandidate(init)["catch"](function () {});
|
|
2071
|
-
} catch (err) {}
|
|
2072
|
-
});
|
|
2073
|
-
})["catch"](function (err) {
|
|
2074
|
-
console.error('Failed to apply the WebRTC answer.', err);
|
|
2075
|
-
});
|
|
2076
|
-
});
|
|
2077
|
-
connection.on('ReceiveRealtimeIceCandidate', function (candidate, sdpMid, sdpMLineIndex) {
|
|
2078
|
-
if (!realtimePc || !candidate) {
|
|
2079
|
-
return;
|
|
2080
|
-
}
|
|
2081
|
-
var init = {
|
|
2082
|
-
candidate: candidate,
|
|
2083
|
-
sdpMid: sdpMid,
|
|
2084
|
-
sdpMLineIndex: sdpMLineIndex
|
|
2085
|
-
};
|
|
2086
|
-
// A candidate can only be added after the remote description (answer) is set; buffer until then.
|
|
2087
|
-
if (realtimeRemoteDescriptionSet) {
|
|
2088
|
-
try {
|
|
2089
|
-
realtimePc.addIceCandidate(init)["catch"](function () {});
|
|
2090
|
-
} catch (err) {}
|
|
2091
|
-
} else {
|
|
2092
|
-
realtimePendingIce.push(init);
|
|
2093
|
-
}
|
|
2094
|
-
});
|
|
2095
|
-
}
|
|
2096
|
-
function stopRealtimeWebRtc() {
|
|
2097
|
-
stopWebRtcEchoGuard();
|
|
2098
|
-
realtimeWebRtcMicTrack = null;
|
|
2099
|
-
if (realtimePc) {
|
|
2100
|
-
try {
|
|
2101
|
-
realtimePc.close();
|
|
2102
|
-
} catch (err) {}
|
|
2103
|
-
realtimePc = null;
|
|
2104
|
-
}
|
|
2105
|
-
if (realtimeRemoteAudioEl) {
|
|
2106
|
-
try {
|
|
2107
|
-
realtimeRemoteAudioEl.srcObject = null;
|
|
2108
|
-
realtimeRemoteAudioEl.remove();
|
|
2109
|
-
} catch (err) {}
|
|
2110
|
-
realtimeRemoteAudioEl = null;
|
|
2111
|
-
}
|
|
2112
|
-
realtimeIsWebRtc = false;
|
|
2113
|
-
}
|
|
2114
|
-
function markWebRtcConnected() {
|
|
2115
|
-
if (realtimeWebRtcConnected) {
|
|
2116
|
-
return;
|
|
2117
|
-
}
|
|
2118
|
-
realtimeWebRtcConnected = true;
|
|
2119
|
-
if (realtimeWebRtcConnectTimer) {
|
|
2120
|
-
clearTimeout(realtimeWebRtcConnectTimer);
|
|
2121
|
-
realtimeWebRtcConnectTimer = null;
|
|
2122
|
-
}
|
|
2123
|
-
}
|
|
2124
|
-
|
|
2125
|
-
// --- Microphone gate (thin wrapper over the shared factory) ----------------------------------------
|
|
2126
|
-
|
|
2127
|
-
// How long the gate stays open after the user's level drops. The gate is a safety net around echo, not a
|
|
2128
|
-
// turn detector: the provider's turn detection sees the real speech and should be the one deciding a
|
|
2129
|
-
// turn is over, so this must comfortably outlast the pause it is willing to wait through. The gate emits
|
|
2130
|
-
// digital silence when it closes, and whichever of the two expires first is what actually ends the turn.
|
|
2131
|
-
var REALTIME_GATE_HANGOVER_MS = 2000;
|
|
2132
|
-
function realtimeGateModeMessage() {
|
|
2133
|
-
return {
|
|
2134
|
-
pushToTalk: !!realtimePushToTalk,
|
|
2135
|
-
pttActive: !!realtimePttActive,
|
|
2136
|
-
bargeIn: !!realtimeBargeIn,
|
|
2137
|
-
speechHangoverMs: REALTIME_GATE_HANGOVER_MS
|
|
2138
|
-
};
|
|
2139
|
-
}
|
|
2140
|
-
|
|
2141
|
-
// The gate runs on the audio thread and cannot read this closure's variables, so every settings or
|
|
2142
|
-
// push-to-talk change has to be pushed to it explicitly.
|
|
2143
|
-
function syncRealtimeGateMode() {
|
|
2144
|
-
if (realtimeGate) {
|
|
2145
|
-
realtimeGate.setMode(realtimeGateModeMessage());
|
|
2146
|
-
}
|
|
2147
|
-
}
|
|
2148
|
-
|
|
2149
|
-
// Builds the gate and resolves with the track to send (over the peer, or into the WebSocket capture graph).
|
|
2150
|
-
function setupMicGate(rawStream) {
|
|
2151
|
-
stopWebRtcEchoGuard();
|
|
2152
|
-
var rawTrack = rawStream && rawStream.getAudioTracks()[0] || null;
|
|
2153
|
-
realtimeWebRtcMicTrack = rawTrack;
|
|
2154
|
-
return createMicGate(rawStream, realtimeGateModeMessage()).then(function (gate) {
|
|
2155
|
-
realtimeGate = gate;
|
|
2156
|
-
if (realtimeGatePendingRemoteStream) {
|
|
2157
|
-
gate.attachAssistantStream(realtimeGatePendingRemoteStream);
|
|
2158
|
-
}
|
|
2159
|
-
return gate.track || rawTrack;
|
|
2160
|
-
});
|
|
2161
|
-
}
|
|
2162
|
-
|
|
2163
|
-
// Feeds the assistant's remote stream into the gate so it knows when the assistant is audible — for the
|
|
2164
|
-
// raised threshold with barge-in on, and for half-duplex with barge-in off. The track can arrive before
|
|
2165
|
-
// the gate's worklet has finished loading, so remember it either way.
|
|
2166
|
-
function attachWebRtcAssistantAnalyser(remoteStream) {
|
|
2167
|
-
realtimeGatePendingRemoteStream = remoteStream || null;
|
|
2168
|
-
if (realtimeGate && remoteStream) {
|
|
2169
|
-
realtimeGate.attachAssistantStream(remoteStream);
|
|
2170
|
-
}
|
|
2171
|
-
}
|
|
2172
|
-
function stopWebRtcEchoGuard() {
|
|
2173
|
-
if (realtimeGate) {
|
|
2174
|
-
try {
|
|
2175
|
-
realtimeGate.stop();
|
|
2176
|
-
} catch (e) {}
|
|
2177
|
-
realtimeGate = null;
|
|
2178
|
-
}
|
|
2179
|
-
realtimeGatePendingRemoteStream = null;
|
|
2180
|
-
}
|
|
2181
|
-
|
|
2182
|
-
// Route the assistant playback element so the browser's echo canceller couples to it as an AEC reference,
|
|
2183
|
-
// the way ChatGPT's voice mode does. Preference order:
|
|
2184
|
-
// 1. The "communications" sink (Chrome/Edge). Rendering here doesn't just pick a device — it puts the
|
|
2185
|
-
// browser's audio pipeline in COMMUNICATIONS mode, which is what actually couples playback with the
|
|
2186
|
-
// mic's echo canceller. This is what ChatGPT uses and what keeps full-duplex stable across turns.
|
|
2187
|
-
// 2. The concrete device the "default" alias points at, matched by groupId.
|
|
2188
|
-
// If neither alias exists — Firefox exposes only concrete outputs — we deliberately do nothing and let the
|
|
2189
|
-
// browser use its own default. Picking "the first concrete output" there routed the assistant to whatever
|
|
2190
|
-
// device happened to be enumerated first (often an HDMI monitor), so the assistant appeared silent.
|
|
2191
|
-
// Applies the user's chosen speaker to the playing element, live.
|
|
2192
|
-
function applyRealtimeOutputDevice() {
|
|
2193
|
-
routeOutputToPreferredDevice(realtimeRemoteAudioEl, realtimeOutputDeviceId);
|
|
2194
|
-
}
|
|
2195
|
-
function routeRealtimeOutputToDefaultDevice(el) {
|
|
2196
|
-
routeOutputToPreferredDevice(el, realtimeOutputDeviceId);
|
|
2197
|
-
}
|
|
2198
|
-
function ensureRealtimePlayback(el) {
|
|
2199
|
-
ensurePlayback(el, function () {
|
|
2200
|
-
setRealtimeState('playback-blocked');
|
|
2201
|
-
});
|
|
2202
|
-
}
|
|
2203
|
-
|
|
2204
|
-
// Says why a realtime session is running on the WebSocket transport when it never attempted WebRTC.
|
|
2205
|
-
// The connect-time fallback reports its own reason instead (see fallbackToWebSocket).
|
|
2206
|
-
function logWebSocketTransportReason(reason) {
|
|
2207
|
-
if (window.console && console.warn) {
|
|
2208
|
-
console.warn('Realtime is using the WebSocket transport (' + reason + '); acoustic echo cancellation is weaker than on WebRTC.');
|
|
2209
|
-
}
|
|
2210
|
-
}
|
|
2211
|
-
|
|
2212
|
-
// Connect-time only: tear down the failed WebRTC attempt and restart on the known-good WebSocket path.
|
|
2213
|
-
// Never called once a session is established (see the connection-state handlers), so we never migrate audio
|
|
2214
|
-
// mid-conversation — we only choose the transport before the model starts responding.
|
|
2215
|
-
function fallbackToWebSocket(reason) {
|
|
2216
|
-
if (realtimeFellBack) {
|
|
2217
|
-
return;
|
|
2218
|
-
}
|
|
2219
|
-
realtimeFellBack = true;
|
|
2220
|
-
if (realtimeWebRtcConnectTimer) {
|
|
2221
|
-
clearTimeout(realtimeWebRtcConnectTimer);
|
|
2222
|
-
realtimeWebRtcConnectTimer = null;
|
|
2223
|
-
}
|
|
2224
|
-
if (window.console && console.warn) {
|
|
2225
|
-
console.warn('Realtime WebRTC transport unavailable (' + reason + '); falling back to the WebSocket transport.');
|
|
2226
|
-
}
|
|
2227
|
-
rememberWebRtcBlocked();
|
|
2228
|
-
showCompatibilityModeNotice();
|
|
2229
|
-
try {
|
|
2230
|
-
if (realtimeStream) {
|
|
2231
|
-
realtimeStream.getTracks().forEach(function (t) {
|
|
2232
|
-
t.stop();
|
|
2233
|
-
});
|
|
2234
|
-
}
|
|
2235
|
-
} catch (err) {}
|
|
2236
|
-
realtimeStream = null;
|
|
2237
|
-
stopRealtimeWebRtc();
|
|
2238
|
-
realtimeIsWebRtc = false;
|
|
2239
|
-
// Reset the active flag so the WebSocket start proceeds cleanly; it re-activates immediately.
|
|
2240
|
-
isRealtimeActive = false;
|
|
2241
|
-
startRealtimeWebSocketConversation();
|
|
2242
|
-
}
|
|
2243
|
-
function stopRealtimeConversation() {
|
|
2244
|
-
if (!isRealtimeActive) {
|
|
2245
|
-
return;
|
|
2246
|
-
}
|
|
2247
|
-
if (realtimeWebRtcConnectTimer) {
|
|
2248
|
-
clearTimeout(realtimeWebRtcConnectTimer);
|
|
2249
|
-
realtimeWebRtcConnectTimer = null;
|
|
2250
|
-
}
|
|
2251
|
-
isRealtimeActive = false;
|
|
2252
|
-
realtimeSessionReady = false;
|
|
2253
|
-
onDeactivate();
|
|
2254
|
-
updateRealtimeButton();
|
|
2255
|
-
detachRealtimePushToTalk();
|
|
2256
|
-
removeRealtimePttUi();
|
|
2257
|
-
if (!realtimeEndedNotice) {
|
|
2258
|
-
setRealtimeState('idle');
|
|
2259
|
-
}
|
|
2260
|
-
if (realtimeIsWebRtc) {
|
|
2261
|
-
stopRealtimeWebRtc();
|
|
2262
|
-
try {
|
|
2263
|
-
if (realtimeStream) {
|
|
2264
|
-
realtimeStream.getTracks().forEach(function (t) {
|
|
2265
|
-
t.stop();
|
|
2266
|
-
});
|
|
2267
|
-
}
|
|
2268
|
-
} catch (err) {}
|
|
2269
|
-
realtimeStream = null;
|
|
2270
|
-
return;
|
|
2271
|
-
}
|
|
2272
|
-
try {
|
|
2273
|
-
if (realtimeSubject) {
|
|
2274
|
-
realtimeSubject.complete();
|
|
2275
|
-
}
|
|
2276
|
-
} catch (err) {/* already completed */}
|
|
2277
|
-
realtimeSubject = null;
|
|
2278
|
-
stopWebRtcEchoGuard();
|
|
2279
|
-
|
|
2280
|
-
// Detach both shapes the capture node can take: the worklet delivers frames over its port, the
|
|
2281
|
-
// ScriptProcessorNode fallback over onaudioprocess. Leaving either attached keeps pushing frames at
|
|
2282
|
-
// a subject that is already completed.
|
|
2283
|
-
try {
|
|
2284
|
-
if (realtimeProcessor) {
|
|
2285
|
-
realtimeProcessor.disconnect();
|
|
2286
|
-
realtimeProcessor.onaudioprocess = null;
|
|
2287
|
-
if (realtimeProcessor.port) {
|
|
2288
|
-
realtimeProcessor.port.onmessage = null;
|
|
2289
|
-
realtimeProcessor.port.close();
|
|
2290
|
-
}
|
|
2291
|
-
}
|
|
2292
|
-
} catch (err) {}
|
|
2293
|
-
try {
|
|
2294
|
-
if (realtimeMicSource) {
|
|
2295
|
-
realtimeMicSource.disconnect();
|
|
2296
|
-
}
|
|
2297
|
-
} catch (err) {}
|
|
2298
|
-
try {
|
|
2299
|
-
if (realtimeZeroGain) {
|
|
2300
|
-
realtimeZeroGain.disconnect();
|
|
2301
|
-
}
|
|
2302
|
-
} catch (err) {}
|
|
2303
|
-
try {
|
|
2304
|
-
if (realtimeGain) {
|
|
2305
|
-
realtimeGain.disconnect();
|
|
2306
|
-
}
|
|
2307
|
-
} catch (err) {}
|
|
2308
|
-
try {
|
|
2309
|
-
if (realtimeStream) {
|
|
2310
|
-
realtimeStream.getTracks().forEach(function (t) {
|
|
2311
|
-
t.stop();
|
|
2312
|
-
});
|
|
2313
|
-
}
|
|
2314
|
-
} catch (err) {}
|
|
2315
|
-
realtimeProcessor = null;
|
|
2316
|
-
realtimeMicSource = null;
|
|
2317
|
-
realtimeZeroGain = null;
|
|
2318
|
-
realtimeGain = null;
|
|
2319
|
-
realtimeStream = null;
|
|
2320
|
-
flushRealtimePlayback();
|
|
2321
|
-
try {
|
|
2322
|
-
if (realtimeAudioCtx) {
|
|
2323
|
-
realtimeAudioCtx.close();
|
|
2324
|
-
}
|
|
2325
|
-
} catch (err) {}
|
|
2326
|
-
realtimeAudioCtx = null;
|
|
2327
|
-
}
|
|
2328
|
-
function toggleRealtime() {
|
|
2329
|
-
if (isRealtimeActive) {
|
|
2330
|
-
stopRealtimeConversation();
|
|
2331
|
-
return;
|
|
2332
|
-
}
|
|
2333
|
-
startRealtimeConversation();
|
|
2334
|
-
}
|
|
2335
|
-
|
|
2336
|
-
// Resolves with the receive-side statistics of the assistant's audio track (WebRTC transport only), in
|
|
2337
|
-
// the browser's own words: packets lost, jitter, how much the jitter buffer is holding, and how often it
|
|
2338
|
-
// had to conceal or time-stretch audio. Null on the WebSocket transport.
|
|
2339
|
-
function getTransportStats() {
|
|
2340
|
-
if (!realtimePc || typeof realtimePc.getStats !== 'function') {
|
|
2341
|
-
return Promise.resolve(null);
|
|
2342
|
-
}
|
|
2343
|
-
return realtimePc.getStats().then(function (report) {
|
|
2344
|
-
// Where playback actually goes (the "communications" sink puts Chrome's pipeline in communications
|
|
2345
|
-
// mode; empty means the browser default) and the receiver's requested cushion, so a report of
|
|
2346
|
-
// "sounds different in this browser" can be tied to the output path.
|
|
2347
|
-
var result = {
|
|
2348
|
-
transport: 'webrtc',
|
|
2349
|
-
inbound: null,
|
|
2350
|
-
codec: null,
|
|
2351
|
-
sinkId: realtimeRemoteAudioEl && typeof realtimeRemoteAudioEl.sinkId === 'string' ? realtimeRemoteAudioEl.sinkId : null,
|
|
2352
|
-
jitterBufferTargetMs: function () {
|
|
2353
|
-
try {
|
|
2354
|
-
var r = realtimePc.getReceivers().filter(function (x) {
|
|
2355
|
-
return x.track && x.track.kind === 'audio';
|
|
2356
|
-
})[0];
|
|
2357
|
-
return r && 'jitterBufferTarget' in r ? r.jitterBufferTarget : null;
|
|
2358
|
-
} catch (e) {
|
|
2359
|
-
return null;
|
|
2360
|
-
}
|
|
2361
|
-
}()
|
|
2362
|
-
};
|
|
2363
|
-
report.forEach(function (stat) {
|
|
2364
|
-
if (stat.type === 'inbound-rtp' && stat.kind === 'audio') {
|
|
2365
|
-
result.inbound = {
|
|
2366
|
-
packetsReceived: stat.packetsReceived,
|
|
2367
|
-
packetsLost: stat.packetsLost,
|
|
2368
|
-
packetsDiscarded: stat.packetsDiscarded,
|
|
2369
|
-
jitterMs: typeof stat.jitter === 'number' ? Math.round(stat.jitter * 1000) : null,
|
|
2370
|
-
jitterBufferDelayMs: typeof stat.jitterBufferDelay === 'number' && stat.jitterBufferEmittedCount ? Math.round(stat.jitterBufferDelay / stat.jitterBufferEmittedCount * 1000) : null,
|
|
2371
|
-
totalSamplesReceived: stat.totalSamplesReceived,
|
|
2372
|
-
concealedSamples: stat.concealedSamples,
|
|
2373
|
-
concealmentEvents: stat.concealmentEvents,
|
|
2374
|
-
silentConcealedSamples: stat.silentConcealedSamples,
|
|
2375
|
-
insertedSamplesForDeceleration: stat.insertedSamplesForDeceleration,
|
|
2376
|
-
removedSamplesForAcceleration: stat.removedSamplesForAcceleration,
|
|
2377
|
-
audioLevel: stat.audioLevel
|
|
2378
|
-
};
|
|
2379
|
-
if (stat.codecId) {
|
|
2380
|
-
var codec = report.get(stat.codecId);
|
|
2381
|
-
if (codec) {
|
|
2382
|
-
result.codec = {
|
|
2383
|
-
mimeType: codec.mimeType,
|
|
2384
|
-
clockRate: codec.clockRate,
|
|
2385
|
-
channels: codec.channels,
|
|
2386
|
-
sdpFmtpLine: codec.sdpFmtpLine
|
|
2387
|
-
};
|
|
2388
|
-
}
|
|
2389
|
-
}
|
|
2390
|
-
}
|
|
2391
|
-
});
|
|
2392
|
-
return result;
|
|
2393
|
-
})["catch"](function () {
|
|
2394
|
-
return null;
|
|
2395
|
-
});
|
|
2396
|
-
}
|
|
2397
|
-
function playRealtimePcm(bytes) {
|
|
2398
|
-
if (!realtimeAudioCtx || !bytes || bytes.length < 2) {
|
|
2399
|
-
return;
|
|
2400
|
-
}
|
|
2401
|
-
var ctx = realtimeAudioCtx;
|
|
2402
|
-
var sampleCount = Math.floor(bytes.length / 2);
|
|
2403
|
-
var pcm = new Int16Array(bytes.buffer, bytes.byteOffset, sampleCount);
|
|
2404
|
-
var f32 = new Float32Array(sampleCount);
|
|
2405
|
-
for (var i = 0; i < sampleCount; i++) {
|
|
2406
|
-
f32[i] = pcm[i] / 0x8000;
|
|
2407
|
-
}
|
|
2408
|
-
var buffer = ctx.createBuffer(1, sampleCount, ctx.sampleRate);
|
|
2409
|
-
buffer.copyToChannel(f32, 0);
|
|
2410
|
-
var src = ctx.createBufferSource();
|
|
2411
|
-
src.buffer = buffer;
|
|
2412
|
-
src.connect(realtimeGain || ctx.destination);
|
|
2413
|
-
var now = ctx.currentTime;
|
|
2414
|
-
if (realtimePlayHead < now) {
|
|
2415
|
-
realtimePlayHead = now;
|
|
2416
|
-
}
|
|
2417
|
-
src.start(realtimePlayHead);
|
|
2418
|
-
realtimePlayHead += buffer.duration;
|
|
2419
|
-
realtimeSources.push(src);
|
|
2420
|
-
src.onended = function () {
|
|
2421
|
-
realtimeSources = realtimeSources.filter(function (s) {
|
|
2422
|
-
return s !== src;
|
|
2423
|
-
});
|
|
2424
|
-
};
|
|
2425
|
-
}
|
|
2426
|
-
function flushRealtimePlayback() {
|
|
2427
|
-
realtimeSources.forEach(function (s) {
|
|
2428
|
-
try {
|
|
2429
|
-
s.stop();
|
|
2430
|
-
} catch (err) {}
|
|
2431
|
-
});
|
|
2432
|
-
realtimeSources = [];
|
|
2433
|
-
if (realtimeAudioCtx) {
|
|
2434
|
-
realtimePlayHead = realtimeAudioCtx.currentTime;
|
|
2435
|
-
}
|
|
2436
|
-
}
|
|
2437
|
-
function isRealtimeDeployment(deploymentName) {
|
|
2438
|
-
if (!deploymentName) {
|
|
2439
|
-
return false;
|
|
2440
|
-
}
|
|
2441
|
-
return realtimeCapableDeployments.indexOf(deploymentName.toLowerCase()) !== -1;
|
|
2442
|
-
}
|
|
2443
|
-
|
|
2444
|
-
// --- Wire up ---
|
|
2445
|
-
var realtimeBtn = q('realtimeButton');
|
|
2446
|
-
if (realtimeBtn) {
|
|
2447
|
-
realtimeBtn.addEventListener('click', function (e) {
|
|
2448
|
-
// Ignore keyboard-synthesized clicks (Space/Enter) during an active realtime session, so a
|
|
2449
|
-
// push-to-talk Space press can never toggle the session off.
|
|
2450
|
-
if (e && e.detail === 0 && isRealtimeActive) {
|
|
2451
|
-
return;
|
|
2452
|
-
}
|
|
2453
|
-
toggleRealtime();
|
|
2454
|
-
});
|
|
2455
|
-
setupRealtimeAudioSettings();
|
|
2456
|
-
}
|
|
2457
|
-
var deploymentSelect = q('deploymentSelect');
|
|
2458
|
-
if (deploymentSelect) {
|
|
2459
|
-
deploymentSelect.addEventListener('change', function () {
|
|
2460
|
-
applyRealtimeMode(isRealtimeDeployment(deploymentSelect.value));
|
|
2461
|
-
});
|
|
2462
|
-
}
|
|
2463
|
-
|
|
2464
|
-
// Apply the initial input mode: audio-only when the interaction already uses a realtime deployment.
|
|
2465
|
-
applyRealtimeMode(isRealtimeMode || deploymentSelect && isRealtimeDeployment(deploymentSelect.value));
|
|
2466
|
-
var controller = {
|
|
2467
|
-
toggle: toggleRealtime,
|
|
2468
|
-
start: startRealtimeConversation,
|
|
2469
|
-
stop: stopRealtimeConversation,
|
|
2470
|
-
applyMode: applyRealtimeMode,
|
|
2471
|
-
isRealtimeDeployment: isRealtimeDeployment,
|
|
2472
|
-
receivePcm: playRealtimePcm,
|
|
2473
|
-
isActive: function isActive() {
|
|
2474
|
-
return isRealtimeActive;
|
|
2475
|
-
},
|
|
2476
|
-
getState: function getState() {
|
|
2477
|
-
return realtimeState;
|
|
2478
|
-
},
|
|
2479
|
-
// The gate's most recent measurement (level, tracked noise floor, open/closed, assistant audible), for
|
|
2480
|
-
// hosts that want to render a microphone meter or explain why the mic is muted.
|
|
2481
|
-
getGateLevel: function getGateLevel() {
|
|
2482
|
-
return realtimeGate ? realtimeGate.getLevel() : null;
|
|
2483
|
-
},
|
|
2484
|
-
// The browser's own receive-side statistics for the assistant's audio on the WebRTC transport
|
|
2485
|
-
// (packets lost, jitter, jitter-buffer delay, concealment), for diagnosing choppy playback.
|
|
2486
|
-
getTransportStats: getTransportStats
|
|
2487
|
-
};
|
|
2488
|
-
|
|
2489
|
-
// The most recently attached controller, for diagnostics from the console or a test.
|
|
2490
|
-
window.CoreAIRealtime.activeController = controller;
|
|
2491
|
-
return controller;
|
|
2492
|
-
}
|
|
2493
|
-
window.CoreAIRealtime = {
|
|
2494
|
-
attach: attach,
|
|
2495
|
-
createMicGate: createMicGate,
|
|
2496
|
-
// Exposed for tests: the gate's rules are where every one of its historical failures lived, and they are
|
|
2497
|
-
// far easier to pin down with numbers than with audio.
|
|
2498
|
-
decideGate: coreAiGateDecide,
|
|
2499
|
-
createGateState: createGateState,
|
|
2500
|
-
rmsDb: coreAiRmsDb,
|
|
2501
|
-
// Host-agnostic playback helpers, so a host that drives realtime itself still shares one implementation
|
|
2502
|
-
// of these rather than growing its own.
|
|
2503
|
-
routeOutputToPreferredDevice: routeOutputToPreferredDevice,
|
|
2504
|
-
ensurePlayback: ensurePlayback
|
|
2505
|
-
};
|
|
2506
|
-
})(window, document);
|
|
5
|
+
(function() {
|
|
6
|
+
//#region src/Resources/CrestApps.AI.Resources/Assets/js/realtime-audio.js
|
|
7
|
+
(function(window, document) {
|
|
8
|
+
"use strict";
|
|
9
|
+
var REALTIME_SAMPLE_RATE = 24e3;
|
|
10
|
+
var REALTIME_GATE_LOOKAHEAD_MS = 80;
|
|
11
|
+
function coreAiGateDecide(state, input) {
|
|
12
|
+
var micDb = input.micDb;
|
|
13
|
+
var assistantDb = typeof input.assistantDb === "number" ? input.assistantDb : -100;
|
|
14
|
+
var nowMs = input.nowMs;
|
|
15
|
+
var mode = input.mode || {};
|
|
16
|
+
var dtMs = state.lastMs === null ? 0 : Math.max(0, Math.min(50, nowMs - state.lastMs));
|
|
17
|
+
state.lastMs = nowMs;
|
|
18
|
+
if (assistantDb > -50) state.assistantUntil = nowMs + 400;
|
|
19
|
+
var assistantSpeaking = nowMs < state.assistantUntil;
|
|
20
|
+
if (assistantDb > state.assistantPeakDb) state.assistantPeakDb = assistantDb;
|
|
21
|
+
else state.assistantPeakDb = Math.max(-100, state.assistantPeakDb - .02 * dtMs);
|
|
22
|
+
if (micDb <= -95) {
|
|
23
|
+
if (state.floorDb === null) {
|
|
24
|
+
state.open = false;
|
|
25
|
+
return state;
|
|
26
|
+
}
|
|
27
|
+
} else if (state.floorDb === null) state.floorDb = Math.min(micDb, -50);
|
|
28
|
+
if (micDb < state.floorDb) state.floorDb += (micDb - state.floorDb) * Math.min(1, .0075 * dtMs);
|
|
29
|
+
else if (!assistantSpeaking) {
|
|
30
|
+
var sustained = state.openSince !== null && nowMs - state.openSince > 1e4;
|
|
31
|
+
var risingPerMs = state.open && !sustained ? 1e-4 : 56e-5;
|
|
32
|
+
state.floorDb += (micDb - state.floorDb) * Math.min(1, risingPerMs * dtMs);
|
|
33
|
+
}
|
|
34
|
+
if (state.floorDb < -75) state.floorDb = -75;
|
|
35
|
+
if (assistantSpeaking && !state.open && assistantDb > -50 && !mode.pushToTalk) {
|
|
36
|
+
var erl = micDb - state.assistantPeakDb;
|
|
37
|
+
if (state.echoReturnDb === null) state.echoReturnDb = erl;
|
|
38
|
+
else {
|
|
39
|
+
var warm = state.echoLearnedMs < 1e3;
|
|
40
|
+
var ratePerMs = erl > state.echoReturnDb ? warm ? .02 : .0012 : 6e-4;
|
|
41
|
+
state.echoReturnDb += (erl - state.echoReturnDb) * Math.min(1, ratePerMs * dtMs);
|
|
42
|
+
}
|
|
43
|
+
state.echoLearnedMs += dtMs;
|
|
44
|
+
}
|
|
45
|
+
var expectedEchoDb = state.echoReturnDb === null ? -100 : state.assistantPeakDb + state.echoReturnDb;
|
|
46
|
+
var openThreshold, holdThreshold;
|
|
47
|
+
if (assistantSpeaking) {
|
|
48
|
+
openThreshold = Math.max(state.floorDb + 12, expectedEchoDb + 8, -45);
|
|
49
|
+
holdThreshold = Math.max(state.floorDb + 3, expectedEchoDb + 3);
|
|
50
|
+
} else {
|
|
51
|
+
openThreshold = state.floorDb + 9;
|
|
52
|
+
holdThreshold = state.floorDb + 3;
|
|
53
|
+
}
|
|
54
|
+
var hangoverMs = mode.speechHangoverMs || 2e3;
|
|
55
|
+
if (micDb > openThreshold) {
|
|
56
|
+
if (assistantSpeaking && !state.open) {
|
|
57
|
+
if (state.aboveSince === null) state.aboveSince = nowMs;
|
|
58
|
+
if (nowMs - state.aboveSince >= 250) state.speakUntil = nowMs + hangoverMs;
|
|
59
|
+
} else state.speakUntil = nowMs + hangoverMs;
|
|
60
|
+
} else {
|
|
61
|
+
state.aboveSince = null;
|
|
62
|
+
if (state.open && micDb > holdThreshold) state.speakUntil = Math.max(state.speakUntil, nowMs + 250);
|
|
63
|
+
}
|
|
64
|
+
var userActive = nowMs < state.speakUntil;
|
|
65
|
+
if (!assistantSpeaking && state.wasAssistantSpeaking) state.listenGraceUntil = nowMs + 1500;
|
|
66
|
+
state.wasAssistantSpeaking = assistantSpeaking;
|
|
67
|
+
state.assistantSpeaking = assistantSpeaking;
|
|
68
|
+
if (mode.pushToTalk) state.open = !!mode.pttActive;
|
|
69
|
+
else if (mode.bargeIn) state.open = userActive;
|
|
70
|
+
else state.open = !assistantSpeaking && (userActive || nowMs < state.listenGraceUntil);
|
|
71
|
+
if (state.open && state.openSince === null) state.openSince = nowMs;
|
|
72
|
+
else if (!state.open) state.openSince = null;
|
|
73
|
+
return state;
|
|
74
|
+
}
|
|
75
|
+
function createGateState() {
|
|
76
|
+
return {
|
|
77
|
+
floorDb: null,
|
|
78
|
+
lastMs: null,
|
|
79
|
+
speakUntil: 0,
|
|
80
|
+
assistantUntil: 0,
|
|
81
|
+
assistantPeakDb: -100,
|
|
82
|
+
echoReturnDb: null,
|
|
83
|
+
echoLearnedMs: 0,
|
|
84
|
+
aboveSince: null,
|
|
85
|
+
listenGraceUntil: 0,
|
|
86
|
+
wasAssistantSpeaking: false,
|
|
87
|
+
assistantSpeaking: false,
|
|
88
|
+
open: false,
|
|
89
|
+
openSince: null
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
function coreAiRmsDb(block) {
|
|
93
|
+
if (!block || !block.length) return -100;
|
|
94
|
+
var sum = 0;
|
|
95
|
+
for (var i = 0; i < block.length; i++) sum += block[i] * block[i];
|
|
96
|
+
var rms = Math.sqrt(sum / block.length);
|
|
97
|
+
return rms > 1e-7 ? 20 * Math.log10(rms) : -100;
|
|
98
|
+
}
|
|
99
|
+
var REALTIME_GATE_WORKLET_SOURCE = [
|
|
100
|
+
"var decide = " + coreAiGateDecide.toString() + ";",
|
|
101
|
+
"var newState = " + createGateState.toString() + ";",
|
|
102
|
+
"var rmsDb = " + coreAiRmsDb.toString() + ";",
|
|
103
|
+
"class CoreAiMicGateProcessor extends AudioWorkletProcessor {",
|
|
104
|
+
" constructor(options) {",
|
|
105
|
+
" super();",
|
|
106
|
+
" var o = (options && options.processorOptions) || {};",
|
|
107
|
+
" this.delay = new Float32Array(Math.max(1, Math.round(sampleRate * (o.lookaheadMs || 80) / 1000)));",
|
|
108
|
+
" this.delayPos = 0;",
|
|
109
|
+
" this.gain = 0;",
|
|
110
|
+
" this.mode = o.mode || {};",
|
|
111
|
+
" this.state = newState();",
|
|
112
|
+
" this.lastPost = 0;",
|
|
113
|
+
" this.port.onmessage = function (e) { this.mode = e.data || {}; }.bind(this);",
|
|
114
|
+
" }",
|
|
115
|
+
" process(inputs, outputs) {",
|
|
116
|
+
" var mic = (inputs[0] && inputs[0][0]) || null;",
|
|
117
|
+
" var assistant = (inputs[1] && inputs[1][0]) || null;",
|
|
118
|
+
" var out = (outputs[0] && outputs[0][0]) || null;",
|
|
119
|
+
" if (!out) { return true; }",
|
|
120
|
+
" var micDb = rmsDb(mic);",
|
|
121
|
+
" decide(this.state, { micDb: micDb, assistantDb: rmsDb(assistant), nowMs: currentTime * 1000, mode: this.mode });",
|
|
122
|
+
" var target = this.state.open ? 1 : 0;",
|
|
123
|
+
" var step = 1 / (sampleRate * 0.01);",
|
|
124
|
+
" var delay = this.delay;",
|
|
125
|
+
" var pos = this.delayPos;",
|
|
126
|
+
" var len = delay.length;",
|
|
127
|
+
" for (var i = 0; i < out.length; i++) {",
|
|
128
|
+
" var delayed = delay[pos];",
|
|
129
|
+
" delay[pos] = mic ? mic[i] : 0;",
|
|
130
|
+
" pos = (pos + 1) % len;",
|
|
131
|
+
" if (this.gain < target) { this.gain = Math.min(target, this.gain + step); }",
|
|
132
|
+
" else if (this.gain > target) { this.gain = Math.max(target, this.gain - step); }",
|
|
133
|
+
" out[i] = delayed * this.gain;",
|
|
134
|
+
" }",
|
|
135
|
+
" this.delayPos = pos;",
|
|
136
|
+
" if (currentTime * 1000 - this.lastPost > 100) {",
|
|
137
|
+
" this.lastPost = currentTime * 1000;",
|
|
138
|
+
" this.port.postMessage({ micDb: micDb, floorDb: this.state.floorDb, echoReturnDb: this.state.echoReturnDb, open: this.state.open, assistantSpeaking: this.state.assistantSpeaking });",
|
|
139
|
+
" }",
|
|
140
|
+
" return true;",
|
|
141
|
+
" }",
|
|
142
|
+
"}",
|
|
143
|
+
"registerProcessor(\"coreai-mic-gate\", CoreAiMicGateProcessor);"
|
|
144
|
+
].join("\n");
|
|
145
|
+
var REALTIME_CAPTURE_FRAME_SAMPLES = 4096;
|
|
146
|
+
var REALTIME_CAPTURE_WORKLET_SOURCE = [
|
|
147
|
+
"class CoreAiCaptureProcessor extends AudioWorkletProcessor {",
|
|
148
|
+
" constructor(options) {",
|
|
149
|
+
" super();",
|
|
150
|
+
" var o = (options && options.processorOptions) || {};",
|
|
151
|
+
" this.size = Math.max(128, o.frameSamples || 4096);",
|
|
152
|
+
" this.buf = new Float32Array(this.size);",
|
|
153
|
+
" this.pos = 0;",
|
|
154
|
+
" }",
|
|
155
|
+
" process(inputs) {",
|
|
156
|
+
" var input = (inputs[0] && inputs[0][0]) || null;",
|
|
157
|
+
" var n = input ? input.length : 128;",
|
|
158
|
+
" for (var i = 0; i < n; i++) {",
|
|
159
|
+
" this.buf[this.pos++] = input ? input[i] : 0;",
|
|
160
|
+
" if (this.pos === this.size) {",
|
|
161
|
+
" this.port.postMessage(this.buf);",
|
|
162
|
+
" this.pos = 0;",
|
|
163
|
+
" }",
|
|
164
|
+
" }",
|
|
165
|
+
" return true;",
|
|
166
|
+
" }",
|
|
167
|
+
"}",
|
|
168
|
+
"registerProcessor(\"coreai-pcm-capture\", CoreAiCaptureProcessor);"
|
|
169
|
+
].join("\n");
|
|
170
|
+
function createRealtimeCaptureNode(ctx, onFrame) {
|
|
171
|
+
function scriptProcessorNode() {
|
|
172
|
+
var node = ctx.createScriptProcessor(REALTIME_CAPTURE_FRAME_SAMPLES, 1, 1);
|
|
173
|
+
node.onaudioprocess = function(event) {
|
|
174
|
+
onFrame(event.inputBuffer.getChannelData(0));
|
|
175
|
+
};
|
|
176
|
+
return node;
|
|
177
|
+
}
|
|
178
|
+
var blobUrl = null;
|
|
179
|
+
if (ctx.audioWorklet && typeof ctx.audioWorklet.addModule === "function" && typeof AudioWorkletNode === "function") try {
|
|
180
|
+
blobUrl = URL.createObjectURL(new Blob([REALTIME_CAPTURE_WORKLET_SOURCE], { type: "application/javascript" }));
|
|
181
|
+
} catch (err) {
|
|
182
|
+
blobUrl = null;
|
|
183
|
+
}
|
|
184
|
+
if (!blobUrl) return Promise.resolve(scriptProcessorNode());
|
|
185
|
+
return ctx.audioWorklet.addModule(blobUrl).then(function() {
|
|
186
|
+
try {
|
|
187
|
+
URL.revokeObjectURL(blobUrl);
|
|
188
|
+
} catch (e) {}
|
|
189
|
+
var node = new AudioWorkletNode(ctx, "coreai-pcm-capture", {
|
|
190
|
+
numberOfInputs: 1,
|
|
191
|
+
numberOfOutputs: 1,
|
|
192
|
+
outputChannelCount: [1],
|
|
193
|
+
processorOptions: { frameSamples: REALTIME_CAPTURE_FRAME_SAMPLES }
|
|
194
|
+
});
|
|
195
|
+
node.port.onmessage = function(e) {
|
|
196
|
+
if (e.data) onFrame(e.data);
|
|
197
|
+
};
|
|
198
|
+
return node;
|
|
199
|
+
}).catch(function(err) {
|
|
200
|
+
try {
|
|
201
|
+
URL.revokeObjectURL(blobUrl);
|
|
202
|
+
} catch (e) {}
|
|
203
|
+
if (window.console && console.warn) console.warn("The realtime capture worklet could not be loaded; falling back to a ScriptProcessorNode.", err);
|
|
204
|
+
return scriptProcessorNode();
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
function createMicGate(rawStream, mode) {
|
|
208
|
+
var rawTrack = rawStream && rawStream.getAudioTracks && rawStream.getAudioTracks()[0] || null;
|
|
209
|
+
var state = {
|
|
210
|
+
ctx: null,
|
|
211
|
+
node: null,
|
|
212
|
+
micSource: null,
|
|
213
|
+
remoteSource: null,
|
|
214
|
+
dest: null,
|
|
215
|
+
timer: 0,
|
|
216
|
+
level: null,
|
|
217
|
+
fallback: null,
|
|
218
|
+
track: rawTrack,
|
|
219
|
+
stopped: false
|
|
220
|
+
};
|
|
221
|
+
var gate = {
|
|
222
|
+
track: rawTrack,
|
|
223
|
+
usingWorklet: false,
|
|
224
|
+
getLevel: function() {
|
|
225
|
+
return state.level;
|
|
226
|
+
},
|
|
227
|
+
setMode: function(next) {
|
|
228
|
+
mode = next || mode;
|
|
229
|
+
if (state.node && state.node.port) try {
|
|
230
|
+
state.node.port.postMessage(mode);
|
|
231
|
+
} catch (e) {}
|
|
232
|
+
if (state.fallback) state.fallback.mode = mode;
|
|
233
|
+
},
|
|
234
|
+
attachAssistantStream: function(remoteStream) {
|
|
235
|
+
attachAssistant(state, remoteStream);
|
|
236
|
+
},
|
|
237
|
+
stop: function() {
|
|
238
|
+
state.stopped = true;
|
|
239
|
+
if (state.timer) {
|
|
240
|
+
try {
|
|
241
|
+
window.clearInterval(state.timer);
|
|
242
|
+
} catch (e) {}
|
|
243
|
+
state.timer = 0;
|
|
244
|
+
}
|
|
245
|
+
if (state.node) {
|
|
246
|
+
try {
|
|
247
|
+
state.node.port.onmessage = null;
|
|
248
|
+
state.node.disconnect();
|
|
249
|
+
} catch (e) {}
|
|
250
|
+
state.node = null;
|
|
251
|
+
}
|
|
252
|
+
if (state.micSource) {
|
|
253
|
+
try {
|
|
254
|
+
state.micSource.disconnect();
|
|
255
|
+
} catch (e) {}
|
|
256
|
+
state.micSource = null;
|
|
257
|
+
}
|
|
258
|
+
if (state.remoteSource) {
|
|
259
|
+
try {
|
|
260
|
+
state.remoteSource.disconnect();
|
|
261
|
+
} catch (e) {}
|
|
262
|
+
state.remoteSource = null;
|
|
263
|
+
}
|
|
264
|
+
if (state.dest) {
|
|
265
|
+
try {
|
|
266
|
+
state.dest.disconnect();
|
|
267
|
+
} catch (e) {}
|
|
268
|
+
state.dest = null;
|
|
269
|
+
}
|
|
270
|
+
if (state.ctx) {
|
|
271
|
+
try {
|
|
272
|
+
state.ctx.close();
|
|
273
|
+
} catch (e) {}
|
|
274
|
+
state.ctx = null;
|
|
275
|
+
}
|
|
276
|
+
state.fallback = null;
|
|
277
|
+
}
|
|
278
|
+
};
|
|
279
|
+
var AudioCtor = window.AudioContext || window.webkitAudioContext;
|
|
280
|
+
if (!AudioCtor || !rawTrack) return Promise.resolve(gate);
|
|
281
|
+
try {
|
|
282
|
+
state.ctx = new AudioCtor();
|
|
283
|
+
if (state.ctx.state === "suspended" && typeof state.ctx.resume === "function") state.ctx.resume().catch(function() {});
|
|
284
|
+
} catch (err) {
|
|
285
|
+
state.ctx = null;
|
|
286
|
+
return Promise.resolve(gate);
|
|
287
|
+
}
|
|
288
|
+
var blobUrl = null;
|
|
289
|
+
if (state.ctx.audioWorklet && typeof state.ctx.audioWorklet.addModule === "function") try {
|
|
290
|
+
blobUrl = URL.createObjectURL(new Blob([REALTIME_GATE_WORKLET_SOURCE], { type: "application/javascript" }));
|
|
291
|
+
} catch (err) {
|
|
292
|
+
blobUrl = null;
|
|
293
|
+
}
|
|
294
|
+
if (!blobUrl) {
|
|
295
|
+
gate.track = buildFallbackGate(state, rawStream, rawTrack, mode);
|
|
296
|
+
return Promise.resolve(gate);
|
|
297
|
+
}
|
|
298
|
+
return state.ctx.audioWorklet.addModule(blobUrl).then(function() {
|
|
299
|
+
try {
|
|
300
|
+
URL.revokeObjectURL(blobUrl);
|
|
301
|
+
} catch (e) {}
|
|
302
|
+
if (state.stopped) return gate;
|
|
303
|
+
var node = new AudioWorkletNode(state.ctx, "coreai-mic-gate", {
|
|
304
|
+
numberOfInputs: 2,
|
|
305
|
+
numberOfOutputs: 1,
|
|
306
|
+
outputChannelCount: [1],
|
|
307
|
+
processorOptions: {
|
|
308
|
+
lookaheadMs: REALTIME_GATE_LOOKAHEAD_MS,
|
|
309
|
+
mode
|
|
310
|
+
}
|
|
311
|
+
});
|
|
312
|
+
state.node = node;
|
|
313
|
+
node.port.onmessage = function(e) {
|
|
314
|
+
state.level = e.data || null;
|
|
315
|
+
window.CoreAIRealtime.lastGateLevel = state.level;
|
|
316
|
+
};
|
|
317
|
+
state.micSource = state.ctx.createMediaStreamSource(rawStream);
|
|
318
|
+
state.micSource.connect(node, 0, 0);
|
|
319
|
+
state.dest = state.ctx.createMediaStreamDestination();
|
|
320
|
+
node.connect(state.dest);
|
|
321
|
+
if (state.pendingRemoteStream) attachAssistant(state, state.pendingRemoteStream);
|
|
322
|
+
gate.track = state.dest.stream.getAudioTracks()[0] || rawTrack;
|
|
323
|
+
gate.usingWorklet = true;
|
|
324
|
+
return gate;
|
|
325
|
+
}).catch(function(err) {
|
|
326
|
+
try {
|
|
327
|
+
URL.revokeObjectURL(blobUrl);
|
|
328
|
+
} catch (e) {}
|
|
329
|
+
if (window.console && console.warn) console.warn("The realtime microphone gate worklet could not be loaded; using the simpler gate.", err);
|
|
330
|
+
gate.track = buildFallbackGate(state, rawStream, rawTrack, mode);
|
|
331
|
+
return gate;
|
|
332
|
+
});
|
|
333
|
+
}
|
|
334
|
+
function attachAssistant(state, remoteStream) {
|
|
335
|
+
state.pendingRemoteStream = remoteStream || null;
|
|
336
|
+
if (!state.ctx || !remoteStream || state.stopped) return;
|
|
337
|
+
try {
|
|
338
|
+
if (state.remoteSource) try {
|
|
339
|
+
state.remoteSource.disconnect();
|
|
340
|
+
} catch (e) {}
|
|
341
|
+
var src = state.ctx.createMediaStreamSource(remoteStream);
|
|
342
|
+
state.remoteSource = src;
|
|
343
|
+
if (state.node) {
|
|
344
|
+
src.connect(state.node, 0, 1);
|
|
345
|
+
return;
|
|
346
|
+
}
|
|
347
|
+
if (state.fallback) {
|
|
348
|
+
var analyser = state.ctx.createAnalyser();
|
|
349
|
+
analyser.fftSize = 2048;
|
|
350
|
+
src.connect(analyser);
|
|
351
|
+
state.fallback.remoteAnalyser = analyser;
|
|
352
|
+
state.fallback.remoteData = new Float32Array(analyser.fftSize);
|
|
353
|
+
}
|
|
354
|
+
} catch (e) {}
|
|
355
|
+
}
|
|
356
|
+
function buildFallbackGate(state, rawStream, rawTrack, mode) {
|
|
357
|
+
try {
|
|
358
|
+
var ctx = state.ctx;
|
|
359
|
+
var source = ctx.createMediaStreamSource(rawStream);
|
|
360
|
+
state.micSource = source;
|
|
361
|
+
var analyser = ctx.createAnalyser();
|
|
362
|
+
analyser.fftSize = 2048;
|
|
363
|
+
source.connect(analyser);
|
|
364
|
+
var gain = ctx.createGain();
|
|
365
|
+
gain.gain.value = 0;
|
|
366
|
+
source.connect(gain);
|
|
367
|
+
var dest = ctx.createMediaStreamDestination();
|
|
368
|
+
state.dest = dest;
|
|
369
|
+
gain.connect(dest);
|
|
370
|
+
var f = {
|
|
371
|
+
mode,
|
|
372
|
+
remoteAnalyser: null,
|
|
373
|
+
remoteData: null
|
|
374
|
+
};
|
|
375
|
+
state.fallback = f;
|
|
376
|
+
if (state.pendingRemoteStream) attachAssistant(state, state.pendingRemoteStream);
|
|
377
|
+
var data = new Float32Array(analyser.fftSize);
|
|
378
|
+
var gateState = createGateState();
|
|
379
|
+
state.timer = window.setInterval(function() {
|
|
380
|
+
analyser.getFloatTimeDomainData(data);
|
|
381
|
+
var micDb = coreAiRmsDb(data);
|
|
382
|
+
var assistantDb = -100;
|
|
383
|
+
if (f.remoteAnalyser && f.remoteData) {
|
|
384
|
+
f.remoteAnalyser.getFloatTimeDomainData(f.remoteData);
|
|
385
|
+
assistantDb = coreAiRmsDb(f.remoteData);
|
|
386
|
+
}
|
|
387
|
+
var nowMs = window.performance && performance.now ? performance.now() : Date.now();
|
|
388
|
+
coreAiGateDecide(gateState, {
|
|
389
|
+
micDb,
|
|
390
|
+
assistantDb,
|
|
391
|
+
nowMs,
|
|
392
|
+
mode: f.mode || {}
|
|
393
|
+
});
|
|
394
|
+
state.level = {
|
|
395
|
+
micDb,
|
|
396
|
+
floorDb: gateState.floorDb,
|
|
397
|
+
open: gateState.open,
|
|
398
|
+
assistantSpeaking: gateState.assistantSpeaking
|
|
399
|
+
};
|
|
400
|
+
var target = gateState.open ? 1 : 0;
|
|
401
|
+
try {
|
|
402
|
+
gain.gain.setTargetAtTime(target, ctx.currentTime, .015);
|
|
403
|
+
} catch (e) {
|
|
404
|
+
gain.gain.value = target;
|
|
405
|
+
}
|
|
406
|
+
}, 20);
|
|
407
|
+
return dest.stream.getAudioTracks()[0] || rawTrack;
|
|
408
|
+
} catch (err) {
|
|
409
|
+
return rawTrack;
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
function routeOutputToPreferredDevice(el, chosenDeviceId) {
|
|
413
|
+
if (!el || typeof el.setSinkId !== "function") return;
|
|
414
|
+
if (chosenDeviceId) {
|
|
415
|
+
try {
|
|
416
|
+
el.setSinkId(chosenDeviceId).catch(function() {});
|
|
417
|
+
} catch (e) {}
|
|
418
|
+
return;
|
|
419
|
+
}
|
|
420
|
+
if (!navigator.mediaDevices || !navigator.mediaDevices.enumerateDevices) return;
|
|
421
|
+
navigator.mediaDevices.enumerateDevices().then(function(devices) {
|
|
422
|
+
var outputs = devices.filter(function(d) {
|
|
423
|
+
return d.kind === "audiooutput";
|
|
424
|
+
});
|
|
425
|
+
if (!outputs.length) return;
|
|
426
|
+
var target = "";
|
|
427
|
+
for (var i = 0; i < outputs.length; i++) if (outputs[i].deviceId === "communications") {
|
|
428
|
+
target = "communications";
|
|
429
|
+
break;
|
|
430
|
+
}
|
|
431
|
+
if (!target) {
|
|
432
|
+
var defaultEntry = null;
|
|
433
|
+
for (var d = 0; d < outputs.length; d++) if (outputs[d].deviceId === "default") {
|
|
434
|
+
defaultEntry = outputs[d];
|
|
435
|
+
break;
|
|
436
|
+
}
|
|
437
|
+
if (defaultEntry && defaultEntry.groupId) {
|
|
438
|
+
for (var j = 0; j < outputs.length; j++) if (outputs[j].deviceId !== "default" && outputs[j].deviceId !== "communications" && outputs[j].groupId === defaultEntry.groupId) {
|
|
439
|
+
target = outputs[j].deviceId;
|
|
440
|
+
break;
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
if (target) try {
|
|
445
|
+
el.setSinkId(target).catch(function() {});
|
|
446
|
+
} catch (e) {}
|
|
447
|
+
}).catch(function() {});
|
|
448
|
+
}
|
|
449
|
+
function ensurePlayback(el, onBlocked) {
|
|
450
|
+
if (!el || typeof el.play !== "function") return;
|
|
451
|
+
var played = el.play();
|
|
452
|
+
if (played && typeof played.catch === "function") played.catch(function(err) {
|
|
453
|
+
if (typeof onBlocked === "function") onBlocked(err);
|
|
454
|
+
if (window.console && console.warn) console.warn("The browser blocked realtime audio playback. Interact with the page to enable audio.", err);
|
|
455
|
+
});
|
|
456
|
+
}
|
|
457
|
+
function attach(opts) {
|
|
458
|
+
opts = opts || {};
|
|
459
|
+
var connection = opts.connection;
|
|
460
|
+
var sel = opts.selectors || {};
|
|
461
|
+
var realtimeVoiceName = opts.voiceName || "";
|
|
462
|
+
var realtimeCapableDeployments = (opts.capableDeployments || []).map(function(n) {
|
|
463
|
+
return (n || "").toLowerCase();
|
|
464
|
+
});
|
|
465
|
+
var isRealtimeMode = !!opts.realtimeEnabled;
|
|
466
|
+
var noop = function() {};
|
|
467
|
+
var onActivate = opts.onActivate || noop;
|
|
468
|
+
var onDeactivate = opts.onDeactivate || noop;
|
|
469
|
+
var onSessionStateChanged = opts.onSessionStateChanged || noop;
|
|
470
|
+
var onEnterRealtimeMode = opts.onEnterRealtimeMode || noop;
|
|
471
|
+
var onUserTurnPending = opts.onUserTurnPending || noop;
|
|
472
|
+
var onUserTurnDropped = opts.onUserTurnDropped || noop;
|
|
473
|
+
var strings = opts.strings || {};
|
|
474
|
+
function localize(key, fallback) {
|
|
475
|
+
return typeof strings[key] === "string" ? strings[key] : fallback;
|
|
476
|
+
}
|
|
477
|
+
var ensureConnected = opts.ensureConnected || function() {
|
|
478
|
+
return Promise.resolve(true);
|
|
479
|
+
};
|
|
480
|
+
var sendStart = opts.sendStart || noop;
|
|
481
|
+
var getVoiceName = opts.getVoiceName || null;
|
|
482
|
+
var sendStartWebRtc = opts.sendStartWebRtc || null;
|
|
483
|
+
var webRtcEnabled = opts.webRtcEnabled === true && typeof window.RTCPeerConnection === "function" && !!sendStartWebRtc;
|
|
484
|
+
var webRtcIceServers = Array.isArray(opts.webRtcIceServers) && opts.webRtcIceServers.length ? opts.webRtcIceServers : null;
|
|
485
|
+
var DEFAULT_ICE_SERVERS = [{ urls: "stun:stun.l.google.com:19302" }];
|
|
486
|
+
var onStateChange = opts.onStateChange || noop;
|
|
487
|
+
var realtimeState = "idle";
|
|
488
|
+
function q(name) {
|
|
489
|
+
var s = sel[name];
|
|
490
|
+
return s ? document.querySelector(s) : null;
|
|
491
|
+
}
|
|
492
|
+
var isRealtimeActive = false, realtimeLifecycleBound = false, realtimeEndedNotice = null, realtimeCompatibilityNoticeEl = null, realtimeStatusEl = null;
|
|
493
|
+
var realtimeStream = null, realtimeAudioCtx = null, realtimeSubject = null, realtimeProcessor = null, realtimeMicSource = null, realtimeZeroGain = null, realtimeGain = null, realtimePlayHead = 0, realtimeSources = [];
|
|
494
|
+
var realtimeIsWebRtc = false, realtimePc = null, realtimeRemoteAudioEl = null, realtimeWebRtcHandlersBound = false, realtimeRemoteDescriptionSet = false, realtimePendingIce = [];
|
|
495
|
+
var realtimeWebRtcConnected = false, realtimeWebRtcConnectTimer = null, realtimeFellBack = false;
|
|
496
|
+
var REALTIME_WEBRTC_CONNECT_TIMEOUT_MS = 8e3;
|
|
497
|
+
var REALTIME_WEBRTC_NO_RELAY_TIMEOUT_MS = 5e3;
|
|
498
|
+
var REALTIME_WEBRTC_JITTER_BUFFER_TARGET_MS = 150;
|
|
499
|
+
var REALTIME_WEBRTC_BLOCKED_KEY = "coreai.realtime.webrtcBlocked";
|
|
500
|
+
var realtimeAttemptIceServers = null;
|
|
501
|
+
var realtimeAttemptToken = 0;
|
|
502
|
+
var realtimeSawRelayCandidate = false, realtimeGate = null, realtimeGatePendingRemoteStream = null;
|
|
503
|
+
var realtimeBargeIn = true, realtimePushToTalk = false, realtimePttActive = false, realtimePttBound = false, realtimePttKeyDown = null, realtimePttKeyUp = null, realtimePttButton = null, realtimePttUiEl = null, realtimeVolume = 1, realtimeMicDeviceId = "", realtimeOutputDeviceId = "", realtimeLanguage = "", realtimeSpeechSpeed = null, realtimeSessionSpeechSpeed = 1, realtimeSpeechSpeedAdjustable = false, realtimeAudioSettingsBuilt = false, realtimeAudioSettingsElement = null;
|
|
504
|
+
var REALTIME_WS_ECHO_HANGOVER_SEC = .5;
|
|
505
|
+
var REALTIME_PREFS_VERSION = 3;
|
|
506
|
+
var REALTIME_SPEECH_SPEED_MIN = .75, REALTIME_SPEECH_SPEED_MAX = 1.5, REALTIME_SPEECH_SPEED_DEFAULT = 1;
|
|
507
|
+
function clampRealtimeSpeechSpeed(speed) {
|
|
508
|
+
return Math.round(Math.min(REALTIME_SPEECH_SPEED_MAX, Math.max(REALTIME_SPEECH_SPEED_MIN, speed)) * 100) / 100;
|
|
509
|
+
}
|
|
510
|
+
function loadRealtimeAudioPrefs() {
|
|
511
|
+
var prefs = {
|
|
512
|
+
bargeIn: true,
|
|
513
|
+
pushToTalk: false,
|
|
514
|
+
volume: 1,
|
|
515
|
+
micDeviceId: "",
|
|
516
|
+
outputDeviceId: "",
|
|
517
|
+
language: "",
|
|
518
|
+
speechSpeed: null,
|
|
519
|
+
version: 0
|
|
520
|
+
};
|
|
521
|
+
try {
|
|
522
|
+
var raw = window.localStorage.getItem("coreai.realtime.audioPrefs");
|
|
523
|
+
if (raw) {
|
|
524
|
+
var parsed = JSON.parse(raw);
|
|
525
|
+
if (typeof parsed.bargeIn === "boolean") prefs.bargeIn = parsed.bargeIn;
|
|
526
|
+
if (typeof parsed.pushToTalk === "boolean") prefs.pushToTalk = parsed.pushToTalk;
|
|
527
|
+
if (typeof parsed.volume === "number" && isFinite(parsed.volume)) prefs.volume = Math.min(1, Math.max(0, parsed.volume));
|
|
528
|
+
if (typeof parsed.micDeviceId === "string") prefs.micDeviceId = parsed.micDeviceId;
|
|
529
|
+
if (typeof parsed.outputDeviceId === "string") prefs.outputDeviceId = parsed.outputDeviceId;
|
|
530
|
+
if (typeof parsed.language === "string") prefs.language = parsed.language;
|
|
531
|
+
if (typeof parsed.speechSpeed === "number" && isFinite(parsed.speechSpeed)) prefs.speechSpeed = clampRealtimeSpeechSpeed(parsed.speechSpeed);
|
|
532
|
+
if (typeof parsed.version === "number") prefs.version = parsed.version;
|
|
533
|
+
}
|
|
534
|
+
} catch (err) {}
|
|
535
|
+
return repairRealtimeAudioPrefs(prefs);
|
|
536
|
+
}
|
|
537
|
+
function repairRealtimeAudioPrefs(prefs) {
|
|
538
|
+
if (prefs.version === REALTIME_PREFS_VERSION) return prefs;
|
|
539
|
+
prefs.version = REALTIME_PREFS_VERSION;
|
|
540
|
+
prefs.bargeIn = true;
|
|
541
|
+
prefs.pushToTalk = false;
|
|
542
|
+
saveRealtimeAudioPrefs(prefs);
|
|
543
|
+
return prefs;
|
|
544
|
+
}
|
|
545
|
+
function saveRealtimeAudioPrefs(prefs) {
|
|
546
|
+
try {
|
|
547
|
+
window.localStorage.setItem("coreai.realtime.audioPrefs", JSON.stringify(prefs));
|
|
548
|
+
} catch (err) {}
|
|
549
|
+
}
|
|
550
|
+
function applyRealtimeAudioPrefs(prefs) {
|
|
551
|
+
realtimeBargeIn = prefs.bargeIn !== false;
|
|
552
|
+
realtimePushToTalk = !!prefs.pushToTalk;
|
|
553
|
+
realtimeVolume = typeof prefs.volume === "number" ? prefs.volume : 1;
|
|
554
|
+
realtimeMicDeviceId = prefs.micDeviceId || "";
|
|
555
|
+
realtimeOutputDeviceId = prefs.outputDeviceId || "";
|
|
556
|
+
realtimeLanguage = prefs.language || "";
|
|
557
|
+
realtimeSpeechSpeed = typeof prefs.speechSpeed === "number" ? prefs.speechSpeed : null;
|
|
558
|
+
syncRealtimeGateMode();
|
|
559
|
+
if (realtimeGain) realtimeGain.gain.value = realtimeVolume;
|
|
560
|
+
if (realtimeRemoteAudioEl) realtimeRemoteAudioEl.volume = Math.max(0, Math.min(1, realtimeVolume));
|
|
561
|
+
}
|
|
562
|
+
function isEditableTarget(target) {
|
|
563
|
+
if (!target || !target.tagName) return false;
|
|
564
|
+
var tag = target.tagName.toLowerCase();
|
|
565
|
+
return tag === "input" || tag === "textarea" || tag === "select" || target.isContentEditable === true;
|
|
566
|
+
}
|
|
567
|
+
function attachRealtimePushToTalk() {
|
|
568
|
+
if (realtimePttBound) return;
|
|
569
|
+
realtimePttBound = true;
|
|
570
|
+
realtimePttActive = false;
|
|
571
|
+
realtimePttKeyDown = function(e) {
|
|
572
|
+
if (realtimePushToTalk && !isEditableTarget(e.target) && (e.code === "Space" || e.key === " ")) {
|
|
573
|
+
e.preventDefault();
|
|
574
|
+
if (!e.repeat) setRealtimePttActive(true);
|
|
575
|
+
}
|
|
576
|
+
};
|
|
577
|
+
realtimePttKeyUp = function(e) {
|
|
578
|
+
if (realtimePushToTalk && !isEditableTarget(e.target) && (e.code === "Space" || e.key === " ")) {
|
|
579
|
+
e.preventDefault();
|
|
580
|
+
setRealtimePttActive(false);
|
|
581
|
+
}
|
|
582
|
+
};
|
|
583
|
+
document.addEventListener("keydown", realtimePttKeyDown, true);
|
|
584
|
+
document.addEventListener("keyup", realtimePttKeyUp, true);
|
|
585
|
+
}
|
|
586
|
+
function detachRealtimePushToTalk() {
|
|
587
|
+
if (!realtimePttBound) return;
|
|
588
|
+
realtimePttBound = false;
|
|
589
|
+
realtimePttActive = false;
|
|
590
|
+
try {
|
|
591
|
+
document.removeEventListener("keydown", realtimePttKeyDown, true);
|
|
592
|
+
} catch (err) {}
|
|
593
|
+
try {
|
|
594
|
+
document.removeEventListener("keyup", realtimePttKeyUp, true);
|
|
595
|
+
} catch (err) {}
|
|
596
|
+
}
|
|
597
|
+
function setRealtimePttActive(active) {
|
|
598
|
+
realtimePttActive = active;
|
|
599
|
+
syncRealtimeGateMode();
|
|
600
|
+
if (realtimePttButton) {
|
|
601
|
+
realtimePttButton.classList.toggle("btn-danger", active);
|
|
602
|
+
realtimePttButton.classList.toggle("btn-outline-primary", !active);
|
|
603
|
+
realtimePttButton.innerHTML = active ? "<i class=\"fa-solid fa-microphone me-1\"></i> Listening…" : "<i class=\"fa-solid fa-microphone me-1\"></i> Hold to talk";
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
function buildRealtimePttUi() {
|
|
607
|
+
var realtimeBtn = q("realtimeButton");
|
|
608
|
+
if (!realtimeBtn || realtimePttUiEl) return;
|
|
609
|
+
var coarse = window.matchMedia && window.matchMedia("(pointer: coarse)").matches;
|
|
610
|
+
var bar = document.createElement("div");
|
|
611
|
+
bar.className = "text-center mt-2";
|
|
612
|
+
var btn = document.createElement("button");
|
|
613
|
+
btn.type = "button";
|
|
614
|
+
btn.className = "btn btn-outline-primary btn-sm";
|
|
615
|
+
btn.innerHTML = "<i class=\"fa-solid fa-microphone me-1\"></i> Hold to talk";
|
|
616
|
+
var hint = document.createElement("div");
|
|
617
|
+
hint.className = "form-text mt-1 mb-0";
|
|
618
|
+
hint.innerHTML = coarse ? "Push-to-talk is on — press and hold the button to speak." : "Push-to-talk is on — hold <kbd>Space</kbd> or this button to speak.";
|
|
619
|
+
bar.appendChild(btn);
|
|
620
|
+
bar.appendChild(hint);
|
|
621
|
+
var row = realtimeBtn.parentElement;
|
|
622
|
+
if (row && row.parentElement) row.parentElement.appendChild(bar);
|
|
623
|
+
else realtimeBtn.insertAdjacentElement("afterend", bar);
|
|
624
|
+
realtimePttButton = btn;
|
|
625
|
+
realtimePttUiEl = bar;
|
|
626
|
+
btn.addEventListener("pointerdown", function(e) {
|
|
627
|
+
e.preventDefault();
|
|
628
|
+
setRealtimePttActive(true);
|
|
629
|
+
});
|
|
630
|
+
btn.addEventListener("pointerup", function() {
|
|
631
|
+
setRealtimePttActive(false);
|
|
632
|
+
});
|
|
633
|
+
btn.addEventListener("pointerleave", function() {
|
|
634
|
+
setRealtimePttActive(false);
|
|
635
|
+
});
|
|
636
|
+
btn.addEventListener("pointercancel", function() {
|
|
637
|
+
setRealtimePttActive(false);
|
|
638
|
+
});
|
|
639
|
+
setRealtimePttActive(false);
|
|
640
|
+
}
|
|
641
|
+
function removeRealtimePttUi() {
|
|
642
|
+
realtimePttButton = null;
|
|
643
|
+
if (realtimePttUiEl) {
|
|
644
|
+
try {
|
|
645
|
+
realtimePttUiEl.remove();
|
|
646
|
+
} catch (err) {}
|
|
647
|
+
realtimePttUiEl = null;
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
function setupRealtimeAudioSettings() {
|
|
651
|
+
var realtimeBtn = q("realtimeButton");
|
|
652
|
+
if (!realtimeBtn || realtimeAudioSettingsBuilt) return;
|
|
653
|
+
realtimeAudioSettingsBuilt = true;
|
|
654
|
+
var prefs = loadRealtimeAudioPrefs();
|
|
655
|
+
applyRealtimeAudioPrefs(prefs);
|
|
656
|
+
var wrap = document.createElement("span");
|
|
657
|
+
wrap.id = "realtime-audio-settings";
|
|
658
|
+
wrap.style.position = "relative";
|
|
659
|
+
wrap.style.display = "inline-flex";
|
|
660
|
+
var gear = document.createElement("button");
|
|
661
|
+
gear.type = "button";
|
|
662
|
+
gear.className = "btn btn-outline-secondary";
|
|
663
|
+
gear.title = localize("settingsTitle", "Voice settings");
|
|
664
|
+
gear.setAttribute("aria-label", localize("settingsTitle", "Voice settings"));
|
|
665
|
+
gear.innerHTML = "<i class=\"fa-solid fa-headset\"></i>";
|
|
666
|
+
var langOptions = "<option value=\"\"" + (prefs.language === "" ? " selected" : "") + ">" + localize("languageAuto", "Automatic") + "</option>" + [
|
|
667
|
+
["en", "English"],
|
|
668
|
+
["es", "Spanish"],
|
|
669
|
+
["fr", "French"],
|
|
670
|
+
["de", "German"],
|
|
671
|
+
["it", "Italian"],
|
|
672
|
+
["pt", "Portuguese"],
|
|
673
|
+
["nl", "Dutch"],
|
|
674
|
+
["zh", "Chinese"],
|
|
675
|
+
["ja", "Japanese"],
|
|
676
|
+
["ko", "Korean"],
|
|
677
|
+
["ar", "Arabic"],
|
|
678
|
+
["hi", "Hindi"],
|
|
679
|
+
["ru", "Russian"]
|
|
680
|
+
].map(function(l) {
|
|
681
|
+
return "<option value=\"" + l[0] + "\"" + (prefs.language === l[0] ? " selected" : "") + ">" + l[1] + "</option>";
|
|
682
|
+
}).join("");
|
|
683
|
+
var panel = document.createElement("div");
|
|
684
|
+
panel.className = "card shadow-sm";
|
|
685
|
+
panel.style.cssText = "position:absolute;bottom:calc(100% + 6px);right:0;z-index:1080;width:290px;max-height:70vh;overflow:auto;display:none;";
|
|
686
|
+
panel.innerHTML = "<div class=\"card-body p-3\"><div class=\"fw-semibold mb-2\" style=\"font-size:0.85rem;\">" + localize("settingsTitle", "Voice settings") + "</div><label class=\"form-label mb-1 d-block\" style=\"font-size:0.8rem;\">" + localize("microphone", "Microphone") + "</label><select class=\"form-select form-select-sm js-mic mb-2\"><option value=\"\">" + localize("defaultMicrophone", "Default microphone") + "</option></select><label class=\"form-label mb-1 d-block\" style=\"font-size:0.8rem;\">" + localize("speaker", "Speaker") + "</label><select class=\"form-select form-select-sm js-out mb-1\"><option value=\"\">" + localize("automatic", "Automatic") + "</option></select><button type=\"button\" class=\"btn btn-outline-secondary btn-sm mb-2 js-pick-out\" style=\"display:none;\">" + localize("chooseSpeaker", "Choose speaker…") + "</button><div class=\"form-text mb-2 js-out-help\">" + localize("speakerHelp", "Automatic uses the device your system uses for calls. If you cannot hear the assistant, pick the speakers you are actually listening to.") + "</div><label class=\"form-label mb-1 d-block\" style=\"font-size:0.8rem;\">" + localize("assistantVolume", "Assistant volume") + ": <strong class=\"js-vol-val\">" + Math.round(prefs.volume * 100) + "</strong>%</label><input type=\"range\" class=\"form-range js-vol mb-2\" min=\"0\" max=\"100\" step=\"5\" value=\"" + Math.round(prefs.volume * 100) + "\"><div class=\"js-speed-row\" hidden><label class=\"form-label mb-1 d-flex align-items-baseline\" style=\"font-size:0.8rem;\"><span>" + localize("speechSpeed", "Speaking speed") + ": <strong class=\"js-speed-val\">1.00</strong>×</span><button type=\"button\" class=\"btn btn-link btn-sm p-0 ms-auto js-speed-reset\" style=\"font-size:0.8rem;\" hidden>" + localize("speechSpeedReset", "Reset") + "</button></label><input type=\"range\" class=\"form-range js-speed mb-0\" min=\"" + REALTIME_SPEECH_SPEED_MIN + "\" max=\"" + REALTIME_SPEECH_SPEED_MAX + "\" step=\"0.05\" value=\"" + REALTIME_SPEECH_SPEED_DEFAULT + "\"><div class=\"form-text mt-0 mb-2\">" + localize("speechSpeedHelp", "Applies from the assistant's next reply.") + "</div></div><label class=\"form-label mb-1 d-block\" style=\"font-size:0.8rem;\">" + localize("language", "Language") + "</label><select class=\"form-select form-select-sm js-lang mb-2\">" + langOptions + "</select><div class=\"form-check form-switch mb-1\"><input class=\"form-check-input js-barge\" type=\"checkbox\" id=\"realtime-setting-barge\"" + (prefs.bargeIn ? " checked" : "") + "><label class=\"form-check-label\" for=\"realtime-setting-barge\">" + localize("allowInterruptions", "Allow interruptions") + "</label></div><div class=\"form-text mb-2\">" + localize("allowInterruptionsHelp", "Talk over the assistant to interrupt it. Turn this off if the assistant keeps hearing itself through your speakers.") + "</div><div class=\"form-check form-switch mb-1\"><input class=\"form-check-input js-ptt\" type=\"checkbox\" id=\"realtime-setting-ptt\"" + (prefs.pushToTalk ? " checked" : "") + "><label class=\"form-check-label\" for=\"realtime-setting-ptt\">" + localize("pushToTalk", "Push-to-talk") + "</label></div><div class=\"form-text\">" + localize("pushToTalkHelp", "Hold <kbd>Space</kbd> (or the button) to talk. For very noisy places.") + "</div></div>";
|
|
687
|
+
wrap.appendChild(gear);
|
|
688
|
+
wrap.appendChild(panel);
|
|
689
|
+
realtimeBtn.insertAdjacentElement("afterend", wrap);
|
|
690
|
+
realtimeAudioSettingsElement = wrap;
|
|
691
|
+
var bargeInput = panel.querySelector(".js-barge");
|
|
692
|
+
var pttInput = panel.querySelector(".js-ptt");
|
|
693
|
+
var volInput = panel.querySelector(".js-vol");
|
|
694
|
+
var volVal = panel.querySelector(".js-vol-val");
|
|
695
|
+
var micSelect = panel.querySelector(".js-mic");
|
|
696
|
+
var outSelect = panel.querySelector(".js-out");
|
|
697
|
+
var outPickButton = panel.querySelector(".js-pick-out");
|
|
698
|
+
var langSelect = panel.querySelector(".js-lang");
|
|
699
|
+
var speedInput = panel.querySelector(".js-speed");
|
|
700
|
+
var speedVal = panel.querySelector(".js-speed-val");
|
|
701
|
+
var speedReset = panel.querySelector(".js-speed-reset");
|
|
702
|
+
function readPanelPrefs() {
|
|
703
|
+
return {
|
|
704
|
+
bargeIn: bargeInput.checked,
|
|
705
|
+
pushToTalk: pttInput.checked,
|
|
706
|
+
volume: (parseInt(volInput.value, 10) || 0) / 100,
|
|
707
|
+
micDeviceId: micSelect.value || "",
|
|
708
|
+
outputDeviceId: outSelect.value || "",
|
|
709
|
+
language: langSelect.value || "",
|
|
710
|
+
speechSpeed: realtimeSpeechSpeed,
|
|
711
|
+
version: REALTIME_PREFS_VERSION
|
|
712
|
+
};
|
|
713
|
+
}
|
|
714
|
+
function persist() {
|
|
715
|
+
var next = readPanelPrefs();
|
|
716
|
+
applyRealtimeAudioPrefs(next);
|
|
717
|
+
saveRealtimeAudioPrefs(next);
|
|
718
|
+
pushRealtimeSettingsToServer();
|
|
719
|
+
}
|
|
720
|
+
function persistSpeechSpeed(speed) {
|
|
721
|
+
realtimeSpeechSpeed = speed;
|
|
722
|
+
saveRealtimeAudioPrefs(readPanelPrefs());
|
|
723
|
+
syncRealtimeSpeechSpeedControl();
|
|
724
|
+
pushRealtimeSpeechSpeedToServer();
|
|
725
|
+
}
|
|
726
|
+
function populateDevices() {
|
|
727
|
+
if (!navigator.mediaDevices || !navigator.mediaDevices.enumerateDevices) return;
|
|
728
|
+
navigator.mediaDevices.enumerateDevices().then(function(devices) {
|
|
729
|
+
var options = ["<option value=\"\">" + localize("defaultMicrophone", "Default microphone") + "</option>"];
|
|
730
|
+
devices.filter(function(d) {
|
|
731
|
+
return d.kind === "audioinput";
|
|
732
|
+
}).forEach(function(d, i) {
|
|
733
|
+
var label = d.label || localize("microphone", "Microphone") + " " + (i + 1);
|
|
734
|
+
options.push("<option value=\"" + d.deviceId + "\"" + (d.deviceId === realtimeMicDeviceId ? " selected" : "") + ">" + label.replace(/</g, "") + "</option>");
|
|
735
|
+
});
|
|
736
|
+
micSelect.innerHTML = options.join("");
|
|
737
|
+
var outs = devices.filter(function(d) {
|
|
738
|
+
return d.kind === "audiooutput" && d.deviceId && d.deviceId !== "communications" && d.deviceId !== "default";
|
|
739
|
+
});
|
|
740
|
+
if (outs.length) {
|
|
741
|
+
var outOptions = ["<option value=\"\">" + localize("automatic", "Automatic") + "</option>"];
|
|
742
|
+
outs.forEach(function(d, i) {
|
|
743
|
+
var label = d.label || localize("speaker", "Speaker") + " " + (i + 1);
|
|
744
|
+
outOptions.push("<option value=\"" + d.deviceId + "\"" + (d.deviceId === realtimeOutputDeviceId ? " selected" : "") + ">" + label.replace(/</g, "") + "</option>");
|
|
745
|
+
});
|
|
746
|
+
outSelect.innerHTML = outOptions.join("");
|
|
747
|
+
}
|
|
748
|
+
}).catch(function() {});
|
|
749
|
+
}
|
|
750
|
+
populateDevices();
|
|
751
|
+
if (navigator.mediaDevices && typeof navigator.mediaDevices.addEventListener === "function") navigator.mediaDevices.addEventListener("devicechange", function() {
|
|
752
|
+
populateDevices();
|
|
753
|
+
if (!isRealtimeActive || !realtimeMicDeviceId) return;
|
|
754
|
+
navigator.mediaDevices.enumerateDevices().then(function(devices) {
|
|
755
|
+
if (!devices.some(function(d) {
|
|
756
|
+
return d.kind === "audioinput" && d.deviceId === realtimeMicDeviceId;
|
|
757
|
+
})) endRealtimeSessionFromServer("device_lost");
|
|
758
|
+
}).catch(function() {});
|
|
759
|
+
});
|
|
760
|
+
function constrainPanel() {
|
|
761
|
+
panel.style.right = "0px";
|
|
762
|
+
panel.style.left = "auto";
|
|
763
|
+
var panelRect = panel.getBoundingClientRect();
|
|
764
|
+
if (panelRect.left < 8) panel.style.right = panelRect.left - 8 + "px";
|
|
765
|
+
var rect = gear.getBoundingClientRect();
|
|
766
|
+
var boundaryTop = 8;
|
|
767
|
+
var node = panel.parentElement ? panel.parentElement.parentElement : null;
|
|
768
|
+
while (node && node !== document.body && node !== document.documentElement) {
|
|
769
|
+
var s = window.getComputedStyle(node);
|
|
770
|
+
if (s.overflowY === "auto" || s.overflowY === "scroll" || s.overflowY === "hidden" || s.position === "fixed") {
|
|
771
|
+
boundaryTop = Math.max(boundaryTop, node.getBoundingClientRect().top);
|
|
772
|
+
break;
|
|
773
|
+
}
|
|
774
|
+
node = node.parentElement;
|
|
775
|
+
}
|
|
776
|
+
panel.style.maxHeight = Math.max(160, rect.top - boundaryTop - 10) + "px";
|
|
777
|
+
}
|
|
778
|
+
gear.addEventListener("click", function(e) {
|
|
779
|
+
e.stopPropagation();
|
|
780
|
+
var showing = panel.style.display === "none";
|
|
781
|
+
if (showing) {
|
|
782
|
+
panel.style.display = "block";
|
|
783
|
+
constrainPanel();
|
|
784
|
+
populateDevices();
|
|
785
|
+
}
|
|
786
|
+
panel.style.display = showing ? "block" : "none";
|
|
787
|
+
});
|
|
788
|
+
panel.addEventListener("click", function(e) {
|
|
789
|
+
e.stopPropagation();
|
|
790
|
+
});
|
|
791
|
+
document.addEventListener("click", function() {
|
|
792
|
+
panel.style.display = "none";
|
|
793
|
+
});
|
|
794
|
+
bargeInput.addEventListener("change", persist);
|
|
795
|
+
pttInput.addEventListener("change", persist);
|
|
796
|
+
volInput.addEventListener("input", function() {
|
|
797
|
+
volVal.textContent = volInput.value;
|
|
798
|
+
persist();
|
|
799
|
+
});
|
|
800
|
+
speedInput.addEventListener("input", function() {
|
|
801
|
+
speedVal.textContent = formatRealtimeSpeechSpeed(parseFloat(speedInput.value));
|
|
802
|
+
});
|
|
803
|
+
speedInput.addEventListener("change", function() {
|
|
804
|
+
persistSpeechSpeed(clampRealtimeSpeechSpeed(parseFloat(speedInput.value)));
|
|
805
|
+
});
|
|
806
|
+
speedReset.addEventListener("click", function() {
|
|
807
|
+
persistSpeechSpeed(null);
|
|
808
|
+
});
|
|
809
|
+
micSelect.addEventListener("change", persist);
|
|
810
|
+
langSelect.addEventListener("change", persist);
|
|
811
|
+
outSelect.addEventListener("change", function() {
|
|
812
|
+
persist();
|
|
813
|
+
applyRealtimeOutputDevice();
|
|
814
|
+
});
|
|
815
|
+
if (navigator.mediaDevices && typeof navigator.mediaDevices.selectAudioOutput === "function") {
|
|
816
|
+
outPickButton.style.display = "";
|
|
817
|
+
outPickButton.addEventListener("click", function() {
|
|
818
|
+
navigator.mediaDevices.selectAudioOutput().then(function(device) {
|
|
819
|
+
if (!device || !device.deviceId) return;
|
|
820
|
+
var option = document.createElement("option");
|
|
821
|
+
option.value = device.deviceId;
|
|
822
|
+
option.textContent = device.label || localize("selectedSpeaker", "Selected speaker");
|
|
823
|
+
outSelect.appendChild(option);
|
|
824
|
+
outSelect.value = device.deviceId;
|
|
825
|
+
persist();
|
|
826
|
+
applyRealtimeOutputDevice();
|
|
827
|
+
}).catch(function() {});
|
|
828
|
+
});
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
function setRealtimeButtonHtml(button, html) {
|
|
832
|
+
if (!html) return;
|
|
833
|
+
if (window.DOMPurify && typeof window.DOMPurify.sanitize === "function") {
|
|
834
|
+
button.replaceChildren(window.DOMPurify.sanitize(html, { RETURN_DOM_FRAGMENT: true }));
|
|
835
|
+
return;
|
|
836
|
+
}
|
|
837
|
+
var text = html.replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim();
|
|
838
|
+
button.replaceChildren(document.createTextNode(text));
|
|
839
|
+
}
|
|
840
|
+
function updateRealtimeButton() {
|
|
841
|
+
onSessionStateChanged(isRealtimeActive);
|
|
842
|
+
var realtimeBtn = q("realtimeButton");
|
|
843
|
+
if (!realtimeBtn) return;
|
|
844
|
+
if (isRealtimeActive) {
|
|
845
|
+
realtimeBtn.classList.add("active", "btn-primary");
|
|
846
|
+
realtimeBtn.classList.remove("btn-dark", "btn-outline-dark", "btn-outline-secondary");
|
|
847
|
+
realtimeBtn.title = realtimeBtn.getAttribute("data-end-title") || "End Conversation";
|
|
848
|
+
setRealtimeButtonHtml(realtimeBtn, realtimeBtn.getAttribute("data-end-html"));
|
|
849
|
+
} else {
|
|
850
|
+
realtimeBtn.classList.remove("active", "btn-primary", "btn-dark", "btn-outline-secondary");
|
|
851
|
+
realtimeBtn.classList.add("btn-outline-dark");
|
|
852
|
+
realtimeBtn.blur();
|
|
853
|
+
realtimeBtn.title = realtimeBtn.getAttribute("data-start-title") || "Start speaking";
|
|
854
|
+
setRealtimeButtonHtml(realtimeBtn, realtimeBtn.getAttribute("data-start-html"));
|
|
855
|
+
}
|
|
856
|
+
}
|
|
857
|
+
function applyRealtimeMode(enable) {
|
|
858
|
+
isRealtimeMode = enable;
|
|
859
|
+
var realtimeBtn = q("realtimeButton");
|
|
860
|
+
var micBtn = q("micButton");
|
|
861
|
+
var conversationBtn = q("conversationButton");
|
|
862
|
+
var inputEl = q("input");
|
|
863
|
+
var sendBtn = q("sendButton");
|
|
864
|
+
var audioSettings = realtimeAudioSettingsElement;
|
|
865
|
+
if (enable) {
|
|
866
|
+
onEnterRealtimeMode();
|
|
867
|
+
if (inputEl) inputEl.hidden = true;
|
|
868
|
+
if (sendBtn) sendBtn.hidden = true;
|
|
869
|
+
if (micBtn) micBtn.hidden = true;
|
|
870
|
+
if (conversationBtn) conversationBtn.hidden = true;
|
|
871
|
+
if (realtimeBtn) realtimeBtn.hidden = false;
|
|
872
|
+
if (audioSettings) audioSettings.hidden = false;
|
|
873
|
+
} else {
|
|
874
|
+
if (isRealtimeActive) stopRealtimeConversation();
|
|
875
|
+
if (realtimeBtn) realtimeBtn.hidden = true;
|
|
876
|
+
if (audioSettings) audioSettings.hidden = true;
|
|
877
|
+
removeCompatibilityModeNotice();
|
|
878
|
+
clearRealtimeStatus();
|
|
879
|
+
if (inputEl) inputEl.hidden = false;
|
|
880
|
+
if (sendBtn) sendBtn.hidden = false;
|
|
881
|
+
if (micBtn) micBtn.hidden = false;
|
|
882
|
+
if (conversationBtn) conversationBtn.hidden = false;
|
|
883
|
+
}
|
|
884
|
+
updateRealtimeButton();
|
|
885
|
+
}
|
|
886
|
+
function pushRealtimeSettingsToServer() {
|
|
887
|
+
if (!isRealtimeActive || !connection || typeof connection.send !== "function") return;
|
|
888
|
+
try {
|
|
889
|
+
connection.send("UpdateRealtimeSettings", realtimeBargeIn, null, null);
|
|
890
|
+
} catch (err) {}
|
|
891
|
+
}
|
|
892
|
+
function formatRealtimeSpeechSpeed(speed) {
|
|
893
|
+
return (isFinite(speed) ? speed : REALTIME_SPEECH_SPEED_DEFAULT).toFixed(2);
|
|
894
|
+
}
|
|
895
|
+
function effectiveRealtimeSpeechSpeed() {
|
|
896
|
+
return realtimeSpeechSpeed !== null ? realtimeSpeechSpeed : realtimeSessionSpeechSpeed;
|
|
897
|
+
}
|
|
898
|
+
function syncRealtimeSpeechSpeedControl() {
|
|
899
|
+
if (!realtimeAudioSettingsElement) return;
|
|
900
|
+
var row = realtimeAudioSettingsElement.querySelector(".js-speed-row");
|
|
901
|
+
var input = realtimeAudioSettingsElement.querySelector(".js-speed");
|
|
902
|
+
var value = realtimeAudioSettingsElement.querySelector(".js-speed-val");
|
|
903
|
+
var reset = realtimeAudioSettingsElement.querySelector(".js-speed-reset");
|
|
904
|
+
if (!row || !input || !value || !reset) return;
|
|
905
|
+
var speed = effectiveRealtimeSpeechSpeed();
|
|
906
|
+
row.hidden = !realtimeSpeechSpeedAdjustable;
|
|
907
|
+
input.value = String(speed);
|
|
908
|
+
value.textContent = formatRealtimeSpeechSpeed(speed);
|
|
909
|
+
reset.hidden = realtimeSpeechSpeed === null;
|
|
910
|
+
}
|
|
911
|
+
function pushRealtimeSpeechSpeedToServer() {
|
|
912
|
+
if (!isRealtimeActive || !realtimeSpeechSpeedAdjustable || !connection || typeof connection.send !== "function") return;
|
|
913
|
+
try {
|
|
914
|
+
connection.send("UpdateRealtimeSpeechSpeed", effectiveRealtimeSpeechSpeed());
|
|
915
|
+
} catch (err) {}
|
|
916
|
+
}
|
|
917
|
+
function applySessionSpeechSpeed(payload) {
|
|
918
|
+
var speed = parseFloat(payload);
|
|
919
|
+
realtimeSpeechSpeedAdjustable = isFinite(speed);
|
|
920
|
+
realtimeSessionSpeechSpeed = realtimeSpeechSpeedAdjustable ? speed : REALTIME_SPEECH_SPEED_DEFAULT;
|
|
921
|
+
syncRealtimeSpeechSpeedControl();
|
|
922
|
+
if (realtimeSpeechSpeed !== null && realtimeSpeechSpeed !== realtimeSessionSpeechSpeed) pushRealtimeSpeechSpeedToServer();
|
|
923
|
+
}
|
|
924
|
+
var REALTIME_STATE_TEXT = {
|
|
925
|
+
"requesting-mic": localize("requestingMic", "Waiting for microphone access…"),
|
|
926
|
+
"connecting": localize("connecting", "Connecting…"),
|
|
927
|
+
"listening": localize("listening", "Listening"),
|
|
928
|
+
"playback-blocked": localize("playbackBlocked", "Audio is blocked by the browser — click the page to enable it.")
|
|
929
|
+
};
|
|
930
|
+
function setRealtimeState(state, detail) {
|
|
931
|
+
if (realtimeState === state) return;
|
|
932
|
+
realtimeState = state;
|
|
933
|
+
var text = REALTIME_STATE_TEXT[state];
|
|
934
|
+
if (text) showRealtimeStatus(text, false);
|
|
935
|
+
else if (state === "idle") clearRealtimeStatus();
|
|
936
|
+
try {
|
|
937
|
+
onStateChange({
|
|
938
|
+
state,
|
|
939
|
+
transport: realtimeIsWebRtc ? "webrtc" : "websocket",
|
|
940
|
+
detail: detail || null
|
|
941
|
+
});
|
|
942
|
+
} catch (err) {}
|
|
943
|
+
}
|
|
944
|
+
function bindRealtimeLifecycleHandlers() {
|
|
945
|
+
if (realtimeLifecycleBound || !connection) return;
|
|
946
|
+
realtimeLifecycleBound = true;
|
|
947
|
+
connection.on("ReceiveRealtimeEvent", function(identifier, type, payload) {
|
|
948
|
+
if (!isRealtimeActive) return;
|
|
949
|
+
if (type === "speech_speed") applySessionSpeechSpeed(payload);
|
|
950
|
+
else if (type === "session_ready") setRealtimeState("listening");
|
|
951
|
+
else if (type === "session_ended") endRealtimeSessionFromServer(payload || "completed");
|
|
952
|
+
else if (type === "speech_started") {
|
|
953
|
+
if (realtimeBargeIn) flushRealtimePlayback();
|
|
954
|
+
} else if (type === "playback_flush") flushRealtimePlayback();
|
|
955
|
+
else if (type === "user_turn_pending") onUserTurnPending(payload);
|
|
956
|
+
else if (type === "user_turn_dropped") onUserTurnDropped(payload);
|
|
957
|
+
});
|
|
958
|
+
connection.on("ReceiveError", function() {
|
|
959
|
+
if (!isRealtimeActive) return;
|
|
960
|
+
if (realtimeIsWebRtc && !realtimeWebRtcConnected && !realtimeFellBack) {
|
|
961
|
+
fallbackToWebSocket("server reported an error before the peer connected");
|
|
962
|
+
return;
|
|
963
|
+
}
|
|
964
|
+
endRealtimeSessionFromServer("error");
|
|
965
|
+
});
|
|
966
|
+
if (typeof connection.onclose === "function") connection.onclose(function() {
|
|
967
|
+
if (isRealtimeActive) endRealtimeSessionFromServer("disconnected");
|
|
968
|
+
});
|
|
969
|
+
if (typeof connection.onreconnecting === "function") connection.onreconnecting(function() {
|
|
970
|
+
if (isRealtimeActive) endRealtimeSessionFromServer("disconnected");
|
|
971
|
+
});
|
|
972
|
+
}
|
|
973
|
+
function endRealtimeSessionFromServer(reason) {
|
|
974
|
+
realtimeEndedNotice = reason;
|
|
975
|
+
stopRealtimeConversation();
|
|
976
|
+
setRealtimeState("ended", reason);
|
|
977
|
+
if (window.console && console.info) console.info("The realtime voice session ended (" + reason + ").");
|
|
978
|
+
if (reason === "idle") showRealtimeStatus(localize("endedIdle", "Voice paused after a period of silence."), true);
|
|
979
|
+
else if (reason === "max_duration") showRealtimeStatus(localize("endedMaxDuration", "Voice paused — this session reached its time limit. You can start another."), true);
|
|
980
|
+
else if (reason === "disconnected") showRealtimeStatus(localize("endedDisconnected", "Voice session ended — the connection dropped."), true);
|
|
981
|
+
else if (reason === "error") showRealtimeStatus(localize("endedError", "Voice session ended unexpectedly."), true);
|
|
982
|
+
else if (reason === "device_lost") showRealtimeStatus(localize("endedDeviceLost", "Voice session ended — the microphone was disconnected."), true);
|
|
983
|
+
}
|
|
984
|
+
function showRealtimeStatus(text, offerResume) {
|
|
985
|
+
var realtimeBtn = q("realtimeButton");
|
|
986
|
+
if (!realtimeBtn) return;
|
|
987
|
+
if (!realtimeStatusEl) {
|
|
988
|
+
realtimeStatusEl = document.createElement("div");
|
|
989
|
+
realtimeStatusEl.className = "form-text mt-2 mb-0 text-center";
|
|
990
|
+
realtimeStatusEl.setAttribute("role", "status");
|
|
991
|
+
realtimeStatusEl.setAttribute("aria-live", "polite");
|
|
992
|
+
var row = realtimeBtn.parentElement;
|
|
993
|
+
if (row && row.parentElement) row.parentElement.appendChild(realtimeStatusEl);
|
|
994
|
+
else realtimeBtn.insertAdjacentElement("afterend", realtimeStatusEl);
|
|
995
|
+
}
|
|
996
|
+
realtimeStatusEl.replaceChildren();
|
|
997
|
+
realtimeStatusEl.appendChild(document.createTextNode(text));
|
|
998
|
+
if (offerResume) {
|
|
999
|
+
var resume = document.createElement("button");
|
|
1000
|
+
resume.type = "button";
|
|
1001
|
+
resume.className = "btn btn-link btn-sm p-0 ms-2 align-baseline";
|
|
1002
|
+
resume.textContent = localize("resume", "Resume");
|
|
1003
|
+
resume.addEventListener("click", function() {
|
|
1004
|
+
clearRealtimeStatus();
|
|
1005
|
+
startRealtimeConversation();
|
|
1006
|
+
});
|
|
1007
|
+
realtimeStatusEl.appendChild(resume);
|
|
1008
|
+
}
|
|
1009
|
+
}
|
|
1010
|
+
function clearRealtimeStatus() {
|
|
1011
|
+
if (!realtimeStatusEl) return;
|
|
1012
|
+
try {
|
|
1013
|
+
realtimeStatusEl.remove();
|
|
1014
|
+
} catch (err) {}
|
|
1015
|
+
realtimeStatusEl = null;
|
|
1016
|
+
}
|
|
1017
|
+
function resolveIceServers() {
|
|
1018
|
+
if (webRtcIceServers) return Promise.resolve(webRtcIceServers);
|
|
1019
|
+
if (realtimeAttemptIceServers) return Promise.resolve(realtimeAttemptIceServers);
|
|
1020
|
+
if (!connection || typeof connection.invoke !== "function") return Promise.resolve(DEFAULT_ICE_SERVERS);
|
|
1021
|
+
return connection.invoke("GetRealtimeIceServers").then(function(servers) {
|
|
1022
|
+
realtimeAttemptIceServers = Array.isArray(servers) && servers.length ? servers : DEFAULT_ICE_SERVERS;
|
|
1023
|
+
return realtimeAttemptIceServers;
|
|
1024
|
+
}).catch(function(err) {
|
|
1025
|
+
if (window.console && console.warn) console.warn("Could not resolve the realtime ICE servers; using the default STUN server.", err);
|
|
1026
|
+
realtimeAttemptIceServers = DEFAULT_ICE_SERVERS;
|
|
1027
|
+
return realtimeAttemptIceServers;
|
|
1028
|
+
});
|
|
1029
|
+
}
|
|
1030
|
+
function iceServersFingerprint(servers) {
|
|
1031
|
+
var urls = [];
|
|
1032
|
+
try {
|
|
1033
|
+
(servers || []).forEach(function(server) {
|
|
1034
|
+
var list = server && server.urls;
|
|
1035
|
+
if (typeof list === "string") urls.push(list);
|
|
1036
|
+
else if (Array.isArray(list)) list.forEach(function(url) {
|
|
1037
|
+
if (url) urls.push(url);
|
|
1038
|
+
});
|
|
1039
|
+
});
|
|
1040
|
+
} catch (err) {
|
|
1041
|
+
return "none";
|
|
1042
|
+
}
|
|
1043
|
+
return urls.length ? urls.sort().join("|") : "none";
|
|
1044
|
+
}
|
|
1045
|
+
function isWebRtcKnownBlocked(servers) {
|
|
1046
|
+
try {
|
|
1047
|
+
return window.sessionStorage.getItem(REALTIME_WEBRTC_BLOCKED_KEY) === iceServersFingerprint(servers);
|
|
1048
|
+
} catch (err) {
|
|
1049
|
+
return false;
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
function rememberWebRtcBlocked() {
|
|
1053
|
+
try {
|
|
1054
|
+
window.sessionStorage.setItem(REALTIME_WEBRTC_BLOCKED_KEY, iceServersFingerprint(realtimeAttemptIceServers || webRtcIceServers));
|
|
1055
|
+
} catch (err) {}
|
|
1056
|
+
}
|
|
1057
|
+
function showCompatibilityModeNotice() {
|
|
1058
|
+
if (realtimeCompatibilityNoticeEl) return;
|
|
1059
|
+
var realtimeBtn = q("realtimeButton");
|
|
1060
|
+
if (!realtimeBtn) return;
|
|
1061
|
+
var note = document.createElement("div");
|
|
1062
|
+
note.className = "form-text mt-2 mb-0 text-center";
|
|
1063
|
+
note.setAttribute("role", "status");
|
|
1064
|
+
note.textContent = localize("compatibilityMode", "Using compatibility audio mode. Echo cancellation may be weaker — headphones are recommended.");
|
|
1065
|
+
var row = realtimeBtn.parentElement;
|
|
1066
|
+
if (row && row.parentElement) row.parentElement.appendChild(note);
|
|
1067
|
+
else realtimeBtn.insertAdjacentElement("afterend", note);
|
|
1068
|
+
realtimeCompatibilityNoticeEl = note;
|
|
1069
|
+
}
|
|
1070
|
+
function removeCompatibilityModeNotice() {
|
|
1071
|
+
if (!realtimeCompatibilityNoticeEl) return;
|
|
1072
|
+
try {
|
|
1073
|
+
realtimeCompatibilityNoticeEl.remove();
|
|
1074
|
+
} catch (err) {}
|
|
1075
|
+
realtimeCompatibilityNoticeEl = null;
|
|
1076
|
+
}
|
|
1077
|
+
function startRealtimeConversation() {
|
|
1078
|
+
if (!isRealtimeMode || isRealtimeActive || !connection) return;
|
|
1079
|
+
applyRealtimeAudioPrefs(loadRealtimeAudioPrefs());
|
|
1080
|
+
realtimeAttemptIceServers = null;
|
|
1081
|
+
var attempt = ++realtimeAttemptToken;
|
|
1082
|
+
realtimeFellBack = false;
|
|
1083
|
+
realtimeSpeechSpeedAdjustable = false;
|
|
1084
|
+
syncRealtimeSpeechSpeedControl();
|
|
1085
|
+
realtimeEndedNotice = null;
|
|
1086
|
+
clearRealtimeStatus();
|
|
1087
|
+
bindRealtimeLifecycleHandlers();
|
|
1088
|
+
setRealtimeState("requesting-mic");
|
|
1089
|
+
if (!webRtcEnabled) {
|
|
1090
|
+
logWebSocketTransportReason("the server did not advertise the WebRTC transport");
|
|
1091
|
+
startRealtimeWebSocketConversation();
|
|
1092
|
+
return;
|
|
1093
|
+
}
|
|
1094
|
+
resolveIceServers().then(function(servers) {
|
|
1095
|
+
if (attempt !== realtimeAttemptToken || !isRealtimeMode || isRealtimeActive) return;
|
|
1096
|
+
if (isWebRtcKnownBlocked(servers)) {
|
|
1097
|
+
logWebSocketTransportReason("a WebRTC attempt already failed earlier in this browser session with the same ICE configuration");
|
|
1098
|
+
startRealtimeWebSocketConversation();
|
|
1099
|
+
return;
|
|
1100
|
+
}
|
|
1101
|
+
startRealtimeWebRtcConversation();
|
|
1102
|
+
});
|
|
1103
|
+
}
|
|
1104
|
+
function startRealtimeWebSocketConversation() {
|
|
1105
|
+
if (webRtcEnabled) showCompatibilityModeNotice();
|
|
1106
|
+
attachRealtimePushToTalk();
|
|
1107
|
+
var focusedBtn = q("realtimeButton");
|
|
1108
|
+
if (focusedBtn) try {
|
|
1109
|
+
focusedBtn.blur();
|
|
1110
|
+
} catch (err) {}
|
|
1111
|
+
var audioConstraints = {
|
|
1112
|
+
channelCount: 1,
|
|
1113
|
+
echoCancellation: { ideal: true },
|
|
1114
|
+
noiseSuppression: true,
|
|
1115
|
+
autoGainControl: true,
|
|
1116
|
+
echoCancellationType: { ideal: "system" },
|
|
1117
|
+
voiceIsolation: { ideal: true }
|
|
1118
|
+
};
|
|
1119
|
+
if (realtimeMicDeviceId) audioConstraints.deviceId = { exact: realtimeMicDeviceId };
|
|
1120
|
+
navigator.mediaDevices.getUserMedia({ audio: audioConstraints }).then(function(stream) {
|
|
1121
|
+
ensureConnected().then(function() {
|
|
1122
|
+
realtimeStream = stream;
|
|
1123
|
+
isRealtimeActive = true;
|
|
1124
|
+
onActivate();
|
|
1125
|
+
updateRealtimeButton();
|
|
1126
|
+
setRealtimeState("connecting");
|
|
1127
|
+
realtimeAudioCtx = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: REALTIME_SAMPLE_RATE });
|
|
1128
|
+
if (realtimeAudioCtx.state === "suspended" && typeof realtimeAudioCtx.resume === "function") realtimeAudioCtx.resume().catch(function(err) {
|
|
1129
|
+
setRealtimeState("playback-blocked");
|
|
1130
|
+
if (window.console && console.warn) console.warn("The browser blocked realtime audio playback. Interact with the page to enable audio.", err);
|
|
1131
|
+
});
|
|
1132
|
+
realtimePlayHead = 0;
|
|
1133
|
+
realtimeSources = [];
|
|
1134
|
+
realtimeGain = realtimeAudioCtx.createGain();
|
|
1135
|
+
realtimeGain.gain.value = realtimeVolume != null ? realtimeVolume : 1;
|
|
1136
|
+
realtimeGain.connect(realtimeAudioCtx.destination);
|
|
1137
|
+
realtimeSubject = new window.signalR.Subject();
|
|
1138
|
+
var ctxAtStart = realtimeAudioCtx;
|
|
1139
|
+
var zeroGain = realtimeAudioCtx.createGain();
|
|
1140
|
+
zeroGain.gain.value = 0;
|
|
1141
|
+
realtimeZeroGain = zeroGain;
|
|
1142
|
+
zeroGain.connect(realtimeAudioCtx.destination);
|
|
1143
|
+
var monitorDest = realtimeAudioCtx.createMediaStreamDestination();
|
|
1144
|
+
realtimeGain.connect(monitorDest);
|
|
1145
|
+
var sendCapturedFrame = function(input) {
|
|
1146
|
+
var muted;
|
|
1147
|
+
if (realtimePushToTalk) muted = !realtimePttActive;
|
|
1148
|
+
else muted = !realtimeBargeIn && realtimeAudioCtx && realtimeAudioCtx.currentTime < realtimePlayHead + REALTIME_WS_ECHO_HANGOVER_SEC;
|
|
1149
|
+
var pcm = new Int16Array(input.length);
|
|
1150
|
+
if (!muted) for (var i = 0; i < input.length; i++) {
|
|
1151
|
+
var s = Math.max(-1, Math.min(1, input[i]));
|
|
1152
|
+
pcm[i] = s < 0 ? s * 32768 : s * 32767;
|
|
1153
|
+
}
|
|
1154
|
+
var bytes = new Uint8Array(pcm.buffer);
|
|
1155
|
+
var binary = "";
|
|
1156
|
+
for (var b = 0; b < bytes.length; b++) binary += String.fromCharCode(bytes[b]);
|
|
1157
|
+
try {
|
|
1158
|
+
realtimeSubject.next(btoa(binary));
|
|
1159
|
+
} catch (err) {}
|
|
1160
|
+
};
|
|
1161
|
+
createRealtimeCaptureNode(realtimeAudioCtx, sendCapturedFrame).then(function(captureNode) {
|
|
1162
|
+
if (!isRealtimeActive || realtimeAudioCtx !== ctxAtStart) {
|
|
1163
|
+
try {
|
|
1164
|
+
captureNode.disconnect();
|
|
1165
|
+
} catch (err) {}
|
|
1166
|
+
return;
|
|
1167
|
+
}
|
|
1168
|
+
realtimeProcessor = captureNode;
|
|
1169
|
+
captureNode.connect(zeroGain);
|
|
1170
|
+
setupMicGate(stream).then(function(micTrack) {
|
|
1171
|
+
if (!isRealtimeActive || realtimeAudioCtx !== ctxAtStart || !micTrack) return;
|
|
1172
|
+
var gatedStream = new MediaStream([micTrack]);
|
|
1173
|
+
var source = realtimeAudioCtx.createMediaStreamSource(gatedStream);
|
|
1174
|
+
realtimeMicSource = source;
|
|
1175
|
+
source.connect(captureNode);
|
|
1176
|
+
if (realtimeGate) realtimeGate.attachAssistantStream(monitorDest.stream);
|
|
1177
|
+
});
|
|
1178
|
+
});
|
|
1179
|
+
var language = realtimeLanguage || null;
|
|
1180
|
+
var voice = getVoiceName && getVoiceName() || realtimeVoiceName || "";
|
|
1181
|
+
sendStart(realtimeSubject, voice, language, null, null, realtimeBargeIn);
|
|
1182
|
+
if (realtimePushToTalk) buildRealtimePttUi();
|
|
1183
|
+
}).catch(function(err) {
|
|
1184
|
+
stream.getTracks().forEach(function(track) {
|
|
1185
|
+
track.stop();
|
|
1186
|
+
});
|
|
1187
|
+
isRealtimeActive = false;
|
|
1188
|
+
onDeactivate();
|
|
1189
|
+
updateRealtimeButton();
|
|
1190
|
+
console.error("The realtime conversation could not start because the chat connection is not available.", err);
|
|
1191
|
+
});
|
|
1192
|
+
}).catch(function(err) {
|
|
1193
|
+
console.error("Microphone access denied:", err);
|
|
1194
|
+
isRealtimeActive = false;
|
|
1195
|
+
onDeactivate();
|
|
1196
|
+
updateRealtimeButton();
|
|
1197
|
+
});
|
|
1198
|
+
}
|
|
1199
|
+
function startRealtimeWebRtcConversation() {
|
|
1200
|
+
var micConstraints = {
|
|
1201
|
+
echoCancellation: true,
|
|
1202
|
+
noiseSuppression: true,
|
|
1203
|
+
autoGainControl: true
|
|
1204
|
+
};
|
|
1205
|
+
if (realtimeMicDeviceId) micConstraints.deviceId = { exact: realtimeMicDeviceId };
|
|
1206
|
+
navigator.mediaDevices.getUserMedia({ audio: micConstraints }).then(function(stream) {
|
|
1207
|
+
ensureConnected().then(function() {
|
|
1208
|
+
return resolveIceServers();
|
|
1209
|
+
}).then(function(iceServers) {
|
|
1210
|
+
realtimeStream = stream;
|
|
1211
|
+
isRealtimeActive = true;
|
|
1212
|
+
realtimeIsWebRtc = true;
|
|
1213
|
+
onActivate();
|
|
1214
|
+
updateRealtimeButton();
|
|
1215
|
+
setRealtimeState("connecting");
|
|
1216
|
+
realtimeRemoteDescriptionSet = false;
|
|
1217
|
+
realtimePendingIce = [];
|
|
1218
|
+
realtimeWebRtcConnected = false;
|
|
1219
|
+
var pc = new RTCPeerConnection({ iceServers });
|
|
1220
|
+
realtimePc = pc;
|
|
1221
|
+
attachRealtimePushToTalk();
|
|
1222
|
+
var focusedBtn = q("realtimeButton");
|
|
1223
|
+
if (focusedBtn) try {
|
|
1224
|
+
focusedBtn.blur();
|
|
1225
|
+
} catch (err) {}
|
|
1226
|
+
if (realtimePushToTalk) buildRealtimePttUi();
|
|
1227
|
+
var audioEl = document.createElement("audio");
|
|
1228
|
+
audioEl.autoplay = true;
|
|
1229
|
+
audioEl.style.display = "none";
|
|
1230
|
+
audioEl.volume = Math.max(0, Math.min(1, realtimeVolume != null ? realtimeVolume : 1));
|
|
1231
|
+
document.body.appendChild(audioEl);
|
|
1232
|
+
realtimeRemoteAudioEl = audioEl;
|
|
1233
|
+
routeRealtimeOutputToDefaultDevice(audioEl);
|
|
1234
|
+
pc.ontrack = function(e) {
|
|
1235
|
+
try {
|
|
1236
|
+
if (e.receiver && "jitterBufferTarget" in e.receiver) e.receiver.jitterBufferTarget = REALTIME_WEBRTC_JITTER_BUFFER_TARGET_MS;
|
|
1237
|
+
} catch (err) {}
|
|
1238
|
+
if (e.streams && e.streams[0]) {
|
|
1239
|
+
audioEl.srcObject = e.streams[0];
|
|
1240
|
+
ensureRealtimePlayback(audioEl);
|
|
1241
|
+
attachWebRtcAssistantAnalyser(e.streams[0]);
|
|
1242
|
+
}
|
|
1243
|
+
};
|
|
1244
|
+
realtimeSawRelayCandidate = false;
|
|
1245
|
+
pc.onicecandidate = function(e) {
|
|
1246
|
+
if (e.candidate) {
|
|
1247
|
+
if (e.candidate.candidate && e.candidate.candidate.indexOf(" relay ") !== -1) realtimeSawRelayCandidate = true;
|
|
1248
|
+
try {
|
|
1249
|
+
connection.send("AddRealtimeIceCandidate", e.candidate.candidate, e.candidate.sdpMid || "", e.candidate.sdpMLineIndex || 0);
|
|
1250
|
+
} catch (err) {}
|
|
1251
|
+
return;
|
|
1252
|
+
}
|
|
1253
|
+
if (!realtimeSawRelayCandidate && !realtimeWebRtcConnected && realtimeWebRtcConnectTimer) {
|
|
1254
|
+
clearTimeout(realtimeWebRtcConnectTimer);
|
|
1255
|
+
realtimeWebRtcConnectTimer = setTimeout(function() {
|
|
1256
|
+
if (!realtimeWebRtcConnected && isRealtimeActive) fallbackToWebSocket("no relay candidate and no direct connection");
|
|
1257
|
+
}, REALTIME_WEBRTC_NO_RELAY_TIMEOUT_MS);
|
|
1258
|
+
}
|
|
1259
|
+
};
|
|
1260
|
+
pc.onconnectionstatechange = function() {
|
|
1261
|
+
if (pc.connectionState === "connected") markWebRtcConnected();
|
|
1262
|
+
else if (pc.connectionState === "failed") {
|
|
1263
|
+
if (!realtimeWebRtcConnected) fallbackToWebSocket("connection failed");
|
|
1264
|
+
else if (isRealtimeActive) stopRealtimeConversation();
|
|
1265
|
+
} else if (pc.connectionState === "closed" && isRealtimeActive && realtimeWebRtcConnected) stopRealtimeConversation();
|
|
1266
|
+
};
|
|
1267
|
+
pc.oniceconnectionstatechange = function() {
|
|
1268
|
+
var s = pc.iceConnectionState;
|
|
1269
|
+
if (s === "connected" || s === "completed") markWebRtcConnected();
|
|
1270
|
+
else if (s === "failed" && !realtimeWebRtcConnected) fallbackToWebSocket("ICE failed");
|
|
1271
|
+
};
|
|
1272
|
+
bindWebRtcSignalingHandlers();
|
|
1273
|
+
realtimeWebRtcConnectTimer = setTimeout(function() {
|
|
1274
|
+
if (!realtimeWebRtcConnected && isRealtimeActive) fallbackToWebSocket("connection timed out");
|
|
1275
|
+
}, REALTIME_WEBRTC_CONNECT_TIMEOUT_MS);
|
|
1276
|
+
setupMicGate(stream).then(function(micTrackToSend) {
|
|
1277
|
+
if (realtimePc !== pc) return null;
|
|
1278
|
+
if (micTrackToSend) pc.addTrack(micTrackToSend, stream);
|
|
1279
|
+
return pc.createOffer();
|
|
1280
|
+
}).then(function(offer) {
|
|
1281
|
+
if (!offer || realtimePc !== pc) return null;
|
|
1282
|
+
return pc.setLocalDescription(offer).then(function() {
|
|
1283
|
+
return offer;
|
|
1284
|
+
});
|
|
1285
|
+
}).then(function(offer) {
|
|
1286
|
+
if (!offer) return;
|
|
1287
|
+
var language = realtimeLanguage || null;
|
|
1288
|
+
var voice = getVoiceName && getVoiceName() || realtimeVoiceName || "";
|
|
1289
|
+
sendStartWebRtc(offer.sdp, voice, language, null, null, realtimeBargeIn);
|
|
1290
|
+
}).catch(function(err) {
|
|
1291
|
+
console.error("Failed to create the WebRTC offer; falling back to WebSocket.", err);
|
|
1292
|
+
fallbackToWebSocket("offer failed");
|
|
1293
|
+
});
|
|
1294
|
+
}).catch(function(err) {
|
|
1295
|
+
stream.getTracks().forEach(function(t) {
|
|
1296
|
+
t.stop();
|
|
1297
|
+
});
|
|
1298
|
+
isRealtimeActive = false;
|
|
1299
|
+
realtimeIsWebRtc = false;
|
|
1300
|
+
onDeactivate();
|
|
1301
|
+
updateRealtimeButton();
|
|
1302
|
+
console.error("The realtime WebRTC conversation could not start because the chat connection is not available.", err);
|
|
1303
|
+
});
|
|
1304
|
+
}).catch(function(err) {
|
|
1305
|
+
console.error("Microphone access denied:", err);
|
|
1306
|
+
isRealtimeActive = false;
|
|
1307
|
+
realtimeIsWebRtc = false;
|
|
1308
|
+
onDeactivate();
|
|
1309
|
+
updateRealtimeButton();
|
|
1310
|
+
});
|
|
1311
|
+
}
|
|
1312
|
+
function bindWebRtcSignalingHandlers() {
|
|
1313
|
+
if (realtimeWebRtcHandlersBound) return;
|
|
1314
|
+
realtimeWebRtcHandlersBound = true;
|
|
1315
|
+
connection.on("ReceiveRealtimeAnswer", function(sdp) {
|
|
1316
|
+
if (!realtimePc) return;
|
|
1317
|
+
realtimePc.setRemoteDescription({
|
|
1318
|
+
type: "answer",
|
|
1319
|
+
sdp
|
|
1320
|
+
}).then(function() {
|
|
1321
|
+
realtimeRemoteDescriptionSet = true;
|
|
1322
|
+
var pending = realtimePendingIce;
|
|
1323
|
+
realtimePendingIce = [];
|
|
1324
|
+
pending.forEach(function(init) {
|
|
1325
|
+
try {
|
|
1326
|
+
realtimePc.addIceCandidate(init).catch(function() {});
|
|
1327
|
+
} catch (err) {}
|
|
1328
|
+
});
|
|
1329
|
+
}).catch(function(err) {
|
|
1330
|
+
console.error("Failed to apply the WebRTC answer.", err);
|
|
1331
|
+
});
|
|
1332
|
+
});
|
|
1333
|
+
connection.on("ReceiveRealtimeIceCandidate", function(candidate, sdpMid, sdpMLineIndex) {
|
|
1334
|
+
if (!realtimePc || !candidate) return;
|
|
1335
|
+
var init = {
|
|
1336
|
+
candidate,
|
|
1337
|
+
sdpMid,
|
|
1338
|
+
sdpMLineIndex
|
|
1339
|
+
};
|
|
1340
|
+
if (realtimeRemoteDescriptionSet) try {
|
|
1341
|
+
realtimePc.addIceCandidate(init).catch(function() {});
|
|
1342
|
+
} catch (err) {}
|
|
1343
|
+
else realtimePendingIce.push(init);
|
|
1344
|
+
});
|
|
1345
|
+
}
|
|
1346
|
+
function stopRealtimeWebRtc() {
|
|
1347
|
+
stopWebRtcEchoGuard();
|
|
1348
|
+
if (realtimePc) {
|
|
1349
|
+
try {
|
|
1350
|
+
realtimePc.close();
|
|
1351
|
+
} catch (err) {}
|
|
1352
|
+
realtimePc = null;
|
|
1353
|
+
}
|
|
1354
|
+
if (realtimeRemoteAudioEl) {
|
|
1355
|
+
try {
|
|
1356
|
+
realtimeRemoteAudioEl.srcObject = null;
|
|
1357
|
+
realtimeRemoteAudioEl.remove();
|
|
1358
|
+
} catch (err) {}
|
|
1359
|
+
realtimeRemoteAudioEl = null;
|
|
1360
|
+
}
|
|
1361
|
+
realtimeIsWebRtc = false;
|
|
1362
|
+
}
|
|
1363
|
+
function markWebRtcConnected() {
|
|
1364
|
+
if (realtimeWebRtcConnected) return;
|
|
1365
|
+
realtimeWebRtcConnected = true;
|
|
1366
|
+
if (realtimeWebRtcConnectTimer) {
|
|
1367
|
+
clearTimeout(realtimeWebRtcConnectTimer);
|
|
1368
|
+
realtimeWebRtcConnectTimer = null;
|
|
1369
|
+
}
|
|
1370
|
+
}
|
|
1371
|
+
var REALTIME_GATE_HANGOVER_MS = 2e3;
|
|
1372
|
+
function realtimeGateModeMessage() {
|
|
1373
|
+
return {
|
|
1374
|
+
pushToTalk: !!realtimePushToTalk,
|
|
1375
|
+
pttActive: !!realtimePttActive,
|
|
1376
|
+
bargeIn: !!realtimeBargeIn,
|
|
1377
|
+
speechHangoverMs: REALTIME_GATE_HANGOVER_MS
|
|
1378
|
+
};
|
|
1379
|
+
}
|
|
1380
|
+
function syncRealtimeGateMode() {
|
|
1381
|
+
if (realtimeGate) realtimeGate.setMode(realtimeGateModeMessage());
|
|
1382
|
+
}
|
|
1383
|
+
function setupMicGate(rawStream) {
|
|
1384
|
+
stopWebRtcEchoGuard();
|
|
1385
|
+
var rawTrack = rawStream && rawStream.getAudioTracks()[0] || null;
|
|
1386
|
+
return createMicGate(rawStream, realtimeGateModeMessage()).then(function(gate) {
|
|
1387
|
+
realtimeGate = gate;
|
|
1388
|
+
if (realtimeGatePendingRemoteStream) gate.attachAssistantStream(realtimeGatePendingRemoteStream);
|
|
1389
|
+
return gate.track || rawTrack;
|
|
1390
|
+
});
|
|
1391
|
+
}
|
|
1392
|
+
function attachWebRtcAssistantAnalyser(remoteStream) {
|
|
1393
|
+
realtimeGatePendingRemoteStream = remoteStream || null;
|
|
1394
|
+
if (realtimeGate && remoteStream) realtimeGate.attachAssistantStream(remoteStream);
|
|
1395
|
+
}
|
|
1396
|
+
function stopWebRtcEchoGuard() {
|
|
1397
|
+
if (realtimeGate) {
|
|
1398
|
+
try {
|
|
1399
|
+
realtimeGate.stop();
|
|
1400
|
+
} catch (e) {}
|
|
1401
|
+
realtimeGate = null;
|
|
1402
|
+
}
|
|
1403
|
+
realtimeGatePendingRemoteStream = null;
|
|
1404
|
+
}
|
|
1405
|
+
function applyRealtimeOutputDevice() {
|
|
1406
|
+
routeOutputToPreferredDevice(realtimeRemoteAudioEl, realtimeOutputDeviceId);
|
|
1407
|
+
}
|
|
1408
|
+
function routeRealtimeOutputToDefaultDevice(el) {
|
|
1409
|
+
routeOutputToPreferredDevice(el, realtimeOutputDeviceId);
|
|
1410
|
+
}
|
|
1411
|
+
function ensureRealtimePlayback(el) {
|
|
1412
|
+
ensurePlayback(el, function() {
|
|
1413
|
+
setRealtimeState("playback-blocked");
|
|
1414
|
+
});
|
|
1415
|
+
}
|
|
1416
|
+
function logWebSocketTransportReason(reason) {
|
|
1417
|
+
if (window.console && console.warn) console.warn("Realtime is using the WebSocket transport (" + reason + "); acoustic echo cancellation is weaker than on WebRTC.");
|
|
1418
|
+
}
|
|
1419
|
+
function fallbackToWebSocket(reason) {
|
|
1420
|
+
if (realtimeFellBack) return;
|
|
1421
|
+
realtimeFellBack = true;
|
|
1422
|
+
if (realtimeWebRtcConnectTimer) {
|
|
1423
|
+
clearTimeout(realtimeWebRtcConnectTimer);
|
|
1424
|
+
realtimeWebRtcConnectTimer = null;
|
|
1425
|
+
}
|
|
1426
|
+
if (window.console && console.warn) console.warn("Realtime WebRTC transport unavailable (" + reason + "); falling back to the WebSocket transport.");
|
|
1427
|
+
rememberWebRtcBlocked();
|
|
1428
|
+
showCompatibilityModeNotice();
|
|
1429
|
+
try {
|
|
1430
|
+
if (realtimeStream) realtimeStream.getTracks().forEach(function(t) {
|
|
1431
|
+
t.stop();
|
|
1432
|
+
});
|
|
1433
|
+
} catch (err) {}
|
|
1434
|
+
realtimeStream = null;
|
|
1435
|
+
stopRealtimeWebRtc();
|
|
1436
|
+
realtimeIsWebRtc = false;
|
|
1437
|
+
isRealtimeActive = false;
|
|
1438
|
+
startRealtimeWebSocketConversation();
|
|
1439
|
+
}
|
|
1440
|
+
function stopRealtimeConversation() {
|
|
1441
|
+
if (!isRealtimeActive) return;
|
|
1442
|
+
if (realtimeWebRtcConnectTimer) {
|
|
1443
|
+
clearTimeout(realtimeWebRtcConnectTimer);
|
|
1444
|
+
realtimeWebRtcConnectTimer = null;
|
|
1445
|
+
}
|
|
1446
|
+
isRealtimeActive = false;
|
|
1447
|
+
onDeactivate();
|
|
1448
|
+
updateRealtimeButton();
|
|
1449
|
+
detachRealtimePushToTalk();
|
|
1450
|
+
removeRealtimePttUi();
|
|
1451
|
+
if (!realtimeEndedNotice) setRealtimeState("idle");
|
|
1452
|
+
if (realtimeIsWebRtc) {
|
|
1453
|
+
stopRealtimeWebRtc();
|
|
1454
|
+
try {
|
|
1455
|
+
if (realtimeStream) realtimeStream.getTracks().forEach(function(t) {
|
|
1456
|
+
t.stop();
|
|
1457
|
+
});
|
|
1458
|
+
} catch (err) {}
|
|
1459
|
+
realtimeStream = null;
|
|
1460
|
+
return;
|
|
1461
|
+
}
|
|
1462
|
+
try {
|
|
1463
|
+
if (realtimeSubject) realtimeSubject.complete();
|
|
1464
|
+
} catch (err) {}
|
|
1465
|
+
realtimeSubject = null;
|
|
1466
|
+
stopWebRtcEchoGuard();
|
|
1467
|
+
try {
|
|
1468
|
+
if (realtimeProcessor) {
|
|
1469
|
+
realtimeProcessor.disconnect();
|
|
1470
|
+
realtimeProcessor.onaudioprocess = null;
|
|
1471
|
+
if (realtimeProcessor.port) {
|
|
1472
|
+
realtimeProcessor.port.onmessage = null;
|
|
1473
|
+
realtimeProcessor.port.close();
|
|
1474
|
+
}
|
|
1475
|
+
}
|
|
1476
|
+
} catch (err) {}
|
|
1477
|
+
try {
|
|
1478
|
+
if (realtimeMicSource) realtimeMicSource.disconnect();
|
|
1479
|
+
} catch (err) {}
|
|
1480
|
+
try {
|
|
1481
|
+
if (realtimeZeroGain) realtimeZeroGain.disconnect();
|
|
1482
|
+
} catch (err) {}
|
|
1483
|
+
try {
|
|
1484
|
+
if (realtimeGain) realtimeGain.disconnect();
|
|
1485
|
+
} catch (err) {}
|
|
1486
|
+
try {
|
|
1487
|
+
if (realtimeStream) realtimeStream.getTracks().forEach(function(t) {
|
|
1488
|
+
t.stop();
|
|
1489
|
+
});
|
|
1490
|
+
} catch (err) {}
|
|
1491
|
+
realtimeProcessor = null;
|
|
1492
|
+
realtimeMicSource = null;
|
|
1493
|
+
realtimeZeroGain = null;
|
|
1494
|
+
realtimeGain = null;
|
|
1495
|
+
realtimeStream = null;
|
|
1496
|
+
flushRealtimePlayback();
|
|
1497
|
+
try {
|
|
1498
|
+
if (realtimeAudioCtx) realtimeAudioCtx.close();
|
|
1499
|
+
} catch (err) {}
|
|
1500
|
+
realtimeAudioCtx = null;
|
|
1501
|
+
}
|
|
1502
|
+
function toggleRealtime() {
|
|
1503
|
+
if (isRealtimeActive) {
|
|
1504
|
+
stopRealtimeConversation();
|
|
1505
|
+
return;
|
|
1506
|
+
}
|
|
1507
|
+
startRealtimeConversation();
|
|
1508
|
+
}
|
|
1509
|
+
function getTransportStats() {
|
|
1510
|
+
if (!realtimePc || typeof realtimePc.getStats !== "function") return Promise.resolve(null);
|
|
1511
|
+
return realtimePc.getStats().then(function(report) {
|
|
1512
|
+
var result = {
|
|
1513
|
+
transport: "webrtc",
|
|
1514
|
+
inbound: null,
|
|
1515
|
+
codec: null,
|
|
1516
|
+
sinkId: realtimeRemoteAudioEl && typeof realtimeRemoteAudioEl.sinkId === "string" ? realtimeRemoteAudioEl.sinkId : null,
|
|
1517
|
+
jitterBufferTargetMs: (function() {
|
|
1518
|
+
try {
|
|
1519
|
+
var r = realtimePc.getReceivers().filter(function(x) {
|
|
1520
|
+
return x.track && x.track.kind === "audio";
|
|
1521
|
+
})[0];
|
|
1522
|
+
return r && "jitterBufferTarget" in r ? r.jitterBufferTarget : null;
|
|
1523
|
+
} catch (e) {
|
|
1524
|
+
return null;
|
|
1525
|
+
}
|
|
1526
|
+
})()
|
|
1527
|
+
};
|
|
1528
|
+
report.forEach(function(stat) {
|
|
1529
|
+
if (stat.type === "inbound-rtp" && stat.kind === "audio") {
|
|
1530
|
+
result.inbound = {
|
|
1531
|
+
packetsReceived: stat.packetsReceived,
|
|
1532
|
+
packetsLost: stat.packetsLost,
|
|
1533
|
+
packetsDiscarded: stat.packetsDiscarded,
|
|
1534
|
+
jitterMs: typeof stat.jitter === "number" ? Math.round(stat.jitter * 1e3) : null,
|
|
1535
|
+
jitterBufferDelayMs: typeof stat.jitterBufferDelay === "number" && stat.jitterBufferEmittedCount ? Math.round(stat.jitterBufferDelay / stat.jitterBufferEmittedCount * 1e3) : null,
|
|
1536
|
+
totalSamplesReceived: stat.totalSamplesReceived,
|
|
1537
|
+
concealedSamples: stat.concealedSamples,
|
|
1538
|
+
concealmentEvents: stat.concealmentEvents,
|
|
1539
|
+
silentConcealedSamples: stat.silentConcealedSamples,
|
|
1540
|
+
insertedSamplesForDeceleration: stat.insertedSamplesForDeceleration,
|
|
1541
|
+
removedSamplesForAcceleration: stat.removedSamplesForAcceleration,
|
|
1542
|
+
audioLevel: stat.audioLevel
|
|
1543
|
+
};
|
|
1544
|
+
if (stat.codecId) {
|
|
1545
|
+
var codec = report.get(stat.codecId);
|
|
1546
|
+
if (codec) result.codec = {
|
|
1547
|
+
mimeType: codec.mimeType,
|
|
1548
|
+
clockRate: codec.clockRate,
|
|
1549
|
+
channels: codec.channels,
|
|
1550
|
+
sdpFmtpLine: codec.sdpFmtpLine
|
|
1551
|
+
};
|
|
1552
|
+
}
|
|
1553
|
+
}
|
|
1554
|
+
});
|
|
1555
|
+
return result;
|
|
1556
|
+
}).catch(function() {
|
|
1557
|
+
return null;
|
|
1558
|
+
});
|
|
1559
|
+
}
|
|
1560
|
+
function playRealtimePcm(bytes) {
|
|
1561
|
+
if (!realtimeAudioCtx || !bytes || bytes.length < 2) return;
|
|
1562
|
+
var ctx = realtimeAudioCtx;
|
|
1563
|
+
var sampleCount = Math.floor(bytes.length / 2);
|
|
1564
|
+
var pcm = new Int16Array(bytes.buffer, bytes.byteOffset, sampleCount);
|
|
1565
|
+
var f32 = new Float32Array(sampleCount);
|
|
1566
|
+
for (var i = 0; i < sampleCount; i++) f32[i] = pcm[i] / 32768;
|
|
1567
|
+
var buffer = ctx.createBuffer(1, sampleCount, ctx.sampleRate);
|
|
1568
|
+
buffer.copyToChannel(f32, 0);
|
|
1569
|
+
var src = ctx.createBufferSource();
|
|
1570
|
+
src.buffer = buffer;
|
|
1571
|
+
src.connect(realtimeGain || ctx.destination);
|
|
1572
|
+
var now = ctx.currentTime;
|
|
1573
|
+
if (realtimePlayHead < now) realtimePlayHead = now;
|
|
1574
|
+
src.start(realtimePlayHead);
|
|
1575
|
+
realtimePlayHead += buffer.duration;
|
|
1576
|
+
realtimeSources.push(src);
|
|
1577
|
+
src.onended = function() {
|
|
1578
|
+
realtimeSources = realtimeSources.filter(function(s) {
|
|
1579
|
+
return s !== src;
|
|
1580
|
+
});
|
|
1581
|
+
};
|
|
1582
|
+
}
|
|
1583
|
+
function flushRealtimePlayback() {
|
|
1584
|
+
realtimeSources.forEach(function(s) {
|
|
1585
|
+
try {
|
|
1586
|
+
s.stop();
|
|
1587
|
+
} catch (err) {}
|
|
1588
|
+
});
|
|
1589
|
+
realtimeSources = [];
|
|
1590
|
+
if (realtimeAudioCtx) realtimePlayHead = realtimeAudioCtx.currentTime;
|
|
1591
|
+
}
|
|
1592
|
+
function isRealtimeDeployment(deploymentName) {
|
|
1593
|
+
if (!deploymentName) return false;
|
|
1594
|
+
return realtimeCapableDeployments.indexOf(deploymentName.toLowerCase()) !== -1;
|
|
1595
|
+
}
|
|
1596
|
+
var realtimeBtn = q("realtimeButton");
|
|
1597
|
+
if (realtimeBtn) {
|
|
1598
|
+
realtimeBtn.addEventListener("click", function(e) {
|
|
1599
|
+
if (e && e.detail === 0 && isRealtimeActive) return;
|
|
1600
|
+
toggleRealtime();
|
|
1601
|
+
});
|
|
1602
|
+
setupRealtimeAudioSettings();
|
|
1603
|
+
}
|
|
1604
|
+
var deploymentSelect = q("deploymentSelect");
|
|
1605
|
+
if (deploymentSelect) deploymentSelect.addEventListener("change", function() {
|
|
1606
|
+
applyRealtimeMode(isRealtimeDeployment(deploymentSelect.value));
|
|
1607
|
+
});
|
|
1608
|
+
applyRealtimeMode(isRealtimeMode || deploymentSelect && isRealtimeDeployment(deploymentSelect.value));
|
|
1609
|
+
var controller = {
|
|
1610
|
+
toggle: toggleRealtime,
|
|
1611
|
+
start: startRealtimeConversation,
|
|
1612
|
+
stop: stopRealtimeConversation,
|
|
1613
|
+
applyMode: applyRealtimeMode,
|
|
1614
|
+
isRealtimeDeployment,
|
|
1615
|
+
receivePcm: playRealtimePcm,
|
|
1616
|
+
isActive: function() {
|
|
1617
|
+
return isRealtimeActive;
|
|
1618
|
+
},
|
|
1619
|
+
getState: function() {
|
|
1620
|
+
return realtimeState;
|
|
1621
|
+
},
|
|
1622
|
+
getGateLevel: function() {
|
|
1623
|
+
return realtimeGate ? realtimeGate.getLevel() : null;
|
|
1624
|
+
},
|
|
1625
|
+
getTransportStats
|
|
1626
|
+
};
|
|
1627
|
+
window.CoreAIRealtime.activeController = controller;
|
|
1628
|
+
return controller;
|
|
1629
|
+
}
|
|
1630
|
+
window.CoreAIRealtime = {
|
|
1631
|
+
attach,
|
|
1632
|
+
createMicGate,
|
|
1633
|
+
decideGate: coreAiGateDecide,
|
|
1634
|
+
createGateState,
|
|
1635
|
+
rmsDb: coreAiRmsDb,
|
|
1636
|
+
routeOutputToPreferredDevice,
|
|
1637
|
+
ensurePlayback
|
|
1638
|
+
};
|
|
1639
|
+
})(window, document);
|
|
1640
|
+
//#endregion
|
|
1641
|
+
})();
|
|
2507
1642
|
//# sourceMappingURL=realtime-audio.js.map
|