@tanstack/ai-grok 0.14.9 → 0.14.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/esm/adapters/image.js +228 -193
- package/dist/esm/adapters/image.js.map +1 -1
- package/dist/esm/adapters/summarize.js +49 -12
- package/dist/esm/adapters/summarize.js.map +1 -1
- package/dist/esm/adapters/text.js +82 -38
- package/dist/esm/adapters/text.js.map +1 -1
- package/dist/esm/adapters/transcription.js +133 -114
- package/dist/esm/adapters/transcription.js.map +1 -1
- package/dist/esm/adapters/tts.js +147 -130
- package/dist/esm/adapters/tts.js.map +1 -1
- package/dist/esm/adapters/video.js +275 -226
- package/dist/esm/adapters/video.js.map +1 -1
- package/dist/esm/image/image-provider-options.js +64 -66
- package/dist/esm/image/image-provider-options.js.map +1 -1
- package/dist/esm/index.js +3 -31
- package/dist/esm/model-meta.js +149 -47
- package/dist/esm/model-meta.js.map +1 -1
- package/dist/esm/realtime/adapter.js +729 -812
- package/dist/esm/realtime/adapter.js.map +1 -1
- package/dist/esm/realtime/index.js +3 -0
- package/dist/esm/realtime/token.js +78 -72
- package/dist/esm/realtime/token.js.map +1 -1
- package/dist/esm/tools/index.js +57 -96
- package/dist/esm/tools/index.js.map +1 -1
- package/dist/esm/utils/audio.js +116 -162
- package/dist/esm/utils/audio.js.map +1 -1
- package/dist/esm/utils/client.js +22 -16
- package/dist/esm/utils/client.js.map +1 -1
- package/dist/esm/utils/index.js +5 -0
- package/dist/esm/utils/schema-converter.js +2 -0
- package/dist/esm/video/video-provider-options.js +83 -57
- package/dist/esm/video/video-provider-options.js.map +1 -1
- package/package.json +8 -8
- package/src/realtime/adapter.ts +1 -1
- package/src/utils/audio.ts +1 -1
- package/dist/esm/index.js.map +0 -1
|
@@ -1,823 +1,740 @@
|
|
|
1
1
|
import { resolveDebugOption } from "@tanstack/ai/adapter-internals";
|
|
2
|
-
|
|
2
|
+
//#region src/realtime/adapter.ts
|
|
3
|
+
var GROK_REALTIME_URL = "https://api.x.ai/v1/realtime";
|
|
4
|
+
/**
|
|
5
|
+
* Runtime-checked field readers for untyped server events. Replace the
|
|
6
|
+
* drive-by `event.X as string` / `event.X as Record<string, unknown>` casts
|
|
7
|
+
* with readers that return `undefined` when the shape doesn't match, so a
|
|
8
|
+
* malformed frame can't throw a TypeError inside `handleServerEvent`.
|
|
9
|
+
*/
|
|
3
10
|
function readString(obj, key) {
|
|
4
|
-
|
|
5
|
-
|
|
11
|
+
const value = obj[key];
|
|
12
|
+
return typeof value === "string" ? value : void 0;
|
|
6
13
|
}
|
|
7
14
|
function readObject(obj, key) {
|
|
8
|
-
|
|
9
|
-
|
|
15
|
+
const value = obj[key];
|
|
16
|
+
return value && typeof value === "object" && !Array.isArray(value) ? value : void 0;
|
|
10
17
|
}
|
|
11
18
|
function readObjectArray(obj, key) {
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
(item) => item !== null && typeof item === "object" && !Array.isArray(item)
|
|
16
|
-
);
|
|
19
|
+
const value = obj[key];
|
|
20
|
+
if (!Array.isArray(value)) return void 0;
|
|
21
|
+
return value.filter((item) => item !== null && typeof item === "object" && !Array.isArray(item));
|
|
17
22
|
}
|
|
23
|
+
/**
|
|
24
|
+
* Creates a Grok realtime adapter for client-side use.
|
|
25
|
+
*
|
|
26
|
+
* Uses WebRTC for browser connections (default). Mirrors the OpenAI realtime
|
|
27
|
+
* adapter because xAI's Voice Agent API is OpenAI-realtime-compatible — the
|
|
28
|
+
* only differences are the endpoint URL and default model.
|
|
29
|
+
*
|
|
30
|
+
* @example
|
|
31
|
+
* ```typescript
|
|
32
|
+
* import { RealtimeClient } from '@tanstack/ai-client'
|
|
33
|
+
* import { grokRealtime } from '@tanstack/ai-grok'
|
|
34
|
+
*
|
|
35
|
+
* const client = new RealtimeClient({
|
|
36
|
+
* getToken: () => fetch('/api/realtime-token').then(r => r.json()),
|
|
37
|
+
* adapter: grokRealtime(),
|
|
38
|
+
* })
|
|
39
|
+
* ```
|
|
40
|
+
*/
|
|
18
41
|
function grokRealtime(options = {}) {
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
}
|
|
39
|
-
};
|
|
42
|
+
const connectionMode = options.connectionMode ?? "webrtc";
|
|
43
|
+
const logger = resolveDebugOption(options.debug);
|
|
44
|
+
return {
|
|
45
|
+
provider: "grok",
|
|
46
|
+
async connect(token, _clientTools) {
|
|
47
|
+
const model = token.config.model ?? "grok-voice-fast-1.0";
|
|
48
|
+
logger.request(`activity=realtime provider=grok model=${model}`, {
|
|
49
|
+
provider: "grok",
|
|
50
|
+
model
|
|
51
|
+
});
|
|
52
|
+
if (connectionMode === "webrtc") return createWebRTCConnection(token, logger);
|
|
53
|
+
const error = /* @__PURE__ */ new Error("WebSocket connection mode not yet implemented");
|
|
54
|
+
logger.errors("grok.realtime fatal", {
|
|
55
|
+
error,
|
|
56
|
+
source: "grok.realtime"
|
|
57
|
+
});
|
|
58
|
+
throw error;
|
|
59
|
+
}
|
|
60
|
+
};
|
|
40
61
|
}
|
|
62
|
+
/**
|
|
63
|
+
* Creates a WebRTC connection to xAI's realtime API.
|
|
64
|
+
*/
|
|
41
65
|
async function createWebRTCConnection(token, logger) {
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
sessionUpdate.modalities = config.outputModalities;
|
|
713
|
-
}
|
|
714
|
-
if (config.temperature !== void 0) {
|
|
715
|
-
sessionUpdate.temperature = config.temperature;
|
|
716
|
-
}
|
|
717
|
-
if (config.maxOutputTokens !== void 0) {
|
|
718
|
-
sessionUpdate.max_response_output_tokens = config.maxOutputTokens;
|
|
719
|
-
}
|
|
720
|
-
const providerOptions = config.providerOptions ?? {};
|
|
721
|
-
const callerTranscription = "inputAudioTranscription" in providerOptions ? providerOptions.inputAudioTranscription : "input_audio_transcription" in providerOptions ? providerOptions.input_audio_transcription : void 0;
|
|
722
|
-
if (callerTranscription !== void 0) {
|
|
723
|
-
sessionUpdate.input_audio_transcription = callerTranscription === false ? null : callerTranscription;
|
|
724
|
-
} else if (!hasSentInitialSessionUpdate) {
|
|
725
|
-
sessionUpdate.input_audio_transcription = { model: "grok-stt" };
|
|
726
|
-
}
|
|
727
|
-
if (Object.keys(sessionUpdate).length > 0) {
|
|
728
|
-
sendEvent({
|
|
729
|
-
type: "session.update",
|
|
730
|
-
session: sessionUpdate
|
|
731
|
-
});
|
|
732
|
-
hasSentInitialSessionUpdate = true;
|
|
733
|
-
}
|
|
734
|
-
},
|
|
735
|
-
interrupt() {
|
|
736
|
-
sendEvent({ type: "response.cancel" });
|
|
737
|
-
currentMode = "listening";
|
|
738
|
-
emit("mode_change", { mode: "listening" });
|
|
739
|
-
emit("interrupted", {
|
|
740
|
-
...currentMessageId !== null && { messageId: currentMessageId }
|
|
741
|
-
});
|
|
742
|
-
},
|
|
743
|
-
on(event, handler) {
|
|
744
|
-
let handlers = eventHandlers.get(event);
|
|
745
|
-
if (!handlers) {
|
|
746
|
-
handlers = /* @__PURE__ */ new Set();
|
|
747
|
-
eventHandlers.set(event, handlers);
|
|
748
|
-
}
|
|
749
|
-
handlers.add(handler);
|
|
750
|
-
return () => {
|
|
751
|
-
eventHandlers.get(event)?.delete(handler);
|
|
752
|
-
};
|
|
753
|
-
},
|
|
754
|
-
getAudioVisualization() {
|
|
755
|
-
function calculateLevel(analyser) {
|
|
756
|
-
const data = new Uint8Array(analyser.fftSize);
|
|
757
|
-
analyser.getByteTimeDomainData(data);
|
|
758
|
-
let maxDeviation = 0;
|
|
759
|
-
for (const sample of data) {
|
|
760
|
-
const deviation = Math.abs(sample - 128);
|
|
761
|
-
if (deviation > maxDeviation) {
|
|
762
|
-
maxDeviation = deviation;
|
|
763
|
-
}
|
|
764
|
-
}
|
|
765
|
-
const normalized = maxDeviation / 128;
|
|
766
|
-
return Math.min(1, normalized * 1.5);
|
|
767
|
-
}
|
|
768
|
-
return {
|
|
769
|
-
get inputLevel() {
|
|
770
|
-
if (!inputAnalyser) return 0;
|
|
771
|
-
return calculateLevel(inputAnalyser);
|
|
772
|
-
},
|
|
773
|
-
get outputLevel() {
|
|
774
|
-
if (!outputAnalyser) return 0;
|
|
775
|
-
return calculateLevel(outputAnalyser);
|
|
776
|
-
},
|
|
777
|
-
getInputFrequencyData() {
|
|
778
|
-
if (!inputAnalyser)
|
|
779
|
-
return new Uint8Array(FALLBACK_FREQUENCY_BIN_COUNT);
|
|
780
|
-
const data = new Uint8Array(inputAnalyser.frequencyBinCount);
|
|
781
|
-
inputAnalyser.getByteFrequencyData(data);
|
|
782
|
-
return data;
|
|
783
|
-
},
|
|
784
|
-
getOutputFrequencyData() {
|
|
785
|
-
if (!outputAnalyser)
|
|
786
|
-
return new Uint8Array(FALLBACK_FREQUENCY_BIN_COUNT);
|
|
787
|
-
const data = new Uint8Array(outputAnalyser.frequencyBinCount);
|
|
788
|
-
outputAnalyser.getByteFrequencyData(data);
|
|
789
|
-
return data;
|
|
790
|
-
},
|
|
791
|
-
getInputTimeDomainData() {
|
|
792
|
-
if (!inputAnalyser)
|
|
793
|
-
return new Uint8Array(FALLBACK_TIME_DOMAIN_SIZE).fill(
|
|
794
|
-
FALLBACK_TIME_DOMAIN_FILL
|
|
795
|
-
);
|
|
796
|
-
const data = new Uint8Array(inputAnalyser.fftSize);
|
|
797
|
-
inputAnalyser.getByteTimeDomainData(data);
|
|
798
|
-
return data;
|
|
799
|
-
},
|
|
800
|
-
getOutputTimeDomainData() {
|
|
801
|
-
if (!outputAnalyser)
|
|
802
|
-
return new Uint8Array(FALLBACK_TIME_DOMAIN_SIZE).fill(
|
|
803
|
-
FALLBACK_TIME_DOMAIN_FILL
|
|
804
|
-
);
|
|
805
|
-
const data = new Uint8Array(outputAnalyser.fftSize);
|
|
806
|
-
outputAnalyser.getByteTimeDomainData(data);
|
|
807
|
-
return data;
|
|
808
|
-
},
|
|
809
|
-
get inputSampleRate() {
|
|
810
|
-
return 24e3;
|
|
811
|
-
},
|
|
812
|
-
get outputSampleRate() {
|
|
813
|
-
return 24e3;
|
|
814
|
-
}
|
|
815
|
-
};
|
|
816
|
-
}
|
|
817
|
-
};
|
|
818
|
-
return connection;
|
|
66
|
+
const model = token.config.model ?? "grok-voice-fast-1.0";
|
|
67
|
+
const eventHandlers = /* @__PURE__ */ new Map();
|
|
68
|
+
const pc = new RTCPeerConnection();
|
|
69
|
+
let audioContext = null;
|
|
70
|
+
let inputAnalyser = null;
|
|
71
|
+
let outputAnalyser = null;
|
|
72
|
+
let inputSource = null;
|
|
73
|
+
let outputSource = null;
|
|
74
|
+
let localStream = null;
|
|
75
|
+
let audioElement = null;
|
|
76
|
+
const channel = pc.createDataChannel("oai-events");
|
|
77
|
+
let dataChannel = channel;
|
|
78
|
+
let currentMode = "idle";
|
|
79
|
+
let currentMessageId = null;
|
|
80
|
+
let isTornDown = false;
|
|
81
|
+
const pendingEvents = [];
|
|
82
|
+
let hasSentInitialSessionUpdate = false;
|
|
83
|
+
const FALLBACK_FREQUENCY_BIN_COUNT = 1024;
|
|
84
|
+
const FALLBACK_TIME_DOMAIN_SIZE = 2048;
|
|
85
|
+
const FALLBACK_TIME_DOMAIN_FILL = 128;
|
|
86
|
+
function emit(event, payload) {
|
|
87
|
+
const handlers = eventHandlers.get(event);
|
|
88
|
+
if (handlers) for (const handler of handlers) handler(payload);
|
|
89
|
+
}
|
|
90
|
+
let dataChannelOpened = false;
|
|
91
|
+
let rejectDataChannelReady = null;
|
|
92
|
+
let dataChannelReadyTimeout = null;
|
|
93
|
+
const dataChannelReady = new Promise((resolve, reject) => {
|
|
94
|
+
rejectDataChannelReady = (reason) => {
|
|
95
|
+
if (dataChannelReadyTimeout !== null) {
|
|
96
|
+
clearTimeout(dataChannelReadyTimeout);
|
|
97
|
+
dataChannelReadyTimeout = null;
|
|
98
|
+
}
|
|
99
|
+
rejectDataChannelReady = null;
|
|
100
|
+
reject(reason);
|
|
101
|
+
};
|
|
102
|
+
dataChannelReadyTimeout = setTimeout(() => {
|
|
103
|
+
if (!dataChannelOpened) rejectDataChannelReady?.(/* @__PURE__ */ new Error("Data channel did not open within 15000ms — aborting connection"));
|
|
104
|
+
}, 15e3);
|
|
105
|
+
channel.onopen = () => {
|
|
106
|
+
dataChannelOpened = true;
|
|
107
|
+
if (dataChannelReadyTimeout !== null) {
|
|
108
|
+
clearTimeout(dataChannelReadyTimeout);
|
|
109
|
+
dataChannelReadyTimeout = null;
|
|
110
|
+
}
|
|
111
|
+
rejectDataChannelReady = null;
|
|
112
|
+
flushPendingEvents();
|
|
113
|
+
emit("status_change", { status: "connected" });
|
|
114
|
+
resolve();
|
|
115
|
+
};
|
|
116
|
+
});
|
|
117
|
+
channel.onmessage = (event) => {
|
|
118
|
+
try {
|
|
119
|
+
const message = JSON.parse(event.data);
|
|
120
|
+
const messageRecord = message !== null && typeof message === "object" ? message : {};
|
|
121
|
+
logger.provider(`provider=grok direction=in type=${readString(messageRecord, "type") ?? "<unknown>"}`, { frame: messageRecord });
|
|
122
|
+
handleServerEvent(messageRecord);
|
|
123
|
+
} catch (parseErr) {
|
|
124
|
+
logger.errors("grok.realtime fatal", {
|
|
125
|
+
error: parseErr,
|
|
126
|
+
source: "grok.realtime"
|
|
127
|
+
});
|
|
128
|
+
emit("error", { error: parseErr instanceof Error ? parseErr : new Error(String(parseErr)) });
|
|
129
|
+
}
|
|
130
|
+
};
|
|
131
|
+
channel.onerror = (error) => {
|
|
132
|
+
if (isTornDown) return;
|
|
133
|
+
logger.errors("grok.realtime fatal", {
|
|
134
|
+
error,
|
|
135
|
+
source: "grok.realtime"
|
|
136
|
+
});
|
|
137
|
+
const rtcError = readObject(error, "error");
|
|
138
|
+
const msg = (rtcError && readString(rtcError, "message")) ?? (error.type || "unknown");
|
|
139
|
+
const dcErr = /* @__PURE__ */ new Error(`Data channel error: ${msg}`);
|
|
140
|
+
if (!dataChannelOpened) rejectDataChannelReady?.(dcErr);
|
|
141
|
+
emit("error", { error: dcErr });
|
|
142
|
+
};
|
|
143
|
+
channel.onclose = () => {
|
|
144
|
+
if (isTornDown) return;
|
|
145
|
+
if (!dataChannelOpened) rejectDataChannelReady?.(/* @__PURE__ */ new Error("Data channel closed before opening"));
|
|
146
|
+
};
|
|
147
|
+
pc.ontrack = (event) => {
|
|
148
|
+
if (event.track.kind === "audio" && event.streams[0]) setupOutputAudioAnalysis(event.streams[0]);
|
|
149
|
+
};
|
|
150
|
+
pc.onconnectionstatechange = () => {
|
|
151
|
+
const state = pc.connectionState;
|
|
152
|
+
logger.provider(`provider=grok pc.connectionState=${state}`, { state });
|
|
153
|
+
if (state === "failed" || state === "disconnected" || state === "closed") {
|
|
154
|
+
if (!isTornDown) emit("status_change", { status: state === "failed" ? "error" : "idle" });
|
|
155
|
+
if (!dataChannelOpened) {
|
|
156
|
+
const message = state === "failed" ? `PeerConnection failed before data channel opened` : `PeerConnection entered state '${state}' before data channel opened`;
|
|
157
|
+
rejectDataChannelReady?.(new Error(message));
|
|
158
|
+
}
|
|
159
|
+
if (state === "failed" && !isTornDown) teardownConnection();
|
|
160
|
+
}
|
|
161
|
+
};
|
|
162
|
+
pc.oniceconnectionstatechange = () => {
|
|
163
|
+
const state = pc.iceConnectionState;
|
|
164
|
+
logger.provider(`provider=grok pc.iceConnectionState=${state}`, { state });
|
|
165
|
+
if (!dataChannelOpened && (state === "failed" || state === "closed" || state === "disconnected")) {
|
|
166
|
+
const message = state === "failed" ? `ICE connection failed before data channel opened` : `ICE connection entered state '${state}' before data channel opened`;
|
|
167
|
+
rejectDataChannelReady?.(new Error(message));
|
|
168
|
+
}
|
|
169
|
+
};
|
|
170
|
+
/**
|
|
171
|
+
* Tear down every resource we may have allocated so the mic/pc/audio
|
|
172
|
+
* nodes/audio element don't leak on a failed connect. Safe to call from
|
|
173
|
+
* any point after `new RTCPeerConnection()`; each branch null-guards and
|
|
174
|
+
* swallows errors because cascading closes (e.g. `pc.close()` closing the
|
|
175
|
+
* data channel implicitly) are expected.
|
|
176
|
+
*
|
|
177
|
+
* Shared between the SDP-path catch, the post-SDP catch, and (implicitly
|
|
178
|
+
* via idempotency) the `disconnect()` entry point.
|
|
179
|
+
*/
|
|
180
|
+
async function teardownConnection() {
|
|
181
|
+
isTornDown = true;
|
|
182
|
+
pendingEvents.length = 0;
|
|
183
|
+
rejectDataChannelReady?.(/* @__PURE__ */ new Error("Connection torn down before data channel opened"));
|
|
184
|
+
if (localStream) {
|
|
185
|
+
for (const track of localStream.getTracks()) track.stop();
|
|
186
|
+
localStream = null;
|
|
187
|
+
}
|
|
188
|
+
if (audioElement) {
|
|
189
|
+
try {
|
|
190
|
+
audioElement.pause();
|
|
191
|
+
} catch {}
|
|
192
|
+
audioElement.srcObject = null;
|
|
193
|
+
audioElement = null;
|
|
194
|
+
}
|
|
195
|
+
if (outputSource) {
|
|
196
|
+
try {
|
|
197
|
+
outputSource.disconnect();
|
|
198
|
+
} catch {}
|
|
199
|
+
outputSource = null;
|
|
200
|
+
}
|
|
201
|
+
if (outputAnalyser) {
|
|
202
|
+
try {
|
|
203
|
+
outputAnalyser.disconnect();
|
|
204
|
+
} catch {}
|
|
205
|
+
outputAnalyser = null;
|
|
206
|
+
}
|
|
207
|
+
if (inputSource) {
|
|
208
|
+
try {
|
|
209
|
+
inputSource.disconnect();
|
|
210
|
+
} catch {}
|
|
211
|
+
inputSource = null;
|
|
212
|
+
}
|
|
213
|
+
if (inputAnalyser) {
|
|
214
|
+
try {
|
|
215
|
+
inputAnalyser.disconnect();
|
|
216
|
+
} catch {}
|
|
217
|
+
inputAnalyser = null;
|
|
218
|
+
}
|
|
219
|
+
if (dataChannel) {
|
|
220
|
+
try {
|
|
221
|
+
dataChannel.close();
|
|
222
|
+
} catch {}
|
|
223
|
+
dataChannel = null;
|
|
224
|
+
}
|
|
225
|
+
try {
|
|
226
|
+
pc.close();
|
|
227
|
+
} catch {}
|
|
228
|
+
if (audioContext) {
|
|
229
|
+
try {
|
|
230
|
+
await audioContext.close();
|
|
231
|
+
} catch {}
|
|
232
|
+
audioContext = null;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
try {
|
|
236
|
+
try {
|
|
237
|
+
localStream = await navigator.mediaDevices.getUserMedia({ audio: {
|
|
238
|
+
echoCancellation: true,
|
|
239
|
+
noiseSuppression: true,
|
|
240
|
+
sampleRate: 24e3
|
|
241
|
+
} });
|
|
242
|
+
} catch (error) {
|
|
243
|
+
logger.errors("grok.realtime fatal", {
|
|
244
|
+
error,
|
|
245
|
+
source: "grok.realtime.getUserMedia"
|
|
246
|
+
});
|
|
247
|
+
throw new Error(`Microphone access required for realtime voice: ${error instanceof Error ? error.message : error}`);
|
|
248
|
+
}
|
|
249
|
+
for (const track of localStream.getAudioTracks()) pc.addTrack(track, localStream);
|
|
250
|
+
const offer = await pc.createOffer();
|
|
251
|
+
await pc.setLocalDescription(offer);
|
|
252
|
+
const sdpResponse = await fetch(`${GROK_REALTIME_URL}?model=${model}`, {
|
|
253
|
+
method: "POST",
|
|
254
|
+
headers: {
|
|
255
|
+
Authorization: `Bearer ${token.token}`,
|
|
256
|
+
"Content-Type": "application/sdp"
|
|
257
|
+
},
|
|
258
|
+
body: offer.sdp
|
|
259
|
+
});
|
|
260
|
+
if (!sdpResponse.ok) {
|
|
261
|
+
const errorText = await sdpResponse.text();
|
|
262
|
+
const error = /* @__PURE__ */ new Error(`Failed to establish WebRTC connection: ${sdpResponse.status} - ${errorText}`);
|
|
263
|
+
logger.errors("grok.realtime fatal", {
|
|
264
|
+
error,
|
|
265
|
+
source: "grok.realtime.sdp",
|
|
266
|
+
status: sdpResponse.status
|
|
267
|
+
});
|
|
268
|
+
throw error;
|
|
269
|
+
}
|
|
270
|
+
const answerSdp = await sdpResponse.text();
|
|
271
|
+
await pc.setRemoteDescription({
|
|
272
|
+
type: "answer",
|
|
273
|
+
sdp: answerSdp
|
|
274
|
+
});
|
|
275
|
+
} catch (err) {
|
|
276
|
+
await teardownConnection();
|
|
277
|
+
throw err;
|
|
278
|
+
}
|
|
279
|
+
try {
|
|
280
|
+
setupInputAudioAnalysis(localStream);
|
|
281
|
+
await dataChannelReady;
|
|
282
|
+
} catch (err) {
|
|
283
|
+
await teardownConnection();
|
|
284
|
+
throw err;
|
|
285
|
+
}
|
|
286
|
+
function handleServerEvent(event) {
|
|
287
|
+
switch (readString(event, "type")) {
|
|
288
|
+
case "session.created":
|
|
289
|
+
case "session.updated": break;
|
|
290
|
+
case "input_audio_buffer.speech_started":
|
|
291
|
+
currentMode = "listening";
|
|
292
|
+
emit("mode_change", { mode: "listening" });
|
|
293
|
+
break;
|
|
294
|
+
case "input_audio_buffer.speech_stopped":
|
|
295
|
+
currentMode = "thinking";
|
|
296
|
+
emit("mode_change", { mode: "thinking" });
|
|
297
|
+
break;
|
|
298
|
+
case "input_audio_buffer.committed": break;
|
|
299
|
+
case "conversation.item.input_audio_transcription.completed": {
|
|
300
|
+
const transcript = readString(event, "transcript");
|
|
301
|
+
if (transcript === void 0) break;
|
|
302
|
+
emit("transcript", {
|
|
303
|
+
role: "user",
|
|
304
|
+
transcript,
|
|
305
|
+
isFinal: true
|
|
306
|
+
});
|
|
307
|
+
break;
|
|
308
|
+
}
|
|
309
|
+
case "response.created":
|
|
310
|
+
currentMessageId = null;
|
|
311
|
+
currentMode = "thinking";
|
|
312
|
+
emit("mode_change", { mode: "thinking" });
|
|
313
|
+
break;
|
|
314
|
+
case "response.output_item.added": {
|
|
315
|
+
const item = readObject(event, "item");
|
|
316
|
+
if (item && readString(item, "type") === "message") {
|
|
317
|
+
const id = readString(item, "id");
|
|
318
|
+
if (id !== void 0) currentMessageId = id;
|
|
319
|
+
}
|
|
320
|
+
break;
|
|
321
|
+
}
|
|
322
|
+
case "response.output_audio_transcript.delta":
|
|
323
|
+
case "response.audio_transcript.delta": {
|
|
324
|
+
const delta = readString(event, "delta");
|
|
325
|
+
if (delta === void 0) break;
|
|
326
|
+
emit("transcript", {
|
|
327
|
+
role: "assistant",
|
|
328
|
+
transcript: delta,
|
|
329
|
+
isFinal: false
|
|
330
|
+
});
|
|
331
|
+
break;
|
|
332
|
+
}
|
|
333
|
+
case "response.output_audio_transcript.done":
|
|
334
|
+
case "response.audio_transcript.done": {
|
|
335
|
+
const transcript = readString(event, "transcript");
|
|
336
|
+
if (transcript === void 0) break;
|
|
337
|
+
emit("transcript", {
|
|
338
|
+
role: "assistant",
|
|
339
|
+
transcript,
|
|
340
|
+
isFinal: true
|
|
341
|
+
});
|
|
342
|
+
break;
|
|
343
|
+
}
|
|
344
|
+
case "response.text.delta":
|
|
345
|
+
case "response.output_text.delta": {
|
|
346
|
+
const delta = readString(event, "delta");
|
|
347
|
+
if (delta === void 0) break;
|
|
348
|
+
emit("transcript", {
|
|
349
|
+
role: "assistant",
|
|
350
|
+
transcript: delta,
|
|
351
|
+
isFinal: false
|
|
352
|
+
});
|
|
353
|
+
break;
|
|
354
|
+
}
|
|
355
|
+
case "response.text.done":
|
|
356
|
+
case "response.output_text.done": {
|
|
357
|
+
const text = readString(event, "text");
|
|
358
|
+
if (text === void 0) break;
|
|
359
|
+
emit("transcript", {
|
|
360
|
+
role: "assistant",
|
|
361
|
+
transcript: text,
|
|
362
|
+
isFinal: true
|
|
363
|
+
});
|
|
364
|
+
break;
|
|
365
|
+
}
|
|
366
|
+
case "response.output_audio.delta":
|
|
367
|
+
case "response.audio.delta":
|
|
368
|
+
if (currentMode !== "speaking") {
|
|
369
|
+
currentMode = "speaking";
|
|
370
|
+
emit("mode_change", { mode: "speaking" });
|
|
371
|
+
}
|
|
372
|
+
break;
|
|
373
|
+
case "response.output_audio.done":
|
|
374
|
+
case "response.audio.done": break;
|
|
375
|
+
case "response.function_call_arguments.done": {
|
|
376
|
+
const callId = readString(event, "call_id");
|
|
377
|
+
const name = readString(event, "name") ?? "";
|
|
378
|
+
const args = readString(event, "arguments") ?? "";
|
|
379
|
+
if (!callId) {
|
|
380
|
+
logger.errors("grok.realtime tool_call missing call_id — dropping tool_call", {
|
|
381
|
+
source: "grok.realtime",
|
|
382
|
+
event_type: "response.function_call_arguments.done",
|
|
383
|
+
item_id: event.item_id
|
|
384
|
+
});
|
|
385
|
+
emit("error", { error: /* @__PURE__ */ new Error("Realtime tool call missing call_id; tool will not execute") });
|
|
386
|
+
break;
|
|
387
|
+
}
|
|
388
|
+
try {
|
|
389
|
+
emit("tool_call", {
|
|
390
|
+
toolCallId: callId,
|
|
391
|
+
toolName: name,
|
|
392
|
+
input: JSON.parse(args)
|
|
393
|
+
});
|
|
394
|
+
} catch {
|
|
395
|
+
emit("tool_call", {
|
|
396
|
+
toolCallId: callId,
|
|
397
|
+
toolName: name,
|
|
398
|
+
input: args
|
|
399
|
+
});
|
|
400
|
+
}
|
|
401
|
+
break;
|
|
402
|
+
}
|
|
403
|
+
case "response.done": {
|
|
404
|
+
const output = readObjectArray(readObject(event, "response") ?? {}, "output");
|
|
405
|
+
if (currentMode !== "idle") {
|
|
406
|
+
currentMode = "listening";
|
|
407
|
+
emit("mode_change", { mode: "listening" });
|
|
408
|
+
}
|
|
409
|
+
if (currentMessageId) {
|
|
410
|
+
const message = {
|
|
411
|
+
id: currentMessageId,
|
|
412
|
+
role: "assistant",
|
|
413
|
+
timestamp: Date.now(),
|
|
414
|
+
parts: []
|
|
415
|
+
};
|
|
416
|
+
for (const item of output ?? []) {
|
|
417
|
+
if (readString(item, "type") !== "message") continue;
|
|
418
|
+
const content = readObjectArray(item, "content");
|
|
419
|
+
if (!content) continue;
|
|
420
|
+
for (const part of content) {
|
|
421
|
+
const partType = readString(part, "type");
|
|
422
|
+
if (partType === "audio") {
|
|
423
|
+
const transcript = readString(part, "transcript");
|
|
424
|
+
if (transcript) message.parts.push({
|
|
425
|
+
type: "audio",
|
|
426
|
+
transcript
|
|
427
|
+
});
|
|
428
|
+
} else if (partType === "text") {
|
|
429
|
+
const content = readString(part, "text");
|
|
430
|
+
if (content) message.parts.push({
|
|
431
|
+
type: "text",
|
|
432
|
+
content
|
|
433
|
+
});
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
emit("message_complete", { message });
|
|
438
|
+
currentMessageId = null;
|
|
439
|
+
}
|
|
440
|
+
break;
|
|
441
|
+
}
|
|
442
|
+
case "conversation.item.truncated":
|
|
443
|
+
if (currentMode !== "idle") {
|
|
444
|
+
currentMode = "listening";
|
|
445
|
+
emit("mode_change", { mode: "listening" });
|
|
446
|
+
}
|
|
447
|
+
emit("interrupted", { ...currentMessageId !== null && { messageId: currentMessageId } });
|
|
448
|
+
break;
|
|
449
|
+
case "error": {
|
|
450
|
+
const errorObj = readObject(event, "error") ?? {};
|
|
451
|
+
const message = readString(errorObj, "message") ?? "Unknown realtime server error";
|
|
452
|
+
const err = new Error(message);
|
|
453
|
+
const code = readString(errorObj, "code");
|
|
454
|
+
if (code !== void 0) err.code = code;
|
|
455
|
+
const errType = readString(errorObj, "type");
|
|
456
|
+
if (errType !== void 0) err.type = errType;
|
|
457
|
+
const param = readString(errorObj, "param");
|
|
458
|
+
if (param !== void 0) err.param = param;
|
|
459
|
+
logger.errors("grok.realtime server error", {
|
|
460
|
+
...errorObj,
|
|
461
|
+
source: "grok.realtime server"
|
|
462
|
+
});
|
|
463
|
+
emit("error", { error: err });
|
|
464
|
+
break;
|
|
465
|
+
}
|
|
466
|
+
case void 0:
|
|
467
|
+
default:
|
|
468
|
+
logger.provider("grok.realtime unhandled server event", { type: event.type });
|
|
469
|
+
break;
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
function setupOutputAudioAnalysis(stream) {
|
|
473
|
+
if (isTornDown) return;
|
|
474
|
+
if (audioElement) {
|
|
475
|
+
try {
|
|
476
|
+
audioElement.pause();
|
|
477
|
+
} catch {}
|
|
478
|
+
audioElement.srcObject = null;
|
|
479
|
+
audioElement = null;
|
|
480
|
+
}
|
|
481
|
+
if (outputSource) {
|
|
482
|
+
try {
|
|
483
|
+
outputSource.disconnect();
|
|
484
|
+
} catch {}
|
|
485
|
+
outputSource = null;
|
|
486
|
+
}
|
|
487
|
+
if (outputAnalyser) {
|
|
488
|
+
try {
|
|
489
|
+
outputAnalyser.disconnect();
|
|
490
|
+
} catch {}
|
|
491
|
+
outputAnalyser = null;
|
|
492
|
+
}
|
|
493
|
+
audioElement = new Audio();
|
|
494
|
+
audioElement.srcObject = stream;
|
|
495
|
+
audioElement.autoplay = true;
|
|
496
|
+
audioElement.play().catch((e) => {
|
|
497
|
+
logger.errors("grok.realtime audio autoplay blocked", {
|
|
498
|
+
error: e,
|
|
499
|
+
source: "grok.realtime.audio_permission_required"
|
|
500
|
+
});
|
|
501
|
+
});
|
|
502
|
+
if (!audioContext) audioContext = new AudioContext();
|
|
503
|
+
if (audioContext.state === "suspended") audioContext.resume().catch((err) => {
|
|
504
|
+
logger.errors("grok.realtime audioContext.resume failed", {
|
|
505
|
+
error: err,
|
|
506
|
+
source: "grok.realtime"
|
|
507
|
+
});
|
|
508
|
+
});
|
|
509
|
+
outputAnalyser = audioContext.createAnalyser();
|
|
510
|
+
outputAnalyser.fftSize = 2048;
|
|
511
|
+
outputAnalyser.smoothingTimeConstant = .3;
|
|
512
|
+
outputSource = audioContext.createMediaStreamSource(stream);
|
|
513
|
+
outputSource.connect(outputAnalyser);
|
|
514
|
+
}
|
|
515
|
+
function setupInputAudioAnalysis(stream) {
|
|
516
|
+
if (isTornDown) return;
|
|
517
|
+
if (!audioContext) audioContext = new AudioContext();
|
|
518
|
+
if (audioContext.state === "suspended") audioContext.resume().catch((err) => {
|
|
519
|
+
logger.errors("grok.realtime audioContext.resume failed", {
|
|
520
|
+
error: err,
|
|
521
|
+
source: "grok.realtime"
|
|
522
|
+
});
|
|
523
|
+
});
|
|
524
|
+
inputAnalyser = audioContext.createAnalyser();
|
|
525
|
+
inputAnalyser.fftSize = 2048;
|
|
526
|
+
inputAnalyser.smoothingTimeConstant = .3;
|
|
527
|
+
inputSource = audioContext.createMediaStreamSource(stream);
|
|
528
|
+
inputSource.connect(inputAnalyser);
|
|
529
|
+
}
|
|
530
|
+
function sendEvent(event) {
|
|
531
|
+
if (isTornDown) {
|
|
532
|
+
logger.errors("grok.realtime sendEvent after disconnect", {
|
|
533
|
+
eventType: readString(event, "type") ?? "<unknown>",
|
|
534
|
+
source: "grok.realtime"
|
|
535
|
+
});
|
|
536
|
+
return;
|
|
537
|
+
}
|
|
538
|
+
if (dataChannel?.readyState === "open") {
|
|
539
|
+
logger.provider(`provider=grok direction=out type=${readString(event, "type") ?? "<unknown>"}`, { frame: event });
|
|
540
|
+
try {
|
|
541
|
+
dataChannel.send(JSON.stringify(event));
|
|
542
|
+
} catch (error) {
|
|
543
|
+
logger.errors("grok.realtime sendEvent failed", {
|
|
544
|
+
error,
|
|
545
|
+
eventType: readString(event, "type") ?? "<unknown>",
|
|
546
|
+
source: "grok.realtime"
|
|
547
|
+
});
|
|
548
|
+
emit("error", { error: error instanceof Error ? error : new Error(String(error)) });
|
|
549
|
+
}
|
|
550
|
+
} else pendingEvents.push(event);
|
|
551
|
+
}
|
|
552
|
+
function flushPendingEvents() {
|
|
553
|
+
try {
|
|
554
|
+
for (const event of pendingEvents) {
|
|
555
|
+
logger.provider(`provider=grok direction=out type=${readString(event, "type") ?? "<unknown>"}`, { frame: event });
|
|
556
|
+
channel.send(JSON.stringify(event));
|
|
557
|
+
}
|
|
558
|
+
pendingEvents.length = 0;
|
|
559
|
+
} catch (error) {
|
|
560
|
+
logger.errors("grok.realtime flushPendingEvents failed", {
|
|
561
|
+
error,
|
|
562
|
+
source: "grok.realtime"
|
|
563
|
+
});
|
|
564
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
565
|
+
rejectDataChannelReady?.(err);
|
|
566
|
+
emit("error", { error: err });
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
return {
|
|
570
|
+
async disconnect() {
|
|
571
|
+
await teardownConnection();
|
|
572
|
+
emit("status_change", { status: "idle" });
|
|
573
|
+
},
|
|
574
|
+
async startAudioCapture() {
|
|
575
|
+
if (localStream) for (const track of localStream.getAudioTracks()) track.enabled = true;
|
|
576
|
+
currentMode = "listening";
|
|
577
|
+
emit("mode_change", { mode: "listening" });
|
|
578
|
+
},
|
|
579
|
+
stopAudioCapture() {
|
|
580
|
+
if (localStream) for (const track of localStream.getAudioTracks()) track.enabled = false;
|
|
581
|
+
currentMode = "idle";
|
|
582
|
+
emit("mode_change", { mode: "idle" });
|
|
583
|
+
},
|
|
584
|
+
sendText(text) {
|
|
585
|
+
sendEvent({
|
|
586
|
+
type: "conversation.item.create",
|
|
587
|
+
item: {
|
|
588
|
+
type: "message",
|
|
589
|
+
role: "user",
|
|
590
|
+
content: [{
|
|
591
|
+
type: "input_text",
|
|
592
|
+
text
|
|
593
|
+
}]
|
|
594
|
+
}
|
|
595
|
+
});
|
|
596
|
+
sendEvent({ type: "response.create" });
|
|
597
|
+
},
|
|
598
|
+
sendImage(imageData, mimeType) {
|
|
599
|
+
sendEvent({
|
|
600
|
+
type: "conversation.item.create",
|
|
601
|
+
item: {
|
|
602
|
+
type: "message",
|
|
603
|
+
role: "user",
|
|
604
|
+
content: [{
|
|
605
|
+
type: "input_image",
|
|
606
|
+
image_url: { url: imageData.startsWith("http://") || imageData.startsWith("https://") || imageData.startsWith("data:") ? imageData : `data:${mimeType};base64,${imageData}` }
|
|
607
|
+
}]
|
|
608
|
+
}
|
|
609
|
+
});
|
|
610
|
+
sendEvent({ type: "response.create" });
|
|
611
|
+
},
|
|
612
|
+
sendToolResult(callId, result) {
|
|
613
|
+
sendEvent({
|
|
614
|
+
type: "conversation.item.create",
|
|
615
|
+
item: {
|
|
616
|
+
type: "function_call_output",
|
|
617
|
+
call_id: callId,
|
|
618
|
+
output: result
|
|
619
|
+
}
|
|
620
|
+
});
|
|
621
|
+
sendEvent({ type: "response.create" });
|
|
622
|
+
},
|
|
623
|
+
updateSession(config) {
|
|
624
|
+
const sessionUpdate = {};
|
|
625
|
+
if (config.instructions) sessionUpdate.instructions = config.instructions;
|
|
626
|
+
if (config.voice) sessionUpdate.voice = config.voice;
|
|
627
|
+
if (config.vadMode) if (config.vadMode === "semantic") sessionUpdate.turn_detection = {
|
|
628
|
+
type: "semantic_vad",
|
|
629
|
+
eagerness: config.semanticEagerness ?? "medium"
|
|
630
|
+
};
|
|
631
|
+
else if (config.vadMode === "server") sessionUpdate.turn_detection = {
|
|
632
|
+
type: "server_vad",
|
|
633
|
+
threshold: config.vadConfig?.threshold ?? .5,
|
|
634
|
+
prefix_padding_ms: config.vadConfig?.prefixPaddingMs ?? 300,
|
|
635
|
+
silence_duration_ms: config.vadConfig?.silenceDurationMs ?? 500
|
|
636
|
+
};
|
|
637
|
+
else sessionUpdate.turn_detection = null;
|
|
638
|
+
if (config.tools !== void 0) {
|
|
639
|
+
sessionUpdate.tools = config.tools.map((t) => ({
|
|
640
|
+
type: "function",
|
|
641
|
+
name: t.name,
|
|
642
|
+
description: t.description,
|
|
643
|
+
parameters: t.inputSchema ?? {
|
|
644
|
+
type: "object",
|
|
645
|
+
properties: {}
|
|
646
|
+
}
|
|
647
|
+
}));
|
|
648
|
+
sessionUpdate.tool_choice = "auto";
|
|
649
|
+
}
|
|
650
|
+
if (config.outputModalities) sessionUpdate.modalities = config.outputModalities;
|
|
651
|
+
if (config.temperature !== void 0) sessionUpdate.temperature = config.temperature;
|
|
652
|
+
if (config.maxOutputTokens !== void 0) sessionUpdate.max_response_output_tokens = config.maxOutputTokens;
|
|
653
|
+
const providerOptions = config.providerOptions ?? {};
|
|
654
|
+
const callerTranscription = "inputAudioTranscription" in providerOptions ? providerOptions.inputAudioTranscription : "input_audio_transcription" in providerOptions ? providerOptions.input_audio_transcription : void 0;
|
|
655
|
+
if (callerTranscription !== void 0) sessionUpdate.input_audio_transcription = callerTranscription === false ? null : callerTranscription;
|
|
656
|
+
else if (!hasSentInitialSessionUpdate) sessionUpdate.input_audio_transcription = { model: "grok-stt" };
|
|
657
|
+
if (Object.keys(sessionUpdate).length > 0) {
|
|
658
|
+
sendEvent({
|
|
659
|
+
type: "session.update",
|
|
660
|
+
session: sessionUpdate
|
|
661
|
+
});
|
|
662
|
+
hasSentInitialSessionUpdate = true;
|
|
663
|
+
}
|
|
664
|
+
},
|
|
665
|
+
interrupt() {
|
|
666
|
+
sendEvent({ type: "response.cancel" });
|
|
667
|
+
currentMode = "listening";
|
|
668
|
+
emit("mode_change", { mode: "listening" });
|
|
669
|
+
emit("interrupted", { ...currentMessageId !== null && { messageId: currentMessageId } });
|
|
670
|
+
},
|
|
671
|
+
on(event, handler) {
|
|
672
|
+
let handlers = eventHandlers.get(event);
|
|
673
|
+
if (!handlers) {
|
|
674
|
+
handlers = /* @__PURE__ */ new Set();
|
|
675
|
+
eventHandlers.set(event, handlers);
|
|
676
|
+
}
|
|
677
|
+
handlers.add(handler);
|
|
678
|
+
return () => {
|
|
679
|
+
eventHandlers.get(event)?.delete(handler);
|
|
680
|
+
};
|
|
681
|
+
},
|
|
682
|
+
getAudioVisualization() {
|
|
683
|
+
function calculateLevel(analyser) {
|
|
684
|
+
const data = new Uint8Array(analyser.fftSize);
|
|
685
|
+
analyser.getByteTimeDomainData(data);
|
|
686
|
+
let maxDeviation = 0;
|
|
687
|
+
for (const sample of data) {
|
|
688
|
+
const deviation = Math.abs(sample - 128);
|
|
689
|
+
if (deviation > maxDeviation) maxDeviation = deviation;
|
|
690
|
+
}
|
|
691
|
+
const normalized = maxDeviation / 128;
|
|
692
|
+
return Math.min(1, normalized * 1.5);
|
|
693
|
+
}
|
|
694
|
+
return {
|
|
695
|
+
get inputLevel() {
|
|
696
|
+
if (!inputAnalyser) return 0;
|
|
697
|
+
return calculateLevel(inputAnalyser);
|
|
698
|
+
},
|
|
699
|
+
get outputLevel() {
|
|
700
|
+
if (!outputAnalyser) return 0;
|
|
701
|
+
return calculateLevel(outputAnalyser);
|
|
702
|
+
},
|
|
703
|
+
getInputFrequencyData() {
|
|
704
|
+
if (!inputAnalyser) return new Uint8Array(FALLBACK_FREQUENCY_BIN_COUNT);
|
|
705
|
+
const data = new Uint8Array(inputAnalyser.frequencyBinCount);
|
|
706
|
+
inputAnalyser.getByteFrequencyData(data);
|
|
707
|
+
return data;
|
|
708
|
+
},
|
|
709
|
+
getOutputFrequencyData() {
|
|
710
|
+
if (!outputAnalyser) return new Uint8Array(FALLBACK_FREQUENCY_BIN_COUNT);
|
|
711
|
+
const data = new Uint8Array(outputAnalyser.frequencyBinCount);
|
|
712
|
+
outputAnalyser.getByteFrequencyData(data);
|
|
713
|
+
return data;
|
|
714
|
+
},
|
|
715
|
+
getInputTimeDomainData() {
|
|
716
|
+
if (!inputAnalyser) return new Uint8Array(FALLBACK_TIME_DOMAIN_SIZE).fill(FALLBACK_TIME_DOMAIN_FILL);
|
|
717
|
+
const data = new Uint8Array(inputAnalyser.fftSize);
|
|
718
|
+
inputAnalyser.getByteTimeDomainData(data);
|
|
719
|
+
return data;
|
|
720
|
+
},
|
|
721
|
+
getOutputTimeDomainData() {
|
|
722
|
+
if (!outputAnalyser) return new Uint8Array(FALLBACK_TIME_DOMAIN_SIZE).fill(FALLBACK_TIME_DOMAIN_FILL);
|
|
723
|
+
const data = new Uint8Array(outputAnalyser.fftSize);
|
|
724
|
+
outputAnalyser.getByteTimeDomainData(data);
|
|
725
|
+
return data;
|
|
726
|
+
},
|
|
727
|
+
get inputSampleRate() {
|
|
728
|
+
return 24e3;
|
|
729
|
+
},
|
|
730
|
+
get outputSampleRate() {
|
|
731
|
+
return 24e3;
|
|
732
|
+
}
|
|
733
|
+
};
|
|
734
|
+
}
|
|
735
|
+
};
|
|
819
736
|
}
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
//# sourceMappingURL=adapter.js.map
|
|
737
|
+
//#endregion
|
|
738
|
+
export { grokRealtime };
|
|
739
|
+
|
|
740
|
+
//# sourceMappingURL=adapter.js.map
|