@tanstack/ai-grok 0.14.8 → 0.14.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/dist/esm/adapters/image.js +228 -193
  2. package/dist/esm/adapters/image.js.map +1 -1
  3. package/dist/esm/adapters/summarize.js +49 -12
  4. package/dist/esm/adapters/summarize.js.map +1 -1
  5. package/dist/esm/adapters/text.js +82 -38
  6. package/dist/esm/adapters/text.js.map +1 -1
  7. package/dist/esm/adapters/transcription.js +133 -114
  8. package/dist/esm/adapters/transcription.js.map +1 -1
  9. package/dist/esm/adapters/tts.js +147 -130
  10. package/dist/esm/adapters/tts.js.map +1 -1
  11. package/dist/esm/adapters/video.js +275 -226
  12. package/dist/esm/adapters/video.js.map +1 -1
  13. package/dist/esm/image/image-provider-options.js +64 -66
  14. package/dist/esm/image/image-provider-options.js.map +1 -1
  15. package/dist/esm/index.js +3 -31
  16. package/dist/esm/model-meta.js +149 -47
  17. package/dist/esm/model-meta.js.map +1 -1
  18. package/dist/esm/realtime/adapter.js +729 -812
  19. package/dist/esm/realtime/adapter.js.map +1 -1
  20. package/dist/esm/realtime/index.js +3 -0
  21. package/dist/esm/realtime/token.js +78 -72
  22. package/dist/esm/realtime/token.js.map +1 -1
  23. package/dist/esm/tools/index.js +57 -96
  24. package/dist/esm/tools/index.js.map +1 -1
  25. package/dist/esm/utils/audio.js +116 -162
  26. package/dist/esm/utils/audio.js.map +1 -1
  27. package/dist/esm/utils/client.js +22 -16
  28. package/dist/esm/utils/client.js.map +1 -1
  29. package/dist/esm/utils/index.js +5 -0
  30. package/dist/esm/utils/schema-converter.js +2 -0
  31. package/dist/esm/video/video-provider-options.js +83 -57
  32. package/dist/esm/video/video-provider-options.js.map +1 -1
  33. package/package.json +8 -8
  34. package/src/realtime/adapter.ts +1 -1
  35. package/src/utils/audio.ts +1 -1
  36. package/dist/esm/index.js.map +0 -1
@@ -1,823 +1,740 @@
1
1
  import { resolveDebugOption } from "@tanstack/ai/adapter-internals";
2
- const GROK_REALTIME_URL = "https://api.x.ai/v1/realtime";
2
+ //#region src/realtime/adapter.ts
3
+ var GROK_REALTIME_URL = "https://api.x.ai/v1/realtime";
4
+ /**
5
+ * Runtime-checked field readers for untyped server events. Replace the
6
+ * drive-by `event.X as string` / `event.X as Record<string, unknown>` casts
7
+ * with readers that return `undefined` when the shape doesn't match, so a
8
+ * malformed frame can't throw a TypeError inside `handleServerEvent`.
9
+ */
3
10
  function readString(obj, key) {
4
- const value = obj[key];
5
- return typeof value === "string" ? value : void 0;
11
+ const value = obj[key];
12
+ return typeof value === "string" ? value : void 0;
6
13
  }
7
14
  function readObject(obj, key) {
8
- const value = obj[key];
9
- return value && typeof value === "object" && !Array.isArray(value) ? value : void 0;
15
+ const value = obj[key];
16
+ return value && typeof value === "object" && !Array.isArray(value) ? value : void 0;
10
17
  }
11
18
  function readObjectArray(obj, key) {
12
- const value = obj[key];
13
- if (!Array.isArray(value)) return void 0;
14
- return value.filter(
15
- (item) => item !== null && typeof item === "object" && !Array.isArray(item)
16
- );
19
+ const value = obj[key];
20
+ if (!Array.isArray(value)) return void 0;
21
+ return value.filter((item) => item !== null && typeof item === "object" && !Array.isArray(item));
17
22
  }
23
+ /**
24
+ * Creates a Grok realtime adapter for client-side use.
25
+ *
26
+ * Uses WebRTC for browser connections (default). Mirrors the OpenAI realtime
27
+ * adapter because xAI's Voice Agent API is OpenAI-realtime-compatible — the
28
+ * only differences are the endpoint URL and default model.
29
+ *
30
+ * @example
31
+ * ```typescript
32
+ * import { RealtimeClient } from '@tanstack/ai-client'
33
+ * import { grokRealtime } from '@tanstack/ai-grok'
34
+ *
35
+ * const client = new RealtimeClient({
36
+ * getToken: () => fetch('/api/realtime-token').then(r => r.json()),
37
+ * adapter: grokRealtime(),
38
+ * })
39
+ * ```
40
+ */
18
41
  function grokRealtime(options = {}) {
19
- const connectionMode = options.connectionMode ?? "webrtc";
20
- const logger = resolveDebugOption(options.debug);
21
- return {
22
- provider: "grok",
23
- async connect(token, _clientTools) {
24
- const model = token.config.model ?? "grok-voice-fast-1.0";
25
- logger.request(`activity=realtime provider=grok model=${model}`, {
26
- provider: "grok",
27
- model
28
- });
29
- if (connectionMode === "webrtc") {
30
- return createWebRTCConnection(token, logger);
31
- }
32
- const error = new Error("WebSocket connection mode not yet implemented");
33
- logger.errors("grok.realtime fatal", {
34
- error,
35
- source: "grok.realtime"
36
- });
37
- throw error;
38
- }
39
- };
42
+ const connectionMode = options.connectionMode ?? "webrtc";
43
+ const logger = resolveDebugOption(options.debug);
44
+ return {
45
+ provider: "grok",
46
+ async connect(token, _clientTools) {
47
+ const model = token.config.model ?? "grok-voice-fast-1.0";
48
+ logger.request(`activity=realtime provider=grok model=${model}`, {
49
+ provider: "grok",
50
+ model
51
+ });
52
+ if (connectionMode === "webrtc") return createWebRTCConnection(token, logger);
53
+ const error = /* @__PURE__ */ new Error("WebSocket connection mode not yet implemented");
54
+ logger.errors("grok.realtime fatal", {
55
+ error,
56
+ source: "grok.realtime"
57
+ });
58
+ throw error;
59
+ }
60
+ };
40
61
  }
62
+ /**
63
+ * Creates a WebRTC connection to xAI's realtime API.
64
+ */
41
65
  async function createWebRTCConnection(token, logger) {
42
- const model = token.config.model ?? "grok-voice-fast-1.0";
43
- const eventHandlers = /* @__PURE__ */ new Map();
44
- const pc = new RTCPeerConnection();
45
- let audioContext = null;
46
- let inputAnalyser = null;
47
- let outputAnalyser = null;
48
- let inputSource = null;
49
- let outputSource = null;
50
- let localStream = null;
51
- let audioElement = null;
52
- const channel = pc.createDataChannel("oai-events");
53
- let dataChannel = channel;
54
- let currentMode = "idle";
55
- let currentMessageId = null;
56
- let isTornDown = false;
57
- const pendingEvents = [];
58
- let hasSentInitialSessionUpdate = false;
59
- const FALLBACK_FREQUENCY_BIN_COUNT = 1024;
60
- const FALLBACK_TIME_DOMAIN_SIZE = 2048;
61
- const FALLBACK_TIME_DOMAIN_FILL = 128;
62
- function emit(event, payload) {
63
- const handlers = eventHandlers.get(event);
64
- if (handlers) {
65
- for (const handler of handlers) {
66
- handler(payload);
67
- }
68
- }
69
- }
70
- let dataChannelOpened = false;
71
- let rejectDataChannelReady = null;
72
- let dataChannelReadyTimeout = null;
73
- const dataChannelReady = new Promise((resolve, reject) => {
74
- rejectDataChannelReady = (reason) => {
75
- if (dataChannelReadyTimeout !== null) {
76
- clearTimeout(dataChannelReadyTimeout);
77
- dataChannelReadyTimeout = null;
78
- }
79
- rejectDataChannelReady = null;
80
- reject(reason);
81
- };
82
- dataChannelReadyTimeout = setTimeout(() => {
83
- if (!dataChannelOpened) {
84
- rejectDataChannelReady?.(
85
- new Error(
86
- "Data channel did not open within 15000ms — aborting connection"
87
- )
88
- );
89
- }
90
- }, 15e3);
91
- channel.onopen = () => {
92
- dataChannelOpened = true;
93
- if (dataChannelReadyTimeout !== null) {
94
- clearTimeout(dataChannelReadyTimeout);
95
- dataChannelReadyTimeout = null;
96
- }
97
- rejectDataChannelReady = null;
98
- flushPendingEvents();
99
- emit("status_change", { status: "connected" });
100
- resolve();
101
- };
102
- });
103
- channel.onmessage = (event) => {
104
- try {
105
- const message = JSON.parse(event.data);
106
- const messageRecord = message !== null && typeof message === "object" ? message : {};
107
- logger.provider(
108
- `provider=grok direction=in type=${readString(messageRecord, "type") ?? "<unknown>"}`,
109
- { frame: messageRecord }
110
- );
111
- handleServerEvent(messageRecord);
112
- } catch (parseErr) {
113
- logger.errors("grok.realtime fatal", {
114
- error: parseErr,
115
- source: "grok.realtime"
116
- });
117
- emit("error", {
118
- error: parseErr instanceof Error ? parseErr : new Error(String(parseErr))
119
- });
120
- }
121
- };
122
- channel.onerror = (error) => {
123
- if (isTornDown) return;
124
- logger.errors("grok.realtime fatal", {
125
- error,
126
- source: "grok.realtime"
127
- });
128
- const errorRecord = error;
129
- const rtcError = readObject(errorRecord, "error");
130
- const msg = (rtcError && readString(rtcError, "message")) ?? (error.type || "unknown");
131
- const dcErr = new Error(`Data channel error: ${msg}`);
132
- if (!dataChannelOpened) {
133
- rejectDataChannelReady?.(dcErr);
134
- }
135
- emit("error", { error: dcErr });
136
- };
137
- channel.onclose = () => {
138
- if (isTornDown) return;
139
- if (!dataChannelOpened) {
140
- rejectDataChannelReady?.(new Error("Data channel closed before opening"));
141
- }
142
- };
143
- pc.ontrack = (event) => {
144
- if (event.track.kind === "audio" && event.streams[0]) {
145
- setupOutputAudioAnalysis(event.streams[0]);
146
- }
147
- };
148
- pc.onconnectionstatechange = () => {
149
- const state = pc.connectionState;
150
- logger.provider(`provider=grok pc.connectionState=${state}`, {
151
- state
152
- });
153
- if (state === "failed" || state === "disconnected" || state === "closed") {
154
- if (!isTornDown) {
155
- emit("status_change", {
156
- status: state === "failed" ? "error" : "idle"
157
- });
158
- }
159
- if (!dataChannelOpened) {
160
- const message = state === "failed" ? `PeerConnection failed before data channel opened` : `PeerConnection entered state '${state}' before data channel opened`;
161
- rejectDataChannelReady?.(new Error(message));
162
- }
163
- if (state === "failed" && !isTornDown) {
164
- void teardownConnection();
165
- }
166
- }
167
- };
168
- pc.oniceconnectionstatechange = () => {
169
- const state = pc.iceConnectionState;
170
- logger.provider(`provider=grok pc.iceConnectionState=${state}`, {
171
- state
172
- });
173
- if (!dataChannelOpened && (state === "failed" || state === "closed" || state === "disconnected")) {
174
- const message = state === "failed" ? `ICE connection failed before data channel opened` : `ICE connection entered state '${state}' before data channel opened`;
175
- rejectDataChannelReady?.(new Error(message));
176
- }
177
- };
178
- async function teardownConnection() {
179
- isTornDown = true;
180
- pendingEvents.length = 0;
181
- rejectDataChannelReady?.(
182
- new Error("Connection torn down before data channel opened")
183
- );
184
- if (localStream) {
185
- for (const track of localStream.getTracks()) {
186
- track.stop();
187
- }
188
- localStream = null;
189
- }
190
- if (audioElement) {
191
- try {
192
- audioElement.pause();
193
- } catch {
194
- }
195
- audioElement.srcObject = null;
196
- audioElement = null;
197
- }
198
- if (outputSource) {
199
- try {
200
- outputSource.disconnect();
201
- } catch {
202
- }
203
- outputSource = null;
204
- }
205
- if (outputAnalyser) {
206
- try {
207
- outputAnalyser.disconnect();
208
- } catch {
209
- }
210
- outputAnalyser = null;
211
- }
212
- if (inputSource) {
213
- try {
214
- inputSource.disconnect();
215
- } catch {
216
- }
217
- inputSource = null;
218
- }
219
- if (inputAnalyser) {
220
- try {
221
- inputAnalyser.disconnect();
222
- } catch {
223
- }
224
- inputAnalyser = null;
225
- }
226
- if (dataChannel) {
227
- try {
228
- dataChannel.close();
229
- } catch {
230
- }
231
- dataChannel = null;
232
- }
233
- try {
234
- pc.close();
235
- } catch {
236
- }
237
- if (audioContext) {
238
- try {
239
- await audioContext.close();
240
- } catch {
241
- }
242
- audioContext = null;
243
- }
244
- }
245
- try {
246
- try {
247
- localStream = await navigator.mediaDevices.getUserMedia({
248
- audio: {
249
- echoCancellation: true,
250
- noiseSuppression: true,
251
- sampleRate: 24e3
252
- }
253
- });
254
- } catch (error) {
255
- logger.errors("grok.realtime fatal", {
256
- error,
257
- source: "grok.realtime.getUserMedia"
258
- });
259
- throw new Error(
260
- `Microphone access required for realtime voice: ${error instanceof Error ? error.message : error}`
261
- );
262
- }
263
- for (const track of localStream.getAudioTracks()) {
264
- pc.addTrack(track, localStream);
265
- }
266
- const offer = await pc.createOffer();
267
- await pc.setLocalDescription(offer);
268
- const sdpResponse = await fetch(`${GROK_REALTIME_URL}?model=${model}`, {
269
- method: "POST",
270
- headers: {
271
- Authorization: `Bearer ${token.token}`,
272
- "Content-Type": "application/sdp"
273
- },
274
- body: offer.sdp
275
- });
276
- if (!sdpResponse.ok) {
277
- const errorText = await sdpResponse.text();
278
- const error = new Error(
279
- `Failed to establish WebRTC connection: ${sdpResponse.status} - ${errorText}`
280
- );
281
- logger.errors("grok.realtime fatal", {
282
- error,
283
- source: "grok.realtime.sdp",
284
- status: sdpResponse.status
285
- });
286
- throw error;
287
- }
288
- const answerSdp = await sdpResponse.text();
289
- await pc.setRemoteDescription({ type: "answer", sdp: answerSdp });
290
- } catch (err) {
291
- await teardownConnection();
292
- throw err;
293
- }
294
- try {
295
- setupInputAudioAnalysis(localStream);
296
- await dataChannelReady;
297
- } catch (err) {
298
- await teardownConnection();
299
- throw err;
300
- }
301
- function handleServerEvent(event) {
302
- const type = readString(event, "type");
303
- switch (type) {
304
- case "session.created":
305
- case "session.updated":
306
- break;
307
- case "input_audio_buffer.speech_started":
308
- currentMode = "listening";
309
- emit("mode_change", { mode: "listening" });
310
- break;
311
- case "input_audio_buffer.speech_stopped":
312
- currentMode = "thinking";
313
- emit("mode_change", { mode: "thinking" });
314
- break;
315
- case "input_audio_buffer.committed":
316
- break;
317
- case "conversation.item.input_audio_transcription.completed": {
318
- const transcript = readString(event, "transcript");
319
- if (transcript === void 0) break;
320
- emit("transcript", { role: "user", transcript, isFinal: true });
321
- break;
322
- }
323
- case "response.created":
324
- currentMessageId = null;
325
- currentMode = "thinking";
326
- emit("mode_change", { mode: "thinking" });
327
- break;
328
- case "response.output_item.added": {
329
- const item = readObject(event, "item");
330
- if (item && readString(item, "type") === "message") {
331
- const id = readString(item, "id");
332
- if (id !== void 0) currentMessageId = id;
333
- }
334
- break;
335
- }
336
- // xAI realtime per docs uses `response.output_audio_transcript.*`;
337
- // accept the legacy OpenAI-realtime `response.audio_transcript.*` as
338
- // an alias so this adapter stays compatible across protocol versions.
339
- case "response.output_audio_transcript.delta":
340
- case "response.audio_transcript.delta": {
341
- const delta = readString(event, "delta");
342
- if (delta === void 0) break;
343
- emit("transcript", {
344
- role: "assistant",
345
- transcript: delta,
346
- isFinal: false
347
- });
348
- break;
349
- }
350
- case "response.output_audio_transcript.done":
351
- case "response.audio_transcript.done": {
352
- const transcript = readString(event, "transcript");
353
- if (transcript === void 0) break;
354
- emit("transcript", { role: "assistant", transcript, isFinal: true });
355
- break;
356
- }
357
- // xAI realtime per docs uses `response.text.*`; accept the legacy
358
- // OpenAI-realtime `response.output_text.*` as an alias.
359
- case "response.text.delta":
360
- case "response.output_text.delta": {
361
- const delta = readString(event, "delta");
362
- if (delta === void 0) break;
363
- emit("transcript", {
364
- role: "assistant",
365
- transcript: delta,
366
- isFinal: false
367
- });
368
- break;
369
- }
370
- case "response.text.done":
371
- case "response.output_text.done": {
372
- const text = readString(event, "text");
373
- if (text === void 0) break;
374
- emit("transcript", {
375
- role: "assistant",
376
- transcript: text,
377
- isFinal: true
378
- });
379
- break;
380
- }
381
- // xAI realtime per docs uses `response.output_audio.*`; accept the
382
- // legacy OpenAI-realtime `response.audio.*` as an alias.
383
- case "response.output_audio.delta":
384
- case "response.audio.delta":
385
- if (currentMode !== "speaking") {
386
- currentMode = "speaking";
387
- emit("mode_change", { mode: "speaking" });
388
- }
389
- break;
390
- case "response.output_audio.done":
391
- case "response.audio.done":
392
- break;
393
- case "response.function_call_arguments.done": {
394
- const callId = readString(event, "call_id");
395
- const name = readString(event, "name") ?? "";
396
- const args = readString(event, "arguments") ?? "";
397
- if (!callId) {
398
- logger.errors(
399
- "grok.realtime tool_call missing call_id — dropping tool_call",
400
- {
401
- source: "grok.realtime",
402
- event_type: "response.function_call_arguments.done",
403
- item_id: event.item_id
404
- }
405
- );
406
- emit("error", {
407
- error: new Error(
408
- "Realtime tool call missing call_id; tool will not execute"
409
- )
410
- });
411
- break;
412
- }
413
- try {
414
- const input = JSON.parse(args);
415
- emit("tool_call", { toolCallId: callId, toolName: name, input });
416
- } catch {
417
- emit("tool_call", { toolCallId: callId, toolName: name, input: args });
418
- }
419
- break;
420
- }
421
- case "response.done": {
422
- const response = readObject(event, "response") ?? {};
423
- const output = readObjectArray(response, "output");
424
- if (currentMode !== "idle") {
425
- currentMode = "listening";
426
- emit("mode_change", { mode: "listening" });
427
- }
428
- if (currentMessageId) {
429
- const message = {
430
- id: currentMessageId,
431
- role: "assistant",
432
- timestamp: Date.now(),
433
- parts: []
434
- };
435
- for (const item of output ?? []) {
436
- if (readString(item, "type") !== "message") continue;
437
- const content = readObjectArray(item, "content");
438
- if (!content) continue;
439
- for (const part of content) {
440
- const partType = readString(part, "type");
441
- if (partType === "audio") {
442
- const transcript = readString(part, "transcript");
443
- if (transcript) {
444
- message.parts.push({ type: "audio", transcript });
445
- }
446
- } else if (partType === "text") {
447
- const content2 = readString(part, "text");
448
- if (content2) {
449
- message.parts.push({ type: "text", content: content2 });
450
- }
451
- }
452
- }
453
- }
454
- emit("message_complete", { message });
455
- currentMessageId = null;
456
- }
457
- break;
458
- }
459
- case "conversation.item.truncated":
460
- if (currentMode !== "idle") {
461
- currentMode = "listening";
462
- emit("mode_change", { mode: "listening" });
463
- }
464
- emit("interrupted", {
465
- ...currentMessageId !== null && { messageId: currentMessageId }
466
- });
467
- break;
468
- case "error": {
469
- const errorObj = readObject(event, "error") ?? {};
470
- const message = readString(errorObj, "message") ?? "Unknown realtime server error";
471
- const err = new Error(message);
472
- const code = readString(errorObj, "code");
473
- if (code !== void 0) err.code = code;
474
- const errType = readString(errorObj, "type");
475
- if (errType !== void 0) err.type = errType;
476
- const param = readString(errorObj, "param");
477
- if (param !== void 0) err.param = param;
478
- logger.errors("grok.realtime server error", {
479
- ...errorObj,
480
- source: "grok.realtime server"
481
- });
482
- emit("error", { error: err });
483
- break;
484
- }
485
- case void 0:
486
- default:
487
- logger.provider("grok.realtime unhandled server event", {
488
- type: event.type
489
- });
490
- break;
491
- }
492
- }
493
- function setupOutputAudioAnalysis(stream) {
494
- if (isTornDown) return;
495
- if (audioElement) {
496
- try {
497
- audioElement.pause();
498
- } catch {
499
- }
500
- audioElement.srcObject = null;
501
- audioElement = null;
502
- }
503
- if (outputSource) {
504
- try {
505
- outputSource.disconnect();
506
- } catch {
507
- }
508
- outputSource = null;
509
- }
510
- if (outputAnalyser) {
511
- try {
512
- outputAnalyser.disconnect();
513
- } catch {
514
- }
515
- outputAnalyser = null;
516
- }
517
- audioElement = new Audio();
518
- audioElement.srcObject = stream;
519
- audioElement.autoplay = true;
520
- audioElement.play().catch((e) => {
521
- logger.errors("grok.realtime audio autoplay blocked", {
522
- error: e,
523
- source: "grok.realtime.audio_permission_required"
524
- });
525
- });
526
- if (!audioContext) {
527
- audioContext = new AudioContext();
528
- }
529
- if (audioContext.state === "suspended") {
530
- audioContext.resume().catch((err) => {
531
- logger.errors("grok.realtime audioContext.resume failed", {
532
- error: err,
533
- source: "grok.realtime"
534
- });
535
- });
536
- }
537
- outputAnalyser = audioContext.createAnalyser();
538
- outputAnalyser.fftSize = 2048;
539
- outputAnalyser.smoothingTimeConstant = 0.3;
540
- outputSource = audioContext.createMediaStreamSource(stream);
541
- outputSource.connect(outputAnalyser);
542
- }
543
- function setupInputAudioAnalysis(stream) {
544
- if (isTornDown) return;
545
- if (!audioContext) {
546
- audioContext = new AudioContext();
547
- }
548
- if (audioContext.state === "suspended") {
549
- audioContext.resume().catch((err) => {
550
- logger.errors("grok.realtime audioContext.resume failed", {
551
- error: err,
552
- source: "grok.realtime"
553
- });
554
- });
555
- }
556
- inputAnalyser = audioContext.createAnalyser();
557
- inputAnalyser.fftSize = 2048;
558
- inputAnalyser.smoothingTimeConstant = 0.3;
559
- inputSource = audioContext.createMediaStreamSource(stream);
560
- inputSource.connect(inputAnalyser);
561
- }
562
- function sendEvent(event) {
563
- if (isTornDown) {
564
- logger.errors("grok.realtime sendEvent after disconnect", {
565
- eventType: readString(event, "type") ?? "<unknown>",
566
- source: "grok.realtime"
567
- });
568
- return;
569
- }
570
- if (dataChannel?.readyState === "open") {
571
- logger.provider(
572
- `provider=grok direction=out type=${readString(event, "type") ?? "<unknown>"}`,
573
- { frame: event }
574
- );
575
- try {
576
- dataChannel.send(JSON.stringify(event));
577
- } catch (error) {
578
- logger.errors("grok.realtime sendEvent failed", {
579
- error,
580
- eventType: readString(event, "type") ?? "<unknown>",
581
- source: "grok.realtime"
582
- });
583
- emit("error", {
584
- error: error instanceof Error ? error : new Error(String(error))
585
- });
586
- }
587
- } else {
588
- pendingEvents.push(event);
589
- }
590
- }
591
- function flushPendingEvents() {
592
- try {
593
- for (const event of pendingEvents) {
594
- logger.provider(
595
- `provider=grok direction=out type=${readString(event, "type") ?? "<unknown>"}`,
596
- { frame: event }
597
- );
598
- channel.send(JSON.stringify(event));
599
- }
600
- pendingEvents.length = 0;
601
- } catch (error) {
602
- logger.errors("grok.realtime flushPendingEvents failed", {
603
- error,
604
- source: "grok.realtime"
605
- });
606
- const err = error instanceof Error ? error : new Error(String(error));
607
- rejectDataChannelReady?.(err);
608
- emit("error", { error: err });
609
- }
610
- }
611
- const connection = {
612
- async disconnect() {
613
- await teardownConnection();
614
- emit("status_change", { status: "idle" });
615
- },
616
- async startAudioCapture() {
617
- if (localStream) {
618
- for (const track of localStream.getAudioTracks()) {
619
- track.enabled = true;
620
- }
621
- }
622
- currentMode = "listening";
623
- emit("mode_change", { mode: "listening" });
624
- },
625
- stopAudioCapture() {
626
- if (localStream) {
627
- for (const track of localStream.getAudioTracks()) {
628
- track.enabled = false;
629
- }
630
- }
631
- currentMode = "idle";
632
- emit("mode_change", { mode: "idle" });
633
- },
634
- sendText(text) {
635
- sendEvent({
636
- type: "conversation.item.create",
637
- item: {
638
- type: "message",
639
- role: "user",
640
- content: [{ type: "input_text", text }]
641
- }
642
- });
643
- sendEvent({ type: "response.create" });
644
- },
645
- sendImage(imageData, mimeType) {
646
- const isAlreadyUrlOrDataUri = imageData.startsWith("http://") || imageData.startsWith("https://") || imageData.startsWith("data:");
647
- const imageContent = {
648
- type: "input_image",
649
- // The OpenAI-realtime content part (which this adapter mirrors) nests
650
- // the URL under an `image_url: { url: ... }` object, not a bare
651
- // string.
652
- image_url: {
653
- url: isAlreadyUrlOrDataUri ? imageData : `data:${mimeType};base64,${imageData}`
654
- }
655
- };
656
- sendEvent({
657
- type: "conversation.item.create",
658
- item: {
659
- type: "message",
660
- role: "user",
661
- content: [imageContent]
662
- }
663
- });
664
- sendEvent({ type: "response.create" });
665
- },
666
- sendToolResult(callId, result) {
667
- sendEvent({
668
- type: "conversation.item.create",
669
- item: {
670
- type: "function_call_output",
671
- call_id: callId,
672
- output: result
673
- }
674
- });
675
- sendEvent({ type: "response.create" });
676
- },
677
- updateSession(config) {
678
- const sessionUpdate = {};
679
- if (config.instructions) {
680
- sessionUpdate.instructions = config.instructions;
681
- }
682
- if (config.voice) {
683
- sessionUpdate.voice = config.voice;
684
- }
685
- if (config.vadMode) {
686
- if (config.vadMode === "semantic") {
687
- sessionUpdate.turn_detection = {
688
- type: "semantic_vad",
689
- eagerness: config.semanticEagerness ?? "medium"
690
- };
691
- } else if (config.vadMode === "server") {
692
- sessionUpdate.turn_detection = {
693
- type: "server_vad",
694
- threshold: config.vadConfig?.threshold ?? 0.5,
695
- prefix_padding_ms: config.vadConfig?.prefixPaddingMs ?? 300,
696
- silence_duration_ms: config.vadConfig?.silenceDurationMs ?? 500
697
- };
698
- } else {
699
- sessionUpdate.turn_detection = null;
700
- }
701
- }
702
- if (config.tools !== void 0) {
703
- sessionUpdate.tools = config.tools.map((t) => ({
704
- type: "function",
705
- name: t.name,
706
- description: t.description,
707
- parameters: t.inputSchema ?? { type: "object", properties: {} }
708
- }));
709
- sessionUpdate.tool_choice = "auto";
710
- }
711
- if (config.outputModalities) {
712
- sessionUpdate.modalities = config.outputModalities;
713
- }
714
- if (config.temperature !== void 0) {
715
- sessionUpdate.temperature = config.temperature;
716
- }
717
- if (config.maxOutputTokens !== void 0) {
718
- sessionUpdate.max_response_output_tokens = config.maxOutputTokens;
719
- }
720
- const providerOptions = config.providerOptions ?? {};
721
- const callerTranscription = "inputAudioTranscription" in providerOptions ? providerOptions.inputAudioTranscription : "input_audio_transcription" in providerOptions ? providerOptions.input_audio_transcription : void 0;
722
- if (callerTranscription !== void 0) {
723
- sessionUpdate.input_audio_transcription = callerTranscription === false ? null : callerTranscription;
724
- } else if (!hasSentInitialSessionUpdate) {
725
- sessionUpdate.input_audio_transcription = { model: "grok-stt" };
726
- }
727
- if (Object.keys(sessionUpdate).length > 0) {
728
- sendEvent({
729
- type: "session.update",
730
- session: sessionUpdate
731
- });
732
- hasSentInitialSessionUpdate = true;
733
- }
734
- },
735
- interrupt() {
736
- sendEvent({ type: "response.cancel" });
737
- currentMode = "listening";
738
- emit("mode_change", { mode: "listening" });
739
- emit("interrupted", {
740
- ...currentMessageId !== null && { messageId: currentMessageId }
741
- });
742
- },
743
- on(event, handler) {
744
- let handlers = eventHandlers.get(event);
745
- if (!handlers) {
746
- handlers = /* @__PURE__ */ new Set();
747
- eventHandlers.set(event, handlers);
748
- }
749
- handlers.add(handler);
750
- return () => {
751
- eventHandlers.get(event)?.delete(handler);
752
- };
753
- },
754
- getAudioVisualization() {
755
- function calculateLevel(analyser) {
756
- const data = new Uint8Array(analyser.fftSize);
757
- analyser.getByteTimeDomainData(data);
758
- let maxDeviation = 0;
759
- for (const sample of data) {
760
- const deviation = Math.abs(sample - 128);
761
- if (deviation > maxDeviation) {
762
- maxDeviation = deviation;
763
- }
764
- }
765
- const normalized = maxDeviation / 128;
766
- return Math.min(1, normalized * 1.5);
767
- }
768
- return {
769
- get inputLevel() {
770
- if (!inputAnalyser) return 0;
771
- return calculateLevel(inputAnalyser);
772
- },
773
- get outputLevel() {
774
- if (!outputAnalyser) return 0;
775
- return calculateLevel(outputAnalyser);
776
- },
777
- getInputFrequencyData() {
778
- if (!inputAnalyser)
779
- return new Uint8Array(FALLBACK_FREQUENCY_BIN_COUNT);
780
- const data = new Uint8Array(inputAnalyser.frequencyBinCount);
781
- inputAnalyser.getByteFrequencyData(data);
782
- return data;
783
- },
784
- getOutputFrequencyData() {
785
- if (!outputAnalyser)
786
- return new Uint8Array(FALLBACK_FREQUENCY_BIN_COUNT);
787
- const data = new Uint8Array(outputAnalyser.frequencyBinCount);
788
- outputAnalyser.getByteFrequencyData(data);
789
- return data;
790
- },
791
- getInputTimeDomainData() {
792
- if (!inputAnalyser)
793
- return new Uint8Array(FALLBACK_TIME_DOMAIN_SIZE).fill(
794
- FALLBACK_TIME_DOMAIN_FILL
795
- );
796
- const data = new Uint8Array(inputAnalyser.fftSize);
797
- inputAnalyser.getByteTimeDomainData(data);
798
- return data;
799
- },
800
- getOutputTimeDomainData() {
801
- if (!outputAnalyser)
802
- return new Uint8Array(FALLBACK_TIME_DOMAIN_SIZE).fill(
803
- FALLBACK_TIME_DOMAIN_FILL
804
- );
805
- const data = new Uint8Array(outputAnalyser.fftSize);
806
- outputAnalyser.getByteTimeDomainData(data);
807
- return data;
808
- },
809
- get inputSampleRate() {
810
- return 24e3;
811
- },
812
- get outputSampleRate() {
813
- return 24e3;
814
- }
815
- };
816
- }
817
- };
818
- return connection;
66
+ const model = token.config.model ?? "grok-voice-fast-1.0";
67
+ const eventHandlers = /* @__PURE__ */ new Map();
68
+ const pc = new RTCPeerConnection();
69
+ let audioContext = null;
70
+ let inputAnalyser = null;
71
+ let outputAnalyser = null;
72
+ let inputSource = null;
73
+ let outputSource = null;
74
+ let localStream = null;
75
+ let audioElement = null;
76
+ const channel = pc.createDataChannel("oai-events");
77
+ let dataChannel = channel;
78
+ let currentMode = "idle";
79
+ let currentMessageId = null;
80
+ let isTornDown = false;
81
+ const pendingEvents = [];
82
+ let hasSentInitialSessionUpdate = false;
83
+ const FALLBACK_FREQUENCY_BIN_COUNT = 1024;
84
+ const FALLBACK_TIME_DOMAIN_SIZE = 2048;
85
+ const FALLBACK_TIME_DOMAIN_FILL = 128;
86
+ function emit(event, payload) {
87
+ const handlers = eventHandlers.get(event);
88
+ if (handlers) for (const handler of handlers) handler(payload);
89
+ }
90
+ let dataChannelOpened = false;
91
+ let rejectDataChannelReady = null;
92
+ let dataChannelReadyTimeout = null;
93
+ const dataChannelReady = new Promise((resolve, reject) => {
94
+ rejectDataChannelReady = (reason) => {
95
+ if (dataChannelReadyTimeout !== null) {
96
+ clearTimeout(dataChannelReadyTimeout);
97
+ dataChannelReadyTimeout = null;
98
+ }
99
+ rejectDataChannelReady = null;
100
+ reject(reason);
101
+ };
102
+ dataChannelReadyTimeout = setTimeout(() => {
103
+ if (!dataChannelOpened) rejectDataChannelReady?.(/* @__PURE__ */ new Error("Data channel did not open within 15000ms — aborting connection"));
104
+ }, 15e3);
105
+ channel.onopen = () => {
106
+ dataChannelOpened = true;
107
+ if (dataChannelReadyTimeout !== null) {
108
+ clearTimeout(dataChannelReadyTimeout);
109
+ dataChannelReadyTimeout = null;
110
+ }
111
+ rejectDataChannelReady = null;
112
+ flushPendingEvents();
113
+ emit("status_change", { status: "connected" });
114
+ resolve();
115
+ };
116
+ });
117
+ channel.onmessage = (event) => {
118
+ try {
119
+ const message = JSON.parse(event.data);
120
+ const messageRecord = message !== null && typeof message === "object" ? message : {};
121
+ logger.provider(`provider=grok direction=in type=${readString(messageRecord, "type") ?? "<unknown>"}`, { frame: messageRecord });
122
+ handleServerEvent(messageRecord);
123
+ } catch (parseErr) {
124
+ logger.errors("grok.realtime fatal", {
125
+ error: parseErr,
126
+ source: "grok.realtime"
127
+ });
128
+ emit("error", { error: parseErr instanceof Error ? parseErr : new Error(String(parseErr)) });
129
+ }
130
+ };
131
+ channel.onerror = (error) => {
132
+ if (isTornDown) return;
133
+ logger.errors("grok.realtime fatal", {
134
+ error,
135
+ source: "grok.realtime"
136
+ });
137
+ const rtcError = readObject(error, "error");
138
+ const msg = (rtcError && readString(rtcError, "message")) ?? (error.type || "unknown");
139
+ const dcErr = /* @__PURE__ */ new Error(`Data channel error: ${msg}`);
140
+ if (!dataChannelOpened) rejectDataChannelReady?.(dcErr);
141
+ emit("error", { error: dcErr });
142
+ };
143
+ channel.onclose = () => {
144
+ if (isTornDown) return;
145
+ if (!dataChannelOpened) rejectDataChannelReady?.(/* @__PURE__ */ new Error("Data channel closed before opening"));
146
+ };
147
+ pc.ontrack = (event) => {
148
+ if (event.track.kind === "audio" && event.streams[0]) setupOutputAudioAnalysis(event.streams[0]);
149
+ };
150
+ pc.onconnectionstatechange = () => {
151
+ const state = pc.connectionState;
152
+ logger.provider(`provider=grok pc.connectionState=${state}`, { state });
153
+ if (state === "failed" || state === "disconnected" || state === "closed") {
154
+ if (!isTornDown) emit("status_change", { status: state === "failed" ? "error" : "idle" });
155
+ if (!dataChannelOpened) {
156
+ const message = state === "failed" ? `PeerConnection failed before data channel opened` : `PeerConnection entered state '${state}' before data channel opened`;
157
+ rejectDataChannelReady?.(new Error(message));
158
+ }
159
+ if (state === "failed" && !isTornDown) teardownConnection();
160
+ }
161
+ };
162
+ pc.oniceconnectionstatechange = () => {
163
+ const state = pc.iceConnectionState;
164
+ logger.provider(`provider=grok pc.iceConnectionState=${state}`, { state });
165
+ if (!dataChannelOpened && (state === "failed" || state === "closed" || state === "disconnected")) {
166
+ const message = state === "failed" ? `ICE connection failed before data channel opened` : `ICE connection entered state '${state}' before data channel opened`;
167
+ rejectDataChannelReady?.(new Error(message));
168
+ }
169
+ };
170
+ /**
171
+ * Tear down every resource we may have allocated so the mic/pc/audio
172
+ * nodes/audio element don't leak on a failed connect. Safe to call from
173
+ * any point after `new RTCPeerConnection()`; each branch null-guards and
174
+ * swallows errors because cascading closes (e.g. `pc.close()` closing the
175
+ * data channel implicitly) are expected.
176
+ *
177
+ * Shared between the SDP-path catch, the post-SDP catch, and (implicitly
178
+ * via idempotency) the `disconnect()` entry point.
179
+ */
180
+ async function teardownConnection() {
181
+ isTornDown = true;
182
+ pendingEvents.length = 0;
183
+ rejectDataChannelReady?.(/* @__PURE__ */ new Error("Connection torn down before data channel opened"));
184
+ if (localStream) {
185
+ for (const track of localStream.getTracks()) track.stop();
186
+ localStream = null;
187
+ }
188
+ if (audioElement) {
189
+ try {
190
+ audioElement.pause();
191
+ } catch {}
192
+ audioElement.srcObject = null;
193
+ audioElement = null;
194
+ }
195
+ if (outputSource) {
196
+ try {
197
+ outputSource.disconnect();
198
+ } catch {}
199
+ outputSource = null;
200
+ }
201
+ if (outputAnalyser) {
202
+ try {
203
+ outputAnalyser.disconnect();
204
+ } catch {}
205
+ outputAnalyser = null;
206
+ }
207
+ if (inputSource) {
208
+ try {
209
+ inputSource.disconnect();
210
+ } catch {}
211
+ inputSource = null;
212
+ }
213
+ if (inputAnalyser) {
214
+ try {
215
+ inputAnalyser.disconnect();
216
+ } catch {}
217
+ inputAnalyser = null;
218
+ }
219
+ if (dataChannel) {
220
+ try {
221
+ dataChannel.close();
222
+ } catch {}
223
+ dataChannel = null;
224
+ }
225
+ try {
226
+ pc.close();
227
+ } catch {}
228
+ if (audioContext) {
229
+ try {
230
+ await audioContext.close();
231
+ } catch {}
232
+ audioContext = null;
233
+ }
234
+ }
235
+ try {
236
+ try {
237
+ localStream = await navigator.mediaDevices.getUserMedia({ audio: {
238
+ echoCancellation: true,
239
+ noiseSuppression: true,
240
+ sampleRate: 24e3
241
+ } });
242
+ } catch (error) {
243
+ logger.errors("grok.realtime fatal", {
244
+ error,
245
+ source: "grok.realtime.getUserMedia"
246
+ });
247
+ throw new Error(`Microphone access required for realtime voice: ${error instanceof Error ? error.message : error}`);
248
+ }
249
+ for (const track of localStream.getAudioTracks()) pc.addTrack(track, localStream);
250
+ const offer = await pc.createOffer();
251
+ await pc.setLocalDescription(offer);
252
+ const sdpResponse = await fetch(`${GROK_REALTIME_URL}?model=${model}`, {
253
+ method: "POST",
254
+ headers: {
255
+ Authorization: `Bearer ${token.token}`,
256
+ "Content-Type": "application/sdp"
257
+ },
258
+ body: offer.sdp
259
+ });
260
+ if (!sdpResponse.ok) {
261
+ const errorText = await sdpResponse.text();
262
+ const error = /* @__PURE__ */ new Error(`Failed to establish WebRTC connection: ${sdpResponse.status} - ${errorText}`);
263
+ logger.errors("grok.realtime fatal", {
264
+ error,
265
+ source: "grok.realtime.sdp",
266
+ status: sdpResponse.status
267
+ });
268
+ throw error;
269
+ }
270
+ const answerSdp = await sdpResponse.text();
271
+ await pc.setRemoteDescription({
272
+ type: "answer",
273
+ sdp: answerSdp
274
+ });
275
+ } catch (err) {
276
+ await teardownConnection();
277
+ throw err;
278
+ }
279
+ try {
280
+ setupInputAudioAnalysis(localStream);
281
+ await dataChannelReady;
282
+ } catch (err) {
283
+ await teardownConnection();
284
+ throw err;
285
+ }
286
+ function handleServerEvent(event) {
287
+ switch (readString(event, "type")) {
288
+ case "session.created":
289
+ case "session.updated": break;
290
+ case "input_audio_buffer.speech_started":
291
+ currentMode = "listening";
292
+ emit("mode_change", { mode: "listening" });
293
+ break;
294
+ case "input_audio_buffer.speech_stopped":
295
+ currentMode = "thinking";
296
+ emit("mode_change", { mode: "thinking" });
297
+ break;
298
+ case "input_audio_buffer.committed": break;
299
+ case "conversation.item.input_audio_transcription.completed": {
300
+ const transcript = readString(event, "transcript");
301
+ if (transcript === void 0) break;
302
+ emit("transcript", {
303
+ role: "user",
304
+ transcript,
305
+ isFinal: true
306
+ });
307
+ break;
308
+ }
309
+ case "response.created":
310
+ currentMessageId = null;
311
+ currentMode = "thinking";
312
+ emit("mode_change", { mode: "thinking" });
313
+ break;
314
+ case "response.output_item.added": {
315
+ const item = readObject(event, "item");
316
+ if (item && readString(item, "type") === "message") {
317
+ const id = readString(item, "id");
318
+ if (id !== void 0) currentMessageId = id;
319
+ }
320
+ break;
321
+ }
322
+ case "response.output_audio_transcript.delta":
323
+ case "response.audio_transcript.delta": {
324
+ const delta = readString(event, "delta");
325
+ if (delta === void 0) break;
326
+ emit("transcript", {
327
+ role: "assistant",
328
+ transcript: delta,
329
+ isFinal: false
330
+ });
331
+ break;
332
+ }
333
+ case "response.output_audio_transcript.done":
334
+ case "response.audio_transcript.done": {
335
+ const transcript = readString(event, "transcript");
336
+ if (transcript === void 0) break;
337
+ emit("transcript", {
338
+ role: "assistant",
339
+ transcript,
340
+ isFinal: true
341
+ });
342
+ break;
343
+ }
344
+ case "response.text.delta":
345
+ case "response.output_text.delta": {
346
+ const delta = readString(event, "delta");
347
+ if (delta === void 0) break;
348
+ emit("transcript", {
349
+ role: "assistant",
350
+ transcript: delta,
351
+ isFinal: false
352
+ });
353
+ break;
354
+ }
355
+ case "response.text.done":
356
+ case "response.output_text.done": {
357
+ const text = readString(event, "text");
358
+ if (text === void 0) break;
359
+ emit("transcript", {
360
+ role: "assistant",
361
+ transcript: text,
362
+ isFinal: true
363
+ });
364
+ break;
365
+ }
366
+ case "response.output_audio.delta":
367
+ case "response.audio.delta":
368
+ if (currentMode !== "speaking") {
369
+ currentMode = "speaking";
370
+ emit("mode_change", { mode: "speaking" });
371
+ }
372
+ break;
373
+ case "response.output_audio.done":
374
+ case "response.audio.done": break;
375
+ case "response.function_call_arguments.done": {
376
+ const callId = readString(event, "call_id");
377
+ const name = readString(event, "name") ?? "";
378
+ const args = readString(event, "arguments") ?? "";
379
+ if (!callId) {
380
+ logger.errors("grok.realtime tool_call missing call_id — dropping tool_call", {
381
+ source: "grok.realtime",
382
+ event_type: "response.function_call_arguments.done",
383
+ item_id: event.item_id
384
+ });
385
+ emit("error", { error: /* @__PURE__ */ new Error("Realtime tool call missing call_id; tool will not execute") });
386
+ break;
387
+ }
388
+ try {
389
+ emit("tool_call", {
390
+ toolCallId: callId,
391
+ toolName: name,
392
+ input: JSON.parse(args)
393
+ });
394
+ } catch {
395
+ emit("tool_call", {
396
+ toolCallId: callId,
397
+ toolName: name,
398
+ input: args
399
+ });
400
+ }
401
+ break;
402
+ }
403
+ case "response.done": {
404
+ const output = readObjectArray(readObject(event, "response") ?? {}, "output");
405
+ if (currentMode !== "idle") {
406
+ currentMode = "listening";
407
+ emit("mode_change", { mode: "listening" });
408
+ }
409
+ if (currentMessageId) {
410
+ const message = {
411
+ id: currentMessageId,
412
+ role: "assistant",
413
+ timestamp: Date.now(),
414
+ parts: []
415
+ };
416
+ for (const item of output ?? []) {
417
+ if (readString(item, "type") !== "message") continue;
418
+ const content = readObjectArray(item, "content");
419
+ if (!content) continue;
420
+ for (const part of content) {
421
+ const partType = readString(part, "type");
422
+ if (partType === "audio") {
423
+ const transcript = readString(part, "transcript");
424
+ if (transcript) message.parts.push({
425
+ type: "audio",
426
+ transcript
427
+ });
428
+ } else if (partType === "text") {
429
+ const content = readString(part, "text");
430
+ if (content) message.parts.push({
431
+ type: "text",
432
+ content
433
+ });
434
+ }
435
+ }
436
+ }
437
+ emit("message_complete", { message });
438
+ currentMessageId = null;
439
+ }
440
+ break;
441
+ }
442
+ case "conversation.item.truncated":
443
+ if (currentMode !== "idle") {
444
+ currentMode = "listening";
445
+ emit("mode_change", { mode: "listening" });
446
+ }
447
+ emit("interrupted", { ...currentMessageId !== null && { messageId: currentMessageId } });
448
+ break;
449
+ case "error": {
450
+ const errorObj = readObject(event, "error") ?? {};
451
+ const message = readString(errorObj, "message") ?? "Unknown realtime server error";
452
+ const err = new Error(message);
453
+ const code = readString(errorObj, "code");
454
+ if (code !== void 0) err.code = code;
455
+ const errType = readString(errorObj, "type");
456
+ if (errType !== void 0) err.type = errType;
457
+ const param = readString(errorObj, "param");
458
+ if (param !== void 0) err.param = param;
459
+ logger.errors("grok.realtime server error", {
460
+ ...errorObj,
461
+ source: "grok.realtime server"
462
+ });
463
+ emit("error", { error: err });
464
+ break;
465
+ }
466
+ case void 0:
467
+ default:
468
+ logger.provider("grok.realtime unhandled server event", { type: event.type });
469
+ break;
470
+ }
471
+ }
472
+ function setupOutputAudioAnalysis(stream) {
473
+ if (isTornDown) return;
474
+ if (audioElement) {
475
+ try {
476
+ audioElement.pause();
477
+ } catch {}
478
+ audioElement.srcObject = null;
479
+ audioElement = null;
480
+ }
481
+ if (outputSource) {
482
+ try {
483
+ outputSource.disconnect();
484
+ } catch {}
485
+ outputSource = null;
486
+ }
487
+ if (outputAnalyser) {
488
+ try {
489
+ outputAnalyser.disconnect();
490
+ } catch {}
491
+ outputAnalyser = null;
492
+ }
493
+ audioElement = new Audio();
494
+ audioElement.srcObject = stream;
495
+ audioElement.autoplay = true;
496
+ audioElement.play().catch((e) => {
497
+ logger.errors("grok.realtime audio autoplay blocked", {
498
+ error: e,
499
+ source: "grok.realtime.audio_permission_required"
500
+ });
501
+ });
502
+ if (!audioContext) audioContext = new AudioContext();
503
+ if (audioContext.state === "suspended") audioContext.resume().catch((err) => {
504
+ logger.errors("grok.realtime audioContext.resume failed", {
505
+ error: err,
506
+ source: "grok.realtime"
507
+ });
508
+ });
509
+ outputAnalyser = audioContext.createAnalyser();
510
+ outputAnalyser.fftSize = 2048;
511
+ outputAnalyser.smoothingTimeConstant = .3;
512
+ outputSource = audioContext.createMediaStreamSource(stream);
513
+ outputSource.connect(outputAnalyser);
514
+ }
515
+ function setupInputAudioAnalysis(stream) {
516
+ if (isTornDown) return;
517
+ if (!audioContext) audioContext = new AudioContext();
518
+ if (audioContext.state === "suspended") audioContext.resume().catch((err) => {
519
+ logger.errors("grok.realtime audioContext.resume failed", {
520
+ error: err,
521
+ source: "grok.realtime"
522
+ });
523
+ });
524
+ inputAnalyser = audioContext.createAnalyser();
525
+ inputAnalyser.fftSize = 2048;
526
+ inputAnalyser.smoothingTimeConstant = .3;
527
+ inputSource = audioContext.createMediaStreamSource(stream);
528
+ inputSource.connect(inputAnalyser);
529
+ }
530
+ function sendEvent(event) {
531
+ if (isTornDown) {
532
+ logger.errors("grok.realtime sendEvent after disconnect", {
533
+ eventType: readString(event, "type") ?? "<unknown>",
534
+ source: "grok.realtime"
535
+ });
536
+ return;
537
+ }
538
+ if (dataChannel?.readyState === "open") {
539
+ logger.provider(`provider=grok direction=out type=${readString(event, "type") ?? "<unknown>"}`, { frame: event });
540
+ try {
541
+ dataChannel.send(JSON.stringify(event));
542
+ } catch (error) {
543
+ logger.errors("grok.realtime sendEvent failed", {
544
+ error,
545
+ eventType: readString(event, "type") ?? "<unknown>",
546
+ source: "grok.realtime"
547
+ });
548
+ emit("error", { error: error instanceof Error ? error : new Error(String(error)) });
549
+ }
550
+ } else pendingEvents.push(event);
551
+ }
552
+ function flushPendingEvents() {
553
+ try {
554
+ for (const event of pendingEvents) {
555
+ logger.provider(`provider=grok direction=out type=${readString(event, "type") ?? "<unknown>"}`, { frame: event });
556
+ channel.send(JSON.stringify(event));
557
+ }
558
+ pendingEvents.length = 0;
559
+ } catch (error) {
560
+ logger.errors("grok.realtime flushPendingEvents failed", {
561
+ error,
562
+ source: "grok.realtime"
563
+ });
564
+ const err = error instanceof Error ? error : new Error(String(error));
565
+ rejectDataChannelReady?.(err);
566
+ emit("error", { error: err });
567
+ }
568
+ }
569
+ return {
570
+ async disconnect() {
571
+ await teardownConnection();
572
+ emit("status_change", { status: "idle" });
573
+ },
574
+ async startAudioCapture() {
575
+ if (localStream) for (const track of localStream.getAudioTracks()) track.enabled = true;
576
+ currentMode = "listening";
577
+ emit("mode_change", { mode: "listening" });
578
+ },
579
+ stopAudioCapture() {
580
+ if (localStream) for (const track of localStream.getAudioTracks()) track.enabled = false;
581
+ currentMode = "idle";
582
+ emit("mode_change", { mode: "idle" });
583
+ },
584
+ sendText(text) {
585
+ sendEvent({
586
+ type: "conversation.item.create",
587
+ item: {
588
+ type: "message",
589
+ role: "user",
590
+ content: [{
591
+ type: "input_text",
592
+ text
593
+ }]
594
+ }
595
+ });
596
+ sendEvent({ type: "response.create" });
597
+ },
598
+ sendImage(imageData, mimeType) {
599
+ sendEvent({
600
+ type: "conversation.item.create",
601
+ item: {
602
+ type: "message",
603
+ role: "user",
604
+ content: [{
605
+ type: "input_image",
606
+ image_url: { url: imageData.startsWith("http://") || imageData.startsWith("https://") || imageData.startsWith("data:") ? imageData : `data:${mimeType};base64,${imageData}` }
607
+ }]
608
+ }
609
+ });
610
+ sendEvent({ type: "response.create" });
611
+ },
612
+ sendToolResult(callId, result) {
613
+ sendEvent({
614
+ type: "conversation.item.create",
615
+ item: {
616
+ type: "function_call_output",
617
+ call_id: callId,
618
+ output: result
619
+ }
620
+ });
621
+ sendEvent({ type: "response.create" });
622
+ },
623
+ updateSession(config) {
624
+ const sessionUpdate = {};
625
+ if (config.instructions) sessionUpdate.instructions = config.instructions;
626
+ if (config.voice) sessionUpdate.voice = config.voice;
627
+ if (config.vadMode) if (config.vadMode === "semantic") sessionUpdate.turn_detection = {
628
+ type: "semantic_vad",
629
+ eagerness: config.semanticEagerness ?? "medium"
630
+ };
631
+ else if (config.vadMode === "server") sessionUpdate.turn_detection = {
632
+ type: "server_vad",
633
+ threshold: config.vadConfig?.threshold ?? .5,
634
+ prefix_padding_ms: config.vadConfig?.prefixPaddingMs ?? 300,
635
+ silence_duration_ms: config.vadConfig?.silenceDurationMs ?? 500
636
+ };
637
+ else sessionUpdate.turn_detection = null;
638
+ if (config.tools !== void 0) {
639
+ sessionUpdate.tools = config.tools.map((t) => ({
640
+ type: "function",
641
+ name: t.name,
642
+ description: t.description,
643
+ parameters: t.inputSchema ?? {
644
+ type: "object",
645
+ properties: {}
646
+ }
647
+ }));
648
+ sessionUpdate.tool_choice = "auto";
649
+ }
650
+ if (config.outputModalities) sessionUpdate.modalities = config.outputModalities;
651
+ if (config.temperature !== void 0) sessionUpdate.temperature = config.temperature;
652
+ if (config.maxOutputTokens !== void 0) sessionUpdate.max_response_output_tokens = config.maxOutputTokens;
653
+ const providerOptions = config.providerOptions ?? {};
654
+ const callerTranscription = "inputAudioTranscription" in providerOptions ? providerOptions.inputAudioTranscription : "input_audio_transcription" in providerOptions ? providerOptions.input_audio_transcription : void 0;
655
+ if (callerTranscription !== void 0) sessionUpdate.input_audio_transcription = callerTranscription === false ? null : callerTranscription;
656
+ else if (!hasSentInitialSessionUpdate) sessionUpdate.input_audio_transcription = { model: "grok-stt" };
657
+ if (Object.keys(sessionUpdate).length > 0) {
658
+ sendEvent({
659
+ type: "session.update",
660
+ session: sessionUpdate
661
+ });
662
+ hasSentInitialSessionUpdate = true;
663
+ }
664
+ },
665
+ interrupt() {
666
+ sendEvent({ type: "response.cancel" });
667
+ currentMode = "listening";
668
+ emit("mode_change", { mode: "listening" });
669
+ emit("interrupted", { ...currentMessageId !== null && { messageId: currentMessageId } });
670
+ },
671
+ on(event, handler) {
672
+ let handlers = eventHandlers.get(event);
673
+ if (!handlers) {
674
+ handlers = /* @__PURE__ */ new Set();
675
+ eventHandlers.set(event, handlers);
676
+ }
677
+ handlers.add(handler);
678
+ return () => {
679
+ eventHandlers.get(event)?.delete(handler);
680
+ };
681
+ },
682
+ getAudioVisualization() {
683
+ function calculateLevel(analyser) {
684
+ const data = new Uint8Array(analyser.fftSize);
685
+ analyser.getByteTimeDomainData(data);
686
+ let maxDeviation = 0;
687
+ for (const sample of data) {
688
+ const deviation = Math.abs(sample - 128);
689
+ if (deviation > maxDeviation) maxDeviation = deviation;
690
+ }
691
+ const normalized = maxDeviation / 128;
692
+ return Math.min(1, normalized * 1.5);
693
+ }
694
+ return {
695
+ get inputLevel() {
696
+ if (!inputAnalyser) return 0;
697
+ return calculateLevel(inputAnalyser);
698
+ },
699
+ get outputLevel() {
700
+ if (!outputAnalyser) return 0;
701
+ return calculateLevel(outputAnalyser);
702
+ },
703
+ getInputFrequencyData() {
704
+ if (!inputAnalyser) return new Uint8Array(FALLBACK_FREQUENCY_BIN_COUNT);
705
+ const data = new Uint8Array(inputAnalyser.frequencyBinCount);
706
+ inputAnalyser.getByteFrequencyData(data);
707
+ return data;
708
+ },
709
+ getOutputFrequencyData() {
710
+ if (!outputAnalyser) return new Uint8Array(FALLBACK_FREQUENCY_BIN_COUNT);
711
+ const data = new Uint8Array(outputAnalyser.frequencyBinCount);
712
+ outputAnalyser.getByteFrequencyData(data);
713
+ return data;
714
+ },
715
+ getInputTimeDomainData() {
716
+ if (!inputAnalyser) return new Uint8Array(FALLBACK_TIME_DOMAIN_SIZE).fill(FALLBACK_TIME_DOMAIN_FILL);
717
+ const data = new Uint8Array(inputAnalyser.fftSize);
718
+ inputAnalyser.getByteTimeDomainData(data);
719
+ return data;
720
+ },
721
+ getOutputTimeDomainData() {
722
+ if (!outputAnalyser) return new Uint8Array(FALLBACK_TIME_DOMAIN_SIZE).fill(FALLBACK_TIME_DOMAIN_FILL);
723
+ const data = new Uint8Array(outputAnalyser.fftSize);
724
+ outputAnalyser.getByteTimeDomainData(data);
725
+ return data;
726
+ },
727
+ get inputSampleRate() {
728
+ return 24e3;
729
+ },
730
+ get outputSampleRate() {
731
+ return 24e3;
732
+ }
733
+ };
734
+ }
735
+ };
819
736
  }
820
- export {
821
- grokRealtime
822
- };
823
- //# sourceMappingURL=adapter.js.map
737
+ //#endregion
738
+ export { grokRealtime };
739
+
740
+ //# sourceMappingURL=adapter.js.map