@tanstack/ai-grok 0.14.3 → 0.14.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/dist/esm/adapters/image.d.ts +89 -0
  2. package/dist/esm/adapters/image.js +205 -0
  3. package/dist/esm/adapters/image.js.map +1 -0
  4. package/dist/esm/adapters/summarize.d.ts +51 -0
  5. package/dist/esm/adapters/summarize.js +20 -0
  6. package/dist/esm/adapters/summarize.js.map +1 -0
  7. package/dist/esm/adapters/text.d.ts +76 -0
  8. package/dist/esm/adapters/text.js +44 -0
  9. package/dist/esm/adapters/text.js.map +1 -0
  10. package/dist/esm/adapters/transcription.d.ts +84 -0
  11. package/dist/esm/adapters/transcription.js +122 -0
  12. package/dist/esm/adapters/transcription.js.map +1 -0
  13. package/dist/esm/adapters/tts.d.ts +70 -0
  14. package/dist/esm/adapters/tts.js +142 -0
  15. package/dist/esm/adapters/tts.js.map +1 -0
  16. package/dist/esm/adapters/video.d.ts +128 -0
  17. package/dist/esm/adapters/video.js +237 -0
  18. package/dist/esm/adapters/video.js.map +1 -0
  19. package/dist/esm/audio/transcription-provider-options.d.ts +41 -0
  20. package/dist/esm/audio/tts-provider-options.d.ts +42 -0
  21. package/dist/esm/image/image-provider-options.d.ts +133 -0
  22. package/dist/esm/image/image-provider-options.js +76 -0
  23. package/dist/esm/image/image-provider-options.js.map +1 -0
  24. package/dist/esm/index.d.ts +16 -0
  25. package/dist/esm/index.js +40 -0
  26. package/dist/esm/index.js.map +1 -0
  27. package/dist/esm/message-types.d.ts +64 -0
  28. package/dist/esm/model-meta.d.ts +99 -0
  29. package/dist/esm/model-meta.js +58 -0
  30. package/dist/esm/model-meta.js.map +1 -0
  31. package/dist/esm/realtime/adapter.d.ts +21 -0
  32. package/dist/esm/realtime/adapter.js +823 -0
  33. package/dist/esm/realtime/adapter.js.map +1 -0
  34. package/dist/esm/realtime/index.d.ts +4 -0
  35. package/dist/esm/realtime/token.d.ts +22 -0
  36. package/dist/esm/realtime/token.js +75 -0
  37. package/dist/esm/realtime/token.js.map +1 -0
  38. package/dist/esm/realtime/types.d.ts +95 -0
  39. package/dist/esm/text/text-provider-options.d.ts +54 -0
  40. package/dist/esm/tools/index.d.ts +45 -0
  41. package/dist/esm/tools/index.js +113 -0
  42. package/dist/esm/tools/index.js.map +1 -0
  43. package/dist/esm/utils/audio.d.ts +23 -0
  44. package/dist/esm/utils/audio.js +172 -0
  45. package/dist/esm/utils/audio.js.map +1 -0
  46. package/dist/esm/utils/client.d.ts +14 -0
  47. package/dist/esm/utils/client.js +21 -0
  48. package/dist/esm/utils/client.js.map +1 -0
  49. package/dist/esm/utils/index.d.ts +4 -0
  50. package/dist/esm/utils/schema-converter.d.ts +2 -0
  51. package/dist/esm/video/video-provider-options.d.ts +135 -0
  52. package/dist/esm/video/video-provider-options.js +67 -0
  53. package/dist/esm/video/video-provider-options.js.map +1 -0
  54. package/package.json +5 -5
@@ -0,0 +1,823 @@
1
+ import { resolveDebugOption } from "@tanstack/ai/adapter-internals";
2
+ const GROK_REALTIME_URL = "https://api.x.ai/v1/realtime";
3
+ function readString(obj, key) {
4
+ const value = obj[key];
5
+ return typeof value === "string" ? value : void 0;
6
+ }
7
+ function readObject(obj, key) {
8
+ const value = obj[key];
9
+ return value && typeof value === "object" && !Array.isArray(value) ? value : void 0;
10
+ }
11
+ function readObjectArray(obj, key) {
12
+ const value = obj[key];
13
+ if (!Array.isArray(value)) return void 0;
14
+ return value.filter(
15
+ (item) => item !== null && typeof item === "object" && !Array.isArray(item)
16
+ );
17
+ }
18
+ function grokRealtime(options = {}) {
19
+ const connectionMode = options.connectionMode ?? "webrtc";
20
+ const logger = resolveDebugOption(options.debug);
21
+ return {
22
+ provider: "grok",
23
+ async connect(token, _clientTools) {
24
+ const model = token.config.model ?? "grok-voice-fast-1.0";
25
+ logger.request(`activity=realtime provider=grok model=${model}`, {
26
+ provider: "grok",
27
+ model
28
+ });
29
+ if (connectionMode === "webrtc") {
30
+ return createWebRTCConnection(token, logger);
31
+ }
32
+ const error = new Error("WebSocket connection mode not yet implemented");
33
+ logger.errors("grok.realtime fatal", {
34
+ error,
35
+ source: "grok.realtime"
36
+ });
37
+ throw error;
38
+ }
39
+ };
40
+ }
41
+ async function createWebRTCConnection(token, logger) {
42
+ const model = token.config.model ?? "grok-voice-fast-1.0";
43
+ const eventHandlers = /* @__PURE__ */ new Map();
44
+ const pc = new RTCPeerConnection();
45
+ let audioContext = null;
46
+ let inputAnalyser = null;
47
+ let outputAnalyser = null;
48
+ let inputSource = null;
49
+ let outputSource = null;
50
+ let localStream = null;
51
+ let audioElement = null;
52
+ const channel = pc.createDataChannel("oai-events");
53
+ let dataChannel = channel;
54
+ let currentMode = "idle";
55
+ let currentMessageId = null;
56
+ let isTornDown = false;
57
+ const pendingEvents = [];
58
+ let hasSentInitialSessionUpdate = false;
59
+ const FALLBACK_FREQUENCY_BIN_COUNT = 1024;
60
+ const FALLBACK_TIME_DOMAIN_SIZE = 2048;
61
+ const FALLBACK_TIME_DOMAIN_FILL = 128;
62
+ function emit(event, payload) {
63
+ const handlers = eventHandlers.get(event);
64
+ if (handlers) {
65
+ for (const handler of handlers) {
66
+ handler(payload);
67
+ }
68
+ }
69
+ }
70
+ let dataChannelOpened = false;
71
+ let rejectDataChannelReady = null;
72
+ let dataChannelReadyTimeout = null;
73
+ const dataChannelReady = new Promise((resolve, reject) => {
74
+ rejectDataChannelReady = (reason) => {
75
+ if (dataChannelReadyTimeout !== null) {
76
+ clearTimeout(dataChannelReadyTimeout);
77
+ dataChannelReadyTimeout = null;
78
+ }
79
+ rejectDataChannelReady = null;
80
+ reject(reason);
81
+ };
82
+ dataChannelReadyTimeout = setTimeout(() => {
83
+ if (!dataChannelOpened) {
84
+ rejectDataChannelReady?.(
85
+ new Error(
86
+ "Data channel did not open within 15000ms — aborting connection"
87
+ )
88
+ );
89
+ }
90
+ }, 15e3);
91
+ channel.onopen = () => {
92
+ dataChannelOpened = true;
93
+ if (dataChannelReadyTimeout !== null) {
94
+ clearTimeout(dataChannelReadyTimeout);
95
+ dataChannelReadyTimeout = null;
96
+ }
97
+ rejectDataChannelReady = null;
98
+ flushPendingEvents();
99
+ emit("status_change", { status: "connected" });
100
+ resolve();
101
+ };
102
+ });
103
+ channel.onmessage = (event) => {
104
+ try {
105
+ const message = JSON.parse(event.data);
106
+ const messageRecord = message !== null && typeof message === "object" ? message : {};
107
+ logger.provider(
108
+ `provider=grok direction=in type=${readString(messageRecord, "type") ?? "<unknown>"}`,
109
+ { frame: messageRecord }
110
+ );
111
+ handleServerEvent(messageRecord);
112
+ } catch (parseErr) {
113
+ logger.errors("grok.realtime fatal", {
114
+ error: parseErr,
115
+ source: "grok.realtime"
116
+ });
117
+ emit("error", {
118
+ error: parseErr instanceof Error ? parseErr : new Error(String(parseErr))
119
+ });
120
+ }
121
+ };
122
+ channel.onerror = (error) => {
123
+ if (isTornDown) return;
124
+ logger.errors("grok.realtime fatal", {
125
+ error,
126
+ source: "grok.realtime"
127
+ });
128
+ const errorRecord = error;
129
+ const rtcError = readObject(errorRecord, "error");
130
+ const msg = (rtcError && readString(rtcError, "message")) ?? (error.type || "unknown");
131
+ const dcErr = new Error(`Data channel error: ${msg}`);
132
+ if (!dataChannelOpened) {
133
+ rejectDataChannelReady?.(dcErr);
134
+ }
135
+ emit("error", { error: dcErr });
136
+ };
137
+ channel.onclose = () => {
138
+ if (isTornDown) return;
139
+ if (!dataChannelOpened) {
140
+ rejectDataChannelReady?.(new Error("Data channel closed before opening"));
141
+ }
142
+ };
143
+ pc.ontrack = (event) => {
144
+ if (event.track.kind === "audio" && event.streams[0]) {
145
+ setupOutputAudioAnalysis(event.streams[0]);
146
+ }
147
+ };
148
+ pc.onconnectionstatechange = () => {
149
+ const state = pc.connectionState;
150
+ logger.provider(`provider=grok pc.connectionState=${state}`, {
151
+ state
152
+ });
153
+ if (state === "failed" || state === "disconnected" || state === "closed") {
154
+ if (!isTornDown) {
155
+ emit("status_change", {
156
+ status: state === "failed" ? "error" : "idle"
157
+ });
158
+ }
159
+ if (!dataChannelOpened) {
160
+ const message = state === "failed" ? `PeerConnection failed before data channel opened` : `PeerConnection entered state '${state}' before data channel opened`;
161
+ rejectDataChannelReady?.(new Error(message));
162
+ }
163
+ if (state === "failed" && !isTornDown) {
164
+ void teardownConnection();
165
+ }
166
+ }
167
+ };
168
+ pc.oniceconnectionstatechange = () => {
169
+ const state = pc.iceConnectionState;
170
+ logger.provider(`provider=grok pc.iceConnectionState=${state}`, {
171
+ state
172
+ });
173
+ if (!dataChannelOpened && (state === "failed" || state === "closed" || state === "disconnected")) {
174
+ const message = state === "failed" ? `ICE connection failed before data channel opened` : `ICE connection entered state '${state}' before data channel opened`;
175
+ rejectDataChannelReady?.(new Error(message));
176
+ }
177
+ };
178
+ async function teardownConnection() {
179
+ isTornDown = true;
180
+ pendingEvents.length = 0;
181
+ rejectDataChannelReady?.(
182
+ new Error("Connection torn down before data channel opened")
183
+ );
184
+ if (localStream) {
185
+ for (const track of localStream.getTracks()) {
186
+ track.stop();
187
+ }
188
+ localStream = null;
189
+ }
190
+ if (audioElement) {
191
+ try {
192
+ audioElement.pause();
193
+ } catch {
194
+ }
195
+ audioElement.srcObject = null;
196
+ audioElement = null;
197
+ }
198
+ if (outputSource) {
199
+ try {
200
+ outputSource.disconnect();
201
+ } catch {
202
+ }
203
+ outputSource = null;
204
+ }
205
+ if (outputAnalyser) {
206
+ try {
207
+ outputAnalyser.disconnect();
208
+ } catch {
209
+ }
210
+ outputAnalyser = null;
211
+ }
212
+ if (inputSource) {
213
+ try {
214
+ inputSource.disconnect();
215
+ } catch {
216
+ }
217
+ inputSource = null;
218
+ }
219
+ if (inputAnalyser) {
220
+ try {
221
+ inputAnalyser.disconnect();
222
+ } catch {
223
+ }
224
+ inputAnalyser = null;
225
+ }
226
+ if (dataChannel) {
227
+ try {
228
+ dataChannel.close();
229
+ } catch {
230
+ }
231
+ dataChannel = null;
232
+ }
233
+ try {
234
+ pc.close();
235
+ } catch {
236
+ }
237
+ if (audioContext) {
238
+ try {
239
+ await audioContext.close();
240
+ } catch {
241
+ }
242
+ audioContext = null;
243
+ }
244
+ }
245
+ try {
246
+ try {
247
+ localStream = await navigator.mediaDevices.getUserMedia({
248
+ audio: {
249
+ echoCancellation: true,
250
+ noiseSuppression: true,
251
+ sampleRate: 24e3
252
+ }
253
+ });
254
+ } catch (error) {
255
+ logger.errors("grok.realtime fatal", {
256
+ error,
257
+ source: "grok.realtime.getUserMedia"
258
+ });
259
+ throw new Error(
260
+ `Microphone access required for realtime voice: ${error instanceof Error ? error.message : error}`
261
+ );
262
+ }
263
+ for (const track of localStream.getAudioTracks()) {
264
+ pc.addTrack(track, localStream);
265
+ }
266
+ const offer = await pc.createOffer();
267
+ await pc.setLocalDescription(offer);
268
+ const sdpResponse = await fetch(`${GROK_REALTIME_URL}?model=${model}`, {
269
+ method: "POST",
270
+ headers: {
271
+ Authorization: `Bearer ${token.token}`,
272
+ "Content-Type": "application/sdp"
273
+ },
274
+ body: offer.sdp
275
+ });
276
+ if (!sdpResponse.ok) {
277
+ const errorText = await sdpResponse.text();
278
+ const error = new Error(
279
+ `Failed to establish WebRTC connection: ${sdpResponse.status} - ${errorText}`
280
+ );
281
+ logger.errors("grok.realtime fatal", {
282
+ error,
283
+ source: "grok.realtime.sdp",
284
+ status: sdpResponse.status
285
+ });
286
+ throw error;
287
+ }
288
+ const answerSdp = await sdpResponse.text();
289
+ await pc.setRemoteDescription({ type: "answer", sdp: answerSdp });
290
+ } catch (err) {
291
+ await teardownConnection();
292
+ throw err;
293
+ }
294
+ try {
295
+ setupInputAudioAnalysis(localStream);
296
+ await dataChannelReady;
297
+ } catch (err) {
298
+ await teardownConnection();
299
+ throw err;
300
+ }
301
+ function handleServerEvent(event) {
302
+ const type = readString(event, "type");
303
+ switch (type) {
304
+ case "session.created":
305
+ case "session.updated":
306
+ break;
307
+ case "input_audio_buffer.speech_started":
308
+ currentMode = "listening";
309
+ emit("mode_change", { mode: "listening" });
310
+ break;
311
+ case "input_audio_buffer.speech_stopped":
312
+ currentMode = "thinking";
313
+ emit("mode_change", { mode: "thinking" });
314
+ break;
315
+ case "input_audio_buffer.committed":
316
+ break;
317
+ case "conversation.item.input_audio_transcription.completed": {
318
+ const transcript = readString(event, "transcript");
319
+ if (transcript === void 0) break;
320
+ emit("transcript", { role: "user", transcript, isFinal: true });
321
+ break;
322
+ }
323
+ case "response.created":
324
+ currentMessageId = null;
325
+ currentMode = "thinking";
326
+ emit("mode_change", { mode: "thinking" });
327
+ break;
328
+ case "response.output_item.added": {
329
+ const item = readObject(event, "item");
330
+ if (item && readString(item, "type") === "message") {
331
+ const id = readString(item, "id");
332
+ if (id !== void 0) currentMessageId = id;
333
+ }
334
+ break;
335
+ }
336
+ // xAI realtime per docs uses `response.output_audio_transcript.*`;
337
+ // accept the legacy OpenAI-realtime `response.audio_transcript.*` as
338
+ // an alias so this adapter stays compatible across protocol versions.
339
+ case "response.output_audio_transcript.delta":
340
+ case "response.audio_transcript.delta": {
341
+ const delta = readString(event, "delta");
342
+ if (delta === void 0) break;
343
+ emit("transcript", {
344
+ role: "assistant",
345
+ transcript: delta,
346
+ isFinal: false
347
+ });
348
+ break;
349
+ }
350
+ case "response.output_audio_transcript.done":
351
+ case "response.audio_transcript.done": {
352
+ const transcript = readString(event, "transcript");
353
+ if (transcript === void 0) break;
354
+ emit("transcript", { role: "assistant", transcript, isFinal: true });
355
+ break;
356
+ }
357
+ // xAI realtime per docs uses `response.text.*`; accept the legacy
358
+ // OpenAI-realtime `response.output_text.*` as an alias.
359
+ case "response.text.delta":
360
+ case "response.output_text.delta": {
361
+ const delta = readString(event, "delta");
362
+ if (delta === void 0) break;
363
+ emit("transcript", {
364
+ role: "assistant",
365
+ transcript: delta,
366
+ isFinal: false
367
+ });
368
+ break;
369
+ }
370
+ case "response.text.done":
371
+ case "response.output_text.done": {
372
+ const text = readString(event, "text");
373
+ if (text === void 0) break;
374
+ emit("transcript", {
375
+ role: "assistant",
376
+ transcript: text,
377
+ isFinal: true
378
+ });
379
+ break;
380
+ }
381
+ // xAI realtime per docs uses `response.output_audio.*`; accept the
382
+ // legacy OpenAI-realtime `response.audio.*` as an alias.
383
+ case "response.output_audio.delta":
384
+ case "response.audio.delta":
385
+ if (currentMode !== "speaking") {
386
+ currentMode = "speaking";
387
+ emit("mode_change", { mode: "speaking" });
388
+ }
389
+ break;
390
+ case "response.output_audio.done":
391
+ case "response.audio.done":
392
+ break;
393
+ case "response.function_call_arguments.done": {
394
+ const callId = readString(event, "call_id");
395
+ const name = readString(event, "name") ?? "";
396
+ const args = readString(event, "arguments") ?? "";
397
+ if (!callId) {
398
+ logger.errors(
399
+ "grok.realtime tool_call missing call_id — dropping tool_call",
400
+ {
401
+ source: "grok.realtime",
402
+ event_type: "response.function_call_arguments.done",
403
+ item_id: event.item_id
404
+ }
405
+ );
406
+ emit("error", {
407
+ error: new Error(
408
+ "Realtime tool call missing call_id; tool will not execute"
409
+ )
410
+ });
411
+ break;
412
+ }
413
+ try {
414
+ const input = JSON.parse(args);
415
+ emit("tool_call", { toolCallId: callId, toolName: name, input });
416
+ } catch {
417
+ emit("tool_call", { toolCallId: callId, toolName: name, input: args });
418
+ }
419
+ break;
420
+ }
421
+ case "response.done": {
422
+ const response = readObject(event, "response") ?? {};
423
+ const output = readObjectArray(response, "output");
424
+ if (currentMode !== "idle") {
425
+ currentMode = "listening";
426
+ emit("mode_change", { mode: "listening" });
427
+ }
428
+ if (currentMessageId) {
429
+ const message = {
430
+ id: currentMessageId,
431
+ role: "assistant",
432
+ timestamp: Date.now(),
433
+ parts: []
434
+ };
435
+ for (const item of output ?? []) {
436
+ if (readString(item, "type") !== "message") continue;
437
+ const content = readObjectArray(item, "content");
438
+ if (!content) continue;
439
+ for (const part of content) {
440
+ const partType = readString(part, "type");
441
+ if (partType === "audio") {
442
+ const transcript = readString(part, "transcript");
443
+ if (transcript) {
444
+ message.parts.push({ type: "audio", transcript });
445
+ }
446
+ } else if (partType === "text") {
447
+ const content2 = readString(part, "text");
448
+ if (content2) {
449
+ message.parts.push({ type: "text", content: content2 });
450
+ }
451
+ }
452
+ }
453
+ }
454
+ emit("message_complete", { message });
455
+ currentMessageId = null;
456
+ }
457
+ break;
458
+ }
459
+ case "conversation.item.truncated":
460
+ if (currentMode !== "idle") {
461
+ currentMode = "listening";
462
+ emit("mode_change", { mode: "listening" });
463
+ }
464
+ emit("interrupted", {
465
+ ...currentMessageId !== null && { messageId: currentMessageId }
466
+ });
467
+ break;
468
+ case "error": {
469
+ const errorObj = readObject(event, "error") ?? {};
470
+ const message = readString(errorObj, "message") ?? "Unknown realtime server error";
471
+ const err = new Error(message);
472
+ const code = readString(errorObj, "code");
473
+ if (code !== void 0) err.code = code;
474
+ const errType = readString(errorObj, "type");
475
+ if (errType !== void 0) err.type = errType;
476
+ const param = readString(errorObj, "param");
477
+ if (param !== void 0) err.param = param;
478
+ logger.errors("grok.realtime server error", {
479
+ ...errorObj,
480
+ source: "grok.realtime server"
481
+ });
482
+ emit("error", { error: err });
483
+ break;
484
+ }
485
+ case void 0:
486
+ default:
487
+ logger.provider("grok.realtime unhandled server event", {
488
+ type: event.type
489
+ });
490
+ break;
491
+ }
492
+ }
493
+ function setupOutputAudioAnalysis(stream) {
494
+ if (isTornDown) return;
495
+ if (audioElement) {
496
+ try {
497
+ audioElement.pause();
498
+ } catch {
499
+ }
500
+ audioElement.srcObject = null;
501
+ audioElement = null;
502
+ }
503
+ if (outputSource) {
504
+ try {
505
+ outputSource.disconnect();
506
+ } catch {
507
+ }
508
+ outputSource = null;
509
+ }
510
+ if (outputAnalyser) {
511
+ try {
512
+ outputAnalyser.disconnect();
513
+ } catch {
514
+ }
515
+ outputAnalyser = null;
516
+ }
517
+ audioElement = new Audio();
518
+ audioElement.srcObject = stream;
519
+ audioElement.autoplay = true;
520
+ audioElement.play().catch((e) => {
521
+ logger.errors("grok.realtime audio autoplay blocked", {
522
+ error: e,
523
+ source: "grok.realtime.audio_permission_required"
524
+ });
525
+ });
526
+ if (!audioContext) {
527
+ audioContext = new AudioContext();
528
+ }
529
+ if (audioContext.state === "suspended") {
530
+ audioContext.resume().catch((err) => {
531
+ logger.errors("grok.realtime audioContext.resume failed", {
532
+ error: err,
533
+ source: "grok.realtime"
534
+ });
535
+ });
536
+ }
537
+ outputAnalyser = audioContext.createAnalyser();
538
+ outputAnalyser.fftSize = 2048;
539
+ outputAnalyser.smoothingTimeConstant = 0.3;
540
+ outputSource = audioContext.createMediaStreamSource(stream);
541
+ outputSource.connect(outputAnalyser);
542
+ }
543
+ function setupInputAudioAnalysis(stream) {
544
+ if (isTornDown) return;
545
+ if (!audioContext) {
546
+ audioContext = new AudioContext();
547
+ }
548
+ if (audioContext.state === "suspended") {
549
+ audioContext.resume().catch((err) => {
550
+ logger.errors("grok.realtime audioContext.resume failed", {
551
+ error: err,
552
+ source: "grok.realtime"
553
+ });
554
+ });
555
+ }
556
+ inputAnalyser = audioContext.createAnalyser();
557
+ inputAnalyser.fftSize = 2048;
558
+ inputAnalyser.smoothingTimeConstant = 0.3;
559
+ inputSource = audioContext.createMediaStreamSource(stream);
560
+ inputSource.connect(inputAnalyser);
561
+ }
562
+ function sendEvent(event) {
563
+ if (isTornDown) {
564
+ logger.errors("grok.realtime sendEvent after disconnect", {
565
+ eventType: readString(event, "type") ?? "<unknown>",
566
+ source: "grok.realtime"
567
+ });
568
+ return;
569
+ }
570
+ if (dataChannel?.readyState === "open") {
571
+ logger.provider(
572
+ `provider=grok direction=out type=${readString(event, "type") ?? "<unknown>"}`,
573
+ { frame: event }
574
+ );
575
+ try {
576
+ dataChannel.send(JSON.stringify(event));
577
+ } catch (error) {
578
+ logger.errors("grok.realtime sendEvent failed", {
579
+ error,
580
+ eventType: readString(event, "type") ?? "<unknown>",
581
+ source: "grok.realtime"
582
+ });
583
+ emit("error", {
584
+ error: error instanceof Error ? error : new Error(String(error))
585
+ });
586
+ }
587
+ } else {
588
+ pendingEvents.push(event);
589
+ }
590
+ }
591
+ function flushPendingEvents() {
592
+ try {
593
+ for (const event of pendingEvents) {
594
+ logger.provider(
595
+ `provider=grok direction=out type=${readString(event, "type") ?? "<unknown>"}`,
596
+ { frame: event }
597
+ );
598
+ channel.send(JSON.stringify(event));
599
+ }
600
+ pendingEvents.length = 0;
601
+ } catch (error) {
602
+ logger.errors("grok.realtime flushPendingEvents failed", {
603
+ error,
604
+ source: "grok.realtime"
605
+ });
606
+ const err = error instanceof Error ? error : new Error(String(error));
607
+ rejectDataChannelReady?.(err);
608
+ emit("error", { error: err });
609
+ }
610
+ }
611
+ const connection = {
612
+ async disconnect() {
613
+ await teardownConnection();
614
+ emit("status_change", { status: "idle" });
615
+ },
616
+ async startAudioCapture() {
617
+ if (localStream) {
618
+ for (const track of localStream.getAudioTracks()) {
619
+ track.enabled = true;
620
+ }
621
+ }
622
+ currentMode = "listening";
623
+ emit("mode_change", { mode: "listening" });
624
+ },
625
+ stopAudioCapture() {
626
+ if (localStream) {
627
+ for (const track of localStream.getAudioTracks()) {
628
+ track.enabled = false;
629
+ }
630
+ }
631
+ currentMode = "idle";
632
+ emit("mode_change", { mode: "idle" });
633
+ },
634
+ sendText(text) {
635
+ sendEvent({
636
+ type: "conversation.item.create",
637
+ item: {
638
+ type: "message",
639
+ role: "user",
640
+ content: [{ type: "input_text", text }]
641
+ }
642
+ });
643
+ sendEvent({ type: "response.create" });
644
+ },
645
+ sendImage(imageData, mimeType) {
646
+ const isAlreadyUrlOrDataUri = imageData.startsWith("http://") || imageData.startsWith("https://") || imageData.startsWith("data:");
647
+ const imageContent = {
648
+ type: "input_image",
649
+ // The OpenAI-realtime content part (which this adapter mirrors) nests
650
+ // the URL under an `image_url: { url: ... }` object, not a bare
651
+ // string.
652
+ image_url: {
653
+ url: isAlreadyUrlOrDataUri ? imageData : `data:${mimeType};base64,${imageData}`
654
+ }
655
+ };
656
+ sendEvent({
657
+ type: "conversation.item.create",
658
+ item: {
659
+ type: "message",
660
+ role: "user",
661
+ content: [imageContent]
662
+ }
663
+ });
664
+ sendEvent({ type: "response.create" });
665
+ },
666
+ sendToolResult(callId, result) {
667
+ sendEvent({
668
+ type: "conversation.item.create",
669
+ item: {
670
+ type: "function_call_output",
671
+ call_id: callId,
672
+ output: result
673
+ }
674
+ });
675
+ sendEvent({ type: "response.create" });
676
+ },
677
+ updateSession(config) {
678
+ const sessionUpdate = {};
679
+ if (config.instructions) {
680
+ sessionUpdate.instructions = config.instructions;
681
+ }
682
+ if (config.voice) {
683
+ sessionUpdate.voice = config.voice;
684
+ }
685
+ if (config.vadMode) {
686
+ if (config.vadMode === "semantic") {
687
+ sessionUpdate.turn_detection = {
688
+ type: "semantic_vad",
689
+ eagerness: config.semanticEagerness ?? "medium"
690
+ };
691
+ } else if (config.vadMode === "server") {
692
+ sessionUpdate.turn_detection = {
693
+ type: "server_vad",
694
+ threshold: config.vadConfig?.threshold ?? 0.5,
695
+ prefix_padding_ms: config.vadConfig?.prefixPaddingMs ?? 300,
696
+ silence_duration_ms: config.vadConfig?.silenceDurationMs ?? 500
697
+ };
698
+ } else {
699
+ sessionUpdate.turn_detection = null;
700
+ }
701
+ }
702
+ if (config.tools !== void 0) {
703
+ sessionUpdate.tools = config.tools.map((t) => ({
704
+ type: "function",
705
+ name: t.name,
706
+ description: t.description,
707
+ parameters: t.inputSchema ?? { type: "object", properties: {} }
708
+ }));
709
+ sessionUpdate.tool_choice = "auto";
710
+ }
711
+ if (config.outputModalities) {
712
+ sessionUpdate.modalities = config.outputModalities;
713
+ }
714
+ if (config.temperature !== void 0) {
715
+ sessionUpdate.temperature = config.temperature;
716
+ }
717
+ if (config.maxOutputTokens !== void 0) {
718
+ sessionUpdate.max_response_output_tokens = config.maxOutputTokens;
719
+ }
720
+ const providerOptions = config.providerOptions ?? {};
721
+ const callerTranscription = "inputAudioTranscription" in providerOptions ? providerOptions.inputAudioTranscription : "input_audio_transcription" in providerOptions ? providerOptions.input_audio_transcription : void 0;
722
+ if (callerTranscription !== void 0) {
723
+ sessionUpdate.input_audio_transcription = callerTranscription === false ? null : callerTranscription;
724
+ } else if (!hasSentInitialSessionUpdate) {
725
+ sessionUpdate.input_audio_transcription = { model: "grok-stt" };
726
+ }
727
+ if (Object.keys(sessionUpdate).length > 0) {
728
+ sendEvent({
729
+ type: "session.update",
730
+ session: sessionUpdate
731
+ });
732
+ hasSentInitialSessionUpdate = true;
733
+ }
734
+ },
735
+ interrupt() {
736
+ sendEvent({ type: "response.cancel" });
737
+ currentMode = "listening";
738
+ emit("mode_change", { mode: "listening" });
739
+ emit("interrupted", {
740
+ ...currentMessageId !== null && { messageId: currentMessageId }
741
+ });
742
+ },
743
+ on(event, handler) {
744
+ let handlers = eventHandlers.get(event);
745
+ if (!handlers) {
746
+ handlers = /* @__PURE__ */ new Set();
747
+ eventHandlers.set(event, handlers);
748
+ }
749
+ handlers.add(handler);
750
+ return () => {
751
+ eventHandlers.get(event)?.delete(handler);
752
+ };
753
+ },
754
+ getAudioVisualization() {
755
+ function calculateLevel(analyser) {
756
+ const data = new Uint8Array(analyser.fftSize);
757
+ analyser.getByteTimeDomainData(data);
758
+ let maxDeviation = 0;
759
+ for (const sample of data) {
760
+ const deviation = Math.abs(sample - 128);
761
+ if (deviation > maxDeviation) {
762
+ maxDeviation = deviation;
763
+ }
764
+ }
765
+ const normalized = maxDeviation / 128;
766
+ return Math.min(1, normalized * 1.5);
767
+ }
768
+ return {
769
+ get inputLevel() {
770
+ if (!inputAnalyser) return 0;
771
+ return calculateLevel(inputAnalyser);
772
+ },
773
+ get outputLevel() {
774
+ if (!outputAnalyser) return 0;
775
+ return calculateLevel(outputAnalyser);
776
+ },
777
+ getInputFrequencyData() {
778
+ if (!inputAnalyser)
779
+ return new Uint8Array(FALLBACK_FREQUENCY_BIN_COUNT);
780
+ const data = new Uint8Array(inputAnalyser.frequencyBinCount);
781
+ inputAnalyser.getByteFrequencyData(data);
782
+ return data;
783
+ },
784
+ getOutputFrequencyData() {
785
+ if (!outputAnalyser)
786
+ return new Uint8Array(FALLBACK_FREQUENCY_BIN_COUNT);
787
+ const data = new Uint8Array(outputAnalyser.frequencyBinCount);
788
+ outputAnalyser.getByteFrequencyData(data);
789
+ return data;
790
+ },
791
+ getInputTimeDomainData() {
792
+ if (!inputAnalyser)
793
+ return new Uint8Array(FALLBACK_TIME_DOMAIN_SIZE).fill(
794
+ FALLBACK_TIME_DOMAIN_FILL
795
+ );
796
+ const data = new Uint8Array(inputAnalyser.fftSize);
797
+ inputAnalyser.getByteTimeDomainData(data);
798
+ return data;
799
+ },
800
+ getOutputTimeDomainData() {
801
+ if (!outputAnalyser)
802
+ return new Uint8Array(FALLBACK_TIME_DOMAIN_SIZE).fill(
803
+ FALLBACK_TIME_DOMAIN_FILL
804
+ );
805
+ const data = new Uint8Array(outputAnalyser.fftSize);
806
+ outputAnalyser.getByteTimeDomainData(data);
807
+ return data;
808
+ },
809
+ get inputSampleRate() {
810
+ return 24e3;
811
+ },
812
+ get outputSampleRate() {
813
+ return 24e3;
814
+ }
815
+ };
816
+ }
817
+ };
818
+ return connection;
819
+ }
820
+ export {
821
+ grokRealtime
822
+ };
823
+ //# sourceMappingURL=adapter.js.map