@tanstack/ai-client 0.22.1 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/README.md +15 -1
  2. package/dist/esm/audio-recorder.js +190 -213
  3. package/dist/esm/audio-recorder.js.map +1 -1
  4. package/dist/esm/chat-client.d.ts +172 -3
  5. package/dist/esm/chat-client.js +1656 -1386
  6. package/dist/esm/chat-client.js.map +1 -1
  7. package/dist/esm/cleared-stream-tracker.d.ts +23 -0
  8. package/dist/esm/cleared-stream-tracker.js +97 -0
  9. package/dist/esm/cleared-stream-tracker.js.map +1 -0
  10. package/dist/esm/client-persistor.d.ts +25 -12
  11. package/dist/esm/client-persistor.js +260 -235
  12. package/dist/esm/client-persistor.js.map +1 -1
  13. package/dist/esm/connection-adapters.d.ts +231 -10
  14. package/dist/esm/connection-adapters.js +989 -574
  15. package/dist/esm/connection-adapters.js.map +1 -1
  16. package/dist/esm/devtools-noop.d.ts +1 -0
  17. package/dist/esm/devtools-noop.js +79 -139
  18. package/dist/esm/devtools-noop.js.map +1 -1
  19. package/dist/esm/devtools.d.ts +31 -1
  20. package/dist/esm/devtools.js +977 -1127
  21. package/dist/esm/devtools.js.map +1 -1
  22. package/dist/esm/events.js +224 -226
  23. package/dist/esm/events.js.map +1 -1
  24. package/dist/esm/generation-client.d.ts +145 -2
  25. package/dist/esm/generation-client.js +659 -321
  26. package/dist/esm/generation-client.js.map +1 -1
  27. package/dist/esm/generation-reconstruct.d.ts +21 -0
  28. package/dist/esm/generation-reconstruct.js +85 -0
  29. package/dist/esm/generation-reconstruct.js.map +1 -0
  30. package/dist/esm/generation-types.d.ts +289 -3
  31. package/dist/esm/generation-types.js +356 -13
  32. package/dist/esm/generation-types.js.map +1 -1
  33. package/dist/esm/index.d.ts +9 -4
  34. package/dist/esm/index.js +7 -39
  35. package/dist/esm/interrupt-manager.d.ts +77 -0
  36. package/dist/esm/interrupt-manager.js +787 -0
  37. package/dist/esm/interrupt-manager.js.map +1 -0
  38. package/dist/esm/mcp-app-bridge.js +56 -64
  39. package/dist/esm/mcp-app-bridge.js.map +1 -1
  40. package/dist/esm/realtime-client.js +366 -440
  41. package/dist/esm/realtime-client.js.map +1 -1
  42. package/dist/esm/response-stream.js +19 -26
  43. package/dist/esm/response-stream.js.map +1 -1
  44. package/dist/esm/sse-parser.js +44 -47
  45. package/dist/esm/sse-parser.js.map +1 -1
  46. package/dist/esm/sse-utils.js +8 -9
  47. package/dist/esm/sse-utils.js.map +1 -1
  48. package/dist/esm/storage-adapters.d.ts +62 -0
  49. package/dist/esm/storage-adapters.js +174 -0
  50. package/dist/esm/storage-adapters.js.map +1 -0
  51. package/dist/esm/types.d.ts +212 -10
  52. package/dist/esm/types.js +38 -7
  53. package/dist/esm/types.js.map +1 -1
  54. package/dist/esm/video-generation-client.d.ts +113 -2
  55. package/dist/esm/video-generation-client.js +665 -379
  56. package/dist/esm/video-generation-client.js.map +1 -1
  57. package/package.json +7 -7
  58. package/src/chat-client.ts +1079 -61
  59. package/src/cleared-stream-tracker.ts +151 -0
  60. package/src/client-persistor.ts +102 -33
  61. package/src/connection-adapters.ts +1185 -142
  62. package/src/devtools-noop.ts +4 -3
  63. package/src/devtools.ts +121 -3
  64. package/src/generation-client.ts +563 -13
  65. package/src/generation-reconstruct.ts +121 -0
  66. package/src/generation-types.ts +727 -3
  67. package/src/index.ts +56 -1
  68. package/src/interrupt-manager.ts +1440 -0
  69. package/src/storage-adapters.ts +242 -0
  70. package/src/types.ts +301 -9
  71. package/src/video-generation-client.ts +479 -13
  72. package/dist/esm/index.js.map +0 -1
@@ -1,442 +1,368 @@
1
1
  import { convertSchemaToJsonSchema } from "@tanstack/ai/client";
2
- const TOKEN_REFRESH_BUFFER_MS = 6e4;
3
- class RealtimeClient {
4
- options;
5
- connection = null;
6
- token = null;
7
- tokenRefreshTimeout = null;
8
- clientTools;
9
- stateChangeCallbacks = /* @__PURE__ */ new Set();
10
- unsubscribers = [];
11
- state = {
12
- status: "idle",
13
- mode: "idle",
14
- messages: [],
15
- pendingUserTranscript: null,
16
- pendingAssistantTranscript: null,
17
- error: null
18
- };
19
- constructor(options) {
20
- this.options = {
21
- autoPlayback: true,
22
- autoCapture: true,
23
- vadMode: "server",
24
- ...options
25
- };
26
- this.clientTools = /* @__PURE__ */ new Map();
27
- if (options.tools) {
28
- for (const tool of options.tools) {
29
- this.clientTools.set(tool.name, tool);
30
- }
31
- }
32
- }
33
- // ============================================================================
34
- // Connection Lifecycle
35
- // ============================================================================
36
- /**
37
- * Connect to the realtime session.
38
- * Fetches a token and establishes the connection.
39
- */
40
- async connect() {
41
- if (this.state.status === "connected") {
42
- return;
43
- }
44
- this.updateState({ status: "connecting", error: null });
45
- try {
46
- this.token = await this.options.getToken();
47
- this.scheduleTokenRefresh();
48
- const toolsList = this.clientTools.size > 0 ? Array.from(this.clientTools.values()) : void 0;
49
- this.connection = await this.options.adapter.connect(
50
- this.token,
51
- toolsList
52
- );
53
- this.subscribeToConnectionEvents();
54
- this.applySessionConfig();
55
- if (this.options.autoCapture) {
56
- await this.connection.startAudioCapture();
57
- }
58
- this.updateState({ status: "connected", mode: "listening" });
59
- this.options.onConnect?.();
60
- } catch (error) {
61
- const err = error instanceof Error ? error : new Error(String(error));
62
- this.updateState({ status: "error", error: err });
63
- this.options.onError?.(err);
64
- throw err;
65
- }
66
- }
67
- /**
68
- * Disconnect from the realtime session.
69
- */
70
- async disconnect() {
71
- if (this.tokenRefreshTimeout) {
72
- clearTimeout(this.tokenRefreshTimeout);
73
- this.tokenRefreshTimeout = null;
74
- }
75
- for (const unsub of this.unsubscribers) {
76
- unsub();
77
- }
78
- this.unsubscribers = [];
79
- if (this.connection) {
80
- await this.connection.disconnect();
81
- this.connection = null;
82
- }
83
- this.token = null;
84
- this.updateState({
85
- status: "idle",
86
- mode: "idle",
87
- pendingUserTranscript: null,
88
- pendingAssistantTranscript: null
89
- });
90
- this.options.onDisconnect?.();
91
- }
92
- // ============================================================================
93
- // Voice Control
94
- // ============================================================================
95
- /**
96
- * Start listening for voice input.
97
- * Only needed when vadMode is 'manual'.
98
- */
99
- startListening() {
100
- if (!this.connection || this.state.status !== "connected") {
101
- return;
102
- }
103
- void this.connection.startAudioCapture();
104
- this.updateState({ mode: "listening" });
105
- }
106
- /**
107
- * Stop listening for voice input.
108
- * Only needed when vadMode is 'manual'.
109
- */
110
- stopListening() {
111
- if (!this.connection) {
112
- return;
113
- }
114
- this.connection.stopAudioCapture();
115
- this.updateState({ mode: "idle" });
116
- }
117
- /**
118
- * Interrupt the current assistant response.
119
- */
120
- interrupt() {
121
- if (!this.connection) {
122
- return;
123
- }
124
- this.connection.interrupt();
125
- }
126
- // ============================================================================
127
- // Text Input
128
- // ============================================================================
129
- /**
130
- * Send a text message instead of voice.
131
- */
132
- sendText(text) {
133
- if (!this.connection || this.state.status !== "connected") {
134
- return;
135
- }
136
- const userMessage = {
137
- id: this.generateId(),
138
- role: "user",
139
- timestamp: Date.now(),
140
- parts: [{ type: "text", content: text }]
141
- };
142
- this.addMessage(userMessage);
143
- this.connection.sendText(text);
144
- }
145
- /**
146
- * Send an image to the conversation.
147
- * @param imageData - Base64-encoded image data or a URL
148
- * @param mimeType - MIME type of the image (e.g., 'image/png', 'image/jpeg')
149
- */
150
- sendImage(imageData, mimeType) {
151
- if (!this.connection || this.state.status !== "connected") {
152
- return;
153
- }
154
- const userMessage = {
155
- id: this.generateId(),
156
- role: "user",
157
- timestamp: Date.now(),
158
- parts: [{ type: "image", data: imageData, mimeType }]
159
- };
160
- this.addMessage(userMessage);
161
- this.connection.sendImage(imageData, mimeType);
162
- }
163
- // ============================================================================
164
- // State Access
165
- // ============================================================================
166
- /** Get current connection status */
167
- get status() {
168
- return this.state.status;
169
- }
170
- /** Get current mode */
171
- get mode() {
172
- return this.state.mode;
173
- }
174
- /** Get conversation messages */
175
- get messages() {
176
- return this.state.messages;
177
- }
178
- /** Get current error, if any */
179
- get error() {
180
- return this.state.error;
181
- }
182
- /** Get pending user transcript (while user is speaking) */
183
- get pendingUserTranscript() {
184
- return this.state.pendingUserTranscript;
185
- }
186
- /** Get pending assistant transcript (while assistant is speaking) */
187
- get pendingAssistantTranscript() {
188
- return this.state.pendingAssistantTranscript;
189
- }
190
- /** Get audio visualization data */
191
- get audio() {
192
- return this.connection?.getAudioVisualization() ?? null;
193
- }
194
- /**
195
- * Update the session configuration.
196
- * This applies changes to the active connection and persists them for future reconnections.
197
- */
198
- updateSession(config) {
199
- const o = this.options;
200
- if (config.instructions !== void 0) o.instructions = config.instructions;
201
- if (config.voice !== void 0) o.voice = config.voice;
202
- if (config.vadMode !== void 0) o.vadMode = config.vadMode;
203
- if (config.outputModalities !== void 0) {
204
- o.outputModalities = config.outputModalities;
205
- }
206
- if (config.temperature !== void 0) o.temperature = config.temperature;
207
- if (config.maxOutputTokens !== void 0) {
208
- o.maxOutputTokens = config.maxOutputTokens;
209
- }
210
- if (config.semanticEagerness !== void 0) {
211
- o.semanticEagerness = config.semanticEagerness;
212
- }
213
- if (config.providerOptions !== void 0) {
214
- o.providerOptions = config.providerOptions;
215
- }
216
- if (this.connection) {
217
- this.applySessionConfig();
218
- }
219
- }
220
- // ============================================================================
221
- // State Subscription
222
- // ============================================================================
223
- /**
224
- * Subscribe to state changes.
225
- * @returns Unsubscribe function
226
- */
227
- onStateChange(callback) {
228
- this.stateChangeCallbacks.add(callback);
229
- return () => {
230
- this.stateChangeCallbacks.delete(callback);
231
- };
232
- }
233
- // ============================================================================
234
- // Cleanup
235
- // ============================================================================
236
- /**
237
- * Clean up resources.
238
- * Call this when disposing of the client.
239
- */
240
- destroy() {
241
- void this.disconnect();
242
- this.stateChangeCallbacks.clear();
243
- }
244
- // ============================================================================
245
- // Private Methods
246
- // ============================================================================
247
- updateState(updates) {
248
- this.state = { ...this.state, ...updates };
249
- for (const callback of this.stateChangeCallbacks) {
250
- callback(this.state);
251
- }
252
- if ("status" in updates && updates.status !== void 0) {
253
- this.options.onStatusChange?.(updates.status);
254
- }
255
- if ("mode" in updates && updates.mode !== void 0) {
256
- this.options.onModeChange?.(updates.mode);
257
- }
258
- }
259
- addMessage(message) {
260
- this.updateState({
261
- messages: [...this.state.messages, message]
262
- });
263
- this.options.onMessage?.(message);
264
- }
265
- scheduleTokenRefresh() {
266
- if (!this.token) return;
267
- const timeUntilExpiry = this.token.expiresAt - Date.now();
268
- const refreshIn = Math.max(0, timeUntilExpiry - TOKEN_REFRESH_BUFFER_MS);
269
- this.tokenRefreshTimeout = setTimeout(() => {
270
- void this.refreshToken();
271
- }, refreshIn);
272
- }
273
- async refreshToken() {
274
- try {
275
- this.token = await this.options.getToken();
276
- this.scheduleTokenRefresh();
277
- this.connection?.updateToken?.(this.token);
278
- } catch (error) {
279
- const err = error instanceof Error ? error : new Error(String(error));
280
- this.updateState({ error: err });
281
- this.options.onError?.(err);
282
- }
283
- }
284
- subscribeToConnectionEvents() {
285
- if (!this.connection) return;
286
- this.unsubscribers.push(
287
- this.connection.on("status_change", ({ status }) => {
288
- this.updateState({ status });
289
- })
290
- );
291
- this.unsubscribers.push(
292
- this.connection.on("mode_change", ({ mode }) => {
293
- this.updateState({ mode });
294
- })
295
- );
296
- this.unsubscribers.push(
297
- this.connection.on("transcript", ({ role, transcript, isFinal }) => {
298
- if (role === "user") {
299
- this.updateState({
300
- pendingUserTranscript: isFinal ? null : transcript
301
- });
302
- if (isFinal && transcript) {
303
- this.addMessage({
304
- id: this.generateId(),
305
- role: "user",
306
- timestamp: Date.now(),
307
- parts: [{ type: "audio", transcript, durationMs: 0 }]
308
- });
309
- }
310
- } else {
311
- this.updateState({
312
- pendingAssistantTranscript: isFinal ? null : transcript
313
- });
314
- }
315
- })
316
- );
317
- this.unsubscribers.push(
318
- this.connection.on("tool_call", ({ toolCallId, toolName, input }) => {
319
- if (!toolCallId) {
320
- console.error(
321
- "[RealtimeClient] tool_call missing toolCallId",
322
- toolName
323
- );
324
- return;
325
- }
326
- const tool = this.clientTools.get(toolName);
327
- const execute = tool?.execute;
328
- if (execute) {
329
- void (async () => {
330
- try {
331
- const output = await execute(input);
332
- this.connection?.sendToolResult(
333
- toolCallId,
334
- typeof output === "string" ? output : JSON.stringify(output)
335
- );
336
- } catch (error) {
337
- const errMsg = error instanceof Error ? error.message : String(error);
338
- this.connection?.sendToolResult(
339
- toolCallId,
340
- JSON.stringify({ error: errMsg })
341
- );
342
- }
343
- })();
344
- } else {
345
- this.connection?.sendToolResult(
346
- toolCallId,
347
- JSON.stringify({
348
- error: `Unknown or non-executable tool: ${String(toolName)}`
349
- })
350
- );
351
- }
352
- })
353
- );
354
- this.unsubscribers.push(
355
- this.connection.on("message_complete", ({ message }) => {
356
- const existingIndex = this.state.messages.findIndex(
357
- (m) => m.id === message.id
358
- );
359
- if (existingIndex >= 0) {
360
- const newMessages = [...this.state.messages];
361
- newMessages[existingIndex] = message;
362
- this.updateState({ messages: newMessages });
363
- } else {
364
- this.addMessage(message);
365
- }
366
- })
367
- );
368
- this.unsubscribers.push(
369
- this.connection.on("interrupted", ({ messageId }) => {
370
- if (messageId) {
371
- const newMessages = this.state.messages.map(
372
- (m) => m.id === messageId ? { ...m, interrupted: true } : m
373
- );
374
- this.updateState({ messages: newMessages });
375
- }
376
- this.updateState({
377
- mode: "listening",
378
- pendingAssistantTranscript: null
379
- });
380
- this.options.onInterrupted?.();
381
- })
382
- );
383
- this.unsubscribers.push(
384
- this.connection.on("error", ({ error }) => {
385
- this.updateState({ error });
386
- this.options.onError?.(error);
387
- })
388
- );
389
- this.unsubscribers.push(
390
- this.connection.on("go_away", ({ timeLeft }) => {
391
- this.options.onGoAway?.(timeLeft);
392
- })
393
- );
394
- this.unsubscribers.push(
395
- this.connection.on("usage", (usage) => {
396
- this.options.onUsage?.(usage);
397
- })
398
- );
399
- }
400
- applySessionConfig() {
401
- if (!this.connection) return;
402
- const {
403
- instructions,
404
- voice,
405
- vadMode,
406
- tools,
407
- outputModalities,
408
- temperature,
409
- maxOutputTokens,
410
- semanticEagerness
411
- } = this.options;
412
- const hasConfig = instructions || voice || vadMode || tools && tools.length > 0 || outputModalities || temperature !== void 0 || maxOutputTokens !== void 0 || semanticEagerness;
413
- if (!hasConfig) return;
414
- const toolsConfig = tools ? Array.from(this.clientTools.values()).map((t) => {
415
- const inputSchema = t.inputSchema ? convertSchemaToJsonSchema(t.inputSchema) : void 0;
416
- const outputSchema = t.outputSchema ? convertSchemaToJsonSchema(t.outputSchema) : void 0;
417
- return {
418
- name: t.name,
419
- description: t.description,
420
- ...inputSchema ? { inputSchema } : {},
421
- ...outputSchema ? { outputSchema } : {}
422
- };
423
- }) : void 0;
424
- this.connection.updateSession({
425
- ...instructions !== void 0 ? { instructions } : {},
426
- ...voice !== void 0 ? { voice } : {},
427
- ...vadMode !== void 0 ? { vadMode } : {},
428
- ...toolsConfig !== void 0 ? { tools: toolsConfig } : {},
429
- ...outputModalities !== void 0 ? { outputModalities } : {},
430
- ...temperature !== void 0 ? { temperature } : {},
431
- ...maxOutputTokens !== void 0 ? { maxOutputTokens } : {},
432
- ...semanticEagerness !== void 0 ? { semanticEagerness } : {}
433
- });
434
- }
435
- generateId() {
436
- return `msg-${Date.now()}-${Math.random().toString(36).substring(7)}`;
437
- }
438
- }
439
- export {
440
- RealtimeClient
2
+ //#region src/realtime-client.ts
3
+ var TOKEN_REFRESH_BUFFER_MS = 6e4;
4
+ /**
5
+ * Client for managing realtime voice conversations.
6
+ *
7
+ * Handles connection lifecycle, audio I/O, message state,
8
+ * and tool execution for realtime voice-to-voice AI interactions.
9
+ *
10
+ * @example
11
+ * ```typescript
12
+ * import { RealtimeClient } from '@tanstack/ai-client'
13
+ * import { openaiRealtime } from '@tanstack/ai-openai'
14
+ *
15
+ * const client = new RealtimeClient({
16
+ * getToken: () => fetch('/api/realtime-token').then(r => r.json()),
17
+ * adapter: openaiRealtime(),
18
+ * tools: [myTool.client(handler)],
19
+ * onMessage: (msg) => console.log('Message:', msg),
20
+ * })
21
+ *
22
+ * await client.connect()
23
+ * ```
24
+ */
25
+ var RealtimeClient = class {
26
+ options;
27
+ connection = null;
28
+ token = null;
29
+ tokenRefreshTimeout = null;
30
+ clientTools;
31
+ stateChangeCallbacks = /* @__PURE__ */ new Set();
32
+ unsubscribers = [];
33
+ state = {
34
+ status: "idle",
35
+ mode: "idle",
36
+ messages: [],
37
+ pendingUserTranscript: null,
38
+ pendingAssistantTranscript: null,
39
+ error: null
40
+ };
41
+ constructor(options) {
42
+ this.options = {
43
+ autoPlayback: true,
44
+ autoCapture: true,
45
+ vadMode: "server",
46
+ ...options
47
+ };
48
+ this.clientTools = /* @__PURE__ */ new Map();
49
+ if (options.tools) for (const tool of options.tools) this.clientTools.set(tool.name, tool);
50
+ }
51
+ /**
52
+ * Connect to the realtime session.
53
+ * Fetches a token and establishes the connection.
54
+ */
55
+ async connect() {
56
+ if (this.state.status === "connected") return;
57
+ this.updateState({
58
+ status: "connecting",
59
+ error: null
60
+ });
61
+ try {
62
+ this.token = await this.options.getToken();
63
+ this.scheduleTokenRefresh();
64
+ const toolsList = this.clientTools.size > 0 ? Array.from(this.clientTools.values()) : void 0;
65
+ this.connection = await this.options.adapter.connect(this.token, toolsList);
66
+ this.subscribeToConnectionEvents();
67
+ this.applySessionConfig();
68
+ if (this.options.autoCapture) await this.connection.startAudioCapture();
69
+ this.updateState({
70
+ status: "connected",
71
+ mode: "listening"
72
+ });
73
+ this.options.onConnect?.();
74
+ } catch (error) {
75
+ const err = error instanceof Error ? error : new Error(String(error));
76
+ this.updateState({
77
+ status: "error",
78
+ error: err
79
+ });
80
+ this.options.onError?.(err);
81
+ throw err;
82
+ }
83
+ }
84
+ /**
85
+ * Disconnect from the realtime session.
86
+ */
87
+ async disconnect() {
88
+ if (this.tokenRefreshTimeout) {
89
+ clearTimeout(this.tokenRefreshTimeout);
90
+ this.tokenRefreshTimeout = null;
91
+ }
92
+ for (const unsub of this.unsubscribers) unsub();
93
+ this.unsubscribers = [];
94
+ if (this.connection) {
95
+ await this.connection.disconnect();
96
+ this.connection = null;
97
+ }
98
+ this.token = null;
99
+ this.updateState({
100
+ status: "idle",
101
+ mode: "idle",
102
+ pendingUserTranscript: null,
103
+ pendingAssistantTranscript: null
104
+ });
105
+ this.options.onDisconnect?.();
106
+ }
107
+ /**
108
+ * Start listening for voice input.
109
+ * Only needed when vadMode is 'manual'.
110
+ */
111
+ startListening() {
112
+ if (!this.connection || this.state.status !== "connected") return;
113
+ this.connection.startAudioCapture();
114
+ this.updateState({ mode: "listening" });
115
+ }
116
+ /**
117
+ * Stop listening for voice input.
118
+ * Only needed when vadMode is 'manual'.
119
+ */
120
+ stopListening() {
121
+ if (!this.connection) return;
122
+ this.connection.stopAudioCapture();
123
+ this.updateState({ mode: "idle" });
124
+ }
125
+ /**
126
+ * Interrupt the current assistant response.
127
+ */
128
+ interrupt() {
129
+ if (!this.connection) return;
130
+ this.connection.interrupt();
131
+ }
132
+ /**
133
+ * Send a text message instead of voice.
134
+ */
135
+ sendText(text) {
136
+ if (!this.connection || this.state.status !== "connected") return;
137
+ const userMessage = {
138
+ id: this.generateId(),
139
+ role: "user",
140
+ timestamp: Date.now(),
141
+ parts: [{
142
+ type: "text",
143
+ content: text
144
+ }]
145
+ };
146
+ this.addMessage(userMessage);
147
+ this.connection.sendText(text);
148
+ }
149
+ /**
150
+ * Send an image to the conversation.
151
+ * @param imageData - Base64-encoded image data or a URL
152
+ * @param mimeType - MIME type of the image (e.g., 'image/png', 'image/jpeg')
153
+ */
154
+ sendImage(imageData, mimeType) {
155
+ if (!this.connection || this.state.status !== "connected") return;
156
+ const userMessage = {
157
+ id: this.generateId(),
158
+ role: "user",
159
+ timestamp: Date.now(),
160
+ parts: [{
161
+ type: "image",
162
+ data: imageData,
163
+ mimeType
164
+ }]
165
+ };
166
+ this.addMessage(userMessage);
167
+ this.connection.sendImage(imageData, mimeType);
168
+ }
169
+ /** Get current connection status */
170
+ get status() {
171
+ return this.state.status;
172
+ }
173
+ /** Get current mode */
174
+ get mode() {
175
+ return this.state.mode;
176
+ }
177
+ /** Get conversation messages */
178
+ get messages() {
179
+ return this.state.messages;
180
+ }
181
+ /** Get current error, if any */
182
+ get error() {
183
+ return this.state.error;
184
+ }
185
+ /** Get pending user transcript (while user is speaking) */
186
+ get pendingUserTranscript() {
187
+ return this.state.pendingUserTranscript;
188
+ }
189
+ /** Get pending assistant transcript (while assistant is speaking) */
190
+ get pendingAssistantTranscript() {
191
+ return this.state.pendingAssistantTranscript;
192
+ }
193
+ /** Get audio visualization data */
194
+ get audio() {
195
+ return this.connection?.getAudioVisualization() ?? null;
196
+ }
197
+ /**
198
+ * Update the session configuration.
199
+ * This applies changes to the active connection and persists them for future reconnections.
200
+ */
201
+ updateSession(config) {
202
+ const o = this.options;
203
+ if (config.instructions !== void 0) o.instructions = config.instructions;
204
+ if (config.voice !== void 0) o.voice = config.voice;
205
+ if (config.vadMode !== void 0) o.vadMode = config.vadMode;
206
+ if (config.outputModalities !== void 0) o.outputModalities = config.outputModalities;
207
+ if (config.temperature !== void 0) o.temperature = config.temperature;
208
+ if (config.maxOutputTokens !== void 0) o.maxOutputTokens = config.maxOutputTokens;
209
+ if (config.semanticEagerness !== void 0) o.semanticEagerness = config.semanticEagerness;
210
+ if (config.providerOptions !== void 0) o.providerOptions = config.providerOptions;
211
+ if (this.connection) this.applySessionConfig();
212
+ }
213
+ /**
214
+ * Subscribe to state changes.
215
+ * @returns Unsubscribe function
216
+ */
217
+ onStateChange(callback) {
218
+ this.stateChangeCallbacks.add(callback);
219
+ return () => {
220
+ this.stateChangeCallbacks.delete(callback);
221
+ };
222
+ }
223
+ /**
224
+ * Clean up resources.
225
+ * Call this when disposing of the client.
226
+ */
227
+ destroy() {
228
+ this.disconnect();
229
+ this.stateChangeCallbacks.clear();
230
+ }
231
+ updateState(updates) {
232
+ this.state = {
233
+ ...this.state,
234
+ ...updates
235
+ };
236
+ for (const callback of this.stateChangeCallbacks) callback(this.state);
237
+ if ("status" in updates && updates.status !== void 0) this.options.onStatusChange?.(updates.status);
238
+ if ("mode" in updates && updates.mode !== void 0) this.options.onModeChange?.(updates.mode);
239
+ }
240
+ addMessage(message) {
241
+ this.updateState({ messages: [...this.state.messages, message] });
242
+ this.options.onMessage?.(message);
243
+ }
244
+ scheduleTokenRefresh() {
245
+ if (!this.token) return;
246
+ const timeUntilExpiry = this.token.expiresAt - Date.now();
247
+ const refreshIn = Math.max(0, timeUntilExpiry - TOKEN_REFRESH_BUFFER_MS);
248
+ this.tokenRefreshTimeout = setTimeout(() => {
249
+ this.refreshToken();
250
+ }, refreshIn);
251
+ }
252
+ async refreshToken() {
253
+ try {
254
+ this.token = await this.options.getToken();
255
+ this.scheduleTokenRefresh();
256
+ this.connection?.updateToken?.(this.token);
257
+ } catch (error) {
258
+ const err = error instanceof Error ? error : new Error(String(error));
259
+ this.updateState({ error: err });
260
+ this.options.onError?.(err);
261
+ }
262
+ }
263
+ subscribeToConnectionEvents() {
264
+ if (!this.connection) return;
265
+ this.unsubscribers.push(this.connection.on("status_change", ({ status }) => {
266
+ this.updateState({ status });
267
+ }));
268
+ this.unsubscribers.push(this.connection.on("mode_change", ({ mode }) => {
269
+ this.updateState({ mode });
270
+ }));
271
+ this.unsubscribers.push(this.connection.on("transcript", ({ role, transcript, isFinal }) => {
272
+ if (role === "user") {
273
+ this.updateState({ pendingUserTranscript: isFinal ? null : transcript });
274
+ if (isFinal && transcript) this.addMessage({
275
+ id: this.generateId(),
276
+ role: "user",
277
+ timestamp: Date.now(),
278
+ parts: [{
279
+ type: "audio",
280
+ transcript,
281
+ durationMs: 0
282
+ }]
283
+ });
284
+ } else this.updateState({ pendingAssistantTranscript: isFinal ? null : transcript });
285
+ }));
286
+ this.unsubscribers.push(this.connection.on("tool_call", ({ toolCallId, toolName, input }) => {
287
+ if (!toolCallId) {
288
+ console.error("[RealtimeClient] tool_call missing toolCallId", toolName);
289
+ return;
290
+ }
291
+ const execute = this.clientTools.get(toolName)?.execute;
292
+ if (execute) (async () => {
293
+ try {
294
+ const output = await execute(input);
295
+ this.connection?.sendToolResult(toolCallId, typeof output === "string" ? output : JSON.stringify(output));
296
+ } catch (error) {
297
+ const errMsg = error instanceof Error ? error.message : String(error);
298
+ this.connection?.sendToolResult(toolCallId, JSON.stringify({ error: errMsg }));
299
+ }
300
+ })();
301
+ else this.connection?.sendToolResult(toolCallId, JSON.stringify({ error: `Unknown or non-executable tool: ${String(toolName)}` }));
302
+ }));
303
+ this.unsubscribers.push(this.connection.on("message_complete", ({ message }) => {
304
+ const existingIndex = this.state.messages.findIndex((m) => m.id === message.id);
305
+ if (existingIndex >= 0) {
306
+ const newMessages = [...this.state.messages];
307
+ newMessages[existingIndex] = message;
308
+ this.updateState({ messages: newMessages });
309
+ } else this.addMessage(message);
310
+ }));
311
+ this.unsubscribers.push(this.connection.on("interrupted", ({ messageId }) => {
312
+ if (messageId) {
313
+ const newMessages = this.state.messages.map((m) => m.id === messageId ? {
314
+ ...m,
315
+ interrupted: true
316
+ } : m);
317
+ this.updateState({ messages: newMessages });
318
+ }
319
+ this.updateState({
320
+ mode: "listening",
321
+ pendingAssistantTranscript: null
322
+ });
323
+ this.options.onInterrupted?.();
324
+ }));
325
+ this.unsubscribers.push(this.connection.on("error", ({ error }) => {
326
+ this.updateState({ error });
327
+ this.options.onError?.(error);
328
+ }));
329
+ this.unsubscribers.push(this.connection.on("go_away", ({ timeLeft }) => {
330
+ this.options.onGoAway?.(timeLeft);
331
+ }));
332
+ this.unsubscribers.push(this.connection.on("usage", (usage) => {
333
+ this.options.onUsage?.(usage);
334
+ }));
335
+ }
336
+ applySessionConfig() {
337
+ if (!this.connection) return;
338
+ const { instructions, voice, vadMode, tools, outputModalities, temperature, maxOutputTokens, semanticEagerness } = this.options;
339
+ if (!(instructions || voice || vadMode || tools && tools.length > 0 || outputModalities || temperature !== void 0 || maxOutputTokens !== void 0 || semanticEagerness)) return;
340
+ const toolsConfig = tools ? Array.from(this.clientTools.values()).map((t) => {
341
+ const inputSchema = t.inputSchema ? convertSchemaToJsonSchema(t.inputSchema) : void 0;
342
+ const outputSchema = t.outputSchema ? convertSchemaToJsonSchema(t.outputSchema) : void 0;
343
+ return {
344
+ name: t.name,
345
+ description: t.description,
346
+ ...inputSchema ? { inputSchema } : {},
347
+ ...outputSchema ? { outputSchema } : {}
348
+ };
349
+ }) : void 0;
350
+ this.connection.updateSession({
351
+ ...instructions !== void 0 ? { instructions } : {},
352
+ ...voice !== void 0 ? { voice } : {},
353
+ ...vadMode !== void 0 ? { vadMode } : {},
354
+ ...toolsConfig !== void 0 ? { tools: toolsConfig } : {},
355
+ ...outputModalities !== void 0 ? { outputModalities } : {},
356
+ ...temperature !== void 0 ? { temperature } : {},
357
+ ...maxOutputTokens !== void 0 ? { maxOutputTokens } : {},
358
+ ...semanticEagerness !== void 0 ? { semanticEagerness } : {}
359
+ });
360
+ }
361
+ generateId() {
362
+ return `msg-${Date.now()}-${Math.random().toString(36).substring(7)}`;
363
+ }
441
364
  };
442
- //# sourceMappingURL=realtime-client.js.map
365
+ //#endregion
366
+ export { RealtimeClient };
367
+
368
+ //# sourceMappingURL=realtime-client.js.map