@tanstack/ai-client 0.5.3 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,98 @@
1
+ import { AudioVisualization, RealtimeMessage, RealtimeMode, RealtimeStatus } from '@tanstack/ai';
2
+ import { RealtimeClientOptions, RealtimeStateChangeCallback } from './realtime-types.js';
3
+ /**
4
+ * Client for managing realtime voice conversations.
5
+ *
6
+ * Handles connection lifecycle, audio I/O, message state,
7
+ * and tool execution for realtime voice-to-voice AI interactions.
8
+ *
9
+ * @example
10
+ * ```typescript
11
+ * import { RealtimeClient } from '@tanstack/ai-client'
12
+ * import { openaiRealtime } from '@tanstack/ai-openai'
13
+ *
14
+ * const client = new RealtimeClient({
15
+ * getToken: () => fetch('/api/realtime-token').then(r => r.json()),
16
+ * adapter: openaiRealtime(),
17
+ * tools: [myTool.client(handler)],
18
+ * onMessage: (msg) => console.log('Message:', msg),
19
+ * })
20
+ *
21
+ * await client.connect()
22
+ * ```
23
+ */
24
+ export declare class RealtimeClient {
25
+ private options;
26
+ private connection;
27
+ private token;
28
+ private tokenRefreshTimeout;
29
+ private clientTools;
30
+ private stateChangeCallbacks;
31
+ private unsubscribers;
32
+ private state;
33
+ constructor(options: RealtimeClientOptions);
34
+ /**
35
+ * Connect to the realtime session.
36
+ * Fetches a token and establishes the connection.
37
+ */
38
+ connect(): Promise<void>;
39
+ /**
40
+ * Disconnect from the realtime session.
41
+ */
42
+ disconnect(): Promise<void>;
43
+ /**
44
+ * Start listening for voice input.
45
+ * Only needed when vadMode is 'manual'.
46
+ */
47
+ startListening(): void;
48
+ /**
49
+ * Stop listening for voice input.
50
+ * Only needed when vadMode is 'manual'.
51
+ */
52
+ stopListening(): void;
53
+ /**
54
+ * Interrupt the current assistant response.
55
+ */
56
+ interrupt(): void;
57
+ /**
58
+ * Send a text message instead of voice.
59
+ */
60
+ sendText(text: string): void;
61
+ /**
62
+ * Send an image to the conversation.
63
+ * @param imageData - Base64-encoded image data or a URL
64
+ * @param mimeType - MIME type of the image (e.g., 'image/png', 'image/jpeg')
65
+ */
66
+ sendImage(imageData: string, mimeType: string): void;
67
+ /** Get current connection status */
68
+ get status(): RealtimeStatus;
69
+ /** Get current mode */
70
+ get mode(): RealtimeMode;
71
+ /** Get conversation messages */
72
+ get messages(): Array<RealtimeMessage>;
73
+ /** Get current error, if any */
74
+ get error(): Error | null;
75
+ /** Get pending user transcript (while user is speaking) */
76
+ get pendingUserTranscript(): string | null;
77
+ /** Get pending assistant transcript (while assistant is speaking) */
78
+ get pendingAssistantTranscript(): string | null;
79
+ /** Get audio visualization data */
80
+ get audio(): AudioVisualization | null;
81
+ /**
82
+ * Subscribe to state changes.
83
+ * @returns Unsubscribe function
84
+ */
85
+ onStateChange(callback: RealtimeStateChangeCallback): () => void;
86
+ /**
87
+ * Clean up resources.
88
+ * Call this when disposing of the client.
89
+ */
90
+ destroy(): void;
91
+ private updateState;
92
+ private addMessage;
93
+ private scheduleTokenRefresh;
94
+ private refreshToken;
95
+ private subscribeToConnectionEvents;
96
+ private applySessionConfig;
97
+ private generateId;
98
+ }
@@ -0,0 +1,384 @@
1
+ import { convertSchemaToJsonSchema } from "@tanstack/ai";
2
+ const TOKEN_REFRESH_BUFFER_MS = 6e4;
3
+ class RealtimeClient {
4
+ constructor(options) {
5
+ this.connection = null;
6
+ this.token = null;
7
+ this.tokenRefreshTimeout = null;
8
+ this.stateChangeCallbacks = /* @__PURE__ */ new Set();
9
+ this.unsubscribers = [];
10
+ this.state = {
11
+ status: "idle",
12
+ mode: "idle",
13
+ messages: [],
14
+ pendingUserTranscript: null,
15
+ pendingAssistantTranscript: null,
16
+ error: null
17
+ };
18
+ this.options = {
19
+ autoPlayback: true,
20
+ autoCapture: true,
21
+ vadMode: "server",
22
+ ...options
23
+ };
24
+ this.clientTools = /* @__PURE__ */ new Map();
25
+ if (options.tools) {
26
+ for (const tool of options.tools) {
27
+ this.clientTools.set(tool.name, tool);
28
+ }
29
+ }
30
+ }
31
+ // ============================================================================
32
+ // Connection Lifecycle
33
+ // ============================================================================
34
+ /**
35
+ * Connect to the realtime session.
36
+ * Fetches a token and establishes the connection.
37
+ */
38
+ async connect() {
39
+ if (this.state.status === "connected") {
40
+ return;
41
+ }
42
+ this.updateState({ status: "connecting", error: null });
43
+ try {
44
+ this.token = await this.options.getToken();
45
+ this.scheduleTokenRefresh();
46
+ const toolsList = this.clientTools.size > 0 ? Array.from(this.clientTools.values()) : void 0;
47
+ this.connection = await this.options.adapter.connect(
48
+ this.token,
49
+ toolsList
50
+ );
51
+ this.subscribeToConnectionEvents();
52
+ this.applySessionConfig();
53
+ if (this.options.autoCapture) {
54
+ await this.connection.startAudioCapture();
55
+ }
56
+ this.updateState({ status: "connected", mode: "listening" });
57
+ this.options.onConnect?.();
58
+ } catch (error) {
59
+ const err = error instanceof Error ? error : new Error(String(error));
60
+ this.updateState({ status: "error", error: err });
61
+ this.options.onError?.(err);
62
+ throw err;
63
+ }
64
+ }
65
+ /**
66
+ * Disconnect from the realtime session.
67
+ */
68
+ async disconnect() {
69
+ if (this.tokenRefreshTimeout) {
70
+ clearTimeout(this.tokenRefreshTimeout);
71
+ this.tokenRefreshTimeout = null;
72
+ }
73
+ for (const unsub of this.unsubscribers) {
74
+ unsub();
75
+ }
76
+ this.unsubscribers = [];
77
+ if (this.connection) {
78
+ await this.connection.disconnect();
79
+ this.connection = null;
80
+ }
81
+ this.token = null;
82
+ this.updateState({
83
+ status: "idle",
84
+ mode: "idle",
85
+ pendingUserTranscript: null,
86
+ pendingAssistantTranscript: null
87
+ });
88
+ this.options.onDisconnect?.();
89
+ }
90
+ // ============================================================================
91
+ // Voice Control
92
+ // ============================================================================
93
+ /**
94
+ * Start listening for voice input.
95
+ * Only needed when vadMode is 'manual'.
96
+ */
97
+ startListening() {
98
+ if (!this.connection || this.state.status !== "connected") {
99
+ return;
100
+ }
101
+ this.connection.startAudioCapture();
102
+ this.updateState({ mode: "listening" });
103
+ }
104
+ /**
105
+ * Stop listening for voice input.
106
+ * Only needed when vadMode is 'manual'.
107
+ */
108
+ stopListening() {
109
+ if (!this.connection) {
110
+ return;
111
+ }
112
+ this.connection.stopAudioCapture();
113
+ this.updateState({ mode: "idle" });
114
+ }
115
+ /**
116
+ * Interrupt the current assistant response.
117
+ */
118
+ interrupt() {
119
+ if (!this.connection) {
120
+ return;
121
+ }
122
+ this.connection.interrupt();
123
+ }
124
+ // ============================================================================
125
+ // Text Input
126
+ // ============================================================================
127
+ /**
128
+ * Send a text message instead of voice.
129
+ */
130
+ sendText(text) {
131
+ if (!this.connection || this.state.status !== "connected") {
132
+ return;
133
+ }
134
+ const userMessage = {
135
+ id: this.generateId(),
136
+ role: "user",
137
+ timestamp: Date.now(),
138
+ parts: [{ type: "text", content: text }]
139
+ };
140
+ this.addMessage(userMessage);
141
+ this.connection.sendText(text);
142
+ }
143
+ /**
144
+ * Send an image to the conversation.
145
+ * @param imageData - Base64-encoded image data or a URL
146
+ * @param mimeType - MIME type of the image (e.g., 'image/png', 'image/jpeg')
147
+ */
148
+ sendImage(imageData, mimeType) {
149
+ if (!this.connection || this.state.status !== "connected") {
150
+ return;
151
+ }
152
+ const userMessage = {
153
+ id: this.generateId(),
154
+ role: "user",
155
+ timestamp: Date.now(),
156
+ parts: [{ type: "image", data: imageData, mimeType }]
157
+ };
158
+ this.addMessage(userMessage);
159
+ this.connection.sendImage(imageData, mimeType);
160
+ }
161
+ // ============================================================================
162
+ // State Access
163
+ // ============================================================================
164
+ /** Get current connection status */
165
+ get status() {
166
+ return this.state.status;
167
+ }
168
+ /** Get current mode */
169
+ get mode() {
170
+ return this.state.mode;
171
+ }
172
+ /** Get conversation messages */
173
+ get messages() {
174
+ return this.state.messages;
175
+ }
176
+ /** Get current error, if any */
177
+ get error() {
178
+ return this.state.error;
179
+ }
180
+ /** Get pending user transcript (while user is speaking) */
181
+ get pendingUserTranscript() {
182
+ return this.state.pendingUserTranscript;
183
+ }
184
+ /** Get pending assistant transcript (while assistant is speaking) */
185
+ get pendingAssistantTranscript() {
186
+ return this.state.pendingAssistantTranscript;
187
+ }
188
+ /** Get audio visualization data */
189
+ get audio() {
190
+ return this.connection?.getAudioVisualization() ?? null;
191
+ }
192
+ // ============================================================================
193
+ // State Subscription
194
+ // ============================================================================
195
+ /**
196
+ * Subscribe to state changes.
197
+ * @returns Unsubscribe function
198
+ */
199
+ onStateChange(callback) {
200
+ this.stateChangeCallbacks.add(callback);
201
+ return () => {
202
+ this.stateChangeCallbacks.delete(callback);
203
+ };
204
+ }
205
+ // ============================================================================
206
+ // Cleanup
207
+ // ============================================================================
208
+ /**
209
+ * Clean up resources.
210
+ * Call this when disposing of the client.
211
+ */
212
+ destroy() {
213
+ this.disconnect();
214
+ this.stateChangeCallbacks.clear();
215
+ }
216
+ // ============================================================================
217
+ // Private Methods
218
+ // ============================================================================
219
+ updateState(updates) {
220
+ this.state = { ...this.state, ...updates };
221
+ for (const callback of this.stateChangeCallbacks) {
222
+ callback(this.state);
223
+ }
224
+ if ("status" in updates && updates.status !== void 0) {
225
+ this.options.onStatusChange?.(updates.status);
226
+ }
227
+ if ("mode" in updates && updates.mode !== void 0) {
228
+ this.options.onModeChange?.(updates.mode);
229
+ }
230
+ }
231
+ addMessage(message) {
232
+ this.updateState({
233
+ messages: [...this.state.messages, message]
234
+ });
235
+ this.options.onMessage?.(message);
236
+ }
237
+ scheduleTokenRefresh() {
238
+ if (!this.token) return;
239
+ const timeUntilExpiry = this.token.expiresAt - Date.now();
240
+ const refreshIn = Math.max(0, timeUntilExpiry - TOKEN_REFRESH_BUFFER_MS);
241
+ this.tokenRefreshTimeout = setTimeout(() => {
242
+ this.refreshToken();
243
+ }, refreshIn);
244
+ }
245
+ async refreshToken() {
246
+ try {
247
+ this.token = await this.options.getToken();
248
+ this.scheduleTokenRefresh();
249
+ } catch (error) {
250
+ const err = error instanceof Error ? error : new Error(String(error));
251
+ this.updateState({ error: err });
252
+ this.options.onError?.(err);
253
+ }
254
+ }
255
+ subscribeToConnectionEvents() {
256
+ if (!this.connection) return;
257
+ this.unsubscribers.push(
258
+ this.connection.on("status_change", ({ status }) => {
259
+ this.updateState({ status });
260
+ })
261
+ );
262
+ this.unsubscribers.push(
263
+ this.connection.on("mode_change", ({ mode }) => {
264
+ this.updateState({ mode });
265
+ })
266
+ );
267
+ this.unsubscribers.push(
268
+ this.connection.on("transcript", ({ role, transcript, isFinal }) => {
269
+ if (role === "user") {
270
+ this.updateState({
271
+ pendingUserTranscript: isFinal ? null : transcript
272
+ });
273
+ if (isFinal && transcript) {
274
+ this.addMessage({
275
+ id: this.generateId(),
276
+ role: "user",
277
+ timestamp: Date.now(),
278
+ parts: [{ type: "audio", transcript, durationMs: 0 }]
279
+ });
280
+ }
281
+ } else {
282
+ this.updateState({
283
+ pendingAssistantTranscript: isFinal ? null : transcript
284
+ });
285
+ }
286
+ })
287
+ );
288
+ this.unsubscribers.push(
289
+ this.connection.on(
290
+ "tool_call",
291
+ async ({ toolCallId, toolName, input }) => {
292
+ const tool = this.clientTools.get(toolName);
293
+ if (tool?.execute) {
294
+ try {
295
+ const output = await tool.execute(input);
296
+ this.connection?.sendToolResult(
297
+ toolCallId,
298
+ typeof output === "string" ? output : JSON.stringify(output)
299
+ );
300
+ } catch (error) {
301
+ const errMsg = error instanceof Error ? error.message : String(error);
302
+ this.connection?.sendToolResult(
303
+ toolCallId,
304
+ JSON.stringify({ error: errMsg })
305
+ );
306
+ }
307
+ }
308
+ }
309
+ )
310
+ );
311
+ this.unsubscribers.push(
312
+ this.connection.on("message_complete", ({ message }) => {
313
+ const existingIndex = this.state.messages.findIndex(
314
+ (m) => m.id === message.id
315
+ );
316
+ if (existingIndex >= 0) {
317
+ const newMessages = [...this.state.messages];
318
+ newMessages[existingIndex] = message;
319
+ this.updateState({ messages: newMessages });
320
+ } else {
321
+ this.addMessage(message);
322
+ }
323
+ })
324
+ );
325
+ this.unsubscribers.push(
326
+ this.connection.on("interrupted", ({ messageId }) => {
327
+ if (messageId) {
328
+ const newMessages = this.state.messages.map(
329
+ (m) => m.id === messageId ? { ...m, interrupted: true } : m
330
+ );
331
+ this.updateState({ messages: newMessages });
332
+ }
333
+ this.updateState({
334
+ mode: "listening",
335
+ pendingAssistantTranscript: null
336
+ });
337
+ this.options.onInterrupted?.();
338
+ })
339
+ );
340
+ this.unsubscribers.push(
341
+ this.connection.on("error", ({ error }) => {
342
+ this.updateState({ error });
343
+ this.options.onError?.(error);
344
+ })
345
+ );
346
+ }
347
+ applySessionConfig() {
348
+ if (!this.connection) return;
349
+ const {
350
+ instructions,
351
+ voice,
352
+ vadMode,
353
+ tools,
354
+ outputModalities,
355
+ temperature,
356
+ maxOutputTokens,
357
+ semanticEagerness
358
+ } = this.options;
359
+ const hasConfig = instructions || voice || vadMode || tools && tools.length > 0 || outputModalities || temperature !== void 0 || maxOutputTokens !== void 0 || semanticEagerness;
360
+ if (!hasConfig) return;
361
+ const toolsConfig = tools ? Array.from(this.clientTools.values()).map((t) => ({
362
+ name: t.name,
363
+ description: t.description,
364
+ inputSchema: t.inputSchema ? convertSchemaToJsonSchema(t.inputSchema) : void 0
365
+ })) : void 0;
366
+ this.connection.updateSession({
367
+ instructions,
368
+ voice,
369
+ vadMode,
370
+ tools: toolsConfig,
371
+ outputModalities,
372
+ temperature,
373
+ maxOutputTokens,
374
+ semanticEagerness
375
+ });
376
+ }
377
+ generateId() {
378
+ return `msg-${Date.now()}-${Math.random().toString(36).substring(7)}`;
379
+ }
380
+ }
381
+ export {
382
+ RealtimeClient
383
+ };
384
+ //# sourceMappingURL=realtime-client.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"realtime-client.js","sources":["../../src/realtime-client.ts"],"sourcesContent":["import { convertSchemaToJsonSchema } from '@tanstack/ai'\nimport type {\n AnyClientTool,\n AudioVisualization,\n RealtimeMessage,\n RealtimeMode,\n RealtimeStatus,\n RealtimeToken,\n} from '@tanstack/ai'\nimport type {\n RealtimeClientOptions,\n RealtimeClientState,\n RealtimeConnection,\n RealtimeStateChangeCallback,\n} from './realtime-types'\n\n// Token refresh buffer - refresh 1 minute before expiry\nconst TOKEN_REFRESH_BUFFER_MS = 60_000\n\n/**\n * Client for managing realtime voice conversations.\n *\n * Handles connection lifecycle, audio I/O, message state,\n * and tool execution for realtime voice-to-voice AI interactions.\n *\n * @example\n * ```typescript\n * import { RealtimeClient } from '@tanstack/ai-client'\n * import { openaiRealtime } from '@tanstack/ai-openai'\n *\n * const client = new RealtimeClient({\n * getToken: () => fetch('/api/realtime-token').then(r => r.json()),\n * adapter: openaiRealtime(),\n * tools: [myTool.client(handler)],\n * onMessage: (msg) => console.log('Message:', msg),\n * })\n *\n * await client.connect()\n * ```\n */\nexport class RealtimeClient {\n private options: RealtimeClientOptions\n private connection: RealtimeConnection | null = null\n private token: RealtimeToken | null = null\n private tokenRefreshTimeout: ReturnType<typeof setTimeout> | null = null\n private clientTools: Map<string, AnyClientTool>\n private stateChangeCallbacks: Set<RealtimeStateChangeCallback> = new Set()\n private unsubscribers: Array<() => void> = []\n\n private state: RealtimeClientState = {\n status: 'idle',\n mode: 'idle',\n messages: [],\n pendingUserTranscript: null,\n pendingAssistantTranscript: null,\n error: null,\n }\n\n constructor(options: RealtimeClientOptions) {\n this.options = {\n autoPlayback: true,\n autoCapture: true,\n vadMode: 'server',\n ...options,\n }\n\n // Build client tools map\n this.clientTools = new Map()\n if (options.tools) {\n for (const tool of options.tools) {\n this.clientTools.set(tool.name, tool)\n }\n }\n }\n\n // ============================================================================\n // Connection Lifecycle\n // ============================================================================\n\n /**\n * Connect to the realtime session.\n * Fetches a token and establishes the connection.\n */\n async connect(): Promise<void> {\n if (this.state.status === 'connected') {\n return\n }\n\n this.updateState({ status: 'connecting', error: null })\n\n try {\n // Fetch token from server\n this.token = await this.options.getToken()\n\n // Schedule token refresh\n this.scheduleTokenRefresh()\n\n // Connect via adapter (pass tools for providers like ElevenLabs that need them at connect time)\n const toolsList =\n this.clientTools.size > 0\n ? Array.from(this.clientTools.values())\n : undefined\n this.connection = await this.options.adapter.connect(\n this.token,\n toolsList,\n )\n\n // Subscribe to connection events\n this.subscribeToConnectionEvents()\n\n // Auto-configure session with client-provided settings\n this.applySessionConfig()\n\n // Start audio capture if configured\n if (this.options.autoCapture) {\n await this.connection.startAudioCapture()\n }\n\n this.updateState({ status: 'connected', mode: 'listening' })\n this.options.onConnect?.()\n } catch (error) {\n const err = error instanceof Error ? error : new Error(String(error))\n this.updateState({ status: 'error', error: err })\n this.options.onError?.(err)\n throw err\n }\n }\n\n /**\n * Disconnect from the realtime session.\n */\n async disconnect(): Promise<void> {\n if (this.tokenRefreshTimeout) {\n clearTimeout(this.tokenRefreshTimeout)\n this.tokenRefreshTimeout = null\n }\n\n // Unsubscribe from all events\n for (const unsub of this.unsubscribers) {\n unsub()\n }\n this.unsubscribers = []\n\n if (this.connection) {\n await this.connection.disconnect()\n this.connection = null\n }\n\n this.token = null\n this.updateState({\n status: 'idle',\n mode: 'idle',\n pendingUserTranscript: null,\n pendingAssistantTranscript: null,\n })\n this.options.onDisconnect?.()\n }\n\n // ============================================================================\n // Voice Control\n // ============================================================================\n\n /**\n * Start listening for voice input.\n * Only needed when vadMode is 'manual'.\n */\n startListening(): void {\n if (!this.connection || this.state.status !== 'connected') {\n return\n }\n this.connection.startAudioCapture()\n this.updateState({ mode: 'listening' })\n }\n\n /**\n * Stop listening for voice input.\n * Only needed when vadMode is 'manual'.\n */\n stopListening(): void {\n if (!this.connection) {\n return\n }\n this.connection.stopAudioCapture()\n this.updateState({ mode: 'idle' })\n }\n\n /**\n * Interrupt the current assistant response.\n */\n interrupt(): void {\n if (!this.connection) {\n return\n }\n this.connection.interrupt()\n }\n\n // ============================================================================\n // Text Input\n // ============================================================================\n\n /**\n * Send a text message instead of voice.\n */\n sendText(text: string): void {\n if (!this.connection || this.state.status !== 'connected') {\n return\n }\n\n // Add user message\n const userMessage: RealtimeMessage = {\n id: this.generateId(),\n role: 'user',\n timestamp: Date.now(),\n parts: [{ type: 'text', content: text }],\n }\n this.addMessage(userMessage)\n\n // Send to provider\n this.connection.sendText(text)\n }\n\n /**\n * Send an image to the conversation.\n * @param imageData - Base64-encoded image data or a URL\n * @param mimeType - MIME type of the image (e.g., 'image/png', 'image/jpeg')\n */\n sendImage(imageData: string, mimeType: string): void {\n if (!this.connection || this.state.status !== 'connected') {\n return\n }\n\n // Add user message with image part\n const userMessage: RealtimeMessage = {\n id: this.generateId(),\n role: 'user',\n timestamp: Date.now(),\n parts: [{ type: 'image', data: imageData, mimeType }],\n }\n this.addMessage(userMessage)\n\n // Send to provider\n this.connection.sendImage(imageData, mimeType)\n }\n\n // ============================================================================\n // State Access\n // ============================================================================\n\n /** Get current connection status */\n get status(): RealtimeStatus {\n return this.state.status\n }\n\n /** Get current mode */\n get mode(): RealtimeMode {\n return this.state.mode\n }\n\n /** Get conversation messages */\n get messages(): Array<RealtimeMessage> {\n return this.state.messages\n }\n\n /** Get current error, if any */\n get error(): Error | null {\n return this.state.error\n }\n\n /** Get pending user transcript (while user is speaking) */\n get pendingUserTranscript(): string | null {\n return this.state.pendingUserTranscript\n }\n\n /** Get pending assistant transcript (while assistant is speaking) */\n get pendingAssistantTranscript(): string | null {\n return this.state.pendingAssistantTranscript\n }\n\n /** Get audio visualization data */\n get audio(): AudioVisualization | null {\n return this.connection?.getAudioVisualization() ?? null\n }\n\n // ============================================================================\n // State Subscription\n // ============================================================================\n\n /**\n * Subscribe to state changes.\n * @returns Unsubscribe function\n */\n onStateChange(callback: RealtimeStateChangeCallback): () => void {\n this.stateChangeCallbacks.add(callback)\n return () => {\n this.stateChangeCallbacks.delete(callback)\n }\n }\n\n // ============================================================================\n // Cleanup\n // ============================================================================\n\n /**\n * Clean up resources.\n * Call this when disposing of the client.\n */\n destroy(): void {\n this.disconnect()\n this.stateChangeCallbacks.clear()\n }\n\n // ============================================================================\n // Private Methods\n // ============================================================================\n\n private updateState(updates: Partial<RealtimeClientState>): void {\n this.state = { ...this.state, ...updates }\n\n // Notify callbacks\n for (const callback of this.stateChangeCallbacks) {\n callback(this.state)\n }\n\n // Notify specific callbacks\n if ('status' in updates && updates.status !== undefined) {\n this.options.onStatusChange?.(updates.status)\n }\n if ('mode' in updates && updates.mode !== undefined) {\n this.options.onModeChange?.(updates.mode)\n }\n }\n\n private addMessage(message: RealtimeMessage): void {\n this.updateState({\n messages: [...this.state.messages, message],\n })\n this.options.onMessage?.(message)\n }\n\n private scheduleTokenRefresh(): void {\n if (!this.token) return\n\n const timeUntilExpiry = this.token.expiresAt - Date.now()\n const refreshIn = Math.max(0, timeUntilExpiry - TOKEN_REFRESH_BUFFER_MS)\n\n this.tokenRefreshTimeout = setTimeout(() => {\n this.refreshToken()\n }, refreshIn)\n }\n\n private async refreshToken(): Promise<void> {\n try {\n this.token = await this.options.getToken()\n this.scheduleTokenRefresh()\n // Note: Some providers may require reconnection with new token\n // This is handled by the adapter implementation\n } catch (error) {\n const err = error instanceof Error ? error : new Error(String(error))\n this.updateState({ error: err })\n this.options.onError?.(err)\n }\n }\n\n private subscribeToConnectionEvents(): void {\n if (!this.connection) return\n\n // Status changes\n this.unsubscribers.push(\n this.connection.on('status_change', ({ status }) => {\n this.updateState({ status })\n }),\n )\n\n // Mode changes\n this.unsubscribers.push(\n this.connection.on('mode_change', ({ mode }) => {\n this.updateState({ mode })\n }),\n )\n\n // Transcripts (streaming)\n // User transcripts are added as messages when final (no separate message_complete for user input)\n // Assistant transcripts are streamed, final message comes via message_complete\n this.unsubscribers.push(\n this.connection.on('transcript', ({ role, transcript, isFinal }) => {\n if (role === 'user') {\n this.updateState({\n pendingUserTranscript: isFinal ? null : transcript,\n })\n // Add user message when transcript is finalized\n if (isFinal && transcript) {\n this.addMessage({\n id: this.generateId(),\n role: 'user',\n timestamp: Date.now(),\n parts: [{ type: 'audio', transcript, durationMs: 0 }],\n })\n }\n } else {\n // Assistant transcripts - just update pending, message_complete handles final\n this.updateState({\n pendingAssistantTranscript: isFinal ? null : transcript,\n })\n }\n }),\n )\n\n // Tool calls\n this.unsubscribers.push(\n this.connection.on(\n 'tool_call',\n async ({ toolCallId, toolName, input }) => {\n const tool = this.clientTools.get(toolName)\n if (tool?.execute) {\n try {\n const output = await tool.execute(input)\n this.connection?.sendToolResult(\n toolCallId,\n typeof output === 'string' ? output : JSON.stringify(output),\n )\n } catch (error) {\n const errMsg =\n error instanceof Error ? error.message : String(error)\n this.connection?.sendToolResult(\n toolCallId,\n JSON.stringify({ error: errMsg }),\n )\n }\n }\n },\n ),\n )\n\n // Message complete\n this.unsubscribers.push(\n this.connection.on('message_complete', ({ message }) => {\n // Replace pending message with final version if needed\n const existingIndex = this.state.messages.findIndex(\n (m) => m.id === message.id,\n )\n if (existingIndex >= 0) {\n const newMessages = [...this.state.messages]\n newMessages[existingIndex] = message\n this.updateState({ messages: newMessages })\n } else {\n this.addMessage(message)\n }\n }),\n )\n\n // Interruption\n this.unsubscribers.push(\n this.connection.on('interrupted', ({ messageId }) => {\n if (messageId) {\n const newMessages = this.state.messages.map((m) =>\n m.id === messageId ? { ...m, interrupted: true } : m,\n )\n this.updateState({ messages: newMessages })\n }\n this.updateState({\n mode: 'listening',\n pendingAssistantTranscript: null,\n })\n this.options.onInterrupted?.()\n }),\n )\n\n // Errors\n this.unsubscribers.push(\n this.connection.on('error', ({ error }) => {\n this.updateState({ error })\n this.options.onError?.(error)\n }),\n )\n }\n\n private applySessionConfig(): void {\n if (!this.connection) return\n\n const {\n instructions,\n voice,\n vadMode,\n tools,\n outputModalities,\n temperature,\n maxOutputTokens,\n semanticEagerness,\n } = this.options\n const hasConfig =\n instructions ||\n voice ||\n vadMode ||\n (tools && tools.length > 0) ||\n outputModalities ||\n temperature !== undefined ||\n maxOutputTokens !== undefined ||\n semanticEagerness\n if (!hasConfig) return\n\n const toolsConfig = tools\n ? Array.from(this.clientTools.values()).map((t) => ({\n name: t.name,\n description: t.description,\n inputSchema: t.inputSchema\n ? convertSchemaToJsonSchema(t.inputSchema)\n : undefined,\n }))\n : undefined\n\n this.connection.updateSession({\n instructions,\n voice,\n vadMode,\n tools: toolsConfig,\n outputModalities,\n temperature,\n maxOutputTokens,\n semanticEagerness,\n })\n }\n\n private generateId(): string {\n return `msg-${Date.now()}-${Math.random().toString(36).substring(7)}`\n }\n}\n"],"names":[],"mappings":";AAiBA,MAAM,0BAA0B;AAuBzB,MAAM,eAAe;AAAA,EAkB1B,YAAY,SAAgC;AAhB5C,SAAQ,aAAwC;AAChD,SAAQ,QAA8B;AACtC,SAAQ,sBAA4D;AAEpE,SAAQ,2CAA6D,IAAA;AACrE,SAAQ,gBAAmC,CAAA;AAE3C,SAAQ,QAA6B;AAAA,MACnC,QAAQ;AAAA,MACR,MAAM;AAAA,MACN,UAAU,CAAA;AAAA,MACV,uBAAuB;AAAA,MACvB,4BAA4B;AAAA,MAC5B,OAAO;AAAA,IAAA;AAIP,SAAK,UAAU;AAAA,MACb,cAAc;AAAA,MACd,aAAa;AAAA,MACb,SAAS;AAAA,MACT,GAAG;AAAA,IAAA;AAIL,SAAK,kCAAkB,IAAA;AACvB,QAAI,QAAQ,OAAO;AACjB,iBAAW,QAAQ,QAAQ,OAAO;AAChC,aAAK,YAAY,IAAI,KAAK,MAAM,IAAI;AAAA,MACtC;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,MAAM,UAAyB;AAC7B,QAAI,KAAK,MAAM,WAAW,aAAa;AACrC;AAAA,IACF;AAEA,SAAK,YAAY,EAAE,QAAQ,cAAc,OAAO,MAAM;AAEtD,QAAI;AAEF,WAAK,QAAQ,MAAM,KAAK,QAAQ,SAAA;AAGhC,WAAK,qBAAA;AAGL,YAAM,YACJ,KAAK,YAAY,OAAO,IACpB,MAAM,KAAK,KAAK,YAAY,OAAA,CAAQ,IACpC;AACN,WAAK,aAAa,MAAM,KAAK,QAAQ,QAAQ;AAAA,QAC3C,KAAK;AAAA,QACL;AAAA,MAAA;AAIF,WAAK,4BAAA;AAGL,WAAK,mBAAA;AAGL,UAAI,KAAK,QAAQ,aAAa;AAC5B,cAAM,KAAK,WAAW,kBAAA;AAAA,MACxB;AAEA,WAAK,YAAY,EAAE,QAAQ,aAAa,MAAM,aAAa;AAC3D,WAAK,QAAQ,YAAA;AAAA,IACf,SAAS,OAAO;AACd,YAAM,MAAM,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;AACpE,WAAK,YAAY,EAAE,QAAQ,SAAS,OAAO,KAAK;AAChD,WAAK,QAAQ,UAAU,GAAG;AAC1B,YAAM;AAAA,IACR;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,aAA4B;AAChC,QAAI,KAAK,qBAAqB;AAC5B,mBAAa,KAAK,mBAAmB;AACrC,WAAK,sBAAsB;AAAA,IAC7B;AAGA,eAAW,SAAS,KAAK,eAAe;AACtC,YAAA;AAAA,IACF;AACA,SAAK,gBAAgB,CAAA;AAErB,QAAI,KAAK,YAAY;AACnB,YAAM,KAAK,WAAW,WAAA;AACtB,WAAK,aAAa;AAAA,IACpB;AAEA,SAAK,QAAQ;AACb,SAAK,YAAY;AAAA,MACf,QAAQ;AAAA,MACR,MAAM;AAAA,MACN,uBAAuB;AAAA,MACvB,4BAA4B;AAAA,IAAA,CAC7B;AACD,SAAK,QAAQ,eAAA;AAAA,EACf;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,iBAAuB;AACrB,QAAI,CAAC,KAAK,cAAc,KAAK,MAAM,WAAW,aAAa;AACzD;AAAA,IACF;AACA,SAAK,WAAW,kBAAA;AAChB,SAAK,YAAY,EAAE,MAAM,YAAA,CAAa;AAAA,EACxC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,gBAAsB;AACpB,QAAI,CAAC,KAAK,YAAY;AACpB;AAAA,IACF;AACA,SAAK,WAAW,iBAAA;AAChB,SAAK,YAAY,EAAE,MAAM,OAAA,CAAQ;AAAA,EACnC;AAAA;AAAA;AAAA;AAAA,EAKA,YAAkB;AAChB,QAAI,CAAC,KAAK,YAAY;AACpB;AAAA,IACF;AACA,SAAK,WAAW,UAAA;AAAA,EAClB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,SAAS,MAAoB;AAC3B,QAAI,CAAC,KAAK,cAAc,KAAK,MAAM,WAAW,aAAa;AACzD;AAAA,IACF;AAGA,UAAM,cAA+B;AAAA,MACnC,IAAI,KAAK,WAAA;AAAA,MACT,MAAM;AAAA,MACN,WAAW,KAAK,IAAA;AAAA,MAChB,OAAO,CAAC,EAAE,MAAM,QAAQ,SAAS,MAAM;AAAA,IAAA;AAEzC,SAAK,WAAW,WAAW;AAG3B,SAAK,WAAW,SAAS,IAAI;AAAA,EAC/B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,UAAU,WAAmB,UAAwB;AACnD,QAAI,CAAC,KAAK,cAAc,KAAK,MAAM,WAAW,aAAa;AACzD;AAAA,IACF;AAGA,UAAM,cAA+B;AAAA,MACnC,IAAI,KAAK,WAAA;AAAA,MACT,MAAM;AAAA,MACN,WAAW,KAAK,IAAA;AAAA,MAChB,OAAO,CAAC,EAAE,MAAM,SAAS,MAAM,WAAW,UAAU;AAAA,IAAA;AAEtD,SAAK,WAAW,WAAW;AAG3B,SAAK,WAAW,UAAU,WAAW,QAAQ;AAAA,EAC/C;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,IAAI,SAAyB;AAC3B,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,IAAI,OAAqB;AACvB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,IAAI,WAAmC;AACrC,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,IAAI,QAAsB;AACxB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,IAAI,wBAAuC;AACzC,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,IAAI,6BAA4C;AAC9C,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,IAAI,QAAmC;AACrC,WAAO,KAAK,YAAY,sBAAA,KAA2B;AAAA,EACrD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,cAAc,UAAmD;AAC/D,SAAK,qBAAqB,IAAI,QAAQ;AACtC,WAAO,MAAM;AACX,WAAK,qBAAqB,OAAO,QAAQ;AAAA,IAC3C;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,UAAgB;AACd,SAAK,WAAA;AACL,SAAK,qBAAqB,MAAA;AAAA,EAC5B;AAAA;AAAA;AAAA;AAAA,EAMQ,YAAY,SAA6C;AAC/D,SAAK,QAAQ,EAAE,GAAG,KAAK,OAAO,GAAG,QAAA;AAGjC,eAAW,YAAY,KAAK,sBAAsB;AAChD,eAAS,KAAK,KAAK;AAAA,IACrB;AAGA,QAAI,YAAY,WAAW,QAAQ,WAAW,QAAW;AACvD,WAAK,QAAQ,iBAAiB,QAAQ,MAAM;AAAA,IAC9C;AACA,QAAI,UAAU,WAAW,QAAQ,SAAS,QAAW;AACnD,WAAK,QAAQ,eAAe,QAAQ,IAAI;AAAA,IAC1C;AAAA,EACF;AAAA,EAEQ,WAAW,SAAgC;AACjD,SAAK,YAAY;AAAA,MACf,UAAU,CAAC,GAAG,KAAK,MAAM,UAAU,OAAO;AAAA,IAAA,CAC3C;AACD,SAAK,QAAQ,YAAY,OAAO;AAAA,EAClC;AAAA,EAEQ,uBAA6B;AACnC,QAAI,CAAC,KAAK,MAAO;AAEjB,UAAM,kBAAkB,KAAK,MAAM,YAAY,KAAK,IAAA;AACpD,UAAM,YAAY,KAAK,IAAI,GAAG,kBAAkB,uBAAuB;AAEvE,SAAK,sBAAsB,WAAW,MAAM;AAC1C,WAAK,aAAA;AAAA,IACP,GAAG,SAAS;AAAA,EACd;AAAA,EAEA,MAAc,eAA8B;AAC1C,QAAI;AACF,WAAK,QAAQ,MAAM,KAAK,QAAQ,SAAA;AAChC,WAAK,qBAAA;AAAA,IAGP,SAAS,OAAO;AACd,YAAM,MAAM,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;AACpE,WAAK,YAAY,EAAE,OAAO,IAAA,CAAK;AAC/B,WAAK,QAAQ,UAAU,GAAG;AAAA,IAC5B;AAAA,EACF;AAAA,EAEQ,8BAAoC;AAC1C,QAAI,CAAC,KAAK,WAAY;AAGtB,SAAK,cAAc;AAAA,MACjB,KAAK,WAAW,GAAG,iBAAiB,CAAC,EAAE,aAAa;AAClD,aAAK,YAAY,EAAE,QAAQ;AAAA,MAC7B,CAAC;AAAA,IAAA;AAIH,SAAK,cAAc;AAAA,MACjB,KAAK,WAAW,GAAG,eAAe,CAAC,EAAE,WAAW;AAC9C,aAAK,YAAY,EAAE,MAAM;AAAA,MAC3B,CAAC;AAAA,IAAA;AAMH,SAAK,cAAc;AAAA,MACjB,KAAK,WAAW,GAAG,cAAc,CAAC,EAAE,MAAM,YAAY,cAAc;AAClE,YAAI,SAAS,QAAQ;AACnB,eAAK,YAAY;AAAA,YACf,uBAAuB,UAAU,OAAO;AAAA,UAAA,CACzC;AAED,cAAI,WAAW,YAAY;AACzB,iBAAK,WAAW;AAAA,cACd,IAAI,KAAK,WAAA;AAAA,cACT,MAAM;AAAA,cACN,WAAW,KAAK,IAAA;AAAA,cAChB,OAAO,CAAC,EAAE,MAAM,SAAS,YAAY,YAAY,GAAG;AAAA,YAAA,CACrD;AAAA,UACH;AAAA,QACF,OAAO;AAEL,eAAK,YAAY;AAAA,YACf,4BAA4B,UAAU,OAAO;AAAA,UAAA,CAC9C;AAAA,QACH;AAAA,MACF,CAAC;AAAA,IAAA;AAIH,SAAK,cAAc;AAAA,MACjB,KAAK,WAAW;AAAA,QACd;AAAA,QACA,OAAO,EAAE,YAAY,UAAU,YAAY;AACzC,gBAAM,OAAO,KAAK,YAAY,IAAI,QAAQ;AAC1C,cAAI,MAAM,SAAS;AACjB,gBAAI;AACF,oBAAM,SAAS,MAAM,KAAK,QAAQ,KAAK;AACvC,mBAAK,YAAY;AAAA,gBACf;AAAA,gBACA,OAAO,WAAW,WAAW,SAAS,KAAK,UAAU,MAAM;AAAA,cAAA;AAAA,YAE/D,SAAS,OAAO;AACd,oBAAM,SACJ,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AACvD,mBAAK,YAAY;AAAA,gBACf;AAAA,gBACA,KAAK,UAAU,EAAE,OAAO,QAAQ;AAAA,cAAA;AAAA,YAEpC;AAAA,UACF;AAAA,QACF;AAAA,MAAA;AAAA,IACF;AAIF,SAAK,cAAc;AAAA,MACjB,KAAK,WAAW,GAAG,oBAAoB,CAAC,EAAE,cAAc;AAEtD,cAAM,gBAAgB,KAAK,MAAM,SAAS;AAAA,UACxC,CAAC,MAAM,EAAE,OAAO,QAAQ;AAAA,QAAA;AAE1B,YAAI,iBAAiB,GAAG;AACtB,gBAAM,cAAc,CAAC,GAAG,KAAK,MAAM,QAAQ;AAC3C,sBAAY,aAAa,IAAI;AAC7B,eAAK,YAAY,EAAE,UAAU,YAAA,CAAa;AAAA,QAC5C,OAAO;AACL,eAAK,WAAW,OAAO;AAAA,QACzB;AAAA,MACF,CAAC;AAAA,IAAA;AAIH,SAAK,cAAc;AAAA,MACjB,KAAK,WAAW,GAAG,eAAe,CAAC,EAAE,gBAAgB;AACnD,YAAI,WAAW;AACb,gBAAM,cAAc,KAAK,MAAM,SAAS;AAAA,YAAI,CAAC,MAC3C,EAAE,OAAO,YAAY,EAAE,GAAG,GAAG,aAAa,SAAS;AAAA,UAAA;AAErD,eAAK,YAAY,EAAE,UAAU,YAAA,CAAa;AAAA,QAC5C;AACA,aAAK,YAAY;AAAA,UACf,MAAM;AAAA,UACN,4BAA4B;AAAA,QAAA,CAC7B;AACD,aAAK,QAAQ,gBAAA;AAAA,MACf,CAAC;AAAA,IAAA;AAIH,SAAK,cAAc;AAAA,MACjB,KAAK,WAAW,GAAG,SAAS,CAAC,EAAE,YAAY;AACzC,aAAK,YAAY,EAAE,OAAO;AAC1B,aAAK,QAAQ,UAAU,KAAK;AAAA,MAC9B,CAAC;AAAA,IAAA;AAAA,EAEL;AAAA,EAEQ,qBAA2B;AACjC,QAAI,CAAC,KAAK,WAAY;AAEtB,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA,IACE,KAAK;AACT,UAAM,YACJ,gBACA,SACA,WACC,SAAS,MAAM,SAAS,KACzB,oBACA,gBAAgB,UAChB,oBAAoB,UACpB;AACF,QAAI,CAAC,UAAW;AAEhB,UAAM,cAAc,QAChB,MAAM,KAAK,KAAK,YAAY,OAAA,CAAQ,EAAE,IAAI,CAAC,OAAO;AAAA,MAChD,MAAM,EAAE;AAAA,MACR,aAAa,EAAE;AAAA,MACf,aAAa,EAAE,cACX,0BAA0B,EAAE,WAAW,IACvC;AAAA,IAAA,EACJ,IACF;AAEJ,SAAK,WAAW,cAAc;AAAA,MAC5B;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA,CACD;AAAA,EACH;AAAA,EAEQ,aAAqB;AAC3B,WAAO,OAAO,KAAK,IAAA,CAAK,IAAI,KAAK,OAAA,EAAS,SAAS,EAAE,EAAE,UAAU,CAAC,CAAC;AAAA,EACrE;AACF;"}
@@ -0,0 +1,118 @@
1
+ import { AnyClientTool, AudioVisualization, RealtimeEvent, RealtimeEventHandler, RealtimeMessage, RealtimeMode, RealtimeSessionConfig, RealtimeStatus, RealtimeToken } from '@tanstack/ai';
2
+ /**
3
+ * Adapter interface for connecting to realtime providers.
4
+ * Each provider (OpenAI, ElevenLabs, etc.) implements this interface.
5
+ */
6
+ export interface RealtimeAdapter {
7
+ /** Provider identifier */
8
+ provider: string;
9
+ /**
10
+ * Create a connection using the provided token
11
+ * @param token - The ephemeral token from the server
12
+ * @param clientTools - Optional client-side tools to register with the provider
13
+ * @returns A connection instance
14
+ */
15
+ connect: (token: RealtimeToken, clientTools?: ReadonlyArray<AnyClientTool>) => Promise<RealtimeConnection>;
16
+ }
17
+ /**
18
+ * Connection interface representing an active realtime session.
19
+ * Handles audio I/O, events, and session management.
20
+ */
21
+ export interface RealtimeConnection {
22
+ /** Disconnect from the realtime session */
23
+ disconnect: () => Promise<void>;
24
+ /** Start capturing audio from the microphone */
25
+ startAudioCapture: () => Promise<void>;
26
+ /** Stop capturing audio */
27
+ stopAudioCapture: () => void;
28
+ /** Send a text message (fallback for when voice isn't available) */
29
+ sendText: (text: string) => void;
30
+ /** Send an image to the conversation */
31
+ sendImage: (imageData: string, mimeType: string) => void;
32
+ /** Send a tool execution result back to the provider */
33
+ sendToolResult: (callId: string, result: string) => void;
34
+ /** Update session configuration */
35
+ updateSession: (config: Partial<RealtimeSessionConfig>) => void;
36
+ /** Interrupt the current response */
37
+ interrupt: () => void;
38
+ /** Subscribe to connection events */
39
+ on: <TEvent extends RealtimeEvent>(event: TEvent, handler: RealtimeEventHandler<TEvent>) => () => void;
40
+ /** Get audio visualization data */
41
+ getAudioVisualization: () => AudioVisualization;
42
+ }
43
+ /**
44
+ * Options for the RealtimeClient
45
+ */
46
+ export interface RealtimeClientOptions {
47
+ /**
48
+ * Function to fetch a realtime token from the server.
49
+ * Called on connect and when token needs refresh.
50
+ */
51
+ getToken: () => Promise<RealtimeToken>;
52
+ /**
53
+ * The realtime adapter to use (e.g., openaiRealtime())
54
+ */
55
+ adapter: RealtimeAdapter;
56
+ /**
57
+ * Client-side tools with execution logic
58
+ */
59
+ tools?: ReadonlyArray<AnyClientTool>;
60
+ /**
61
+ * Auto-play assistant audio (default: true)
62
+ */
63
+ autoPlayback?: boolean;
64
+ /**
65
+ * Request microphone access on connect (default: true)
66
+ */
67
+ autoCapture?: boolean;
68
+ /**
69
+ * System instructions for the assistant
70
+ */
71
+ instructions?: string;
72
+ /**
73
+ * Voice to use for audio output
74
+ */
75
+ voice?: string;
76
+ /**
77
+ * Voice activity detection mode (default: 'server')
78
+ */
79
+ vadMode?: 'server' | 'semantic' | 'manual';
80
+ /**
81
+ * Output modalities for responses (e.g., ['audio', 'text'])
82
+ */
83
+ outputModalities?: Array<'audio' | 'text'>;
84
+ /**
85
+ * Temperature for generation (provider-specific range)
86
+ */
87
+ temperature?: number;
88
+ /**
89
+ * Maximum number of tokens in a response
90
+ */
91
+ maxOutputTokens?: number | 'inf';
92
+ /**
93
+ * Eagerness level for semantic VAD ('low', 'medium', 'high')
94
+ */
95
+ semanticEagerness?: 'low' | 'medium' | 'high';
96
+ onStatusChange?: (status: RealtimeStatus) => void;
97
+ onModeChange?: (mode: RealtimeMode) => void;
98
+ onMessage?: (message: RealtimeMessage) => void;
99
+ onError?: (error: Error) => void;
100
+ onConnect?: () => void;
101
+ onDisconnect?: () => void;
102
+ onInterrupted?: () => void;
103
+ }
104
+ /**
105
+ * Internal state of the RealtimeClient
106
+ */
107
+ export interface RealtimeClientState {
108
+ status: RealtimeStatus;
109
+ mode: RealtimeMode;
110
+ messages: Array<RealtimeMessage>;
111
+ pendingUserTranscript: string | null;
112
+ pendingAssistantTranscript: string | null;
113
+ error: Error | null;
114
+ }
115
+ /**
116
+ * Callback type for state changes
117
+ */
118
+ export type RealtimeStateChangeCallback = (state: RealtimeClientState) => void;
@@ -0,0 +1,8 @@
1
+ import { StreamChunk } from '@tanstack/ai';
2
+ /**
3
+ * Parse a Response body as Server-Sent Events, yielding StreamChunks.
4
+ *
5
+ * Used by GenerationClient to parse SSE Responses returned from fetchers
6
+ * (e.g., TanStack Start server functions using `toServerSentEventsResponse()`).
7
+ */
8
+ export declare function parseSSEResponse(response: Response, abortSignal?: AbortSignal): AsyncGenerator<StreamChunk>;