@tanstack/ai-client 0.18.4 → 0.18.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -3
- package/dist/esm/chat-client.d.ts +0 -264
- package/dist/esm/chat-client.js +0 -1097
- package/dist/esm/chat-client.js.map +0 -1
- package/dist/esm/client-persistor.d.ts +0 -86
- package/dist/esm/client-persistor.js +0 -243
- package/dist/esm/client-persistor.js.map +0 -1
- package/dist/esm/connection-adapters.d.ts +0 -203
- package/dist/esm/connection-adapters.js +0 -611
- package/dist/esm/connection-adapters.js.map +0 -1
- package/dist/esm/devtools-noop.d.ts +0 -63
- package/dist/esm/devtools-noop.js +0 -141
- package/dist/esm/devtools-noop.js.map +0 -1
- package/dist/esm/devtools.d.ts +0 -276
- package/dist/esm/devtools.js +0 -1178
- package/dist/esm/devtools.js.map +0 -1
- package/dist/esm/events.d.ts +0 -129
- package/dist/esm/events.js +0 -227
- package/dist/esm/events.js.map +0 -1
- package/dist/esm/generation-client.d.ts +0 -99
- package/dist/esm/generation-client.js +0 -325
- package/dist/esm/generation-client.js.map +0 -1
- package/dist/esm/generation-types.d.ts +0 -242
- package/dist/esm/generation-types.js +0 -14
- package/dist/esm/generation-types.js.map +0 -1
- package/dist/esm/index.d.ts +0 -16
- package/dist/esm/index.js +0 -44
- package/dist/esm/index.js.map +0 -1
- package/dist/esm/realtime-client.d.ts +0 -98
- package/dist/esm/realtime-client.js +0 -403
- package/dist/esm/realtime-client.js.map +0 -1
- package/dist/esm/realtime-types.d.ts +0 -78
- package/dist/esm/response-stream.d.ts +0 -7
- package/dist/esm/response-stream.js +0 -32
- package/dist/esm/response-stream.js.map +0 -1
- package/dist/esm/sse-parser.d.ts +0 -8
- package/dist/esm/sse-parser.js +0 -52
- package/dist/esm/sse-parser.js.map +0 -1
- package/dist/esm/sse-utils.d.ts +0 -1
- package/dist/esm/sse-utils.js +0 -11
- package/dist/esm/sse-utils.js.map +0 -1
- package/dist/esm/tool-types.d.ts +0 -20
- package/dist/esm/types.d.ts +0 -419
- package/dist/esm/types.js +0 -11
- package/dist/esm/types.js.map +0 -1
- package/dist/esm/video-generation-client.d.ts +0 -110
- package/dist/esm/video-generation-client.js +0 -382
- package/dist/esm/video-generation-client.js.map +0 -1
|
@@ -1,403 +0,0 @@
|
|
|
1
|
-
import { convertSchemaToJsonSchema } from "@tanstack/ai/client";
|
|
2
|
-
const TOKEN_REFRESH_BUFFER_MS = 6e4;
|
|
3
|
-
class RealtimeClient {
|
|
4
|
-
options;
|
|
5
|
-
connection = null;
|
|
6
|
-
token = null;
|
|
7
|
-
tokenRefreshTimeout = null;
|
|
8
|
-
clientTools;
|
|
9
|
-
stateChangeCallbacks = /* @__PURE__ */ new Set();
|
|
10
|
-
unsubscribers = [];
|
|
11
|
-
state = {
|
|
12
|
-
status: "idle",
|
|
13
|
-
mode: "idle",
|
|
14
|
-
messages: [],
|
|
15
|
-
pendingUserTranscript: null,
|
|
16
|
-
pendingAssistantTranscript: null,
|
|
17
|
-
error: null
|
|
18
|
-
};
|
|
19
|
-
constructor(options) {
|
|
20
|
-
this.options = {
|
|
21
|
-
autoPlayback: true,
|
|
22
|
-
autoCapture: true,
|
|
23
|
-
vadMode: "server",
|
|
24
|
-
...options
|
|
25
|
-
};
|
|
26
|
-
this.clientTools = /* @__PURE__ */ new Map();
|
|
27
|
-
if (options.tools) {
|
|
28
|
-
for (const tool of options.tools) {
|
|
29
|
-
this.clientTools.set(tool.name, tool);
|
|
30
|
-
}
|
|
31
|
-
}
|
|
32
|
-
}
|
|
33
|
-
// ============================================================================
|
|
34
|
-
// Connection Lifecycle
|
|
35
|
-
// ============================================================================
|
|
36
|
-
/**
|
|
37
|
-
* Connect to the realtime session.
|
|
38
|
-
* Fetches a token and establishes the connection.
|
|
39
|
-
*/
|
|
40
|
-
async connect() {
|
|
41
|
-
if (this.state.status === "connected") {
|
|
42
|
-
return;
|
|
43
|
-
}
|
|
44
|
-
this.updateState({ status: "connecting", error: null });
|
|
45
|
-
try {
|
|
46
|
-
this.token = await this.options.getToken();
|
|
47
|
-
this.scheduleTokenRefresh();
|
|
48
|
-
const toolsList = this.clientTools.size > 0 ? Array.from(this.clientTools.values()) : void 0;
|
|
49
|
-
this.connection = await this.options.adapter.connect(
|
|
50
|
-
this.token,
|
|
51
|
-
toolsList
|
|
52
|
-
);
|
|
53
|
-
this.subscribeToConnectionEvents();
|
|
54
|
-
this.applySessionConfig();
|
|
55
|
-
if (this.options.autoCapture) {
|
|
56
|
-
await this.connection.startAudioCapture();
|
|
57
|
-
}
|
|
58
|
-
this.updateState({ status: "connected", mode: "listening" });
|
|
59
|
-
this.options.onConnect?.();
|
|
60
|
-
} catch (error) {
|
|
61
|
-
const err = error instanceof Error ? error : new Error(String(error));
|
|
62
|
-
this.updateState({ status: "error", error: err });
|
|
63
|
-
this.options.onError?.(err);
|
|
64
|
-
throw err;
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
/**
|
|
68
|
-
* Disconnect from the realtime session.
|
|
69
|
-
*/
|
|
70
|
-
async disconnect() {
|
|
71
|
-
if (this.tokenRefreshTimeout) {
|
|
72
|
-
clearTimeout(this.tokenRefreshTimeout);
|
|
73
|
-
this.tokenRefreshTimeout = null;
|
|
74
|
-
}
|
|
75
|
-
for (const unsub of this.unsubscribers) {
|
|
76
|
-
unsub();
|
|
77
|
-
}
|
|
78
|
-
this.unsubscribers = [];
|
|
79
|
-
if (this.connection) {
|
|
80
|
-
await this.connection.disconnect();
|
|
81
|
-
this.connection = null;
|
|
82
|
-
}
|
|
83
|
-
this.token = null;
|
|
84
|
-
this.updateState({
|
|
85
|
-
status: "idle",
|
|
86
|
-
mode: "idle",
|
|
87
|
-
pendingUserTranscript: null,
|
|
88
|
-
pendingAssistantTranscript: null
|
|
89
|
-
});
|
|
90
|
-
this.options.onDisconnect?.();
|
|
91
|
-
}
|
|
92
|
-
// ============================================================================
|
|
93
|
-
// Voice Control
|
|
94
|
-
// ============================================================================
|
|
95
|
-
/**
|
|
96
|
-
* Start listening for voice input.
|
|
97
|
-
* Only needed when vadMode is 'manual'.
|
|
98
|
-
*/
|
|
99
|
-
startListening() {
|
|
100
|
-
if (!this.connection || this.state.status !== "connected") {
|
|
101
|
-
return;
|
|
102
|
-
}
|
|
103
|
-
void this.connection.startAudioCapture();
|
|
104
|
-
this.updateState({ mode: "listening" });
|
|
105
|
-
}
|
|
106
|
-
/**
|
|
107
|
-
* Stop listening for voice input.
|
|
108
|
-
* Only needed when vadMode is 'manual'.
|
|
109
|
-
*/
|
|
110
|
-
stopListening() {
|
|
111
|
-
if (!this.connection) {
|
|
112
|
-
return;
|
|
113
|
-
}
|
|
114
|
-
this.connection.stopAudioCapture();
|
|
115
|
-
this.updateState({ mode: "idle" });
|
|
116
|
-
}
|
|
117
|
-
/**
|
|
118
|
-
* Interrupt the current assistant response.
|
|
119
|
-
*/
|
|
120
|
-
interrupt() {
|
|
121
|
-
if (!this.connection) {
|
|
122
|
-
return;
|
|
123
|
-
}
|
|
124
|
-
this.connection.interrupt();
|
|
125
|
-
}
|
|
126
|
-
// ============================================================================
|
|
127
|
-
// Text Input
|
|
128
|
-
// ============================================================================
|
|
129
|
-
/**
|
|
130
|
-
* Send a text message instead of voice.
|
|
131
|
-
*/
|
|
132
|
-
sendText(text) {
|
|
133
|
-
if (!this.connection || this.state.status !== "connected") {
|
|
134
|
-
return;
|
|
135
|
-
}
|
|
136
|
-
const userMessage = {
|
|
137
|
-
id: this.generateId(),
|
|
138
|
-
role: "user",
|
|
139
|
-
timestamp: Date.now(),
|
|
140
|
-
parts: [{ type: "text", content: text }]
|
|
141
|
-
};
|
|
142
|
-
this.addMessage(userMessage);
|
|
143
|
-
this.connection.sendText(text);
|
|
144
|
-
}
|
|
145
|
-
/**
|
|
146
|
-
* Send an image to the conversation.
|
|
147
|
-
* @param imageData - Base64-encoded image data or a URL
|
|
148
|
-
* @param mimeType - MIME type of the image (e.g., 'image/png', 'image/jpeg')
|
|
149
|
-
*/
|
|
150
|
-
sendImage(imageData, mimeType) {
|
|
151
|
-
if (!this.connection || this.state.status !== "connected") {
|
|
152
|
-
return;
|
|
153
|
-
}
|
|
154
|
-
const userMessage = {
|
|
155
|
-
id: this.generateId(),
|
|
156
|
-
role: "user",
|
|
157
|
-
timestamp: Date.now(),
|
|
158
|
-
parts: [{ type: "image", data: imageData, mimeType }]
|
|
159
|
-
};
|
|
160
|
-
this.addMessage(userMessage);
|
|
161
|
-
this.connection.sendImage(imageData, mimeType);
|
|
162
|
-
}
|
|
163
|
-
// ============================================================================
|
|
164
|
-
// State Access
|
|
165
|
-
// ============================================================================
|
|
166
|
-
/** Get current connection status */
|
|
167
|
-
get status() {
|
|
168
|
-
return this.state.status;
|
|
169
|
-
}
|
|
170
|
-
/** Get current mode */
|
|
171
|
-
get mode() {
|
|
172
|
-
return this.state.mode;
|
|
173
|
-
}
|
|
174
|
-
/** Get conversation messages */
|
|
175
|
-
get messages() {
|
|
176
|
-
return this.state.messages;
|
|
177
|
-
}
|
|
178
|
-
/** Get current error, if any */
|
|
179
|
-
get error() {
|
|
180
|
-
return this.state.error;
|
|
181
|
-
}
|
|
182
|
-
/** Get pending user transcript (while user is speaking) */
|
|
183
|
-
get pendingUserTranscript() {
|
|
184
|
-
return this.state.pendingUserTranscript;
|
|
185
|
-
}
|
|
186
|
-
/** Get pending assistant transcript (while assistant is speaking) */
|
|
187
|
-
get pendingAssistantTranscript() {
|
|
188
|
-
return this.state.pendingAssistantTranscript;
|
|
189
|
-
}
|
|
190
|
-
/** Get audio visualization data */
|
|
191
|
-
get audio() {
|
|
192
|
-
return this.connection?.getAudioVisualization() ?? null;
|
|
193
|
-
}
|
|
194
|
-
// ============================================================================
|
|
195
|
-
// State Subscription
|
|
196
|
-
// ============================================================================
|
|
197
|
-
/**
|
|
198
|
-
* Subscribe to state changes.
|
|
199
|
-
* @returns Unsubscribe function
|
|
200
|
-
*/
|
|
201
|
-
onStateChange(callback) {
|
|
202
|
-
this.stateChangeCallbacks.add(callback);
|
|
203
|
-
return () => {
|
|
204
|
-
this.stateChangeCallbacks.delete(callback);
|
|
205
|
-
};
|
|
206
|
-
}
|
|
207
|
-
// ============================================================================
|
|
208
|
-
// Cleanup
|
|
209
|
-
// ============================================================================
|
|
210
|
-
/**
|
|
211
|
-
* Clean up resources.
|
|
212
|
-
* Call this when disposing of the client.
|
|
213
|
-
*/
|
|
214
|
-
destroy() {
|
|
215
|
-
void this.disconnect();
|
|
216
|
-
this.stateChangeCallbacks.clear();
|
|
217
|
-
}
|
|
218
|
-
// ============================================================================
|
|
219
|
-
// Private Methods
|
|
220
|
-
// ============================================================================
|
|
221
|
-
updateState(updates) {
|
|
222
|
-
this.state = { ...this.state, ...updates };
|
|
223
|
-
for (const callback of this.stateChangeCallbacks) {
|
|
224
|
-
callback(this.state);
|
|
225
|
-
}
|
|
226
|
-
if ("status" in updates && updates.status !== void 0) {
|
|
227
|
-
this.options.onStatusChange?.(updates.status);
|
|
228
|
-
}
|
|
229
|
-
if ("mode" in updates && updates.mode !== void 0) {
|
|
230
|
-
this.options.onModeChange?.(updates.mode);
|
|
231
|
-
}
|
|
232
|
-
}
|
|
233
|
-
addMessage(message) {
|
|
234
|
-
this.updateState({
|
|
235
|
-
messages: [...this.state.messages, message]
|
|
236
|
-
});
|
|
237
|
-
this.options.onMessage?.(message);
|
|
238
|
-
}
|
|
239
|
-
scheduleTokenRefresh() {
|
|
240
|
-
if (!this.token) return;
|
|
241
|
-
const timeUntilExpiry = this.token.expiresAt - Date.now();
|
|
242
|
-
const refreshIn = Math.max(0, timeUntilExpiry - TOKEN_REFRESH_BUFFER_MS);
|
|
243
|
-
this.tokenRefreshTimeout = setTimeout(() => {
|
|
244
|
-
void this.refreshToken();
|
|
245
|
-
}, refreshIn);
|
|
246
|
-
}
|
|
247
|
-
async refreshToken() {
|
|
248
|
-
try {
|
|
249
|
-
this.token = await this.options.getToken();
|
|
250
|
-
this.scheduleTokenRefresh();
|
|
251
|
-
} catch (error) {
|
|
252
|
-
const err = error instanceof Error ? error : new Error(String(error));
|
|
253
|
-
this.updateState({ error: err });
|
|
254
|
-
this.options.onError?.(err);
|
|
255
|
-
}
|
|
256
|
-
}
|
|
257
|
-
subscribeToConnectionEvents() {
|
|
258
|
-
if (!this.connection) return;
|
|
259
|
-
this.unsubscribers.push(
|
|
260
|
-
this.connection.on("status_change", ({ status }) => {
|
|
261
|
-
this.updateState({ status });
|
|
262
|
-
})
|
|
263
|
-
);
|
|
264
|
-
this.unsubscribers.push(
|
|
265
|
-
this.connection.on("mode_change", ({ mode }) => {
|
|
266
|
-
this.updateState({ mode });
|
|
267
|
-
})
|
|
268
|
-
);
|
|
269
|
-
this.unsubscribers.push(
|
|
270
|
-
this.connection.on("transcript", ({ role, transcript, isFinal }) => {
|
|
271
|
-
if (role === "user") {
|
|
272
|
-
this.updateState({
|
|
273
|
-
pendingUserTranscript: isFinal ? null : transcript
|
|
274
|
-
});
|
|
275
|
-
if (isFinal && transcript) {
|
|
276
|
-
this.addMessage({
|
|
277
|
-
id: this.generateId(),
|
|
278
|
-
role: "user",
|
|
279
|
-
timestamp: Date.now(),
|
|
280
|
-
parts: [{ type: "audio", transcript, durationMs: 0 }]
|
|
281
|
-
});
|
|
282
|
-
}
|
|
283
|
-
} else {
|
|
284
|
-
this.updateState({
|
|
285
|
-
pendingAssistantTranscript: isFinal ? null : transcript
|
|
286
|
-
});
|
|
287
|
-
}
|
|
288
|
-
})
|
|
289
|
-
);
|
|
290
|
-
this.unsubscribers.push(
|
|
291
|
-
this.connection.on("tool_call", ({ toolCallId, toolName, input }) => {
|
|
292
|
-
if (!toolCallId) {
|
|
293
|
-
console.error(
|
|
294
|
-
"[RealtimeClient] tool_call missing toolCallId",
|
|
295
|
-
toolName
|
|
296
|
-
);
|
|
297
|
-
return;
|
|
298
|
-
}
|
|
299
|
-
const tool = this.clientTools.get(toolName);
|
|
300
|
-
const execute = tool?.execute;
|
|
301
|
-
if (execute) {
|
|
302
|
-
void (async () => {
|
|
303
|
-
try {
|
|
304
|
-
const output = await execute(input);
|
|
305
|
-
this.connection?.sendToolResult(
|
|
306
|
-
toolCallId,
|
|
307
|
-
typeof output === "string" ? output : JSON.stringify(output)
|
|
308
|
-
);
|
|
309
|
-
} catch (error) {
|
|
310
|
-
const errMsg = error instanceof Error ? error.message : String(error);
|
|
311
|
-
this.connection?.sendToolResult(
|
|
312
|
-
toolCallId,
|
|
313
|
-
JSON.stringify({ error: errMsg })
|
|
314
|
-
);
|
|
315
|
-
}
|
|
316
|
-
})();
|
|
317
|
-
} else {
|
|
318
|
-
this.connection?.sendToolResult(
|
|
319
|
-
toolCallId,
|
|
320
|
-
JSON.stringify({
|
|
321
|
-
error: `Unknown or non-executable tool: ${String(toolName)}`
|
|
322
|
-
})
|
|
323
|
-
);
|
|
324
|
-
}
|
|
325
|
-
})
|
|
326
|
-
);
|
|
327
|
-
this.unsubscribers.push(
|
|
328
|
-
this.connection.on("message_complete", ({ message }) => {
|
|
329
|
-
const existingIndex = this.state.messages.findIndex(
|
|
330
|
-
(m) => m.id === message.id
|
|
331
|
-
);
|
|
332
|
-
if (existingIndex >= 0) {
|
|
333
|
-
const newMessages = [...this.state.messages];
|
|
334
|
-
newMessages[existingIndex] = message;
|
|
335
|
-
this.updateState({ messages: newMessages });
|
|
336
|
-
} else {
|
|
337
|
-
this.addMessage(message);
|
|
338
|
-
}
|
|
339
|
-
})
|
|
340
|
-
);
|
|
341
|
-
this.unsubscribers.push(
|
|
342
|
-
this.connection.on("interrupted", ({ messageId }) => {
|
|
343
|
-
if (messageId) {
|
|
344
|
-
const newMessages = this.state.messages.map(
|
|
345
|
-
(m) => m.id === messageId ? { ...m, interrupted: true } : m
|
|
346
|
-
);
|
|
347
|
-
this.updateState({ messages: newMessages });
|
|
348
|
-
}
|
|
349
|
-
this.updateState({
|
|
350
|
-
mode: "listening",
|
|
351
|
-
pendingAssistantTranscript: null
|
|
352
|
-
});
|
|
353
|
-
this.options.onInterrupted?.();
|
|
354
|
-
})
|
|
355
|
-
);
|
|
356
|
-
this.unsubscribers.push(
|
|
357
|
-
this.connection.on("error", ({ error }) => {
|
|
358
|
-
this.updateState({ error });
|
|
359
|
-
this.options.onError?.(error);
|
|
360
|
-
})
|
|
361
|
-
);
|
|
362
|
-
}
|
|
363
|
-
applySessionConfig() {
|
|
364
|
-
if (!this.connection) return;
|
|
365
|
-
const {
|
|
366
|
-
instructions,
|
|
367
|
-
voice,
|
|
368
|
-
vadMode,
|
|
369
|
-
tools,
|
|
370
|
-
outputModalities,
|
|
371
|
-
temperature,
|
|
372
|
-
maxOutputTokens,
|
|
373
|
-
semanticEagerness
|
|
374
|
-
} = this.options;
|
|
375
|
-
const hasConfig = instructions || voice || vadMode || tools && tools.length > 0 || outputModalities || temperature !== void 0 || maxOutputTokens !== void 0 || semanticEagerness;
|
|
376
|
-
if (!hasConfig) return;
|
|
377
|
-
const toolsConfig = tools ? Array.from(this.clientTools.values()).map((t) => {
|
|
378
|
-
const inputSchema = t.inputSchema ? convertSchemaToJsonSchema(t.inputSchema) : void 0;
|
|
379
|
-
return {
|
|
380
|
-
name: t.name,
|
|
381
|
-
description: t.description,
|
|
382
|
-
...inputSchema ? { inputSchema } : {}
|
|
383
|
-
};
|
|
384
|
-
}) : void 0;
|
|
385
|
-
this.connection.updateSession({
|
|
386
|
-
...instructions !== void 0 ? { instructions } : {},
|
|
387
|
-
...voice !== void 0 ? { voice } : {},
|
|
388
|
-
...vadMode !== void 0 ? { vadMode } : {},
|
|
389
|
-
...toolsConfig !== void 0 ? { tools: toolsConfig } : {},
|
|
390
|
-
...outputModalities !== void 0 ? { outputModalities } : {},
|
|
391
|
-
...temperature !== void 0 ? { temperature } : {},
|
|
392
|
-
...maxOutputTokens !== void 0 ? { maxOutputTokens } : {},
|
|
393
|
-
...semanticEagerness !== void 0 ? { semanticEagerness } : {}
|
|
394
|
-
});
|
|
395
|
-
}
|
|
396
|
-
generateId() {
|
|
397
|
-
return `msg-${Date.now()}-${Math.random().toString(36).substring(7)}`;
|
|
398
|
-
}
|
|
399
|
-
}
|
|
400
|
-
export {
|
|
401
|
-
RealtimeClient
|
|
402
|
-
};
|
|
403
|
-
//# sourceMappingURL=realtime-client.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"realtime-client.js","sources":["../../src/realtime-client.ts"],"sourcesContent":["import { convertSchemaToJsonSchema } from '@tanstack/ai/client'\nimport type {\n AnyClientTool,\n AudioVisualization,\n RealtimeMessage,\n RealtimeMode,\n RealtimeStatus,\n RealtimeToken,\n} from '@tanstack/ai/client'\nimport type {\n RealtimeClientOptions,\n RealtimeClientState,\n RealtimeConnection,\n RealtimeStateChangeCallback,\n} from './realtime-types'\n\n// Token refresh buffer - refresh 1 minute before expiry\nconst TOKEN_REFRESH_BUFFER_MS = 60_000\n\n/**\n * Client for managing realtime voice conversations.\n *\n * Handles connection lifecycle, audio I/O, message state,\n * and tool execution for realtime voice-to-voice AI interactions.\n *\n * @example\n * ```typescript\n * import { RealtimeClient } from '@tanstack/ai-client'\n * import { openaiRealtime } from '@tanstack/ai-openai'\n *\n * const client = new RealtimeClient({\n * getToken: () => fetch('/api/realtime-token').then(r => r.json()),\n * adapter: openaiRealtime(),\n * tools: [myTool.client(handler)],\n * onMessage: (msg) => console.log('Message:', msg),\n * })\n *\n * await client.connect()\n * ```\n */\nexport class RealtimeClient {\n private readonly options: RealtimeClientOptions\n private connection: RealtimeConnection | null = null\n private token: RealtimeToken | null = null\n private tokenRefreshTimeout: ReturnType<typeof setTimeout> | null = null\n private readonly clientTools: Map<string, AnyClientTool>\n private readonly stateChangeCallbacks: Set<RealtimeStateChangeCallback> =\n new Set()\n private unsubscribers: Array<() => void> = []\n\n private state: RealtimeClientState = {\n status: 'idle',\n mode: 'idle',\n messages: [],\n pendingUserTranscript: null,\n pendingAssistantTranscript: null,\n error: null,\n }\n\n constructor(options: RealtimeClientOptions) {\n this.options = {\n autoPlayback: true,\n autoCapture: true,\n vadMode: 'server',\n ...options,\n }\n\n // Build client tools map\n this.clientTools = new Map()\n if (options.tools) {\n for (const tool of options.tools) {\n this.clientTools.set(tool.name, tool)\n }\n }\n }\n\n // ============================================================================\n // Connection Lifecycle\n // ============================================================================\n\n /**\n * Connect to the realtime session.\n * Fetches a token and establishes the connection.\n */\n async connect(): Promise<void> {\n if (this.state.status === 'connected') {\n return\n }\n\n this.updateState({ status: 'connecting', error: null })\n\n try {\n // Fetch token from server\n this.token = await this.options.getToken()\n\n // Schedule token refresh\n this.scheduleTokenRefresh()\n\n // Connect via adapter (pass tools for providers like ElevenLabs that need them at connect time)\n const toolsList =\n this.clientTools.size > 0\n ? Array.from(this.clientTools.values())\n : undefined\n this.connection = await this.options.adapter.connect(\n this.token,\n toolsList,\n )\n\n // Subscribe to connection events\n this.subscribeToConnectionEvents()\n\n // Auto-configure session with client-provided settings\n this.applySessionConfig()\n\n // Start audio capture if configured\n if (this.options.autoCapture) {\n await this.connection.startAudioCapture()\n }\n\n this.updateState({ status: 'connected', mode: 'listening' })\n this.options.onConnect?.()\n } catch (error) {\n const err = error instanceof Error ? error : new Error(String(error))\n this.updateState({ status: 'error', error: err })\n this.options.onError?.(err)\n throw err\n }\n }\n\n /**\n * Disconnect from the realtime session.\n */\n async disconnect(): Promise<void> {\n if (this.tokenRefreshTimeout) {\n clearTimeout(this.tokenRefreshTimeout)\n this.tokenRefreshTimeout = null\n }\n\n // Unsubscribe from all events\n for (const unsub of this.unsubscribers) {\n unsub()\n }\n this.unsubscribers = []\n\n if (this.connection) {\n await this.connection.disconnect()\n this.connection = null\n }\n\n this.token = null\n this.updateState({\n status: 'idle',\n mode: 'idle',\n pendingUserTranscript: null,\n pendingAssistantTranscript: null,\n })\n this.options.onDisconnect?.()\n }\n\n // ============================================================================\n // Voice Control\n // ============================================================================\n\n /**\n * Start listening for voice input.\n * Only needed when vadMode is 'manual'.\n */\n startListening(): void {\n if (!this.connection || this.state.status !== 'connected') {\n return\n }\n void this.connection.startAudioCapture()\n this.updateState({ mode: 'listening' })\n }\n\n /**\n * Stop listening for voice input.\n * Only needed when vadMode is 'manual'.\n */\n stopListening(): void {\n if (!this.connection) {\n return\n }\n this.connection.stopAudioCapture()\n this.updateState({ mode: 'idle' })\n }\n\n /**\n * Interrupt the current assistant response.\n */\n interrupt(): void {\n if (!this.connection) {\n return\n }\n this.connection.interrupt()\n }\n\n // ============================================================================\n // Text Input\n // ============================================================================\n\n /**\n * Send a text message instead of voice.\n */\n sendText(text: string): void {\n if (!this.connection || this.state.status !== 'connected') {\n return\n }\n\n // Add user message\n const userMessage: RealtimeMessage = {\n id: this.generateId(),\n role: 'user',\n timestamp: Date.now(),\n parts: [{ type: 'text', content: text }],\n }\n this.addMessage(userMessage)\n\n // Send to provider\n this.connection.sendText(text)\n }\n\n /**\n * Send an image to the conversation.\n * @param imageData - Base64-encoded image data or a URL\n * @param mimeType - MIME type of the image (e.g., 'image/png', 'image/jpeg')\n */\n sendImage(imageData: string, mimeType: string): void {\n if (!this.connection || this.state.status !== 'connected') {\n return\n }\n\n // Add user message with image part\n const userMessage: RealtimeMessage = {\n id: this.generateId(),\n role: 'user',\n timestamp: Date.now(),\n parts: [{ type: 'image', data: imageData, mimeType }],\n }\n this.addMessage(userMessage)\n\n // Send to provider\n this.connection.sendImage(imageData, mimeType)\n }\n\n // ============================================================================\n // State Access\n // ============================================================================\n\n /** Get current connection status */\n get status(): RealtimeStatus {\n return this.state.status\n }\n\n /** Get current mode */\n get mode(): RealtimeMode {\n return this.state.mode\n }\n\n /** Get conversation messages */\n get messages(): Array<RealtimeMessage> {\n return this.state.messages\n }\n\n /** Get current error, if any */\n get error(): Error | null {\n return this.state.error\n }\n\n /** Get pending user transcript (while user is speaking) */\n get pendingUserTranscript(): string | null {\n return this.state.pendingUserTranscript\n }\n\n /** Get pending assistant transcript (while assistant is speaking) */\n get pendingAssistantTranscript(): string | null {\n return this.state.pendingAssistantTranscript\n }\n\n /** Get audio visualization data */\n get audio(): AudioVisualization | null {\n return this.connection?.getAudioVisualization() ?? null\n }\n\n // ============================================================================\n // State Subscription\n // ============================================================================\n\n /**\n * Subscribe to state changes.\n * @returns Unsubscribe function\n */\n onStateChange(callback: RealtimeStateChangeCallback): () => void {\n this.stateChangeCallbacks.add(callback)\n return () => {\n this.stateChangeCallbacks.delete(callback)\n }\n }\n\n // ============================================================================\n // Cleanup\n // ============================================================================\n\n /**\n * Clean up resources.\n * Call this when disposing of the client.\n */\n destroy(): void {\n void this.disconnect()\n this.stateChangeCallbacks.clear()\n }\n\n // ============================================================================\n // Private Methods\n // ============================================================================\n\n private updateState(updates: Partial<RealtimeClientState>): void {\n this.state = { ...this.state, ...updates }\n\n // Notify callbacks\n for (const callback of this.stateChangeCallbacks) {\n callback(this.state)\n }\n\n // Notify specific callbacks\n if ('status' in updates && updates.status !== undefined) {\n this.options.onStatusChange?.(updates.status)\n }\n if ('mode' in updates && updates.mode !== undefined) {\n this.options.onModeChange?.(updates.mode)\n }\n }\n\n private addMessage(message: RealtimeMessage): void {\n this.updateState({\n messages: [...this.state.messages, message],\n })\n this.options.onMessage?.(message)\n }\n\n private scheduleTokenRefresh(): void {\n if (!this.token) return\n\n const timeUntilExpiry = this.token.expiresAt - Date.now()\n const refreshIn = Math.max(0, timeUntilExpiry - TOKEN_REFRESH_BUFFER_MS)\n\n this.tokenRefreshTimeout = setTimeout(() => {\n void this.refreshToken()\n }, refreshIn)\n }\n\n private async refreshToken(): Promise<void> {\n try {\n this.token = await this.options.getToken()\n this.scheduleTokenRefresh()\n // Note: Some providers may require reconnection with new token\n // This is handled by the adapter implementation\n } catch (error) {\n const err = error instanceof Error ? error : new Error(String(error))\n this.updateState({ error: err })\n this.options.onError?.(err)\n }\n }\n\n private subscribeToConnectionEvents(): void {\n if (!this.connection) return\n\n // Status changes\n this.unsubscribers.push(\n this.connection.on('status_change', ({ status }) => {\n this.updateState({ status })\n }),\n )\n\n // Mode changes\n this.unsubscribers.push(\n this.connection.on('mode_change', ({ mode }) => {\n this.updateState({ mode })\n }),\n )\n\n // Transcripts (streaming)\n // User transcripts are added as messages when final (no separate message_complete for user input)\n // Assistant transcripts are streamed, final message comes via message_complete\n this.unsubscribers.push(\n this.connection.on('transcript', ({ role, transcript, isFinal }) => {\n if (role === 'user') {\n this.updateState({\n pendingUserTranscript: isFinal ? null : transcript,\n })\n // Add user message when transcript is finalized\n if (isFinal && transcript) {\n this.addMessage({\n id: this.generateId(),\n role: 'user',\n timestamp: Date.now(),\n parts: [{ type: 'audio', transcript, durationMs: 0 }],\n })\n }\n } else {\n // Assistant transcripts - just update pending, message_complete handles final\n this.updateState({\n pendingAssistantTranscript: isFinal ? null : transcript,\n })\n }\n }),\n )\n\n // Tool calls\n this.unsubscribers.push(\n this.connection.on('tool_call', ({ toolCallId, toolName, input }) => {\n if (!toolCallId) {\n console.error(\n '[RealtimeClient] tool_call missing toolCallId',\n toolName,\n )\n return\n }\n const tool = this.clientTools.get(toolName)\n const execute = tool?.execute\n if (execute) {\n void (async () => {\n try {\n const output = await execute(input)\n this.connection?.sendToolResult(\n toolCallId,\n typeof output === 'string' ? output : JSON.stringify(output),\n )\n } catch (error) {\n const errMsg =\n error instanceof Error ? error.message : String(error)\n this.connection?.sendToolResult(\n toolCallId,\n JSON.stringify({ error: errMsg }),\n )\n }\n })()\n } else {\n this.connection?.sendToolResult(\n toolCallId,\n JSON.stringify({\n error: `Unknown or non-executable tool: ${String(toolName)}`,\n }),\n )\n }\n }),\n )\n\n // Message complete\n this.unsubscribers.push(\n this.connection.on('message_complete', ({ message }) => {\n // Replace pending message with final version if needed\n const existingIndex = this.state.messages.findIndex(\n (m) => m.id === message.id,\n )\n if (existingIndex >= 0) {\n const newMessages = [...this.state.messages]\n newMessages[existingIndex] = message\n this.updateState({ messages: newMessages })\n } else {\n this.addMessage(message)\n }\n }),\n )\n\n // Interruption\n this.unsubscribers.push(\n this.connection.on('interrupted', ({ messageId }) => {\n if (messageId) {\n const newMessages = this.state.messages.map((m) =>\n m.id === messageId ? { ...m, interrupted: true } : m,\n )\n this.updateState({ messages: newMessages })\n }\n this.updateState({\n mode: 'listening',\n pendingAssistantTranscript: null,\n })\n this.options.onInterrupted?.()\n }),\n )\n\n // Errors\n this.unsubscribers.push(\n this.connection.on('error', ({ error }) => {\n this.updateState({ error })\n this.options.onError?.(error)\n }),\n )\n }\n\n private applySessionConfig(): void {\n if (!this.connection) return\n\n const {\n instructions,\n voice,\n vadMode,\n tools,\n outputModalities,\n temperature,\n maxOutputTokens,\n semanticEagerness,\n } = this.options\n const hasConfig =\n instructions ||\n voice ||\n vadMode ||\n (tools && tools.length > 0) ||\n outputModalities ||\n temperature !== undefined ||\n maxOutputTokens !== undefined ||\n semanticEagerness\n if (!hasConfig) return\n\n // `RealtimeToolConfig.inputSchema` is `Record<string, any>` (no\n // `undefined` under `exactOptionalPropertyTypes`). Conditionally spread\n // it so we don't pass `undefined` when the tool has no input schema.\n const toolsConfig = tools\n ? Array.from(this.clientTools.values()).map((t) => {\n const inputSchema = t.inputSchema\n ? convertSchemaToJsonSchema(t.inputSchema)\n : undefined\n return {\n name: t.name,\n description: t.description,\n ...(inputSchema ? { inputSchema } : {}),\n }\n })\n : undefined\n\n // `RealtimeSessionConfig` declares each field as `field?: T` (no\n // `undefined`). Spread each one conditionally so we don't violate\n // `exactOptionalPropertyTypes` when the source is `T | undefined`.\n this.connection.updateSession({\n ...(instructions !== undefined ? { instructions } : {}),\n ...(voice !== undefined ? { voice } : {}),\n ...(vadMode !== undefined ? { vadMode } : {}),\n ...(toolsConfig !== undefined ? { tools: toolsConfig } : {}),\n ...(outputModalities !== undefined ? { outputModalities } : {}),\n ...(temperature !== undefined ? { temperature } : {}),\n ...(maxOutputTokens !== undefined ? { maxOutputTokens } : {}),\n ...(semanticEagerness !== undefined ? { semanticEagerness } : {}),\n })\n }\n\n private generateId(): string {\n return `msg-${Date.now()}-${Math.random().toString(36).substring(7)}`\n }\n}\n"],"names":[],"mappings":";AAiBA,MAAM,0BAA0B;AAuBzB,MAAM,eAAe;AAAA,EACT;AAAA,EACT,aAAwC;AAAA,EACxC,QAA8B;AAAA,EAC9B,sBAA4D;AAAA,EACnD;AAAA,EACA,2CACX,IAAA;AAAA,EACE,gBAAmC,CAAA;AAAA,EAEnC,QAA6B;AAAA,IACnC,QAAQ;AAAA,IACR,MAAM;AAAA,IACN,UAAU,CAAA;AAAA,IACV,uBAAuB;AAAA,IACvB,4BAA4B;AAAA,IAC5B,OAAO;AAAA,EAAA;AAAA,EAGT,YAAY,SAAgC;AAC1C,SAAK,UAAU;AAAA,MACb,cAAc;AAAA,MACd,aAAa;AAAA,MACb,SAAS;AAAA,MACT,GAAG;AAAA,IAAA;AAIL,SAAK,kCAAkB,IAAA;AACvB,QAAI,QAAQ,OAAO;AACjB,iBAAW,QAAQ,QAAQ,OAAO;AAChC,aAAK,YAAY,IAAI,KAAK,MAAM,IAAI;AAAA,MACtC;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,MAAM,UAAyB;AAC7B,QAAI,KAAK,MAAM,WAAW,aAAa;AACrC;AAAA,IACF;AAEA,SAAK,YAAY,EAAE,QAAQ,cAAc,OAAO,MAAM;AAEtD,QAAI;AAEF,WAAK,QAAQ,MAAM,KAAK,QAAQ,SAAA;AAGhC,WAAK,qBAAA;AAGL,YAAM,YACJ,KAAK,YAAY,OAAO,IACpB,MAAM,KAAK,KAAK,YAAY,OAAA,CAAQ,IACpC;AACN,WAAK,aAAa,MAAM,KAAK,QAAQ,QAAQ;AAAA,QAC3C,KAAK;AAAA,QACL;AAAA,MAAA;AAIF,WAAK,4BAAA;AAGL,WAAK,mBAAA;AAGL,UAAI,KAAK,QAAQ,aAAa;AAC5B,cAAM,KAAK,WAAW,kBAAA;AAAA,MACxB;AAEA,WAAK,YAAY,EAAE,QAAQ,aAAa,MAAM,aAAa;AAC3D,WAAK,QAAQ,YAAA;AAAA,IACf,SAAS,OAAO;AACd,YAAM,MAAM,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;AACpE,WAAK,YAAY,EAAE,QAAQ,SAAS,OAAO,KAAK;AAChD,WAAK,QAAQ,UAAU,GAAG;AAC1B,YAAM;AAAA,IACR;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,aAA4B;AAChC,QAAI,KAAK,qBAAqB;AAC5B,mBAAa,KAAK,mBAAmB;AACrC,WAAK,sBAAsB;AAAA,IAC7B;AAGA,eAAW,SAAS,KAAK,eAAe;AACtC,YAAA;AAAA,IACF;AACA,SAAK,gBAAgB,CAAA;AAErB,QAAI,KAAK,YAAY;AACnB,YAAM,KAAK,WAAW,WAAA;AACtB,WAAK,aAAa;AAAA,IACpB;AAEA,SAAK,QAAQ;AACb,SAAK,YAAY;AAAA,MACf,QAAQ;AAAA,MACR,MAAM;AAAA,MACN,uBAAuB;AAAA,MACvB,4BAA4B;AAAA,IAAA,CAC7B;AACD,SAAK,QAAQ,eAAA;AAAA,EACf;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,iBAAuB;AACrB,QAAI,CAAC,KAAK,cAAc,KAAK,MAAM,WAAW,aAAa;AACzD;AAAA,IACF;AACA,SAAK,KAAK,WAAW,kBAAA;AACrB,SAAK,YAAY,EAAE,MAAM,YAAA,CAAa;AAAA,EACxC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,gBAAsB;AACpB,QAAI,CAAC,KAAK,YAAY;AACpB;AAAA,IACF;AACA,SAAK,WAAW,iBAAA;AAChB,SAAK,YAAY,EAAE,MAAM,OAAA,CAAQ;AAAA,EACnC;AAAA;AAAA;AAAA;AAAA,EAKA,YAAkB;AAChB,QAAI,CAAC,KAAK,YAAY;AACpB;AAAA,IACF;AACA,SAAK,WAAW,UAAA;AAAA,EAClB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,SAAS,MAAoB;AAC3B,QAAI,CAAC,KAAK,cAAc,KAAK,MAAM,WAAW,aAAa;AACzD;AAAA,IACF;AAGA,UAAM,cAA+B;AAAA,MACnC,IAAI,KAAK,WAAA;AAAA,MACT,MAAM;AAAA,MACN,WAAW,KAAK,IAAA;AAAA,MAChB,OAAO,CAAC,EAAE,MAAM,QAAQ,SAAS,MAAM;AAAA,IAAA;AAEzC,SAAK,WAAW,WAAW;AAG3B,SAAK,WAAW,SAAS,IAAI;AAAA,EAC/B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,UAAU,WAAmB,UAAwB;AACnD,QAAI,CAAC,KAAK,cAAc,KAAK,MAAM,WAAW,aAAa;AACzD;AAAA,IACF;AAGA,UAAM,cAA+B;AAAA,MACnC,IAAI,KAAK,WAAA;AAAA,MACT,MAAM;AAAA,MACN,WAAW,KAAK,IAAA;AAAA,MAChB,OAAO,CAAC,EAAE,MAAM,SAAS,MAAM,WAAW,UAAU;AAAA,IAAA;AAEtD,SAAK,WAAW,WAAW;AAG3B,SAAK,WAAW,UAAU,WAAW,QAAQ;AAAA,EAC/C;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,IAAI,SAAyB;AAC3B,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,IAAI,OAAqB;AACvB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,IAAI,WAAmC;AACrC,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,IAAI,QAAsB;AACxB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,IAAI,wBAAuC;AACzC,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,IAAI,6BAA4C;AAC9C,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,IAAI,QAAmC;AACrC,WAAO,KAAK,YAAY,sBAAA,KAA2B;AAAA,EACrD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,cAAc,UAAmD;AAC/D,SAAK,qBAAqB,IAAI,QAAQ;AACtC,WAAO,MAAM;AACX,WAAK,qBAAqB,OAAO,QAAQ;AAAA,IAC3C;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,UAAgB;AACd,SAAK,KAAK,WAAA;AACV,SAAK,qBAAqB,MAAA;AAAA,EAC5B;AAAA;AAAA;AAAA;AAAA,EAMQ,YAAY,SAA6C;AAC/D,SAAK,QAAQ,EAAE,GAAG,KAAK,OAAO,GAAG,QAAA;AAGjC,eAAW,YAAY,KAAK,sBAAsB;AAChD,eAAS,KAAK,KAAK;AAAA,IACrB;AAGA,QAAI,YAAY,WAAW,QAAQ,WAAW,QAAW;AACvD,WAAK,QAAQ,iBAAiB,QAAQ,MAAM;AAAA,IAC9C;AACA,QAAI,UAAU,WAAW,QAAQ,SAAS,QAAW;AACnD,WAAK,QAAQ,eAAe,QAAQ,IAAI;AAAA,IAC1C;AAAA,EACF;AAAA,EAEQ,WAAW,SAAgC;AACjD,SAAK,YAAY;AAAA,MACf,UAAU,CAAC,GAAG,KAAK,MAAM,UAAU,OAAO;AAAA,IAAA,CAC3C;AACD,SAAK,QAAQ,YAAY,OAAO;AAAA,EAClC;AAAA,EAEQ,uBAA6B;AACnC,QAAI,CAAC,KAAK,MAAO;AAEjB,UAAM,kBAAkB,KAAK,MAAM,YAAY,KAAK,IAAA;AACpD,UAAM,YAAY,KAAK,IAAI,GAAG,kBAAkB,uBAAuB;AAEvE,SAAK,sBAAsB,WAAW,MAAM;AAC1C,WAAK,KAAK,aAAA;AAAA,IACZ,GAAG,SAAS;AAAA,EACd;AAAA,EAEA,MAAc,eAA8B;AAC1C,QAAI;AACF,WAAK,QAAQ,MAAM,KAAK,QAAQ,SAAA;AAChC,WAAK,qBAAA;AAAA,IAGP,SAAS,OAAO;AACd,YAAM,MAAM,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;AACpE,WAAK,YAAY,EAAE,OAAO,IAAA,CAAK;AAC/B,WAAK,QAAQ,UAAU,GAAG;AAAA,IAC5B;AAAA,EACF;AAAA,EAEQ,8BAAoC;AAC1C,QAAI,CAAC,KAAK,WAAY;AAGtB,SAAK,cAAc;AAAA,MACjB,KAAK,WAAW,GAAG,iBAAiB,CAAC,EAAE,aAAa;AAClD,aAAK,YAAY,EAAE,QAAQ;AAAA,MAC7B,CAAC;AAAA,IAAA;AAIH,SAAK,cAAc;AAAA,MACjB,KAAK,WAAW,GAAG,eAAe,CAAC,EAAE,WAAW;AAC9C,aAAK,YAAY,EAAE,MAAM;AAAA,MAC3B,CAAC;AAAA,IAAA;AAMH,SAAK,cAAc;AAAA,MACjB,KAAK,WAAW,GAAG,cAAc,CAAC,EAAE,MAAM,YAAY,cAAc;AAClE,YAAI,SAAS,QAAQ;AACnB,eAAK,YAAY;AAAA,YACf,uBAAuB,UAAU,OAAO;AAAA,UAAA,CACzC;AAED,cAAI,WAAW,YAAY;AACzB,iBAAK,WAAW;AAAA,cACd,IAAI,KAAK,WAAA;AAAA,cACT,MAAM;AAAA,cACN,WAAW,KAAK,IAAA;AAAA,cAChB,OAAO,CAAC,EAAE,MAAM,SAAS,YAAY,YAAY,GAAG;AAAA,YAAA,CACrD;AAAA,UACH;AAAA,QACF,OAAO;AAEL,eAAK,YAAY;AAAA,YACf,4BAA4B,UAAU,OAAO;AAAA,UAAA,CAC9C;AAAA,QACH;AAAA,MACF,CAAC;AAAA,IAAA;AAIH,SAAK,cAAc;AAAA,MACjB,KAAK,WAAW,GAAG,aAAa,CAAC,EAAE,YAAY,UAAU,YAAY;AACnE,YAAI,CAAC,YAAY;AACf,kBAAQ;AAAA,YACN;AAAA,YACA;AAAA,UAAA;AAEF;AAAA,QACF;AACA,cAAM,OAAO,KAAK,YAAY,IAAI,QAAQ;AAC1C,cAAM,UAAU,MAAM;AACtB,YAAI,SAAS;AACX,gBAAM,YAAY;AAChB,gBAAI;AACF,oBAAM,SAAS,MAAM,QAAQ,KAAK;AAClC,mBAAK,YAAY;AAAA,gBACf;AAAA,gBACA,OAAO,WAAW,WAAW,SAAS,KAAK,UAAU,MAAM;AAAA,cAAA;AAAA,YAE/D,SAAS,OAAO;AACd,oBAAM,SACJ,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AACvD,mBAAK,YAAY;AAAA,gBACf;AAAA,gBACA,KAAK,UAAU,EAAE,OAAO,QAAQ;AAAA,cAAA;AAAA,YAEpC;AAAA,UACF,GAAA;AAAA,QACF,OAAO;AACL,eAAK,YAAY;AAAA,YACf;AAAA,YACA,KAAK,UAAU;AAAA,cACb,OAAO,mCAAmC,OAAO,QAAQ,CAAC;AAAA,YAAA,CAC3D;AAAA,UAAA;AAAA,QAEL;AAAA,MACF,CAAC;AAAA,IAAA;AAIH,SAAK,cAAc;AAAA,MACjB,KAAK,WAAW,GAAG,oBAAoB,CAAC,EAAE,cAAc;AAEtD,cAAM,gBAAgB,KAAK,MAAM,SAAS;AAAA,UACxC,CAAC,MAAM,EAAE,OAAO,QAAQ;AAAA,QAAA;AAE1B,YAAI,iBAAiB,GAAG;AACtB,gBAAM,cAAc,CAAC,GAAG,KAAK,MAAM,QAAQ;AAC3C,sBAAY,aAAa,IAAI;AAC7B,eAAK,YAAY,EAAE,UAAU,YAAA,CAAa;AAAA,QAC5C,OAAO;AACL,eAAK,WAAW,OAAO;AAAA,QACzB;AAAA,MACF,CAAC;AAAA,IAAA;AAIH,SAAK,cAAc;AAAA,MACjB,KAAK,WAAW,GAAG,eAAe,CAAC,EAAE,gBAAgB;AACnD,YAAI,WAAW;AACb,gBAAM,cAAc,KAAK,MAAM,SAAS;AAAA,YAAI,CAAC,MAC3C,EAAE,OAAO,YAAY,EAAE,GAAG,GAAG,aAAa,SAAS;AAAA,UAAA;AAErD,eAAK,YAAY,EAAE,UAAU,YAAA,CAAa;AAAA,QAC5C;AACA,aAAK,YAAY;AAAA,UACf,MAAM;AAAA,UACN,4BAA4B;AAAA,QAAA,CAC7B;AACD,aAAK,QAAQ,gBAAA;AAAA,MACf,CAAC;AAAA,IAAA;AAIH,SAAK,cAAc;AAAA,MACjB,KAAK,WAAW,GAAG,SAAS,CAAC,EAAE,YAAY;AACzC,aAAK,YAAY,EAAE,OAAO;AAC1B,aAAK,QAAQ,UAAU,KAAK;AAAA,MAC9B,CAAC;AAAA,IAAA;AAAA,EAEL;AAAA,EAEQ,qBAA2B;AACjC,QAAI,CAAC,KAAK,WAAY;AAEtB,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA,IACE,KAAK;AACT,UAAM,YACJ,gBACA,SACA,WACC,SAAS,MAAM,SAAS,KACzB,oBACA,gBAAgB,UAChB,oBAAoB,UACpB;AACF,QAAI,CAAC,UAAW;AAKhB,UAAM,cAAc,QAChB,MAAM,KAAK,KAAK,YAAY,OAAA,CAAQ,EAAE,IAAI,CAAC,MAAM;AAC/C,YAAM,cAAc,EAAE,cAClB,0BAA0B,EAAE,WAAW,IACvC;AACJ,aAAO;AAAA,QACL,MAAM,EAAE;AAAA,QACR,aAAa,EAAE;AAAA,QACf,GAAI,cAAc,EAAE,gBAAgB,CAAA;AAAA,MAAC;AAAA,IAEzC,CAAC,IACD;AAKJ,SAAK,WAAW,cAAc;AAAA,MAC5B,GAAI,iBAAiB,SAAY,EAAE,aAAA,IAAiB,CAAA;AAAA,MACpD,GAAI,UAAU,SAAY,EAAE,MAAA,IAAU,CAAA;AAAA,MACtC,GAAI,YAAY,SAAY,EAAE,QAAA,IAAY,CAAA;AAAA,MAC1C,GAAI,gBAAgB,SAAY,EAAE,OAAO,YAAA,IAAgB,CAAA;AAAA,MACzD,GAAI,qBAAqB,SAAY,EAAE,iBAAA,IAAqB,CAAA;AAAA,MAC5D,GAAI,gBAAgB,SAAY,EAAE,YAAA,IAAgB,CAAA;AAAA,MAClD,GAAI,oBAAoB,SAAY,EAAE,gBAAA,IAAoB,CAAA;AAAA,MAC1D,GAAI,sBAAsB,SAAY,EAAE,sBAAsB,CAAA;AAAA,IAAC,CAChE;AAAA,EACH;AAAA,EAEQ,aAAqB;AAC3B,WAAO,OAAO,KAAK,IAAA,CAAK,IAAI,KAAK,OAAA,EAAS,SAAS,EAAE,EAAE,UAAU,CAAC,CAAC;AAAA,EACrE;AACF;"}
|
|
@@ -1,78 +0,0 @@
|
|
|
1
|
-
import { AnyClientTool, RealtimeAdapter, RealtimeMessage, RealtimeMode, RealtimeStatus, RealtimeToken } from '@tanstack/ai/client';
|
|
2
|
-
export type { RealtimeAdapter, RealtimeConnection } from '@tanstack/ai/client';
|
|
3
|
-
/**
|
|
4
|
-
* Options for the RealtimeClient
|
|
5
|
-
*/
|
|
6
|
-
export interface RealtimeClientOptions {
|
|
7
|
-
/**
|
|
8
|
-
* Function to fetch a realtime token from the server.
|
|
9
|
-
* Called on connect and when token needs refresh.
|
|
10
|
-
*/
|
|
11
|
-
getToken: () => Promise<RealtimeToken>;
|
|
12
|
-
/**
|
|
13
|
-
* The realtime adapter to use (e.g., openaiRealtime())
|
|
14
|
-
*/
|
|
15
|
-
adapter: RealtimeAdapter;
|
|
16
|
-
/**
|
|
17
|
-
* Client-side tools with execution logic
|
|
18
|
-
*/
|
|
19
|
-
tools?: ReadonlyArray<AnyClientTool>;
|
|
20
|
-
/**
|
|
21
|
-
* Auto-play assistant audio (default: true)
|
|
22
|
-
*/
|
|
23
|
-
autoPlayback?: boolean;
|
|
24
|
-
/**
|
|
25
|
-
* Request microphone access on connect (default: true)
|
|
26
|
-
*/
|
|
27
|
-
autoCapture?: boolean;
|
|
28
|
-
/**
|
|
29
|
-
* System instructions for the assistant
|
|
30
|
-
*/
|
|
31
|
-
instructions?: string;
|
|
32
|
-
/**
|
|
33
|
-
* Voice to use for audio output
|
|
34
|
-
*/
|
|
35
|
-
voice?: string;
|
|
36
|
-
/**
|
|
37
|
-
* Voice activity detection mode (default: 'server')
|
|
38
|
-
*/
|
|
39
|
-
vadMode?: 'server' | 'semantic' | 'manual';
|
|
40
|
-
/**
|
|
41
|
-
* Output modalities for responses (e.g., ['audio', 'text'])
|
|
42
|
-
*/
|
|
43
|
-
outputModalities?: Array<'audio' | 'text'>;
|
|
44
|
-
/**
|
|
45
|
-
* Temperature for generation (provider-specific range)
|
|
46
|
-
*/
|
|
47
|
-
temperature?: number;
|
|
48
|
-
/**
|
|
49
|
-
* Maximum number of tokens in a response
|
|
50
|
-
*/
|
|
51
|
-
maxOutputTokens?: number | 'inf';
|
|
52
|
-
/**
|
|
53
|
-
* Eagerness level for semantic VAD ('low', 'medium', 'high')
|
|
54
|
-
*/
|
|
55
|
-
semanticEagerness?: 'low' | 'medium' | 'high';
|
|
56
|
-
onStatusChange?: (status: RealtimeStatus) => void;
|
|
57
|
-
onModeChange?: (mode: RealtimeMode) => void;
|
|
58
|
-
onMessage?: (message: RealtimeMessage) => void;
|
|
59
|
-
onError?: (error: Error) => void;
|
|
60
|
-
onConnect?: () => void;
|
|
61
|
-
onDisconnect?: () => void;
|
|
62
|
-
onInterrupted?: () => void;
|
|
63
|
-
}
|
|
64
|
-
/**
|
|
65
|
-
* Internal state of the RealtimeClient
|
|
66
|
-
*/
|
|
67
|
-
export interface RealtimeClientState {
|
|
68
|
-
status: RealtimeStatus;
|
|
69
|
-
mode: RealtimeMode;
|
|
70
|
-
messages: Array<RealtimeMessage>;
|
|
71
|
-
pendingUserTranscript: string | null;
|
|
72
|
-
pendingAssistantTranscript: string | null;
|
|
73
|
-
error: Error | null;
|
|
74
|
-
}
|
|
75
|
-
/**
|
|
76
|
-
* Callback type for state changes
|
|
77
|
-
*/
|
|
78
|
-
export type RealtimeStateChangeCallback = (state: RealtimeClientState) => void;
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
export type MissingResponseStreamFeature = 'TextDecoder' | 'Response.body' | 'Response.body.getReader';
|
|
2
|
-
export declare class UnsupportedResponseStreamError extends Error {
|
|
3
|
-
readonly missingFeature: MissingResponseStreamFeature;
|
|
4
|
-
constructor(missingFeature: MissingResponseStreamFeature);
|
|
5
|
-
}
|
|
6
|
-
export declare function getResponseStreamReader(response: Response): ReadableStreamDefaultReader<Uint8Array>;
|
|
7
|
-
export declare function createResponseStreamTextDecoder(): TextDecoder;
|
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
class UnsupportedResponseStreamError extends Error {
|
|
2
|
-
missingFeature;
|
|
3
|
-
constructor(missingFeature) {
|
|
4
|
-
super(createUnsupportedResponseStreamMessage(missingFeature));
|
|
5
|
-
this.name = "UnsupportedResponseStreamError";
|
|
6
|
-
this.missingFeature = missingFeature;
|
|
7
|
-
}
|
|
8
|
-
}
|
|
9
|
-
function createUnsupportedResponseStreamMessage(missingFeature) {
|
|
10
|
-
return `Streaming fetch responses are not supported in this runtime because ${missingFeature} is unavailable. React Native users need a compatible fetch/stream/TextDecoder polyfill, or should use a fetch, XHR, or custom transport that can deliver streaming chunks.`;
|
|
11
|
-
}
|
|
12
|
-
function getResponseStreamReader(response) {
|
|
13
|
-
if (!response.body) {
|
|
14
|
-
throw new UnsupportedResponseStreamError("Response.body");
|
|
15
|
-
}
|
|
16
|
-
if (typeof response.body.getReader !== "function") {
|
|
17
|
-
throw new UnsupportedResponseStreamError("Response.body.getReader");
|
|
18
|
-
}
|
|
19
|
-
return response.body.getReader();
|
|
20
|
-
}
|
|
21
|
-
function createResponseStreamTextDecoder() {
|
|
22
|
-
if (typeof globalThis.TextDecoder !== "function") {
|
|
23
|
-
throw new UnsupportedResponseStreamError("TextDecoder");
|
|
24
|
-
}
|
|
25
|
-
return new globalThis.TextDecoder();
|
|
26
|
-
}
|
|
27
|
-
export {
|
|
28
|
-
UnsupportedResponseStreamError,
|
|
29
|
-
createResponseStreamTextDecoder,
|
|
30
|
-
getResponseStreamReader
|
|
31
|
-
};
|
|
32
|
-
//# sourceMappingURL=response-stream.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"response-stream.js","sources":["../../src/response-stream.ts"],"sourcesContent":["export type MissingResponseStreamFeature =\n | 'TextDecoder'\n | 'Response.body'\n | 'Response.body.getReader'\n\nexport class UnsupportedResponseStreamError extends Error {\n readonly missingFeature: MissingResponseStreamFeature\n\n constructor(missingFeature: MissingResponseStreamFeature) {\n super(createUnsupportedResponseStreamMessage(missingFeature))\n this.name = 'UnsupportedResponseStreamError'\n this.missingFeature = missingFeature\n }\n}\n\nfunction createUnsupportedResponseStreamMessage(\n missingFeature: MissingResponseStreamFeature,\n): string {\n return `Streaming fetch responses are not supported in this runtime because ${missingFeature} is unavailable. React Native users need a compatible fetch/stream/TextDecoder polyfill, or should use a fetch, XHR, or custom transport that can deliver streaming chunks.`\n}\n\nexport function getResponseStreamReader(\n response: Response,\n): ReadableStreamDefaultReader<Uint8Array> {\n if (!response.body) {\n throw new UnsupportedResponseStreamError('Response.body')\n }\n\n if (typeof response.body.getReader !== 'function') {\n throw new UnsupportedResponseStreamError('Response.body.getReader')\n }\n\n return response.body.getReader()\n}\n\nexport function createResponseStreamTextDecoder(): TextDecoder {\n if (typeof globalThis.TextDecoder !== 'function') {\n throw new UnsupportedResponseStreamError('TextDecoder')\n }\n\n return new globalThis.TextDecoder()\n}\n"],"names":[],"mappings":"AAKO,MAAM,uCAAuC,MAAM;AAAA,EAC/C;AAAA,EAET,YAAY,gBAA8C;AACxD,UAAM,uCAAuC,cAAc,CAAC;AAC5D,SAAK,OAAO;AACZ,SAAK,iBAAiB;AAAA,EACxB;AACF;AAEA,SAAS,uCACP,gBACQ;AACR,SAAO,uEAAuE,cAAc;AAC9F;AAEO,SAAS,wBACd,UACyC;AACzC,MAAI,CAAC,SAAS,MAAM;AAClB,UAAM,IAAI,+BAA+B,eAAe;AAAA,EAC1D;AAEA,MAAI,OAAO,SAAS,KAAK,cAAc,YAAY;AACjD,UAAM,IAAI,+BAA+B,yBAAyB;AAAA,EACpE;AAEA,SAAO,SAAS,KAAK,UAAA;AACvB;AAEO,SAAS,kCAA+C;AAC7D,MAAI,OAAO,WAAW,gBAAgB,YAAY;AAChD,UAAM,IAAI,+BAA+B,aAAa;AAAA,EACxD;AAEA,SAAO,IAAI,WAAW,YAAA;AACxB;"}
|
package/dist/esm/sse-parser.d.ts
DELETED
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import { StreamChunk } from '@tanstack/ai/client';
|
|
2
|
-
/**
|
|
3
|
-
* Parse a Response body as Server-Sent Events, yielding StreamChunks.
|
|
4
|
-
*
|
|
5
|
-
* Used by GenerationClient to parse SSE Responses returned from fetchers
|
|
6
|
-
* (e.g., TanStack Start server functions using `toServerSentEventsResponse()`).
|
|
7
|
-
*/
|
|
8
|
-
export declare function parseSSEResponse(response: Response, abortSignal?: AbortSignal): AsyncGenerator<StreamChunk>;
|
package/dist/esm/sse-parser.js
DELETED
|
@@ -1,52 +0,0 @@
|
|
|
1
|
-
import { getResponseStreamReader, createResponseStreamTextDecoder } from "./response-stream.js";
|
|
2
|
-
import { parseSseDataLine } from "./sse-utils.js";
|
|
3
|
-
async function* readStreamLines(reader, abortSignal) {
|
|
4
|
-
try {
|
|
5
|
-
const decoder = createResponseStreamTextDecoder();
|
|
6
|
-
let buffer = "";
|
|
7
|
-
while (!abortSignal?.aborted) {
|
|
8
|
-
const { done, value } = await reader.read();
|
|
9
|
-
if (done) break;
|
|
10
|
-
buffer += decoder.decode(value, { stream: true });
|
|
11
|
-
const lines = buffer.split("\n");
|
|
12
|
-
buffer = lines.pop() || "";
|
|
13
|
-
for (const line of lines) {
|
|
14
|
-
if (line.trim()) {
|
|
15
|
-
yield line;
|
|
16
|
-
}
|
|
17
|
-
}
|
|
18
|
-
}
|
|
19
|
-
if (buffer.trim()) {
|
|
20
|
-
yield buffer;
|
|
21
|
-
}
|
|
22
|
-
} finally {
|
|
23
|
-
reader.releaseLock();
|
|
24
|
-
}
|
|
25
|
-
}
|
|
26
|
-
async function* parseSSEResponse(response, abortSignal) {
|
|
27
|
-
if (!response.ok) {
|
|
28
|
-
throw new Error(
|
|
29
|
-
`HTTP error! status: ${response.status} ${response.statusText}`
|
|
30
|
-
);
|
|
31
|
-
}
|
|
32
|
-
const reader = getResponseStreamReader(response);
|
|
33
|
-
for await (const line of readStreamLines(reader, abortSignal)) {
|
|
34
|
-
const data = parseSseDataLine(line);
|
|
35
|
-
if (data === "[DONE]") {
|
|
36
|
-
console.warn(
|
|
37
|
-
"[@tanstack/ai-client] Received [DONE] sentinel. This is deprecated — upgrade your @tanstack/ai server package. RUN_FINISHED is the stream terminator."
|
|
38
|
-
);
|
|
39
|
-
continue;
|
|
40
|
-
}
|
|
41
|
-
try {
|
|
42
|
-
const parsed = JSON.parse(data);
|
|
43
|
-
yield parsed;
|
|
44
|
-
} catch (parseError) {
|
|
45
|
-
console.warn("Failed to parse SSE chunk:", data);
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
export {
|
|
50
|
-
parseSSEResponse
|
|
51
|
-
};
|
|
52
|
-
//# sourceMappingURL=sse-parser.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"sse-parser.js","sources":["../../src/sse-parser.ts"],"sourcesContent":["import {\n createResponseStreamTextDecoder,\n getResponseStreamReader,\n} from './response-stream'\nimport { parseSseDataLine } from './sse-utils'\nimport type { StreamChunk } from '@tanstack/ai/client'\n\n/**\n * Read lines from a stream (newline-delimited)\n */\nasync function* readStreamLines(\n reader: ReadableStreamDefaultReader<Uint8Array>,\n abortSignal?: AbortSignal,\n): AsyncGenerator<string> {\n try {\n const decoder = createResponseStreamTextDecoder()\n let buffer = ''\n\n while (!abortSignal?.aborted) {\n const { done, value } = await reader.read()\n if (done) break\n\n buffer += decoder.decode(value, { stream: true })\n const lines = buffer.split('\\n')\n\n buffer = lines.pop() || ''\n\n for (const line of lines) {\n if (line.trim()) {\n yield line\n }\n }\n }\n\n if (buffer.trim()) {\n yield buffer\n }\n } finally {\n reader.releaseLock()\n }\n}\n\n/**\n * Parse a Response body as Server-Sent Events, yielding StreamChunks.\n *\n * Used by GenerationClient to parse SSE Responses returned from fetchers\n * (e.g., TanStack Start server functions using `toServerSentEventsResponse()`).\n */\nexport async function* parseSSEResponse(\n response: Response,\n abortSignal?: AbortSignal,\n): AsyncGenerator<StreamChunk> {\n if (!response.ok) {\n throw new Error(\n `HTTP error! status: ${response.status} ${response.statusText}`,\n )\n }\n\n const reader = getResponseStreamReader(response)\n\n for await (const line of readStreamLines(reader, abortSignal)) {\n const data = parseSseDataLine(line)\n\n if (data === '[DONE]') {\n console.warn(\n '[@tanstack/ai-client] Received [DONE] sentinel. This is deprecated — upgrade your @tanstack/ai server package. RUN_FINISHED is the stream terminator.',\n )\n continue\n }\n\n try {\n const parsed: StreamChunk = JSON.parse(data)\n yield parsed\n } catch (parseError) {\n console.warn('Failed to parse SSE chunk:', data)\n }\n }\n}\n"],"names":[],"mappings":";;AAUA,gBAAgB,gBACd,QACA,aACwB;AACxB,MAAI;AACF,UAAM,UAAU,gCAAA;AAChB,QAAI,SAAS;AAEb,WAAO,CAAC,aAAa,SAAS;AAC5B,YAAM,EAAE,MAAM,MAAA,IAAU,MAAM,OAAO,KAAA;AACrC,UAAI,KAAM;AAEV,gBAAU,QAAQ,OAAO,OAAO,EAAE,QAAQ,MAAM;AAChD,YAAM,QAAQ,OAAO,MAAM,IAAI;AAE/B,eAAS,MAAM,SAAS;AAExB,iBAAW,QAAQ,OAAO;AACxB,YAAI,KAAK,QAAQ;AACf,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,QAAI,OAAO,QAAQ;AACjB,YAAM;AAAA,IACR;AAAA,EACF,UAAA;AACE,WAAO,YAAA;AAAA,EACT;AACF;AAQA,gBAAuB,iBACrB,UACA,aAC6B;AAC7B,MAAI,CAAC,SAAS,IAAI;AAChB,UAAM,IAAI;AAAA,MACR,uBAAuB,SAAS,MAAM,IAAI,SAAS,UAAU;AAAA,IAAA;AAAA,EAEjE;AAEA,QAAM,SAAS,wBAAwB,QAAQ;AAE/C,mBAAiB,QAAQ,gBAAgB,QAAQ,WAAW,GAAG;AAC7D,UAAM,OAAO,iBAAiB,IAAI;AAElC,QAAI,SAAS,UAAU;AACrB,cAAQ;AAAA,QACN;AAAA,MAAA;AAEF;AAAA,IACF;AAEA,QAAI;AACF,YAAM,SAAsB,KAAK,MAAM,IAAI;AAC3C,YAAM;AAAA,IACR,SAAS,YAAY;AACnB,cAAQ,KAAK,8BAA8B,IAAI;AAAA,IACjD;AAAA,EACF;AACF;"}
|
package/dist/esm/sse-utils.d.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export declare function parseSseDataLine(line: string): string;
|
package/dist/esm/sse-utils.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"sse-utils.js","sources":["../../src/sse-utils.ts"],"sourcesContent":["export function parseSseDataLine(line: string): string {\n if (!line.startsWith('data:')) {\n return line\n }\n\n const data = line.slice(5)\n return data.startsWith(' ') ? data.slice(1) : data\n}\n"],"names":[],"mappings":"AAAO,SAAS,iBAAiB,MAAsB;AACrD,MAAI,CAAC,KAAK,WAAW,OAAO,GAAG;AAC7B,WAAO;AAAA,EACT;AAEA,QAAM,OAAO,KAAK,MAAM,CAAC;AACzB,SAAO,KAAK,WAAW,GAAG,IAAI,KAAK,MAAM,CAAC,IAAI;AAChD;"}
|