@tanstack/ai-client 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -1
- package/dist/esm/audio-recorder.js +190 -213
- package/dist/esm/audio-recorder.js.map +1 -1
- package/dist/esm/chat-client.d.ts +172 -3
- package/dist/esm/chat-client.js +1656 -1386
- package/dist/esm/chat-client.js.map +1 -1
- package/dist/esm/cleared-stream-tracker.d.ts +23 -0
- package/dist/esm/cleared-stream-tracker.js +97 -0
- package/dist/esm/cleared-stream-tracker.js.map +1 -0
- package/dist/esm/client-persistor.d.ts +25 -12
- package/dist/esm/client-persistor.js +260 -235
- package/dist/esm/client-persistor.js.map +1 -1
- package/dist/esm/connection-adapters.d.ts +231 -10
- package/dist/esm/connection-adapters.js +989 -574
- package/dist/esm/connection-adapters.js.map +1 -1
- package/dist/esm/devtools-noop.d.ts +1 -0
- package/dist/esm/devtools-noop.js +79 -139
- package/dist/esm/devtools-noop.js.map +1 -1
- package/dist/esm/devtools.d.ts +31 -1
- package/dist/esm/devtools.js +977 -1127
- package/dist/esm/devtools.js.map +1 -1
- package/dist/esm/events.js +224 -226
- package/dist/esm/events.js.map +1 -1
- package/dist/esm/generation-client.d.ts +145 -2
- package/dist/esm/generation-client.js +659 -321
- package/dist/esm/generation-client.js.map +1 -1
- package/dist/esm/generation-reconstruct.d.ts +21 -0
- package/dist/esm/generation-reconstruct.js +85 -0
- package/dist/esm/generation-reconstruct.js.map +1 -0
- package/dist/esm/generation-types.d.ts +289 -3
- package/dist/esm/generation-types.js +356 -13
- package/dist/esm/generation-types.js.map +1 -1
- package/dist/esm/index.d.ts +9 -4
- package/dist/esm/index.js +7 -39
- package/dist/esm/interrupt-manager.d.ts +77 -0
- package/dist/esm/interrupt-manager.js +787 -0
- package/dist/esm/interrupt-manager.js.map +1 -0
- package/dist/esm/mcp-app-bridge.js +56 -64
- package/dist/esm/mcp-app-bridge.js.map +1 -1
- package/dist/esm/realtime-client.js +366 -440
- package/dist/esm/realtime-client.js.map +1 -1
- package/dist/esm/response-stream.js +19 -26
- package/dist/esm/response-stream.js.map +1 -1
- package/dist/esm/sse-parser.js +44 -47
- package/dist/esm/sse-parser.js.map +1 -1
- package/dist/esm/sse-utils.js +8 -9
- package/dist/esm/sse-utils.js.map +1 -1
- package/dist/esm/storage-adapters.d.ts +62 -0
- package/dist/esm/storage-adapters.js +174 -0
- package/dist/esm/storage-adapters.js.map +1 -0
- package/dist/esm/types.d.ts +212 -10
- package/dist/esm/types.js +38 -7
- package/dist/esm/types.js.map +1 -1
- package/dist/esm/video-generation-client.d.ts +113 -2
- package/dist/esm/video-generation-client.js +665 -379
- package/dist/esm/video-generation-client.js.map +1 -1
- package/package.json +7 -7
- package/src/chat-client.ts +1079 -61
- package/src/cleared-stream-tracker.ts +151 -0
- package/src/client-persistor.ts +102 -33
- package/src/connection-adapters.ts +1185 -142
- package/src/devtools-noop.ts +4 -3
- package/src/devtools.ts +121 -3
- package/src/generation-client.ts +563 -13
- package/src/generation-reconstruct.ts +121 -0
- package/src/generation-types.ts +727 -3
- package/src/index.ts +56 -1
- package/src/interrupt-manager.ts +1440 -0
- package/src/storage-adapters.ts +242 -0
- package/src/types.ts +301 -9
- package/src/video-generation-client.ts +479 -13
- package/dist/esm/index.js.map +0 -1
|
@@ -1,442 +1,368 @@
|
|
|
1
1
|
import { convertSchemaToJsonSchema } from "@tanstack/ai/client";
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
this.addMessage(message);
|
|
365
|
-
}
|
|
366
|
-
})
|
|
367
|
-
);
|
|
368
|
-
this.unsubscribers.push(
|
|
369
|
-
this.connection.on("interrupted", ({ messageId }) => {
|
|
370
|
-
if (messageId) {
|
|
371
|
-
const newMessages = this.state.messages.map(
|
|
372
|
-
(m) => m.id === messageId ? { ...m, interrupted: true } : m
|
|
373
|
-
);
|
|
374
|
-
this.updateState({ messages: newMessages });
|
|
375
|
-
}
|
|
376
|
-
this.updateState({
|
|
377
|
-
mode: "listening",
|
|
378
|
-
pendingAssistantTranscript: null
|
|
379
|
-
});
|
|
380
|
-
this.options.onInterrupted?.();
|
|
381
|
-
})
|
|
382
|
-
);
|
|
383
|
-
this.unsubscribers.push(
|
|
384
|
-
this.connection.on("error", ({ error }) => {
|
|
385
|
-
this.updateState({ error });
|
|
386
|
-
this.options.onError?.(error);
|
|
387
|
-
})
|
|
388
|
-
);
|
|
389
|
-
this.unsubscribers.push(
|
|
390
|
-
this.connection.on("go_away", ({ timeLeft }) => {
|
|
391
|
-
this.options.onGoAway?.(timeLeft);
|
|
392
|
-
})
|
|
393
|
-
);
|
|
394
|
-
this.unsubscribers.push(
|
|
395
|
-
this.connection.on("usage", (usage) => {
|
|
396
|
-
this.options.onUsage?.(usage);
|
|
397
|
-
})
|
|
398
|
-
);
|
|
399
|
-
}
|
|
400
|
-
applySessionConfig() {
|
|
401
|
-
if (!this.connection) return;
|
|
402
|
-
const {
|
|
403
|
-
instructions,
|
|
404
|
-
voice,
|
|
405
|
-
vadMode,
|
|
406
|
-
tools,
|
|
407
|
-
outputModalities,
|
|
408
|
-
temperature,
|
|
409
|
-
maxOutputTokens,
|
|
410
|
-
semanticEagerness
|
|
411
|
-
} = this.options;
|
|
412
|
-
const hasConfig = instructions || voice || vadMode || tools && tools.length > 0 || outputModalities || temperature !== void 0 || maxOutputTokens !== void 0 || semanticEagerness;
|
|
413
|
-
if (!hasConfig) return;
|
|
414
|
-
const toolsConfig = tools ? Array.from(this.clientTools.values()).map((t) => {
|
|
415
|
-
const inputSchema = t.inputSchema ? convertSchemaToJsonSchema(t.inputSchema) : void 0;
|
|
416
|
-
const outputSchema = t.outputSchema ? convertSchemaToJsonSchema(t.outputSchema) : void 0;
|
|
417
|
-
return {
|
|
418
|
-
name: t.name,
|
|
419
|
-
description: t.description,
|
|
420
|
-
...inputSchema ? { inputSchema } : {},
|
|
421
|
-
...outputSchema ? { outputSchema } : {}
|
|
422
|
-
};
|
|
423
|
-
}) : void 0;
|
|
424
|
-
this.connection.updateSession({
|
|
425
|
-
...instructions !== void 0 ? { instructions } : {},
|
|
426
|
-
...voice !== void 0 ? { voice } : {},
|
|
427
|
-
...vadMode !== void 0 ? { vadMode } : {},
|
|
428
|
-
...toolsConfig !== void 0 ? { tools: toolsConfig } : {},
|
|
429
|
-
...outputModalities !== void 0 ? { outputModalities } : {},
|
|
430
|
-
...temperature !== void 0 ? { temperature } : {},
|
|
431
|
-
...maxOutputTokens !== void 0 ? { maxOutputTokens } : {},
|
|
432
|
-
...semanticEagerness !== void 0 ? { semanticEagerness } : {}
|
|
433
|
-
});
|
|
434
|
-
}
|
|
435
|
-
generateId() {
|
|
436
|
-
return `msg-${Date.now()}-${Math.random().toString(36).substring(7)}`;
|
|
437
|
-
}
|
|
438
|
-
}
|
|
439
|
-
export {
|
|
440
|
-
RealtimeClient
|
|
2
|
+
//#region src/realtime-client.ts
|
|
3
|
+
var TOKEN_REFRESH_BUFFER_MS = 6e4;
|
|
4
|
+
/**
|
|
5
|
+
* Client for managing realtime voice conversations.
|
|
6
|
+
*
|
|
7
|
+
* Handles connection lifecycle, audio I/O, message state,
|
|
8
|
+
* and tool execution for realtime voice-to-voice AI interactions.
|
|
9
|
+
*
|
|
10
|
+
* @example
|
|
11
|
+
* ```typescript
|
|
12
|
+
* import { RealtimeClient } from '@tanstack/ai-client'
|
|
13
|
+
* import { openaiRealtime } from '@tanstack/ai-openai'
|
|
14
|
+
*
|
|
15
|
+
* const client = new RealtimeClient({
|
|
16
|
+
* getToken: () => fetch('/api/realtime-token').then(r => r.json()),
|
|
17
|
+
* adapter: openaiRealtime(),
|
|
18
|
+
* tools: [myTool.client(handler)],
|
|
19
|
+
* onMessage: (msg) => console.log('Message:', msg),
|
|
20
|
+
* })
|
|
21
|
+
*
|
|
22
|
+
* await client.connect()
|
|
23
|
+
* ```
|
|
24
|
+
*/
|
|
25
|
+
var RealtimeClient = class {
|
|
26
|
+
options;
|
|
27
|
+
connection = null;
|
|
28
|
+
token = null;
|
|
29
|
+
tokenRefreshTimeout = null;
|
|
30
|
+
clientTools;
|
|
31
|
+
stateChangeCallbacks = /* @__PURE__ */ new Set();
|
|
32
|
+
unsubscribers = [];
|
|
33
|
+
state = {
|
|
34
|
+
status: "idle",
|
|
35
|
+
mode: "idle",
|
|
36
|
+
messages: [],
|
|
37
|
+
pendingUserTranscript: null,
|
|
38
|
+
pendingAssistantTranscript: null,
|
|
39
|
+
error: null
|
|
40
|
+
};
|
|
41
|
+
constructor(options) {
|
|
42
|
+
this.options = {
|
|
43
|
+
autoPlayback: true,
|
|
44
|
+
autoCapture: true,
|
|
45
|
+
vadMode: "server",
|
|
46
|
+
...options
|
|
47
|
+
};
|
|
48
|
+
this.clientTools = /* @__PURE__ */ new Map();
|
|
49
|
+
if (options.tools) for (const tool of options.tools) this.clientTools.set(tool.name, tool);
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Connect to the realtime session.
|
|
53
|
+
* Fetches a token and establishes the connection.
|
|
54
|
+
*/
|
|
55
|
+
async connect() {
|
|
56
|
+
if (this.state.status === "connected") return;
|
|
57
|
+
this.updateState({
|
|
58
|
+
status: "connecting",
|
|
59
|
+
error: null
|
|
60
|
+
});
|
|
61
|
+
try {
|
|
62
|
+
this.token = await this.options.getToken();
|
|
63
|
+
this.scheduleTokenRefresh();
|
|
64
|
+
const toolsList = this.clientTools.size > 0 ? Array.from(this.clientTools.values()) : void 0;
|
|
65
|
+
this.connection = await this.options.adapter.connect(this.token, toolsList);
|
|
66
|
+
this.subscribeToConnectionEvents();
|
|
67
|
+
this.applySessionConfig();
|
|
68
|
+
if (this.options.autoCapture) await this.connection.startAudioCapture();
|
|
69
|
+
this.updateState({
|
|
70
|
+
status: "connected",
|
|
71
|
+
mode: "listening"
|
|
72
|
+
});
|
|
73
|
+
this.options.onConnect?.();
|
|
74
|
+
} catch (error) {
|
|
75
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
76
|
+
this.updateState({
|
|
77
|
+
status: "error",
|
|
78
|
+
error: err
|
|
79
|
+
});
|
|
80
|
+
this.options.onError?.(err);
|
|
81
|
+
throw err;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Disconnect from the realtime session.
|
|
86
|
+
*/
|
|
87
|
+
async disconnect() {
|
|
88
|
+
if (this.tokenRefreshTimeout) {
|
|
89
|
+
clearTimeout(this.tokenRefreshTimeout);
|
|
90
|
+
this.tokenRefreshTimeout = null;
|
|
91
|
+
}
|
|
92
|
+
for (const unsub of this.unsubscribers) unsub();
|
|
93
|
+
this.unsubscribers = [];
|
|
94
|
+
if (this.connection) {
|
|
95
|
+
await this.connection.disconnect();
|
|
96
|
+
this.connection = null;
|
|
97
|
+
}
|
|
98
|
+
this.token = null;
|
|
99
|
+
this.updateState({
|
|
100
|
+
status: "idle",
|
|
101
|
+
mode: "idle",
|
|
102
|
+
pendingUserTranscript: null,
|
|
103
|
+
pendingAssistantTranscript: null
|
|
104
|
+
});
|
|
105
|
+
this.options.onDisconnect?.();
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Start listening for voice input.
|
|
109
|
+
* Only needed when vadMode is 'manual'.
|
|
110
|
+
*/
|
|
111
|
+
startListening() {
|
|
112
|
+
if (!this.connection || this.state.status !== "connected") return;
|
|
113
|
+
this.connection.startAudioCapture();
|
|
114
|
+
this.updateState({ mode: "listening" });
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Stop listening for voice input.
|
|
118
|
+
* Only needed when vadMode is 'manual'.
|
|
119
|
+
*/
|
|
120
|
+
stopListening() {
|
|
121
|
+
if (!this.connection) return;
|
|
122
|
+
this.connection.stopAudioCapture();
|
|
123
|
+
this.updateState({ mode: "idle" });
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Interrupt the current assistant response.
|
|
127
|
+
*/
|
|
128
|
+
interrupt() {
|
|
129
|
+
if (!this.connection) return;
|
|
130
|
+
this.connection.interrupt();
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Send a text message instead of voice.
|
|
134
|
+
*/
|
|
135
|
+
sendText(text) {
|
|
136
|
+
if (!this.connection || this.state.status !== "connected") return;
|
|
137
|
+
const userMessage = {
|
|
138
|
+
id: this.generateId(),
|
|
139
|
+
role: "user",
|
|
140
|
+
timestamp: Date.now(),
|
|
141
|
+
parts: [{
|
|
142
|
+
type: "text",
|
|
143
|
+
content: text
|
|
144
|
+
}]
|
|
145
|
+
};
|
|
146
|
+
this.addMessage(userMessage);
|
|
147
|
+
this.connection.sendText(text);
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Send an image to the conversation.
|
|
151
|
+
* @param imageData - Base64-encoded image data or a URL
|
|
152
|
+
* @param mimeType - MIME type of the image (e.g., 'image/png', 'image/jpeg')
|
|
153
|
+
*/
|
|
154
|
+
sendImage(imageData, mimeType) {
|
|
155
|
+
if (!this.connection || this.state.status !== "connected") return;
|
|
156
|
+
const userMessage = {
|
|
157
|
+
id: this.generateId(),
|
|
158
|
+
role: "user",
|
|
159
|
+
timestamp: Date.now(),
|
|
160
|
+
parts: [{
|
|
161
|
+
type: "image",
|
|
162
|
+
data: imageData,
|
|
163
|
+
mimeType
|
|
164
|
+
}]
|
|
165
|
+
};
|
|
166
|
+
this.addMessage(userMessage);
|
|
167
|
+
this.connection.sendImage(imageData, mimeType);
|
|
168
|
+
}
|
|
169
|
+
/** Get current connection status */
|
|
170
|
+
get status() {
|
|
171
|
+
return this.state.status;
|
|
172
|
+
}
|
|
173
|
+
/** Get current mode */
|
|
174
|
+
get mode() {
|
|
175
|
+
return this.state.mode;
|
|
176
|
+
}
|
|
177
|
+
/** Get conversation messages */
|
|
178
|
+
get messages() {
|
|
179
|
+
return this.state.messages;
|
|
180
|
+
}
|
|
181
|
+
/** Get current error, if any */
|
|
182
|
+
get error() {
|
|
183
|
+
return this.state.error;
|
|
184
|
+
}
|
|
185
|
+
/** Get pending user transcript (while user is speaking) */
|
|
186
|
+
get pendingUserTranscript() {
|
|
187
|
+
return this.state.pendingUserTranscript;
|
|
188
|
+
}
|
|
189
|
+
/** Get pending assistant transcript (while assistant is speaking) */
|
|
190
|
+
get pendingAssistantTranscript() {
|
|
191
|
+
return this.state.pendingAssistantTranscript;
|
|
192
|
+
}
|
|
193
|
+
/** Get audio visualization data */
|
|
194
|
+
get audio() {
|
|
195
|
+
return this.connection?.getAudioVisualization() ?? null;
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* Update the session configuration.
|
|
199
|
+
* This applies changes to the active connection and persists them for future reconnections.
|
|
200
|
+
*/
|
|
201
|
+
updateSession(config) {
|
|
202
|
+
const o = this.options;
|
|
203
|
+
if (config.instructions !== void 0) o.instructions = config.instructions;
|
|
204
|
+
if (config.voice !== void 0) o.voice = config.voice;
|
|
205
|
+
if (config.vadMode !== void 0) o.vadMode = config.vadMode;
|
|
206
|
+
if (config.outputModalities !== void 0) o.outputModalities = config.outputModalities;
|
|
207
|
+
if (config.temperature !== void 0) o.temperature = config.temperature;
|
|
208
|
+
if (config.maxOutputTokens !== void 0) o.maxOutputTokens = config.maxOutputTokens;
|
|
209
|
+
if (config.semanticEagerness !== void 0) o.semanticEagerness = config.semanticEagerness;
|
|
210
|
+
if (config.providerOptions !== void 0) o.providerOptions = config.providerOptions;
|
|
211
|
+
if (this.connection) this.applySessionConfig();
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* Subscribe to state changes.
|
|
215
|
+
* @returns Unsubscribe function
|
|
216
|
+
*/
|
|
217
|
+
onStateChange(callback) {
|
|
218
|
+
this.stateChangeCallbacks.add(callback);
|
|
219
|
+
return () => {
|
|
220
|
+
this.stateChangeCallbacks.delete(callback);
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
/**
|
|
224
|
+
* Clean up resources.
|
|
225
|
+
* Call this when disposing of the client.
|
|
226
|
+
*/
|
|
227
|
+
destroy() {
|
|
228
|
+
this.disconnect();
|
|
229
|
+
this.stateChangeCallbacks.clear();
|
|
230
|
+
}
|
|
231
|
+
updateState(updates) {
|
|
232
|
+
this.state = {
|
|
233
|
+
...this.state,
|
|
234
|
+
...updates
|
|
235
|
+
};
|
|
236
|
+
for (const callback of this.stateChangeCallbacks) callback(this.state);
|
|
237
|
+
if ("status" in updates && updates.status !== void 0) this.options.onStatusChange?.(updates.status);
|
|
238
|
+
if ("mode" in updates && updates.mode !== void 0) this.options.onModeChange?.(updates.mode);
|
|
239
|
+
}
|
|
240
|
+
addMessage(message) {
|
|
241
|
+
this.updateState({ messages: [...this.state.messages, message] });
|
|
242
|
+
this.options.onMessage?.(message);
|
|
243
|
+
}
|
|
244
|
+
scheduleTokenRefresh() {
|
|
245
|
+
if (!this.token) return;
|
|
246
|
+
const timeUntilExpiry = this.token.expiresAt - Date.now();
|
|
247
|
+
const refreshIn = Math.max(0, timeUntilExpiry - TOKEN_REFRESH_BUFFER_MS);
|
|
248
|
+
this.tokenRefreshTimeout = setTimeout(() => {
|
|
249
|
+
this.refreshToken();
|
|
250
|
+
}, refreshIn);
|
|
251
|
+
}
|
|
252
|
+
async refreshToken() {
|
|
253
|
+
try {
|
|
254
|
+
this.token = await this.options.getToken();
|
|
255
|
+
this.scheduleTokenRefresh();
|
|
256
|
+
this.connection?.updateToken?.(this.token);
|
|
257
|
+
} catch (error) {
|
|
258
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
259
|
+
this.updateState({ error: err });
|
|
260
|
+
this.options.onError?.(err);
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
subscribeToConnectionEvents() {
|
|
264
|
+
if (!this.connection) return;
|
|
265
|
+
this.unsubscribers.push(this.connection.on("status_change", ({ status }) => {
|
|
266
|
+
this.updateState({ status });
|
|
267
|
+
}));
|
|
268
|
+
this.unsubscribers.push(this.connection.on("mode_change", ({ mode }) => {
|
|
269
|
+
this.updateState({ mode });
|
|
270
|
+
}));
|
|
271
|
+
this.unsubscribers.push(this.connection.on("transcript", ({ role, transcript, isFinal }) => {
|
|
272
|
+
if (role === "user") {
|
|
273
|
+
this.updateState({ pendingUserTranscript: isFinal ? null : transcript });
|
|
274
|
+
if (isFinal && transcript) this.addMessage({
|
|
275
|
+
id: this.generateId(),
|
|
276
|
+
role: "user",
|
|
277
|
+
timestamp: Date.now(),
|
|
278
|
+
parts: [{
|
|
279
|
+
type: "audio",
|
|
280
|
+
transcript,
|
|
281
|
+
durationMs: 0
|
|
282
|
+
}]
|
|
283
|
+
});
|
|
284
|
+
} else this.updateState({ pendingAssistantTranscript: isFinal ? null : transcript });
|
|
285
|
+
}));
|
|
286
|
+
this.unsubscribers.push(this.connection.on("tool_call", ({ toolCallId, toolName, input }) => {
|
|
287
|
+
if (!toolCallId) {
|
|
288
|
+
console.error("[RealtimeClient] tool_call missing toolCallId", toolName);
|
|
289
|
+
return;
|
|
290
|
+
}
|
|
291
|
+
const execute = this.clientTools.get(toolName)?.execute;
|
|
292
|
+
if (execute) (async () => {
|
|
293
|
+
try {
|
|
294
|
+
const output = await execute(input);
|
|
295
|
+
this.connection?.sendToolResult(toolCallId, typeof output === "string" ? output : JSON.stringify(output));
|
|
296
|
+
} catch (error) {
|
|
297
|
+
const errMsg = error instanceof Error ? error.message : String(error);
|
|
298
|
+
this.connection?.sendToolResult(toolCallId, JSON.stringify({ error: errMsg }));
|
|
299
|
+
}
|
|
300
|
+
})();
|
|
301
|
+
else this.connection?.sendToolResult(toolCallId, JSON.stringify({ error: `Unknown or non-executable tool: ${String(toolName)}` }));
|
|
302
|
+
}));
|
|
303
|
+
this.unsubscribers.push(this.connection.on("message_complete", ({ message }) => {
|
|
304
|
+
const existingIndex = this.state.messages.findIndex((m) => m.id === message.id);
|
|
305
|
+
if (existingIndex >= 0) {
|
|
306
|
+
const newMessages = [...this.state.messages];
|
|
307
|
+
newMessages[existingIndex] = message;
|
|
308
|
+
this.updateState({ messages: newMessages });
|
|
309
|
+
} else this.addMessage(message);
|
|
310
|
+
}));
|
|
311
|
+
this.unsubscribers.push(this.connection.on("interrupted", ({ messageId }) => {
|
|
312
|
+
if (messageId) {
|
|
313
|
+
const newMessages = this.state.messages.map((m) => m.id === messageId ? {
|
|
314
|
+
...m,
|
|
315
|
+
interrupted: true
|
|
316
|
+
} : m);
|
|
317
|
+
this.updateState({ messages: newMessages });
|
|
318
|
+
}
|
|
319
|
+
this.updateState({
|
|
320
|
+
mode: "listening",
|
|
321
|
+
pendingAssistantTranscript: null
|
|
322
|
+
});
|
|
323
|
+
this.options.onInterrupted?.();
|
|
324
|
+
}));
|
|
325
|
+
this.unsubscribers.push(this.connection.on("error", ({ error }) => {
|
|
326
|
+
this.updateState({ error });
|
|
327
|
+
this.options.onError?.(error);
|
|
328
|
+
}));
|
|
329
|
+
this.unsubscribers.push(this.connection.on("go_away", ({ timeLeft }) => {
|
|
330
|
+
this.options.onGoAway?.(timeLeft);
|
|
331
|
+
}));
|
|
332
|
+
this.unsubscribers.push(this.connection.on("usage", (usage) => {
|
|
333
|
+
this.options.onUsage?.(usage);
|
|
334
|
+
}));
|
|
335
|
+
}
|
|
336
|
+
applySessionConfig() {
|
|
337
|
+
if (!this.connection) return;
|
|
338
|
+
const { instructions, voice, vadMode, tools, outputModalities, temperature, maxOutputTokens, semanticEagerness } = this.options;
|
|
339
|
+
if (!(instructions || voice || vadMode || tools && tools.length > 0 || outputModalities || temperature !== void 0 || maxOutputTokens !== void 0 || semanticEagerness)) return;
|
|
340
|
+
const toolsConfig = tools ? Array.from(this.clientTools.values()).map((t) => {
|
|
341
|
+
const inputSchema = t.inputSchema ? convertSchemaToJsonSchema(t.inputSchema) : void 0;
|
|
342
|
+
const outputSchema = t.outputSchema ? convertSchemaToJsonSchema(t.outputSchema) : void 0;
|
|
343
|
+
return {
|
|
344
|
+
name: t.name,
|
|
345
|
+
description: t.description,
|
|
346
|
+
...inputSchema ? { inputSchema } : {},
|
|
347
|
+
...outputSchema ? { outputSchema } : {}
|
|
348
|
+
};
|
|
349
|
+
}) : void 0;
|
|
350
|
+
this.connection.updateSession({
|
|
351
|
+
...instructions !== void 0 ? { instructions } : {},
|
|
352
|
+
...voice !== void 0 ? { voice } : {},
|
|
353
|
+
...vadMode !== void 0 ? { vadMode } : {},
|
|
354
|
+
...toolsConfig !== void 0 ? { tools: toolsConfig } : {},
|
|
355
|
+
...outputModalities !== void 0 ? { outputModalities } : {},
|
|
356
|
+
...temperature !== void 0 ? { temperature } : {},
|
|
357
|
+
...maxOutputTokens !== void 0 ? { maxOutputTokens } : {},
|
|
358
|
+
...semanticEagerness !== void 0 ? { semanticEagerness } : {}
|
|
359
|
+
});
|
|
360
|
+
}
|
|
361
|
+
generateId() {
|
|
362
|
+
return `msg-${Date.now()}-${Math.random().toString(36).substring(7)}`;
|
|
363
|
+
}
|
|
441
364
|
};
|
|
442
|
-
//#
|
|
365
|
+
//#endregion
|
|
366
|
+
export { RealtimeClient };
|
|
367
|
+
|
|
368
|
+
//# sourceMappingURL=realtime-client.js.map
|