@charivo/realtime 0.7.0 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -0
- package/dist/{chunk-RRXU3IC5.mjs → chunk-E656Y2PO.mjs} +5 -44
- package/dist/chunk-E656Y2PO.mjs.map +1 -0
- package/dist/{chunk-LOHYIBEP.mjs → chunk-FRTYFCPV.mjs} +4 -3
- package/dist/chunk-FRTYFCPV.mjs.map +1 -0
- package/dist/{chunk-M7TPQNHF.mjs → chunk-GJ2NA3ZT.mjs} +4 -3
- package/dist/chunk-GJ2NA3ZT.mjs.map +1 -0
- package/dist/chunk-X6YRHDTW.mjs +93 -0
- package/dist/chunk-X6YRHDTW.mjs.map +1 -0
- package/dist/index.d.mts +11 -21
- package/dist/index.d.ts +11 -21
- package/dist/index.js +452 -276
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +450 -275
- package/dist/index.mjs.map +1 -1
- package/dist/openai/index.mjs +3 -3
- package/dist/openai-agents/index.js +2 -3
- package/dist/openai-agents/index.js.map +1 -1
- package/dist/openai-agents/index.mjs +3 -3
- package/dist/remote/index.js +2 -3
- package/dist/remote/index.js.map +1 -1
- package/dist/remote/index.mjs +4 -4
- package/package.json +1 -1
- package/dist/chunk-LBKX5PJ6.mjs +0 -54
- package/dist/chunk-LBKX5PJ6.mjs.map +0 -1
- package/dist/chunk-LOHYIBEP.mjs.map +0 -1
- package/dist/chunk-M7TPQNHF.mjs.map +0 -1
- package/dist/chunk-RRXU3IC5.mjs.map +0 -1
package/dist/index.js
CHANGED
|
@@ -9,10 +9,9 @@ var DEFAULT_REALTIME_AGENT_INSTRUCTIONS = `
|
|
|
9
9
|
You are speaking in a realtime voice conversation.
|
|
10
10
|
Respond naturally and keep replies concise enough for spoken delivery.
|
|
11
11
|
Use available tools proactively when they make the conversation clearer or more present.
|
|
12
|
-
|
|
12
|
+
Let each tool call earn its place; don't chain rapid-fire actions just to look responsive.
|
|
13
13
|
Never break character. Never refer to yourself as an AI, model, or assistant.
|
|
14
|
-
|
|
15
|
-
Speak only the words you want heard aloud. Use tools for actions. Never say tool names or tool arguments out loud. Do not output bracketed or parenthetical action notes. If a tool is unavailable, omit the action entirely rather than narrating it.
|
|
14
|
+
Speak only the words you want heard aloud. Use tools for actions, but never mention tool calls, tool names, tool arguments, or bracketed/parenthetical action notes. If a tool is unavailable, omit the action instead of narrating it.
|
|
16
15
|
`.trim();
|
|
17
16
|
function buildRealtimeSessionConfig({
|
|
18
17
|
character,
|
|
@@ -55,10 +54,419 @@ function ensureSentenceFragment(text) {
|
|
|
55
54
|
return text.trim().replace(/[.!?]+$/, "");
|
|
56
55
|
}
|
|
57
56
|
|
|
57
|
+
// src/internal/audio-output.ts
|
|
58
|
+
var RealtimeAudioOutput = class {
|
|
59
|
+
constructor(emit) {
|
|
60
|
+
this.emit = emit;
|
|
61
|
+
}
|
|
62
|
+
active = false;
|
|
63
|
+
isActive() {
|
|
64
|
+
return this.active;
|
|
65
|
+
}
|
|
66
|
+
start() {
|
|
67
|
+
if (this.active) {
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
this.active = true;
|
|
71
|
+
this.emit("tts:audio:start", {});
|
|
72
|
+
}
|
|
73
|
+
end() {
|
|
74
|
+
if (!this.active) {
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
this.active = false;
|
|
78
|
+
this.emit("tts:lipsync:update", { rms: 0 });
|
|
79
|
+
this.emit("tts:audio:end", {});
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
var RealtimeBrowserLifecycle = class {
|
|
83
|
+
constructor(getState) {
|
|
84
|
+
this.getState = getState;
|
|
85
|
+
}
|
|
86
|
+
wakeLockSentinel = null;
|
|
87
|
+
teardownBrowserLifecycle;
|
|
88
|
+
install() {
|
|
89
|
+
if (this.teardownBrowserLifecycle) {
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
this.teardownBrowserLifecycle = core.subscribeBrowserLifecycle({
|
|
93
|
+
onHidden: () => {
|
|
94
|
+
void this.releaseWakeLock();
|
|
95
|
+
},
|
|
96
|
+
onPageHide: () => {
|
|
97
|
+
void this.releaseWakeLock();
|
|
98
|
+
},
|
|
99
|
+
onPageShow: () => {
|
|
100
|
+
void this.requestWakeLock();
|
|
101
|
+
},
|
|
102
|
+
onVisible: () => {
|
|
103
|
+
void this.requestWakeLock();
|
|
104
|
+
}
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
dispose() {
|
|
108
|
+
this.teardownBrowserLifecycle?.();
|
|
109
|
+
this.teardownBrowserLifecycle = void 0;
|
|
110
|
+
void this.releaseWakeLock();
|
|
111
|
+
}
|
|
112
|
+
async requestWakeLock() {
|
|
113
|
+
const state = this.getState();
|
|
114
|
+
if (state.session.status !== "active" || state.connection === "disconnecting" || typeof navigator === "undefined" || typeof document === "undefined" || document.visibilityState !== "visible" || !("wakeLock" in navigator)) {
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
if (this.wakeLockSentinel) {
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
const wakeLockApi = navigator;
|
|
121
|
+
try {
|
|
122
|
+
this.wakeLockSentinel = await wakeLockApi.wakeLock?.request("screen");
|
|
123
|
+
this.wakeLockSentinel?.addEventListener?.(
|
|
124
|
+
"release",
|
|
125
|
+
() => {
|
|
126
|
+
this.wakeLockSentinel = null;
|
|
127
|
+
},
|
|
128
|
+
{ once: true }
|
|
129
|
+
);
|
|
130
|
+
} catch {
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
async releaseWakeLock() {
|
|
134
|
+
if (!this.wakeLockSentinel) {
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
const wakeLockSentinel = this.wakeLockSentinel;
|
|
138
|
+
this.wakeLockSentinel = null;
|
|
139
|
+
try {
|
|
140
|
+
await wakeLockSentinel.release();
|
|
141
|
+
} catch {
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
};
|
|
145
|
+
|
|
146
|
+
// src/internal/manager-state.ts
|
|
147
|
+
var nextFallbackSessionId = 1;
|
|
148
|
+
function mergeSessionConfig(baseConfig, overrideConfig) {
|
|
149
|
+
if (!baseConfig && !overrideConfig) {
|
|
150
|
+
return void 0;
|
|
151
|
+
}
|
|
152
|
+
const { tools: _ignoredOverrideTools, ...overrideWithoutTools } = overrideConfig ?? {};
|
|
153
|
+
return {
|
|
154
|
+
...baseConfig,
|
|
155
|
+
...overrideWithoutTools
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
function mergeRealtimeState(current, partial) {
|
|
159
|
+
return {
|
|
160
|
+
connection: partial.connection ?? current.connection,
|
|
161
|
+
session: {
|
|
162
|
+
status: partial.session?.status ?? current.session.status,
|
|
163
|
+
config: partial.session?.config ?? current.session.config,
|
|
164
|
+
characterId: partial.session?.characterId ?? current.session.characterId
|
|
165
|
+
},
|
|
166
|
+
response: {
|
|
167
|
+
status: partial.response?.status ?? current.response.status,
|
|
168
|
+
text: partial.response?.text ?? current.response.text
|
|
169
|
+
},
|
|
170
|
+
lastError: partial.lastError ?? current.lastError
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
function createRealtimeSessionId() {
|
|
174
|
+
const randomUuid = globalThis.crypto?.randomUUID?.();
|
|
175
|
+
if (randomUuid) {
|
|
176
|
+
return randomUuid;
|
|
177
|
+
}
|
|
178
|
+
return `realtime-session-${Date.now()}-${nextFallbackSessionId++}`;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
// src/internal/timing.ts
|
|
182
|
+
function delay(ms) {
|
|
183
|
+
return new Promise((resolve) => {
|
|
184
|
+
setTimeout(resolve, ms);
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// src/internal/tool-registry.ts
|
|
189
|
+
var RealtimeToolRegistry = class {
|
|
190
|
+
tools = /* @__PURE__ */ new Map();
|
|
191
|
+
register(tool) {
|
|
192
|
+
this.tools.set(tool.definition.name, tool);
|
|
193
|
+
}
|
|
194
|
+
unregister(name) {
|
|
195
|
+
this.tools.delete(name);
|
|
196
|
+
}
|
|
197
|
+
get(name) {
|
|
198
|
+
return this.tools.get(name);
|
|
199
|
+
}
|
|
200
|
+
getDefinitions() {
|
|
201
|
+
return Array.from(this.tools.values(), (tool) => ({
|
|
202
|
+
...tool.definition,
|
|
203
|
+
parameters: {
|
|
204
|
+
...tool.definition.parameters,
|
|
205
|
+
properties: { ...tool.definition.parameters.properties },
|
|
206
|
+
required: tool.definition.parameters.required ? [...tool.definition.parameters.required] : void 0
|
|
207
|
+
}
|
|
208
|
+
}));
|
|
209
|
+
}
|
|
210
|
+
};
|
|
211
|
+
function isRecord(value) {
|
|
212
|
+
return typeof value === "object" && value !== null;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
// src/internal/tool-args-validation.ts
|
|
216
|
+
function validateToolArguments(definition, args) {
|
|
217
|
+
if (!isRecord(args) || Array.isArray(args)) {
|
|
218
|
+
throw new Error(
|
|
219
|
+
`Realtime tool "${definition.name}" arguments failed schema validation: arguments must be an object`
|
|
220
|
+
);
|
|
221
|
+
}
|
|
222
|
+
const { parameters } = definition;
|
|
223
|
+
const required = parameters.required ?? [];
|
|
224
|
+
for (const propertyName of required) {
|
|
225
|
+
if (!hasOwnProperty(args, propertyName)) {
|
|
226
|
+
throw new Error(
|
|
227
|
+
`Realtime tool "${definition.name}" arguments failed schema validation: missing required property "${propertyName}"`
|
|
228
|
+
);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
for (const [propertyName, schema] of Object.entries(parameters.properties)) {
|
|
232
|
+
if (!hasOwnProperty(args, propertyName) || !isRecord(schema)) {
|
|
233
|
+
continue;
|
|
234
|
+
}
|
|
235
|
+
validateToolArgumentValue(
|
|
236
|
+
definition.name,
|
|
237
|
+
propertyName,
|
|
238
|
+
args[propertyName],
|
|
239
|
+
schema
|
|
240
|
+
);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
function validateToolArgumentValue(toolName, propertyName, value, schema) {
|
|
244
|
+
const enumValues = schema.enum;
|
|
245
|
+
if (Array.isArray(enumValues) && !enumValues.some((item) => Object.is(item, value))) {
|
|
246
|
+
throw new Error(
|
|
247
|
+
`Realtime tool "${toolName}" arguments failed schema validation: property "${propertyName}" must be one of ${formatEnumValues(enumValues)}`
|
|
248
|
+
);
|
|
249
|
+
}
|
|
250
|
+
const expectedType = schema.type;
|
|
251
|
+
if (typeof expectedType !== "string") {
|
|
252
|
+
return;
|
|
253
|
+
}
|
|
254
|
+
if (!isSupportedSchemaType(expectedType)) {
|
|
255
|
+
throw new Error(
|
|
256
|
+
`Realtime tool "${toolName}" arguments failed schema validation: property "${propertyName}" uses unsupported schema type "${expectedType}"`
|
|
257
|
+
);
|
|
258
|
+
}
|
|
259
|
+
if (!matchesSchemaType(value, expectedType)) {
|
|
260
|
+
throw new Error(
|
|
261
|
+
`Realtime tool "${toolName}" arguments failed schema validation: property "${propertyName}" must be ${expectedType}`
|
|
262
|
+
);
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
function matchesSchemaType(value, expectedType) {
|
|
266
|
+
switch (expectedType) {
|
|
267
|
+
case "string":
|
|
268
|
+
return typeof value === "string";
|
|
269
|
+
case "number":
|
|
270
|
+
return typeof value === "number" && Number.isFinite(value);
|
|
271
|
+
case "integer":
|
|
272
|
+
return typeof value === "number" && Number.isInteger(value);
|
|
273
|
+
case "boolean":
|
|
274
|
+
return typeof value === "boolean";
|
|
275
|
+
case "object":
|
|
276
|
+
return isRecord(value) && !Array.isArray(value);
|
|
277
|
+
case "array":
|
|
278
|
+
return Array.isArray(value);
|
|
279
|
+
case "null":
|
|
280
|
+
return value === null;
|
|
281
|
+
}
|
|
282
|
+
return false;
|
|
283
|
+
}
|
|
284
|
+
function isSupportedSchemaType(expectedType) {
|
|
285
|
+
return expectedType === "string" || expectedType === "number" || expectedType === "integer" || expectedType === "boolean" || expectedType === "object" || expectedType === "array" || expectedType === "null";
|
|
286
|
+
}
|
|
287
|
+
function formatEnumValues(values) {
|
|
288
|
+
return values.map((value) => JSON.stringify(value)).join(", ");
|
|
289
|
+
}
|
|
290
|
+
function hasOwnProperty(record, propertyName) {
|
|
291
|
+
return Object.prototype.hasOwnProperty.call(record, propertyName);
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
// src/internal/tool-execution.ts
|
|
295
|
+
function createFailureOutput(error) {
|
|
296
|
+
return {
|
|
297
|
+
success: false,
|
|
298
|
+
error: error.message
|
|
299
|
+
};
|
|
300
|
+
}
|
|
301
|
+
async function withTimeout(promise, timeoutMs, toolName) {
|
|
302
|
+
let timeoutId;
|
|
303
|
+
try {
|
|
304
|
+
return await Promise.race([
|
|
305
|
+
promise,
|
|
306
|
+
new Promise((_, reject) => {
|
|
307
|
+
timeoutId = setTimeout(() => {
|
|
308
|
+
reject(
|
|
309
|
+
new Error(
|
|
310
|
+
`Realtime tool "${toolName}" timed out after ${timeoutMs}ms`
|
|
311
|
+
)
|
|
312
|
+
);
|
|
313
|
+
}, timeoutMs);
|
|
314
|
+
})
|
|
315
|
+
]);
|
|
316
|
+
} finally {
|
|
317
|
+
if (timeoutId !== void 0) {
|
|
318
|
+
clearTimeout(timeoutId);
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
// src/internal/tool-runner.ts
|
|
324
|
+
async function executeRealtimeToolCall({
|
|
325
|
+
event,
|
|
326
|
+
tool,
|
|
327
|
+
client,
|
|
328
|
+
character,
|
|
329
|
+
state,
|
|
330
|
+
defaultToolTimeoutMs,
|
|
331
|
+
resultProjectors,
|
|
332
|
+
emit,
|
|
333
|
+
log
|
|
334
|
+
}) {
|
|
335
|
+
if (!event.callId) {
|
|
336
|
+
emitToolError(
|
|
337
|
+
event.name,
|
|
338
|
+
new Error(`Tool "${event.name}" is missing a call ID`),
|
|
339
|
+
event.callId,
|
|
340
|
+
emit,
|
|
341
|
+
log
|
|
342
|
+
);
|
|
343
|
+
return;
|
|
344
|
+
}
|
|
345
|
+
if (!tool) {
|
|
346
|
+
await handleToolExecutionFailure(
|
|
347
|
+
event.name,
|
|
348
|
+
event.callId,
|
|
349
|
+
new Error(`No realtime tool registered for "${event.name}"`),
|
|
350
|
+
client,
|
|
351
|
+
emit,
|
|
352
|
+
log
|
|
353
|
+
);
|
|
354
|
+
return;
|
|
355
|
+
}
|
|
356
|
+
try {
|
|
357
|
+
log("debug", "Realtime tool execution started", {
|
|
358
|
+
name: event.name,
|
|
359
|
+
callId: event.callId
|
|
360
|
+
});
|
|
361
|
+
const output = await runToolHandler({
|
|
362
|
+
tool,
|
|
363
|
+
event,
|
|
364
|
+
character,
|
|
365
|
+
state,
|
|
366
|
+
defaultToolTimeoutMs
|
|
367
|
+
});
|
|
368
|
+
await client.sendToolResult(event.callId, output);
|
|
369
|
+
emit("realtime:tool:result", {
|
|
370
|
+
name: event.name,
|
|
371
|
+
output,
|
|
372
|
+
callId: event.callId
|
|
373
|
+
});
|
|
374
|
+
projectToolResult(
|
|
375
|
+
event.name,
|
|
376
|
+
output,
|
|
377
|
+
resultProjectors,
|
|
378
|
+
emit,
|
|
379
|
+
log,
|
|
380
|
+
event.callId
|
|
381
|
+
);
|
|
382
|
+
log("info", "Realtime tool execution succeeded", {
|
|
383
|
+
name: event.name,
|
|
384
|
+
callId: event.callId
|
|
385
|
+
});
|
|
386
|
+
} catch (error) {
|
|
387
|
+
await handleToolExecutionFailure(
|
|
388
|
+
event.name,
|
|
389
|
+
event.callId,
|
|
390
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
391
|
+
client,
|
|
392
|
+
emit,
|
|
393
|
+
log
|
|
394
|
+
);
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
async function runToolHandler({
|
|
398
|
+
tool,
|
|
399
|
+
event,
|
|
400
|
+
character,
|
|
401
|
+
state,
|
|
402
|
+
defaultToolTimeoutMs
|
|
403
|
+
}) {
|
|
404
|
+
validateToolArguments(tool.definition, event.args);
|
|
405
|
+
const result = await withTimeout(
|
|
406
|
+
tool.handler(event.args, {
|
|
407
|
+
character,
|
|
408
|
+
state,
|
|
409
|
+
callId: event.callId
|
|
410
|
+
}),
|
|
411
|
+
tool.timeoutMs ?? defaultToolTimeoutMs,
|
|
412
|
+
tool.definition.name
|
|
413
|
+
);
|
|
414
|
+
if (!isRecord(result)) {
|
|
415
|
+
throw new Error(
|
|
416
|
+
`Realtime tool "${tool.definition.name}" must return an object`
|
|
417
|
+
);
|
|
418
|
+
}
|
|
419
|
+
return result;
|
|
420
|
+
}
|
|
421
|
+
async function handleToolExecutionFailure(name, callId, error, client, emit, log) {
|
|
422
|
+
emitToolError(name, error, callId, emit, log);
|
|
423
|
+
try {
|
|
424
|
+
await client.sendToolResult(callId, createFailureOutput(error));
|
|
425
|
+
} catch (sendError) {
|
|
426
|
+
emitToolError(
|
|
427
|
+
name,
|
|
428
|
+
sendError instanceof Error ? sendError : new Error(String(sendError)),
|
|
429
|
+
callId,
|
|
430
|
+
emit,
|
|
431
|
+
log
|
|
432
|
+
);
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
function projectToolResult(name, output, resultProjectors, emit, log, callId) {
|
|
436
|
+
for (const projector of resultProjectors ?? []) {
|
|
437
|
+
try {
|
|
438
|
+
projector({
|
|
439
|
+
name,
|
|
440
|
+
output,
|
|
441
|
+
callId,
|
|
442
|
+
emit
|
|
443
|
+
});
|
|
444
|
+
} catch (error) {
|
|
445
|
+
const projectorError = error instanceof Error ? error : new Error(String(error));
|
|
446
|
+
log("warn", "Realtime result projector failed", {
|
|
447
|
+
name,
|
|
448
|
+
callId,
|
|
449
|
+
error: projectorError.message
|
|
450
|
+
});
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
function emitToolError(name, error, callId, emit, log) {
|
|
455
|
+
log("warn", "Realtime tool execution failed", {
|
|
456
|
+
name,
|
|
457
|
+
callId,
|
|
458
|
+
error: error.message
|
|
459
|
+
});
|
|
460
|
+
emit("realtime:tool:error", {
|
|
461
|
+
name,
|
|
462
|
+
error,
|
|
463
|
+
callId
|
|
464
|
+
});
|
|
465
|
+
}
|
|
466
|
+
|
|
58
467
|
// src/realtime-manager.ts
|
|
59
468
|
var DEFAULT_TOOL_TIMEOUT_MS = 1e4;
|
|
60
469
|
var RECONNECT_DELAYS_MS = [500, 1e3, 2e3, 4e3, 5e3];
|
|
61
|
-
var nextFallbackSessionId = 1;
|
|
62
470
|
var RealtimeManagerImpl = class {
|
|
63
471
|
constructor(client, options = {}) {
|
|
64
472
|
this.client = client;
|
|
@@ -66,7 +474,7 @@ var RealtimeManagerImpl = class {
|
|
|
66
474
|
for (const tool of options.tools ?? []) {
|
|
67
475
|
this.registerTool(tool);
|
|
68
476
|
}
|
|
69
|
-
this.
|
|
477
|
+
this.browserLifecycle.install();
|
|
70
478
|
this.client.onEvent((event) => {
|
|
71
479
|
void this.handleClientEvent(event).catch((error) => {
|
|
72
480
|
this.applyError(
|
|
@@ -81,17 +489,20 @@ var RealtimeManagerImpl = class {
|
|
|
81
489
|
isStoppingSession = false;
|
|
82
490
|
isRefreshingSession = false;
|
|
83
491
|
isRecoveringConnection = false;
|
|
84
|
-
isAudioPlaybackActive = false;
|
|
85
492
|
sessionBaseConfig;
|
|
86
493
|
refreshInFlight = null;
|
|
87
494
|
reconnectInFlight = null;
|
|
88
495
|
hasQueuedRefresh = false;
|
|
89
496
|
stopRequestedDuringRefresh = false;
|
|
90
497
|
reconnectToken = 0;
|
|
91
|
-
wakeLockSentinel = null;
|
|
92
|
-
teardownBrowserLifecycle;
|
|
93
498
|
sessionId;
|
|
94
|
-
toolRegistry =
|
|
499
|
+
toolRegistry = new RealtimeToolRegistry();
|
|
500
|
+
audioOutput = new RealtimeAudioOutput((event, payload) => {
|
|
501
|
+
this.eventEmitter?.emit(event, payload);
|
|
502
|
+
});
|
|
503
|
+
browserLifecycle = new RealtimeBrowserLifecycle(
|
|
504
|
+
() => this.state
|
|
505
|
+
);
|
|
95
506
|
state = {
|
|
96
507
|
connection: "idle",
|
|
97
508
|
session: {
|
|
@@ -119,20 +530,13 @@ var RealtimeManagerImpl = class {
|
|
|
119
530
|
this.emitState();
|
|
120
531
|
}
|
|
121
532
|
registerTool(tool) {
|
|
122
|
-
this.toolRegistry.
|
|
533
|
+
this.toolRegistry.register(tool);
|
|
123
534
|
}
|
|
124
535
|
unregisterTool(name) {
|
|
125
|
-
this.toolRegistry.
|
|
536
|
+
this.toolRegistry.unregister(name);
|
|
126
537
|
}
|
|
127
538
|
getRegisteredTools() {
|
|
128
|
-
return
|
|
129
|
-
...tool.definition,
|
|
130
|
-
parameters: {
|
|
131
|
-
...tool.definition.parameters,
|
|
132
|
-
properties: { ...tool.definition.parameters.properties },
|
|
133
|
-
required: tool.definition.parameters.required ? [...tool.definition.parameters.required] : void 0
|
|
134
|
-
}
|
|
135
|
-
}));
|
|
539
|
+
return this.toolRegistry.getDefinitions();
|
|
136
540
|
}
|
|
137
541
|
getState() {
|
|
138
542
|
return {
|
|
@@ -170,12 +574,12 @@ var RealtimeManagerImpl = class {
|
|
|
170
574
|
},
|
|
171
575
|
lastError: null
|
|
172
576
|
};
|
|
173
|
-
this.
|
|
577
|
+
this.browserLifecycle.install();
|
|
174
578
|
this.emitState();
|
|
175
579
|
try {
|
|
176
580
|
await this.client.connect(sessionConfig);
|
|
177
581
|
this.commitActiveSession(sessionConfig);
|
|
178
|
-
void this.requestWakeLock();
|
|
582
|
+
void this.browserLifecycle.requestWakeLock();
|
|
179
583
|
this.emitSessionStart("user");
|
|
180
584
|
} catch (error) {
|
|
181
585
|
const typedError = core.toCharivoError("transport", error);
|
|
@@ -267,7 +671,7 @@ var RealtimeManagerImpl = class {
|
|
|
267
671
|
status: "interrupted"
|
|
268
672
|
}
|
|
269
673
|
};
|
|
270
|
-
this.
|
|
674
|
+
this.audioOutput.end();
|
|
271
675
|
this.emitState();
|
|
272
676
|
}
|
|
273
677
|
async handleClientEvent(event) {
|
|
@@ -346,14 +750,14 @@ var RealtimeManagerImpl = class {
|
|
|
346
750
|
return;
|
|
347
751
|
}
|
|
348
752
|
case "audio.output.started":
|
|
349
|
-
this.
|
|
753
|
+
this.audioOutput.start();
|
|
350
754
|
return;
|
|
351
755
|
case "audio.output.ended":
|
|
352
|
-
this.
|
|
756
|
+
this.audioOutput.end();
|
|
353
757
|
return;
|
|
354
758
|
case "audio.lipsync":
|
|
355
|
-
if (event.rms > 1e-3 && !this.
|
|
356
|
-
this.
|
|
759
|
+
if (event.rms > 1e-3 && !this.audioOutput.isActive()) {
|
|
760
|
+
this.audioOutput.start();
|
|
357
761
|
}
|
|
358
762
|
this.eventEmitter?.emit("tts:lipsync:update", { rms: event.rms });
|
|
359
763
|
return;
|
|
@@ -363,7 +767,21 @@ var RealtimeManagerImpl = class {
|
|
|
363
767
|
args: event.args,
|
|
364
768
|
callId: event.callId
|
|
365
769
|
});
|
|
366
|
-
await
|
|
770
|
+
await executeRealtimeToolCall({
|
|
771
|
+
event,
|
|
772
|
+
tool: this.toolRegistry.get(event.name),
|
|
773
|
+
client: this.client,
|
|
774
|
+
character: this.character,
|
|
775
|
+
state: this.getState(),
|
|
776
|
+
defaultToolTimeoutMs: this.options.defaultToolTimeoutMs ?? DEFAULT_TOOL_TIMEOUT_MS,
|
|
777
|
+
resultProjectors: this.options.resultProjectors,
|
|
778
|
+
emit: (eventName, payload) => {
|
|
779
|
+
this.eventEmitter?.emit(eventName, payload);
|
|
780
|
+
},
|
|
781
|
+
log: (level, message, context) => {
|
|
782
|
+
this.log(level, message, context);
|
|
783
|
+
}
|
|
784
|
+
});
|
|
367
785
|
return;
|
|
368
786
|
case "tool.result":
|
|
369
787
|
this.eventEmitter?.emit("realtime:tool:result", {
|
|
@@ -389,112 +807,6 @@ var RealtimeManagerImpl = class {
|
|
|
389
807
|
return;
|
|
390
808
|
}
|
|
391
809
|
}
|
|
392
|
-
async executeToolCall(event) {
|
|
393
|
-
const tool = this.toolRegistry.get(event.name);
|
|
394
|
-
if (!event.callId) {
|
|
395
|
-
this.emitToolError(
|
|
396
|
-
event.name,
|
|
397
|
-
new Error(`Tool "${event.name}" is missing a call ID`),
|
|
398
|
-
event.callId
|
|
399
|
-
);
|
|
400
|
-
return;
|
|
401
|
-
}
|
|
402
|
-
if (!tool) {
|
|
403
|
-
await this.handleToolExecutionFailure(
|
|
404
|
-
event.name,
|
|
405
|
-
event.callId,
|
|
406
|
-
new Error(`No realtime tool registered for "${event.name}"`)
|
|
407
|
-
);
|
|
408
|
-
return;
|
|
409
|
-
}
|
|
410
|
-
try {
|
|
411
|
-
this.log("debug", "Realtime tool execution started", {
|
|
412
|
-
name: event.name,
|
|
413
|
-
callId: event.callId
|
|
414
|
-
});
|
|
415
|
-
const output = await this.runToolHandler(tool, event);
|
|
416
|
-
await this.client.sendToolResult(event.callId, output);
|
|
417
|
-
this.eventEmitter?.emit("realtime:tool:result", {
|
|
418
|
-
name: event.name,
|
|
419
|
-
output,
|
|
420
|
-
callId: event.callId
|
|
421
|
-
});
|
|
422
|
-
this.projectToolResult(event.name, output, event.callId);
|
|
423
|
-
this.log("info", "Realtime tool execution succeeded", {
|
|
424
|
-
name: event.name,
|
|
425
|
-
callId: event.callId
|
|
426
|
-
});
|
|
427
|
-
} catch (error) {
|
|
428
|
-
await this.handleToolExecutionFailure(
|
|
429
|
-
event.name,
|
|
430
|
-
event.callId,
|
|
431
|
-
error instanceof Error ? error : new Error(String(error))
|
|
432
|
-
);
|
|
433
|
-
}
|
|
434
|
-
}
|
|
435
|
-
async runToolHandler(tool, event) {
|
|
436
|
-
const timeoutMs = tool.timeoutMs ?? this.options.defaultToolTimeoutMs ?? DEFAULT_TOOL_TIMEOUT_MS;
|
|
437
|
-
const result = await withTimeout(
|
|
438
|
-
tool.handler(event.args, {
|
|
439
|
-
character: this.character,
|
|
440
|
-
state: this.getState(),
|
|
441
|
-
callId: event.callId
|
|
442
|
-
}),
|
|
443
|
-
timeoutMs,
|
|
444
|
-
tool.definition.name
|
|
445
|
-
);
|
|
446
|
-
if (!isRecord(result)) {
|
|
447
|
-
throw new Error(
|
|
448
|
-
`Realtime tool "${tool.definition.name}" must return an object`
|
|
449
|
-
);
|
|
450
|
-
}
|
|
451
|
-
return result;
|
|
452
|
-
}
|
|
453
|
-
async handleToolExecutionFailure(name, callId, error) {
|
|
454
|
-
this.emitToolError(name, error, callId);
|
|
455
|
-
try {
|
|
456
|
-
await this.client.sendToolResult(callId, createFailureOutput(error));
|
|
457
|
-
} catch (sendError) {
|
|
458
|
-
this.emitToolError(
|
|
459
|
-
name,
|
|
460
|
-
sendError instanceof Error ? sendError : new Error(String(sendError)),
|
|
461
|
-
callId
|
|
462
|
-
);
|
|
463
|
-
}
|
|
464
|
-
}
|
|
465
|
-
projectToolResult(name, output, callId) {
|
|
466
|
-
for (const projector of this.options.resultProjectors ?? []) {
|
|
467
|
-
try {
|
|
468
|
-
projector({
|
|
469
|
-
name,
|
|
470
|
-
output,
|
|
471
|
-
callId,
|
|
472
|
-
emit: (event, payload) => {
|
|
473
|
-
this.eventEmitter?.emit(event, payload);
|
|
474
|
-
}
|
|
475
|
-
});
|
|
476
|
-
} catch (error) {
|
|
477
|
-
const projectorError = error instanceof Error ? error : new Error(String(error));
|
|
478
|
-
this.log("warn", "Realtime result projector failed", {
|
|
479
|
-
name,
|
|
480
|
-
callId,
|
|
481
|
-
error: projectorError.message
|
|
482
|
-
});
|
|
483
|
-
}
|
|
484
|
-
}
|
|
485
|
-
}
|
|
486
|
-
emitToolError(name, error, callId) {
|
|
487
|
-
this.log("warn", "Realtime tool execution failed", {
|
|
488
|
-
name,
|
|
489
|
-
callId,
|
|
490
|
-
error: error.message
|
|
491
|
-
});
|
|
492
|
-
this.eventEmitter?.emit("realtime:tool:error", {
|
|
493
|
-
name,
|
|
494
|
-
error,
|
|
495
|
-
callId
|
|
496
|
-
});
|
|
497
|
-
}
|
|
498
810
|
ensureResponseStarted() {
|
|
499
811
|
if (this.state.response.status === "responding") {
|
|
500
812
|
return;
|
|
@@ -518,7 +830,7 @@ var RealtimeManagerImpl = class {
|
|
|
518
830
|
this.reconnectToken += 1;
|
|
519
831
|
const reconnectToken = this.reconnectToken;
|
|
520
832
|
if (this.state.response.status === "responding") {
|
|
521
|
-
this.
|
|
833
|
+
this.audioOutput.end();
|
|
522
834
|
this.state = {
|
|
523
835
|
...this.state,
|
|
524
836
|
connection: "connecting",
|
|
@@ -587,7 +899,7 @@ var RealtimeManagerImpl = class {
|
|
|
587
899
|
lastError: null
|
|
588
900
|
};
|
|
589
901
|
this.emitState();
|
|
590
|
-
void this.requestWakeLock();
|
|
902
|
+
void this.browserLifecycle.requestWakeLock();
|
|
591
903
|
this.eventEmitter?.emit("realtime:reconnect:success", {
|
|
592
904
|
attempts: attempt,
|
|
593
905
|
totalMs: Date.now() - startedAt,
|
|
@@ -641,7 +953,7 @@ var RealtimeManagerImpl = class {
|
|
|
641
953
|
}
|
|
642
954
|
applyError(error, connection) {
|
|
643
955
|
const typedError = core.toCharivoError("transport", error);
|
|
644
|
-
this.
|
|
956
|
+
this.audioOutput.end();
|
|
645
957
|
this.state = {
|
|
646
958
|
...this.state,
|
|
647
959
|
connection,
|
|
@@ -654,71 +966,6 @@ var RealtimeManagerImpl = class {
|
|
|
654
966
|
error: typedError.message
|
|
655
967
|
});
|
|
656
968
|
}
|
|
657
|
-
emitAudioStart() {
|
|
658
|
-
if (this.isAudioPlaybackActive) {
|
|
659
|
-
return;
|
|
660
|
-
}
|
|
661
|
-
this.isAudioPlaybackActive = true;
|
|
662
|
-
this.eventEmitter?.emit("tts:audio:start", {});
|
|
663
|
-
}
|
|
664
|
-
emitAudioEnd() {
|
|
665
|
-
if (!this.isAudioPlaybackActive) {
|
|
666
|
-
return;
|
|
667
|
-
}
|
|
668
|
-
this.isAudioPlaybackActive = false;
|
|
669
|
-
this.eventEmitter?.emit("tts:lipsync:update", { rms: 0 });
|
|
670
|
-
this.eventEmitter?.emit("tts:audio:end", {});
|
|
671
|
-
}
|
|
672
|
-
installBrowserLifecycleListeners() {
|
|
673
|
-
if (this.teardownBrowserLifecycle) {
|
|
674
|
-
return;
|
|
675
|
-
}
|
|
676
|
-
this.teardownBrowserLifecycle = core.subscribeBrowserLifecycle({
|
|
677
|
-
onHidden: () => {
|
|
678
|
-
void this.releaseWakeLock();
|
|
679
|
-
},
|
|
680
|
-
onPageHide: () => {
|
|
681
|
-
void this.releaseWakeLock();
|
|
682
|
-
},
|
|
683
|
-
onPageShow: () => {
|
|
684
|
-
void this.requestWakeLock();
|
|
685
|
-
},
|
|
686
|
-
onVisible: () => {
|
|
687
|
-
void this.requestWakeLock();
|
|
688
|
-
}
|
|
689
|
-
});
|
|
690
|
-
}
|
|
691
|
-
async requestWakeLock() {
|
|
692
|
-
if (this.state.session.status !== "active" || this.state.connection === "disconnecting" || typeof navigator === "undefined" || typeof document === "undefined" || document.visibilityState !== "visible" || !("wakeLock" in navigator)) {
|
|
693
|
-
return;
|
|
694
|
-
}
|
|
695
|
-
if (this.wakeLockSentinel) {
|
|
696
|
-
return;
|
|
697
|
-
}
|
|
698
|
-
const wakeLockApi = navigator;
|
|
699
|
-
try {
|
|
700
|
-
this.wakeLockSentinel = await wakeLockApi.wakeLock?.request("screen");
|
|
701
|
-
this.wakeLockSentinel?.addEventListener?.(
|
|
702
|
-
"release",
|
|
703
|
-
() => {
|
|
704
|
-
this.wakeLockSentinel = null;
|
|
705
|
-
},
|
|
706
|
-
{ once: true }
|
|
707
|
-
);
|
|
708
|
-
} catch {
|
|
709
|
-
}
|
|
710
|
-
}
|
|
711
|
-
async releaseWakeLock() {
|
|
712
|
-
if (!this.wakeLockSentinel) {
|
|
713
|
-
return;
|
|
714
|
-
}
|
|
715
|
-
const wakeLockSentinel = this.wakeLockSentinel;
|
|
716
|
-
this.wakeLockSentinel = null;
|
|
717
|
-
try {
|
|
718
|
-
await wakeLockSentinel.release();
|
|
719
|
-
} catch {
|
|
720
|
-
}
|
|
721
|
-
}
|
|
722
969
|
resolveNextSessionBaseConfig(config) {
|
|
723
970
|
return mergeSessionConfig(
|
|
724
971
|
this.sessionBaseConfig ?? this.options.defaultSessionConfig,
|
|
@@ -851,10 +1098,8 @@ var RealtimeManagerImpl = class {
|
|
|
851
1098
|
}
|
|
852
1099
|
finalizeFailedSession(error) {
|
|
853
1100
|
this.cancelReconnectLoop();
|
|
854
|
-
this.
|
|
855
|
-
this.
|
|
856
|
-
void this.releaseWakeLock();
|
|
857
|
-
this.emitAudioEnd();
|
|
1101
|
+
this.browserLifecycle.dispose();
|
|
1102
|
+
this.audioOutput.end();
|
|
858
1103
|
this.state = {
|
|
859
1104
|
...this.state,
|
|
860
1105
|
connection: "error",
|
|
@@ -878,10 +1123,8 @@ var RealtimeManagerImpl = class {
|
|
|
878
1123
|
}
|
|
879
1124
|
finalizeStoppedSession(emitSessionEnd, reason) {
|
|
880
1125
|
this.cancelReconnectLoop();
|
|
881
|
-
this.
|
|
882
|
-
this.
|
|
883
|
-
void this.releaseWakeLock();
|
|
884
|
-
this.emitAudioEnd();
|
|
1126
|
+
this.browserLifecycle.dispose();
|
|
1127
|
+
this.audioOutput.end();
|
|
885
1128
|
this.state = {
|
|
886
1129
|
...this.state,
|
|
887
1130
|
connection: "idle",
|
|
@@ -929,73 +1172,6 @@ var RealtimeManagerImpl = class {
|
|
|
929
1172
|
function createRealtimeManager(client, options) {
|
|
930
1173
|
return new RealtimeManagerImpl(client, options);
|
|
931
1174
|
}
|
|
932
|
-
function mergeSessionConfig(baseConfig, overrideConfig) {
|
|
933
|
-
if (!baseConfig && !overrideConfig) {
|
|
934
|
-
return void 0;
|
|
935
|
-
}
|
|
936
|
-
const { tools: _ignoredOverrideTools, ...overrideWithoutTools } = overrideConfig ?? {};
|
|
937
|
-
return {
|
|
938
|
-
...baseConfig,
|
|
939
|
-
...overrideWithoutTools
|
|
940
|
-
};
|
|
941
|
-
}
|
|
942
|
-
function mergeRealtimeState(current, partial) {
|
|
943
|
-
return {
|
|
944
|
-
connection: partial.connection ?? current.connection,
|
|
945
|
-
session: {
|
|
946
|
-
status: partial.session?.status ?? current.session.status,
|
|
947
|
-
config: partial.session?.config ?? current.session.config,
|
|
948
|
-
characterId: partial.session?.characterId ?? current.session.characterId
|
|
949
|
-
},
|
|
950
|
-
response: {
|
|
951
|
-
status: partial.response?.status ?? current.response.status,
|
|
952
|
-
text: partial.response?.text ?? current.response.text
|
|
953
|
-
},
|
|
954
|
-
lastError: partial.lastError ?? current.lastError
|
|
955
|
-
};
|
|
956
|
-
}
|
|
957
|
-
function createRealtimeSessionId() {
|
|
958
|
-
const randomUuid = globalThis.crypto?.randomUUID?.();
|
|
959
|
-
if (randomUuid) {
|
|
960
|
-
return randomUuid;
|
|
961
|
-
}
|
|
962
|
-
return `realtime-session-${Date.now()}-${nextFallbackSessionId++}`;
|
|
963
|
-
}
|
|
964
|
-
function isRecord(value) {
|
|
965
|
-
return typeof value === "object" && value !== null;
|
|
966
|
-
}
|
|
967
|
-
function createFailureOutput(error) {
|
|
968
|
-
return {
|
|
969
|
-
success: false,
|
|
970
|
-
error: error.message
|
|
971
|
-
};
|
|
972
|
-
}
|
|
973
|
-
async function withTimeout(promise, timeoutMs, toolName) {
|
|
974
|
-
let timeoutId;
|
|
975
|
-
try {
|
|
976
|
-
return await Promise.race([
|
|
977
|
-
promise,
|
|
978
|
-
new Promise((_, reject) => {
|
|
979
|
-
timeoutId = setTimeout(() => {
|
|
980
|
-
reject(
|
|
981
|
-
new Error(
|
|
982
|
-
`Realtime tool "${toolName}" timed out after ${timeoutMs}ms`
|
|
983
|
-
)
|
|
984
|
-
);
|
|
985
|
-
}, timeoutMs);
|
|
986
|
-
})
|
|
987
|
-
]);
|
|
988
|
-
} finally {
|
|
989
|
-
if (timeoutId !== void 0) {
|
|
990
|
-
clearTimeout(timeoutId);
|
|
991
|
-
}
|
|
992
|
-
}
|
|
993
|
-
}
|
|
994
|
-
function delay(ms) {
|
|
995
|
-
return new Promise((resolve) => {
|
|
996
|
-
setTimeout(resolve, ms);
|
|
997
|
-
});
|
|
998
|
-
}
|
|
999
1175
|
|
|
1000
1176
|
exports.DEFAULT_REALTIME_AGENT_INSTRUCTIONS = DEFAULT_REALTIME_AGENT_INSTRUCTIONS;
|
|
1001
1177
|
exports.RealtimeManagerImpl = RealtimeManagerImpl;
|