@charivo/realtime 0.7.1 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -0
- package/dist/{chunk-Q5T2AV7Q.mjs → chunk-E656Y2PO.mjs} +5 -44
- package/dist/chunk-E656Y2PO.mjs.map +1 -0
- package/dist/{chunk-D4BW4H2X.mjs → chunk-FRTYFCPV.mjs} +4 -3
- package/dist/chunk-FRTYFCPV.mjs.map +1 -0
- package/dist/{chunk-GSRVMEKE.mjs → chunk-GJ2NA3ZT.mjs} +4 -3
- package/dist/chunk-GJ2NA3ZT.mjs.map +1 -0
- package/dist/{chunk-AIQJUH3Y.mjs → chunk-X6YRHDTW.mjs} +43 -3
- package/dist/chunk-X6YRHDTW.mjs.map +1 -0
- package/dist/index.d.mts +11 -21
- package/dist/index.d.ts +11 -21
- package/dist/index.js +450 -273
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +450 -275
- package/dist/index.mjs.map +1 -1
- package/dist/openai/index.mjs +3 -3
- package/dist/openai-agents/index.mjs +3 -3
- package/dist/remote/index.mjs +4 -4
- package/package.json +1 -1
- package/dist/chunk-AIQJUH3Y.mjs.map +0 -1
- package/dist/chunk-D4BW4H2X.mjs.map +0 -1
- package/dist/chunk-GSRVMEKE.mjs.map +0 -1
- package/dist/chunk-Q5T2AV7Q.mjs.map +0 -1
package/dist/index.js
CHANGED
|
@@ -54,10 +54,419 @@ function ensureSentenceFragment(text) {
|
|
|
54
54
|
return text.trim().replace(/[.!?]+$/, "");
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
+
// src/internal/audio-output.ts
|
|
58
|
+
var RealtimeAudioOutput = class {
|
|
59
|
+
constructor(emit) {
|
|
60
|
+
this.emit = emit;
|
|
61
|
+
}
|
|
62
|
+
active = false;
|
|
63
|
+
isActive() {
|
|
64
|
+
return this.active;
|
|
65
|
+
}
|
|
66
|
+
start() {
|
|
67
|
+
if (this.active) {
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
this.active = true;
|
|
71
|
+
this.emit("tts:audio:start", {});
|
|
72
|
+
}
|
|
73
|
+
end() {
|
|
74
|
+
if (!this.active) {
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
this.active = false;
|
|
78
|
+
this.emit("tts:lipsync:update", { rms: 0 });
|
|
79
|
+
this.emit("tts:audio:end", {});
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
var RealtimeBrowserLifecycle = class {
|
|
83
|
+
constructor(getState) {
|
|
84
|
+
this.getState = getState;
|
|
85
|
+
}
|
|
86
|
+
wakeLockSentinel = null;
|
|
87
|
+
teardownBrowserLifecycle;
|
|
88
|
+
install() {
|
|
89
|
+
if (this.teardownBrowserLifecycle) {
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
this.teardownBrowserLifecycle = core.subscribeBrowserLifecycle({
|
|
93
|
+
onHidden: () => {
|
|
94
|
+
void this.releaseWakeLock();
|
|
95
|
+
},
|
|
96
|
+
onPageHide: () => {
|
|
97
|
+
void this.releaseWakeLock();
|
|
98
|
+
},
|
|
99
|
+
onPageShow: () => {
|
|
100
|
+
void this.requestWakeLock();
|
|
101
|
+
},
|
|
102
|
+
onVisible: () => {
|
|
103
|
+
void this.requestWakeLock();
|
|
104
|
+
}
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
dispose() {
|
|
108
|
+
this.teardownBrowserLifecycle?.();
|
|
109
|
+
this.teardownBrowserLifecycle = void 0;
|
|
110
|
+
void this.releaseWakeLock();
|
|
111
|
+
}
|
|
112
|
+
async requestWakeLock() {
|
|
113
|
+
const state = this.getState();
|
|
114
|
+
if (state.session.status !== "active" || state.connection === "disconnecting" || typeof navigator === "undefined" || typeof document === "undefined" || document.visibilityState !== "visible" || !("wakeLock" in navigator)) {
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
if (this.wakeLockSentinel) {
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
const wakeLockApi = navigator;
|
|
121
|
+
try {
|
|
122
|
+
this.wakeLockSentinel = await wakeLockApi.wakeLock?.request("screen");
|
|
123
|
+
this.wakeLockSentinel?.addEventListener?.(
|
|
124
|
+
"release",
|
|
125
|
+
() => {
|
|
126
|
+
this.wakeLockSentinel = null;
|
|
127
|
+
},
|
|
128
|
+
{ once: true }
|
|
129
|
+
);
|
|
130
|
+
} catch {
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
async releaseWakeLock() {
|
|
134
|
+
if (!this.wakeLockSentinel) {
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
const wakeLockSentinel = this.wakeLockSentinel;
|
|
138
|
+
this.wakeLockSentinel = null;
|
|
139
|
+
try {
|
|
140
|
+
await wakeLockSentinel.release();
|
|
141
|
+
} catch {
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
};
|
|
145
|
+
|
|
146
|
+
// src/internal/manager-state.ts
|
|
147
|
+
var nextFallbackSessionId = 1;
|
|
148
|
+
function mergeSessionConfig(baseConfig, overrideConfig) {
|
|
149
|
+
if (!baseConfig && !overrideConfig) {
|
|
150
|
+
return void 0;
|
|
151
|
+
}
|
|
152
|
+
const { tools: _ignoredOverrideTools, ...overrideWithoutTools } = overrideConfig ?? {};
|
|
153
|
+
return {
|
|
154
|
+
...baseConfig,
|
|
155
|
+
...overrideWithoutTools
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
function mergeRealtimeState(current, partial) {
|
|
159
|
+
return {
|
|
160
|
+
connection: partial.connection ?? current.connection,
|
|
161
|
+
session: {
|
|
162
|
+
status: partial.session?.status ?? current.session.status,
|
|
163
|
+
config: partial.session?.config ?? current.session.config,
|
|
164
|
+
characterId: partial.session?.characterId ?? current.session.characterId
|
|
165
|
+
},
|
|
166
|
+
response: {
|
|
167
|
+
status: partial.response?.status ?? current.response.status,
|
|
168
|
+
text: partial.response?.text ?? current.response.text
|
|
169
|
+
},
|
|
170
|
+
lastError: partial.lastError ?? current.lastError
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
function createRealtimeSessionId() {
|
|
174
|
+
const randomUuid = globalThis.crypto?.randomUUID?.();
|
|
175
|
+
if (randomUuid) {
|
|
176
|
+
return randomUuid;
|
|
177
|
+
}
|
|
178
|
+
return `realtime-session-${Date.now()}-${nextFallbackSessionId++}`;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
// src/internal/timing.ts
|
|
182
|
+
function delay(ms) {
|
|
183
|
+
return new Promise((resolve) => {
|
|
184
|
+
setTimeout(resolve, ms);
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// src/internal/tool-registry.ts
|
|
189
|
+
var RealtimeToolRegistry = class {
|
|
190
|
+
tools = /* @__PURE__ */ new Map();
|
|
191
|
+
register(tool) {
|
|
192
|
+
this.tools.set(tool.definition.name, tool);
|
|
193
|
+
}
|
|
194
|
+
unregister(name) {
|
|
195
|
+
this.tools.delete(name);
|
|
196
|
+
}
|
|
197
|
+
get(name) {
|
|
198
|
+
return this.tools.get(name);
|
|
199
|
+
}
|
|
200
|
+
getDefinitions() {
|
|
201
|
+
return Array.from(this.tools.values(), (tool) => ({
|
|
202
|
+
...tool.definition,
|
|
203
|
+
parameters: {
|
|
204
|
+
...tool.definition.parameters,
|
|
205
|
+
properties: { ...tool.definition.parameters.properties },
|
|
206
|
+
required: tool.definition.parameters.required ? [...tool.definition.parameters.required] : void 0
|
|
207
|
+
}
|
|
208
|
+
}));
|
|
209
|
+
}
|
|
210
|
+
};
|
|
211
|
+
function isRecord(value) {
|
|
212
|
+
return typeof value === "object" && value !== null;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
// src/internal/tool-args-validation.ts
|
|
216
|
+
function validateToolArguments(definition, args) {
|
|
217
|
+
if (!isRecord(args) || Array.isArray(args)) {
|
|
218
|
+
throw new Error(
|
|
219
|
+
`Realtime tool "${definition.name}" arguments failed schema validation: arguments must be an object`
|
|
220
|
+
);
|
|
221
|
+
}
|
|
222
|
+
const { parameters } = definition;
|
|
223
|
+
const required = parameters.required ?? [];
|
|
224
|
+
for (const propertyName of required) {
|
|
225
|
+
if (!hasOwnProperty(args, propertyName)) {
|
|
226
|
+
throw new Error(
|
|
227
|
+
`Realtime tool "${definition.name}" arguments failed schema validation: missing required property "${propertyName}"`
|
|
228
|
+
);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
for (const [propertyName, schema] of Object.entries(parameters.properties)) {
|
|
232
|
+
if (!hasOwnProperty(args, propertyName) || !isRecord(schema)) {
|
|
233
|
+
continue;
|
|
234
|
+
}
|
|
235
|
+
validateToolArgumentValue(
|
|
236
|
+
definition.name,
|
|
237
|
+
propertyName,
|
|
238
|
+
args[propertyName],
|
|
239
|
+
schema
|
|
240
|
+
);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
function validateToolArgumentValue(toolName, propertyName, value, schema) {
|
|
244
|
+
const enumValues = schema.enum;
|
|
245
|
+
if (Array.isArray(enumValues) && !enumValues.some((item) => Object.is(item, value))) {
|
|
246
|
+
throw new Error(
|
|
247
|
+
`Realtime tool "${toolName}" arguments failed schema validation: property "${propertyName}" must be one of ${formatEnumValues(enumValues)}`
|
|
248
|
+
);
|
|
249
|
+
}
|
|
250
|
+
const expectedType = schema.type;
|
|
251
|
+
if (typeof expectedType !== "string") {
|
|
252
|
+
return;
|
|
253
|
+
}
|
|
254
|
+
if (!isSupportedSchemaType(expectedType)) {
|
|
255
|
+
throw new Error(
|
|
256
|
+
`Realtime tool "${toolName}" arguments failed schema validation: property "${propertyName}" uses unsupported schema type "${expectedType}"`
|
|
257
|
+
);
|
|
258
|
+
}
|
|
259
|
+
if (!matchesSchemaType(value, expectedType)) {
|
|
260
|
+
throw new Error(
|
|
261
|
+
`Realtime tool "${toolName}" arguments failed schema validation: property "${propertyName}" must be ${expectedType}`
|
|
262
|
+
);
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
function matchesSchemaType(value, expectedType) {
|
|
266
|
+
switch (expectedType) {
|
|
267
|
+
case "string":
|
|
268
|
+
return typeof value === "string";
|
|
269
|
+
case "number":
|
|
270
|
+
return typeof value === "number" && Number.isFinite(value);
|
|
271
|
+
case "integer":
|
|
272
|
+
return typeof value === "number" && Number.isInteger(value);
|
|
273
|
+
case "boolean":
|
|
274
|
+
return typeof value === "boolean";
|
|
275
|
+
case "object":
|
|
276
|
+
return isRecord(value) && !Array.isArray(value);
|
|
277
|
+
case "array":
|
|
278
|
+
return Array.isArray(value);
|
|
279
|
+
case "null":
|
|
280
|
+
return value === null;
|
|
281
|
+
}
|
|
282
|
+
return false;
|
|
283
|
+
}
|
|
284
|
+
function isSupportedSchemaType(expectedType) {
|
|
285
|
+
return expectedType === "string" || expectedType === "number" || expectedType === "integer" || expectedType === "boolean" || expectedType === "object" || expectedType === "array" || expectedType === "null";
|
|
286
|
+
}
|
|
287
|
+
function formatEnumValues(values) {
|
|
288
|
+
return values.map((value) => JSON.stringify(value)).join(", ");
|
|
289
|
+
}
|
|
290
|
+
function hasOwnProperty(record, propertyName) {
|
|
291
|
+
return Object.prototype.hasOwnProperty.call(record, propertyName);
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
// src/internal/tool-execution.ts
|
|
295
|
+
function createFailureOutput(error) {
|
|
296
|
+
return {
|
|
297
|
+
success: false,
|
|
298
|
+
error: error.message
|
|
299
|
+
};
|
|
300
|
+
}
|
|
301
|
+
async function withTimeout(promise, timeoutMs, toolName) {
|
|
302
|
+
let timeoutId;
|
|
303
|
+
try {
|
|
304
|
+
return await Promise.race([
|
|
305
|
+
promise,
|
|
306
|
+
new Promise((_, reject) => {
|
|
307
|
+
timeoutId = setTimeout(() => {
|
|
308
|
+
reject(
|
|
309
|
+
new Error(
|
|
310
|
+
`Realtime tool "${toolName}" timed out after ${timeoutMs}ms`
|
|
311
|
+
)
|
|
312
|
+
);
|
|
313
|
+
}, timeoutMs);
|
|
314
|
+
})
|
|
315
|
+
]);
|
|
316
|
+
} finally {
|
|
317
|
+
if (timeoutId !== void 0) {
|
|
318
|
+
clearTimeout(timeoutId);
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
// src/internal/tool-runner.ts
|
|
324
|
+
async function executeRealtimeToolCall({
|
|
325
|
+
event,
|
|
326
|
+
tool,
|
|
327
|
+
client,
|
|
328
|
+
character,
|
|
329
|
+
state,
|
|
330
|
+
defaultToolTimeoutMs,
|
|
331
|
+
resultProjectors,
|
|
332
|
+
emit,
|
|
333
|
+
log
|
|
334
|
+
}) {
|
|
335
|
+
if (!event.callId) {
|
|
336
|
+
emitToolError(
|
|
337
|
+
event.name,
|
|
338
|
+
new Error(`Tool "${event.name}" is missing a call ID`),
|
|
339
|
+
event.callId,
|
|
340
|
+
emit,
|
|
341
|
+
log
|
|
342
|
+
);
|
|
343
|
+
return;
|
|
344
|
+
}
|
|
345
|
+
if (!tool) {
|
|
346
|
+
await handleToolExecutionFailure(
|
|
347
|
+
event.name,
|
|
348
|
+
event.callId,
|
|
349
|
+
new Error(`No realtime tool registered for "${event.name}"`),
|
|
350
|
+
client,
|
|
351
|
+
emit,
|
|
352
|
+
log
|
|
353
|
+
);
|
|
354
|
+
return;
|
|
355
|
+
}
|
|
356
|
+
try {
|
|
357
|
+
log("debug", "Realtime tool execution started", {
|
|
358
|
+
name: event.name,
|
|
359
|
+
callId: event.callId
|
|
360
|
+
});
|
|
361
|
+
const output = await runToolHandler({
|
|
362
|
+
tool,
|
|
363
|
+
event,
|
|
364
|
+
character,
|
|
365
|
+
state,
|
|
366
|
+
defaultToolTimeoutMs
|
|
367
|
+
});
|
|
368
|
+
await client.sendToolResult(event.callId, output);
|
|
369
|
+
emit("realtime:tool:result", {
|
|
370
|
+
name: event.name,
|
|
371
|
+
output,
|
|
372
|
+
callId: event.callId
|
|
373
|
+
});
|
|
374
|
+
projectToolResult(
|
|
375
|
+
event.name,
|
|
376
|
+
output,
|
|
377
|
+
resultProjectors,
|
|
378
|
+
emit,
|
|
379
|
+
log,
|
|
380
|
+
event.callId
|
|
381
|
+
);
|
|
382
|
+
log("info", "Realtime tool execution succeeded", {
|
|
383
|
+
name: event.name,
|
|
384
|
+
callId: event.callId
|
|
385
|
+
});
|
|
386
|
+
} catch (error) {
|
|
387
|
+
await handleToolExecutionFailure(
|
|
388
|
+
event.name,
|
|
389
|
+
event.callId,
|
|
390
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
391
|
+
client,
|
|
392
|
+
emit,
|
|
393
|
+
log
|
|
394
|
+
);
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
async function runToolHandler({
|
|
398
|
+
tool,
|
|
399
|
+
event,
|
|
400
|
+
character,
|
|
401
|
+
state,
|
|
402
|
+
defaultToolTimeoutMs
|
|
403
|
+
}) {
|
|
404
|
+
validateToolArguments(tool.definition, event.args);
|
|
405
|
+
const result = await withTimeout(
|
|
406
|
+
tool.handler(event.args, {
|
|
407
|
+
character,
|
|
408
|
+
state,
|
|
409
|
+
callId: event.callId
|
|
410
|
+
}),
|
|
411
|
+
tool.timeoutMs ?? defaultToolTimeoutMs,
|
|
412
|
+
tool.definition.name
|
|
413
|
+
);
|
|
414
|
+
if (!isRecord(result)) {
|
|
415
|
+
throw new Error(
|
|
416
|
+
`Realtime tool "${tool.definition.name}" must return an object`
|
|
417
|
+
);
|
|
418
|
+
}
|
|
419
|
+
return result;
|
|
420
|
+
}
|
|
421
|
+
async function handleToolExecutionFailure(name, callId, error, client, emit, log) {
|
|
422
|
+
emitToolError(name, error, callId, emit, log);
|
|
423
|
+
try {
|
|
424
|
+
await client.sendToolResult(callId, createFailureOutput(error));
|
|
425
|
+
} catch (sendError) {
|
|
426
|
+
emitToolError(
|
|
427
|
+
name,
|
|
428
|
+
sendError instanceof Error ? sendError : new Error(String(sendError)),
|
|
429
|
+
callId,
|
|
430
|
+
emit,
|
|
431
|
+
log
|
|
432
|
+
);
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
function projectToolResult(name, output, resultProjectors, emit, log, callId) {
|
|
436
|
+
for (const projector of resultProjectors ?? []) {
|
|
437
|
+
try {
|
|
438
|
+
projector({
|
|
439
|
+
name,
|
|
440
|
+
output,
|
|
441
|
+
callId,
|
|
442
|
+
emit
|
|
443
|
+
});
|
|
444
|
+
} catch (error) {
|
|
445
|
+
const projectorError = error instanceof Error ? error : new Error(String(error));
|
|
446
|
+
log("warn", "Realtime result projector failed", {
|
|
447
|
+
name,
|
|
448
|
+
callId,
|
|
449
|
+
error: projectorError.message
|
|
450
|
+
});
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
function emitToolError(name, error, callId, emit, log) {
|
|
455
|
+
log("warn", "Realtime tool execution failed", {
|
|
456
|
+
name,
|
|
457
|
+
callId,
|
|
458
|
+
error: error.message
|
|
459
|
+
});
|
|
460
|
+
emit("realtime:tool:error", {
|
|
461
|
+
name,
|
|
462
|
+
error,
|
|
463
|
+
callId
|
|
464
|
+
});
|
|
465
|
+
}
|
|
466
|
+
|
|
57
467
|
// src/realtime-manager.ts
|
|
58
468
|
var DEFAULT_TOOL_TIMEOUT_MS = 1e4;
|
|
59
469
|
var RECONNECT_DELAYS_MS = [500, 1e3, 2e3, 4e3, 5e3];
|
|
60
|
-
var nextFallbackSessionId = 1;
|
|
61
470
|
var RealtimeManagerImpl = class {
|
|
62
471
|
constructor(client, options = {}) {
|
|
63
472
|
this.client = client;
|
|
@@ -65,7 +474,7 @@ var RealtimeManagerImpl = class {
|
|
|
65
474
|
for (const tool of options.tools ?? []) {
|
|
66
475
|
this.registerTool(tool);
|
|
67
476
|
}
|
|
68
|
-
this.
|
|
477
|
+
this.browserLifecycle.install();
|
|
69
478
|
this.client.onEvent((event) => {
|
|
70
479
|
void this.handleClientEvent(event).catch((error) => {
|
|
71
480
|
this.applyError(
|
|
@@ -80,17 +489,20 @@ var RealtimeManagerImpl = class {
|
|
|
80
489
|
isStoppingSession = false;
|
|
81
490
|
isRefreshingSession = false;
|
|
82
491
|
isRecoveringConnection = false;
|
|
83
|
-
isAudioPlaybackActive = false;
|
|
84
492
|
sessionBaseConfig;
|
|
85
493
|
refreshInFlight = null;
|
|
86
494
|
reconnectInFlight = null;
|
|
87
495
|
hasQueuedRefresh = false;
|
|
88
496
|
stopRequestedDuringRefresh = false;
|
|
89
497
|
reconnectToken = 0;
|
|
90
|
-
wakeLockSentinel = null;
|
|
91
|
-
teardownBrowserLifecycle;
|
|
92
498
|
sessionId;
|
|
93
|
-
toolRegistry =
|
|
499
|
+
toolRegistry = new RealtimeToolRegistry();
|
|
500
|
+
audioOutput = new RealtimeAudioOutput((event, payload) => {
|
|
501
|
+
this.eventEmitter?.emit(event, payload);
|
|
502
|
+
});
|
|
503
|
+
browserLifecycle = new RealtimeBrowserLifecycle(
|
|
504
|
+
() => this.state
|
|
505
|
+
);
|
|
94
506
|
state = {
|
|
95
507
|
connection: "idle",
|
|
96
508
|
session: {
|
|
@@ -118,20 +530,13 @@ var RealtimeManagerImpl = class {
|
|
|
118
530
|
this.emitState();
|
|
119
531
|
}
|
|
120
532
|
registerTool(tool) {
|
|
121
|
-
this.toolRegistry.
|
|
533
|
+
this.toolRegistry.register(tool);
|
|
122
534
|
}
|
|
123
535
|
unregisterTool(name) {
|
|
124
|
-
this.toolRegistry.
|
|
536
|
+
this.toolRegistry.unregister(name);
|
|
125
537
|
}
|
|
126
538
|
getRegisteredTools() {
|
|
127
|
-
return
|
|
128
|
-
...tool.definition,
|
|
129
|
-
parameters: {
|
|
130
|
-
...tool.definition.parameters,
|
|
131
|
-
properties: { ...tool.definition.parameters.properties },
|
|
132
|
-
required: tool.definition.parameters.required ? [...tool.definition.parameters.required] : void 0
|
|
133
|
-
}
|
|
134
|
-
}));
|
|
539
|
+
return this.toolRegistry.getDefinitions();
|
|
135
540
|
}
|
|
136
541
|
getState() {
|
|
137
542
|
return {
|
|
@@ -169,12 +574,12 @@ var RealtimeManagerImpl = class {
|
|
|
169
574
|
},
|
|
170
575
|
lastError: null
|
|
171
576
|
};
|
|
172
|
-
this.
|
|
577
|
+
this.browserLifecycle.install();
|
|
173
578
|
this.emitState();
|
|
174
579
|
try {
|
|
175
580
|
await this.client.connect(sessionConfig);
|
|
176
581
|
this.commitActiveSession(sessionConfig);
|
|
177
|
-
void this.requestWakeLock();
|
|
582
|
+
void this.browserLifecycle.requestWakeLock();
|
|
178
583
|
this.emitSessionStart("user");
|
|
179
584
|
} catch (error) {
|
|
180
585
|
const typedError = core.toCharivoError("transport", error);
|
|
@@ -266,7 +671,7 @@ var RealtimeManagerImpl = class {
|
|
|
266
671
|
status: "interrupted"
|
|
267
672
|
}
|
|
268
673
|
};
|
|
269
|
-
this.
|
|
674
|
+
this.audioOutput.end();
|
|
270
675
|
this.emitState();
|
|
271
676
|
}
|
|
272
677
|
async handleClientEvent(event) {
|
|
@@ -345,14 +750,14 @@ var RealtimeManagerImpl = class {
|
|
|
345
750
|
return;
|
|
346
751
|
}
|
|
347
752
|
case "audio.output.started":
|
|
348
|
-
this.
|
|
753
|
+
this.audioOutput.start();
|
|
349
754
|
return;
|
|
350
755
|
case "audio.output.ended":
|
|
351
|
-
this.
|
|
756
|
+
this.audioOutput.end();
|
|
352
757
|
return;
|
|
353
758
|
case "audio.lipsync":
|
|
354
|
-
if (event.rms > 1e-3 && !this.
|
|
355
|
-
this.
|
|
759
|
+
if (event.rms > 1e-3 && !this.audioOutput.isActive()) {
|
|
760
|
+
this.audioOutput.start();
|
|
356
761
|
}
|
|
357
762
|
this.eventEmitter?.emit("tts:lipsync:update", { rms: event.rms });
|
|
358
763
|
return;
|
|
@@ -362,7 +767,21 @@ var RealtimeManagerImpl = class {
|
|
|
362
767
|
args: event.args,
|
|
363
768
|
callId: event.callId
|
|
364
769
|
});
|
|
365
|
-
await
|
|
770
|
+
await executeRealtimeToolCall({
|
|
771
|
+
event,
|
|
772
|
+
tool: this.toolRegistry.get(event.name),
|
|
773
|
+
client: this.client,
|
|
774
|
+
character: this.character,
|
|
775
|
+
state: this.getState(),
|
|
776
|
+
defaultToolTimeoutMs: this.options.defaultToolTimeoutMs ?? DEFAULT_TOOL_TIMEOUT_MS,
|
|
777
|
+
resultProjectors: this.options.resultProjectors,
|
|
778
|
+
emit: (eventName, payload) => {
|
|
779
|
+
this.eventEmitter?.emit(eventName, payload);
|
|
780
|
+
},
|
|
781
|
+
log: (level, message, context) => {
|
|
782
|
+
this.log(level, message, context);
|
|
783
|
+
}
|
|
784
|
+
});
|
|
366
785
|
return;
|
|
367
786
|
case "tool.result":
|
|
368
787
|
this.eventEmitter?.emit("realtime:tool:result", {
|
|
@@ -388,112 +807,6 @@ var RealtimeManagerImpl = class {
|
|
|
388
807
|
return;
|
|
389
808
|
}
|
|
390
809
|
}
|
|
391
|
-
async executeToolCall(event) {
|
|
392
|
-
const tool = this.toolRegistry.get(event.name);
|
|
393
|
-
if (!event.callId) {
|
|
394
|
-
this.emitToolError(
|
|
395
|
-
event.name,
|
|
396
|
-
new Error(`Tool "${event.name}" is missing a call ID`),
|
|
397
|
-
event.callId
|
|
398
|
-
);
|
|
399
|
-
return;
|
|
400
|
-
}
|
|
401
|
-
if (!tool) {
|
|
402
|
-
await this.handleToolExecutionFailure(
|
|
403
|
-
event.name,
|
|
404
|
-
event.callId,
|
|
405
|
-
new Error(`No realtime tool registered for "${event.name}"`)
|
|
406
|
-
);
|
|
407
|
-
return;
|
|
408
|
-
}
|
|
409
|
-
try {
|
|
410
|
-
this.log("debug", "Realtime tool execution started", {
|
|
411
|
-
name: event.name,
|
|
412
|
-
callId: event.callId
|
|
413
|
-
});
|
|
414
|
-
const output = await this.runToolHandler(tool, event);
|
|
415
|
-
await this.client.sendToolResult(event.callId, output);
|
|
416
|
-
this.eventEmitter?.emit("realtime:tool:result", {
|
|
417
|
-
name: event.name,
|
|
418
|
-
output,
|
|
419
|
-
callId: event.callId
|
|
420
|
-
});
|
|
421
|
-
this.projectToolResult(event.name, output, event.callId);
|
|
422
|
-
this.log("info", "Realtime tool execution succeeded", {
|
|
423
|
-
name: event.name,
|
|
424
|
-
callId: event.callId
|
|
425
|
-
});
|
|
426
|
-
} catch (error) {
|
|
427
|
-
await this.handleToolExecutionFailure(
|
|
428
|
-
event.name,
|
|
429
|
-
event.callId,
|
|
430
|
-
error instanceof Error ? error : new Error(String(error))
|
|
431
|
-
);
|
|
432
|
-
}
|
|
433
|
-
}
|
|
434
|
-
async runToolHandler(tool, event) {
|
|
435
|
-
const timeoutMs = tool.timeoutMs ?? this.options.defaultToolTimeoutMs ?? DEFAULT_TOOL_TIMEOUT_MS;
|
|
436
|
-
const result = await withTimeout(
|
|
437
|
-
tool.handler(event.args, {
|
|
438
|
-
character: this.character,
|
|
439
|
-
state: this.getState(),
|
|
440
|
-
callId: event.callId
|
|
441
|
-
}),
|
|
442
|
-
timeoutMs,
|
|
443
|
-
tool.definition.name
|
|
444
|
-
);
|
|
445
|
-
if (!isRecord(result)) {
|
|
446
|
-
throw new Error(
|
|
447
|
-
`Realtime tool "${tool.definition.name}" must return an object`
|
|
448
|
-
);
|
|
449
|
-
}
|
|
450
|
-
return result;
|
|
451
|
-
}
|
|
452
|
-
async handleToolExecutionFailure(name, callId, error) {
|
|
453
|
-
this.emitToolError(name, error, callId);
|
|
454
|
-
try {
|
|
455
|
-
await this.client.sendToolResult(callId, createFailureOutput(error));
|
|
456
|
-
} catch (sendError) {
|
|
457
|
-
this.emitToolError(
|
|
458
|
-
name,
|
|
459
|
-
sendError instanceof Error ? sendError : new Error(String(sendError)),
|
|
460
|
-
callId
|
|
461
|
-
);
|
|
462
|
-
}
|
|
463
|
-
}
|
|
464
|
-
projectToolResult(name, output, callId) {
|
|
465
|
-
for (const projector of this.options.resultProjectors ?? []) {
|
|
466
|
-
try {
|
|
467
|
-
projector({
|
|
468
|
-
name,
|
|
469
|
-
output,
|
|
470
|
-
callId,
|
|
471
|
-
emit: (event, payload) => {
|
|
472
|
-
this.eventEmitter?.emit(event, payload);
|
|
473
|
-
}
|
|
474
|
-
});
|
|
475
|
-
} catch (error) {
|
|
476
|
-
const projectorError = error instanceof Error ? error : new Error(String(error));
|
|
477
|
-
this.log("warn", "Realtime result projector failed", {
|
|
478
|
-
name,
|
|
479
|
-
callId,
|
|
480
|
-
error: projectorError.message
|
|
481
|
-
});
|
|
482
|
-
}
|
|
483
|
-
}
|
|
484
|
-
}
|
|
485
|
-
emitToolError(name, error, callId) {
|
|
486
|
-
this.log("warn", "Realtime tool execution failed", {
|
|
487
|
-
name,
|
|
488
|
-
callId,
|
|
489
|
-
error: error.message
|
|
490
|
-
});
|
|
491
|
-
this.eventEmitter?.emit("realtime:tool:error", {
|
|
492
|
-
name,
|
|
493
|
-
error,
|
|
494
|
-
callId
|
|
495
|
-
});
|
|
496
|
-
}
|
|
497
810
|
ensureResponseStarted() {
|
|
498
811
|
if (this.state.response.status === "responding") {
|
|
499
812
|
return;
|
|
@@ -517,7 +830,7 @@ var RealtimeManagerImpl = class {
|
|
|
517
830
|
this.reconnectToken += 1;
|
|
518
831
|
const reconnectToken = this.reconnectToken;
|
|
519
832
|
if (this.state.response.status === "responding") {
|
|
520
|
-
this.
|
|
833
|
+
this.audioOutput.end();
|
|
521
834
|
this.state = {
|
|
522
835
|
...this.state,
|
|
523
836
|
connection: "connecting",
|
|
@@ -586,7 +899,7 @@ var RealtimeManagerImpl = class {
|
|
|
586
899
|
lastError: null
|
|
587
900
|
};
|
|
588
901
|
this.emitState();
|
|
589
|
-
void this.requestWakeLock();
|
|
902
|
+
void this.browserLifecycle.requestWakeLock();
|
|
590
903
|
this.eventEmitter?.emit("realtime:reconnect:success", {
|
|
591
904
|
attempts: attempt,
|
|
592
905
|
totalMs: Date.now() - startedAt,
|
|
@@ -640,7 +953,7 @@ var RealtimeManagerImpl = class {
|
|
|
640
953
|
}
|
|
641
954
|
applyError(error, connection) {
|
|
642
955
|
const typedError = core.toCharivoError("transport", error);
|
|
643
|
-
this.
|
|
956
|
+
this.audioOutput.end();
|
|
644
957
|
this.state = {
|
|
645
958
|
...this.state,
|
|
646
959
|
connection,
|
|
@@ -653,71 +966,6 @@ var RealtimeManagerImpl = class {
|
|
|
653
966
|
error: typedError.message
|
|
654
967
|
});
|
|
655
968
|
}
|
|
656
|
-
emitAudioStart() {
|
|
657
|
-
if (this.isAudioPlaybackActive) {
|
|
658
|
-
return;
|
|
659
|
-
}
|
|
660
|
-
this.isAudioPlaybackActive = true;
|
|
661
|
-
this.eventEmitter?.emit("tts:audio:start", {});
|
|
662
|
-
}
|
|
663
|
-
emitAudioEnd() {
|
|
664
|
-
if (!this.isAudioPlaybackActive) {
|
|
665
|
-
return;
|
|
666
|
-
}
|
|
667
|
-
this.isAudioPlaybackActive = false;
|
|
668
|
-
this.eventEmitter?.emit("tts:lipsync:update", { rms: 0 });
|
|
669
|
-
this.eventEmitter?.emit("tts:audio:end", {});
|
|
670
|
-
}
|
|
671
|
-
installBrowserLifecycleListeners() {
|
|
672
|
-
if (this.teardownBrowserLifecycle) {
|
|
673
|
-
return;
|
|
674
|
-
}
|
|
675
|
-
this.teardownBrowserLifecycle = core.subscribeBrowserLifecycle({
|
|
676
|
-
onHidden: () => {
|
|
677
|
-
void this.releaseWakeLock();
|
|
678
|
-
},
|
|
679
|
-
onPageHide: () => {
|
|
680
|
-
void this.releaseWakeLock();
|
|
681
|
-
},
|
|
682
|
-
onPageShow: () => {
|
|
683
|
-
void this.requestWakeLock();
|
|
684
|
-
},
|
|
685
|
-
onVisible: () => {
|
|
686
|
-
void this.requestWakeLock();
|
|
687
|
-
}
|
|
688
|
-
});
|
|
689
|
-
}
|
|
690
|
-
async requestWakeLock() {
|
|
691
|
-
if (this.state.session.status !== "active" || this.state.connection === "disconnecting" || typeof navigator === "undefined" || typeof document === "undefined" || document.visibilityState !== "visible" || !("wakeLock" in navigator)) {
|
|
692
|
-
return;
|
|
693
|
-
}
|
|
694
|
-
if (this.wakeLockSentinel) {
|
|
695
|
-
return;
|
|
696
|
-
}
|
|
697
|
-
const wakeLockApi = navigator;
|
|
698
|
-
try {
|
|
699
|
-
this.wakeLockSentinel = await wakeLockApi.wakeLock?.request("screen");
|
|
700
|
-
this.wakeLockSentinel?.addEventListener?.(
|
|
701
|
-
"release",
|
|
702
|
-
() => {
|
|
703
|
-
this.wakeLockSentinel = null;
|
|
704
|
-
},
|
|
705
|
-
{ once: true }
|
|
706
|
-
);
|
|
707
|
-
} catch {
|
|
708
|
-
}
|
|
709
|
-
}
|
|
710
|
-
async releaseWakeLock() {
|
|
711
|
-
if (!this.wakeLockSentinel) {
|
|
712
|
-
return;
|
|
713
|
-
}
|
|
714
|
-
const wakeLockSentinel = this.wakeLockSentinel;
|
|
715
|
-
this.wakeLockSentinel = null;
|
|
716
|
-
try {
|
|
717
|
-
await wakeLockSentinel.release();
|
|
718
|
-
} catch {
|
|
719
|
-
}
|
|
720
|
-
}
|
|
721
969
|
resolveNextSessionBaseConfig(config) {
|
|
722
970
|
return mergeSessionConfig(
|
|
723
971
|
this.sessionBaseConfig ?? this.options.defaultSessionConfig,
|
|
@@ -850,10 +1098,8 @@ var RealtimeManagerImpl = class {
|
|
|
850
1098
|
}
|
|
851
1099
|
finalizeFailedSession(error) {
|
|
852
1100
|
this.cancelReconnectLoop();
|
|
853
|
-
this.
|
|
854
|
-
this.
|
|
855
|
-
void this.releaseWakeLock();
|
|
856
|
-
this.emitAudioEnd();
|
|
1101
|
+
this.browserLifecycle.dispose();
|
|
1102
|
+
this.audioOutput.end();
|
|
857
1103
|
this.state = {
|
|
858
1104
|
...this.state,
|
|
859
1105
|
connection: "error",
|
|
@@ -877,10 +1123,8 @@ var RealtimeManagerImpl = class {
|
|
|
877
1123
|
}
|
|
878
1124
|
finalizeStoppedSession(emitSessionEnd, reason) {
|
|
879
1125
|
this.cancelReconnectLoop();
|
|
880
|
-
this.
|
|
881
|
-
this.
|
|
882
|
-
void this.releaseWakeLock();
|
|
883
|
-
this.emitAudioEnd();
|
|
1126
|
+
this.browserLifecycle.dispose();
|
|
1127
|
+
this.audioOutput.end();
|
|
884
1128
|
this.state = {
|
|
885
1129
|
...this.state,
|
|
886
1130
|
connection: "idle",
|
|
@@ -928,73 +1172,6 @@ var RealtimeManagerImpl = class {
|
|
|
928
1172
|
function createRealtimeManager(client, options) {
|
|
929
1173
|
return new RealtimeManagerImpl(client, options);
|
|
930
1174
|
}
|
|
931
|
-
function mergeSessionConfig(baseConfig, overrideConfig) {
|
|
932
|
-
if (!baseConfig && !overrideConfig) {
|
|
933
|
-
return void 0;
|
|
934
|
-
}
|
|
935
|
-
const { tools: _ignoredOverrideTools, ...overrideWithoutTools } = overrideConfig ?? {};
|
|
936
|
-
return {
|
|
937
|
-
...baseConfig,
|
|
938
|
-
...overrideWithoutTools
|
|
939
|
-
};
|
|
940
|
-
}
|
|
941
|
-
function mergeRealtimeState(current, partial) {
|
|
942
|
-
return {
|
|
943
|
-
connection: partial.connection ?? current.connection,
|
|
944
|
-
session: {
|
|
945
|
-
status: partial.session?.status ?? current.session.status,
|
|
946
|
-
config: partial.session?.config ?? current.session.config,
|
|
947
|
-
characterId: partial.session?.characterId ?? current.session.characterId
|
|
948
|
-
},
|
|
949
|
-
response: {
|
|
950
|
-
status: partial.response?.status ?? current.response.status,
|
|
951
|
-
text: partial.response?.text ?? current.response.text
|
|
952
|
-
},
|
|
953
|
-
lastError: partial.lastError ?? current.lastError
|
|
954
|
-
};
|
|
955
|
-
}
|
|
956
|
-
function createRealtimeSessionId() {
|
|
957
|
-
const randomUuid = globalThis.crypto?.randomUUID?.();
|
|
958
|
-
if (randomUuid) {
|
|
959
|
-
return randomUuid;
|
|
960
|
-
}
|
|
961
|
-
return `realtime-session-${Date.now()}-${nextFallbackSessionId++}`;
|
|
962
|
-
}
|
|
963
|
-
function isRecord(value) {
|
|
964
|
-
return typeof value === "object" && value !== null;
|
|
965
|
-
}
|
|
966
|
-
function createFailureOutput(error) {
|
|
967
|
-
return {
|
|
968
|
-
success: false,
|
|
969
|
-
error: error.message
|
|
970
|
-
};
|
|
971
|
-
}
|
|
972
|
-
async function withTimeout(promise, timeoutMs, toolName) {
|
|
973
|
-
let timeoutId;
|
|
974
|
-
try {
|
|
975
|
-
return await Promise.race([
|
|
976
|
-
promise,
|
|
977
|
-
new Promise((_, reject) => {
|
|
978
|
-
timeoutId = setTimeout(() => {
|
|
979
|
-
reject(
|
|
980
|
-
new Error(
|
|
981
|
-
`Realtime tool "${toolName}" timed out after ${timeoutMs}ms`
|
|
982
|
-
)
|
|
983
|
-
);
|
|
984
|
-
}, timeoutMs);
|
|
985
|
-
})
|
|
986
|
-
]);
|
|
987
|
-
} finally {
|
|
988
|
-
if (timeoutId !== void 0) {
|
|
989
|
-
clearTimeout(timeoutId);
|
|
990
|
-
}
|
|
991
|
-
}
|
|
992
|
-
}
|
|
993
|
-
function delay(ms) {
|
|
994
|
-
return new Promise((resolve) => {
|
|
995
|
-
setTimeout(resolve, ms);
|
|
996
|
-
});
|
|
997
|
-
}
|
|
998
1175
|
|
|
999
1176
|
exports.DEFAULT_REALTIME_AGENT_INSTRUCTIONS = DEFAULT_REALTIME_AGENT_INSTRUCTIONS;
|
|
1000
1177
|
exports.RealtimeManagerImpl = RealtimeManagerImpl;
|