@charivo/realtime 0.7.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -0
- package/dist/{chunk-AIQJUH3Y.mjs → chunk-PNBYFAER.mjs} +45 -4
- package/dist/chunk-PNBYFAER.mjs.map +1 -0
- package/dist/{chunk-D4BW4H2X.mjs → chunk-QD4QSGPO.mjs} +20 -10
- package/dist/chunk-QD4QSGPO.mjs.map +1 -0
- package/dist/{chunk-GSRVMEKE.mjs → chunk-VYMT3Z2W.mjs} +18 -8
- package/dist/chunk-VYMT3Z2W.mjs.map +1 -0
- package/dist/{chunk-Q5T2AV7Q.mjs → chunk-YQWOBEE5.mjs} +5 -44
- package/dist/chunk-YQWOBEE5.mjs.map +1 -0
- package/dist/index.d.mts +15 -21
- package/dist/index.d.ts +15 -21
- package/dist/index.js +502 -275
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +500 -276
- package/dist/index.mjs.map +1 -1
- package/dist/openai/index.js +16 -7
- package/dist/openai/index.js.map +1 -1
- package/dist/openai/index.mjs +3 -3
- package/dist/openai-agents/index.js +14 -5
- package/dist/openai-agents/index.js.map +1 -1
- package/dist/openai-agents/index.mjs +3 -3
- package/dist/remote/index.js +30 -12
- package/dist/remote/index.js.map +1 -1
- package/dist/remote/index.mjs +4 -4
- package/package.json +2 -2
- package/dist/chunk-AIQJUH3Y.mjs.map +0 -1
- package/dist/chunk-D4BW4H2X.mjs.map +0 -1
- package/dist/chunk-GSRVMEKE.mjs.map +0 -1
- package/dist/chunk-Q5T2AV7Q.mjs.map +0 -1
package/dist/index.mjs
CHANGED
|
@@ -1,10 +1,417 @@
|
|
|
1
|
-
import { buildRealtimeSessionConfig } from './chunk-
|
|
2
|
-
export { DEFAULT_REALTIME_AGENT_INSTRUCTIONS, buildRealtimeSessionConfig } from './chunk-
|
|
1
|
+
import { buildRealtimeSessionConfig, isRecord } from './chunk-PNBYFAER.mjs';
|
|
2
|
+
export { DEFAULT_REALTIME_AGENT_INSTRUCTIONS, buildRealtimeSessionConfig } from './chunk-PNBYFAER.mjs';
|
|
3
3
|
import { CharivoStateError, toCharivoError, subscribeBrowserLifecycle } from '@charivo/core';
|
|
4
4
|
|
|
5
|
+
// src/internal/audio-output.ts
|
|
6
|
+
var RealtimeAudioOutput = class {
|
|
7
|
+
constructor(emit) {
|
|
8
|
+
this.emit = emit;
|
|
9
|
+
}
|
|
10
|
+
active = false;
|
|
11
|
+
isActive() {
|
|
12
|
+
return this.active;
|
|
13
|
+
}
|
|
14
|
+
start() {
|
|
15
|
+
if (this.active) {
|
|
16
|
+
return;
|
|
17
|
+
}
|
|
18
|
+
this.active = true;
|
|
19
|
+
this.emit("tts:audio:start", {});
|
|
20
|
+
}
|
|
21
|
+
end() {
|
|
22
|
+
if (!this.active) {
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
this.active = false;
|
|
26
|
+
this.emit("tts:lipsync:update", { rms: 0 });
|
|
27
|
+
this.emit("tts:audio:end", {});
|
|
28
|
+
}
|
|
29
|
+
};
|
|
30
|
+
var RealtimeBrowserLifecycle = class {
|
|
31
|
+
constructor(getState) {
|
|
32
|
+
this.getState = getState;
|
|
33
|
+
}
|
|
34
|
+
wakeLockSentinel = null;
|
|
35
|
+
teardownBrowserLifecycle;
|
|
36
|
+
install() {
|
|
37
|
+
if (this.teardownBrowserLifecycle) {
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
this.teardownBrowserLifecycle = subscribeBrowserLifecycle({
|
|
41
|
+
onHidden: () => {
|
|
42
|
+
void this.releaseWakeLock();
|
|
43
|
+
},
|
|
44
|
+
onPageHide: () => {
|
|
45
|
+
void this.releaseWakeLock();
|
|
46
|
+
},
|
|
47
|
+
onPageShow: () => {
|
|
48
|
+
void this.requestWakeLock();
|
|
49
|
+
},
|
|
50
|
+
onVisible: () => {
|
|
51
|
+
void this.requestWakeLock();
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
dispose() {
|
|
56
|
+
this.teardownBrowserLifecycle?.();
|
|
57
|
+
this.teardownBrowserLifecycle = void 0;
|
|
58
|
+
void this.releaseWakeLock();
|
|
59
|
+
}
|
|
60
|
+
async requestWakeLock() {
|
|
61
|
+
const state = this.getState();
|
|
62
|
+
if (state.session.status !== "active" || state.connection === "disconnecting" || typeof navigator === "undefined" || typeof document === "undefined" || document.visibilityState !== "visible" || !("wakeLock" in navigator)) {
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
65
|
+
if (this.wakeLockSentinel) {
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
const wakeLockApi = navigator;
|
|
69
|
+
try {
|
|
70
|
+
this.wakeLockSentinel = await wakeLockApi.wakeLock?.request("screen");
|
|
71
|
+
this.wakeLockSentinel?.addEventListener?.(
|
|
72
|
+
"release",
|
|
73
|
+
() => {
|
|
74
|
+
this.wakeLockSentinel = null;
|
|
75
|
+
},
|
|
76
|
+
{ once: true }
|
|
77
|
+
);
|
|
78
|
+
} catch {
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
async releaseWakeLock() {
|
|
82
|
+
if (!this.wakeLockSentinel) {
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
const wakeLockSentinel = this.wakeLockSentinel;
|
|
86
|
+
this.wakeLockSentinel = null;
|
|
87
|
+
try {
|
|
88
|
+
await wakeLockSentinel.release();
|
|
89
|
+
} catch {
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
// src/internal/manager-state.ts
|
|
95
|
+
var nextFallbackSessionId = 1;
|
|
96
|
+
function mergeSessionConfig(baseConfig, overrideConfig) {
|
|
97
|
+
if (!baseConfig && !overrideConfig) {
|
|
98
|
+
return void 0;
|
|
99
|
+
}
|
|
100
|
+
const { tools: _ignoredOverrideTools, ...overrideWithoutTools } = overrideConfig ?? {};
|
|
101
|
+
return {
|
|
102
|
+
...baseConfig,
|
|
103
|
+
...overrideWithoutTools
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
function mergeRealtimeState(current, partial) {
|
|
107
|
+
return {
|
|
108
|
+
connection: partial.connection ?? current.connection,
|
|
109
|
+
session: {
|
|
110
|
+
status: partial.session?.status ?? current.session.status,
|
|
111
|
+
config: partial.session?.config ?? current.session.config,
|
|
112
|
+
characterId: partial.session?.characterId ?? current.session.characterId
|
|
113
|
+
},
|
|
114
|
+
response: {
|
|
115
|
+
status: partial.response?.status ?? current.response.status,
|
|
116
|
+
text: partial.response?.text ?? current.response.text
|
|
117
|
+
},
|
|
118
|
+
lastError: partial.lastError ?? current.lastError
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
function createRealtimeSessionId() {
|
|
122
|
+
const randomUuid = globalThis.crypto?.randomUUID?.();
|
|
123
|
+
if (randomUuid) {
|
|
124
|
+
return randomUuid;
|
|
125
|
+
}
|
|
126
|
+
return `realtime-session-${Date.now()}-${nextFallbackSessionId++}`;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// src/internal/timing.ts
|
|
130
|
+
function delay(ms) {
|
|
131
|
+
return new Promise((resolve) => {
|
|
132
|
+
setTimeout(resolve, ms);
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// src/internal/tool-registry.ts
|
|
137
|
+
var RealtimeToolRegistry = class {
|
|
138
|
+
tools = /* @__PURE__ */ new Map();
|
|
139
|
+
register(tool) {
|
|
140
|
+
this.tools.set(tool.definition.name, tool);
|
|
141
|
+
}
|
|
142
|
+
unregister(name) {
|
|
143
|
+
this.tools.delete(name);
|
|
144
|
+
}
|
|
145
|
+
get(name) {
|
|
146
|
+
return this.tools.get(name);
|
|
147
|
+
}
|
|
148
|
+
getDefinitions() {
|
|
149
|
+
return Array.from(this.tools.values(), (tool) => ({
|
|
150
|
+
...tool.definition,
|
|
151
|
+
parameters: {
|
|
152
|
+
...tool.definition.parameters,
|
|
153
|
+
properties: { ...tool.definition.parameters.properties },
|
|
154
|
+
required: tool.definition.parameters.required ? [...tool.definition.parameters.required] : void 0
|
|
155
|
+
}
|
|
156
|
+
}));
|
|
157
|
+
}
|
|
158
|
+
};
|
|
159
|
+
|
|
160
|
+
// src/internal/tool-args-validation.ts
|
|
161
|
+
function validateToolArguments(definition, args) {
|
|
162
|
+
if (!isRecord(args) || Array.isArray(args)) {
|
|
163
|
+
throw new Error(
|
|
164
|
+
`Realtime tool "${definition.name}" arguments failed schema validation: arguments must be an object`
|
|
165
|
+
);
|
|
166
|
+
}
|
|
167
|
+
const { parameters } = definition;
|
|
168
|
+
const required = parameters.required ?? [];
|
|
169
|
+
for (const propertyName of required) {
|
|
170
|
+
if (!hasOwnProperty(args, propertyName)) {
|
|
171
|
+
throw new Error(
|
|
172
|
+
`Realtime tool "${definition.name}" arguments failed schema validation: missing required property "${propertyName}"`
|
|
173
|
+
);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
for (const [propertyName, schema] of Object.entries(parameters.properties)) {
|
|
177
|
+
if (!hasOwnProperty(args, propertyName) || !isRecord(schema)) {
|
|
178
|
+
continue;
|
|
179
|
+
}
|
|
180
|
+
validateToolArgumentValue(
|
|
181
|
+
definition.name,
|
|
182
|
+
propertyName,
|
|
183
|
+
args[propertyName],
|
|
184
|
+
schema
|
|
185
|
+
);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
function validateToolArgumentValue(toolName, propertyName, value, schema) {
|
|
189
|
+
const enumValues = schema.enum;
|
|
190
|
+
if (Array.isArray(enumValues) && !enumValues.some((item) => Object.is(item, value))) {
|
|
191
|
+
throw new Error(
|
|
192
|
+
`Realtime tool "${toolName}" arguments failed schema validation: property "${propertyName}" must be one of ${formatEnumValues(enumValues)}`
|
|
193
|
+
);
|
|
194
|
+
}
|
|
195
|
+
const expectedType = schema.type;
|
|
196
|
+
if (typeof expectedType !== "string") {
|
|
197
|
+
return;
|
|
198
|
+
}
|
|
199
|
+
if (!isSupportedSchemaType(expectedType)) {
|
|
200
|
+
throw new Error(
|
|
201
|
+
`Realtime tool "${toolName}" arguments failed schema validation: property "${propertyName}" uses unsupported schema type "${expectedType}"`
|
|
202
|
+
);
|
|
203
|
+
}
|
|
204
|
+
if (!matchesSchemaType(value, expectedType)) {
|
|
205
|
+
throw new Error(
|
|
206
|
+
`Realtime tool "${toolName}" arguments failed schema validation: property "${propertyName}" must be ${expectedType}`
|
|
207
|
+
);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
function matchesSchemaType(value, expectedType) {
|
|
211
|
+
switch (expectedType) {
|
|
212
|
+
case "string":
|
|
213
|
+
return typeof value === "string";
|
|
214
|
+
case "number":
|
|
215
|
+
return typeof value === "number" && Number.isFinite(value);
|
|
216
|
+
case "integer":
|
|
217
|
+
return typeof value === "number" && Number.isInteger(value);
|
|
218
|
+
case "boolean":
|
|
219
|
+
return typeof value === "boolean";
|
|
220
|
+
case "object":
|
|
221
|
+
return isRecord(value) && !Array.isArray(value);
|
|
222
|
+
case "array":
|
|
223
|
+
return Array.isArray(value);
|
|
224
|
+
case "null":
|
|
225
|
+
return value === null;
|
|
226
|
+
}
|
|
227
|
+
return false;
|
|
228
|
+
}
|
|
229
|
+
function isSupportedSchemaType(expectedType) {
|
|
230
|
+
return expectedType === "string" || expectedType === "number" || expectedType === "integer" || expectedType === "boolean" || expectedType === "object" || expectedType === "array" || expectedType === "null";
|
|
231
|
+
}
|
|
232
|
+
function formatEnumValues(values) {
|
|
233
|
+
return values.map((value) => JSON.stringify(value)).join(", ");
|
|
234
|
+
}
|
|
235
|
+
function hasOwnProperty(record, propertyName) {
|
|
236
|
+
return Object.prototype.hasOwnProperty.call(record, propertyName);
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
// src/internal/tool-execution.ts
|
|
240
|
+
function createFailureOutput(error) {
|
|
241
|
+
return {
|
|
242
|
+
success: false,
|
|
243
|
+
error: error.message
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
async function withTimeout(promise, timeoutMs, toolName) {
|
|
247
|
+
let timeoutId;
|
|
248
|
+
try {
|
|
249
|
+
return await Promise.race([
|
|
250
|
+
promise,
|
|
251
|
+
new Promise((_, reject) => {
|
|
252
|
+
timeoutId = setTimeout(() => {
|
|
253
|
+
reject(
|
|
254
|
+
new Error(
|
|
255
|
+
`Realtime tool "${toolName}" timed out after ${timeoutMs}ms`
|
|
256
|
+
)
|
|
257
|
+
);
|
|
258
|
+
}, timeoutMs);
|
|
259
|
+
})
|
|
260
|
+
]);
|
|
261
|
+
} finally {
|
|
262
|
+
if (timeoutId !== void 0) {
|
|
263
|
+
clearTimeout(timeoutId);
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
// src/internal/tool-runner.ts
|
|
269
|
+
async function executeRealtimeToolCall({
|
|
270
|
+
event,
|
|
271
|
+
tool,
|
|
272
|
+
client,
|
|
273
|
+
character,
|
|
274
|
+
state,
|
|
275
|
+
defaultToolTimeoutMs,
|
|
276
|
+
resultProjectors,
|
|
277
|
+
emit,
|
|
278
|
+
log
|
|
279
|
+
}) {
|
|
280
|
+
if (!event.callId) {
|
|
281
|
+
emitToolError(
|
|
282
|
+
event.name,
|
|
283
|
+
new Error(`Tool "${event.name}" is missing a call ID`),
|
|
284
|
+
event.callId,
|
|
285
|
+
emit,
|
|
286
|
+
log
|
|
287
|
+
);
|
|
288
|
+
return;
|
|
289
|
+
}
|
|
290
|
+
if (!tool) {
|
|
291
|
+
await handleToolExecutionFailure(
|
|
292
|
+
event.name,
|
|
293
|
+
event.callId,
|
|
294
|
+
new Error(`No realtime tool registered for "${event.name}"`),
|
|
295
|
+
client,
|
|
296
|
+
emit,
|
|
297
|
+
log
|
|
298
|
+
);
|
|
299
|
+
return;
|
|
300
|
+
}
|
|
301
|
+
try {
|
|
302
|
+
log("debug", "Realtime tool execution started", {
|
|
303
|
+
name: event.name,
|
|
304
|
+
callId: event.callId
|
|
305
|
+
});
|
|
306
|
+
const output = await runToolHandler({
|
|
307
|
+
tool,
|
|
308
|
+
event,
|
|
309
|
+
character,
|
|
310
|
+
state,
|
|
311
|
+
defaultToolTimeoutMs
|
|
312
|
+
});
|
|
313
|
+
await client.sendToolResult(event.callId, output);
|
|
314
|
+
emit("realtime:tool:result", {
|
|
315
|
+
name: event.name,
|
|
316
|
+
output,
|
|
317
|
+
callId: event.callId
|
|
318
|
+
});
|
|
319
|
+
projectToolResult(
|
|
320
|
+
event.name,
|
|
321
|
+
output,
|
|
322
|
+
resultProjectors,
|
|
323
|
+
emit,
|
|
324
|
+
log,
|
|
325
|
+
event.callId
|
|
326
|
+
);
|
|
327
|
+
log("info", "Realtime tool execution succeeded", {
|
|
328
|
+
name: event.name,
|
|
329
|
+
callId: event.callId
|
|
330
|
+
});
|
|
331
|
+
} catch (error) {
|
|
332
|
+
await handleToolExecutionFailure(
|
|
333
|
+
event.name,
|
|
334
|
+
event.callId,
|
|
335
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
336
|
+
client,
|
|
337
|
+
emit,
|
|
338
|
+
log
|
|
339
|
+
);
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
async function runToolHandler({
|
|
343
|
+
tool,
|
|
344
|
+
event,
|
|
345
|
+
character,
|
|
346
|
+
state,
|
|
347
|
+
defaultToolTimeoutMs
|
|
348
|
+
}) {
|
|
349
|
+
validateToolArguments(tool.definition, event.args);
|
|
350
|
+
const result = await withTimeout(
|
|
351
|
+
tool.handler(event.args, {
|
|
352
|
+
character,
|
|
353
|
+
state,
|
|
354
|
+
callId: event.callId
|
|
355
|
+
}),
|
|
356
|
+
tool.timeoutMs ?? defaultToolTimeoutMs,
|
|
357
|
+
tool.definition.name
|
|
358
|
+
);
|
|
359
|
+
if (!isRecord(result)) {
|
|
360
|
+
throw new Error(
|
|
361
|
+
`Realtime tool "${tool.definition.name}" must return an object`
|
|
362
|
+
);
|
|
363
|
+
}
|
|
364
|
+
return result;
|
|
365
|
+
}
|
|
366
|
+
async function handleToolExecutionFailure(name, callId, error, client, emit, log) {
|
|
367
|
+
emitToolError(name, error, callId, emit, log);
|
|
368
|
+
try {
|
|
369
|
+
await client.sendToolResult(callId, createFailureOutput(error));
|
|
370
|
+
} catch (sendError) {
|
|
371
|
+
emitToolError(
|
|
372
|
+
name,
|
|
373
|
+
sendError instanceof Error ? sendError : new Error(String(sendError)),
|
|
374
|
+
callId,
|
|
375
|
+
emit,
|
|
376
|
+
log
|
|
377
|
+
);
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
function projectToolResult(name, output, resultProjectors, emit, log, callId) {
|
|
381
|
+
for (const projector of resultProjectors ?? []) {
|
|
382
|
+
try {
|
|
383
|
+
projector({
|
|
384
|
+
name,
|
|
385
|
+
output,
|
|
386
|
+
callId,
|
|
387
|
+
emit
|
|
388
|
+
});
|
|
389
|
+
} catch (error) {
|
|
390
|
+
const projectorError = error instanceof Error ? error : new Error(String(error));
|
|
391
|
+
log("warn", "Realtime result projector failed", {
|
|
392
|
+
name,
|
|
393
|
+
callId,
|
|
394
|
+
error: projectorError.message
|
|
395
|
+
});
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
function emitToolError(name, error, callId, emit, log) {
|
|
400
|
+
log("warn", "Realtime tool execution failed", {
|
|
401
|
+
name,
|
|
402
|
+
callId,
|
|
403
|
+
error: error.message
|
|
404
|
+
});
|
|
405
|
+
emit("realtime:tool:error", {
|
|
406
|
+
name,
|
|
407
|
+
error,
|
|
408
|
+
callId
|
|
409
|
+
});
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
// src/realtime-manager.ts
|
|
5
413
|
var DEFAULT_TOOL_TIMEOUT_MS = 1e4;
|
|
6
414
|
var RECONNECT_DELAYS_MS = [500, 1e3, 2e3, 4e3, 5e3];
|
|
7
|
-
var nextFallbackSessionId = 1;
|
|
8
415
|
var RealtimeManagerImpl = class {
|
|
9
416
|
constructor(client, options = {}) {
|
|
10
417
|
this.client = client;
|
|
@@ -12,7 +419,7 @@ var RealtimeManagerImpl = class {
|
|
|
12
419
|
for (const tool of options.tools ?? []) {
|
|
13
420
|
this.registerTool(tool);
|
|
14
421
|
}
|
|
15
|
-
this.
|
|
422
|
+
this.browserLifecycle.install();
|
|
16
423
|
this.client.onEvent((event) => {
|
|
17
424
|
void this.handleClientEvent(event).catch((error) => {
|
|
18
425
|
this.applyError(
|
|
@@ -27,17 +434,22 @@ var RealtimeManagerImpl = class {
|
|
|
27
434
|
isStoppingSession = false;
|
|
28
435
|
isRefreshingSession = false;
|
|
29
436
|
isRecoveringConnection = false;
|
|
30
|
-
isAudioPlaybackActive = false;
|
|
31
437
|
sessionBaseConfig;
|
|
32
438
|
refreshInFlight = null;
|
|
33
439
|
reconnectInFlight = null;
|
|
34
440
|
hasQueuedRefresh = false;
|
|
35
441
|
stopRequestedDuringRefresh = false;
|
|
36
442
|
reconnectToken = 0;
|
|
37
|
-
|
|
38
|
-
|
|
443
|
+
responseOutstanding = false;
|
|
444
|
+
hasPendingToolRefresh = false;
|
|
39
445
|
sessionId;
|
|
40
|
-
toolRegistry =
|
|
446
|
+
toolRegistry = new RealtimeToolRegistry();
|
|
447
|
+
audioOutput = new RealtimeAudioOutput((event, payload) => {
|
|
448
|
+
this.eventEmitter?.emit(event, payload);
|
|
449
|
+
});
|
|
450
|
+
browserLifecycle = new RealtimeBrowserLifecycle(
|
|
451
|
+
() => this.state
|
|
452
|
+
);
|
|
41
453
|
state = {
|
|
42
454
|
connection: "idle",
|
|
43
455
|
session: {
|
|
@@ -65,20 +477,15 @@ var RealtimeManagerImpl = class {
|
|
|
65
477
|
this.emitState();
|
|
66
478
|
}
|
|
67
479
|
registerTool(tool) {
|
|
68
|
-
this.toolRegistry.
|
|
480
|
+
this.toolRegistry.register(tool);
|
|
481
|
+
this.requestToolRefresh();
|
|
69
482
|
}
|
|
70
483
|
unregisterTool(name) {
|
|
71
|
-
this.toolRegistry.
|
|
484
|
+
this.toolRegistry.unregister(name);
|
|
485
|
+
this.requestToolRefresh();
|
|
72
486
|
}
|
|
73
487
|
getRegisteredTools() {
|
|
74
|
-
return
|
|
75
|
-
...tool.definition,
|
|
76
|
-
parameters: {
|
|
77
|
-
...tool.definition.parameters,
|
|
78
|
-
properties: { ...tool.definition.parameters.properties },
|
|
79
|
-
required: tool.definition.parameters.required ? [...tool.definition.parameters.required] : void 0
|
|
80
|
-
}
|
|
81
|
-
}));
|
|
488
|
+
return this.toolRegistry.getDefinitions();
|
|
82
489
|
}
|
|
83
490
|
getState() {
|
|
84
491
|
return {
|
|
@@ -116,12 +523,12 @@ var RealtimeManagerImpl = class {
|
|
|
116
523
|
},
|
|
117
524
|
lastError: null
|
|
118
525
|
};
|
|
119
|
-
this.
|
|
526
|
+
this.browserLifecycle.install();
|
|
120
527
|
this.emitState();
|
|
121
528
|
try {
|
|
122
529
|
await this.client.connect(sessionConfig);
|
|
123
530
|
this.commitActiveSession(sessionConfig);
|
|
124
|
-
void this.requestWakeLock();
|
|
531
|
+
void this.browserLifecycle.requestWakeLock();
|
|
125
532
|
this.emitSessionStart("user");
|
|
126
533
|
} catch (error) {
|
|
127
534
|
const typedError = toCharivoError("transport", error);
|
|
@@ -187,7 +594,16 @@ var RealtimeManagerImpl = class {
|
|
|
187
594
|
if (this.state.connection !== "connected") {
|
|
188
595
|
throw new CharivoStateError("Realtime session is reconnecting");
|
|
189
596
|
}
|
|
190
|
-
|
|
597
|
+
if (this.state.response.status === "responding" || this.responseOutstanding) {
|
|
598
|
+
throw new CharivoStateError(
|
|
599
|
+
"Response already in progress. Call interrupt() before sending a new message."
|
|
600
|
+
);
|
|
601
|
+
}
|
|
602
|
+
this.responseOutstanding = true;
|
|
603
|
+
await this.client.sendText(text).catch((error) => {
|
|
604
|
+
this.responseOutstanding = false;
|
|
605
|
+
return Promise.reject(toCharivoError("transport", error));
|
|
606
|
+
});
|
|
191
607
|
}
|
|
192
608
|
async sendAudioChunk(audio) {
|
|
193
609
|
if (this.state.session.status !== "active") {
|
|
@@ -206,6 +622,7 @@ var RealtimeManagerImpl = class {
|
|
|
206
622
|
throw new CharivoStateError("Realtime session is reconnecting");
|
|
207
623
|
}
|
|
208
624
|
await this.client.interrupt().catch((error) => Promise.reject(toCharivoError("transport", error)));
|
|
625
|
+
this.responseOutstanding = false;
|
|
209
626
|
this.state = {
|
|
210
627
|
...this.state,
|
|
211
628
|
response: {
|
|
@@ -213,8 +630,9 @@ var RealtimeManagerImpl = class {
|
|
|
213
630
|
status: "interrupted"
|
|
214
631
|
}
|
|
215
632
|
};
|
|
216
|
-
this.
|
|
633
|
+
this.audioOutput.end();
|
|
217
634
|
this.emitState();
|
|
635
|
+
this.flushPendingToolRefresh();
|
|
218
636
|
}
|
|
219
637
|
async handleClientEvent(event) {
|
|
220
638
|
switch (event.type) {
|
|
@@ -275,7 +693,9 @@ var RealtimeManagerImpl = class {
|
|
|
275
693
|
this.emitState();
|
|
276
694
|
return;
|
|
277
695
|
case "assistant.response.completed": {
|
|
696
|
+
this.responseOutstanding = false;
|
|
278
697
|
if (this.state.response.status === "interrupted") {
|
|
698
|
+
this.flushPendingToolRefresh();
|
|
279
699
|
return;
|
|
280
700
|
}
|
|
281
701
|
const text = event.text || this.state.response.text;
|
|
@@ -289,17 +709,18 @@ var RealtimeManagerImpl = class {
|
|
|
289
709
|
};
|
|
290
710
|
this.eventEmitter?.emit("realtime:assistant:done", { text });
|
|
291
711
|
this.emitState();
|
|
712
|
+
this.flushPendingToolRefresh();
|
|
292
713
|
return;
|
|
293
714
|
}
|
|
294
715
|
case "audio.output.started":
|
|
295
|
-
this.
|
|
716
|
+
this.audioOutput.start();
|
|
296
717
|
return;
|
|
297
718
|
case "audio.output.ended":
|
|
298
|
-
this.
|
|
719
|
+
this.audioOutput.end();
|
|
299
720
|
return;
|
|
300
721
|
case "audio.lipsync":
|
|
301
|
-
if (event.rms > 1e-3 && !this.
|
|
302
|
-
this.
|
|
722
|
+
if (event.rms > 1e-3 && !this.audioOutput.isActive()) {
|
|
723
|
+
this.audioOutput.start();
|
|
303
724
|
}
|
|
304
725
|
this.eventEmitter?.emit("tts:lipsync:update", { rms: event.rms });
|
|
305
726
|
return;
|
|
@@ -309,7 +730,21 @@ var RealtimeManagerImpl = class {
|
|
|
309
730
|
args: event.args,
|
|
310
731
|
callId: event.callId
|
|
311
732
|
});
|
|
312
|
-
await
|
|
733
|
+
await executeRealtimeToolCall({
|
|
734
|
+
event,
|
|
735
|
+
tool: this.toolRegistry.get(event.name),
|
|
736
|
+
client: this.client,
|
|
737
|
+
character: this.character,
|
|
738
|
+
state: this.getState(),
|
|
739
|
+
defaultToolTimeoutMs: this.options.defaultToolTimeoutMs ?? DEFAULT_TOOL_TIMEOUT_MS,
|
|
740
|
+
resultProjectors: this.options.resultProjectors,
|
|
741
|
+
emit: (eventName, payload) => {
|
|
742
|
+
this.eventEmitter?.emit(eventName, payload);
|
|
743
|
+
},
|
|
744
|
+
log: (level, message, context) => {
|
|
745
|
+
this.log(level, message, context);
|
|
746
|
+
}
|
|
747
|
+
});
|
|
313
748
|
return;
|
|
314
749
|
case "tool.result":
|
|
315
750
|
this.eventEmitter?.emit("realtime:tool:result", {
|
|
@@ -331,116 +766,21 @@ var RealtimeManagerImpl = class {
|
|
|
331
766
|
this.emitState();
|
|
332
767
|
return;
|
|
333
768
|
}
|
|
769
|
+
this.responseOutstanding = false;
|
|
770
|
+
if (this.state.response.status === "responding") {
|
|
771
|
+
this.audioOutput.end();
|
|
772
|
+
this.state = {
|
|
773
|
+
...this.state,
|
|
774
|
+
response: { status: "interrupted", text: this.state.response.text }
|
|
775
|
+
};
|
|
776
|
+
}
|
|
334
777
|
this.applyError(event.error, this.state.connection);
|
|
778
|
+
if (this.state.session.status === "active") {
|
|
779
|
+
this.flushPendingToolRefresh();
|
|
780
|
+
}
|
|
335
781
|
return;
|
|
336
782
|
}
|
|
337
783
|
}
|
|
338
|
-
async executeToolCall(event) {
|
|
339
|
-
const tool = this.toolRegistry.get(event.name);
|
|
340
|
-
if (!event.callId) {
|
|
341
|
-
this.emitToolError(
|
|
342
|
-
event.name,
|
|
343
|
-
new Error(`Tool "${event.name}" is missing a call ID`),
|
|
344
|
-
event.callId
|
|
345
|
-
);
|
|
346
|
-
return;
|
|
347
|
-
}
|
|
348
|
-
if (!tool) {
|
|
349
|
-
await this.handleToolExecutionFailure(
|
|
350
|
-
event.name,
|
|
351
|
-
event.callId,
|
|
352
|
-
new Error(`No realtime tool registered for "${event.name}"`)
|
|
353
|
-
);
|
|
354
|
-
return;
|
|
355
|
-
}
|
|
356
|
-
try {
|
|
357
|
-
this.log("debug", "Realtime tool execution started", {
|
|
358
|
-
name: event.name,
|
|
359
|
-
callId: event.callId
|
|
360
|
-
});
|
|
361
|
-
const output = await this.runToolHandler(tool, event);
|
|
362
|
-
await this.client.sendToolResult(event.callId, output);
|
|
363
|
-
this.eventEmitter?.emit("realtime:tool:result", {
|
|
364
|
-
name: event.name,
|
|
365
|
-
output,
|
|
366
|
-
callId: event.callId
|
|
367
|
-
});
|
|
368
|
-
this.projectToolResult(event.name, output, event.callId);
|
|
369
|
-
this.log("info", "Realtime tool execution succeeded", {
|
|
370
|
-
name: event.name,
|
|
371
|
-
callId: event.callId
|
|
372
|
-
});
|
|
373
|
-
} catch (error) {
|
|
374
|
-
await this.handleToolExecutionFailure(
|
|
375
|
-
event.name,
|
|
376
|
-
event.callId,
|
|
377
|
-
error instanceof Error ? error : new Error(String(error))
|
|
378
|
-
);
|
|
379
|
-
}
|
|
380
|
-
}
|
|
381
|
-
async runToolHandler(tool, event) {
|
|
382
|
-
const timeoutMs = tool.timeoutMs ?? this.options.defaultToolTimeoutMs ?? DEFAULT_TOOL_TIMEOUT_MS;
|
|
383
|
-
const result = await withTimeout(
|
|
384
|
-
tool.handler(event.args, {
|
|
385
|
-
character: this.character,
|
|
386
|
-
state: this.getState(),
|
|
387
|
-
callId: event.callId
|
|
388
|
-
}),
|
|
389
|
-
timeoutMs,
|
|
390
|
-
tool.definition.name
|
|
391
|
-
);
|
|
392
|
-
if (!isRecord(result)) {
|
|
393
|
-
throw new Error(
|
|
394
|
-
`Realtime tool "${tool.definition.name}" must return an object`
|
|
395
|
-
);
|
|
396
|
-
}
|
|
397
|
-
return result;
|
|
398
|
-
}
|
|
399
|
-
async handleToolExecutionFailure(name, callId, error) {
|
|
400
|
-
this.emitToolError(name, error, callId);
|
|
401
|
-
try {
|
|
402
|
-
await this.client.sendToolResult(callId, createFailureOutput(error));
|
|
403
|
-
} catch (sendError) {
|
|
404
|
-
this.emitToolError(
|
|
405
|
-
name,
|
|
406
|
-
sendError instanceof Error ? sendError : new Error(String(sendError)),
|
|
407
|
-
callId
|
|
408
|
-
);
|
|
409
|
-
}
|
|
410
|
-
}
|
|
411
|
-
projectToolResult(name, output, callId) {
|
|
412
|
-
for (const projector of this.options.resultProjectors ?? []) {
|
|
413
|
-
try {
|
|
414
|
-
projector({
|
|
415
|
-
name,
|
|
416
|
-
output,
|
|
417
|
-
callId,
|
|
418
|
-
emit: (event, payload) => {
|
|
419
|
-
this.eventEmitter?.emit(event, payload);
|
|
420
|
-
}
|
|
421
|
-
});
|
|
422
|
-
} catch (error) {
|
|
423
|
-
const projectorError = error instanceof Error ? error : new Error(String(error));
|
|
424
|
-
this.log("warn", "Realtime result projector failed", {
|
|
425
|
-
name,
|
|
426
|
-
callId,
|
|
427
|
-
error: projectorError.message
|
|
428
|
-
});
|
|
429
|
-
}
|
|
430
|
-
}
|
|
431
|
-
}
|
|
432
|
-
emitToolError(name, error, callId) {
|
|
433
|
-
this.log("warn", "Realtime tool execution failed", {
|
|
434
|
-
name,
|
|
435
|
-
callId,
|
|
436
|
-
error: error.message
|
|
437
|
-
});
|
|
438
|
-
this.eventEmitter?.emit("realtime:tool:error", {
|
|
439
|
-
name,
|
|
440
|
-
error,
|
|
441
|
-
callId
|
|
442
|
-
});
|
|
443
|
-
}
|
|
444
784
|
ensureResponseStarted() {
|
|
445
785
|
if (this.state.response.status === "responding") {
|
|
446
786
|
return;
|
|
@@ -460,11 +800,13 @@ var RealtimeManagerImpl = class {
|
|
|
460
800
|
if (this.state.session.status !== "active" || this.isStoppingSession || this.state.connection === "disconnecting" || this.isRecoveringConnection) {
|
|
461
801
|
return;
|
|
462
802
|
}
|
|
803
|
+
this.responseOutstanding = false;
|
|
804
|
+
this.hasPendingToolRefresh = false;
|
|
463
805
|
this.isRecoveringConnection = true;
|
|
464
806
|
this.reconnectToken += 1;
|
|
465
807
|
const reconnectToken = this.reconnectToken;
|
|
466
808
|
if (this.state.response.status === "responding") {
|
|
467
|
-
this.
|
|
809
|
+
this.audioOutput.end();
|
|
468
810
|
this.state = {
|
|
469
811
|
...this.state,
|
|
470
812
|
connection: "connecting",
|
|
@@ -533,7 +875,7 @@ var RealtimeManagerImpl = class {
|
|
|
533
875
|
lastError: null
|
|
534
876
|
};
|
|
535
877
|
this.emitState();
|
|
536
|
-
void this.requestWakeLock();
|
|
878
|
+
void this.browserLifecycle.requestWakeLock();
|
|
537
879
|
this.eventEmitter?.emit("realtime:reconnect:success", {
|
|
538
880
|
attempts: attempt,
|
|
539
881
|
totalMs: Date.now() - startedAt,
|
|
@@ -587,7 +929,7 @@ var RealtimeManagerImpl = class {
|
|
|
587
929
|
}
|
|
588
930
|
applyError(error, connection) {
|
|
589
931
|
const typedError = toCharivoError("transport", error);
|
|
590
|
-
this.
|
|
932
|
+
this.audioOutput.end();
|
|
591
933
|
this.state = {
|
|
592
934
|
...this.state,
|
|
593
935
|
connection,
|
|
@@ -600,71 +942,6 @@ var RealtimeManagerImpl = class {
|
|
|
600
942
|
error: typedError.message
|
|
601
943
|
});
|
|
602
944
|
}
|
|
603
|
-
emitAudioStart() {
|
|
604
|
-
if (this.isAudioPlaybackActive) {
|
|
605
|
-
return;
|
|
606
|
-
}
|
|
607
|
-
this.isAudioPlaybackActive = true;
|
|
608
|
-
this.eventEmitter?.emit("tts:audio:start", {});
|
|
609
|
-
}
|
|
610
|
-
emitAudioEnd() {
|
|
611
|
-
if (!this.isAudioPlaybackActive) {
|
|
612
|
-
return;
|
|
613
|
-
}
|
|
614
|
-
this.isAudioPlaybackActive = false;
|
|
615
|
-
this.eventEmitter?.emit("tts:lipsync:update", { rms: 0 });
|
|
616
|
-
this.eventEmitter?.emit("tts:audio:end", {});
|
|
617
|
-
}
|
|
618
|
-
installBrowserLifecycleListeners() {
|
|
619
|
-
if (this.teardownBrowserLifecycle) {
|
|
620
|
-
return;
|
|
621
|
-
}
|
|
622
|
-
this.teardownBrowserLifecycle = subscribeBrowserLifecycle({
|
|
623
|
-
onHidden: () => {
|
|
624
|
-
void this.releaseWakeLock();
|
|
625
|
-
},
|
|
626
|
-
onPageHide: () => {
|
|
627
|
-
void this.releaseWakeLock();
|
|
628
|
-
},
|
|
629
|
-
onPageShow: () => {
|
|
630
|
-
void this.requestWakeLock();
|
|
631
|
-
},
|
|
632
|
-
onVisible: () => {
|
|
633
|
-
void this.requestWakeLock();
|
|
634
|
-
}
|
|
635
|
-
});
|
|
636
|
-
}
|
|
637
|
-
async requestWakeLock() {
|
|
638
|
-
if (this.state.session.status !== "active" || this.state.connection === "disconnecting" || typeof navigator === "undefined" || typeof document === "undefined" || document.visibilityState !== "visible" || !("wakeLock" in navigator)) {
|
|
639
|
-
return;
|
|
640
|
-
}
|
|
641
|
-
if (this.wakeLockSentinel) {
|
|
642
|
-
return;
|
|
643
|
-
}
|
|
644
|
-
const wakeLockApi = navigator;
|
|
645
|
-
try {
|
|
646
|
-
this.wakeLockSentinel = await wakeLockApi.wakeLock?.request("screen");
|
|
647
|
-
this.wakeLockSentinel?.addEventListener?.(
|
|
648
|
-
"release",
|
|
649
|
-
() => {
|
|
650
|
-
this.wakeLockSentinel = null;
|
|
651
|
-
},
|
|
652
|
-
{ once: true }
|
|
653
|
-
);
|
|
654
|
-
} catch {
|
|
655
|
-
}
|
|
656
|
-
}
|
|
657
|
-
async releaseWakeLock() {
|
|
658
|
-
if (!this.wakeLockSentinel) {
|
|
659
|
-
return;
|
|
660
|
-
}
|
|
661
|
-
const wakeLockSentinel = this.wakeLockSentinel;
|
|
662
|
-
this.wakeLockSentinel = null;
|
|
663
|
-
try {
|
|
664
|
-
await wakeLockSentinel.release();
|
|
665
|
-
} catch {
|
|
666
|
-
}
|
|
667
|
-
}
|
|
668
945
|
resolveNextSessionBaseConfig(config) {
|
|
669
946
|
return mergeSessionConfig(
|
|
670
947
|
this.sessionBaseConfig ?? this.options.defaultSessionConfig,
|
|
@@ -730,6 +1007,20 @@ var RealtimeManagerImpl = class {
|
|
|
730
1007
|
this.isStoppingSession = false;
|
|
731
1008
|
}
|
|
732
1009
|
}
|
|
1010
|
+
requestToolRefresh() {
|
|
1011
|
+
if (this.state.session.status !== "active") return;
|
|
1012
|
+
if (this.state.response.status === "responding" || this.responseOutstanding) {
|
|
1013
|
+
this.hasPendingToolRefresh = true;
|
|
1014
|
+
return;
|
|
1015
|
+
}
|
|
1016
|
+
this.updateSession().catch(() => {
|
|
1017
|
+
});
|
|
1018
|
+
}
|
|
1019
|
+
flushPendingToolRefresh() {
|
|
1020
|
+
if (!this.hasPendingToolRefresh) return;
|
|
1021
|
+
this.hasPendingToolRefresh = false;
|
|
1022
|
+
this.requestToolRefresh();
|
|
1023
|
+
}
|
|
733
1024
|
async runRefreshLoop() {
|
|
734
1025
|
do {
|
|
735
1026
|
this.hasQueuedRefresh = false;
|
|
@@ -796,11 +1087,11 @@ var RealtimeManagerImpl = class {
|
|
|
796
1087
|
});
|
|
797
1088
|
}
|
|
798
1089
|
finalizeFailedSession(error) {
|
|
1090
|
+
this.responseOutstanding = false;
|
|
1091
|
+
this.hasPendingToolRefresh = false;
|
|
799
1092
|
this.cancelReconnectLoop();
|
|
800
|
-
this.
|
|
801
|
-
this.
|
|
802
|
-
void this.releaseWakeLock();
|
|
803
|
-
this.emitAudioEnd();
|
|
1093
|
+
this.browserLifecycle.dispose();
|
|
1094
|
+
this.audioOutput.end();
|
|
804
1095
|
this.state = {
|
|
805
1096
|
...this.state,
|
|
806
1097
|
connection: "error",
|
|
@@ -823,11 +1114,11 @@ var RealtimeManagerImpl = class {
|
|
|
823
1114
|
this.sessionId = void 0;
|
|
824
1115
|
}
|
|
825
1116
|
finalizeStoppedSession(emitSessionEnd, reason) {
|
|
1117
|
+
this.responseOutstanding = false;
|
|
1118
|
+
this.hasPendingToolRefresh = false;
|
|
826
1119
|
this.cancelReconnectLoop();
|
|
827
|
-
this.
|
|
828
|
-
this.
|
|
829
|
-
void this.releaseWakeLock();
|
|
830
|
-
this.emitAudioEnd();
|
|
1120
|
+
this.browserLifecycle.dispose();
|
|
1121
|
+
this.audioOutput.end();
|
|
831
1122
|
this.state = {
|
|
832
1123
|
...this.state,
|
|
833
1124
|
connection: "idle",
|
|
@@ -875,73 +1166,6 @@ var RealtimeManagerImpl = class {
|
|
|
875
1166
|
function createRealtimeManager(client, options) {
|
|
876
1167
|
return new RealtimeManagerImpl(client, options);
|
|
877
1168
|
}
|
|
878
|
-
function mergeSessionConfig(baseConfig, overrideConfig) {
|
|
879
|
-
if (!baseConfig && !overrideConfig) {
|
|
880
|
-
return void 0;
|
|
881
|
-
}
|
|
882
|
-
const { tools: _ignoredOverrideTools, ...overrideWithoutTools } = overrideConfig ?? {};
|
|
883
|
-
return {
|
|
884
|
-
...baseConfig,
|
|
885
|
-
...overrideWithoutTools
|
|
886
|
-
};
|
|
887
|
-
}
|
|
888
|
-
function mergeRealtimeState(current, partial) {
|
|
889
|
-
return {
|
|
890
|
-
connection: partial.connection ?? current.connection,
|
|
891
|
-
session: {
|
|
892
|
-
status: partial.session?.status ?? current.session.status,
|
|
893
|
-
config: partial.session?.config ?? current.session.config,
|
|
894
|
-
characterId: partial.session?.characterId ?? current.session.characterId
|
|
895
|
-
},
|
|
896
|
-
response: {
|
|
897
|
-
status: partial.response?.status ?? current.response.status,
|
|
898
|
-
text: partial.response?.text ?? current.response.text
|
|
899
|
-
},
|
|
900
|
-
lastError: partial.lastError ?? current.lastError
|
|
901
|
-
};
|
|
902
|
-
}
|
|
903
|
-
function createRealtimeSessionId() {
|
|
904
|
-
const randomUuid = globalThis.crypto?.randomUUID?.();
|
|
905
|
-
if (randomUuid) {
|
|
906
|
-
return randomUuid;
|
|
907
|
-
}
|
|
908
|
-
return `realtime-session-${Date.now()}-${nextFallbackSessionId++}`;
|
|
909
|
-
}
|
|
910
|
-
function isRecord(value) {
|
|
911
|
-
return typeof value === "object" && value !== null;
|
|
912
|
-
}
|
|
913
|
-
function createFailureOutput(error) {
|
|
914
|
-
return {
|
|
915
|
-
success: false,
|
|
916
|
-
error: error.message
|
|
917
|
-
};
|
|
918
|
-
}
|
|
919
|
-
async function withTimeout(promise, timeoutMs, toolName) {
|
|
920
|
-
let timeoutId;
|
|
921
|
-
try {
|
|
922
|
-
return await Promise.race([
|
|
923
|
-
promise,
|
|
924
|
-
new Promise((_, reject) => {
|
|
925
|
-
timeoutId = setTimeout(() => {
|
|
926
|
-
reject(
|
|
927
|
-
new Error(
|
|
928
|
-
`Realtime tool "${toolName}" timed out after ${timeoutMs}ms`
|
|
929
|
-
)
|
|
930
|
-
);
|
|
931
|
-
}, timeoutMs);
|
|
932
|
-
})
|
|
933
|
-
]);
|
|
934
|
-
} finally {
|
|
935
|
-
if (timeoutId !== void 0) {
|
|
936
|
-
clearTimeout(timeoutId);
|
|
937
|
-
}
|
|
938
|
-
}
|
|
939
|
-
}
|
|
940
|
-
function delay(ms) {
|
|
941
|
-
return new Promise((resolve) => {
|
|
942
|
-
setTimeout(resolve, ms);
|
|
943
|
-
});
|
|
944
|
-
}
|
|
945
1169
|
|
|
946
1170
|
export { RealtimeManagerImpl, createRealtimeManager };
|
|
947
1171
|
//# sourceMappingURL=index.mjs.map
|