@charivo/core 0.15.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -4
- package/dist/index.cjs +267 -0
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +164 -12
- package/dist/index.d.ts +164 -12
- package/dist/index.js +265 -1
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -8,6 +8,12 @@ It exports:
|
|
|
8
8
|
- shared domain types such as `Character`, `Message`, and realtime session types
|
|
9
9
|
- interface contracts for LLM, render, TTS, STT, and realtime managers
|
|
10
10
|
- the typed `EventBus`
|
|
11
|
+
- `createLipSyncAnalyzer`: the shared audio-analysis utility the TTS manager
|
|
12
|
+
and realtime clients use to compute mouth-open RMS and emit
|
|
13
|
+
`tts:lipsync:update`
|
|
14
|
+
- modality-neutral tool contracts (`ToolDefinition`, `ToolRegistration`, ...)
|
|
15
|
+
and validation helpers (`validateToolArguments`, `assertToolResultObject`)
|
|
16
|
+
shared by `@charivo/llm` and `@charivo/realtime`
|
|
11
17
|
|
|
12
18
|
## Install
|
|
13
19
|
|
|
@@ -67,7 +73,7 @@ for driving avatar state from the app layer:
|
|
|
67
73
|
|
|
68
74
|
- `setLocalGaze(coords: GazeCoordinates): boolean` — drives local-presence gaze
|
|
69
75
|
(e.g. webcam face tracking). Returns `false` (no-op) while AI gaze owns the
|
|
70
|
-
avatar (the `
|
|
76
|
+
avatar (the `avatar:gaze` suspend window is active) or when the renderer has
|
|
71
77
|
no `lookAt`.
|
|
72
78
|
|
|
73
79
|
## Errors
|
|
@@ -83,6 +89,55 @@ Public methods throw typed errors exported from `@charivo/core`:
|
|
|
83
89
|
Prefer `instanceof CharivoError` or `error.code` checks over
|
|
84
90
|
`error.message.includes(...)`.
|
|
85
91
|
|
|
92
|
+
## Tool Contracts
|
|
93
|
+
|
|
94
|
+
`@charivo/core` owns a modality-neutral tool contract used by both
|
|
95
|
+
`@charivo/llm` (`LLMManager`) and `@charivo/realtime` (`RealtimeManager`), so a
|
|
96
|
+
tool built once (e.g. by `@charivo/avatar`) registers with either manager:
|
|
97
|
+
|
|
98
|
+
- `ToolDefinition`: `{ type: "function", name, description, parameters }`, a
|
|
99
|
+
JSON Schema-shaped function definition
|
|
100
|
+
- `ToolContext`: `{ character?, callId?, state? }` passed to a handler;
|
|
101
|
+
`state` (a `RealtimeState`) is present only for realtime sessions
|
|
102
|
+
- `ToolHandler`: `(args, context) => Promise<Record<string, unknown>>` — must
|
|
103
|
+
resolve to a plain object; arrays and primitives are rejected by the runners
|
|
104
|
+
- `ToolRegistration`: `{ definition, handler, timeoutMs? }`
|
|
105
|
+
- `ToolResultProjector` / `ToolResultProjectorContext`: `({ name, output,
|
|
106
|
+
callId?, emit }) => void`, run by a manager after a successful tool call so
|
|
107
|
+
app code can turn tool output into Charivo events (e.g. `avatar:expression`)
|
|
108
|
+
|
|
109
|
+
Validation helpers, used by both managers before/after a handler runs:
|
|
110
|
+
|
|
111
|
+
- `validateToolArguments(definition, args, toolLabel?)`: throws a plain
|
|
112
|
+
`Error` on the first schema violation. Enforces only required-key presence,
|
|
113
|
+
`enum` membership, and each property's top-level `type` — nested schemas,
|
|
114
|
+
`additionalProperties`, and numeric-length constraints are not validated.
|
|
115
|
+
- `assertToolResultObject(result, toolName, toolLabel?)`: throws a plain
|
|
116
|
+
`Error` unless `result` is a plain object.
|
|
117
|
+
|
|
118
|
+
### Deprecated Realtime Tool Aliases
|
|
119
|
+
|
|
120
|
+
`RealtimeTool`, `RealtimeToolContext`, `RealtimeToolHandler`, and
|
|
121
|
+
`RealtimeToolRegistration` are deprecated aliases of the neutral contracts
|
|
122
|
+
above, kept for backward compatibility. Prefer `ToolDefinition`, `ToolContext`,
|
|
123
|
+
`ToolHandler`, and `ToolRegistration`. Note `RealtimeToolContext.state` is
|
|
124
|
+
required, while the neutral `ToolContext.state` is optional.
|
|
125
|
+
|
|
126
|
+
### LLM Tool-Calling Contracts
|
|
127
|
+
|
|
128
|
+
- `LLMMessage`: role-discriminated union (`system`/`user`, `assistant` with
|
|
129
|
+
optional `toolCalls`, or `tool` with a required `toolCallId`) so
|
|
130
|
+
protocol-invalid combinations are unrepresentable for typed callers
|
|
131
|
+
- `LLMToolCall`: `{ id, name, arguments }`
|
|
132
|
+
- `LLMToolResponse`: `{ content, toolCalls? }`
|
|
133
|
+
- `LLMProvider.generateResponseWithTools?(messages, tools)`: optional
|
|
134
|
+
tool-calling variant a provider implements alongside `generateResponse`
|
|
135
|
+
- `LLMClient.callWithTools?(messages, tools)`: optional tool-calling variant a
|
|
136
|
+
client implements alongside `call`
|
|
137
|
+
- `LLMManager.registerTool?`, `unregisterTool?`, `getRegisteredTools?`,
|
|
138
|
+
`setToolInstructions?`: optional manager methods for managing the tool
|
|
139
|
+
registry and the tool-only system-prompt addendum
|
|
140
|
+
|
|
86
141
|
## Events
|
|
87
142
|
|
|
88
143
|
Important event names include:
|
|
@@ -111,8 +166,8 @@ Important event names include:
|
|
|
111
166
|
- `realtime:tool:result`
|
|
112
167
|
- `realtime:tool:error`
|
|
113
168
|
- `realtime:usage`
|
|
114
|
-
- `
|
|
115
|
-
- `
|
|
116
|
-
- `
|
|
169
|
+
- `avatar:expression`
|
|
170
|
+
- `avatar:motion`
|
|
171
|
+
- `avatar:gaze`
|
|
117
172
|
- `realtime:text:delta`
|
|
118
173
|
- `realtime:error`
|
package/dist/index.cjs
CHANGED
|
@@ -181,6 +181,256 @@ function subscribeBrowserLifecycle(callbacks) {
|
|
|
181
181
|
};
|
|
182
182
|
}
|
|
183
183
|
|
|
184
|
+
// src/lipsync-analyzer.ts
|
|
185
|
+
var LIPSYNC_FFT_SIZE = 256;
|
|
186
|
+
var LIPSYNC_SMOOTHING_TIME_CONSTANT = 0.8;
|
|
187
|
+
function computeLipSyncRms(frequencyData) {
|
|
188
|
+
const speechBandStart = Math.floor(frequencyData.length * 0.1);
|
|
189
|
+
const speechBandEnd = Math.floor(frequencyData.length * 0.6);
|
|
190
|
+
let sum = 0;
|
|
191
|
+
for (let index = speechBandStart; index < speechBandEnd; index += 1) {
|
|
192
|
+
const normalized = frequencyData[index] / 255;
|
|
193
|
+
sum += normalized * normalized;
|
|
194
|
+
}
|
|
195
|
+
const rms = Math.sqrt(sum / (speechBandEnd - speechBandStart));
|
|
196
|
+
return Math.min(rms * 1.7, 1);
|
|
197
|
+
}
|
|
198
|
+
var LipSyncAnalyzerImpl = class {
|
|
199
|
+
constructor(options) {
|
|
200
|
+
this.options = options;
|
|
201
|
+
}
|
|
202
|
+
audioContext;
|
|
203
|
+
analyser;
|
|
204
|
+
source;
|
|
205
|
+
attachedStream;
|
|
206
|
+
dataArray;
|
|
207
|
+
isActive = false;
|
|
208
|
+
animationFrameId;
|
|
209
|
+
async prepare() {
|
|
210
|
+
if (this.audioContext) {
|
|
211
|
+
return;
|
|
212
|
+
}
|
|
213
|
+
const audioContextConstructor = typeof window === "undefined" ? void 0 : window.AudioContext || window.webkitAudioContext;
|
|
214
|
+
if (!audioContextConstructor) {
|
|
215
|
+
throw new Error("AudioContext is not supported in this browser");
|
|
216
|
+
}
|
|
217
|
+
this.audioContext = new audioContextConstructor();
|
|
218
|
+
}
|
|
219
|
+
attachMediaElement(element) {
|
|
220
|
+
this.stop();
|
|
221
|
+
this.resetNodes();
|
|
222
|
+
try {
|
|
223
|
+
const audioContext = this.ensureAudioContext();
|
|
224
|
+
if (!audioContext) {
|
|
225
|
+
return;
|
|
226
|
+
}
|
|
227
|
+
const analyser = this.createAnalyser(audioContext);
|
|
228
|
+
const source = audioContext.createMediaElementSource(element);
|
|
229
|
+
source.connect(analyser);
|
|
230
|
+
analyser.connect(audioContext.destination);
|
|
231
|
+
this.startAnalysis(source, analyser);
|
|
232
|
+
if (audioContext.state === "suspended") {
|
|
233
|
+
element.addEventListener(
|
|
234
|
+
"play",
|
|
235
|
+
() => {
|
|
236
|
+
this.audioContext?.resume();
|
|
237
|
+
},
|
|
238
|
+
{ once: true }
|
|
239
|
+
);
|
|
240
|
+
}
|
|
241
|
+
} catch (error) {
|
|
242
|
+
this.options.onError?.(error);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
attachMediaStream(stream) {
|
|
246
|
+
if (this.attachedStream === stream) {
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
249
|
+
this.stop();
|
|
250
|
+
this.resetNodes();
|
|
251
|
+
try {
|
|
252
|
+
const audioContext = this.ensureAudioContext();
|
|
253
|
+
if (!audioContext) {
|
|
254
|
+
return;
|
|
255
|
+
}
|
|
256
|
+
const analyser = this.createAnalyser(audioContext);
|
|
257
|
+
const source = audioContext.createMediaStreamSource(stream);
|
|
258
|
+
source.connect(analyser);
|
|
259
|
+
this.startAnalysis(source, analyser);
|
|
260
|
+
this.attachedStream = stream;
|
|
261
|
+
} catch (error) {
|
|
262
|
+
this.options.onError?.(error);
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
pause() {
|
|
266
|
+
this.cancelLoop();
|
|
267
|
+
this.options.onRms(0);
|
|
268
|
+
}
|
|
269
|
+
resume() {
|
|
270
|
+
if (!this.isActive || this.animationFrameId !== void 0 || !this.analyser || !this.dataArray) {
|
|
271
|
+
return;
|
|
272
|
+
}
|
|
273
|
+
this.analyzeFrame();
|
|
274
|
+
}
|
|
275
|
+
stop() {
|
|
276
|
+
this.isActive = false;
|
|
277
|
+
this.attachedStream = void 0;
|
|
278
|
+
this.cancelLoop();
|
|
279
|
+
this.options.onRms(0);
|
|
280
|
+
}
|
|
281
|
+
async cleanup() {
|
|
282
|
+
this.stop();
|
|
283
|
+
this.resetNodes();
|
|
284
|
+
this.dataArray = void 0;
|
|
285
|
+
const audioContext = this.audioContext;
|
|
286
|
+
this.audioContext = void 0;
|
|
287
|
+
await audioContext?.close();
|
|
288
|
+
}
|
|
289
|
+
/**
|
|
290
|
+
* Create the context lazily so callers can attach without awaiting prepare().
|
|
291
|
+
*
|
|
292
|
+
* This relies on prepare() staying await-free before it assigns
|
|
293
|
+
* `this.audioContext`: the void call below runs its body synchronously, so
|
|
294
|
+
* the context is already visible when this returns on the same tick.
|
|
295
|
+
*/
|
|
296
|
+
ensureAudioContext() {
|
|
297
|
+
if (!this.audioContext) {
|
|
298
|
+
void this.prepare().catch((error) => {
|
|
299
|
+
this.options.onError?.(error);
|
|
300
|
+
});
|
|
301
|
+
}
|
|
302
|
+
return this.audioContext;
|
|
303
|
+
}
|
|
304
|
+
createAnalyser(audioContext) {
|
|
305
|
+
const analyser = audioContext.createAnalyser();
|
|
306
|
+
analyser.fftSize = LIPSYNC_FFT_SIZE;
|
|
307
|
+
analyser.smoothingTimeConstant = LIPSYNC_SMOOTHING_TIME_CONSTANT;
|
|
308
|
+
return analyser;
|
|
309
|
+
}
|
|
310
|
+
startAnalysis(source, analyser) {
|
|
311
|
+
this.source = source;
|
|
312
|
+
this.analyser = analyser;
|
|
313
|
+
this.dataArray = new Uint8Array(analyser.frequencyBinCount);
|
|
314
|
+
this.isActive = true;
|
|
315
|
+
this.analyzeFrame();
|
|
316
|
+
}
|
|
317
|
+
analyzeFrame() {
|
|
318
|
+
if (!this.isActive || !this.analyser || !this.dataArray) {
|
|
319
|
+
return;
|
|
320
|
+
}
|
|
321
|
+
this.analyser.getByteFrequencyData(this.dataArray);
|
|
322
|
+
this.options.onRms(computeLipSyncRms(this.dataArray));
|
|
323
|
+
this.animationFrameId = requestAnimationFrame(() => this.analyzeFrame());
|
|
324
|
+
}
|
|
325
|
+
cancelLoop() {
|
|
326
|
+
if (this.animationFrameId !== void 0) {
|
|
327
|
+
cancelAnimationFrame(this.animationFrameId);
|
|
328
|
+
this.animationFrameId = void 0;
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
resetNodes() {
|
|
332
|
+
if (this.source) {
|
|
333
|
+
this.source.disconnect();
|
|
334
|
+
this.source = void 0;
|
|
335
|
+
}
|
|
336
|
+
if (this.analyser) {
|
|
337
|
+
this.analyser.disconnect();
|
|
338
|
+
this.analyser = void 0;
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
};
|
|
342
|
+
function createLipSyncAnalyzer(options) {
|
|
343
|
+
return new LipSyncAnalyzerImpl(options);
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
// src/tool-validation.ts
|
|
347
|
+
function validateToolArguments(definition, args, toolLabel = "Tool") {
|
|
348
|
+
if (!isRecord(args) || Array.isArray(args)) {
|
|
349
|
+
throw new Error(
|
|
350
|
+
`${toolLabel} "${definition.name}" arguments failed schema validation: arguments must be an object`
|
|
351
|
+
);
|
|
352
|
+
}
|
|
353
|
+
const { parameters } = definition;
|
|
354
|
+
const required = parameters.required ?? [];
|
|
355
|
+
for (const propertyName of required) {
|
|
356
|
+
if (!hasOwnProperty(args, propertyName)) {
|
|
357
|
+
throw new Error(
|
|
358
|
+
`${toolLabel} "${definition.name}" arguments failed schema validation: missing required property "${propertyName}"`
|
|
359
|
+
);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
for (const [propertyName, schema] of Object.entries(parameters.properties)) {
|
|
363
|
+
if (!hasOwnProperty(args, propertyName) || !isRecord(schema)) {
|
|
364
|
+
continue;
|
|
365
|
+
}
|
|
366
|
+
validateToolArgumentValue(
|
|
367
|
+
definition.name,
|
|
368
|
+
propertyName,
|
|
369
|
+
args[propertyName],
|
|
370
|
+
schema,
|
|
371
|
+
toolLabel
|
|
372
|
+
);
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
function assertToolResultObject(result, toolName, toolLabel = "Tool") {
|
|
376
|
+
if (!isRecord(result) || Array.isArray(result)) {
|
|
377
|
+
throw new Error(`${toolLabel} "${toolName}" must return an object`);
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
function validateToolArgumentValue(toolName, propertyName, value, schema, toolLabel) {
|
|
381
|
+
const enumValues = schema.enum;
|
|
382
|
+
if (Array.isArray(enumValues) && !enumValues.some((item) => Object.is(item, value))) {
|
|
383
|
+
throw new Error(
|
|
384
|
+
`${toolLabel} "${toolName}" arguments failed schema validation: property "${propertyName}" must be one of ${formatEnumValues(enumValues)}`
|
|
385
|
+
);
|
|
386
|
+
}
|
|
387
|
+
const expectedType = schema.type;
|
|
388
|
+
if (typeof expectedType !== "string") {
|
|
389
|
+
return;
|
|
390
|
+
}
|
|
391
|
+
if (!isSupportedSchemaType(expectedType)) {
|
|
392
|
+
throw new Error(
|
|
393
|
+
`${toolLabel} "${toolName}" arguments failed schema validation: property "${propertyName}" uses unsupported schema type "${expectedType}"`
|
|
394
|
+
);
|
|
395
|
+
}
|
|
396
|
+
if (!matchesSchemaType(value, expectedType)) {
|
|
397
|
+
throw new Error(
|
|
398
|
+
`${toolLabel} "${toolName}" arguments failed schema validation: property "${propertyName}" must be ${expectedType}`
|
|
399
|
+
);
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
function matchesSchemaType(value, expectedType) {
|
|
403
|
+
switch (expectedType) {
|
|
404
|
+
case "string":
|
|
405
|
+
return typeof value === "string";
|
|
406
|
+
case "number":
|
|
407
|
+
return typeof value === "number" && Number.isFinite(value);
|
|
408
|
+
case "integer":
|
|
409
|
+
return typeof value === "number" && Number.isInteger(value);
|
|
410
|
+
case "boolean":
|
|
411
|
+
return typeof value === "boolean";
|
|
412
|
+
case "object":
|
|
413
|
+
return isRecord(value) && !Array.isArray(value);
|
|
414
|
+
case "array":
|
|
415
|
+
return Array.isArray(value);
|
|
416
|
+
case "null":
|
|
417
|
+
return value === null;
|
|
418
|
+
}
|
|
419
|
+
return false;
|
|
420
|
+
}
|
|
421
|
+
function isSupportedSchemaType(expectedType) {
|
|
422
|
+
return expectedType === "string" || expectedType === "number" || expectedType === "integer" || expectedType === "boolean" || expectedType === "object" || expectedType === "array" || expectedType === "null";
|
|
423
|
+
}
|
|
424
|
+
function formatEnumValues(values) {
|
|
425
|
+
return values.map((value) => JSON.stringify(value)).join(", ");
|
|
426
|
+
}
|
|
427
|
+
function hasOwnProperty(record, propertyName) {
|
|
428
|
+
return Object.prototype.hasOwnProperty.call(record, propertyName);
|
|
429
|
+
}
|
|
430
|
+
function isRecord(value) {
|
|
431
|
+
return typeof value === "object" && value !== null;
|
|
432
|
+
}
|
|
433
|
+
|
|
184
434
|
// src/index.ts
|
|
185
435
|
var Charivo = class {
|
|
186
436
|
eventBus;
|
|
@@ -220,6 +470,7 @@ var Charivo = class {
|
|
|
220
470
|
*/
|
|
221
471
|
attachLLM(manager) {
|
|
222
472
|
this.llmManager = manager;
|
|
473
|
+
this.connectLLMManagerEventEmitter(manager);
|
|
223
474
|
if (this.character) {
|
|
224
475
|
manager.setCharacter(this.character);
|
|
225
476
|
}
|
|
@@ -281,6 +532,14 @@ var Charivo = class {
|
|
|
281
532
|
manager.setEventEmitter(this.eventBus);
|
|
282
533
|
}
|
|
283
534
|
}
|
|
535
|
+
/**
|
|
536
|
+
* Connects the LLM manager to the event bus for tool-call event emission.
|
|
537
|
+
*/
|
|
538
|
+
connectLLMManagerEventEmitter(manager) {
|
|
539
|
+
if (manager.setEventEmitter) {
|
|
540
|
+
manager.setEventEmitter(this.eventBus);
|
|
541
|
+
}
|
|
542
|
+
}
|
|
284
543
|
/**
|
|
285
544
|
* Set the character for this Charivo instance.
|
|
286
545
|
* Automatically propagates to all attached managers (LLM, Renderer).
|
|
@@ -480,6 +739,11 @@ var Charivo = class {
|
|
|
480
739
|
} catch (error) {
|
|
481
740
|
recordError(error, "Failed to stop TTS during dispose");
|
|
482
741
|
}
|
|
742
|
+
try {
|
|
743
|
+
await this.ttsManager.dispose?.();
|
|
744
|
+
} catch (error) {
|
|
745
|
+
recordError(error, "Failed to release TTS resources during dispose");
|
|
746
|
+
}
|
|
483
747
|
}
|
|
484
748
|
if (this.sttManager) {
|
|
485
749
|
try {
|
|
@@ -531,9 +795,12 @@ exports.CharivoTransportError = CharivoTransportError;
|
|
|
531
795
|
exports.EventBus = EventBus;
|
|
532
796
|
exports.OPENAI_REALTIME_ADAPTER = OPENAI_REALTIME_ADAPTER;
|
|
533
797
|
exports.OPENAI_REALTIME_AGENTS_ADAPTER = OPENAI_REALTIME_AGENTS_ADAPTER;
|
|
798
|
+
exports.assertToolResultObject = assertToolResultObject;
|
|
799
|
+
exports.createLipSyncAnalyzer = createLipSyncAnalyzer;
|
|
534
800
|
exports.getErrorMessage = getErrorMessage;
|
|
535
801
|
exports.isCharivoError = isCharivoError;
|
|
536
802
|
exports.subscribeBrowserLifecycle = subscribeBrowserLifecycle;
|
|
537
803
|
exports.toCharivoError = toCharivoError;
|
|
804
|
+
exports.validateToolArguments = validateToolArguments;
|
|
538
805
|
//# sourceMappingURL=index.cjs.map
|
|
539
806
|
//# sourceMappingURL=index.cjs.map
|