@charivo/core 0.15.0 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -8,6 +8,12 @@ It exports:
8
8
  - shared domain types such as `Character`, `Message`, and realtime session types
9
9
  - interface contracts for LLM, render, TTS, STT, and realtime managers
10
10
  - the typed `EventBus`
11
+ - `createLipSyncAnalyzer`: the shared audio-analysis utility the TTS manager
12
+ and realtime clients use to compute mouth-open RMS and emit
13
+ `tts:lipsync:update`
14
+ - modality-neutral tool contracts (`ToolDefinition`, `ToolRegistration`, ...)
15
+ and validation helpers (`validateToolArguments`, `assertToolResultObject`)
16
+ shared by `@charivo/llm` and `@charivo/realtime`
11
17
 
12
18
  ## Install
13
19
 
@@ -67,7 +73,7 @@ for driving avatar state from the app layer:
67
73
 
68
74
  - `setLocalGaze(coords: GazeCoordinates): boolean` — drives local-presence gaze
69
75
  (e.g. webcam face tracking). Returns `false` (no-op) while AI gaze owns the
70
- avatar (the `realtime:gaze` suspend window is active) or when the renderer has
76
+ avatar (the `avatar:gaze` suspend window is active) or when the renderer has
71
77
  no `lookAt`.
72
78
 
73
79
  ## Errors
@@ -83,6 +89,55 @@ Public methods throw typed errors exported from `@charivo/core`:
83
89
  Prefer `instanceof CharivoError` or `error.code` checks over
84
90
  `error.message.includes(...)`.
85
91
 
92
+ ## Tool Contracts
93
+
94
+ `@charivo/core` owns a modality-neutral tool contract used by both
95
+ `@charivo/llm` (`LLMManager`) and `@charivo/realtime` (`RealtimeManager`), so a
96
+ tool built once (e.g. by `@charivo/avatar`) registers with either manager:
97
+
98
+ - `ToolDefinition`: `{ type: "function", name, description, parameters }`, a
99
+ JSON Schema-shaped function definition
100
+ - `ToolContext`: `{ character?, callId?, state? }` passed to a handler;
101
+ `state` (a `RealtimeState`) is present only for realtime sessions
102
+ - `ToolHandler`: `(args, context) => Promise<Record<string, unknown>>` — must
103
+ resolve to a plain object; arrays and primitives are rejected by the runners
104
+ - `ToolRegistration`: `{ definition, handler, timeoutMs? }`
105
+ - `ToolResultProjector` / `ToolResultProjectorContext`: `({ name, output,
106
+ callId?, emit }) => void`, run by a manager after a successful tool call so
107
+ app code can turn tool output into Charivo events (e.g. `avatar:expression`)
108
+
109
+ Validation helpers, used by both managers before/after a handler runs:
110
+
111
+ - `validateToolArguments(definition, args, toolLabel?)`: throws a plain
112
+ `Error` on the first schema violation. Enforces only required-key presence,
113
+ `enum` membership, and each property's top-level `type` — nested schemas,
114
+ `additionalProperties`, and numeric-length constraints are not validated.
115
+ - `assertToolResultObject(result, toolName, toolLabel?)`: throws a plain
116
+ `Error` unless `result` is a plain object.
117
+
118
+ ### Deprecated Realtime Tool Aliases
119
+
120
+ `RealtimeTool`, `RealtimeToolContext`, `RealtimeToolHandler`, and
121
+ `RealtimeToolRegistration` are deprecated aliases of the neutral contracts
122
+ above, kept for backward compatibility. Prefer `ToolDefinition`, `ToolContext`,
123
+ `ToolHandler`, and `ToolRegistration`. Note `RealtimeToolContext.state` is
124
+ required, while the neutral `ToolContext.state` is optional.
125
+
126
+ ### LLM Tool-Calling Contracts
127
+
128
+ - `LLMMessage`: role-discriminated union (`system`/`user`, `assistant` with
129
+ optional `toolCalls`, or `tool` with a required `toolCallId`) so
130
+ protocol-invalid combinations are unrepresentable for typed callers
131
+ - `LLMToolCall`: `{ id, name, arguments }`
132
+ - `LLMToolResponse`: `{ content, toolCalls? }`
133
+ - `LLMProvider.generateResponseWithTools?(messages, tools)`: optional
134
+ tool-calling variant a provider implements alongside `generateResponse`
135
+ - `LLMClient.callWithTools?(messages, tools)`: optional tool-calling variant a
136
+ client implements alongside `call`
137
+ - `LLMManager.registerTool?`, `unregisterTool?`, `getRegisteredTools?`,
138
+ `setToolInstructions?`: optional manager methods for managing the tool
139
+ registry and the tool-only system-prompt addendum
140
+
86
141
  ## Events
87
142
 
88
143
  Important event names include:
@@ -111,8 +166,8 @@ Important event names include:
111
166
  - `realtime:tool:result`
112
167
  - `realtime:tool:error`
113
168
  - `realtime:usage`
114
- - `realtime:expression`
115
- - `realtime:motion`
116
- - `realtime:gaze`
169
+ - `avatar:expression`
170
+ - `avatar:motion`
171
+ - `avatar:gaze`
117
172
  - `realtime:text:delta`
118
173
  - `realtime:error`
package/dist/index.cjs CHANGED
@@ -181,6 +181,256 @@ function subscribeBrowserLifecycle(callbacks) {
181
181
  };
182
182
  }
183
183
 
184
+ // src/lipsync-analyzer.ts
185
+ var LIPSYNC_FFT_SIZE = 256;
186
+ var LIPSYNC_SMOOTHING_TIME_CONSTANT = 0.8;
187
+ function computeLipSyncRms(frequencyData) {
188
+ const speechBandStart = Math.floor(frequencyData.length * 0.1);
189
+ const speechBandEnd = Math.floor(frequencyData.length * 0.6);
190
+ let sum = 0;
191
+ for (let index = speechBandStart; index < speechBandEnd; index += 1) {
192
+ const normalized = frequencyData[index] / 255;
193
+ sum += normalized * normalized;
194
+ }
195
+ const rms = Math.sqrt(sum / (speechBandEnd - speechBandStart));
196
+ return Math.min(rms * 1.7, 1);
197
+ }
198
+ var LipSyncAnalyzerImpl = class {
199
+ constructor(options) {
200
+ this.options = options;
201
+ }
202
+ audioContext;
203
+ analyser;
204
+ source;
205
+ attachedStream;
206
+ dataArray;
207
+ isActive = false;
208
+ animationFrameId;
209
+ async prepare() {
210
+ if (this.audioContext) {
211
+ return;
212
+ }
213
+ const audioContextConstructor = typeof window === "undefined" ? void 0 : window.AudioContext || window.webkitAudioContext;
214
+ if (!audioContextConstructor) {
215
+ throw new Error("AudioContext is not supported in this browser");
216
+ }
217
+ this.audioContext = new audioContextConstructor();
218
+ }
219
+ attachMediaElement(element) {
220
+ this.stop();
221
+ this.resetNodes();
222
+ try {
223
+ const audioContext = this.ensureAudioContext();
224
+ if (!audioContext) {
225
+ return;
226
+ }
227
+ const analyser = this.createAnalyser(audioContext);
228
+ const source = audioContext.createMediaElementSource(element);
229
+ source.connect(analyser);
230
+ analyser.connect(audioContext.destination);
231
+ this.startAnalysis(source, analyser);
232
+ if (audioContext.state === "suspended") {
233
+ element.addEventListener(
234
+ "play",
235
+ () => {
236
+ this.audioContext?.resume();
237
+ },
238
+ { once: true }
239
+ );
240
+ }
241
+ } catch (error) {
242
+ this.options.onError?.(error);
243
+ }
244
+ }
245
+ attachMediaStream(stream) {
246
+ if (this.attachedStream === stream) {
247
+ return;
248
+ }
249
+ this.stop();
250
+ this.resetNodes();
251
+ try {
252
+ const audioContext = this.ensureAudioContext();
253
+ if (!audioContext) {
254
+ return;
255
+ }
256
+ const analyser = this.createAnalyser(audioContext);
257
+ const source = audioContext.createMediaStreamSource(stream);
258
+ source.connect(analyser);
259
+ this.startAnalysis(source, analyser);
260
+ this.attachedStream = stream;
261
+ } catch (error) {
262
+ this.options.onError?.(error);
263
+ }
264
+ }
265
+ pause() {
266
+ this.cancelLoop();
267
+ this.options.onRms(0);
268
+ }
269
+ resume() {
270
+ if (!this.isActive || this.animationFrameId !== void 0 || !this.analyser || !this.dataArray) {
271
+ return;
272
+ }
273
+ this.analyzeFrame();
274
+ }
275
+ stop() {
276
+ this.isActive = false;
277
+ this.attachedStream = void 0;
278
+ this.cancelLoop();
279
+ this.options.onRms(0);
280
+ }
281
+ async cleanup() {
282
+ this.stop();
283
+ this.resetNodes();
284
+ this.dataArray = void 0;
285
+ const audioContext = this.audioContext;
286
+ this.audioContext = void 0;
287
+ await audioContext?.close();
288
+ }
289
+ /**
290
+ * Create the context lazily so callers can attach without awaiting prepare().
291
+ *
292
+ * This relies on prepare() staying await-free before it assigns
293
+ * `this.audioContext`: the void call below runs its body synchronously, so
294
+ * the context is already visible when this returns on the same tick.
295
+ */
296
+ ensureAudioContext() {
297
+ if (!this.audioContext) {
298
+ void this.prepare().catch((error) => {
299
+ this.options.onError?.(error);
300
+ });
301
+ }
302
+ return this.audioContext;
303
+ }
304
+ createAnalyser(audioContext) {
305
+ const analyser = audioContext.createAnalyser();
306
+ analyser.fftSize = LIPSYNC_FFT_SIZE;
307
+ analyser.smoothingTimeConstant = LIPSYNC_SMOOTHING_TIME_CONSTANT;
308
+ return analyser;
309
+ }
310
+ startAnalysis(source, analyser) {
311
+ this.source = source;
312
+ this.analyser = analyser;
313
+ this.dataArray = new Uint8Array(analyser.frequencyBinCount);
314
+ this.isActive = true;
315
+ this.analyzeFrame();
316
+ }
317
+ analyzeFrame() {
318
+ if (!this.isActive || !this.analyser || !this.dataArray) {
319
+ return;
320
+ }
321
+ this.analyser.getByteFrequencyData(this.dataArray);
322
+ this.options.onRms(computeLipSyncRms(this.dataArray));
323
+ this.animationFrameId = requestAnimationFrame(() => this.analyzeFrame());
324
+ }
325
+ cancelLoop() {
326
+ if (this.animationFrameId !== void 0) {
327
+ cancelAnimationFrame(this.animationFrameId);
328
+ this.animationFrameId = void 0;
329
+ }
330
+ }
331
+ resetNodes() {
332
+ if (this.source) {
333
+ this.source.disconnect();
334
+ this.source = void 0;
335
+ }
336
+ if (this.analyser) {
337
+ this.analyser.disconnect();
338
+ this.analyser = void 0;
339
+ }
340
+ }
341
+ };
342
+ function createLipSyncAnalyzer(options) {
343
+ return new LipSyncAnalyzerImpl(options);
344
+ }
345
+
346
+ // src/tool-validation.ts
347
+ function validateToolArguments(definition, args, toolLabel = "Tool") {
348
+ if (!isRecord(args) || Array.isArray(args)) {
349
+ throw new Error(
350
+ `${toolLabel} "${definition.name}" arguments failed schema validation: arguments must be an object`
351
+ );
352
+ }
353
+ const { parameters } = definition;
354
+ const required = parameters.required ?? [];
355
+ for (const propertyName of required) {
356
+ if (!hasOwnProperty(args, propertyName)) {
357
+ throw new Error(
358
+ `${toolLabel} "${definition.name}" arguments failed schema validation: missing required property "${propertyName}"`
359
+ );
360
+ }
361
+ }
362
+ for (const [propertyName, schema] of Object.entries(parameters.properties)) {
363
+ if (!hasOwnProperty(args, propertyName) || !isRecord(schema)) {
364
+ continue;
365
+ }
366
+ validateToolArgumentValue(
367
+ definition.name,
368
+ propertyName,
369
+ args[propertyName],
370
+ schema,
371
+ toolLabel
372
+ );
373
+ }
374
+ }
375
+ function assertToolResultObject(result, toolName, toolLabel = "Tool") {
376
+ if (!isRecord(result) || Array.isArray(result)) {
377
+ throw new Error(`${toolLabel} "${toolName}" must return an object`);
378
+ }
379
+ }
380
+ function validateToolArgumentValue(toolName, propertyName, value, schema, toolLabel) {
381
+ const enumValues = schema.enum;
382
+ if (Array.isArray(enumValues) && !enumValues.some((item) => Object.is(item, value))) {
383
+ throw new Error(
384
+ `${toolLabel} "${toolName}" arguments failed schema validation: property "${propertyName}" must be one of ${formatEnumValues(enumValues)}`
385
+ );
386
+ }
387
+ const expectedType = schema.type;
388
+ if (typeof expectedType !== "string") {
389
+ return;
390
+ }
391
+ if (!isSupportedSchemaType(expectedType)) {
392
+ throw new Error(
393
+ `${toolLabel} "${toolName}" arguments failed schema validation: property "${propertyName}" uses unsupported schema type "${expectedType}"`
394
+ );
395
+ }
396
+ if (!matchesSchemaType(value, expectedType)) {
397
+ throw new Error(
398
+ `${toolLabel} "${toolName}" arguments failed schema validation: property "${propertyName}" must be ${expectedType}`
399
+ );
400
+ }
401
+ }
402
+ function matchesSchemaType(value, expectedType) {
403
+ switch (expectedType) {
404
+ case "string":
405
+ return typeof value === "string";
406
+ case "number":
407
+ return typeof value === "number" && Number.isFinite(value);
408
+ case "integer":
409
+ return typeof value === "number" && Number.isInteger(value);
410
+ case "boolean":
411
+ return typeof value === "boolean";
412
+ case "object":
413
+ return isRecord(value) && !Array.isArray(value);
414
+ case "array":
415
+ return Array.isArray(value);
416
+ case "null":
417
+ return value === null;
418
+ }
419
+ return false;
420
+ }
421
+ function isSupportedSchemaType(expectedType) {
422
+ return expectedType === "string" || expectedType === "number" || expectedType === "integer" || expectedType === "boolean" || expectedType === "object" || expectedType === "array" || expectedType === "null";
423
+ }
424
+ function formatEnumValues(values) {
425
+ return values.map((value) => JSON.stringify(value)).join(", ");
426
+ }
427
+ function hasOwnProperty(record, propertyName) {
428
+ return Object.prototype.hasOwnProperty.call(record, propertyName);
429
+ }
430
+ function isRecord(value) {
431
+ return typeof value === "object" && value !== null;
432
+ }
433
+
184
434
  // src/index.ts
185
435
  var Charivo = class {
186
436
  eventBus;
@@ -220,6 +470,7 @@ var Charivo = class {
220
470
  */
221
471
  attachLLM(manager) {
222
472
  this.llmManager = manager;
473
+ this.connectLLMManagerEventEmitter(manager);
223
474
  if (this.character) {
224
475
  manager.setCharacter(this.character);
225
476
  }
@@ -281,6 +532,14 @@ var Charivo = class {
281
532
  manager.setEventEmitter(this.eventBus);
282
533
  }
283
534
  }
535
+ /**
536
+ * Connects the LLM manager to the event bus for tool-call event emission.
537
+ */
538
+ connectLLMManagerEventEmitter(manager) {
539
+ if (manager.setEventEmitter) {
540
+ manager.setEventEmitter(this.eventBus);
541
+ }
542
+ }
284
543
  /**
285
544
  * Set the character for this Charivo instance.
286
545
  * Automatically propagates to all attached managers (LLM, Renderer).
@@ -480,6 +739,11 @@ var Charivo = class {
480
739
  } catch (error) {
481
740
  recordError(error, "Failed to stop TTS during dispose");
482
741
  }
742
+ try {
743
+ await this.ttsManager.dispose?.();
744
+ } catch (error) {
745
+ recordError(error, "Failed to release TTS resources during dispose");
746
+ }
483
747
  }
484
748
  if (this.sttManager) {
485
749
  try {
@@ -531,9 +795,12 @@ exports.CharivoTransportError = CharivoTransportError;
531
795
  exports.EventBus = EventBus;
532
796
  exports.OPENAI_REALTIME_ADAPTER = OPENAI_REALTIME_ADAPTER;
533
797
  exports.OPENAI_REALTIME_AGENTS_ADAPTER = OPENAI_REALTIME_AGENTS_ADAPTER;
798
+ exports.assertToolResultObject = assertToolResultObject;
799
+ exports.createLipSyncAnalyzer = createLipSyncAnalyzer;
534
800
  exports.getErrorMessage = getErrorMessage;
535
801
  exports.isCharivoError = isCharivoError;
536
802
  exports.subscribeBrowserLifecycle = subscribeBrowserLifecycle;
537
803
  exports.toCharivoError = toCharivoError;
804
+ exports.validateToolArguments = validateToolArguments;
538
805
  //# sourceMappingURL=index.cjs.map
539
806
  //# sourceMappingURL=index.cjs.map