@charivo/core 0.15.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -4
- package/dist/index.cjs +267 -0
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +164 -12
- package/dist/index.d.ts +164 -12
- package/dist/index.js +265 -1
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -53,16 +53,43 @@ type RealtimeTransportKind = "webrtc" | "websocket";
|
|
|
53
53
|
type RealtimeToolChoice = "auto" | "none" | "required";
|
|
54
54
|
declare const OPENAI_REALTIME_ADAPTER = "openai-webrtc";
|
|
55
55
|
declare const OPENAI_REALTIME_AGENTS_ADAPTER = "openai-agents-webrtc";
|
|
56
|
-
interface
|
|
56
|
+
interface ToolDefinition {
|
|
57
57
|
type: "function";
|
|
58
58
|
name: string;
|
|
59
59
|
description: string;
|
|
60
|
+
/**
|
|
61
|
+
* JSON Schema-shaped. `validateToolArguments` only enforces required-key
|
|
62
|
+
* presence, enum membership, and each property's top-level `type`; nested
|
|
63
|
+
* schemas, `additionalProperties`, and numeric-length constraints are not
|
|
64
|
+
* validated.
|
|
65
|
+
*/
|
|
60
66
|
parameters: {
|
|
61
67
|
type: "object";
|
|
62
68
|
properties: Record<string, unknown>;
|
|
63
69
|
required?: string[];
|
|
64
70
|
};
|
|
65
71
|
}
|
|
72
|
+
interface ToolContext {
|
|
73
|
+
character?: Character | null;
|
|
74
|
+
callId?: string;
|
|
75
|
+
/** Present only for realtime sessions. */
|
|
76
|
+
state?: RealtimeState;
|
|
77
|
+
}
|
|
78
|
+
/** Must resolve to a plain object; arrays and primitives are rejected by the runners. */
|
|
79
|
+
type ToolHandler = (args: Record<string, unknown>, context: ToolContext) => Promise<Record<string, unknown>>;
|
|
80
|
+
interface ToolRegistration {
|
|
81
|
+
definition: ToolDefinition;
|
|
82
|
+
handler: ToolHandler;
|
|
83
|
+
/** Timeout in ms; falls back to the manager default when omitted. */
|
|
84
|
+
timeoutMs?: number;
|
|
85
|
+
}
|
|
86
|
+
interface ToolResultProjectorContext {
|
|
87
|
+
name: string;
|
|
88
|
+
output: Record<string, unknown>;
|
|
89
|
+
callId?: string;
|
|
90
|
+
emit<K extends keyof EventMap>(event: K, payload: EventMap[K]): void;
|
|
91
|
+
}
|
|
92
|
+
type ToolResultProjector = (context: ToolResultProjectorContext) => void;
|
|
66
93
|
interface RealtimeSessionConfig {
|
|
67
94
|
provider?: string;
|
|
68
95
|
transport?: RealtimeTransportKind;
|
|
@@ -71,7 +98,7 @@ interface RealtimeSessionConfig {
|
|
|
71
98
|
instructions?: string;
|
|
72
99
|
temperature?: number;
|
|
73
100
|
maxTokens?: number;
|
|
74
|
-
tools?:
|
|
101
|
+
tools?: ToolDefinition[];
|
|
75
102
|
toolChoice?: RealtimeToolChoice;
|
|
76
103
|
inputAudioTranscription?: {
|
|
77
104
|
model?: string;
|
|
@@ -119,12 +146,16 @@ interface RealtimeState {
|
|
|
119
146
|
};
|
|
120
147
|
lastError: Error | null;
|
|
121
148
|
}
|
|
122
|
-
|
|
123
|
-
|
|
149
|
+
/** @deprecated Use ToolDefinition */
|
|
150
|
+
interface RealtimeTool extends ToolDefinition {
|
|
151
|
+
}
|
|
152
|
+
/** @deprecated Use ToolContext (note: context.state becomes optional in the neutral contract) */
|
|
153
|
+
interface RealtimeToolContext extends ToolContext {
|
|
124
154
|
state: RealtimeState;
|
|
125
|
-
callId?: string;
|
|
126
155
|
}
|
|
156
|
+
/** @deprecated Use ToolHandler (note: context.state becomes optional in the neutral contract) */
|
|
127
157
|
type RealtimeToolHandler = (args: Record<string, unknown>, context: RealtimeToolContext) => Promise<Record<string, unknown>>;
|
|
158
|
+
/** @deprecated Use ToolRegistration (note: context.state becomes optional in the neutral contract) */
|
|
128
159
|
interface RealtimeToolRegistration {
|
|
129
160
|
definition: RealtimeTool;
|
|
130
161
|
handler: RealtimeToolHandler;
|
|
@@ -141,17 +172,48 @@ interface LLMAdapter {
|
|
|
141
172
|
setCharacter(character: Character): void;
|
|
142
173
|
clearHistory(): void;
|
|
143
174
|
}
|
|
175
|
+
interface LLMToolCall {
|
|
176
|
+
id: string;
|
|
177
|
+
name: string;
|
|
178
|
+
/** JSON-decoded; the provider/client adapter parses the raw tool-call arguments before this contract is used. */
|
|
179
|
+
arguments: Record<string, unknown>;
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Role-discriminated union so protocol-invalid combinations are unrepresentable
|
|
183
|
+
* for typed direct callers: unknown roles, tool turns without an ID, and tool
|
|
184
|
+
* calls on user turns cannot be expressed.
|
|
185
|
+
*/
|
|
186
|
+
type LLMMessage = {
|
|
187
|
+
role: "system" | "user";
|
|
188
|
+
content: string;
|
|
189
|
+
} | {
|
|
190
|
+
role: "assistant";
|
|
191
|
+
content: string;
|
|
192
|
+
toolCalls?: LLMToolCall[];
|
|
193
|
+
} | {
|
|
194
|
+
role: "tool";
|
|
195
|
+
content: string;
|
|
196
|
+
toolCallId: string;
|
|
197
|
+
};
|
|
198
|
+
interface LLMToolResponse {
|
|
199
|
+
content: string;
|
|
200
|
+
toolCalls?: LLMToolCall[];
|
|
201
|
+
}
|
|
144
202
|
interface LLMProvider {
|
|
145
203
|
generateResponse(messages: Array<{
|
|
146
204
|
role: string;
|
|
147
205
|
content: string;
|
|
148
206
|
}>): Promise<string>;
|
|
207
|
+
/** Tool-calling variant; providers that support function calling implement this alongside generateResponse. */
|
|
208
|
+
generateResponseWithTools?(messages: LLMMessage[], tools: ToolDefinition[]): Promise<LLMToolResponse>;
|
|
149
209
|
}
|
|
150
210
|
interface LLMClient {
|
|
151
211
|
call(messages: Array<{
|
|
152
212
|
role: string;
|
|
153
213
|
content: string;
|
|
154
214
|
}>): Promise<string>;
|
|
215
|
+
/** Tool-calling variant; clients that support function calling implement this alongside call. */
|
|
216
|
+
callWithTools?(messages: LLMMessage[], tools: ToolDefinition[]): Promise<LLMToolResponse>;
|
|
155
217
|
}
|
|
156
218
|
interface LLMManager {
|
|
157
219
|
setCharacter(character: Character): void;
|
|
@@ -159,6 +221,24 @@ interface LLMManager {
|
|
|
159
221
|
clearHistory(): void;
|
|
160
222
|
getHistory(): Message[];
|
|
161
223
|
generateResponse(message: Message): Promise<string>;
|
|
224
|
+
setEventEmitter?(eventEmitter: CharivoEventEmitter): void;
|
|
225
|
+
registerTool?(tool: ToolRegistration): void;
|
|
226
|
+
unregisterTool?(name: string): void;
|
|
227
|
+
getRegisteredTools?(): ToolDefinition[];
|
|
228
|
+
/** System-prompt-level instructions injected only when tools are registered; pass null to clear. */
|
|
229
|
+
setToolInstructions?(instructions: string | null): void;
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* `LLMManager` with the tool-calling members required instead of optional.
|
|
233
|
+
* Third-party `LLMManager` implementations may omit tool support, but the
|
|
234
|
+
* built-in manager returned by `createLLMManager` always provides it.
|
|
235
|
+
*/
|
|
236
|
+
interface LLMManagerWithTools extends LLMManager {
|
|
237
|
+
setEventEmitter(eventEmitter: CharivoEventEmitter): void;
|
|
238
|
+
registerTool(tool: ToolRegistration): void;
|
|
239
|
+
unregisterTool(name: string): void;
|
|
240
|
+
getRegisteredTools(): ToolDefinition[];
|
|
241
|
+
setToolInstructions(instructions: string | null): void;
|
|
162
242
|
}
|
|
163
243
|
interface Renderer {
|
|
164
244
|
initialize(): Promise<void>;
|
|
@@ -183,7 +263,6 @@ interface RenderManager {
|
|
|
183
263
|
setEventBus(eventBus: CharivoEventBus): void;
|
|
184
264
|
/** Removes all event-bus listeners registered by setEventBus. */
|
|
185
265
|
disconnect(): void;
|
|
186
|
-
prepareAudio?(): Promise<void>;
|
|
187
266
|
loadModel?(modelPath: string): Promise<void>;
|
|
188
267
|
setMessageCallback?(callback: (message: Message, character?: Character) => void): void;
|
|
189
268
|
}
|
|
@@ -208,6 +287,11 @@ interface TTSPlayer {
|
|
|
208
287
|
stop(): Promise<void>;
|
|
209
288
|
setVoice(voice: string): void;
|
|
210
289
|
isSupported(): boolean;
|
|
290
|
+
/**
|
|
291
|
+
* Stateless audio generation. Required for the `"audio"` playback mode: the
|
|
292
|
+
* manager creates the audio element itself so it can analyze playback for
|
|
293
|
+
* lip-sync. Players that can only `speak()` must use `"web-speech"` mode.
|
|
294
|
+
*/
|
|
211
295
|
generateAudio?(text: string, options?: TTSOptions): Promise<ArrayBuffer>;
|
|
212
296
|
}
|
|
213
297
|
interface TTSProvider {
|
|
@@ -219,7 +303,11 @@ interface TTSManager {
|
|
|
219
303
|
stop(): Promise<void>;
|
|
220
304
|
setVoice(voice: string): void;
|
|
221
305
|
isSupported(): boolean;
|
|
306
|
+
/** Creates the audio analysis context up front; call from a user gesture handler so browsers allow playback later. */
|
|
307
|
+
prepareAudio?(): Promise<void>;
|
|
222
308
|
setEventEmitter?(eventEmitter: CharivoEventEmitter): void;
|
|
309
|
+
/** Final resource release; call stop() first - dispose() does not stop playback. */
|
|
310
|
+
dispose?(): Promise<void>;
|
|
223
311
|
}
|
|
224
312
|
interface STTOptions {
|
|
225
313
|
language?: string;
|
|
@@ -243,6 +331,7 @@ interface STTManager {
|
|
|
243
331
|
interface RealtimeManager {
|
|
244
332
|
setCharacter(character: Character): void;
|
|
245
333
|
getState(): RealtimeState;
|
|
334
|
+
prepareAudio?(config?: RealtimeSessionConfig): Promise<void>;
|
|
246
335
|
startSession(config?: RealtimeSessionConfig): Promise<void>;
|
|
247
336
|
updateSession(config?: RealtimeSessionConfig): Promise<void>;
|
|
248
337
|
stopSession(): Promise<void>;
|
|
@@ -251,7 +340,7 @@ interface RealtimeManager {
|
|
|
251
340
|
interrupt(): Promise<void>;
|
|
252
341
|
registerTool(tool: RealtimeToolRegistration): void;
|
|
253
342
|
unregisterTool(name: string): void;
|
|
254
|
-
getRegisteredTools():
|
|
343
|
+
getRegisteredTools(): ToolDefinition[];
|
|
255
344
|
setEventEmitter?(eventEmitter: CharivoEventEmitter): void;
|
|
256
345
|
}
|
|
257
346
|
type EventMap = {
|
|
@@ -276,7 +365,6 @@ type EventMap = {
|
|
|
276
365
|
error: Error;
|
|
277
366
|
};
|
|
278
367
|
"tts:audio:start": {
|
|
279
|
-
audioElement?: HTMLAudioElement;
|
|
280
368
|
characterId?: string;
|
|
281
369
|
};
|
|
282
370
|
"tts:audio:end": {
|
|
@@ -353,14 +441,14 @@ type EventMap = {
|
|
|
353
441
|
lastError: Error;
|
|
354
442
|
};
|
|
355
443
|
"realtime:usage": RealtimeUsageEvent;
|
|
356
|
-
"
|
|
444
|
+
"avatar:expression": {
|
|
357
445
|
expressionId: string;
|
|
358
446
|
};
|
|
359
|
-
"
|
|
447
|
+
"avatar:motion": {
|
|
360
448
|
group: string;
|
|
361
449
|
index: number;
|
|
362
450
|
};
|
|
363
|
-
"
|
|
451
|
+
"avatar:gaze": GazeCoordinates;
|
|
364
452
|
"realtime:text:delta": {
|
|
365
453
|
text: string;
|
|
366
454
|
};
|
|
@@ -424,6 +512,66 @@ declare function isCharivoError(error: unknown): error is CharivoError;
|
|
|
424
512
|
declare function getErrorMessage(error: unknown, fallback?: string): string;
|
|
425
513
|
declare function toCharivoError(kind: CharivoErrorKind, error: unknown, fallbackMessage?: string): CharivoError;
|
|
426
514
|
|
|
515
|
+
interface LipSyncAnalyzerOptions {
|
|
516
|
+
onRms: (rms: number) => void;
|
|
517
|
+
onError?: (error: unknown) => void;
|
|
518
|
+
}
|
|
519
|
+
interface LipSyncAnalyzer {
|
|
520
|
+
/**
|
|
521
|
+
* Create the AudioContext up front (idempotent).
|
|
522
|
+
*/
|
|
523
|
+
prepare(): Promise<void>;
|
|
524
|
+
/**
|
|
525
|
+
* Analyze an audio element while keeping its playback audible.
|
|
526
|
+
*/
|
|
527
|
+
attachMediaElement(element: HTMLAudioElement): void;
|
|
528
|
+
/**
|
|
529
|
+
* Analyze a media stream without routing it to the speakers.
|
|
530
|
+
*/
|
|
531
|
+
attachMediaStream(stream: MediaStream): void;
|
|
532
|
+
/**
|
|
533
|
+
* Temporarily halt analysis; `resume()` restarts it.
|
|
534
|
+
*/
|
|
535
|
+
pause(): void;
|
|
536
|
+
resume(): void;
|
|
537
|
+
/**
|
|
538
|
+
* Halt analysis for the current source. `resume()` no-ops until the next attach.
|
|
539
|
+
*/
|
|
540
|
+
stop(): void;
|
|
541
|
+
cleanup(): Promise<void>;
|
|
542
|
+
}
|
|
543
|
+
declare function createLipSyncAnalyzer(options: LipSyncAnalyzerOptions): LipSyncAnalyzer;
|
|
544
|
+
|
|
545
|
+
/**
|
|
546
|
+
* Shared tool validation: the single argument/result contract enforced before
|
|
547
|
+
* and after a tool handler runs, regardless of the modality that calls it.
|
|
548
|
+
*
|
|
549
|
+
* Errors here are plain `Error`, not a `CharivoErrorKind` variant, on purpose:
|
|
550
|
+
* this preserves message-string compatibility with the pre-move realtime
|
|
551
|
+
* validator, and no existing error kind fits schema validation.
|
|
552
|
+
*/
|
|
553
|
+
|
|
554
|
+
/**
|
|
555
|
+
* Validates tool call arguments against a definition's parameter schema.
|
|
556
|
+
*
|
|
557
|
+
* Enforces only: required-key presence, `enum` membership, and each
|
|
558
|
+
* property's top-level `type`. Nested schemas, `additionalProperties`, and
|
|
559
|
+
* numeric-length constraints (`minLength`, `maximum`, etc.) are not
|
|
560
|
+
* validated. Throws a plain `Error` on the first violation found.
|
|
561
|
+
*
|
|
562
|
+
* @param toolLabel - Prefixes thrown messages (e.g. `${toolLabel} "name" ...`);
|
|
563
|
+
* callers use this to distinguish realtime tools from other modalities.
|
|
564
|
+
*/
|
|
565
|
+
declare function validateToolArguments(definition: ToolDefinition, args: unknown, toolLabel?: string): void;
|
|
566
|
+
/**
|
|
567
|
+
* Tool handlers must resolve to a plain object; arrays and primitives are not
|
|
568
|
+
* serializable tool outputs. Throws a plain `Error` on violation.
|
|
569
|
+
*
|
|
570
|
+
* @param toolLabel - Prefixes the thrown message (e.g. `${toolLabel} "name" ...`);
|
|
571
|
+
* callers use this to distinguish realtime tools from other modalities.
|
|
572
|
+
*/
|
|
573
|
+
declare function assertToolResultObject(result: unknown, toolName: string, toolLabel?: string): asserts result is Record<string, unknown>;
|
|
574
|
+
|
|
427
575
|
declare class Charivo {
|
|
428
576
|
private eventBus;
|
|
429
577
|
private llmManager?;
|
|
@@ -482,6 +630,10 @@ declare class Charivo {
|
|
|
482
630
|
* Connects the STT manager to the event bus for speech recognition event emission.
|
|
483
631
|
*/
|
|
484
632
|
private connectSTTManagerEventEmitter;
|
|
633
|
+
/**
|
|
634
|
+
* Connects the LLM manager to the event bus for tool-call event emission.
|
|
635
|
+
*/
|
|
636
|
+
private connectLLMManagerEventEmitter;
|
|
485
637
|
/**
|
|
486
638
|
* Set the character for this Charivo instance.
|
|
487
639
|
* Automatically propagates to all attached managers (LLM, Renderer).
|
|
@@ -549,4 +701,4 @@ declare class Charivo {
|
|
|
549
701
|
private finishDispose;
|
|
550
702
|
}
|
|
551
703
|
|
|
552
|
-
export { type AvatarControlCatalog, type BrowserLifecycleCallbacks, type Character, Charivo, type CharivoConfig, CharivoDisposeError, CharivoError, type CharivoErrorCode, type CharivoErrorKind, type CharivoErrorOptions, type CharivoEventBus, type CharivoEventEmitter, CharivoProviderError, CharivoStateError, CharivoTimeoutError, CharivoTransportError, type Conversation, EventBus, type EventMap, type GazeCoordinates, type LLMAdapter, type LLMClient, type LLMManager, type LLMProvider, type Message, type MotionSelection, OPENAI_REALTIME_ADAPTER, OPENAI_REALTIME_AGENTS_ADAPTER, type Plugin, type RealtimeConnectionState, type RealtimeManager, type RealtimeProvider, type RealtimeReconnectCause, type RealtimeResponseStatus, type RealtimeSessionBootstrap, type RealtimeSessionConfig, type RealtimeSessionRequest, type RealtimeSessionStatus, type RealtimeSessionTransitionReason, type RealtimeState, type RealtimeTool, type RealtimeToolChoice, type RealtimeToolContext, type RealtimeToolHandler, type RealtimeToolRegistration, type RealtimeTransportKind, type RealtimeUsageEvent, type RenderManager, type Renderer, type STTManager, type STTOptions, type STTProvider, type STTTranscriber, type TTSManager, type TTSOptions, type TTSPlaybackMode, type TTSPlayer, type TTSProvider, getErrorMessage, isCharivoError, subscribeBrowserLifecycle, toCharivoError };
|
|
704
|
+
export { type AvatarControlCatalog, type BrowserLifecycleCallbacks, type Character, Charivo, type CharivoConfig, CharivoDisposeError, CharivoError, type CharivoErrorCode, type CharivoErrorKind, type CharivoErrorOptions, type CharivoEventBus, type CharivoEventEmitter, CharivoProviderError, CharivoStateError, CharivoTimeoutError, CharivoTransportError, type Conversation, EventBus, type EventMap, type GazeCoordinates, type LLMAdapter, type LLMClient, type LLMManager, type LLMManagerWithTools, type LLMMessage, type LLMProvider, type LLMToolCall, type LLMToolResponse, type LipSyncAnalyzer, type LipSyncAnalyzerOptions, type Message, type MotionSelection, OPENAI_REALTIME_ADAPTER, OPENAI_REALTIME_AGENTS_ADAPTER, type Plugin, type RealtimeConnectionState, type RealtimeManager, type RealtimeProvider, type RealtimeReconnectCause, type RealtimeResponseStatus, type RealtimeSessionBootstrap, type RealtimeSessionConfig, type RealtimeSessionRequest, type RealtimeSessionStatus, type RealtimeSessionTransitionReason, type RealtimeState, type RealtimeTool, type RealtimeToolChoice, type RealtimeToolContext, type RealtimeToolHandler, type RealtimeToolRegistration, type RealtimeTransportKind, type RealtimeUsageEvent, type RenderManager, type Renderer, type STTManager, type STTOptions, type STTProvider, type STTTranscriber, type TTSManager, type TTSOptions, type TTSPlaybackMode, type TTSPlayer, type TTSProvider, type ToolContext, type ToolDefinition, type ToolHandler, type ToolRegistration, type ToolResultProjector, type ToolResultProjectorContext, assertToolResultObject, createLipSyncAnalyzer, getErrorMessage, isCharivoError, subscribeBrowserLifecycle, toCharivoError, validateToolArguments };
|
package/dist/index.js
CHANGED
|
@@ -179,6 +179,256 @@ function subscribeBrowserLifecycle(callbacks) {
|
|
|
179
179
|
};
|
|
180
180
|
}
|
|
181
181
|
|
|
182
|
+
// src/lipsync-analyzer.ts
|
|
183
|
+
var LIPSYNC_FFT_SIZE = 256;
|
|
184
|
+
var LIPSYNC_SMOOTHING_TIME_CONSTANT = 0.8;
|
|
185
|
+
function computeLipSyncRms(frequencyData) {
|
|
186
|
+
const speechBandStart = Math.floor(frequencyData.length * 0.1);
|
|
187
|
+
const speechBandEnd = Math.floor(frequencyData.length * 0.6);
|
|
188
|
+
let sum = 0;
|
|
189
|
+
for (let index = speechBandStart; index < speechBandEnd; index += 1) {
|
|
190
|
+
const normalized = frequencyData[index] / 255;
|
|
191
|
+
sum += normalized * normalized;
|
|
192
|
+
}
|
|
193
|
+
const rms = Math.sqrt(sum / (speechBandEnd - speechBandStart));
|
|
194
|
+
return Math.min(rms * 1.7, 1);
|
|
195
|
+
}
|
|
196
|
+
var LipSyncAnalyzerImpl = class {
|
|
197
|
+
constructor(options) {
|
|
198
|
+
this.options = options;
|
|
199
|
+
}
|
|
200
|
+
audioContext;
|
|
201
|
+
analyser;
|
|
202
|
+
source;
|
|
203
|
+
attachedStream;
|
|
204
|
+
dataArray;
|
|
205
|
+
isActive = false;
|
|
206
|
+
animationFrameId;
|
|
207
|
+
async prepare() {
|
|
208
|
+
if (this.audioContext) {
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
const audioContextConstructor = typeof window === "undefined" ? void 0 : window.AudioContext || window.webkitAudioContext;
|
|
212
|
+
if (!audioContextConstructor) {
|
|
213
|
+
throw new Error("AudioContext is not supported in this browser");
|
|
214
|
+
}
|
|
215
|
+
this.audioContext = new audioContextConstructor();
|
|
216
|
+
}
|
|
217
|
+
attachMediaElement(element) {
|
|
218
|
+
this.stop();
|
|
219
|
+
this.resetNodes();
|
|
220
|
+
try {
|
|
221
|
+
const audioContext = this.ensureAudioContext();
|
|
222
|
+
if (!audioContext) {
|
|
223
|
+
return;
|
|
224
|
+
}
|
|
225
|
+
const analyser = this.createAnalyser(audioContext);
|
|
226
|
+
const source = audioContext.createMediaElementSource(element);
|
|
227
|
+
source.connect(analyser);
|
|
228
|
+
analyser.connect(audioContext.destination);
|
|
229
|
+
this.startAnalysis(source, analyser);
|
|
230
|
+
if (audioContext.state === "suspended") {
|
|
231
|
+
element.addEventListener(
|
|
232
|
+
"play",
|
|
233
|
+
() => {
|
|
234
|
+
this.audioContext?.resume();
|
|
235
|
+
},
|
|
236
|
+
{ once: true }
|
|
237
|
+
);
|
|
238
|
+
}
|
|
239
|
+
} catch (error) {
|
|
240
|
+
this.options.onError?.(error);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
attachMediaStream(stream) {
|
|
244
|
+
if (this.attachedStream === stream) {
|
|
245
|
+
return;
|
|
246
|
+
}
|
|
247
|
+
this.stop();
|
|
248
|
+
this.resetNodes();
|
|
249
|
+
try {
|
|
250
|
+
const audioContext = this.ensureAudioContext();
|
|
251
|
+
if (!audioContext) {
|
|
252
|
+
return;
|
|
253
|
+
}
|
|
254
|
+
const analyser = this.createAnalyser(audioContext);
|
|
255
|
+
const source = audioContext.createMediaStreamSource(stream);
|
|
256
|
+
source.connect(analyser);
|
|
257
|
+
this.startAnalysis(source, analyser);
|
|
258
|
+
this.attachedStream = stream;
|
|
259
|
+
} catch (error) {
|
|
260
|
+
this.options.onError?.(error);
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
pause() {
|
|
264
|
+
this.cancelLoop();
|
|
265
|
+
this.options.onRms(0);
|
|
266
|
+
}
|
|
267
|
+
resume() {
|
|
268
|
+
if (!this.isActive || this.animationFrameId !== void 0 || !this.analyser || !this.dataArray) {
|
|
269
|
+
return;
|
|
270
|
+
}
|
|
271
|
+
this.analyzeFrame();
|
|
272
|
+
}
|
|
273
|
+
stop() {
|
|
274
|
+
this.isActive = false;
|
|
275
|
+
this.attachedStream = void 0;
|
|
276
|
+
this.cancelLoop();
|
|
277
|
+
this.options.onRms(0);
|
|
278
|
+
}
|
|
279
|
+
async cleanup() {
|
|
280
|
+
this.stop();
|
|
281
|
+
this.resetNodes();
|
|
282
|
+
this.dataArray = void 0;
|
|
283
|
+
const audioContext = this.audioContext;
|
|
284
|
+
this.audioContext = void 0;
|
|
285
|
+
await audioContext?.close();
|
|
286
|
+
}
|
|
287
|
+
/**
|
|
288
|
+
* Create the context lazily so callers can attach without awaiting prepare().
|
|
289
|
+
*
|
|
290
|
+
* This relies on prepare() staying await-free before it assigns
|
|
291
|
+
* `this.audioContext`: the void call below runs its body synchronously, so
|
|
292
|
+
* the context is already visible when this returns on the same tick.
|
|
293
|
+
*/
|
|
294
|
+
ensureAudioContext() {
|
|
295
|
+
if (!this.audioContext) {
|
|
296
|
+
void this.prepare().catch((error) => {
|
|
297
|
+
this.options.onError?.(error);
|
|
298
|
+
});
|
|
299
|
+
}
|
|
300
|
+
return this.audioContext;
|
|
301
|
+
}
|
|
302
|
+
createAnalyser(audioContext) {
|
|
303
|
+
const analyser = audioContext.createAnalyser();
|
|
304
|
+
analyser.fftSize = LIPSYNC_FFT_SIZE;
|
|
305
|
+
analyser.smoothingTimeConstant = LIPSYNC_SMOOTHING_TIME_CONSTANT;
|
|
306
|
+
return analyser;
|
|
307
|
+
}
|
|
308
|
+
startAnalysis(source, analyser) {
|
|
309
|
+
this.source = source;
|
|
310
|
+
this.analyser = analyser;
|
|
311
|
+
this.dataArray = new Uint8Array(analyser.frequencyBinCount);
|
|
312
|
+
this.isActive = true;
|
|
313
|
+
this.analyzeFrame();
|
|
314
|
+
}
|
|
315
|
+
analyzeFrame() {
|
|
316
|
+
if (!this.isActive || !this.analyser || !this.dataArray) {
|
|
317
|
+
return;
|
|
318
|
+
}
|
|
319
|
+
this.analyser.getByteFrequencyData(this.dataArray);
|
|
320
|
+
this.options.onRms(computeLipSyncRms(this.dataArray));
|
|
321
|
+
this.animationFrameId = requestAnimationFrame(() => this.analyzeFrame());
|
|
322
|
+
}
|
|
323
|
+
cancelLoop() {
|
|
324
|
+
if (this.animationFrameId !== void 0) {
|
|
325
|
+
cancelAnimationFrame(this.animationFrameId);
|
|
326
|
+
this.animationFrameId = void 0;
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
resetNodes() {
|
|
330
|
+
if (this.source) {
|
|
331
|
+
this.source.disconnect();
|
|
332
|
+
this.source = void 0;
|
|
333
|
+
}
|
|
334
|
+
if (this.analyser) {
|
|
335
|
+
this.analyser.disconnect();
|
|
336
|
+
this.analyser = void 0;
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
};
|
|
340
|
+
function createLipSyncAnalyzer(options) {
|
|
341
|
+
return new LipSyncAnalyzerImpl(options);
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
// src/tool-validation.ts
|
|
345
|
+
function validateToolArguments(definition, args, toolLabel = "Tool") {
|
|
346
|
+
if (!isRecord(args) || Array.isArray(args)) {
|
|
347
|
+
throw new Error(
|
|
348
|
+
`${toolLabel} "${definition.name}" arguments failed schema validation: arguments must be an object`
|
|
349
|
+
);
|
|
350
|
+
}
|
|
351
|
+
const { parameters } = definition;
|
|
352
|
+
const required = parameters.required ?? [];
|
|
353
|
+
for (const propertyName of required) {
|
|
354
|
+
if (!hasOwnProperty(args, propertyName)) {
|
|
355
|
+
throw new Error(
|
|
356
|
+
`${toolLabel} "${definition.name}" arguments failed schema validation: missing required property "${propertyName}"`
|
|
357
|
+
);
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
for (const [propertyName, schema] of Object.entries(parameters.properties)) {
|
|
361
|
+
if (!hasOwnProperty(args, propertyName) || !isRecord(schema)) {
|
|
362
|
+
continue;
|
|
363
|
+
}
|
|
364
|
+
validateToolArgumentValue(
|
|
365
|
+
definition.name,
|
|
366
|
+
propertyName,
|
|
367
|
+
args[propertyName],
|
|
368
|
+
schema,
|
|
369
|
+
toolLabel
|
|
370
|
+
);
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
function assertToolResultObject(result, toolName, toolLabel = "Tool") {
|
|
374
|
+
if (!isRecord(result) || Array.isArray(result)) {
|
|
375
|
+
throw new Error(`${toolLabel} "${toolName}" must return an object`);
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
function validateToolArgumentValue(toolName, propertyName, value, schema, toolLabel) {
|
|
379
|
+
const enumValues = schema.enum;
|
|
380
|
+
if (Array.isArray(enumValues) && !enumValues.some((item) => Object.is(item, value))) {
|
|
381
|
+
throw new Error(
|
|
382
|
+
`${toolLabel} "${toolName}" arguments failed schema validation: property "${propertyName}" must be one of ${formatEnumValues(enumValues)}`
|
|
383
|
+
);
|
|
384
|
+
}
|
|
385
|
+
const expectedType = schema.type;
|
|
386
|
+
if (typeof expectedType !== "string") {
|
|
387
|
+
return;
|
|
388
|
+
}
|
|
389
|
+
if (!isSupportedSchemaType(expectedType)) {
|
|
390
|
+
throw new Error(
|
|
391
|
+
`${toolLabel} "${toolName}" arguments failed schema validation: property "${propertyName}" uses unsupported schema type "${expectedType}"`
|
|
392
|
+
);
|
|
393
|
+
}
|
|
394
|
+
if (!matchesSchemaType(value, expectedType)) {
|
|
395
|
+
throw new Error(
|
|
396
|
+
`${toolLabel} "${toolName}" arguments failed schema validation: property "${propertyName}" must be ${expectedType}`
|
|
397
|
+
);
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
function matchesSchemaType(value, expectedType) {
|
|
401
|
+
switch (expectedType) {
|
|
402
|
+
case "string":
|
|
403
|
+
return typeof value === "string";
|
|
404
|
+
case "number":
|
|
405
|
+
return typeof value === "number" && Number.isFinite(value);
|
|
406
|
+
case "integer":
|
|
407
|
+
return typeof value === "number" && Number.isInteger(value);
|
|
408
|
+
case "boolean":
|
|
409
|
+
return typeof value === "boolean";
|
|
410
|
+
case "object":
|
|
411
|
+
return isRecord(value) && !Array.isArray(value);
|
|
412
|
+
case "array":
|
|
413
|
+
return Array.isArray(value);
|
|
414
|
+
case "null":
|
|
415
|
+
return value === null;
|
|
416
|
+
}
|
|
417
|
+
return false;
|
|
418
|
+
}
|
|
419
|
+
function isSupportedSchemaType(expectedType) {
|
|
420
|
+
return expectedType === "string" || expectedType === "number" || expectedType === "integer" || expectedType === "boolean" || expectedType === "object" || expectedType === "array" || expectedType === "null";
|
|
421
|
+
}
|
|
422
|
+
function formatEnumValues(values) {
|
|
423
|
+
return values.map((value) => JSON.stringify(value)).join(", ");
|
|
424
|
+
}
|
|
425
|
+
function hasOwnProperty(record, propertyName) {
|
|
426
|
+
return Object.prototype.hasOwnProperty.call(record, propertyName);
|
|
427
|
+
}
|
|
428
|
+
function isRecord(value) {
|
|
429
|
+
return typeof value === "object" && value !== null;
|
|
430
|
+
}
|
|
431
|
+
|
|
182
432
|
// src/index.ts
|
|
183
433
|
var Charivo = class {
|
|
184
434
|
eventBus;
|
|
@@ -218,6 +468,7 @@ var Charivo = class {
|
|
|
218
468
|
*/
|
|
219
469
|
attachLLM(manager) {
|
|
220
470
|
this.llmManager = manager;
|
|
471
|
+
this.connectLLMManagerEventEmitter(manager);
|
|
221
472
|
if (this.character) {
|
|
222
473
|
manager.setCharacter(this.character);
|
|
223
474
|
}
|
|
@@ -279,6 +530,14 @@ var Charivo = class {
|
|
|
279
530
|
manager.setEventEmitter(this.eventBus);
|
|
280
531
|
}
|
|
281
532
|
}
|
|
533
|
+
/**
|
|
534
|
+
* Connects the LLM manager to the event bus for tool-call event emission.
|
|
535
|
+
*/
|
|
536
|
+
connectLLMManagerEventEmitter(manager) {
|
|
537
|
+
if (manager.setEventEmitter) {
|
|
538
|
+
manager.setEventEmitter(this.eventBus);
|
|
539
|
+
}
|
|
540
|
+
}
|
|
282
541
|
/**
|
|
283
542
|
* Set the character for this Charivo instance.
|
|
284
543
|
* Automatically propagates to all attached managers (LLM, Renderer).
|
|
@@ -478,6 +737,11 @@ var Charivo = class {
|
|
|
478
737
|
} catch (error) {
|
|
479
738
|
recordError(error, "Failed to stop TTS during dispose");
|
|
480
739
|
}
|
|
740
|
+
try {
|
|
741
|
+
await this.ttsManager.dispose?.();
|
|
742
|
+
} catch (error) {
|
|
743
|
+
recordError(error, "Failed to release TTS resources during dispose");
|
|
744
|
+
}
|
|
481
745
|
}
|
|
482
746
|
if (this.sttManager) {
|
|
483
747
|
try {
|
|
@@ -519,6 +783,6 @@ var Charivo = class {
|
|
|
519
783
|
}
|
|
520
784
|
};
|
|
521
785
|
|
|
522
|
-
export { Charivo, CharivoDisposeError, CharivoError, CharivoProviderError, CharivoStateError, CharivoTimeoutError, CharivoTransportError, EventBus, OPENAI_REALTIME_ADAPTER, OPENAI_REALTIME_AGENTS_ADAPTER, getErrorMessage, isCharivoError, subscribeBrowserLifecycle, toCharivoError };
|
|
786
|
+
export { Charivo, CharivoDisposeError, CharivoError, CharivoProviderError, CharivoStateError, CharivoTimeoutError, CharivoTransportError, EventBus, OPENAI_REALTIME_ADAPTER, OPENAI_REALTIME_AGENTS_ADAPTER, assertToolResultObject, createLipSyncAnalyzer, getErrorMessage, isCharivoError, subscribeBrowserLifecycle, toCharivoError, validateToolArguments };
|
|
523
787
|
//# sourceMappingURL=index.js.map
|
|
524
788
|
//# sourceMappingURL=index.js.map
|