@omote/babylon 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -0
- package/dist/index.cjs +217 -39
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +84 -31
- package/dist/index.d.ts +84 -31
- package/dist/index.js +224 -40
- package/dist/index.js.map +1 -1
- package/package.json +11 -5
package/dist/index.d.cts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import * as _omote_core from '@omote/core';
|
|
2
|
-
import { FaceCompositorConfig, CharacterControllerConfig, EmotionWeights, ConversationalState,
|
|
2
|
+
import { FaceCompositorConfig, CharacterControllerConfig, FrameSource, TTSBackend, TTSSpeakerConfig, SpeechListenerConfig, TranscriptResult, VoiceOrchestratorConfig, EmotionWeights, ConversationalState, CharacterProfile, TTSSpeaker, SpeechListener } from '@omote/core';
|
|
3
|
+
export { FrameSource, TTSSpeakerConfig as TTSConfig } from '@omote/core';
|
|
3
4
|
import { AbstractMesh, TransformNode, Scene, Camera } from '@babylonjs/core';
|
|
4
5
|
|
|
5
6
|
/**
|
|
@@ -47,13 +48,6 @@ interface SceneDiscoveryResult {
|
|
|
47
48
|
*/
|
|
48
49
|
declare function discoverScene(root: AbstractMesh): SceneDiscoveryResult;
|
|
49
50
|
|
|
50
|
-
/** Generic frame source -- any object that emits 'frame' events */
|
|
51
|
-
interface FrameSource {
|
|
52
|
-
on(event: 'frame', callback: (frame: {
|
|
53
|
-
blendshapes: Float32Array;
|
|
54
|
-
}) => void): void;
|
|
55
|
-
off?(event: 'frame', callback: (...args: any[]) => void): void;
|
|
56
|
-
}
|
|
57
51
|
interface OmoteAvatarOptions {
|
|
58
52
|
/** Root mesh of the avatar (typically loaded via SceneLoader) */
|
|
59
53
|
target: AbstractMesh;
|
|
@@ -82,6 +76,9 @@ declare class OmoteAvatar {
|
|
|
82
76
|
private _camera;
|
|
83
77
|
private frameSourceCallback;
|
|
84
78
|
private connectedSource;
|
|
79
|
+
private ttsSpeaker;
|
|
80
|
+
private speechListener;
|
|
81
|
+
private voiceOrchestrator;
|
|
85
82
|
private renderCallback;
|
|
86
83
|
private lastTime;
|
|
87
84
|
constructor(options: OmoteAvatarOptions);
|
|
@@ -107,6 +104,70 @@ declare class OmoteAvatar {
|
|
|
107
104
|
connectFrameSource(source: FrameSource): void;
|
|
108
105
|
/** Disconnect the current frame source (if any). */
|
|
109
106
|
disconnectFrameSource(): void;
|
|
107
|
+
/**
|
|
108
|
+
* Connect a TTS backend for speak() / streamText() support.
|
|
109
|
+
* Loads LAM model and creates internal PlaybackPipeline.
|
|
110
|
+
*
|
|
111
|
+
* @param tts - TTS backend (e.g., KokoroTTSInference, ElevenLabs adapter)
|
|
112
|
+
* @param config - A2E, expression profile, and playback configuration
|
|
113
|
+
*/
|
|
114
|
+
connectSpeaker(tts: TTSBackend, config?: TTSSpeakerConfig): Promise<void>;
|
|
115
|
+
/**
|
|
116
|
+
* Synthesize text and play with lip sync.
|
|
117
|
+
* Auto-aborts previous speak if still in progress.
|
|
118
|
+
*
|
|
119
|
+
* @param text - Text to synthesize
|
|
120
|
+
* @param options - Optional voice override and abort signal
|
|
121
|
+
*/
|
|
122
|
+
speak(text: string, options?: {
|
|
123
|
+
signal?: AbortSignal;
|
|
124
|
+
voice?: string;
|
|
125
|
+
}): Promise<void>;
|
|
126
|
+
/**
|
|
127
|
+
* Stream LLM tokens with sentence-buffered TTS + lip sync.
|
|
128
|
+
* Returns a sink: call push(token) for each token, end() when done.
|
|
129
|
+
*/
|
|
130
|
+
streamText(options?: {
|
|
131
|
+
signal?: AbortSignal;
|
|
132
|
+
voice?: string;
|
|
133
|
+
}): Promise<{
|
|
134
|
+
push: (token: string) => void;
|
|
135
|
+
end: () => Promise<void>;
|
|
136
|
+
}>;
|
|
137
|
+
/** Stop current TTS playback. */
|
|
138
|
+
stopSpeaking(): void;
|
|
139
|
+
/** Disconnect speaker and dispose its resources. */
|
|
140
|
+
disconnectSpeaker(): Promise<void>;
|
|
141
|
+
/** @deprecated Use connectSpeaker(). Will be removed in v1.0. */
|
|
142
|
+
connectTTS(tts: TTSBackend, config?: TTSSpeakerConfig): Promise<void>;
|
|
143
|
+
/** @deprecated Use disconnectSpeaker(). Will be removed in v1.0. */
|
|
144
|
+
disconnectTTS(): Promise<void>;
|
|
145
|
+
/**
|
|
146
|
+
* Connect a speech listener for startListening() / onTranscript() support.
|
|
147
|
+
* Loads ASR + VAD models.
|
|
148
|
+
*/
|
|
149
|
+
connectListener(config?: SpeechListenerConfig): Promise<void>;
|
|
150
|
+
/** Start listening for user speech. Requires connectListener() or connectVoice() first. */
|
|
151
|
+
startListening(): Promise<void>;
|
|
152
|
+
/** Stop listening. */
|
|
153
|
+
stopListening(): void;
|
|
154
|
+
/**
|
|
155
|
+
* Subscribe to transcript events. Returns an unsubscribe function.
|
|
156
|
+
* Requires connectListener() first.
|
|
157
|
+
*/
|
|
158
|
+
onTranscript(callback: (result: TranscriptResult) => void): () => void;
|
|
159
|
+
/** Disconnect listener and dispose its resources. */
|
|
160
|
+
disconnectListener(): Promise<void>;
|
|
161
|
+
/**
|
|
162
|
+
* Connect voice with automatic speaker + listener + interruption wiring.
|
|
163
|
+
* Supports both local TTS (mode: 'local') and cloud TTS (mode: 'cloud').
|
|
164
|
+
* Does NOT auto-start listening — call startListening() when ready.
|
|
165
|
+
*
|
|
166
|
+
* Backward compatible: `mode` defaults to 'local' when not specified.
|
|
167
|
+
*/
|
|
168
|
+
connectVoice(config: VoiceOrchestratorConfig): Promise<void>;
|
|
169
|
+
/** Disconnect voice (speaker + listener + interruption). */
|
|
170
|
+
disconnectVoice(): Promise<void>;
|
|
110
171
|
/** Set blendshapes directly (alternative to connectFrameSource). */
|
|
111
172
|
setFrame(blendshapes: Float32Array): void;
|
|
112
173
|
/** Set emotion (string preset like 'happy' or EmotionWeights object). */
|
|
@@ -117,6 +178,8 @@ declare class OmoteAvatar {
|
|
|
117
178
|
setState(state: ConversationalState): void;
|
|
118
179
|
/** Set audio energy level (0-1, drives emphasis/gesture intensity). */
|
|
119
180
|
setAudioEnergy(energy: number): void;
|
|
181
|
+
/** Update character expression profile at runtime. */
|
|
182
|
+
setProfile(profile: CharacterProfile): void;
|
|
120
183
|
/**
|
|
121
184
|
* Set the active camera for gaze tracking.
|
|
122
185
|
* Required when using autoUpdate. Can also be passed directly to update().
|
|
@@ -130,10 +193,20 @@ declare class OmoteAvatar {
|
|
|
130
193
|
get hasMorphTargets(): boolean;
|
|
131
194
|
/** Number of successfully mapped ARKit blendshapes. */
|
|
132
195
|
get mappedBlendshapeCount(): number;
|
|
196
|
+
/** Whether the avatar is currently speaking via TTS. */
|
|
197
|
+
get isSpeaking(): boolean;
|
|
198
|
+
/** Whether the avatar is currently listening for speech. */
|
|
199
|
+
get isListening(): boolean;
|
|
200
|
+
/** Current conversational state. */
|
|
201
|
+
get conversationalState(): ConversationalState;
|
|
202
|
+
/** Access the internal TTSSpeaker (null if not connected). */
|
|
203
|
+
get speaker(): TTSSpeaker | null;
|
|
204
|
+
/** Access the internal SpeechListener (null if not connected). */
|
|
205
|
+
get listener(): SpeechListener | null;
|
|
133
206
|
/** Reset all state (smoothing, life layer, emotions). */
|
|
134
207
|
reset(): void;
|
|
135
|
-
/**
|
|
136
|
-
dispose(): void
|
|
208
|
+
/** Disconnect all voice resources, frame sources, unregister render loop, dispose controller. */
|
|
209
|
+
dispose(): Promise<void>;
|
|
137
210
|
private registerAutoUpdate;
|
|
138
211
|
}
|
|
139
212
|
|
|
@@ -191,24 +264,4 @@ declare class BlendshapeController {
|
|
|
191
264
|
dispose(): void;
|
|
192
265
|
}
|
|
193
266
|
|
|
194
|
-
|
|
195
|
-
target: AbstractMesh;
|
|
196
|
-
scene: Scene;
|
|
197
|
-
controllerOptions?: BlendshapeControllerOptions;
|
|
198
|
-
}
|
|
199
|
-
/** @deprecated Use {@link OmoteAvatar} instead. OmoteA2E will be removed in v0.8.0. */
|
|
200
|
-
declare class OmoteA2E {
|
|
201
|
-
private orchestrator;
|
|
202
|
-
private controller;
|
|
203
|
-
constructor(options: OmoteA2EOptions);
|
|
204
|
-
load(): Promise<void>;
|
|
205
|
-
start(): Promise<void>;
|
|
206
|
-
stop(): void;
|
|
207
|
-
update(): void;
|
|
208
|
-
dispose(): Promise<void>;
|
|
209
|
-
get isReady(): boolean;
|
|
210
|
-
get isStreaming(): boolean;
|
|
211
|
-
get backend(): string | null;
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
export { BlendshapeController, type BlendshapeControllerOptions, type FrameSource, type MorphIndexEntry, OmoteA2E, type OmoteA2EOptions, OmoteAvatar, type OmoteAvatarOptions, type SceneDiscoveryResult, discoverScene, writeBlendshapes };
|
|
267
|
+
export { BlendshapeController, type BlendshapeControllerOptions, type MorphIndexEntry, OmoteAvatar, type OmoteAvatarOptions, type SceneDiscoveryResult, discoverScene, writeBlendshapes };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import * as _omote_core from '@omote/core';
|
|
2
|
-
import { FaceCompositorConfig, CharacterControllerConfig, EmotionWeights, ConversationalState,
|
|
2
|
+
import { FaceCompositorConfig, CharacterControllerConfig, FrameSource, TTSBackend, TTSSpeakerConfig, SpeechListenerConfig, TranscriptResult, VoiceOrchestratorConfig, EmotionWeights, ConversationalState, CharacterProfile, TTSSpeaker, SpeechListener } from '@omote/core';
|
|
3
|
+
export { FrameSource, TTSSpeakerConfig as TTSConfig } from '@omote/core';
|
|
3
4
|
import { AbstractMesh, TransformNode, Scene, Camera } from '@babylonjs/core';
|
|
4
5
|
|
|
5
6
|
/**
|
|
@@ -47,13 +48,6 @@ interface SceneDiscoveryResult {
|
|
|
47
48
|
*/
|
|
48
49
|
declare function discoverScene(root: AbstractMesh): SceneDiscoveryResult;
|
|
49
50
|
|
|
50
|
-
/** Generic frame source -- any object that emits 'frame' events */
|
|
51
|
-
interface FrameSource {
|
|
52
|
-
on(event: 'frame', callback: (frame: {
|
|
53
|
-
blendshapes: Float32Array;
|
|
54
|
-
}) => void): void;
|
|
55
|
-
off?(event: 'frame', callback: (...args: any[]) => void): void;
|
|
56
|
-
}
|
|
57
51
|
interface OmoteAvatarOptions {
|
|
58
52
|
/** Root mesh of the avatar (typically loaded via SceneLoader) */
|
|
59
53
|
target: AbstractMesh;
|
|
@@ -82,6 +76,9 @@ declare class OmoteAvatar {
|
|
|
82
76
|
private _camera;
|
|
83
77
|
private frameSourceCallback;
|
|
84
78
|
private connectedSource;
|
|
79
|
+
private ttsSpeaker;
|
|
80
|
+
private speechListener;
|
|
81
|
+
private voiceOrchestrator;
|
|
85
82
|
private renderCallback;
|
|
86
83
|
private lastTime;
|
|
87
84
|
constructor(options: OmoteAvatarOptions);
|
|
@@ -107,6 +104,70 @@ declare class OmoteAvatar {
|
|
|
107
104
|
connectFrameSource(source: FrameSource): void;
|
|
108
105
|
/** Disconnect the current frame source (if any). */
|
|
109
106
|
disconnectFrameSource(): void;
|
|
107
|
+
/**
|
|
108
|
+
* Connect a TTS backend for speak() / streamText() support.
|
|
109
|
+
* Loads LAM model and creates internal PlaybackPipeline.
|
|
110
|
+
*
|
|
111
|
+
* @param tts - TTS backend (e.g., KokoroTTSInference, ElevenLabs adapter)
|
|
112
|
+
* @param config - A2E, expression profile, and playback configuration
|
|
113
|
+
*/
|
|
114
|
+
connectSpeaker(tts: TTSBackend, config?: TTSSpeakerConfig): Promise<void>;
|
|
115
|
+
/**
|
|
116
|
+
* Synthesize text and play with lip sync.
|
|
117
|
+
* Auto-aborts previous speak if still in progress.
|
|
118
|
+
*
|
|
119
|
+
* @param text - Text to synthesize
|
|
120
|
+
* @param options - Optional voice override and abort signal
|
|
121
|
+
*/
|
|
122
|
+
speak(text: string, options?: {
|
|
123
|
+
signal?: AbortSignal;
|
|
124
|
+
voice?: string;
|
|
125
|
+
}): Promise<void>;
|
|
126
|
+
/**
|
|
127
|
+
* Stream LLM tokens with sentence-buffered TTS + lip sync.
|
|
128
|
+
* Returns a sink: call push(token) for each token, end() when done.
|
|
129
|
+
*/
|
|
130
|
+
streamText(options?: {
|
|
131
|
+
signal?: AbortSignal;
|
|
132
|
+
voice?: string;
|
|
133
|
+
}): Promise<{
|
|
134
|
+
push: (token: string) => void;
|
|
135
|
+
end: () => Promise<void>;
|
|
136
|
+
}>;
|
|
137
|
+
/** Stop current TTS playback. */
|
|
138
|
+
stopSpeaking(): void;
|
|
139
|
+
/** Disconnect speaker and dispose its resources. */
|
|
140
|
+
disconnectSpeaker(): Promise<void>;
|
|
141
|
+
/** @deprecated Use connectSpeaker(). Will be removed in v1.0. */
|
|
142
|
+
connectTTS(tts: TTSBackend, config?: TTSSpeakerConfig): Promise<void>;
|
|
143
|
+
/** @deprecated Use disconnectSpeaker(). Will be removed in v1.0. */
|
|
144
|
+
disconnectTTS(): Promise<void>;
|
|
145
|
+
/**
|
|
146
|
+
* Connect a speech listener for startListening() / onTranscript() support.
|
|
147
|
+
* Loads ASR + VAD models.
|
|
148
|
+
*/
|
|
149
|
+
connectListener(config?: SpeechListenerConfig): Promise<void>;
|
|
150
|
+
/** Start listening for user speech. Requires connectListener() or connectVoice() first. */
|
|
151
|
+
startListening(): Promise<void>;
|
|
152
|
+
/** Stop listening. */
|
|
153
|
+
stopListening(): void;
|
|
154
|
+
/**
|
|
155
|
+
* Subscribe to transcript events. Returns an unsubscribe function.
|
|
156
|
+
* Requires connectListener() first.
|
|
157
|
+
*/
|
|
158
|
+
onTranscript(callback: (result: TranscriptResult) => void): () => void;
|
|
159
|
+
/** Disconnect listener and dispose its resources. */
|
|
160
|
+
disconnectListener(): Promise<void>;
|
|
161
|
+
/**
|
|
162
|
+
* Connect voice with automatic speaker + listener + interruption wiring.
|
|
163
|
+
* Supports both local TTS (mode: 'local') and cloud TTS (mode: 'cloud').
|
|
164
|
+
* Does NOT auto-start listening — call startListening() when ready.
|
|
165
|
+
*
|
|
166
|
+
* Backward compatible: `mode` defaults to 'local' when not specified.
|
|
167
|
+
*/
|
|
168
|
+
connectVoice(config: VoiceOrchestratorConfig): Promise<void>;
|
|
169
|
+
/** Disconnect voice (speaker + listener + interruption). */
|
|
170
|
+
disconnectVoice(): Promise<void>;
|
|
110
171
|
/** Set blendshapes directly (alternative to connectFrameSource). */
|
|
111
172
|
setFrame(blendshapes: Float32Array): void;
|
|
112
173
|
/** Set emotion (string preset like 'happy' or EmotionWeights object). */
|
|
@@ -117,6 +178,8 @@ declare class OmoteAvatar {
|
|
|
117
178
|
setState(state: ConversationalState): void;
|
|
118
179
|
/** Set audio energy level (0-1, drives emphasis/gesture intensity). */
|
|
119
180
|
setAudioEnergy(energy: number): void;
|
|
181
|
+
/** Update character expression profile at runtime. */
|
|
182
|
+
setProfile(profile: CharacterProfile): void;
|
|
120
183
|
/**
|
|
121
184
|
* Set the active camera for gaze tracking.
|
|
122
185
|
* Required when using autoUpdate. Can also be passed directly to update().
|
|
@@ -130,10 +193,20 @@ declare class OmoteAvatar {
|
|
|
130
193
|
get hasMorphTargets(): boolean;
|
|
131
194
|
/** Number of successfully mapped ARKit blendshapes. */
|
|
132
195
|
get mappedBlendshapeCount(): number;
|
|
196
|
+
/** Whether the avatar is currently speaking via TTS. */
|
|
197
|
+
get isSpeaking(): boolean;
|
|
198
|
+
/** Whether the avatar is currently listening for speech. */
|
|
199
|
+
get isListening(): boolean;
|
|
200
|
+
/** Current conversational state. */
|
|
201
|
+
get conversationalState(): ConversationalState;
|
|
202
|
+
/** Access the internal TTSSpeaker (null if not connected). */
|
|
203
|
+
get speaker(): TTSSpeaker | null;
|
|
204
|
+
/** Access the internal SpeechListener (null if not connected). */
|
|
205
|
+
get listener(): SpeechListener | null;
|
|
133
206
|
/** Reset all state (smoothing, life layer, emotions). */
|
|
134
207
|
reset(): void;
|
|
135
|
-
/**
|
|
136
|
-
dispose(): void
|
|
208
|
+
/** Disconnect all voice resources, frame sources, unregister render loop, dispose controller. */
|
|
209
|
+
dispose(): Promise<void>;
|
|
137
210
|
private registerAutoUpdate;
|
|
138
211
|
}
|
|
139
212
|
|
|
@@ -191,24 +264,4 @@ declare class BlendshapeController {
|
|
|
191
264
|
dispose(): void;
|
|
192
265
|
}
|
|
193
266
|
|
|
194
|
-
|
|
195
|
-
target: AbstractMesh;
|
|
196
|
-
scene: Scene;
|
|
197
|
-
controllerOptions?: BlendshapeControllerOptions;
|
|
198
|
-
}
|
|
199
|
-
/** @deprecated Use {@link OmoteAvatar} instead. OmoteA2E will be removed in v0.8.0. */
|
|
200
|
-
declare class OmoteA2E {
|
|
201
|
-
private orchestrator;
|
|
202
|
-
private controller;
|
|
203
|
-
constructor(options: OmoteA2EOptions);
|
|
204
|
-
load(): Promise<void>;
|
|
205
|
-
start(): Promise<void>;
|
|
206
|
-
stop(): void;
|
|
207
|
-
update(): void;
|
|
208
|
-
dispose(): Promise<void>;
|
|
209
|
-
get isReady(): boolean;
|
|
210
|
-
get isStreaming(): boolean;
|
|
211
|
-
get backend(): string | null;
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
export { BlendshapeController, type BlendshapeControllerOptions, type FrameSource, type MorphIndexEntry, OmoteA2E, type OmoteA2EOptions, OmoteAvatar, type OmoteAvatarOptions, type SceneDiscoveryResult, discoverScene, writeBlendshapes };
|
|
267
|
+
export { BlendshapeController, type BlendshapeControllerOptions, type MorphIndexEntry, OmoteAvatar, type OmoteAvatarOptions, type SceneDiscoveryResult, discoverScene, writeBlendshapes };
|
package/dist/index.js
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
// src/OmoteAvatar.ts
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
CharacterController,
|
|
4
|
+
TTSSpeaker,
|
|
5
|
+
SpeechListener,
|
|
6
|
+
VoiceOrchestrator,
|
|
7
|
+
createLogger as createLogger2
|
|
8
|
+
} from "@omote/core";
|
|
3
9
|
|
|
4
10
|
// src/SceneDiscovery.ts
|
|
5
11
|
import { LAM_BLENDSHAPES, createLogger } from "@omote/core";
|
|
@@ -129,6 +135,12 @@ var OmoteAvatar = class {
|
|
|
129
135
|
// Frame source connection
|
|
130
136
|
this.frameSourceCallback = null;
|
|
131
137
|
this.connectedSource = null;
|
|
138
|
+
// TTS integration
|
|
139
|
+
this.ttsSpeaker = null;
|
|
140
|
+
// Speech listener
|
|
141
|
+
this.speechListener = null;
|
|
142
|
+
// Voice orchestrator
|
|
143
|
+
this.voiceOrchestrator = null;
|
|
132
144
|
// Auto-update
|
|
133
145
|
this.renderCallback = null;
|
|
134
146
|
this.lastTime = 0;
|
|
@@ -207,9 +219,15 @@ var OmoteAvatar = class {
|
|
|
207
219
|
* Only one source can be connected at a time; calling again disconnects the previous.
|
|
208
220
|
*/
|
|
209
221
|
connectFrameSource(source) {
|
|
222
|
+
if (this.ttsSpeaker && source !== this.ttsSpeaker.frameSource) {
|
|
223
|
+
this.ttsSpeaker.stop();
|
|
224
|
+
}
|
|
210
225
|
this.disconnectFrameSource();
|
|
211
226
|
this.frameSourceCallback = (frame) => {
|
|
212
227
|
this.currentBlendshapes = frame.blendshapes;
|
|
228
|
+
if (frame.emotion !== void 0) {
|
|
229
|
+
this._emotion = frame.emotion;
|
|
230
|
+
}
|
|
213
231
|
};
|
|
214
232
|
source.on("frame", this.frameSourceCallback);
|
|
215
233
|
this.connectedSource = source;
|
|
@@ -225,6 +243,182 @@ var OmoteAvatar = class {
|
|
|
225
243
|
this.frameSourceCallback = null;
|
|
226
244
|
}
|
|
227
245
|
// ---------------------------------------------------------------------------
|
|
246
|
+
// Speaker (TTS → lip sync)
|
|
247
|
+
// ---------------------------------------------------------------------------
|
|
248
|
+
/**
|
|
249
|
+
* Connect a TTS backend for speak() / streamText() support.
|
|
250
|
+
* Loads LAM model and creates internal PlaybackPipeline.
|
|
251
|
+
*
|
|
252
|
+
* @param tts - TTS backend (e.g., KokoroTTSInference, ElevenLabs adapter)
|
|
253
|
+
* @param config - A2E, expression profile, and playback configuration
|
|
254
|
+
*/
|
|
255
|
+
async connectSpeaker(tts, config) {
|
|
256
|
+
await this.disconnectSpeaker();
|
|
257
|
+
this.ttsSpeaker = new TTSSpeaker();
|
|
258
|
+
await this.ttsSpeaker.connect(tts, config);
|
|
259
|
+
this.connectFrameSource(this.ttsSpeaker.frameSource);
|
|
260
|
+
}
|
|
261
|
+
/**
|
|
262
|
+
* Synthesize text and play with lip sync.
|
|
263
|
+
* Auto-aborts previous speak if still in progress.
|
|
264
|
+
*
|
|
265
|
+
* @param text - Text to synthesize
|
|
266
|
+
* @param options - Optional voice override and abort signal
|
|
267
|
+
*/
|
|
268
|
+
async speak(text, options) {
|
|
269
|
+
if (this.voiceOrchestrator) {
|
|
270
|
+
await this.voiceOrchestrator.speak(text, options);
|
|
271
|
+
return;
|
|
272
|
+
}
|
|
273
|
+
if (!this.ttsSpeaker) {
|
|
274
|
+
throw new Error("No speaker connected. Call connectSpeaker() first.");
|
|
275
|
+
}
|
|
276
|
+
this._isSpeaking = true;
|
|
277
|
+
this._state = "speaking";
|
|
278
|
+
try {
|
|
279
|
+
await this.ttsSpeaker.speak(text, options);
|
|
280
|
+
} finally {
|
|
281
|
+
this._isSpeaking = false;
|
|
282
|
+
if (this._state === "speaking") {
|
|
283
|
+
this._state = "idle";
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
/**
|
|
288
|
+
* Stream LLM tokens with sentence-buffered TTS + lip sync.
|
|
289
|
+
* Returns a sink: call push(token) for each token, end() when done.
|
|
290
|
+
*/
|
|
291
|
+
async streamText(options) {
|
|
292
|
+
if (this.voiceOrchestrator) {
|
|
293
|
+
return this.voiceOrchestrator.streamText(options);
|
|
294
|
+
}
|
|
295
|
+
if (!this.ttsSpeaker) {
|
|
296
|
+
throw new Error("No speaker connected. Call connectSpeaker() first.");
|
|
297
|
+
}
|
|
298
|
+
this._isSpeaking = true;
|
|
299
|
+
this._state = "speaking";
|
|
300
|
+
const stream = await this.ttsSpeaker.streamText(options ?? {});
|
|
301
|
+
return {
|
|
302
|
+
push: stream.push,
|
|
303
|
+
end: async () => {
|
|
304
|
+
try {
|
|
305
|
+
await stream.end();
|
|
306
|
+
} finally {
|
|
307
|
+
this._isSpeaking = false;
|
|
308
|
+
if (this._state === "speaking") this._state = "idle";
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
};
|
|
312
|
+
}
|
|
313
|
+
/** Stop current TTS playback. */
|
|
314
|
+
stopSpeaking() {
|
|
315
|
+
if (this.voiceOrchestrator) {
|
|
316
|
+
this.voiceOrchestrator.stopSpeaking();
|
|
317
|
+
return;
|
|
318
|
+
}
|
|
319
|
+
this.ttsSpeaker?.stop();
|
|
320
|
+
}
|
|
321
|
+
/** Disconnect speaker and dispose its resources. */
|
|
322
|
+
async disconnectSpeaker() {
|
|
323
|
+
if (this.ttsSpeaker) {
|
|
324
|
+
this.disconnectFrameSource();
|
|
325
|
+
await this.ttsSpeaker.dispose();
|
|
326
|
+
this.ttsSpeaker = null;
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
/** @deprecated Use connectSpeaker(). Will be removed in v1.0. */
|
|
330
|
+
async connectTTS(tts, config) {
|
|
331
|
+
return this.connectSpeaker(tts, config);
|
|
332
|
+
}
|
|
333
|
+
/** @deprecated Use disconnectSpeaker(). Will be removed in v1.0. */
|
|
334
|
+
async disconnectTTS() {
|
|
335
|
+
return this.disconnectSpeaker();
|
|
336
|
+
}
|
|
337
|
+
// ---------------------------------------------------------------------------
|
|
338
|
+
// Listener (mic → VAD → ASR → transcript)
|
|
339
|
+
// ---------------------------------------------------------------------------
|
|
340
|
+
/**
|
|
341
|
+
* Connect a speech listener for startListening() / onTranscript() support.
|
|
342
|
+
* Loads ASR + VAD models.
|
|
343
|
+
*/
|
|
344
|
+
async connectListener(config) {
|
|
345
|
+
await this.disconnectListener();
|
|
346
|
+
this.speechListener = new SpeechListener(config);
|
|
347
|
+
await this.speechListener.loadModels();
|
|
348
|
+
}
|
|
349
|
+
/** Start listening for user speech. Requires connectListener() or connectVoice() first. */
|
|
350
|
+
async startListening() {
|
|
351
|
+
if (this.voiceOrchestrator) {
|
|
352
|
+
await this.voiceOrchestrator.startListening();
|
|
353
|
+
return;
|
|
354
|
+
}
|
|
355
|
+
if (!this.speechListener) {
|
|
356
|
+
throw new Error("No listener connected. Call connectListener() first.");
|
|
357
|
+
}
|
|
358
|
+
this._state = "listening";
|
|
359
|
+
await this.speechListener.start();
|
|
360
|
+
}
|
|
361
|
+
/** Stop listening. */
|
|
362
|
+
stopListening() {
|
|
363
|
+
if (this.voiceOrchestrator) {
|
|
364
|
+
this.voiceOrchestrator.stopListening();
|
|
365
|
+
return;
|
|
366
|
+
}
|
|
367
|
+
this.speechListener?.stop();
|
|
368
|
+
if (this._state === "listening") this._state = "idle";
|
|
369
|
+
}
|
|
370
|
+
/**
|
|
371
|
+
* Subscribe to transcript events. Returns an unsubscribe function.
|
|
372
|
+
* Requires connectListener() first.
|
|
373
|
+
*/
|
|
374
|
+
onTranscript(callback) {
|
|
375
|
+
const listener = this.speechListener ?? this.voiceOrchestrator?.listener;
|
|
376
|
+
if (!listener) {
|
|
377
|
+
throw new Error("No listener connected. Call connectListener() or connectVoice() first.");
|
|
378
|
+
}
|
|
379
|
+
listener.on("transcript", callback);
|
|
380
|
+
return () => {
|
|
381
|
+
listener.off?.("transcript", callback);
|
|
382
|
+
};
|
|
383
|
+
}
|
|
384
|
+
/** Disconnect listener and dispose its resources. */
|
|
385
|
+
async disconnectListener() {
|
|
386
|
+
if (this.speechListener) {
|
|
387
|
+
await this.speechListener.dispose();
|
|
388
|
+
this.speechListener = null;
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
// ---------------------------------------------------------------------------
|
|
392
|
+
// Voice (combined speaker + listener + interruption)
|
|
393
|
+
// ---------------------------------------------------------------------------
|
|
394
|
+
/**
|
|
395
|
+
* Connect voice with automatic speaker + listener + interruption wiring.
|
|
396
|
+
* Supports both local TTS (mode: 'local') and cloud TTS (mode: 'cloud').
|
|
397
|
+
* Does NOT auto-start listening — call startListening() when ready.
|
|
398
|
+
*
|
|
399
|
+
* Backward compatible: `mode` defaults to 'local' when not specified.
|
|
400
|
+
*/
|
|
401
|
+
async connectVoice(config) {
|
|
402
|
+
await this.disconnectVoice();
|
|
403
|
+
this.voiceOrchestrator = new VoiceOrchestrator();
|
|
404
|
+
await this.voiceOrchestrator.connect(config);
|
|
405
|
+
if (this.voiceOrchestrator.frameSource) {
|
|
406
|
+
this.connectFrameSource(this.voiceOrchestrator.frameSource);
|
|
407
|
+
}
|
|
408
|
+
this.voiceOrchestrator.on("state", (state) => {
|
|
409
|
+
this._state = state;
|
|
410
|
+
this._isSpeaking = state === "speaking";
|
|
411
|
+
});
|
|
412
|
+
}
|
|
413
|
+
/** Disconnect voice (speaker + listener + interruption). */
|
|
414
|
+
async disconnectVoice() {
|
|
415
|
+
if (this.voiceOrchestrator) {
|
|
416
|
+
this.disconnectFrameSource();
|
|
417
|
+
await this.voiceOrchestrator.disconnect();
|
|
418
|
+
this.voiceOrchestrator = null;
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
// ---------------------------------------------------------------------------
|
|
228
422
|
// State setters
|
|
229
423
|
// ---------------------------------------------------------------------------
|
|
230
424
|
/** Set blendshapes directly (alternative to connectFrameSource). */
|
|
@@ -247,6 +441,10 @@ var OmoteAvatar = class {
|
|
|
247
441
|
setAudioEnergy(energy) {
|
|
248
442
|
this._audioEnergy = energy;
|
|
249
443
|
}
|
|
444
|
+
/** Update character expression profile at runtime. */
|
|
445
|
+
setProfile(profile) {
|
|
446
|
+
this.controller.setProfile(profile);
|
|
447
|
+
}
|
|
250
448
|
/**
|
|
251
449
|
* Set the active camera for gaze tracking.
|
|
252
450
|
* Required when using autoUpdate. Can also be passed directly to update().
|
|
@@ -273,6 +471,26 @@ var OmoteAvatar = class {
|
|
|
273
471
|
get mappedBlendshapeCount() {
|
|
274
472
|
return this.discovery.mappedBlendshapeCount;
|
|
275
473
|
}
|
|
474
|
+
/** Whether the avatar is currently speaking via TTS. */
|
|
475
|
+
get isSpeaking() {
|
|
476
|
+
return this._isSpeaking;
|
|
477
|
+
}
|
|
478
|
+
/** Whether the avatar is currently listening for speech. */
|
|
479
|
+
get isListening() {
|
|
480
|
+
return this._state === "listening";
|
|
481
|
+
}
|
|
482
|
+
/** Current conversational state. */
|
|
483
|
+
get conversationalState() {
|
|
484
|
+
return this._state;
|
|
485
|
+
}
|
|
486
|
+
/** Access the internal TTSSpeaker (null if not connected). */
|
|
487
|
+
get speaker() {
|
|
488
|
+
return this.ttsSpeaker ?? this.voiceOrchestrator?.speaker ?? null;
|
|
489
|
+
}
|
|
490
|
+
/** Access the internal SpeechListener (null if not connected). */
|
|
491
|
+
get listener() {
|
|
492
|
+
return this.speechListener ?? this.voiceOrchestrator?.listener ?? null;
|
|
493
|
+
}
|
|
276
494
|
// ---------------------------------------------------------------------------
|
|
277
495
|
// Lifecycle
|
|
278
496
|
// ---------------------------------------------------------------------------
|
|
@@ -285,8 +503,11 @@ var OmoteAvatar = class {
|
|
|
285
503
|
this._audioEnergy = 0;
|
|
286
504
|
this.controller.reset();
|
|
287
505
|
}
|
|
288
|
-
/**
|
|
289
|
-
dispose() {
|
|
506
|
+
/** Disconnect all voice resources, frame sources, unregister render loop, dispose controller. */
|
|
507
|
+
async dispose() {
|
|
508
|
+
await this.disconnectVoice();
|
|
509
|
+
await this.disconnectSpeaker();
|
|
510
|
+
await this.disconnectListener();
|
|
290
511
|
this.disconnectFrameSource();
|
|
291
512
|
if (this.renderCallback) {
|
|
292
513
|
this.scene.unregisterBeforeRender(this.renderCallback);
|
|
@@ -419,45 +640,8 @@ var BlendshapeController = class {
|
|
|
419
640
|
this.scene = null;
|
|
420
641
|
}
|
|
421
642
|
};
|
|
422
|
-
|
|
423
|
-
// src/OmoteA2E.ts
|
|
424
|
-
import { A2EOrchestrator } from "@omote/core";
|
|
425
|
-
var OmoteA2E = class {
|
|
426
|
-
constructor(options) {
|
|
427
|
-
const { target, scene, controllerOptions, ...orchestratorConfig } = options;
|
|
428
|
-
this.controller = new BlendshapeController(target, scene, controllerOptions);
|
|
429
|
-
this.orchestrator = new A2EOrchestrator(orchestratorConfig);
|
|
430
|
-
}
|
|
431
|
-
async load() {
|
|
432
|
-
return this.orchestrator.load();
|
|
433
|
-
}
|
|
434
|
-
async start() {
|
|
435
|
-
return this.orchestrator.start();
|
|
436
|
-
}
|
|
437
|
-
stop() {
|
|
438
|
-
this.orchestrator.stop();
|
|
439
|
-
}
|
|
440
|
-
update() {
|
|
441
|
-
const w = this.orchestrator.latestWeights;
|
|
442
|
-
if (w) this.controller.update(w);
|
|
443
|
-
}
|
|
444
|
-
async dispose() {
|
|
445
|
-
await this.orchestrator.dispose();
|
|
446
|
-
this.controller.dispose();
|
|
447
|
-
}
|
|
448
|
-
get isReady() {
|
|
449
|
-
return this.orchestrator.isReady;
|
|
450
|
-
}
|
|
451
|
-
get isStreaming() {
|
|
452
|
-
return this.orchestrator.isStreaming;
|
|
453
|
-
}
|
|
454
|
-
get backend() {
|
|
455
|
-
return this.orchestrator.backend;
|
|
456
|
-
}
|
|
457
|
-
};
|
|
458
643
|
export {
|
|
459
644
|
BlendshapeController,
|
|
460
|
-
OmoteA2E,
|
|
461
645
|
OmoteAvatar,
|
|
462
646
|
discoverScene,
|
|
463
647
|
writeBlendshapes
|