@usamanawaz/react-native-ai-avatar 0.0.0-stage → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +86 -2
  3. package/assets/models/male-avatar-clothed.glb +0 -0
  4. package/assets/models/male-avatar.glb +0 -0
  5. package/dist/AIAvatar.d.ts +20 -0
  6. package/dist/AvatarView.d.ts +3 -0
  7. package/dist/animation/animator.d.ts +44 -0
  8. package/dist/animation/channels.d.ts +32 -0
  9. package/dist/animation/expressions.d.ts +4 -0
  10. package/dist/index.d.ts +13 -0
  11. package/dist/model/bakeSkinning.d.ts +8 -0
  12. package/dist/model/blendshapes.d.ts +12 -0
  13. package/dist/model/boneMap.d.ts +9 -0
  14. package/dist/model/catalog.d.ts +2 -0
  15. package/dist/model/cpuMorphs.d.ts +24 -0
  16. package/dist/model/cpuSkin.d.ts +19 -0
  17. package/dist/model/loadAvatar.d.ts +21 -0
  18. package/dist/model/skeleton.d.ts +17 -0
  19. package/dist/model/visemes.d.ts +19 -0
  20. package/dist/rendering/AvatarStage.d.ts +7 -0
  21. package/dist/rendering/applyPose.d.ts +3 -0
  22. package/dist/speech/ExpoSpeechProvider.d.ts +13 -0
  23. package/dist/speech/approximateMouth.d.ts +15 -0
  24. package/dist/types.d.ts +35 -0
  25. package/dist/voice/types.d.ts +18 -0
  26. package/package.json +72 -4
  27. package/src/avatar/AIAvatar.tsx +137 -0
  28. package/src/avatar/AvatarView.tsx +11 -0
  29. package/src/avatar/animation/animator.ts +344 -0
  30. package/src/avatar/animation/channels.ts +59 -0
  31. package/src/avatar/animation/expressions.ts +51 -0
  32. package/src/avatar/index.ts +23 -0
  33. package/src/avatar/model/bakeSkinning.ts +46 -0
  34. package/src/avatar/model/blendshapes.ts +70 -0
  35. package/src/avatar/model/boneMap.ts +17 -0
  36. package/src/avatar/model/catalog.ts +15 -0
  37. package/src/avatar/model/cpuMorphs.ts +124 -0
  38. package/src/avatar/model/cpuSkin.ts +105 -0
  39. package/src/avatar/model/jpeg-js.d.ts +6 -0
  40. package/src/avatar/model/loadAvatar.ts +497 -0
  41. package/src/avatar/model/skeleton.ts +21 -0
  42. package/src/avatar/model/visemes.ts +55 -0
  43. package/src/avatar/rendering/AvatarStage.tsx +119 -0
  44. package/src/avatar/rendering/applyPose.ts +59 -0
  45. package/src/avatar/speech/ExpoSpeechProvider.ts +42 -0
  46. package/src/avatar/speech/approximateMouth.ts +244 -0
  47. package/src/avatar/types.ts +45 -0
  48. package/src/avatar/voice/types.ts +20 -0
@@ -0,0 +1,137 @@
1
+ import { forwardRef, useEffect, useImperativeHandle, useRef } from 'react';
2
+ import { StyleSheet, Text, View, type StyleProp, type ViewStyle } from 'react-native';
3
+
4
+ import { AvatarAnimator } from './animation/animator';
5
+ import { avatarModelSource } from './model/catalog';
6
+ import { AvatarStage } from './rendering/AvatarStage';
7
+ import { createExpoSpeechProvider } from './speech/ExpoSpeechProvider';
8
+ import type { AvatarModelId, AvatarOutfit, ExpressionName } from './types';
9
+ import type { VoiceProvider } from './voice/types';
10
+
11
+ export type AIAvatarProps = {
12
+ model?: AvatarModelId;
13
+ /** `casual` wears the black crew-neck. Omit it to keep the default male model. */
14
+ outfit?: AvatarOutfit;
15
+ enableBlinking?: boolean;
16
+ enableHeadMovement?: boolean;
17
+ enableEyeMovement?: boolean;
18
+ expression?: ExpressionName;
19
+ style?: StyleProp<ViewStyle>;
20
+ voice?: VoiceProvider;
21
+ };
22
+
23
+ export type AIAvatarHandle = {
24
+ /** Speaks with the system voice and an approximate mouth rhythm. Resolves when speech ends or is stopped. */
25
+ speak(text: string): Promise<void>;
26
+ stop(): Promise<void>;
27
+ };
28
+
29
+ export const AIAvatar = forwardRef<AIAvatarHandle, AIAvatarProps>(function AIAvatar(
30
+ {
31
+ model = 'male',
32
+ outfit,
33
+ enableBlinking = true,
34
+ enableHeadMovement = true,
35
+ enableEyeMovement = true,
36
+ expression = 'neutral',
37
+ style,
38
+ voice,
39
+ },
40
+ ref,
41
+ ) {
42
+ const animatorRef = useRef<AvatarAnimator | null>(null);
43
+ if (animatorRef.current === null) animatorRef.current = new AvatarAnimator();
44
+ const animator = animatorRef.current;
45
+ const voiceRef = useRef<VoiceProvider>(voice ?? createExpoSpeechProvider());
46
+ const runRef = useRef(0);
47
+
48
+ useEffect(() => {
49
+ animator.setExpression(expression);
50
+ }, [animator, expression]);
51
+
52
+ useEffect(() => {
53
+ animator.setFlags({
54
+ blink: enableBlinking,
55
+ head: enableHeadMovement,
56
+ eyes: enableEyeMovement,
57
+ });
58
+ }, [animator, enableBlinking, enableEyeMovement, enableHeadMovement]);
59
+
60
+ useEffect(() => {
61
+ if (voice) voiceRef.current = voice;
62
+ }, [voice]);
63
+
64
+ useImperativeHandle(
65
+ ref,
66
+ () => ({
67
+ async speak(text: string) {
68
+ const spoken = text.trim();
69
+ if (!spoken) return;
70
+ const run = ++runRef.current;
71
+ await voiceRef.current.stop();
72
+ if (run !== runRef.current) return;
73
+ animator.beginSpeech(spoken);
74
+ try {
75
+ await voiceRef.current.speak({
76
+ text: spoken,
77
+ onBoundary: ({ charIndex, charLength }) => {
78
+ if (run !== runRef.current) return;
79
+ animator.setSpeechBoundary(charIndex, charLength);
80
+ },
81
+ });
82
+ } finally {
83
+ if (run === runRef.current) animator.endSpeech();
84
+ }
85
+ },
86
+ async stop() {
87
+ runRef.current += 1;
88
+ animator.endSpeech();
89
+ await voiceRef.current.stop();
90
+ },
91
+ }),
92
+ [animator],
93
+ );
94
+
95
+ const source = avatarModelSource(model, outfit);
96
+ if (source === null) {
97
+ return (
98
+ <View style={[styles.missing, style]}>
99
+ <Text style={styles.missingTitle}>{model === 'female' ? 'Female' : 'Male'} avatar</Text>
100
+ <Text style={styles.missingBody}>This model is not bundled yet.</Text>
101
+ </View>
102
+ );
103
+ }
104
+
105
+ return (
106
+ <View style={[styles.fill, style]}>
107
+ <AvatarStage source={source} animator={animator} />
108
+ </View>
109
+ );
110
+ });
111
+
112
+ const styles = StyleSheet.create({
113
+ fill: {
114
+ flex: 1,
115
+ minHeight: 320,
116
+ },
117
+ missing: {
118
+ flex: 1,
119
+ minHeight: 320,
120
+ borderRadius: 16,
121
+ backgroundColor: '#243044',
122
+ alignItems: 'center',
123
+ justifyContent: 'center',
124
+ padding: 24,
125
+ gap: 8,
126
+ },
127
+ missingTitle: {
128
+ color: '#ffffff',
129
+ fontSize: 22,
130
+ fontWeight: '600',
131
+ },
132
+ missingBody: {
133
+ color: '#d5deea',
134
+ fontSize: 15,
135
+ textAlign: 'center',
136
+ },
137
+ });
@@ -0,0 +1,11 @@
1
+ import { AIAvatar } from './AIAvatar';
2
+ import type { AvatarViewProps } from './types';
3
+
4
+ /** Older surface. Prefer `AIAvatar` when you need speech or animation switches. */
5
+ export function AvatarView({
6
+ model,
7
+ sex = 'male',
8
+ expression = 'neutral',
9
+ }: AvatarViewProps) {
10
+ return <AIAvatar model={model?.sex ?? sex} expression={expression} />;
11
+ }
@@ -0,0 +1,344 @@
1
+ import { BLENDSHAPE_CHANNELS, BLINK_DURATION_MS, BLINK_INTERVAL_MS, HEAD_LIMITS_DEG } from './channels';
2
+ import { EXPRESSION_PRESETS } from './expressions';
3
+ import type { BlendshapeName, BlendshapeWeights } from '../model/blendshapes';
4
+ import type { BonePose } from '../model/skeleton';
5
+ import type { ExpressionName } from '../types';
6
+ import {
7
+ buildMouthWords,
8
+ mouthWeightsAt,
9
+ scaledWordCues,
10
+ type MouthWord,
11
+ } from '../speech/approximateMouth';
12
+ import type { VisemeCue } from '../model/visemes';
13
+
14
+ export type AvatarPose = {
15
+ weights: BlendshapeWeights;
16
+ bones: BonePose;
17
+ };
18
+
19
+ type Flags = {
20
+ blink: boolean;
21
+ head: boolean;
22
+ eyes: boolean;
23
+ };
24
+
25
+ const EXPRESSION_RATE = 6;
26
+
27
+ /**
28
+ * Frame-independent face pose. Rendering only applies the result.
29
+ * Mouth cues follow spoken word boundaries from the voice provider.
30
+ */
31
+ export class AvatarAnimator {
32
+ private expression: ExpressionName = 'neutral';
33
+ private flags: Flags = { blink: true, head: true, eyes: true };
34
+ private weights: BlendshapeWeights = {};
35
+ private bones: Required<BonePose> = {
36
+ Neck: { x: 0, y: 0, z: 0 },
37
+ Head: { x: 0, y: 0, z: 0 },
38
+ LeftEye: { x: 0, y: 0, z: 0 },
39
+ RightEye: { x: 0, y: 0, z: 0 },
40
+ Jaw: { x: 0, y: 0, z: 0 },
41
+ };
42
+ private lastTime: number | null = null;
43
+ private nextBlinkAt = 0;
44
+ private blinkStarted = false;
45
+ private blinkAt = 0;
46
+ private gaze = { x: 0, y: 0 };
47
+ private gazeTarget = { x: 0, y: 0 };
48
+ private nextSaccadeAt = 0;
49
+ private speech: {
50
+ words: MouthWord[];
51
+ wordIndex: number;
52
+ cues: VisemeCue[];
53
+ cueStartedAt: number | null;
54
+ armedAt: number | null;
55
+ secondsPerUnit: number;
56
+ boundarySeen: boolean;
57
+ text: string;
58
+ } | null = null;
59
+
60
+ setExpression(expression: ExpressionName) {
61
+ this.expression = expression;
62
+ }
63
+
64
+ setFlags(flags: Flags) {
65
+ this.flags = flags;
66
+ }
67
+
68
+ beginSpeech(text: string) {
69
+ const words = buildMouthWords(text);
70
+ this.speech = {
71
+ words,
72
+ wordIndex: -1,
73
+ cues: [{ time: 0, viseme: 'sil', weight: 1 }],
74
+ cueStartedAt: null,
75
+ armedAt: null,
76
+ secondsPerUnit: 1,
77
+ boundarySeen: false,
78
+ text,
79
+ };
80
+ }
81
+
82
+ /** Lock mouth shapes to the word the TTS engine is about to speak. */
83
+ setSpeechBoundary(charIndex: number, charLength: number) {
84
+ if (!this.speech) return;
85
+ const index = wordIndexAt(this.speech.words, charIndex, charLength);
86
+ if (index < 0) return;
87
+ const now = this.lastTime;
88
+ if (
89
+ now !== null &&
90
+ this.speech.wordIndex >= 0 &&
91
+ this.speech.wordIndex !== index &&
92
+ this.speech.cueStartedAt !== null
93
+ ) {
94
+ const previous = this.speech.words[this.speech.wordIndex];
95
+ const spoken = Math.max(0.06, now - this.speech.cueStartedAt);
96
+ if (previous && previous.duration > 0.04) {
97
+ const measured = spoken / previous.duration;
98
+ this.speech.secondsPerUnit = this.speech.secondsPerUnit * 0.4 + measured * 0.6;
99
+ }
100
+ }
101
+ this.speech.boundarySeen = true;
102
+ this.startWord(index, now);
103
+ }
104
+
105
+ endSpeech() {
106
+ this.speech = null;
107
+ }
108
+
109
+ get speaking() {
110
+ return this.speech !== null;
111
+ }
112
+
113
+ sample(now: number): AvatarPose {
114
+ const dt = this.lastTime === null ? 0 : Math.min(0.05, Math.max(0, now - this.lastTime));
115
+ this.lastTime = now;
116
+ if (this.nextBlinkAt === 0) this.scheduleBlink(now);
117
+ if (this.nextSaccadeAt === 0) this.nextSaccadeAt = now + 2.4;
118
+
119
+ const target = this.targetWeights(now);
120
+ for (const name of keyedNames(target, this.weights)) {
121
+ const next = target[name] ?? 0;
122
+ const blink = (BLENDSHAPE_CHANNELS.blink as readonly string[]).includes(name);
123
+ const rate = isLipsync(name) ? mouthEase(name) : EXPRESSION_RATE;
124
+ // The jaw has weight. Blink still cuts in immediately.
125
+ this.weights[name] = blink ? next : approach(this.weights[name] ?? 0, next, dt, rate);
126
+ if ((this.weights[name] ?? 0) < 0.001) delete this.weights[name];
127
+ }
128
+
129
+ const head = this.flags.head ? idleHead(now, this.speech !== null) : { neck: zero(), head: zero() };
130
+ const eyes = this.flags.eyes ? this.eyePose(dt, now) : { x: 0, y: 0, z: 0 };
131
+ this.bones.Neck = approachRotation(this.bones.Neck ?? zero(), head.neck, dt, 3);
132
+ this.bones.Head = approachRotation(this.bones.Head ?? zero(), head.head, dt, 3);
133
+ this.bones.LeftEye = approachRotation(this.bones.LeftEye ?? zero(), eyes, dt, 12);
134
+ this.bones.RightEye = approachRotation(this.bones.RightEye ?? zero(), eyes, dt, 12);
135
+ this.bones.Jaw = zero();
136
+
137
+ return { weights: { ...this.weights }, bones: clonePose(this.bones) };
138
+ }
139
+
140
+ private startWord(index: number, now: number | null) {
141
+ if (!this.speech || index < 0 || index >= this.speech.words.length) return;
142
+ if (index === this.speech.wordIndex && this.speech.cueStartedAt !== null) return;
143
+ const word = this.speech.words[index];
144
+ const duration = clamp(word.duration * this.speech.secondsPerUnit, 0.16, 0.9);
145
+ this.speech.wordIndex = index;
146
+ this.speech.cues = scaledWordCues(word, duration);
147
+ this.speech.cueStartedAt = now;
148
+ }
149
+
150
+ private targetWeights(now: number): BlendshapeWeights {
151
+ const target: BlendshapeWeights = { ...EXPRESSION_PRESETS[this.expression] };
152
+
153
+ if (this.flags.eyes) {
154
+ for (const name of BLENDSHAPE_CHANNELS.gaze) target[name] = 0;
155
+ const x = this.gaze.x;
156
+ const y = this.gaze.y;
157
+ target.eyeLookOutLeft = clamp01(x);
158
+ target.eyeLookInLeft = clamp01(-x);
159
+ target.eyeLookInRight = clamp01(x);
160
+ target.eyeLookOutRight = clamp01(-x);
161
+ target.eyeLookUpLeft = clamp01(y);
162
+ target.eyeLookUpRight = clamp01(y);
163
+ target.eyeLookDownLeft = clamp01(-y);
164
+ target.eyeLookDownRight = clamp01(-y);
165
+ // A little inward look so both eyes meet the person at the phone.
166
+ target.eyeLookInLeft = Math.max(target.eyeLookInLeft ?? 0, 0.16);
167
+ target.eyeLookInRight = Math.max(target.eyeLookInRight ?? 0, 0.16);
168
+ }
169
+
170
+ if (this.speech) {
171
+ if (this.speech.armedAt === null) this.speech.armedAt = now;
172
+ if (this.speech.wordIndex < 0) {
173
+ // The first boundary starts the mouth with the voice. If none arrives, start anyway.
174
+ if (!this.speech.boundarySeen && now - this.speech.armedAt > 0.22) this.startWord(0, now);
175
+ } else if (this.speech.cueStartedAt !== null && !this.speech.boundarySeen) {
176
+ const elapsed = now - this.speech.cueStartedAt;
177
+ const end = this.speech.cues[this.speech.cues.length - 1]?.time ?? 0;
178
+ if (elapsed > end && this.speech.wordIndex < this.speech.words.length - 1) {
179
+ this.startWord(this.speech.wordIndex + 1, now);
180
+ }
181
+ }
182
+ if (this.speech.cueStartedAt === null) this.speech.cueStartedAt = now;
183
+ const mouth = mouthWeightsAt(this.speech.cues, now - this.speech.cueStartedAt);
184
+ for (const name of BLENDSHAPE_CHANNELS.lipsync) target[name] = mouth[name] ?? 0;
185
+ // Eyes meet the person in front of the camera.
186
+ target.eyeLookOutLeft = 0;
187
+ target.eyeLookOutRight = 0;
188
+ target.eyeLookUpLeft = 0;
189
+ target.eyeLookUpRight = 0;
190
+ target.eyeLookDownLeft = 0;
191
+ target.eyeLookDownRight = 0;
192
+ target.eyeLookInLeft = 0.2;
193
+ target.eyeLookInRight = 0.2;
194
+ }
195
+
196
+ const blink = this.flags.blink ? this.blinkWeight(now) : 0;
197
+ target.eyeBlinkLeft = Math.max(target.eyeBlinkLeft ?? 0, blink);
198
+ target.eyeBlinkRight = Math.max(target.eyeBlinkRight ?? 0, blink);
199
+ return target;
200
+ }
201
+
202
+ private blinkWeight(now: number): number {
203
+ const duration = BLINK_DURATION_MS / 1000;
204
+ if (!this.blinkStarted && now >= this.nextBlinkAt) {
205
+ this.blinkStarted = true;
206
+ this.blinkAt = now;
207
+ }
208
+ if (!this.blinkStarted) return 0;
209
+
210
+ const elapsed = now - this.blinkAt;
211
+ const total = duration * 2;
212
+ if (elapsed >= total) {
213
+ this.blinkStarted = false;
214
+ this.scheduleBlink(now);
215
+ return 0;
216
+ }
217
+ if (elapsed < duration) return smoothstep(elapsed / duration);
218
+ return 1 - smoothstep((elapsed - duration) / duration);
219
+ }
220
+
221
+ private eyePose(dt: number, now: number) {
222
+ // Mostly look straight at the user/camera; rare tiny glances only.
223
+ if (now >= this.nextSaccadeAt) {
224
+ const glance = Math.random() < 0.12;
225
+ this.gazeTarget = glance
226
+ ? { x: (Math.random() * 2 - 1) * 0.035, y: (Math.random() * 2 - 1) * 0.02 }
227
+ : { x: 0, y: 0 };
228
+ this.nextSaccadeAt = now + (glance ? 0.14 + Math.random() * 0.16 : 2.8 + Math.random() * 3.2);
229
+ }
230
+ this.gaze.x = approach(this.gaze.x, this.gazeTarget.x, dt, 10);
231
+ this.gaze.y = approach(this.gaze.y, this.gazeTarget.y, dt, 10);
232
+ return {
233
+ x: this.gaze.y * HEAD_LIMITS_DEG.eye.x,
234
+ y: this.gaze.x * HEAD_LIMITS_DEG.eye.y,
235
+ z: 0,
236
+ };
237
+ }
238
+
239
+ private scheduleBlink(now: number) {
240
+ const span = BLINK_INTERVAL_MS.max - BLINK_INTERVAL_MS.min;
241
+ this.nextBlinkAt = now + (BLINK_INTERVAL_MS.min + Math.random() * span) / 1000;
242
+ }
243
+ }
244
+
245
+ function wordIndexAt(words: MouthWord[], charIndex: number, charLength: number): number {
246
+ const end = charIndex + Math.max(charLength, 1);
247
+ const inside = words.findIndex((word) => charIndex >= word.start && charIndex < word.end);
248
+ if (inside >= 0) return inside;
249
+ const ahead = words.findIndex((word) => word.start >= charIndex && word.start < end);
250
+ if (ahead >= 0) return ahead;
251
+ let nearest = -1;
252
+ for (let index = 0; index < words.length; index += 1) {
253
+ if (words[index].start <= charIndex) nearest = index;
254
+ }
255
+ return nearest;
256
+ }
257
+
258
+ function idleHead(now: number, speaking: boolean) {
259
+ // Face the camera. A nod while speaking, almost no turn away.
260
+ if (speaking) {
261
+ const nod = Math.sin(now * 2.1) * 1.6;
262
+ return {
263
+ neck: { x: nod * 0.35, y: 0, z: 0 },
264
+ head: { x: nod * 0.7, y: 0, z: 0 },
265
+ };
266
+ }
267
+ const yaw = Math.sin(now * 0.22) * 1.1;
268
+ const pitch = Math.sin(now * 0.18) * 1;
269
+ return {
270
+ neck: clampRotation({ x: pitch * 0.3, y: yaw * 0.2, z: 0 }, HEAD_LIMITS_DEG.neck),
271
+ head: clampRotation({ x: pitch * 0.4, y: yaw * 0.3, z: 0 }, HEAD_LIMITS_DEG.head),
272
+ };
273
+ }
274
+
275
+ function isLipsync(name: BlendshapeName) {
276
+ return (BLENDSHAPE_CHANNELS.lipsync as readonly BlendshapeName[]).includes(name);
277
+ }
278
+
279
+ function mouthEase(name: string): number {
280
+ if (name === 'jawOpen') return 12;
281
+ if (name === 'mouthClose' || name.startsWith('mouthPress')) return 22;
282
+ return 16;
283
+ }
284
+
285
+ function keyedNames(target: BlendshapeWeights, current: BlendshapeWeights): BlendshapeName[] {
286
+ return [...new Set([...Object.keys(target), ...Object.keys(current)])] as BlendshapeName[];
287
+ }
288
+
289
+ function approach(current: number, target: number, dt: number, rate: number): number {
290
+ if (dt <= 0) return target;
291
+ const step = 1 - Math.exp(-rate * dt);
292
+ return current + (target - current) * step;
293
+ }
294
+
295
+ function approachRotation(
296
+ current: { x: number; y: number; z: number },
297
+ target: { x: number; y: number; z: number },
298
+ dt: number,
299
+ rate: number,
300
+ ) {
301
+ return {
302
+ x: approach(current.x, target.x, dt, rate),
303
+ y: approach(current.y, target.y, dt, rate),
304
+ z: approach(current.z, target.z, dt, rate),
305
+ };
306
+ }
307
+
308
+ function clampRotation(
309
+ rotation: { x: number; y: number; z: number },
310
+ limits: { x: number; y: number; z: number },
311
+ ) {
312
+ return {
313
+ x: clamp(rotation.x, -limits.x, limits.x),
314
+ y: clamp(rotation.y, -limits.y, limits.y),
315
+ z: clamp(rotation.z, -limits.z, limits.z),
316
+ };
317
+ }
318
+
319
+ function zero() {
320
+ return { x: 0, y: 0, z: 0 };
321
+ }
322
+
323
+ function clonePose(pose: Required<BonePose>): BonePose {
324
+ return {
325
+ Neck: { ...pose.Neck! },
326
+ Head: { ...pose.Head! },
327
+ LeftEye: { ...pose.LeftEye! },
328
+ RightEye: { ...pose.RightEye! },
329
+ Jaw: { ...pose.Jaw! },
330
+ };
331
+ }
332
+
333
+ function clamp(value: number, min: number, max: number) {
334
+ return Math.min(max, Math.max(min, value));
335
+ }
336
+
337
+ function clamp01(value: number) {
338
+ return clamp(value, 0, 1);
339
+ }
340
+
341
+ function smoothstep(t: number) {
342
+ const x = clamp01(t);
343
+ return x * x * (3 - 2 * x);
344
+ }
@@ -0,0 +1,59 @@
1
+ import type { BlendshapeName } from '../model/blendshapes';
2
+
3
+ /**
4
+ * Later writers replace earlier ones on the same shape key.
5
+ * Lip-sync owns the mouth. Blink owns the lids. Gaze owns eye direction.
6
+ */
7
+ export const BLENDSHAPE_CHANNELS = {
8
+ blink: ['eyeBlinkLeft', 'eyeBlinkRight'],
9
+ gaze: [
10
+ 'eyeLookDownLeft',
11
+ 'eyeLookDownRight',
12
+ 'eyeLookInLeft',
13
+ 'eyeLookInRight',
14
+ 'eyeLookOutLeft',
15
+ 'eyeLookOutRight',
16
+ 'eyeLookUpLeft',
17
+ 'eyeLookUpRight',
18
+ ],
19
+ lipsync: [
20
+ 'jawOpen',
21
+ 'jawForward',
22
+ 'jawLeft',
23
+ 'jawRight',
24
+ 'mouthClose',
25
+ 'mouthFunnel',
26
+ 'mouthPucker',
27
+ 'mouthLeft',
28
+ 'mouthRight',
29
+ 'mouthSmileLeft',
30
+ 'mouthSmileRight',
31
+ 'mouthFrownLeft',
32
+ 'mouthFrownRight',
33
+ 'mouthDimpleLeft',
34
+ 'mouthDimpleRight',
35
+ 'mouthStretchLeft',
36
+ 'mouthStretchRight',
37
+ 'mouthRollLower',
38
+ 'mouthRollUpper',
39
+ 'mouthShrugLower',
40
+ 'mouthShrugUpper',
41
+ 'mouthPressLeft',
42
+ 'mouthPressRight',
43
+ 'mouthLowerDownLeft',
44
+ 'mouthLowerDownRight',
45
+ 'mouthUpperUpLeft',
46
+ 'mouthUpperUpRight',
47
+ 'tongueOut',
48
+ ],
49
+ } as const satisfies Record<string, readonly BlendshapeName[]>;
50
+
51
+ export const BLINK_INTERVAL_MS = { min: 2400, max: 6000 } as const;
52
+ export const BLINK_DURATION_MS = 140;
53
+
54
+ /** Head idle and gaze stay inside these Euler degrees. */
55
+ export const HEAD_LIMITS_DEG = {
56
+ neck: { x: 12, y: 18, z: 8 },
57
+ head: { x: 16, y: 24, z: 10 },
58
+ eye: { x: 18, y: 22, z: 0 },
59
+ } as const;
@@ -0,0 +1,51 @@
1
+ import type { BlendshapeWeights } from '../model/blendshapes';
2
+ import type { ExpressionName } from '../types';
3
+
4
+ /** Shared by male and female meshes. Mouth shapes yield to lip-sync while speaking. */
5
+ export const EXPRESSION_PRESETS: Record<ExpressionName, BlendshapeWeights> = {
6
+ neutral: {},
7
+ happy: {
8
+ mouthSmileLeft: 0.7,
9
+ mouthSmileRight: 0.7,
10
+ cheekSquintLeft: 0.3,
11
+ cheekSquintRight: 0.3,
12
+ eyeSquintLeft: 0.2,
13
+ eyeSquintRight: 0.2,
14
+ },
15
+ sad: {
16
+ mouthFrownLeft: 0.6,
17
+ mouthFrownRight: 0.6,
18
+ browInnerUp: 0.6,
19
+ browDownLeft: 0.25,
20
+ browDownRight: 0.25,
21
+ eyeSquintLeft: 0.15,
22
+ eyeSquintRight: 0.15,
23
+ },
24
+ angry: {
25
+ browDownLeft: 0.8,
26
+ browDownRight: 0.8,
27
+ browInnerUp: 0.35,
28
+ eyeSquintLeft: 0.35,
29
+ eyeSquintRight: 0.35,
30
+ noseSneerLeft: 0.3,
31
+ noseSneerRight: 0.3,
32
+ jawForward: 0.2,
33
+ mouthFrownLeft: 0.25,
34
+ mouthFrownRight: 0.25,
35
+ },
36
+ surprised: {
37
+ eyeWideLeft: 0.7,
38
+ eyeWideRight: 0.7,
39
+ browInnerUp: 0.7,
40
+ browOuterUpLeft: 0.6,
41
+ browOuterUpRight: 0.6,
42
+ jawOpen: 0.35,
43
+ },
44
+ thinking: {
45
+ browInnerUp: 0.25,
46
+ browDownLeft: 0.2,
47
+ eyeLookUpLeft: 0.25,
48
+ eyeLookUpRight: 0.25,
49
+ mouthPucker: 0.12,
50
+ },
51
+ };
@@ -0,0 +1,23 @@
1
+ export { AIAvatar } from './AIAvatar';
2
+ export { AvatarView } from './AvatarView';
3
+ export { EXPRESSION_PRESETS } from './animation/expressions';
4
+ export { BLENDSHAPE_CHANNELS, BLINK_DURATION_MS, BLINK_INTERVAL_MS, HEAD_LIMITS_DEG } from './animation/channels';
5
+ export { ARKIT_BLENDSHAPES, BLENDSHAPES } from './model/blendshapes';
6
+ export { OPTIONAL_BONES, REQUIRED_BONES } from './model/skeleton';
7
+ export { VISEME_BLENDSHAPES, VISEMES } from './model/visemes';
8
+
9
+ export type { BlendshapeName, BlendshapeWeights } from './model/blendshapes';
10
+ export type { BoneName, BonePose, BoneRotation } from './model/skeleton';
11
+ export type { VisemeCue, VisemeId } from './model/visemes';
12
+ export type { SpeechResult, VoiceProvider } from './voice/types';
13
+ export type { AIAvatarHandle, AIAvatarProps } from './AIAvatar';
14
+ export type {
15
+ AvatarModel,
16
+ AvatarModelId,
17
+ AvatarOutfit,
18
+ AvatarSex,
19
+ AvatarViewProps,
20
+ ExpressionName,
21
+ GazeTarget,
22
+ SpeakRequest,
23
+ } from './types';
@@ -0,0 +1,46 @@
1
+ import * as THREE from 'three';
2
+
3
+ /**
4
+ * Expo GL on the iOS Simulator cannot sample the float bone textures Three.js
5
+ * uses for SkinnedMesh. Convert every skinned mesh to a plain Mesh at bind pose
6
+ * so the avatar still draws. Face animation stays on CPU morphs; head turn is a
7
+ * whole-body group rotation instead of bone skinning.
8
+ */
9
+ export function bakeSkinningToStaticMeshes(scene: THREE.Object3D) {
10
+ scene.updateMatrixWorld(true);
11
+ const skinned: THREE.SkinnedMesh[] = [];
12
+ scene.traverse((object) => {
13
+ if ((object as THREE.SkinnedMesh).isSkinnedMesh) {
14
+ skinned.push(object as THREE.SkinnedMesh);
15
+ }
16
+ });
17
+
18
+ for (const source of skinned) {
19
+ const parent = source.parent;
20
+ if (!parent) continue;
21
+
22
+ // Rest/bind pose → bone*inverse is identity, so buffer positions are correct.
23
+ source.skeleton?.pose();
24
+ source.updateMatrixWorld(true);
25
+
26
+ const mesh = new THREE.Mesh(source.geometry, source.material);
27
+ mesh.name = source.name;
28
+ mesh.frustumCulled = false;
29
+ mesh.castShadow = source.castShadow;
30
+ mesh.receiveShadow = source.receiveShadow;
31
+ mesh.visible = source.visible;
32
+ mesh.layers.mask = source.layers.mask;
33
+ mesh.position.copy(source.position);
34
+ mesh.quaternion.copy(source.quaternion);
35
+ mesh.scale.copy(source.scale);
36
+ mesh.morphTargetDictionary = source.morphTargetDictionary
37
+ ? { ...source.morphTargetDictionary }
38
+ : undefined;
39
+ mesh.morphTargetInfluences = source.morphTargetInfluences
40
+ ? source.morphTargetInfluences.slice()
41
+ : undefined;
42
+
43
+ parent.add(mesh);
44
+ parent.remove(source);
45
+ }
46
+ }