@usamanawaz/react-native-ai-avatar 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +86 -2
- package/assets/models/male-avatar-clothed.glb +0 -0
- package/assets/models/male-avatar.glb +0 -0
- package/dist/AIAvatar.d.ts +20 -0
- package/dist/AvatarView.d.ts +3 -0
- package/dist/animation/animator.d.ts +44 -0
- package/dist/animation/channels.d.ts +32 -0
- package/dist/animation/expressions.d.ts +4 -0
- package/dist/index.d.ts +13 -0
- package/dist/model/bakeSkinning.d.ts +8 -0
- package/dist/model/blendshapes.d.ts +12 -0
- package/dist/model/boneMap.d.ts +9 -0
- package/dist/model/catalog.d.ts +2 -0
- package/dist/model/cpuMorphs.d.ts +24 -0
- package/dist/model/cpuSkin.d.ts +19 -0
- package/dist/model/loadAvatar.d.ts +21 -0
- package/dist/model/skeleton.d.ts +17 -0
- package/dist/model/visemes.d.ts +19 -0
- package/dist/rendering/AvatarStage.d.ts +7 -0
- package/dist/rendering/applyPose.d.ts +3 -0
- package/dist/speech/ExpoSpeechProvider.d.ts +13 -0
- package/dist/speech/approximateMouth.d.ts +15 -0
- package/dist/types.d.ts +35 -0
- package/dist/voice/types.d.ts +18 -0
- package/package.json +72 -4
- package/src/avatar/AIAvatar.tsx +137 -0
- package/src/avatar/AvatarView.tsx +11 -0
- package/src/avatar/animation/animator.ts +344 -0
- package/src/avatar/animation/channels.ts +59 -0
- package/src/avatar/animation/expressions.ts +51 -0
- package/src/avatar/index.ts +23 -0
- package/src/avatar/model/bakeSkinning.ts +46 -0
- package/src/avatar/model/blendshapes.ts +70 -0
- package/src/avatar/model/boneMap.ts +17 -0
- package/src/avatar/model/catalog.ts +15 -0
- package/src/avatar/model/cpuMorphs.ts +124 -0
- package/src/avatar/model/cpuSkin.ts +105 -0
- package/src/avatar/model/jpeg-js.d.ts +6 -0
- package/src/avatar/model/loadAvatar.ts +497 -0
- package/src/avatar/model/skeleton.ts +21 -0
- package/src/avatar/model/visemes.ts +55 -0
- package/src/avatar/rendering/AvatarStage.tsx +119 -0
- package/src/avatar/rendering/applyPose.ts +59 -0
- package/src/avatar/speech/ExpoSpeechProvider.ts +42 -0
- package/src/avatar/speech/approximateMouth.ts +244 -0
- package/src/avatar/types.ts +45 -0
- package/src/avatar/voice/types.ts +20 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2015-present 650 Industries, Inc. (aka Expo)
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,87 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @usamanawaz/react-native-ai-avatar
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
A 3D avatar for Expo and React Native. It blinks, looks toward the camera, changes expression, and moves its mouth while the device speaks.
|
|
4
|
+
|
|
5
|
+
Works with Expo SDK 57 on Android, iOS, and web.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
npm install @usamanawaz/react-native-ai-avatar
|
|
11
|
+
npx expo install expo-gl expo-asset expo-file-system expo-speech three @react-three/fiber jpeg-js
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
`expo-gl` includes native code. Use a [development build](https://docs.expo.dev/develop/development-builds/introduction/) if Expo Go does not include it. On web, `react-dom` and `react-native-web` are also required.
|
|
15
|
+
|
|
16
|
+
## Metro
|
|
17
|
+
|
|
18
|
+
Add `glb` as an asset, and load Three from its ESM build. The CommonJS build calls `process.emitWarning` and crashes in React Native.
|
|
19
|
+
|
|
20
|
+
```js
|
|
21
|
+
const path = require('path');
|
|
22
|
+
const { getDefaultConfig } = require('expo/metro-config');
|
|
23
|
+
|
|
24
|
+
const config = getDefaultConfig(__dirname);
|
|
25
|
+
config.resolver.assetExts.push('glb', 'gltf');
|
|
26
|
+
|
|
27
|
+
const threeEsmEntry = path.resolve(__dirname, 'node_modules/three/build/three.module.js');
|
|
28
|
+
config.resolver.resolveRequest = (context, moduleName, platform) => {
|
|
29
|
+
if (moduleName === 'three') {
|
|
30
|
+
return { type: 'sourceFile', filePath: threeEsmEntry };
|
|
31
|
+
}
|
|
32
|
+
return context.resolveRequest(context, moduleName, platform);
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
module.exports = config;
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## Use
|
|
39
|
+
|
|
40
|
+
```tsx
|
|
41
|
+
import { useRef } from 'react';
|
|
42
|
+
import { Button, View } from 'react-native';
|
|
43
|
+
import { AIAvatar, type AIAvatarHandle } from '@usamanawaz/react-native-ai-avatar';
|
|
44
|
+
|
|
45
|
+
export function Screen() {
|
|
46
|
+
const avatar = useRef<AIAvatarHandle>(null);
|
|
47
|
+
|
|
48
|
+
return (
|
|
49
|
+
<View style={{ flex: 1 }}>
|
|
50
|
+
<AIAvatar
|
|
51
|
+
ref={avatar}
|
|
52
|
+
model="male"
|
|
53
|
+
outfit="casual"
|
|
54
|
+
expression="neutral"
|
|
55
|
+
enableBlinking
|
|
56
|
+
enableHeadMovement
|
|
57
|
+
enableEyeMovement
|
|
58
|
+
/>
|
|
59
|
+
<Button title="Speak" onPress={() => avatar.current?.speak('Hello. How can I help you today?')} />
|
|
60
|
+
<Button title="Stop" onPress={() => avatar.current?.stop()} />
|
|
61
|
+
</View>
|
|
62
|
+
);
|
|
63
|
+
}
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
`speak` uses the system voice and resolves when speech ends. `stop` cuts it off.
|
|
67
|
+
|
|
68
|
+
## Props
|
|
69
|
+
|
|
70
|
+
| Prop | Default | What it does |
|
|
71
|
+
| --- | --- | --- |
|
|
72
|
+
| `model` | `"male"` | Bundled character. `"female"` is reserved and not included yet. |
|
|
73
|
+
| `outfit` | none | `"casual"` wears the black crew-neck. Leave it unset for the default male model. |
|
|
74
|
+
| `expression` | `"neutral"` | `neutral`, `happy`, `sad`, `angry`, `surprised`, or `thinking`. |
|
|
75
|
+
| `enableBlinking` | `true` | Blinks on its own. |
|
|
76
|
+
| `enableHeadMovement` | `true` | Small idle motion. While speaking, the head stays aimed at the camera. |
|
|
77
|
+
| `enableEyeMovement` | `true` | Eyes look toward the person in front of the phone. |
|
|
78
|
+
| `voice` | system speech | Your own `VoiceProvider`, if you do not want `expo-speech`. |
|
|
79
|
+
| `style` | | Style for the avatar frame. The frame needs a height, or `flex: 1` inside a parent that has one. |
|
|
80
|
+
|
|
81
|
+
## Notes
|
|
82
|
+
|
|
83
|
+
Lip sync follows the words from on-device speech. It is timed from spelling and word boundaries, not from a phoneme track.
|
|
84
|
+
|
|
85
|
+
Give the avatar a tall area. A face-and-chest framing is built in, and a short box will crop the head.
|
|
86
|
+
|
|
87
|
+
If the panel stays a flat color on the iOS Simulator, run the same build on a phone. Simulator OpenGL often cannot draw this scene.
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { type StyleProp, type ViewStyle } from 'react-native';
|
|
2
|
+
import type { AvatarModelId, AvatarOutfit, ExpressionName } from './types';
|
|
3
|
+
import type { VoiceProvider } from './voice/types';
|
|
4
|
+
export type AIAvatarProps = {
|
|
5
|
+
model?: AvatarModelId;
|
|
6
|
+
/** `casual` wears the black crew-neck. Omit it to keep the default male model. */
|
|
7
|
+
outfit?: AvatarOutfit;
|
|
8
|
+
enableBlinking?: boolean;
|
|
9
|
+
enableHeadMovement?: boolean;
|
|
10
|
+
enableEyeMovement?: boolean;
|
|
11
|
+
expression?: ExpressionName;
|
|
12
|
+
style?: StyleProp<ViewStyle>;
|
|
13
|
+
voice?: VoiceProvider;
|
|
14
|
+
};
|
|
15
|
+
export type AIAvatarHandle = {
|
|
16
|
+
/** Speaks with the system voice and an approximate mouth rhythm. Resolves when speech ends or is stopped. */
|
|
17
|
+
speak(text: string): Promise<void>;
|
|
18
|
+
stop(): Promise<void>;
|
|
19
|
+
};
|
|
20
|
+
export declare const AIAvatar: import("react").ForwardRefExoticComponent<AIAvatarProps & import("react").RefAttributes<AIAvatarHandle>>;
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { BlendshapeWeights } from '../model/blendshapes';
|
|
2
|
+
import type { BonePose } from '../model/skeleton';
|
|
3
|
+
import type { ExpressionName } from '../types';
|
|
4
|
+
export type AvatarPose = {
|
|
5
|
+
weights: BlendshapeWeights;
|
|
6
|
+
bones: BonePose;
|
|
7
|
+
};
|
|
8
|
+
type Flags = {
|
|
9
|
+
blink: boolean;
|
|
10
|
+
head: boolean;
|
|
11
|
+
eyes: boolean;
|
|
12
|
+
};
|
|
13
|
+
/**
|
|
14
|
+
* Frame-independent face pose. Rendering only applies the result.
|
|
15
|
+
* Mouth cues follow spoken word boundaries from the voice provider.
|
|
16
|
+
*/
|
|
17
|
+
export declare class AvatarAnimator {
|
|
18
|
+
private expression;
|
|
19
|
+
private flags;
|
|
20
|
+
private weights;
|
|
21
|
+
private bones;
|
|
22
|
+
private lastTime;
|
|
23
|
+
private nextBlinkAt;
|
|
24
|
+
private blinkStarted;
|
|
25
|
+
private blinkAt;
|
|
26
|
+
private gaze;
|
|
27
|
+
private gazeTarget;
|
|
28
|
+
private nextSaccadeAt;
|
|
29
|
+
private speech;
|
|
30
|
+
setExpression(expression: ExpressionName): void;
|
|
31
|
+
setFlags(flags: Flags): void;
|
|
32
|
+
beginSpeech(text: string): void;
|
|
33
|
+
/** Lock mouth shapes to the word the TTS engine is about to speak. */
|
|
34
|
+
setSpeechBoundary(charIndex: number, charLength: number): void;
|
|
35
|
+
endSpeech(): void;
|
|
36
|
+
get speaking(): boolean;
|
|
37
|
+
sample(now: number): AvatarPose;
|
|
38
|
+
private startWord;
|
|
39
|
+
private targetWeights;
|
|
40
|
+
private blinkWeight;
|
|
41
|
+
private eyePose;
|
|
42
|
+
private scheduleBlink;
|
|
43
|
+
}
|
|
44
|
+
export {};
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Later writers replace earlier ones on the same shape key.
|
|
3
|
+
* Lip-sync owns the mouth. Blink owns the lids. Gaze owns eye direction.
|
|
4
|
+
*/
|
|
5
|
+
export declare const BLENDSHAPE_CHANNELS: {
|
|
6
|
+
readonly blink: readonly ["eyeBlinkLeft", "eyeBlinkRight"];
|
|
7
|
+
readonly gaze: readonly ["eyeLookDownLeft", "eyeLookDownRight", "eyeLookInLeft", "eyeLookInRight", "eyeLookOutLeft", "eyeLookOutRight", "eyeLookUpLeft", "eyeLookUpRight"];
|
|
8
|
+
readonly lipsync: readonly ["jawOpen", "jawForward", "jawLeft", "jawRight", "mouthClose", "mouthFunnel", "mouthPucker", "mouthLeft", "mouthRight", "mouthSmileLeft", "mouthSmileRight", "mouthFrownLeft", "mouthFrownRight", "mouthDimpleLeft", "mouthDimpleRight", "mouthStretchLeft", "mouthStretchRight", "mouthRollLower", "mouthRollUpper", "mouthShrugLower", "mouthShrugUpper", "mouthPressLeft", "mouthPressRight", "mouthLowerDownLeft", "mouthLowerDownRight", "mouthUpperUpLeft", "mouthUpperUpRight", "tongueOut"];
|
|
9
|
+
};
|
|
10
|
+
export declare const BLINK_INTERVAL_MS: {
|
|
11
|
+
readonly min: 2400;
|
|
12
|
+
readonly max: 6000;
|
|
13
|
+
};
|
|
14
|
+
export declare const BLINK_DURATION_MS = 140;
|
|
15
|
+
/** Head idle and gaze stay inside these Euler degrees. */
|
|
16
|
+
export declare const HEAD_LIMITS_DEG: {
|
|
17
|
+
readonly neck: {
|
|
18
|
+
readonly x: 12;
|
|
19
|
+
readonly y: 18;
|
|
20
|
+
readonly z: 8;
|
|
21
|
+
};
|
|
22
|
+
readonly head: {
|
|
23
|
+
readonly x: 16;
|
|
24
|
+
readonly y: 24;
|
|
25
|
+
readonly z: 10;
|
|
26
|
+
};
|
|
27
|
+
readonly eye: {
|
|
28
|
+
readonly x: 18;
|
|
29
|
+
readonly y: 22;
|
|
30
|
+
readonly z: 0;
|
|
31
|
+
};
|
|
32
|
+
};
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { BlendshapeWeights } from '../model/blendshapes';
|
|
2
|
+
import type { ExpressionName } from '../types';
|
|
3
|
+
/** Shared by male and female meshes. Mouth shapes yield to lip-sync while speaking. */
|
|
4
|
+
export declare const EXPRESSION_PRESETS: Record<ExpressionName, BlendshapeWeights>;
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export { AIAvatar } from './AIAvatar';
|
|
2
|
+
export { AvatarView } from './AvatarView';
|
|
3
|
+
export { EXPRESSION_PRESETS } from './animation/expressions';
|
|
4
|
+
export { BLENDSHAPE_CHANNELS, BLINK_DURATION_MS, BLINK_INTERVAL_MS, HEAD_LIMITS_DEG } from './animation/channels';
|
|
5
|
+
export { ARKIT_BLENDSHAPES, BLENDSHAPES } from './model/blendshapes';
|
|
6
|
+
export { OPTIONAL_BONES, REQUIRED_BONES } from './model/skeleton';
|
|
7
|
+
export { VISEME_BLENDSHAPES, VISEMES } from './model/visemes';
|
|
8
|
+
export type { BlendshapeName, BlendshapeWeights } from './model/blendshapes';
|
|
9
|
+
export type { BoneName, BonePose, BoneRotation } from './model/skeleton';
|
|
10
|
+
export type { VisemeCue, VisemeId } from './model/visemes';
|
|
11
|
+
export type { SpeechResult, VoiceProvider } from './voice/types';
|
|
12
|
+
export type { AIAvatarHandle, AIAvatarProps } from './AIAvatar';
|
|
13
|
+
export type { AvatarModel, AvatarModelId, AvatarOutfit, AvatarSex, AvatarViewProps, ExpressionName, GazeTarget, SpeakRequest, } from './types';
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import * as THREE from 'three';
|
|
2
|
+
/**
|
|
3
|
+
* Expo GL on the iOS Simulator cannot sample the float bone textures Three.js
|
|
4
|
+
* uses for SkinnedMesh. Convert every skinned mesh to a plain Mesh at bind pose
|
|
5
|
+
* so the avatar still draws. Face animation stays on CPU morphs; head turn is a
|
|
6
|
+
* whole-body group rotation instead of bone skinning.
|
|
7
|
+
*/
|
|
8
|
+
export declare function bakeSkinningToStaticMeshes(scene: THREE.Object3D): void;
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Face contract for every GLB this library loads.
|
|
3
|
+
* Names match Apple's ARKit blend shapes so one mesh works for blink,
|
|
4
|
+
* gaze, expression, and lip-sync.
|
|
5
|
+
*/
|
|
6
|
+
export declare const BLENDSHAPES: readonly ["browDownLeft", "browDownRight", "browInnerUp", "browOuterUpLeft", "browOuterUpRight", "cheekPuff", "cheekSquintLeft", "cheekSquintRight", "eyeBlinkLeft", "eyeBlinkRight", "eyeLookDownLeft", "eyeLookDownRight", "eyeLookInLeft", "eyeLookInRight", "eyeLookOutLeft", "eyeLookOutRight", "eyeLookUpLeft", "eyeLookUpRight", "eyeSquintLeft", "eyeSquintRight", "eyeWideLeft", "eyeWideRight", "jawForward", "jawLeft", "jawRight", "jawOpen", "mouthClose", "mouthFunnel", "mouthPucker", "mouthLeft", "mouthRight", "mouthSmileLeft", "mouthSmileRight", "mouthFrownLeft", "mouthFrownRight", "mouthDimpleLeft", "mouthDimpleRight", "mouthStretchLeft", "mouthStretchRight", "mouthRollLower", "mouthRollUpper", "mouthShrugLower", "mouthShrugUpper", "mouthPressLeft", "mouthPressRight", "mouthLowerDownLeft", "mouthLowerDownRight", "mouthUpperUpLeft", "mouthUpperUpRight", "noseSneerLeft", "noseSneerRight", "tongueOut"];
|
|
7
|
+
type AssertLength<T extends readonly unknown[], N extends number> = T['length'] extends N ? T : never;
|
|
8
|
+
export declare const ARKIT_BLENDSHAPES: AssertLength<typeof BLENDSHAPES, 52>;
|
|
9
|
+
export type BlendshapeName = (typeof BLENDSHAPES)[number];
|
|
10
|
+
/** Morph target weights. Missing keys stay at 0. Values are 0 to 1. */
|
|
11
|
+
export type BlendshapeWeights = Partial<Record<BlendshapeName, number>>;
|
|
12
|
+
export {};
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { BoneName } from './skeleton';
|
|
2
|
+
/**
|
|
3
|
+
* MPFB default-rig names, then the names Three.js actually puts on the
|
|
4
|
+
* objects. Dots are stripped on load (`eye.L` becomes `eyeL`) because they
|
|
5
|
+
* are reserved in animation paths. The GLB itself is left unchanged.
|
|
6
|
+
*/
|
|
7
|
+
export declare const MPFB_BONE_SOURCES: Record<BoneName, readonly string[]>;
|
|
8
|
+
/** Neck rotation is shared across the three neck bones so one control does not triple the bend. */
|
|
9
|
+
export declare const NECK_BONE_SHARE: number;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import * as THREE from 'three';
|
|
2
|
+
import type { BlendshapeWeights } from './blendshapes';
|
|
3
|
+
/**
|
|
4
|
+
* Expo GL on iOS (especially the Simulator) often cannot sample the float
|
|
5
|
+
* DataArrayTexture Three.js uses for morph targets. We strip GPU morphs and
|
|
6
|
+
* apply the same deltas on the CPU so the body mesh still draws.
|
|
7
|
+
*/
|
|
8
|
+
export type CpuMorphMesh = {
|
|
9
|
+
mesh: THREE.Mesh;
|
|
10
|
+
dictionary: Record<string, number>;
|
|
11
|
+
basePosition: Float32Array;
|
|
12
|
+
baseNormal: Float32Array | null;
|
|
13
|
+
positionDeltas: Float32Array[];
|
|
14
|
+
normalDeltas: Float32Array[] | null;
|
|
15
|
+
position: THREE.BufferAttribute;
|
|
16
|
+
normal: THREE.BufferAttribute | null;
|
|
17
|
+
scratchPosition: Float32Array;
|
|
18
|
+
scratchNormal: Float32Array | null;
|
|
19
|
+
lastKey: string;
|
|
20
|
+
/** When set, expression offsets are stored here so CPU skinning can run after them. */
|
|
21
|
+
morphOut?: Float32Array;
|
|
22
|
+
};
|
|
23
|
+
export declare function extractCpuMorphs(mesh: THREE.Mesh): CpuMorphMesh | null;
|
|
24
|
+
export declare function applyCpuMorphs(targets: CpuMorphMesh[], weights: BlendshapeWeights): void;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import * as THREE from 'three';
|
|
2
|
+
/**
|
|
3
|
+
* Expo GL cannot sample the float bone texture Three.js uses for SkinnedMesh.
|
|
4
|
+
* The mesh is drawn as a plain Mesh, and bone motion is applied on the CPU.
|
|
5
|
+
*/
|
|
6
|
+
export type CpuSkin = {
|
|
7
|
+
position: THREE.BufferAttribute;
|
|
8
|
+
bind: Float32Array;
|
|
9
|
+
/** Added to the bind position before skinning. Face shapes write this. */
|
|
10
|
+
morph: Float32Array;
|
|
11
|
+
skinIndex: THREE.BufferAttribute;
|
|
12
|
+
skinWeight: THREE.BufferAttribute;
|
|
13
|
+
skeleton: THREE.Skeleton;
|
|
14
|
+
bindMatrix: THREE.Matrix4;
|
|
15
|
+
bindMatrixInverse: THREE.Matrix4;
|
|
16
|
+
boneMatrices: THREE.Matrix4[];
|
|
17
|
+
};
|
|
18
|
+
export declare function createCpuSkins(scene: THREE.Object3D): CpuSkin[];
|
|
19
|
+
export declare function updateCpuSkins(skins: CpuSkin[]): void;
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import * as THREE from 'three';
|
|
2
|
+
import { type CpuSkin } from './cpuSkin';
|
|
3
|
+
import { type CpuMorphMesh } from './cpuMorphs';
|
|
4
|
+
import type { BoneName } from './skeleton';
|
|
5
|
+
export type BoundBone = {
|
|
6
|
+
bone: THREE.Bone;
|
|
7
|
+
rest: THREE.Quaternion;
|
|
8
|
+
};
|
|
9
|
+
export type LoadedAvatar = {
|
|
10
|
+
scene: THREE.Group;
|
|
11
|
+
/** Rotated for subtle head-turn without GPU skinning. */
|
|
12
|
+
lookGroup: THREE.Group;
|
|
13
|
+
/** @deprecated Prefer cpuMorphs — kept for older call sites. */
|
|
14
|
+
morphMeshes: THREE.Mesh[];
|
|
15
|
+
cpuMorphs: CpuMorphMesh[];
|
|
16
|
+
cpuSkins: CpuSkin[];
|
|
17
|
+
bones: Partial<Record<BoneName, BoundBone[]>>;
|
|
18
|
+
frameCamera: (camera: THREE.PerspectiveCamera) => boolean;
|
|
19
|
+
dispose: () => void;
|
|
20
|
+
};
|
|
21
|
+
export declare function loadAvatar(moduleId: number, onProgress?: (status: string) => void): Promise<LoadedAvatar>;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bones a GLB must drive. Both sexes use these contract names.
|
|
3
|
+
* MPFB files keep their own bone names; the loader aliases them onto this list.
|
|
4
|
+
*/
|
|
5
|
+
export declare const REQUIRED_BONES: readonly ["Neck", "Head", "LeftEye", "RightEye"];
|
|
6
|
+
/** Present when the mesh also drives the jaw with a bone. Lip-sync can use jawOpen alone. */
|
|
7
|
+
export declare const OPTIONAL_BONES: readonly ["Jaw"];
|
|
8
|
+
export type RequiredBoneName = (typeof REQUIRED_BONES)[number];
|
|
9
|
+
export type OptionalBoneName = (typeof OPTIONAL_BONES)[number];
|
|
10
|
+
export type BoneName = RequiredBoneName | OptionalBoneName;
|
|
11
|
+
/** Euler degrees, applied on top of the bind pose. */
|
|
12
|
+
export type BoneRotation = {
|
|
13
|
+
x: number;
|
|
14
|
+
y: number;
|
|
15
|
+
z: number;
|
|
16
|
+
};
|
|
17
|
+
export type BonePose = Partial<Record<BoneName, BoneRotation>>;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { BlendshapeWeights } from './blendshapes';
|
|
2
|
+
/**
|
|
3
|
+
* Oculus viseme set (15). Speech text and audio both resolve to this list
|
|
4
|
+
* before they touch the mesh.
|
|
5
|
+
*/
|
|
6
|
+
export declare const VISEMES: readonly ["sil", "PP", "FF", "TH", "DD", "kk", "CH", "SS", "nn", "RR", "aa", "E", "ih", "oh", "ou"];
|
|
7
|
+
export type VisemeId = (typeof VISEMES)[number];
|
|
8
|
+
export type VisemeCue = {
|
|
9
|
+
/** Seconds from the start of the utterance. */
|
|
10
|
+
time: number;
|
|
11
|
+
viseme: VisemeId;
|
|
12
|
+
/** 0 to 1. Neighboring cues are crossfaded. */
|
|
13
|
+
weight: number;
|
|
14
|
+
};
|
|
15
|
+
/**
|
|
16
|
+
* Starting weights for each viseme. Tuned once a real mesh is loaded.
|
|
17
|
+
* `sil` is silence and leaves the mouth on the current expression.
|
|
18
|
+
*/
|
|
19
|
+
export declare const VISEME_BLENDSHAPES: Record<VisemeId, BlendshapeWeights>;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { SpeakRequest } from '../types';
|
|
2
|
+
import type { VoiceProvider } from '../voice/types';
|
|
3
|
+
export type ExpoSpeakRequest = SpeakRequest & {
|
|
4
|
+
onBoundary?: (event: {
|
|
5
|
+
charIndex: number;
|
|
6
|
+
charLength: number;
|
|
7
|
+
}) => void;
|
|
8
|
+
};
|
|
9
|
+
/**
|
|
10
|
+
* On-device system voice. Word boundaries keep mouth shapes aligned with
|
|
11
|
+
* speech; phoneme-accurate tracks still come from the animator.
|
|
12
|
+
*/
|
|
13
|
+
export declare function createExpoSpeechProvider(): VoiceProvider;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { BlendshapeWeights } from '../model/blendshapes';
|
|
2
|
+
import { type VisemeCue } from '../model/visemes';
|
|
3
|
+
export type MouthWord = {
|
|
4
|
+
/** Inclusive start character index in the original spoken string. */
|
|
5
|
+
start: number;
|
|
6
|
+
/** Exclusive end character index. */
|
|
7
|
+
end: number;
|
|
8
|
+
cues: VisemeCue[];
|
|
9
|
+
duration: number;
|
|
10
|
+
};
|
|
11
|
+
export declare function approximateMouthCues(text: string): VisemeCue[];
|
|
12
|
+
export declare function buildMouthWords(text: string): MouthWord[];
|
|
13
|
+
export declare function mouthWeightsAt(cues: VisemeCue[], elapsedSec: number): BlendshapeWeights;
|
|
14
|
+
/** Stretch a word's cues so they fill `durationSec`, then rest the mouth. */
|
|
15
|
+
export declare function scaledWordCues(word: MouthWord, durationSec: number): VisemeCue[];
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
export type AvatarSex = 'female' | 'male';
|
|
2
|
+
/** Which bundled GLB to show. Female is reserved until that model is exported. */
|
|
3
|
+
export type AvatarModelId = AvatarSex;
|
|
4
|
+
/** Clothes bundled with the male human. `casual` is the black crew-neck. */
|
|
5
|
+
export type AvatarOutfit = 'casual';
|
|
6
|
+
export type ExpressionName = 'neutral' | 'happy' | 'sad' | 'angry' | 'surprised' | 'thinking';
|
|
7
|
+
/** Gaze direction in view space. 0 is toward the camera. Each axis is -1 to 1. */
|
|
8
|
+
export type GazeTarget = {
|
|
9
|
+
x: number;
|
|
10
|
+
y: number;
|
|
11
|
+
};
|
|
12
|
+
/**
|
|
13
|
+
* A GLB/GLTF avatar. `source` is a Metro asset module from `require()`
|
|
14
|
+
* or a `file://` URI already on device.
|
|
15
|
+
*/
|
|
16
|
+
export type AvatarModel = {
|
|
17
|
+
sex: AvatarSex;
|
|
18
|
+
source: number | string;
|
|
19
|
+
};
|
|
20
|
+
export type SpeakRequest = {
|
|
21
|
+
text: string;
|
|
22
|
+
voiceId?: string;
|
|
23
|
+
/** Fired when TTS reaches a word (iOS/Android). Used to lock lip-sync. */
|
|
24
|
+
onBoundary?: (event: {
|
|
25
|
+
charIndex: number;
|
|
26
|
+
charLength: number;
|
|
27
|
+
}) => void;
|
|
28
|
+
};
|
|
29
|
+
export type AvatarViewProps = {
|
|
30
|
+
model?: AvatarModel;
|
|
31
|
+
sex?: AvatarSex;
|
|
32
|
+
expression?: ExpressionName;
|
|
33
|
+
gaze?: GazeTarget;
|
|
34
|
+
speak?: SpeakRequest | null;
|
|
35
|
+
};
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { VisemeCue } from '../model/visemes';
|
|
2
|
+
import type { SpeakRequest } from '../types';
|
|
3
|
+
export type SpeechResult = {
|
|
4
|
+
/** Playable file when the provider synthesizes audio. System TTS may omit this. */
|
|
5
|
+
audioUri?: string;
|
|
6
|
+
durationMs?: number;
|
|
7
|
+
visemes?: VisemeCue[];
|
|
8
|
+
};
|
|
9
|
+
/**
|
|
10
|
+
* One implementation per engine. The avatar never calls a TTS SDK directly.
|
|
11
|
+
* First planned adapter is on-device system speech. A later adapter can be
|
|
12
|
+
* an offline neural voice that also returns a viseme track.
|
|
13
|
+
*/
|
|
14
|
+
export type VoiceProvider = {
|
|
15
|
+
readonly id: string;
|
|
16
|
+
speak(request: SpeakRequest): Promise<SpeechResult>;
|
|
17
|
+
stop(): Promise<void>;
|
|
18
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,74 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@usamanawaz/react-native-ai-avatar",
|
|
3
|
-
"version": "
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "Reusable 3D AI avatar for Expo and React Native on Android, iOS, and web.",
|
|
5
|
+
"main": "./src/avatar/index.ts",
|
|
6
|
+
"types": "./dist/index.d.ts",
|
|
7
|
+
"react-native": "./src/avatar/index.ts",
|
|
8
|
+
"source": "./src/avatar/index.ts",
|
|
9
|
+
"exports": {
|
|
10
|
+
".": {
|
|
11
|
+
"types": "./dist/index.d.ts",
|
|
12
|
+
"react-native": "./src/avatar/index.ts",
|
|
13
|
+
"browser": "./src/avatar/index.ts",
|
|
14
|
+
"import": "./src/avatar/index.ts",
|
|
15
|
+
"default": "./src/avatar/index.ts"
|
|
16
|
+
},
|
|
17
|
+
"./package.json": "./package.json"
|
|
18
|
+
},
|
|
19
|
+
"files": [
|
|
20
|
+
"src/avatar",
|
|
21
|
+
"assets/models/male-avatar.glb",
|
|
22
|
+
"assets/models/male-avatar-clothed.glb",
|
|
23
|
+
"dist"
|
|
24
|
+
],
|
|
25
|
+
"dependencies": {
|
|
26
|
+
"jpeg-js": "^0.4.4"
|
|
27
|
+
},
|
|
28
|
+
"devDependencies": {
|
|
29
|
+
"@react-three/fiber": "^9.8.1",
|
|
30
|
+
"@types/react": "~19.2.2",
|
|
31
|
+
"@types/three": "^0.186.0",
|
|
32
|
+
"expo": "~57.0.28",
|
|
33
|
+
"expo-asset": "~57.0.19",
|
|
34
|
+
"expo-file-system": "~57.0.7",
|
|
35
|
+
"expo-gl": "~57.0.2",
|
|
36
|
+
"expo-speech": "~57.0.3",
|
|
37
|
+
"expo-status-bar": "~57.0.1",
|
|
38
|
+
"react": "19.2.3",
|
|
39
|
+
"react-dom": "19.2.3",
|
|
40
|
+
"react-native": "0.86.3",
|
|
41
|
+
"react-native-web": "^0.21.2",
|
|
42
|
+
"three": "^0.186.1",
|
|
43
|
+
"typescript": "~6.0.3"
|
|
44
|
+
},
|
|
45
|
+
"peerDependencies": {
|
|
46
|
+
"@react-three/fiber": ">=9.8.1",
|
|
47
|
+
"expo": ">=57",
|
|
48
|
+
"expo-asset": ">=57",
|
|
49
|
+
"expo-file-system": ">=57",
|
|
50
|
+
"expo-gl": ">=57",
|
|
51
|
+
"expo-speech": ">=57",
|
|
52
|
+
"react": ">=19",
|
|
53
|
+
"react-dom": ">=19",
|
|
54
|
+
"react-native": ">=0.81",
|
|
55
|
+
"react-native-web": ">=0.21.0",
|
|
56
|
+
"three": ">=0.170.0"
|
|
57
|
+
},
|
|
58
|
+
"peerDependenciesMeta": {
|
|
59
|
+
"react-dom": {
|
|
60
|
+
"optional": true
|
|
61
|
+
},
|
|
62
|
+
"react-native-web": {
|
|
63
|
+
"optional": true
|
|
64
|
+
}
|
|
65
|
+
},
|
|
66
|
+
"scripts": {
|
|
67
|
+
"start": "expo start",
|
|
68
|
+
"android": "expo start --android",
|
|
69
|
+
"ios": "expo start --ios",
|
|
70
|
+
"web": "expo start --web",
|
|
71
|
+
"build:types": "tsc -p tsconfig.build.json",
|
|
72
|
+
"prepublishOnly": "npm run build:types"
|
|
73
|
+
}
|
|
74
|
+
}
|