@memori.ai/memori-react 7.6.1 → 7.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/dist/components/AttachmentMediaModal/AttachmentMediaModal.d.ts +14 -0
- package/dist/components/AttachmentMediaModal/AttachmentMediaModal.js +66 -0
- package/dist/components/AttachmentMediaModal/AttachmentMediaModal.js.map +1 -0
- package/dist/components/Avatar/Avatar.js +2 -2
- package/dist/components/Avatar/Avatar.js.map +1 -1
- package/dist/components/Avatar/AvatarView/AvatarComponent/avatarComponent.d.ts +4 -3
- package/dist/components/Avatar/AvatarView/AvatarComponent/avatarComponent.js +10 -6
- package/dist/components/Avatar/AvatarView/AvatarComponent/avatarComponent.js.map +1 -1
- package/dist/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.d.ts +11 -17
- package/dist/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.js +128 -104
- package/dist/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.js.map +1 -1
- package/dist/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.d.ts +1 -4
- package/dist/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.js +2 -4
- package/dist/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.js.map +1 -1
- package/dist/components/Avatar/AvatarView/components/fullbodyAvatar.js +8 -6
- package/dist/components/Avatar/AvatarView/components/fullbodyAvatar.js.map +1 -1
- package/dist/components/Avatar/AvatarView/index.d.ts +5 -3
- package/dist/components/Avatar/AvatarView/index.js +2 -2
- package/dist/components/Avatar/AvatarView/index.js.map +1 -1
- package/dist/components/Avatar/AvatarView/utils/useMouthSpeaking.js +1 -1
- package/dist/components/Avatar/AvatarView/utils/useMouthSpeaking.js.map +1 -1
- package/dist/components/ImageUpload/ImageUpload.css +168 -0
- package/dist/components/ImageUpload/ImageUpload.d.ts +28 -0
- package/dist/components/ImageUpload/ImageUpload.js +163 -0
- package/dist/components/ImageUpload/ImageUpload.js.map +1 -0
- package/dist/components/MemoriWidget/MemoriWidget.js +117 -118
- package/dist/components/MemoriWidget/MemoriWidget.js.map +1 -1
- package/dist/components/layouts/Default.d.ts +17 -0
- package/dist/components/layouts/Default.js +8 -0
- package/dist/components/layouts/Default.js.map +1 -0
- package/dist/components/layouts/ZoomedFullBody.d.ts +2 -2
- package/dist/components/layouts/ZoomedFullBody.js +11 -2
- package/dist/components/layouts/ZoomedFullBody.js.map +1 -1
- package/dist/components/layouts/zoomed-full-body.css +16 -0
- package/dist/components/ui/Message.d.ts +17 -0
- package/dist/components/ui/Message.js +13 -0
- package/dist/components/ui/Message.js.map +1 -0
- package/dist/context/visemeContext.d.ts +8 -15
- package/dist/context/visemeContext.js +64 -166
- package/dist/context/visemeContext.js.map +1 -1
- package/dist/helpers/translations.js +10 -2
- package/dist/helpers/translations.js.map +1 -1
- package/dist/helpers/utils.js +5 -6
- package/dist/helpers/utils.js.map +1 -1
- package/dist/styles.css +1 -0
- package/esm/components/AttachmentMediaModal/AttachmentMediaModal.d.ts +14 -0
- package/esm/components/AttachmentMediaModal/AttachmentMediaModal.js +63 -0
- package/esm/components/AttachmentMediaModal/AttachmentMediaModal.js.map +1 -0
- package/esm/components/Avatar/Avatar.js +2 -2
- package/esm/components/Avatar/Avatar.js.map +1 -1
- package/esm/components/Avatar/AvatarView/AvatarComponent/avatarComponent.d.ts +4 -3
- package/esm/components/Avatar/AvatarView/AvatarComponent/avatarComponent.js +10 -6
- package/esm/components/Avatar/AvatarView/AvatarComponent/avatarComponent.js.map +1 -1
- package/esm/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.d.ts +11 -17
- package/esm/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.js +131 -107
- package/esm/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.js.map +1 -1
- package/esm/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.d.ts +1 -4
- package/esm/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.js +2 -4
- package/esm/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.js.map +1 -1
- package/esm/components/Avatar/AvatarView/components/fullbodyAvatar.js +8 -6
- package/esm/components/Avatar/AvatarView/components/fullbodyAvatar.js.map +1 -1
- package/esm/components/Avatar/AvatarView/index.d.ts +5 -3
- package/esm/components/Avatar/AvatarView/index.js +2 -2
- package/esm/components/Avatar/AvatarView/index.js.map +1 -1
- package/esm/components/Avatar/AvatarView/utils/useMouthSpeaking.js +1 -1
- package/esm/components/Avatar/AvatarView/utils/useMouthSpeaking.js.map +1 -1
- package/esm/components/ImageUpload/ImageUpload.css +168 -0
- package/esm/components/ImageUpload/ImageUpload.d.ts +28 -0
- package/esm/components/ImageUpload/ImageUpload.js +160 -0
- package/esm/components/ImageUpload/ImageUpload.js.map +1 -0
- package/esm/components/MemoriWidget/MemoriWidget.js +117 -118
- package/esm/components/MemoriWidget/MemoriWidget.js.map +1 -1
- package/esm/components/layouts/Default.d.ts +17 -0
- package/esm/components/layouts/Default.js +5 -0
- package/esm/components/layouts/Default.js.map +1 -0
- package/esm/components/layouts/ZoomedFullBody.d.ts +2 -2
- package/esm/components/layouts/ZoomedFullBody.js +11 -2
- package/esm/components/layouts/ZoomedFullBody.js.map +1 -1
- package/esm/components/layouts/zoomed-full-body.css +16 -0
- package/esm/components/ui/Message.d.ts +17 -0
- package/esm/components/ui/Message.js +10 -0
- package/esm/components/ui/Message.js.map +1 -0
- package/esm/context/visemeContext.d.ts +8 -15
- package/esm/context/visemeContext.js +65 -167
- package/esm/context/visemeContext.js.map +1 -1
- package/esm/helpers/translations.js +10 -2
- package/esm/helpers/translations.js.map +1 -1
- package/esm/helpers/utils.js +5 -6
- package/esm/helpers/utils.js.map +1 -1
- package/esm/styles.css +1 -0
- package/package.json +1 -1
- package/src/components/Avatar/Avatar.stories.tsx +7 -5
- package/src/components/Avatar/Avatar.tsx +3 -5
- package/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx +20 -19
- package/src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx +206 -140
- package/src/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.tsx +1 -7
- package/src/components/Avatar/AvatarView/AvatarView.stories.tsx +36 -24
- package/src/components/Avatar/AvatarView/index.tsx +3 -8
- package/src/components/MemoriWidget/MemoriWidget.tsx +140 -160
- package/src/components/layouts/ZoomedFullBody.tsx +38 -29
- package/src/components/layouts/__snapshots__/ZoomedFullBody.test.tsx.snap +25 -21
- package/src/components/layouts/zoomed-full-body.css +16 -0
- package/src/context/visemeContext.tsx +90 -260
- package/src/helpers/translations.ts +11 -8
- package/src/helpers/utils.ts +9 -8
- package/src/styles.css +1 -0
- package/dist/components/Avatar/AvatarView/components/avatar.d.ts +0 -9
- package/dist/components/Avatar/AvatarView/components/avatar.js +0 -39
- package/dist/components/Avatar/AvatarView/components/avatar.js.map +0 -1
- package/dist/components/Avatar/AvatarView/utils/useViseme.d.ts +0 -18
- package/dist/components/Avatar/AvatarView/utils/useViseme.js +0 -141
- package/dist/components/Avatar/AvatarView/utils/useViseme.js.map +0 -1
- package/dist/components/Avatar/AvatarView/utils/visemeContext.d.ts +0 -24
- package/dist/components/Avatar/AvatarView/utils/visemeContext.js +0 -157
- package/dist/components/Avatar/AvatarView/utils/visemeContext.js.map +0 -1
- package/dist/components/MemoriWidget/enhanceSSML/enhanceSSML.d.ts +0 -4
- package/dist/components/MemoriWidget/enhanceSSML/enhanceSSML.js +0 -157
- package/dist/components/MemoriWidget/enhanceSSML/enhanceSSML.js.map +0 -1
- package/dist/components/layouts/ZoomedHalfBody.d.ts +0 -4
- package/dist/components/layouts/ZoomedHalfBody.js +0 -8
- package/dist/components/layouts/ZoomedHalfBody.js.map +0 -1
- package/dist/components/layouts/zoomed-half-body.css +0 -3
- package/esm/components/Avatar/AvatarView/components/avatar.d.ts +0 -9
- package/esm/components/Avatar/AvatarView/components/avatar.js +0 -35
- package/esm/components/Avatar/AvatarView/components/avatar.js.map +0 -1
- package/esm/components/Avatar/AvatarView/utils/useViseme.d.ts +0 -18
- package/esm/components/Avatar/AvatarView/utils/useViseme.js +0 -137
- package/esm/components/Avatar/AvatarView/utils/useViseme.js.map +0 -1
- package/esm/components/Avatar/AvatarView/utils/visemeContext.d.ts +0 -24
- package/esm/components/Avatar/AvatarView/utils/visemeContext.js +0 -152
- package/esm/components/Avatar/AvatarView/utils/visemeContext.js.map +0 -1
- package/esm/components/MemoriWidget/enhanceSSML/enhanceSSML.d.ts +0 -4
- package/esm/components/MemoriWidget/enhanceSSML/enhanceSSML.js +0 -152
- package/esm/components/MemoriWidget/enhanceSSML/enhanceSSML.js.map +0 -1
- package/esm/components/layouts/ZoomedHalfBody.d.ts +0 -4
- package/esm/components/layouts/ZoomedHalfBody.js +0 -5
- package/esm/components/layouts/ZoomedHalfBody.js.map +0 -1
- package/esm/components/layouts/zoomed-half-body.css +0 -3
|
@@ -1,21 +1,17 @@
|
|
|
1
|
-
import React, { useEffect, useRef, useState
|
|
1
|
+
import React, { useEffect, useRef, useState } from 'react';
|
|
2
2
|
import {
|
|
3
3
|
Vector3,
|
|
4
4
|
Euler,
|
|
5
5
|
AnimationMixer,
|
|
6
6
|
SkinnedMesh,
|
|
7
7
|
Object3D,
|
|
8
|
+
MathUtils,
|
|
8
9
|
AnimationAction,
|
|
10
|
+
LoopOnce,
|
|
9
11
|
} from 'three';
|
|
10
12
|
import { useAnimations, useGLTF } from '@react-three/drei';
|
|
11
|
-
import { useGraph,
|
|
13
|
+
import { useGraph, useFrame } from '@react-three/fiber';
|
|
12
14
|
import { correctMaterials, isSkinnedMesh } from '../../../../../helpers/utils';
|
|
13
|
-
import { useAvatarBlink } from '../../utils/useEyeBlink';
|
|
14
|
-
import { useViseme } from '../../../../../context/visemeContext';
|
|
15
|
-
|
|
16
|
-
const lerp = (start: number, end: number, alpha: number): number => {
|
|
17
|
-
return start * (1 - alpha) + end * alpha;
|
|
18
|
-
};
|
|
19
15
|
|
|
20
16
|
interface FullbodyAvatarProps {
|
|
21
17
|
url: string;
|
|
@@ -26,16 +22,21 @@ interface FullbodyAvatarProps {
|
|
|
26
22
|
weight: number;
|
|
27
23
|
};
|
|
28
24
|
timeScale: number;
|
|
29
|
-
loading?: boolean;
|
|
30
|
-
speaking?: boolean;
|
|
31
25
|
isZoomed?: boolean;
|
|
32
|
-
setMorphTargetInfluences: (influences: { [key: string]: number }) => void;
|
|
33
|
-
setMorphTargetDictionary: (dictionary: { [key: string]: number }) => void;
|
|
34
|
-
morphTargetInfluences: { [key: string]: number };
|
|
35
|
-
morphTargetDictionary: { [key: string]: number };
|
|
36
|
-
setMeshRef: any;
|
|
37
26
|
eyeBlink?: boolean;
|
|
38
|
-
|
|
27
|
+
updateCurrentViseme: (
|
|
28
|
+
currentTime: number
|
|
29
|
+
) => { name: string; weight: number } | null;
|
|
30
|
+
smoothMorphTarget?: boolean;
|
|
31
|
+
morphTargetSmoothing?: number;
|
|
32
|
+
morphTargetInfluences: Record<string, number>;
|
|
33
|
+
setMorphTargetDictionary: (
|
|
34
|
+
morphTargetDictionary: Record<string, number>
|
|
35
|
+
) => void;
|
|
36
|
+
setMorphTargetInfluences: (
|
|
37
|
+
morphTargetInfluences: Record<string, number>
|
|
38
|
+
) => void;
|
|
39
|
+
emotionMorphTargets: Record<string, number>;
|
|
39
40
|
}
|
|
40
41
|
|
|
41
42
|
const AVATAR_POSITION = new Vector3(0, -1, 0);
|
|
@@ -43,11 +44,19 @@ const AVATAR_ROTATION = new Euler(0.175, 0, 0);
|
|
|
43
44
|
const AVATAR_POSITION_ZOOMED = new Vector3(0, -1.45, 0);
|
|
44
45
|
|
|
45
46
|
const ANIMATION_URLS = {
|
|
46
|
-
MALE: 'https://assets.memori.ai/api/v2/asset/
|
|
47
|
+
MALE: 'https://assets.memori.ai/api/v2/asset/2c5e88a4-cf62-408b-9ef0-518b099dfcb2.glb',
|
|
47
48
|
FEMALE:
|
|
48
|
-
'https://assets.memori.ai/api/v2/asset/
|
|
49
|
+
'https://assets.memori.ai/api/v2/asset/0e49aa5d-f757-4292-a170-d843c2839a41.glb',
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
// Blink configuration
|
|
53
|
+
const BLINK_CONFIG = {
|
|
54
|
+
minInterval: 1000,
|
|
55
|
+
maxInterval: 5000,
|
|
56
|
+
blinkDuration: 150,
|
|
49
57
|
};
|
|
50
|
-
|
|
58
|
+
|
|
59
|
+
const EMOTION_TRANSITION_SPEED = 0.1; // Adjust this value to control emotion transition speed
|
|
51
60
|
|
|
52
61
|
export default function FullbodyAvatar({
|
|
53
62
|
url,
|
|
@@ -56,85 +65,65 @@ export default function FullbodyAvatar({
|
|
|
56
65
|
currentBaseAction,
|
|
57
66
|
timeScale,
|
|
58
67
|
isZoomed,
|
|
59
|
-
setMorphTargetInfluences,
|
|
60
|
-
setMorphTargetDictionary,
|
|
61
|
-
morphTargetInfluences,
|
|
62
68
|
eyeBlink,
|
|
63
|
-
|
|
64
|
-
|
|
69
|
+
morphTargetSmoothing = 0.5,
|
|
70
|
+
updateCurrentViseme,
|
|
71
|
+
setMorphTargetDictionary,
|
|
72
|
+
setMorphTargetInfluences,
|
|
73
|
+
emotionMorphTargets,
|
|
65
74
|
}: FullbodyAvatarProps) {
|
|
66
75
|
const { scene } = useGLTF(url);
|
|
67
76
|
const { animations } = useGLTF(ANIMATION_URLS[sex]);
|
|
68
77
|
const { nodes, materials } = useGraph(scene);
|
|
69
78
|
const { actions } = useAnimations(animations, scene);
|
|
70
|
-
const [mixer] = useState(() => new AnimationMixer(scene));
|
|
71
79
|
|
|
72
|
-
const
|
|
80
|
+
const mixer = useRef(new AnimationMixer(scene));
|
|
81
|
+
const headMeshRef = useRef<SkinnedMesh>();
|
|
73
82
|
const currentActionRef = useRef<AnimationAction | null>(null);
|
|
74
|
-
const
|
|
75
|
-
|
|
76
|
-
// Blink animation
|
|
77
|
-
useAvatarBlink({
|
|
78
|
-
enabled: eyeBlink || false,
|
|
79
|
-
setMorphTargetInfluences,
|
|
80
|
-
config: {
|
|
81
|
-
minInterval: 1500,
|
|
82
|
-
maxInterval: 4000,
|
|
83
|
-
blinkDuration: 120,
|
|
84
|
-
},
|
|
85
|
-
});
|
|
83
|
+
const [isTransitioningToIdle, setIsTransitioningToIdle] = useState(false);
|
|
86
84
|
|
|
87
|
-
//
|
|
88
|
-
const
|
|
89
|
-
|
|
85
|
+
// Blink state
|
|
86
|
+
const lastBlinkTime = useRef(0);
|
|
87
|
+
const nextBlinkTime = useRef(0);
|
|
88
|
+
const isBlinking = useRef(false);
|
|
89
|
+
const blinkStartTime = useRef(0);
|
|
90
90
|
|
|
91
|
-
|
|
91
|
+
// Morph targets
|
|
92
|
+
const currentEmotionRef = useRef<Record<string, number>>({});
|
|
93
|
+
const previousEmotionKeysRef = useRef<Set<string>>(new Set());
|
|
92
94
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
const remainingTime = (currentActionRef.current.getClip().duration - currentActionRef.current.time) * 1000;
|
|
96
|
-
setTimeout(() => {
|
|
97
|
-
startIdleAnimation();
|
|
98
|
-
}, remainingTime);
|
|
99
|
-
} else {
|
|
100
|
-
startIdleAnimation();
|
|
101
|
-
}
|
|
102
|
-
};
|
|
103
|
-
|
|
104
|
-
const startIdleAnimation = () => {
|
|
105
|
-
const idleAnimations = Object.keys(actions).filter(key =>
|
|
106
|
-
key.startsWith('Idle')
|
|
107
|
-
);
|
|
108
|
-
const randomIdle =
|
|
109
|
-
idleAnimations[Math.floor(Math.random() * idleAnimations.length)];
|
|
95
|
+
useEffect(() => {
|
|
96
|
+
correctMaterials(materials);
|
|
110
97
|
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
98
|
+
scene.traverse((object: Object3D) => {
|
|
99
|
+
if (object instanceof SkinnedMesh) {
|
|
100
|
+
if (object.name === 'GBNL__Head' || object.name === 'Wolf3D_Avatar') {
|
|
101
|
+
headMeshRef.current = object;
|
|
102
|
+
if (object.morphTargetDictionary && object.morphTargetInfluences) {
|
|
103
|
+
setMorphTargetDictionary(object.morphTargetDictionary);
|
|
114
104
|
|
|
115
|
-
|
|
116
|
-
|
|
105
|
+
const initialInfluences = Object.keys(
|
|
106
|
+
object.morphTargetDictionary
|
|
107
|
+
).reduce((acc, key) => ({ ...acc, [key]: 0 }), {});
|
|
108
|
+
setMorphTargetInfluences(initialInfluences);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
117
111
|
}
|
|
112
|
+
});
|
|
118
113
|
|
|
119
|
-
|
|
120
|
-
currentActionRef.current = idleAction;
|
|
114
|
+
onLoaded?.();
|
|
121
115
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
116
|
+
return () => {
|
|
117
|
+
Object.values(materials).forEach(material => material.dispose());
|
|
118
|
+
Object.values(nodes)
|
|
119
|
+
.filter(isSkinnedMesh)
|
|
120
|
+
.forEach(mesh => mesh.geometry.dispose());
|
|
125
121
|
};
|
|
122
|
+
}, [materials, nodes, url, onLoaded, scene]);
|
|
126
123
|
|
|
127
|
-
|
|
128
|
-
finishCurrentAnimation();
|
|
129
|
-
} else {
|
|
130
|
-
startIdleAnimation();
|
|
131
|
-
}
|
|
132
|
-
}, [actions]);
|
|
133
|
-
|
|
134
|
-
// Base animation
|
|
124
|
+
// Handle base animation changes
|
|
135
125
|
useEffect(() => {
|
|
136
|
-
if (!actions || !currentBaseAction.action
|
|
137
|
-
return;
|
|
126
|
+
if (!actions || !currentBaseAction.action) return;
|
|
138
127
|
|
|
139
128
|
const newAction = actions[currentBaseAction.action];
|
|
140
129
|
if (!newAction) {
|
|
@@ -147,79 +136,156 @@ export default function FullbodyAvatar({
|
|
|
147
136
|
const fadeOutDuration = 0.8;
|
|
148
137
|
const fadeInDuration = 0.8;
|
|
149
138
|
|
|
150
|
-
if (!currentBaseAction.action.startsWith('Idle')) {
|
|
151
|
-
setTimeout(() => {
|
|
152
|
-
transitionToIdle();
|
|
153
|
-
}, ANIMATION_DURATION);
|
|
154
|
-
}
|
|
155
|
-
|
|
156
139
|
if (currentActionRef.current) {
|
|
157
140
|
currentActionRef.current.fadeOut(fadeOutDuration);
|
|
158
141
|
}
|
|
159
|
-
|
|
160
|
-
newAction
|
|
142
|
+
|
|
143
|
+
console.log(newAction);
|
|
161
144
|
newAction.reset().fadeIn(fadeInDuration).play();
|
|
162
145
|
currentActionRef.current = newAction;
|
|
163
|
-
}, [currentBaseAction, timeScale, actions, transitionToIdle]);
|
|
164
146
|
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
correctMaterials(materials);
|
|
147
|
+
// Set the time scale for the new action
|
|
148
|
+
newAction.timeScale = timeScale;
|
|
168
149
|
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
)
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
150
|
+
// If it's an emotion animation, set it to play once and then transition to idle
|
|
151
|
+
if (
|
|
152
|
+
currentBaseAction.action.startsWith('Gioia') ||
|
|
153
|
+
currentBaseAction.action.startsWith('Rabbia') ||
|
|
154
|
+
currentBaseAction.action.startsWith('Sorpresa') ||
|
|
155
|
+
currentBaseAction.action.startsWith('Timore') ||
|
|
156
|
+
currentBaseAction.action.startsWith('Tristezza')
|
|
157
|
+
) {
|
|
158
|
+
newAction.setLoop(LoopOnce, 1);
|
|
159
|
+
newAction.clampWhenFinished = true;
|
|
160
|
+
setIsTransitioningToIdle(true);
|
|
161
|
+
}
|
|
162
|
+
}, [actions, currentBaseAction, timeScale]);
|
|
163
|
+
|
|
164
|
+
useFrame(state => {
|
|
165
|
+
if (
|
|
166
|
+
headMeshRef.current &&
|
|
167
|
+
headMeshRef.current.morphTargetDictionary &&
|
|
168
|
+
headMeshRef.current.morphTargetInfluences
|
|
169
|
+
) {
|
|
170
|
+
const currentTime = state.clock.getElapsedTime() * 1000; // Convert to milliseconds
|
|
171
|
+
|
|
172
|
+
// Handle blinking
|
|
173
|
+
let blinkValue = 0;
|
|
174
|
+
if (eyeBlink) {
|
|
175
|
+
if (currentTime >= nextBlinkTime.current && !isBlinking.current) {
|
|
176
|
+
isBlinking.current = true;
|
|
177
|
+
blinkStartTime.current = currentTime;
|
|
178
|
+
lastBlinkTime.current = currentTime;
|
|
179
|
+
nextBlinkTime.current =
|
|
180
|
+
currentTime +
|
|
181
|
+
Math.random() *
|
|
182
|
+
(BLINK_CONFIG.maxInterval - BLINK_CONFIG.minInterval) +
|
|
183
|
+
BLINK_CONFIG.minInterval;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
if (isBlinking.current) {
|
|
187
|
+
const blinkProgress =
|
|
188
|
+
(currentTime - blinkStartTime.current) / BLINK_CONFIG.blinkDuration;
|
|
189
|
+
if (blinkProgress <= 0.5) {
|
|
190
|
+
// Eyes closing
|
|
191
|
+
blinkValue = blinkProgress * 2;
|
|
192
|
+
} else if (blinkProgress <= 1) {
|
|
193
|
+
// Eyes opening
|
|
194
|
+
blinkValue = 2 - blinkProgress * 2;
|
|
195
|
+
} else {
|
|
196
|
+
// Blink finished
|
|
197
|
+
isBlinking.current = false;
|
|
198
|
+
blinkValue = 0;
|
|
199
|
+
}
|
|
184
200
|
}
|
|
185
201
|
}
|
|
186
|
-
});
|
|
187
202
|
|
|
188
|
-
|
|
203
|
+
const currentViseme = updateCurrentViseme(currentTime / 1000);
|
|
189
204
|
|
|
190
|
-
|
|
191
|
-
Object.
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
clearVisemes,
|
|
204
|
-
]);
|
|
205
|
-
|
|
206
|
-
// Update morph target influences
|
|
207
|
-
useFrame((_, delta) => {
|
|
208
|
-
if (avatarMeshRef.current && avatarMeshRef.current.morphTargetDictionary) {
|
|
209
|
-
updateMorphTargetInfluences();
|
|
210
|
-
}
|
|
211
|
-
mixer.update(delta * 0.001);
|
|
212
|
-
|
|
213
|
-
function updateMorphTargetInfluences() {
|
|
214
|
-
Object.entries(morphTargetInfluences).forEach(([key, value]) => {
|
|
215
|
-
const index = avatarMeshRef.current!.morphTargetDictionary![key];
|
|
216
|
-
if (typeof index === 'number' &&
|
|
217
|
-
avatarMeshRef.current!.morphTargetInfluences) {
|
|
218
|
-
const currentValue = avatarMeshRef.current!.morphTargetInfluences[index];
|
|
219
|
-
const smoothValue = lerp(currentValue, value, 0.1);
|
|
220
|
-
avatarMeshRef.current!.morphTargetInfluences[index] = smoothValue;
|
|
205
|
+
// Create a set of current emotion keys
|
|
206
|
+
const currentEmotionKeys = new Set(Object.keys(emotionMorphTargets));
|
|
207
|
+
|
|
208
|
+
// Reset old emotion morph targets
|
|
209
|
+
previousEmotionKeysRef.current.forEach(key => {
|
|
210
|
+
if (!currentEmotionKeys.has(key)) {
|
|
211
|
+
const index = headMeshRef.current!.morphTargetDictionary![key];
|
|
212
|
+
if (typeof index === 'number') {
|
|
213
|
+
currentEmotionRef.current[key] = 0;
|
|
214
|
+
if (headMeshRef.current && headMeshRef.current.morphTargetInfluences) {
|
|
215
|
+
headMeshRef.current.morphTargetInfluences[index] = 0;
|
|
216
|
+
}
|
|
217
|
+
}
|
|
221
218
|
}
|
|
222
219
|
});
|
|
220
|
+
|
|
221
|
+
// Update morph targets
|
|
222
|
+
Object.entries(headMeshRef.current.morphTargetDictionary).forEach(
|
|
223
|
+
([key, index]) => {
|
|
224
|
+
if (typeof index === 'number') {
|
|
225
|
+
let targetValue = 0;
|
|
226
|
+
|
|
227
|
+
// Handle emotions (base layer)
|
|
228
|
+
if (Object.prototype.hasOwnProperty.call(emotionMorphTargets, key)) {
|
|
229
|
+
const targetEmotionValue = emotionMorphTargets[key];
|
|
230
|
+
const currentEmotionValue = currentEmotionRef.current[key] || 0;
|
|
231
|
+
const newEmotionValue = MathUtils.lerp(
|
|
232
|
+
currentEmotionValue,
|
|
233
|
+
targetEmotionValue * 2,
|
|
234
|
+
EMOTION_TRANSITION_SPEED
|
|
235
|
+
);
|
|
236
|
+
currentEmotionRef.current[key] = newEmotionValue;
|
|
237
|
+
targetValue += newEmotionValue;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// Handle visemes (additive layer)
|
|
241
|
+
if (currentViseme && key === currentViseme.name) {
|
|
242
|
+
targetValue += currentViseme.weight * 1.2; // Amplify the effect
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
// Handle blinking (additive layer, only for 'eyesClosed')
|
|
246
|
+
if (key === 'eyesClosed' && eyeBlink) {
|
|
247
|
+
targetValue += blinkValue;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// Clamp the final value between 0 and 1
|
|
251
|
+
targetValue = MathUtils.clamp(targetValue, 0, 1);
|
|
252
|
+
|
|
253
|
+
// Apply smoothing
|
|
254
|
+
if (headMeshRef.current && headMeshRef.current.morphTargetInfluences) {
|
|
255
|
+
headMeshRef.current.morphTargetInfluences[index] = MathUtils.lerp(
|
|
256
|
+
headMeshRef.current.morphTargetInfluences[index],
|
|
257
|
+
targetValue,
|
|
258
|
+
morphTargetSmoothing
|
|
259
|
+
);
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
);
|
|
264
|
+
|
|
265
|
+
// Update the set of previous emotion keys for the next frame
|
|
266
|
+
previousEmotionKeysRef.current = currentEmotionKeys;
|
|
267
|
+
|
|
268
|
+
// Handle transition from emotion animation to idle
|
|
269
|
+
if (isTransitioningToIdle && currentActionRef.current) {
|
|
270
|
+
if (
|
|
271
|
+
currentActionRef.current.time >=
|
|
272
|
+
currentActionRef.current.getClip().duration
|
|
273
|
+
) {
|
|
274
|
+
// Transition to the idle animation
|
|
275
|
+
const idleNumber = Math.floor(Math.random() * 5) + 1; // Randomly choose 1, 2, 3, 4 or 5
|
|
276
|
+
const idleAction = actions[`Idle${idleNumber == 3 ? 4 : idleNumber}`];
|
|
277
|
+
|
|
278
|
+
if (idleAction) {
|
|
279
|
+
currentActionRef.current.fadeOut(0.5);
|
|
280
|
+
idleAction.reset().fadeIn(0.5).play();
|
|
281
|
+
currentActionRef.current = idleAction;
|
|
282
|
+
setIsTransitioningToIdle(false);
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
// Update the animation mixer
|
|
288
|
+
mixer.current.update(0.01); // Fixed delta time for consistent animation speed
|
|
223
289
|
}
|
|
224
290
|
});
|
|
225
291
|
|
|
@@ -231,4 +297,4 @@ export default function FullbodyAvatar({
|
|
|
231
297
|
<primitive object={scene} />
|
|
232
298
|
</group>
|
|
233
299
|
);
|
|
234
|
-
}
|
|
300
|
+
}
|
|
@@ -14,8 +14,6 @@ interface HalfBodyAvatarProps {
|
|
|
14
14
|
headMovement?: boolean;
|
|
15
15
|
speaking?: boolean;
|
|
16
16
|
onLoaded?: () => void;
|
|
17
|
-
setMeshRef: (mesh: Object3D) => void;
|
|
18
|
-
clearVisemes: () => void;
|
|
19
17
|
setMorphTargetDictionary: (morphTargetDictionary: any) => void;
|
|
20
18
|
eyeBlink?: boolean;
|
|
21
19
|
morphTargetInfluences: any;
|
|
@@ -31,9 +29,7 @@ export default function HalfBodyAvatar({
|
|
|
31
29
|
setMorphTargetDictionary,
|
|
32
30
|
headMovement,
|
|
33
31
|
eyeBlink,
|
|
34
|
-
setMeshRef,
|
|
35
32
|
onLoaded,
|
|
36
|
-
clearVisemes,
|
|
37
33
|
morphTargetInfluences,
|
|
38
34
|
}: HalfBodyAvatarProps) {
|
|
39
35
|
const { scene } = useGLTF(url);
|
|
@@ -58,7 +54,6 @@ export default function HalfBodyAvatar({
|
|
|
58
54
|
// Set mesh reference for the first SkinnedMesh found
|
|
59
55
|
const firstSkinnedMesh = Object.values(nodes).find(isSkinnedMesh) as SkinnedMesh;
|
|
60
56
|
if (firstSkinnedMesh) {
|
|
61
|
-
setMeshRef(firstSkinnedMesh);
|
|
62
57
|
avatarMeshRef.current = firstSkinnedMesh;
|
|
63
58
|
if (firstSkinnedMesh.morphTargetDictionary && firstSkinnedMesh.morphTargetInfluences) {
|
|
64
59
|
setMorphTargetDictionary(firstSkinnedMesh.morphTargetDictionary);
|
|
@@ -77,12 +72,11 @@ export default function HalfBodyAvatar({
|
|
|
77
72
|
const disposeObjects = () => {
|
|
78
73
|
Object.values(materials).forEach(dispose);
|
|
79
74
|
Object.values(nodes).filter(isSkinnedMesh).forEach(dispose);
|
|
80
|
-
clearVisemes();
|
|
81
75
|
};
|
|
82
76
|
|
|
83
77
|
disposeObjects();
|
|
84
78
|
};
|
|
85
|
-
}, [materials, nodes, url, onLoaded
|
|
79
|
+
}, [materials, nodes, url, onLoaded]);
|
|
86
80
|
|
|
87
81
|
const skinnedMeshes = useMemo(
|
|
88
82
|
() => Object.values(nodes).filter(isSkinnedMesh),
|
|
@@ -96,8 +96,9 @@ Default.args = {
|
|
|
96
96
|
headMovement: false,
|
|
97
97
|
rotateAvatar: false,
|
|
98
98
|
speaking: false,
|
|
99
|
-
|
|
100
|
-
|
|
99
|
+
stopProcessing: () => {},
|
|
100
|
+
resetVisemeQueue: () => {},
|
|
101
|
+
updateCurrentViseme: () => null,
|
|
101
102
|
url: 'https://assets.memori.ai/api/v2/asset/b791f77c-1a94-4272-829e-eca82fcc62b7.glb',
|
|
102
103
|
fallbackImg:
|
|
103
104
|
'https://assets.memori.ai/api/v2/asset/d8035229-08cf-42a7-a532-ab051df2603d.png',
|
|
@@ -105,8 +106,9 @@ Default.args = {
|
|
|
105
106
|
|
|
106
107
|
export const EyeBlink = Template.bind({});
|
|
107
108
|
EyeBlink.args = {
|
|
108
|
-
|
|
109
|
-
|
|
109
|
+
stopProcessing: () => {},
|
|
110
|
+
resetVisemeQueue: () => {},
|
|
111
|
+
updateCurrentViseme: () => null,
|
|
110
112
|
eyeBlink: true,
|
|
111
113
|
headMovement: false,
|
|
112
114
|
rotateAvatar: false,
|
|
@@ -118,8 +120,9 @@ EyeBlink.args = {
|
|
|
118
120
|
|
|
119
121
|
export const HeadMovement = Template.bind({});
|
|
120
122
|
HeadMovement.args = {
|
|
121
|
-
|
|
122
|
-
|
|
123
|
+
stopProcessing: () => {},
|
|
124
|
+
resetVisemeQueue: () => {},
|
|
125
|
+
updateCurrentViseme: () => null,
|
|
123
126
|
eyeBlink: false,
|
|
124
127
|
headMovement: true,
|
|
125
128
|
rotateAvatar: false,
|
|
@@ -131,8 +134,9 @@ HeadMovement.args = {
|
|
|
131
134
|
|
|
132
135
|
export const RotateAvatar = Template.bind({});
|
|
133
136
|
RotateAvatar.args = {
|
|
134
|
-
|
|
135
|
-
|
|
137
|
+
stopProcessing: () => {},
|
|
138
|
+
resetVisemeQueue: () => {},
|
|
139
|
+
updateCurrentViseme: () => null,
|
|
136
140
|
eyeBlink: false,
|
|
137
141
|
headMovement: false,
|
|
138
142
|
rotateAvatar: true,
|
|
@@ -144,8 +148,9 @@ RotateAvatar.args = {
|
|
|
144
148
|
|
|
145
149
|
export const Speaking = Template.bind({});
|
|
146
150
|
Speaking.args = {
|
|
147
|
-
|
|
148
|
-
|
|
151
|
+
stopProcessing: () => {},
|
|
152
|
+
resetVisemeQueue: () => {},
|
|
153
|
+
updateCurrentViseme: () => null,
|
|
149
154
|
eyeBlink: false,
|
|
150
155
|
headMovement: false,
|
|
151
156
|
rotateAvatar: false,
|
|
@@ -162,8 +167,9 @@ Fullbody.args = {
|
|
|
162
167
|
headMovement: true,
|
|
163
168
|
rotateAvatar: true,
|
|
164
169
|
speaking: false,
|
|
165
|
-
|
|
166
|
-
|
|
170
|
+
stopProcessing: () => {},
|
|
171
|
+
resetVisemeQueue: () => {},
|
|
172
|
+
updateCurrentViseme: () => null,
|
|
167
173
|
url: 'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb',
|
|
168
174
|
fallbackImg:
|
|
169
175
|
'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
|
|
@@ -178,8 +184,9 @@ FullbodyZoomed.args = {
|
|
|
178
184
|
rotateAvatar: true,
|
|
179
185
|
speaking: false,
|
|
180
186
|
isZoomed: true,
|
|
181
|
-
|
|
182
|
-
|
|
187
|
+
stopProcessing: () => {},
|
|
188
|
+
resetVisemeQueue: () => {},
|
|
189
|
+
updateCurrentViseme: () => null,
|
|
183
190
|
url: 'https://assets.memori.ai/api/v2/asset/3f5ef41c-6c4c-449c-888d-cf9c89782528.glb',
|
|
184
191
|
fallbackImg:
|
|
185
192
|
'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
|
|
@@ -193,8 +200,9 @@ FullbodyAnimatedIdle.args = {
|
|
|
193
200
|
headMovement: true,
|
|
194
201
|
rotateAvatar: true,
|
|
195
202
|
speaking: false,
|
|
196
|
-
|
|
197
|
-
|
|
203
|
+
stopProcessing: () => {},
|
|
204
|
+
resetVisemeQueue: () => {},
|
|
205
|
+
updateCurrentViseme: () => null,
|
|
198
206
|
url: 'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb',
|
|
199
207
|
fallbackImg:
|
|
200
208
|
'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
|
|
@@ -209,8 +217,9 @@ FullbodyAnimatedLoading.args = {
|
|
|
209
217
|
headMovement: true,
|
|
210
218
|
rotateAvatar: true,
|
|
211
219
|
speaking: false,
|
|
212
|
-
|
|
213
|
-
|
|
220
|
+
stopProcessing: () => {},
|
|
221
|
+
resetVisemeQueue: () => {},
|
|
222
|
+
updateCurrentViseme: () => null,
|
|
214
223
|
url: 'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb',
|
|
215
224
|
fallbackImg:
|
|
216
225
|
'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
|
|
@@ -225,8 +234,9 @@ FullbodyAnimatedSpeaking.args = {
|
|
|
225
234
|
headMovement: true,
|
|
226
235
|
rotateAvatar: true,
|
|
227
236
|
speaking: true,
|
|
228
|
-
|
|
229
|
-
|
|
237
|
+
stopProcessing: () => {},
|
|
238
|
+
resetVisemeQueue: () => {},
|
|
239
|
+
updateCurrentViseme: () => null,
|
|
230
240
|
url: 'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb',
|
|
231
241
|
fallbackImg:
|
|
232
242
|
'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
|
|
@@ -241,8 +251,9 @@ FullbodyFemale.args = {
|
|
|
241
251
|
headMovement: true,
|
|
242
252
|
rotateAvatar: true,
|
|
243
253
|
speaking: false,
|
|
244
|
-
|
|
245
|
-
|
|
254
|
+
stopProcessing: () => {},
|
|
255
|
+
resetVisemeQueue: () => {},
|
|
256
|
+
updateCurrentViseme: () => null,
|
|
246
257
|
url: 'https://models.readyplayer.me/650d50c2663b19e0d2831b2b.glb',
|
|
247
258
|
fallbackImg:
|
|
248
259
|
'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
|
|
@@ -256,8 +267,9 @@ FullbodyAnimatedFemale.args = {
|
|
|
256
267
|
headMovement: true,
|
|
257
268
|
rotateAvatar: true,
|
|
258
269
|
speaking: true,
|
|
259
|
-
|
|
260
|
-
|
|
270
|
+
stopProcessing: () => {},
|
|
271
|
+
resetVisemeQueue: () => {},
|
|
272
|
+
updateCurrentViseme: () => null,
|
|
261
273
|
url: 'https://models.readyplayer.me/650d50c2663b19e0d2831b2b.glb',
|
|
262
274
|
fallbackImg:
|
|
263
275
|
'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
|
|
@@ -23,8 +23,7 @@ export interface Props {
|
|
|
23
23
|
isZoomed?: boolean;
|
|
24
24
|
chatEmission?: any;
|
|
25
25
|
setMeshRef?: any;
|
|
26
|
-
|
|
27
|
-
setEmotion: (emotion: string) => void;
|
|
26
|
+
updateCurrentViseme: (currentTime: number) => { name: string; weight: number } | null;
|
|
28
27
|
}
|
|
29
28
|
|
|
30
29
|
const defaultStyles = {
|
|
@@ -88,9 +87,7 @@ export default function ContainerAvatarView({
|
|
|
88
87
|
showControls = false,
|
|
89
88
|
isZoomed,
|
|
90
89
|
chatEmission,
|
|
91
|
-
|
|
92
|
-
clearVisemes,
|
|
93
|
-
setEmotion,
|
|
90
|
+
updateCurrentViseme,
|
|
94
91
|
}: Props) {
|
|
95
92
|
return (
|
|
96
93
|
<Canvas
|
|
@@ -114,9 +111,7 @@ export default function ContainerAvatarView({
|
|
|
114
111
|
speaking={speaking || false}
|
|
115
112
|
halfBody={halfBody || false}
|
|
116
113
|
chatEmission={chatEmission}
|
|
117
|
-
|
|
118
|
-
clearVisemes={clearVisemes}
|
|
119
|
-
setEmotion={setEmotion}
|
|
114
|
+
updateCurrentViseme={updateCurrentViseme}
|
|
120
115
|
/>
|
|
121
116
|
</Suspense>
|
|
122
117
|
</Canvas>
|