@memori.ai/memori-react 7.6.1 → 7.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/dist/components/AttachmentMediaModal/AttachmentMediaModal.d.ts +14 -0
  3. package/dist/components/AttachmentMediaModal/AttachmentMediaModal.js +66 -0
  4. package/dist/components/AttachmentMediaModal/AttachmentMediaModal.js.map +1 -0
  5. package/dist/components/Avatar/Avatar.js +2 -2
  6. package/dist/components/Avatar/Avatar.js.map +1 -1
  7. package/dist/components/Avatar/AvatarView/AvatarComponent/avatarComponent.d.ts +4 -3
  8. package/dist/components/Avatar/AvatarView/AvatarComponent/avatarComponent.js +10 -6
  9. package/dist/components/Avatar/AvatarView/AvatarComponent/avatarComponent.js.map +1 -1
  10. package/dist/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.d.ts +11 -17
  11. package/dist/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.js +128 -104
  12. package/dist/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.js.map +1 -1
  13. package/dist/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.d.ts +1 -4
  14. package/dist/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.js +2 -4
  15. package/dist/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.js.map +1 -1
  16. package/dist/components/Avatar/AvatarView/components/fullbodyAvatar.js +8 -6
  17. package/dist/components/Avatar/AvatarView/components/fullbodyAvatar.js.map +1 -1
  18. package/dist/components/Avatar/AvatarView/index.d.ts +5 -3
  19. package/dist/components/Avatar/AvatarView/index.js +2 -2
  20. package/dist/components/Avatar/AvatarView/index.js.map +1 -1
  21. package/dist/components/Avatar/AvatarView/utils/useMouthSpeaking.js +1 -1
  22. package/dist/components/Avatar/AvatarView/utils/useMouthSpeaking.js.map +1 -1
  23. package/dist/components/ImageUpload/ImageUpload.css +168 -0
  24. package/dist/components/ImageUpload/ImageUpload.d.ts +28 -0
  25. package/dist/components/ImageUpload/ImageUpload.js +163 -0
  26. package/dist/components/ImageUpload/ImageUpload.js.map +1 -0
  27. package/dist/components/MemoriWidget/MemoriWidget.js +117 -118
  28. package/dist/components/MemoriWidget/MemoriWidget.js.map +1 -1
  29. package/dist/components/layouts/Default.d.ts +17 -0
  30. package/dist/components/layouts/Default.js +8 -0
  31. package/dist/components/layouts/Default.js.map +1 -0
  32. package/dist/components/layouts/ZoomedFullBody.d.ts +2 -2
  33. package/dist/components/layouts/ZoomedFullBody.js +11 -2
  34. package/dist/components/layouts/ZoomedFullBody.js.map +1 -1
  35. package/dist/components/layouts/zoomed-full-body.css +16 -0
  36. package/dist/components/ui/Message.d.ts +17 -0
  37. package/dist/components/ui/Message.js +13 -0
  38. package/dist/components/ui/Message.js.map +1 -0
  39. package/dist/context/visemeContext.d.ts +8 -15
  40. package/dist/context/visemeContext.js +64 -166
  41. package/dist/context/visemeContext.js.map +1 -1
  42. package/dist/helpers/translations.js +10 -2
  43. package/dist/helpers/translations.js.map +1 -1
  44. package/dist/helpers/utils.js +5 -6
  45. package/dist/helpers/utils.js.map +1 -1
  46. package/dist/styles.css +1 -0
  47. package/esm/components/AttachmentMediaModal/AttachmentMediaModal.d.ts +14 -0
  48. package/esm/components/AttachmentMediaModal/AttachmentMediaModal.js +63 -0
  49. package/esm/components/AttachmentMediaModal/AttachmentMediaModal.js.map +1 -0
  50. package/esm/components/Avatar/Avatar.js +2 -2
  51. package/esm/components/Avatar/Avatar.js.map +1 -1
  52. package/esm/components/Avatar/AvatarView/AvatarComponent/avatarComponent.d.ts +4 -3
  53. package/esm/components/Avatar/AvatarView/AvatarComponent/avatarComponent.js +10 -6
  54. package/esm/components/Avatar/AvatarView/AvatarComponent/avatarComponent.js.map +1 -1
  55. package/esm/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.d.ts +11 -17
  56. package/esm/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.js +131 -107
  57. package/esm/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.js.map +1 -1
  58. package/esm/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.d.ts +1 -4
  59. package/esm/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.js +2 -4
  60. package/esm/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.js.map +1 -1
  61. package/esm/components/Avatar/AvatarView/components/fullbodyAvatar.js +8 -6
  62. package/esm/components/Avatar/AvatarView/components/fullbodyAvatar.js.map +1 -1
  63. package/esm/components/Avatar/AvatarView/index.d.ts +5 -3
  64. package/esm/components/Avatar/AvatarView/index.js +2 -2
  65. package/esm/components/Avatar/AvatarView/index.js.map +1 -1
  66. package/esm/components/Avatar/AvatarView/utils/useMouthSpeaking.js +1 -1
  67. package/esm/components/Avatar/AvatarView/utils/useMouthSpeaking.js.map +1 -1
  68. package/esm/components/ImageUpload/ImageUpload.css +168 -0
  69. package/esm/components/ImageUpload/ImageUpload.d.ts +28 -0
  70. package/esm/components/ImageUpload/ImageUpload.js +160 -0
  71. package/esm/components/ImageUpload/ImageUpload.js.map +1 -0
  72. package/esm/components/MemoriWidget/MemoriWidget.js +117 -118
  73. package/esm/components/MemoriWidget/MemoriWidget.js.map +1 -1
  74. package/esm/components/layouts/Default.d.ts +17 -0
  75. package/esm/components/layouts/Default.js +5 -0
  76. package/esm/components/layouts/Default.js.map +1 -0
  77. package/esm/components/layouts/ZoomedFullBody.d.ts +2 -2
  78. package/esm/components/layouts/ZoomedFullBody.js +11 -2
  79. package/esm/components/layouts/ZoomedFullBody.js.map +1 -1
  80. package/esm/components/layouts/zoomed-full-body.css +16 -0
  81. package/esm/components/ui/Message.d.ts +17 -0
  82. package/esm/components/ui/Message.js +10 -0
  83. package/esm/components/ui/Message.js.map +1 -0
  84. package/esm/context/visemeContext.d.ts +8 -15
  85. package/esm/context/visemeContext.js +65 -167
  86. package/esm/context/visemeContext.js.map +1 -1
  87. package/esm/helpers/translations.js +10 -2
  88. package/esm/helpers/translations.js.map +1 -1
  89. package/esm/helpers/utils.js +5 -6
  90. package/esm/helpers/utils.js.map +1 -1
  91. package/esm/styles.css +1 -0
  92. package/package.json +1 -1
  93. package/src/components/Avatar/Avatar.stories.tsx +7 -5
  94. package/src/components/Avatar/Avatar.tsx +3 -5
  95. package/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx +20 -19
  96. package/src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx +206 -140
  97. package/src/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.tsx +1 -7
  98. package/src/components/Avatar/AvatarView/AvatarView.stories.tsx +36 -24
  99. package/src/components/Avatar/AvatarView/index.tsx +3 -8
  100. package/src/components/MemoriWidget/MemoriWidget.tsx +140 -160
  101. package/src/components/layouts/ZoomedFullBody.tsx +38 -29
  102. package/src/components/layouts/__snapshots__/ZoomedFullBody.test.tsx.snap +25 -21
  103. package/src/components/layouts/zoomed-full-body.css +16 -0
  104. package/src/context/visemeContext.tsx +90 -260
  105. package/src/helpers/translations.ts +11 -8
  106. package/src/helpers/utils.ts +9 -8
  107. package/src/styles.css +1 -0
  108. package/dist/components/Avatar/AvatarView/components/avatar.d.ts +0 -9
  109. package/dist/components/Avatar/AvatarView/components/avatar.js +0 -39
  110. package/dist/components/Avatar/AvatarView/components/avatar.js.map +0 -1
  111. package/dist/components/Avatar/AvatarView/utils/useViseme.d.ts +0 -18
  112. package/dist/components/Avatar/AvatarView/utils/useViseme.js +0 -141
  113. package/dist/components/Avatar/AvatarView/utils/useViseme.js.map +0 -1
  114. package/dist/components/Avatar/AvatarView/utils/visemeContext.d.ts +0 -24
  115. package/dist/components/Avatar/AvatarView/utils/visemeContext.js +0 -157
  116. package/dist/components/Avatar/AvatarView/utils/visemeContext.js.map +0 -1
  117. package/dist/components/MemoriWidget/enhanceSSML/enhanceSSML.d.ts +0 -4
  118. package/dist/components/MemoriWidget/enhanceSSML/enhanceSSML.js +0 -157
  119. package/dist/components/MemoriWidget/enhanceSSML/enhanceSSML.js.map +0 -1
  120. package/dist/components/layouts/ZoomedHalfBody.d.ts +0 -4
  121. package/dist/components/layouts/ZoomedHalfBody.js +0 -8
  122. package/dist/components/layouts/ZoomedHalfBody.js.map +0 -1
  123. package/dist/components/layouts/zoomed-half-body.css +0 -3
  124. package/esm/components/Avatar/AvatarView/components/avatar.d.ts +0 -9
  125. package/esm/components/Avatar/AvatarView/components/avatar.js +0 -35
  126. package/esm/components/Avatar/AvatarView/components/avatar.js.map +0 -1
  127. package/esm/components/Avatar/AvatarView/utils/useViseme.d.ts +0 -18
  128. package/esm/components/Avatar/AvatarView/utils/useViseme.js +0 -137
  129. package/esm/components/Avatar/AvatarView/utils/useViseme.js.map +0 -1
  130. package/esm/components/Avatar/AvatarView/utils/visemeContext.d.ts +0 -24
  131. package/esm/components/Avatar/AvatarView/utils/visemeContext.js +0 -152
  132. package/esm/components/Avatar/AvatarView/utils/visemeContext.js.map +0 -1
  133. package/esm/components/MemoriWidget/enhanceSSML/enhanceSSML.d.ts +0 -4
  134. package/esm/components/MemoriWidget/enhanceSSML/enhanceSSML.js +0 -152
  135. package/esm/components/MemoriWidget/enhanceSSML/enhanceSSML.js.map +0 -1
  136. package/esm/components/layouts/ZoomedHalfBody.d.ts +0 -4
  137. package/esm/components/layouts/ZoomedHalfBody.js +0 -5
  138. package/esm/components/layouts/ZoomedHalfBody.js.map +0 -1
  139. package/esm/components/layouts/zoomed-half-body.css +0 -3
@@ -1,21 +1,17 @@
1
- import React, { useEffect, useRef, useState, useCallback } from 'react';
1
+ import React, { useEffect, useRef, useState } from 'react';
2
2
  import {
3
3
  Vector3,
4
4
  Euler,
5
5
  AnimationMixer,
6
6
  SkinnedMesh,
7
7
  Object3D,
8
+ MathUtils,
8
9
  AnimationAction,
10
+ LoopOnce,
9
11
  } from 'three';
10
12
  import { useAnimations, useGLTF } from '@react-three/drei';
11
- import { useGraph, dispose, useFrame } from '@react-three/fiber';
13
+ import { useGraph, useFrame } from '@react-three/fiber';
12
14
  import { correctMaterials, isSkinnedMesh } from '../../../../../helpers/utils';
13
- import { useAvatarBlink } from '../../utils/useEyeBlink';
14
- import { useViseme } from '../../../../../context/visemeContext';
15
-
16
- const lerp = (start: number, end: number, alpha: number): number => {
17
- return start * (1 - alpha) + end * alpha;
18
- };
19
15
 
20
16
  interface FullbodyAvatarProps {
21
17
  url: string;
@@ -26,16 +22,21 @@ interface FullbodyAvatarProps {
26
22
  weight: number;
27
23
  };
28
24
  timeScale: number;
29
- loading?: boolean;
30
- speaking?: boolean;
31
25
  isZoomed?: boolean;
32
- setMorphTargetInfluences: (influences: { [key: string]: number }) => void;
33
- setMorphTargetDictionary: (dictionary: { [key: string]: number }) => void;
34
- morphTargetInfluences: { [key: string]: number };
35
- morphTargetDictionary: { [key: string]: number };
36
- setMeshRef: any;
37
26
  eyeBlink?: boolean;
38
- clearVisemes: () => void;
27
+ updateCurrentViseme: (
28
+ currentTime: number
29
+ ) => { name: string; weight: number } | null;
30
+ smoothMorphTarget?: boolean;
31
+ morphTargetSmoothing?: number;
32
+ morphTargetInfluences: Record<string, number>;
33
+ setMorphTargetDictionary: (
34
+ morphTargetDictionary: Record<string, number>
35
+ ) => void;
36
+ setMorphTargetInfluences: (
37
+ morphTargetInfluences: Record<string, number>
38
+ ) => void;
39
+ emotionMorphTargets: Record<string, number>;
39
40
  }
40
41
 
41
42
  const AVATAR_POSITION = new Vector3(0, -1, 0);
@@ -43,11 +44,19 @@ const AVATAR_ROTATION = new Euler(0.175, 0, 0);
43
44
  const AVATAR_POSITION_ZOOMED = new Vector3(0, -1.45, 0);
44
45
 
45
46
  const ANIMATION_URLS = {
46
- MALE: 'https://assets.memori.ai/api/v2/asset/1c350a21-97d8-4add-82cc-9dc10767a26b.glb',
47
+ MALE: 'https://assets.memori.ai/api/v2/asset/2c5e88a4-cf62-408b-9ef0-518b099dfcb2.glb',
47
48
  FEMALE:
48
- 'https://assets.memori.ai/api/v2/asset/c2b07166-de10-4c66-918b-7b7cd380cca7.glb',
49
+ 'https://assets.memori.ai/api/v2/asset/0e49aa5d-f757-4292-a170-d843c2839a41.glb',
50
+ };
51
+
52
+ // Blink configuration
53
+ const BLINK_CONFIG = {
54
+ minInterval: 1000,
55
+ maxInterval: 5000,
56
+ blinkDuration: 150,
49
57
  };
50
- const ANIMATION_DURATION = 3000; // Duration in milliseconds for non-idle animations
58
+
59
+ const EMOTION_TRANSITION_SPEED = 0.1; // Adjust this value to control emotion transition speed
51
60
 
52
61
  export default function FullbodyAvatar({
53
62
  url,
@@ -56,85 +65,65 @@ export default function FullbodyAvatar({
56
65
  currentBaseAction,
57
66
  timeScale,
58
67
  isZoomed,
59
- setMorphTargetInfluences,
60
- setMorphTargetDictionary,
61
- morphTargetInfluences,
62
68
  eyeBlink,
63
- setMeshRef,
64
- clearVisemes,
69
+ morphTargetSmoothing = 0.5,
70
+ updateCurrentViseme,
71
+ setMorphTargetDictionary,
72
+ setMorphTargetInfluences,
73
+ emotionMorphTargets,
65
74
  }: FullbodyAvatarProps) {
66
75
  const { scene } = useGLTF(url);
67
76
  const { animations } = useGLTF(ANIMATION_URLS[sex]);
68
77
  const { nodes, materials } = useGraph(scene);
69
78
  const { actions } = useAnimations(animations, scene);
70
- const [mixer] = useState(() => new AnimationMixer(scene));
71
79
 
72
- const avatarMeshRef = useRef<SkinnedMesh>();
80
+ const mixer = useRef(new AnimationMixer(scene));
81
+ const headMeshRef = useRef<SkinnedMesh>();
73
82
  const currentActionRef = useRef<AnimationAction | null>(null);
74
- const isTransitioningRef = useRef(false);
75
-
76
- // Blink animation
77
- useAvatarBlink({
78
- enabled: eyeBlink || false,
79
- setMorphTargetInfluences,
80
- config: {
81
- minInterval: 1500,
82
- maxInterval: 4000,
83
- blinkDuration: 120,
84
- },
85
- });
83
+ const [isTransitioningToIdle, setIsTransitioningToIdle] = useState(false);
86
84
 
87
- // Idle animation when emotion animation is finished
88
- const transitionToIdle = useCallback(() => {
89
- if (!actions || isTransitioningRef.current) return;
85
+ // Blink state
86
+ const lastBlinkTime = useRef(0);
87
+ const nextBlinkTime = useRef(0);
88
+ const isBlinking = useRef(false);
89
+ const blinkStartTime = useRef(0);
90
90
 
91
- isTransitioningRef.current = true;
91
+ // Morph targets
92
+ const currentEmotionRef = useRef<Record<string, number>>({});
93
+ const previousEmotionKeysRef = useRef<Set<string>>(new Set());
92
94
 
93
- const finishCurrentAnimation = () => {
94
- if (currentActionRef.current && !currentActionRef.current.paused) {
95
- const remainingTime = (currentActionRef.current.getClip().duration - currentActionRef.current.time) * 1000;
96
- setTimeout(() => {
97
- startIdleAnimation();
98
- }, remainingTime);
99
- } else {
100
- startIdleAnimation();
101
- }
102
- };
103
-
104
- const startIdleAnimation = () => {
105
- const idleAnimations = Object.keys(actions).filter(key =>
106
- key.startsWith('Idle')
107
- );
108
- const randomIdle =
109
- idleAnimations[Math.floor(Math.random() * idleAnimations.length)];
95
+ useEffect(() => {
96
+ correctMaterials(materials);
110
97
 
111
- const idleAction = actions[randomIdle];
112
- const fadeOutDuration = 0.5;
113
- const fadeInDuration = 0.5;
98
+ scene.traverse((object: Object3D) => {
99
+ if (object instanceof SkinnedMesh) {
100
+ if (object.name === 'GBNL__Head' || object.name === 'Wolf3D_Avatar') {
101
+ headMeshRef.current = object;
102
+ if (object.morphTargetDictionary && object.morphTargetInfluences) {
103
+ setMorphTargetDictionary(object.morphTargetDictionary);
114
104
 
115
- if (currentActionRef.current) {
116
- currentActionRef.current.fadeOut(fadeOutDuration);
105
+ const initialInfluences = Object.keys(
106
+ object.morphTargetDictionary
107
+ ).reduce((acc, key) => ({ ...acc, [key]: 0 }), {});
108
+ setMorphTargetInfluences(initialInfluences);
109
+ }
110
+ }
117
111
  }
112
+ });
118
113
 
119
- idleAction?.reset().fadeIn(fadeInDuration).play();
120
- currentActionRef.current = idleAction;
114
+ onLoaded?.();
121
115
 
122
- setTimeout(() => {
123
- isTransitioningRef.current = false;
124
- }, (fadeOutDuration + fadeInDuration) * 1000);
116
+ return () => {
117
+ Object.values(materials).forEach(material => material.dispose());
118
+ Object.values(nodes)
119
+ .filter(isSkinnedMesh)
120
+ .forEach(mesh => mesh.geometry.dispose());
125
121
  };
122
+ }, [materials, nodes, url, onLoaded, scene]);
126
123
 
127
- if (currentActionRef.current && !currentActionRef.current.getClip().name.startsWith('Idle')) {
128
- finishCurrentAnimation();
129
- } else {
130
- startIdleAnimation();
131
- }
132
- }, [actions]);
133
-
134
- // Base animation
124
+ // Handle base animation changes
135
125
  useEffect(() => {
136
- if (!actions || !currentBaseAction.action || isTransitioningRef.current)
137
- return;
126
+ if (!actions || !currentBaseAction.action) return;
138
127
 
139
128
  const newAction = actions[currentBaseAction.action];
140
129
  if (!newAction) {
@@ -147,79 +136,156 @@ export default function FullbodyAvatar({
147
136
  const fadeOutDuration = 0.8;
148
137
  const fadeInDuration = 0.8;
149
138
 
150
- if (!currentBaseAction.action.startsWith('Idle')) {
151
- setTimeout(() => {
152
- transitionToIdle();
153
- }, ANIMATION_DURATION);
154
- }
155
-
156
139
  if (currentActionRef.current) {
157
140
  currentActionRef.current.fadeOut(fadeOutDuration);
158
141
  }
159
-
160
- newAction.timeScale = timeScale;
142
+
143
+ console.log(newAction);
161
144
  newAction.reset().fadeIn(fadeInDuration).play();
162
145
  currentActionRef.current = newAction;
163
- }, [currentBaseAction, timeScale, actions, transitionToIdle]);
164
146
 
165
- // Set up the mesh reference and morph target influences
166
- useEffect(() => {
167
- correctMaterials(materials);
147
+ // Set the time scale for the new action
148
+ newAction.timeScale = timeScale;
168
149
 
169
- scene.traverse((object: Object3D) => {
170
- if (
171
- object instanceof SkinnedMesh &&
172
- (object.name === 'GBNL__Head' || object.name === 'Wolf3D_Avatar')
173
- ) {
174
- avatarMeshRef.current = object;
175
- setMeshRef(object);
176
-
177
- if (object.morphTargetDictionary && object.morphTargetInfluences) {
178
- setMorphTargetDictionary(object.morphTargetDictionary);
179
-
180
- const initialInfluences = Object.keys(
181
- object.morphTargetDictionary
182
- ).reduce((acc, key) => ({ ...acc, [key]: 0 }), {});
183
- setMorphTargetInfluences(initialInfluences);
150
+ // If it's an emotion animation, set it to play once and then transition to idle
151
+ if (
152
+ currentBaseAction.action.startsWith('Gioia') ||
153
+ currentBaseAction.action.startsWith('Rabbia') ||
154
+ currentBaseAction.action.startsWith('Sorpresa') ||
155
+ currentBaseAction.action.startsWith('Timore') ||
156
+ currentBaseAction.action.startsWith('Tristezza')
157
+ ) {
158
+ newAction.setLoop(LoopOnce, 1);
159
+ newAction.clampWhenFinished = true;
160
+ setIsTransitioningToIdle(true);
161
+ }
162
+ }, [actions, currentBaseAction, timeScale]);
163
+
164
+ useFrame(state => {
165
+ if (
166
+ headMeshRef.current &&
167
+ headMeshRef.current.morphTargetDictionary &&
168
+ headMeshRef.current.morphTargetInfluences
169
+ ) {
170
+ const currentTime = state.clock.getElapsedTime() * 1000; // Convert to milliseconds
171
+
172
+ // Handle blinking
173
+ let blinkValue = 0;
174
+ if (eyeBlink) {
175
+ if (currentTime >= nextBlinkTime.current && !isBlinking.current) {
176
+ isBlinking.current = true;
177
+ blinkStartTime.current = currentTime;
178
+ lastBlinkTime.current = currentTime;
179
+ nextBlinkTime.current =
180
+ currentTime +
181
+ Math.random() *
182
+ (BLINK_CONFIG.maxInterval - BLINK_CONFIG.minInterval) +
183
+ BLINK_CONFIG.minInterval;
184
+ }
185
+
186
+ if (isBlinking.current) {
187
+ const blinkProgress =
188
+ (currentTime - blinkStartTime.current) / BLINK_CONFIG.blinkDuration;
189
+ if (blinkProgress <= 0.5) {
190
+ // Eyes closing
191
+ blinkValue = blinkProgress * 2;
192
+ } else if (blinkProgress <= 1) {
193
+ // Eyes opening
194
+ blinkValue = 2 - blinkProgress * 2;
195
+ } else {
196
+ // Blink finished
197
+ isBlinking.current = false;
198
+ blinkValue = 0;
199
+ }
184
200
  }
185
201
  }
186
- });
187
202
 
188
- onLoaded?.();
203
+ const currentViseme = updateCurrentViseme(currentTime / 1000);
189
204
 
190
- return () => {
191
- Object.values(materials).forEach(dispose);
192
- Object.values(nodes).filter(isSkinnedMesh).forEach(dispose);
193
- clearVisemes();
194
- };
195
- }, [
196
- materials,
197
- nodes,
198
- url,
199
- onLoaded,
200
- setMorphTargetDictionary,
201
- setMorphTargetInfluences,
202
- setMeshRef,
203
- clearVisemes,
204
- ]);
205
-
206
- // Update morph target influences
207
- useFrame((_, delta) => {
208
- if (avatarMeshRef.current && avatarMeshRef.current.morphTargetDictionary) {
209
- updateMorphTargetInfluences();
210
- }
211
- mixer.update(delta * 0.001);
212
-
213
- function updateMorphTargetInfluences() {
214
- Object.entries(morphTargetInfluences).forEach(([key, value]) => {
215
- const index = avatarMeshRef.current!.morphTargetDictionary![key];
216
- if (typeof index === 'number' &&
217
- avatarMeshRef.current!.morphTargetInfluences) {
218
- const currentValue = avatarMeshRef.current!.morphTargetInfluences[index];
219
- const smoothValue = lerp(currentValue, value, 0.1);
220
- avatarMeshRef.current!.morphTargetInfluences[index] = smoothValue;
205
+ // Create a set of current emotion keys
206
+ const currentEmotionKeys = new Set(Object.keys(emotionMorphTargets));
207
+
208
+ // Reset old emotion morph targets
209
+ previousEmotionKeysRef.current.forEach(key => {
210
+ if (!currentEmotionKeys.has(key)) {
211
+ const index = headMeshRef.current!.morphTargetDictionary![key];
212
+ if (typeof index === 'number') {
213
+ currentEmotionRef.current[key] = 0;
214
+ if (headMeshRef.current && headMeshRef.current.morphTargetInfluences) {
215
+ headMeshRef.current.morphTargetInfluences[index] = 0;
216
+ }
217
+ }
221
218
  }
222
219
  });
220
+
221
+ // Update morph targets
222
+ Object.entries(headMeshRef.current.morphTargetDictionary).forEach(
223
+ ([key, index]) => {
224
+ if (typeof index === 'number') {
225
+ let targetValue = 0;
226
+
227
+ // Handle emotions (base layer)
228
+ if (Object.prototype.hasOwnProperty.call(emotionMorphTargets, key)) {
229
+ const targetEmotionValue = emotionMorphTargets[key];
230
+ const currentEmotionValue = currentEmotionRef.current[key] || 0;
231
+ const newEmotionValue = MathUtils.lerp(
232
+ currentEmotionValue,
233
+ targetEmotionValue * 2,
234
+ EMOTION_TRANSITION_SPEED
235
+ );
236
+ currentEmotionRef.current[key] = newEmotionValue;
237
+ targetValue += newEmotionValue;
238
+ }
239
+
240
+ // Handle visemes (additive layer)
241
+ if (currentViseme && key === currentViseme.name) {
242
+ targetValue += currentViseme.weight * 1.2; // Amplify the effect
243
+ }
244
+
245
+ // Handle blinking (additive layer, only for 'eyesClosed')
246
+ if (key === 'eyesClosed' && eyeBlink) {
247
+ targetValue += blinkValue;
248
+ }
249
+
250
+ // Clamp the final value between 0 and 1
251
+ targetValue = MathUtils.clamp(targetValue, 0, 1);
252
+
253
+ // Apply smoothing
254
+ if (headMeshRef.current && headMeshRef.current.morphTargetInfluences) {
255
+ headMeshRef.current.morphTargetInfluences[index] = MathUtils.lerp(
256
+ headMeshRef.current.morphTargetInfluences[index],
257
+ targetValue,
258
+ morphTargetSmoothing
259
+ );
260
+ }
261
+ }
262
+ }
263
+ );
264
+
265
+ // Update the set of previous emotion keys for the next frame
266
+ previousEmotionKeysRef.current = currentEmotionKeys;
267
+
268
+ // Handle transition from emotion animation to idle
269
+ if (isTransitioningToIdle && currentActionRef.current) {
270
+ if (
271
+ currentActionRef.current.time >=
272
+ currentActionRef.current.getClip().duration
273
+ ) {
274
+ // Transition to the idle animation
275
+ const idleNumber = Math.floor(Math.random() * 5) + 1; // Randomly choose 1, 2, 3, 4 or 5
276
+ const idleAction = actions[`Idle${idleNumber == 3 ? 4 : idleNumber}`];
277
+
278
+ if (idleAction) {
279
+ currentActionRef.current.fadeOut(0.5);
280
+ idleAction.reset().fadeIn(0.5).play();
281
+ currentActionRef.current = idleAction;
282
+ setIsTransitioningToIdle(false);
283
+ }
284
+ }
285
+ }
286
+
287
+ // Update the animation mixer
288
+ mixer.current.update(0.01); // Fixed delta time for consistent animation speed
223
289
  }
224
290
  });
225
291
 
@@ -231,4 +297,4 @@ export default function FullbodyAvatar({
231
297
  <primitive object={scene} />
232
298
  </group>
233
299
  );
234
- }
300
+ }
@@ -14,8 +14,6 @@ interface HalfBodyAvatarProps {
14
14
  headMovement?: boolean;
15
15
  speaking?: boolean;
16
16
  onLoaded?: () => void;
17
- setMeshRef: (mesh: Object3D) => void;
18
- clearVisemes: () => void;
19
17
  setMorphTargetDictionary: (morphTargetDictionary: any) => void;
20
18
  eyeBlink?: boolean;
21
19
  morphTargetInfluences: any;
@@ -31,9 +29,7 @@ export default function HalfBodyAvatar({
31
29
  setMorphTargetDictionary,
32
30
  headMovement,
33
31
  eyeBlink,
34
- setMeshRef,
35
32
  onLoaded,
36
- clearVisemes,
37
33
  morphTargetInfluences,
38
34
  }: HalfBodyAvatarProps) {
39
35
  const { scene } = useGLTF(url);
@@ -58,7 +54,6 @@ export default function HalfBodyAvatar({
58
54
  // Set mesh reference for the first SkinnedMesh found
59
55
  const firstSkinnedMesh = Object.values(nodes).find(isSkinnedMesh) as SkinnedMesh;
60
56
  if (firstSkinnedMesh) {
61
- setMeshRef(firstSkinnedMesh);
62
57
  avatarMeshRef.current = firstSkinnedMesh;
63
58
  if (firstSkinnedMesh.morphTargetDictionary && firstSkinnedMesh.morphTargetInfluences) {
64
59
  setMorphTargetDictionary(firstSkinnedMesh.morphTargetDictionary);
@@ -77,12 +72,11 @@ export default function HalfBodyAvatar({
77
72
  const disposeObjects = () => {
78
73
  Object.values(materials).forEach(dispose);
79
74
  Object.values(nodes).filter(isSkinnedMesh).forEach(dispose);
80
- clearVisemes();
81
75
  };
82
76
 
83
77
  disposeObjects();
84
78
  };
85
- }, [materials, nodes, url, onLoaded, clearVisemes]);
79
+ }, [materials, nodes, url, onLoaded]);
86
80
 
87
81
  const skinnedMeshes = useMemo(
88
82
  () => Object.values(nodes).filter(isSkinnedMesh),
@@ -96,8 +96,9 @@ Default.args = {
96
96
  headMovement: false,
97
97
  rotateAvatar: false,
98
98
  speaking: false,
99
- clearVisemes: () => {},
100
- setMeshRef: () => {},
99
+ stopProcessing: () => {},
100
+ resetVisemeQueue: () => {},
101
+ updateCurrentViseme: () => null,
101
102
  url: 'https://assets.memori.ai/api/v2/asset/b791f77c-1a94-4272-829e-eca82fcc62b7.glb',
102
103
  fallbackImg:
103
104
  'https://assets.memori.ai/api/v2/asset/d8035229-08cf-42a7-a532-ab051df2603d.png',
@@ -105,8 +106,9 @@ Default.args = {
105
106
 
106
107
  export const EyeBlink = Template.bind({});
107
108
  EyeBlink.args = {
108
- clearVisemes: () => {},
109
- setMeshRef: () => {},
109
+ stopProcessing: () => {},
110
+ resetVisemeQueue: () => {},
111
+ updateCurrentViseme: () => null,
110
112
  eyeBlink: true,
111
113
  headMovement: false,
112
114
  rotateAvatar: false,
@@ -118,8 +120,9 @@ EyeBlink.args = {
118
120
 
119
121
  export const HeadMovement = Template.bind({});
120
122
  HeadMovement.args = {
121
- clearVisemes: () => {},
122
- setMeshRef: () => {},
123
+ stopProcessing: () => {},
124
+ resetVisemeQueue: () => {},
125
+ updateCurrentViseme: () => null,
123
126
  eyeBlink: false,
124
127
  headMovement: true,
125
128
  rotateAvatar: false,
@@ -131,8 +134,9 @@ HeadMovement.args = {
131
134
 
132
135
  export const RotateAvatar = Template.bind({});
133
136
  RotateAvatar.args = {
134
- clearVisemes: () => {},
135
- setMeshRef: () => {},
137
+ stopProcessing: () => {},
138
+ resetVisemeQueue: () => {},
139
+ updateCurrentViseme: () => null,
136
140
  eyeBlink: false,
137
141
  headMovement: false,
138
142
  rotateAvatar: true,
@@ -144,8 +148,9 @@ RotateAvatar.args = {
144
148
 
145
149
  export const Speaking = Template.bind({});
146
150
  Speaking.args = {
147
- clearVisemes: () => {},
148
- setMeshRef: () => {},
151
+ stopProcessing: () => {},
152
+ resetVisemeQueue: () => {},
153
+ updateCurrentViseme: () => null,
149
154
  eyeBlink: false,
150
155
  headMovement: false,
151
156
  rotateAvatar: false,
@@ -162,8 +167,9 @@ Fullbody.args = {
162
167
  headMovement: true,
163
168
  rotateAvatar: true,
164
169
  speaking: false,
165
- clearVisemes: () => {},
166
- setMeshRef: () => {},
170
+ stopProcessing: () => {},
171
+ resetVisemeQueue: () => {},
172
+ updateCurrentViseme: () => null,
167
173
  url: 'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb',
168
174
  fallbackImg:
169
175
  'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
@@ -178,8 +184,9 @@ FullbodyZoomed.args = {
178
184
  rotateAvatar: true,
179
185
  speaking: false,
180
186
  isZoomed: true,
181
- clearVisemes: () => {},
182
- setMeshRef: () => {},
187
+ stopProcessing: () => {},
188
+ resetVisemeQueue: () => {},
189
+ updateCurrentViseme: () => null,
183
190
  url: 'https://assets.memori.ai/api/v2/asset/3f5ef41c-6c4c-449c-888d-cf9c89782528.glb',
184
191
  fallbackImg:
185
192
  'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
@@ -193,8 +200,9 @@ FullbodyAnimatedIdle.args = {
193
200
  headMovement: true,
194
201
  rotateAvatar: true,
195
202
  speaking: false,
196
- clearVisemes: () => {},
197
- setMeshRef: () => {},
203
+ stopProcessing: () => {},
204
+ resetVisemeQueue: () => {},
205
+ updateCurrentViseme: () => null,
198
206
  url: 'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb',
199
207
  fallbackImg:
200
208
  'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
@@ -209,8 +217,9 @@ FullbodyAnimatedLoading.args = {
209
217
  headMovement: true,
210
218
  rotateAvatar: true,
211
219
  speaking: false,
212
- clearVisemes: () => {},
213
- setMeshRef: () => {},
220
+ stopProcessing: () => {},
221
+ resetVisemeQueue: () => {},
222
+ updateCurrentViseme: () => null,
214
223
  url: 'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb',
215
224
  fallbackImg:
216
225
  'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
@@ -225,8 +234,9 @@ FullbodyAnimatedSpeaking.args = {
225
234
  headMovement: true,
226
235
  rotateAvatar: true,
227
236
  speaking: true,
228
- clearVisemes: () => {},
229
- setMeshRef: () => {},
237
+ stopProcessing: () => {},
238
+ resetVisemeQueue: () => {},
239
+ updateCurrentViseme: () => null,
230
240
  url: 'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb',
231
241
  fallbackImg:
232
242
  'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
@@ -241,8 +251,9 @@ FullbodyFemale.args = {
241
251
  headMovement: true,
242
252
  rotateAvatar: true,
243
253
  speaking: false,
244
- clearVisemes: () => {},
245
- setMeshRef: () => {},
254
+ stopProcessing: () => {},
255
+ resetVisemeQueue: () => {},
256
+ updateCurrentViseme: () => null,
246
257
  url: 'https://models.readyplayer.me/650d50c2663b19e0d2831b2b.glb',
247
258
  fallbackImg:
248
259
  'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
@@ -256,8 +267,9 @@ FullbodyAnimatedFemale.args = {
256
267
  headMovement: true,
257
268
  rotateAvatar: true,
258
269
  speaking: true,
259
- clearVisemes: () => {},
260
- setMeshRef: () => {},
270
+ stopProcessing: () => {},
271
+ resetVisemeQueue: () => {},
272
+ updateCurrentViseme: () => null,
261
273
  url: 'https://models.readyplayer.me/650d50c2663b19e0d2831b2b.glb',
262
274
  fallbackImg:
263
275
  'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
@@ -23,8 +23,7 @@ export interface Props {
23
23
  isZoomed?: boolean;
24
24
  chatEmission?: any;
25
25
  setMeshRef?: any;
26
- clearVisemes: () => void;
27
- setEmotion: (emotion: string) => void;
26
+ updateCurrentViseme: (currentTime: number) => { name: string; weight: number } | null;
28
27
  }
29
28
 
30
29
  const defaultStyles = {
@@ -88,9 +87,7 @@ export default function ContainerAvatarView({
88
87
  showControls = false,
89
88
  isZoomed,
90
89
  chatEmission,
91
- setMeshRef,
92
- clearVisemes,
93
- setEmotion,
90
+ updateCurrentViseme,
94
91
  }: Props) {
95
92
  return (
96
93
  <Canvas
@@ -114,9 +111,7 @@ export default function ContainerAvatarView({
114
111
  speaking={speaking || false}
115
112
  halfBody={halfBody || false}
116
113
  chatEmission={chatEmission}
117
- setMeshRef={setMeshRef}
118
- clearVisemes={clearVisemes}
119
- setEmotion={setEmotion}
114
+ updateCurrentViseme={updateCurrentViseme}
120
115
  />
121
116
  </Suspense>
122
117
  </Canvas>