@memori.ai/memori-react 7.7.0 → 7.8.0-rc.0

package/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx

@@ -14,6 +14,8 @@ interface Props {
   speaking: boolean;
   isZoomed: boolean;
   chatEmission: any;
+  stopProcessing: () => void;
+  resetVisemeQueue: () => void;
   updateCurrentViseme: (currentTime: number) => { name: string; weight: number } | null;
 }
 
@@ -50,6 +52,7 @@ const baseActions: Record<string, BaseAction> = {
 
 
 export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
+  stopProcessing,
   chatEmission,
   showControls,
   animation,
@@ -62,6 +65,7 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
   loading,
   isZoomed,
   updateCurrentViseme,
+  resetVisemeQueue,
 }) => {
   const [currentBaseAction, setCurrentBaseAction] = useState({
     action: animation || 'Idle1',
@@ -178,6 +182,12 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
     }
   }, [loading]);
 
+  // useEffect(() => {
+  //   if (speaking && currentBaseAction.action !== 'Idle1') {
+  //     const animation = `Idle1`;
+  //     onBaseActionChange(animation);
+  //   }
+  // }, [speaking]);
 
   return (
     <>
@@ -202,17 +212,20 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
           morphTargetInfluences={morphTargetInfluences}
           setMorphTargetInfluences={setMorphTargetInfluences}
           setMorphTargetDictionary={setMorphTargetDictionary}
+          updateCurrentViseme={updateCurrentViseme}
         />
       ) : (
         <FullbodyAvatar
           url={url}
           sex={sex}
+          resetVisemeQueue={resetVisemeQueue}
           eyeBlink={eyeBlink}
           currentBaseAction={currentBaseAction}
           timeScale={timeScale}
           morphTargetInfluences={morphTargetInfluences}
           isZoomed={isZoomed}
           updateCurrentViseme={updateCurrentViseme}
+          stopProcessing={stopProcessing}
           setMorphTargetDictionary={setMorphTargetDictionary}
           setMorphTargetInfluences={setMorphTargetInfluences}
           emotionMorphTargets={emotionMorphTargets}

package/src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx

@@ -24,6 +24,8 @@ interface FullbodyAvatarProps {
   timeScale: number;
   isZoomed?: boolean;
   eyeBlink?: boolean;
+  stopProcessing: () => void;
+  resetVisemeQueue: () => void;
   updateCurrentViseme: (
     currentTime: number
   ) => { name: string; weight: number } | null;
@@ -239,7 +241,7 @@ export default function FullbodyAvatar({
 
             // Handle visemes (additive layer)
             if (currentViseme && key === currentViseme.name) {
-              targetValue += currentViseme.weight * 1.2; // Amplify the effect
+              targetValue += currentViseme.weight * 1.3; // Amplify the effect
             }
 
             // Handle blinking (additive layer, only for 'eyesClosed')

package/src/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.tsx

@@ -6,7 +6,7 @@ import { useGraph, dispose, useFrame } from '@react-three/fiber';
 import { useAvatarBlink } from '../../utils/useEyeBlink';
 import useHeadMovement from '../../utils/useHeadMovement';
 import { hideHands } from '../../utils/utils';
-
+import { AnimationMixer, MathUtils } from 'three';
 
 interface HalfBodyAvatarProps {
   url: string;
@@ -17,101 +17,152 @@ interface HalfBodyAvatarProps {
   setMorphTargetDictionary: (morphTargetDictionary: any) => void;
   eyeBlink?: boolean;
   morphTargetInfluences: any;
+  updateCurrentViseme: (currentTime: number) => any;
+  morphTargetSmoothing?: number;
 }
 
 const AVATAR_POSITION = new Vector3(0, -0.6, 0);
-const lerp = (start: number, end: number, alpha: number): number => {
-  return start * (1 - alpha) + end * alpha;
+// Blink configuration
+const BLINK_CONFIG = {
+  minInterval: 1000,
+  maxInterval: 5000,
+  blinkDuration: 150,
 };
+
 export default function HalfBodyAvatar({
   url,
   setMorphTargetInfluences,
   setMorphTargetDictionary,
-  headMovement,
   eyeBlink,
   onLoaded,
-  morphTargetInfluences,
+  morphTargetSmoothing = 0.5,
+  updateCurrentViseme,
 }: HalfBodyAvatarProps) {
   const { scene } = useGLTF(url);
   const { nodes, materials } = useGraph(scene);
+  const mixer = useRef(new AnimationMixer(scene));
   const avatarMeshRef = useRef<SkinnedMesh | null>(null);
 
-  useAvatarBlink({
-    enabled: eyeBlink || false,
-    setMorphTargetInfluences,
-    config: {
-      minInterval: 1500,
-      maxInterval: 4000,
-      blinkDuration: 120
-    }
-  });
-  useHeadMovement(headMovement, nodes);
+  // Blink state
+  const lastBlinkTime = useRef(0);
+  const nextBlinkTime = useRef(0);
+  const isBlinking = useRef(false);
+  const blinkStartTime = useRef(0);
+
+  const headMeshRef = useRef<SkinnedMesh>();
 
   useEffect(() => {
-    const setupAvatar = () => {
-      hideHands(nodes);
-      correctMaterials(materials);
-      // Set mesh reference for the first SkinnedMesh found
-      const firstSkinnedMesh = Object.values(nodes).find(isSkinnedMesh) as SkinnedMesh;
-      if (firstSkinnedMesh) {
-        avatarMeshRef.current = firstSkinnedMesh;
-        if (firstSkinnedMesh.morphTargetDictionary && firstSkinnedMesh.morphTargetInfluences) {
-          setMorphTargetDictionary(firstSkinnedMesh.morphTargetDictionary);
-          const initialInfluences = Object.keys(
-            firstSkinnedMesh.morphTargetDictionary
-          ).reduce((acc, key) => ({ ...acc, [key]: 0 }), {});
-          setMorphTargetInfluences(initialInfluences);
+    correctMaterials(materials);
+
+    scene.traverse((object: Object3D) => {
+      if (object instanceof SkinnedMesh) {
+        if (object.name === 'GBNL__Head' || object.name === 'Wolf3D_Avatar') {
+          headMeshRef.current = object;
+          if (object.morphTargetDictionary && object.morphTargetInfluences) {
+            setMorphTargetDictionary(object.morphTargetDictionary);
+
+            const initialInfluences = Object.keys(
+              object.morphTargetDictionary
+            ).reduce((acc, key) => ({ ...acc, [key]: 0 }), {});
+            setMorphTargetInfluences(initialInfluences);
+          }
         }
       }
-      onLoaded?.();
-    };
+    });
 
-    setupAvatar();
+    onLoaded?.();
 
     return () => {
-      const disposeObjects = () => {
-        Object.values(materials).forEach(dispose);
-        Object.values(nodes).filter(isSkinnedMesh).forEach(dispose);
-      };
-
-      disposeObjects();
+      Object.values(materials).forEach(material => material.dispose());
+      Object.values(nodes)
+        .filter(isSkinnedMesh)
+        .forEach(mesh => mesh.geometry.dispose());
     };
-  }, [materials, nodes, url, onLoaded]);
+  }, [materials, nodes, url, onLoaded, scene]);
+  useFrame(state => {
 
-  const skinnedMeshes = useMemo(
-    () => Object.values(nodes).filter(isSkinnedMesh),
-    [nodes]
-  );
+    if (
+      headMeshRef.current &&
+      headMeshRef.current.morphTargetDictionary &&
+      headMeshRef.current.morphTargetInfluences
+    ) {
+      const currentTime = state.clock.getElapsedTime() * 1000; // Convert to milliseconds
 
-    // Update morph target influences
-    useFrame((_) => {
-      if (avatarMeshRef.current && avatarMeshRef.current.morphTargetDictionary) {
-        updateMorphTargetInfluences();
-      }
-  
-      function updateMorphTargetInfluences() {
-        Object.entries(morphTargetInfluences).forEach(([key, value]) => {
-          const index = avatarMeshRef.current!.morphTargetDictionary![key];
-          if (typeof index === 'number' &&
-            avatarMeshRef.current!.morphTargetInfluences) {
-            const currentValue = avatarMeshRef.current!.morphTargetInfluences[index];
-            const smoothValue = lerp(currentValue, value as number, 0.1);
-            avatarMeshRef.current!.morphTargetInfluences[index] = smoothValue;
+      // Handle blinking
+      let blinkValue = 0;
+      if (eyeBlink) {
+        if (currentTime >= nextBlinkTime.current && !isBlinking.current) {
+          isBlinking.current = true;
+          blinkStartTime.current = currentTime;
+          lastBlinkTime.current = currentTime;
+          nextBlinkTime.current =
+            currentTime +
+            Math.random() *
+              (BLINK_CONFIG.maxInterval - BLINK_CONFIG.minInterval) +
+            BLINK_CONFIG.minInterval;
+        }
+
+        if (isBlinking.current) {
+          const blinkProgress =
+            (currentTime - blinkStartTime.current) / BLINK_CONFIG.blinkDuration;
+          if (blinkProgress <= 0.5) {
+            // Eyes closing
+            blinkValue = blinkProgress * 2;
+          } else if (blinkProgress <= 1) {
+            // Eyes opening
+            blinkValue = 2 - blinkProgress * 2;
+          } else {
+            // Blink finished
+            isBlinking.current = false;
+            blinkValue = 0;
           }
-        });
+        }
       }
-  });
 
+      const currentViseme = updateCurrentViseme(currentTime / 1000);
+
+      // Update morph targets
+      Object.entries(headMeshRef.current.morphTargetDictionary).forEach(
+        ([key, index]) => {
+          if (typeof index === 'number') {
+            let targetValue = 0;
+
+            // Handle visemes (additive layer)
+            if (currentViseme && key === currentViseme.name) {
+              targetValue += currentViseme.weight * 1.3; // Amplify the effect
+            }
+
+            // Handle blinking (additive layer, only for 'eyesClosed')
+            if (key === 'eyesClosed' && eyeBlink) {
+              targetValue += blinkValue;
+            }
+
+            // Clamp the final value between 0 and 1
+            targetValue = MathUtils.clamp(targetValue, 0, 1);
+
+            // Apply smoothing
+            if (
+              headMeshRef.current &&
+              headMeshRef.current.morphTargetInfluences
+            ) {
+              headMeshRef.current.morphTargetInfluences[index] = MathUtils.lerp(
+                headMeshRef.current.morphTargetInfluences[index],
+                targetValue,
+                morphTargetSmoothing
+              );
+            }
+          }
+        }
+      );
+
+      // Update the animation mixer
+      mixer.current.update(0.01); // Fixed delta time for consistent animation speed
+    }
+  });
 
   return (
     <group position={AVATAR_POSITION}>
-      {nodes.Hips && <primitive key="armature" object={nodes.Hips} />}
-      {skinnedMeshes.map(
-        (node: Object3D) =>
-          node && (
-            <primitive key={node.name} object={node} receiveShadow castShadow />
-          )
-      )}
+      <primitive object={scene} />
     </group>
   );
 }

package/src/components/Avatar/AvatarView/AvatarView.stories.tsx

@@ -96,9 +96,7 @@ Default.args = {
   headMovement: false,
   rotateAvatar: false,
   speaking: false,
-  stopProcessing: () => {},
-  resetVisemeQueue: () => {},
-  updateCurrentViseme: () => null,
+  updateCurrentViseme: () => {},
   url: 'https://assets.memori.ai/api/v2/asset/b791f77c-1a94-4272-829e-eca82fcc62b7.glb',
   fallbackImg:
     'https://assets.memori.ai/api/v2/asset/d8035229-08cf-42a7-a532-ab051df2603d.png',
@@ -106,9 +104,7 @@ Default.args = {
 
 export const EyeBlink = Template.bind({});
 EyeBlink.args = {
-  stopProcessing: () => {},
-  resetVisemeQueue: () => {},
-  updateCurrentViseme: () => null,
+  updateCurrentViseme: () => {},
   eyeBlink: true,
   headMovement: false,
   rotateAvatar: false,
@@ -120,9 +116,7 @@ EyeBlink.args = {
 
 export const HeadMovement = Template.bind({});
 HeadMovement.args = {
-  stopProcessing: () => {},
-  resetVisemeQueue: () => {},
-  updateCurrentViseme: () => null,
+  updateCurrentViseme: () => {},
   eyeBlink: false,
   headMovement: true,
   rotateAvatar: false,
@@ -134,9 +128,7 @@ HeadMovement.args = {
 
 export const RotateAvatar = Template.bind({});
 RotateAvatar.args = {
-  stopProcessing: () => {},
-  resetVisemeQueue: () => {},
-  updateCurrentViseme: () => null,
+  updateCurrentViseme: () => {},
   eyeBlink: false,
   headMovement: false,
   rotateAvatar: true,
@@ -148,9 +140,7 @@ RotateAvatar.args = {
 
 export const Speaking = Template.bind({});
 Speaking.args = {
-  stopProcessing: () => {},
-  resetVisemeQueue: () => {},
-  updateCurrentViseme: () => null,
+  updateCurrentViseme: () => {},
   eyeBlink: false,
   headMovement: false,
   rotateAvatar: false,
@@ -167,9 +157,7 @@ Fullbody.args = {
   headMovement: true,
   rotateAvatar: true,
   speaking: false,
-  stopProcessing: () => {},
-  resetVisemeQueue: () => {},
-  updateCurrentViseme: () => null,
+  updateCurrentViseme: () => {},
   url: 'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb',
   fallbackImg:
     'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
@@ -184,9 +172,7 @@ FullbodyZoomed.args = {
   rotateAvatar: true,
   speaking: false,
   isZoomed: true,
-  stopProcessing: () => {},
-  resetVisemeQueue: () => {},
-  updateCurrentViseme: () => null,
+  updateCurrentViseme: () => {},
   url: 'https://assets.memori.ai/api/v2/asset/3f5ef41c-6c4c-449c-888d-cf9c89782528.glb',
   fallbackImg:
     'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
@@ -200,9 +186,7 @@ FullbodyAnimatedIdle.args = {
   headMovement: true,
   rotateAvatar: true,
   speaking: false,
-  stopProcessing: () => {},
-  resetVisemeQueue: () => {},
-  updateCurrentViseme: () => null,
+  updateCurrentViseme: () => {},
   url: 'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb',
   fallbackImg:
     'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
@@ -217,9 +201,7 @@ FullbodyAnimatedLoading.args = {
   headMovement: true,
   rotateAvatar: true,
   speaking: false,
-  stopProcessing: () => {},
-  resetVisemeQueue: () => {},
-  updateCurrentViseme: () => null,
+  updateCurrentViseme: () => {},
   url: 'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb',
   fallbackImg:
     'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
@@ -234,9 +216,7 @@ FullbodyAnimatedSpeaking.args = {
   headMovement: true,
   rotateAvatar: true,
   speaking: true,
-  stopProcessing: () => {},
-  resetVisemeQueue: () => {},
-  updateCurrentViseme: () => null,
+  updateCurrentViseme: () => {},
   url: 'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb',
   fallbackImg:
     'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
@@ -251,9 +231,7 @@ FullbodyFemale.args = {
   headMovement: true,
   rotateAvatar: true,
   speaking: false,
-  stopProcessing: () => {},
-  resetVisemeQueue: () => {},
-  updateCurrentViseme: () => null,
+  updateCurrentViseme: () => {},
   url: 'https://models.readyplayer.me/650d50c2663b19e0d2831b2b.glb',
   fallbackImg:
     'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
@@ -267,9 +245,7 @@ FullbodyAnimatedFemale.args = {
   headMovement: true,
   rotateAvatar: true,
   speaking: true,
-  stopProcessing: () => {},
-  resetVisemeQueue: () => {},
-  updateCurrentViseme: () => null,
+  updateCurrentViseme: () => {},
   url: 'https://models.readyplayer.me/650d50c2663b19e0d2831b2b.glb',
   fallbackImg:
     'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',

package/src/components/Avatar/AvatarView/index.tsx

@@ -23,6 +23,8 @@ export interface Props {
   isZoomed?: boolean;
   chatEmission?: any;
   setMeshRef?: any;
+  stopProcessing: () => void;
+  resetVisemeQueue: () => void;
   updateCurrentViseme: (currentTime: number) => { name: string; weight: number } | null;
 }
 
@@ -87,6 +89,8 @@ export default function ContainerAvatarView({
   showControls = false,
   isZoomed,
   chatEmission,
+  stopProcessing,
+  resetVisemeQueue,
   updateCurrentViseme,
 }: Props) {
   return (
@@ -112,6 +116,8 @@ export default function ContainerAvatarView({
           halfBody={halfBody || false}
           chatEmission={chatEmission}
           updateCurrentViseme={updateCurrentViseme}
+          stopProcessing={stopProcessing}
+          resetVisemeQueue={resetVisemeQueue}
         />
       </Suspense>
     </Canvas>