@memori.ai/memori-react 7.7.1 → 7.8.0-rc.0

package/src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx

@@ -1,22 +1,17 @@
-import React, { useEffect, useRef, useState, useCallback } from 'react';
+import React, { useEffect, useRef, useState } from 'react';
 import {
   Vector3,
   Euler,
   AnimationMixer,
   SkinnedMesh,
   Object3D,
+  MathUtils,
   AnimationAction,
   LoopOnce,
 } from 'three';
 import { useAnimations, useGLTF } from '@react-three/drei';
 import { useGraph, useFrame } from '@react-three/fiber';
 import { correctMaterials, isSkinnedMesh } from '../../../../../helpers/utils';
-import { useAvatarBlink } from '../../utils/useEyeBlink';
-import { useViseme } from '../../../../../context/visemeContext';
-
-const lerp = (start: number, end: number, alpha: number): number => {
-  return start * (1 - alpha) + end * alpha;
-};
 
 interface FullbodyAvatarProps {
   url: string;
@@ -27,16 +22,23 @@ interface FullbodyAvatarProps {
     weight: number;
   };
   timeScale: number;
-  loading?: boolean;
-  speaking?: boolean;
   isZoomed?: boolean;
-  setMorphTargetInfluences: (influences: { [key: string]: number }) => void;
-  setMorphTargetDictionary: (dictionary: { [key: string]: number }) => void;
-  morphTargetInfluences: { [key: string]: number };
-  morphTargetDictionary: { [key: string]: number };
-  setMeshRef: any;
   eyeBlink?: boolean;
-  clearVisemes: () => void;
+  stopProcessing: () => void;
+  resetVisemeQueue: () => void;
+  updateCurrentViseme: (
+    currentTime: number
+  ) => { name: string; weight: number } | null;
+  smoothMorphTarget?: boolean;
+  morphTargetSmoothing?: number;
+  morphTargetInfluences: Record<string, number>;
+  setMorphTargetDictionary: (
+    morphTargetDictionary: Record<string, number>
+  ) => void;
+  setMorphTargetInfluences: (
+    morphTargetInfluences: Record<string, number>
+  ) => void;
+  emotionMorphTargets: Record<string, number>;
 }
 
 const AVATAR_POSITION = new Vector3(0, -1, 0);
@@ -49,7 +51,14 @@ const ANIMATION_URLS = {
     'https://assets.memori.ai/api/v2/asset/0e49aa5d-f757-4292-a170-d843c2839a41.glb',
 };
 
-const TRANSITION_DURATION = 0.5; // Duration for transitioning between animations
+// Blink configuration
+const BLINK_CONFIG = {
+  minInterval: 1000,
+  maxInterval: 5000,
+  blinkDuration: 150,
+};
+
+const EMOTION_TRANSITION_SPEED = 0.1; // Adjust this value to control emotion transition speed
 
 export default function FullbodyAvatar({
   url,
@@ -58,92 +67,63 @@ export default function FullbodyAvatar({
   currentBaseAction,
   timeScale,
   isZoomed,
-  setMorphTargetInfluences,
-  setMorphTargetDictionary,
-  morphTargetInfluences,
   eyeBlink,
-  setMeshRef,
- //clearVisemes,
+  morphTargetSmoothing = 0.5,
+  updateCurrentViseme,
+  setMorphTargetDictionary,
+  setMorphTargetInfluences,
+  emotionMorphTargets,
 }: FullbodyAvatarProps) {
   const { scene } = useGLTF(url);
   const { animations } = useGLTF(ANIMATION_URLS[sex]);
   const { nodes, materials } = useGraph(scene);
   const { actions } = useAnimations(animations, scene);
-  const [mixer] = useState(() => new AnimationMixer(scene));
 
-  const avatarMeshRef = useRef<SkinnedMesh>();
+  const mixer = useRef(new AnimationMixer(scene));
+  const headMeshRef = useRef<SkinnedMesh>();
   const currentActionRef = useRef<AnimationAction | null>(null);
-  const isTransitioningRef = useRef(false);
-  const lastActionTimeRef = useRef(0);
   const [isTransitioningToIdle, setIsTransitioningToIdle] = useState(false);
-  // Blink animation
-  useAvatarBlink({
-    enabled: eyeBlink || false,
-    setMorphTargetInfluences,
-    config: {
-      minInterval: 1500,
-      maxInterval: 4000,
-      blinkDuration: 120,
-    },
-  });
 
-  // Idle animation when emotion animation is finished
-  const transitionToIdle = useCallback(() => {
-    if (!actions || isTransitioningRef.current) return;
-
-    isTransitioningRef.current = true;
-
-    const finishCurrentAnimation = () => {
-      if (currentActionRef.current && !currentActionRef.current.paused) {
-        const remainingTime =
-          (currentActionRef.current.getClip().duration -
-            currentActionRef.current.time) *
-          1000;
-        setTimeout(() => {
-          startIdleAnimation();
-        }, remainingTime);
-      } else {
-        startIdleAnimation();
-      }
-    };
+  // Blink state
+  const lastBlinkTime = useRef(0);
+  const nextBlinkTime = useRef(0);
+  const isBlinking = useRef(false);
+  const blinkStartTime = useRef(0);
 
-    const startIdleAnimation = () => {
-      const idleAnimations = Object.keys(actions).filter(key =>
-        key.startsWith('Idle')
-      );
-      const randomIdle =
-        idleAnimations[Math.floor(Math.random() * idleAnimations.length)];
-
-      const idleAction = actions[randomIdle];
-      if (currentActionRef.current && idleAction) {
-        currentActionRef.current.crossFadeTo(
-          idleAction,
-          TRANSITION_DURATION,
-          true
-        );
-      }
+  // Morph targets
+  const currentEmotionRef = useRef<Record<string, number>>({});
+  const previousEmotionKeysRef = useRef<Set<string>>(new Set());
+
+  useEffect(() => {
+    correctMaterials(materials);
 
-      if (idleAction) {
-        idleAction.reset().fadeIn(TRANSITION_DURATION).play();
-        currentActionRef.current = idleAction;
+    scene.traverse((object: Object3D) => {
+      if (object instanceof SkinnedMesh) {
+        if (object.name === 'GBNL__Head' || object.name === 'Wolf3D_Avatar') {
+          headMeshRef.current = object;
+          if (object.morphTargetDictionary && object.morphTargetInfluences) {
+            setMorphTargetDictionary(object.morphTargetDictionary);
+
+            const initialInfluences = Object.keys(
+              object.morphTargetDictionary
+            ).reduce((acc, key) => ({ ...acc, [key]: 0 }), {});
+            setMorphTargetInfluences(initialInfluences);
+          }
+        }
       }
+    });
 
-      setTimeout(() => {
-        isTransitioningRef.current = false;
-      }, TRANSITION_DURATION * 1000);
-    };
+    onLoaded?.();
 
-    if (
-      currentActionRef.current &&
-      !currentActionRef.current.getClip().name.startsWith('Idle')
-    ) {
-      finishCurrentAnimation();
-    } else {
-      startIdleAnimation();
-    }
-  }, [actions]);
+    return () => {
+      Object.values(materials).forEach(material => material.dispose());
+      Object.values(nodes)
+        .filter(isSkinnedMesh)
+        .forEach(mesh => mesh.geometry.dispose());
+    };
+  }, [materials, nodes, url, onLoaded, scene]);
 
-  // Base animation
+  // Handle base animation changes
   useEffect(() => {
     if (!actions || !currentBaseAction.action) return;
 
@@ -183,88 +163,131 @@ export default function FullbodyAvatar({
     }
   }, [actions, currentBaseAction, timeScale]);
 
-  // Set up the mesh reference and morph target influences
-  useEffect(() => {
-    correctMaterials(materials);
+  useFrame(state => {
+    if (
+      headMeshRef.current &&
+      headMeshRef.current.morphTargetDictionary &&
+      headMeshRef.current.morphTargetInfluences
+    ) {
+      const currentTime = state.clock.getElapsedTime() * 1000; // Convert to milliseconds
+
+      // Handle blinking
+      let blinkValue = 0;
+      if (eyeBlink) {
+        if (currentTime >= nextBlinkTime.current && !isBlinking.current) {
+          isBlinking.current = true;
+          blinkStartTime.current = currentTime;
+          lastBlinkTime.current = currentTime;
+          nextBlinkTime.current =
+            currentTime +
+            Math.random() *
+              (BLINK_CONFIG.maxInterval - BLINK_CONFIG.minInterval) +
+            BLINK_CONFIG.minInterval;
+        }
 
-    scene.traverse((object: Object3D) => {
-      if (
-        object instanceof SkinnedMesh &&
-        (object.name === 'GBNL__Head' || object.name === 'Wolf3D_Avatar')
-      ) {
-        avatarMeshRef.current = object;
-        setMeshRef(object);
-
-        if (object.morphTargetDictionary && object.morphTargetInfluences) {
-          setMorphTargetDictionary(object.morphTargetDictionary);
-
-          const initialInfluences = Object.keys(
-            object.morphTargetDictionary
-          ).reduce((acc, key) => ({ ...acc, [key]: 0 }), {});
-          setMorphTargetInfluences(initialInfluences);
+        if (isBlinking.current) {
+          const blinkProgress =
+            (currentTime - blinkStartTime.current) / BLINK_CONFIG.blinkDuration;
+          if (blinkProgress <= 0.5) {
+            // Eyes closing
+            blinkValue = blinkProgress * 2;
+          } else if (blinkProgress <= 1) {
+            // Eyes opening
+            blinkValue = 2 - blinkProgress * 2;
+          } else {
+            // Blink finished
+            isBlinking.current = false;
+            blinkValue = 0;
+          }
         }
       }
-    });
 
-    onLoaded?.();
+      const currentViseme = updateCurrentViseme(currentTime / 1000);
+
+      // Create a set of current emotion keys
+      const currentEmotionKeys = new Set(Object.keys(emotionMorphTargets));
+
+      // Reset old emotion morph targets
+      previousEmotionKeysRef.current.forEach(key => {
+        if (!currentEmotionKeys.has(key)) {
+          const index = headMeshRef.current!.morphTargetDictionary![key];
+          if (typeof index === 'number') {
+            currentEmotionRef.current[key] = 0;
+            if (headMeshRef.current && headMeshRef.current.morphTargetInfluences) {
+              headMeshRef.current.morphTargetInfluences[index] = 0;
+            }
+          }
+        }
+      });
 
-    return () => {
-      Object.values(materials).forEach(material => material.dispose());
-      Object.values(nodes)
-        .filter(isSkinnedMesh)
-        .forEach(mesh => mesh.geometry.dispose());
-      mixer.stopAllAction();
-    };
-  }, [
-    materials,
-    nodes,
-    url,
-    onLoaded,
-    setMorphTargetDictionary,
-    setMorphTargetInfluences,
-    setMeshRef,
-    mixer,
-  ]);
-
-  // Update morph target influences
-  useFrame((_, delta) => {
-    if (avatarMeshRef.current && avatarMeshRef.current.morphTargetDictionary) {
-      updateMorphTargetInfluences();
-    }
-    mixer.update(delta);
+      // Update morph targets
+      Object.entries(headMeshRef.current.morphTargetDictionary).forEach(
+        ([key, index]) => {
+          if (typeof index === 'number') {
+            let targetValue = 0;
+
+            // Handle emotions (base layer)
+            if (Object.prototype.hasOwnProperty.call(emotionMorphTargets, key)) {
+              const targetEmotionValue = emotionMorphTargets[key];
+              const currentEmotionValue = currentEmotionRef.current[key] || 0;
+              const newEmotionValue = MathUtils.lerp(
+                currentEmotionValue,
+                targetEmotionValue * 2,
+                EMOTION_TRANSITION_SPEED
+              );
+              currentEmotionRef.current[key] = newEmotionValue;
+              targetValue += newEmotionValue;
+            }
+
+            // Handle visemes (additive layer)
+            if (currentViseme && key === currentViseme.name) {
+              targetValue += currentViseme.weight * 1.3; // Amplify the effect
+            }
+
+            // Handle blinking (additive layer, only for 'eyesClosed')
+            if (key === 'eyesClosed' && eyeBlink) {
+              targetValue += blinkValue;
+            }
+
+            // Clamp the final value between 0 and 1
+            targetValue = MathUtils.clamp(targetValue, 0, 1);
+
+            // Apply smoothing
+            if (headMeshRef.current && headMeshRef.current.morphTargetInfluences) {
+              headMeshRef.current.morphTargetInfluences[index] = MathUtils.lerp(
+                headMeshRef.current.morphTargetInfluences[index],
+                targetValue,
+                morphTargetSmoothing
+              );
+            }
+          }
+        }
+      );
 
-    function updateMorphTargetInfluences() {
-      Object.entries(morphTargetInfluences).forEach(([key, value]) => {
-        const index = avatarMeshRef.current!.morphTargetDictionary![key];
+      // Update the set of previous emotion keys for the next frame
+      previousEmotionKeysRef.current = currentEmotionKeys;
+
+      // Handle transition from emotion animation to idle
+      if (isTransitioningToIdle && currentActionRef.current) {
         if (
-          typeof index === 'number' &&
-          avatarMeshRef.current!.morphTargetInfluences
+          currentActionRef.current.time >=
+          currentActionRef.current.getClip().duration
         ) {
-          const currentValue =
-            avatarMeshRef.current!.morphTargetInfluences[index];
-          const smoothValue = lerp(currentValue, value, 0.1);
-          avatarMeshRef.current!.morphTargetInfluences[index] = smoothValue;
-        }
-      });
-    }
-
-    // Handle transition from emotion animation to idle
-    if (isTransitioningToIdle && currentActionRef.current) {
-      if (
-        currentActionRef.current.time >=
-        currentActionRef.current.getClip().duration
-      ) {
-        // Transition to the idle animation
-        const idleNumber = Math.floor(Math.random() * 5) + 1; // Randomly choose 1, 2, 3, 4 or 5
-        const idleAction = actions[`Idle${idleNumber == 3 ? 4 : idleNumber}`];
-
-        if (idleAction) {
-          currentActionRef.current.fadeOut(0.5);
-          idleAction.reset().fadeIn(0.5).play();
-          currentActionRef.current = idleAction;
-          setIsTransitioningToIdle(false);
+          // Transition to the idle animation
+          const idleNumber = Math.floor(Math.random() * 5) + 1; // Randomly choose 1, 2, 3, 4 or 5
+          const idleAction = actions[`Idle${idleNumber == 3 ? 4 : idleNumber}`];
+
+          if (idleAction) {
+            currentActionRef.current.fadeOut(0.5);
+            idleAction.reset().fadeIn(0.5).play();
+            currentActionRef.current = idleAction;
+            setIsTransitioningToIdle(false);
+          }
         }
       }
+
+      // Update the animation mixer
+      mixer.current.update(0.01); // Fixed delta time for consistent animation speed
     }
   });
 

package/src/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.tsx

@@ -6,7 +6,7 @@ import { useGraph, dispose, useFrame } from '@react-three/fiber';
 import { useAvatarBlink } from '../../utils/useEyeBlink';
 import useHeadMovement from '../../utils/useHeadMovement';
 import { hideHands } from '../../utils/utils';
-
+import { AnimationMixer, MathUtils } from 'three';
 
 interface HalfBodyAvatarProps {
   url: string;
@@ -14,110 +14,155 @@ interface HalfBodyAvatarProps {
   headMovement?: boolean;
   speaking?: boolean;
   onLoaded?: () => void;
-  setMeshRef: (mesh: Object3D) => void;
-  clearVisemes: () => void;
   setMorphTargetDictionary: (morphTargetDictionary: any) => void;
   eyeBlink?: boolean;
   morphTargetInfluences: any;
+  updateCurrentViseme: (currentTime: number) => any;
+  morphTargetSmoothing?: number;
 }
 
 const AVATAR_POSITION = new Vector3(0, -0.6, 0);
-const lerp = (start: number, end: number, alpha: number): number => {
-  return start * (1 - alpha) + end * alpha;
+// Blink configuration
+const BLINK_CONFIG = {
+  minInterval: 1000,
+  maxInterval: 5000,
+  blinkDuration: 150,
 };
+
 export default function HalfBodyAvatar({
   url,
   setMorphTargetInfluences,
   setMorphTargetDictionary,
-  headMovement,
   eyeBlink,
-  setMeshRef,
   onLoaded,
-  clearVisemes,
-  morphTargetInfluences,
+  morphTargetSmoothing = 0.5,
+  updateCurrentViseme,
 }: HalfBodyAvatarProps) {
   const { scene } = useGLTF(url);
   const { nodes, materials } = useGraph(scene);
+  const mixer = useRef(new AnimationMixer(scene));
   const avatarMeshRef = useRef<SkinnedMesh | null>(null);
 
-  useAvatarBlink({
-    enabled: eyeBlink || false,
-    setMorphTargetInfluences,
-    config: {
-      minInterval: 1500,
-      maxInterval: 4000,
-      blinkDuration: 120
-    }
-  });
-  useHeadMovement(headMovement, nodes);
+  // Blink state
+  const lastBlinkTime = useRef(0);
+  const nextBlinkTime = useRef(0);
+  const isBlinking = useRef(false);
+  const blinkStartTime = useRef(0);
+
+  const headMeshRef = useRef<SkinnedMesh>();
 
   useEffect(() => {
-    const setupAvatar = () => {
-      hideHands(nodes);
-      correctMaterials(materials);
-      // Set mesh reference for the first SkinnedMesh found
-      const firstSkinnedMesh = Object.values(nodes).find(isSkinnedMesh) as SkinnedMesh;
-      if (firstSkinnedMesh) {
-        setMeshRef(firstSkinnedMesh);
-        avatarMeshRef.current = firstSkinnedMesh;
-        if (firstSkinnedMesh.morphTargetDictionary && firstSkinnedMesh.morphTargetInfluences) {
-          setMorphTargetDictionary(firstSkinnedMesh.morphTargetDictionary);
-          const initialInfluences = Object.keys(
-            firstSkinnedMesh.morphTargetDictionary
-          ).reduce((acc, key) => ({ ...acc, [key]: 0 }), {});
-          setMorphTargetInfluences(initialInfluences);
+    correctMaterials(materials);
+
+    scene.traverse((object: Object3D) => {
+      if (object instanceof SkinnedMesh) {
+        if (object.name === 'GBNL__Head' || object.name === 'Wolf3D_Avatar') {
+          headMeshRef.current = object;
+          if (object.morphTargetDictionary && object.morphTargetInfluences) {
+            setMorphTargetDictionary(object.morphTargetDictionary);
+
+            const initialInfluences = Object.keys(
+              object.morphTargetDictionary
+            ).reduce((acc, key) => ({ ...acc, [key]: 0 }), {});
+            setMorphTargetInfluences(initialInfluences);
+          }
         }
       }
-      onLoaded?.();
-    };
+    });
 
-    setupAvatar();
+    onLoaded?.();
 
     return () => {
-      const disposeObjects = () => {
-        Object.values(materials).forEach(dispose);
-        Object.values(nodes).filter(isSkinnedMesh).forEach(dispose);
-        clearVisemes();
-      };
-
-      disposeObjects();
+      Object.values(materials).forEach(material => material.dispose());
+      Object.values(nodes)
+        .filter(isSkinnedMesh)
+        .forEach(mesh => mesh.geometry.dispose());
     };
-  }, [materials, nodes, url, onLoaded, clearVisemes]);
+  }, [materials, nodes, url, onLoaded, scene]);
+  useFrame(state => {
 
-  const skinnedMeshes = useMemo(
-    () => Object.values(nodes).filter(isSkinnedMesh),
-    [nodes]
-  );
+    if (
+      headMeshRef.current &&
+      headMeshRef.current.morphTargetDictionary &&
+      headMeshRef.current.morphTargetInfluences
+    ) {
+      const currentTime = state.clock.getElapsedTime() * 1000; // Convert to milliseconds
 
-    // Update morph target influences
-    useFrame((_) => {
-      if (avatarMeshRef.current && avatarMeshRef.current.morphTargetDictionary) {
-        updateMorphTargetInfluences();
-      }
-  
-      function updateMorphTargetInfluences() {
-        Object.entries(morphTargetInfluences).forEach(([key, value]) => {
-          const index = avatarMeshRef.current!.morphTargetDictionary![key];
-          if (typeof index === 'number' &&
-            avatarMeshRef.current!.morphTargetInfluences) {
-            const currentValue = avatarMeshRef.current!.morphTargetInfluences[index];
-            const smoothValue = lerp(currentValue, value as number, 0.1);
-            avatarMeshRef.current!.morphTargetInfluences[index] = smoothValue;
+      // Handle blinking
+      let blinkValue = 0;
+      if (eyeBlink) {
+        if (currentTime >= nextBlinkTime.current && !isBlinking.current) {
+          isBlinking.current = true;
+          blinkStartTime.current = currentTime;
+          lastBlinkTime.current = currentTime;
+          nextBlinkTime.current =
+            currentTime +
+            Math.random() *
+              (BLINK_CONFIG.maxInterval - BLINK_CONFIG.minInterval) +
+            BLINK_CONFIG.minInterval;
+        }
+
+        if (isBlinking.current) {
+          const blinkProgress =
+            (currentTime - blinkStartTime.current) / BLINK_CONFIG.blinkDuration;
+          if (blinkProgress <= 0.5) {
+            // Eyes closing
+            blinkValue = blinkProgress * 2;
+          } else if (blinkProgress <= 1) {
+            // Eyes opening
+            blinkValue = 2 - blinkProgress * 2;
+          } else {
+            // Blink finished
+            isBlinking.current = false;
+            blinkValue = 0;
           }
-        });
+        }
       }
-  });
 
+      const currentViseme = updateCurrentViseme(currentTime / 1000);
+
+      // Update morph targets
+      Object.entries(headMeshRef.current.morphTargetDictionary).forEach(
+        ([key, index]) => {
+          if (typeof index === 'number') {
+            let targetValue = 0;
+
+            // Handle visemes (additive layer)
+            if (currentViseme && key === currentViseme.name) {
+              targetValue += currentViseme.weight * 1.3; // Amplify the effect
+            }
+
+            // Handle blinking (additive layer, only for 'eyesClosed')
+            if (key === 'eyesClosed' && eyeBlink) {
+              targetValue += blinkValue;
+            }
+
+            // Clamp the final value between 0 and 1
+            targetValue = MathUtils.clamp(targetValue, 0, 1);
+
+            // Apply smoothing
+            if (
+              headMeshRef.current &&
+              headMeshRef.current.morphTargetInfluences
+            ) {
+              headMeshRef.current.morphTargetInfluences[index] = MathUtils.lerp(
+                headMeshRef.current.morphTargetInfluences[index],
+                targetValue,
+                morphTargetSmoothing
+              );
+            }
+          }
+        }
+      );
+
+      // Update the animation mixer
+      mixer.current.update(0.01); // Fixed delta time for consistent animation speed
+    }
+  });
 
   return (
     <group position={AVATAR_POSITION}>
-      {nodes.Hips && <primitive key="armature" object={nodes.Hips} />}
-      {skinnedMeshes.map(
-        (node: Object3D) =>
-          node && (
-            <primitive key={node.name} object={node} receiveShadow castShadow />
-          )
-      )}
+      <primitive object={scene} />
     </group>
   );
 }