@memori.ai/memori-react 7.6.0 → 7.7.0

package/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx

@@ -14,9 +14,7 @@ interface Props {
   speaking: boolean;
   isZoomed: boolean;
   chatEmission: any;
-  setMeshRef: any;
-  clearVisemes: () => void;
-  setEmotion: (emotion: string) => void;
+  updateCurrentViseme: (currentTime: number) => { name: string; weight: number } | null;
 }
 
 interface BaseAction {
@@ -50,9 +48,8 @@ const baseActions: Record<string, BaseAction> = {
   Loading3: { weight: 0 },
 };
 
+
 export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
-  setMeshRef,
-  clearVisemes,
   chatEmission,
   showControls,
   animation,
@@ -64,7 +61,7 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
   halfBody,
   loading,
   isZoomed,
-  setEmotion,
+  updateCurrentViseme,
 }) => {
   const [currentBaseAction, setCurrentBaseAction] = useState({
     action: animation || 'Idle1',
@@ -77,6 +74,9 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
   const [morphTargetDictionary, setMorphTargetDictionary] = useState<{
     [key: string]: number;
   }>({});
+  const [emotionMorphTargets, setEmotionMorphTargets] = useState<{
+    [key: string]: number;
+  }>({});
 
   const [timeScale, setTimeScale] = useState(0.8);
 
@@ -93,7 +93,7 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
 
     //remove the last character from the action
     const newEmotion = action.slice(0, -1);
-    setEmotion(newEmotion);
+    // setEmotion(newEmotion);
 
     const defaultEmotions = Object.keys(emotionMap).reduce((acc, key) => {
       acc[key] = 0;
@@ -105,9 +105,8 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
     const emotionValues =
       emotion === 'default' ? defaultEmotions : emotionMap[emotion];
 
-    setMorphTargetInfluences(prevInfluences => ({
-      ...prevInfluences,
-      ...defaultEmotions,
+    setEmotionMorphTargets(prevEmotions => ({
+      ...prevEmotions,
       ...emotionValues,
     }));
   }, []);
@@ -162,6 +161,11 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
       const emotion = `${outputContent}${randomNumber}`;
 
       onBaseActionChange(emotion);
+    } else {
+      //Set a random idle animation
+      const randomNumber = Math.floor(Math.random() * 5) + 1;
+      const animation = `Idle${randomNumber === 3 ? 4 : randomNumber}`;
+      onBaseActionChange(animation);
     }
   }, [chatEmission]);
 
@@ -174,6 +178,7 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
     }
   }, [loading]);
 
+
   return (
     <>
       {showControls && (
@@ -191,13 +196,11 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
       {halfBody ? (
         <HalfBodyAvatar
           url={url}
-          setMeshRef={setMeshRef}
-          setMorphTargetInfluences={setMorphTargetInfluences}
           headMovement={headMovement}
           speaking={speaking}
           eyeBlink={eyeBlink}
           morphTargetInfluences={morphTargetInfluences}
-          clearVisemes={clearVisemes}
+          setMorphTargetInfluences={setMorphTargetInfluences}
           setMorphTargetDictionary={setMorphTargetDictionary}
         />
       ) : (
@@ -205,16 +208,14 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
           url={url}
           sex={sex}
           eyeBlink={eyeBlink}
-          speaking={speaking}
           currentBaseAction={currentBaseAction}
           timeScale={timeScale}
-          setMorphTargetInfluences={setMorphTargetInfluences}
-          setMorphTargetDictionary={setMorphTargetDictionary}
           morphTargetInfluences={morphTargetInfluences}
-          morphTargetDictionary={morphTargetDictionary}
           isZoomed={isZoomed}
-          setMeshRef={setMeshRef}
-          clearVisemes={clearVisemes}
+          updateCurrentViseme={updateCurrentViseme}
+          setMorphTargetDictionary={setMorphTargetDictionary}
+          setMorphTargetInfluences={setMorphTargetInfluences}
+          emotionMorphTargets={emotionMorphTargets}
         />
       )}
     </>

package/src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx

@@ -1,21 +1,17 @@
-import React, { useEffect, useRef, useState, useCallback } from 'react';
+import React, { useEffect, useRef, useState } from 'react';
 import {
   Vector3,
   Euler,
   AnimationMixer,
   SkinnedMesh,
   Object3D,
+  MathUtils,
   AnimationAction,
+  LoopOnce,
 } from 'three';
 import { useAnimations, useGLTF } from '@react-three/drei';
-import { useGraph, dispose, useFrame } from '@react-three/fiber';
+import { useGraph, useFrame } from '@react-three/fiber';
 import { correctMaterials, isSkinnedMesh } from '../../../../../helpers/utils';
-import { useAvatarBlink } from '../../utils/useEyeBlink';
-import { useViseme } from '../../../../../context/visemeContext';
-
-const lerp = (start: number, end: number, alpha: number): number => {
-  return start * (1 - alpha) + end * alpha;
-};
 
 interface FullbodyAvatarProps {
   url: string;
@@ -26,16 +22,21 @@ interface FullbodyAvatarProps {
     weight: number;
   };
   timeScale: number;
-  loading?: boolean;
-  speaking?: boolean;
   isZoomed?: boolean;
-  setMorphTargetInfluences: (influences: { [key: string]: number }) => void;
-  setMorphTargetDictionary: (dictionary: { [key: string]: number }) => void;
-  morphTargetInfluences: { [key: string]: number };
-  morphTargetDictionary: { [key: string]: number };
-  setMeshRef: any;
   eyeBlink?: boolean;
-  clearVisemes: () => void;
+  updateCurrentViseme: (
+    currentTime: number
+  ) => { name: string; weight: number } | null;
+  smoothMorphTarget?: boolean;
+  morphTargetSmoothing?: number;
+  morphTargetInfluences: Record<string, number>;
+  setMorphTargetDictionary: (
+    morphTargetDictionary: Record<string, number>
+  ) => void;
+  setMorphTargetInfluences: (
+    morphTargetInfluences: Record<string, number>
+  ) => void;
+  emotionMorphTargets: Record<string, number>;
 }
 
 const AVATAR_POSITION = new Vector3(0, -1, 0);
@@ -43,11 +44,19 @@ const AVATAR_ROTATION = new Euler(0.175, 0, 0);
 const AVATAR_POSITION_ZOOMED = new Vector3(0, -1.45, 0);
 
 const ANIMATION_URLS = {
-  MALE: 'https://assets.memori.ai/api/v2/asset/1c350a21-97d8-4add-82cc-9dc10767a26b.glb',
+  MALE: 'https://assets.memori.ai/api/v2/asset/2c5e88a4-cf62-408b-9ef0-518b099dfcb2.glb',
   FEMALE:
-    'https://assets.memori.ai/api/v2/asset/c2b07166-de10-4c66-918b-7b7cd380cca7.glb',
+    'https://assets.memori.ai/api/v2/asset/0e49aa5d-f757-4292-a170-d843c2839a41.glb',
+};
+
+// Blink configuration
+const BLINK_CONFIG = {
+  minInterval: 1000,
+  maxInterval: 5000,
+  blinkDuration: 150,
 };
-const ANIMATION_DURATION = 3000; // Duration in milliseconds for non-idle animations
+
+const EMOTION_TRANSITION_SPEED = 0.1; // Adjust this value to control emotion transition speed
 
 export default function FullbodyAvatar({
   url,
@@ -56,85 +65,65 @@ export default function FullbodyAvatar({
   currentBaseAction,
   timeScale,
   isZoomed,
-  setMorphTargetInfluences,
-  setMorphTargetDictionary,
-  morphTargetInfluences,
   eyeBlink,
-  setMeshRef,
-  clearVisemes,
+  morphTargetSmoothing = 0.5,
+  updateCurrentViseme,
+  setMorphTargetDictionary,
+  setMorphTargetInfluences,
+  emotionMorphTargets,
 }: FullbodyAvatarProps) {
   const { scene } = useGLTF(url);
   const { animations } = useGLTF(ANIMATION_URLS[sex]);
   const { nodes, materials } = useGraph(scene);
   const { actions } = useAnimations(animations, scene);
-  const [mixer] = useState(() => new AnimationMixer(scene));
 
-  const avatarMeshRef = useRef<SkinnedMesh>();
+  const mixer = useRef(new AnimationMixer(scene));
+  const headMeshRef = useRef<SkinnedMesh>();
   const currentActionRef = useRef<AnimationAction | null>(null);
-  const isTransitioningRef = useRef(false);
-   
-  // Blink animation
-  useAvatarBlink({
-    enabled: eyeBlink || false,
-    setMorphTargetInfluences,
-    config: {
-      minInterval: 1500,
-      maxInterval: 4000,
-      blinkDuration: 120,
-    },
-  });
+  const [isTransitioningToIdle, setIsTransitioningToIdle] = useState(false);
 
-  // Idle animation when emotion animation is finished
-  const transitionToIdle = useCallback(() => {
-    if (!actions || isTransitioningRef.current) return;
+  // Blink state
+  const lastBlinkTime = useRef(0);
+  const nextBlinkTime = useRef(0);
+  const isBlinking = useRef(false);
+  const blinkStartTime = useRef(0);
 
-    isTransitioningRef.current = true;
+  // Morph targets
+  const currentEmotionRef = useRef<Record<string, number>>({});
+  const previousEmotionKeysRef = useRef<Set<string>>(new Set());
 
-    const finishCurrentAnimation = () => {
-      if (currentActionRef.current && !currentActionRef.current.paused) {
-        const remainingTime = (currentActionRef.current.getClip().duration - currentActionRef.current.time) * 1000;
-        setTimeout(() => {
-          startIdleAnimation();
-        }, remainingTime);
-      } else {
-        startIdleAnimation();
-      }
-    };
-
-    const startIdleAnimation = () => {
-      const idleAnimations = Object.keys(actions).filter(key =>
-        key.startsWith('Idle')
-      );
-      const randomIdle =
-        idleAnimations[Math.floor(Math.random() * idleAnimations.length)];
+  useEffect(() => {
+    correctMaterials(materials);
 
-      const idleAction = actions[randomIdle];
-      const fadeOutDuration = 0.5;
-      const fadeInDuration = 0.5;
+    scene.traverse((object: Object3D) => {
+      if (object instanceof SkinnedMesh) {
+        if (object.name === 'GBNL__Head' || object.name === 'Wolf3D_Avatar') {
+          headMeshRef.current = object;
+          if (object.morphTargetDictionary && object.morphTargetInfluences) {
+            setMorphTargetDictionary(object.morphTargetDictionary);
 
-      if (currentActionRef.current) {
-        currentActionRef.current.fadeOut(fadeOutDuration);
+            const initialInfluences = Object.keys(
+              object.morphTargetDictionary
+            ).reduce((acc, key) => ({ ...acc, [key]: 0 }), {});
+            setMorphTargetInfluences(initialInfluences);
+          }
+        }
       }
+    });
 
-      idleAction?.reset().fadeIn(fadeInDuration).play();
-      currentActionRef.current = idleAction;
+    onLoaded?.();
 
-      setTimeout(() => {
-        isTransitioningRef.current = false;
-      }, (fadeOutDuration + fadeInDuration) * 1000);
+    return () => {
+      Object.values(materials).forEach(material => material.dispose());
+      Object.values(nodes)
+        .filter(isSkinnedMesh)
+        .forEach(mesh => mesh.geometry.dispose());
     };
+  }, [materials, nodes, url, onLoaded, scene]);
 
-    if (currentActionRef.current && !currentActionRef.current.getClip().name.startsWith('Idle')) {
-      finishCurrentAnimation();
-    } else {
-      startIdleAnimation();
-    }
-  }, [actions]);
-
-  // Base animation
+  // Handle base animation changes
   useEffect(() => {
-    if (!actions || !currentBaseAction.action || isTransitioningRef.current)
-      return;
+    if (!actions || !currentBaseAction.action) return;
 
     const newAction = actions[currentBaseAction.action];
     if (!newAction) {
@@ -147,79 +136,156 @@ export default function FullbodyAvatar({
     const fadeOutDuration = 0.8;
     const fadeInDuration = 0.8;
 
-    if (!currentBaseAction.action.startsWith('Idle')) {
-      setTimeout(() => {
-        transitionToIdle();
-      }, ANIMATION_DURATION);
-    }
-
     if (currentActionRef.current) {
       currentActionRef.current.fadeOut(fadeOutDuration);
     }
-
-    newAction.timeScale = timeScale;
+    
+    console.log(newAction);
     newAction.reset().fadeIn(fadeInDuration).play();
     currentActionRef.current = newAction;
-  }, [currentBaseAction, timeScale, actions, transitionToIdle]);
 
-  // Set up the mesh reference and morph target influences
-  useEffect(() => {
-    correctMaterials(materials);
+    // Set the time scale for the new action
+    newAction.timeScale = timeScale;
 
-    scene.traverse((object: Object3D) => {
-      if (
-        object instanceof SkinnedMesh &&
-        (object.name === 'GBNL__Head' || object.name === 'Wolf3D_Avatar')
-      ) {
-        avatarMeshRef.current = object;
-        setMeshRef(object);
-
-        if (object.morphTargetDictionary && object.morphTargetInfluences) {
-          setMorphTargetDictionary(object.morphTargetDictionary);
-
-          const initialInfluences = Object.keys(
-            object.morphTargetDictionary
-          ).reduce((acc, key) => ({ ...acc, [key]: 0 }), {});
-          setMorphTargetInfluences(initialInfluences);
+    // If it's an emotion animation, set it to play once and then transition to idle
+    if (
+      currentBaseAction.action.startsWith('Gioia') ||
+      currentBaseAction.action.startsWith('Rabbia') ||
+      currentBaseAction.action.startsWith('Sorpresa') ||
+      currentBaseAction.action.startsWith('Timore') ||
+      currentBaseAction.action.startsWith('Tristezza')
+    ) {
+      newAction.setLoop(LoopOnce, 1);
+      newAction.clampWhenFinished = true;
+      setIsTransitioningToIdle(true);
+    }
+  }, [actions, currentBaseAction, timeScale]);
+
+  useFrame(state => {
+    if (
+      headMeshRef.current &&
+      headMeshRef.current.morphTargetDictionary &&
+      headMeshRef.current.morphTargetInfluences
+    ) {
+      const currentTime = state.clock.getElapsedTime() * 1000; // Convert to milliseconds
+
+      // Handle blinking
+      let blinkValue = 0;
+      if (eyeBlink) {
+        if (currentTime >= nextBlinkTime.current && !isBlinking.current) {
+          isBlinking.current = true;
+          blinkStartTime.current = currentTime;
+          lastBlinkTime.current = currentTime;
+          nextBlinkTime.current =
+            currentTime +
+            Math.random() *
+              (BLINK_CONFIG.maxInterval - BLINK_CONFIG.minInterval) +
+            BLINK_CONFIG.minInterval;
+        }
+
+        if (isBlinking.current) {
+          const blinkProgress =
+            (currentTime - blinkStartTime.current) / BLINK_CONFIG.blinkDuration;
+          if (blinkProgress <= 0.5) {
+            // Eyes closing
+            blinkValue = blinkProgress * 2;
+          } else if (blinkProgress <= 1) {
+            // Eyes opening
+            blinkValue = 2 - blinkProgress * 2;
+          } else {
+            // Blink finished
+            isBlinking.current = false;
+            blinkValue = 0;
+          }
         }
       }
-    });
 
-    onLoaded?.();
+      const currentViseme = updateCurrentViseme(currentTime / 1000);
 
-    return () => {
-      Object.values(materials).forEach(dispose);
-      Object.values(nodes).filter(isSkinnedMesh).forEach(dispose);
-      clearVisemes();
-    };
-  }, [
-    materials,
-    nodes,
-    url,
-    onLoaded,
-    setMorphTargetDictionary,
-    setMorphTargetInfluences,
-    setMeshRef,
-    clearVisemes,
-  ]);
-
-  // Update morph target influences
-  useFrame((_, delta) => {
-    if (avatarMeshRef.current && avatarMeshRef.current.morphTargetDictionary) {
-      updateMorphTargetInfluences();
-    }
-    mixer.update(delta * 0.001);
-
-    function updateMorphTargetInfluences() {
-      Object.entries(morphTargetInfluences).forEach(([key, value]) => {
-        const index = avatarMeshRef.current!.morphTargetDictionary![key];
-        if (typeof index === 'number' &&
-          avatarMeshRef.current!.morphTargetInfluences) {
-          const currentValue = avatarMeshRef.current!.morphTargetInfluences[index];
-          const smoothValue = lerp(currentValue, value, 0.1);
-          avatarMeshRef.current!.morphTargetInfluences[index] = smoothValue;
+      // Create a set of current emotion keys
+      const currentEmotionKeys = new Set(Object.keys(emotionMorphTargets));
+
+      // Reset old emotion morph targets
+      previousEmotionKeysRef.current.forEach(key => {
+        if (!currentEmotionKeys.has(key)) {
+          const index = headMeshRef.current!.morphTargetDictionary![key];
+          if (typeof index === 'number') {
+            currentEmotionRef.current[key] = 0;
+            if (headMeshRef.current && headMeshRef.current.morphTargetInfluences) {
+              headMeshRef.current.morphTargetInfluences[index] = 0;
+            }
+          }
         }
       });
+
+      // Update morph targets
+      Object.entries(headMeshRef.current.morphTargetDictionary).forEach(
+        ([key, index]) => {
+          if (typeof index === 'number') {
+            let targetValue = 0;
+
+            // Handle emotions (base layer)
+            if (Object.prototype.hasOwnProperty.call(emotionMorphTargets, key)) {
+              const targetEmotionValue = emotionMorphTargets[key];
+              const currentEmotionValue = currentEmotionRef.current[key] || 0;
+              const newEmotionValue = MathUtils.lerp(
+                currentEmotionValue,
+                targetEmotionValue * 2,
+                EMOTION_TRANSITION_SPEED
+              );
+              currentEmotionRef.current[key] = newEmotionValue;
+              targetValue += newEmotionValue;
+            }
+
+            // Handle visemes (additive layer)
+            if (currentViseme && key === currentViseme.name) {
+              targetValue += currentViseme.weight * 1.2; // Amplify the effect
+            }
+
+            // Handle blinking (additive layer, only for 'eyesClosed')
+            if (key === 'eyesClosed' && eyeBlink) {
+              targetValue += blinkValue;
+            }
+
+            // Clamp the final value between 0 and 1
+            targetValue = MathUtils.clamp(targetValue, 0, 1);
+
+            // Apply smoothing
+            if (headMeshRef.current && headMeshRef.current.morphTargetInfluences) {
+              headMeshRef.current.morphTargetInfluences[index] = MathUtils.lerp(
+                headMeshRef.current.morphTargetInfluences[index],
+                targetValue,
+                morphTargetSmoothing
+              );
+            }
+          }
+        }
+      );
+
+      // Update the set of previous emotion keys for the next frame
+      previousEmotionKeysRef.current = currentEmotionKeys;
+
+      // Handle transition from emotion animation to idle
+      if (isTransitioningToIdle && currentActionRef.current) {
+        if (
+          currentActionRef.current.time >=
+          currentActionRef.current.getClip().duration
+        ) {
+          // Transition to the idle animation
+          const idleNumber = Math.floor(Math.random() * 5) + 1; // Randomly choose 1, 2, 3, 4 or 5
+          const idleAction = actions[`Idle${idleNumber == 3 ? 4 : idleNumber}`];
+
+          if (idleAction) {
+            currentActionRef.current.fadeOut(0.5);
+            idleAction.reset().fadeIn(0.5).play();
+            currentActionRef.current = idleAction;
+            setIsTransitioningToIdle(false);
+          }
+        }
+      }
+
+      // Update the animation mixer
+      mixer.current.update(0.01); // Fixed delta time for consistent animation speed
     }
   });
 
@@ -231,4 +297,4 @@ export default function FullbodyAvatar({
       <primitive object={scene} />
     </group>
   );
-}
+}

package/src/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.tsx

@@ -14,8 +14,6 @@ interface HalfBodyAvatarProps {
   headMovement?: boolean;
   speaking?: boolean;
   onLoaded?: () => void;
-  setMeshRef: (mesh: Object3D) => void;
-  clearVisemes: () => void;
   setMorphTargetDictionary: (morphTargetDictionary: any) => void;
   eyeBlink?: boolean;
   morphTargetInfluences: any;
@@ -31,9 +29,7 @@ export default function HalfBodyAvatar({
   setMorphTargetDictionary,
   headMovement,
   eyeBlink,
-  setMeshRef,
   onLoaded,
-  clearVisemes,
   morphTargetInfluences,
 }: HalfBodyAvatarProps) {
   const { scene } = useGLTF(url);
@@ -58,7 +54,6 @@ export default function HalfBodyAvatar({
       // Set mesh reference for the first SkinnedMesh found
       const firstSkinnedMesh = Object.values(nodes).find(isSkinnedMesh) as SkinnedMesh;
       if (firstSkinnedMesh) {
-        setMeshRef(firstSkinnedMesh);
         avatarMeshRef.current = firstSkinnedMesh;
         if (firstSkinnedMesh.morphTargetDictionary && firstSkinnedMesh.morphTargetInfluences) {
           setMorphTargetDictionary(firstSkinnedMesh.morphTargetDictionary);
@@ -77,12 +72,11 @@ export default function HalfBodyAvatar({
       const disposeObjects = () => {
         Object.values(materials).forEach(dispose);
         Object.values(nodes).filter(isSkinnedMesh).forEach(dispose);
-        clearVisemes();
       };
 
       disposeObjects();
     };
-  }, [materials, nodes, url, onLoaded, clearVisemes]);
+  }, [materials, nodes, url, onLoaded]);
 
   const skinnedMeshes = useMemo(
     () => Object.values(nodes).filter(isSkinnedMesh),