@memori.ai/memori-react 7.5.1 → 7.6.0

package/src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx

@@ -0,0 +1,234 @@
+import React, { useEffect, useRef, useState, useCallback } from 'react';
+import {
+  Vector3,
+  Euler,
+  AnimationMixer,
+  SkinnedMesh,
+  Object3D,
+  AnimationAction,
+} from 'three';
+import { useAnimations, useGLTF } from '@react-three/drei';
+import { useGraph, dispose, useFrame } from '@react-three/fiber';
+import { correctMaterials, isSkinnedMesh } from '../../../../../helpers/utils';
+import { useAvatarBlink } from '../../utils/useEyeBlink';
+import { useViseme } from '../../../../../context/visemeContext';
+
+const lerp = (start: number, end: number, alpha: number): number => {
+  return start * (1 - alpha) + end * alpha;
+};
+
+interface FullbodyAvatarProps {
+  url: string;
+  sex: 'MALE' | 'FEMALE';
+  onLoaded?: () => void;
+  currentBaseAction: {
+    action: string;
+    weight: number;
+  };
+  timeScale: number;
+  loading?: boolean;
+  speaking?: boolean;
+  isZoomed?: boolean;
+  setMorphTargetInfluences: (influences: { [key: string]: number }) => void;
+  setMorphTargetDictionary: (dictionary: { [key: string]: number }) => void;
+  morphTargetInfluences: { [key: string]: number };
+  morphTargetDictionary: { [key: string]: number };
+  setMeshRef: any;
+  eyeBlink?: boolean;
+  clearVisemes: () => void;
+}
+
+const AVATAR_POSITION = new Vector3(0, -1, 0);
+const AVATAR_ROTATION = new Euler(0.175, 0, 0);
+const AVATAR_POSITION_ZOOMED = new Vector3(0, -1.45, 0);
+
+const ANIMATION_URLS = {
+  MALE: 'https://assets.memori.ai/api/v2/asset/1c350a21-97d8-4add-82cc-9dc10767a26b.glb',
+  FEMALE:
+    'https://assets.memori.ai/api/v2/asset/c2b07166-de10-4c66-918b-7b7cd380cca7.glb',
+};
+const ANIMATION_DURATION = 3000; // Duration in milliseconds for non-idle animations
+
+export default function FullbodyAvatar({
+  url,
+  sex,
+  onLoaded,
+  currentBaseAction,
+  timeScale,
+  isZoomed,
+  setMorphTargetInfluences,
+  setMorphTargetDictionary,
+  morphTargetInfluences,
+  eyeBlink,
+  setMeshRef,
+  clearVisemes,
+}: FullbodyAvatarProps) {
+  const { scene } = useGLTF(url);
+  const { animations } = useGLTF(ANIMATION_URLS[sex]);
+  const { nodes, materials } = useGraph(scene);
+  const { actions } = useAnimations(animations, scene);
+  const [mixer] = useState(() => new AnimationMixer(scene));
+
+  const avatarMeshRef = useRef<SkinnedMesh>();
+  const currentActionRef = useRef<AnimationAction | null>(null);
+  const isTransitioningRef = useRef(false);
+   
+  // Blink animation
+  useAvatarBlink({
+    enabled: eyeBlink || false,
+    setMorphTargetInfluences,
+    config: {
+      minInterval: 1500,
+      maxInterval: 4000,
+      blinkDuration: 120,
+    },
+  });
+
+  // Idle animation when emotion animation is finished
+  const transitionToIdle = useCallback(() => {
+    if (!actions || isTransitioningRef.current) return;
+
+    isTransitioningRef.current = true;
+
+    const finishCurrentAnimation = () => {
+      if (currentActionRef.current && !currentActionRef.current.paused) {
+        const remainingTime = (currentActionRef.current.getClip().duration - currentActionRef.current.time) * 1000;
+        setTimeout(() => {
+          startIdleAnimation();
+        }, remainingTime);
+      } else {
+        startIdleAnimation();
+      }
+    };
+
+    const startIdleAnimation = () => {
+      const idleAnimations = Object.keys(actions).filter(key =>
+        key.startsWith('Idle')
+      );
+      const randomIdle =
+        idleAnimations[Math.floor(Math.random() * idleAnimations.length)];
+
+      const idleAction = actions[randomIdle];
+      const fadeOutDuration = 0.5;
+      const fadeInDuration = 0.5;
+
+      if (currentActionRef.current) {
+        currentActionRef.current.fadeOut(fadeOutDuration);
+      }
+
+      idleAction?.reset().fadeIn(fadeInDuration).play();
+      currentActionRef.current = idleAction;
+
+      setTimeout(() => {
+        isTransitioningRef.current = false;
+      }, (fadeOutDuration + fadeInDuration) * 1000);
+    };
+
+    if (currentActionRef.current && !currentActionRef.current.getClip().name.startsWith('Idle')) {
+      finishCurrentAnimation();
+    } else {
+      startIdleAnimation();
+    }
+  }, [actions]);
+
+  // Base animation
+  useEffect(() => {
+    if (!actions || !currentBaseAction.action || isTransitioningRef.current)
+      return;
+
+    const newAction = actions[currentBaseAction.action];
+    if (!newAction) {
+      console.warn(
+        `Animation "${currentBaseAction.action}" not found in actions.`
+      );
+      return;
+    }
+
+    const fadeOutDuration = 0.8;
+    const fadeInDuration = 0.8;
+
+    if (!currentBaseAction.action.startsWith('Idle')) {
+      setTimeout(() => {
+        transitionToIdle();
+      }, ANIMATION_DURATION);
+    }
+
+    if (currentActionRef.current) {
+      currentActionRef.current.fadeOut(fadeOutDuration);
+    }
+
+    newAction.timeScale = timeScale;
+    newAction.reset().fadeIn(fadeInDuration).play();
+    currentActionRef.current = newAction;
+  }, [currentBaseAction, timeScale, actions, transitionToIdle]);
+
+  // Set up the mesh reference and morph target influences
+  useEffect(() => {
+    correctMaterials(materials);
+
+    scene.traverse((object: Object3D) => {
+      if (
+        object instanceof SkinnedMesh &&
+        (object.name === 'GBNL__Head' || object.name === 'Wolf3D_Avatar')
+      ) {
+        avatarMeshRef.current = object;
+        setMeshRef(object);
+
+        if (object.morphTargetDictionary && object.morphTargetInfluences) {
+          setMorphTargetDictionary(object.morphTargetDictionary);
+
+          const initialInfluences = Object.keys(
+            object.morphTargetDictionary
+          ).reduce((acc, key) => ({ ...acc, [key]: 0 }), {});
+          setMorphTargetInfluences(initialInfluences);
+        }
+      }
+    });
+
+    onLoaded?.();
+
+    return () => {
+      Object.values(materials).forEach(dispose);
+      Object.values(nodes).filter(isSkinnedMesh).forEach(dispose);
+      clearVisemes();
+    };
+  }, [
+    materials,
+    nodes,
+    url,
+    onLoaded,
+    setMorphTargetDictionary,
+    setMorphTargetInfluences,
+    setMeshRef,
+    clearVisemes,
+  ]);
+
+  // Update morph target influences
+  useFrame((_, delta) => {
+    if (avatarMeshRef.current && avatarMeshRef.current.morphTargetDictionary) {
+      updateMorphTargetInfluences();
+    }
+    mixer.update(delta * 0.001);
+
+    function updateMorphTargetInfluences() {
+      Object.entries(morphTargetInfluences).forEach(([key, value]) => {
+        const index = avatarMeshRef.current!.morphTargetDictionary![key];
+        if (typeof index === 'number' &&
+          avatarMeshRef.current!.morphTargetInfluences) {
+          const currentValue = avatarMeshRef.current!.morphTargetInfluences[index];
+          const smoothValue = lerp(currentValue, value, 0.1);
+          avatarMeshRef.current!.morphTargetInfluences[index] = smoothValue;
+        }
+      });
+    }
+  });
+
+  return (
+    <group
+      position={isZoomed ? AVATAR_POSITION_ZOOMED : AVATAR_POSITION}
+      rotation={AVATAR_ROTATION}
+    >
+      <primitive object={scene} />
+    </group>
+  );
+}

package/src/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.tsx

@@ -0,0 +1,123 @@
+import React, { useEffect, useMemo, useRef } from 'react';
+import { Object3D, SkinnedMesh, Vector3 } from 'three';
+import { useGLTF } from '@react-three/drei';
+import { correctMaterials, isSkinnedMesh } from '../../../../../helpers/utils';
+import { useGraph, dispose, useFrame } from '@react-three/fiber';
+import { useAvatarBlink } from '../../utils/useEyeBlink';
+import useHeadMovement from '../../utils/useHeadMovement';
+import { hideHands } from '../../utils/utils';
+
+
+interface HalfBodyAvatarProps {
+  url: string;
+  setMorphTargetInfluences: (morphTargetInfluences: any) => void;
+  headMovement?: boolean;
+  speaking?: boolean;
+  onLoaded?: () => void;
+  setMeshRef: (mesh: Object3D) => void;
+  clearVisemes: () => void;
+  setMorphTargetDictionary: (morphTargetDictionary: any) => void;
+  eyeBlink?: boolean;
+  morphTargetInfluences: any;
+}
+
+const AVATAR_POSITION = new Vector3(0, -0.6, 0);
+const lerp = (start: number, end: number, alpha: number): number => {
+  return start * (1 - alpha) + end * alpha;
+};
+export default function HalfBodyAvatar({
+  url,
+  setMorphTargetInfluences,
+  setMorphTargetDictionary,
+  headMovement,
+  eyeBlink,
+  setMeshRef,
+  onLoaded,
+  clearVisemes,
+  morphTargetInfluences,
+}: HalfBodyAvatarProps) {
+  const { scene } = useGLTF(url);
+  const { nodes, materials } = useGraph(scene);
+  const avatarMeshRef = useRef<SkinnedMesh | null>(null);
+
+  useAvatarBlink({
+    enabled: eyeBlink || false,
+    setMorphTargetInfluences,
+    config: {
+      minInterval: 1500,
+      maxInterval: 4000,
+      blinkDuration: 120
+    }
+  });
+  useHeadMovement(headMovement, nodes);
+
+  useEffect(() => {
+    const setupAvatar = () => {
+      hideHands(nodes);
+      correctMaterials(materials);
+      // Set mesh reference for the first SkinnedMesh found
+      const firstSkinnedMesh = Object.values(nodes).find(isSkinnedMesh) as SkinnedMesh;
+      if (firstSkinnedMesh) {
+        setMeshRef(firstSkinnedMesh);
+        avatarMeshRef.current = firstSkinnedMesh;
+        if (firstSkinnedMesh.morphTargetDictionary && firstSkinnedMesh.morphTargetInfluences) {
+          setMorphTargetDictionary(firstSkinnedMesh.morphTargetDictionary);
+          const initialInfluences = Object.keys(
+            firstSkinnedMesh.morphTargetDictionary
+          ).reduce((acc, key) => ({ ...acc, [key]: 0 }), {});
+          setMorphTargetInfluences(initialInfluences);
+        }
+      }
+      onLoaded?.();
+    };
+
+    setupAvatar();
+
+    return () => {
+      const disposeObjects = () => {
+        Object.values(materials).forEach(dispose);
+        Object.values(nodes).filter(isSkinnedMesh).forEach(dispose);
+        clearVisemes();
+      };
+
+      disposeObjects();
+    };
+  }, [materials, nodes, url, onLoaded, clearVisemes]);
+
+  const skinnedMeshes = useMemo(
+    () => Object.values(nodes).filter(isSkinnedMesh),
+    [nodes]
+  );
+
+    // Update morph target influences
+    useFrame((_) => {
+      if (avatarMeshRef.current && avatarMeshRef.current.morphTargetDictionary) {
+        updateMorphTargetInfluences();
+      }
+  
+      function updateMorphTargetInfluences() {
+        Object.entries(morphTargetInfluences).forEach(([key, value]) => {
+          const index = avatarMeshRef.current!.morphTargetDictionary![key];
+          if (typeof index === 'number' &&
+            avatarMeshRef.current!.morphTargetInfluences) {
+            const currentValue = avatarMeshRef.current!.morphTargetInfluences[index];
+            const smoothValue = lerp(currentValue, value as number, 0.1);
+            avatarMeshRef.current!.morphTargetInfluences[index] = smoothValue;
+          }
+        });
+      }
+  });
+
+
+  return (
+    <group position={AVATAR_POSITION}>
+      {nodes.Hips && <primitive key="armature" object={nodes.Hips} />}
+      {skinnedMeshes.map(
+        (node: Object3D) =>
+          node && (
+            <primitive key={node.name} object={node} receiveShadow castShadow />
+          )
+      )}
+    </group>
+  );
+}

package/src/components/Avatar/AvatarView/{components → AvatarComponent/components}/loader.tsx

@@ -1,6 +1,6 @@
 import React from 'react';
 import { Html, useProgress } from '@react-three/drei';
-import Spin from '../../../ui/Spin';
+import Spin from '../../../../ui/Spin';
 
 interface Props {
   fallbackImg?: string;

package/src/components/Avatar/AvatarView/AvatarView.stories.tsx

@@ -2,6 +2,7 @@ import React from 'react';
 import { Meta, Story } from '@storybook/react';
 import I18nWrapper from '../../../I18nWrapper';
 import AvatarView, { Props } from './index';
+import { VisemeProvider } from '../../../context/visemeContext';
 
 const meta: Meta = {
   title: 'RPM 3D Avatar',
@@ -60,9 +61,7 @@ const meta: Meta = {
     },
   },
   parameters: {
-    controls: { expanded: false,
-    },
-    
+    controls: { expanded: false },
   },
 };
 
@@ -76,11 +75,13 @@ const Template: Story<Props> = args => {
 
   return hydrated ? (
     <I18nWrapper>
-      <AvatarView
-        {...args}
-        url={args.url + `#${new Date(Date.now()).toISOString()}`}
-        key={Date.now()}
-      />
+      <VisemeProvider>
+        <AvatarView
+          {...args}
+          url={args.url + `#${new Date(Date.now()).toISOString()}`}
+          key={Date.now()}
+        />
+      </VisemeProvider>
     </I18nWrapper>
   ) : (
     <></>
@@ -95,6 +96,8 @@ Default.args = {
   headMovement: false,
   rotateAvatar: false,
   speaking: false,
+  clearVisemes: () => {},
+  setMeshRef: () => {},
   url: 'https://assets.memori.ai/api/v2/asset/b791f77c-1a94-4272-829e-eca82fcc62b7.glb',
   fallbackImg:
     'https://assets.memori.ai/api/v2/asset/d8035229-08cf-42a7-a532-ab051df2603d.png',
@@ -102,6 +105,8 @@ Default.args = {
 
 export const EyeBlink = Template.bind({});
 EyeBlink.args = {
+  clearVisemes: () => {},
+  setMeshRef: () => {},
   eyeBlink: true,
   headMovement: false,
   rotateAvatar: false,
@@ -113,6 +118,8 @@ EyeBlink.args = {
 
 export const HeadMovement = Template.bind({});
 HeadMovement.args = {
+  clearVisemes: () => {},
+  setMeshRef: () => {},
   eyeBlink: false,
   headMovement: true,
   rotateAvatar: false,
@@ -124,6 +131,8 @@ HeadMovement.args = {
 
 export const RotateAvatar = Template.bind({});
 RotateAvatar.args = {
+  clearVisemes: () => {},
+  setMeshRef: () => {},
   eyeBlink: false,
   headMovement: false,
   rotateAvatar: true,
@@ -135,6 +144,8 @@ RotateAvatar.args = {
 
 export const Speaking = Template.bind({});
 Speaking.args = {
+  clearVisemes: () => {},
+  setMeshRef: () => {},
   eyeBlink: false,
   headMovement: false,
   rotateAvatar: false,
@@ -151,12 +162,30 @@ Fullbody.args = {
   headMovement: true,
   rotateAvatar: true,
   speaking: false,
+  clearVisemes: () => {},
+  setMeshRef: () => {},
   url: 'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb',
   fallbackImg:
     'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
   halfBody: false,
 };
 
+export const FullbodyZoomed = Template.bind({});
+FullbodyZoomed.args = {
+  sex: 'FEMALE',
+  eyeBlink: true,
+  headMovement: true,
+  rotateAvatar: true,
+  speaking: false,
+  isZoomed: true,
+  clearVisemes: () => {},
+  setMeshRef: () => {},
+  url: 'https://assets.memori.ai/api/v2/asset/3f5ef41c-6c4c-449c-888d-cf9c89782528.glb',
+  fallbackImg:
+    'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
+  halfBody: false,
+};
+
 export const FullbodyAnimatedIdle = Template.bind({});
 FullbodyAnimatedIdle.args = {
   sex: 'MALE',
@@ -164,6 +193,8 @@ FullbodyAnimatedIdle.args = {
   headMovement: true,
   rotateAvatar: true,
   speaking: false,
+  clearVisemes: () => {},
+  setMeshRef: () => {},
   url: 'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb',
   fallbackImg:
     'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
@@ -178,6 +209,8 @@ FullbodyAnimatedLoading.args = {
   headMovement: true,
   rotateAvatar: true,
   speaking: false,
+  clearVisemes: () => {},
+  setMeshRef: () => {},
   url: 'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb',
   fallbackImg:
     'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
@@ -192,6 +225,8 @@ FullbodyAnimatedSpeaking.args = {
   headMovement: true,
   rotateAvatar: true,
   speaking: true,
+  clearVisemes: () => {},
+  setMeshRef: () => {},
   url: 'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb',
   fallbackImg:
     'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
@@ -206,6 +241,8 @@ FullbodyFemale.args = {
   headMovement: true,
   rotateAvatar: true,
   speaking: false,
+  clearVisemes: () => {},
+  setMeshRef: () => {},
   url: 'https://models.readyplayer.me/650d50c2663b19e0d2831b2b.glb',
   fallbackImg:
     'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
@@ -219,6 +256,8 @@ FullbodyAnimatedFemale.args = {
   headMovement: true,
   rotateAvatar: true,
   speaking: true,
+  clearVisemes: () => {},
+  setMeshRef: () => {},
   url: 'https://models.readyplayer.me/650d50c2663b19e0d2831b2b.glb',
   fallbackImg:
     'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',