@memori.ai/memori-react 6.8.1 → 6.8.4

package/src/components/{AvatarView → Avatar/AvatarView}/AvatarView.stories.tsx

@@ -1,7 +1,8 @@
 import React from 'react';
 import { Meta, Story } from '@storybook/react';
-import I18nWrapper from '../../I18nWrapper';
+import I18nWrapper from '../../../I18nWrapper';
 import AvatarView, { Props } from './index';
+import { VisemeProvider } from '../../../context/visemeContext';
 
 const meta: Meta = {
   title: 'RPM 3D Avatar',
@@ -37,6 +38,11 @@ const meta: Meta = {
         type: 'boolean',
       },
     },
+    showControls: {
+      control: {
+        type: 'boolean',
+      },
+    },
     animation: {
       control: {
         type: 'select',
@@ -55,7 +61,7 @@ const meta: Meta = {
     },
   },
   parameters: {
-    controls: { expanded: true },
+    controls: { expanded: false },
   },
 };
 
@@ -69,11 +75,13 @@ const Template: Story<Props> = args => {
 
   return hydrated ? (
     <I18nWrapper>
-      <AvatarView
-        {...args}
-        url={args.url + `#${new Date(Date.now()).toISOString()}`}
-        key={Date.now()}
-      />
+      <VisemeProvider>
+        <AvatarView
+          {...args}
+          url={args.url + `#${new Date(Date.now()).toISOString()}`}
+          key={Date.now()}
+        />
+      </VisemeProvider>
     </I18nWrapper>
   ) : (
     <></>
@@ -88,6 +96,7 @@ Default.args = {
   headMovement: false,
   rotateAvatar: false,
   speaking: false,
+  updateCurrentViseme: () => {},
   url: 'https://assets.memori.ai/api/v2/asset/b791f77c-1a94-4272-829e-eca82fcc62b7.glb',
   fallbackImg:
     'https://assets.memori.ai/api/v2/asset/d8035229-08cf-42a7-a532-ab051df2603d.png',
@@ -95,6 +104,7 @@ Default.args = {
 
 export const EyeBlink = Template.bind({});
 EyeBlink.args = {
+  updateCurrentViseme: () => {},
   eyeBlink: true,
   headMovement: false,
   rotateAvatar: false,
@@ -106,6 +116,7 @@ EyeBlink.args = {
 
 export const HeadMovement = Template.bind({});
 HeadMovement.args = {
+  updateCurrentViseme: () => {},
   eyeBlink: false,
   headMovement: true,
   rotateAvatar: false,
@@ -117,6 +128,7 @@ HeadMovement.args = {
 
 export const RotateAvatar = Template.bind({});
 RotateAvatar.args = {
+  updateCurrentViseme: () => {},
   eyeBlink: false,
   headMovement: false,
   rotateAvatar: true,
@@ -128,6 +140,7 @@ RotateAvatar.args = {
 
 export const Speaking = Template.bind({});
 Speaking.args = {
+  updateCurrentViseme: () => {},
   eyeBlink: false,
   headMovement: false,
   rotateAvatar: false,
@@ -144,12 +157,28 @@ Fullbody.args = {
   headMovement: true,
   rotateAvatar: true,
   speaking: false,
+  updateCurrentViseme: () => {},
   url: 'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb',
   fallbackImg:
     'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
   halfBody: false,
 };
 
+export const FullbodyZoomed = Template.bind({});
+FullbodyZoomed.args = {
+  sex: 'FEMALE',
+  eyeBlink: true,
+  headMovement: true,
+  rotateAvatar: true,
+  speaking: false,
+  isZoomed: true,
+  updateCurrentViseme: () => {},
+  url: 'https://assets.memori.ai/api/v2/asset/3f5ef41c-6c4c-449c-888d-cf9c89782528.glb',
+  fallbackImg:
+    'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
+  halfBody: false,
+};
+
 export const FullbodyAnimatedIdle = Template.bind({});
 FullbodyAnimatedIdle.args = {
   sex: 'MALE',
@@ -157,6 +186,7 @@ FullbodyAnimatedIdle.args = {
   headMovement: true,
   rotateAvatar: true,
   speaking: false,
+  updateCurrentViseme: () => {},
   url: 'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb',
   fallbackImg:
     'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
@@ -171,6 +201,7 @@ FullbodyAnimatedLoading.args = {
   headMovement: true,
   rotateAvatar: true,
   speaking: false,
+  updateCurrentViseme: () => {},
   url: 'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb',
   fallbackImg:
     'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
@@ -185,6 +216,7 @@ FullbodyAnimatedSpeaking.args = {
   headMovement: true,
   rotateAvatar: true,
   speaking: true,
+  updateCurrentViseme: () => {},
   url: 'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb',
   fallbackImg:
     'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
@@ -199,6 +231,7 @@ FullbodyFemale.args = {
   headMovement: true,
   rotateAvatar: true,
   speaking: false,
+  updateCurrentViseme: () => {},
   url: 'https://models.readyplayer.me/650d50c2663b19e0d2831b2b.glb',
   fallbackImg:
     'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
@@ -212,6 +245,7 @@ FullbodyAnimatedFemale.args = {
   headMovement: true,
   rotateAvatar: true,
   speaking: true,
+  updateCurrentViseme: () => {},
   url: 'https://models.readyplayer.me/650d50c2663b19e0d2831b2b.glb',
   fallbackImg:
     'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',

package/src/components/Avatar/AvatarView/index.tsx

@@ -0,0 +1,194 @@
+import { CSSProperties, useState } from 'react';
+import React, { Suspense } from 'react';
+import { Canvas } from '@react-three/fiber';
+import {
+  OrbitControls,
+  SpotLight,
+  Environment,
+  PerspectiveCamera,
+} from '@react-three/drei';
+import { isAndroid, isiOS } from '../../../helpers/utils';
+import { AvatarView } from './AvatarComponent/avatarComponent';
+import PositionControls from './AvatarComponent/positionControls/positionControls';
+import { getLocalConfig } from '../../../helpers/configuration';
+import Loader from './AvatarComponent/components/loader';
+import Lights from './AvatarComponent/lights/Lights';
+
+export interface Props {
+  url: string;
+  sex: 'MALE' | 'FEMALE';
+  fallbackImg?: string;
+  eyeBlink?: boolean;
+  headMovement?: boolean;
+  rotateAvatar?: boolean;
+  speaking?: boolean;
+  style?: CSSProperties;
+  fallback?: React.ReactNode;
+  halfBody?: boolean;
+  loading?: boolean;
+  animation?: string;
+  showControls?: boolean;
+  isZoomed?: boolean;
+  chatEmission?: any;
+  enablePositionControls?: boolean;
+  setEnablePositionControls: (value: boolean) => void;
+  isTotem?: boolean;
+  setMeshRef?: any;
+  stopProcessing: () => void;
+  resetVisemeQueue: () => void;
+  updateCurrentViseme: (
+    currentTime: number
+  ) => { name: string; weight: number } | null;
+}
+
+const defaultStyles = {
+  halfBody: {
+    width: '100%',
+    height: '100%',
+    minHeight: '500px', // Ensure minimum height
+    backgroundColor: 'white',
+    borderRadius: '100%',
+  },
+  fullBody: {
+    width: '500px',
+    height: '500px',
+    backgroundColor: 'white',
+  },
+};
+
+
+const getCameraSettings = (halfBody: boolean, isZoomed: boolean) => {
+  const baseZ = halfBody ? 0.6 : 3;
+
+  return {
+    fov: isZoomed ? 44 : 40,
+    position: [0, 0, baseZ],
+    target: [0, 0, 0],
+  };
+};
+
+export default function ContainerAvatarView({
+  url,
+  sex,
+  style,
+  rotateAvatar,
+  eyeBlink,
+  headMovement,
+  speaking,
+  fallback,
+  fallbackImg,
+  halfBody = true,
+  loading,
+  // animation,
+  showControls = false,
+  isZoomed,
+  chatEmission,
+  updateCurrentViseme,
+  enablePositionControls,
+  setEnablePositionControls,
+  isTotem = false,
+
+}: Props) {
+  const [cameraZ, setCameraZ] = useState(
+    () => getCameraSettings(halfBody, isZoomed || false).position[2]
+  );
+
+  const getAvatarHeight = () => {
+    // For totem avatars, use stored config value or default to 50
+    if (isTotem) {
+      const height = getLocalConfig('avatarHeight', 50);
+      return height;
+    }
+
+    // For half body avatars, use fixed height of 100
+    if (halfBody) {
+      return 80;
+    }
+
+    // For full body avatars, adjust height based on zoom
+    return isZoomed ? 16 : 65;
+  };
+
+  const getAvatarDepth = () => {
+    // For totem avatars, use stored config value or default to 50
+    if (isTotem) {
+      const depth = getLocalConfig('avatarDepth', 50);
+      return depth;
+    }
+
+    // For half body avatars, use fixed depth of 50
+    if (halfBody) {
+      return 50;
+    }
+
+    // For full body avatars, adjust depth based on zoom
+    return isZoomed ? 5 : 100;
+  };
+
+  const [avatarHeight, setAvatarHeight] = useState(getAvatarHeight());
+  const [avatarDepth, setAvatarDepth] = useState(getAvatarDepth());
+
+  return (
+    <>
+          {enablePositionControls && (
+        <PositionControls
+          avatarHeight={avatarHeight}
+          avatarDepth={avatarDepth}
+          halfBody={halfBody}
+          setAvatarHeight={setAvatarHeight}
+          setAvatarDepth={setAvatarDepth}
+          setEnablePositionControls={setEnablePositionControls}
+        />
+      )}
+      <Canvas 
+        style={style || defaultStyles.fullBody}
+        shadows={true}
+        dpr={[1, 2]} // Optimize pixel ratio
+        gl={{
+          antialias: true,
+          alpha: true,
+          powerPreference: "high-performance",
+          stencil: false
+        }}
+      >
+        <PerspectiveCamera
+          makeDefault
+          position={[0, 0, cameraZ]}
+          fov={getCameraSettings(halfBody, isZoomed || false).fov}
+          near={0.1}
+          far={1000}
+        />
+
+        {rotateAvatar && (
+          <OrbitControls
+            enablePan={false} 
+            enableZoom={false}
+            target={[0, 0, 0]}
+            enableDamping
+            dampingFactor={0.05}
+          />
+        )}
+
+        <Suspense fallback={fallback || <Loader fallbackImg={fallbackImg} />}>
+          <Lights />
+          
+          <AvatarView
+            url={url}
+            sex={sex}
+            showControls={showControls}
+            loading={loading || false}
+            eyeBlink={eyeBlink || false}
+            headMovement={headMovement || false}
+            speaking={speaking || false}
+            halfBody={halfBody}
+            chatEmission={chatEmission}
+            updateCurrentViseme={updateCurrentViseme}
+            setCameraZ={setCameraZ}
+            avatarHeight={avatarHeight}
+            avatarDepth={avatarDepth}
+          />
+        </Suspense>
+      </Canvas>
+    </>
+  );
+}

package/src/components/Avatar/AvatarView/utils/hideHands.ts

@@ -0,0 +1,11 @@
+import { Nodes } from "./utils";
+
+export const hideHands = (nodes: Nodes) => {
+  if (nodes.Wolf3D_Hands) {
+    nodes.Wolf3D_Hands.visible = false;
+  }
+  if (nodes.RightHand && nodes.LeftHand) {
+    nodes.RightHand.position.set(0, -2, 0);
+    nodes.LeftHand.position.set(0, -2, 0);
+  }
+};

package/src/components/Avatar/AvatarView/utils/useEyeBlink.ts

@@ -0,0 +1,100 @@
+import { useCallback, useEffect, useRef } from 'react';
+
+interface BlinkConfig {
+  minInterval: number;
+  maxInterval: number;
+  blinkDuration: number;
+}
+
+const DEFAULT_BLINK_CONFIG: BlinkConfig = {
+  minInterval: 1000,    // Minimum time between blinks in milliseconds
+  maxInterval: 5000,    // Maximum time between blinks in milliseconds
+  blinkDuration: 150,   // Duration of a single blink in milliseconds
+};
+
+interface UseAvatarBlinkProps {
+  enabled: boolean;
+  setMorphTargetInfluences: (morphTargetInfluences: any) => void;
+  config?: Partial<BlinkConfig>;
+}
+
+export function useAvatarBlink({
+  enabled,
+  setMorphTargetInfluences,
+  config = {}
+}: UseAvatarBlinkProps) {
+  const blinkTimeoutRef = useRef<NodeJS.Timeout>();
+  const isBlinkingRef = useRef(false);
+  const lastBlinkTime = useRef(0);
+  
+  const blinkConfig = {
+    ...DEFAULT_BLINK_CONFIG,
+    ...config
+  };
+  
+  const blink = useCallback(() => {
+    if (!enabled || isBlinkingRef.current) return;
+    
+    isBlinkingRef.current = true;
+    // Close eyes
+    setMorphTargetInfluences((prev: any) => ({
+      ...prev,
+      eyesClosed: 1
+    }));
+    
+    // Open eyes after blinkDuration
+    setTimeout(() => {
+      setMorphTargetInfluences((prev: any) => ({
+        ...prev,
+        eyesClosed: 0
+      }));
+      isBlinkingRef.current = false;
+      lastBlinkTime.current = Date.now();
+      
+      // Schedule next blink
+      scheduleNextBlink();
+    }, blinkConfig.blinkDuration);
+  }, [enabled, blinkConfig.blinkDuration, setMorphTargetInfluences]);
+  
+  const scheduleNextBlink = useCallback(() => {
+    if (blinkTimeoutRef.current) {
+      clearTimeout(blinkTimeoutRef.current);
+    }
+    
+    // Randomize the next blink delay between min and max interval
+    const nextBlinkDelay = Math.random() * 
+      (blinkConfig.maxInterval - blinkConfig.minInterval) + 
+      blinkConfig.minInterval;
+    
+    blinkTimeoutRef.current = setTimeout(blink, nextBlinkDelay);
+  }, [blink, blinkConfig.maxInterval, blinkConfig.minInterval]);
+
+  // Handle enabled state changes
+  useEffect(() => {
+    if (enabled) {
+      scheduleNextBlink();
+    } else {
+      if (blinkTimeoutRef.current) {
+        clearTimeout(blinkTimeoutRef.current);
+      }
+      // Reset eyes to open
+      setMorphTargetInfluences((prevInfluences: any) => ({
+        ...prevInfluences,
+        eyesClosed: 0
+      }));
+    }
+    
+    // Cleanup
+    return () => {
+      if (blinkTimeoutRef.current) {
+        clearTimeout(blinkTimeoutRef.current);
+      }
+    };
+  }, [enabled, scheduleNextBlink, setMorphTargetInfluences]);
+
+  return {
+    isBlinking: isBlinkingRef.current,
+    lastBlinkTime: lastBlinkTime.current,
+    triggerBlink: blink
+  };
+}

package/src/components/Avatar/AvatarView/utils/useMouthAnimation.ts

@@ -0,0 +1,93 @@
+import React, { useRef } from 'react';
+import { SkinnedMesh } from 'three';
+
+const VISEME_SMOOTHING = 0.2;
+const VISEME_SPEED_FACTOR = 0.44;
+
+interface Viseme {
+  name: string;
+  duration: number;
+  weight: number;
+  startTime: number;
+}
+
+interface UseMouthAnimationProps {
+  currentVisemes: Viseme[];
+  avatarMeshRef: React.RefObject<SkinnedMesh>;
+}
+
+const lerp = (start: number, end: number, alpha: number): number => {
+  return start * (1 - alpha) + end * alpha;
+};
+
+export function useMouthAnimation({ currentVisemes, avatarMeshRef }: UseMouthAnimationProps) {
+  const visemeStartTimeRef = useRef(0);
+  const currentVisemeWeightRef = useRef<{ [key: string]: number }>({});
+
+  // Helper function to get current viseme information
+  const getCurrentVisemeInfo = (elapsedTime: number) => {
+    let currentVisemeIndex = 0;
+    let accumulatedDuration = 0;
+
+    while (
+      currentVisemeIndex < currentVisemes.length &&
+      accumulatedDuration <= elapsedTime
+    ) {
+      accumulatedDuration += currentVisemes[currentVisemeIndex].duration;
+      currentVisemeIndex++;
+    }
+
+    return { currentVisemeIndex, accumulatedDuration };
+  };
+
+  // Helper function to apply the current viseme
+  const applyCurrentViseme = (index: number, elapsedTime: number, accumulatedDuration: number) => {
+    const currentViseme = currentVisemes[index - 1];
+    const visemeProgress = (elapsedTime - (accumulatedDuration - currentViseme.duration)) / currentViseme.duration;
+    const targetWeight = Math.sin(visemeProgress * Math.PI) * currentViseme.weight;
+
+    // Smooth the transition between visemes
+    if (!currentVisemeWeightRef.current[currentViseme.name]) {
+      currentVisemeWeightRef.current[currentViseme.name] = 0;
+    }
+    currentVisemeWeightRef.current[currentViseme.name] = lerp(
+      currentVisemeWeightRef.current[currentViseme.name],
+      targetWeight,
+      VISEME_SMOOTHING
+    );
+
+    const visemeIndex = avatarMeshRef.current?.morphTargetDictionary?.[currentViseme.name];
+    if (typeof visemeIndex === 'number' && avatarMeshRef.current?.morphTargetInfluences) {
+      avatarMeshRef.current.morphTargetInfluences[visemeIndex] = currentVisemeWeightRef.current[currentViseme.name];
+    }
+  };
+
+  // Helper function to reset viseme animation
+  const resetVisemeAnimation = (currentTime: number) => {
+    visemeStartTimeRef.current = currentTime;
+    currentVisemeWeightRef.current = {};
+  };
+
+  // Main function to handle viseme-based mouth movement
+  const handleMouthMovement = (elapsedTime: number) => {
+    if (currentVisemes.length === 0) return;
+
+    const currentTime = elapsedTime * VISEME_SPEED_FACTOR;
+    const visemeElapsedTime = currentTime - visemeStartTimeRef.current;
+
+    const { currentVisemeIndex, accumulatedDuration } = getCurrentVisemeInfo(visemeElapsedTime);
+
+    if (currentVisemeIndex > 0) {
+      applyCurrentViseme(currentVisemeIndex, visemeElapsedTime, accumulatedDuration);
+    }
+
+    // Reset viseme animation if we've reached the end
+    if (visemeElapsedTime > accumulatedDuration) {
+      resetVisemeAnimation(currentTime);
+    }
+  };
+
+  return {
+    handleMouthMovement,
+  };
+}

package/src/components/Avatar/AvatarView/utils/useSmile.ts

@@ -0,0 +1,39 @@
+import { Nodes } from './utils';
+import { SkinnedMesh } from 'three';
+import { useEffect, useRef } from 'react';
+import { useFrame } from '@react-three/fiber';
+
+interface SmileState {
+  headMesh: SkinnedMesh | null;
+  mouthSmileMorphIndex: number;
+}
+
+const SMILE_INTENSITY = 1 / 3;
+
+export default function useSmile(smiling: boolean | undefined, nodes: Nodes) {
+  const smileStateRef = useRef<SmileState>({
+    headMesh: null,
+    mouthSmileMorphIndex: 0,
+  });
+
+  useEffect(() => {
+    const headMesh = (nodes.Wolf3D_Head || nodes.Wolf3D_Avatar020 || nodes.Wolf3D_Avatar001) as SkinnedMesh;
+    smileStateRef.current.headMesh = headMesh;
+
+    if (headMesh?.morphTargetDictionary && headMesh?.morphTargetInfluences) {
+      smileStateRef.current.mouthSmileMorphIndex = headMesh.morphTargetDictionary.mouthSmile;
+    }
+  }, [nodes]);
+
+  useFrame(() => {
+    const { headMesh, mouthSmileMorphIndex } = smileStateRef.current;
+
+    if (!headMesh?.morphTargetInfluences) return;
+
+    const targetSmileIntensity = smiling ? SMILE_INTENSITY : 0;
+    const currentSmileIntensity = headMesh.morphTargetInfluences[mouthSmileMorphIndex];
+
+    // Smoothly interpolate between current and target smile intensity
+    headMesh.morphTargetInfluences[mouthSmileMorphIndex] += (targetSmileIntensity - currentSmileIntensity) * 0.1;
+  });
+}