@memori.ai/memori-react 7.8.0-rc.0 → 7.8.0

package/src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx

@@ -1,4 +1,4 @@
-import React, { useEffect, useRef, useState } from 'react';
+import React, { useEffect, useRef, useMemo, useCallback } from 'react';
 import {
   Vector3,
   Euler,
@@ -11,7 +11,6 @@ import {
 } from 'three';
 import { useAnimations, useGLTF } from '@react-three/drei';
 import { useGraph, useFrame } from '@react-three/fiber';
-import { correctMaterials, isSkinnedMesh } from '../../../../../helpers/utils';
 
 interface FullbodyAvatarProps {
   url: string;
@@ -41,6 +40,7 @@ interface FullbodyAvatarProps {
   emotionMorphTargets: Record<string, number>;
 }
 
+
 const AVATAR_POSITION = new Vector3(0, -1, 0);
 const AVATAR_ROTATION = new Euler(0.175, 0, 0);
 const AVATAR_POSITION_ZOOMED = new Vector3(0, -1.45, 0);
@@ -51,24 +51,23 @@ const ANIMATION_URLS = {
     'https://assets.memori.ai/api/v2/asset/0e49aa5d-f757-4292-a170-d843c2839a41.glb',
 };
 
-// Blink configuration
 const BLINK_CONFIG = {
   minInterval: 1000,
   maxInterval: 5000,
   blinkDuration: 150,
 };
 
-const EMOTION_TRANSITION_SPEED = 0.1; // Adjust this value to control emotion transition speed
+const EMOTION_SMOOTHING = 0.3;
+const VISME_SMOOTHING = 0.5;
+
 
 export default function FullbodyAvatar({
   url,
   sex,
-  onLoaded,
   currentBaseAction,
   timeScale,
   isZoomed,
   eyeBlink,
-  morphTargetSmoothing = 0.5,
   updateCurrentViseme,
   setMorphTargetDictionary,
   setMorphTargetInfluences,
@@ -76,55 +75,51 @@ export default function FullbodyAvatar({
 }: FullbodyAvatarProps) {
   const { scene } = useGLTF(url);
   const { animations } = useGLTF(ANIMATION_URLS[sex]);
-  const { nodes, materials } = useGraph(scene);
   const { actions } = useAnimations(animations, scene);
 
-  const mixer = useRef(new AnimationMixer(scene));
+  const mixerRef = useRef<AnimationMixer>();
   const headMeshRef = useRef<SkinnedMesh>();
   const currentActionRef = useRef<AnimationAction | null>(null);
-  const [isTransitioningToIdle, setIsTransitioningToIdle] = useState(false);
+  const isTransitioningToIdleRef = useRef(false);
 
-  // Blink state
-  const lastBlinkTime = useRef(0);
-  const nextBlinkTime = useRef(0);
-  const isBlinking = useRef(false);
-  const blinkStartTime = useRef(0);
+  const lastBlinkTimeRef = useRef(0);
+  const nextBlinkTimeRef = useRef(0);
+  const isBlinkingRef = useRef(false);
+  const blinkStartTimeRef = useRef(0);
 
-  // Morph targets
   const currentEmotionRef = useRef<Record<string, number>>({});
   const previousEmotionKeysRef = useRef<Set<string>>(new Set());
 
-  useEffect(() => {
-    correctMaterials(materials);
-
+  // Memoize the scene traversal
+  const headMesh = useMemo(() => {
+    let foundMesh: SkinnedMesh | undefined;
     scene.traverse((object: Object3D) => {
-      if (object instanceof SkinnedMesh) {
-        if (object.name === 'GBNL__Head' || object.name === 'Wolf3D_Avatar') {
-          headMeshRef.current = object;
-          if (object.morphTargetDictionary && object.morphTargetInfluences) {
-            setMorphTargetDictionary(object.morphTargetDictionary);
-
-            const initialInfluences = Object.keys(
-              object.morphTargetDictionary
-            ).reduce((acc, key) => ({ ...acc, [key]: 0 }), {});
-            setMorphTargetInfluences(initialInfluences);
-          }
-        }
+      if (
+        object instanceof SkinnedMesh &&
+        (object.name === 'GBNL__Head' || object.name === 'Wolf3D_Avatar')
+      ) {
+        foundMesh = object;
       }
     });
+    return foundMesh;
+  }, [scene]);
 
-    onLoaded?.();
-
-    return () => {
-      Object.values(materials).forEach(material => material.dispose());
-      Object.values(nodes)
-        .filter(isSkinnedMesh)
-        .forEach(mesh => mesh.geometry.dispose());
-    };
-  }, [materials, nodes, url, onLoaded, scene]);
-
-  // Handle base animation changes
   useEffect(() => {
+    if (headMesh) {
+      headMeshRef.current = headMesh;
+      if (headMesh.morphTargetDictionary && headMesh.morphTargetInfluences) {
+        setMorphTargetDictionary(headMesh.morphTargetDictionary);
+        const initialInfluences = Object.keys(
+          headMesh.morphTargetDictionary
+        ).reduce((acc, key) => ({ ...acc, [key]: 0 }), {});
+        setMorphTargetInfluences(initialInfluences);
+      }
+    }
+    mixerRef.current = new AnimationMixer(scene);
+  }, [headMesh, scene, setMorphTargetDictionary, setMorphTargetInfluences]);
+
+  // Memoize the animation change handler
+  const handleAnimationChange = useCallback(() => {
     if (!actions || !currentBaseAction.action) return;
 
     const newAction = actions[currentBaseAction.action];
@@ -141,15 +136,11 @@ export default function FullbodyAvatar({
     if (currentActionRef.current) {
       currentActionRef.current.fadeOut(fadeOutDuration);
     }
-    
-    console.log(newAction);
+
     newAction.reset().fadeIn(fadeInDuration).play();
     currentActionRef.current = newAction;
-
-    // Set the time scale for the new action
     newAction.timeScale = timeScale;
 
-    // If it's an emotion animation, set it to play once and then transition to idle
     if (
       currentBaseAction.action.startsWith('Gioia') ||
       currentBaseAction.action.startsWith('Rabbia') ||
@@ -159,55 +150,55 @@ export default function FullbodyAvatar({
     ) {
       newAction.setLoop(LoopOnce, 1);
       newAction.clampWhenFinished = true;
-      setIsTransitioningToIdle(true);
+      isTransitioningToIdleRef.current = true;
     }
   }, [actions, currentBaseAction, timeScale]);
 
-  useFrame(state => {
-    if (
-      headMeshRef.current &&
-      headMeshRef.current.morphTargetDictionary &&
-      headMeshRef.current.morphTargetInfluences
-    ) {
-      const currentTime = state.clock.getElapsedTime() * 1000; // Convert to milliseconds
+  useEffect(() => {
+    handleAnimationChange();
+  }, [handleAnimationChange]);
+
+  // Optimize the frame update function
+  const updateFrame = useCallback(
+    (currentTime: number) => {
+      if (
+        !headMeshRef.current ||
+        !headMeshRef.current.morphTargetDictionary ||
+        !headMeshRef.current.morphTargetInfluences
+      )
+        return;
 
-      // Handle blinking
       let blinkValue = 0;
       if (eyeBlink) {
-        if (currentTime >= nextBlinkTime.current && !isBlinking.current) {
-          isBlinking.current = true;
-          blinkStartTime.current = currentTime;
-          lastBlinkTime.current = currentTime;
-          nextBlinkTime.current =
+        if (currentTime >= nextBlinkTimeRef.current && !isBlinkingRef.current) {
+          isBlinkingRef.current = true;
+          blinkStartTimeRef.current = currentTime;
+          lastBlinkTimeRef.current = currentTime;
+          nextBlinkTimeRef.current =
             currentTime +
             Math.random() *
               (BLINK_CONFIG.maxInterval - BLINK_CONFIG.minInterval) +
             BLINK_CONFIG.minInterval;
         }
 
-        if (isBlinking.current) {
+        if (isBlinkingRef.current) {
           const blinkProgress =
-            (currentTime - blinkStartTime.current) / BLINK_CONFIG.blinkDuration;
+            (currentTime - blinkStartTimeRef.current) /
+            BLINK_CONFIG.blinkDuration;
           if (blinkProgress <= 0.5) {
-            // Eyes closing
             blinkValue = blinkProgress * 2;
           } else if (blinkProgress <= 1) {
-            // Eyes opening
             blinkValue = 2 - blinkProgress * 2;
           } else {
-            // Blink finished
-            isBlinking.current = false;
+            isBlinkingRef.current = false;
             blinkValue = 0;
           }
         }
       }
 
       const currentViseme = updateCurrentViseme(currentTime / 1000);
-
-      // Create a set of current emotion keys
       const currentEmotionKeys = new Set(Object.keys(emotionMorphTargets));
 
-      // Reset old emotion morph targets
       previousEmotionKeysRef.current.forEach(key => {
         if (!currentEmotionKeys.has(key)) {
           const index = headMeshRef.current!.morphTargetDictionary![key];
@@ -220,75 +211,78 @@ export default function FullbodyAvatar({
         }
       });
 
-      // Update morph targets
       Object.entries(headMeshRef.current.morphTargetDictionary).forEach(
         ([key, index]) => {
           if (typeof index === 'number') {
             let targetValue = 0;
 
-            // Handle emotions (base layer)
-            if (Object.prototype.hasOwnProperty.call(emotionMorphTargets, key)) {
+            if (currentEmotionKeys.has(key)) {
               const targetEmotionValue = emotionMorphTargets[key];
               const currentEmotionValue = currentEmotionRef.current[key] || 0;
               const newEmotionValue = MathUtils.lerp(
                 currentEmotionValue,
-                targetEmotionValue * 2,
-                EMOTION_TRANSITION_SPEED
+                targetEmotionValue * 2.5,
+                EMOTION_SMOOTHING
               );
               currentEmotionRef.current[key] = newEmotionValue;
               targetValue += newEmotionValue;
             }
 
-            // Handle visemes (additive layer)
             if (currentViseme && key === currentViseme.name) {
-              targetValue += currentViseme.weight * 1.3; // Amplify the effect
+              targetValue += currentViseme.weight * 1;
             }
 
-            // Handle blinking (additive layer, only for 'eyesClosed')
             if (key === 'eyesClosed' && eyeBlink) {
               targetValue += blinkValue;
             }
 
-            // Clamp the final value between 0 and 1
             targetValue = MathUtils.clamp(targetValue, 0, 1);
-
-            // Apply smoothing
-            if (headMeshRef.current && headMeshRef.current.morphTargetInfluences) {
+            if (
+              headMeshRef.current &&
+              headMeshRef.current.morphTargetInfluences
+            ) {
               headMeshRef.current.morphTargetInfluences[index] = MathUtils.lerp(
                 headMeshRef.current.morphTargetInfluences[index],
                 targetValue,
-                morphTargetSmoothing
+                VISME_SMOOTHING
               );
             }
           }
         }
       );
 
-      // Update the set of previous emotion keys for the next frame
       previousEmotionKeysRef.current = currentEmotionKeys;
 
-      // Handle transition from emotion animation to idle
-      if (isTransitioningToIdle && currentActionRef.current) {
+      if (isTransitioningToIdleRef.current && currentActionRef.current) {
         if (
           currentActionRef.current.time >=
           currentActionRef.current.getClip().duration
         ) {
-          // Transition to the idle animation
-          const idleNumber = Math.floor(Math.random() * 5) + 1; // Randomly choose 1, 2, 3, 4 or 5
-          const idleAction = actions[`Idle${idleNumber == 3 ? 4 : idleNumber}`];
+          const idleNumber = Math.floor(Math.random() * 5) + 1;
+          const idleAction =
+            actions[`Idle${idleNumber === 3 ? 4 : idleNumber}`];
 
           if (idleAction) {
             currentActionRef.current.fadeOut(0.5);
             idleAction.reset().fadeIn(0.5).play();
             currentActionRef.current = idleAction;
-            setIsTransitioningToIdle(false);
+            isTransitioningToIdleRef.current = false;
           }
         }
       }
 
-      // Update the animation mixer
-      mixer.current.update(0.01); // Fixed delta time for consistent animation speed
-    }
+      mixerRef.current?.update(0.01);
+    },
+    [
+      actions,
+      emotionMorphTargets,
+      eyeBlink,
+      updateCurrentViseme,
+    ]
+  );
+
+  useFrame(state => {
+    updateFrame(state.clock.getElapsedTime() * 1000);
   });
 
   return (

package/src/components/MemoriWidget/MemoriWidget.tsx

@@ -543,13 +543,10 @@ const MemoriWidget = ({
   const [hideEmissions, setHideEmissions] = useState(false);
 
   const {
-    updateCurrentViseme,
     startProcessing,
-    resetAndStartProcessing,
-    isProcessing,
     addViseme,
     stopProcessing,
-    resetVisemeQueue,
+    resetVisemeQueue
   } = useViseme();
 
   useEffect(() => {
@@ -1963,7 +1960,7 @@ const MemoriWidget = ({
       onEndSpeakStartListen();
     };
 
-    // Add this before starting new speech synthesis
+    // Clear any existing visemes before starting new speech
     resetVisemeQueue();
 
     // Set up the viseme event handler
@@ -1978,7 +1975,9 @@ const MemoriWidget = ({
     speechSynthesizer.speakSsmlAsync(
       `<speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis" xmlns:mstts="https://www.w3.org/2001/mstts" xmlns:emo="http://www.w3.org/2009/10/emotionml" xml:lang="${getCultureCodeByLanguage(
         userLang
-      )}"><voice name="${getTTSVoice(userLang)}"><s>${replaceTextWithPhonemes(
+      )}"><voice name="${getTTSVoice(
+        userLang
+      )}"><s>${replaceTextWithPhonemes(
         textToSpeak,
         userLang.toLowerCase()
       )}</s></voice></speak>`,
@@ -1987,24 +1986,18 @@ const MemoriWidget = ({
           setIsPlayingAudio(true);
           memoriSpeaking = true;
 
+          // Process the viseme data
+          startProcessing();
+
           try {
             // Decode the audio data
-            audioContext!.decodeAudioData(result.audioData, function (buffer) {
-              console.log('decoded');
-              const currentSource = audioContext!.createBufferSource();
-              currentSource.buffer = buffer;
-              currentSource.connect(audioContext!.destination);
-              startProcessing();
-              currentSource.start();
-
-              currentSource.onended = () => {
-                console.log('ended');
-                setIsPlayingAudio(false);
-                memoriSpeaking = false;
-                stopProcessing();
-                resetVisemeQueue();
-                emitEndSpeakEvent();
-              };
+            audioContext.decodeAudioData(result.audioData, function (buffer) {
+              source.buffer = buffer;
+              source.connect(audioContext.destination);
+
+              if (history.length < 1 || (isSafari && isIOS)) {
+                source.start(0);
+              }
             });
 
             // Handle the audio context state changes
@@ -2013,28 +2006,29 @@ const MemoriWidget = ({
                 audioContext.state === 'suspended' ||
                 audioContext.state === 'closed'
               ) {
-                console.log('suspended');
                 source.disconnect();
                 setIsPlayingAudio(false);
                 stopProcessing();
                 resetVisemeQueue();
                 memoriSpeaking = false;
-                emitEndSpeakEvent();
               } else if ((audioContext.state as string) === 'interrupted') {
-                console.log('interrupted');
-                stopProcessing();
-                resetVisemeQueue();
-
                 audioContext.resume();
               }
             };
+
+            audioContext.resume();
+
+            if (speechSynthesizer) {
+              speechSynthesizer.close();
+              speechSynthesizer = null;
+            }
           } catch (e) {
             console.warn('speak error: ', e);
             window.speechSynthesis.speak(new SpeechSynthesisUtterance(text));
-            setIsPlayingAudio(false);
-            memoriSpeaking = false;
             stopProcessing();
             resetVisemeQueue();
+            setIsPlayingAudio(false);
+            memoriSpeaking = false;
 
             if (speechSynthesizer) {
               speechSynthesizer.close();
@@ -2045,7 +2039,6 @@ const MemoriWidget = ({
         } else {
           audioContext.resume();
           stopProcessing();
-          resetVisemeQueue();
           setIsPlayingAudio(false);
           memoriSpeaking = false;
           emitEndSpeakEvent();
@@ -2055,9 +2048,9 @@ const MemoriWidget = ({
         console.error('speak:', error);
         window.speechSynthesis.speak(new SpeechSynthesisUtterance(text));
         setIsPlayingAudio(false);
-        memoriSpeaking = false;
         stopProcessing();
         resetVisemeQueue();
+        memoriSpeaking = false;
         emitEndSpeakEvent();
       }
     );
@@ -3425,4 +3418,4 @@ const MemoriWidget = ({
   );
 };
 
-export default MemoriWidget;
+export default MemoriWidget;

package/src/components/layouts/ZoomedFullBody.tsx

@@ -1,4 +1,4 @@
-import React from 'react';
+import React, { useEffect } from 'react';
 import Spin from '../ui/Spin';
 import { LayoutProps } from '../MemoriWidget/MemoriWidget';
 
@@ -21,6 +21,15 @@ const ZoomedFullBodyLayout: React.FC<LayoutProps> = ({
   loading = false,
   poweredBy,
 }) => {
+
+  useEffect(() => {
+    document.body.style.overflow = 'hidden';
+    return () => {
+      document.body.style.overflow = '';
+    };
+  }, []);
+
+
   return (
     <>
       {integrationStyle}

package/src/components/layouts/zoomed-full-body.css

@@ -12,7 +12,7 @@
 }
 
 .memori--grid-column--zoomed-full-body {
-    max-height: calc(100% - 80px) !important;
+    max-height: calc(100% - 130px) !important;
 }
 
 .memori.memori-widget.memori-layout-zoomed_full_body .memori--grid {

package/src/context/visemeContext.tsx

@@ -53,9 +53,9 @@ const VISEME_MAP: { [key: number]: string } = {
 };
 
 const DEFAULT_VISEME_DURATION = 0.04; //0; // Reduced from 0.4 for smoother transitions
-const VISEME_OVERLAP = 0.15; // Slightly increased from 0.04 for more overlap
+const VISEME_OVERLAP = 0.02; // Slightly increased from 0.04 for more overlap
 const SMOOTHING_FACTOR = 0.35; // New constant for weight smoothing
-const TIME_OFFSET = -0.04; // Adjust this value as needed (in seconds)
+const TIME_OFFSET = 0; // Adjust this value as needed (in seconds)
 const PRELOAD_TIME = 1; // Preload visemes 0.5 seconds in advance
 
 export const VisemeProvider: React.FC<{ children: React.ReactNode }> = ({

package/src/index.stories.tsx

@@ -28,6 +28,31 @@ export const Anonymous = Template.bind({});
 Anonymous.args = {
   ownerUserName: 'nzambello',
   memoriName: 'Nicola',
+  tenantID: 'www.aisuru.com',
+  engineURL: 'https://engine.memori.ai',
+  apiURL: 'https://backend.memori.ai',
+  baseURL: 'https://www.aisuru.com',
+  uiLang: 'IT',
+  spokenLang: 'IT',
+  layout: 'ZOOMED_FULL_BODY',
+  showInstruct: 'false',
+  showSettings: 'true',
+  showClear: 'false',
+  showAIicon: 'true',
+  showWhyThisAnswer: 'true',
+  showTypingText: 'false',
+  showOnlyLastMessages: 'false',
+  showTranslationOriginal: 'false',
+  showCopyButton: 'false',
+  showShare: 'true',
+  showLogin: 'false',
+  enableAudio: 'true',
+};
+
+export const Nunzio = Template.bind({});
+Nunzio.args = {
+  ownerUserName: 'nunzio.fiore',
+  memoriName: 'Nunzio',
   tenantID: 'aisuru.com',
   apiURL: 'https://backend.memori.ai',
   baseURL: 'https://www.aisuru.com',
@@ -95,18 +120,6 @@ Giovanna.args = {
   uiLang: 'IT',
   spokenLang: 'IT',
   layout: 'ZOOMED_FULL_BODY',
-  showInstruct: 'false',
-  showSettings: 'true',
-  showClear: 'false',
-  showAIicon: 'true',
-  showWhyThisAnswer: 'true',
-  showTypingText: 'false',
-  showOnlyLastMessages: 'false',
-  showTranslationOriginal: 'false',
-  showCopyButton: 'false',
-  showShare: 'true',
-  showLogin: 'false',
-  enableAudio: 'true',
 };
 
 export const GiovannaProva = Template.bind({});
@@ -122,6 +135,21 @@ GiovannaProva.args = {
   uiLang: 'EN',
   spokenLang: 'IT',
   layout: 'ZOOMED_FULL_BODY',
+  integrationID: 'e92ac275-39b5-474d-8f9e-826cc5284f1e',
+  initialQuestion: 'inizio simulazione',
+};
+
+export const NunzioFiore = Template.bind({});
+NunzioFiore.args = {
+  ownerUserName: 'nunzio.fiore',
+  memoriName: 'Nunzio',
+  tenantID: 'www.aisuru.com',
+  engineURL: 'https://engine.memori.ai',
+  apiURL: 'https://backend.memori.ai',
+  baseURL: 'https://www.aisuru.com',
+  uiLang: 'IT',
+  spokenLang: 'IT',
+  layout: 'ZOOMED_FULL_BODY',
   showInstruct: 'false',
   showSettings: 'true',
   showClear: 'false',
@@ -134,10 +162,9 @@ GiovannaProva.args = {
   showShare: 'true',
   showLogin: 'false',
   enableAudio: 'true',
-  integrationID: 'e92ac275-39b5-474d-8f9e-826cc5284f1e',
-  initialQuestion: 'inizio simulazione',
 };
 
+
 const TemplateWithBatchButton: Story<Props> = args => (
   <div>
     <button