@memori.ai/memori-react 7.7.0 → 7.7.1

package/src/context/visemeContext.tsx

@@ -1,145 +1,332 @@
-import React, { createContext, useContext, useRef, useCallback, useState, useEffect } from 'react';
+import React, {
+  createContext,
+  useContext,
+  useState,
+  useCallback,
+  useRef,
+  useEffect,
+} from 'react';
+import { MathUtils, SkinnedMesh } from 'three';
 
-type Viseme = { 
-  name: string; 
-  weight: number; 
-  startTime: number; 
-  endTime: number;
+type AzureViseme = { visemeId: number; audioOffset: number };
+
+type ProcessedViseme = {
+  name: string;
+  duration: number;
+  weight: number;
+  startTime: number;
 };
 
 interface VisemeContextType {
-  addViseme: (visemeId: number, audioOffset: number) => void;
-  updateCurrentViseme: (currentTime: number) => Viseme | null;
-  startProcessing: () => void;
-  stopProcessing: () => void;
-  resetVisemeQueue: () => void;
-  isProcessing: boolean;
+  setMeshRef: (mesh: SkinnedMesh | null) => void;
+  addVisemeToQueue: (viseme: AzureViseme) => void;
+  processVisemeQueue: () => ProcessedViseme[];
+  clearVisemes: () => void;
+  isMeshSet: boolean;
+  setEmotion: (emotion: string) => void;
+  emotion: string;
+  getAzureStyleForEmotion: (emotion: string) => string;
 }
 
 const VisemeContext = createContext<VisemeContextType | undefined>(undefined);
 
+const VISEME_SMOOTHING = 0.3;
+const DEFAULT_VISEME_DURATION = 0.1;
+const MINIMUM_ELAPSED_TIME = 0.01;
+const VISEME_SPEED_FACTOR = 1;
+const AUDIO_PLAYBACK_RATE = 1;
+const VISEME_BASE_SPEED = 1;
+
 const VISEME_MAP: { [key: number]: string } = {
-  0: 'viseme_sil', // silence
-  1: 'viseme_PP', // p, b, m
-  2: 'viseme_FF', // f, v
-  3: 'viseme_TH', // th, dh
-  4: 'viseme_DD', // t, d, n, l
-  5: 'viseme_kk', // k, g, ng
-  6: 'viseme_CH', // tS, dZ, S, Z
-  7: 'viseme_SS', // s, z
-  8: 'viseme_nn', // Not explicitly defined in Azure mapping, keeping for compatibility
-  9: 'viseme_RR', // r
-  10: 'viseme_aa', // A:
-  11: 'viseme_E', // e
-  12: 'viseme_I', // I
-  13: 'viseme_O', // O
-  14: 'viseme_U', // u
-  // Mapping the rest based on closest matches or keeping them as in the original mapping
-  15: 'viseme_kk', // g, k (same as 5)
-  16: 'viseme_CH', // ch, j, sh, zh (same as 6)
-  17: 'viseme_SS', // s, z (same as 7)
-  18: 'viseme_TH', // th, dh (same as 3)
-  19: 'viseme_RR', // r (same as 9)
-  20: 'viseme_kk', // w (closest match, could be debated)
-  21: 'viseme_PP', // y (closest match, could be debated)
+  0: 'viseme_sil',
+  1: 'viseme_PP',
+  2: 'viseme_FF',
+  3: 'viseme_TH',
+  4: 'viseme_DD',
+  5: 'viseme_kk',
+  6: 'viseme_CH',
+  7: 'viseme_SS',
+  8: 'viseme_nn',
+  9: 'viseme_RR',
+  10: 'viseme_aa',
+  11: 'viseme_E',
+  12: 'viseme_I',
+  13: 'viseme_O',
+  14: 'viseme_U',
+  15: 'viseme_kk',
+  16: 'viseme_CH',
+  17: 'viseme_SS',
+  18: 'viseme_TH',
+  19: 'viseme_RR',
+  20: 'viseme_kk',
+  21: 'viseme_PP',
 };
 
-const DEFAULT_VISEME_DURATION = 0.04 //0; // Reduced from 0.4 for smoother transitions
-const VISEME_OVERLAP = 0.35; // Slightly increased from 0.04 for more overlap
-const SMOOTHING_FACTOR = 0.35 // New constant for weight smoothing
-const TIME_OFFSET =-0.15; // Adjust this value as needed (in seconds)
-const PRELOAD_TIME = 0.515; // Preload visemes 0.5 seconds in advance
-
-export const VisemeProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => {
-  const visemeQueueRef = useRef<Viseme[]>([]);
+export const VisemeProvider: React.FC<{ children: React.ReactNode }> = ({
+  children,
+}) => {
+  const [isMeshSet, setIsMeshSet] = useState(false);
+  const [emotion, setEmotion] = useState('Neutral');
+  const isAnimatingRef = useRef(false);
+  const currentVisemesRef = useRef<ProcessedViseme[]>([]);
+  const visemeQueueRef = useRef<AzureViseme[]>([]);
+  const animationFrameRef = useRef<number | null>(null);
   const startTimeRef = useRef<number | null>(null);
-  const [isProcessing, setIsProcessing] = useState(false);
-  const lastVisemeRef = useRef<Viseme | null>(null);
-
-  const addViseme = useCallback((visemeId: number, audioOffset: number) => {
-    const visemeName = VISEME_MAP[visemeId] || 'viseme_sil';
-    const startTime = audioOffset / 10000000 + TIME_OFFSET;
-    const endTime = startTime + DEFAULT_VISEME_DURATION;
-    const newViseme: Viseme = {
-      name: visemeName,
-      weight: 0,
-      startTime,
-      endTime,
-    };
-    visemeQueueRef.current.push(newViseme);
+  const currentVisemeWeightRef = useRef<{ [key: string]: number }>({});
+  const meshRef = useRef<SkinnedMesh | null>(null);
 
-    if (!isProcessing) {
-      startProcessing();
-    }
-  }, [isProcessing]);
+  const lerp = (start: number, end: number, alpha: number): number => {
+    return start * (1 - alpha) + end * alpha;
+  };
 
-  const updateCurrentViseme = useCallback((currentTime: number): Viseme | null => {
-    if (!isProcessing || startTimeRef.current === null) {
-      console.log('StartTimeRef not set');
-      return null;
-    }
+  const easeInOutQuad = (x: number): number => {
+    return x < 0.5 ? 2 * x * x : 1 - Math.pow(-2 * x + 2, 2) / 2;
+  };
 
-    const elapsedTime = currentTime - startTimeRef.current + PRELOAD_TIME
+  const setMeshRef = useCallback(
+    (mesh: SkinnedMesh | null) => {
+      if (mesh && mesh.morphTargetDictionary && mesh.morphTargetInfluences) {
+        meshRef.current = mesh;
+        setIsMeshSet(true);
+      } else {
+        console.error('Invalid mesh provided:', mesh);
+      }
+    },
+    [meshRef]
+  );
+
+  const addVisemeToQueue = useCallback((viseme: AzureViseme) => {
+    visemeQueueRef.current.push(viseme);
+  }, []);
 
-    // Remove expired visemes
-    visemeQueueRef.current = visemeQueueRef.current.filter(v => v.endTime > elapsedTime);
+  const getCurrentViseme = useCallback((elapsedTime: number) => {
+    if (elapsedTime < MINIMUM_ELAPSED_TIME) return null;
 
-    const currentViseme = visemeQueueRef.current.find(v => 
-      v.startTime <= elapsedTime && v.endTime > elapsedTime - VISEME_OVERLAP
+    return currentVisemesRef.current.find((viseme, index) => {
+      const nextViseme = currentVisemesRef.current[index + 1];
+      return (
+        elapsedTime >= viseme.startTime &&
+        (!nextViseme || elapsedTime < nextViseme.startTime)
+      );
+    });
+  }, []);
+
+  const getDynamicSpeedFactor = (visemeDuration: number): number => {
+    const baseDuration = 0.1;
+    return (
+      VISEME_BASE_SPEED * (baseDuration / visemeDuration) * AUDIO_PLAYBACK_RATE
     );
+  };
+
+  const applyViseme = useCallback(
+    (viseme: ProcessedViseme, elapsedTime: number) => {
+      if (!meshRef.current) {
+        console.error('Mesh not set');
+        return;
+      }
+
+      const visemeProgress = Math.min(
+        (elapsedTime - viseme.startTime) / viseme.duration,
+        1
+      );
+
+      const dynamicSpeedFactor = getDynamicSpeedFactor(viseme.duration);
+      const adjustedProgress = visemeProgress * dynamicSpeedFactor;
+
+      const easedProgress = easeInOutQuad(adjustedProgress);
+      const targetWeight = Math.sin(easedProgress * Math.PI) * viseme.weight;
+
+      currentVisemeWeightRef.current[viseme.name] = MathUtils.lerp(
+        currentVisemeWeightRef.current[viseme.name] || 0,
+        targetWeight,
+        VISEME_SMOOTHING
+      );
+
+      const visemeIndex = meshRef.current.morphTargetDictionary?.[viseme.name];
+      if (
+        typeof visemeIndex === 'number' &&
+        meshRef.current.morphTargetInfluences
+      ) {
+        meshRef.current.morphTargetInfluences[visemeIndex] =
+          currentVisemeWeightRef.current[viseme.name];
+      } else {
+        console.error(
+          `Viseme not found in morph target dictionary: ${viseme.name}`
+        );
+      }
+
+      Object.keys(currentVisemeWeightRef.current).forEach((visemeName) => {
+        if (visemeName !== viseme.name) {
+          currentVisemeWeightRef.current[visemeName] = lerp(
+            currentVisemeWeightRef.current[visemeName],
+            0,
+            VISEME_SMOOTHING * 0.5
+          );
+
+          const index = meshRef.current!.morphTargetDictionary?.[visemeName];
+          if (typeof index === 'number' && meshRef.current!.morphTargetInfluences) {
+            meshRef.current!.morphTargetInfluences[index] = currentVisemeWeightRef.current[visemeName];
+          }
+        }
+      });
+    },
+    []
+  );
+
+  const animate = useCallback(
+    (time: number) => {
+      if (startTimeRef.current === null) {
+        startTimeRef.current = time;
+      }
+
+      const elapsedTime =
+        ((time - startTimeRef.current) / 1000) * VISEME_SPEED_FACTOR;
+
+      const currentViseme = getCurrentViseme(elapsedTime);
 
-    if (currentViseme) {
-      console.log('CurrentViseme Found!')
-      const visemeProgress = (elapsedTime - currentViseme.startTime) / (currentViseme.endTime - currentViseme.startTime);
-      const targetWeight = Math.sin(Math.PI * Math.min(visemeProgress, 1));
+      if (currentViseme) {
+        applyViseme(currentViseme, elapsedTime);
+      }
+
+      if (
+        currentVisemesRef.current.length > 0 &&
+        elapsedTime <
+          currentVisemesRef.current[currentVisemesRef.current.length - 1]
+            .startTime +
+            currentVisemesRef.current[currentVisemesRef.current.length - 1]
+              .duration
+      ) {
+        animationFrameRef.current = requestAnimationFrame(animate);
+      } else {
+        clearVisemes();
+      }
+    },
+    [getCurrentViseme, applyViseme]
+  );
+
+  const processVisemeQueue = useCallback(() => {
+    const azureVisemes = [...visemeQueueRef.current];
+    visemeQueueRef.current = [];
+
+    if (azureVisemes.length === 0) {
+      return [];
+    }
+
+    const processedVisemes: ProcessedViseme[] = azureVisemes.map(
+      (currentViseme, i) => {
+        const nextViseme = azureVisemes[i + 1];
+        const duration = nextViseme
+          ? (nextViseme.audioOffset - currentViseme.audioOffset) / 10000000
+          : DEFAULT_VISEME_DURATION;
+
+        const processedViseme = {
+          name: VISEME_MAP[currentViseme.visemeId] || 'viseme_sil',
+          duration,
+          weight: 1,
+          startTime: currentViseme.audioOffset / 10000000,
+        };
+        return processedViseme;
+      }
+    );
+
+    const interpolatedVisemes: ProcessedViseme[] = [];
+    for (let i = 0; i < processedVisemes.length - 1; i++) {
+      const currentViseme = processedVisemes[i];
+      const nextViseme = processedVisemes[i + 1];
+      
+      interpolatedVisemes.push(currentViseme);
       
-      // Smooth the weight transition
-      const smoothedWeight = lastVisemeRef.current
-        ? lastVisemeRef.current.weight + (targetWeight - lastVisemeRef.current.weight) * SMOOTHING_FACTOR
-        : targetWeight;
-
-      const updatedViseme = { ...currentViseme, weight: smoothedWeight };
-      lastVisemeRef.current = updatedViseme;
-      return updatedViseme;
+      const intermediateDuration = (nextViseme.startTime - currentViseme.startTime) * 0.5;
+      interpolatedVisemes.push({
+        name: 'viseme_sil',
+        duration: intermediateDuration,
+        weight: 0.5,
+        startTime: currentViseme.startTime + intermediateDuration,
+      });
     }
+    interpolatedVisemes.push(processedVisemes[processedVisemes.length - 1]);
+
+    currentVisemesRef.current = interpolatedVisemes;
 
-    // Gradually reduce weight when no viseme is active
-    if (lastVisemeRef.current) {
-      const reducedWeight = lastVisemeRef.current.weight * (1 - SMOOTHING_FACTOR);
-      if (reducedWeight > 0.01) {
-        lastVisemeRef.current = { ...lastVisemeRef.current, weight: reducedWeight };
-        return lastVisemeRef.current;
+    if (!isAnimatingRef.current) {
+      isAnimatingRef.current = true;
+      startTimeRef.current = performance.now();
+      animationFrameRef.current = requestAnimationFrame(animate);
+    } else {
+      if (startTimeRef.current !== null) {
+        const currentTime = performance.now();
+        const elapsedTime =
+          ((currentTime - startTimeRef.current) / 1000) * VISEME_SPEED_FACTOR;
+        startTimeRef.current =
+          currentTime - (elapsedTime / VISEME_SPEED_FACTOR) * 1000;
       }
     }
 
-    lastVisemeRef.current = null;
-    return null;
-  }, [isProcessing]);
+    return interpolatedVisemes;
+  }, [animate]);
 
-  const startProcessing = useCallback(() => {
-    if (isProcessing) return;
-    startTimeRef.current = performance.now() / 1000;
-    setIsProcessing(true);
-  }, [isProcessing]);
+  const clearVisemes = useCallback(() => {
+    currentVisemesRef.current = [];
+    visemeQueueRef.current = [];
+
+    if (animationFrameRef.current !== null) {
+      cancelAnimationFrame(animationFrameRef.current);
+      animationFrameRef.current = null;
+    }
 
-  const stopProcessing = useCallback(() => {
-    setIsProcessing(false);
+    if (
+      meshRef.current?.morphTargetDictionary &&
+      meshRef.current?.morphTargetInfluences
+    ) {
+      Object.values(meshRef.current.morphTargetDictionary).forEach(index => {
+        if (typeof index === 'number') {
+          meshRef.current!.morphTargetInfluences![index] = 0;
+        }
+      });
+    }
+
+    currentVisemeWeightRef.current = {};
     startTimeRef.current = null;
-    lastVisemeRef.current = null;
+    isAnimatingRef.current = false;
   }, []);
 
-  const resetVisemeQueue = useCallback(() => {
-    visemeQueueRef.current = [];
-    lastVisemeRef.current = null;
+  const emotionMap: Record<string, Record<string, number>> = {
+    Gioia: { Gioria: 1 },
+    Rabbia: { Rabbia: 1 },
+    Sorpresa: { Sorpresa: 1 },
+    Tristezza: { Tristezza: 1 },
+    Timore: { Timore: 1 },
+  };
+
+  const emotionToAzureStyleMap: Record<string, string> = {
+    Gioia: 'cheerful',
+    Rabbia: 'angry',
+    Sorpresa: 'excited',
+    Tristezza: 'sad',
+    Timore: 'terrified',
+  };
+
+  function getAzureStyleForEmotion(emotion: string): string {
+    return emotionToAzureStyleMap[emotion] || 'neutral';
+  }
+
+  useEffect(() => {
+    return () => {
+      if (animationFrameRef.current !== null) {
+        cancelAnimationFrame(animationFrameRef.current);
+      }
+    };
   }, []);
 
-  const contextValue = {
-    addViseme,
-    updateCurrentViseme,
-    startProcessing,
-    stopProcessing,
-    resetVisemeQueue,
-    isProcessing,
+  const contextValue: VisemeContextType = {
+    setMeshRef,
+    addVisemeToQueue,
+    processVisemeQueue,
+    clearVisemes,
+    isMeshSet,
+    setEmotion,
+    emotion,
+    getAzureStyleForEmotion,
   };
 
   return (
@@ -151,7 +338,7 @@ export const VisemeProvider: React.FC<{ children: React.ReactNode }> = ({ childr
 
 export const useViseme = (): VisemeContextType => {
   const context = useContext(VisemeContext);
-  if (!context) {
+  if (context === undefined) {
     throw new Error('useViseme must be used within a VisemeProvider');
   }
   return context;

package/src/index.stories.tsx

@@ -111,29 +111,31 @@ Giovanna.args = {
 
 export const GiovannaProva = Template.bind({});
 GiovannaProva.args = {
-  memoriName: 'GiovannaProva',
-  ownerUserName: 'patini929',
-  memoriID: 'aee4c0ab-66c0-4a4e-acf5-e7be0a3a8ddf',
-  ownerUserID: '1941d326-6986-4fa1-872b-458d09fb654c',
-  tenantID: 'www.aisuru.com',
-  engineURL: 'https://engine.memori.ai',
-  apiURL: 'https://backend.memori.ai',
-  baseURL: 'https://www.aisuru.com',
-  uiLang: 'IT',
+  memoriName: 'Giovanna Test',
+  ownerUserName: 'andrea.patini3',
+  memoriID: '431d9819-c958-442c-a799-f90617371c0c',
+  ownerUserID: '58770358-a5db-4b49-b3a4-734fc468e745',
+  tenantID: 'aisuru-staging.aclambda.online',
+  engineURL: 'https://engine-staging.memori.ai',
+  apiURL: 'https://backend-staging.memori.ai',
+  baseURL: 'https://aisuru-staging.aclambda.online',
+  uiLang: 'EN',
   spokenLang: 'IT',
   layout: 'ZOOMED_FULL_BODY',
-  showInstruct: false,
-  showSettings: true,
-  showClear: false,
-  showAIicon: true,
-  showWhyThisAnswer: true,
-  showTypingText: false,
-  showOnlyLastMessages: false,
-  showTranslationOriginal: false,
-  showCopyButton: false,
-  showShare: true,
-  showLogin: false,
-  enableAudio: true,
+  showInstruct: 'false',
+  showSettings: 'true',
+  showClear: 'false',
+  showAIicon: 'true',
+  showWhyThisAnswer: 'true',
+  showTypingText: 'false',
+  showOnlyLastMessages: 'false',
+  showTranslationOriginal: 'false',
+  showCopyButton: 'false',
+  showShare: 'true',
+  showLogin: 'false',
+  enableAudio: 'true',
+  integrationID: 'e92ac275-39b5-474d-8f9e-826cc5284f1e',
+  initialQuestion: 'inizio simulazione',
 };
 
 const TemplateWithBatchButton: Story<Props> = args => (

package/src/styles.css

@@ -53,7 +53,6 @@
 @import url('./components/layouts/website-assistant.css');
 @import url('./components/layouts/chat.css');
 @import url('./components/layouts/zoomed-full-body.css');
-
 @import url('https://cdn.jsdelivr.net/npm/katex@0.16.10/dist/katex.min.css');
 
 .sr-only {