@memori.ai/memori-react 7.19.2 → 7.21.1

package/src/components/Avatar/AvatarView/AvatarComponent/components/controllers/MorphTargetController.ts

@@ -1,6 +1,14 @@
 import { SkinnedMesh } from 'three';
 import { MathUtils } from 'three';
-import { EMOTION_SMOOTHING, VISEME_SMOOTHING, BLINK_CONFIG } from '../../constants';
+import {
+  EMOTION_SMOOTHING,
+  VISEME_SMOOTHING,
+  BLINK_CONFIG,
+  MAPPING_EMOTIONS_ITALIAN_TO_ENGLISH,
+  MAPPING_BLEND_SHAPE_TO_EMOTION_RPM,
+  EmotionMapping,
+  BlendShapeMap,
+} from '../../constants';
 
 /**
  * Controller class for handling morph target animations including emotions, visemes and blinking
@@ -9,16 +17,155 @@ export class MorphTargetController {
   private headMesh: SkinnedMesh;
   private currentEmotionValues: Record<string, number> = {};
   private previousEmotionKeys: Set<string> = new Set();
+  private isRPM: boolean = false;
 
-  constructor(headMesh: SkinnedMesh) {
+  // Default RPM blend shape mappings
+  private rpmBlendShapes: BlendShapeMap =
+    MAPPING_BLEND_SHAPE_TO_EMOTION_RPM as BlendShapeMap;
+
+  // Default custom GLB emotion mappings
+  private customGlbMapping: EmotionMapping =
+    MAPPING_EMOTIONS_ITALIAN_TO_ENGLISH as EmotionMapping;
+
+  /**
+   * Constructor for MorphTargetController
+   * @param headMesh SkinnedMesh representing the head
+   * @param rpmBlendShapes Optional custom RPM blend shapes
+   * @param customGlbMapping Optional custom GLB emotion mapping
+   */
+  constructor(
+    headMesh: SkinnedMesh,
+    rpmBlendShapes?: BlendShapeMap,
+    customGlbMapping?: EmotionMapping
+  ) {
     this.headMesh = headMesh;
+
+    // Detect if this is an RPM avatar based on mesh name
+    this.isRPM = headMesh.name !== 'GBNL__Head';
+
+    // Override default maps if provided
+    if (rpmBlendShapes) {
+      this.rpmBlendShapes = rpmBlendShapes;
+    }
+
+    if (customGlbMapping) {
+      this.customGlbMapping = customGlbMapping;
+    }
+  }
+
+  /**
+   * Process chat emission to extract emotion data
+   * @param chatEmission Chat emission text
+   * @param isLoading Loading state
+   * @returns Object with emotion morph targets
+   */
+  processChatEmission(
+    chatEmission: any,
+    isLoading: boolean
+  ): Record<string, number> {
+    // Default empty emotion targets
+    const defaultEmotions = this.getDefaultEmotionMorphTargets();
+
+    // If loading or no chat emission, return default emotions
+    if (isLoading || !chatEmission) {
+      return defaultEmotions;
+    }
+
+    // Check if chat emission contains emotion tag
+    const hasOutputTagEmotion = chatEmission?.includes(
+      '<output class="memori-emotion">'
+    );
+
+    if (!hasOutputTagEmotion) {
+      return defaultEmotions;
+    }
+
+    // Extract emotion name
+    const outputContentEmotion = chatEmission
+      ?.split('<output class="memori-emotion">')[1]
+      ?.split('</output>')[0]
+      ?.trim();
+
+    if (!outputContentEmotion) {
+      return defaultEmotions;
+    }
+
+    // Process emotion based on avatar type
+    return this.processEmotion(outputContentEmotion);
+  }
+
+  /**
+   * Process an emotion name into morph target values
+   * @param emotionName Emotion name (in Italian or English)
+   * @returns Object with morph target values
+   */
+  private processEmotion(emotionName: string): Record<string, number> {
+    // Get default empty emotion targets
+    const defaultEmotions = this.getDefaultEmotionMorphTargets();
+
+    // First, try to find the emotion regardless of language
+    if (this.isRPM) {
+      // For RPM avatars, find the matching emotion in the rpmBlendShapes array
+      const foundEmotion = this.rpmBlendShapes.find(
+        item =>
+          item.emotion.italian.toLowerCase() === emotionName.toLowerCase() ||
+          item.emotion.english.toLowerCase() === emotionName.toLowerCase()
+      );
+
+      if (foundEmotion) {
+        return { ...defaultEmotions, ...foundEmotion.blendShapes };
+      }
+    } else {
+      // For custom GLB, find the matching emotion in the customGlbMapping array
+      const foundEmotion = this.customGlbMapping.find(
+        item =>
+          item.italian.toLowerCase() === emotionName.toLowerCase() ||
+          item.english.toLowerCase() === emotionName.toLowerCase()
+      );
+
+      if (foundEmotion) {
+        return { ...defaultEmotions, [foundEmotion.english]: 1 };
+      }
+    }
+
+    console.log('[MorphTargetController] No emotion found:', emotionName);
+    return defaultEmotions;
+  }
+
+  /**
+   * Get default emotion morph targets (all set to 0)
+   */
+  private getDefaultEmotionMorphTargets(): Record<string, number> {
+    // For RPM, collect all blend shape keys
+    if (this.isRPM) {
+      const allBlendShapeKeys = new Set<string>();
+
+      this.rpmBlendShapes.forEach(item => {
+        Object.keys(item.blendShapes).forEach(key =>
+          allBlendShapeKeys.add(key)
+        );
+      });
+
+      return Array.from(allBlendShapeKeys).reduce(
+        (acc, key) => ({ ...acc, [key]: 0 }),
+        {}
+      );
+    }
+    // For custom GLB, use English emotion names
+    else {
+      return this.customGlbMapping.reduce(
+        (acc, emotion) => ({ ...acc, [emotion.english]: 0 }),
+        {}
+      );
+    }
   }
   /**
    * Updates the morph target influences for emotions, visemes and blinking
    */
   updateMorphTargets(
     currentTime: number,
-    emotionMorphTargets: Record<string, number>,
+    chatEmission: any,
+    isLoading: boolean,
     currentViseme: { name: string; weight: number } | null,
     eyeBlink: boolean,
     blinkState: {
@@ -33,17 +180,25 @@ export class MorphTargetController {
       !this.headMesh.morphTargetDictionary ||
       !this.headMesh.morphTargetInfluences
     ) {
-      console.error('[MorphTargetController] Missing morphTargetDictionary or morphTargetInfluences');
+      console.error(
+        '[MorphTargetController] Missing morphTargetDictionary or morphTargetInfluences'
+      );
       return;
     }
 
+    // Process chat emission to get emotion morph targets
+    const emotionMorphTargets = this.processChatEmission(
+      chatEmission,
+      isLoading
+    );
+
     // Calculate blink value for this frame
     const blinkValue = this.calculateBlinkValue(
       currentTime,
       blinkState,
       eyeBlink
     );
-    
+
     const currentEmotionKeys = new Set(Object.keys(emotionMorphTargets));
 
     // Process each morph target
@@ -55,6 +210,7 @@ export class MorphTargetController {
 
         // Handle emotion morphs with smoothing
         if (currentEmotionKeys.has(key)) {
+          // console.log('[MorphTargetController] Processing morph target:', key);
           const targetEmotionValue = emotionMorphTargets[key];
           const currentEmotionValue = this.currentEmotionValues[key] || 0;
           const newEmotionValue = MathUtils.lerp(
@@ -124,15 +280,15 @@ export class MorphTargetController {
     if (blinkState.isBlinking) {
       const blinkProgress =
         (currentTime - blinkState.blinkStartTime) / BLINK_CONFIG.blinkDuration;
-      
+
       // First half of blink - closing eyes
       if (blinkProgress <= 0.5) {
         blinkValue = blinkProgress * 2;
-      } 
+      }
       // Second half of blink - opening eyes
       else if (blinkProgress <= 1) {
         blinkValue = 2 - blinkProgress * 2;
-      } 
+      }
       // Blink complete
       else {
         blinkState.isBlinking = false;

package/src/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.tsx

@@ -15,22 +15,19 @@ import useHeadMovement from '../../utils/useHeadMovement';
 
 interface HalfBodyAvatarProps {
   url: string;
-  setMorphTargetInfluences: (morphTargetInfluences: any) => void;
-  setMorphTargetDictionary: (morphTargetDictionary: any) => void;
   updateCurrentViseme: (currentTime: number) => any;
   eyeBlink?: boolean;
-  heightValue?: number; // 0-100 slider value
   avatarHeight: number;
   avatarDepth: number;
   onLoaded?: () => void;
   onCameraZChange: (value: number) => void;
   headMovement?: boolean;
+  chatEmission?: any;
+  loading?: boolean;
 }
 
 export default function HalfBodyAvatar({
   url,
-  setMorphTargetInfluences,
-  setMorphTargetDictionary,
   updateCurrentViseme,
   eyeBlink = false,
   avatarHeight = 50,
@@ -38,18 +35,22 @@ export default function HalfBodyAvatar({
   headMovement = false,
   onLoaded,
   onCameraZChange,
+  chatEmission,
+  loading = false,
 }: HalfBodyAvatarProps) {
   const { scene } = useGLTF(url);
   const { nodes, materials } = useGraph(scene);
   const { camera } = useThree();
 
-  const morphTargetControllerRef = useRef<MorphTargetController>();
-  const positionControllerRef = useRef<AvatarPositionController>();
+  const morphTargetControllerRef = useRef<MorphTargetController | null>(null);
+  const positionControllerRef = useRef<AvatarPositionController | null>(null);
   const targetCameraZRef = useRef(camera.position.z);
 
-
+  // Apply head movement if enabled
   useHeadMovement(headMovement, nodes);
   
+
+  // Eye blinking state
   const blinkStateRef = useRef({
     isBlinking: false,
     lastBlinkTime: 0,
@@ -63,7 +64,7 @@ export default function HalfBodyAvatar({
     scene?.traverse((object: Object3D) => {
       if (
         object instanceof SkinnedMesh &&
-        (object.name === 'GBNL__Head' || object.name === 'Wolf3D_Avatar')
+        (object.name === 'GBNL__Head' || object.name === 'Wolf3D_Avatar' || object.name === 'Wolf3D_Avatar006_1')
       ) {
         foundMesh = object;
       }
@@ -80,21 +81,17 @@ export default function HalfBodyAvatar({
       );
     }
 
+    // Initialize MorphTargetController if head mesh exists
     if (headMesh) {
       morphTargetControllerRef.current = new MorphTargetController(headMesh);
-
-      if (headMesh.morphTargetDictionary && headMesh.morphTargetInfluences) {
-        setMorphTargetDictionary(headMesh.morphTargetDictionary);
-        const initialInfluences = Object.keys(headMesh.morphTargetDictionary)
-          .reduce((acc, key) => ({ ...acc, [key]: 0 }), {});
-        setMorphTargetInfluences(initialInfluences);
-      }
     }
 
+    // Correct materials and perform other initialization tasks
     correctMaterials(materials);
     onLoaded?.();
     hideHands(nodes);
     
+    // Cleanup on unmount
     return () => {
       Object.values(materials).forEach(material => material.dispose());
       Object.values(nodes)
@@ -103,12 +100,14 @@ export default function HalfBodyAvatar({
     };
   }, [materials, nodes, url, onLoaded, scene, headMesh]);
 
+  // Handle avatar height changes
   useEffect(() => {
     if (positionControllerRef.current) {
       positionControllerRef.current.updateHeight(avatarHeight, true);
     }
   }, [avatarHeight]);
 
+  // Handle avatar depth changes
   useEffect(() => {
     if (positionControllerRef.current && onCameraZChange) {
       const newCameraZ = positionControllerRef.current.updateDepth(avatarDepth, true);
@@ -123,9 +122,12 @@ export default function HalfBodyAvatar({
     // Update morph targets
     if (morphTargetControllerRef.current) {
       const currentViseme = updateCurrentViseme(currentTime / 1000);
+      
+      // Use the updated MorphTargetController that handles chat emission directly
       morphTargetControllerRef.current.updateMorphTargets(
         currentTime,
-        {},
+        chatEmission,
+        loading,
         currentViseme,
         eyeBlink,
         blinkStateRef.current,

package/src/components/Avatar/AvatarView/AvatarComponent/constants.ts

@@ -11,94 +11,95 @@ export const SCALE_LERP_FACTOR = 0.1;
 
 // Maximum number of idle loops before forcing change
 export const MAX_IDLE_LOOPS_DEFAULT = 5;
-
-
-interface BaseAction {
-  weight: number;
-  action?: string;
+// Type definitions for emotion handling
+interface EmotionMappingItem {
+  italian: string;
+  english: string;
 }
+export type EmotionMapping = EmotionMappingItem[];
 
-// Base animations for the avatar
-export const BASE_ACTIONS: Record<string, BaseAction> = {
-  Gioia1: { weight: 0 },
-  Gioia2: { weight: 0 },
-  Gioia3: { weight: 0 },
-  Idle1: { weight: 1 },
-  Idle2: { weight: 0 },
-  Idle3: { weight: 0 },
-  Idle4: { weight: 0 },
-  Idle5: { weight: 0 },
-  Rabbia1: { weight: 0 },
-  Rabbia2: { weight: 0 },
-  Rabbia3: { weight: 0 },
-  Sorpresa1: { weight: 0 },
-  Sorpresa2: { weight: 0 },
-  Sorpresa3: { weight: 0 },
-  Timore1: { weight: 0 },
-  Timore2: { weight: 0 },
-  Timore3: { weight: 0 },
-  Tristezza1: { weight: 0 },
-  Tristezza2: { weight: 0 },
-  Tristezza3: { weight: 0 },
-  Loading1: { weight: 0 },
-  Loading2: { weight: 0 },
-  Loading3: { weight: 0 },
-};
-
-// Mapping of emotions from Italian to English
-export const MAPPING_EMOTIONS_ITALIAN_TO_ENGLISH = {
-  Gioia: 'Joy',
-  Rabbia: 'Anger',
-  Sorpresa: 'Surprise',
-  Tristezza: 'Sadness',
-  Timore: 'Fear',
-};
+interface BlendShapeMapItem {
+  emotion: { italian: string; english: string };
+  blendShapes: Record<string, number>;
+}
+export type BlendShapeMap = BlendShapeMapItem[];
 
-// Mapping of blend shapes to emotions
-export const MAPPING_BLEND_SHAPE_TO_EMOTION_RPM = {
-  Rabbia: {
-    'browDownLeft': 0.5,
-    'browDownRight': 0.5,
-    'browOuterUpLeft': 0.5,
-    'browOuterUpRight': 0.5,
-    'mouthSmile': -0.2,
+// Italian to English emotion mapping
+// Italian to English emotion mapping
+export const MAPPING_EMOTIONS_ITALIAN_TO_ENGLISH: EmotionMapping = [
+  {
+    italian: 'Gioia',
+    english: 'Joy',
   },
-  Timore: {
-    'browOuterUpLeft': -0.5,
-    'browOuterUpRight': -0.5,
-    'eyeWideLeft': -0.5,
-    'eyeWideRight': -0.5,
+  {
+    italian: 'Rabbia',
+    english: 'Anger',
   },
-  Tristezza: {
-    'browDownLeft': -0.5,
-    'browDownRight': -0.5,
-    'eyeSquintLeft': 0.5,
-    'eyeSquintRight': 0.5,
-    'mouthSmile': -0.6,
+  {
+    italian: 'Sorpresa',
+    english: 'Surprise',
   },
-  Sorpresa: {
-    'browInnerUp': 0.5,
-    'browOuterUpLeft': 0.5,
-    'browOuterUpRight': 0.5,
-    'eyeWideLeft': 0.5,
-    'eyeWideRight': 0.5,
+  {
+    italian: 'Tristezza',
+    english: 'Sadness',
   },
-  Gioia: {
-    'browDownLeft': 0.5,
-    'browDownRight': 0.5,
-    'browInnerUp': 0.5,
-    'mouthSmile': 0.8,
+  {
+    italian: 'Timore',
+    english: 'Fear',
   },
-};
+];
 
-// Mapping of blend shapes to emotions for custom GLB
-export const MAPPING_BLEND_SHAPE_TO_EMOTION_CUSTOM_GLB = {
-  Gioia: 'Joy',
-  Rabbia: 'Anger',
-  Sorpresa: 'Surprise',
-  Tristezza: 'Sadness',
-  Timore: 'Fear',
-};
+// Mapping of blend shapes to emotions for RPM avatars
+export const MAPPING_BLEND_SHAPE_TO_EMOTION_RPM: BlendShapeMap = [
+  {
+    emotion: { italian: 'Rabbia', english: 'Anger' },
+    blendShapes: {
+      'browDownLeft': 0.5,
+      'browDownRight': 0.5,
+      'browOuterUpLeft': 0.5,
+      'browOuterUpRight': 0.5,
+      'mouthSmile': -0.2,
+    },
+  },
+  {
+    emotion: { italian: 'Timore', english: 'Fear' },
+    blendShapes: {
+      'browOuterUpLeft': -0.5,
+      'browOuterUpRight': -0.5,
+      'eyeWideLeft': -0.5,
+      'eyeWideRight': -0.5,
+    },
+  },
+  {
+    emotion: { italian: 'Tristezza', english: 'Sadness' },
+    blendShapes: {
+      'browDownLeft': -0.5,
+      'browDownRight': -0.5,
+      'eyeSquintLeft': 0.5,
+      'eyeSquintRight': 0.5,
+      'mouthSmile': -0.6,
+    },
+  },
+  {
+    emotion: { italian: 'Sorpresa', english: 'Surprise' },
+    blendShapes: {
+      'browInnerUp': 0.5,
+      'browOuterUpLeft': 0.5,
+      'browOuterUpRight': 0.5,
+      'eyeWideLeft': 0.5,
+      'eyeWideRight': 0.5,
+    },
+  },
+  {
+    emotion: { italian: 'Gioia', english: 'Joy' },
+    blendShapes: {
+      'browDownLeft': 0.5,
+      'browDownRight': 0.5,
+      'browInnerUp': 0.5,
+      'mouthSmile': 0.8,
+    },
+  },
+];
 
 // URL for the male avatar
 export const ANIMATION_URLS = {

package/src/components/Avatar/AvatarView/index.tsx

@@ -79,12 +79,10 @@ export default function ContainerAvatarView({
   fallbackImg,
   halfBody = true,
   loading,
-  animation,
+  // animation,
   showControls = false,
   isZoomed,
   chatEmission,
-  stopProcessing,
-  resetVisemeQueue,
   updateCurrentViseme,
   enablePositionControls,
   setEnablePositionControls,
@@ -179,16 +177,12 @@ export default function ContainerAvatarView({
             sex={sex}
             showControls={showControls}
             loading={loading || false}
-            animation={animation}
-            isZoomed={isZoomed || false}
             eyeBlink={eyeBlink || false}
             headMovement={headMovement || false}
             speaking={speaking || false}
             halfBody={halfBody}
             chatEmission={chatEmission}
             updateCurrentViseme={updateCurrentViseme}
-            stopProcessing={stopProcessing}
-            resetVisemeQueue={resetVisemeQueue}
             setCameraZ={setCameraZ}
             avatarHeight={avatarHeight}
             avatarDepth={avatarDepth}

package/src/components/ChatBubble/ChatBubble.css

@@ -98,6 +98,15 @@
     color: var(--memori-primary-text);
 }
 
+.memori-chat--bubble h1,
+.memori-chat--bubble h2,
+.memori-chat--bubble h3,
+.memori-chat--bubble h4,
+.memori-chat--bubble h5,
+.memori-chat--bubble h6 {
+    color: inherit;
+}
+
 @media (max-width: 600px) {
     .memori-chat--bubble p {
         font-size: 12px;

package/src/components/ChatBubble/ChatBubble.tsx

@@ -396,11 +396,23 @@ const ChatBubble: React.FC<Props> = ({
               {message.generatedByAI && showAIicon && (
                 <Tooltip
                   align="left"
-                  content={t('generatedByAI')}
+                  content={
+                    t('generatedByAI') ||
+                    (lang === 'it'
+                      ? 'Risposta generata da IA, può talvolta generare informazioni non corrette'
+                      : 'Answer generated by AI, may occasionally generate incorrect informations')
+                  }
                   className="memori-chat--bubble-action-icon memori-chat--bubble-action-icon--ai"
                 >
                   <span>
-                    <AI title={t('generatedByAI') || undefined} />
+                    <AI
+                      title={
+                        t('generatedByAI') ||
+                        (lang === 'it'
+                          ? 'Risposta generata da IA, può talvolta generare informazioni non corrette'
+                          : 'Answer generated by AI, may occasionally generate incorrect informations')
+                      }
+                    />
                   </span>
                 </Tooltip>
               )}