@memori.ai/memori-react 6.8.1 → 6.8.4

package/src/components/Avatar/AvatarView/AvatarComponent/components/controllers/AvatarPositionController.ts

@@ -0,0 +1,83 @@
+import { Vector3, MathUtils } from 'three';
+import { AVATAR_POSITION, AVATAR_POSITION_ZOOMED } from '../../constants';
+
+export class AvatarPositionController {
+  private currentScale: Vector3;
+  private targetScale: Vector3;
+  private currentPosition: Vector3;
+  private basePosition: Vector3;
+  private defaultPosition: Vector3;
+  private zoomedPosition: Vector3;
+  private initialCameraPosition: Vector3;
+
+  constructor(
+    defaultPosition: Vector3 = AVATAR_POSITION.clone(),
+    zoomedPosition: Vector3 = AVATAR_POSITION_ZOOMED.clone(),
+    initialCameraZ: number = 0.6
+  ) {
+    this.defaultPosition = defaultPosition;
+    this.zoomedPosition = zoomedPosition;
+    this.currentScale = new Vector3(1, 1, 1);
+    this.targetScale = new Vector3(1, 1, 1);
+    this.currentPosition = defaultPosition.clone();
+    this.basePosition = defaultPosition.clone();
+    this.initialCameraPosition = new Vector3(0, 0, initialCameraZ);
+  }
+
+  // Map height slider value (0 to 100) to scale (0.8 to 1.2)
+  private mapHeightToScale(sliderValue: number, isHalfBody: boolean): number {
+    // Convert slider value to scale factor
+    if (isHalfBody) {
+      return MathUtils.lerp(1.4, 2.1, sliderValue / 100);
+    } else {
+      return MathUtils.lerp(0.5, 1.5, sliderValue / 100);
+    }
+  }
+
+  // Map depth slider value (0 to 100) to camera Z position
+  private mapDepthToCamera(depthValue: number, isHalfBody: boolean): number {
+    const baseZ = this.initialCameraPosition.z;
+    if (isHalfBody) {
+      return MathUtils.lerp(baseZ, baseZ + 3, depthValue / 100);
+    } else {
+      return MathUtils.lerp(baseZ, baseZ + 3, depthValue / 100);
+    }
+  }
+
+  // Update height from GUI control (0-100)
+  updateHeight(heightValue: number, isHalfBody: boolean): void {
+    const heightScale = this.mapHeightToScale(heightValue, isHalfBody);
+    this.targetScale.set(heightScale, heightScale, heightScale);
+  }
+
+  // Update depth and return new camera Z position
+  updateDepth(depthValue: number, isHalfBody: boolean): number {
+    return this.mapDepthToCamera(depthValue, isHalfBody);
+  }
+
+  // Update base position for zooming
+  updateBasePosition(isZoomed: boolean): void {
+    const newPosition = isZoomed ? this.zoomedPosition : this.defaultPosition;
+    this.basePosition.copy(newPosition);
+    this.currentPosition.copy(newPosition);
+  }
+
+  // Smoothly interpolate current scale to target scale
+  updateScale(lerpFactor: number): Vector3 {
+    this.currentScale.lerp(this.targetScale, lerpFactor);
+    return this.currentScale;
+  }
+
+  // Get current position
+  getPosition(): Vector3 {
+    return this.currentPosition;
+  }
+
+  // Reset to default values
+  reset(): void {
+    this.currentScale.set(1, 1, 1);
+    this.targetScale.set(1, 1, 1);
+    this.currentPosition.copy(this.defaultPosition);
+    this.basePosition.copy(this.defaultPosition);
+  }
+}

package/src/components/Avatar/AvatarView/AvatarComponent/components/controllers/MorphTargetController.ts

@@ -0,0 +1,301 @@
+import { SkinnedMesh } from 'three';
+import { MathUtils } from 'three';
+import {
+  EMOTION_SMOOTHING,
+  VISEME_SMOOTHING,
+  BLINK_CONFIG,
+  MAPPING_EMOTIONS_ITALIAN_TO_ENGLISH,
+  MAPPING_BLEND_SHAPE_TO_EMOTION_RPM,
+  EmotionMapping,
+  BlendShapeMap,
+} from '../../constants';
+
+/**
+ * Controller class for handling morph target animations including emotions, visemes and blinking
+ */
+export class MorphTargetController {
+  private headMesh: SkinnedMesh;
+  private currentEmotionValues: Record<string, number> = {};
+  private previousEmotionKeys: Set<string> = new Set();
+  private isRPM: boolean = false;
+
+  // Default RPM blend shape mappings
+  private rpmBlendShapes: BlendShapeMap =
+    MAPPING_BLEND_SHAPE_TO_EMOTION_RPM as BlendShapeMap;
+
+  // Default custom GLB emotion mappings
+  private customGlbMapping: EmotionMapping =
+    MAPPING_EMOTIONS_ITALIAN_TO_ENGLISH as EmotionMapping;
+
+  /**
+   * Constructor for MorphTargetController
+   * @param headMesh SkinnedMesh representing the head
+   * @param rpmBlendShapes Optional custom RPM blend shapes
+   * @param customGlbMapping Optional custom GLB emotion mapping
+   */
+  constructor(
+    headMesh: SkinnedMesh,
+    rpmBlendShapes?: BlendShapeMap,
+    customGlbMapping?: EmotionMapping
+  ) {
+    this.headMesh = headMesh;
+
+    // Detect if this is an RPM avatar based on mesh name
+    this.isRPM = headMesh.name !== 'GBNL__Head';
+
+    // Override default maps if provided
+    if (rpmBlendShapes) {
+      this.rpmBlendShapes = rpmBlendShapes;
+    }
+
+    if (customGlbMapping) {
+      this.customGlbMapping = customGlbMapping;
+    }
+  }
+
+  /**
+   * Process chat emission to extract emotion data
+   * @param chatEmission Chat emission text
+   * @param isLoading Loading state
+   * @returns Object with emotion morph targets
+   */
+  processChatEmission(
+    chatEmission: any,
+    isLoading: boolean
+  ): Record<string, number> {
+    // Default empty emotion targets
+    const defaultEmotions = this.getDefaultEmotionMorphTargets();
+
+    // If loading or no chat emission, return default emotions
+    if (isLoading || !chatEmission) {
+      return defaultEmotions;
+    }
+
+    // Check if chat emission contains emotion tag
+    const hasOutputTagEmotion = chatEmission?.includes(
+      '<output class="memori-emotion">'
+    );
+
+    if (!hasOutputTagEmotion) {
+      return defaultEmotions;
+    }
+
+    // Extract emotion name
+    const outputContentEmotion = chatEmission
+      ?.split('<output class="memori-emotion">')[1]
+      ?.split('</output>')[0]
+      ?.trim();
+
+    if (!outputContentEmotion) {
+      return defaultEmotions;
+    }
+
+    // Process emotion based on avatar type
+    return this.processEmotion(outputContentEmotion);
+  }
+
+  /**
+   * Process an emotion name into morph target values
+   * @param emotionName Emotion name (in Italian or English)
+   * @returns Object with morph target values
+   */
+  private processEmotion(emotionName: string): Record<string, number> {
+    // Get default empty emotion targets
+    const defaultEmotions = this.getDefaultEmotionMorphTargets();
+
+    // First, try to find the emotion regardless of language
+    if (this.isRPM) {
+      // For RPM avatars, find the matching emotion in the rpmBlendShapes array
+      const foundEmotion = this.rpmBlendShapes.find(
+        item =>
+          item.emotion.italian.toLowerCase() === emotionName.toLowerCase() ||
+          item.emotion.english.toLowerCase() === emotionName.toLowerCase()
+      );
+
+      if (foundEmotion) {
+        return { ...defaultEmotions, ...foundEmotion.blendShapes };
+      }
+    } else {
+      // For custom GLB, find the matching emotion in the customGlbMapping array
+      const foundEmotion = this.customGlbMapping.find(
+        item =>
+          item.italian.toLowerCase() === emotionName.toLowerCase() ||
+          item.english.toLowerCase() === emotionName.toLowerCase()
+      );
+
+      if (foundEmotion) {
+        return { ...defaultEmotions, [foundEmotion.english]: 1 };
+      }
+    }
+
+    console.log('[MorphTargetController] No emotion found:', emotionName);
+    return defaultEmotions;
+  }
+
+  /**
+   * Get default emotion morph targets (all set to 0)
+   */
+  private getDefaultEmotionMorphTargets(): Record<string, number> {
+    // For RPM, collect all blend shape keys
+    if (this.isRPM) {
+      const allBlendShapeKeys = new Set<string>();
+
+      this.rpmBlendShapes.forEach(item => {
+        Object.keys(item.blendShapes).forEach(key =>
+          allBlendShapeKeys.add(key)
+        );
+      });
+
+      return Array.from(allBlendShapeKeys).reduce(
+        (acc, key) => ({ ...acc, [key]: 0 }),
+        {}
+      );
+    }
+    // For custom GLB, use English emotion names
+    else {
+      return this.customGlbMapping.reduce(
+        (acc, emotion) => ({ ...acc, [emotion.english]: 0 }),
+        {}
+      );
+    }
+  }
+  /**
+   * Updates the morph target influences for emotions, visemes and blinking
+   */
+  updateMorphTargets(
+    currentTime: number,
+    chatEmission: any,
+    isLoading: boolean,
+    currentViseme: { name: string; weight: number } | null,
+    eyeBlink: boolean,
+    blinkState: {
+      isBlinking: boolean;
+      lastBlinkTime: number;
+      nextBlinkTime: number;
+      blinkStartTime: number;
+    }
+  ) {
+    // Validate required mesh properties exist
+    if (
+      !this.headMesh.morphTargetDictionary ||
+      !this.headMesh.morphTargetInfluences
+    ) {
+      console.error(
+        '[MorphTargetController] Missing morphTargetDictionary or morphTargetInfluences'
+      );
+      return;
+    }
+
+    // Process chat emission to get emotion morph targets
+    const emotionMorphTargets = this.processChatEmission(
+      chatEmission,
+      isLoading
+    );
+
+    // Calculate blink value for this frame
+    const blinkValue = this.calculateBlinkValue(
+      currentTime,
+      blinkState,
+      eyeBlink
+    );
+
+    const currentEmotionKeys = new Set(Object.keys(emotionMorphTargets));
+
+    // Process each morph target
+    Object.entries(this.headMesh.morphTargetDictionary).forEach(
+      ([key, index]) => {
+        if (typeof index !== 'number') return;
+
+        let targetValue = 0;
+
+        // Handle emotion morphs with smoothing
+        if (currentEmotionKeys.has(key)) {
+          // console.log('[MorphTargetController] Processing morph target:', key);
+          const targetEmotionValue = emotionMorphTargets[key];
+          const currentEmotionValue = this.currentEmotionValues[key] || 0;
+          const newEmotionValue = MathUtils.lerp(
+            currentEmotionValue,
+            targetEmotionValue * 3, // Amplify emotion by 3x
+            EMOTION_SMOOTHING
+          );
+          this.currentEmotionValues[key] = newEmotionValue;
+          targetValue += newEmotionValue;
+        }
+
+        // Add viseme influence if active
+        if (currentViseme && key === currentViseme.name) {
+          targetValue += currentViseme.weight;
+        }
+
+        // Add blink influence if active
+        if (key === 'eyesClosed' && eyeBlink) {
+          targetValue += blinkValue;
+        }
+
+        // Clamp and smooth final value
+        targetValue = MathUtils.clamp(targetValue, 0, 1);
+        if (this.headMesh.morphTargetInfluences) {
+          const finalValue = MathUtils.lerp(
+            this.headMesh.morphTargetInfluences[index] || 0,
+            targetValue,
+            VISEME_SMOOTHING
+          );
+          this.headMesh.morphTargetInfluences[index] = finalValue;
+        }
+      }
+    );
+
+    this.previousEmotionKeys = currentEmotionKeys;
+  }
+
+  /**
+   * Calculates the blink value based on timing and state
+   */
+  private calculateBlinkValue(
+    currentTime: number,
+    blinkState: {
+      isBlinking: boolean;
+      lastBlinkTime: number;
+      nextBlinkTime: number;
+      blinkStartTime: number;
+    },
+    eyeBlink: boolean
+  ): number {
+    if (!eyeBlink) return 0;
+
+    let blinkValue = 0;
+
+    // Start new blink if it's time
+    if (currentTime >= blinkState.nextBlinkTime && !blinkState.isBlinking) {
+      blinkState.isBlinking = true;
+      blinkState.blinkStartTime = currentTime;
+      blinkState.lastBlinkTime = currentTime;
+      blinkState.nextBlinkTime =
+        currentTime +
+        Math.random() * (BLINK_CONFIG.maxInterval - BLINK_CONFIG.minInterval) +
+        BLINK_CONFIG.minInterval;
+    }
+
+    // Calculate blink animation progress
+    if (blinkState.isBlinking) {
+      const blinkProgress =
+        (currentTime - blinkState.blinkStartTime) / BLINK_CONFIG.blinkDuration;
+
+      // First half of blink - closing eyes
+      if (blinkProgress <= 0.5) {
+        blinkValue = blinkProgress * 2;
+      }
+      // Second half of blink - opening eyes
+      else if (blinkProgress <= 1) {
+        blinkValue = 2 - blinkProgress * 2;
+      }
+      // Blink complete
+      else {
+        blinkState.isBlinking = false;
+        blinkValue = 0;
+      }
+    }
+
+    return blinkValue;
+  }
+}

package/src/components/Avatar/AvatarView/AvatarComponent/components/controls.tsx

@@ -0,0 +1,100 @@
+import React, { useEffect, useRef, useState } from 'react';
+import GUI from 'lil-gui';
+
+interface BaseAction {
+  weight: number;
+  action?: string;
+}
+
+interface AdditiveAction {
+  weight: number;
+  action?: string;
+}
+
+export interface AnimationControlPanelProps {
+  baseActions: Record<string, BaseAction>;
+  onBaseActionChange: (action: string, outputContent: string) => void;
+  currentBaseAction: {
+    action: string;
+    weight: number;
+  };
+  onMorphTargetInfluencesChange: (influences: { [key: string]: number }) => void;
+  onMorphTargetDictionaryChange: (dictionary: { [key: string]: number }) => void;
+  morphTargetDictionary: { [key: string]: number };
+  modifyTimeScale: (value: number) => void;
+  timeScale: number;
+}
+
+const AnimationControlPanel: React.FC<AnimationControlPanelProps> = ({
+  onBaseActionChange,
+  baseActions,
+  modifyTimeScale,
+  onMorphTargetInfluencesChange,
+  onMorphTargetDictionaryChange,
+  morphTargetDictionary,
+  timeScale,
+}) => {
+  const guiRef = useRef<GUI | null>(null);
+  const panelSettingsRef = useRef<Record<string, any>>({
+    'modify time scale': timeScale,
+  });
+  const crossFadeControlsRef = useRef<any[]>([]);
+
+  useEffect(() => {
+
+    const gui = new GUI({ width: 310 });
+    guiRef.current = gui;
+
+    const folder1 = gui.addFolder('Base Actions');
+    const folder2 = gui.addFolder('Additive Action Weights');
+    const folder3 = gui.addFolder('General Speed');
+
+    const baseNames = ['None', ...Object.keys(baseActions)];
+
+    baseNames.forEach(name => {
+      const settings = baseActions[name];
+      panelSettingsRef.current[name] = () => {
+        onBaseActionChange(name, '');
+      };
+
+      const control = folder1.add(panelSettingsRef.current, name);
+      crossFadeControlsRef.current.push(control);
+    });
+
+    Object.entries(morphTargetDictionary).forEach(([name, settings]) => {
+      panelSettingsRef.current[name] = settings/100;
+      folder2
+        .add(panelSettingsRef.current, name, -1.0, 1.0, 0.01)
+        .listen()
+        .onChange((weight: number) => {
+          onMorphTargetInfluencesChange({ [name]: weight });
+        });
+    });
+
+    folder3
+      .add(panelSettingsRef.current, 'modify time scale', 0.0, 1.5, 0.01)
+      .onChange((value: number) => {
+        modifyTimeScale(value);
+      });
+
+    folder1.open();
+    folder2.open();
+    folder3.open();
+
+    return () => {
+      gui.destroy();
+    };
+  }, [
+    onBaseActionChange,
+    onMorphTargetInfluencesChange,
+    onMorphTargetDictionaryChange,
+    modifyTimeScale,
+    baseActions,
+    morphTargetDictionary,
+    timeScale
+  ]);
+
+  return null; // This component doesn't render anything itself
+};
+
+export default AnimationControlPanel;

package/src/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.tsx

@@ -0,0 +1,152 @@
+import { useEffect, useMemo, useRef } from 'react';
+import { Object3D, SkinnedMesh } from 'three';
+import { useGLTF } from '@react-three/drei';
+import { useGraph, useFrame, useThree } from '@react-three/fiber';
+import { correctMaterials, isSkinnedMesh } from '../../../../../helpers/utils';
+import { MorphTargetController } from './controllers/MorphTargetController';
+import { AvatarPositionController } from './controllers/AvatarPositionController';
+import {
+  AVATAR_POSITION,
+  SCALE_LERP_FACTOR,
+  AVATAR_POSITION_ZOOMED,
+} from '../constants';
+import { hideHands } from '../../utils/utils';
+import useHeadMovement from '../../utils/useHeadMovement';
+
+interface HalfBodyAvatarProps {
+  url: string;
+  updateCurrentViseme: (currentTime: number) => any;
+  eyeBlink?: boolean;
+  avatarHeight: number;
+  avatarDepth: number;
+  onLoaded?: () => void;
+  onCameraZChange: (value: number) => void;
+  headMovement?: boolean;
+  chatEmission?: any;
+  loading?: boolean;
+}
+
+export default function HalfBodyAvatar({
+  url,
+  updateCurrentViseme,
+  eyeBlink = false,
+  avatarHeight = 50,
+  avatarDepth = 0,
+  headMovement = false,
+  onLoaded,
+  onCameraZChange,
+  chatEmission,
+  loading = false,
+}: HalfBodyAvatarProps) {
+  const { scene } = useGLTF(url);
+  const { nodes, materials } = useGraph(scene);
+  const { camera } = useThree();
+
+  const morphTargetControllerRef = useRef<MorphTargetController | null>(null);
+  const positionControllerRef = useRef<AvatarPositionController | null>(null);
+  const targetCameraZRef = useRef(camera.position.z);
+
+  // Apply head movement if enabled
+  useHeadMovement(headMovement, nodes);
+  
+
+  // Eye blinking state
+  const blinkStateRef = useRef({
+    isBlinking: false,
+    lastBlinkTime: 0,
+    nextBlinkTime: 0,
+    blinkStartTime: 0,
+  });
+
+  // Find head mesh
+  const headMesh = useMemo(() => {
+    let foundMesh: SkinnedMesh | undefined;
+    scene?.traverse((object: Object3D) => {
+      if (
+        object instanceof SkinnedMesh &&
+        (object.name === 'GBNL__Head' || object.name === 'Wolf3D_Avatar' || object.name === 'Wolf3D_Avatar006_1')
+      ) {
+        foundMesh = object;
+      }
+    });
+    return foundMesh;
+  }, [scene]);
+
+  // Initialize controllers
+  useEffect(() => {
+    if (!positionControllerRef.current) {
+      positionControllerRef.current = new AvatarPositionController(
+        AVATAR_POSITION,
+        AVATAR_POSITION_ZOOMED,
+      );
+    }
+
+    // Initialize MorphTargetController if head mesh exists
+    if (headMesh) {
+      morphTargetControllerRef.current = new MorphTargetController(headMesh);
+    }
+
+    // Correct materials and perform other initialization tasks
+    correctMaterials(materials);
+    onLoaded?.();
+    hideHands(nodes);
+    
+    // Cleanup on unmount
+    return () => {
+      Object.values(materials).forEach(material => material.dispose());
+      Object.values(nodes)
+        .filter(isSkinnedMesh)
+        .forEach(mesh => mesh.geometry.dispose());
+    };
+  }, [materials, nodes, url, onLoaded, scene, headMesh]);
+
+  // Handle avatar height changes
+  useEffect(() => {
+    if (positionControllerRef.current) {
+      positionControllerRef.current.updateHeight(avatarHeight, true);
+    }
+  }, [avatarHeight]);
+
+  // Handle avatar depth changes
+  useEffect(() => {
+    if (positionControllerRef.current && onCameraZChange) {
+      const newCameraZ = positionControllerRef.current.updateDepth(avatarDepth, true);
+      onCameraZChange(newCameraZ);
+    }
+  }, [avatarDepth, onCameraZChange]);
+
+  // Animation and morphing update loop
+  useFrame((state) => {
+    const currentTime = state.clock.elapsedTime * 1000;
+
+    // Update morph targets
+    if (morphTargetControllerRef.current) {
+      const currentViseme = updateCurrentViseme(currentTime / 1000);
+      
+      // Use the updated MorphTargetController that handles chat emission directly
+      morphTargetControllerRef.current.updateMorphTargets(
+        currentTime,
+        chatEmission,
+        loading,
+        currentViseme,
+        eyeBlink,
+        blinkStateRef.current,
+      );
+    }
+
+    // Update scale with smooth transition
+    if (scene && positionControllerRef.current) {
+      const newScale = positionControllerRef.current.updateScale(SCALE_LERP_FACTOR);
+      scene.scale.copy(newScale);
+    }
+  });
+
+  // Get current position from controller
+  const position = positionControllerRef.current?.getPosition() || AVATAR_POSITION;
+
+  return (
+    <group position={position}>
+      <primitive object={scene} />
+    </group>
+  );
+}

package/src/components/{AvatarView → Avatar/AvatarView/AvatarComponent}/components/loader.tsx

@@ -1,6 +1,6 @@
 import React from 'react';
 import { Html, useProgress } from '@react-three/drei';
-import Spin from '../../ui/Spin';
+import Spin from '../../../../ui/Spin';
 
 interface Props {
   fallbackImg?: string;