@memori.ai/memori-react 7.5.1 → 7.6.1

package/src/components/layouts/hidden-chat.css

@@ -0,0 +1,184 @@
+.memori-sidebar-container {
+    position: fixed;
+    z-index: 1000;
+    top: 0;
+    right: 0;
+    height: 100%;
+}
+
+.memori-sidebar-toggle {
+    display: none;
+}
+
+.memori-sidebar-toggle-label {
+    position: absolute;
+    top: 50%;
+    right: 0;
+    display: flex;
+    width: auto;
+    height: 80px;
+    align-items: center;
+    justify-content: center;
+    padding: 10px 5px;
+    background-color: var(--memori-primary);
+    color: white;
+    cursor: pointer;
+    font-size: 18px;
+    font-weight: bold;
+    text-orientation: mixed;
+    transform: translateY(-50%);
+    transition: all 0.3s ease-in-out;
+    writing-mode: vertical-rl;
+}
+
+.memori-open-label {
+    border-radius: 5px 0 0 5px;
+}
+
+.memori-close-label {
+    right: 350px;
+    border-radius: 0 5px 5px 0;
+    opacity: 0;
+    pointer-events: none;
+    transform: rotate(180deg);
+}
+
+.memori-sidebar {
+    position: fixed;
+    z-index: 1000;
+    top: 0;
+    right: -350px;
+    width: 350px;
+    height: 100%;
+    background-color: white;
+    box-shadow: -2px 0 5px rgba(0, 0, 0, 0.1);
+    transition: right 0.3s ease-in-out;
+}
+
+.memori-sidebar-content h2 {
+    margin-bottom: 20px;
+    color: var(--memori-primary);
+    font-size: 24px;
+    font-weight: bold;
+}
+
+.memori-sidebar-content ul {
+    padding: 0;
+    list-style-type: none;
+}
+
+.memori-sidebar-content li {
+    margin-bottom: 10px;
+}
+
+.memori-sidebar-content a {
+    display: flex;
+    align-items: center;
+    color: #4b5563;
+    text-decoration: none;
+    transition: color 0.2s ease-in-out;
+}
+
+.memori-sidebar-content a:hover {
+    color: var(--memori-primary);
+}
+
+.memori-sidebar-content svg {
+    width: 20px;
+    height: 20px;
+    margin-right: 10px;
+}
+
+.memori-sidebar-toggle:checked ~ .memori-sidebar-container .memori-sidebar {
+    right: 0;
+}
+
+.memori-sidebar-toggle:checked ~ .memori-sidebar-container .memori-open-label {
+    opacity: 0;
+    pointer-events: none;
+    transform: translateY(-50%) translateX(350px);
+}
+
+.memori-loading {
+    display: flex;
+    height: 100%;
+    align-items: center;
+    justify-content: center;
+}
+
+
+.memori-sidebar-content {
+    position: absolute;
+    top: -10px;
+    width: 100%;
+    padding: 20px;
+}
+
+.memori-sidebar-toggle:checked ~ .memori-sidebar-container .memori-close-label {
+    opacity: 1;
+    pointer-events: auto;
+}
+
+.memori-hidden-chat-layout--controls {
+    display: flex;
+    height: 90%;
+    padding: 10px;
+    margin-top: 50px;
+}
+
+.memori-hidden-chat-layout-header--layout {
+    display: flex;
+    width: 100%;
+    align-items: center;
+    justify-content: flex-end;
+    padding: 10px;
+    margin: 0;
+    background-color: white;
+    color: white;
+}
+
+.memori-hidden-chat-layout-header--layout > button, .memori-hidden-chat-layout-header--layout > div > button {
+    display: flex;
+    width: 38px;
+    height: 38px;
+    align-items: center;
+    justify-content: center;
+    border-radius: 50%;
+    aspect-ratio: 1;
+    background-color: var(--memori-primary);
+    cursor: pointer;
+    transition: background-color 0.3s ease-in-out;
+}
+
+.memori-button--icon > svg {
+    margin: 0;
+}
+
+.memori-icon,.memori-icon-close {
+    width: 25px;
+    height: 25px;
+}
+
+.memori-icon-close{
+  fill: white;
+}
+
+
+@media (max-width: 768px) {
+    .memori-sidebar-container {
+        display: none;
+    }
+
+    .memori-sidebar {
+        right: -100%;
+        width: 100%;
+    }
+
+    .memori-sidebar-toggle:checked ~ .memori-sidebar-container .memori-open-label {
+        transform: translateY(-50%) translateX(100%);
+    }
+
+    .memori-close-label {
+        right: 100%;
+    }
+}

package/src/components/layouts/layouts.stories.tsx

@@ -4,6 +4,7 @@ import { memori, tenant, integration } from '../../mocks/data';
 import Memori, { LayoutProps, Props } from '../MemoriWidget/MemoriWidget';
 import I18nWrapper from '../../I18nWrapper';
 import Spin from '../ui/Spin';
+import { VisemeProvider } from '../../context/visemeContext';
 
 const meta: Meta = {
   title: 'General/Layouts',
@@ -62,7 +63,9 @@ export const CustomLayout: React.FC<LayoutProps> = ({
 
 const Template: Story<Props> = args => (
   <I18nWrapper>
-    <Memori {...args} />
+    <VisemeProvider>
+      <Memori {...args} />
+    </VisemeProvider>
   </I18nWrapper>
 );
 // By passing using the Args format for exported stories, you can control the props for a component for reuse in a test
@@ -197,8 +200,20 @@ ChatOnly.args = {
   layout: 'CHAT',
 };
 
-export const Totem = Template.bind({});
-Totem.args = {
+
+export const Custom = Template.bind({});
+Custom.args = {
+  uiLang: 'it',
+  showShare: true,
+  showSettings: true,
+  memori,
+  tenant,
+  layout: 'FULLPAGE',
+  customLayout: CustomLayout,
+};
+
+export const WebsiteAssistant = Template.bind({});
+WebsiteAssistant.args = {
   uiLang: 'it',
   showShare: true,
   showSettings: true,
@@ -303,25 +318,16 @@ Totem.args = {
     }),
   },
   tenant,
-  layout: 'TOTEM',
-};
-
-export const Custom = Template.bind({});
-Custom.args = {
-  uiLang: 'it',
-  showShare: true,
-  showSettings: true,
-  memori,
-  tenant,
-  layout: 'FULLPAGE',
-  customLayout: CustomLayout,
+  layout: 'WEBSITE_ASSISTANT',
 };
 
-export const WebsiteAssistant = Template.bind({});
-WebsiteAssistant.args = {
+export const HiddenChat = Template.bind({});
+HiddenChat.args = {
   uiLang: 'it',
   showShare: true,
   showSettings: true,
+  // memori,
+  // tenant,
   memori: {
     memoriID: '6573844d-a7cd-47ef-9e78-840d82020c21',
     name: 'Nicola',
@@ -422,6 +428,116 @@ WebsiteAssistant.args = {
         new Date(Date.now()).getTime(),
     }),
   },
-  tenant,
-  layout: 'WEBSITE_ASSISTANT',
+  layout: 'HIDDEN_CHAT',
+};
+
+
+export const ZoomedFullBody = Template.bind({});
+ZoomedFullBody.args = {
+  uiLang: 'it',
+  showShare: true,
+  showSettings: true,
+  showAudio: true,
+  enableAudio: true,
+  memori: {
+    memoriID: '6573844d-a7cd-47ef-9e78-840d82020c21',
+    name: 'Nicola',
+    password: null,
+    recoveryTokens: null,
+    newPassword: null,
+    ownerUserID: null,
+    ownerUserName: 'nzambello',
+    ownerTenantName: 'aisuru.com',
+    memoriConfigurationID: 'fd10bb42-98d9-4c08-8e02-2b08bd4e4975',
+    description:
+      'Sono Nicola Zambello, sviluppatore e attivista per un web etico e sostenibile',
+    completionDescription: null,
+    engineMemoriID: '9b0a2913-d3d8-4e98-a49d-6e1c99479e1b',
+    isOwner: false,
+    isGiver: false,
+    isReceiver: false,
+    giverTag: null,
+    giverPIN: null,
+    privacyType: 'PUBLIC',
+    enableAudio: true,
+    secretToken: null,
+    minimumNumberOfRecoveryTokens: null,
+    totalNumberOfRecoveryTokens: null,
+    sentInvitations: [],
+    receivedInvitations: [],
+    integrations: [
+      {
+        integrationID: '62de8c99-0ac2-4cbe-bd95-a39ad7dc6b32',
+        memoriID: '6573844d-a7cd-47ef-9e78-840d82020c21',
+        type: 'LANDING_EXPERIENCE',
+        state: 'NEW',
+        deviceEmails: null,
+        invocationText: null,
+        jobID: null,
+        customData:
+          '{"textColor":"#000000","buttonBgColor":"#007eb6","buttonTextColor":"#ffffff","globalBackground":"https://assets.memori.ai/api/v2/asset/cade3b9c-0437-4342-b2bd-8db9c2a3a20e.png","blurBackground":true,"innerBgColor":"light","multilanguage":true,"avatar":"readyplayerme","avatarURL":"https://assets.memori.ai/api/v2/asset/893c41df-7619-436d-9e86-fe1d406fc933.glb#1681736752156","name":"Pagina pubblica","contextVars":"ANIMALE:CANE","personTag":"☠️","personPIN":"666666","personName":"Pirata","showShare":true,"avatarFullBodyURL":"https://models.readyplayer.me/63b55751f17e295642bf07a2.glb"}',
+        resources: [],
+        publish: true,
+        creationTimestamp: '2022-06-13T14:44:52.833573Z',
+        lastChangeTimestamp: '2022-06-13T14:44:52.833573Z',
+      },
+    ],
+    avatarURL:
+      'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
+    coverURL:
+      'https://assets.memori.ai/api/v2/asset/e9bb9f6d-8f34-45ab-af9e-6d630d9a51a8.png',
+    avatar3DURL:
+      'https://assets.memori.ai/api/v2/asset/3932bf70-e953-4e8a-b63a-f316544c283e.glb',
+    avatarOriginal3DURL:
+      'https://assets.memori.ai/api/v2/asset/3932bf70-e953-4e8a-b63a-f316544c283e.glb',
+    needsPosition: false,
+    voiceType: 'FEMALE',
+    culture: 'it-IT',
+    categories: [
+      'biografico',
+      'tecnologia',
+      'web',
+      'open-source',
+      'green',
+      'privacy',
+    ],
+    exposed: true,
+    disableR2R3Loop: null,
+    disableR4Loop: null,
+    disableR5Loop: null,
+    enableCompletions: true,
+    completionModel: null,
+    chainingMemoriID: null,
+    chainingBaseURL: null,
+    chainingPassword: null,
+    contentQualityIndex: 210.8,
+    contentQualityIndexTimestamp: '2023-04-17T00:01:32.194744Z',
+    publishedInTheMetaverse: true,
+    metaverseEnvironment: 'apartment',
+    blockedUntil: null,
+    creationTimestamp: '2022-06-13T14:21:55.793034Z',
+    lastChangeTimestamp: '2023-04-15T08:15:36.403546Z',
+  },
+  integration: {
+    integrationID: '62de8c99-0ac2-4cbe-bd95-a39ad7dc6b32',
+    memoriID: '6573844d-a7cd-47ef-9e78-840d82020c21',
+    type: 'LANDING_EXPERIENCE',
+    state: 'NEW',
+    deviceEmails: null,
+    invocationText: null,
+    jobID: null,
+    publish: true,
+    creationTimestamp: '2022-06-13T14:44:52.833573Z',
+    lastChangeTimestamp: '2022-06-13T14:44:52.833573Z',
+    customData: JSON.stringify({
+      ...JSON.parse(
+        '{"textColor":"#000000","buttonBgColor":"#007eb6","buttonTextColor":"#ffffff","globalBackground":"https://assets.memori.ai/api/v2/asset/cade3b9c-0437-4342-b2bd-8db9c2a3a20e.png","blurBackground":true,"innerBgColor":"light","multilanguage":true,"avatar":"readyplayerme","avatarURL":"https://assets.memori.ai/api/v2/asset/893c41df-7619-436d-9e86-fe1d406fc933.glb#1681736752156","name":"Pagina pubblica","contextVars":"ANIMALE:CANE","personTag":"☠️","personPIN":"666666","personName":"Pirata","showShare":true,"avatarFullBodyURL":"https://models.readyplayer.me/63b55751f17e295642bf07a2.glb"}'
+      ),
+      avatar: 'readyplayerme-full',
+      avatarURL:
+        'https://assets.memori.ai/api/v2/asset/3932bf70-e953-4e8a-b63a-f316544c283e.glb'+
+        new Date(Date.now()).getTime(),
+    }),
+  },
+  layout: 'ZOOMED_FULL_BODY',
 };

package/src/context/visemeContext.tsx

@@ -0,0 +1,328 @@
+import React, {
+  createContext,
+  useContext,
+  useState,
+  useCallback,
+  useRef,
+  useEffect,
+} from 'react';
+import { SkinnedMesh } from 'three';
+
+type AzureViseme = { visemeId: number; audioOffset: number };
+
+type ProcessedViseme = {
+  name: string;
+  duration: number;
+  weight: number;
+  startTime: number;
+};
+
+interface VisemeContextType {
+  setMeshRef: (mesh: SkinnedMesh | null) => void;
+  addVisemeToQueue: (viseme: AzureViseme) => void;
+  processVisemeQueue: () => ProcessedViseme[];
+  clearVisemes: () => void;
+  isMeshSet: boolean;
+  setEmotion: (emotion: string) => void;
+  emotion: string;
+  getAzureStyleForEmotion: (emotion: string) => string;
+}
+
+const VisemeContext = createContext<VisemeContextType | undefined>(undefined);
+
+const VISEME_SMOOTHING = 0.5;
+const DEFAULT_VISEME_DURATION = 0.1;
+const MINIMUM_ELAPSED_TIME = 0.01;
+const VISEME_SPEED_FACTOR = 1.0;
+const AUDIO_PLAYBACK_RATE = 1.0;
+const VISEME_BASE_SPEED = 1.0;
+
+const VISEME_MAP: { [key: number]: string } = {
+  0: 'viseme_sil', // silence
+  1: 'viseme_PP', // p, b, m
+  2: 'viseme_FF', // f, v
+  3: 'viseme_TH', // th, dh
+  4: 'viseme_DD', // t, d, n, l
+  5: 'viseme_kk', // k, g, ng
+  6: 'viseme_CH', // tS, dZ, S, Z
+  7: 'viseme_SS', // s, z
+  8: 'viseme_nn', // Not explicitly defined in Azure mapping, keeping for compatibility
+  9: 'viseme_RR', // r
+  10: 'viseme_aa', // A:
+  11: 'viseme_E', // e
+  12: 'viseme_I', // I
+  13: 'viseme_O', // O
+  14: 'viseme_U', // u
+  // Mapping the rest based on closest matches or keeping them as in the original mapping
+  15: 'viseme_kk', // g, k (same as 5)
+  16: 'viseme_CH', // ch, j, sh, zh (same as 6)
+  17: 'viseme_SS', // s, z (same as 7)
+  18: 'viseme_TH', // th, dh (same as 3)
+  19: 'viseme_RR', // r (same as 9)
+  20: 'viseme_kk', // w (closest match, could be debated)
+  21: 'viseme_PP', // y (closest match, could be debated)
+};
+
+export const VisemeProvider: React.FC<{ children: React.ReactNode }> = ({
+  children,
+}) => {
+  const [isMeshSet, setIsMeshSet] = useState(false);
+  const [emotion, setEmotion] = useState('Neutral');
+  const isAnimatingRef = useRef(false);
+  const currentVisemesRef = useRef<ProcessedViseme[]>([]);
+  const visemeQueueRef = useRef<AzureViseme[]>([]);
+  const animationFrameRef = useRef<number | null>(null);
+  const startTimeRef = useRef<number | null>(null);
+  const currentVisemeWeightRef = useRef<{ [key: string]: number }>({});
+  const meshRef = useRef<SkinnedMesh | null>(null);
+
+  const lerp = (start: number, end: number, alpha: number): number => {
+    return start * (1 - alpha) + end * alpha;
+  };
+
+  const easeInOutCubic = (x: number): number => {
+    return x < 0.5 ? 4 * x * x * x : 1 - Math.pow(-2 * x + 2, 3) / 2;
+  };
+
+  const setMeshRef = useCallback(
+    (mesh: SkinnedMesh | null) => {
+      if (mesh && mesh.morphTargetDictionary && mesh.morphTargetInfluences) {
+        meshRef.current = mesh;
+        setIsMeshSet(true);
+        // console.log('Mesh set successfully:', mesh);
+      } else {
+        console.error('Invalid mesh provided:', mesh);
+      }
+    },
+    [meshRef]
+  );
+
+  const addVisemeToQueue = useCallback((viseme: AzureViseme) => {
+    visemeQueueRef.current.push(viseme);
+    // console.log('Viseme added to queue:', viseme);
+  }, []);
+
+  const getCurrentViseme = useCallback((elapsedTime: number) => {
+    if (elapsedTime < MINIMUM_ELAPSED_TIME) return null;
+
+    return currentVisemesRef.current.find((viseme, index) => {
+      const nextViseme = currentVisemesRef.current[index + 1];
+      return (
+        elapsedTime >= viseme.startTime &&
+        (!nextViseme || elapsedTime < nextViseme.startTime)
+      );
+    });
+  }, []);
+
+  const getDynamicSpeedFactor = (visemeDuration: number): number => {
+    const baseDuration = 0.1; // Average expected viseme duration
+    return (
+      VISEME_BASE_SPEED * (baseDuration / visemeDuration) * AUDIO_PLAYBACK_RATE
+    );
+  };
+
+  const applyViseme = useCallback(
+    (viseme: ProcessedViseme, elapsedTime: number) => {
+      if (!meshRef.current) {
+        console.error('Mesh not set');
+        return;
+      }
+
+      const visemeProgress = Math.min(
+        (elapsedTime - viseme.startTime) / viseme.duration,
+        1
+      );
+
+      const dynamicSpeedFactor = getDynamicSpeedFactor(viseme.duration);
+      const adjustedProgress = visemeProgress * dynamicSpeedFactor;
+
+      // Use a cubic easing function for smoother transitions
+      const easedProgress = easeInOutCubic(adjustedProgress);
+      const targetWeight = Math.sin(easedProgress * Math.PI) * viseme.weight;
+
+      currentVisemeWeightRef.current[viseme.name] = lerp(
+        currentVisemeWeightRef.current[viseme.name] || 0,
+        targetWeight,
+        VISEME_SMOOTHING
+      );
+
+      const visemeIndex = meshRef.current.morphTargetDictionary?.[viseme.name];
+      if (
+        typeof visemeIndex === 'number' &&
+        meshRef.current.morphTargetInfluences
+      ) {
+        meshRef.current.morphTargetInfluences[visemeIndex] =
+          currentVisemeWeightRef.current[viseme.name];
+        // console.log(`Applied viseme: ${viseme.name}, weight: ${currentVisemeWeightRef.current[viseme.name]}`);
+      } else {
+        console.error(
+          `Viseme not found in morph target dictionary: ${viseme.name}`
+        );
+      }
+    },
+    []
+  );
+
+  const animate = useCallback(
+    (time: number) => {
+      if (startTimeRef.current === null) {
+        startTimeRef.current = time;
+      }
+
+      const elapsedTime =
+        ((time - startTimeRef.current) / 1000) * VISEME_SPEED_FACTOR;
+
+      const currentViseme = getCurrentViseme(elapsedTime);
+
+      if (currentViseme) {
+        applyViseme(currentViseme, elapsedTime);
+      }
+
+      if (
+        currentVisemesRef.current.length > 0 &&
+        elapsedTime <
+          currentVisemesRef.current[currentVisemesRef.current.length - 1]
+            .startTime +
+            currentVisemesRef.current[currentVisemesRef.current.length - 1]
+              .duration
+      ) {
+        animationFrameRef.current = requestAnimationFrame(animate);
+      } else {
+        clearVisemes();
+      }
+    },
+    [getCurrentViseme, applyViseme]
+  );
+
+  const processVisemeQueue = useCallback(() => {
+    const azureVisemes = [...visemeQueueRef.current];
+    visemeQueueRef.current = [];
+
+    if (azureVisemes.length === 0) {
+      // console.log('No visemes to process');
+      return [];
+    }
+
+    const processedVisemes: ProcessedViseme[] = azureVisemes.map(
+      (currentViseme, i) => {
+        const nextViseme = azureVisemes[i + 1];
+        const duration = nextViseme
+          ? (nextViseme.audioOffset - currentViseme.audioOffset) / 10000000
+          : DEFAULT_VISEME_DURATION;
+
+        const processedViseme = {
+          name: VISEME_MAP[currentViseme.visemeId] || 'viseme_sil',
+          duration,
+          weight: 1,
+          startTime: currentViseme.audioOffset / 10000000,
+        };
+        //console.log('Processed viseme:', processedViseme);
+        return processedViseme;
+      }
+    );
+
+    currentVisemesRef.current = processedVisemes;
+
+    // Start animation immediately if not already animating
+    if (!isAnimatingRef.current) {
+      isAnimatingRef.current = true;
+      startTimeRef.current = performance.now();
+      // console.log('Starting animation');
+      animationFrameRef.current = requestAnimationFrame(animate);
+    } else {
+      // If already animating, adjust the start time for the new visemes
+      if (startTimeRef.current !== null) {
+        const currentTime = performance.now();
+        const elapsedTime =
+          ((currentTime - startTimeRef.current) / 1000) * VISEME_SPEED_FACTOR;
+        startTimeRef.current =
+          currentTime - (elapsedTime / VISEME_SPEED_FACTOR) * 1000;
+      }
+    }
+
+    return processedVisemes;
+  }, [isMeshSet, animate]);
+
+  const clearVisemes = useCallback(() => {
+    currentVisemesRef.current = [];
+    visemeQueueRef.current = [];
+
+    if (animationFrameRef.current !== null) {
+      cancelAnimationFrame(animationFrameRef.current);
+      animationFrameRef.current = null;
+    }
+
+    if (
+      meshRef.current?.morphTargetDictionary &&
+      meshRef.current?.morphTargetInfluences
+    ) {
+      Object.values(meshRef.current.morphTargetDictionary).forEach(index => {
+        if (typeof index === 'number') {
+          meshRef.current!.morphTargetInfluences![index] = 0;
+        }
+      });
+    }
+
+    currentVisemeWeightRef.current = {};
+    startTimeRef.current = null;
+    isAnimatingRef.current = false;
+    // console.log('Visemes cleared');
+  }, []);
+
+  // Your existing emotion map
+  const emotionMap: Record<string, Record<string, number>> = {
+    Gioia: { Gioria: 1 },
+    Rabbia: { Rabbia: 1 },
+    Sorpresa: { Sorpresa: 1 },
+    Tristezza: { Tristezza: 1 },
+    Timore: { Timore: 1 },
+  };
+
+  // Mapping from your emotions to Azure styles
+  const emotionToAzureStyleMap: Record<string, string> = {
+    Gioia: 'cheerful',
+    Rabbia: 'angry',
+    Sorpresa: 'excited',
+    Tristezza: 'sad',
+    Timore: 'terrified',
+  };
+
+  // Function to get Azure style from emotion
+  function getAzureStyleForEmotion(emotion: string): string {
+    return emotionToAzureStyleMap[emotion] || 'neutral';
+  }
+
+
+  useEffect(() => {
+    return () => {
+      if (animationFrameRef.current !== null) {
+        cancelAnimationFrame(animationFrameRef.current);
+      }
+    };
+  }, []);
+
+  const contextValue: VisemeContextType = {
+    setMeshRef,
+    addVisemeToQueue,
+    processVisemeQueue,
+    clearVisemes,
+    isMeshSet,
+    setEmotion,
+    emotion,
+    getAzureStyleForEmotion,
+  };
+
+  return (
+    <VisemeContext.Provider value={contextValue}>
+      {children}
+    </VisemeContext.Provider>
+  );
+};
+
+export const useViseme = (): VisemeContextType => {
+  const context = useContext(VisemeContext);
+  if (context === undefined) {
+    throw new Error('useViseme must be used within a VisemeProvider');
+  }
+  return context;
+};