@memori.ai/memori-react 7.7.0 → 7.8.0-rc.0

package/src/components/MemoriWidget/MemoriWidget.tsx

@@ -543,19 +543,14 @@ const MemoriWidget = ({
   const [hideEmissions, setHideEmissions] = useState(false);
 
   const {
+    updateCurrentViseme,
     startProcessing,
-    stopProcessing,
+    resetAndStartProcessing,
+    isProcessing,
     addViseme,
+    stopProcessing,
     resetVisemeQueue,
-    isProcessing,
   } = useViseme();
-  const audioContextRef = useRef<AudioContext | null>(null);
-  const speechSynthesizerRef = useRef<speechSdk.SpeechSynthesizer | null>(null);
-  const audioDestinationRef = useRef<speechSdk.SpeakerAudioDestination | null>(
-    null
-  );
-  const currentSpeechRef = useRef<{ cancel: () => void } | null>(null);
-
 
   useEffect(() => {
     setIsPlayingAudio(!!speechSynthesizer);
@@ -1890,167 +1885,204 @@ const MemoriWidget = ({
     const e = new CustomEvent('MemoriEndSpeak');
     document.dispatchEvent(e);
   };
-  const initializeAudioContext = useCallback(() => {
-    if (!audioContextRef.current || audioContextRef.current.state === 'closed') {
-      audioContextRef.current = new (window.AudioContext || (window as any).webkitAudioContext)() as unknown as IAudioContext;
-    }
-    return audioContextRef.current;
-  }, []);
-
-  const initializeSpeechSynthesizer = useCallback((audioConfig: speechSdk.AudioConfig) => {
-    if (!speechSynthesizerRef.current && AZURE_COGNITIVE_SERVICES_TTS_KEY) {
-      const speechConfig = speechSdk.SpeechConfig.fromSubscription(
-        AZURE_COGNITIVE_SERVICES_TTS_KEY,
-        'eastus'
-      );
-      speechSynthesizerRef.current = new speechSdk.SpeechSynthesizer(speechConfig, audioConfig);
-    }
-    return speechSynthesizerRef.current;
-  }, []);
-
-  const stopCurrentSpeech = useCallback(() => {
-    if (currentSpeechRef.current) {
-      currentSpeechRef.current.cancel();
-      currentSpeechRef.current = null;
-    }
-    if (audioContextRef.current) {
-      audioContextRef.current.suspend();
-    }
-    if (audioDestinationRef.current) {
-      audioDestinationRef.current.pause();
-    }
-    setIsPlayingAudio(false);
-    stopProcessing();
-    resetVisemeQueue();
-  }, []);
 
-  const speak = useCallback(async (text: string): Promise<void> => {
+  const speak = (text: string): void => {
     if (!AZURE_COGNITIVE_SERVICES_TTS_KEY || preview) {
       emitEndSpeakEvent();
       return;
     }
-
     stopListening();
-    stopCurrentSpeech(); // Stop any ongoing speech
+    // stopAudio();
 
-    if (preview || muteSpeaker || speakerMuted) {
-      setIsPlayingAudio(false);
+    if (preview) return;
+
+    if (muteSpeaker || speakerMuted) {
+      memoriSpeaking = false;
       setMemoriTyping(false);
+
       emitEndSpeakEvent();
+
+      // trigger start continuous listening if set, see MemoriChat
       if (continuousSpeech) {
         setListeningTimeout();
       }
       return;
     }
 
-    try {
-      const audioContext = initializeAudioContext();
-      await audioContext.resume();
+    if (audioDestination) audioDestination.pause();
 
-      if (!audioDestinationRef.current) {
-        audioDestinationRef.current = new speechSdk.SpeakerAudioDestination();
+    let isSafari =
+      window.navigator.userAgent.includes('Safari') &&
+      !window.navigator.userAgent.includes('Chrome');
+    let isIOS = /iPad|iPhone|iPod/.test(navigator.userAgent);
+    if ((audioContext.state as string) === 'interrupted') {
+      audioContext.resume().then(() => speak(text));
+      return;
+    }
+    if (audioContext.state === 'closed') {
+      audioContext = new AudioContext();
+      let buffer = audioContext.createBuffer(1, 10000, 22050);
+      let source = audioContext.createBufferSource();
+      source.buffer = buffer;
+      source.connect(audioContext.destination);
+    } else if (audioContext.state === 'suspended') {
+      stopAudio();
+
+      audioContext = new AudioContext();
+      let buffer = audioContext.createBuffer(1, 10000, 22050);
+      let source = audioContext.createBufferSource();
+      source.buffer = buffer;
+      source.connect(audioContext.destination);
+    }
+
+    if (!speechSynthesizer) {
+      if (!isIOS) {
+        audioDestination = new speechSdk.SpeakerAudioDestination();
       }
+      let audioConfig =
+        speechSdk.AudioConfig.fromSpeakerOutput(audioDestination);
+      speechSynthesizer = new speechSdk.SpeechSynthesizer(
+        speechConfig,
+        audioConfig
+      );
+    }
 
-      const audioConfig = speechSdk.AudioConfig.fromSpeakerOutput(audioDestinationRef.current);
-      const speechSynthesizer = initializeSpeechSynthesizer(audioConfig);
+    const source = audioContext.createBufferSource();
+    source.addEventListener('ended', () => {
+      setIsPlayingAudio(false);
+      memoriSpeaking = false;
+    });
+    audioDestination.onAudioEnd = () => {
+      setIsPlayingAudio(false);
+      memoriSpeaking = false;
+      source.disconnect();
 
-      if (speechSynthesizer) {
+      emitEndSpeakEvent();
 
-        // Add the new visemeReceived event listener
-        speechSynthesizer.visemeReceived = (_, e) => {
-          addViseme(e.visemeId, e.audioOffset);
-          console.log('viseme added')
-        };
-      }
-      startProcessing();
-
-      const textToSpeak = escapeHTML(stripMarkdown(stripEmojis(stripHTML(stripOutputTags(text)))));
-
-      const ssml = `
-        <speak version="1.0" 
-               xmlns="http://www.w3.org/2001/10/synthesis" 
-               xmlns:mstts="https://www.w3.org/2001/mstts" 
-               xml:lang="${getCultureCodeByLanguage(userLang)}">
-          <voice name="${getTTSVoice(userLang)}">
-              <s>${replaceTextWithPhonemes(textToSpeak, userLang.toLowerCase())}</s>
-          </voice>
-        </speak>
-      `;
-
-      const speakPromise = new Promise<speechSdk.SpeechSynthesisResult>((resolve, reject) => {
-        speechSynthesizer?.speakSsmlAsync(
-          ssml,
-          result => resolve(result),
-          error => reject(error)
-        );
-      });
+      // trigger start continuous listening if set
+      onEndSpeakStartListen();
+    };
 
-      currentSpeechRef.current = {
-        cancel: () => {
-          speechSynthesizer?.close();
-          audioDestinationRef.current?.pause();
-        }
-      };
+    // Add this before starting new speech synthesis
+    resetVisemeQueue();
 
-      const result = await speakPromise;
+    // Set up the viseme event handler
+    speechSynthesizer.visemeReceived = function (_, e) {
+      addViseme(e.visemeId, e.audioOffset);
+    };
 
-      setIsPlayingAudio(true);
+    const textToSpeak = escapeHTML(
+      stripMarkdown(stripEmojis(stripHTML(stripOutputTags(text))))
+    );
 
-      if (audioContext && result) {
-        const audioBuffer = await audioContext.decodeAudioData(result.audioData);
-        const source = audioContext.createBufferSource();
-        source.buffer = audioBuffer;
-        source.connect(audioContext.destination);
+    speechSynthesizer.speakSsmlAsync(
+      `<speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis" xmlns:mstts="https://www.w3.org/2001/mstts" xmlns:emo="http://www.w3.org/2009/10/emotionml" xml:lang="${getCultureCodeByLanguage(
+        userLang
+      )}"><voice name="${getTTSVoice(userLang)}"><s>${replaceTextWithPhonemes(
+        textToSpeak,
+        userLang.toLowerCase()
+      )}</s></voice></speak>`,
+      result => {
+        if (result) {
+          setIsPlayingAudio(true);
+          memoriSpeaking = true;
 
-        source.onended = () => {
-          setIsPlayingAudio(false);
+          try {
+            // Decode the audio data
+            audioContext!.decodeAudioData(result.audioData, function (buffer) {
+              console.log('decoded');
+              const currentSource = audioContext!.createBufferSource();
+              currentSource.buffer = buffer;
+              currentSource.connect(audioContext!.destination);
+              startProcessing();
+              currentSource.start();
+
+              currentSource.onended = () => {
+                console.log('ended');
+                setIsPlayingAudio(false);
+                memoriSpeaking = false;
+                stopProcessing();
+                resetVisemeQueue();
+                emitEndSpeakEvent();
+              };
+            });
+
+            // Handle the audio context state changes
+            audioContext.onstatechange = () => {
+              if (
+                audioContext.state === 'suspended' ||
+                audioContext.state === 'closed'
+              ) {
+                console.log('suspended');
+                source.disconnect();
+                setIsPlayingAudio(false);
+                stopProcessing();
+                resetVisemeQueue();
+                memoriSpeaking = false;
+                emitEndSpeakEvent();
+              } else if ((audioContext.state as string) === 'interrupted') {
+                console.log('interrupted');
+                stopProcessing();
+                resetVisemeQueue();
+
+                audioContext.resume();
+              }
+            };
+          } catch (e) {
+            console.warn('speak error: ', e);
+            window.speechSynthesis.speak(new SpeechSynthesisUtterance(text));
+            setIsPlayingAudio(false);
+            memoriSpeaking = false;
+            stopProcessing();
+            resetVisemeQueue();
+
+            if (speechSynthesizer) {
+              speechSynthesizer.close();
+              speechSynthesizer = null;
+            }
+            emitEndSpeakEvent();
+          }
+        } else {
+          audioContext.resume();
           stopProcessing();
           resetVisemeQueue();
-          currentSpeechRef.current = null;
+          setIsPlayingAudio(false);
+          memoriSpeaking = false;
           emitEndSpeakEvent();
-          onEndSpeakStartListen();
-        };
-
-        await audioContext.resume();
-        source.start(0);
-      } else {
+        }
+      },
+      error => {
+        console.error('speak:', error);
+        window.speechSynthesis.speak(new SpeechSynthesisUtterance(text));
+        setIsPlayingAudio(false);
+        memoriSpeaking = false;
         stopProcessing();
         resetVisemeQueue();
-        throw new Error('No result from speech synthesis');
+        emitEndSpeakEvent();
       }
-    } catch (error) {
-      console.error('Speech synthesis error:', error);
-      stopProcessing();
-      resetVisemeQueue();
-      // Fallback to browser's speech synthesis
-      const utterance = new SpeechSynthesisUtterance(text);
-      window.speechSynthesis.speak(utterance);
-    } finally {
-      setMemoriTyping(false);
-    }
-  }, [initializeAudioContext, initializeSpeechSynthesizer, stopCurrentSpeech]);
+    );
 
-  const stopAudio = useCallback(() => {
+    setMemoriTyping(false);
+  };
+  const stopAudio = () => {
     setIsPlayingAudio(false);
     memoriSpeaking = false;
     try {
-      if (speechSynthesizerRef.current) {
-        speechSynthesizerRef.current.close();
-        speechSynthesizerRef.current = null;
+      if (speechSynthesizer) {
+        speechSynthesizer.close();
+        speechSynthesizer = null;
       }
-      if (audioContextRef.current && audioContextRef.current.state !== 'closed') {
-        audioContextRef.current.close();
+      if (audioContext.state !== 'closed') {
+        audioContext.close();
       }
-      if (audioDestinationRef.current) {
-        audioDestinationRef.current.pause();
-        audioDestinationRef.current.close();
+      if (audioDestination) {
+        audioDestination.pause();
+        audioDestination.close();
       }
-      stopCurrentSpeech();
     } catch (e) {
       console.debug('stopAudio error: ', e);
     }
-  }, [stopCurrentSpeech]);
-
+  };
 
   /**
    * Focus on the chat input on mount
@@ -2214,12 +2246,6 @@ const MemoriWidget = ({
   useEffect(() => {
     return () => {
       resetUIEffects();
-      if (speechSynthesizerRef.current) {
-        speechSynthesizerRef.current.close();
-      }
-      if (audioContextRef.current) {
-        audioContextRef.current.close();
-      }
     };
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, []);
@@ -2973,7 +2999,9 @@ const MemoriWidget = ({
 
   const showFullHistory =
     showOnlyLastMessages === undefined
-      ? layout !== 'TOTEM' && layout !== 'WEBSITE_ASSISTANT'
+      ? layout !== 'TOTEM' &&
+        layout !== 'WEBSITE_ASSISTANT' &&
+        layout !== 'HIDDEN_CHAT'
       : !showOnlyLastMessages;
 
   const headerProps: HeaderProps = {

package/src/components/layouts/ZoomedFullBody.test.tsx

@@ -34,4 +34,4 @@ it('renders ZOOMED_FULL_BODY layout unchanged', () => {
     </I18nWrapper>
   );
   expect(container).toMatchSnapshot();
-});
+});

package/src/components/layouts/ZoomedFullBody.tsx

@@ -1,4 +1,4 @@
-import React, { useEffect, useCallback } from 'react';
+import React from 'react';
 import Spin from '../ui/Spin';
 import { LayoutProps } from '../MemoriWidget/MemoriWidget';
 
@@ -21,20 +21,15 @@ const ZoomedFullBodyLayout: React.FC<LayoutProps> = ({
   loading = false,
   poweredBy,
 }) => {
-  useEffect(() => {
-    document.body.style.overflow = 'hidden';
-    return () => {
-      document.body.style.overflow = '';
-    };
-  }, []);
-
   return (
     <>
       {integrationStyle}
       {integrationBackground}
 
       <Spin className="memori-spin--zoomed-full-body" spinning={loading}>
-        {showInstruct && ChangeMode && changeModeProps && <ChangeMode {...changeModeProps} />}
+        {showInstruct && ChangeMode && changeModeProps && (
+          <ChangeMode {...changeModeProps} />
+        )}
 
         {Header && headerProps && <Header {...headerProps} />}
 

package/src/components/layouts/zoomed-full-body.css

@@ -1,5 +1,5 @@
 .memori-spin--zoomed-full-body {
-    height: calc(100vh) !important;
+    height: 100vh !important;
 }
 
 .memori--powered-by-custom {
@@ -13,4 +13,8 @@
 
 .memori--grid-column--zoomed-full-body {
     max-height: calc(100% - 80px) !important;
-}
+}
+
+.memori.memori-widget.memori-layout-zoomed_full_body .memori--grid {
+    height: calc(100% - 25px);
+}

package/src/context/visemeContext.tsx

@@ -1,9 +1,16 @@
-import React, { createContext, useContext, useRef, useCallback, useState, useEffect } from 'react';
-
-type Viseme = { 
-  name: string; 
-  weight: number; 
-  startTime: number; 
+import React, {
+  createContext,
+  useContext,
+  useRef,
+  useCallback,
+  useState,
+  useEffect,
+} from 'react';
+
+type Viseme = {
+  name: string;
+  weight: number;
+  startTime: number;
   endTime: number;
 };
 
@@ -13,6 +20,7 @@ interface VisemeContextType {
   startProcessing: () => void;
   stopProcessing: () => void;
   resetVisemeQueue: () => void;
+  resetAndStartProcessing: () => void;
   isProcessing: boolean;
 }
 
@@ -44,80 +52,98 @@ const VISEME_MAP: { [key: number]: string } = {
   21: 'viseme_PP', // y (closest match, could be debated)
 };
 
-const DEFAULT_VISEME_DURATION = 0.04 //0; // Reduced from 0.4 for smoother transitions
-const VISEME_OVERLAP = 0.35; // Slightly increased from 0.04 for more overlap
-const SMOOTHING_FACTOR = 0.35 // New constant for weight smoothing
-const TIME_OFFSET =-0.15; // Adjust this value as needed (in seconds)
-const PRELOAD_TIME = 0.515; // Preload visemes 0.5 seconds in advance
+const DEFAULT_VISEME_DURATION = 0.04; //0; // Reduced from 0.4 for smoother transitions
+const VISEME_OVERLAP = 0.15; // Slightly increased from 0.04 for more overlap
+const SMOOTHING_FACTOR = 0.35; // New constant for weight smoothing
+const TIME_OFFSET = -0.04; // Adjust this value as needed (in seconds)
+const PRELOAD_TIME = 1; // Preload visemes 0.5 seconds in advance
 
-export const VisemeProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => {
+export const VisemeProvider: React.FC<{ children: React.ReactNode }> = ({
+  children,
+}) => {
   const visemeQueueRef = useRef<Viseme[]>([]);
   const startTimeRef = useRef<number | null>(null);
   const [isProcessing, setIsProcessing] = useState(false);
   const lastVisemeRef = useRef<Viseme | null>(null);
 
-  const addViseme = useCallback((visemeId: number, audioOffset: number) => {
-    const visemeName = VISEME_MAP[visemeId] || 'viseme_sil';
-    const startTime = audioOffset / 10000000 + TIME_OFFSET;
-    const endTime = startTime + DEFAULT_VISEME_DURATION;
-    const newViseme: Viseme = {
-      name: visemeName,
-      weight: 0,
-      startTime,
-      endTime,
-    };
-    visemeQueueRef.current.push(newViseme);
-
-    if (!isProcessing) {
-      startProcessing();
-    }
-  }, [isProcessing]);
+  const addViseme = useCallback(
+    (visemeId: number, audioOffset: number) => {
+      const visemeName = VISEME_MAP[visemeId] || 'viseme_sil';
+      const startTime = audioOffset / 10000000 + TIME_OFFSET;
+      const endTime = startTime + DEFAULT_VISEME_DURATION;
+      const newViseme: Viseme = {
+        name: visemeName,
+        weight: 0,
+        startTime,
+        endTime,
+      };
+      visemeQueueRef.current.push(newViseme);
+
+      if (!isProcessing) {
+        startProcessing();
+      }
+    },
+    [isProcessing]
+  );
 
-  const updateCurrentViseme = useCallback((currentTime: number): Viseme | null => {
-    if (!isProcessing || startTimeRef.current === null) {
-      console.log('StartTimeRef not set');
-      return null;
-    }
-
-    const elapsedTime = currentTime - startTimeRef.current + PRELOAD_TIME
-
-    // Remove expired visemes
-    visemeQueueRef.current = visemeQueueRef.current.filter(v => v.endTime > elapsedTime);
-
-    const currentViseme = visemeQueueRef.current.find(v => 
-      v.startTime <= elapsedTime && v.endTime > elapsedTime - VISEME_OVERLAP
-    );
-
-    if (currentViseme) {
-      console.log('CurrentViseme Found!')
-      const visemeProgress = (elapsedTime - currentViseme.startTime) / (currentViseme.endTime - currentViseme.startTime);
-      const targetWeight = Math.sin(Math.PI * Math.min(visemeProgress, 1));
-      
-      // Smooth the weight transition
-      const smoothedWeight = lastVisemeRef.current
-        ? lastVisemeRef.current.weight + (targetWeight - lastVisemeRef.current.weight) * SMOOTHING_FACTOR
-        : targetWeight;
-
-      const updatedViseme = { ...currentViseme, weight: smoothedWeight };
-      lastVisemeRef.current = updatedViseme;
-      return updatedViseme;
-    }
-
-    // Gradually reduce weight when no viseme is active
-    if (lastVisemeRef.current) {
-      const reducedWeight = lastVisemeRef.current.weight * (1 - SMOOTHING_FACTOR);
-      if (reducedWeight > 0.01) {
-        lastVisemeRef.current = { ...lastVisemeRef.current, weight: reducedWeight };
-        return lastVisemeRef.current;
+  const updateCurrentViseme = useCallback(
+    (currentTime: number): Viseme | null => {
+      if (!isProcessing || startTimeRef.current === null) {
+        console.log('StartTimeRef not set');
+        return null;
       }
-    }
 
-    lastVisemeRef.current = null;
-    return null;
-  }, [isProcessing]);
+      const elapsedTime = currentTime - startTimeRef.current + PRELOAD_TIME;
+
+      // Remove expired visemes
+      visemeQueueRef.current = visemeQueueRef.current.filter(
+        v => v.endTime > elapsedTime
+      );
+
+      const currentViseme = visemeQueueRef.current.find(
+        v =>
+          v.startTime <= elapsedTime && v.endTime > elapsedTime - VISEME_OVERLAP
+      );
+
+      if (currentViseme) {
+        console.log('CurrentViseme Found!');
+        const visemeProgress =
+          (elapsedTime - currentViseme.startTime) /
+          (currentViseme.endTime - currentViseme.startTime);
+        const targetWeight = Math.sin(Math.PI * Math.min(visemeProgress, 1));
+
+        // Smooth the weight transition
+        const smoothedWeight = lastVisemeRef.current
+          ? lastVisemeRef.current.weight +
+            (targetWeight - lastVisemeRef.current.weight) * SMOOTHING_FACTOR
+          : targetWeight;
+
+        const updatedViseme = { ...currentViseme, weight: smoothedWeight };
+        lastVisemeRef.current = updatedViseme;
+        return updatedViseme;
+      }
+
+      // Gradually reduce weight when no viseme is active
+      if (lastVisemeRef.current) {
+        const reducedWeight =
+          lastVisemeRef.current.weight * (1 - SMOOTHING_FACTOR);
+        if (reducedWeight > 0.01) {
+          lastVisemeRef.current = {
+            ...lastVisemeRef.current,
+            weight: reducedWeight,
+          };
+          return lastVisemeRef.current;
+        }
+      }
+
+      lastVisemeRef.current = null;
+      return null;
+    },
+    [isProcessing]
+  );
 
   const startProcessing = useCallback(() => {
-    if (isProcessing) return;
+    // if (isProcessing) return;
     startTimeRef.current = performance.now() / 1000;
     setIsProcessing(true);
   }, [isProcessing]);
@@ -133,11 +159,19 @@ export const VisemeProvider: React.FC<{ children: React.ReactNode }> = ({ childr
     lastVisemeRef.current = null;
   }, []);
 
+  const resetAndStartProcessing = useCallback(() => {
+    stopProcessing();
+    resetVisemeQueue();
+    startTimeRef.current = performance.now() / 1000;
+    setIsProcessing(true);
+  }, [stopProcessing, resetVisemeQueue]);
+
   const contextValue = {
     addViseme,
     updateCurrentViseme,
     startProcessing,
     stopProcessing,
+    resetAndStartProcessing,
     resetVisemeQueue,
     isProcessing,
   };
@@ -155,4 +189,4 @@ export const useViseme = (): VisemeContextType => {
     throw new Error('useViseme must be used within a VisemeProvider');
   }
   return context;
-};
+};