@memori.ai/memori-react 7.7.0 → 7.8.0-rc.0

package/esm/components/MemoriWidget/MemoriWidget.js

@@ -216,11 +216,7 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
     const [isPlayingAudio, setIsPlayingAudio] = useState(false);
     const [controlsPosition, setControlsPosition] = useState('center');
     const [hideEmissions, setHideEmissions] = useState(false);
-    const { startProcessing, stopProcessing, addViseme, resetVisemeQueue, isProcessing, } = useViseme();
-    const audioContextRef = useRef(null);
-    const speechSynthesizerRef = useRef(null);
-    const audioDestinationRef = useRef(null);
-    const currentSpeechRef = useRef(null);
+    const { updateCurrentViseme, startProcessing, resetAndStartProcessing, isProcessing, addViseme, stopProcessing, resetVisemeQueue, } = useViseme();
     useEffect(() => {
         setIsPlayingAudio(!!speechSynthesizer);
         memoriSpeaking = !!speechSynthesizer;
@@ -1247,43 +1243,16 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
         const e = new CustomEvent('MemoriEndSpeak');
         document.dispatchEvent(e);
     };
-    const initializeAudioContext = useCallback(() => {
-        if (!audioContextRef.current || audioContextRef.current.state === 'closed') {
-            audioContextRef.current = new (window.AudioContext || window.webkitAudioContext)();
-        }
-        return audioContextRef.current;
-    }, []);
-    const initializeSpeechSynthesizer = useCallback((audioConfig) => {
-        if (!speechSynthesizerRef.current && AZURE_COGNITIVE_SERVICES_TTS_KEY) {
-            const speechConfig = speechSdk.SpeechConfig.fromSubscription(AZURE_COGNITIVE_SERVICES_TTS_KEY, 'eastus');
-            speechSynthesizerRef.current = new speechSdk.SpeechSynthesizer(speechConfig, audioConfig);
-        }
-        return speechSynthesizerRef.current;
-    }, []);
-    const stopCurrentSpeech = useCallback(() => {
-        if (currentSpeechRef.current) {
-            currentSpeechRef.current.cancel();
-            currentSpeechRef.current = null;
-        }
-        if (audioContextRef.current) {
-            audioContextRef.current.suspend();
-        }
-        if (audioDestinationRef.current) {
-            audioDestinationRef.current.pause();
-        }
-        setIsPlayingAudio(false);
-        stopProcessing();
-        resetVisemeQueue();
-    }, []);
-    const speak = useCallback(async (text) => {
+    const speak = (text) => {
         if (!AZURE_COGNITIVE_SERVICES_TTS_KEY || preview) {
             emitEndSpeakEvent();
             return;
         }
         stopListening();
-        stopCurrentSpeech();
-        if (preview || muteSpeaker || speakerMuted) {
-            setIsPlayingAudio(false);
+        if (preview)
+            return;
+        if (muteSpeaker || speakerMuted) {
+            memoriSpeaking = false;
             setMemoriTyping(false);
             emitEndSpeakEvent();
             if (continuousSpeech) {
@@ -1291,98 +1260,147 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
             }
             return;
         }
-        try {
-            const audioContext = initializeAudioContext();
-            await audioContext.resume();
-            if (!audioDestinationRef.current) {
-                audioDestinationRef.current = new speechSdk.SpeakerAudioDestination();
-            }
-            const audioConfig = speechSdk.AudioConfig.fromSpeakerOutput(audioDestinationRef.current);
-            const speechSynthesizer = initializeSpeechSynthesizer(audioConfig);
-            if (speechSynthesizer) {
-                speechSynthesizer.visemeReceived = (_, e) => {
-                    addViseme(e.visemeId, e.audioOffset);
-                    console.log('viseme added');
-                };
+        if (audioDestination)
+            audioDestination.pause();
+        let isSafari = window.navigator.userAgent.includes('Safari') &&
+            !window.navigator.userAgent.includes('Chrome');
+        let isIOS = /iPad|iPhone|iPod/.test(navigator.userAgent);
+        if (audioContext.state === 'interrupted') {
+            audioContext.resume().then(() => speak(text));
+            return;
+        }
+        if (audioContext.state === 'closed') {
+            audioContext = new AudioContext();
+            let buffer = audioContext.createBuffer(1, 10000, 22050);
+            let source = audioContext.createBufferSource();
+            source.buffer = buffer;
+            source.connect(audioContext.destination);
+        }
+        else if (audioContext.state === 'suspended') {
+            stopAudio();
+            audioContext = new AudioContext();
+            let buffer = audioContext.createBuffer(1, 10000, 22050);
+            let source = audioContext.createBufferSource();
+            source.buffer = buffer;
+            source.connect(audioContext.destination);
+        }
+        if (!speechSynthesizer) {
+            if (!isIOS) {
+                audioDestination = new speechSdk.SpeakerAudioDestination();
             }
-            startProcessing();
-            const textToSpeak = escapeHTML(stripMarkdown(stripEmojis(stripHTML(stripOutputTags(text)))));
-            const ssml = `
-        <speak version="1.0" 
-               xmlns="http://www.w3.org/2001/10/synthesis" 
-               xmlns:mstts="https://www.w3.org/2001/mstts" 
-               xml:lang="${getCultureCodeByLanguage(userLang)}">
-          <voice name="${getTTSVoice(userLang)}">
-              <s>${replaceTextWithPhonemes(textToSpeak, userLang.toLowerCase())}</s>
-          </voice>
-        </speak>
-      `;
-            const speakPromise = new Promise((resolve, reject) => {
-                speechSynthesizer === null || speechSynthesizer === void 0 ? void 0 : speechSynthesizer.speakSsmlAsync(ssml, result => resolve(result), error => reject(error));
-            });
-            currentSpeechRef.current = {
-                cancel: () => {
-                    var _a;
-                    speechSynthesizer === null || speechSynthesizer === void 0 ? void 0 : speechSynthesizer.close();
-                    (_a = audioDestinationRef.current) === null || _a === void 0 ? void 0 : _a.pause();
+            let audioConfig = speechSdk.AudioConfig.fromSpeakerOutput(audioDestination);
+            speechSynthesizer = new speechSdk.SpeechSynthesizer(speechConfig, audioConfig);
+        }
+        const source = audioContext.createBufferSource();
+        source.addEventListener('ended', () => {
+            setIsPlayingAudio(false);
+            memoriSpeaking = false;
+        });
+        audioDestination.onAudioEnd = () => {
+            setIsPlayingAudio(false);
+            memoriSpeaking = false;
+            source.disconnect();
+            emitEndSpeakEvent();
+            onEndSpeakStartListen();
+        };
+        resetVisemeQueue();
+        speechSynthesizer.visemeReceived = function (_, e) {
+            addViseme(e.visemeId, e.audioOffset);
+        };
+        const textToSpeak = escapeHTML(stripMarkdown(stripEmojis(stripHTML(stripOutputTags(text)))));
+        speechSynthesizer.speakSsmlAsync(`<speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis" xmlns:mstts="https://www.w3.org/2001/mstts" xmlns:emo="http://www.w3.org/2009/10/emotionml" xml:lang="${getCultureCodeByLanguage(userLang)}"><voice name="${getTTSVoice(userLang)}"><s>${replaceTextWithPhonemes(textToSpeak, userLang.toLowerCase())}</s></voice></speak>`, result => {
+            if (result) {
+                setIsPlayingAudio(true);
+                memoriSpeaking = true;
+                try {
+                    audioContext.decodeAudioData(result.audioData, function (buffer) {
+                        console.log('decoded');
+                        const currentSource = audioContext.createBufferSource();
+                        currentSource.buffer = buffer;
+                        currentSource.connect(audioContext.destination);
+                        startProcessing();
+                        currentSource.start();
+                        currentSource.onended = () => {
+                            console.log('ended');
+                            setIsPlayingAudio(false);
+                            memoriSpeaking = false;
+                            stopProcessing();
+                            resetVisemeQueue();
+                            emitEndSpeakEvent();
+                        };
+                    });
+                    audioContext.onstatechange = () => {
+                        if (audioContext.state === 'suspended' ||
+                            audioContext.state === 'closed') {
+                            console.log('suspended');
+                            source.disconnect();
+                            setIsPlayingAudio(false);
+                            stopProcessing();
+                            resetVisemeQueue();
+                            memoriSpeaking = false;
+                            emitEndSpeakEvent();
+                        }
+                        else if (audioContext.state === 'interrupted') {
+                            console.log('interrupted');
+                            stopProcessing();
+                            resetVisemeQueue();
+                            audioContext.resume();
+                        }
+                    };
                 }
-            };
-            const result = await speakPromise;
-            setIsPlayingAudio(true);
-            if (audioContext && result) {
-                const audioBuffer = await audioContext.decodeAudioData(result.audioData);
-                const source = audioContext.createBufferSource();
-                source.buffer = audioBuffer;
-                source.connect(audioContext.destination);
-                source.onended = () => {
+                catch (e) {
+                    console.warn('speak error: ', e);
+                    window.speechSynthesis.speak(new SpeechSynthesisUtterance(text));
                     setIsPlayingAudio(false);
+                    memoriSpeaking = false;
                     stopProcessing();
                     resetVisemeQueue();
-                    currentSpeechRef.current = null;
+                    if (speechSynthesizer) {
+                        speechSynthesizer.close();
+                        speechSynthesizer = null;
+                    }
                     emitEndSpeakEvent();
-                    onEndSpeakStartListen();
-                };
-                await audioContext.resume();
-                source.start(0);
+                }
             }
             else {
+                audioContext.resume();
                 stopProcessing();
                 resetVisemeQueue();
-                throw new Error('No result from speech synthesis');
+                setIsPlayingAudio(false);
+                memoriSpeaking = false;
+                emitEndSpeakEvent();
             }
-        }
-        catch (error) {
-            console.error('Speech synthesis error:', error);
+        }, error => {
+            console.error('speak:', error);
+            window.speechSynthesis.speak(new SpeechSynthesisUtterance(text));
+            setIsPlayingAudio(false);
+            memoriSpeaking = false;
             stopProcessing();
             resetVisemeQueue();
-            const utterance = new SpeechSynthesisUtterance(text);
-            window.speechSynthesis.speak(utterance);
-        }
-        finally {
-            setMemoriTyping(false);
-        }
-    }, [initializeAudioContext, initializeSpeechSynthesizer, stopCurrentSpeech]);
-    const stopAudio = useCallback(() => {
+            emitEndSpeakEvent();
+        });
+        setMemoriTyping(false);
+    };
+    const stopAudio = () => {
         setIsPlayingAudio(false);
         memoriSpeaking = false;
         try {
-            if (speechSynthesizerRef.current) {
-                speechSynthesizerRef.current.close();
-                speechSynthesizerRef.current = null;
+            if (speechSynthesizer) {
+                speechSynthesizer.close();
+                speechSynthesizer = null;
             }
-            if (audioContextRef.current && audioContextRef.current.state !== 'closed') {
-                audioContextRef.current.close();
+            if (audioContext.state !== 'closed') {
+                audioContext.close();
             }
-            if (audioDestinationRef.current) {
-                audioDestinationRef.current.pause();
-                audioDestinationRef.current.close();
+            if (audioDestination) {
+                audioDestination.pause();
+                audioDestination.close();
             }
-            stopCurrentSpeech();
         }
         catch (e) {
             console.debug('stopAudio error: ', e);
         }
-    }, [stopCurrentSpeech]);
+    };
     useEffect(() => {
         let textarea = document.querySelector('#chat-fieldset textarea');
         if (textarea)
@@ -1515,12 +1533,6 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
     useEffect(() => {
         return () => {
             resetUIEffects();
-            if (speechSynthesizerRef.current) {
-                speechSynthesizerRef.current.close();
-            }
-            if (audioContextRef.current) {
-                audioContextRef.current.close();
-            }
         };
     }, []);
     useEffect(() => {
@@ -2082,7 +2094,9 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
         }
     }, [tenant === null || tenant === void 0 ? void 0 : tenant.billingDelegation, deepThoughtEnabled]);
     const showFullHistory = showOnlyLastMessages === undefined
-        ? layout !== 'TOTEM' && layout !== 'WEBSITE_ASSISTANT'
+        ? layout !== 'TOTEM' &&
+            layout !== 'WEBSITE_ASSISTANT' &&
+            layout !== 'HIDDEN_CHAT'
         : !showOnlyLastMessages;
     const headerProps = {
         memori,