@memori.ai/memori-react 8.0.2 → 8.1.0-rc.0

package/esm/components/MemoriWidget/MemoriWidget.js

@@ -3,7 +3,6 @@ import { useState, useEffect, useCallback, useRef, useMemo, } from 'react';
 import { useTranslation } from 'react-i18next';
 import memoriApiClient from '@memori.ai/memori-api-client';
 import { AudioContext } from 'standardized-audio-context';
-import * as speechSdk from 'microsoft-cognitiveservices-speech-sdk';
 import cx from 'classnames';
 import { DateTime } from 'luxon';
 import toast from 'react-hot-toast';
@@ -38,6 +37,7 @@ import { sanitizeText } from '../../helpers/sanitizer';
 import { useTTS } from '../../helpers/tts/useTTS';
 import Alert from '../ui/Alert';
 import ChatHistoryDrawer from '../ChatHistoryDrawer/ChatHistory';
+import { useSTT } from '../../helpers/stt/useSTT';
 const getMemoriState = (integrationId) => {
     var _a, _b, _c, _d, _f;
     let widget = integrationId
@@ -152,9 +152,6 @@ window.getMemoriState = getMemoriState;
 window.typeMessage = typeMessage;
 window.typeMessageHidden = typeMessageHidden;
 window.typeBatchMessages = typeBatchMessages;
-let recognizer;
-let speechConfig;
-let audioDestination;
 let audioContext;
 let memoriPassword;
 let speakerMuted = false;
@@ -383,7 +380,9 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
             }
             const mediaDocuments = media === null || media === void 0 ? void 0 : media.filter(m => { var _a; return !m.mediumID && ((_a = m.properties) === null || _a === void 0 ? void 0 : _a.isAttachedFile); });
             if (mediaDocuments && mediaDocuments.length > 0) {
-                const documentContents = mediaDocuments.map(doc => doc.content).join(' ');
+                const documentContents = mediaDocuments
+                    .map(doc => doc.content)
+                    .join(' ');
                 msg = msg + ' ' + documentContents;
             }
             const { currentState, ...response } = await postTextEnteredEvent({
@@ -568,6 +567,62 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
                 : 0;
     const [birthDate, setBirthDate] = useState();
     const [showAgeVerification, setShowAgeVerification] = useState(false);
+    const getCultureCodeByLanguage = (lang) => {
+        var _a, _b;
+        let voice = '';
+        let voiceLang = (lang ||
+            ((_b = (_a = memori.culture) === null || _a === void 0 ? void 0 : _a.split('-')) === null || _b === void 0 ? void 0 : _b[0]) ||
+            i18n.language ||
+            'IT').toUpperCase();
+        switch (voiceLang) {
+            case 'IT':
+                voice = 'it-IT';
+                break;
+            case 'DE':
+                voice = 'de-DE';
+                break;
+            case 'EN':
+                voice = 'en-GB';
+                break;
+            case 'ES':
+                voice = 'es-ES';
+                break;
+            case 'FR':
+                voice = 'fr-FR';
+                break;
+            case 'PT':
+                voice = 'pt-PT';
+                break;
+            case 'UK':
+                voice = 'uk-UK';
+                break;
+            case 'RU':
+                voice = 'ru-RU';
+                break;
+            case 'PL':
+                voice = 'pl-PL';
+                break;
+            case 'FI':
+                voice = 'fi-FI';
+                break;
+            case 'EL':
+                voice = 'el-GR';
+                break;
+            case 'AR':
+                voice = 'ar-SA';
+                break;
+            case 'ZH':
+                voice = 'zh-CN';
+                break;
+            case 'JA':
+                voice = 'ja-JP';
+                break;
+            default:
+                voice = 'it-IT';
+                break;
+        }
+        return voice;
+    };
     const [sessionId, setSessionId] = useState(initialSessionID);
     const [currentDialogState, _setCurrentDialogState] = useState();
     const setCurrentDialogState = (state) => {
@@ -1041,17 +1096,43 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
             tenant: tenantID,
             region: 'westeurope',
             voiceType: memori.voiceType,
-            layout: selectedLayout
+            layout: selectedLayout,
         });
     }, [ttsProvider, userLang, memori.culture, memori.voiceType]);
+    const sttConfig = useMemo(() => ({
+        provider: ttsProvider,
+        language: getCultureCodeByLanguage(userLang),
+        tenant: tenantID,
+    }), [ttsProvider, userLang]);
     const { speak: ttsSpeak, stop: ttsStop, isPlaying: isPlayingAudio, speakerMuted, toggleMute, hasUserActivatedSpeak, setHasUserActivatedSpeak, error, setError, } = useTTS(ttsConfig, {
         apiUrl: `${baseUrl}/api/tts`,
         continuousSpeech: continuousSpeech,
-        onEndSpeakStartListen: () => {
-            console.log('[MemoriWidget] onEndSpeakStartListen called');
-        },
         preview: preview,
     }, autoStart, defaultEnableAudio, defaultSpeakerActive);
+    const processSpeechAndSendMessage = (text) => {
+        console.log('processSpeechAndSendMessage', text);
+        if (!text || text.trim().length === 0) {
+            return;
+        }
+        try {
+            const message = stripDuplicates(text);
+            console.debug('Processing speech message:', message);
+            if (message.length > 0) {
+                setUserMessage('');
+                console.debug('Sending message:', message);
+                sendMessage(message);
+            }
+        }
+        catch (error) {
+            console.error('Error in processSpeechAndSendMessage:', error);
+        }
+    };
+    const { isListening, startRecording, stopRecording, } = useSTT(sttConfig, processSpeechAndSendMessage, {
+        apiUrl: `${baseUrl}/api/stt`,
+        continuousRecording: continuousSpeech,
+        silenceTimeout: continuousSpeechTimeout,
+        autoStart: autoStart,
+    }, defaultEnableAudio);
     const resetInteractionTimeout = () => {
         clearInteractionTimeout();
         if (!isPlayingAudio && !userMessage.length && !memoriTyping && !listening)
@@ -1153,73 +1234,14 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
         memoriTyping,
         hasUserActivatedSpeak,
     ]);
-    const getCultureCodeByLanguage = (lang) => {
-        var _a, _b;
-        let voice = '';
-        let voiceLang = (lang ||
-            ((_b = (_a = memori.culture) === null || _a === void 0 ? void 0 : _a.split('-')) === null || _b === void 0 ? void 0 : _b[0]) ||
-            i18n.language ||
-            'IT').toUpperCase();
-        switch (voiceLang) {
-            case 'IT':
-                voice = 'it-IT';
-                break;
-            case 'DE':
-                voice = 'de-DE';
-                break;
-            case 'EN':
-                voice = 'en-GB';
-                break;
-            case 'ES':
-                voice = 'es-ES';
-                break;
-            case 'FR':
-                voice = 'fr-FR';
-                break;
-            case 'PT':
-                voice = 'pt-PT';
-                break;
-            case 'UK':
-                voice = 'uk-UK';
-                break;
-            case 'RU':
-                voice = 'ru-RU';
-                break;
-            case 'PL':
-                voice = 'pl-PL';
-                break;
-            case 'FI':
-                voice = 'fi-FI';
-                break;
-            case 'EL':
-                voice = 'el-GR';
-                break;
-            case 'AR':
-                voice = 'ar-SA';
-                break;
-            case 'ZH':
-                voice = 'zh-CN';
-                break;
-            case 'JA':
-                voice = 'ja-JP';
-                break;
-            default:
-                voice = 'it-IT';
-                break;
-        }
-        return voice;
-    };
     const handleSpeak = async (text) => {
         if (!text || preview || speakerMuted || !defaultEnableAudio) {
             const e = new CustomEvent('MemoriEndSpeak');
             document.dispatchEvent(e);
-            if (continuousSpeech) {
-                setListeningTimeout();
-            }
             return Promise.resolve();
         }
-        if (typeof stopListening === 'function') {
-            stopListening();
+        if (typeof stopRecording === 'function') {
+            stopRecording();
         }
         setMemoriTyping(true);
         const processedText = sanitizeText(text);
@@ -1261,25 +1283,6 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
         hasUserActivatedSpeak,
         setHasUserActivatedSpeak,
     ]);
-    const handleFallback = (text) => {
-        if (defaultEnableAudio) {
-            window.speechSynthesis.speak(new SpeechSynthesisUtterance(text));
-        }
-        cleanup();
-    };
-    const cleanup = () => {
-        if (recognizer) {
-            recognizer.stopContinuousRecognitionAsync();
-            recognizer.close();
-            recognizer = null;
-        }
-        if (speechSynthesizerRef.current) {
-            speechSynthesizerRef.current.close();
-            speechSynthesizerRef.current = null;
-        }
-        setListening(false);
-        clearListeningTimeout();
-    };
     const stopAudio = useCallback(async () => {
         ttsStop();
     }, [ttsStop]);
@@ -1297,230 +1300,8 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
             focusChatInput();
         }
     }, [currentDialogState === null || currentDialogState === void 0 ? void 0 : currentDialogState.emission]);
-    const [transcript, setTranscript] = useState('');
-    const [transcriptTimeout, setTranscriptTimeout] = useState(null);
-    const [isSpeaking, setIsSpeaking] = useState(false);
-    const resetTranscript = () => {
-        setTranscript('');
-    };
-    const setListeningTimeout = () => {
-        clearListeningTimeout();
-        console.debug('Setting speech processing timeout');
-        const timeout = setTimeout(() => {
-            console.debug('Speech timeout triggered, processing transcript');
-            handleTranscriptProcessing();
-        }, continuousSpeechTimeout * 1000 + 300);
-        setTranscriptTimeout(timeout);
-    };
-    const clearListeningTimeout = () => {
-        if (transcriptTimeout) {
-            console.debug('Clearing transcript timeout');
-            clearTimeout(transcriptTimeout);
-            setTranscriptTimeout(null);
-        }
-    };
-    const resetListeningTimeout = () => {
-        clearListeningTimeout();
-        if (continuousSpeech && !isProcessingSTT) {
-            console.debug('Setting new listening timeout');
-            setListeningTimeout();
-        }
-    };
-    useEffect(() => {
-        if (!isSpeaking && transcript && transcript.length > 0) {
-            console.debug('Transcript updated while not speaking, resetting timeout');
-            resetListeningTimeout();
-            resetInteractionTimeout();
-        }
-    }, [transcript, isSpeaking]);
-    useEffect(() => {
-        return () => {
-            clearListeningTimeout();
-        };
-    }, []);
-    let microphoneStream = null;
-    const startListening = async () => {
-        console.debug('Starting speech recognition...');
-        if (!sessionId) {
-            console.error('No session ID available');
-            throw new Error('No session ID available');
-        }
-        if (recognizer) {
-            console.debug('Cleaning up existing recognizer...');
-            try {
-                await new Promise((resolve, _) => {
-                    recognizer === null || recognizer === void 0 ? void 0 : recognizer.stopContinuousRecognitionAsync(resolve, error => {
-                        console.error('Error stopping recognition:', error);
-                        resolve();
-                    });
-                });
-                console.debug('Closing existing recognizer...');
-                recognizer.close();
-                recognizer = null;
-            }
-            catch (error) {
-                console.error('Error during recognizer cleanup:', error);
-            }
-        }
-        console.debug('Resetting transcript and STT state...');
-        resetTranscript();
-        setIsProcessingSTT(false);
-        console.debug('Adding delay for Azure services cleanup...');
-        await new Promise(resolve => setTimeout(resolve, 500));
-        try {
-            console.debug('Requesting microphone access...');
-            if (microphoneStream) {
-                microphoneStream.getTracks().forEach(track => track.stop());
-                microphoneStream = null;
-            }
-            const stream = await navigator.mediaDevices.getUserMedia({
-                audio: true,
-            });
-            setHasUserActivatedListening(true);
-            console.debug('Setting up speech config...');
-            console.debug('Creating audio config and recognizer...');
-            const audioConfig = speechSdk.AudioConfig.fromDefaultMicrophoneInput();
-            recognizer = new speechSdk.SpeechRecognizer(speechConfig, audioConfig);
-            console.debug('Setting up recognizer handlers...');
-            setupRecognizerHandlers(recognizer);
-            console.debug('Starting continuous recognition...');
-            await new Promise((resolve, reject) => {
-                recognizer === null || recognizer === void 0 ? void 0 : recognizer.startContinuousRecognitionAsync(resolve, error => {
-                    console.error('Failed to start recognition:', error);
-                    reject(error);
-                });
-            });
-            console.debug('Speech recognition started successfully');
-            setListening(true);
-        }
-        catch (error) {
-            console.error('Error in startListening:', error);
-            if (recognizer) {
-                console.debug('Cleaning up recognizer after error...');
-                recognizer.close();
-                recognizer = null;
-            }
-            setListening(false);
-            throw error;
-        }
-    };
-    startListeningRef.current = startListening;
-    const onEndSpeakStartListen = useCallback((_e) => {
-        if (isPlayingAudio && speechSynthesizerRef.current) {
-            speechSynthesizerRef.current.close();
-            speechSynthesizerRef.current = null;
-        }
-        if (continuousSpeech &&
-            (hasUserActivatedListening || !requestedListening)) {
-            setRequestedListening(true);
-            if (startListeningRef.current) {
-                startListeningRef.current();
-            }
-        }
-    }, [continuousSpeech, hasUserActivatedListening, isPlayingAudio, requestedListening]);
-    const setupSpeechConfig = (AZURE_COGNITIVE_SERVICES_TTS_KEY) => {
-        console.debug('Creating speech config...');
-        speechConfig = speechSdk.SpeechConfig.fromSubscription(AZURE_COGNITIVE_SERVICES_TTS_KEY, 'westeurope');
-        console.debug('Setting speech recognition language:', userLang);
-        speechConfig.speechRecognitionLanguage = getCultureCodeByLanguage(userLang);
-        speechConfig.speechSynthesisLanguage = getCultureCodeByLanguage(userLang);
-        speechConfig.speechSynthesisVoiceName = getTTSVoice(userLang);
-        return speechConfig;
-    };
-    const [isProcessingSTT, setIsProcessingSTT] = useState(false);
-    const setupRecognizerHandlers = (recognizer) => {
-        if (recognizer) {
-            console.debug('Setting up recognizer event handlers...');
-            recognizer.recognized = (_, event) => {
-                console.debug('Recognition event received');
-                handleRecognizedSpeech(event.result.text);
-            };
-            console.debug('Configuring recognizer properties...');
-            recognizer.properties.setProperty('SpeechServiceResponse_JsonResult', 'true');
-            recognizer.properties.setProperty('SpeechServiceConnection_NoiseSuppression', 'true');
-            recognizer.properties.setProperty('SpeechServiceConnection_SNRThresholdDb', '10.0');
-        }
-    };
-    let isProcessingSpeech = false;
-    const processSpeechAndSendMessage = (text) => {
-        if (isProcessingSpeech || !text || text.trim().length === 0) {
-            console.debug('Skipping speech processing: already processing or empty text');
-            return;
-        }
-        try {
-            isProcessingSpeech = true;
-            const message = stripDuplicates(text);
-            console.debug('Processing speech message:', message);
-            if (message.length > 0) {
-                setIsProcessingSTT(true);
-                setUserMessage('');
-                console.debug('Sending message:', message);
-                sendMessage(message);
-                resetTranscript();
-                clearListening();
-            }
-        }
-        finally {
-            setTimeout(() => {
-                isProcessingSpeech = false;
-            }, 1000);
-        }
-    };
-    const handleRecognizedSpeech = (text) => {
-        console.debug('Speech recognized:', text);
-        setTranscript(text);
-        setIsSpeaking(false);
-        if (!continuousSpeech) {
-            processSpeechAndSendMessage(text);
-        }
-    };
-    const handleTranscriptProcessing = () => {
-        if (transcript && transcript.length > 0 && listening) {
-            processSpeechAndSendMessage(transcript);
-        }
-        else if (listening) {
-            resetInteractionTimeout();
-        }
-    };
-    const stopListening = async () => {
-        console.debug('Stopping speech recognition');
-        if (recognizer) {
-            try {
-                recognizer.stopContinuousRecognitionAsync();
-                recognizer.close();
-            }
-            catch (error) {
-                console.error('Error stopping recognizer:', error);
-            }
-            recognizer = null;
-        }
-        if (microphoneStream) {
-            try {
-                microphoneStream.getTracks().forEach(track => track.stop());
-            }
-            catch (error) {
-                console.error('Error stopping microphone stream:', error);
-            }
-            microphoneStream = null;
-        }
-        setListening(false);
-    };
-    const clearListening = () => {
-        stopListening();
-        clearListeningTimeout();
-        setIsSpeaking(false);
-    };
-    const resetListening = () => {
-        if (listening) {
-            clearListening();
-            resetTranscript();
-            setUserMessage('');
-            startListening();
-        }
-    };
     const resetUIEffects = () => {
         try {
-            clearListening();
             clearInteractionTimeout();
             setClickedStart(false);
             timeoutRef.current = undefined;
@@ -1541,23 +1322,19 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
             document.removeEventListener('MemoriResetUIEffects', resetUIEffects);
         };
     }, []);
-    useEffect(() => {
-        if ((currentDialogState === null || currentDialogState === void 0 ? void 0 : currentDialogState.state) === 'Z0')
-            clearListening();
-    }, [currentDialogState === null || currentDialogState === void 0 ? void 0 : currentDialogState.state]);
     useEffect(() => {
         if (!isPlayingAudio &&
             continuousSpeech &&
             (hasUserActivatedListening || !requestedListening) &&
             sessionId) {
-            startListening();
+            startRecording();
         }
-        else if (isPlayingAudio && listening) {
-            stopListening();
+        else if (isPlayingAudio && isListening) {
+            stopRecording();
         }
     }, [isPlayingAudio, hasUserActivatedListening]);
     useEffect(() => {
-        resetListening();
+        stopRecording();
     }, [language]);
     const [sendOnEnter, setSendOnEnter] = useState('keypress');
     useEffect(() => {
@@ -1661,7 +1438,6 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
         }
     }, [integrationConfig, memori.avatarURL, ogImage]);
     const simulateUserPrompt = (text, translatedText) => {
-        stopListening();
         stopAudio();
         sendMessage(text, undefined, undefined, false, translatedText);
     };
@@ -1678,7 +1454,6 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
                 }, 1000);
             }
             else {
-                stopListening();
                 stopAudio();
                 sendMessage(text, undefined, undefined, undefined, undefined, hidden, typingText, useLoaderTextAsMsg, hasBatchQueued);
             }
@@ -2107,7 +1882,9 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
         avatar3dVisible,
         setAvatar3dVisible,
         hasUserActivatedSpeak,
-        isPlayingAudio: isPlayingAudio && !speakerMuted && ((_y = enableAudio !== null && enableAudio !== void 0 ? enableAudio : integrationConfig === null || integrationConfig === void 0 ? void 0 : integrationConfig.enableAudio) !== null && _y !== void 0 ? _y : true),
+        isPlayingAudio: isPlayingAudio &&
+            !speakerMuted &&
+            ((_y = enableAudio !== null && enableAudio !== void 0 ? enableAudio : integrationConfig === null || integrationConfig === void 0 ? void 0 : integrationConfig.enableAudio) !== null && _y !== void 0 ? _y : true),
         loading: !!memoriTyping,
         baseUrl,
         apiUrl: client.constants.BACKEND_URL,
@@ -2142,7 +1919,6 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
         memori,
         sessionID: sessionId || '',
         tenant,
-        provider: ttsProvider,
         translateTo: isMultilanguageEnabled &&
             userLang.toUpperCase() !==
                 ((_3 = ((_2 = (_1 = (_0 = (_z = memori.culture) === null || _z === void 0 ? void 0 : _z.split('-')) === null || _0 === void 0 ? void 0 : _0[0]) !== null && _1 !== void 0 ? _1 : i18n.language) !== null && _2 !== void 0 ? _2 : 'IT')) === null || _3 === void 0 ? void 0 : _3.toUpperCase())
@@ -2182,16 +1958,14 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
         onChangeUserMessage,
         sendMessage: (msg, media) => {
             stopAudio();
-            stopListening();
+            stopRecording();
             sendMessage(msg, media);
             setUserMessage('');
-            resetTranscript();
         },
-        stopListening: clearListening,
-        startListening,
+        stopListening: stopRecording,
+        startListening: startRecording,
         stopAudio,
-        resetTranscript,
-        listening,
+        listening: isListening,
         setEnableFocusChatInput,
         isPlayingAudio,
         customMediaRenderer,