@memori.ai/memori-react 8.0.2 → 8.1.0-rc.0

package/dist/components/MemoriWidget/MemoriWidget.js

@@ -6,7 +6,6 @@ const react_1 = require("react");
 const react_i18next_1 = require("react-i18next");
 const memori_api_client_1 = tslib_1.__importDefault(require("@memori.ai/memori-api-client"));
 const standardized_audio_context_1 = require("standardized-audio-context");
-const speechSdk = tslib_1.__importStar(require("microsoft-cognitiveservices-speech-sdk"));
 const classnames_1 = tslib_1.__importDefault(require("classnames"));
 const luxon_1 = require("luxon");
 const react_hot_toast_1 = tslib_1.__importDefault(require("react-hot-toast"));
@@ -41,6 +40,7 @@ const sanitizer_1 = require("../../helpers/sanitizer");
 const useTTS_1 = require("../../helpers/tts/useTTS");
 const Alert_1 = tslib_1.__importDefault(require("../ui/Alert"));
 const ChatHistory_1 = tslib_1.__importDefault(require("../ChatHistoryDrawer/ChatHistory"));
+const useSTT_1 = require("../../helpers/stt/useSTT");
 const getMemoriState = (integrationId) => {
     var _a, _b, _c, _d, _f;
     let widget = integrationId
@@ -155,9 +155,6 @@ window.getMemoriState = getMemoriState;
 window.typeMessage = typeMessage;
 window.typeMessageHidden = typeMessageHidden;
 window.typeBatchMessages = typeBatchMessages;
-let recognizer;
-let speechConfig;
-let audioDestination;
 let audioContext;
 let memoriPassword;
 let speakerMuted = false;
@@ -386,7 +383,9 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
             }
             const mediaDocuments = media === null || media === void 0 ? void 0 : media.filter(m => { var _a; return !m.mediumID && ((_a = m.properties) === null || _a === void 0 ? void 0 : _a.isAttachedFile); });
             if (mediaDocuments && mediaDocuments.length > 0) {
-                const documentContents = mediaDocuments.map(doc => doc.content).join(' ');
+                const documentContents = mediaDocuments
+                    .map(doc => doc.content)
+                    .join(' ');
                 msg = msg + ' ' + documentContents;
             }
             const { currentState, ...response } = await postTextEnteredEvent({
@@ -571,6 +570,62 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
                 : 0;
     const [birthDate, setBirthDate] = (0, react_1.useState)();
     const [showAgeVerification, setShowAgeVerification] = (0, react_1.useState)(false);
+    const getCultureCodeByLanguage = (lang) => {
+        var _a, _b;
+        let voice = '';
+        let voiceLang = (lang ||
+            ((_b = (_a = memori.culture) === null || _a === void 0 ? void 0 : _a.split('-')) === null || _b === void 0 ? void 0 : _b[0]) ||
+            i18n.language ||
+            'IT').toUpperCase();
+        switch (voiceLang) {
+            case 'IT':
+                voice = 'it-IT';
+                break;
+            case 'DE':
+                voice = 'de-DE';
+                break;
+            case 'EN':
+                voice = 'en-GB';
+                break;
+            case 'ES':
+                voice = 'es-ES';
+                break;
+            case 'FR':
+                voice = 'fr-FR';
+                break;
+            case 'PT':
+                voice = 'pt-PT';
+                break;
+            case 'UK':
+                voice = 'uk-UK';
+                break;
+            case 'RU':
+                voice = 'ru-RU';
+                break;
+            case 'PL':
+                voice = 'pl-PL';
+                break;
+            case 'FI':
+                voice = 'fi-FI';
+                break;
+            case 'EL':
+                voice = 'el-GR';
+                break;
+            case 'AR':
+                voice = 'ar-SA';
+                break;
+            case 'ZH':
+                voice = 'zh-CN';
+                break;
+            case 'JA':
+                voice = 'ja-JP';
+                break;
+            default:
+                voice = 'it-IT';
+                break;
+        }
+        return voice;
+    };
     const [sessionId, setSessionId] = (0, react_1.useState)(initialSessionID);
     const [currentDialogState, _setCurrentDialogState] = (0, react_1.useState)();
     const setCurrentDialogState = (state) => {
@@ -1044,17 +1099,43 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
             tenant: tenantID,
             region: 'westeurope',
             voiceType: memori.voiceType,
-            layout: selectedLayout
+            layout: selectedLayout,
         });
     }, [ttsProvider, userLang, memori.culture, memori.voiceType]);
+    const sttConfig = (0, react_1.useMemo)(() => ({
+        provider: ttsProvider,
+        language: getCultureCodeByLanguage(userLang),
+        tenant: tenantID,
+    }), [ttsProvider, userLang]);
     const { speak: ttsSpeak, stop: ttsStop, isPlaying: isPlayingAudio, speakerMuted, toggleMute, hasUserActivatedSpeak, setHasUserActivatedSpeak, error, setError, } = (0, useTTS_1.useTTS)(ttsConfig, {
         apiUrl: `${baseUrl}/api/tts`,
         continuousSpeech: continuousSpeech,
-        onEndSpeakStartListen: () => {
-            console.log('[MemoriWidget] onEndSpeakStartListen called');
-        },
         preview: preview,
     }, autoStart, defaultEnableAudio, defaultSpeakerActive);
+    const processSpeechAndSendMessage = (text) => {
+        console.log('processSpeechAndSendMessage', text);
+        if (!text || text.trim().length === 0) {
+            return;
+        }
+        try {
+            const message = (0, utils_1.stripDuplicates)(text);
+            console.debug('Processing speech message:', message);
+            if (message.length > 0) {
+                setUserMessage('');
+                console.debug('Sending message:', message);
+                sendMessage(message);
+            }
+        }
+        catch (error) {
+            console.error('Error in processSpeechAndSendMessage:', error);
+        }
+    };
+    const { isListening, startRecording, stopRecording, } = (0, useSTT_1.useSTT)(sttConfig, processSpeechAndSendMessage, {
+        apiUrl: `${baseUrl}/api/stt`,
+        continuousRecording: continuousSpeech,
+        silenceTimeout: continuousSpeechTimeout,
+        autoStart: autoStart,
+    }, defaultEnableAudio);
     const resetInteractionTimeout = () => {
         clearInteractionTimeout();
         if (!isPlayingAudio && !userMessage.length && !memoriTyping && !listening)
@@ -1156,73 +1237,14 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
         memoriTyping,
         hasUserActivatedSpeak,
     ]);
-    const getCultureCodeByLanguage = (lang) => {
-        var _a, _b;
-        let voice = '';
-        let voiceLang = (lang ||
-            ((_b = (_a = memori.culture) === null || _a === void 0 ? void 0 : _a.split('-')) === null || _b === void 0 ? void 0 : _b[0]) ||
-            i18n.language ||
-            'IT').toUpperCase();
-        switch (voiceLang) {
-            case 'IT':
-                voice = 'it-IT';
-                break;
-            case 'DE':
-                voice = 'de-DE';
-                break;
-            case 'EN':
-                voice = 'en-GB';
-                break;
-            case 'ES':
-                voice = 'es-ES';
-                break;
-            case 'FR':
-                voice = 'fr-FR';
-                break;
-            case 'PT':
-                voice = 'pt-PT';
-                break;
-            case 'UK':
-                voice = 'uk-UK';
-                break;
-            case 'RU':
-                voice = 'ru-RU';
-                break;
-            case 'PL':
-                voice = 'pl-PL';
-                break;
-            case 'FI':
-                voice = 'fi-FI';
-                break;
-            case 'EL':
-                voice = 'el-GR';
-                break;
-            case 'AR':
-                voice = 'ar-SA';
-                break;
-            case 'ZH':
-                voice = 'zh-CN';
-                break;
-            case 'JA':
-                voice = 'ja-JP';
-                break;
-            default:
-                voice = 'it-IT';
-                break;
-        }
-        return voice;
-    };
     const handleSpeak = async (text) => {
         if (!text || preview || speakerMuted || !defaultEnableAudio) {
             const e = new CustomEvent('MemoriEndSpeak');
             document.dispatchEvent(e);
-            if (continuousSpeech) {
-                setListeningTimeout();
-            }
             return Promise.resolve();
         }
-        if (typeof stopListening === 'function') {
-            stopListening();
+        if (typeof stopRecording === 'function') {
+            stopRecording();
         }
         setMemoriTyping(true);
         const processedText = (0, sanitizer_1.sanitizeText)(text);
@@ -1264,25 +1286,6 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
         hasUserActivatedSpeak,
         setHasUserActivatedSpeak,
     ]);
-    const handleFallback = (text) => {
-        if (defaultEnableAudio) {
-            window.speechSynthesis.speak(new SpeechSynthesisUtterance(text));
-        }
-        cleanup();
-    };
-    const cleanup = () => {
-        if (recognizer) {
-            recognizer.stopContinuousRecognitionAsync();
-            recognizer.close();
-            recognizer = null;
-        }
-        if (speechSynthesizerRef.current) {
-            speechSynthesizerRef.current.close();
-            speechSynthesizerRef.current = null;
-        }
-        setListening(false);
-        clearListeningTimeout();
-    };
     const stopAudio = (0, react_1.useCallback)(async () => {
         ttsStop();
     }, [ttsStop]);
@@ -1300,230 +1303,8 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
             focusChatInput();
         }
     }, [currentDialogState === null || currentDialogState === void 0 ? void 0 : currentDialogState.emission]);
-    const [transcript, setTranscript] = (0, react_1.useState)('');
-    const [transcriptTimeout, setTranscriptTimeout] = (0, react_1.useState)(null);
-    const [isSpeaking, setIsSpeaking] = (0, react_1.useState)(false);
-    const resetTranscript = () => {
-        setTranscript('');
-    };
-    const setListeningTimeout = () => {
-        clearListeningTimeout();
-        console.debug('Setting speech processing timeout');
-        const timeout = setTimeout(() => {
-            console.debug('Speech timeout triggered, processing transcript');
-            handleTranscriptProcessing();
-        }, continuousSpeechTimeout * 1000 + 300);
-        setTranscriptTimeout(timeout);
-    };
-    const clearListeningTimeout = () => {
-        if (transcriptTimeout) {
-            console.debug('Clearing transcript timeout');
-            clearTimeout(transcriptTimeout);
-            setTranscriptTimeout(null);
-        }
-    };
-    const resetListeningTimeout = () => {
-        clearListeningTimeout();
-        if (continuousSpeech && !isProcessingSTT) {
-            console.debug('Setting new listening timeout');
-            setListeningTimeout();
-        }
-    };
-    (0, react_1.useEffect)(() => {
-        if (!isSpeaking && transcript && transcript.length > 0) {
-            console.debug('Transcript updated while not speaking, resetting timeout');
-            resetListeningTimeout();
-            resetInteractionTimeout();
-        }
-    }, [transcript, isSpeaking]);
-    (0, react_1.useEffect)(() => {
-        return () => {
-            clearListeningTimeout();
-        };
-    }, []);
-    let microphoneStream = null;
-    const startListening = async () => {
-        console.debug('Starting speech recognition...');
-        if (!sessionId) {
-            console.error('No session ID available');
-            throw new Error('No session ID available');
-        }
-        if (recognizer) {
-            console.debug('Cleaning up existing recognizer...');
-            try {
-                await new Promise((resolve, _) => {
-                    recognizer === null || recognizer === void 0 ? void 0 : recognizer.stopContinuousRecognitionAsync(resolve, error => {
-                        console.error('Error stopping recognition:', error);
-                        resolve();
-                    });
-                });
-                console.debug('Closing existing recognizer...');
-                recognizer.close();
-                recognizer = null;
-            }
-            catch (error) {
-                console.error('Error during recognizer cleanup:', error);
-            }
-        }
-        console.debug('Resetting transcript and STT state...');
-        resetTranscript();
-        setIsProcessingSTT(false);
-        console.debug('Adding delay for Azure services cleanup...');
-        await new Promise(resolve => setTimeout(resolve, 500));
-        try {
-            console.debug('Requesting microphone access...');
-            if (microphoneStream) {
-                microphoneStream.getTracks().forEach(track => track.stop());
-                microphoneStream = null;
-            }
-            const stream = await navigator.mediaDevices.getUserMedia({
-                audio: true,
-            });
-            setHasUserActivatedListening(true);
-            console.debug('Setting up speech config...');
-            console.debug('Creating audio config and recognizer...');
-            const audioConfig = speechSdk.AudioConfig.fromDefaultMicrophoneInput();
-            recognizer = new speechSdk.SpeechRecognizer(speechConfig, audioConfig);
-            console.debug('Setting up recognizer handlers...');
-            setupRecognizerHandlers(recognizer);
-            console.debug('Starting continuous recognition...');
-            await new Promise((resolve, reject) => {
-                recognizer === null || recognizer === void 0 ? void 0 : recognizer.startContinuousRecognitionAsync(resolve, error => {
-                    console.error('Failed to start recognition:', error);
-                    reject(error);
-                });
-            });
-            console.debug('Speech recognition started successfully');
-            setListening(true);
-        }
-        catch (error) {
-            console.error('Error in startListening:', error);
-            if (recognizer) {
-                console.debug('Cleaning up recognizer after error...');
-                recognizer.close();
-                recognizer = null;
-            }
-            setListening(false);
-            throw error;
-        }
-    };
-    startListeningRef.current = startListening;
-    const onEndSpeakStartListen = (0, react_1.useCallback)((_e) => {
-        if (isPlayingAudio && speechSynthesizerRef.current) {
-            speechSynthesizerRef.current.close();
-            speechSynthesizerRef.current = null;
-        }
-        if (continuousSpeech &&
-            (hasUserActivatedListening || !requestedListening)) {
-            setRequestedListening(true);
-            if (startListeningRef.current) {
-                startListeningRef.current();
-            }
-        }
-    }, [continuousSpeech, hasUserActivatedListening, isPlayingAudio, requestedListening]);
-    const setupSpeechConfig = (AZURE_COGNITIVE_SERVICES_TTS_KEY) => {
-        console.debug('Creating speech config...');
-        speechConfig = speechSdk.SpeechConfig.fromSubscription(AZURE_COGNITIVE_SERVICES_TTS_KEY, 'westeurope');
-        console.debug('Setting speech recognition language:', userLang);
-        speechConfig.speechRecognitionLanguage = getCultureCodeByLanguage(userLang);
-        speechConfig.speechSynthesisLanguage = getCultureCodeByLanguage(userLang);
-        speechConfig.speechSynthesisVoiceName = (0, ttsVoiceUtility_1.getTTSVoice)(userLang);
-        return speechConfig;
-    };
-    const [isProcessingSTT, setIsProcessingSTT] = (0, react_1.useState)(false);
-    const setupRecognizerHandlers = (recognizer) => {
-        if (recognizer) {
-            console.debug('Setting up recognizer event handlers...');
-            recognizer.recognized = (_, event) => {
-                console.debug('Recognition event received');
-                handleRecognizedSpeech(event.result.text);
-            };
-            console.debug('Configuring recognizer properties...');
-            recognizer.properties.setProperty('SpeechServiceResponse_JsonResult', 'true');
-            recognizer.properties.setProperty('SpeechServiceConnection_NoiseSuppression', 'true');
-            recognizer.properties.setProperty('SpeechServiceConnection_SNRThresholdDb', '10.0');
-        }
-    };
-    let isProcessingSpeech = false;
-    const processSpeechAndSendMessage = (text) => {
-        if (isProcessingSpeech || !text || text.trim().length === 0) {
-            console.debug('Skipping speech processing: already processing or empty text');
-            return;
-        }
-        try {
-            isProcessingSpeech = true;
-            const message = (0, utils_1.stripDuplicates)(text);
-            console.debug('Processing speech message:', message);
-            if (message.length > 0) {
-                setIsProcessingSTT(true);
-                setUserMessage('');
-                console.debug('Sending message:', message);
-                sendMessage(message);
-                resetTranscript();
-                clearListening();
-            }
-        }
-        finally {
-            setTimeout(() => {
-                isProcessingSpeech = false;
-            }, 1000);
-        }
-    };
-    const handleRecognizedSpeech = (text) => {
-        console.debug('Speech recognized:', text);
-        setTranscript(text);
-        setIsSpeaking(false);
-        if (!continuousSpeech) {
-            processSpeechAndSendMessage(text);
-        }
-    };
-    const handleTranscriptProcessing = () => {
-        if (transcript && transcript.length > 0 && listening) {
-            processSpeechAndSendMessage(transcript);
-        }
-        else if (listening) {
-            resetInteractionTimeout();
-        }
-    };
-    const stopListening = async () => {
-        console.debug('Stopping speech recognition');
-        if (recognizer) {
-            try {
-                recognizer.stopContinuousRecognitionAsync();
-                recognizer.close();
-            }
-            catch (error) {
-                console.error('Error stopping recognizer:', error);
-            }
-            recognizer = null;
-        }
-        if (microphoneStream) {
-            try {
-                microphoneStream.getTracks().forEach(track => track.stop());
-            }
-            catch (error) {
-                console.error('Error stopping microphone stream:', error);
-            }
-            microphoneStream = null;
-        }
-        setListening(false);
-    };
-    const clearListening = () => {
-        stopListening();
-        clearListeningTimeout();
-        setIsSpeaking(false);
-    };
-    const resetListening = () => {
-        if (listening) {
-            clearListening();
-            resetTranscript();
-            setUserMessage('');
-            startListening();
-        }
-    };
     const resetUIEffects = () => {
         try {
-            clearListening();
             clearInteractionTimeout();
             setClickedStart(false);
             timeoutRef.current = undefined;
@@ -1544,23 +1325,19 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
             document.removeEventListener('MemoriResetUIEffects', resetUIEffects);
         };
     }, []);
-    (0, react_1.useEffect)(() => {
-        if ((currentDialogState === null || currentDialogState === void 0 ? void 0 : currentDialogState.state) === 'Z0')
-            clearListening();
-    }, [currentDialogState === null || currentDialogState === void 0 ? void 0 : currentDialogState.state]);
     (0, react_1.useEffect)(() => {
         if (!isPlayingAudio &&
             continuousSpeech &&
             (hasUserActivatedListening || !requestedListening) &&
             sessionId) {
-            startListening();
+            startRecording();
         }
-        else if (isPlayingAudio && listening) {
-            stopListening();
+        else if (isPlayingAudio && isListening) {
+            stopRecording();
         }
     }, [isPlayingAudio, hasUserActivatedListening]);
     (0, react_1.useEffect)(() => {
-        resetListening();
+        stopRecording();
     }, [language]);
     const [sendOnEnter, setSendOnEnter] = (0, react_1.useState)('keypress');
     (0, react_1.useEffect)(() => {
@@ -1664,7 +1441,6 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
         }
     }, [integrationConfig, memori.avatarURL, ogImage]);
     const simulateUserPrompt = (text, translatedText) => {
-        stopListening();
         stopAudio();
         sendMessage(text, undefined, undefined, false, translatedText);
     };
@@ -1681,7 +1457,6 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
                 }, 1000);
             }
             else {
-                stopListening();
                 stopAudio();
                 sendMessage(text, undefined, undefined, undefined, undefined, hidden, typingText, useLoaderTextAsMsg, hasBatchQueued);
             }
@@ -2110,7 +1885,9 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
         avatar3dVisible,
         setAvatar3dVisible,
         hasUserActivatedSpeak,
-        isPlayingAudio: isPlayingAudio && !speakerMuted && ((_y = enableAudio !== null && enableAudio !== void 0 ? enableAudio : integrationConfig === null || integrationConfig === void 0 ? void 0 : integrationConfig.enableAudio) !== null && _y !== void 0 ? _y : true),
+        isPlayingAudio: isPlayingAudio &&
+            !speakerMuted &&
+            ((_y = enableAudio !== null && enableAudio !== void 0 ? enableAudio : integrationConfig === null || integrationConfig === void 0 ? void 0 : integrationConfig.enableAudio) !== null && _y !== void 0 ? _y : true),
         loading: !!memoriTyping,
         baseUrl,
         apiUrl: client.constants.BACKEND_URL,
@@ -2145,7 +1922,6 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
         memori,
         sessionID: sessionId || '',
         tenant,
-        provider: ttsProvider,
         translateTo: isMultilanguageEnabled &&
             userLang.toUpperCase() !==
                 ((_3 = ((_2 = (_1 = (_0 = (_z = memori.culture) === null || _z === void 0 ? void 0 : _z.split('-')) === null || _0 === void 0 ? void 0 : _0[0]) !== null && _1 !== void 0 ? _1 : i18n.language) !== null && _2 !== void 0 ? _2 : 'IT')) === null || _3 === void 0 ? void 0 : _3.toUpperCase())
@@ -2185,16 +1961,14 @@ const MemoriWidget = ({ memori, memoriConfigs, ownerUserID, ownerUserName, tenan
         onChangeUserMessage,
         sendMessage: (msg, media) => {
             stopAudio();
-            stopListening();
+            stopRecording();
             sendMessage(msg, media);
             setUserMessage('');
-            resetTranscript();
         },
-        stopListening: clearListening,
-        startListening,
+        stopListening: stopRecording,
+        startListening: startRecording,
         stopAudio,
-        resetTranscript,
-        listening,
+        listening: isListening,
         setEnableFocusChatInput,
         isPlayingAudio,
         customMediaRenderer,