@memori.ai/memori-react 7.19.2 → 7.21.1

package/src/components/MemoriWidget/MemoriWidget.tsx

@@ -2202,26 +2202,20 @@ const MemoriWidget = ({
     cleanup();
   };
 
-  const cleanup = (): void => {
-    setIsPlayingAudio(false);
-    stopProcessing();
-    resetVisemeQueue();
-    memoriSpeaking = false;
+  const cleanup = () => {
+    if (recognizer) {
+      recognizer.stopContinuousRecognitionAsync();
+      recognizer.close();
+      recognizer = null;
+    }
 
-    try {
-      if (speechSynthesizer) {
-        const currentSynthesizer = speechSynthesizer;
-        speechSynthesizer = null; // Clear reference first
-        console.debug('Closing speech synthesizer');
-        currentSynthesizer.close();
-      }
-    } catch (error) {
-      console.debug('Error during synthesizer cleanup:', error);
-      // Even if close fails, ensure synthesizer is nullified
+    if (speechSynthesizer) {
+      speechSynthesizer.close();
       speechSynthesizer = null;
     }
 
-    emitEndSpeakEvent();
+    setListening(false);
+    clearListeningTimeout();
   };
 
   // Modify stopAudio to include speech state reset
@@ -2282,34 +2276,41 @@ const MemoriWidget = ({
 
   const resetTranscript = () => {
     setTranscript('');
-    // setIsProcessingSTT(false);
   };
-
+  // Modify setListeningTimeout to be more robust
   const setListeningTimeout = () => {
-    clearListeningTimeout();
-    const timeout = setTimeout(
-      handleTranscriptProcessing,
-      continuousSpeechTimeout * 1000 + 300
-    );
+    clearListeningTimeout(); // Clear any existing timeout
+
+    console.debug('Setting speech processing timeout');
+    const timeout = setTimeout(() => {
+      console.debug('Speech timeout triggered, processing transcript');
+      handleTranscriptProcessing();
+    }, continuousSpeechTimeout * 1000 + 300);
+
     setTranscriptTimeout(timeout as unknown as NodeJS.Timeout);
   };
 
   const clearListeningTimeout = () => {
     if (transcriptTimeout) {
+      console.debug('Clearing transcript timeout');
       clearTimeout(transcriptTimeout);
       setTranscriptTimeout(null);
     }
   };
 
+  // Add safety check in resetListeningTimeout
   const resetListeningTimeout = () => {
     clearListeningTimeout();
-    if (continuousSpeech) {
+    if (continuousSpeech && !isProcessingSTT) {
+      console.debug('Setting new listening timeout');
       setListeningTimeout();
     }
   };
-  // Modified useEffect to handle transcript changes
+
+  // Make sure only one path can trigger message sending
   useEffect(() => {
-    if (!isSpeaking) {
+    if (!isSpeaking && transcript && transcript.length > 0) {
+      console.debug('Transcript updated while not speaking, resetting timeout');
       resetListeningTimeout();
       resetInteractionTimeout();
     }
@@ -2325,87 +2326,126 @@ const MemoriWidget = ({
   /**
    * Listening methods
    */
-  /**
-   * Starts speech recognition using Azure Cognitive Services
-   * Sets up recognizer and begins continuous recognition
-   */
+  let microphoneStream: MediaStream | null = null;
+  // Modify startListening to ensure full cleanup before starting
   const startListening = async (): Promise<void> => {
+    console.debug('Starting speech recognition...');
+
     if (!AZURE_COGNITIVE_SERVICES_TTS_KEY) {
+      console.error('No TTS key available');
       throw new Error('No TTS key available');
     }
 
     if (!sessionId) {
+      console.error('No session ID available');
       throw new Error('No session ID available');
     }
 
-    // Ensure complete cleanup before starting, if it's already listening, stop it
-    cleanup();
+    // First, ensure any existing recognizer is fully closed
+    if (recognizer) {
+      console.debug('Cleaning up existing recognizer...');
+      try {
+        // Stop the recognizer properly
+        await new Promise<void>((resolve, _) => {
+          recognizer?.stopContinuousRecognitionAsync(resolve, error => {
+            console.error('Error stopping recognition:', error);
+            resolve(); // Resolve anyway to continue cleanup
+          });
+        });
+
+        console.debug('Closing existing recognizer...');
+        recognizer.close();
+        recognizer = null;
+      } catch (error) {
+        console.error('Error during recognizer cleanup:', error);
+        // Continue with initialization anyway
+      }
+    }
+
+    // Clear any existing state
+    console.debug('Resetting transcript and STT state...');
     resetTranscript();
+    setIsProcessingSTT(false);
+
+    // Add a small delay to ensure Azure services have time to release resources
+    console.debug('Adding delay for Azure services cleanup...');
+    await new Promise(resolve => setTimeout(resolve, 500));
 
     try {
-      // Add delay to ensure previous instance is fully cleaned up
-      // await new Promise(resolve => setTimeout(resolve, 300));
+      console.debug('Requesting microphone access...');
+      // Release previous microphone stream if it exists
+      if (microphoneStream) {
+        microphoneStream.getTracks().forEach(track => track.stop());
+        microphoneStream = null;
+      }
 
-      const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+      const stream = await navigator.mediaDevices.getUserMedia({
+        audio: true,
+      });
       setHasUserActivatedListening(true);
 
       // Recreate speech config each time
+      console.debug('Setting up speech config...');
       speechConfig = setupSpeechConfig(AZURE_COGNITIVE_SERVICES_TTS_KEY);
 
+      console.debug('Creating audio config and recognizer...');
       const audioConfig = speechSdk.AudioConfig.fromDefaultMicrophoneInput();
       recognizer = new speechSdk.SpeechRecognizer(speechConfig, audioConfig);
 
       // Set up recognizer event handlers
+      console.debug('Setting up recognizer handlers...');
       setupRecognizerHandlers(recognizer);
 
-      // Start recognition
-      setListening(true);
-      recognizer.startContinuousRecognitionAsync();
-
-      recognizer.canceled = (_s, e) => {
-        if (e.reason === speechSdk.CancellationReason.Error) {
-          console.debug(`"CANCELED: ErrorCode=${e.errorCode}`);
-          console.debug(`"CANCELED: ErrorDetails=${e.errorDetails}`);
-          console.debug(
-            'CANCELED: Did you set the speech resource key and region values?'
-          );
-          stopListening();
-          cleanup();
-        }
-
-        stopListening();
-      };
+      // Start recognition - use promises for better error handling
+      console.debug('Starting continuous recognition...');
+      await new Promise<void>((resolve, reject) => {
+        recognizer?.startContinuousRecognitionAsync(resolve, error => {
+          console.error('Failed to start recognition:', error);
+          reject(error);
+        });
+      });
 
-      recognizer.sessionStopped = (_s, _e) => {
-        stopListening();
-        resetTranscript();
-      };
+      console.debug('Speech recognition started successfully');
+      setListening(true);
     } catch (error) {
       console.error('Error in startListening:', error);
-      stopListening();
+      // Ensure cleanup happens even on error
+      if (recognizer) {
+        console.debug('Cleaning up recognizer after error...');
+        recognizer.close();
+        recognizer = null;
+      }
+      setListening(false);
       throw error;
     }
   };
 
   const setupSpeechConfig = (AZURE_COGNITIVE_SERVICES_TTS_KEY: string) => {
+    console.debug('Creating speech config...');
     speechConfig = speechSdk.SpeechConfig.fromSubscription(
       AZURE_COGNITIVE_SERVICES_TTS_KEY,
       'westeurope'
     );
+    console.debug('Setting speech recognition language:', userLang);
     speechConfig.speechRecognitionLanguage = getCultureCodeByLanguage(userLang);
     speechConfig.speechSynthesisLanguage = getCultureCodeByLanguage(userLang);
     speechConfig.speechSynthesisVoiceName = getTTSVoice(userLang); // https://docs.microsoft.com/it-it/azure/cognitive-services/speech-service/language-support#text-to-speech
     return speechConfig;
   };
 
+  const [isProcessingSTT, setIsProcessingSTT] = useState(false);
+
   const setupRecognizerHandlers = (recognizer: speechSdk.SpeechRecognizer) => {
     if (recognizer) {
+      console.debug('Setting up recognizer event handlers...');
       recognizer.recognized = (_, event) => {
         // Process the recognized speech result
+        console.debug('Recognition event received');
         handleRecognizedSpeech(event.result.text);
       };
 
       // Configure speech recognition properties directly on the recognizer
+      console.debug('Configuring recognizer properties...');
       recognizer.properties.setProperty(
         'SpeechServiceResponse_JsonResult',
         'true'
@@ -2423,32 +2463,66 @@ const MemoriWidget = ({
     }
   };
 
-  const handleRecognizedSpeech = (text: string) => {
-    console.debug('Handling recognized speech:', text);
+  // Add a mutex-like flag to prevent duplicate processing
+  let isProcessingSpeech = false;
 
-    if (!text || text.trim().length === 0) {
-      console.debug('No valid text received from speech recognition');
+  // Create a single, centralized function to process and send messages
+  const processSpeechAndSendMessage = (text: string) => {
+    // Skip if already processing or no text
+    if (isProcessingSpeech || !text || text.trim().length === 0) {
+      console.debug(
+        'Skipping speech processing: already processing or empty text'
+      );
       return;
     }
 
+    try {
+      // Set processing flag immediately
+      isProcessingSpeech = true;
+
+      // Process the text
+      const message = stripDuplicates(text);
+      console.debug('Processing speech message:', message);
+
+      if (message.length > 0) {
+        // Update UI states
+        setIsProcessingSTT(true);
+        setUserMessage('');
+
+        // Send the message
+        console.debug('Sending message:', message);
+        sendMessage(message);
+
+        // Reset states
+        resetTranscript();
+        clearListening();
+      }
+    } finally {
+      // Reset processing flag after a short delay to prevent race conditions
+      setTimeout(() => {
+        isProcessingSpeech = false;
+      }, 1000);
+    }
+  };
+
+  // Update handleRecognizedSpeech to use the centralized function
+  const handleRecognizedSpeech = (text: string) => {
+    console.debug('Speech recognized:', text);
     setTranscript(text);
     setIsSpeaking(false);
 
-    const message = stripDuplicates(text);
-    console.debug('Stripped message:', message);
-    if (message.length > 0) {
-      setUserMessage(message);
+    // Don't process here - wait for timeout or explicit processing
+    if (!continuousSpeech) {
+      // For manual mode, process immediately
+      processSpeechAndSendMessage(text);
     }
+    // For continuous mode, rely on the timeout
   };
 
-  // Helper function to handle transcript processing
+  // Update handleTranscriptProcessing to use the centralized function
   const handleTranscriptProcessing = () => {
-    const message = stripDuplicates(transcript);
-    if (message.length > 0 && listening) {
-      sendMessage(message);
-      resetTranscript();
-      setUserMessage('');
-      clearListening();
+    if (transcript && transcript.length > 0 && listening) {
+      processSpeechAndSendMessage(transcript);
     } else if (listening) {
       resetInteractionTimeout();
     }
@@ -2458,14 +2532,32 @@ const MemoriWidget = ({
    * Stops the speech recognition process
    * Closes recognizer and cleans up resources
    */
-  const stopListening = () => {
+  // Similarly, modify stopListening to use promises
+  // Enhance stopListening to fully release resources
+  const stopListening = async () => {
     console.debug('Stopping speech recognition');
+
+    // Stop the recognizer
     if (recognizer) {
-      // Stop continuous recognition and close the recognizer
-      recognizer.stopContinuousRecognitionAsync();
-      recognizer.close();
+      try {
+        recognizer.stopContinuousRecognitionAsync();
+        recognizer.close();
+      } catch (error) {
+        console.error('Error stopping recognizer:', error);
+      }
       recognizer = null;
     }
+
+    // Release the microphone stream
+    if (microphoneStream) {
+      try {
+        microphoneStream.getTracks().forEach(track => track.stop());
+      } catch (error) {
+        console.error('Error stopping microphone stream:', error);
+      }
+      microphoneStream = null;
+    }
+
     setListening(false);
   };
 

package/src/components/UploadButton/UploadButton.tsx

@@ -39,7 +39,7 @@ const FileUploadButton = ({
 }) => {
   // State for loading indicator
   const [isLoading, setIsLoading] = useState(false);
-  // State for tracking upload errors  
+  // State for tracking upload errors
   const [errors, setErrors] = useState<UploadError[]>([]);
   // Reference to hidden file input
   const fileInputRef = useRef<HTMLInputElement>(null);
@@ -122,7 +122,7 @@ const FileUploadButton = ({
   const validateFile = (file: File): boolean => {
     const fileExt = `.${file.name.split('.').pop()?.toLowerCase()}`;
     const ALLOWED_FILE_TYPES = ['.pdf', '.txt'];
-    
+
     if (!ALLOWED_FILE_TYPES.includes(fileExt)) {
       addError({
         message: `File type "${fileExt}" is not supported. Please use: ${ALLOWED_FILE_TYPES.join(
@@ -161,7 +161,7 @@ const FileUploadButton = ({
 
       if (fileExt === 'pdf') {
         text = await extractTextFromPDF(file);
-      } else if (fileExt === 'txt') {
+      } else if (fileExt === 'txt' || fileExt === 'json') {
         text = await file.text();
       }
 
@@ -241,7 +241,7 @@ const FileUploadButton = ({
       <input
         ref={fileInputRef}
         type="file"
-        accept=".pdf,.txt"
+        accept=".pdf,.txt,.json"
         className="memori--upload-file-input"
         onChange={handleFileSelect}
         multiple

package/src/components/UploadButton/__snapshots__/UploadButton.test.tsx.snap

@@ -6,7 +6,7 @@ exports[`renders UploadButton unchanged 1`] = `
     class="relative file-upload-wrapper"
   >
     <input
-      accept=".pdf,.txt"
+      accept=".pdf,.txt,.json"
       class="memori--upload-file-input"
       multiple=""
       type="file"

package/src/components/WhyThisAnswer/WhyThisAnswer.css

@@ -126,3 +126,46 @@ div.memori--whythisanswer-skeleton-text {
     margin-bottom: 0.5rem;
     background: #ccc;
 }
+
+.memori--whythisanswer-title-text {
+    width: 100%;
+}
+
+.memori--whythisanswer-title-text-top {
+    display: flex;
+    align-items: space-between;
+    gap: 0.5rem;
+}
+
+.memori--whythisanswer-title-text-top p {
+    margin: 0;
+    font-size: 0.75rem;
+}
+
+.memori--whythisanswer-contextvars {
+    display: flex;
+    flex-wrap: wrap;
+    margin-top: 0.5rem;
+}
+
+.memori--whythisanswer-contextvars-card {
+    max-width: max-content;
+    padding: 0.5rem;
+    box-shadow: 0 0 0.25rem var(--memori-primary);
+    font-size: 0.75rem;
+}
+
+.memori--whythisanswer-contextvars-card + .memori--whythisanswer-contextvars-card {
+    margin-left: 0.5rem;
+}
+
+.memori--whythisanswer-contextvars-card > div > div {
+    padding: 0 !important;
+}
+
+.memori--whythisanswer-title-text-top-container {
+    display: flex;
+    justify-content: space-between;
+    margin-bottom: 8px;
+    gap: 0.5rem;
+}

package/src/components/WhyThisAnswer/WhyThisAnswer.stories.tsx

@@ -1,11 +1,11 @@
 import React from 'react';
 import { Meta, Story } from '@storybook/react';
-import { sessionID, memoryQuestion } from '../../mocks/data';
+import { sessionID } from '../../mocks/data';
 import I18nWrapper from '../../I18nWrapper';
 import WhyThisAnswer, { Props } from './WhyThisAnswer';
+import { SearchMatches } from '@memori.ai/memori-api-client/dist/types';
 
 import './WhyThisAnswer.css';
-import { SearchMatches } from '@memori.ai/memori-api-client/dist/types';
 
 const meta: Meta = {
   title: 'Why This Answer',
@@ -69,6 +69,7 @@ WithData.args = {
       confidenceLevel: 'HIGH',
       memory: {
         memoryID: '1',
+        memoryType: 'Question',
         title: 'This is the title of the content',
         titleVariants: [
           "This is a variant of the content's title",
@@ -89,6 +90,7 @@ WithData.args = {
       confidenceLevel: 'LOW',
       memory: {
         memoryID: '2',
+        memoryType: 'Question',
         title: 'Content with a long answer',
         titleVariants: undefined,
         answers: [
@@ -105,6 +107,7 @@ WithData.args = {
         memoryID: '3',
         title: 'Content with sources',
         titleVariants: undefined,
+        memoryType: 'Question',
         answers: [
           {
             text: 'This is a test answer',
@@ -130,9 +133,9 @@ WithData.args = {
       confidence: 0.7,
       confidenceLevel: 'MEDIUM',
       memory: {
-        ...memoryQuestion,
         memoryID: '4',
         title: 'Content with links',
+        memoryType: 'Question',
         titleVariants: undefined,
         answers: [
           {
@@ -155,5 +158,43 @@ WithData.args = {
         ],
       },
     } as SearchMatches,
+    {
+      confidence: 0.7,
+      confidenceLevel: 'MEDIUM',
+      memory: {
+        memoryID: '5',
+        memoryType: 'Question',
+        title: 'Content with receiver',
+        titleVariants: undefined,
+        receiverName: 'receiver',
+        receiverTag: '🧑‍💻',
+        answers: [
+          {
+            text: 'This is a an answer',
+          },
+        ],
+        media: [],
+      },
+    } as SearchMatches,
+    {
+      confidence: 0.5,
+      confidenceLevel: 'MEDIUM',
+      memory: {
+        memoryID: '5',
+        memoryType: 'Question',
+        title: 'Content with context',
+        titleVariants: undefined,
+        contextVars: {
+          KEY1: 'VALUE1',
+          KEY2: 'VALUE2',
+        },
+        answers: [
+          {
+            text: 'This is a an answer',
+          },
+        ],
+        media: [],
+      },
+    } as SearchMatches,
   ],
 };