@memori.ai/memori-react 8.6.1 → 8.6.3

package/src/helpers/stt/useSTT.ts

@@ -49,28 +49,50 @@ export interface UseSTTOptions {
  */
 export type RecordingState = 'idle' | 'recording' | 'processing' | 'error';
 
-/**
- * Convert audio blob to WAV format
- */
+  /**
+   * Convert audio blob to WAV format
+   */
 async function convertToWav(audioBlob: Blob): Promise<Blob> {
   return new Promise((resolve, reject) => {
-    const audioContext = new (window.AudioContext || (window as any).webkitAudioContext)();
+    // Safari compatibility: check for AudioContext support
+    const AudioContextClass = window.AudioContext || (window as any).webkitAudioContext;
+    if (!AudioContextClass) {
+      reject(new Error('AudioContext not supported in this browser'));
+      return;
+    }
+
+    const audioContext = new AudioContextClass();
     const fileReader = new FileReader();
     
     fileReader.onload = async () => {
       try {
         const arrayBuffer = fileReader.result as ArrayBuffer;
+        
+        // Resume context if suspended (required for Safari)
+        if (audioContext.state === 'suspended') {
+          await audioContext.resume();
+        }
+        
         const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
         
         // Convert to WAV format
         const wavBlob = audioBufferToWav(audioBuffer);
+        
+        // Close the audio context to free resources
+        await audioContext.close();
+        
         resolve(wavBlob);
       } catch (error) {
+        console.error('Error converting audio to WAV:', error);
         reject(error);
       }
     };
     
-    fileReader.onerror = () => reject(new Error('Failed to read audio file'));
+    fileReader.onerror = () => {
+      console.error('Failed to read audio file');
+      reject(new Error('Failed to read audio file'));
+    };
+    
     fileReader.readAsArrayBuffer(audioBlob);
   });
 }
@@ -150,11 +172,15 @@ export function useSTT(
   const audioContextRef = useRef<AudioContext | null>(null);
   const analyserRef = useRef<AnalyserNode | null>(null);
   const dataArrayRef = useRef<Uint8Array | null>(null);
+  const backgroundNoiseRef = useRef<number>(0);
+  const audioActivityHistoryRef = useRef<number[]>([]);
+  const lastStopTimeRef = useRef<number>(0); // Track when recording was last stopped
   const apiUrl = options.apiUrl || '/api/stt';
-  const silenceTimeout = options.silenceTimeout || 2; // Default 2 seconds
+  const silenceTimeout = options.silenceTimeout || 3; // Increased default to 3 seconds
 
   const initializeRecording = useCallback(async (): Promise<boolean> => {
     try {
+      
       if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
         throw new Error('Media recording is not supported in this browser');
       }
@@ -170,49 +196,73 @@ export function useSTT(
 
       audioStreamRef.current = stream;
 
-      // Initialize audio context for silence detection if continuous recording is enabled
+      // Initialize audio context for silence detection only if continuous recording is enabled
       if (options.continuousRecording) {
         try {
-          audioContextRef.current = new (window.AudioContext || (window as any).webkitAudioContext)();
-          analyserRef.current = audioContextRef.current.createAnalyser();
-          analyserRef.current.fftSize = 256;
-          const bufferLength = analyserRef.current.frequencyBinCount;
-          dataArrayRef.current = new Uint8Array(bufferLength);
-          
-          const source = audioContextRef.current.createMediaStreamSource(stream);
-          source.connect(analyserRef.current);
+          // Safari compatibility: check for AudioContext support
+          const AudioContextClass = window.AudioContext || (window as any).webkitAudioContext;
+          if (AudioContextClass) {
+            audioContextRef.current = new AudioContextClass({
+              sampleRate: 16000, // Match the audio input sample rate
+              latencyHint: 'interactive' // Better for real-time analysis
+            });
+            
+            
+            // Resume context if suspended (required for Safari)
+            if (audioContextRef.current.state === 'suspended') {
+              await audioContextRef.current.resume();
+            }
+            
+            // Wait a bit for Safari to stabilize the AudioContext
+            await new Promise(resolve => setTimeout(resolve, 100));
+            
+            analyserRef.current = audioContextRef.current.createAnalyser();
+            analyserRef.current.fftSize = 512; // Increased for better frequency resolution
+            analyserRef.current.smoothingTimeConstant = 0.3; // Reduced for more responsive detection
+            analyserRef.current.minDecibels = -90;
+            analyserRef.current.maxDecibels = -10;
+            
+            const bufferLength = analyserRef.current.frequencyBinCount;
+            dataArrayRef.current = new Uint8Array(bufferLength);
+            
+            const source = audioContextRef.current.createMediaStreamSource(stream);
+            source.connect(analyserRef.current);
+            
+            // Initialize audio analysis state
+            backgroundNoiseRef.current = 0;
+            audioActivityHistoryRef.current = [];
+            
+          } else {
+            console.warn('🎤 [INIT] AudioContext not supported in this browser');
+          }
         } catch (err) {
           // Silence detection initialization failed but we can continue
+          console.error('🎤 [INIT] Silence detection initialization failed:', err);
         }
+      } else {
+        console.log('🎤 [INIT] Continuous recording disabled, skipping silence detection setup');
       }
 
-      // Format selection based on provider
+      // Format selection based on provider with Safari compatibility
       let mimeType = '';
 
-      if (config.provider === 'azure') {
-        // For Azure, we'll record in a supported format and convert to WAV
-        if (MediaRecorder.isTypeSupported('audio/webm;codecs=opus')) {
-          mimeType = 'audio/webm;codecs=opus';
-        } else if (MediaRecorder.isTypeSupported('audio/webm')) {
-          mimeType = 'audio/webm';
-        } else if (MediaRecorder.isTypeSupported('audio/mp4')) {
-          mimeType = 'audio/mp4';
-        } else if (MediaRecorder.isTypeSupported('audio/ogg;codecs=opus')) {
-          mimeType = 'audio/ogg;codecs=opus';
-        }
-      } else {
-        // For OpenAI, use the best available format
-        if (MediaRecorder.isTypeSupported('audio/webm;codecs=opus')) {
-          mimeType = 'audio/webm;codecs=opus';
-        } else if (MediaRecorder.isTypeSupported('audio/webm')) {
-          mimeType = 'audio/webm';
-        } else if (MediaRecorder.isTypeSupported('audio/mp4')) {
-          mimeType = 'audio/mp4';
-        } else if (MediaRecorder.isTypeSupported('audio/ogg;codecs=opus')) {
-          mimeType = 'audio/ogg;codecs=opus';
+      // Safari compatibility: prefer formats that work well on Safari
+      const supportedFormats = [
+        'audio/mp4', // Best Safari support
+        'audio/webm;codecs=opus',
+        'audio/webm',
+        'audio/ogg;codecs=opus',
+        'audio/wav' // Fallback
+      ];
+
+      for (const format of supportedFormats) {
+        if (MediaRecorder.isTypeSupported(format)) {
+          mimeType = format;
+          break;
         }
       }
 
+
       const mediaRecorder = new MediaRecorder(
         stream,
         mimeType ? { mimeType } : {}
@@ -225,11 +275,10 @@ export function useSTT(
       };
 
       mediaRecorder.onstop = async () => {
-        if (!isRecordingRef.current || !isMountedRef.current) {
-          return;
-        }
 
+        // Immediately set processing state to prevent ghost messages
         setRecordingState('processing');
+        setIsListening(false);
 
         try {
           if (chunksRef.current.length === 0) {
@@ -244,28 +293,31 @@ export function useSTT(
             throw new Error('Recorded audio is empty');
           }
 
+
           // Convert to WAV if using Azure
           if (config.provider === 'azure') {
             try {
-              console.log('Converting audio to WAV format for Azure');
               audioBlob = await convertToWav(audioBlob);
-              console.log('Audio converted to WAV successfully');
             } catch (conversionError) {
-              console.error('Failed to convert audio to WAV:', conversionError);
               throw new Error('Failed to convert audio to WAV format for Azure');
             }
           }
 
           const result = await transcribeAudio(audioBlob);
 
-          if (processSpeechAndSendMessage) {
-            processSpeechAndSendMessage(result.text);
-          }
+          // Only process if we have meaningful text
+          if (result.text && result.text.trim().length > 0) {
+            if (processSpeechAndSendMessage) {
+              processSpeechAndSendMessage(result.text);
+            }
 
-          setLastTranscription(result);
+            setLastTranscription(result);
 
-          if (options.onTranscriptionComplete) {
-            options.onTranscriptionComplete(result);
+            if (options.onTranscriptionComplete) {
+              options.onTranscriptionComplete(result);
+            }
+          } else {
+            console.log('No meaningful text transcribed, skipping message processing');
           }
 
           setRecordingState('idle');
@@ -312,30 +364,49 @@ export function useSTT(
 
   /**
    * Detect if there's audio activity (not silence)
+   * Only works when continuous recording is enabled
    */
   const detectAudioActivity = useCallback((): boolean => {
+    if (!options.continuousRecording) {
+      return false;
+    }
+    
     if (!analyserRef.current || !dataArrayRef.current) {
       return false;
     }
 
-    analyserRef.current.getByteFrequencyData(dataArrayRef.current);
-    
-    // Calculate average volume level
-    const average = dataArrayRef.current.reduce((sum, value) => sum + value, 0) / dataArrayRef.current.length;
-    
-    // Consider audio active if average volume is above threshold
-    const threshold = 10; // Adjust this value as needed
-    return average > threshold;
-  }, []);
+    try {
+      analyserRef.current.getByteFrequencyData(dataArrayRef.current);
+      
+      // Calculate simple average volume
+      let sum = 0;
+      for (let i = 0; i < dataArrayRef.current.length; i++) {
+        sum += dataArrayRef.current[i];
+      }
+      const averageVolume = sum / dataArrayRef.current.length;
+      
+      // Simple threshold - if volume is above 20, consider it activity
+      const hasActivity = averageVolume > 20;
+      
+      
+      return hasActivity;
+    } catch (error) {
+      return false;
+    }
+  }, [options.continuousRecording]);
 
   /**
    * Start silence detection monitoring
+   * Only works when continuous recording is enabled
    */
   const startSilenceDetection = useCallback(() => {
     if (!options.continuousRecording || !analyserRef.current) {
       return;
     }
     
+    let silenceCount = 0;
+    const maxSilenceCount = 30; // 3 seconds of silence (30 * 100ms)
+    
     const checkAudioActivity = () => {
       if (!isRecordingRef.current || !isMountedRef.current) {
         return;
@@ -344,36 +415,33 @@ export function useSTT(
       const hasActivity = detectAudioActivity();
       
       if (hasActivity) {
-        // Reset silence timeout when audio activity is detected
-        if (silenceTimeoutRef.current) {
-          clearTimeout(silenceTimeoutRef.current);
-        }
+        // Reset silence counter when activity is detected
+        silenceCount = 0;
+      } else {
+        // Increment silence counter
+        silenceCount++;
         
-        // Set new timeout for when user stops speaking
-        silenceTimeoutRef.current = setTimeout(() => {
-          if (isRecordingRef.current && isMountedRef.current) {
-            stopRecording();
+        // Stop recording after 3 seconds of silence
+        if (silenceCount >= maxSilenceCount) {
+          isRecordingRef.current = false;
+          if (mediaRecorderRef.current && mediaRecorderRef.current.state === 'recording') {
+            mediaRecorderRef.current.stop();
           }
-        }, silenceTimeout * 1000);
+          return;
+        }
       }
     };
 
-    // Check audio activity every 50ms for responsive detection
-    const intervalId = setInterval(checkAudioActivity, 50);
-    
-    // Store interval ID for cleanup
+    // Check audio activity every 100ms
+    const intervalId = setInterval(checkAudioActivity, 100);
     (window as any).memoriSilenceDetectionInterval = intervalId;
-  }, [options.continuousRecording, detectAudioActivity, silenceTimeout]);
+  }, [options.continuousRecording, detectAudioActivity]);
 
   /**
    * Stop silence detection monitoring
+   * Only works when continuous recording is enabled
    */
   const stopSilenceDetection = useCallback(() => {
-    if (silenceTimeoutRef.current) {
-      clearTimeout(silenceTimeoutRef.current);
-      silenceTimeoutRef.current = null;
-    }
-    
     if ((window as any).memoriSilenceDetectionInterval) {
       clearInterval((window as any).memoriSilenceDetectionInterval);
       (window as any).memoriSilenceDetectionInterval = null;
@@ -442,49 +510,81 @@ export function useSTT(
    * Start recording audio
    */
   const startRecording = useCallback(async (): Promise<void> => {
+    console.log('🎤 [START] Starting recording...');
+    console.log('🎤 [START] Mounted:', isMountedRef.current, 'Muted:', microphoneMuted, 'State:', recordingState);
+    
+    // Prevent immediate restart after stopping (cooldown period)
+    const timeSinceLastStop = Date.now() - lastStopTimeRef.current;
+    if (timeSinceLastStop < 1000) { // 1 second cooldown
+      console.log('🎤 [START] Too soon after last stop, waiting...', timeSinceLastStop + 'ms');
+      return;
+    }
+    
     if (
-      !isMountedRef.current ||
       microphoneMuted ||
       recordingState === 'recording'
     ) {
+      console.log('🎤 [START] Cannot start recording - conditions not met');
       return;
     }
 
     if (!hasUserActivatedRecord) {
+      console.log('🎤 [START] Setting user activated record flag');
       setHasUserActivatedRecord(true);
     }
 
     try {
       setError(null);
       setRecordingState('recording');
+      console.log('🎤 [START] Recording state set to recording');
 
       // Initialize recording if needed
       if (!mediaRecorderRef.current) {
+        console.log('🎤 [START] MediaRecorder not initialized, initializing...');
         const initialized = await initializeRecording();
         if (!initialized) {
+          console.error('🎤 [START] Failed to initialize recording');
           return;
         }
+        console.log('🎤 [START] Recording initialized successfully');
       }
 
       // Reset chunks and start recording
       chunksRef.current = [];
       isRecordingRef.current = true;
+      console.log('🎤 [START] Reset chunks and set recording flag');
 
       if (
         mediaRecorderRef.current &&
         mediaRecorderRef.current.state === 'inactive'
       ) {
-        mediaRecorderRef.current.start(100); // Collect data every 100ms
+        // Use different timeslice based on recording mode and browser
+        // For Safari, use longer timeslice to avoid issues with short recordings
+        // For other browsers, use shorter timeslice for real-time analysis
+        const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent);
+        const timeslice = isSafari ? 500 : 100; // 500ms for Safari, 100ms for others
+        
+        console.log(`🎤 [START] Starting MediaRecorder with ${timeslice}ms timeslice (Safari: ${isSafari})`);
+        console.log('🎤 [START] Continuous recording enabled:', options.continuousRecording);
+        
+        mediaRecorderRef.current.start(timeslice);
         setIsListening(true);
+        console.log('🎤 [START] MediaRecorder started, listening state set to true');
         
         // Start silence detection if continuous recording is enabled
         if (options.continuousRecording) {
+          console.log('🎤 [START] Starting silence detection for continuous recording');
           startSilenceDetection();
+        } else {
+          console.log('🎤 [START] Continuous recording disabled, skipping silence detection');
         }
+      } else {
+        console.log('🎤 [START] MediaRecorder not available or not inactive, state:', mediaRecorderRef.current?.state);
       }
     } catch (err) {
       const errorMsg =
         err instanceof Error ? err : new Error('Failed to start recording');
+      console.error('🎤 [START] Error starting recording:', errorMsg);
       setError(errorMsg);
       setRecordingState('error');
       isRecordingRef.current = false;
@@ -506,25 +606,44 @@ export function useSTT(
    * Stop recording audio
    */
   const stopRecording = useCallback((): void => {
+    console.log('🛑 [STOP] Stop recording called');
+    console.log('🛑 [STOP] isRecordingRef:', isRecordingRef.current, 'continuousRecording:', options.continuousRecording);
+    
     if (!isRecordingRef.current) {
+      console.log('🛑 [STOP] Not currently recording, ignoring stop request');
       return;
     }
 
     try {
+      console.log('🛑 [STOP] Setting listening to false');
       setIsListening(false);
       
-      // Stop silence detection
-      stopSilenceDetection();
+      // Record the stop time for cooldown
+      lastStopTimeRef.current = Date.now();
+      
+      // Stop silence detection only if continuous recording was enabled
+      if (options.continuousRecording) {
+        console.log('🛑 [STOP] Stopping silence detection for continuous recording');
+        stopSilenceDetection();
+      } else {
+        console.log('🛑 [STOP] Continuous recording disabled, skipping silence detection stop');
+      }
 
       if (
         mediaRecorderRef.current &&
         mediaRecorderRef.current.state === 'recording'
       ) {
+        console.log('🛑 [STOP] Stopping MediaRecorder');
         mediaRecorderRef.current.stop();
+      } else {
+        console.log('🛑 [STOP] MediaRecorder not available or not recording, state:', mediaRecorderRef.current?.state);
       }
+      
+      console.log('🛑 [STOP] Recording stop completed');
     } catch (err) {
       const errorMsg =
         err instanceof Error ? err : new Error('Failed to stop recording');
+      console.error('🛑 [STOP] Error stopping recording:', errorMsg);
       setError(errorMsg);
       setRecordingState('error');
       isRecordingRef.current = false;
@@ -539,10 +658,16 @@ export function useSTT(
    * Toggle recording state
    */
   const toggleRecording = useCallback(async (): Promise<void> => {
+    console.log('🔄 [TOGGLE] Toggle recording called, current state:', recordingState);
+    
     if (recordingState === 'recording') {
+      console.log('🔄 [TOGGLE] Currently recording, stopping...');
       stopRecording();
     } else if (recordingState === 'idle') {
+      console.log('🔄 [TOGGLE] Currently idle, starting recording...');
       await startRecording();
+    } else {
+      console.log('🔄 [TOGGLE] Cannot toggle from state:', recordingState);
     }
   }, [recordingState, startRecording, stopRecording]);
 
@@ -579,19 +704,32 @@ export function useSTT(
       audioStreamRef.current = null;
     }
 
-    // Clean up audio context
-    if (audioContextRef.current) {
-      audioContextRef.current.close();
+    // Clean up audio context only if continuous recording was enabled
+    if (options.continuousRecording && audioContextRef.current) {
+      try {
+        // Check if AudioContext is still valid before closing
+        if (audioContextRef.current.state !== 'closed') {
+          audioContextRef.current.close();
+        }
+      } catch (error) {
+        console.warn('Error closing AudioContext:', error);
+      }
       audioContextRef.current = null;
     }
 
-    // Stop silence detection
-    stopSilenceDetection();
+    // Stop silence detection only if continuous recording was enabled
+    if (options.continuousRecording) {
+      stopSilenceDetection();
+    }
+
+    // Reset audio analysis state
+    analyserRef.current = null;
+    dataArrayRef.current = null;
 
     chunksRef.current = [];
     setIsListening(false);
     setRecordingState('idle');
-  }, [stopSilenceDetection]);
+  }, [options.continuousRecording, stopSilenceDetection]);
 
   /**
    * Cleanup on unmount

package/src/helpers/translations.ts

@@ -18,13 +18,18 @@ const stripOutputTags = (text: string) => {
   return text.replaceAll(/<output.*?>(.*?)<\/output>/g, '');
 };
 
+const stripThinkTags = (text: string) => {
+  return text.replaceAll(/<think.*?>(.*?)<\/think>/gs, '');
+};
+
 export const getTranslation = async (
   text: string,
   to: string,
   from?: string,
   baseUrl?: string
 ): Promise<DeeplTranslation> => {
-  let textToTranslate = stripOutputTags(text);
+  let textToTranslate = stripOutputTags(stripThinkTags(text));
+  const justTheThinkTags = text.match(/<think.*?>(.*?)<\/think>/gs);
 
   const isReservedKeyword = dialogKeywords.indexOf(text.toLowerCase()) > -1;
   const fromLanguage = isReservedKeyword ? 'IT' : from?.toUpperCase();
@@ -47,5 +52,11 @@ export const getTranslation = async (
     }
   );
   const deeplResponse = await deeplResult.json();
+
+  //reapply the think tags to the translated text
+  if(deeplResponse && deeplResponse.translations && deeplResponse.translations[0]) {
+    deeplResponse.translations[0].text = (justTheThinkTags ? `<think>${justTheThinkTags}</think>` : '') + deeplResponse?.translations?.[0]?.text;
+  }
+
   return deeplResponse?.translations?.[0] ?? { text: textToTranslate };
 };

package/src/helpers/tts/useTTS.ts

@@ -199,6 +199,10 @@ export function useTTS(
     
     // Only reset speaking flag after cleanup
     isSpeakingRef.current = false;
+    
+    // Dispatch custom event to notify MemoriWidget that audio has ended
+    const e = new CustomEvent('MemoriAudioEnded');
+    document.dispatchEvent(e);
   }, [cleanup]);
 
   /**
@@ -223,7 +227,14 @@ export function useTTS(
       }
 
       // Early exit conditions before setting speaking flag
-      if (!text || options.preview || speakerMuted) {
+      if (!text || !text.trim() || options.preview) {
+        emitEndSpeakEvent();
+        return;
+      }
+
+      // If speaker is muted, completely disable TTS functionality
+      if (speakerMuted) {
+        console.log('[useTTS] TTS disabled - speaker is muted');
         emitEndSpeakEvent();
         return;
       }
@@ -320,6 +331,10 @@ export function useTTS(
             cleanup();
             isSpeakingRef.current = false;
             emitEndSpeakEvent();
+            
+            // Dispatch custom event to notify MemoriWidget that audio has ended
+            const event = new CustomEvent('MemoriAudioEnded');
+            document.dispatchEvent(event);
           }
         };
         
@@ -327,6 +342,10 @@ export function useTTS(
           setIsPlaying(false);
           isSpeakingRef.current = false;
           emitEndSpeakEvent();
+          
+          // Dispatch custom event to notify MemoriWidget that audio has ended
+          const e = new CustomEvent('MemoriAudioEnded');
+          document.dispatchEvent(e);
         };
 
         audioRef.current.onerror = (e) => {
@@ -338,6 +357,10 @@ export function useTTS(
           const errorMsg = new Error(`Audio playback failed. This may be due to a network issue or audio format problem.`);
           setError(errorMsg);
           emitEndSpeakEvent();
+          
+          // Dispatch custom event to notify MemoriWidget that audio has ended
+          const event = new CustomEvent('MemoriAudioEnded');
+          document.dispatchEvent(event);
         };
         
         audioRef.current.load();
@@ -357,6 +380,10 @@ export function useTTS(
         setError(errorMsg);
 
         emitEndSpeakEvent();
+        
+        // Dispatch custom event to notify MemoriWidget that audio has ended
+        const e = new CustomEvent('MemoriAudioEnded');
+        document.dispatchEvent(e);
       }
     },
     [

package/src/index.tsx

@@ -180,7 +180,7 @@ const Memori: React.FC<Props> = ({
       if (data.provider) {
         setProvider(data.provider);
       } else {
-          console.warn('Provider not found in speech key response');
+        console.warn('Provider not found in speech key response');
       }
     } catch (error) {
       console.error('Error fetching speech key:', error);
@@ -273,13 +273,31 @@ const Memori: React.FC<Props> = ({
   if (whiteListedDomains) {
     // check if we are client side
     if (typeof window !== 'undefined') {
-      // check if the current domain is in the whiteListedDomains with Regex
-      if (
-        !whiteListedDomains.some((domain: string) =>
-          new RegExp(domain).test(window.location.hostname)
-        )
-      ) {
-        return null;
+      const referrer = document.referrer;
+      const currentHostname = window.location.hostname;
+      const referrerHostname = referrer ? new URL(referrer).hostname : null;
+      
+      // Check if this is a preview context
+      const isInIframe = window.parent !== window;
+      const hasReferrerFromParent = referrer && referrer.length > 0;
+      const isSrcDocIframe = !currentHostname || currentHostname === '';
+      
+      // Preview detection: iframe with referrer from parent (especially srcDoc)
+      const isPreview = isInIframe && hasReferrerFromParent && (
+        referrerHostname === currentHostname ||  // Normal same-origin
+        isSrcDocIframe ||  // srcDoc iframe case
+        window.location.search.includes('_preview=')  // Preview token
+      );
+      // Skip whitelist check for preview
+      if (!isPreview) {
+        // check if the current domain is in the whiteListedDomains with Regex
+        if (
+          !whiteListedDomains.some((domain: string) =>
+            new RegExp(domain).test(window.location.hostname)
+          )
+        ) {
+          return null;
+        }
       }
     }
   }