@usecrow/ui 0.1.57 → 0.1.58

package/dist/index.cjs

@@ -1769,6 +1769,186 @@ function usePreviewCopilotStyles(previewStyles) {
     styles: mergeCopilotStyles(void 0, previewStyles)
   };
 }
+function useTTSOutput({
+  backendUrl,
+  voiceId = "YTpq7expH9539ERJ"
+}) {
+  const [isSpeaking, setIsSpeaking] = React3.useState(false);
+  const [error, setError] = React3.useState(null);
+  const wsRef = React3.useRef(null);
+  const audioContextRef = React3.useRef(null);
+  const nextTimeRef = React3.useRef(0);
+  const streamCompleteRef = React3.useRef(false);
+  const completionCheckIntervalRef = React3.useRef(null);
+  const cleanupAudioContext = React3.useCallback(() => {
+    setIsSpeaking(false);
+    if (audioContextRef.current && audioContextRef.current.state !== "closed") {
+      audioContextRef.current.close();
+      audioContextRef.current = null;
+    }
+    if (completionCheckIntervalRef.current) {
+      clearInterval(completionCheckIntervalRef.current);
+      completionCheckIntervalRef.current = null;
+    }
+  }, []);
+  const closeWebSocket = React3.useCallback(() => {
+    if (wsRef.current && wsRef.current.readyState === WebSocket.OPEN) {
+      try {
+        wsRef.current.send(JSON.stringify({ type: "stop" }));
+        wsRef.current.close();
+      } catch (e) {
+      }
+    }
+    wsRef.current = null;
+  }, []);
+  const cleanupTTS = React3.useCallback(() => {
+    setIsSpeaking(false);
+    setError(null);
+    closeWebSocket();
+    cleanupAudioContext();
+  }, [closeWebSocket, cleanupAudioContext]);
+  const waitForAudioComplete = React3.useCallback(() => {
+    if (completionCheckIntervalRef.current) {
+      clearInterval(completionCheckIntervalRef.current);
+    }
+    completionCheckIntervalRef.current = setInterval(() => {
+      if (!audioContextRef.current) {
+        if (completionCheckIntervalRef.current) {
+          clearInterval(completionCheckIntervalRef.current);
+          completionCheckIntervalRef.current = null;
+        }
+        return;
+      }
+      const now = audioContextRef.current.currentTime;
+      if (now >= nextTimeRef.current) {
+        if (completionCheckIntervalRef.current) {
+          clearInterval(completionCheckIntervalRef.current);
+          completionCheckIntervalRef.current = null;
+        }
+        cleanupAudioContext();
+      }
+    }, 100);
+  }, [cleanupAudioContext]);
+  const playAudioChunk = React3.useCallback((base64Audio) => {
+    if (!audioContextRef.current || audioContextRef.current.state === "closed") {
+      console.error("TTS: AudioContext not available");
+      return;
+    }
+    try {
+      const binary = atob(base64Audio);
+      const bytes = new Uint8Array(binary.length);
+      for (let i = 0; i < binary.length; i++) {
+        bytes[i] = binary.charCodeAt(i);
+      }
+      const pcm16 = new Int16Array(bytes.buffer);
+      const float32 = new Float32Array(pcm16.length);
+      for (let i = 0; i < pcm16.length; i++) {
+        float32[i] = pcm16[i] / 32768;
+      }
+      const buffer = audioContextRef.current.createBuffer(1, float32.length, 48e3);
+      buffer.getChannelData(0).set(float32);
+      const source = audioContextRef.current.createBufferSource();
+      source.buffer = buffer;
+      source.connect(audioContextRef.current.destination);
+      const now = audioContextRef.current.currentTime;
+      if (nextTimeRef.current < now) {
+        nextTimeRef.current = now;
+      }
+      source.start(nextTimeRef.current);
+      nextTimeRef.current += buffer.duration;
+    } catch (err) {
+      console.error("TTS: Error playing audio chunk:", err);
+      setError(err instanceof Error ? err.message : "Failed to play audio chunk");
+    }
+  }, []);
+  const speak = React3.useCallback(
+    (text) => {
+      console.log("[TTS Hook] speak called with:", text.substring(0, 50), "backendUrl:", backendUrl);
+      if (!text.trim()) {
+        console.log("[TTS Hook] No text to speak");
+        setError("No text to speak");
+        return;
+      }
+      if (isSpeaking || wsRef.current) {
+        console.log("[TTS Hook] Already playing");
+        setError("Already playing, stop first");
+        return;
+      }
+      setError(null);
+      nextTimeRef.current = 0;
+      streamCompleteRef.current = false;
+      try {
+        audioContextRef.current = new (window.AudioContext || window.webkitAudioContext)({
+          sampleRate: 48e3
+        });
+        const url = backendUrl.startsWith("http") ? backendUrl.replace(/^http/, "ws") : backendUrl;
+        const wsUrl = `${url}/api/tts/stream`;
+        console.log("[TTS Hook] Connecting to:", wsUrl);
+        const ws = new WebSocket(wsUrl);
+        wsRef.current = ws;
+        ws.onopen = () => {
+          ws.send(
+            JSON.stringify({
+              type: "setup",
+              voice_id: voiceId,
+              output_format: "pcm"
+            })
+          );
+        };
+        ws.onmessage = (event) => {
+          const msg = JSON.parse(event.data);
+          if (msg.type === "ready") {
+            ws.send(JSON.stringify({ type: "text", text }));
+            ws.send(JSON.stringify({ type: "end_of_stream" }));
+          } else if (msg.type === "audio") {
+            playAudioChunk(msg.audio);
+          } else if (msg.type === "done") {
+            streamCompleteRef.current = true;
+            closeWebSocket();
+            waitForAudioComplete();
+          } else if (msg.type === "error") {
+            setError(msg.message || "TTS error");
+            cleanupTTS();
+          }
+        };
+        ws.onerror = () => {
+          setError("WebSocket error");
+          cleanupTTS();
+        };
+        ws.onclose = () => {
+          wsRef.current = null;
+        };
+        setIsSpeaking(true);
+      } catch (err) {
+        setError(err instanceof Error ? err.message : "Failed to start TTS");
+        cleanupTTS();
+      }
+    },
+    [
+      isSpeaking,
+      backendUrl,
+      voiceId,
+      playAudioChunk,
+      closeWebSocket,
+      waitForAudioComplete,
+      cleanupTTS
+    ]
+  );
+  const stop = React3.useCallback(() => {
+    cleanupTTS();
+  }, [cleanupTTS]);
+  React3.useEffect(() => {
+    return () => {
+      cleanupTTS();
+    };
+  }, [cleanupTTS]);
+  return {
+    speak,
+    stop,
+    isSpeaking,
+    error
+  };
+}
 var WidgetStyleContext = React3.createContext(null);
 function WidgetStyleProvider({
   children,
@@ -2756,80 +2936,176 @@ var ModelSelector = ({
     ] }, provider)) })
   ] });
 };
-var getSpeechRecognition = () => {
-  if (typeof window === "undefined") return null;
-  return window.SpeechRecognition || window.webkitSpeechRecognition || null;
+var isMediaRecorderSupported = () => {
+  if (typeof window === "undefined") return false;
+  return !!(navigator.mediaDevices && typeof navigator.mediaDevices.getUserMedia === "function" && (window.AudioContext || window.webkitAudioContext));
 };
-function useVoiceInput(options = {}) {
-  const { lang, silenceTimeoutMs } = options;
-  const [supported] = React3.useState(() => getSpeechRecognition() !== null);
+function useVoiceInput(options) {
+  const { backendUrl, silenceTimeoutMs } = options;
+  const [supported] = React3.useState(() => isMediaRecorderSupported());
   const [isRecording, setIsRecording] = React3.useState(false);
   const [transcript, setTranscript] = React3.useState("");
-  const recognitionRef = React3.useRef(null);
+  const [error, setError] = React3.useState(null);
+  const wsRef = React3.useRef(null);
+  const streamRef = React3.useRef(null);
+  const audioContextRef = React3.useRef(null);
+  const processorRef = React3.useRef(null);
   const silenceTimerRef = React3.useRef(null);
-  const finalTranscriptRef = React3.useRef("");
+  const transcriptRef = React3.useRef("");
+  const interimRef = React3.useRef("");
+  const isRecordingRef = React3.useRef(false);
   const clearSilenceTimer = React3.useCallback(() => {
     if (silenceTimerRef.current) {
       clearTimeout(silenceTimerRef.current);
       silenceTimerRef.current = null;
     }
   }, []);
-  const stop = React3.useCallback(() => {
+  const cleanup = React3.useCallback(() => {
     clearSilenceTimer();
-    if (recognitionRef.current) {
-      recognitionRef.current.stop();
+    isRecordingRef.current = false;
+    if (interimRef.current) {
+      transcriptRef.current += interimRef.current + " ";
+      setTranscript(transcriptRef.current.trim());
+      interimRef.current = "";
     }
+    if (wsRef.current) {
+      try {
+        if (wsRef.current.readyState === WebSocket.OPEN) {
+          wsRef.current.send(JSON.stringify({ type: "stop" }));
+        }
+        wsRef.current.close();
+      } catch (e) {
+      }
+      wsRef.current = null;
+    }
+    if (processorRef.current) {
+      processorRef.current.disconnect();
+      processorRef.current = null;
+    }
+    if (audioContextRef.current) {
+      audioContextRef.current.close();
+      audioContextRef.current = null;
+    }
+    if (streamRef.current) {
+      streamRef.current.getTracks().forEach((track) => track.stop());
+      streamRef.current = null;
+    }
+    setIsRecording(false);
   }, [clearSilenceTimer]);
+  const stop = React3.useCallback(() => {
+    cleanup();
+  }, [cleanup]);
   const clear = React3.useCallback(() => {
     setTranscript("");
-    finalTranscriptRef.current = "";
+    transcriptRef.current = "";
+    setError(null);
   }, []);
-  const start = React3.useCallback(() => {
-    const SpeechRecognition = getSpeechRecognition();
-    if (!SpeechRecognition) return;
-    if (recognitionRef.current) {
-      recognitionRef.current.abort();
-    }
-    finalTranscriptRef.current = "";
-    setTranscript("");
-    const recognition = new SpeechRecognition();
-    recognition.continuous = true;
-    recognition.interimResults = true;
-    recognition.lang = lang || navigator.language || "en-US";
-    recognition.onresult = (event) => {
-      let interim = "";
-      let final = "";
-      for (let i = 0; i < event.results.length; i++) {
-        const result = event.results[i];
-        if (result.isFinal) {
-          final += result[0].transcript;
-        } else {
-          interim += result[0].transcript;
-        }
+  const startAudioCapture = React3.useCallback(() => {
+    if (!streamRef.current || !wsRef.current) return;
+    audioContextRef.current = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: 24e3 });
+    const source = audioContextRef.current.createMediaStreamSource(
+      streamRef.current
+    );
+    processorRef.current = audioContextRef.current.createScriptProcessor(
+      4096,
+      1,
+      1
+    );
+    processorRef.current.onaudioprocess = (event) => {
+      if (!isRecordingRef.current || !wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) {
+        return;
       }
-      finalTranscriptRef.current = final;
-      setTranscript(final + interim);
-      if (silenceTimeoutMs) {
-        clearSilenceTimer();
-        silenceTimerRef.current = setTimeout(() => {
-          stop();
-        }, silenceTimeoutMs);
+      const inputData = event.inputBuffer.getChannelData(0);
+      const pcm16 = new Int16Array(inputData.length);
+      for (let i = 0; i < inputData.length; i++) {
+        const s = Math.max(-1, Math.min(1, inputData[i]));
+        pcm16[i] = s < 0 ? s * 32768 : s * 32767;
       }
-    };
-    recognition.onerror = (event) => {
-      if (event.error !== "aborted") {
-        console.warn("[Crow Voice] Speech recognition error:", event.error);
+      const bytes = new Uint8Array(pcm16.buffer);
+      let binary = "";
+      for (let i = 0; i < bytes.length; i++) {
+        binary += String.fromCharCode(bytes[i]);
       }
-      setIsRecording(false);
-    };
-    recognition.onend = () => {
-      setIsRecording(false);
-      recognitionRef.current = null;
+      wsRef.current.send(
+        JSON.stringify({ type: "audio", data: btoa(binary) })
+      );
     };
-    recognitionRef.current = recognition;
-    recognition.start();
-    setIsRecording(true);
-  }, [lang, silenceTimeoutMs, clearSilenceTimer, stop]);
+    source.connect(processorRef.current);
+    processorRef.current.connect(audioContextRef.current.destination);
+  }, []);
+  const start = React3.useCallback(async () => {
+    if (!supported) {
+      setError("Audio recording not supported in this browser");
+      return;
+    }
+    setError(null);
+    transcriptRef.current = "";
+    setTranscript("");
+    try {
+      streamRef.current = await navigator.mediaDevices.getUserMedia({
+        audio: {
+          echoCancellation: true,
+          noiseSuppression: true,
+          sampleRate: 24e3
+        }
+      });
+      const wsProtocol = backendUrl.startsWith("https") ? "wss" : "ws";
+      const wsHost = backendUrl.replace(/^https?:\/\//, "");
+      const wsUrl = `${wsProtocol}://${wsHost}/api/stt/stream`;
+      wsRef.current = new WebSocket(wsUrl);
+      wsRef.current.onopen = () => {
+        wsRef.current?.send(JSON.stringify({ type: "setup" }));
+      };
+      wsRef.current.onmessage = (event) => {
+        const msg = JSON.parse(event.data);
+        if (msg.type === "ready") {
+          startAudioCapture();
+          isRecordingRef.current = true;
+          setIsRecording(true);
+        } else if (msg.type === "transcript") {
+          if (msg.is_final && msg.text) {
+            transcriptRef.current += msg.text + " ";
+            interimRef.current = "";
+            setTranscript(transcriptRef.current.trim());
+            if (silenceTimeoutMs) {
+              clearSilenceTimer();
+              silenceTimerRef.current = setTimeout(() => {
+                stop();
+              }, silenceTimeoutMs);
+            }
+          } else if (!msg.is_final && msg.text) {
+            interimRef.current = msg.text;
+            setTranscript((transcriptRef.current + msg.text).trim());
+          }
+        } else if (msg.type === "error") {
+          setError(msg.message || "STT error");
+          cleanup();
+        }
+      };
+      wsRef.current.onerror = () => {
+        setError("WebSocket connection error");
+        cleanup();
+      };
+      wsRef.current.onclose = () => {
+        if (isRecordingRef.current) {
+          cleanup();
+        }
+      };
+    } catch (err) {
+      setError(
+        err instanceof Error ? err.message : "Failed to start recording"
+      );
+      cleanup();
+    }
+  }, [
+    supported,
+    backendUrl,
+    startAudioCapture,
+    silenceTimeoutMs,
+    clearSilenceTimer,
+    stop,
+    cleanup
+  ]);
   const toggle = React3.useCallback(() => {
     if (isRecording) {
       stop();
@@ -2839,13 +3115,19 @@ function useVoiceInput(options = {}) {
   }, [isRecording, start, stop]);
   React3.useEffect(() => {
     return () => {
-      clearSilenceTimer();
-      if (recognitionRef.current) {
-        recognitionRef.current.abort();
-      }
+      cleanup();
     };
-  }, [clearSilenceTimer]);
-  return { supported, isRecording, transcript, start, stop, toggle, clear };
+  }, [cleanup]);
+  return {
+    supported,
+    isRecording,
+    transcript,
+    error,
+    start,
+    stop,
+    toggle,
+    clear
+  };
 }
 var Textarea = React3__default.default.forwardRef(
   ({ className, ...props }, ref) => /* @__PURE__ */ jsxRuntime.jsx(
@@ -3039,11 +3321,23 @@ var PromptInputBox = React3__default.default.forwardRef(
     selectedModel = "gpt-4o",
     onModelChange,
     availableModels = [],
-    highlighted = false
+    highlighted = false,
+    backendUrl = "",
+    triggerVoiceRecording = 0
   }, ref) => {
     const [input, setInput] = React3__default.default.useState("");
     const promptBoxRef = React3__default.default.useRef(null);
-    const voice = useVoiceInput();
+    const voice = useVoiceInput({ backendUrl, silenceTimeoutMs: 1500 });
+    const lastTriggerRef = React3__default.default.useRef(0);
+    const voiceRef = React3__default.default.useRef(voice);
+    voiceRef.current = voice;
+    React3__default.default.useEffect(() => {
+      if (triggerVoiceRecording > 0 && triggerVoiceRecording !== lastTriggerRef.current) {
+        console.log("[Voice] Auto-starting recording from trigger");
+        voiceRef.current.start();
+      }
+      lastTriggerRef.current = triggerVoiceRecording;
+    }, [triggerVoiceRecording]);
     React3__default.default.useEffect(() => {
       if (voice.isRecording && voice.transcript) {
         setInput(voice.transcript);
@@ -3052,11 +3346,16 @@ var PromptInputBox = React3__default.default.forwardRef(
     const wasRecordingRef = React3__default.default.useRef(false);
     React3__default.default.useEffect(() => {
       if (wasRecordingRef.current && !voice.isRecording && voice.transcript) {
-        setInput(voice.transcript);
+        const messageToSend = voice.transcript.trim();
+        if (messageToSend) {
+          console.log("[Voice] Auto-sending:", messageToSend);
+          onSend(messageToSend);
+          setInput("");
+        }
         voice.clear();
       }
       wasRecordingRef.current = voice.isRecording;
-    }, [voice.isRecording, voice.transcript, voice.clear]);
+    }, [voice.isRecording, voice.transcript, voice.clear, onSend]);
     const handleSubmit = () => {
       if (input.trim()) {
         if (voice.isRecording) {
@@ -3667,6 +3966,25 @@ function CrowWidget({
       setShouldRestoreHistory(true);
     }
   });
+  const tts = useTTSOutput({ backendUrl: apiUrl });
+  const ttsRef = React3.useRef(tts);
+  ttsRef.current = tts;
+  const wasLoadingRef = React3.useRef(false);
+  React3.useEffect(() => {
+    console.log("[Crow TTS] isLoading changed:", chat.isLoading, "wasLoading:", wasLoadingRef.current);
+    if (wasLoadingRef.current && !chat.isLoading) {
+      const lastMessage = [...chat.messages].reverse().find((m) => m.isBot);
+      console.log("[Crow TTS] Last bot message:", lastMessage?.content?.substring(0, 50));
+      if (lastMessage?.content) {
+        const textToSpeak = lastMessage.content.replace(/\*\*/g, "").replace(/\*/g, "").replace(/`[^`]+`/g, "").replace(/\[([^\]]+)\]\([^)]+\)/g, "$1").trim();
+        if (textToSpeak) {
+          console.log("[Crow TTS] Speaking:", textToSpeak.substring(0, 50));
+          ttsRef.current.speak(textToSpeak);
+        }
+      }
+    }
+    wasLoadingRef.current = chat.isLoading;
+  }, [chat.isLoading, chat.messages]);
   React3.useEffect(() => {
     if (initialSuggestions.length > 0 && chat.suggestedActions.length === 0) {
       chat.setSuggestedActions(initialSuggestions);
@@ -4061,7 +4379,8 @@ function CrowWidget({
           isLoading: chat.isLoading,
           showStopButton: isBrowserUseActive || !!askUserResolver || !!pendingConfirmation,
           highlighted: !!askUserResolver,
-          className: "crow-backdrop-blur-md"
+          className: "crow-backdrop-blur-md",
+          backendUrl: apiUrl
         }
       )
     ] })