@smart-cloud/ai-kit-ui 1.2.7 → 1.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@smart-cloud/ai-kit-ui",
-  "version": "1.2.7",
+  "version": "1.3.0",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",
@@ -20,7 +20,7 @@
     "@emotion/cache": "^11.14.0",
     "@emotion/react": "^11.14.0",
     "@mantine/colors-generator": "^8.3.15",
-    "@smart-cloud/ai-kit-core": "^1.2.7",
+    "@smart-cloud/ai-kit-core": "^1.3.0",
     "@smart-cloud/wpsuite-core": "^2.2.6",
     "@tabler/icons-react": "^3.36.1",
     "chroma-js": "^3.2.0",

package/src/doc-search/DocSearch.tsx

@@ -26,6 +26,7 @@ import rehypeRaw from "rehype-raw";
 import remarkGfm from "remark-gfm";
 
 import { IconSearch } from "@tabler/icons-react";
+import { IconMicrophone, IconMicrophoneOff } from "@tabler/icons-react";
 
 import { AiFeatureBorder } from "../ai-feature/AiFeatureBorder";
 import { translations } from "../i18n";
@@ -81,6 +82,11 @@ const DocSearchBase: FC<Props> = (props) => {
   } = props;
 
   const [query, setQuery] = useState<string>("");
+  const [recording, setRecording] = useState<boolean>(false);
+  const [audioBlob, setAudioBlob] = useState<Blob | null>(null);
+  const mediaRecorderRef = useRef<MediaRecorder | null>(null);
+  const audioChunksRef = useRef<Blob[]>([]);
+  
   const { busy, error, statusEvent, result, run, cancel, reset } =
     useAiRun<SearchResult>();
 
@@ -127,25 +133,95 @@ const DocSearchBase: FC<Props> = (props) => {
 
   const canSearch = useMemo(() => {
     if (busy) return false;
+    // Can search if we have text OR audio
     const text = typeof inputText === "function" ? inputText() : inputText;
-    return Boolean(text && text.trim().length > 0);
-  }, [inputText, busy]);
+    return Boolean((text && text.trim().length > 0) || audioBlob);
+  }, [inputText, busy, audioBlob]);
+
+  const startRecording = useCallback(async () => {
+    try {
+      const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+      const mediaRecorder = new MediaRecorder(stream, {
+        mimeType: "audio/webm",
+      });
+      
+      audioChunksRef.current = [];
+      
+      mediaRecorder.ondataavailable = (event) => {
+        if (event.data.size > 0) {
+          audioChunksRef.current.push(event.data);
+        }
+      };
+      
+      mediaRecorder.onstop = () => {
+        const audioBlob = new Blob(audioChunksRef.current, { type: "audio/webm" });
+        setAudioBlob(audioBlob);
+        stream.getTracks().forEach(track => track.stop());
+      };
+      
+      mediaRecorderRef.current = mediaRecorder;
+      mediaRecorder.start();
+      setRecording(true);
+    } catch (error) {
+      console.error("Failed to start recording:", error);
+    }
+  }, []);
+
+  const stopRecording = useCallback(() => {
+    if (mediaRecorderRef.current && recording) {
+      mediaRecorderRef.current.stop();
+      setRecording(false);
+    }
+  }, [recording]);
+
+  const clearAudio = useCallback(() => {
+    setAudioBlob(null);
+    audioChunksRef.current = [];
+  }, []);
 
   const onSearch = useCallback(async () => {
-    const q =
-      typeof inputText === "function"
+    let q: string | undefined;
+    let audio: { format: string; data: string } | undefined;
+
+    // Get text query if available
+    if (!audioBlob) {
+      q = typeof inputText === "function"
         ? (inputText as () => string)()
         : inputText;
-    if (!q) return;
-    setQuery(q);
+      if (!q) return;
+      setQuery(q);
+    }
+
+    // Convert audio blob to base64 if available
+    if (audioBlob) {
+      const reader = new FileReader();
+      await new Promise<void>((resolve, reject) => {
+        reader.onload = () => {
+          const base64 = (reader.result as string).split(",")[1];
+          audio = {
+            format: "audio/webm",
+            data: base64,
+          };
+          resolve();
+        };
+        reader.onerror = reject;
+        reader.readAsDataURL(audioBlob);
+      });
+    }
+
     reset();
     await run(async ({ signal, onStatus }) => {
       return await sendSearchMessage(
-        { sessionId, query: q, topK },
+        { 
+          sessionId, 
+          ...(q && { query: q }),
+          audio, 
+          topK 
+        },
         { signal, onStatus, context },
       );
     });
-  }, [context, inputText, run, reset, topK, sessionId]);
+  }, [context, inputText, audioBlob, run, reset, topK, sessionId]);
 
   const close = useCallback(async () => {
     reset();
@@ -313,9 +389,19 @@ const DocSearchBase: FC<Props> = (props) => {
                   <TextInput
                     style={{ flex: 1 }}
                     value={query}
-                    onChange={(e) => setQuery(e.currentTarget.value)}
-                    placeholder={I18n.get("Search the documentation…")}
-                    disabled={busy}
+                    onChange={(e) => {
+                      setQuery(e.currentTarget.value);
+                      // Clear audio when typing
+                      if (audioBlob) {
+                        clearAudio();
+                      }
+                    }}
+                    placeholder={
+                      audioBlob
+                        ? I18n.get("Audio recorded")
+                        : I18n.get("Search the documentation…")
+                    }
+                    disabled={busy || recording || !!audioBlob}
                     onKeyDown={(e) => {
                       if (e.key === "Enter" && canSearch) {
                         e.preventDefault();
@@ -324,6 +410,35 @@ const DocSearchBase: FC<Props> = (props) => {
                     }}
                   />
 
+                  {/* Microphone button */}
+                  {audioBlob ? (
+                    <Button
+                      variant="outline"
+                      size="sm"
+                      color="red"
+                      onClick={clearAudio}
+                      disabled={busy}
+                      title={I18n.get("Clear audio")}
+                    >
+                      <IconMicrophoneOff size={18} />
+                    </Button>
+                  ) : (
+                    <Button
+                      variant={recording ? "filled" : "outline"}
+                      size="sm"
+                      color={recording ? "red" : "gray"}
+                      onClick={recording ? stopRecording : startRecording}
+                      disabled={busy}
+                      title={
+                        recording
+                          ? I18n.get("Stop recording")
+                          : I18n.get("Record audio")
+                      }
+                    >
+                      <IconMicrophone size={18} />
+                    </Button>
+                  )}
+
                   <Button
                     variant="filled"
                     size="sm"