@mastra/playground-ui 5.1.14 → 5.1.16-alpha.0

package/dist/index.cjs.js

@@ -4422,17 +4422,21 @@ const AssistantMessage = ({ ToolFallback: ToolFallbackCustom }) => {
   ] });
 };
 const AssistantActionBar$1 = () => {
-  return /* @__PURE__ */ jsxRuntime.jsx(
+  return /* @__PURE__ */ jsxRuntime.jsxs(
     react.ActionBarPrimitive.Root,
     {
       hideWhenRunning: true,
       autohide: "always",
       autohideFloat: "single-branch",
       className: "flex gap-1 items-center transition-all relative",
-      children: /* @__PURE__ */ jsxRuntime.jsx(react.ActionBarPrimitive.Copy, { asChild: true, children: /* @__PURE__ */ jsxRuntime.jsxs(TooltipIconButton, { tooltip: "Copy", className: "bg-transparent text-icon3 hover:text-icon6", children: [
-        /* @__PURE__ */ jsxRuntime.jsx(react.MessagePrimitive.If, { copied: true, children: /* @__PURE__ */ jsxRuntime.jsx(lucideReact.CheckIcon, {}) }),
-        /* @__PURE__ */ jsxRuntime.jsx(react.MessagePrimitive.If, { copied: false, children: /* @__PURE__ */ jsxRuntime.jsx(lucideReact.CopyIcon, {}) })
-      ] }) })
+      children: [
+        /* @__PURE__ */ jsxRuntime.jsx(react.MessagePrimitive.If, { speaking: false, children: /* @__PURE__ */ jsxRuntime.jsx(react.ActionBarPrimitive.Speak, { asChild: true, children: /* @__PURE__ */ jsxRuntime.jsx(TooltipIconButton, { tooltip: "Read aloud", children: /* @__PURE__ */ jsxRuntime.jsx(lucideReact.AudioLinesIcon, {}) }) }) }),
+        /* @__PURE__ */ jsxRuntime.jsx(react.MessagePrimitive.If, { speaking: true, children: /* @__PURE__ */ jsxRuntime.jsx(react.ActionBarPrimitive.StopSpeaking, { asChild: true, children: /* @__PURE__ */ jsxRuntime.jsx(TooltipIconButton, { tooltip: "Stop", children: /* @__PURE__ */ jsxRuntime.jsx(lucideReact.StopCircleIcon, {}) }) }) }),
+        /* @__PURE__ */ jsxRuntime.jsx(react.ActionBarPrimitive.Copy, { asChild: true, children: /* @__PURE__ */ jsxRuntime.jsxs(TooltipIconButton, { tooltip: "Copy", className: "bg-transparent text-icon3 hover:text-icon6", children: [
+          /* @__PURE__ */ jsxRuntime.jsx(react.MessagePrimitive.If, { copied: true, children: /* @__PURE__ */ jsxRuntime.jsx(lucideReact.CheckIcon, {}) }),
+          /* @__PURE__ */ jsxRuntime.jsx(react.MessagePrimitive.If, { copied: false, children: /* @__PURE__ */ jsxRuntime.jsx(lucideReact.CopyIcon, {}) })
+        ] }) })
+      ]
     }
   );
 };
@@ -4703,7 +4707,67 @@ const Txt = ({ as: Root = "p", className, variant = "ui-md", font, ...props }) =
   return /* @__PURE__ */ jsxRuntime.jsx(Root, { className: clsx(variants[variant], font && fonts[font], className), ...props });
 };
 
-const useSpeechRecognition = ({ language = "en-US" } = {}) => {
+async function recordMicrophoneToFile(onFinish) {
+  const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+  const mediaRecorder = new MediaRecorder(stream);
+  let chunks = [];
+  mediaRecorder.ondataavailable = (e) => {
+    chunks.push(e.data);
+  };
+  mediaRecorder.onstop = () => {
+    const blob = new Blob(chunks, { type: "audio/webm" });
+    const file = new File([blob], `recording-${Date.now()}.webm`, {
+      type: "audio/webm",
+      lastModified: Date.now()
+    });
+    stream.getTracks().forEach((track) => track.stop());
+    onFinish(file);
+  };
+  return mediaRecorder;
+}
+
+const useSpeechRecognition = ({
+  language = "en-US",
+  agentId
+}) => {
+  const client = useMastraClient();
+  const [agent, setAgent] = React.useState(null);
+  React.useEffect(() => {
+    if (!agentId) return;
+    const agent2 = client.getAgent(agentId);
+    const check = async () => {
+      try {
+        await agent2.voice.getSpeakers();
+        setAgent(agent2);
+      } catch (error) {
+        setAgent(null);
+      }
+    };
+    check();
+  }, [agentId]);
+  const {
+    start: startBrowser,
+    stop: stopBrowser,
+    isListening: isListeningBrowser,
+    transcript: transcriptBrowser
+  } = useBrowserSpeechRecognition({ language });
+  const {
+    start: startMastra,
+    stop: stopMastra,
+    isListening: isListeningMastra,
+    transcript: transcriptMastra
+  } = useMastraSpeechToText({ agent });
+  if (!agent) {
+    return {
+      start: startBrowser,
+      stop: stopBrowser,
+      isListening: isListeningBrowser,
+      transcript: transcriptBrowser
+    };
+  }
+  return { start: startMastra, stop: stopMastra, isListening: isListeningMastra, transcript: transcriptMastra };
+};
+const useBrowserSpeechRecognition = ({ language = "en-US" }) => {
   const speechRecognitionRef = React.useRef(null);
   const [state, setState] = React.useState({
     isListening: false,
@@ -4752,6 +4816,41 @@ const useSpeechRecognition = ({ language = "en-US" } = {}) => {
     stop
   };
 };
+const useMastraSpeechToText = ({ agent }) => {
+  const [transcript, setTranscript] = React.useState("");
+  const [recorder, setRecorder] = React.useState(null);
+  if (!agent) {
+    return {
+      start: () => {
+      },
+      stop: () => {
+      },
+      isListening: false,
+      transcript: ""
+    };
+  }
+  const handleFinish = (file) => {
+    agent.voice.listen(file).then((res) => {
+      setTranscript(res.text);
+    });
+  };
+  const start = () => {
+    recordMicrophoneToFile(handleFinish).then((recorder2) => {
+      setRecorder(recorder2);
+      recorder2.start();
+    });
+  };
+  const stop = () => {
+    recorder?.stop();
+    setRecorder(null);
+  };
+  return {
+    start,
+    stop,
+    isListening: Boolean(recorder),
+    transcript
+  };
+};
 
 const useHasAttachments = () => {
   const composer = react.useComposerRuntime();
@@ -4870,7 +4969,7 @@ const ComposerAttachments = () => {
   return /* @__PURE__ */ jsxRuntime.jsx("div", { className: "flex w-full flex-row items-center gap-4 pb-2", children: /* @__PURE__ */ jsxRuntime.jsx(react.ComposerPrimitive.Attachments, { components: { Attachment: AttachmentThumbnail } }) });
 };
 
-const Thread = ({ ToolFallback, agentName, hasMemory, onInputChange }) => {
+const Thread = ({ ToolFallback, agentName, agentId, hasMemory, onInputChange }) => {
   const areaRef = React.useRef(null);
   useAutoscroll(areaRef, { enabled: true });
   const WrappedAssistantMessage = (props) => {
@@ -4891,7 +4990,7 @@ const Thread = ({ ToolFallback, agentName, hasMemory, onInputChange }) => {
       ) }),
       /* @__PURE__ */ jsxRuntime.jsx(react.ThreadPrimitive.If, { empty: false, children: /* @__PURE__ */ jsxRuntime.jsx("div", {}) })
     ] }),
-    /* @__PURE__ */ jsxRuntime.jsx(Composer$1, { hasMemory, onInputChange })
+    /* @__PURE__ */ jsxRuntime.jsx(Composer$1, { hasMemory, onInputChange, agentId })
   ] });
 };
 const ThreadWrapper$1 = ({ children }) => {
@@ -4913,7 +5012,7 @@ const ThreadWelcome$1 = ({ agentName }) => {
     /* @__PURE__ */ jsxRuntime.jsx("p", { className: "mt-4 font-medium", children: "How can I help you today?" })
   ] }) });
 };
-const Composer$1 = ({ hasMemory, onInputChange }) => {
+const Composer$1 = ({ hasMemory, onInputChange, agentId }) => {
   return /* @__PURE__ */ jsxRuntime.jsxs("div", { className: "mx-4", children: [
     /* @__PURE__ */ jsxRuntime.jsxs(react.ComposerPrimitive.Root, { children: [
       /* @__PURE__ */ jsxRuntime.jsx("div", { className: "max-w-[568px] w-full mx-auto pb-2", children: /* @__PURE__ */ jsxRuntime.jsx(ComposerAttachments, {}) }),
@@ -4930,7 +5029,7 @@ const Composer$1 = ({ hasMemory, onInputChange }) => {
           }
         ) }),
         /* @__PURE__ */ jsxRuntime.jsxs("div", { className: "flex justify-end gap-2", children: [
-          /* @__PURE__ */ jsxRuntime.jsx(SpeechInput$1, {}),
+          /* @__PURE__ */ jsxRuntime.jsx(SpeechInput$1, { agentId }),
           /* @__PURE__ */ jsxRuntime.jsx(ComposerAction$1, {})
         ] })
       ] })
@@ -4941,9 +5040,9 @@ const Composer$1 = ({ hasMemory, onInputChange }) => {
     ] })
   ] });
 };
-const SpeechInput$1 = () => {
+const SpeechInput$1 = ({ agentId }) => {
   const composerRuntime = react.useComposerRuntime();
-  const { start, stop, isListening, transcript } = useSpeechRecognition();
+  const { start, stop, isListening, transcript } = useSpeechRecognition({ agentId });
   React.useEffect(() => {
     if (!transcript) return;
     composerRuntime.setText(transcript);
@@ -5090,6 +5189,74 @@ function useWorkingMemory() {
   return ctx;
 }
 
+class VoiceAttachmentAdapter {
+  constructor(agent) {
+    this.agent = agent;
+  }
+  speak(text) {
+    let _cleanup = () => {
+    };
+    const handleEnd = (reason, error) => {
+      if (res.status.type === "ended") return;
+      res.status = { type: "ended", reason, error };
+      _cleanup();
+    };
+    const res = {
+      status: { type: "running" },
+      cancel: () => {
+        handleEnd("cancelled");
+      },
+      subscribe: (callback) => {
+        this.agent.voice.speak(text).then((res2) => {
+          if (res2) {
+            return res2.body;
+          }
+        }).then((readableStream) => {
+          if (readableStream) {
+            return playStreamWithWebAudio(readableStream);
+          }
+        }).then((cleanup) => {
+          if (cleanup) {
+            _cleanup = cleanup;
+          }
+          callback();
+        }).catch((error) => {
+          handleEnd("error", error);
+        });
+        return () => {
+        };
+      }
+    };
+    return res;
+  }
+}
+async function playStreamWithWebAudio(stream) {
+  const audioContext = new window.AudioContext();
+  const reader = stream.getReader();
+  const chunks = [];
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    chunks.push(value);
+  }
+  const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
+  const combinedBuffer = new Uint8Array(totalLength);
+  let offset = 0;
+  for (const chunk of chunks) {
+    combinedBuffer.set(chunk, offset);
+    offset += chunk.length;
+  }
+  const audioBuffer = await audioContext.decodeAudioData(combinedBuffer.buffer);
+  const source = audioContext.createBufferSource();
+  source.buffer = audioBuffer;
+  source.connect(audioContext.destination);
+  source.start();
+  return () => {
+    source.stop();
+    audioContext.close();
+  };
+}
+
 class PDFAttachmentAdapter {
   accept = "application/pdf";
   async add({ file }) {
@@ -5143,6 +5310,37 @@ class PDFAttachmentAdapter {
   }
 }
 
+const useAdapters = (agentId) => {
+  const [isReady, setIsReady] = React.useState(false);
+  const [speechAdapter, setSpeechAdapter] = React.useState(void 0);
+  const baseClient = useMastraClient();
+  React.useEffect(() => {
+    const check = async () => {
+      const agent = baseClient.getAgent(agentId);
+      try {
+        await agent.voice.getSpeakers();
+        setSpeechAdapter(new VoiceAttachmentAdapter(agent));
+        setIsReady(true);
+      } catch {
+        setSpeechAdapter(new react.WebSpeechSynthesisAdapter());
+        setIsReady(true);
+      }
+    };
+    check();
+  }, [agentId]);
+  return {
+    isReady,
+    adapters: {
+      attachments: new react.CompositeAttachmentAdapter([
+        new react.SimpleImageAttachmentAdapter(),
+        new react.SimpleTextAttachmentAdapter(),
+        new PDFAttachmentAdapter()
+      ]),
+      speech: speechAdapter
+    }
+  };
+};
+
 const convertMessage$2 = (message) => {
   return message;
 };
@@ -5567,20 +5765,16 @@ function MastraRuntimeProvider({
       setIsRunning(false);
     }
   };
+  const { adapters, isReady } = useAdapters(agentId);
   const runtime = react.useExternalStoreRuntime({
     isRunning,
     messages,
     convertMessage: convertMessage$2,
     onNew,
     onCancel,
-    adapters: {
-      attachments: new react.CompositeAttachmentAdapter([
-        new react.SimpleImageAttachmentAdapter(),
-        new react.SimpleTextAttachmentAdapter(),
-        new PDFAttachmentAdapter()
-      ])
-    }
+    adapters: isReady ? adapters : void 0
   });
+  if (!isReady) return null;
   return /* @__PURE__ */ jsxRuntime.jsxs(react.AssistantRuntimeProvider, { runtime, children: [
     " ",
     children,
@@ -5681,7 +5875,7 @@ const AgentChat = ({
       refreshThreadList,
       settings,
       runtimeContext,
-      children: /* @__PURE__ */ jsxRuntime.jsx(Thread, { agentName: agentName ?? "", hasMemory: memory, onInputChange })
+      children: /* @__PURE__ */ jsxRuntime.jsx(Thread, { agentName: agentName ?? "", hasMemory: memory, onInputChange, agentId })
     }
   );
 };
@@ -10487,7 +10681,7 @@ const Composer = ({ hasMemory }) => {
 };
 const SpeechInput = () => {
   const composerRuntime = react.useComposerRuntime();
-  const { start, stop, isListening, transcript } = useSpeechRecognition();
+  const { start, stop, isListening, transcript } = useSpeechRecognition({});
   React.useEffect(() => {
     if (!transcript) return;
     composerRuntime.setText(transcript);