@mastra/playground-ui 5.1.14 → 5.1.16-alpha.0

package/dist/index.es.js

@@ -2,8 +2,8 @@ import { jsx, jsxs, Fragment } from 'react/jsx-runtime';
 import * as React from 'react';
 import React__default, { createContext, useContext, forwardRef, memo, useState, useEffect, useRef, useCallback, useMemo, Suspense, Fragment as Fragment$1, startTransition } from 'react';
 import { MastraClient } from '@mastra/client-js';
-import { useMessage, MessagePrimitive, ActionBarPrimitive, useAttachment, AttachmentPrimitive, useComposerRuntime, ComposerPrimitive, ThreadPrimitive, useExternalStoreRuntime, CompositeAttachmentAdapter, SimpleImageAttachmentAdapter, SimpleTextAttachmentAdapter, AssistantRuntimeProvider } from '@assistant-ui/react';
-import { CheckIcon as CheckIcon$1, CopyIcon, Check, Copy, ChevronUpIcon, BrainIcon, X, FileText, CircleXIcon, Mic, PlusIcon, ArrowUp, Search, RefreshCcwIcon, ChevronRight, SortAsc, SortDesc, Circle, ChevronDown, Braces, SaveIcon, RefreshCw, ExternalLink, InfoIcon as InfoIcon$1, GaugeIcon, Plus, LoaderCircle, ChevronDownIcon, ExternalLinkIcon, Loader2, Network, PauseIcon, HourglassIcon, CircleDashed, Footprints, CircleCheck, CircleX, Minus, Maximize, Workflow, AlertCircleIcon, Users, Brain, NetworkIcon, SearchIcon, AlertCircle, CalendarIcon, Brackets, TrashIcon, CirclePause, StopCircle, ChevronUp } from 'lucide-react';
+import { useMessage, MessagePrimitive, ActionBarPrimitive, useAttachment, AttachmentPrimitive, useComposerRuntime, ComposerPrimitive, ThreadPrimitive, CompositeAttachmentAdapter, SimpleImageAttachmentAdapter, SimpleTextAttachmentAdapter, WebSpeechSynthesisAdapter, useExternalStoreRuntime, AssistantRuntimeProvider } from '@assistant-ui/react';
+import { CheckIcon as CheckIcon$1, CopyIcon, Check, Copy, ChevronUpIcon, BrainIcon, AudioLinesIcon, StopCircleIcon, X, FileText, CircleXIcon, Mic, PlusIcon, ArrowUp, Search, RefreshCcwIcon, ChevronRight, SortAsc, SortDesc, Circle, ChevronDown, Braces, SaveIcon, RefreshCw, ExternalLink, InfoIcon as InfoIcon$1, GaugeIcon, Plus, LoaderCircle, ChevronDownIcon, ExternalLinkIcon, Loader2, Network, PauseIcon, HourglassIcon, CircleDashed, Footprints, CircleCheck, CircleX, Minus, Maximize, Workflow, AlertCircleIcon, Users, Brain, NetworkIcon, SearchIcon, AlertCircle, CalendarIcon, Brackets, TrashIcon, CirclePause, StopCircle, ChevronUp } from 'lucide-react';
 import { Slot } from '@radix-ui/react-slot';
 import * as TooltipPrimitive from '@radix-ui/react-tooltip';
 import { TooltipProvider as TooltipProvider$1 } from '@radix-ui/react-tooltip';
@@ -4389,17 +4389,21 @@ const AssistantMessage = ({ ToolFallback: ToolFallbackCustom }) => {
   ] });
 };
 const AssistantActionBar$1 = () => {
-  return /* @__PURE__ */ jsx(
+  return /* @__PURE__ */ jsxs(
     ActionBarPrimitive.Root,
     {
       hideWhenRunning: true,
       autohide: "always",
       autohideFloat: "single-branch",
       className: "flex gap-1 items-center transition-all relative",
-      children: /* @__PURE__ */ jsx(ActionBarPrimitive.Copy, { asChild: true, children: /* @__PURE__ */ jsxs(TooltipIconButton, { tooltip: "Copy", className: "bg-transparent text-icon3 hover:text-icon6", children: [
-        /* @__PURE__ */ jsx(MessagePrimitive.If, { copied: true, children: /* @__PURE__ */ jsx(CheckIcon$1, {}) }),
-        /* @__PURE__ */ jsx(MessagePrimitive.If, { copied: false, children: /* @__PURE__ */ jsx(CopyIcon, {}) })
-      ] }) })
+      children: [
+        /* @__PURE__ */ jsx(MessagePrimitive.If, { speaking: false, children: /* @__PURE__ */ jsx(ActionBarPrimitive.Speak, { asChild: true, children: /* @__PURE__ */ jsx(TooltipIconButton, { tooltip: "Read aloud", children: /* @__PURE__ */ jsx(AudioLinesIcon, {}) }) }) }),
+        /* @__PURE__ */ jsx(MessagePrimitive.If, { speaking: true, children: /* @__PURE__ */ jsx(ActionBarPrimitive.StopSpeaking, { asChild: true, children: /* @__PURE__ */ jsx(TooltipIconButton, { tooltip: "Stop", children: /* @__PURE__ */ jsx(StopCircleIcon, {}) }) }) }),
+        /* @__PURE__ */ jsx(ActionBarPrimitive.Copy, { asChild: true, children: /* @__PURE__ */ jsxs(TooltipIconButton, { tooltip: "Copy", className: "bg-transparent text-icon3 hover:text-icon6", children: [
+          /* @__PURE__ */ jsx(MessagePrimitive.If, { copied: true, children: /* @__PURE__ */ jsx(CheckIcon$1, {}) }),
+          /* @__PURE__ */ jsx(MessagePrimitive.If, { copied: false, children: /* @__PURE__ */ jsx(CopyIcon, {}) })
+        ] }) })
+      ]
     }
   );
 };
@@ -4670,7 +4674,67 @@ const Txt = ({ as: Root = "p", className, variant = "ui-md", font, ...props }) =
   return /* @__PURE__ */ jsx(Root, { className: clsx(variants[variant], font && fonts[font], className), ...props });
 };
 
-const useSpeechRecognition = ({ language = "en-US" } = {}) => {
+async function recordMicrophoneToFile(onFinish) {
+  const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+  const mediaRecorder = new MediaRecorder(stream);
+  let chunks = [];
+  mediaRecorder.ondataavailable = (e) => {
+    chunks.push(e.data);
+  };
+  mediaRecorder.onstop = () => {
+    const blob = new Blob(chunks, { type: "audio/webm" });
+    const file = new File([blob], `recording-${Date.now()}.webm`, {
+      type: "audio/webm",
+      lastModified: Date.now()
+    });
+    stream.getTracks().forEach((track) => track.stop());
+    onFinish(file);
+  };
+  return mediaRecorder;
+}
+
+const useSpeechRecognition = ({
+  language = "en-US",
+  agentId
+}) => {
+  const client = useMastraClient();
+  const [agent, setAgent] = useState(null);
+  useEffect(() => {
+    if (!agentId) return;
+    const agent2 = client.getAgent(agentId);
+    const check = async () => {
+      try {
+        await agent2.voice.getSpeakers();
+        setAgent(agent2);
+      } catch (error) {
+        setAgent(null);
+      }
+    };
+    check();
+  }, [agentId]);
+  const {
+    start: startBrowser,
+    stop: stopBrowser,
+    isListening: isListeningBrowser,
+    transcript: transcriptBrowser
+  } = useBrowserSpeechRecognition({ language });
+  const {
+    start: startMastra,
+    stop: stopMastra,
+    isListening: isListeningMastra,
+    transcript: transcriptMastra
+  } = useMastraSpeechToText({ agent });
+  if (!agent) {
+    return {
+      start: startBrowser,
+      stop: stopBrowser,
+      isListening: isListeningBrowser,
+      transcript: transcriptBrowser
+    };
+  }
+  return { start: startMastra, stop: stopMastra, isListening: isListeningMastra, transcript: transcriptMastra };
+};
+const useBrowserSpeechRecognition = ({ language = "en-US" }) => {
   const speechRecognitionRef = useRef(null);
   const [state, setState] = useState({
     isListening: false,
@@ -4719,6 +4783,41 @@ const useSpeechRecognition = ({ language = "en-US" } = {}) => {
     stop
   };
 };
+const useMastraSpeechToText = ({ agent }) => {
+  const [transcript, setTranscript] = useState("");
+  const [recorder, setRecorder] = useState(null);
+  if (!agent) {
+    return {
+      start: () => {
+      },
+      stop: () => {
+      },
+      isListening: false,
+      transcript: ""
+    };
+  }
+  const handleFinish = (file) => {
+    agent.voice.listen(file).then((res) => {
+      setTranscript(res.text);
+    });
+  };
+  const start = () => {
+    recordMicrophoneToFile(handleFinish).then((recorder2) => {
+      setRecorder(recorder2);
+      recorder2.start();
+    });
+  };
+  const stop = () => {
+    recorder?.stop();
+    setRecorder(null);
+  };
+  return {
+    start,
+    stop,
+    isListening: Boolean(recorder),
+    transcript
+  };
+};
 
 const useHasAttachments = () => {
   const composer = useComposerRuntime();
@@ -4837,7 +4936,7 @@ const ComposerAttachments = () => {
   return /* @__PURE__ */ jsx("div", { className: "flex w-full flex-row items-center gap-4 pb-2", children: /* @__PURE__ */ jsx(ComposerPrimitive.Attachments, { components: { Attachment: AttachmentThumbnail } }) });
 };
 
-const Thread = ({ ToolFallback, agentName, hasMemory, onInputChange }) => {
+const Thread = ({ ToolFallback, agentName, agentId, hasMemory, onInputChange }) => {
   const areaRef = useRef(null);
   useAutoscroll(areaRef, { enabled: true });
   const WrappedAssistantMessage = (props) => {
@@ -4858,7 +4957,7 @@ const Thread = ({ ToolFallback, agentName, hasMemory, onInputChange }) => {
       ) }),
       /* @__PURE__ */ jsx(ThreadPrimitive.If, { empty: false, children: /* @__PURE__ */ jsx("div", {}) })
     ] }),
-    /* @__PURE__ */ jsx(Composer$1, { hasMemory, onInputChange })
+    /* @__PURE__ */ jsx(Composer$1, { hasMemory, onInputChange, agentId })
   ] });
 };
 const ThreadWrapper$1 = ({ children }) => {
@@ -4880,7 +4979,7 @@ const ThreadWelcome$1 = ({ agentName }) => {
     /* @__PURE__ */ jsx("p", { className: "mt-4 font-medium", children: "How can I help you today?" })
   ] }) });
 };
-const Composer$1 = ({ hasMemory, onInputChange }) => {
+const Composer$1 = ({ hasMemory, onInputChange, agentId }) => {
   return /* @__PURE__ */ jsxs("div", { className: "mx-4", children: [
     /* @__PURE__ */ jsxs(ComposerPrimitive.Root, { children: [
       /* @__PURE__ */ jsx("div", { className: "max-w-[568px] w-full mx-auto pb-2", children: /* @__PURE__ */ jsx(ComposerAttachments, {}) }),
@@ -4897,7 +4996,7 @@ const Composer$1 = ({ hasMemory, onInputChange }) => {
           }
         ) }),
         /* @__PURE__ */ jsxs("div", { className: "flex justify-end gap-2", children: [
-          /* @__PURE__ */ jsx(SpeechInput$1, {}),
+          /* @__PURE__ */ jsx(SpeechInput$1, { agentId }),
           /* @__PURE__ */ jsx(ComposerAction$1, {})
         ] })
       ] })
@@ -4908,9 +5007,9 @@ const Composer$1 = ({ hasMemory, onInputChange }) => {
     ] })
   ] });
 };
-const SpeechInput$1 = () => {
+const SpeechInput$1 = ({ agentId }) => {
   const composerRuntime = useComposerRuntime();
-  const { start, stop, isListening, transcript } = useSpeechRecognition();
+  const { start, stop, isListening, transcript } = useSpeechRecognition({ agentId });
   useEffect(() => {
     if (!transcript) return;
     composerRuntime.setText(transcript);
@@ -5057,6 +5156,74 @@ function useWorkingMemory() {
   return ctx;
 }
 
+class VoiceAttachmentAdapter {
+  constructor(agent) {
+    this.agent = agent;
+  }
+  speak(text) {
+    let _cleanup = () => {
+    };
+    const handleEnd = (reason, error) => {
+      if (res.status.type === "ended") return;
+      res.status = { type: "ended", reason, error };
+      _cleanup();
+    };
+    const res = {
+      status: { type: "running" },
+      cancel: () => {
+        handleEnd("cancelled");
+      },
+      subscribe: (callback) => {
+        this.agent.voice.speak(text).then((res2) => {
+          if (res2) {
+            return res2.body;
+          }
+        }).then((readableStream) => {
+          if (readableStream) {
+            return playStreamWithWebAudio(readableStream);
+          }
+        }).then((cleanup) => {
+          if (cleanup) {
+            _cleanup = cleanup;
+          }
+          callback();
+        }).catch((error) => {
+          handleEnd("error", error);
+        });
+        return () => {
+        };
+      }
+    };
+    return res;
+  }
+}
+async function playStreamWithWebAudio(stream) {
+  const audioContext = new window.AudioContext();
+  const reader = stream.getReader();
+  const chunks = [];
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    chunks.push(value);
+  }
+  const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
+  const combinedBuffer = new Uint8Array(totalLength);
+  let offset = 0;
+  for (const chunk of chunks) {
+    combinedBuffer.set(chunk, offset);
+    offset += chunk.length;
+  }
+  const audioBuffer = await audioContext.decodeAudioData(combinedBuffer.buffer);
+  const source = audioContext.createBufferSource();
+  source.buffer = audioBuffer;
+  source.connect(audioContext.destination);
+  source.start();
+  return () => {
+    source.stop();
+    audioContext.close();
+  };
+}
+
 class PDFAttachmentAdapter {
   accept = "application/pdf";
   async add({ file }) {
@@ -5110,6 +5277,37 @@ class PDFAttachmentAdapter {
   }
 }
 
+const useAdapters = (agentId) => {
+  const [isReady, setIsReady] = useState(false);
+  const [speechAdapter, setSpeechAdapter] = useState(void 0);
+  const baseClient = useMastraClient();
+  useEffect(() => {
+    const check = async () => {
+      const agent = baseClient.getAgent(agentId);
+      try {
+        await agent.voice.getSpeakers();
+        setSpeechAdapter(new VoiceAttachmentAdapter(agent));
+        setIsReady(true);
+      } catch {
+        setSpeechAdapter(new WebSpeechSynthesisAdapter());
+        setIsReady(true);
+      }
+    };
+    check();
+  }, [agentId]);
+  return {
+    isReady,
+    adapters: {
+      attachments: new CompositeAttachmentAdapter([
+        new SimpleImageAttachmentAdapter(),
+        new SimpleTextAttachmentAdapter(),
+        new PDFAttachmentAdapter()
+      ]),
+      speech: speechAdapter
+    }
+  };
+};
+
 const convertMessage$2 = (message) => {
   return message;
 };
@@ -5534,20 +5732,16 @@ function MastraRuntimeProvider({
       setIsRunning(false);
     }
   };
+  const { adapters, isReady } = useAdapters(agentId);
   const runtime = useExternalStoreRuntime({
     isRunning,
     messages,
     convertMessage: convertMessage$2,
     onNew,
     onCancel,
-    adapters: {
-      attachments: new CompositeAttachmentAdapter([
-        new SimpleImageAttachmentAdapter(),
-        new SimpleTextAttachmentAdapter(),
-        new PDFAttachmentAdapter()
-      ])
-    }
+    adapters: isReady ? adapters : void 0
   });
+  if (!isReady) return null;
   return /* @__PURE__ */ jsxs(AssistantRuntimeProvider, { runtime, children: [
     " ",
     children,
@@ -5648,7 +5842,7 @@ const AgentChat = ({
       refreshThreadList,
       settings,
       runtimeContext,
-      children: /* @__PURE__ */ jsx(Thread, { agentName: agentName ?? "", hasMemory: memory, onInputChange })
+      children: /* @__PURE__ */ jsx(Thread, { agentName: agentName ?? "", hasMemory: memory, onInputChange, agentId })
     }
   );
 };
@@ -10454,7 +10648,7 @@ const Composer = ({ hasMemory }) => {
 };
 const SpeechInput = () => {
   const composerRuntime = useComposerRuntime();
-  const { start, stop, isListening, transcript } = useSpeechRecognition();
+  const { start, stop, isListening, transcript } = useSpeechRecognition({});
   useEffect(() => {
     if (!transcript) return;
     composerRuntime.setText(transcript);