@mastra/react 0.5.3-alpha.3 → 0.6.0-alpha.4

package/dist/index.d.ts

@@ -7,4 +7,5 @@ export * from './lib/mastra-db/index.js';
 export type { MastraDBMessage, MastraMessageContentV2, MastraMessagePart, MastraToolApproval, MastraToolInvocation, MastraToolInvocationPart, MessageSource, MemoryInfo, } from '@mastra/core/agent/message-list';
 export * from './ui/index.js';
 export * from './workflows/index.js';
+export * from './voice/index.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC;AAC9B,YAAY,EAAE,uBAAuB,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AAClG,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,cAAc,iBAAiB,CAAC;AAChC,YAAY,EACV,eAAe,EACf,sBAAsB,EACtB,iBAAiB,EACjB,kBAAkB,EAClB,oBAAoB,EACpB,wBAAwB,EACxB,aAAa,EACb,UAAU,GACX,MAAM,iCAAiC,CAAC;AACzC,cAAc,MAAM,CAAC;AACrB,cAAc,aAAa,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC;AAC9B,YAAY,EAAE,uBAAuB,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AAClG,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,cAAc,iBAAiB,CAAC;AAChC,YAAY,EACV,eAAe,EACf,sBAAsB,EACtB,iBAAiB,EACjB,kBAAkB,EAClB,oBAAoB,EACpB,wBAAwB,EACxB,aAAa,EACb,UAAU,GACX,MAAM,iCAAiC,CAAC;AACzC,cAAc,MAAM,CAAC;AACrB,cAAc,aAAa,CAAC;AAC5B,cAAc,SAAS,CAAC"}

package/dist/index.js

@@ -113,6 +113,9 @@ var formatStreamCompletionFeedback = (result, maxIterationReached) => {
   );
 };
 
+// src/lib/mastra-db/types.ts
+var CLIENT_MESSAGE_ID_KEY = "clientMessageId";
+
 // src/lib/mastra-db/accumulator.ts
 var cloneMetadata = (metadata) => metadata ? { ...metadata } : {};
 var withParts = (message, parts) => ({
@@ -129,6 +132,14 @@ var withMetadata = (message, metadata) => ({
     metadata
   }
 });
+var clearPendingStatus = (message) => {
+  const { status: _status, [CLIENT_MESSAGE_ID_KEY]: _clientMessageId, ...rest } = message.content.metadata ?? {};
+  return withMetadata(message, rest);
+};
+var clearPendingStatusKeepClientId = (message) => {
+  const { status: _status, ...rest } = message.content.metadata ?? {};
+  return withMetadata(message, rest);
+};
 var replaceLast = (conversation, message) => [
   ...conversation.slice(0, -1),
   message
@@ -155,6 +166,7 @@ var partState = (part) => part.state;
 var finishStreamingAssistantMessage = (conversation) => {
   const lastMessage = conversation[conversation.length - 1];
   if (!lastMessage || lastMessage.role !== "assistant") return conversation;
+  if (lastMessage.content.parts.length === 0) return conversation.slice(0, -1);
   const nextParts = lastMessage.content.parts.map((part) => {
     if ((part.type === "text" || part.type === "reasoning") && partState(part) === "streaming") {
       return {
@@ -355,8 +367,22 @@ var accumulateChunk = ({ chunk, conversation, metadata }) => {
   if (isDataChunk(chunk)) {
     if (chunk.type === "data-user-message" && "data" in chunk && (chunk.data?.type === "user-message" || chunk.data?.type === "user")) {
       const signalId = chunk.data.id;
+      const echoedClientMessageId = chunk.data?.metadata?.[CLIENT_MESSAGE_ID_KEY];
+      if (typeof echoedClientMessageId === "string" && result.some(
+        (message) => message.content.metadata?.status === "pending" && message.content.metadata[CLIENT_MESSAGE_ID_KEY] === echoedClientMessageId
+      )) {
+        return finishStreamingAssistantMessage(
+          result.map(
+            (message) => message.content.metadata?.status === "pending" && message.content.metadata[CLIENT_MESSAGE_ID_KEY] === echoedClientMessageId ? clearPendingStatusKeepClientId(typeof signalId === "string" ? { ...message, id: signalId } : message) : message
+          )
+        );
+      }
       if (typeof signalId === "string" && result.some((message) => message.id === signalId)) {
-        return result;
+        return finishStreamingAssistantMessage(
+          result.map(
+            (message) => message.id === signalId && message.content.metadata?.status === "pending" ? clearPendingStatus(message) : message
+          )
+        );
       }
       const userMessages = signalContentsToUserMessages(chunk.data.contents, metadata);
       if (!userMessages.length) return result;
@@ -1630,46 +1656,45 @@ var accumulateNetworkChunk = ({
 };
 
 // src/lib/mastra-db/fromCoreUserMessage.ts
-var fromCoreUserMessageToMastraDBMessage = (coreUserMessage) => {
-  const id = `user-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
-  const parts = typeof coreUserMessage.content === "string" ? [{ type: "text", text: coreUserMessage.content }] : coreUserMessage.content.map((part) => {
-    switch (part.type) {
-      case "text": {
-        return { type: "text", text: part.text };
-      }
-      case "image": {
-        const url = typeof part.image === "string" ? part.image : part.image instanceof URL ? part.image.toString() : "";
-        return {
-          type: "file",
-          mediaType: part.mimeType ?? "image/*",
-          url
-        };
-      }
-      case "file": {
-        const url = typeof part.data === "string" ? part.data : part.data instanceof URL ? part.data.toString() : "";
-        return {
-          type: "file",
-          mediaType: part.mimeType,
-          url,
-          ...part.filename !== void 0 ? { filename: part.filename } : {}
-        };
-      }
-      default: {
-        const exhaustiveCheck = part;
-        throw new Error(`Unhandled content part type: ${exhaustiveCheck.type}`);
-      }
+var coreUserMessageToParts = (coreUserMessage) => typeof coreUserMessage.content === "string" ? [{ type: "text", text: coreUserMessage.content }] : coreUserMessage.content.map((part) => {
+  switch (part.type) {
+    case "text": {
+      return { type: "text", text: part.text };
     }
-  });
-  return {
-    id,
-    role: "user",
-    createdAt: /* @__PURE__ */ new Date(),
-    content: {
-      format: 2,
-      parts
+    case "image": {
+      const data = typeof part.image === "string" ? part.image : part.image instanceof URL ? part.image.toString() : "";
+      return {
+        type: "file",
+        mimeType: part.mimeType ?? "image/*",
+        data
+      };
     }
-  };
-};
+    case "file": {
+      const data = typeof part.data === "string" ? part.data : part.data instanceof URL ? part.data.toString() : "";
+      return {
+        type: "file",
+        mimeType: part.mimeType,
+        data,
+        ...part.filename !== void 0 ? { filename: part.filename } : {}
+      };
+    }
+    default: {
+      const exhaustiveCheck = part;
+      throw new Error(`Unhandled content part type: ${exhaustiveCheck.type}`);
+    }
+  }
+});
+var newUserMessage = (parts) => ({
+  id: `user-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`,
+  role: "user",
+  createdAt: /* @__PURE__ */ new Date(),
+  content: {
+    format: 2,
+    parts
+  }
+});
+var fromCoreUserMessageToMastraDBMessage = (coreUserMessage) => newUserMessage(coreUserMessageToParts(coreUserMessage));
+var fromCoreUserMessagesToMastraDBMessage = (coreUserMessages) => newUserMessage(coreUserMessages.flatMap(coreUserMessageToParts));
 
 // src/agent/extractRunIdFromMessages.ts
 var isRecord = (value) => value !== null && typeof value === "object";
@@ -1749,21 +1774,33 @@ var resolveInitialMessages = (messages) => messages.filter((message) => {
   return true;
 }).map((message) => {
   const metadata = message.content?.metadata;
-  const pendingToolApprovals = metadata?.pendingToolApprovals;
+  const normalizedMessage = metadata && (metadata.status === "pending" || CLIENT_MESSAGE_ID_KEY in metadata) ? (() => {
+    const { [CLIENT_MESSAGE_ID_KEY]: _omitClientMessageId, ...rest } = metadata;
+    const { status: _omitStatus, ...restWithoutStatus } = rest;
+    return {
+      ...message,
+      content: {
+        ...message.content,
+        metadata: metadata.status === "pending" ? restWithoutStatus : rest
+      }
+    };
+  })() : message;
+  const normalizedMetadata = normalizedMessage.content?.metadata;
+  const pendingToolApprovals = normalizedMetadata?.pendingToolApprovals;
   if (!pendingToolApprovals || typeof pendingToolApprovals !== "object") {
-    return message;
+    return normalizedMessage;
   }
   const stillPending = Object.fromEntries(
     Object.entries(pendingToolApprovals).filter(
-      ([, approval]) => approval && typeof approval === "object" && typeof approval.toolCallId === "string" && !toolCallHasOutput(message.content.parts, approval.toolCallId)
+      ([, approval]) => approval && typeof approval === "object" && typeof approval.toolCallId === "string" && !toolCallHasOutput(normalizedMessage.content.parts, approval.toolCallId)
     )
   );
-  const { pendingToolApprovals: _omit, ...restMetadata } = metadata;
+  const { pendingToolApprovals: _omit, ...restMetadata } = normalizedMetadata;
   const hasStillPending = Object.keys(stillPending).length > 0;
   return {
-    ...message,
+    ...normalizedMessage,
     content: {
-      ...message.content,
+      ...normalizedMessage.content,
       metadata: {
         ...restMetadata,
         mode: "stream",
@@ -2090,7 +2127,8 @@ var useChat = ({
     signal,
     tracingOptions,
     clientTools,
-    signalId
+    signalId,
+    clientMessageId
   }) => {
     const {
       frequencyPenalty,
@@ -2201,7 +2239,7 @@ var useChat = ({
     };
     try {
       const result = await agent.sendMessage({
-        message: messageContents,
+        message: clientMessageId ? { contents: messageContents, metadata: { [CLIENT_MESSAGE_ID_KEY]: clientMessageId } } : messageContents,
         resourceId: resourceId || agentId,
         threadId: threadId2,
         ifIdle: {
@@ -2236,7 +2274,7 @@ var useChat = ({
           onSignalEcho?.(resolvedSignalId);
           if (isThreadSignalUnsupportedError(signalError)) {
             markThreadSignalsUnsupported();
-            setMessages((prev) => [...prev, ...coreUserMessages.map(fromCoreUserMessageToMastraDBMessage)]);
+            setMessages((prev) => [...prev, fromCoreUserMessagesToMastraDBMessage(coreUserMessages)]);
             await streamWithLegacyRoute();
             return;
           }
@@ -2497,15 +2535,26 @@ var useChat = ({
     if (args.coreUserMessages) {
       coreUserMessages.push(...args.coreUserMessages);
     }
-    const dbUserMessages = coreUserMessages.map(fromCoreUserMessageToMastraDBMessage);
-    const signalId = mode === "stream" && args.threadId && !_threadSignalsUnsupportedRef.current && !threadSignalsDisabled ? dbUserMessages[0]?.id : void 0;
-    if (!signalId) {
-      setMessages((s) => [...s, ...dbUserMessages]);
+    const dbUserMessage = fromCoreUserMessagesToMastraDBMessage(coreUserMessages);
+    const clientSetId = mode === "stream" && args.threadId && !_threadSignalsUnsupportedRef.current && !threadSignalsDisabled ? `client-set-${v4()}` : void 0;
+    const signalId = clientSetId;
+    const clientMessageId = clientSetId;
+    if (signalId) {
+      const metadata = {
+        ...dbUserMessage.content.metadata,
+        mode: "stream",
+        status: "pending",
+        [CLIENT_MESSAGE_ID_KEY]: clientMessageId
+      };
+      const pendingMessage = { ...dbUserMessage, id: clientSetId, content: { ...dbUserMessage.content, metadata } };
+      setMessages((s) => [...s, pendingMessage]);
+    } else {
+      setMessages((s) => [...s, dbUserMessage]);
     }
     if (mode === "generate") {
       await generate({ ...args, coreUserMessages });
     } else if (mode === "stream") {
-      await stream({ ...args, coreUserMessages, signalId });
+      await stream({ ...args, coreUserMessages, signalId, clientMessageId });
     } else if (mode === "network") {
       await network({ ...args, coreUserMessages });
     }
@@ -2924,6 +2973,9 @@ var MessageFactoryComponent = ({ message, roles, status, fallback, ...renderers
     content = status.Error({ text: joinText(parts), message });
   } else {
     content = /* @__PURE__ */ jsx(Fragment, { children: parts.map((part, index) => /* @__PURE__ */ jsx(PartRenderer, { part, renderers, fallback }, getPartKey(part, index))) });
+    if (metadata?.status === "pending" && status?.Pending) {
+      content = status.Pending({ children: content, text: joinText(parts), message });
+    }
     const verdict = resolveTaskVerdict(metadata);
     if (verdict && status?.Task) {
       content = /* @__PURE__ */ jsxs(Fragment, { children: [
@@ -3350,6 +3402,233 @@ function useCancelWorkflowRun() {
   });
 }
 
-export { AgentIcon, CodeBlock, CodeBlockClass, CodeCopyButton, Entity, EntityCaret, EntityContent, EntityContentClass, EntityTrigger, EntityTriggerClass, EntityTriggerVariantClasses, Entry, EntryClass, EntryTitle, EntryTitleClass, Icon, IconButton, IconButtonClass, IconSizes, MastraReactProvider, Message, MessageActions, MessageActionsClass, MessageClass, MessageContent, MessageContentClass, MessageFactory, MessageList, MessageListClass, MessageStreaming, MessageStreamingClass, MessageUsage, MessageUsageClass, MessageUsageEntry, MessageUsageEntryClass, MessageUsageValue, MessageUsageValueClass, MessageUsages, MessageUsagesClass, ToolApproval, ToolApprovalActions, ToolApprovalActionsClass, ToolApprovalClass, ToolApprovalContent, ToolApprovalContentClass, ToolApprovalHeader, ToolApprovalHeaderClass, ToolApprovalTitle, ToolApprovalTitleClass, ToolsIcon, Tooltip, TooltipContent, TooltipContentClass, TooltipTrigger, WorkflowIcon, accumulateChunk, accumulateNetworkChunk, finishStreamingAssistantMessage, fromCoreUserMessageToMastraDBMessage, mapWorkflowStreamChunkToWatchResult, useCancelWorkflowRun, useChat, useCreateWorkflowRun, useEntity, useMastraClient, useStreamWorkflow };
+// src/voice/record-mic-to-file.ts
+async function recordMicrophoneToFile(onFinish) {
+  const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+  const mediaRecorder = new MediaRecorder(stream);
+  let chunks = [];
+  mediaRecorder.ondataavailable = (e) => {
+    chunks.push(e.data);
+  };
+  mediaRecorder.onstop = () => {
+    const blob = new Blob(chunks, { type: "audio/webm" });
+    const file = new File([blob], `recording-${Date.now()}.webm`, {
+      type: "audio/webm",
+      lastModified: Date.now()
+    });
+    stream.getTracks().forEach((track) => track.stop());
+    onFinish(file);
+  };
+  return mediaRecorder;
+}
+
+// src/voice/play-stream-with-web-audio.ts
+async function playStreamWithWebAudio(stream, onEnded) {
+  const audioContext = new window.AudioContext();
+  const reader = stream.getReader();
+  const chunks = [];
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      chunks.push(value);
+    }
+    const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
+    const combinedBuffer = new Uint8Array(totalLength);
+    let offset = 0;
+    for (const chunk of chunks) {
+      combinedBuffer.set(chunk, offset);
+      offset += chunk.length;
+    }
+    const audioBuffer = await audioContext.decodeAudioData(combinedBuffer.buffer);
+    const source = audioContext.createBufferSource();
+    source.buffer = audioBuffer;
+    source.onended = onEnded ?? null;
+    source.connect(audioContext.destination);
+    source.start();
+    return () => {
+      source.onended = null;
+      source.stop();
+      void audioContext.close();
+    };
+  } catch (error) {
+    await reader.cancel().catch(() => void 0);
+    await audioContext.close().catch(() => void 0);
+    throw error;
+  } finally {
+    reader.releaseLock();
+  }
+}
+var useSpeechRecognition = ({
+  language = "en-US",
+  agentId,
+  requestContext
+}) => {
+  const client = useMastraClient();
+  const [agent, setAgent] = useState(null);
+  useEffect(() => {
+    let cancelled = false;
+    if (!agentId) {
+      setAgent(null);
+      return () => {
+        cancelled = true;
+      };
+    }
+    const agent2 = client.getAgent(agentId);
+    const check = async () => {
+      try {
+        const speakers = await agent2.voice.getSpeakers(requestContext);
+        if (!cancelled) {
+          setAgent(speakers.length > 0 ? agent2 : null);
+        }
+      } catch {
+        if (!cancelled) {
+          setAgent(null);
+        }
+      }
+    };
+    void check();
+    return () => {
+      cancelled = true;
+    };
+  }, [agentId, client, requestContext]);
+  const browserSpeechRecognition = useBrowserSpeechRecognition({ language });
+  const mastraSpeechRecognition = useMastraSpeechToText({ agent, language });
+  if (!agent) {
+    return browserSpeechRecognition;
+  }
+  return mastraSpeechRecognition;
+};
+var useBrowserSpeechRecognition = ({ language = "en-US" }) => {
+  const speechRecognitionRef = useRef(null);
+  const [state, setState] = useState({
+    isListening: false,
+    transcript: "",
+    error: null
+  });
+  const start = () => {
+    if (!speechRecognitionRef.current) return;
+    speechRecognitionRef.current.start();
+  };
+  const stop = () => {
+    if (!speechRecognitionRef.current) return;
+    speechRecognitionRef.current.stop();
+  };
+  useEffect(() => {
+    if (!("webkitSpeechRecognition" in window) && !("SpeechRecognition" in window)) {
+      setState((prev) => ({ ...prev, error: "Speech Recognition not supported in this browser" }));
+      return;
+    }
+    const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition;
+    const recognition = new SpeechRecognition();
+    speechRecognitionRef.current = recognition;
+    recognition.continuous = true;
+    recognition.lang = language;
+    recognition.onstart = () => {
+      setState((prev) => ({ ...prev, isListening: true, error: null }));
+    };
+    recognition.onresult = (event) => {
+      let finalTranscript = "";
+      for (let i = event.resultIndex; i < event.results.length; i++) {
+        const transcript = event.results[i][0].transcript;
+        if (event.results[i].isFinal) {
+          finalTranscript += transcript + " ";
+        }
+      }
+      setState((prev) => ({ ...prev, transcript: finalTranscript }));
+    };
+    recognition.onerror = (event) => {
+      setState((prev) => ({ ...prev, error: `Error: ${event.error}` }));
+    };
+    recognition.onend = () => setState((prev) => ({ ...prev, isListening: false }));
+    return () => {
+      try {
+        recognition.stop();
+      } catch {
+      }
+      recognition.onstart = null;
+      recognition.onresult = null;
+      recognition.onerror = null;
+      recognition.onend = null;
+      speechRecognitionRef.current = null;
+    };
+  }, [language]);
+  return {
+    ...state,
+    start,
+    stop
+  };
+};
+var useMastraSpeechToText = ({
+  agent,
+  language
+}) => {
+  const [state, setState] = useState({
+    isListening: false,
+    transcript: "",
+    error: null
+  });
+  const recorderRef = useRef(null);
+  const sessionRef = useRef(0);
+  const startInFlightRef = useRef(false);
+  useEffect(() => {
+    return () => {
+      sessionRef.current += 1;
+      startInFlightRef.current = false;
+      recorderRef.current?.stop();
+      recorderRef.current = null;
+    };
+  }, [agent]);
+  const handleFinish = (session) => (file) => {
+    if (!agent || session !== sessionRef.current) return;
+    recorderRef.current = null;
+    setState((prev) => ({ ...prev, isListening: false }));
+    void agent.voice.listen(file, { language }).then((res) => {
+      if (session !== sessionRef.current) return;
+      setState((prev) => ({ ...prev, transcript: res.text, error: null }));
+    }).catch((error) => {
+      if (session !== sessionRef.current) return;
+      const message = error instanceof Error ? error.message : "Failed to transcribe speech";
+      setState((prev) => ({ ...prev, error: message }));
+    });
+  };
+  const start = () => {
+    if (!agent || startInFlightRef.current || recorderRef.current) return;
+    startInFlightRef.current = true;
+    const session = sessionRef.current;
+    void recordMicrophoneToFile(handleFinish(session)).then((recorder) => {
+      startInFlightRef.current = false;
+      if (session !== sessionRef.current) {
+        try {
+          recorder.stop();
+        } catch {
+        }
+        return;
+      }
+      recorderRef.current = recorder;
+      setState((prev) => ({ ...prev, isListening: true, error: null }));
+      recorder.start();
+    }).catch((error) => {
+      startInFlightRef.current = false;
+      if (session !== sessionRef.current) return;
+      const message = error instanceof Error ? error.message : "Failed to start speech recording";
+      setState((prev) => ({ ...prev, isListening: false, error: message }));
+    });
+  };
+  const stop = () => {
+    sessionRef.current += 1;
+    startInFlightRef.current = false;
+    recorderRef.current?.stop();
+    recorderRef.current = null;
+    setState((prev) => ({ ...prev, isListening: false }));
+  };
+  return {
+    ...state,
+    start,
+    stop
+  };
+};
+
+export { AgentIcon, CLIENT_MESSAGE_ID_KEY, CodeBlock, CodeBlockClass, CodeCopyButton, Entity, EntityCaret, EntityContent, EntityContentClass, EntityTrigger, EntityTriggerClass, EntityTriggerVariantClasses, Entry, EntryClass, EntryTitle, EntryTitleClass, Icon, IconButton, IconButtonClass, IconSizes, MastraReactProvider, Message, MessageActions, MessageActionsClass, MessageClass, MessageContent, MessageContentClass, MessageFactory, MessageList, MessageListClass, MessageStreaming, MessageStreamingClass, MessageUsage, MessageUsageClass, MessageUsageEntry, MessageUsageEntryClass, MessageUsageValue, MessageUsageValueClass, MessageUsages, MessageUsagesClass, ToolApproval, ToolApprovalActions, ToolApprovalActionsClass, ToolApprovalClass, ToolApprovalContent, ToolApprovalContentClass, ToolApprovalHeader, ToolApprovalHeaderClass, ToolApprovalTitle, ToolApprovalTitleClass, ToolsIcon, Tooltip, TooltipContent, TooltipContentClass, TooltipTrigger, WorkflowIcon, accumulateChunk, accumulateNetworkChunk, finishStreamingAssistantMessage, fromCoreUserMessageToMastraDBMessage, fromCoreUserMessagesToMastraDBMessage, mapWorkflowStreamChunkToWatchResult, playStreamWithWebAudio, recordMicrophoneToFile, useCancelWorkflowRun, useChat, useCreateWorkflowRun, useEntity, useMastraClient, useSpeechRecognition, useStreamWorkflow };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map