@mastra/react 0.5.3-alpha.2 → 0.6.0-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -0
- package/dist/agent/hooks.d.ts +5 -0
- package/dist/agent/hooks.d.ts.map +1 -1
- package/dist/index.cjs +337 -53
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +333 -54
- package/dist/index.js.map +1 -1
- package/dist/lib/mastra-db/accumulator.d.ts.map +1 -1
- package/dist/lib/mastra-db/fromCoreUserMessage.d.ts +8 -5
- package/dist/lib/mastra-db/fromCoreUserMessage.d.ts.map +1 -1
- package/dist/lib/mastra-db/index.d.ts +2 -1
- package/dist/lib/mastra-db/index.d.ts.map +1 -1
- package/dist/lib/mastra-db/types.d.ts +17 -2
- package/dist/lib/mastra-db/types.d.ts.map +1 -1
- package/dist/ui/MessageFactory/MessageFactory.d.ts.map +1 -1
- package/dist/ui/MessageFactory/index.d.ts +1 -1
- package/dist/ui/MessageFactory/index.d.ts.map +1 -1
- package/dist/ui/MessageFactory/types.d.ts +26 -9
- package/dist/ui/MessageFactory/types.d.ts.map +1 -1
- package/dist/voice/index.d.ts +5 -0
- package/dist/voice/index.d.ts.map +1 -0
- package/dist/voice/play-stream-with-web-audio.d.ts +2 -0
- package/dist/voice/play-stream-with-web-audio.d.ts.map +1 -0
- package/dist/voice/record-mic-to-file.d.ts +2 -0
- package/dist/voice/record-mic-to-file.d.ts.map +1 -0
- package/dist/voice/use-speech-recognition.d.ts +20 -0
- package/dist/voice/use-speech-recognition.d.ts.map +1 -0
- package/package.json +4 -4
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,44 @@
|
|
|
1
1
|
# @mastra/react
|
|
2
2
|
|
|
3
|
+
## 0.6.0-alpha.4
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- Added voice helpers to the React SDK and made agent text-to-speech audible. ([#17663](https://github.com/mastra-ai/mastra/pull/17663))
|
|
8
|
+
|
|
9
|
+
The React SDK now exposes voice helpers: `useSpeechRecognition` for speech-to-text, `playStreamWithWebAudio` for buffered Web Audio playback from a `ReadableStream`, and `recordMicrophoneToFile` for capturing microphone audio. The `useSpeechRecognition` hook automatically uses an agent's voice provider when one is configured, and falls back to the browser's built-in speech recognition otherwise.
|
|
10
|
+
|
|
11
|
+
```tsx
|
|
12
|
+
import { useSpeechRecognition } from '@mastra/react';
|
|
13
|
+
|
|
14
|
+
function VoiceInput({ agentId }: { agentId?: string }) {
|
|
15
|
+
const { start, stop, isListening, transcript } = useSpeechRecognition({ agentId });
|
|
16
|
+
return <button onClick={isListening ? stop : start}>{transcript}</button>;
|
|
17
|
+
}
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
Also fixed agent text-to-speech being inaudible. The `voice/speak` route now returns a web-standard audio response (`Content-Type: audio/mpeg`) so server-side adapters pass raw audio bytes through to the client instead of JSON-encoding them. This also resolves `getReader` crashes when an agent speaks using providers like OpenAI voice.
|
|
21
|
+
|
|
22
|
+
- Show a sent user message instantly as a pending bubble in the chat thread, then settle it in place when the server confirms it — instead of staging it near the composer. The optimistic bubble and the outgoing message share a client-generated correlation id, so the server echo reconciles the exact bubble with no duplicate message. ([#17688](https://github.com/mastra-ai/mastra/pull/17688))
|
|
23
|
+
|
|
24
|
+
### Patch Changes
|
|
25
|
+
|
|
26
|
+
- Fixed MessageFactory Reasoning renderer types to expose streaming state metadata. ([#17774](https://github.com/mastra-ai/mastra/pull/17774))
|
|
27
|
+
|
|
28
|
+
- Fix streamed user messages with attachments in React chat state ([#17774](https://github.com/mastra-ai/mastra/pull/17774))
|
|
29
|
+
|
|
30
|
+
- Updated dependencies [[`575f815`](https://github.com/mastra-ai/mastra/commit/575f815c5c3567b71c0b83cbb7fa98c8253a9d9c), [`306909a`](https://github.com/mastra-ai/mastra/commit/306909a693de77d709b38706e2673c9547d24a28), [`5191af8`](https://github.com/mastra-ai/mastra/commit/5191af80c799eea25357c545fc05d91b3883531d), [`43bd3d4`](https://github.com/mastra-ai/mastra/commit/43bd3d421987463fdf35386a45199c49499ed069), [`e6fa79e`](https://github.com/mastra-ai/mastra/commit/e6fa79ec72a2ddffdd25e85270398951e9d552a4), [`904bcdf`](https://github.com/mastra-ai/mastra/commit/904bcdf7b8004aa7be823f9f70ca63580e47e470), [`7f5ee1d`](https://github.com/mastra-ai/mastra/commit/7f5ee1dca46daee8d2817f2ebe49e6335da81956), [`1e9aab5`](https://github.com/mastra-ai/mastra/commit/1e9aab50ff11e6e88fde4d7cbf512c44a9fe8d61), [`bf8eb6d`](https://github.com/mastra-ai/mastra/commit/bf8eb6d0ec213a403eb9265a594ad283c44ab3dc), [`493a328`](https://github.com/mastra-ai/mastra/commit/493a328f4346a1deeb9f1e2e44c8f2a3a4d7591b), [`029a414`](https://github.com/mastra-ai/mastra/commit/029a4141719793bd3e898a39eb5a0466a55f5f3a), [`b147b29`](https://github.com/mastra-ai/mastra/commit/b147b2907f0cd1aa812efe6d6e3f58d22e66fc88), [`d371ac1`](https://github.com/mastra-ai/mastra/commit/d371ac1d9820afaaf7cfdbc380a475946a994d8f), [`cf182b7`](https://github.com/mastra-ai/mastra/commit/cf182b7fb495767946d9840ef29f19cfa906f31f), [`a049c2a`](https://github.com/mastra-ai/mastra/commit/a049c2a9dfb41d0ee2e7a28874a88cd64fd5669f), [`b147b29`](https://github.com/mastra-ai/mastra/commit/b147b2907f0cd1aa812efe6d6e3f58d22e66fc88), [`2a96528`](https://github.com/mastra-ai/mastra/commit/2a9652848dfa3c5a2426f952e9d93554c26fd90f), [`61f5491`](https://github.com/mastra-ai/mastra/commit/61f54912e6453cc706bb5d7df9f6c7aad78d428f), [`2656d9c`](https://github.com/mastra-ai/mastra/commit/2656d9c2976d4f3354253bfbbbf9b88a1b2bbf34), [`63e3fe1`](https://github.com/mastra-ai/mastra/commit/63e3fe13cc1ea96f91d7c68aea92f400faf9e4da), [`1d4ce8d`](https://github.com/mastra-ai/mastra/commit/1d4ce8daaa54511f325c1b609d31b8e54009d677), [`8c68372`](https://github.com/mastra-ai/mastra/commit/8c68372e85fe0b066ec12c58bd29ffb93e54c552)]:
|
|
31
|
+
- @mastra/core@1.42.0-alpha.4
|
|
32
|
+
- @mastra/client-js@1.24.0-alpha.4
|
|
33
|
+
|
|
34
|
+
## 0.5.3-alpha.3
|
|
35
|
+
|
|
36
|
+
### Patch Changes
|
|
37
|
+
|
|
38
|
+
- Updated dependencies [[`34839c1`](https://github.com/mastra-ai/mastra/commit/34839c1910b6964bf59ed0cee58844efebbb684e), [`053735a`](https://github.com/mastra-ai/mastra/commit/053735a75c2c18e23ce34d9468007efa4a45f4c4), [`34839c1`](https://github.com/mastra-ai/mastra/commit/34839c1910b6964bf59ed0cee58844efebbb684e), [`34839c1`](https://github.com/mastra-ai/mastra/commit/34839c1910b6964bf59ed0cee58844efebbb684e), [`a952852`](https://github.com/mastra-ai/mastra/commit/a952852c971a21fb646cd907c75fcf4443cdc963)]:
|
|
39
|
+
- @mastra/client-js@1.24.0-alpha.3
|
|
40
|
+
- @mastra/core@1.42.0-alpha.3
|
|
41
|
+
|
|
3
42
|
## 0.5.3-alpha.2
|
|
4
43
|
|
|
5
44
|
### Patch Changes
|
package/dist/agent/hooks.d.ts
CHANGED
|
@@ -54,6 +54,11 @@ export type StreamArgs = SharedArgs & {
|
|
|
54
54
|
onChunk?: (chunk: ChunkType) => Promise<void>;
|
|
55
55
|
clientTools?: ClientToolsInput;
|
|
56
56
|
signalId?: string;
|
|
57
|
+
/**
|
|
58
|
+
* Client-generated correlation id stamped on the optimistic pending bubble
|
|
59
|
+
* and the outgoing message metadata so the server echo can reconcile them.
|
|
60
|
+
*/
|
|
61
|
+
clientMessageId?: string;
|
|
57
62
|
};
|
|
58
63
|
export type NetworkArgs = SharedArgs & {
|
|
59
64
|
onNetworkChunk?: (chunk: NetworkChunkType) => Promise<void>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hooks.d.ts","sourceRoot":"","sources":["../../src/agent/hooks.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAY,eAAe,EAA4B,MAAM,iCAAiC,CAAC;AAE3G,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AACjE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AACnE,OAAO,KAAK,EAAE,SAAS,EAAiB,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"hooks.d.ts","sourceRoot":"","sources":["../../src/agent/hooks.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAY,eAAe,EAA4B,MAAM,iCAAiC,CAAC;AAE3G,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AACjE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AACnE,OAAO,KAAK,EAAE,SAAS,EAAiB,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAatF,OAAO,KAAK,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAmI/D,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,eAAe,EAAE,CAAC;IACpC,6FAA6F;IAC7F,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC;;;;OAIG;IACH,WAAW,CAAC,EAAE,gBAAgB,CAAC;IAC/B,YAAY,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAC3D,YAAY,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IAC1C,0BAA0B,CAAC,EAAE,MAAM,IAAI,CAAC;IACxC;;;OAGG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC/B;AAED,UAAU,UAAU;IAClB,gBAAgB,EAAE,eAAe,EAAE,CAAC;IACpC,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,cAAc,CAAC,EAAE,cAAc,CAAC;CACjC;AAED,MAAM,MAAM,eAAe,GAAG;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,gBAAgB,CAAC,EAAE,eAAe,EAAE,CAAA;CAAE,GAAG,CACtF,CAAC;IAAE,IAAI,EAAE,UAAU,CAAA;CAAE,GAAG,IAAI,CAAC,YAAY,EAAE,kBAAkB,CAAC,CAAC,GAC/D,CAAC;IAAE,IAAI,EAAE,QAAQ,CAAA;CAAE,GAAG,IAAI,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAC,GAC3D,CAAC;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,GAAG,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC,GAC7D,CAAC;IAAE,IAAI,CAAC,EAAE,SAAS,CAAA;CAAE,GAAG,IAAI,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAC,CAChE,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG,UAAU,GAAG;IACtC,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,eAAe,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1D,WAAW,CAAC,EAAE,gBAAgB,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG,UAAU,GAAG;IACpC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9C,WAAW,CAAC,EAAE,gBAAgB,CAAC;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG,UAAU,GAAG;IACrC,cAAc,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7D,CAAC;AAoDF,eAAO,MAAM,OAAO,GAAI,qMAWrB,eAAe;;qCAg0ByC,eAAe;;;;kCArN7B,MAAM;kCAiDN,MAAM;0CAgDE,MAAM;0CA0BN,MAAM;;;;oBAhtBvB,UAAU,GAAG,UAAU;;;uCA0uBT,MAAM,UAAU,MAAM;uCAgCtB,MAAM,UAAU,MAAM;;;oBAvwBtC,UAAU,GAAG,UAAU;;;CAm2BxD,CAAC"}
|
package/dist/index.cjs
CHANGED
|
@@ -115,6 +115,9 @@ var formatStreamCompletionFeedback = (result, maxIterationReached) => {
|
|
|
115
115
|
);
|
|
116
116
|
};
|
|
117
117
|
|
|
118
|
+
// src/lib/mastra-db/types.ts
|
|
119
|
+
var CLIENT_MESSAGE_ID_KEY = "clientMessageId";
|
|
120
|
+
|
|
118
121
|
// src/lib/mastra-db/accumulator.ts
|
|
119
122
|
var cloneMetadata = (metadata) => metadata ? { ...metadata } : {};
|
|
120
123
|
var withParts = (message, parts) => ({
|
|
@@ -131,6 +134,14 @@ var withMetadata = (message, metadata) => ({
|
|
|
131
134
|
metadata
|
|
132
135
|
}
|
|
133
136
|
});
|
|
137
|
+
var clearPendingStatus = (message) => {
|
|
138
|
+
const { status: _status, [CLIENT_MESSAGE_ID_KEY]: _clientMessageId, ...rest } = message.content.metadata ?? {};
|
|
139
|
+
return withMetadata(message, rest);
|
|
140
|
+
};
|
|
141
|
+
var clearPendingStatusKeepClientId = (message) => {
|
|
142
|
+
const { status: _status, ...rest } = message.content.metadata ?? {};
|
|
143
|
+
return withMetadata(message, rest);
|
|
144
|
+
};
|
|
134
145
|
var replaceLast = (conversation, message) => [
|
|
135
146
|
...conversation.slice(0, -1),
|
|
136
147
|
message
|
|
@@ -157,6 +168,7 @@ var partState = (part) => part.state;
|
|
|
157
168
|
var finishStreamingAssistantMessage = (conversation) => {
|
|
158
169
|
const lastMessage = conversation[conversation.length - 1];
|
|
159
170
|
if (!lastMessage || lastMessage.role !== "assistant") return conversation;
|
|
171
|
+
if (lastMessage.content.parts.length === 0) return conversation.slice(0, -1);
|
|
160
172
|
const nextParts = lastMessage.content.parts.map((part) => {
|
|
161
173
|
if ((part.type === "text" || part.type === "reasoning") && partState(part) === "streaming") {
|
|
162
174
|
return {
|
|
@@ -357,8 +369,22 @@ var accumulateChunk = ({ chunk, conversation, metadata }) => {
|
|
|
357
369
|
if (isDataChunk(chunk)) {
|
|
358
370
|
if (chunk.type === "data-user-message" && "data" in chunk && (chunk.data?.type === "user-message" || chunk.data?.type === "user")) {
|
|
359
371
|
const signalId = chunk.data.id;
|
|
372
|
+
const echoedClientMessageId = chunk.data?.metadata?.[CLIENT_MESSAGE_ID_KEY];
|
|
373
|
+
if (typeof echoedClientMessageId === "string" && result.some(
|
|
374
|
+
(message) => message.content.metadata?.status === "pending" && message.content.metadata[CLIENT_MESSAGE_ID_KEY] === echoedClientMessageId
|
|
375
|
+
)) {
|
|
376
|
+
return finishStreamingAssistantMessage(
|
|
377
|
+
result.map(
|
|
378
|
+
(message) => message.content.metadata?.status === "pending" && message.content.metadata[CLIENT_MESSAGE_ID_KEY] === echoedClientMessageId ? clearPendingStatusKeepClientId(typeof signalId === "string" ? { ...message, id: signalId } : message) : message
|
|
379
|
+
)
|
|
380
|
+
);
|
|
381
|
+
}
|
|
360
382
|
if (typeof signalId === "string" && result.some((message) => message.id === signalId)) {
|
|
361
|
-
return
|
|
383
|
+
return finishStreamingAssistantMessage(
|
|
384
|
+
result.map(
|
|
385
|
+
(message) => message.id === signalId && message.content.metadata?.status === "pending" ? clearPendingStatus(message) : message
|
|
386
|
+
)
|
|
387
|
+
);
|
|
362
388
|
}
|
|
363
389
|
const userMessages = signalContentsToUserMessages(chunk.data.contents, metadata);
|
|
364
390
|
if (!userMessages.length) return result;
|
|
@@ -1632,46 +1658,45 @@ var accumulateNetworkChunk = ({
|
|
|
1632
1658
|
};
|
|
1633
1659
|
|
|
1634
1660
|
// src/lib/mastra-db/fromCoreUserMessage.ts
|
|
1635
|
-
var
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
case "text": {
|
|
1640
|
-
return { type: "text", text: part.text };
|
|
1641
|
-
}
|
|
1642
|
-
case "image": {
|
|
1643
|
-
const url = typeof part.image === "string" ? part.image : part.image instanceof URL ? part.image.toString() : "";
|
|
1644
|
-
return {
|
|
1645
|
-
type: "file",
|
|
1646
|
-
mediaType: part.mimeType ?? "image/*",
|
|
1647
|
-
url
|
|
1648
|
-
};
|
|
1649
|
-
}
|
|
1650
|
-
case "file": {
|
|
1651
|
-
const url = typeof part.data === "string" ? part.data : part.data instanceof URL ? part.data.toString() : "";
|
|
1652
|
-
return {
|
|
1653
|
-
type: "file",
|
|
1654
|
-
mediaType: part.mimeType,
|
|
1655
|
-
url,
|
|
1656
|
-
...part.filename !== void 0 ? { filename: part.filename } : {}
|
|
1657
|
-
};
|
|
1658
|
-
}
|
|
1659
|
-
default: {
|
|
1660
|
-
const exhaustiveCheck = part;
|
|
1661
|
-
throw new Error(`Unhandled content part type: ${exhaustiveCheck.type}`);
|
|
1662
|
-
}
|
|
1661
|
+
var coreUserMessageToParts = (coreUserMessage) => typeof coreUserMessage.content === "string" ? [{ type: "text", text: coreUserMessage.content }] : coreUserMessage.content.map((part) => {
|
|
1662
|
+
switch (part.type) {
|
|
1663
|
+
case "text": {
|
|
1664
|
+
return { type: "text", text: part.text };
|
|
1663
1665
|
}
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
parts
|
|
1666
|
+
case "image": {
|
|
1667
|
+
const data = typeof part.image === "string" ? part.image : part.image instanceof URL ? part.image.toString() : "";
|
|
1668
|
+
return {
|
|
1669
|
+
type: "file",
|
|
1670
|
+
mimeType: part.mimeType ?? "image/*",
|
|
1671
|
+
data
|
|
1672
|
+
};
|
|
1672
1673
|
}
|
|
1673
|
-
|
|
1674
|
-
|
|
1674
|
+
case "file": {
|
|
1675
|
+
const data = typeof part.data === "string" ? part.data : part.data instanceof URL ? part.data.toString() : "";
|
|
1676
|
+
return {
|
|
1677
|
+
type: "file",
|
|
1678
|
+
mimeType: part.mimeType,
|
|
1679
|
+
data,
|
|
1680
|
+
...part.filename !== void 0 ? { filename: part.filename } : {}
|
|
1681
|
+
};
|
|
1682
|
+
}
|
|
1683
|
+
default: {
|
|
1684
|
+
const exhaustiveCheck = part;
|
|
1685
|
+
throw new Error(`Unhandled content part type: ${exhaustiveCheck.type}`);
|
|
1686
|
+
}
|
|
1687
|
+
}
|
|
1688
|
+
});
|
|
1689
|
+
var newUserMessage = (parts) => ({
|
|
1690
|
+
id: `user-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`,
|
|
1691
|
+
role: "user",
|
|
1692
|
+
createdAt: /* @__PURE__ */ new Date(),
|
|
1693
|
+
content: {
|
|
1694
|
+
format: 2,
|
|
1695
|
+
parts
|
|
1696
|
+
}
|
|
1697
|
+
});
|
|
1698
|
+
var fromCoreUserMessageToMastraDBMessage = (coreUserMessage) => newUserMessage(coreUserMessageToParts(coreUserMessage));
|
|
1699
|
+
var fromCoreUserMessagesToMastraDBMessage = (coreUserMessages) => newUserMessage(coreUserMessages.flatMap(coreUserMessageToParts));
|
|
1675
1700
|
|
|
1676
1701
|
// src/agent/extractRunIdFromMessages.ts
|
|
1677
1702
|
var isRecord = (value) => value !== null && typeof value === "object";
|
|
@@ -1751,21 +1776,33 @@ var resolveInitialMessages = (messages) => messages.filter((message) => {
|
|
|
1751
1776
|
return true;
|
|
1752
1777
|
}).map((message) => {
|
|
1753
1778
|
const metadata = message.content?.metadata;
|
|
1754
|
-
const
|
|
1779
|
+
const normalizedMessage = metadata && (metadata.status === "pending" || CLIENT_MESSAGE_ID_KEY in metadata) ? (() => {
|
|
1780
|
+
const { [CLIENT_MESSAGE_ID_KEY]: _omitClientMessageId, ...rest } = metadata;
|
|
1781
|
+
const { status: _omitStatus, ...restWithoutStatus } = rest;
|
|
1782
|
+
return {
|
|
1783
|
+
...message,
|
|
1784
|
+
content: {
|
|
1785
|
+
...message.content,
|
|
1786
|
+
metadata: metadata.status === "pending" ? restWithoutStatus : rest
|
|
1787
|
+
}
|
|
1788
|
+
};
|
|
1789
|
+
})() : message;
|
|
1790
|
+
const normalizedMetadata = normalizedMessage.content?.metadata;
|
|
1791
|
+
const pendingToolApprovals = normalizedMetadata?.pendingToolApprovals;
|
|
1755
1792
|
if (!pendingToolApprovals || typeof pendingToolApprovals !== "object") {
|
|
1756
|
-
return
|
|
1793
|
+
return normalizedMessage;
|
|
1757
1794
|
}
|
|
1758
1795
|
const stillPending = Object.fromEntries(
|
|
1759
1796
|
Object.entries(pendingToolApprovals).filter(
|
|
1760
|
-
([, approval]) => approval && typeof approval === "object" && typeof approval.toolCallId === "string" && !toolCallHasOutput(
|
|
1797
|
+
([, approval]) => approval && typeof approval === "object" && typeof approval.toolCallId === "string" && !toolCallHasOutput(normalizedMessage.content.parts, approval.toolCallId)
|
|
1761
1798
|
)
|
|
1762
1799
|
);
|
|
1763
|
-
const { pendingToolApprovals: _omit, ...restMetadata } =
|
|
1800
|
+
const { pendingToolApprovals: _omit, ...restMetadata } = normalizedMetadata;
|
|
1764
1801
|
const hasStillPending = Object.keys(stillPending).length > 0;
|
|
1765
1802
|
return {
|
|
1766
|
-
...
|
|
1803
|
+
...normalizedMessage,
|
|
1767
1804
|
content: {
|
|
1768
|
-
...
|
|
1805
|
+
...normalizedMessage.content,
|
|
1769
1806
|
metadata: {
|
|
1770
1807
|
...restMetadata,
|
|
1771
1808
|
mode: "stream",
|
|
@@ -2092,7 +2129,8 @@ var useChat = ({
|
|
|
2092
2129
|
signal,
|
|
2093
2130
|
tracingOptions,
|
|
2094
2131
|
clientTools,
|
|
2095
|
-
signalId
|
|
2132
|
+
signalId,
|
|
2133
|
+
clientMessageId
|
|
2096
2134
|
}) => {
|
|
2097
2135
|
const {
|
|
2098
2136
|
frequencyPenalty,
|
|
@@ -2203,7 +2241,7 @@ var useChat = ({
|
|
|
2203
2241
|
};
|
|
2204
2242
|
try {
|
|
2205
2243
|
const result = await agent.sendMessage({
|
|
2206
|
-
message: messageContents,
|
|
2244
|
+
message: clientMessageId ? { contents: messageContents, metadata: { [CLIENT_MESSAGE_ID_KEY]: clientMessageId } } : messageContents,
|
|
2207
2245
|
resourceId: resourceId || agentId,
|
|
2208
2246
|
threadId: threadId2,
|
|
2209
2247
|
ifIdle: {
|
|
@@ -2238,7 +2276,7 @@ var useChat = ({
|
|
|
2238
2276
|
onSignalEcho?.(resolvedSignalId);
|
|
2239
2277
|
if (isThreadSignalUnsupportedError(signalError)) {
|
|
2240
2278
|
markThreadSignalsUnsupported();
|
|
2241
|
-
setMessages((prev) => [...prev,
|
|
2279
|
+
setMessages((prev) => [...prev, fromCoreUserMessagesToMastraDBMessage(coreUserMessages)]);
|
|
2242
2280
|
await streamWithLegacyRoute();
|
|
2243
2281
|
return;
|
|
2244
2282
|
}
|
|
@@ -2499,15 +2537,26 @@ var useChat = ({
|
|
|
2499
2537
|
if (args.coreUserMessages) {
|
|
2500
2538
|
coreUserMessages.push(...args.coreUserMessages);
|
|
2501
2539
|
}
|
|
2502
|
-
const
|
|
2503
|
-
const
|
|
2504
|
-
|
|
2505
|
-
|
|
2540
|
+
const dbUserMessage = fromCoreUserMessagesToMastraDBMessage(coreUserMessages);
|
|
2541
|
+
const clientSetId = mode === "stream" && args.threadId && !_threadSignalsUnsupportedRef.current && !threadSignalsDisabled ? `client-set-${uuid.v4()}` : void 0;
|
|
2542
|
+
const signalId = clientSetId;
|
|
2543
|
+
const clientMessageId = clientSetId;
|
|
2544
|
+
if (signalId) {
|
|
2545
|
+
const metadata = {
|
|
2546
|
+
...dbUserMessage.content.metadata,
|
|
2547
|
+
mode: "stream",
|
|
2548
|
+
status: "pending",
|
|
2549
|
+
[CLIENT_MESSAGE_ID_KEY]: clientMessageId
|
|
2550
|
+
};
|
|
2551
|
+
const pendingMessage = { ...dbUserMessage, id: clientSetId, content: { ...dbUserMessage.content, metadata } };
|
|
2552
|
+
setMessages((s) => [...s, pendingMessage]);
|
|
2553
|
+
} else {
|
|
2554
|
+
setMessages((s) => [...s, dbUserMessage]);
|
|
2506
2555
|
}
|
|
2507
2556
|
if (mode === "generate") {
|
|
2508
2557
|
await generate({ ...args, coreUserMessages });
|
|
2509
2558
|
} else if (mode === "stream") {
|
|
2510
|
-
await stream({ ...args, coreUserMessages, signalId });
|
|
2559
|
+
await stream({ ...args, coreUserMessages, signalId, clientMessageId });
|
|
2511
2560
|
} else if (mode === "network") {
|
|
2512
2561
|
await network({ ...args, coreUserMessages });
|
|
2513
2562
|
}
|
|
@@ -2926,6 +2975,9 @@ var MessageFactoryComponent = ({ message, roles, status, fallback, ...renderers
|
|
|
2926
2975
|
content = status.Error({ text: joinText(parts), message });
|
|
2927
2976
|
} else {
|
|
2928
2977
|
content = /* @__PURE__ */ jsxRuntime.jsx(jsxRuntime.Fragment, { children: parts.map((part, index) => /* @__PURE__ */ jsxRuntime.jsx(PartRenderer, { part, renderers, fallback }, getPartKey(part, index))) });
|
|
2978
|
+
if (metadata?.status === "pending" && status?.Pending) {
|
|
2979
|
+
content = status.Pending({ children: content, text: joinText(parts), message });
|
|
2980
|
+
}
|
|
2929
2981
|
const verdict = resolveTaskVerdict(metadata);
|
|
2930
2982
|
if (verdict && status?.Task) {
|
|
2931
2983
|
content = /* @__PURE__ */ jsxRuntime.jsxs(jsxRuntime.Fragment, { children: [
|
|
@@ -3352,7 +3404,235 @@ function useCancelWorkflowRun() {
|
|
|
3352
3404
|
});
|
|
3353
3405
|
}
|
|
3354
3406
|
|
|
3407
|
+
// src/voice/record-mic-to-file.ts
|
|
3408
|
+
async function recordMicrophoneToFile(onFinish) {
|
|
3409
|
+
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
3410
|
+
const mediaRecorder = new MediaRecorder(stream);
|
|
3411
|
+
let chunks = [];
|
|
3412
|
+
mediaRecorder.ondataavailable = (e) => {
|
|
3413
|
+
chunks.push(e.data);
|
|
3414
|
+
};
|
|
3415
|
+
mediaRecorder.onstop = () => {
|
|
3416
|
+
const blob = new Blob(chunks, { type: "audio/webm" });
|
|
3417
|
+
const file = new File([blob], `recording-${Date.now()}.webm`, {
|
|
3418
|
+
type: "audio/webm",
|
|
3419
|
+
lastModified: Date.now()
|
|
3420
|
+
});
|
|
3421
|
+
stream.getTracks().forEach((track) => track.stop());
|
|
3422
|
+
onFinish(file);
|
|
3423
|
+
};
|
|
3424
|
+
return mediaRecorder;
|
|
3425
|
+
}
|
|
3426
|
+
|
|
3427
|
+
// src/voice/play-stream-with-web-audio.ts
|
|
3428
|
+
async function playStreamWithWebAudio(stream, onEnded) {
|
|
3429
|
+
const audioContext = new window.AudioContext();
|
|
3430
|
+
const reader = stream.getReader();
|
|
3431
|
+
const chunks = [];
|
|
3432
|
+
try {
|
|
3433
|
+
while (true) {
|
|
3434
|
+
const { done, value } = await reader.read();
|
|
3435
|
+
if (done) break;
|
|
3436
|
+
chunks.push(value);
|
|
3437
|
+
}
|
|
3438
|
+
const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
|
|
3439
|
+
const combinedBuffer = new Uint8Array(totalLength);
|
|
3440
|
+
let offset = 0;
|
|
3441
|
+
for (const chunk of chunks) {
|
|
3442
|
+
combinedBuffer.set(chunk, offset);
|
|
3443
|
+
offset += chunk.length;
|
|
3444
|
+
}
|
|
3445
|
+
const audioBuffer = await audioContext.decodeAudioData(combinedBuffer.buffer);
|
|
3446
|
+
const source = audioContext.createBufferSource();
|
|
3447
|
+
source.buffer = audioBuffer;
|
|
3448
|
+
source.onended = onEnded ?? null;
|
|
3449
|
+
source.connect(audioContext.destination);
|
|
3450
|
+
source.start();
|
|
3451
|
+
return () => {
|
|
3452
|
+
source.onended = null;
|
|
3453
|
+
source.stop();
|
|
3454
|
+
void audioContext.close();
|
|
3455
|
+
};
|
|
3456
|
+
} catch (error) {
|
|
3457
|
+
await reader.cancel().catch(() => void 0);
|
|
3458
|
+
await audioContext.close().catch(() => void 0);
|
|
3459
|
+
throw error;
|
|
3460
|
+
} finally {
|
|
3461
|
+
reader.releaseLock();
|
|
3462
|
+
}
|
|
3463
|
+
}
|
|
3464
|
+
var useSpeechRecognition = ({
|
|
3465
|
+
language = "en-US",
|
|
3466
|
+
agentId,
|
|
3467
|
+
requestContext
|
|
3468
|
+
}) => {
|
|
3469
|
+
const client = useMastraClient();
|
|
3470
|
+
const [agent, setAgent] = react.useState(null);
|
|
3471
|
+
react.useEffect(() => {
|
|
3472
|
+
let cancelled = false;
|
|
3473
|
+
if (!agentId) {
|
|
3474
|
+
setAgent(null);
|
|
3475
|
+
return () => {
|
|
3476
|
+
cancelled = true;
|
|
3477
|
+
};
|
|
3478
|
+
}
|
|
3479
|
+
const agent2 = client.getAgent(agentId);
|
|
3480
|
+
const check = async () => {
|
|
3481
|
+
try {
|
|
3482
|
+
const speakers = await agent2.voice.getSpeakers(requestContext);
|
|
3483
|
+
if (!cancelled) {
|
|
3484
|
+
setAgent(speakers.length > 0 ? agent2 : null);
|
|
3485
|
+
}
|
|
3486
|
+
} catch {
|
|
3487
|
+
if (!cancelled) {
|
|
3488
|
+
setAgent(null);
|
|
3489
|
+
}
|
|
3490
|
+
}
|
|
3491
|
+
};
|
|
3492
|
+
void check();
|
|
3493
|
+
return () => {
|
|
3494
|
+
cancelled = true;
|
|
3495
|
+
};
|
|
3496
|
+
}, [agentId, client, requestContext]);
|
|
3497
|
+
const browserSpeechRecognition = useBrowserSpeechRecognition({ language });
|
|
3498
|
+
const mastraSpeechRecognition = useMastraSpeechToText({ agent, language });
|
|
3499
|
+
if (!agent) {
|
|
3500
|
+
return browserSpeechRecognition;
|
|
3501
|
+
}
|
|
3502
|
+
return mastraSpeechRecognition;
|
|
3503
|
+
};
|
|
3504
|
+
var useBrowserSpeechRecognition = ({ language = "en-US" }) => {
|
|
3505
|
+
const speechRecognitionRef = react.useRef(null);
|
|
3506
|
+
const [state, setState] = react.useState({
|
|
3507
|
+
isListening: false,
|
|
3508
|
+
transcript: "",
|
|
3509
|
+
error: null
|
|
3510
|
+
});
|
|
3511
|
+
const start = () => {
|
|
3512
|
+
if (!speechRecognitionRef.current) return;
|
|
3513
|
+
speechRecognitionRef.current.start();
|
|
3514
|
+
};
|
|
3515
|
+
const stop = () => {
|
|
3516
|
+
if (!speechRecognitionRef.current) return;
|
|
3517
|
+
speechRecognitionRef.current.stop();
|
|
3518
|
+
};
|
|
3519
|
+
react.useEffect(() => {
|
|
3520
|
+
if (!("webkitSpeechRecognition" in window) && !("SpeechRecognition" in window)) {
|
|
3521
|
+
setState((prev) => ({ ...prev, error: "Speech Recognition not supported in this browser" }));
|
|
3522
|
+
return;
|
|
3523
|
+
}
|
|
3524
|
+
const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition;
|
|
3525
|
+
const recognition = new SpeechRecognition();
|
|
3526
|
+
speechRecognitionRef.current = recognition;
|
|
3527
|
+
recognition.continuous = true;
|
|
3528
|
+
recognition.lang = language;
|
|
3529
|
+
recognition.onstart = () => {
|
|
3530
|
+
setState((prev) => ({ ...prev, isListening: true, error: null }));
|
|
3531
|
+
};
|
|
3532
|
+
recognition.onresult = (event) => {
|
|
3533
|
+
let finalTranscript = "";
|
|
3534
|
+
for (let i = event.resultIndex; i < event.results.length; i++) {
|
|
3535
|
+
const transcript = event.results[i][0].transcript;
|
|
3536
|
+
if (event.results[i].isFinal) {
|
|
3537
|
+
finalTranscript += transcript + " ";
|
|
3538
|
+
}
|
|
3539
|
+
}
|
|
3540
|
+
setState((prev) => ({ ...prev, transcript: finalTranscript }));
|
|
3541
|
+
};
|
|
3542
|
+
recognition.onerror = (event) => {
|
|
3543
|
+
setState((prev) => ({ ...prev, error: `Error: ${event.error}` }));
|
|
3544
|
+
};
|
|
3545
|
+
recognition.onend = () => setState((prev) => ({ ...prev, isListening: false }));
|
|
3546
|
+
return () => {
|
|
3547
|
+
try {
|
|
3548
|
+
recognition.stop();
|
|
3549
|
+
} catch {
|
|
3550
|
+
}
|
|
3551
|
+
recognition.onstart = null;
|
|
3552
|
+
recognition.onresult = null;
|
|
3553
|
+
recognition.onerror = null;
|
|
3554
|
+
recognition.onend = null;
|
|
3555
|
+
speechRecognitionRef.current = null;
|
|
3556
|
+
};
|
|
3557
|
+
}, [language]);
|
|
3558
|
+
return {
|
|
3559
|
+
...state,
|
|
3560
|
+
start,
|
|
3561
|
+
stop
|
|
3562
|
+
};
|
|
3563
|
+
};
|
|
3564
|
+
var useMastraSpeechToText = ({
|
|
3565
|
+
agent,
|
|
3566
|
+
language
|
|
3567
|
+
}) => {
|
|
3568
|
+
const [state, setState] = react.useState({
|
|
3569
|
+
isListening: false,
|
|
3570
|
+
transcript: "",
|
|
3571
|
+
error: null
|
|
3572
|
+
});
|
|
3573
|
+
const recorderRef = react.useRef(null);
|
|
3574
|
+
const sessionRef = react.useRef(0);
|
|
3575
|
+
const startInFlightRef = react.useRef(false);
|
|
3576
|
+
react.useEffect(() => {
|
|
3577
|
+
return () => {
|
|
3578
|
+
sessionRef.current += 1;
|
|
3579
|
+
startInFlightRef.current = false;
|
|
3580
|
+
recorderRef.current?.stop();
|
|
3581
|
+
recorderRef.current = null;
|
|
3582
|
+
};
|
|
3583
|
+
}, [agent]);
|
|
3584
|
+
const handleFinish = (session) => (file) => {
|
|
3585
|
+
if (!agent || session !== sessionRef.current) return;
|
|
3586
|
+
recorderRef.current = null;
|
|
3587
|
+
setState((prev) => ({ ...prev, isListening: false }));
|
|
3588
|
+
void agent.voice.listen(file, { language }).then((res) => {
|
|
3589
|
+
if (session !== sessionRef.current) return;
|
|
3590
|
+
setState((prev) => ({ ...prev, transcript: res.text, error: null }));
|
|
3591
|
+
}).catch((error) => {
|
|
3592
|
+
if (session !== sessionRef.current) return;
|
|
3593
|
+
const message = error instanceof Error ? error.message : "Failed to transcribe speech";
|
|
3594
|
+
setState((prev) => ({ ...prev, error: message }));
|
|
3595
|
+
});
|
|
3596
|
+
};
|
|
3597
|
+
const start = () => {
|
|
3598
|
+
if (!agent || startInFlightRef.current || recorderRef.current) return;
|
|
3599
|
+
startInFlightRef.current = true;
|
|
3600
|
+
const session = sessionRef.current;
|
|
3601
|
+
void recordMicrophoneToFile(handleFinish(session)).then((recorder) => {
|
|
3602
|
+
startInFlightRef.current = false;
|
|
3603
|
+
if (session !== sessionRef.current) {
|
|
3604
|
+
try {
|
|
3605
|
+
recorder.stop();
|
|
3606
|
+
} catch {
|
|
3607
|
+
}
|
|
3608
|
+
return;
|
|
3609
|
+
}
|
|
3610
|
+
recorderRef.current = recorder;
|
|
3611
|
+
setState((prev) => ({ ...prev, isListening: true, error: null }));
|
|
3612
|
+
recorder.start();
|
|
3613
|
+
}).catch((error) => {
|
|
3614
|
+
startInFlightRef.current = false;
|
|
3615
|
+
if (session !== sessionRef.current) return;
|
|
3616
|
+
const message = error instanceof Error ? error.message : "Failed to start speech recording";
|
|
3617
|
+
setState((prev) => ({ ...prev, isListening: false, error: message }));
|
|
3618
|
+
});
|
|
3619
|
+
};
|
|
3620
|
+
const stop = () => {
|
|
3621
|
+
sessionRef.current += 1;
|
|
3622
|
+
startInFlightRef.current = false;
|
|
3623
|
+
recorderRef.current?.stop();
|
|
3624
|
+
recorderRef.current = null;
|
|
3625
|
+
setState((prev) => ({ ...prev, isListening: false }));
|
|
3626
|
+
};
|
|
3627
|
+
return {
|
|
3628
|
+
...state,
|
|
3629
|
+
start,
|
|
3630
|
+
stop
|
|
3631
|
+
};
|
|
3632
|
+
};
|
|
3633
|
+
|
|
3355
3634
|
exports.AgentIcon = AgentIcon;
|
|
3635
|
+
exports.CLIENT_MESSAGE_ID_KEY = CLIENT_MESSAGE_ID_KEY;
|
|
3356
3636
|
exports.CodeBlock = CodeBlock;
|
|
3357
3637
|
exports.CodeBlockClass = CodeBlockClass;
|
|
3358
3638
|
exports.CodeCopyButton = CodeCopyButton;
|
|
@@ -3411,12 +3691,16 @@ exports.accumulateChunk = accumulateChunk;
|
|
|
3411
3691
|
exports.accumulateNetworkChunk = accumulateNetworkChunk;
|
|
3412
3692
|
exports.finishStreamingAssistantMessage = finishStreamingAssistantMessage;
|
|
3413
3693
|
exports.fromCoreUserMessageToMastraDBMessage = fromCoreUserMessageToMastraDBMessage;
|
|
3694
|
+
exports.fromCoreUserMessagesToMastraDBMessage = fromCoreUserMessagesToMastraDBMessage;
|
|
3414
3695
|
exports.mapWorkflowStreamChunkToWatchResult = mapWorkflowStreamChunkToWatchResult;
|
|
3696
|
+
exports.playStreamWithWebAudio = playStreamWithWebAudio;
|
|
3697
|
+
exports.recordMicrophoneToFile = recordMicrophoneToFile;
|
|
3415
3698
|
exports.useCancelWorkflowRun = useCancelWorkflowRun;
|
|
3416
3699
|
exports.useChat = useChat;
|
|
3417
3700
|
exports.useCreateWorkflowRun = useCreateWorkflowRun;
|
|
3418
3701
|
exports.useEntity = useEntity;
|
|
3419
3702
|
exports.useMastraClient = useMastraClient;
|
|
3703
|
+
exports.useSpeechRecognition = useSpeechRecognition;
|
|
3420
3704
|
exports.useStreamWorkflow = useStreamWorkflow;
|
|
3421
3705
|
//# sourceMappingURL=index.cjs.map
|
|
3422
3706
|
//# sourceMappingURL=index.cjs.map
|