@mastra/voice-aws-nova-sonic 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/_types/@internal_voice/dist/_types/@internal_ai-sdk-v5/dist/index.d.ts +110 -221
- package/dist/_types/@internal_voice/dist/_types/@internal_core/dist/base/index.d.ts +27 -27
- package/dist/_types/@internal_voice/dist/_types/@internal_core/dist/index-S1lgaKO7.d.ts +218 -0
- package/dist/_types/@internal_voice/dist/_types/@internal_core/dist/request-context/index.d.ts +122 -95
- package/dist/_types/@internal_voice/dist/_types/@internal_core/dist/types/index.d.ts +4 -2
- package/dist/docs/SKILL.md +6 -6
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/docs/references/{docs-voice-overview.md → guides-voice-overview.md} +90 -134
- package/dist/docs/references/{docs-voice-speech-to-speech.md → guides-voice-speech-to-speech.md} +90 -8
- package/dist/docs/references/{reference-voice-aws-nova-sonic.md → integrations-voice-aws-nova-sonic.md} +4 -2
- package/dist/index.cjs +1679 -1860
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1678 -1858
- package/dist/index.js.map +1 -1
- package/package.json +17 -16
- package/dist/_types/@internal_voice/dist/_types/@internal_core/dist/logger/index.d.ts +0 -217
|
@@ -1,10 +1,12 @@
|
|
|
1
|
+
> Discover all available pages from the documentation index: https://mastra.ai/llms.txt
|
|
2
|
+
|
|
1
3
|
# Voice in Mastra
|
|
2
4
|
|
|
3
5
|
Mastra's Voice system provides a unified interface for voice interactions, enabling text-to-speech (TTS), speech-to-text (STT), and real-time speech-to-speech (STS) capabilities in your applications.
|
|
4
6
|
|
|
5
|
-
##
|
|
7
|
+
## Add voice to agents
|
|
6
8
|
|
|
7
|
-
|
|
9
|
+
Pass a voice provider to an agent with the `voice` property. The same property supports text-to-speech (TTS), speech-to-text (STT), and real-time speech-to-speech (STS), depending on the provider you configure.
|
|
8
10
|
|
|
9
11
|
```typescript
|
|
10
12
|
import { Agent } from '@mastra/core/agent'
|
|
@@ -16,7 +18,7 @@ const voiceAgent = new Agent({
|
|
|
16
18
|
id: 'voice-agent',
|
|
17
19
|
name: 'Voice Agent',
|
|
18
20
|
instructions: 'You are a voice assistant that can help users with their tasks.',
|
|
19
|
-
model: 'openai/gpt-5.
|
|
21
|
+
model: 'openai/gpt-5.6-sol',
|
|
20
22
|
voice: new OpenAIVoice(),
|
|
21
23
|
})
|
|
22
24
|
```
|
|
@@ -27,7 +29,7 @@ You can then use the following voice capabilities:
|
|
|
27
29
|
|
|
28
30
|
Turn your agent's responses into natural-sounding speech using Mastra's TTS capabilities. Choose from multiple providers like OpenAI, ElevenLabs, and more.
|
|
29
31
|
|
|
30
|
-
For detailed configuration options and advanced features, check out our [Text-to-Speech guide](https://mastra.ai/
|
|
32
|
+
For detailed configuration options and advanced features, check out our [Text-to-Speech guide](https://mastra.ai/guides/voice/text-to-speech).
|
|
31
33
|
|
|
32
34
|
**OpenAI**:
|
|
33
35
|
|
|
@@ -40,7 +42,7 @@ const voiceAgent = new Agent({
|
|
|
40
42
|
id: 'voice-agent',
|
|
41
43
|
name: 'Voice Agent',
|
|
42
44
|
instructions: 'You are a voice assistant that can help users with their tasks.',
|
|
43
|
-
model: 'openai/gpt-5.
|
|
45
|
+
model: 'openai/gpt-5.6-sol',
|
|
44
46
|
voice: new OpenAIVoice(),
|
|
45
47
|
})
|
|
46
48
|
|
|
@@ -55,7 +57,7 @@ const audioStream = await voiceAgent.voice.speak(text, {
|
|
|
55
57
|
playAudio(audioStream)
|
|
56
58
|
```
|
|
57
59
|
|
|
58
|
-
Visit the [OpenAI Voice Reference](https://mastra.ai/
|
|
60
|
+
Visit the [OpenAI Voice Reference](https://mastra.ai/integrations/voice/openai) for more information on the OpenAI voice provider.
|
|
59
61
|
|
|
60
62
|
**Azure**:
|
|
61
63
|
|
|
@@ -68,7 +70,7 @@ const voiceAgent = new Agent({
|
|
|
68
70
|
id: 'voice-agent',
|
|
69
71
|
name: 'Voice Agent',
|
|
70
72
|
instructions: 'You are a voice assistant that can help users with their tasks.',
|
|
71
|
-
model: 'openai/gpt-5.
|
|
73
|
+
model: 'openai/gpt-5.6-sol',
|
|
72
74
|
voice: new AzureVoice(),
|
|
73
75
|
})
|
|
74
76
|
|
|
@@ -82,7 +84,7 @@ const audioStream = await voiceAgent.voice.speak(text, {
|
|
|
82
84
|
playAudio(audioStream)
|
|
83
85
|
```
|
|
84
86
|
|
|
85
|
-
Visit the [Azure Voice Reference](https://mastra.ai/
|
|
87
|
+
Visit the [Azure Voice Reference](https://mastra.ai/integrations/voice/azure) for more information on the Azure voice provider.
|
|
86
88
|
|
|
87
89
|
**ElevenLabs**:
|
|
88
90
|
|
|
@@ -95,7 +97,7 @@ const voiceAgent = new Agent({
|
|
|
95
97
|
id: 'voice-agent',
|
|
96
98
|
name: 'Voice Agent',
|
|
97
99
|
instructions: 'You are a voice assistant that can help users with their tasks.',
|
|
98
|
-
model: 'openai/gpt-5.
|
|
100
|
+
model: 'openai/gpt-5.6-sol',
|
|
99
101
|
voice: new ElevenLabsVoice(),
|
|
100
102
|
})
|
|
101
103
|
|
|
@@ -109,34 +111,7 @@ const audioStream = await voiceAgent.voice.speak(text, {
|
|
|
109
111
|
playAudio(audioStream)
|
|
110
112
|
```
|
|
111
113
|
|
|
112
|
-
Visit the [ElevenLabs Voice Reference](https://mastra.ai/
|
|
113
|
-
|
|
114
|
-
**PlayAI**:
|
|
115
|
-
|
|
116
|
-
```typescript
|
|
117
|
-
import { Agent } from '@mastra/core/agent'
|
|
118
|
-
import { PlayAIVoice } from '@mastra/voice-playai'
|
|
119
|
-
import { playAudio } from '@mastra/node-audio'
|
|
120
|
-
|
|
121
|
-
const voiceAgent = new Agent({
|
|
122
|
-
id: 'voice-agent',
|
|
123
|
-
name: 'Voice Agent',
|
|
124
|
-
instructions: 'You are a voice assistant that can help users with their tasks.',
|
|
125
|
-
model: 'openai/gpt-5.5',
|
|
126
|
-
voice: new PlayAIVoice(),
|
|
127
|
-
})
|
|
128
|
-
|
|
129
|
-
const { text } = await voiceAgent.generate('What color is the sky?')
|
|
130
|
-
|
|
131
|
-
// Convert text to speech to an Audio Stream
|
|
132
|
-
const audioStream = await voiceAgent.voice.speak(text, {
|
|
133
|
-
speaker: 'default', // Optional: specify a speaker
|
|
134
|
-
})
|
|
135
|
-
|
|
136
|
-
playAudio(audioStream)
|
|
137
|
-
```
|
|
138
|
-
|
|
139
|
-
Visit the [PlayAI Voice Reference](https://mastra.ai/reference/voice/playai) for more information on the PlayAI voice provider.
|
|
114
|
+
Visit the [ElevenLabs Voice Reference](https://mastra.ai/integrations/voice/elevenlabs) for more information on the ElevenLabs voice provider.
|
|
140
115
|
|
|
141
116
|
**Google**:
|
|
142
117
|
|
|
@@ -149,7 +124,7 @@ const voiceAgent = new Agent({
|
|
|
149
124
|
id: 'voice-agent',
|
|
150
125
|
name: 'Voice Agent',
|
|
151
126
|
instructions: 'You are a voice assistant that can help users with their tasks.',
|
|
152
|
-
model: 'openai/gpt-5.
|
|
127
|
+
model: 'openai/gpt-5.6-sol',
|
|
153
128
|
voice: new GoogleVoice(),
|
|
154
129
|
})
|
|
155
130
|
|
|
@@ -163,7 +138,7 @@ const audioStream = await voiceAgent.voice.speak(text, {
|
|
|
163
138
|
playAudio(audioStream)
|
|
164
139
|
```
|
|
165
140
|
|
|
166
|
-
Visit the [Google Voice Reference](https://mastra.ai/
|
|
141
|
+
Visit the [Google Voice Reference](https://mastra.ai/integrations/voice/google) for more information on the Google voice provider.
|
|
167
142
|
|
|
168
143
|
**Cloudflare**:
|
|
169
144
|
|
|
@@ -176,7 +151,7 @@ const voiceAgent = new Agent({
|
|
|
176
151
|
id: 'voice-agent',
|
|
177
152
|
name: 'Voice Agent',
|
|
178
153
|
instructions: 'You are a voice assistant that can help users with their tasks.',
|
|
179
|
-
model: 'openai/gpt-5.
|
|
154
|
+
model: 'openai/gpt-5.6-sol',
|
|
180
155
|
voice: new CloudflareVoice(),
|
|
181
156
|
})
|
|
182
157
|
|
|
@@ -190,7 +165,7 @@ const audioStream = await voiceAgent.voice.speak(text, {
|
|
|
190
165
|
playAudio(audioStream)
|
|
191
166
|
```
|
|
192
167
|
|
|
193
|
-
Visit the [Cloudflare Voice Reference](https://mastra.ai/
|
|
168
|
+
Visit the [Cloudflare Voice Reference](https://mastra.ai/integrations/voice/cloudflare) for more information on the Cloudflare voice provider.
|
|
194
169
|
|
|
195
170
|
**Deepgram**:
|
|
196
171
|
|
|
@@ -203,7 +178,7 @@ const voiceAgent = new Agent({
|
|
|
203
178
|
id: 'voice-agent',
|
|
204
179
|
name: 'Voice Agent',
|
|
205
180
|
instructions: 'You are a voice assistant that can help users with their tasks.',
|
|
206
|
-
model: 'openai/gpt-5.
|
|
181
|
+
model: 'openai/gpt-5.6-sol',
|
|
207
182
|
voice: new DeepgramVoice(),
|
|
208
183
|
})
|
|
209
184
|
|
|
@@ -217,7 +192,7 @@ const audioStream = await voiceAgent.voice.speak(text, {
|
|
|
217
192
|
playAudio(audioStream)
|
|
218
193
|
```
|
|
219
194
|
|
|
220
|
-
Visit the [Deepgram Voice Reference](https://mastra.ai/
|
|
195
|
+
Visit the [Deepgram Voice Reference](https://mastra.ai/integrations/voice/deepgram) for more information on the Deepgram voice provider.
|
|
221
196
|
|
|
222
197
|
**Inworld**:
|
|
223
198
|
|
|
@@ -230,7 +205,7 @@ const voiceAgent = new Agent({
|
|
|
230
205
|
id: 'voice-agent',
|
|
231
206
|
name: 'Voice Agent',
|
|
232
207
|
instructions: 'You are a voice assistant that can help users with their tasks.',
|
|
233
|
-
model: 'openai/gpt-5.
|
|
208
|
+
model: 'openai/gpt-5.6-sol',
|
|
234
209
|
voice: new InworldVoice(),
|
|
235
210
|
})
|
|
236
211
|
|
|
@@ -244,7 +219,7 @@ const audioStream = await voiceAgent.voice.speak(text, {
|
|
|
244
219
|
playAudio(audioStream)
|
|
245
220
|
```
|
|
246
221
|
|
|
247
|
-
Visit the [Inworld Voice Reference](https://mastra.ai/
|
|
222
|
+
Visit the [Inworld Voice Reference](https://mastra.ai/integrations/voice/inworld) for more information on the Inworld voice provider.
|
|
248
223
|
|
|
249
224
|
**Speechify**:
|
|
250
225
|
|
|
@@ -257,7 +232,7 @@ const voiceAgent = new Agent({
|
|
|
257
232
|
id: 'voice-agent',
|
|
258
233
|
name: 'Voice Agent',
|
|
259
234
|
instructions: 'You are a voice assistant that can help users with their tasks.',
|
|
260
|
-
model: 'openai/gpt-5.
|
|
235
|
+
model: 'openai/gpt-5.6-sol',
|
|
261
236
|
voice: new SpeechifyVoice(),
|
|
262
237
|
})
|
|
263
238
|
|
|
@@ -271,7 +246,7 @@ const audioStream = await voiceAgent.voice.speak(text, {
|
|
|
271
246
|
playAudio(audioStream)
|
|
272
247
|
```
|
|
273
248
|
|
|
274
|
-
Visit the [Speechify Voice Reference](https://mastra.ai/
|
|
249
|
+
Visit the [Speechify Voice Reference](https://mastra.ai/integrations/voice/speechify) for more information on the Speechify voice provider.
|
|
275
250
|
|
|
276
251
|
**Sarvam**:
|
|
277
252
|
|
|
@@ -284,7 +259,7 @@ const voiceAgent = new Agent({
|
|
|
284
259
|
id: 'voice-agent',
|
|
285
260
|
name: 'Voice Agent',
|
|
286
261
|
instructions: 'You are a voice assistant that can help users with their tasks.',
|
|
287
|
-
model: 'openai/gpt-5.
|
|
262
|
+
model: 'openai/gpt-5.6-sol',
|
|
288
263
|
voice: new SarvamVoice(),
|
|
289
264
|
})
|
|
290
265
|
|
|
@@ -298,7 +273,7 @@ const audioStream = await voiceAgent.voice.speak(text, {
|
|
|
298
273
|
playAudio(audioStream)
|
|
299
274
|
```
|
|
300
275
|
|
|
301
|
-
Visit the [Sarvam Voice Reference](https://mastra.ai/
|
|
276
|
+
Visit the [Sarvam Voice Reference](https://mastra.ai/integrations/voice/sarvam) for more information on the Sarvam voice provider.
|
|
302
277
|
|
|
303
278
|
**Murf**:
|
|
304
279
|
|
|
@@ -311,7 +286,7 @@ const voiceAgent = new Agent({
|
|
|
311
286
|
id: 'voice-agent',
|
|
312
287
|
name: 'Voice Agent',
|
|
313
288
|
instructions: 'You are a voice assistant that can help users with their tasks.',
|
|
314
|
-
model: 'openai/gpt-5.
|
|
289
|
+
model: 'openai/gpt-5.6-sol',
|
|
315
290
|
voice: new MurfVoice(),
|
|
316
291
|
})
|
|
317
292
|
|
|
@@ -325,11 +300,11 @@ const audioStream = await voiceAgent.voice.speak(text, {
|
|
|
325
300
|
playAudio(audioStream)
|
|
326
301
|
```
|
|
327
302
|
|
|
328
|
-
Visit the [Murf Voice Reference](https://mastra.ai/
|
|
303
|
+
Visit the [Murf Voice Reference](https://mastra.ai/integrations/voice/murf) for more information on the Murf voice provider.
|
|
329
304
|
|
|
330
305
|
### Speech to Text (STT)
|
|
331
306
|
|
|
332
|
-
Transcribe spoken content using
|
|
307
|
+
Transcribe spoken content using providers like OpenAI, ElevenLabs, and more. For detailed configuration options and more, check out [Speech to Text](https://mastra.ai/guides/voice/speech-to-text).
|
|
333
308
|
|
|
334
309
|
You can download a sample audio file from [here](https://github.com/mastra-ai/realtime-voice-demo/raw/refs/heads/main/how_can_i_help_you.mp3).
|
|
335
310
|
|
|
@@ -346,7 +321,7 @@ const voiceAgent = new Agent({
|
|
|
346
321
|
id: 'voice-agent',
|
|
347
322
|
name: 'Voice Agent',
|
|
348
323
|
instructions: 'You are a voice assistant that can help users with their tasks.',
|
|
349
|
-
model: 'openai/gpt-5.
|
|
324
|
+
model: 'openai/gpt-5.6-sol',
|
|
350
325
|
voice: new OpenAIVoice(),
|
|
351
326
|
})
|
|
352
327
|
|
|
@@ -361,7 +336,7 @@ console.log(`User said: ${transcript}`)
|
|
|
361
336
|
const { text } = await voiceAgent.generate(transcript)
|
|
362
337
|
```
|
|
363
338
|
|
|
364
|
-
Visit the [OpenAI Voice Reference](https://mastra.ai/
|
|
339
|
+
Visit the [OpenAI Voice Reference](https://mastra.ai/integrations/voice/openai) for more information on the OpenAI voice provider.
|
|
365
340
|
|
|
366
341
|
**Azure**:
|
|
367
342
|
|
|
@@ -375,7 +350,7 @@ const voiceAgent = new Agent({
|
|
|
375
350
|
id: 'voice-agent',
|
|
376
351
|
name: 'Voice Agent',
|
|
377
352
|
instructions: 'You are a voice assistant that can help users with their tasks.',
|
|
378
|
-
model: 'openai/gpt-5.
|
|
353
|
+
model: 'openai/gpt-5.6-sol',
|
|
379
354
|
voice: new AzureVoice(),
|
|
380
355
|
})
|
|
381
356
|
|
|
@@ -390,7 +365,7 @@ console.log(`User said: ${transcript}`)
|
|
|
390
365
|
const { text } = await voiceAgent.generate(transcript)
|
|
391
366
|
```
|
|
392
367
|
|
|
393
|
-
Visit the [Azure Voice Reference](https://mastra.ai/
|
|
368
|
+
Visit the [Azure Voice Reference](https://mastra.ai/integrations/voice/azure) for more information on the Azure voice provider.
|
|
394
369
|
|
|
395
370
|
**ElevenLabs**:
|
|
396
371
|
|
|
@@ -403,7 +378,7 @@ const voiceAgent = new Agent({
|
|
|
403
378
|
id: 'voice-agent',
|
|
404
379
|
name: 'Voice Agent',
|
|
405
380
|
instructions: 'You are a voice assistant that can help users with their tasks.',
|
|
406
|
-
model: 'openai/gpt-5.
|
|
381
|
+
model: 'openai/gpt-5.6-sol',
|
|
407
382
|
voice: new ElevenLabsVoice(),
|
|
408
383
|
})
|
|
409
384
|
|
|
@@ -418,7 +393,7 @@ console.log(`User said: ${transcript}`)
|
|
|
418
393
|
const { text } = await voiceAgent.generate(transcript)
|
|
419
394
|
```
|
|
420
395
|
|
|
421
|
-
Visit the [ElevenLabs Voice Reference](https://mastra.ai/
|
|
396
|
+
Visit the [ElevenLabs Voice Reference](https://mastra.ai/integrations/voice/elevenlabs) for more information on the ElevenLabs voice provider.
|
|
422
397
|
|
|
423
398
|
**Google**:
|
|
424
399
|
|
|
@@ -431,7 +406,7 @@ const voiceAgent = new Agent({
|
|
|
431
406
|
id: 'voice-agent',
|
|
432
407
|
name: 'Voice Agent',
|
|
433
408
|
instructions: 'You are a voice assistant that can help users with their tasks.',
|
|
434
|
-
model: 'openai/gpt-5.
|
|
409
|
+
model: 'openai/gpt-5.6-sol',
|
|
435
410
|
voice: new GoogleVoice(),
|
|
436
411
|
})
|
|
437
412
|
|
|
@@ -446,7 +421,7 @@ console.log(`User said: ${transcript}`)
|
|
|
446
421
|
const { text } = await voiceAgent.generate(transcript)
|
|
447
422
|
```
|
|
448
423
|
|
|
449
|
-
Visit the [Google Voice Reference](https://mastra.ai/
|
|
424
|
+
Visit the [Google Voice Reference](https://mastra.ai/integrations/voice/google) for more information on the Google voice provider.
|
|
450
425
|
|
|
451
426
|
**Cloudflare**:
|
|
452
427
|
|
|
@@ -459,7 +434,7 @@ const voiceAgent = new Agent({
|
|
|
459
434
|
id: 'voice-agent',
|
|
460
435
|
name: 'Voice Agent',
|
|
461
436
|
instructions: 'You are a voice assistant that can help users with their tasks.',
|
|
462
|
-
model: 'openai/gpt-5.
|
|
437
|
+
model: 'openai/gpt-5.6-sol',
|
|
463
438
|
voice: new CloudflareVoice(),
|
|
464
439
|
})
|
|
465
440
|
|
|
@@ -474,7 +449,7 @@ console.log(`User said: ${transcript}`)
|
|
|
474
449
|
const { text } = await voiceAgent.generate(transcript)
|
|
475
450
|
```
|
|
476
451
|
|
|
477
|
-
Visit the [Cloudflare Voice Reference](https://mastra.ai/
|
|
452
|
+
Visit the [Cloudflare Voice Reference](https://mastra.ai/integrations/voice/cloudflare) for more information on the Cloudflare voice provider.
|
|
478
453
|
|
|
479
454
|
**Deepgram**:
|
|
480
455
|
|
|
@@ -487,7 +462,7 @@ const voiceAgent = new Agent({
|
|
|
487
462
|
id: 'voice-agent',
|
|
488
463
|
name: 'Voice Agent',
|
|
489
464
|
instructions: 'You are a voice assistant that can help users with their tasks.',
|
|
490
|
-
model: 'openai/gpt-5.
|
|
465
|
+
model: 'openai/gpt-5.6-sol',
|
|
491
466
|
voice: new DeepgramVoice(),
|
|
492
467
|
})
|
|
493
468
|
|
|
@@ -502,7 +477,7 @@ console.log(`User said: ${transcript}`)
|
|
|
502
477
|
const { text } = await voiceAgent.generate(transcript)
|
|
503
478
|
```
|
|
504
479
|
|
|
505
|
-
Visit the [Deepgram Voice Reference](https://mastra.ai/
|
|
480
|
+
Visit the [Deepgram Voice Reference](https://mastra.ai/integrations/voice/deepgram) for more information on the Deepgram voice provider.
|
|
506
481
|
|
|
507
482
|
**Inworld**:
|
|
508
483
|
|
|
@@ -515,7 +490,7 @@ const voiceAgent = new Agent({
|
|
|
515
490
|
id: 'voice-agent',
|
|
516
491
|
name: 'Voice Agent',
|
|
517
492
|
instructions: 'You are a voice assistant that can help users with their tasks.',
|
|
518
|
-
model: 'openai/gpt-5.
|
|
493
|
+
model: 'openai/gpt-5.6-sol',
|
|
519
494
|
voice: new InworldVoice(),
|
|
520
495
|
})
|
|
521
496
|
|
|
@@ -530,7 +505,7 @@ console.log(`User said: ${transcript}`)
|
|
|
530
505
|
const { text } = await voiceAgent.generate(transcript)
|
|
531
506
|
```
|
|
532
507
|
|
|
533
|
-
Visit the [Inworld Voice Reference](https://mastra.ai/
|
|
508
|
+
Visit the [Inworld Voice Reference](https://mastra.ai/integrations/voice/inworld) for more information on the Inworld voice provider.
|
|
534
509
|
|
|
535
510
|
**Sarvam**:
|
|
536
511
|
|
|
@@ -543,7 +518,7 @@ const voiceAgent = new Agent({
|
|
|
543
518
|
id: 'voice-agent',
|
|
544
519
|
name: 'Voice Agent',
|
|
545
520
|
instructions: 'You are a voice assistant that can help users with their tasks.',
|
|
546
|
-
model: 'openai/gpt-5.
|
|
521
|
+
model: 'openai/gpt-5.6-sol',
|
|
547
522
|
voice: new SarvamVoice(),
|
|
548
523
|
})
|
|
549
524
|
|
|
@@ -558,11 +533,11 @@ console.log(`User said: ${transcript}`)
|
|
|
558
533
|
const { text } = await voiceAgent.generate(transcript)
|
|
559
534
|
```
|
|
560
535
|
|
|
561
|
-
Visit the [Sarvam Voice Reference](https://mastra.ai/
|
|
536
|
+
Visit the [Sarvam Voice Reference](https://mastra.ai/integrations/voice/sarvam) for more information on the Sarvam voice provider.
|
|
562
537
|
|
|
563
538
|
### Speech to Speech (STS)
|
|
564
539
|
|
|
565
|
-
Create conversational experiences with speech-to-speech capabilities. The unified API enables real-time voice interactions between users and AI agents. For detailed configuration options and advanced features, check out [Speech to Speech](https://mastra.ai/
|
|
540
|
+
Create conversational experiences with speech-to-speech capabilities. The unified API enables real-time voice interactions between users and AI agents. For detailed configuration options and advanced features, check out [Speech to Speech](https://mastra.ai/guides/voice/speech-to-speech).
|
|
566
541
|
|
|
567
542
|
**OpenAI**:
|
|
568
543
|
|
|
@@ -575,7 +550,7 @@ const voiceAgent = new Agent({
|
|
|
575
550
|
id: 'voice-agent',
|
|
576
551
|
name: 'Voice Agent',
|
|
577
552
|
instructions: 'You are a voice assistant that can help users with their tasks.',
|
|
578
|
-
model: 'openai/gpt-5.
|
|
553
|
+
model: 'openai/gpt-5.6-sol',
|
|
579
554
|
voice: new OpenAIRealtimeVoice(),
|
|
580
555
|
})
|
|
581
556
|
|
|
@@ -592,7 +567,7 @@ const micStream = getMicrophoneStream()
|
|
|
592
567
|
await voiceAgent.voice.send(micStream)
|
|
593
568
|
```
|
|
594
569
|
|
|
595
|
-
Visit the [OpenAI Voice Reference](https://mastra.ai/
|
|
570
|
+
Visit the [OpenAI Voice Reference](https://mastra.ai/integrations/voice/openai) for more information on the OpenAI voice provider.
|
|
596
571
|
|
|
597
572
|
**Google**:
|
|
598
573
|
|
|
@@ -605,7 +580,7 @@ const voiceAgent = new Agent({
|
|
|
605
580
|
id: 'voice-agent',
|
|
606
581
|
name: 'Voice Agent',
|
|
607
582
|
instructions: 'You are a voice assistant that can help users with their tasks.',
|
|
608
|
-
model: 'openai/gpt-5.
|
|
583
|
+
model: 'openai/gpt-5.6-sol',
|
|
609
584
|
voice: new GeminiLiveVoice({
|
|
610
585
|
// Live API mode
|
|
611
586
|
apiKey: process.env.GOOGLE_API_KEY,
|
|
@@ -641,7 +616,7 @@ const micStream = getMicrophoneStream()
|
|
|
641
616
|
await voiceAgent.voice.send(micStream)
|
|
642
617
|
```
|
|
643
618
|
|
|
644
|
-
Visit the [Google Gemini Live Reference](https://mastra.ai/
|
|
619
|
+
Visit the [Google Gemini Live Reference](https://mastra.ai/integrations/voice/google) for more information on the Google Gemini Live voice provider.
|
|
645
620
|
|
|
646
621
|
**AWS Nova Sonic**:
|
|
647
622
|
|
|
@@ -654,7 +629,7 @@ const voiceAgent = new Agent({
|
|
|
654
629
|
id: 'voice-agent',
|
|
655
630
|
name: 'Voice Agent',
|
|
656
631
|
instructions: 'You are a voice assistant that can help users with their tasks.',
|
|
657
|
-
model: 'openai/gpt-5.
|
|
632
|
+
model: 'openai/gpt-5.6-sol',
|
|
658
633
|
voice: new NovaSonicVoice({
|
|
659
634
|
region: 'us-east-1',
|
|
660
635
|
speaker: 'matthew',
|
|
@@ -684,7 +659,7 @@ const micStream = getMicrophoneStream()
|
|
|
684
659
|
await voiceAgent.voice.send(micStream)
|
|
685
660
|
```
|
|
686
661
|
|
|
687
|
-
Visit the [AWS Nova Sonic Reference](https://mastra.ai/
|
|
662
|
+
Visit the [AWS Nova Sonic Reference](https://mastra.ai/integrations/voice/aws-nova-sonic) for more information on the AWS Nova Sonic voice provider.
|
|
688
663
|
|
|
689
664
|
**Inworld Realtime**:
|
|
690
665
|
|
|
@@ -697,7 +672,7 @@ const voiceAgent = new Agent({
|
|
|
697
672
|
id: 'voice-agent',
|
|
698
673
|
name: 'Voice Agent',
|
|
699
674
|
instructions: 'You are a voice assistant that can help users with their tasks.',
|
|
700
|
-
model: 'openai/gpt-5.
|
|
675
|
+
model: 'openai/gpt-5.6-sol',
|
|
701
676
|
voice: new InworldRealtimeVoice({
|
|
702
677
|
apiKey: process.env.INWORLD_API_KEY,
|
|
703
678
|
model: 'inworld/models/gemma-4-26b-a4b-it',
|
|
@@ -726,7 +701,7 @@ const micStream = getMicrophoneStream()
|
|
|
726
701
|
await voiceAgent.voice.send(micStream)
|
|
727
702
|
```
|
|
728
703
|
|
|
729
|
-
Visit the [Inworld Realtime Reference](https://mastra.ai/
|
|
704
|
+
Visit the [Inworld Realtime Reference](https://mastra.ai/integrations/voice/inworld) for more information on the Inworld Realtime voice provider.
|
|
730
705
|
|
|
731
706
|
**xAI**:
|
|
732
707
|
|
|
@@ -769,7 +744,11 @@ const micStream = getMicrophoneStream()
|
|
|
769
744
|
await voiceAgent.voice.send(micStream)
|
|
770
745
|
```
|
|
771
746
|
|
|
772
|
-
Visit the [xAI Realtime Voice Reference](https://mastra.ai/
|
|
747
|
+
Visit the [xAI Realtime Voice Reference](https://mastra.ai/integrations/voice/xai) for more information on the xAI voice provider.
|
|
748
|
+
|
|
749
|
+
### Realtime voice
|
|
750
|
+
|
|
751
|
+
Run live calls a user can talk over, in the browser or over the phone. Mastra hands the audio loop to LiveKit, which covers voice activity detection, semantic turn detection, and barge-in, while your agent generates each reply with its own model, tools, and memory. For setup and configuration options, check out [Realtime voice](https://mastra.ai/integrations/voice/livekit).
|
|
773
752
|
|
|
774
753
|
## Voice configuration
|
|
775
754
|
|
|
@@ -796,7 +775,7 @@ const voice = new OpenAIVoice({
|
|
|
796
775
|
})
|
|
797
776
|
```
|
|
798
777
|
|
|
799
|
-
Visit the [OpenAI Voice Reference](https://mastra.ai/
|
|
778
|
+
Visit the [OpenAI Voice Reference](https://mastra.ai/integrations/voice/openai) for more information on the OpenAI voice provider.
|
|
800
779
|
|
|
801
780
|
**Azure**:
|
|
802
781
|
|
|
@@ -821,7 +800,7 @@ const voice = new AzureVoice({
|
|
|
821
800
|
})
|
|
822
801
|
```
|
|
823
802
|
|
|
824
|
-
Visit the [Azure Voice Reference](https://mastra.ai/
|
|
803
|
+
Visit the [Azure Voice Reference](https://mastra.ai/integrations/voice/azure) for more information on the Azure voice provider.
|
|
825
804
|
|
|
826
805
|
**ElevenLabs**:
|
|
827
806
|
|
|
@@ -839,25 +818,7 @@ const voice = new ElevenLabsVoice({
|
|
|
839
818
|
})
|
|
840
819
|
```
|
|
841
820
|
|
|
842
|
-
Visit the [ElevenLabs Voice Reference](https://mastra.ai/
|
|
843
|
-
|
|
844
|
-
**PlayAI**:
|
|
845
|
-
|
|
846
|
-
```typescript
|
|
847
|
-
// PlayAI Voice Configuration
|
|
848
|
-
const voice = new PlayAIVoice({
|
|
849
|
-
speechModel: {
|
|
850
|
-
name: 'playai-voice', // Example model name
|
|
851
|
-
speaker: 'emma', // Example speaker name
|
|
852
|
-
apiKey: process.env.PLAYAI_API_KEY,
|
|
853
|
-
language: 'en-US', // Language code
|
|
854
|
-
speed: 1.0, // Speech speed
|
|
855
|
-
},
|
|
856
|
-
// PlayAI may not have a separate listening model
|
|
857
|
-
})
|
|
858
|
-
```
|
|
859
|
-
|
|
860
|
-
Visit the [PlayAI Voice Reference](https://mastra.ai/reference/voice/playai) for more information on the PlayAI voice provider.
|
|
821
|
+
Visit the [ElevenLabs Voice Reference](https://mastra.ai/integrations/voice/elevenlabs) for more information on the ElevenLabs voice provider.
|
|
861
822
|
|
|
862
823
|
**Google**:
|
|
863
824
|
|
|
@@ -878,7 +839,7 @@ const voice = new GoogleVoice({
|
|
|
878
839
|
})
|
|
879
840
|
```
|
|
880
841
|
|
|
881
|
-
Visit the [Google Voice Reference](https://mastra.ai/
|
|
842
|
+
Visit the [Google Voice Reference](https://mastra.ai/integrations/voice/google) for more information on the Google voice provider.
|
|
882
843
|
|
|
883
844
|
**Cloudflare**:
|
|
884
845
|
|
|
@@ -896,7 +857,7 @@ const voice = new CloudflareVoice({
|
|
|
896
857
|
})
|
|
897
858
|
```
|
|
898
859
|
|
|
899
|
-
Visit the [Cloudflare Voice Reference](https://mastra.ai/
|
|
860
|
+
Visit the [Cloudflare Voice Reference](https://mastra.ai/integrations/voice/cloudflare) for more information on the Cloudflare voice provider.
|
|
900
861
|
|
|
901
862
|
**Deepgram**:
|
|
902
863
|
|
|
@@ -917,7 +878,7 @@ const voice = new DeepgramVoice({
|
|
|
917
878
|
})
|
|
918
879
|
```
|
|
919
880
|
|
|
920
|
-
Visit the [Deepgram Voice Reference](https://mastra.ai/
|
|
881
|
+
Visit the [Deepgram Voice Reference](https://mastra.ai/integrations/voice/deepgram) for more information on the Deepgram voice provider.
|
|
921
882
|
|
|
922
883
|
**Inworld**:
|
|
923
884
|
|
|
@@ -945,7 +906,7 @@ const audioStream = await voice.speak('Hello!', {
|
|
|
945
906
|
})
|
|
946
907
|
```
|
|
947
908
|
|
|
948
|
-
Visit the [Inworld Voice Reference](https://mastra.ai/
|
|
909
|
+
Visit the [Inworld Voice Reference](https://mastra.ai/integrations/voice/inworld) for more information on the Inworld voice provider.
|
|
949
910
|
|
|
950
911
|
**Speechify**:
|
|
951
912
|
|
|
@@ -963,7 +924,7 @@ const voice = new SpeechifyVoice({
|
|
|
963
924
|
})
|
|
964
925
|
```
|
|
965
926
|
|
|
966
|
-
Visit the [Speechify Voice Reference](https://mastra.ai/
|
|
927
|
+
Visit the [Speechify Voice Reference](https://mastra.ai/integrations/voice/speechify) for more information on the Speechify voice provider.
|
|
967
928
|
|
|
968
929
|
**Sarvam**:
|
|
969
930
|
|
|
@@ -983,7 +944,7 @@ const voice = new SarvamVoice({
|
|
|
983
944
|
})
|
|
984
945
|
```
|
|
985
946
|
|
|
986
|
-
Visit the [Sarvam Voice Reference](https://mastra.ai/
|
|
947
|
+
Visit the [Sarvam Voice Reference](https://mastra.ai/integrations/voice/sarvam) for more information on the Sarvam voice provider.
|
|
987
948
|
|
|
988
949
|
**Murf**:
|
|
989
950
|
|
|
@@ -1000,7 +961,7 @@ const voice = new MurfVoice({
|
|
|
1000
961
|
})
|
|
1001
962
|
```
|
|
1002
963
|
|
|
1003
|
-
Visit the [Murf Voice Reference](https://mastra.ai/
|
|
964
|
+
Visit the [Murf Voice Reference](https://mastra.ai/integrations/voice/murf) for more information on the Murf voice provider.
|
|
1004
965
|
|
|
1005
966
|
**OpenAI Realtime**:
|
|
1006
967
|
|
|
@@ -1021,7 +982,7 @@ const voice = new OpenAIRealtimeVoice({
|
|
|
1021
982
|
})
|
|
1022
983
|
```
|
|
1023
984
|
|
|
1024
|
-
For more information on the OpenAI Realtime voice provider, refer to the [OpenAI Realtime Voice Reference](https://mastra.ai/
|
|
985
|
+
For more information on the OpenAI Realtime voice provider, refer to the [OpenAI Realtime Voice Reference](https://mastra.ai/integrations/voice/openai).
|
|
1025
986
|
|
|
1026
987
|
**xAI Realtime**:
|
|
1027
988
|
|
|
@@ -1053,7 +1014,7 @@ const voice = new XAIRealtimeVoice({
|
|
|
1053
1014
|
})
|
|
1054
1015
|
```
|
|
1055
1016
|
|
|
1056
|
-
Visit the [xAI Realtime Voice Reference](https://mastra.ai/
|
|
1017
|
+
Visit the [xAI Realtime Voice Reference](https://mastra.ai/integrations/voice/xai) for more information on the xAI realtime voice provider.
|
|
1057
1018
|
|
|
1058
1019
|
**Google Gemini Live**:
|
|
1059
1020
|
|
|
@@ -1069,7 +1030,7 @@ const voice = new GeminiLiveVoice({
|
|
|
1069
1030
|
})
|
|
1070
1031
|
```
|
|
1071
1032
|
|
|
1072
|
-
Visit the [Google Gemini Live Reference](https://mastra.ai/
|
|
1033
|
+
Visit the [Google Gemini Live Reference](https://mastra.ai/integrations/voice/google) for more information on the Google Gemini Live voice provider.
|
|
1073
1034
|
|
|
1074
1035
|
**AWS Nova Sonic**:
|
|
1075
1036
|
|
|
@@ -1091,7 +1052,7 @@ const voice = new NovaSonicVoice({
|
|
|
1091
1052
|
})
|
|
1092
1053
|
```
|
|
1093
1054
|
|
|
1094
|
-
Visit the [AWS Nova Sonic Reference](https://mastra.ai/
|
|
1055
|
+
Visit the [AWS Nova Sonic Reference](https://mastra.ai/integrations/voice/aws-nova-sonic) for more information on the AWS Nova Sonic voice provider.
|
|
1095
1056
|
|
|
1096
1057
|
**Inworld Realtime**:
|
|
1097
1058
|
|
|
@@ -1111,7 +1072,7 @@ const voice = new InworldRealtimeVoice({
|
|
|
1111
1072
|
})
|
|
1112
1073
|
```
|
|
1113
1074
|
|
|
1114
|
-
Visit the [Inworld Realtime Reference](https://mastra.ai/
|
|
1075
|
+
Visit the [Inworld Realtime Reference](https://mastra.ai/integrations/voice/inworld) for more information on the Inworld Realtime voice provider.
|
|
1115
1076
|
|
|
1116
1077
|
**AI SDK**:
|
|
1117
1078
|
|
|
@@ -1132,20 +1093,20 @@ const voiceAgent = new Agent({
|
|
|
1132
1093
|
id: 'aisdk-voice-agent',
|
|
1133
1094
|
name: 'AI SDK Voice Agent',
|
|
1134
1095
|
instructions: 'You are a helpful assistant with voice capabilities.',
|
|
1135
|
-
model: 'openai/gpt-5.
|
|
1096
|
+
model: 'openai/gpt-5.6-sol',
|
|
1136
1097
|
voice,
|
|
1137
1098
|
})
|
|
1138
1099
|
```
|
|
1139
1100
|
|
|
1140
|
-
### Using
|
|
1101
|
+
### Using multiple voice providers
|
|
1141
1102
|
|
|
1142
|
-
This example demonstrates how to create and use two different voice providers in Mastra: OpenAI for speech-to-text (STT) and
|
|
1103
|
+
This example demonstrates how to create and use two different voice providers in Mastra: OpenAI for speech-to-text (STT) and ElevenLabs for text-to-speech (TTS).
|
|
1143
1104
|
|
|
1144
1105
|
Start by creating instances of the voice providers with any necessary configuration.
|
|
1145
1106
|
|
|
1146
1107
|
```typescript
|
|
1147
1108
|
import { OpenAIVoice } from '@mastra/voice-openai'
|
|
1148
|
-
import {
|
|
1109
|
+
import { ElevenLabsVoice } from '@mastra/voice-elevenlabs'
|
|
1149
1110
|
import { CompositeVoice } from '@mastra/core/voice'
|
|
1150
1111
|
import { playAudio, getMicrophoneStream } from '@mastra/node-audio'
|
|
1151
1112
|
|
|
@@ -1157,13 +1118,8 @@ const input = new OpenAIVoice({
|
|
|
1157
1118
|
},
|
|
1158
1119
|
})
|
|
1159
1120
|
|
|
1160
|
-
// Initialize
|
|
1161
|
-
const output = new
|
|
1162
|
-
speechModel: {
|
|
1163
|
-
name: 'playai-voice',
|
|
1164
|
-
apiKey: process.env.PLAYAI_API_KEY,
|
|
1165
|
-
},
|
|
1166
|
-
})
|
|
1121
|
+
// Initialize ElevenLabs voice for TTS
|
|
1122
|
+
const output = new ElevenLabsVoice()
|
|
1167
1123
|
|
|
1168
1124
|
// Combine the providers using CompositeVoice
|
|
1169
1125
|
const voice = new CompositeVoice({
|
|
@@ -1222,12 +1178,12 @@ You can also mix AI SDK models with Mastra providers:
|
|
|
1222
1178
|
|
|
1223
1179
|
```typescript
|
|
1224
1180
|
import { CompositeVoice } from '@mastra/core/voice'
|
|
1225
|
-
import {
|
|
1181
|
+
import { ElevenLabsVoice } from '@mastra/voice-elevenlabs'
|
|
1226
1182
|
import { groq } from '@ai-sdk/groq'
|
|
1227
1183
|
|
|
1228
1184
|
const voice = new CompositeVoice({
|
|
1229
1185
|
input: groq.transcription('whisper-large-v3'), // AI SDK for STT
|
|
1230
|
-
output: new
|
|
1186
|
+
output: new ElevenLabsVoice(), // Mastra provider for TTS
|
|
1231
1187
|
})
|
|
1232
1188
|
```
|
|
1233
1189
|
|
|
@@ -1237,14 +1193,14 @@ For more information on the CompositeVoice, refer to the [CompositeVoice Referen
|
|
|
1237
1193
|
|
|
1238
1194
|
- [CompositeVoice](https://mastra.ai/reference/voice/composite-voice)
|
|
1239
1195
|
- [MastraVoice](https://mastra.ai/reference/voice/mastra-voice)
|
|
1240
|
-
- [OpenAI Voice](https://mastra.ai/
|
|
1241
|
-
- [OpenAI Realtime Voice](https://mastra.ai/
|
|
1242
|
-
- [xAI Realtime Voice](https://mastra.ai/
|
|
1243
|
-
- [Azure Voice](https://mastra.ai/
|
|
1244
|
-
- [Google Voice](https://mastra.ai/
|
|
1245
|
-
- [Google Gemini Live Voice](https://mastra.ai/
|
|
1246
|
-
- [AWS Nova Sonic Voice](https://mastra.ai/
|
|
1247
|
-
- [Deepgram Voice](https://mastra.ai/
|
|
1248
|
-
- [Inworld Voice](https://mastra.ai/
|
|
1249
|
-
- [
|
|
1196
|
+
- [OpenAI Voice](https://mastra.ai/integrations/voice/openai)
|
|
1197
|
+
- [OpenAI Realtime Voice](https://mastra.ai/integrations/voice/openai)
|
|
1198
|
+
- [xAI Realtime Voice](https://mastra.ai/integrations/voice/xai)
|
|
1199
|
+
- [Azure Voice](https://mastra.ai/integrations/voice/azure)
|
|
1200
|
+
- [Google Voice](https://mastra.ai/integrations/voice/google)
|
|
1201
|
+
- [Google Gemini Live Voice](https://mastra.ai/integrations/voice/google)
|
|
1202
|
+
- [AWS Nova Sonic Voice](https://mastra.ai/integrations/voice/aws-nova-sonic)
|
|
1203
|
+
- [Deepgram Voice](https://mastra.ai/integrations/voice/deepgram)
|
|
1204
|
+
- [Inworld Voice](https://mastra.ai/integrations/voice/inworld)
|
|
1205
|
+
- [LiveKit](https://mastra.ai/integrations/voice/livekit)
|
|
1250
1206
|
- [Voice Examples](https://github.com/mastra-ai/voice-examples)
|