@mindstone/mcp-server-elevenlabs 0.2.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -4
- package/dist/bridge.d.ts +1 -0
- package/dist/client.d.ts +34 -3
- package/dist/client.js +141 -35
- package/dist/endpoints.d.ts +34 -0
- package/dist/endpoints.js +37 -0
- package/dist/error-detail.d.ts +5 -0
- package/dist/error-detail.js +20 -0
- package/dist/server.js +9 -1
- package/dist/tools/account.d.ts +3 -0
- package/dist/tools/account.js +105 -0
- package/dist/tools/alignment.d.ts +3 -0
- package/dist/tools/alignment.js +55 -0
- package/dist/tools/audio-isolation.d.ts +3 -0
- package/dist/tools/audio-isolation.js +51 -0
- package/dist/tools/configure.js +18 -6
- package/dist/tools/dialogue.d.ts +3 -0
- package/dist/tools/dialogue.js +70 -0
- package/dist/tools/dubbing.d.ts +3 -0
- package/dist/tools/dubbing.js +234 -0
- package/dist/tools/file-input.d.ts +28 -0
- package/dist/tools/file-input.js +77 -0
- package/dist/tools/index.d.ts +8 -0
- package/dist/tools/index.js +8 -0
- package/dist/tools/music.js +152 -27
- package/dist/tools/path-safety.d.ts +1 -0
- package/dist/tools/path-safety.js +4 -1
- package/dist/tools/speech.js +90 -34
- package/dist/tools/transcription.js +34 -61
- package/dist/tools/voice-changer.d.ts +3 -0
- package/dist/tools/voice-changer.js +62 -0
- package/dist/tools/voice-clone.d.ts +3 -0
- package/dist/tools/voice-clone.js +104 -0
- package/dist/tools/voice-design.d.ts +3 -0
- package/dist/tools/voice-design.js +153 -0
- package/dist/tools/voices.js +167 -13
- package/dist/types.d.ts +110 -13
- package/dist/types.js +20 -4
- package/dist/untrusted-content.d.ts +45 -0
- package/dist/untrusted-content.js +104 -0
- package/package.json +2 -2
package/dist/tools/music.js
CHANGED
|
@@ -1,24 +1,121 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { getApiKey } from '../auth.js';
|
|
3
3
|
import { elevenLabsJson, elevenLabsAudio } from '../client.js';
|
|
4
|
+
import { ENDPOINTS } from '../endpoints.js';
|
|
4
5
|
import { ElevenLabsError } from '../types.js';
|
|
6
|
+
import { unwrapUntrustedJsonStrings, wrapUntrusted } from '../untrusted-content.js';
|
|
5
7
|
import { withErrorHandling } from '../utils.js';
|
|
6
8
|
const OUTPUT_FORMAT_ENUM = z.enum([
|
|
7
9
|
'mp3_44100_128', 'mp3_44100_192',
|
|
8
10
|
'pcm_16000', 'pcm_22050', 'pcm_24000', 'pcm_44100',
|
|
9
11
|
'ulaw_8000',
|
|
10
12
|
]).optional();
|
|
13
|
+
// Matches lyric section markers on their own line, with optional `]`, to
|
|
14
|
+
// avoid false-positives on prose like `[intro to jazz]`.
|
|
15
|
+
const LYRIC_MARKER_PATTERN = /(^|\n)\s*\[(verse|chorus|bridge|intro|outro|pre[- ]?chorus|hook|refrain)(\s\d+)?\s*\]/i;
|
|
16
|
+
const MIN_TOTAL_MUSIC_MS = 3_000;
|
|
17
|
+
const MAX_TOTAL_MUSIC_MS = 10 * 60 * 1_000; // 10 minutes per the API contract.
|
|
18
|
+
/**
|
|
19
|
+
* Composition section schema — matches the live ElevenLabs API shape.
|
|
20
|
+
* Field names are NOT cosmetic: the API enforces these exact names with
|
|
21
|
+
* `additionalProperties: false`, so renaming any of them breaks generation.
|
|
22
|
+
*
|
|
23
|
+
* `positive_local_styles`, `negative_local_styles`, and `lines` are all
|
|
24
|
+
* required by the API even when empty. We accept them as optional in the
|
|
25
|
+
* input schema but `.default([])` them so a hand-written plan that omits an
|
|
26
|
+
* empty array still passes the API.
|
|
27
|
+
*
|
|
28
|
+
* `.strict()` rejects unknown keys — important because the prior connector
|
|
29
|
+
* version accepted a `{style, lyrics, ...}` shape, and we want to error fast
|
|
30
|
+
* if an LLM agent or older caller sends those (rather than silently strip
|
|
31
|
+
* them and ship a 422 from upstream). See planning doc
|
|
32
|
+
* 260520_elevenlabs_oss_connector_fix.md.
|
|
33
|
+
*/
|
|
34
|
+
const COMPOSITION_SECTION_SCHEMA = z.object({
|
|
35
|
+
section_name: z.string().min(1).max(100).describe('Section label like "Verse 1", "Chorus", "Bridge".'),
|
|
36
|
+
duration_ms: z.number().min(3000).max(120_000).describe('Section duration in milliseconds (3000-120000).'),
|
|
37
|
+
positive_local_styles: z.array(z.string()).max(50).default([])
|
|
38
|
+
.describe('Section-specific styles to include (max 50). Use for stage directions like "whispered vocals" or "male voice"; do NOT put these in lyrics. Pass [] for sections that have no special direction.'),
|
|
39
|
+
negative_local_styles: z.array(z.string()).max(50).default([])
|
|
40
|
+
.describe('Section-specific styles to avoid (max 50). Pass [] when nothing to avoid.'),
|
|
41
|
+
lines: z.array(z.string().max(200)).max(30).default([])
|
|
42
|
+
.describe('Lyric lines for this section (max 30 lines, 200 chars each). Singable/speakable text only — performance directions belong in positive_local_styles. Pass [] for instrumental sections.'),
|
|
43
|
+
}).strict();
|
|
44
|
+
const COMPOSITION_PLAN_SCHEMA = z.object({
|
|
45
|
+
positive_global_styles: z.array(z.string()).optional()
|
|
46
|
+
.describe('Styles to apply globally (genre, mood, tempo, key, BPM).'),
|
|
47
|
+
negative_global_styles: z.array(z.string()).optional()
|
|
48
|
+
.describe('Styles to avoid globally.'),
|
|
49
|
+
sections: z.array(COMPOSITION_SECTION_SCHEMA).min(1).max(30)
|
|
50
|
+
.describe('Array of sections (max 30). Total duration must be between 3 seconds and 10 minutes.'),
|
|
51
|
+
})
|
|
52
|
+
.strict()
|
|
53
|
+
.superRefine((plan, ctx) => {
|
|
54
|
+
const total = plan.sections.reduce((acc, s) => acc + (s.duration_ms || 0), 0);
|
|
55
|
+
if (total < MIN_TOTAL_MUSIC_MS || total > MAX_TOTAL_MUSIC_MS) {
|
|
56
|
+
ctx.addIssue({
|
|
57
|
+
code: z.ZodIssueCode.custom,
|
|
58
|
+
path: ['sections'],
|
|
59
|
+
message: `Total duration across sections must be between ${MIN_TOTAL_MUSIC_MS}ms (3s) and ${MAX_TOTAL_MUSIC_MS}ms (10min); got ${total}ms.`,
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
})
|
|
63
|
+
.describe('Composition plan object — must match the shape returned by create_music_plan exactly. ' +
|
|
64
|
+
'Pass the plan through verbatim or edit individual fields. The legacy {style, lyrics} ' +
|
|
65
|
+
'shape used by ≤0.2.2 is no longer accepted (the API rejects it with HTTP 422).');
|
|
66
|
+
function wrapMusicPlanArray(values, source) {
|
|
67
|
+
return values?.map((value, index) => wrapUntrusted(value, `${source}[${index}]`) ?? value);
|
|
68
|
+
}
|
|
69
|
+
function wrapMusicPlanForDisplay(plan) {
|
|
70
|
+
return {
|
|
71
|
+
...plan,
|
|
72
|
+
positive_global_styles: wrapMusicPlanArray(plan.positive_global_styles, 'elevenlabs:create_music_plan:composition_plan:positive_global_styles'),
|
|
73
|
+
negative_global_styles: wrapMusicPlanArray(plan.negative_global_styles, 'elevenlabs:create_music_plan:composition_plan:negative_global_styles'),
|
|
74
|
+
sections: plan.sections.map((section, index) => ({
|
|
75
|
+
...section,
|
|
76
|
+
section_name: wrapUntrusted(section.section_name, `elevenlabs:create_music_plan:composition_plan:sections[${index}]:section_name`) ?? section.section_name,
|
|
77
|
+
positive_local_styles: wrapMusicPlanArray(section.positive_local_styles, `elevenlabs:create_music_plan:composition_plan:sections[${index}]:positive_local_styles`),
|
|
78
|
+
negative_local_styles: wrapMusicPlanArray(section.negative_local_styles, `elevenlabs:create_music_plan:composition_plan:sections[${index}]:negative_local_styles`),
|
|
79
|
+
lines: wrapMusicPlanArray(section.lines, `elevenlabs:create_music_plan:composition_plan:sections[${index}]:lines`),
|
|
80
|
+
})),
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
function formatIssuePath(path) {
|
|
84
|
+
return path.length > 0 ? path.join('.') : 'composition_plan';
|
|
85
|
+
}
|
|
86
|
+
function parseCompositionPlanInput(value) {
|
|
87
|
+
const unwrapped = unwrapUntrustedJsonStrings(value);
|
|
88
|
+
const parsed = COMPOSITION_PLAN_SCHEMA.safeParse(unwrapped);
|
|
89
|
+
if (!parsed.success) {
|
|
90
|
+
const issues = parsed.error.issues
|
|
91
|
+
.map((issue) => `${formatIssuePath(issue.path)}: ${issue.message}`)
|
|
92
|
+
.join('; ');
|
|
93
|
+
throw new ElevenLabsError(`Invalid composition_plan: ${issues}`, 'INVALID_INPUT', 'Pass a composition_plan returned by create_music_plan, or provide sections with section_name, duration_ms, positive_local_styles, negative_local_styles, and lines.');
|
|
94
|
+
}
|
|
95
|
+
return parsed.data;
|
|
96
|
+
}
|
|
11
97
|
export function registerMusicTools(server) {
|
|
12
98
|
// ── generate_music ────────────────────────────────────────────────────
|
|
13
99
|
server.registerTool('generate_music', {
|
|
14
|
-
description:
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
100
|
+
description: `Generate music from a text prompt using ElevenLabs Music API.
|
|
101
|
+
|
|
102
|
+
WHEN TO USE:
|
|
103
|
+
- Quick music bed from a genre/mood description
|
|
104
|
+
- Vocal songs when the prompt includes [Verse]/[Chorus] lyric markers (do not set force_instrumental: true)
|
|
105
|
+
|
|
106
|
+
EXAMPLE: {"prompt": "Upbeat jazz piano, 30 seconds", "duration_seconds": 30}
|
|
107
|
+
|
|
108
|
+
RELATED TOOLS:
|
|
109
|
+
- create_music_plan / generate_music_from_plan: per-section lyrics and styles
|
|
110
|
+
- check_subscription: confirm credits before long tracks
|
|
111
|
+
|
|
112
|
+
RETURNS: file_path, size_bytes, duration_seconds, format; warnings[] when force_instrumental conflicts with lyric markers.
|
|
113
|
+
|
|
114
|
+
COST: Credits based on duration (3–600 seconds).`,
|
|
18
115
|
inputSchema: z.object({
|
|
19
|
-
prompt: z.string().min(1).describe('Describe the music: genre, mood, instruments, style
|
|
116
|
+
prompt: z.string().min(1).describe('Describe the music: genre, mood, instruments, style. Include `[Verse]`/`[Chorus]`/`[Bridge]` blocks with lyrics if you want a vocal song.'),
|
|
20
117
|
duration_seconds: z.number().min(3).max(600).optional().describe('Duration in seconds (3-600). Default: 30.'),
|
|
21
|
-
force_instrumental: z.boolean().optional().describe('Force instrumental-only output (no vocals). Default: false.'),
|
|
118
|
+
force_instrumental: z.boolean().optional().describe('Force instrumental-only output (no vocals). Default: false. WARNING: setting this to true overrides any lyrics in the prompt — do not enable for vocal songs.'),
|
|
22
119
|
output_format: OUTPUT_FORMAT_ENUM.describe('Audio output format. Default: mp3_44100_128.'),
|
|
23
120
|
seed: z.number().int().optional().describe('Random seed for reproducibility.'),
|
|
24
121
|
}),
|
|
@@ -28,6 +125,15 @@ export function registerMusicTools(server) {
|
|
|
28
125
|
if (!apiKey) {
|
|
29
126
|
throw new ElevenLabsError('ElevenLabs API key not configured', 'AUTH_REQUIRED', 'Ask the user for their API key, then call configure_elevenlabs_api_key.');
|
|
30
127
|
}
|
|
128
|
+
// Pre-call warning: force_instrumental + lyric markers means vocals get
|
|
129
|
+
// silently dropped. This bit a real conversation (260520) — emit a
|
|
130
|
+
// warning so the agent can self-correct rather than ship instrumental.
|
|
131
|
+
const warnings = [];
|
|
132
|
+
if (args.force_instrumental === true && LYRIC_MARKER_PATTERN.test(args.prompt)) {
|
|
133
|
+
warnings.push('force_instrumental: true was set, but the prompt contains lyric section ' +
|
|
134
|
+
'markers like [Verse]/[Chorus]. Vocals will be dropped. ' +
|
|
135
|
+
'Set force_instrumental: false (or omit it) to keep the vocal performance.');
|
|
136
|
+
}
|
|
31
137
|
const durationSeconds = args.duration_seconds ?? 30;
|
|
32
138
|
const durationMs = Math.max(3000, Math.min(600000, durationSeconds * 1000));
|
|
33
139
|
const outputFormat = args.output_format ?? 'mp3_44100_128';
|
|
@@ -41,7 +147,7 @@ export function registerMusicTools(server) {
|
|
|
41
147
|
if (args.seed !== undefined)
|
|
42
148
|
body.seed = args.seed;
|
|
43
149
|
const ext = outputFormat.startsWith('mp3') ? 'mp3' : 'wav';
|
|
44
|
-
const result = await elevenLabsAudio(apiKey,
|
|
150
|
+
const result = await elevenLabsAudio(apiKey, `${ENDPOINTS.MUSIC}?output_format=${outputFormat}`, { method: 'POST', body: JSON.stringify(body) }, ext);
|
|
45
151
|
return JSON.stringify({
|
|
46
152
|
ok: true,
|
|
47
153
|
file_path: result.filePath,
|
|
@@ -49,13 +155,26 @@ export function registerMusicTools(server) {
|
|
|
49
155
|
duration_seconds: durationSeconds,
|
|
50
156
|
format: outputFormat,
|
|
51
157
|
message: `Music generated and saved to ${result.filePath} (${(result.sizeBytes / 1024).toFixed(1)} KB, ${durationSeconds}s).`,
|
|
158
|
+
warnings: warnings.length > 0 ? warnings : undefined,
|
|
52
159
|
});
|
|
53
160
|
}));
|
|
54
161
|
// ── create_music_plan ─────────────────────────────────────────────────
|
|
55
162
|
server.registerTool('create_music_plan', {
|
|
56
|
-
description:
|
|
57
|
-
|
|
58
|
-
|
|
163
|
+
description: `Create a composition plan for music generation.
|
|
164
|
+
|
|
165
|
+
WHEN TO USE:
|
|
166
|
+
- Structure a song with per-section lyrics and styles before paying for generation
|
|
167
|
+
- Review or edit sections before calling generate_music_from_plan
|
|
168
|
+
|
|
169
|
+
EXAMPLE: {"prompt": "Acoustic folk ballad about the sea", "duration_seconds": 45}
|
|
170
|
+
|
|
171
|
+
RELATED TOOLS:
|
|
172
|
+
- generate_music_from_plan: generate audio from the returned composition_plan
|
|
173
|
+
- generate_music: faster one-shot generation without a plan
|
|
174
|
+
|
|
175
|
+
RETURNS: composition_plan (sections with section_name, duration_ms, styles, lines), total_duration_seconds.
|
|
176
|
+
|
|
177
|
+
COST: FREE — no credits consumed.`,
|
|
59
178
|
inputSchema: z.object({
|
|
60
179
|
prompt: z.string().min(1).describe('Describe the music you want.'),
|
|
61
180
|
duration_seconds: z.number().min(3).max(600).optional().describe('Target duration in seconds (3-600). Default: 30.'),
|
|
@@ -68,7 +187,7 @@ export function registerMusicTools(server) {
|
|
|
68
187
|
}
|
|
69
188
|
const durationSeconds = args.duration_seconds ?? 30;
|
|
70
189
|
const durationMs = Math.max(3000, Math.min(600000, durationSeconds * 1000));
|
|
71
|
-
const plan = await elevenLabsJson(apiKey,
|
|
190
|
+
const plan = await elevenLabsJson(apiKey, ENDPOINTS.MUSIC_PLAN, {
|
|
72
191
|
method: 'POST',
|
|
73
192
|
body: JSON.stringify({
|
|
74
193
|
prompt: args.prompt,
|
|
@@ -77,30 +196,36 @@ export function registerMusicTools(server) {
|
|
|
77
196
|
}),
|
|
78
197
|
});
|
|
79
198
|
const totalDurationMs = plan.sections.reduce((sum, s) => sum + (s.duration_ms || 0), 0);
|
|
199
|
+
const displayPlan = wrapMusicPlanForDisplay(plan);
|
|
80
200
|
return JSON.stringify({
|
|
81
201
|
ok: true,
|
|
82
|
-
composition_plan:
|
|
202
|
+
composition_plan: displayPlan,
|
|
83
203
|
total_duration_seconds: totalDurationMs / 1000,
|
|
84
204
|
num_sections: plan.sections.length,
|
|
85
205
|
cost: 'FREE — no credits consumed',
|
|
86
206
|
message: `Composition plan created with ${plan.sections.length} sections (${(totalDurationMs / 1000).toFixed(1)}s total). Review the plan and pass it to generate_music_from_plan when ready.`,
|
|
87
|
-
hint: 'You can modify positive_global_styles, negative_global_styles, section
|
|
207
|
+
hint: 'You can modify positive_global_styles, negative_global_styles, and per-section positive_local_styles, negative_local_styles, lines (lyric lines), or duration_ms before generating.',
|
|
88
208
|
});
|
|
89
209
|
}));
|
|
90
210
|
// ── generate_music_from_plan ──────────────────────────────────────────
|
|
91
211
|
server.registerTool('generate_music_from_plan', {
|
|
92
|
-
description:
|
|
93
|
-
|
|
212
|
+
description: `Generate music from a composition plan.
|
|
213
|
+
|
|
214
|
+
WHEN TO USE:
|
|
215
|
+
- After create_music_plan when you want per-section control over lyrics and styles
|
|
216
|
+
- When generate_music's single prompt is not precise enough
|
|
217
|
+
|
|
218
|
+
EXAMPLE: pass the composition_plan object from create_music_plan verbatim
|
|
219
|
+
|
|
220
|
+
RELATED TOOLS:
|
|
221
|
+
- create_music_plan: produces the plan shape this tool expects
|
|
222
|
+
- check_subscription: confirm credits before generation
|
|
223
|
+
|
|
224
|
+
RETURNS: file_path, size_bytes, duration_seconds, format.
|
|
225
|
+
|
|
226
|
+
COST: Credits based on total plan duration (3s–10min).`,
|
|
94
227
|
inputSchema: z.object({
|
|
95
|
-
composition_plan: z.object
|
|
96
|
-
positive_global_styles: z.array(z.string()).optional().describe('Styles to apply globally.'),
|
|
97
|
-
negative_global_styles: z.array(z.string()).optional().describe('Styles to avoid globally.'),
|
|
98
|
-
sections: z.array(z.object({
|
|
99
|
-
style: z.string().optional(),
|
|
100
|
-
lyrics: z.string().optional(),
|
|
101
|
-
duration_ms: z.number().optional(),
|
|
102
|
-
})).min(1).describe('Array of sections, each with style, lyrics, and duration_ms.'),
|
|
103
|
-
}).describe('The composition plan object.'),
|
|
228
|
+
composition_plan: z.unknown().describe('Composition plan object from create_music_plan. Wrapped display text and raw manually edited text are both accepted; the plan is unwrapped and strictly validated before generation.'),
|
|
104
229
|
seed: z.number().int().optional().describe('Random seed for reproducibility.'),
|
|
105
230
|
output_format: OUTPUT_FORMAT_ENUM.describe('Audio output format. Default: mp3_44100_128.'),
|
|
106
231
|
}),
|
|
@@ -110,7 +235,7 @@ export function registerMusicTools(server) {
|
|
|
110
235
|
if (!apiKey) {
|
|
111
236
|
throw new ElevenLabsError('ElevenLabs API key not configured', 'AUTH_REQUIRED', 'Ask the user for their API key, then call configure_elevenlabs_api_key.');
|
|
112
237
|
}
|
|
113
|
-
const compositionPlan = args.composition_plan;
|
|
238
|
+
const compositionPlan = parseCompositionPlanInput(args.composition_plan);
|
|
114
239
|
const outputFormat = args.output_format ?? 'mp3_44100_128';
|
|
115
240
|
const body = {
|
|
116
241
|
composition_plan: compositionPlan,
|
|
@@ -119,8 +244,8 @@ export function registerMusicTools(server) {
|
|
|
119
244
|
if (args.seed !== undefined)
|
|
120
245
|
body.seed = args.seed;
|
|
121
246
|
const ext = outputFormat.startsWith('mp3') ? 'mp3' : 'wav';
|
|
122
|
-
const result = await elevenLabsAudio(apiKey,
|
|
123
|
-
const totalDurationMs =
|
|
247
|
+
const result = await elevenLabsAudio(apiKey, `${ENDPOINTS.MUSIC}?output_format=${outputFormat}`, { method: 'POST', body: JSON.stringify(body) }, ext);
|
|
248
|
+
const totalDurationMs = compositionPlan.sections.reduce((sum, s) => sum + (s.duration_ms || 0), 0);
|
|
124
249
|
return JSON.stringify({
|
|
125
250
|
ok: true,
|
|
126
251
|
file_path: result.filePath,
|
|
@@ -28,6 +28,7 @@ export declare function isRemoteUrl(input: string): boolean;
|
|
|
28
28
|
* than crashing.
|
|
29
29
|
*/
|
|
30
30
|
export declare function getAudioWorkspaceRoot(): string;
|
|
31
|
+
export declare function isInsideAudioWorkspaceRoot(resolvedPath: string, root?: string): boolean;
|
|
31
32
|
/**
|
|
32
33
|
* Validate that an LLM-supplied `file_path` argument resolves to a real
|
|
33
34
|
* file under the audio workspace sandbox root, even after symlink
|
|
@@ -36,6 +36,9 @@ export function getAudioWorkspaceRoot() {
|
|
|
36
36
|
return lexical;
|
|
37
37
|
}
|
|
38
38
|
}
|
|
39
|
+
export function isInsideAudioWorkspaceRoot(resolvedPath, root = getAudioWorkspaceRoot()) {
|
|
40
|
+
return resolvedPath === root || resolvedPath.startsWith(root + path.sep);
|
|
41
|
+
}
|
|
39
42
|
/**
|
|
40
43
|
* Canonicalise the deepest existing ancestor of an absolute path and
|
|
41
44
|
* re-append the missing tail (M3-fix-C). This lets the lexical-prefix
|
|
@@ -90,7 +93,7 @@ function canonicalisePrefix(absoluteLexical) {
|
|
|
90
93
|
export function resolveAudioPath(filePath) {
|
|
91
94
|
const root = getAudioWorkspaceRoot();
|
|
92
95
|
const denyMessage = `file_path is outside the workspace sandbox root (${root}). Got: ${filePath}`;
|
|
93
|
-
const isInsideRoot = (p) => p
|
|
96
|
+
const isInsideRoot = (p) => isInsideAudioWorkspaceRoot(p, root);
|
|
94
97
|
// Step 1: lexical normalisation — expand `~`, collapse `..`, absolutise.
|
|
95
98
|
const expanded = filePath.startsWith('~')
|
|
96
99
|
? path.join(os.homedir(), filePath.slice(1))
|
package/dist/tools/speech.js
CHANGED
|
@@ -1,31 +1,76 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { getApiKey } from '../auth.js';
|
|
3
3
|
import { elevenLabsJson, elevenLabsAudio } from '../client.js';
|
|
4
|
-
import {
|
|
4
|
+
import { ENDPOINTS, voicesV2Url } from '../endpoints.js';
|
|
5
|
+
import { ElevenLabsError, VOICE_NOT_FOUND_RESOLUTION } from '../types.js';
|
|
6
|
+
import { wrapUntrusted } from '../untrusted-content.js';
|
|
5
7
|
import { withErrorHandling } from '../utils.js';
|
|
6
8
|
/**
|
|
7
9
|
* Look up a voice by name via the ElevenLabs v2 voices API.
|
|
8
10
|
*/
|
|
9
11
|
async function lookupVoiceByName(apiKey, name) {
|
|
10
|
-
const
|
|
12
|
+
const params = new URLSearchParams({
|
|
13
|
+
search: name,
|
|
14
|
+
page_size: '5',
|
|
15
|
+
});
|
|
16
|
+
const data = await elevenLabsJson(apiKey, voicesV2Url(params));
|
|
11
17
|
if (!data.voices || data.voices.length === 0) {
|
|
12
|
-
throw new ElevenLabsError(`No voice found matching "${name}"`, 'VOICE_NOT_FOUND',
|
|
18
|
+
throw new ElevenLabsError(`No voice found matching "${name}"`, 'VOICE_NOT_FOUND', VOICE_NOT_FOUND_RESOLUTION);
|
|
13
19
|
}
|
|
14
20
|
return data.voices[0];
|
|
15
21
|
}
|
|
22
|
+
/**
|
|
23
|
+
* Pick a sensible default voice from the account.
|
|
24
|
+
*
|
|
25
|
+
* The previous version of this connector hardcoded a "Rachel" lookup, which
|
|
26
|
+
* silently fails on accounts that don't have Rachel in their library (most
|
|
27
|
+
* personal/business accounts). We now ask the API for the first page of
|
|
28
|
+
* voices, prefer premade voices, and fall back to whatever the account has.
|
|
29
|
+
*
|
|
30
|
+
* If the account has zero voices we surface a clear VOICE_NOT_FOUND with a
|
|
31
|
+
* resolution that tells the agent to call list_voices first.
|
|
32
|
+
*/
|
|
33
|
+
async function pickDefaultVoice(apiKey) {
|
|
34
|
+
const params = new URLSearchParams({ page_size: '20' });
|
|
35
|
+
const data = await elevenLabsJson(apiKey, voicesV2Url(params));
|
|
36
|
+
if (!data.voices || data.voices.length === 0) {
|
|
37
|
+
throw new ElevenLabsError('No voice specified and the account has no voices.', 'VOICE_NOT_FOUND', VOICE_NOT_FOUND_RESOLUTION);
|
|
38
|
+
}
|
|
39
|
+
// Prefer premade voices (curated, generally suitable as defaults).
|
|
40
|
+
const premade = data.voices.find((v) => v.category === 'premade');
|
|
41
|
+
return premade ?? data.voices[0];
|
|
42
|
+
}
|
|
16
43
|
export function registerSpeechTools(server) {
|
|
17
44
|
// ── generate_speech ───────────────────────────────────────────────────
|
|
18
45
|
server.registerTool('generate_speech', {
|
|
19
|
-
description:
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
46
|
+
description: `Generate spoken audio from text using ElevenLabs text-to-speech.
|
|
47
|
+
|
|
48
|
+
WHEN TO USE:
|
|
49
|
+
- Turn user text into a playable speech file
|
|
50
|
+
- Narration, voiceovers, or reading content aloud
|
|
51
|
+
|
|
52
|
+
EXAMPLE: {"text": "Hello world.", "voice_id": "21m00Tcm4TlvDq8ikWAM", "model_id": "eleven_v3"}
|
|
53
|
+
|
|
54
|
+
RELATED TOOLS:
|
|
55
|
+
- list_voices / search_shared_voices / get_voice: find voice_id
|
|
56
|
+
- list_models: pick model_id (default eleven_v3)
|
|
57
|
+
- check_subscription: confirm credits before long text
|
|
58
|
+
|
|
59
|
+
RETURNS: file_path, size_bytes, voice_id, model, format. API-resolved voice names are enveloped.
|
|
60
|
+
|
|
61
|
+
COST: ~1 credit per 100 characters.`,
|
|
23
62
|
inputSchema: z.object({
|
|
24
63
|
text: z.string().min(1).describe('Text to speak. Maximum ~5000 characters per request.'),
|
|
25
|
-
voice_id: z.string().optional().describe('Direct voice ID. Takes priority over voice_name.'),
|
|
26
|
-
voice_name: z.string().optional().describe('Voice name for fuzzy search (e.g., "
|
|
27
|
-
model_id: z.enum([
|
|
28
|
-
|
|
64
|
+
voice_id: z.string().optional().describe('Direct voice ID (from list_voices). Takes priority over voice_name.'),
|
|
65
|
+
voice_name: z.string().optional().describe('Voice name for fuzzy search (e.g., "Bella", "Sarah"). Use list_voices first to find a name that exists on the account.'),
|
|
66
|
+
model_id: z.enum([
|
|
67
|
+
'eleven_v3',
|
|
68
|
+
'eleven_multilingual_v2',
|
|
69
|
+
'eleven_flash_v2_5',
|
|
70
|
+
'eleven_turbo_v2_5',
|
|
71
|
+
'eleven_monolingual_v1',
|
|
72
|
+
]).optional()
|
|
73
|
+
.describe('TTS model. Default: eleven_v3.'),
|
|
29
74
|
stability: z.number().min(0).max(1).optional().describe('Voice stability 0-1. Default: 0.5.'),
|
|
30
75
|
similarity_boost: z.number().min(0).max(1).optional().describe('Voice similarity 0-1. Default: 0.75.'),
|
|
31
76
|
output_format: z.enum(['mp3_44100_128', 'mp3_44100_192', 'pcm_16000', 'pcm_22050', 'pcm_24000', 'pcm_44100']).optional()
|
|
@@ -39,12 +84,17 @@ export function registerSpeechTools(server) {
|
|
|
39
84
|
}
|
|
40
85
|
let voiceId = args.voice_id;
|
|
41
86
|
const voiceName = args.voice_name;
|
|
42
|
-
const modelId = args.model_id ?? '
|
|
87
|
+
const modelId = args.model_id ?? 'eleven_v3';
|
|
43
88
|
const stability = args.stability ?? 0.5;
|
|
44
89
|
const similarityBoost = args.similarity_boost ?? 0.75;
|
|
45
90
|
const outputFormat = args.output_format ?? 'mp3_44100_128';
|
|
46
|
-
// Voice lookup
|
|
91
|
+
// Voice lookup. When the name comes back from the API it is external,
|
|
92
|
+
// possibly attacker-authored text (e.g. a shared/cloned voice named to
|
|
93
|
+
// carry a prompt injection) and must be enveloped before being returned
|
|
94
|
+
// (AGENTS.md invariant #6). When voice_id is passed directly, the name
|
|
95
|
+
// is just the caller's own input echoed back — no envelope needed.
|
|
47
96
|
let resolvedVoiceName = voiceName || 'default';
|
|
97
|
+
let voiceNameFromApi = false;
|
|
48
98
|
if (!voiceId) {
|
|
49
99
|
if (voiceName) {
|
|
50
100
|
const voice = await lookupVoiceByName(apiKey, voiceName);
|
|
@@ -52,20 +102,12 @@ export function registerSpeechTools(server) {
|
|
|
52
102
|
resolvedVoiceName = voice.name;
|
|
53
103
|
}
|
|
54
104
|
else {
|
|
55
|
-
//
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
resolvedVoiceName = voice.name;
|
|
60
|
-
}
|
|
61
|
-
catch (err) {
|
|
62
|
-
// Propagate the error so withErrorHandling emits isError: true
|
|
63
|
-
if (err instanceof ElevenLabsError) {
|
|
64
|
-
throw err;
|
|
65
|
-
}
|
|
66
|
-
throw new ElevenLabsError('No voice specified and default voice lookup failed.', 'VOICE_NOT_FOUND', 'Provide a voice_id or voice_name. Use list_voices to find available voices.');
|
|
67
|
-
}
|
|
105
|
+
// No voice specified — pick a sensible default from the account.
|
|
106
|
+
const voice = await pickDefaultVoice(apiKey);
|
|
107
|
+
voiceId = voice.voice_id;
|
|
108
|
+
resolvedVoiceName = voice.name;
|
|
68
109
|
}
|
|
110
|
+
voiceNameFromApi = true;
|
|
69
111
|
}
|
|
70
112
|
const body = {
|
|
71
113
|
text: args.text,
|
|
@@ -76,24 +118,38 @@ export function registerSpeechTools(server) {
|
|
|
76
118
|
},
|
|
77
119
|
};
|
|
78
120
|
const ext = outputFormat.startsWith('mp3') ? 'mp3' : 'wav';
|
|
79
|
-
const result = await elevenLabsAudio(apiKey,
|
|
121
|
+
const result = await elevenLabsAudio(apiKey, ENDPOINTS.textToSpeech(voiceId, outputFormat), { method: 'POST', body: JSON.stringify(body) }, ext);
|
|
122
|
+
// The voice name lives ONLY in the (enveloped) `voice` field — never in
|
|
123
|
+
// `message`, so no unwrapped API-authored substring can reach the model.
|
|
80
124
|
return JSON.stringify({
|
|
81
125
|
ok: true,
|
|
82
126
|
file_path: result.filePath,
|
|
83
127
|
size_bytes: result.sizeBytes,
|
|
84
|
-
voice:
|
|
128
|
+
voice: voiceNameFromApi
|
|
129
|
+
? wrapUntrusted(resolvedVoiceName, 'elevenlabs:generate_speech:voice_name')
|
|
130
|
+
: resolvedVoiceName,
|
|
85
131
|
voice_id: voiceId,
|
|
86
132
|
model: modelId,
|
|
87
133
|
format: outputFormat,
|
|
88
|
-
message: `Speech generated
|
|
134
|
+
message: `Speech generated and saved to ${result.filePath} (${(result.sizeBytes / 1024).toFixed(1)} KB).`,
|
|
89
135
|
});
|
|
90
136
|
}));
|
|
91
137
|
// ── generate_sound_effect ─────────────────────────────────────────────
|
|
92
138
|
server.registerTool('generate_sound_effect', {
|
|
93
|
-
description:
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
139
|
+
description: `Generate sound effects from a text description.
|
|
140
|
+
|
|
141
|
+
WHEN TO USE:
|
|
142
|
+
- Short ambient or UI sounds from a natural-language prompt
|
|
143
|
+
- Effects for video, games, or presentations
|
|
144
|
+
|
|
145
|
+
EXAMPLE: {"prompt": "Soft rain on a tin roof", "duration_seconds": 3}
|
|
146
|
+
|
|
147
|
+
RELATED TOOLS:
|
|
148
|
+
- check_subscription: confirm credits before generation
|
|
149
|
+
|
|
150
|
+
RETURNS: file_path, size_bytes, duration_seconds.
|
|
151
|
+
|
|
152
|
+
COST: Credits based on duration (0.5–22 seconds).`,
|
|
97
153
|
inputSchema: z.object({
|
|
98
154
|
prompt: z.string().min(1).describe('Describe the sound effect. Be specific about characteristics.'),
|
|
99
155
|
duration_seconds: z.number().min(0.5).max(22).optional().describe('Duration in seconds (0.5-22). Auto if omitted.'),
|
|
@@ -113,7 +169,7 @@ export function registerSpeechTools(server) {
|
|
|
113
169
|
if (args.duration_seconds !== undefined) {
|
|
114
170
|
body.duration_seconds = Math.max(0.5, Math.min(22, args.duration_seconds));
|
|
115
171
|
}
|
|
116
|
-
const result = await elevenLabsAudio(apiKey,
|
|
172
|
+
const result = await elevenLabsAudio(apiKey, ENDPOINTS.SOUND_GENERATION, { method: 'POST', body: JSON.stringify(body) }, 'mp3');
|
|
117
173
|
return JSON.stringify({
|
|
118
174
|
ok: true,
|
|
119
175
|
file_path: result.filePath,
|
|
@@ -1,20 +1,32 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
-
import * as fs from 'fs';
|
|
3
|
-
import * as path from 'path';
|
|
4
2
|
import { getApiKey } from '../auth.js';
|
|
5
3
|
import { elevenLabsJson } from '../client.js';
|
|
4
|
+
import { ENDPOINTS } from '../endpoints.js';
|
|
6
5
|
import { ElevenLabsError } from '../types.js';
|
|
6
|
+
import { wrapUntrusted } from '../untrusted-content.js';
|
|
7
7
|
import { withErrorHandling } from '../utils.js';
|
|
8
|
-
import {
|
|
8
|
+
import { readSandboxedFile, sandboxedFileToBlob } from './file-input.js';
|
|
9
9
|
export function registerTranscriptionTools(server) {
|
|
10
10
|
server.registerTool('transcribe_audio', {
|
|
11
|
-
description:
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
11
|
+
description: `Transcribe speech from a local audio file to text.
|
|
12
|
+
|
|
13
|
+
WHEN TO USE:
|
|
14
|
+
- Convert meeting recordings or voice memos to text
|
|
15
|
+
- Extract quotes from audio the user provides as a file path
|
|
16
|
+
|
|
17
|
+
EXAMPLE: {"file_path": "/path/to/recording.mp3", "language_code": "en"}
|
|
18
|
+
|
|
19
|
+
RELATED TOOLS:
|
|
20
|
+
- generate_speech: the inverse operation (text to audio)
|
|
21
|
+
|
|
22
|
+
RETURNS: enveloped text, word_count, language. File path must be inside MCP_WORKSPACE_PATH (or os.tmpdir()).
|
|
23
|
+
|
|
24
|
+
COST: Credits based on audio duration.`,
|
|
15
25
|
inputSchema: z.object({
|
|
16
26
|
file_path: z.string().min(1).describe('Absolute path to local audio file to transcribe.'),
|
|
17
27
|
language_code: z.string().optional().describe('Language code (e.g., "en", "es", "fr"). Auto-detected if omitted.'),
|
|
28
|
+
model_id: z.enum(['scribe_v1']).optional().describe('STT model. Default: scribe_v1.'),
|
|
29
|
+
tag_audio_events: z.boolean().optional().describe('When true, include non-speech events like "(laughter)" in the transcript. Default: false.'),
|
|
18
30
|
}),
|
|
19
31
|
annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: true },
|
|
20
32
|
}, withErrorHandling(async (args) => {
|
|
@@ -22,69 +34,30 @@ export function registerTranscriptionTools(server) {
|
|
|
22
34
|
if (!apiKey) {
|
|
23
35
|
throw new ElevenLabsError('ElevenLabs API key not configured', 'AUTH_REQUIRED', 'Ask the user for their API key, then call configure_elevenlabs_api_key.');
|
|
24
36
|
}
|
|
25
|
-
|
|
26
|
-
//
|
|
27
|
-
|
|
28
|
-
//
|
|
29
|
-
//
|
|
30
|
-
//
|
|
31
|
-
//
|
|
32
|
-
//
|
|
33
|
-
// - Remote URL inputs bypass the sandbox: `transcribe_audio`
|
|
34
|
-
// currently only handles local-file inputs, so a URL string falls
|
|
35
|
-
// through to the existing FILE_NOT_FOUND branch (a non-sandbox
|
|
36
|
-
// error code, preserving pre-fix behaviour for URL inputs).
|
|
37
|
-
// - Local paths run through `resolveAudioPath`, which:
|
|
38
|
-
// 1. Lexically resolves `~` and `..` and rejects paths outside
|
|
39
|
-
// the workspace root before any disk read.
|
|
40
|
-
// 2. Canonicalises the file via `fs.realpathSync` so a symlink
|
|
41
|
-
// inside the workspace pointing OUTSIDE the workspace is
|
|
42
|
-
// refused.
|
|
43
|
-
// ----------------------------------------------------------------
|
|
44
|
-
let filePath;
|
|
45
|
-
if (isRemoteUrl(rawFilePath)) {
|
|
46
|
-
// Preserve pre-existing behaviour for URL inputs: the
|
|
47
|
-
// existsSync branch below will return a clean FILE_NOT_FOUND
|
|
48
|
-
// (a non-sandbox error code).
|
|
49
|
-
filePath = rawFilePath;
|
|
50
|
-
}
|
|
51
|
-
else {
|
|
52
|
-
const resolution = resolveAudioPath(rawFilePath);
|
|
53
|
-
if (!resolution.ok) {
|
|
54
|
-
// Surface the sandbox / not-found error via the standard
|
|
55
|
-
// ElevenLabsError path so withErrorHandling renders it with a
|
|
56
|
-
// stable shape and a code.
|
|
57
|
-
const code = resolution.error.startsWith('File not found')
|
|
58
|
-
? 'FILE_NOT_FOUND'
|
|
59
|
-
: 'PATH_SANDBOX_VIOLATION';
|
|
60
|
-
throw new ElevenLabsError(resolution.error, code, 'Provide a path to an existing audio file inside MCP_WORKSPACE_PATH (or os.tmpdir() when unset). Remote URLs are not supported.');
|
|
61
|
-
}
|
|
62
|
-
filePath = resolution.path;
|
|
63
|
-
}
|
|
64
|
-
// Read local file
|
|
65
|
-
if (!fs.existsSync(filePath)) {
|
|
66
|
-
throw new ElevenLabsError(`File not found: ${rawFilePath}`, 'FILE_NOT_FOUND', 'Provide an absolute path to an existing audio file.');
|
|
67
|
-
}
|
|
68
|
-
// Defence-in-depth: re-canonicalise via realpathSync at the very
|
|
69
|
-
// last moment to close the (vanishingly small) TOCTOU window
|
|
70
|
-
// between sandbox validation and the readFileSync call. URL inputs
|
|
71
|
-
// skip this since they did not go through the sandbox.
|
|
72
|
-
const verifiedPath = isRemoteUrl(rawFilePath) ? filePath : fs.realpathSync(filePath);
|
|
73
|
-
const fileBuffer = fs.readFileSync(verifiedPath);
|
|
74
|
-
const fileName = path.basename(verifiedPath);
|
|
75
|
-
// Use FormData to send as multipart
|
|
37
|
+
// Sandbox local reads via shared file-input helper (MCP_WORKSPACE_PATH +
|
|
38
|
+
// realpathSync TOCTOU guard — see file-input.ts).
|
|
39
|
+
const fileInput = readSandboxedFile(args.file_path);
|
|
40
|
+
// Multipart upload. ElevenLabs Speech-to-Text v1 requires:
|
|
41
|
+
// - field name `file` (NOT `audio`)
|
|
42
|
+
// - `model_id` is mandatory (only `scribe_v1` is currently supported)
|
|
43
|
+
// - `tag_audio_events=false` to avoid `(mouse click)` style noise
|
|
76
44
|
const formData = new FormData();
|
|
77
|
-
formData.append('
|
|
45
|
+
formData.append('file', sandboxedFileToBlob(fileInput), fileInput.fileName);
|
|
46
|
+
formData.append('model_id', args.model_id ?? 'scribe_v1');
|
|
47
|
+
formData.append('tag_audio_events', String(args.tag_audio_events ?? false));
|
|
78
48
|
if (args.language_code) {
|
|
79
49
|
formData.append('language_code', args.language_code);
|
|
80
50
|
}
|
|
81
|
-
const data = await elevenLabsJson(apiKey,
|
|
51
|
+
const data = await elevenLabsJson(apiKey, ENDPOINTS.SPEECH_TO_TEXT, {
|
|
82
52
|
method: 'POST',
|
|
83
53
|
body: formData,
|
|
84
54
|
});
|
|
55
|
+
// AGENTS.md invariant #6: the transcript is whatever was SPOKEN in the
|
|
56
|
+
// audio — the most attacker-controllable text this connector returns.
|
|
57
|
+
// `message` stays numeric-only; never echo transcript substrings into it.
|
|
85
58
|
return JSON.stringify({
|
|
86
59
|
ok: true,
|
|
87
|
-
text: data.text,
|
|
60
|
+
text: wrapUntrusted(data.text, 'elevenlabs:transcribe_audio:text'),
|
|
88
61
|
word_count: data.words?.length || 0,
|
|
89
62
|
language: args.language_code || 'auto-detected',
|
|
90
63
|
message: `Transcription complete: ${data.text.length} characters, ${data.words?.length || 0} words.`,
|