dsh-audiogen 0.4.13 → 0.4.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/README.zh-CN.md +2 -1
- package/lib/client.js +350 -348
- package/lib/client.js.map +1 -1
- package/lib/index.js +588 -27
- package/package.json +1 -1
- package/skills/sfx/SKILL.md +6 -0
- package/skills/voice-management/SKILL.md +48 -0
- package/src/agent-audio-tools.ts +123 -0
- package/src/audio-engine.ts +170 -10
- package/src/client/api.ts +2 -0
- package/src/client/audio-panel.module.css +7 -0
- package/src/client/field-specs.ts +1 -1
- package/src/client/studio-view.tsx +9 -1
- package/src/index.ts +1 -1
- package/src/protocol.ts +1 -1
- package/src/routes.ts +6 -1
- package/src/voice-manager.ts +378 -0
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-audiogen",
|
|
3
3
|
"description": "AI audio generation plugin for the dsh web GUI: multi-vendor TTS/music/sound-effect channels (OpenAI-compatible, ElevenLabs, MiniMax, Stability AI and custom), per-channel model/voice catalogs, Agent tool and a sidebar AI 音频 panel.",
|
|
4
|
-
"version": "0.4.
|
|
4
|
+
"version": "0.4.15",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/index.js",
|
|
7
7
|
"exports": {
|
package/skills/sfx/SKILL.md
CHANGED
|
@@ -33,3 +33,9 @@ whenToUse: 用户请求生成音效、提示音、环境音、UI 音,或触发
|
|
|
33
33
|
| prompt_influence | prompt_influence | 0-1,默认 0.3;越高越贴提示词,越低越多样 |
|
|
34
34
|
|
|
35
35
|
> 响应为 audio/mpeg 二进制。请求同时携带 `xi-api-key` 与 `Authorization: Bearer`,兼容 New API 类网关。
|
|
36
|
+
>
|
|
37
|
+
> 网关自动回退:当渠道 API 地址不是 ElevenLabs 官方域名且官方协议被网关拒绝时
|
|
38
|
+
> (404 Invalid URL / 401 Invalid token,例如 ai.farmmx.com 未映射 `/v1/sound-generation`),
|
|
39
|
+
> 引擎自动改用 OpenAI 兼容形态重试:`POST {base}/audio/speech` + `Authorization: Bearer` +
|
|
40
|
+
> `model=eleven_text_to_sound_v2` + `text/duration_seconds/prompt_influence/loop` 字段。
|
|
41
|
+
> 官方地址(api.elevenlabs.io)直连时不触发回退。音色设计在网关无对应兼容端点,会明确报错不便死等。
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: dsh-audiogen-voice-management
|
|
3
|
+
description: 浏览/筛选/删除厂商音色库(MiniMax、ElevenLabs)并复用选定音色生成 TTS。当用户提到「音色管理 / 音色列表 / 查看有哪些音色 / 删除音色 / 筛选音色 / 用某个音色 TTS / 声音列表」时触发。与资源库(本地生成结果)无关,请勿混淆。
|
|
4
|
+
whenToUse: 需要查看某个渠道(MiniMax / ElevenLabs)有哪些可用音色、按语言/关键词筛选音色、删除某个自建音色,或拿到音色 voice_id 后用它生成语音时。
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
# 厂商音色管理(dsh-audiogen)
|
|
8
|
+
|
|
9
|
+
插件提供 `manage_audio_voices` 工具,对接厂商官方音色管理接口:
|
|
10
|
+
|
|
11
|
+
| 渠道 | 浏览 | 删除 |
|
|
12
|
+
|---|---|---|
|
|
13
|
+
| MiniMax | `POST /v1/get_voice`(system 预置 + voice_cloning/voice_generation 自建) | `POST /v1/delete_voice`(仅自建) |
|
|
14
|
+
| ElevenLabs | `GET /v1/voices`(自有)+ `GET /v1/shared-voices`(社区共享库) | `DELETE /v1/voices/{voice_id}`(仅自有) |
|
|
15
|
+
|
|
16
|
+
## 浏览音色(action=list)
|
|
17
|
+
|
|
18
|
+
```json
|
|
19
|
+
{"action": "list", "channel": "MiniMax", "language": "zh", "keyword": "少女", "source": "custom"}
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
- `channel`:设置中的渠道名称或 id;缺省用默认渠道,多渠道时必须指定。
|
|
23
|
+
- `language`:语言子串(`en` / `zh` / `ja`,或 `Chinese (Mandarin)` 等)。
|
|
24
|
+
- `keyword`:音色名/描述/口音/用途中的自由词。
|
|
25
|
+
- `source`:`system`(官方预置)| `custom`(MiniMax 自建)| `owned`(ElevenLabs 自有)| `shared`(ElevenLabs 社区库)。
|
|
26
|
+
- 返回每条含 `voice_id` / `name` / `source` / `deletable` / `description` / `preview_url`。
|
|
27
|
+
|
|
28
|
+
## 删除音色(action=delete)
|
|
29
|
+
|
|
30
|
+
```json
|
|
31
|
+
{"action": "delete", "channel": "MiniMax", "voice_id": "voice_abc123", "confirm": true}
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
- 仅 `deletable=true`(custom/owned)的音色可删;官方/共享/系统音色会被拒绝。
|
|
35
|
+
- 删除不可逆:必须显式传 `confirm: true`,且 `voice_id` 用 `action=list` 返回的精确值。
|
|
36
|
+
|
|
37
|
+
## 闭环:选定音色 → 生成 TTS
|
|
38
|
+
|
|
39
|
+
1. `manage_audio_voices(list)` 找到符合需求的音色,记录其 `voice_id`。
|
|
40
|
+
2. `generate_audio(mode="tts", voice="<voice_id>", channel="<渠道名>", prompt="...")` 生成语音。
|
|
41
|
+
|
|
42
|
+
MiniMax TTS 的 `voice` 参数需要官方 voice_id;ElevenLabs 的 `voice` 参数可填音色 name 或 voice_id(网关渠道强制 voice_id)。
|
|
43
|
+
|
|
44
|
+
## 常见错误
|
|
45
|
+
|
|
46
|
+
- `channel-choice-required`:多个渠道未指定 → 先 `manage_audio_voices(list, channel=...)` 或询问用户。
|
|
47
|
+
- 删除被拒(`system`/`shared` 只读):换个 `source=custom`/`owned` 的音色。
|
|
48
|
+
- 拉取失败:渠道 API 地址/密钥错误,或网关不支持音色管理端点(Stability、自定义 OpenAI 兼容渠道无此能力,会明确提示不支持)。
|
package/src/agent-audio-tools.ts
CHANGED
|
@@ -11,6 +11,10 @@ import type { AudioChannel } from './audio-engine.ts'
|
|
|
11
11
|
import { generateAudio, AudioGenError } from './audio-engine.ts'
|
|
12
12
|
import type { GenerationBudget } from './audio-scheduler.ts'
|
|
13
13
|
import { appendHistory, saveAudioFile, saveToLibrary, listLibrary } from './audio-store.ts'
|
|
14
|
+
import {
|
|
15
|
+
listVendorVoices,
|
|
16
|
+
deleteVendorVoice,
|
|
17
|
+
} from './voice-manager.ts'
|
|
14
18
|
import type { AudioMode, GenerateAudioRequest, LibraryType } from './protocol.ts'
|
|
15
19
|
|
|
16
20
|
export interface AgentAudioToolConfig {
|
|
@@ -546,8 +550,127 @@ export function registerAgentAudioTools(ctx: Context, resolve: () => AgentAudioT
|
|
|
546
550
|
return { status: 'ok' as const, count: entries.length, entries }
|
|
547
551
|
},
|
|
548
552
|
}))
|
|
553
|
+
|
|
554
|
+
/** 厂商音色管理:浏览/筛选 + 删除(仅自建)。删除不可逆,必须 confirm=true。 */
|
|
555
|
+
const managementDisposer = ctx.tools.register(defineTool({
|
|
556
|
+
name: 'manage_audio_voices',
|
|
557
|
+
description: 'Manage vendor voice libraries (MiniMax / ElevenLabs). action=list: browse available TTS voices of a channel — official/shared voices plus voices designed/cloned by the account, with language/keyword/source filtering; returns voice_id/name/source/description/preview_url and whether each voice is deletable. action=delete: delete one OWNED voice (custom/owned only; official/shared/system voices are read-only and refused) — irreversible, so confirm must be true (pass the exact voice_id from action=list). Use the returned voice_id with generate_audio (mode=tts, voice=<voice_id>) to speak with the selected voice.',
|
|
558
|
+
parameters: {
|
|
559
|
+
action: { type: 'string', enum: ['list', 'delete'], required: true, description: 'list = browse/filter voices; delete = remove an owned voice.' },
|
|
560
|
+
channel: { type: 'string', description: 'Channel name or id (e.g. the channel shown in settings). Defaults to the default channel; required when more than one channel is configured.' },
|
|
561
|
+
language: { type: 'string', description: 'Filter for list: language substring (ISO code like en/zh/ja, or a label like Chinese (Mandarin)).' },
|
|
562
|
+
keyword: { type: 'string', description: 'Filter for list: free text over voice name/description/accent/use_case.' },
|
|
563
|
+
source: { type: 'string', enum: ['system', 'custom', 'owned', 'shared'], description: 'Filter for list: MiniMax system/custom; ElevenLabs owned (account) / shared (community).' },
|
|
564
|
+
voice_id: { type: 'string', description: 'Required for delete: the exact voice_id from action=list.' },
|
|
565
|
+
confirm: { type: 'boolean', description: 'Required for delete: must be true; deletion is irreversible.' },
|
|
566
|
+
},
|
|
567
|
+
output: {
|
|
568
|
+
schema: {
|
|
569
|
+
type: 'object',
|
|
570
|
+
additionalProperties: false,
|
|
571
|
+
properties: {
|
|
572
|
+
status: { type: 'string', required: true, enum: ['ok'] },
|
|
573
|
+
kind: { type: 'string', required: true, enum: ['list', 'delete'] },
|
|
574
|
+
vendor: { type: 'string', required: true },
|
|
575
|
+
channel: { type: 'string', required: true },
|
|
576
|
+
count: { type: 'integer' },
|
|
577
|
+
truncated: { type: 'boolean' },
|
|
578
|
+
voices: {
|
|
579
|
+
type: 'array',
|
|
580
|
+
items: {
|
|
581
|
+
type: 'object',
|
|
582
|
+
additionalProperties: false,
|
|
583
|
+
properties: {
|
|
584
|
+
voice_id: { type: 'string', required: true },
|
|
585
|
+
name: { type: 'string', required: true },
|
|
586
|
+
source: { type: 'string', required: true },
|
|
587
|
+
deletable: { type: 'boolean', required: true },
|
|
588
|
+
language: { type: 'string' },
|
|
589
|
+
accent: { type: 'string' },
|
|
590
|
+
gender: { type: 'string' },
|
|
591
|
+
age: { type: 'string' },
|
|
592
|
+
use_case: { type: 'string' },
|
|
593
|
+
description: { type: 'string' },
|
|
594
|
+
preview_url: { type: 'string' },
|
|
595
|
+
},
|
|
596
|
+
},
|
|
597
|
+
},
|
|
598
|
+
voice_id: { type: 'string' },
|
|
599
|
+
deleted: { type: 'boolean' },
|
|
600
|
+
message: { type: 'string' },
|
|
601
|
+
note: { type: 'string' },
|
|
602
|
+
},
|
|
603
|
+
} as const,
|
|
604
|
+
render: (_args, value) => [{ type: 'text', text: JSON.stringify(value) }],
|
|
605
|
+
},
|
|
606
|
+
timeoutMs: 120_000,
|
|
607
|
+
isConcurrencySafe: () => true,
|
|
608
|
+
async execute(args) {
|
|
609
|
+
const config = resolve()
|
|
610
|
+
if (!config.enabled) {
|
|
611
|
+
throw new AudioGenError('AI audio is disabled. Open Settings > Plugins > AI Audio and enable it.', 'plugin-disabled')
|
|
612
|
+
}
|
|
613
|
+
const channel = resolveVoiceManagementChannel(config, args.channel)
|
|
614
|
+
const action = args.action === 'delete' ? 'delete' : 'list'
|
|
615
|
+
if (action === 'list') {
|
|
616
|
+
const source = typeof args.source === 'string' && ['system', 'custom', 'owned', 'shared'].includes(args.source) ? args.source : undefined
|
|
617
|
+
const result = await listVendorVoices(channel, {
|
|
618
|
+
...(typeof args.language === 'string' && args.language.trim() !== '' ? { language: args.language.trim() } : {}),
|
|
619
|
+
...(typeof args.keyword === 'string' && args.keyword.trim() !== '' ? { keyword: args.keyword.trim() } : {}),
|
|
620
|
+
...(source === undefined ? {} : { source }),
|
|
621
|
+
})
|
|
622
|
+
return {
|
|
623
|
+
status: 'ok' as const,
|
|
624
|
+
kind: 'list' as const,
|
|
625
|
+
vendor: result.vendor,
|
|
626
|
+
channel: channel.name,
|
|
627
|
+
count: result.voices.length,
|
|
628
|
+
...(result.truncated ? { truncated: true } : {}),
|
|
629
|
+
voices: result.voices,
|
|
630
|
+
...(result.note === undefined ? {} : { note: result.note }),
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
const voiceId = typeof args.voice_id === 'string' ? args.voice_id.trim() : ''
|
|
634
|
+
if (voiceId === '') throw new AudioGenError('delete requires voice_id (the exact voice_id from action=list).', 'voice-id-required')
|
|
635
|
+
if (args.confirm !== true) throw new AudioGenError('Deletion is irreversible: pass confirm=true after verifying the voice_id.', 'voice-delete-requires-confirm')
|
|
636
|
+
const deleted = await deleteVendorVoice(channel, voiceId)
|
|
637
|
+
return {
|
|
638
|
+
status: 'ok' as const,
|
|
639
|
+
kind: 'delete' as const,
|
|
640
|
+
vendor: deleted.vendor,
|
|
641
|
+
channel: channel.name,
|
|
642
|
+
voice_id: deleted.voice_id,
|
|
643
|
+
deleted: true,
|
|
644
|
+
message: `已删除音色 ${deleted.voice_id}(${channel.name})`,
|
|
645
|
+
}
|
|
646
|
+
},
|
|
647
|
+
}))
|
|
648
|
+
|
|
549
649
|
return () => {
|
|
550
650
|
disposer()
|
|
551
651
|
searchDisposer()
|
|
652
|
+
managementDisposer()
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
/** 解析工具目标渠道:默认渠道 > 唯一可用渠道;多渠道未指定时要求显式选择。 */
|
|
657
|
+
function resolveVoiceManagementChannel(config: AgentAudioToolConfig, requested: unknown): AudioChannel {
|
|
658
|
+
const usable = config.channels.filter(channel => channel.apiUrl.trim() !== '' && channel.apiKey.trim() !== '')
|
|
659
|
+
if (usable.length === 0) {
|
|
660
|
+
throw new AudioGenError('Audio API credentials are not configured. Open Settings > Plugins > AI Audio, add a channel and fill its API URL and API key.', 'audio-api-not-configured')
|
|
661
|
+
}
|
|
662
|
+
const wanted = typeof requested === 'string' ? requested.trim() : ''
|
|
663
|
+
if (wanted === '') {
|
|
664
|
+
if (usable.length === 1) return usable[0]!
|
|
665
|
+
const target = usable.find(channel => channel.id === config.defaultChannelId)
|
|
666
|
+
if (target !== undefined) return target
|
|
667
|
+
const options = usable.map(channel => `"${channel.name}"`).join(', ')
|
|
668
|
+
throw new AudioGenError(`Multiple audio channels are configured — specify the channel (one of: ${options}).`, 'channel-choice-required')
|
|
552
669
|
}
|
|
670
|
+
const direct = usable.find(channel => channel.name === wanted || channel.id === wanted)
|
|
671
|
+
if (direct !== undefined) return direct
|
|
672
|
+
const partial = usable.filter(channel => channel.name.toLowerCase().includes(wanted.toLowerCase()))
|
|
673
|
+
if (partial.length === 1) return partial[0]!
|
|
674
|
+
const options = usable.map(channel => channel.name).join(', ')
|
|
675
|
+
throw new AudioGenError(`Audio channel "${wanted}" is not configured. Choose one of: ${options}.`, 'channel-not-configured')
|
|
553
676
|
}
|
package/src/audio-engine.ts
CHANGED
|
@@ -245,7 +245,7 @@ async function openAITTS(channel: AudioChannel, request: GenerateAudioRequest, s
|
|
|
245
245
|
return normalizeAudioResponse(response, { apiKey: channel.apiKey, fallbackMime: 'audio/mpeg' })
|
|
246
246
|
}
|
|
247
247
|
|
|
248
|
-
async function
|
|
248
|
+
async function elevenLabsOfficial(channel: AudioChannel, request: GenerateAudioRequest, signal?: AbortSignal): Promise<Array<{ data: Uint8Array; mime: string; voiceId?: string }>> {
|
|
249
249
|
const base = endpointBase(channel.apiUrl)
|
|
250
250
|
const model = (request.upstream ?? request.model) || 'eleven_multilingual_v2'
|
|
251
251
|
// 官方使用 xi-api-key;额外携带 Authorization Bearer 以兼容 New API 类网关。
|
|
@@ -299,17 +299,20 @@ async function elevenLabs(channel: AudioChannel, request: GenerateAudioRequest,
|
|
|
299
299
|
|
|
300
300
|
// ------------- ElevenLabs Music(POST /v1/music) -------------
|
|
301
301
|
// 模型:music_v1 / music_v2;prompt 与 composition_plan 二选一(引擎用 prompt)。
|
|
302
|
+
// 未提供歌词时按纯音乐处理(force_instrumental=true),不再要求必须有歌词。
|
|
302
303
|
if (request.mode === 'music') {
|
|
303
304
|
const endpoint = `${base}/music`
|
|
304
305
|
const musicModel = (request.upstream ?? request.model) || 'music_v1'
|
|
306
|
+
const lyrics = request.lyrics?.trim() ?? ''
|
|
307
|
+
const instrumental = request.isInstrumental === true || lyrics === ''
|
|
305
308
|
const body: Record<string, unknown> = {
|
|
306
309
|
model_id: musicModel,
|
|
307
310
|
prompt: request.prompt,
|
|
308
311
|
...(request.duration !== undefined && Number.isFinite(request.duration)
|
|
309
312
|
? { music_length_ms: Math.round(Math.min(600_000, Math.max(3_000, request.duration * 1000))) }
|
|
310
313
|
: {}),
|
|
311
|
-
...(
|
|
312
|
-
...(
|
|
314
|
+
...(lyrics === '' ? {} : { lyrics_text: lyrics }),
|
|
315
|
+
...(instrumental ? { force_instrumental: true } : {}),
|
|
313
316
|
}
|
|
314
317
|
const response = await fetchWithTimeout(endpoint, {
|
|
315
318
|
method: 'POST',
|
|
@@ -379,6 +382,166 @@ async function elevenLabs(channel: AudioChannel, request: GenerateAudioRequest,
|
|
|
379
382
|
return normalizeAudioResponse(response, { apiKey: channel.apiKey, fallbackMime: 'audio/mpeg' })
|
|
380
383
|
}
|
|
381
384
|
|
|
385
|
+
/**
|
|
386
|
+
* 官方 ElevenLabs 请求被网关拒绝的信号:官方路径未映射(404 Invalid URL)或
|
|
387
|
+
* 网关要求 Bearer 认证而非 xi-api-key(401/403 Invalid token / Invalid API key)。
|
|
388
|
+
* New API 类中转(如 ai.farmmx.com)对 ElevenLabs 官方协议通常返回这类错误。
|
|
389
|
+
*/
|
|
390
|
+
function isGatewayRouteMiss(error: unknown): boolean {
|
|
391
|
+
return error instanceof AudioGenError
|
|
392
|
+
&& error.code === 'audio-api-error'
|
|
393
|
+
&& /\bHTTP (404|401|403)\b/.test(error.message)
|
|
394
|
+
&& /\bInvalid URL\b|\bInvalid token\b|\bInvalid API key\b/i.test(error.message)
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
/** 网关兼容形态的请求头:仅 Bearer(携带 xi-api-key 会被网关按官方协议校验而 401)。 */
|
|
398
|
+
function gatewayHeaders(apiKey: string): Record<string, string> {
|
|
399
|
+
return {
|
|
400
|
+
authorization: `Bearer ${apiKey.trim()}`,
|
|
401
|
+
'content-type': 'application/json',
|
|
402
|
+
accept: 'audio/mpeg, application/json',
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
/** /audio/speech 兼容端点:base 已以此结尾时直接复用,否则拼接。 */
|
|
407
|
+
function speechGatewayEndpoint(base: string): string {
|
|
408
|
+
return /\/audio\/speech(\?|$)/i.test(base) ? base : `${base}/audio/speech`
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
/**
|
|
412
|
+
* ElevenLabs 渠道的网关兼容形态(OpenAI 风格):路径用 /audio/speech(音效/TTS)
|
|
413
|
+
* 或 /music(音乐),认证用 Bearer、模型用 `model` 字段。
|
|
414
|
+
*
|
|
415
|
+
* 适配未映射 ElevenLabs 官方端点(404 Invalid URL)或要求 Bearer 认证
|
|
416
|
+
* (401 Invalid token)的 New API 类中转,如 ai.farmmx.com。
|
|
417
|
+
*/
|
|
418
|
+
async function elevenLabsGatewayCompat(
|
|
419
|
+
channel: AudioChannel,
|
|
420
|
+
request: GenerateAudioRequest,
|
|
421
|
+
signal?: AbortSignal,
|
|
422
|
+
): Promise<Array<{ data: Uint8Array; mime: string; voiceId?: string }>> {
|
|
423
|
+
const base = endpointBase(channel.apiUrl)
|
|
424
|
+
const headers = gatewayHeaders(channel.apiKey)
|
|
425
|
+
|
|
426
|
+
// music → POST /music(Bearer + model;官方形态在此类网关上是 401 Invalid token)。
|
|
427
|
+
if (request.mode === 'music') {
|
|
428
|
+
const endpoint = /\/music(\?|$)/i.test(base) ? base : `${base}/music`
|
|
429
|
+
const musicModel = (request.upstream ?? request.model) || 'music_v1'
|
|
430
|
+
const lyrics = request.lyrics?.trim() ?? ''
|
|
431
|
+
const instrumental = request.isInstrumental === true || lyrics === ''
|
|
432
|
+
const body: Record<string, unknown> = {
|
|
433
|
+
model: musicModel,
|
|
434
|
+
prompt: request.prompt,
|
|
435
|
+
...(request.duration !== undefined && Number.isFinite(request.duration)
|
|
436
|
+
? { music_length_ms: Math.round(Math.min(600_000, Math.max(3_000, request.duration * 1000))) }
|
|
437
|
+
: {}),
|
|
438
|
+
...(lyrics === '' ? {} : { lyrics_text: lyrics }),
|
|
439
|
+
...(instrumental ? { force_instrumental: true } : {}),
|
|
440
|
+
}
|
|
441
|
+
const response = await fetchWithTimeout(endpoint, {
|
|
442
|
+
method: 'POST',
|
|
443
|
+
redirect: 'follow',
|
|
444
|
+
headers,
|
|
445
|
+
body: JSON.stringify(body),
|
|
446
|
+
signal,
|
|
447
|
+
}, UPSTREAM_TIMEOUT_MS)
|
|
448
|
+
if (!response.ok) {
|
|
449
|
+
const detail = await response.text().catch(() => '')
|
|
450
|
+
throw new AudioGenError(`ElevenLabs music gateway-compatible API error (HTTP ${response.status})${detail === '' ? '' : `: ${detail.slice(0, 300)}`}`, 'audio-api-error')
|
|
451
|
+
}
|
|
452
|
+
return normalizeAudioResponse(response, { apiKey: channel.apiKey, fallbackMime: 'audio/mpeg' })
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
// 音色设计在网关兼容层没有对应端点,直接给出可操作的错误说明。
|
|
456
|
+
if (request.mode === 'voice_design') {
|
|
457
|
+
throw new AudioGenError(
|
|
458
|
+
'当前网关不支持 ElevenLabs 音色设计端点(POST /v1/text-to-voice/design);该模式请改用 MiniMax 渠道或 ElevenLabs 官方 API。',
|
|
459
|
+
'voice-design-unsupported',
|
|
460
|
+
)
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
// sfx / tts → POST /audio/speech(OpenAI 兼容形态;网关把 model=eleven_text_to_sound_v2 映射到音效生成)。
|
|
464
|
+
// 音效必须传 OpenAI 标准的 `input`(实测 ai.farmmx.com 类 new-api 网关:input+duration_seconds
|
|
465
|
+
// 会走 ElevenLabs 音效生成;若传 ElevenLabs 官方字段 `text`,网关会把它当 TTS 文本朗读出来)。
|
|
466
|
+
const endpoint = speechGatewayEndpoint(base)
|
|
467
|
+
const isSfx = request.mode === 'sfx'
|
|
468
|
+
const model = (request.upstream ?? request.model)
|
|
469
|
+
|| (isSfx ? 'eleven_text_to_sound_v2' : 'eleven_multilingual_v2')
|
|
470
|
+
// 网关(new-api 类中转)对 ElevenLabs TTS 强制校验 voice/voice_id,缺失直接 400;
|
|
471
|
+
// 在发送前拦截,给出可操作的提示,而不是把网关 400 原样抛给用户。
|
|
472
|
+
if (!isSfx) {
|
|
473
|
+
const voice = request.voice?.trim() ?? ''
|
|
474
|
+
if (voice === '') {
|
|
475
|
+
const voiceEntries = (channel.models ?? []).filter(entry => {
|
|
476
|
+
const candidate = entry as { alias: string; id: string; category?: string }
|
|
477
|
+
return candidate.category === 'tts' && candidate.id.trim() !== '' && candidate.id !== candidate.alias
|
|
478
|
+
})
|
|
479
|
+
const suggestions = voiceEntries.map(entry => `${entry.alias}(${entry.id})`)
|
|
480
|
+
throw new AudioGenError(
|
|
481
|
+
`ElevenLabs 网关渠道的 TTS 必须携带音色 voice_id(网关强制校验,缺失会返回 400 voice or voice_id is required):请在「音色」字段填入官方音色 ID${
|
|
482
|
+
suggestions.length === 0 ? '' : `,可选用以下音色:${suggestions.slice(0, 4).join('、')}`
|
|
483
|
+
}。`,
|
|
484
|
+
'voice-required',
|
|
485
|
+
)
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
const body: Record<string, unknown> = isSfx
|
|
489
|
+
? {
|
|
490
|
+
model,
|
|
491
|
+
input: request.prompt,
|
|
492
|
+
...(request.duration !== undefined && Number.isFinite(request.duration)
|
|
493
|
+
? { duration_seconds: Math.min(30, Math.max(0.5, request.duration)) }
|
|
494
|
+
: {}),
|
|
495
|
+
...(request.loop !== undefined ? { loop: request.loop } : {}),
|
|
496
|
+
...(request.promptInfluence !== undefined && Number.isFinite(request.promptInfluence)
|
|
497
|
+
? { prompt_influence: Math.min(1, Math.max(0, request.promptInfluence)) }
|
|
498
|
+
: {}),
|
|
499
|
+
}
|
|
500
|
+
: {
|
|
501
|
+
model,
|
|
502
|
+
input: request.prompt,
|
|
503
|
+
voice: request.voice!.trim(),
|
|
504
|
+
response_format: request.format ?? 'mp3',
|
|
505
|
+
...(request.speed !== undefined ? { speed: request.speed } : {}),
|
|
506
|
+
}
|
|
507
|
+
const response = await fetchWithTimeout(endpoint, {
|
|
508
|
+
method: 'POST',
|
|
509
|
+
redirect: 'follow',
|
|
510
|
+
headers,
|
|
511
|
+
body: JSON.stringify(body),
|
|
512
|
+
signal,
|
|
513
|
+
}, UPSTREAM_TIMEOUT_MS)
|
|
514
|
+
if (!response.ok) {
|
|
515
|
+
const detail = await response.text().catch(() => '')
|
|
516
|
+
throw new AudioGenError(
|
|
517
|
+
`ElevenLabs ${isSfx ? 'sound effects' : 'TTS'} gateway-compatible API error (HTTP ${response.status})${detail === '' ? '' : `: ${detail.slice(0, 300)}`}`,
|
|
518
|
+
'audio-api-error',
|
|
519
|
+
)
|
|
520
|
+
}
|
|
521
|
+
return normalizeAudioResponse(response, { apiKey: channel.apiKey, fallbackMime: 'audio/mpeg' })
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
/**
|
|
525
|
+
* ElevenLabs 渠道入口:官方端点优先;官方协议被网关(New API 类中转)拒绝时,
|
|
526
|
+
* 自动改用 OpenAI 兼容形态重试,使同一渠道同时兼容 ElevenLabs 官方 API 与
|
|
527
|
+
* ai.farmmx.com 类中转。官方地址(api.elevenlabs.io)直连不触发回退。
|
|
528
|
+
*/
|
|
529
|
+
async function elevenLabs(
|
|
530
|
+
channel: AudioChannel,
|
|
531
|
+
request: GenerateAudioRequest,
|
|
532
|
+
signal?: AbortSignal,
|
|
533
|
+
): Promise<Array<{ data: Uint8Array; mime: string; voiceId?: string }>> {
|
|
534
|
+
if (/elevenlabs\.io/i.test(channel.apiUrl)) {
|
|
535
|
+
return elevenLabsOfficial(channel, request, signal)
|
|
536
|
+
}
|
|
537
|
+
try {
|
|
538
|
+
return await elevenLabsOfficial(channel, request, signal)
|
|
539
|
+
} catch (error) {
|
|
540
|
+
if (!isGatewayRouteMiss(error)) throw error
|
|
541
|
+
}
|
|
542
|
+
return elevenLabsGatewayCompat(channel, request, signal)
|
|
543
|
+
}
|
|
544
|
+
|
|
382
545
|
function minimaxApiBase(base: string): string {
|
|
383
546
|
const trimmed = endpointBase(base)
|
|
384
547
|
return /\/v1$/i.test(trimmed) ? trimmed : `${trimmed}/v1`
|
|
@@ -551,22 +714,19 @@ async function minimax(channel: AudioChannel, request: GenerateAudioRequest, sig
|
|
|
551
714
|
// audio_setting{format, sample_rate, bitrate}。音频输出配置为固定枚举:
|
|
552
715
|
// format mp3|wav|pcm;sample_rate 16000|24000|32000|44100;
|
|
553
716
|
// bitrate 32000|64000|128000|256000,超出枚举的值回退默认。
|
|
717
|
+
// 歌词为空时一律按纯音乐生成(is_instrumental=true):面板/Agent 无论是否
|
|
718
|
+
// 显式勾选「纯音乐」都能出结果,不再因缺歌词报错。
|
|
554
719
|
const MUSIC_FORMATS = new Set(['mp3', 'wav', 'pcm'])
|
|
555
720
|
const MUSIC_SAMPLE_RATES = new Set([16000, 24000, 32000, 44100])
|
|
556
721
|
const MUSIC_BITRATES = new Set([32000, 64000, 128000, 256000])
|
|
557
722
|
const lyrics = request.lyrics?.trim() ?? ''
|
|
558
|
-
|
|
559
|
-
throw new AudioGenError(
|
|
560
|
-
'MiniMax 音乐生成需要歌词(lyrics 参数),或在「纯音乐」模式(is_instrumental=true)下生成;也可让面板/Agent 先为提示词创作一段歌词。',
|
|
561
|
-
'lyrics-required',
|
|
562
|
-
)
|
|
563
|
-
}
|
|
723
|
+
const instrumental = request.isInstrumental === true || lyrics === ''
|
|
564
724
|
const endpoint = `${base}/music_generation`
|
|
565
725
|
const body: Record<string, unknown> = {
|
|
566
726
|
model,
|
|
567
727
|
prompt: request.prompt,
|
|
568
728
|
...(lyrics === '' ? {} : { lyrics }),
|
|
569
|
-
...(
|
|
729
|
+
...(instrumental ? { is_instrumental: true } : {}),
|
|
570
730
|
...(request.duration !== undefined ? { duration: request.duration } : {}),
|
|
571
731
|
audio_setting: {
|
|
572
732
|
format: MUSIC_FORMATS.has(request.format ?? 'mp3') ? (request.format ?? 'mp3') : 'mp3',
|
package/src/client/api.ts
CHANGED
|
@@ -16,6 +16,8 @@ export interface GenerateResponse {
|
|
|
16
16
|
historyError?: string
|
|
17
17
|
/** Resource-library entries created by the generation (auto-save). */
|
|
18
18
|
resources?: Array<{ id: string; name: string; type: string }>
|
|
19
|
+
/** 引擎兜底提示(如:未提供歌词时按纯音乐生成)。 */
|
|
20
|
+
note?: string
|
|
19
21
|
code?: string
|
|
20
22
|
message?: string
|
|
21
23
|
}
|
|
@@ -1171,6 +1171,13 @@
|
|
|
1171
1171
|
grid-column: 1 / -1;
|
|
1172
1172
|
}
|
|
1173
1173
|
|
|
1174
|
+
/* 勾选纯音乐后禁用的歌词框 */
|
|
1175
|
+
.textarea:disabled {
|
|
1176
|
+
opacity: 0.55;
|
|
1177
|
+
cursor: not-allowed;
|
|
1178
|
+
background: var(--dsw-alias-bg-layer-2, #f3f4f6);
|
|
1179
|
+
}
|
|
1180
|
+
|
|
1174
1181
|
/* 模式胶囊 */
|
|
1175
1182
|
.modeIcon {
|
|
1176
1183
|
font-size: 13px;
|
|
@@ -91,7 +91,7 @@ export function presetSupports(preset: string, key: FieldKey, mode: AudioMode):
|
|
|
91
91
|
}
|
|
92
92
|
|
|
93
93
|
const SPECS: Record<FieldKey, Omit<FieldSpec, 'key' | 'presets'>> = {
|
|
94
|
-
duration: { label: '时长(秒)', type: 'number', min: 1, max:
|
|
94
|
+
duration: { label: '时长(秒)', type: 'number', min: 1, max: 200, placeholder: '30', hint: 'duration;MiniMax 音乐 ≤190、ElevenLabs 音乐 3-600(转 ms)、Stability 按模型 190/380' },
|
|
95
95
|
format: { label: '输出格式', type: 'select', options: ['mp3', 'wav', 'pcm', 'flac', 'ogg'], hint: 'format / output_format / response_format' },
|
|
96
96
|
lyrics: { label: '歌词(纯音乐模式可留空;多段用空行分隔)', type: 'text', placeholder: '第一段歌词…\n\n第二段歌词…', hint: 'MiniMax lyrics / ElevenLabs lyrics_text' },
|
|
97
97
|
instrumental: { label: '纯音乐(无歌词/人声)', type: 'checkbox', hint: 'MiniMax is_instrumental / ElevenLabs force_instrumental' },
|
|
@@ -424,6 +424,8 @@ export function StudioView(props: {
|
|
|
424
424
|
props.showToast('已保存到资源库')
|
|
425
425
|
props.onLibraryChanged()
|
|
426
426
|
}
|
|
427
|
+
// 引擎兜底提示(如未提供歌词时自动按纯音乐生成),最后展示避免被覆盖。
|
|
428
|
+
if (response.note !== undefined && response.note.trim() !== '') props.showToast(response.note)
|
|
427
429
|
reload()
|
|
428
430
|
return generated
|
|
429
431
|
}
|
|
@@ -756,7 +758,13 @@ export function StudioView(props: {
|
|
|
756
758
|
return (
|
|
757
759
|
<label className={css.label} key={spec.key} title={spec.hint}>
|
|
758
760
|
<span>{spec.label}</span>
|
|
759
|
-
<textarea
|
|
761
|
+
<textarea
|
|
762
|
+
className={css.textarea}
|
|
763
|
+
disabled={instrumental}
|
|
764
|
+
value={lyrics}
|
|
765
|
+
onChange={event => setLyrics(event.target.value)}
|
|
766
|
+
placeholder={instrumental ? '纯音乐模式无需填写歌词' : '第一段歌词…\n\n第二段歌词…'}
|
|
767
|
+
/>
|
|
760
768
|
</label>
|
|
761
769
|
)
|
|
762
770
|
case 'instrumental':
|
package/src/index.ts
CHANGED
|
@@ -82,7 +82,7 @@ const DEFAULT_ALLOW_AGENT_AUDIO = true
|
|
|
82
82
|
|
|
83
83
|
const SECTION_ORDER = 160
|
|
84
84
|
|
|
85
|
-
export const AUDIOGEN_GUIDANCE = '本机已安装 dsh-audiogen 插件(DSH AI 音频):侧边栏「AI 音频」入口。能力:通过「渠道」对接多个音频生成厂商(OpenAI TTS、ElevenLabs、MiniMax、Stability Audio、自定义 OpenAI 兼容接口),支持 TTS 文本转语音、音乐生成和音效生成。API 地址与密钥在 GUI 设置中按渠道配置,密钥仅存于本机设置文档;生成请求由本地宿主代理转发。Agent 可直接调用 `generate_audio` 提交 TTS/音乐/音效任务,默认等待完成并返回同源音频 URL
|
|
85
|
+
export const AUDIOGEN_GUIDANCE = '本机已安装 dsh-audiogen 插件(DSH AI 音频):侧边栏「AI 音频」入口。能力:通过「渠道」对接多个音频生成厂商(OpenAI TTS、ElevenLabs、MiniMax、Stability Audio、自定义 OpenAI 兼容接口),支持 TTS 文本转语音、音乐生成和音效生成。API 地址与密钥在 GUI 设置中按渠道配置,密钥仅存于本机设置文档;生成请求由本地宿主代理转发。Agent 可直接调用 `generate_audio` 提交 TTS/音乐/音效任务,默认等待完成并返回同源音频 URL;可用 `manage_audio_voices` 浏览/筛选/删除厂商音色(MiniMax、ElevenLabs),再用选定音色的 voice_id 调用 `generate_audio` 生成。限制:生成消耗上游 API 额度;音频内容由上游模型生成;模型只能使用用户在各渠道配置目录中的模型。用户提到「音频 / 语音 / TTS / 配乐 / 音效 / AI 音频」时即指本插件,请据此协作。'
|
|
86
86
|
|
|
87
87
|
function guidanceFor(channels: AudioChannel[], defaultChannelId: string): string {
|
|
88
88
|
if (channels.length === 0) {
|
package/src/protocol.ts
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
export const AUDIOGEN_SETTINGS_NAMESPACE = 'dsh-audiogen'
|
|
9
9
|
|
|
10
10
|
/** Published package version shared by the host updater and the client UI. */
|
|
11
|
-
export const PLUGIN_VERSION = '0.4.
|
|
11
|
+
export const PLUGIN_VERSION = '0.4.14'
|
|
12
12
|
|
|
13
13
|
/** Same-origin route family (loopback-only, mirroring dsh-imagegen). */
|
|
14
14
|
export const SETTINGS_API = {
|
package/src/routes.ts
CHANGED
|
@@ -515,7 +515,12 @@ export function makeRoutes(deps: AudiogenRoutesDeps): WebRoute[] {
|
|
|
515
515
|
// library-save is best-effort: generation and history already succeeded
|
|
516
516
|
}
|
|
517
517
|
}
|
|
518
|
-
|
|
518
|
+
// ---- 音乐兜底提示:未提供歌词时引擎按纯音乐生成,告知前端一声 ----
|
|
519
|
+
const instrumentalFallback = request.mode === 'music'
|
|
520
|
+
&& request.isInstrumental !== true
|
|
521
|
+
&& (request.lyrics === undefined || request.lyrics.trim() === '')
|
|
522
|
+
const note = instrumentalFallback ? '未提供歌词,已按纯音乐生成' : undefined
|
|
523
|
+
writeJson(res, 200, { ok: true, outputs: generated, history, ...(resources === undefined ? {} : { resources }), ...(note === undefined ? {} : { note }) })
|
|
519
524
|
} catch (error) {
|
|
520
525
|
const code = error instanceof AudioGenError ? error.code : 'generate-failed'
|
|
521
526
|
writeJson(res, 200, { ok: false, code, message: messageOf(error) })
|