dsh-audiogen 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +67 -0
- package/cordis.patch.yml +8 -0
- package/lib/client.js +2457 -0
- package/lib/client.js.map +1 -0
- package/lib/index.js +1345 -0
- package/package.json +93 -0
- package/skills/design/SKILL.md +13 -0
- package/skills/music/SKILL.md +16 -0
- package/skills/sfx/SKILL.md +16 -0
- package/skills/tts/SKILL.md +18 -0
- package/src/agent-audio-tools.ts +190 -0
- package/src/audio-engine.ts +377 -0
- package/src/audio-presets.ts +80 -0
- package/src/audio-store.ts +131 -0
- package/src/client/AudioGenPanel.tsx +206 -0
- package/src/client/SettingsCard.tsx +337 -0
- package/src/client/api.ts +36 -0
- package/src/client/audio-panel.module.css +198 -0
- package/src/client/audio-toolview.module.css +69 -0
- package/src/client/audio-toolview.tsx +119 -0
- package/src/client/channels-form.ts +263 -0
- package/src/client/controller.ts +44 -0
- package/src/client/css-modules.d.ts +5 -0
- package/src/client/helpers.ts +27 -0
- package/src/client/index.ts +103 -0
- package/src/client/locales.ts +133 -0
- package/src/client/mount.tsx +96 -0
- package/src/client/panel.module.css +1566 -0
- package/src/client/settings-card.module.css +1023 -0
- package/src/client/settings-form.ts +336 -0
- package/src/client/settings-scope.ts +289 -0
- package/src/client/sidebar-entry.ts +115 -0
- package/src/index.ts +201 -0
- package/src/protocol.ts +179 -0
- package/src/routes.ts +386 -0
|
@@ -0,0 +1,377 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Upstream audio proxy engine.
|
|
3
|
+
*
|
|
4
|
+
* Normalizes a provider response into base64 audio payloads so the browser
|
|
5
|
+
* never needs to talk to the upstream directly. Supports a small set of
|
|
6
|
+
* built-in vendor presets plus a best-effort OpenAI-compatible / generic path.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import type { AudioMode, GenerateAudioRequest, GeneratedAudio } from './protocol.ts'
|
|
10
|
+
|
|
11
|
+
/** Resolved channel (key included; never logged). */
|
|
12
|
+
export interface AudioChannel {
|
|
13
|
+
id: string
|
|
14
|
+
preset: string
|
|
15
|
+
name: string
|
|
16
|
+
apiUrl: string
|
|
17
|
+
apiKey: string
|
|
18
|
+
models: Array<{ alias: string; id: string }>
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** An audio generation failure with a user-presentable message. */
|
|
22
|
+
export class AudioGenError extends Error {
|
|
23
|
+
readonly code: string
|
|
24
|
+
|
|
25
|
+
constructor(message: string, code = 'audio-generate-failed') {
|
|
26
|
+
super(message)
|
|
27
|
+
this.name = 'AudioGenError'
|
|
28
|
+
this.code = code
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** Total budget for one upstream generation call. Audio models can be slow. */
|
|
33
|
+
const UPSTREAM_TIMEOUT_MS = 240_000
|
|
34
|
+
/** Budget for downloading one result audio URL. */
|
|
35
|
+
const AUDIO_FETCH_TIMEOUT_MS = 60_000
|
|
36
|
+
|
|
37
|
+
function requestSignal(source: AbortSignal | undefined, timeoutMs: number): { signal: AbortSignal; dispose: () => void } {
|
|
38
|
+
const controller = new AbortController()
|
|
39
|
+
const abortFromSource = () => { controller.abort(source?.reason) }
|
|
40
|
+
if (source?.aborted === true) abortFromSource()
|
|
41
|
+
else source?.addEventListener('abort', abortFromSource, { once: true })
|
|
42
|
+
const timeout = setTimeout(() => { controller.abort(new DOMException('The operation timed out.', 'TimeoutError')) }, timeoutMs)
|
|
43
|
+
timeout.unref?.()
|
|
44
|
+
return {
|
|
45
|
+
signal: controller.signal,
|
|
46
|
+
dispose: () => {
|
|
47
|
+
clearTimeout(timeout)
|
|
48
|
+
source?.removeEventListener('abort', abortFromSource)
|
|
49
|
+
},
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Detect a few common audio container formats from magic bytes. */
|
|
54
|
+
export function detectAudioMime(data: Uint8Array): string | undefined {
|
|
55
|
+
if (data.length >= 4 && data[0] === 0x52 && data[1] === 0x49 && data[2] === 0x46 && data[3] === 0x46) return 'audio/wav'
|
|
56
|
+
if (data.length >= 3 && data[0] === 0x49 && data[1] === 0x44 && data[2] === 0x33) return 'audio/mpeg'
|
|
57
|
+
if (data.length >= 4 && data[0] === 0x66 && data[1] === 0x4c && data[2] === 0x61 && data[3] === 0x43) return 'audio/flac'
|
|
58
|
+
if (data.length >= 4 && data[0] === 0x4f && data[1] === 0x67 && data[2] === 0x67 && data[3] === 0x53) return 'audio/ogg'
|
|
59
|
+
if (data.length >= 4 && data[0] === 0x00 && data[1] === 0x00 && data[2] === 0x00 && data[3] === 0x18) return 'audio/mp4'
|
|
60
|
+
if (data.length >= 4 && data[0] === 0x23 && data[1] === 0x21 && data[2] === 0x41 && data[3] === 0x4d) return 'audio/aiff'
|
|
61
|
+
return undefined
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function mimeFromContentType(value: string | null): string | undefined {
|
|
65
|
+
if (value === null || value === '') return undefined
|
|
66
|
+
return value.split(';')[0]!.trim().toLowerCase()
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function audioMime(data: Uint8Array, contentType: string | null): string {
|
|
70
|
+
return detectAudioMime(data) ?? mimeFromContentType(contentType) ?? 'audio/mpeg'
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function isPreset(channel: AudioChannel, id: string): boolean {
|
|
74
|
+
return channel.preset === id || channel.apiUrl.toLowerCase().includes(id)
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function isOpenAICompatible(channel: AudioChannel, mode: AudioMode): boolean {
|
|
78
|
+
return isPreset(channel, 'openai')
|
|
79
|
+
|| /(^|\/)(v\d+\/)?audio\/speech$/i.test(channel.apiUrl.trim())
|
|
80
|
+
|| (channel.preset === 'custom' && mode === 'tts')
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function isElevenLabs(channel: AudioChannel): boolean {
|
|
84
|
+
return isPreset(channel, 'elevenlabs') || /elevenlabs/i.test(channel.apiUrl)
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function isMiniMax(channel: AudioChannel): boolean {
|
|
88
|
+
return isPreset(channel, 'minimax') || /minimax/i.test(channel.apiUrl)
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function isStability(channel: AudioChannel): boolean {
|
|
92
|
+
return isPreset(channel, 'stability') || /stability\.ai/i.test(channel.apiUrl)
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function endpointBase(url: string): string {
|
|
96
|
+
return url.trim().replace(/\/+$/, '')
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function bytesToBase64(data: Uint8Array): string {
|
|
100
|
+
return Buffer.from(data).toString('base64')
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** Parse a base64 payload that may carry a data: prefix. */
|
|
104
|
+
function bareBase64(value: string): string {
|
|
105
|
+
const match = /^data:([^;,]+)?(;base64)?,(.*)$/s.exec(value.trim())
|
|
106
|
+
if (match !== null && match[3] !== undefined) return match[3]
|
|
107
|
+
return value
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function asBase64(value: unknown): string | undefined {
|
|
111
|
+
if (typeof value === 'string' && value.trim() !== '') return bareBase64(value)
|
|
112
|
+
return undefined
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** Recursively look for the first likely base64 audio string in a JSON payload. */
|
|
116
|
+
function findBase64Audio(value: unknown): string | undefined {
|
|
117
|
+
if (typeof value === 'string' && value.length > 100 && !/^https?:\/\//i.test(value.trim())) return value
|
|
118
|
+
if (Array.isArray(value)) {
|
|
119
|
+
for (const item of value) {
|
|
120
|
+
const found = findBase64Audio(item)
|
|
121
|
+
if (found !== undefined) return found
|
|
122
|
+
}
|
|
123
|
+
return undefined
|
|
124
|
+
}
|
|
125
|
+
if (value === null || typeof value !== 'object') return undefined
|
|
126
|
+
const record = value as Record<string, unknown>
|
|
127
|
+
for (const key of ['audio', 'b64_json', 'base64', 'data', 'output', 'result', 'value']) {
|
|
128
|
+
const candidate = record[key]
|
|
129
|
+
const found = findBase64Audio(candidate)
|
|
130
|
+
if (found !== undefined) return found
|
|
131
|
+
}
|
|
132
|
+
return undefined
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** Find the first provider-returned audio URL in a JSON payload. */
|
|
136
|
+
function findAudioUrl(value: unknown): string | undefined {
|
|
137
|
+
if (typeof value === 'string' && /^https?:\/\//i.test(value)) return value
|
|
138
|
+
if (Array.isArray(value)) {
|
|
139
|
+
for (const item of value) {
|
|
140
|
+
const found = findAudioUrl(item)
|
|
141
|
+
if (found !== undefined) return found
|
|
142
|
+
}
|
|
143
|
+
return undefined
|
|
144
|
+
}
|
|
145
|
+
if (value === null || typeof value !== 'object') return undefined
|
|
146
|
+
const record = value as Record<string, unknown>
|
|
147
|
+
for (const key of ['url', 'audio_url', 'href', 'link', 'audio', 'data', 'output', 'result', 'value']) {
|
|
148
|
+
const candidate = record[key]
|
|
149
|
+
const found = findAudioUrl(candidate)
|
|
150
|
+
if (found !== undefined) return found
|
|
151
|
+
}
|
|
152
|
+
return undefined
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
async function fetchWithTimeout(url: string, init: RequestInit, timeoutMs: number): Promise<Response> {
|
|
156
|
+
const budget = requestSignal(init.signal as AbortSignal | undefined, timeoutMs)
|
|
157
|
+
try {
|
|
158
|
+
return await fetch(url, { ...init, signal: budget.signal })
|
|
159
|
+
} finally {
|
|
160
|
+
budget.dispose()
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
async function normalizeAudioResponse(
|
|
165
|
+
response: Response,
|
|
166
|
+
options: { apiKey: string; fallbackMime?: string },
|
|
167
|
+
): Promise<Array<{ data: Uint8Array; mime: string }>> {
|
|
168
|
+
if (!response.ok) {
|
|
169
|
+
let detail = ''
|
|
170
|
+
try {
|
|
171
|
+
const text = await response.text()
|
|
172
|
+
detail = text.slice(0, 500)
|
|
173
|
+
} catch {
|
|
174
|
+
// keep empty
|
|
175
|
+
}
|
|
176
|
+
throw new AudioGenError(`audio API error (HTTP ${response.status})${detail === '' ? '' : `: ${detail}`}`, 'audio-api-error')
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
const contentType = mimeFromContentType(response.headers.get('content-type')) ?? options.fallbackMime
|
|
180
|
+
const buffer = new Uint8Array(await response.arrayBuffer())
|
|
181
|
+
// Some providers return binary audio directly, others JSON with base64/url.
|
|
182
|
+
const text = new TextDecoder().decode(buffer).trim()
|
|
183
|
+
if (text.startsWith('{') || text.startsWith('[')) {
|
|
184
|
+
let parsed: unknown
|
|
185
|
+
try {
|
|
186
|
+
parsed = JSON.parse(text)
|
|
187
|
+
} catch {
|
|
188
|
+
throw new AudioGenError('audio endpoint returned an unprocessable response body', 'audio-bad-response')
|
|
189
|
+
}
|
|
190
|
+
const base64 = findBase64Audio(parsed)
|
|
191
|
+
if (base64 !== undefined && base64.length > 0) {
|
|
192
|
+
let data: Uint8Array
|
|
193
|
+
try {
|
|
194
|
+
data = new Uint8Array(Buffer.from(base64, 'base64'))
|
|
195
|
+
} catch {
|
|
196
|
+
throw new AudioGenError('audio endpoint returned invalid base64', 'audio-bad-response')
|
|
197
|
+
}
|
|
198
|
+
return [{ data, mime: detectAudioMime(data) ?? contentType ?? 'audio/mpeg' }]
|
|
199
|
+
}
|
|
200
|
+
const url = findAudioUrl(parsed)
|
|
201
|
+
if (url !== undefined) {
|
|
202
|
+
const fetched = await fetchWithTimeout(url, {
|
|
203
|
+
headers: options.apiKey === '' ? {} : { authorization: `Bearer ${options.apiKey}` },
|
|
204
|
+
redirect: 'follow',
|
|
205
|
+
}, AUDIO_FETCH_TIMEOUT_MS)
|
|
206
|
+
if (!fetched.ok) throw new AudioGenError(`failed to fetch generated audio url: HTTP ${fetched.status}`, 'audio-url-fetch-failed')
|
|
207
|
+
const data = new Uint8Array(await fetched.arrayBuffer())
|
|
208
|
+
return [{ data, mime: audioMime(data, fetched.headers.get('content-type')) }]
|
|
209
|
+
}
|
|
210
|
+
throw new AudioGenError('audio endpoint returned neither binary nor base64/url audio', 'audio-empty-result')
|
|
211
|
+
}
|
|
212
|
+
return [{ data: buffer, mime: audioMime(buffer, response.headers.get('content-type') ?? contentType ?? null) }]
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
async function openAITTS(channel: AudioChannel, request: GenerateAudioRequest, signal?: AbortSignal): Promise<Array<{ data: Uint8Array; mime: string }>> {
|
|
216
|
+
const base = endpointBase(channel.apiUrl)
|
|
217
|
+
const endpoint = /\/audio\/speech(\?|$)/i.test(base) ? base : `${base}/audio/speech`
|
|
218
|
+
const model = (request.upstream ?? request.model) || 'tts-1'
|
|
219
|
+
const voice = request.voice ?? 'alloy'
|
|
220
|
+
const body: Record<string, unknown> = {
|
|
221
|
+
model,
|
|
222
|
+
input: request.prompt,
|
|
223
|
+
voice,
|
|
224
|
+
response_format: request.format ?? 'mp3',
|
|
225
|
+
...(request.speed !== undefined ? { speed: request.speed } : {}),
|
|
226
|
+
}
|
|
227
|
+
const response = await fetchWithTimeout(endpoint, {
|
|
228
|
+
method: 'POST',
|
|
229
|
+
redirect: 'error',
|
|
230
|
+
headers: {
|
|
231
|
+
authorization: `Bearer ${channel.apiKey.trim()}`,
|
|
232
|
+
'content-type': 'application/json',
|
|
233
|
+
accept: 'audio/mpeg, application/json',
|
|
234
|
+
},
|
|
235
|
+
body: JSON.stringify(body),
|
|
236
|
+
signal,
|
|
237
|
+
}, UPSTREAM_TIMEOUT_MS)
|
|
238
|
+
return normalizeAudioResponse(response, { apiKey: channel.apiKey, fallbackMime: 'audio/mpeg' })
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
async function elevenLabs(channel: AudioChannel, request: GenerateAudioRequest, signal?: AbortSignal): Promise<Array<{ data: Uint8Array; mime: string }>> {
|
|
242
|
+
const base = endpointBase(channel.apiUrl)
|
|
243
|
+
const model = (request.upstream ?? request.model) || 'eleven_multilingual_v2'
|
|
244
|
+
const voiceId = (request.voice ?? request.model ?? model).trim()
|
|
245
|
+
const endpoint = `${base}/text-to-speech/${encodeURIComponent(voiceId)}`
|
|
246
|
+
const body: Record<string, unknown> = {
|
|
247
|
+
text: request.prompt,
|
|
248
|
+
model_id: model,
|
|
249
|
+
voice_settings: {
|
|
250
|
+
stability: 0.5,
|
|
251
|
+
similarity_boost: 0.75,
|
|
252
|
+
style: 0.0,
|
|
253
|
+
use_speaker_boost: true,
|
|
254
|
+
...(request.speed !== undefined ? { speed: request.speed } : {}),
|
|
255
|
+
},
|
|
256
|
+
}
|
|
257
|
+
const response = await fetchWithTimeout(endpoint, {
|
|
258
|
+
method: 'POST',
|
|
259
|
+
redirect: 'error',
|
|
260
|
+
headers: {
|
|
261
|
+
'xi-api-key': channel.apiKey.trim(),
|
|
262
|
+
'content-type': 'application/json',
|
|
263
|
+
accept: 'audio/mpeg, application/json',
|
|
264
|
+
},
|
|
265
|
+
body: JSON.stringify(body),
|
|
266
|
+
signal,
|
|
267
|
+
}, UPSTREAM_TIMEOUT_MS)
|
|
268
|
+
return normalizeAudioResponse(response, { apiKey: channel.apiKey, fallbackMime: 'audio/mpeg' })
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
async function minimax(channel: AudioChannel, request: GenerateAudioRequest, signal?: AbortSignal): Promise<Array<{ data: Uint8Array; mime: string }>> {
|
|
272
|
+
const base = endpointBase(channel.apiUrl)
|
|
273
|
+
const endpoint = /\/t2a_v2(\?|$)/i.test(base) ? base : `${base}/t2a_v2`
|
|
274
|
+
const model = (request.upstream ?? request.model) || 'speech-01-turbo'
|
|
275
|
+
const voice = request.voice ?? request.model ?? ''
|
|
276
|
+
const body: Record<string, unknown> = {
|
|
277
|
+
model,
|
|
278
|
+
text: request.prompt,
|
|
279
|
+
stream: false,
|
|
280
|
+
...(voice === '' ? {} : { voice_setting: {
|
|
281
|
+
voice_id: voice,
|
|
282
|
+
...(request.speed !== undefined ? { speed: request.speed } : {}),
|
|
283
|
+
vol: 1,
|
|
284
|
+
pitch: 0,
|
|
285
|
+
} }),
|
|
286
|
+
audio_setting: {
|
|
287
|
+
format: request.format ?? 'mp3',
|
|
288
|
+
sample_rate: 32000,
|
|
289
|
+
bitrate: 128000,
|
|
290
|
+
},
|
|
291
|
+
}
|
|
292
|
+
const response = await fetchWithTimeout(endpoint, {
|
|
293
|
+
method: 'POST',
|
|
294
|
+
redirect: 'error',
|
|
295
|
+
headers: {
|
|
296
|
+
authorization: `Bearer ${channel.apiKey.trim()}`,
|
|
297
|
+
'content-type': 'application/json',
|
|
298
|
+
accept: 'application/json, audio/mpeg',
|
|
299
|
+
},
|
|
300
|
+
body: JSON.stringify(body),
|
|
301
|
+
signal,
|
|
302
|
+
}, UPSTREAM_TIMEOUT_MS)
|
|
303
|
+
return normalizeAudioResponse(response, { apiKey: channel.apiKey, fallbackMime: 'audio/mpeg' })
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
async function stabilityAudio(channel: AudioChannel, request: GenerateAudioRequest, signal?: AbortSignal): Promise<Array<{ data: Uint8Array; mime: string }>> {
|
|
307
|
+
const base = endpointBase(channel.apiUrl)
|
|
308
|
+
const endpoint = /\/generation(\?|$)/i.test(base) ? base : `${base}/generation`
|
|
309
|
+
const model = (request.upstream ?? request.model) || 'stable-audio-2.0'
|
|
310
|
+
const body: Record<string, unknown> = {
|
|
311
|
+
model,
|
|
312
|
+
prompt: request.prompt,
|
|
313
|
+
...(request.duration !== undefined ? { duration: request.duration } : {}),
|
|
314
|
+
...(request.format !== undefined ? { output_format: request.format } : {}),
|
|
315
|
+
}
|
|
316
|
+
const response = await fetchWithTimeout(endpoint, {
|
|
317
|
+
method: 'POST',
|
|
318
|
+
redirect: 'error',
|
|
319
|
+
headers: {
|
|
320
|
+
authorization: `Bearer ${channel.apiKey.trim()}`,
|
|
321
|
+
'content-type': 'application/json',
|
|
322
|
+
accept: 'application/json, audio/mpeg, audio/wav',
|
|
323
|
+
},
|
|
324
|
+
body: JSON.stringify(body),
|
|
325
|
+
signal,
|
|
326
|
+
}, UPSTREAM_TIMEOUT_MS)
|
|
327
|
+
return normalizeAudioResponse(response, { apiKey: channel.apiKey, fallbackMime: 'audio/mpeg' })
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
async function genericAudio(channel: AudioChannel, request: GenerateAudioRequest, signal?: AbortSignal): Promise<Array<{ data: Uint8Array; mime: string }>> {
|
|
331
|
+
const base = endpointBase(channel.apiUrl)
|
|
332
|
+
if (request.mode === 'tts' && !/\/generate(\?|$)/i.test(base)) {
|
|
333
|
+
return openAITTS(channel, request, signal)
|
|
334
|
+
}
|
|
335
|
+
const endpoint = /\/generate(\?|$)/i.test(base) ? base : `${base}/generate`
|
|
336
|
+
const model = (request.upstream ?? request.model) || 'default'
|
|
337
|
+
const body: Record<string, unknown> = {
|
|
338
|
+
model,
|
|
339
|
+
prompt: request.prompt,
|
|
340
|
+
mode: request.mode,
|
|
341
|
+
...(request.voice !== undefined ? { voice: request.voice } : {}),
|
|
342
|
+
...(request.duration !== undefined ? { duration: request.duration } : {}),
|
|
343
|
+
...(request.format !== undefined ? { output_format: request.format } : {}),
|
|
344
|
+
}
|
|
345
|
+
const response = await fetchWithTimeout(endpoint, {
|
|
346
|
+
method: 'POST',
|
|
347
|
+
redirect: 'error',
|
|
348
|
+
headers: {
|
|
349
|
+
authorization: `Bearer ${channel.apiKey.trim()}`,
|
|
350
|
+
'content-type': 'application/json',
|
|
351
|
+
accept: 'application/json, audio/mpeg, audio/wav',
|
|
352
|
+
},
|
|
353
|
+
body: JSON.stringify(body),
|
|
354
|
+
signal,
|
|
355
|
+
}, UPSTREAM_TIMEOUT_MS)
|
|
356
|
+
return normalizeAudioResponse(response, { apiKey: channel.apiKey, fallbackMime: 'audio/mpeg' })
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
/**
|
|
360
|
+
* Generate one or more audio outputs from a configured channel.
|
|
361
|
+
* @returns normalized generated audio (base64, mime, bytes).
|
|
362
|
+
*/
|
|
363
|
+
export async function generateAudio(
|
|
364
|
+
channel: AudioChannel,
|
|
365
|
+
request: GenerateAudioRequest,
|
|
366
|
+
signal?: AbortSignal,
|
|
367
|
+
): Promise<Array<{ data: Uint8Array; mime: string }>> {
|
|
368
|
+
if (channel.apiUrl.trim() === '') throw new AudioGenError('channel API URL is not configured', 'audio-no-endpoint')
|
|
369
|
+
if (channel.apiKey.trim() === '') throw new AudioGenError('channel API key is not configured', 'audio-no-key')
|
|
370
|
+
if (request.prompt.trim() === '') throw new AudioGenError('audio prompt/text is required', 'audio-empty-prompt')
|
|
371
|
+
|
|
372
|
+
if (isElevenLabs(channel)) return elevenLabs(channel, request, signal)
|
|
373
|
+
if (isMiniMax(channel)) return minimax(channel, request, signal)
|
|
374
|
+
if (isStability(channel)) return stabilityAudio(channel, request, signal)
|
|
375
|
+
if (isOpenAICompatible(channel, request.mode)) return openAITTS(channel, request, signal)
|
|
376
|
+
return genericAudio(channel, request, signal)
|
|
377
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Built-in audio provider catalog (presets).
|
|
3
|
+
* Framework-free pure data; served to the settings card through a host route.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import type { ModelMapping } from './protocol.ts'
|
|
7
|
+
|
|
8
|
+
/** One built-in provider the settings card can instantiate a channel from. */
|
|
9
|
+
export interface AudioPresetProvider {
|
|
10
|
+
/** Stable preset id stored on channels created from it ('' = custom). */
|
|
11
|
+
id: string
|
|
12
|
+
/** Display name shown in the picker (also the channel's default name). */
|
|
13
|
+
name: string
|
|
14
|
+
/** Official base URL prefilled into the channel. */
|
|
15
|
+
apiUrl: string
|
|
16
|
+
/** One-line description shown in the picker. */
|
|
17
|
+
hint: string
|
|
18
|
+
/** Known model/voice catalog prefilled into the channel. */
|
|
19
|
+
models: ModelMapping[]
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export const AUDIO_PRESETS: AudioPresetProvider[] = [
|
|
23
|
+
{
|
|
24
|
+
id: 'openai-tts',
|
|
25
|
+
name: 'OpenAI · TTS',
|
|
26
|
+
apiUrl: 'https://api.openai.com/v1',
|
|
27
|
+
hint: 'OpenAI 官方语音合成接口(/audio/speech)',
|
|
28
|
+
models: [
|
|
29
|
+
{ alias: 'tts-1', id: 'tts-1' },
|
|
30
|
+
{ alias: 'tts-1-hd', id: 'tts-1-hd' },
|
|
31
|
+
{ alias: 'gpt-4o-mini-tts', id: 'gpt-4o-mini-tts' },
|
|
32
|
+
],
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
id: 'elevenlabs',
|
|
36
|
+
name: 'ElevenLabs',
|
|
37
|
+
apiUrl: 'https://api.elevenlabs.io/v1',
|
|
38
|
+
hint: 'ElevenLabs TTS;模型列表请填写你的 Voice ID(如 Rachel / Adam 等别名)',
|
|
39
|
+
models: [
|
|
40
|
+
{ alias: 'Rachel', id: '21m00Tcm4TlvDq8ikWAM' },
|
|
41
|
+
{ alias: 'Adam', id: 'pNInz6obpgDQGcFmaJgB' },
|
|
42
|
+
{ alias: 'Antoni', id: 'ErXwobaYiN019PkySvjV' },
|
|
43
|
+
{ alias: 'Bella', id: 'EXAVITQu4vr4xnSDxMaL' },
|
|
44
|
+
],
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
id: 'minimax',
|
|
48
|
+
name: 'MiniMax',
|
|
49
|
+
apiUrl: 'https://api.minimax.chat/v1',
|
|
50
|
+
hint: 'MiniMax 语音合成(T2A);需在 API URL 后按官方要求携带 GroupId 或使用完整接口地址',
|
|
51
|
+
models: [
|
|
52
|
+
{ alias: 'speech-01-turbo', id: 'speech-01-turbo' },
|
|
53
|
+
{ alias: 'speech-01-hd', id: 'speech-01-hd' },
|
|
54
|
+
{ alias: 'speech-02-turbo', id: 'speech-02-turbo' },
|
|
55
|
+
{ alias: 'speech-02-hd', id: 'speech-02-hd' },
|
|
56
|
+
],
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
id: 'stability-audio',
|
|
60
|
+
name: 'Stability AI · 音频',
|
|
61
|
+
apiUrl: 'https://api.stability.ai/v2beta/audio',
|
|
62
|
+
hint: 'Stability AI 音乐/音效生成(stable-audio 系列)',
|
|
63
|
+
models: [
|
|
64
|
+
{ alias: 'stable-audio-2.0', id: 'stable-audio-2.0' },
|
|
65
|
+
{ alias: 'stable-audio-1.0', id: 'stable-audio-1.0' },
|
|
66
|
+
],
|
|
67
|
+
},
|
|
68
|
+
{
|
|
69
|
+
id: 'custom',
|
|
70
|
+
name: '自定义渠道',
|
|
71
|
+
apiUrl: '',
|
|
72
|
+
hint: '任意兼容接口;支持 OpenAI 兼容 TTS,或返回音频字节 / JSON 的通用 POST',
|
|
73
|
+
models: [],
|
|
74
|
+
},
|
|
75
|
+
]
|
|
76
|
+
|
|
77
|
+
/** Look up one built-in provider by id. */
|
|
78
|
+
export function audioPresetById(id: string): AudioPresetProvider | undefined {
|
|
79
|
+
return AUDIO_PRESETS.find(preset => preset.id === id)
|
|
80
|
+
}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Host-side persistence for generated audio and generation history.
|
|
3
|
+
* Files live under ~/.dsh/dsh-audiogen/audio/; history is one JSON document.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { mkdir, readFile, writeFile, readdir, unlink } from 'node:fs/promises'
|
|
7
|
+
import { randomUUID } from 'node:crypto'
|
|
8
|
+
import path from 'node:path'
|
|
9
|
+
import os from 'node:os'
|
|
10
|
+
import type { HistoryEntry, HistoryEntryInput } from './protocol.ts'
|
|
11
|
+
import { HISTORY_MAX } from './protocol.ts'
|
|
12
|
+
|
|
13
|
+
function dshHome(): string {
|
|
14
|
+
return process.env.DSH_HOME ?? path.join(os.homedir(), '.dsh')
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export const AUDIO_DATA_DIR = path.join(dshHome(), 'dsh-audiogen', 'audio')
|
|
18
|
+
const HISTORY_FILE = path.join(dshHome(), 'dsh-audiogen', 'history.json')
|
|
19
|
+
|
|
20
|
+
async function ensureDir(): Promise<void> {
|
|
21
|
+
await mkdir(AUDIO_DATA_DIR, { recursive: true })
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function safeName(id: string): string {
|
|
25
|
+
return id.replace(/[^a-zA-Z0-9._-]/g, '_')
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Persist one generated audio file. Returns its metadata and public id. */
|
|
29
|
+
export async function saveAudioFile(data: Uint8Array, mime: string, name?: string): Promise<{
|
|
30
|
+
id: string
|
|
31
|
+
file: string
|
|
32
|
+
mime: string
|
|
33
|
+
bytes: number
|
|
34
|
+
name?: string
|
|
35
|
+
}> {
|
|
36
|
+
await ensureDir()
|
|
37
|
+
const id = randomUUID()
|
|
38
|
+
const extension = mime.split('/')[1]?.replace('mpeg', 'mp3') ?? 'bin'
|
|
39
|
+
const file = `${id}.${extension}`
|
|
40
|
+
await writeFile(path.join(AUDIO_DATA_DIR, file), data)
|
|
41
|
+
return {
|
|
42
|
+
id,
|
|
43
|
+
file,
|
|
44
|
+
mime,
|
|
45
|
+
bytes: data.byteLength,
|
|
46
|
+
...(name === undefined ? {} : { name }),
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** Read a persisted audio file by its id/file name. */
|
|
51
|
+
export async function readAudioFile(file: string): Promise<{ data: Buffer; mime: string; bytes: number } | undefined> {
|
|
52
|
+
const safe = safeName(file)
|
|
53
|
+
const full = path.join(AUDIO_DATA_DIR, safe)
|
|
54
|
+
if (!full.startsWith(AUDIO_DATA_DIR)) return undefined
|
|
55
|
+
try {
|
|
56
|
+
const data = await readFile(full)
|
|
57
|
+
return { data, mime: mimeFromFile(safe), bytes: data.byteLength }
|
|
58
|
+
} catch {
|
|
59
|
+
return undefined
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function mimeFromFile(file: string): string {
|
|
64
|
+
const ext = path.extname(file).toLowerCase()
|
|
65
|
+
switch (ext) {
|
|
66
|
+
case '.wav': return 'audio/wav'
|
|
67
|
+
case '.mp3': return 'audio/mpeg'
|
|
68
|
+
case '.flac': return 'audio/flac'
|
|
69
|
+
case '.ogg': return 'audio/ogg'
|
|
70
|
+
case '.m4a': return 'audio/mp4'
|
|
71
|
+
case '.aac': return 'audio/aac'
|
|
72
|
+
case '.aiff': return 'audio/aiff'
|
|
73
|
+
default: return 'application/octet-stream'
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
async function readHistory(): Promise<HistoryEntry[]> {
|
|
78
|
+
try {
|
|
79
|
+
const text = await readFile(HISTORY_FILE, 'utf8')
|
|
80
|
+
const parsed = JSON.parse(text) as unknown
|
|
81
|
+
return Array.isArray(parsed) ? parsed as HistoryEntry[] : []
|
|
82
|
+
} catch {
|
|
83
|
+
return []
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
async function writeHistory(entries: HistoryEntry[]): Promise<void> {
|
|
88
|
+
await mkdir(path.dirname(HISTORY_FILE), { recursive: true })
|
|
89
|
+
await writeFile(HISTORY_FILE, JSON.stringify(entries, null, 2))
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** Append one history entry and enforce the cap. */
|
|
93
|
+
export async function appendHistory(entry: HistoryEntryInput): Promise<HistoryEntry[]> {
|
|
94
|
+
const list = await readHistory()
|
|
95
|
+
const next: HistoryEntry[] = [{
|
|
96
|
+
id: entry.id,
|
|
97
|
+
createdAt: entry.createdAt,
|
|
98
|
+
mode: entry.mode,
|
|
99
|
+
model: entry.model,
|
|
100
|
+
prompt: entry.prompt,
|
|
101
|
+
...(entry.voice === undefined ? {} : { voice: entry.voice }),
|
|
102
|
+
...(entry.speed === undefined ? {} : { speed: entry.speed }),
|
|
103
|
+
...(entry.duration === undefined ? {} : { duration: entry.duration }),
|
|
104
|
+
...(entry.format === undefined ? {} : { format: entry.format }),
|
|
105
|
+
audio: entry.audio.map(audio => ({
|
|
106
|
+
url: audio.url,
|
|
107
|
+
mime: audio.mime,
|
|
108
|
+
...(audio.duration === undefined ? {} : { duration: audio.duration }),
|
|
109
|
+
})),
|
|
110
|
+
...(entry.channelId === undefined ? {} : { channelId: entry.channelId }),
|
|
111
|
+
...(entry.channel === undefined ? {} : { channel: entry.channel }),
|
|
112
|
+
}, ...list].slice(0, HISTORY_MAX)
|
|
113
|
+
await writeHistory(next)
|
|
114
|
+
return next
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
export async function listHistory(): Promise<HistoryEntry[]> {
|
|
118
|
+
return readHistory()
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
export async function removeHistory(id: string): Promise<HistoryEntry[]> {
|
|
122
|
+
const list = await readHistory()
|
|
123
|
+
const next = list.filter(entry => entry.id !== id)
|
|
124
|
+
await writeHistory(next)
|
|
125
|
+
return next
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export async function clearHistory(): Promise<HistoryEntry[]> {
|
|
129
|
+
await writeHistory([])
|
|
130
|
+
return []
|
|
131
|
+
}
|