dsh-voice 0.8.1 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/README.en.md +17 -15
- package/README.md +17 -15
- package/docs/USAGE.en.md +39 -16
- package/docs/USAGE.md +41 -18
- package/lib/client.js +5067 -0
- package/lib/config.d.ts +7 -0
- package/lib/config.js +21 -0
- package/lib/edge-tts.d.ts +4 -0
- package/lib/edge-tts.js +38 -1
- package/lib/host-config.js +7 -0
- package/lib/index.d.ts +3 -1
- package/lib/index.js +4 -1
- package/lib/openai-tts.d.ts +19 -6
- package/lib/openai-tts.js +163 -2
- package/lib/stt.d.ts +19 -0
- package/lib/stt.js +157 -9
- package/lib/tools.d.ts +4 -2
- package/lib/tools.js +380 -70
- package/lib/tts-text.d.ts +46 -3
- package/lib/tts-text.js +345 -16
- package/lib/web.d.ts +66 -0
- package/lib/web.js +677 -0
- package/locale/en.json +1 -1
- package/locale/zh.json +1 -1
- package/package.json +10 -5
- package/skills/voice-workflow/SKILL.md +51 -19
- package/lib/update-client.js +0 -745
package/lib/tools.js
CHANGED
|
@@ -10,8 +10,8 @@ import { fetchEdgeVoices, synthesizeSpeech, synthesizeSpeechWithMetadata } from
|
|
|
10
10
|
import { rateToSpeed, synthesizeOpenAiSpeech } from './openai-tts.js';
|
|
11
11
|
import { assertAudioFile, assertWorkspaceFile, ensureWorkspaceDirectory, resolveOutputPath, writeWorkspaceFile } from './paths.js';
|
|
12
12
|
import { createProxyFetch } from './proxy-fetch.js';
|
|
13
|
-
import { assertAudioSize, formatSttSubtitles, hasFfmpeg, prepareAudioForAsr, transcribe } from './stt.js';
|
|
14
|
-
import { cleanMarkdownForSpeech, concatMp3Buffers, createSilenceMp3,
|
|
13
|
+
import { assertAudioSize, formatDiarizedTranscript, formatSttSubtitles, hasFfmpeg, prepareAudioForAsr, splitStereoChannelsForDiarization, transcribe } from './stt.js';
|
|
14
|
+
import { applyPronunciationsPlain, cleanMarkdownForSpeech, concatMp3Buffers, createSilenceMp3, formatSubtitles, inspectMp3Buffer, mixAudioWithBgm, normalizeAudioLoudness, offsetCues, parseSubtitleFile, renderPreviewHtml, scaleRatePercent, splitTextIntoChunks, } from './tts-text.js';
|
|
15
15
|
import { filterVoices, isValidVoiceId, resolvePersona, VOICE_PERSONAS, VOICES } from './voices.js';
|
|
16
16
|
function compileParameters(spec) {
|
|
17
17
|
const properties = {};
|
|
@@ -134,11 +134,57 @@ const listSchema = {
|
|
|
134
134
|
},
|
|
135
135
|
additionalProperties: true,
|
|
136
136
|
};
|
|
137
|
+
function parseCallPronunciations(raw) {
|
|
138
|
+
if (Array.isArray(raw)) {
|
|
139
|
+
const out = [];
|
|
140
|
+
for (const item of raw) {
|
|
141
|
+
if (typeof item !== 'object' || item === null || Array.isArray(item))
|
|
142
|
+
continue;
|
|
143
|
+
const rec = item;
|
|
144
|
+
const word = typeof rec.word === 'string' ? rec.word.trim() : '';
|
|
145
|
+
const alias = typeof rec.alias === 'string' ? rec.alias.trim() : '';
|
|
146
|
+
if (word !== '' && alias !== '')
|
|
147
|
+
out.push({ word, alias });
|
|
148
|
+
}
|
|
149
|
+
return out;
|
|
150
|
+
}
|
|
151
|
+
if (typeof raw === 'object' && raw !== null) {
|
|
152
|
+
const out = [];
|
|
153
|
+
for (const [key, val] of Object.entries(raw)) {
|
|
154
|
+
if (typeof val !== 'string')
|
|
155
|
+
continue;
|
|
156
|
+
const word = key.trim();
|
|
157
|
+
const alias = val.trim();
|
|
158
|
+
if (word !== '' && alias !== '')
|
|
159
|
+
out.push({ word, alias });
|
|
160
|
+
}
|
|
161
|
+
return out;
|
|
162
|
+
}
|
|
163
|
+
return [];
|
|
164
|
+
}
|
|
165
|
+
function mergePronunciations(base, overrides) {
|
|
166
|
+
const merged = new Map();
|
|
167
|
+
for (const rule of base ?? []) {
|
|
168
|
+
const word = rule.word.trim();
|
|
169
|
+
const alias = rule.alias.trim();
|
|
170
|
+
if (word !== '' && alias !== '')
|
|
171
|
+
merged.set(word, { word, alias });
|
|
172
|
+
}
|
|
173
|
+
for (const rule of overrides) {
|
|
174
|
+
const word = rule.word.trim();
|
|
175
|
+
const alias = rule.alias.trim();
|
|
176
|
+
if (word !== '' && alias !== '')
|
|
177
|
+
merged.set(word, { word, alias });
|
|
178
|
+
}
|
|
179
|
+
return Array.from(merged.values());
|
|
180
|
+
}
|
|
137
181
|
const ttsSchema = {
|
|
138
182
|
type: 'object',
|
|
139
183
|
properties: {
|
|
140
184
|
output: { type: 'string' },
|
|
141
185
|
bytes: { type: 'integer' },
|
|
186
|
+
durationMs: { type: 'integer' },
|
|
187
|
+
durationSec: { type: 'number' },
|
|
142
188
|
voice: { type: 'string' },
|
|
143
189
|
rate: { type: 'string' },
|
|
144
190
|
pitch: { type: 'string' },
|
|
@@ -157,6 +203,7 @@ const sttSchema = {
|
|
|
157
203
|
audio: { type: 'string' },
|
|
158
204
|
transcriptFile: { type: 'string' },
|
|
159
205
|
subtitleFile: { type: 'string' },
|
|
206
|
+
speakers: { type: 'array', items: { type: 'string' } },
|
|
160
207
|
},
|
|
161
208
|
additionalProperties: true,
|
|
162
209
|
};
|
|
@@ -177,6 +224,8 @@ const FALLBACK_ASR_MODEL = 'whisper-1';
|
|
|
177
224
|
const MAX_TTS_CHARS = 100000;
|
|
178
225
|
/** 多角色对话/播客分段数上限。 */
|
|
179
226
|
const MAX_TTS_SEGMENTS = 50;
|
|
227
|
+
/** 反向字幕卡点配音条目上限。 */
|
|
228
|
+
const MAX_SUBTITLE_INPUT_CUES = 200;
|
|
180
229
|
/** OpenAI 兼容 TTS 默认模型。 */
|
|
181
230
|
const FALLBACK_TTS_MODEL = 'gpt-4o-mini-tts';
|
|
182
231
|
/** ASR 直传上限(25MB)。 */
|
|
@@ -238,7 +287,7 @@ export function buildVoiceTools(config, deps = {}) {
|
|
|
238
287
|
return false;
|
|
239
288
|
return cfg.ttsEngine === 'edge';
|
|
240
289
|
};
|
|
241
|
-
const resolveEffectiveVoiceSpec = (cfg, cwd, input) => {
|
|
290
|
+
const resolveEffectiveVoiceSpec = (cfg, cwd, input, refAudioCache) => {
|
|
242
291
|
let customAlias;
|
|
243
292
|
let voiceConsumedAsAlias = false;
|
|
244
293
|
let personaConsumedAsAlias = false;
|
|
@@ -282,22 +331,28 @@ export function buildVoiceTools(config, deps = {}) {
|
|
|
282
331
|
const hasReferenceAudio = effectiveReferenceAudio !== undefined && effectiveReferenceAudio.trim() !== '';
|
|
283
332
|
if (hasReferenceAudio) {
|
|
284
333
|
const refPath = assertAudioFile(effectiveReferenceAudio, cwd);
|
|
285
|
-
|
|
286
|
-
if (
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
334
|
+
let buf = refAudioCache?.get(refPath);
|
|
335
|
+
if (buf === undefined) {
|
|
336
|
+
const size = statSync(refPath).size;
|
|
337
|
+
if (size <= 0) {
|
|
338
|
+
throw new Error('参考音频文件为空,无法用于音色克隆:' + effectiveReferenceAudio);
|
|
339
|
+
}
|
|
340
|
+
if (size > MAX_REFERENCE_AUDIO_BYTES) {
|
|
341
|
+
throw new Error('参考音频超过 10MB 上限(当前 ' + size + ' 字节),请裁剪为 5–10 秒人声片段:' + effectiveReferenceAudio);
|
|
342
|
+
}
|
|
343
|
+
buf = readFileSync(refPath);
|
|
344
|
+
if (buf.length === 0 || buf.length > MAX_REFERENCE_AUDIO_BYTES) {
|
|
345
|
+
throw new Error('参考音频大小异常(必须为 1 字节至 10MB):' + effectiveReferenceAudio);
|
|
346
|
+
}
|
|
347
|
+
refAudioCache?.set(refPath, buf);
|
|
295
348
|
}
|
|
296
349
|
if (resolveUseEdge(cfg, true)) {
|
|
297
350
|
throw new Error('零样本参考音频克隆(referenceAudio)需要兼容克隆的 TTS 引擎:请在 DSH「设置 → 插件 → 语音」将 ttsEngine 切换为 siliconflow(CosyVoice2-0.5B)或 custom(如本地 GPT-SoVITS / IndexTTS)。');
|
|
298
351
|
}
|
|
299
352
|
references = [{
|
|
300
353
|
audioDataUri: 'data:' + mimeFromPath(refPath) + ';base64,' + buf.toString('base64'),
|
|
354
|
+
rawAudio: buf,
|
|
355
|
+
filename: basename(refPath),
|
|
301
356
|
...(effectiveReferenceText !== undefined && effectiveReferenceText.trim() !== '' ? { text: effectiveReferenceText.trim() } : {}),
|
|
302
357
|
}];
|
|
303
358
|
}
|
|
@@ -327,32 +382,52 @@ export function buildVoiceTools(config, deps = {}) {
|
|
|
327
382
|
throwIfAborted(signal);
|
|
328
383
|
return normalized;
|
|
329
384
|
};
|
|
330
|
-
const
|
|
385
|
+
const finalizeVoiceAudio = async (rawAudio, bgmPath, bgmVolume, bgmDucking, shouldNormalize, signal) => {
|
|
386
|
+
throwIfAborted(signal);
|
|
387
|
+
if (bgmPath !== undefined) {
|
|
388
|
+
if (deps.mixBgm === undefined && !hasFfmpeg()) {
|
|
389
|
+
throw new Error('使用 bgmFile 背景音乐自动闪避混音需要本机安装 ffmpeg(或使用 dsh-ffmpeg)。');
|
|
390
|
+
}
|
|
391
|
+
const mixed = await raceAbort((deps.mixBgm ?? mixAudioWithBgm)(rawAudio, {
|
|
392
|
+
bgmPath,
|
|
393
|
+
bgmVolume,
|
|
394
|
+
ducking: bgmDucking,
|
|
395
|
+
normalizeLoudness: shouldNormalize,
|
|
396
|
+
}, signal), signal);
|
|
397
|
+
throwIfAborted(signal);
|
|
398
|
+
return mixed;
|
|
399
|
+
}
|
|
400
|
+
return await maybeNormalizeAudio(shouldNormalize, rawAudio, signal);
|
|
401
|
+
};
|
|
402
|
+
const synthesizeChunk = async (cfg, useEdge, chunkText, voice, rate, pitch, needSubtitles, signal, instructions, references, pronunciations) => {
|
|
403
|
+
const hasPronunciations = pronunciations !== undefined && pronunciations.length > 0;
|
|
331
404
|
if (useEdge) {
|
|
405
|
+
const pronOpt = hasPronunciations ? { pronunciations } : {};
|
|
332
406
|
if (!needSubtitles) {
|
|
333
407
|
const ttsFn = deps.tts ?? (deps.ttsWithMeta !== undefined
|
|
334
408
|
? async (opts, ttsDeps, timeoutMs, abortSig) => (await deps.ttsWithMeta(opts, ttsDeps, timeoutMs, abortSig)).audio
|
|
335
409
|
: synthesizeSpeech);
|
|
336
|
-
const audio = await raceAbort(ttsFn({ text: chunkText, voice, rate, pitch }, { proxyUrl: cfg.proxyUrl }, cfg.timeoutMs, signal), signal);
|
|
410
|
+
const audio = await raceAbort(ttsFn({ text: chunkText, voice, rate, pitch, ...pronOpt }, { proxyUrl: cfg.proxyUrl }, cfg.timeoutMs, signal), signal);
|
|
337
411
|
return { audio, cues: [] };
|
|
338
412
|
}
|
|
339
413
|
if (deps.ttsWithMeta !== undefined || deps.tts === undefined) {
|
|
340
414
|
const withMetaFn = deps.ttsWithMeta ?? synthesizeSpeechWithMetadata;
|
|
341
|
-
const res = await raceAbort(withMetaFn({ text: chunkText, voice, rate, pitch, boundary: 'sentence' }, { proxyUrl: cfg.proxyUrl }, cfg.timeoutMs, signal), signal);
|
|
415
|
+
const res = await raceAbort(withMetaFn({ text: chunkText, voice, rate, pitch, boundary: 'sentence', ...pronOpt }, { proxyUrl: cfg.proxyUrl }, cfg.timeoutMs, signal), signal);
|
|
342
416
|
const cues = res.cues.length > 0
|
|
343
417
|
? res.cues
|
|
344
|
-
: [{ index: 1, startMs: 0, endMs: Math.max(1,
|
|
418
|
+
: [{ index: 1, startMs: 0, endMs: Math.max(1, inspectMp3Buffer(res.audio).durationMs), text: chunkText }];
|
|
345
419
|
return { audio: res.audio, cues };
|
|
346
420
|
}
|
|
347
|
-
const audio = await raceAbort(deps.tts({ text: chunkText, voice, rate, pitch }, { proxyUrl: cfg.proxyUrl }, cfg.timeoutMs, signal), signal);
|
|
421
|
+
const audio = await raceAbort(deps.tts({ text: chunkText, voice, rate, pitch, ...pronOpt }, { proxyUrl: cfg.proxyUrl }, cfg.timeoutMs, signal), signal);
|
|
348
422
|
return {
|
|
349
423
|
audio,
|
|
350
|
-
cues: [{ index: 1, startMs: 0, endMs: Math.max(1,
|
|
424
|
+
cues: [{ index: 1, startMs: 0, endMs: Math.max(1, inspectMp3Buffer(audio).durationMs), text: chunkText }],
|
|
351
425
|
};
|
|
352
426
|
}
|
|
427
|
+
const spokenText = hasPronunciations ? applyPronunciationsPlain(chunkText, pronunciations) : chunkText;
|
|
353
428
|
const openaiFn = deps.openaiTts ?? synthesizeOpenAiSpeech;
|
|
354
429
|
const audio = await raceAbort(openaiFn(cfg.ttsBaseUrl, cfg.ttsApiKey, {
|
|
355
|
-
text:
|
|
430
|
+
text: spokenText,
|
|
356
431
|
voice,
|
|
357
432
|
model: cfg.ttsModel !== '' ? cfg.ttsModel : FALLBACK_TTS_MODEL,
|
|
358
433
|
speed: rateToSpeed(rate),
|
|
@@ -363,7 +438,7 @@ export function buildVoiceTools(config, deps = {}) {
|
|
|
363
438
|
return {
|
|
364
439
|
audio,
|
|
365
440
|
cues: needSubtitles
|
|
366
|
-
? [{ index: 1, startMs: 0, endMs: Math.max(1,
|
|
441
|
+
? [{ index: 1, startMs: 0, endMs: Math.max(1, inspectMp3Buffer(audio).durationMs), text: chunkText }]
|
|
367
442
|
: [],
|
|
368
443
|
};
|
|
369
444
|
};
|
|
@@ -472,10 +547,11 @@ export function buildVoiceTools(config, deps = {}) {
|
|
|
472
547
|
const voiceTts = {
|
|
473
548
|
name: 'voice_tts',
|
|
474
549
|
isConcurrencySafe: () => !getCfg().overwrite,
|
|
475
|
-
description: '文字转语音:默认调用 edge-tts(微软 Edge 朗读服务,免费)合成 MP3,也可在设置中切换 OpenAI/SiliconFlow/自定义 TTS 引擎。支持内置角色人设 persona、自然语言情绪指令 instructions、5–10 秒参考音频零样本克隆 referenceAudio、多角色播客 segments、读取工作区 inputFile(自动清理 Markdown
|
|
550
|
+
description: '文字转语音:默认调用 edge-tts(微软 Edge 朗读服务,免费)合成 MP3,也可在设置中切换 OpenAI/SiliconFlow/自定义 TTS 引擎。支持内置角色人设 persona、自然语言情绪指令 instructions、5–10 秒参考音频零样本克隆 referenceAudio、自定义发音词典 pronunciations、多角色播客 segments、读取工作区 inputFile(自动清理 Markdown)、按 .srt/.vtt 字幕时间轴反向卡点配音 subtitleInput、背景音乐自动闪避混音 bgmFile、长文自动分段(≤100000 字符)、导出 .srt/.vtt 字幕与 -16 LUFS 响度均衡 normalizeLoudness,并返回精确时长 durationMs/durationSec;输出默认 voice_output.mp3。',
|
|
476
551
|
parameters: compileParameters({
|
|
477
|
-
text: { type: 'string', description: '要合成的文本(若提供 inputFile 或
|
|
552
|
+
text: { type: 'string', description: '要合成的文本(若提供 inputFile、segments 或 subtitleInput 则可选;支持长文本自动分段,≤100000 字符)。' },
|
|
478
553
|
inputFile: { type: 'string', description: '工作区内文稿文件路径(可选,如 article.md;指定时直接从文件读取)。' },
|
|
554
|
+
subtitleInput: { type: 'string', description: '按现有 .srt/.vtt 字幕时间轴反向卡点配音(可选,工作区内字幕路径;每条字幕超出时间窗时自动微调语速加速,句间自动补齐精确静音以对齐原视频画面)。' },
|
|
479
555
|
cleanMarkdown: { type: 'boolean', description: '是否清理 Markdown 语法以便朗读(可选;inputFile 为 .md/.markdown 时默认 true)。' },
|
|
480
556
|
segments: {
|
|
481
557
|
type: 'array',
|
|
@@ -503,6 +579,21 @@ export function buildVoiceTools(config, deps = {}) {
|
|
|
503
579
|
instructions: { type: 'string', description: '自然语言情绪/语气指令(可选,OpenAI gpt-4o-mini-tts 与 SiliconFlow CosyVoice2 可用,如“用开心轻快的语气说”,支持 [laughter] / [breath])。' },
|
|
504
580
|
referenceAudio: { type: 'string', description: '零样本克隆参考音频路径(可选,会话工作区内 5–10 秒干净人声,≤10MB;需 siliconflow 或 custom 引擎)。' },
|
|
505
581
|
referenceText: { type: 'string', description: '参考音频对应的原始台词文本(可选,零样本克隆推荐填写)。' },
|
|
582
|
+
pronunciations: {
|
|
583
|
+
type: 'array',
|
|
584
|
+
description: '发音替读规则数组(可选,如 [{ word: "Nginx", alias: "Engine X" }],与设置中的 pronunciations 合并,同名以本次参数优先;字幕保留原词,朗读按 alias 发音)。',
|
|
585
|
+
items: {
|
|
586
|
+
type: 'object',
|
|
587
|
+
properties: {
|
|
588
|
+
word: { type: 'string' },
|
|
589
|
+
alias: { type: 'string' },
|
|
590
|
+
},
|
|
591
|
+
required: ['word', 'alias'],
|
|
592
|
+
},
|
|
593
|
+
},
|
|
594
|
+
bgmFile: { type: 'string', description: '背景音乐文件路径(可选,工作区内音频文件;需本机安装 ffmpeg,将自动循环铺底并在人声出现时自动闪避降音)。' },
|
|
595
|
+
bgmVolume: { type: 'number', minimum: 0.02, maximum: 1, default: 0.18, description: '背景音乐相对音量(可选,0.02–1.0,默认 0.18)。' },
|
|
596
|
+
bgmDucking: { type: 'boolean', default: true, description: '人声说话时是否自动压低背景音乐音量(Auto-Ducking,可选,默认 true)。' },
|
|
506
597
|
normalizeLoudness: { type: 'boolean', default: false, description: '是否在合成后通过 ffmpeg 执行 -16 LUFS 播客响度均衡(可选,默认 false)。' },
|
|
507
598
|
output: { type: 'string', description: '输出 MP3 路径(可选,相对路径按会话工作目录解析)。' },
|
|
508
599
|
subtitleOutput: { type: 'string', description: '导出同步字幕文件路径(可选,.srt 或 .vtt,相对路径按会话工作目录解析)。' },
|
|
@@ -513,7 +604,8 @@ export function buildVoiceTools(config, deps = {}) {
|
|
|
513
604
|
const rec = asRecord(value);
|
|
514
605
|
const chunksInfo = typeof rec.chunks === 'number' && rec.chunks > 1 ? ',共 ' + rec.chunks + ' 段' : '';
|
|
515
606
|
const voiceLabel = typeof rec.voice === 'string' && rec.voice !== '' ? rec.voice : '参考音频克隆';
|
|
516
|
-
const
|
|
607
|
+
const durationInfo = typeof rec.durationSec === 'number' ? ',时长 ' + rec.durationSec + ' 秒' : '';
|
|
608
|
+
const lines = ['语音合成完成:' + rec.output + '(' + rec.bytes + ' 字节,音色 ' + voiceLabel + chunksInfo + durationInfo + ')'];
|
|
517
609
|
const subtitleFile = typeof rec.subtitleFile === 'string' ? rec.subtitleFile : '';
|
|
518
610
|
if (subtitleFile !== '')
|
|
519
611
|
lines.push('字幕文件已写入:' + subtitleFile);
|
|
@@ -537,6 +629,21 @@ export function buildVoiceTools(config, deps = {}) {
|
|
|
537
629
|
const requested = optionalString(args, 'output');
|
|
538
630
|
const subtitleOutput = optionalString(args, 'subtitleOutput');
|
|
539
631
|
const needSubtitles = subtitleOutput !== undefined;
|
|
632
|
+
const mergedPronunciations = mergePronunciations(cfg.pronunciations, parseCallPronunciations(args.pronunciations));
|
|
633
|
+
const bgmFile = optionalString(args, 'bgmFile');
|
|
634
|
+
let bgmPath;
|
|
635
|
+
let bgmVolume = 0.18;
|
|
636
|
+
if (bgmFile !== undefined) {
|
|
637
|
+
bgmPath = assertAudioFile(bgmFile, cwd);
|
|
638
|
+
if (deps.mixBgm === undefined && !hasFfmpeg()) {
|
|
639
|
+
throw new Error('使用 bgmFile 背景音乐自动闪避混音需要本机安装 ffmpeg(或使用 dsh-ffmpeg)。');
|
|
640
|
+
}
|
|
641
|
+
if (typeof args.bgmVolume === 'number' && Number.isFinite(args.bgmVolume)) {
|
|
642
|
+
bgmVolume = Math.min(1.0, Math.max(0.02, args.bgmVolume));
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
const bgmDucking = args.bgmDucking !== false;
|
|
646
|
+
const refAudioCache = new Map();
|
|
540
647
|
const rawSegments = Array.isArray(args.segments) && args.segments.length > 0 ? args.segments : undefined;
|
|
541
648
|
if (rawSegments !== undefined) {
|
|
542
649
|
if (rawSegments.length > MAX_TTS_SEGMENTS) {
|
|
@@ -550,7 +657,7 @@ export function buildVoiceTools(config, deps = {}) {
|
|
|
550
657
|
instructions: topInstructions,
|
|
551
658
|
referenceAudio: topReferenceAudio,
|
|
552
659
|
referenceText: topReferenceText,
|
|
553
|
-
});
|
|
660
|
+
}, refAudioCache);
|
|
554
661
|
const parsedSegments = [];
|
|
555
662
|
let totalChars = 0;
|
|
556
663
|
for (let i = 0; i < rawSegments.length; i++) {
|
|
@@ -583,7 +690,7 @@ export function buildVoiceTools(config, deps = {}) {
|
|
|
583
690
|
instructions: segInstructions ?? topInstructions,
|
|
584
691
|
referenceAudio: segReferenceAudio ?? topReferenceAudio,
|
|
585
692
|
referenceText: segReferenceText ?? topReferenceText,
|
|
586
|
-
});
|
|
693
|
+
}, refAudioCache);
|
|
587
694
|
}
|
|
588
695
|
else if (segReferenceAudio !== undefined) {
|
|
589
696
|
segSpec = resolveEffectiveVoiceSpec(cfg, cwd, {
|
|
@@ -595,11 +702,11 @@ export function buildVoiceTools(config, deps = {}) {
|
|
|
595
702
|
instructions: segInstructions ?? topInstructions,
|
|
596
703
|
referenceAudio: segReferenceAudio,
|
|
597
704
|
referenceText: segReferenceText ?? topReferenceText,
|
|
598
|
-
});
|
|
705
|
+
}, refAudioCache);
|
|
599
706
|
}
|
|
600
707
|
else {
|
|
601
708
|
const reusedRefs = topSpec.references !== undefined && segReferenceText !== undefined
|
|
602
|
-
? topSpec.references.map((r) => ({
|
|
709
|
+
? topSpec.references.map((r) => ({ ...r, text: segReferenceText }))
|
|
603
710
|
: topSpec.references;
|
|
604
711
|
segSpec = {
|
|
605
712
|
voice: topSpec.voice,
|
|
@@ -651,19 +758,23 @@ export function buildVoiceTools(config, deps = {}) {
|
|
|
651
758
|
});
|
|
652
759
|
}
|
|
653
760
|
}
|
|
654
|
-
const chunkResults = await mapLimit(tasks, TTS_CHUNK_CONCURRENCY, signal, (task) => synthesizeChunk(cfg, task.useEdge, task.text, task.voice, task.rate, task.pitch, needSubtitles, signal, task.instructions, task.references));
|
|
761
|
+
const chunkResults = await mapLimit(tasks, TTS_CHUNK_CONCURRENCY, signal, (task) => synthesizeChunk(cfg, task.useEdge, task.text, task.voice, task.rate, task.pitch, needSubtitles, signal, task.instructions, task.references, mergedPronunciations));
|
|
655
762
|
throwIfAborted(signal);
|
|
656
763
|
const buffers = [];
|
|
657
764
|
const allCues = [];
|
|
658
765
|
let offsetMs = 0;
|
|
659
766
|
let cueIndex = 1;
|
|
660
767
|
let resultIdx = 0;
|
|
768
|
+
let detectedSampleRate = 24000;
|
|
661
769
|
for (let s = 0; s < parsedSegments.length; s++) {
|
|
662
770
|
const seg = parsedSegments[s];
|
|
663
771
|
for (let c = 0; c < seg.chunks.length; c++) {
|
|
664
772
|
const res = chunkResults[resultIdx++];
|
|
665
773
|
buffers.push(res.audio);
|
|
666
|
-
const
|
|
774
|
+
const chunkInfo = inspectMp3Buffer(res.audio);
|
|
775
|
+
if (chunkInfo.frameCount > 0)
|
|
776
|
+
detectedSampleRate = chunkInfo.sampleRate;
|
|
777
|
+
const chunkDurationMs = chunkInfo.durationMs;
|
|
667
778
|
if (needSubtitles) {
|
|
668
779
|
const shifted = offsetCues(res.cues, offsetMs, cueIndex);
|
|
669
780
|
allCues.push(...shifted);
|
|
@@ -673,22 +784,110 @@ export function buildVoiceTools(config, deps = {}) {
|
|
|
673
784
|
}
|
|
674
785
|
}
|
|
675
786
|
if (s < parsedSegments.length - 1 && seg.pauseMs > 0) {
|
|
676
|
-
const silence = createSilenceMp3(seg.pauseMs);
|
|
787
|
+
const silence = createSilenceMp3(seg.pauseMs, detectedSampleRate);
|
|
677
788
|
if (silence.length > 0) {
|
|
678
789
|
buffers.push(silence);
|
|
679
|
-
offsetMs +=
|
|
790
|
+
offsetMs += inspectMp3Buffer(silence).durationMs;
|
|
680
791
|
}
|
|
681
792
|
}
|
|
682
793
|
}
|
|
683
794
|
let audio = buffers.length === 1 ? buffers[0] : concatMp3Buffers(buffers);
|
|
684
|
-
audio = await
|
|
795
|
+
audio = await finalizeVoiceAudio(audio, bgmPath, bgmVolume, bgmDucking, normalizeLoudness, signal);
|
|
796
|
+
const finalInfo = inspectMp3Buffer(audio);
|
|
797
|
+
const durationMs = finalInfo.durationMs;
|
|
798
|
+
const durationSec = Math.round(durationMs / 100) / 10;
|
|
685
799
|
const output = writeWorkspaceFile(resolveOutputPath(requested, 'voice_output.mp3', cfg.overwrite, cwd), audio, cwd, undefined, cfg.overwrite);
|
|
686
800
|
let subtitleFile = '';
|
|
687
801
|
if (subtitleOutput !== undefined) {
|
|
688
802
|
const formatted = formatSubtitles(allCues, subtitleFormatOf(subtitleOutput));
|
|
689
803
|
subtitleFile = writeWorkspaceFile(resolveOutputPath(subtitleOutput, '', cfg.overwrite, cwd), formatted, cwd, undefined, cfg.overwrite);
|
|
690
804
|
}
|
|
691
|
-
return { output, bytes: audio.length, voice: topSpec.voice, rate: topSpec.rate, pitch: topSpec.pitch, textLength: totalChars, chunks: tasks.length, subtitleFile };
|
|
805
|
+
return { output, bytes: audio.length, durationMs, durationSec, voice: topSpec.voice, rate: topSpec.rate, pitch: topSpec.pitch, textLength: totalChars, chunks: tasks.length, subtitleFile };
|
|
806
|
+
}
|
|
807
|
+
const subtitleInput = optionalString(args, 'subtitleInput');
|
|
808
|
+
if (subtitleInput !== undefined) {
|
|
809
|
+
const resolvedSub = assertWorkspaceFile(subtitleInput, cwd, '字幕文件');
|
|
810
|
+
const rawSub = readFileSync(resolvedSub, 'utf8');
|
|
811
|
+
const cues = parseSubtitleFile(rawSub);
|
|
812
|
+
if (cues.length === 0) {
|
|
813
|
+
throw new Error('字幕文件未解析出任何有效字幕条目:' + subtitleInput);
|
|
814
|
+
}
|
|
815
|
+
if (cues.length > MAX_SUBTITLE_INPUT_CUES) {
|
|
816
|
+
throw new Error('subtitleInput 最多支持 ' + MAX_SUBTITLE_INPUT_CUES + ' 条字幕(当前 ' + cues.length + ' 条),请拆分为多段处理。');
|
|
817
|
+
}
|
|
818
|
+
const spec = resolveEffectiveVoiceSpec(cfg, cwd, {
|
|
819
|
+
voice: topVoice,
|
|
820
|
+
persona: topPersona,
|
|
821
|
+
rate: topRate,
|
|
822
|
+
pitch: topPitch,
|
|
823
|
+
instructions: topInstructions,
|
|
824
|
+
referenceAudio: topReferenceAudio,
|
|
825
|
+
referenceText: topReferenceText,
|
|
826
|
+
}, refAudioCache);
|
|
827
|
+
const useEdge = resolveUseEdge(cfg, spec.hasReferenceAudio);
|
|
828
|
+
assertTtsEngineReady(cfg, useEdge);
|
|
829
|
+
assertTtsVoice(spec.voice, useEdge, spec.references !== undefined);
|
|
830
|
+
assertProsody(spec.rate, spec.pitch);
|
|
831
|
+
resolveOutputPath(requested, 'voice_output.mp3', cfg.overwrite, cwd);
|
|
832
|
+
if (subtitleOutput !== undefined)
|
|
833
|
+
resolveOutputPath(subtitleOutput, '', cfg.overwrite, cwd);
|
|
834
|
+
const cueResults = await mapLimit(cues, TTS_CHUNK_CONCURRENCY, signal, async (cue) => {
|
|
835
|
+
const cueText = args.cleanMarkdown === true
|
|
836
|
+
? (cleanMarkdownForSpeech(cue.text).trim() || cue.text.trim())
|
|
837
|
+
: cue.text.trim();
|
|
838
|
+
let res = await synthesizeChunk(cfg, useEdge, cueText, spec.voice, spec.rate, spec.pitch, false, signal, spec.instructions, spec.references, mergedPronunciations);
|
|
839
|
+
const actualMs = inspectMp3Buffer(res.audio).durationMs;
|
|
840
|
+
const windowMs = Math.max(200, cue.endMs - cue.startMs);
|
|
841
|
+
if (actualMs > windowMs * 1.08) {
|
|
842
|
+
const fastRate = scaleRatePercent(spec.rate, actualMs / windowMs);
|
|
843
|
+
if (fastRate !== spec.rate) {
|
|
844
|
+
res = await synthesizeChunk(cfg, useEdge, cueText, spec.voice, fastRate, spec.pitch, false, signal, spec.instructions, spec.references, mergedPronunciations);
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
return { audio: res.audio, textLength: cueText.length };
|
|
848
|
+
});
|
|
849
|
+
throwIfAborted(signal);
|
|
850
|
+
const buffers = [];
|
|
851
|
+
const outCues = [];
|
|
852
|
+
let cursorMs = 0;
|
|
853
|
+
let detectedSampleRate = 24000;
|
|
854
|
+
let totalChars = 0;
|
|
855
|
+
for (let i = 0; i < cues.length; i++) {
|
|
856
|
+
const cue = cues[i];
|
|
857
|
+
const cueAudio = cueResults[i].audio;
|
|
858
|
+
totalChars += cueResults[i].textLength;
|
|
859
|
+
const cueInfo = inspectMp3Buffer(cueAudio);
|
|
860
|
+
if (cueInfo.frameCount > 0)
|
|
861
|
+
detectedSampleRate = cueInfo.sampleRate;
|
|
862
|
+
if (cue.startMs > cursorMs) {
|
|
863
|
+
const pad = createSilenceMp3(cue.startMs - cursorMs, detectedSampleRate);
|
|
864
|
+
if (pad.length > 0) {
|
|
865
|
+
buffers.push(pad);
|
|
866
|
+
cursorMs += inspectMp3Buffer(pad).durationMs;
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
const cueStartMs = cursorMs;
|
|
870
|
+
buffers.push(cueAudio);
|
|
871
|
+
cursorMs += cueInfo.durationMs;
|
|
872
|
+
outCues.push({
|
|
873
|
+
index: i + 1,
|
|
874
|
+
startMs: cueStartMs,
|
|
875
|
+
endMs: Math.max(cueStartMs + 1, cursorMs),
|
|
876
|
+
text: cue.text,
|
|
877
|
+
});
|
|
878
|
+
}
|
|
879
|
+
let audio = buffers.length === 1 ? buffers[0] : concatMp3Buffers(buffers);
|
|
880
|
+
audio = await finalizeVoiceAudio(audio, bgmPath, bgmVolume, bgmDucking, normalizeLoudness, signal);
|
|
881
|
+
const finalInfo = inspectMp3Buffer(audio);
|
|
882
|
+
const durationMs = finalInfo.durationMs;
|
|
883
|
+
const durationSec = Math.round(durationMs / 100) / 10;
|
|
884
|
+
const output = writeWorkspaceFile(resolveOutputPath(requested, 'voice_output.mp3', cfg.overwrite, cwd), audio, cwd, undefined, cfg.overwrite);
|
|
885
|
+
let subtitleFile = '';
|
|
886
|
+
if (subtitleOutput !== undefined) {
|
|
887
|
+
const formatted = formatSubtitles(outCues, subtitleFormatOf(subtitleOutput));
|
|
888
|
+
subtitleFile = writeWorkspaceFile(resolveOutputPath(subtitleOutput, '', cfg.overwrite, cwd), formatted, cwd, undefined, cfg.overwrite);
|
|
889
|
+
}
|
|
890
|
+
return { output, bytes: audio.length, durationMs, durationSec, voice: spec.voice, rate: spec.rate, pitch: spec.pitch, textLength: totalChars, chunks: cues.length, subtitleFile };
|
|
692
891
|
}
|
|
693
892
|
const inputFile = optionalString(args, 'inputFile');
|
|
694
893
|
let finalText;
|
|
@@ -720,7 +919,7 @@ export function buildVoiceTools(config, deps = {}) {
|
|
|
720
919
|
instructions: topInstructions,
|
|
721
920
|
referenceAudio: topReferenceAudio,
|
|
722
921
|
referenceText: topReferenceText,
|
|
723
|
-
});
|
|
922
|
+
}, refAudioCache);
|
|
724
923
|
const useEdge = resolveUseEdge(cfg, spec.hasReferenceAudio);
|
|
725
924
|
assertTtsEngineReady(cfg, useEdge);
|
|
726
925
|
assertTtsVoice(spec.voice, useEdge, spec.references !== undefined);
|
|
@@ -732,13 +931,13 @@ export function buildVoiceTools(config, deps = {}) {
|
|
|
732
931
|
let chunkCount = 1;
|
|
733
932
|
let subtitleFile = '';
|
|
734
933
|
if (finalText.length <= 5000 && !needSubtitles && useEdge) {
|
|
735
|
-
const res = await synthesizeChunk(cfg, true, finalText, spec.voice, spec.rate, spec.pitch, false, signal, spec.instructions, spec.references);
|
|
934
|
+
const res = await synthesizeChunk(cfg, true, finalText, spec.voice, spec.rate, spec.pitch, false, signal, spec.instructions, spec.references, mergedPronunciations);
|
|
736
935
|
audio = res.audio;
|
|
737
936
|
}
|
|
738
937
|
else {
|
|
739
938
|
const chunks = splitTextIntoChunks(finalText, 2800);
|
|
740
939
|
chunkCount = chunks.length;
|
|
741
|
-
const results = await mapLimit(chunks, TTS_CHUNK_CONCURRENCY, signal, (chunk) => synthesizeChunk(cfg, useEdge, chunk, spec.voice, spec.rate, spec.pitch, needSubtitles, signal, spec.instructions, spec.references));
|
|
940
|
+
const results = await mapLimit(chunks, TTS_CHUNK_CONCURRENCY, signal, (chunk) => synthesizeChunk(cfg, useEdge, chunk, spec.voice, spec.rate, spec.pitch, needSubtitles, signal, spec.instructions, spec.references, mergedPronunciations));
|
|
742
941
|
throwIfAborted(signal);
|
|
743
942
|
audio = results.length === 1 ? results[0].audio : concatMp3Buffers(results.map((r) => r.audio));
|
|
744
943
|
if (subtitleOutput !== undefined) {
|
|
@@ -749,7 +948,7 @@ export function buildVoiceTools(config, deps = {}) {
|
|
|
749
948
|
const shifted = offsetCues(res.cues, offsetMs, cueIndex);
|
|
750
949
|
allCues.push(...shifted);
|
|
751
950
|
cueIndex += shifted.length;
|
|
752
|
-
const chunkDurationMs =
|
|
951
|
+
const chunkDurationMs = inspectMp3Buffer(res.audio).durationMs;
|
|
753
952
|
const cueSpan = shifted.length > 0 ? Math.max(0, shifted[shifted.length - 1].endMs - offsetMs) : 0;
|
|
754
953
|
offsetMs += Math.max(chunkDurationMs, cueSpan);
|
|
755
954
|
}
|
|
@@ -758,16 +957,19 @@ export function buildVoiceTools(config, deps = {}) {
|
|
|
758
957
|
}
|
|
759
958
|
}
|
|
760
959
|
throwIfAborted(signal);
|
|
761
|
-
audio = await
|
|
960
|
+
audio = await finalizeVoiceAudio(audio, bgmPath, bgmVolume, bgmDucking, normalizeLoudness, signal);
|
|
961
|
+
const finalInfo = inspectMp3Buffer(audio);
|
|
962
|
+
const durationMs = finalInfo.durationMs;
|
|
963
|
+
const durationSec = Math.round(durationMs / 100) / 10;
|
|
762
964
|
const output = writeWorkspaceFile(resolveOutputPath(requested, 'voice_output.mp3', cfg.overwrite, cwd), audio, cwd, undefined, cfg.overwrite);
|
|
763
|
-
return { output, bytes: audio.length, voice: spec.voice, rate: spec.rate, pitch: spec.pitch, textLength: finalText.length, chunks: chunkCount, subtitleFile };
|
|
965
|
+
return { output, bytes: audio.length, durationMs, durationSec, voice: spec.voice, rate: spec.rate, pitch: spec.pitch, textLength: finalText.length, chunks: chunkCount, subtitleFile };
|
|
764
966
|
},
|
|
765
967
|
get timeoutMs() { return getCfg().timeoutMs + 10000; },
|
|
766
968
|
};
|
|
767
969
|
const voiceStt = {
|
|
768
970
|
name: 'voice_stt',
|
|
769
971
|
isConcurrencySafe: () => !getCfg().overwrite,
|
|
770
|
-
description: '语音转文字:调用 OpenAI 兼容 ASR 接口(默认 Groq whisper-large-v3-turbo;可切 openai/siliconflow/custom)。audio 为音频文件路径(mp3/wav/m4a/ogg/flac,≤25MB;超过 25MB 可设 autoCompress=true 用 ffmpeg
|
|
972
|
+
description: '语音转文字:调用 OpenAI 兼容 ASR 接口(默认 Groq whisper-large-v3-turbo;可切 openai/siliconflow/custom)。audio 为音频文件路径(mp3/wav/m4a/ogg/flac,≤25MB;超过 25MB 可设 autoCompress=true 用 ffmpeg 自动压缩分段);支持多人说话人区分 diarize=true(双声道按声道分离或推断说话人切换);language/prompt 可选;output 可写 .txt,subtitleOutput 可写 .srt/.vtt 字幕。',
|
|
771
973
|
parameters: compileParameters({
|
|
772
974
|
audio: { type: 'string', required: true, description: '音频文件路径(必填)。' },
|
|
773
975
|
engine: { type: 'string', enum: ['groq', 'openai', 'custom'], description: '引擎(可选):只接受与配置 asrEngine 相同的值,用于显式确认;接口地址与密钥始终取配置。' },
|
|
@@ -777,6 +979,7 @@ export function buildVoiceTools(config, deps = {}) {
|
|
|
777
979
|
output: { type: 'string', description: '把转写文本写成 .txt 的路径(可选,相对路径按会话工作目录解析)。' },
|
|
778
980
|
subtitleOutput: { type: 'string', description: '导出字幕路径(可选,.srt 或 .vtt,相对路径按会话工作目录解析;指定时自动开启时间戳)。' },
|
|
779
981
|
timestamps: { type: 'boolean', description: '是否请求分段时间戳(可选,默认 false)。' },
|
|
982
|
+
diarize: { type: 'boolean', description: '是否启用多人说话人区分(可选,默认 false;双声道录音在有 ffmpeg 时可按声道分离,或解析/推断说话人切换并输出 [说话人 A] 标记)。' },
|
|
780
983
|
autoCompress: { type: 'boolean', description: '超过 25MB 时是否自动调用本地 ffmpeg 压缩与分段转写(可选,默认 false)。' },
|
|
781
984
|
}),
|
|
782
985
|
output: {
|
|
@@ -821,15 +1024,29 @@ export function buildVoiceTools(config, deps = {}) {
|
|
|
821
1024
|
const model = optionalString(args, 'model') ?? (cfg.asrModel !== '' ? cfg.asrModel : FALLBACK_ASR_MODEL);
|
|
822
1025
|
const language = optionalString(args, 'language');
|
|
823
1026
|
const prompt = optionalString(args, 'prompt');
|
|
824
|
-
const
|
|
1027
|
+
const diarize = args.diarize === true;
|
|
1028
|
+
const needTimestamps = args.timestamps === true || subtitleOutput !== undefined || diarize;
|
|
825
1029
|
const allowEmptyApiKey = cfg.asrEngine === 'custom' && isLoopbackBaseUrl(baseUrl);
|
|
826
1030
|
const fetchImpl = getProxyFetch(cfg.proxyUrl) ?? globalThis.fetch;
|
|
827
1031
|
const sttFn = deps.stt ?? transcribe;
|
|
1032
|
+
const collectSpeakers = (segs, explicit) => {
|
|
1033
|
+
if (Array.isArray(explicit) && explicit.length > 0)
|
|
1034
|
+
return [...explicit];
|
|
1035
|
+
if (!Array.isArray(segs))
|
|
1036
|
+
return undefined;
|
|
1037
|
+
const uniq = new Set();
|
|
1038
|
+
for (const s of segs) {
|
|
1039
|
+
if (typeof s.speaker === 'string' && s.speaker.trim() !== '')
|
|
1040
|
+
uniq.add(s.speaker.trim());
|
|
1041
|
+
}
|
|
1042
|
+
return uniq.size > 0 ? Array.from(uniq) : undefined;
|
|
1043
|
+
};
|
|
828
1044
|
const fileSize = statSync(audioPath).size;
|
|
829
1045
|
let text = '';
|
|
830
1046
|
let detectedLanguage = '';
|
|
831
1047
|
let duration;
|
|
832
1048
|
let segments;
|
|
1049
|
+
let speakers;
|
|
833
1050
|
if (fileSize > MAX_ASR_BYTES) {
|
|
834
1051
|
if (args.autoCompress !== true)
|
|
835
1052
|
assertAudioSize(fileSize);
|
|
@@ -846,12 +1063,14 @@ export function buildVoiceTools(config, deps = {}) {
|
|
|
846
1063
|
language,
|
|
847
1064
|
prompt,
|
|
848
1065
|
timestamps: needTimestamps,
|
|
1066
|
+
...(diarize ? { diarize: true } : {}),
|
|
849
1067
|
allowEmptyApiKey,
|
|
850
1068
|
}, fetchImpl, cfg.timeoutMs, signal), signal);
|
|
851
1069
|
return { chunk, res };
|
|
852
1070
|
});
|
|
853
1071
|
throwIfAborted(signal);
|
|
854
|
-
|
|
1072
|
+
const rawText = chunkResults.map((item) => item.res.text.trim()).filter(Boolean).join('\n');
|
|
1073
|
+
text = rawText;
|
|
855
1074
|
detectedLanguage = chunkResults.find((item) => typeof item.res.language === 'string' && item.res.language !== '')?.res.language ?? '';
|
|
856
1075
|
let totalDuration = 0;
|
|
857
1076
|
let hasDuration = false;
|
|
@@ -869,6 +1088,7 @@ export function buildVoiceTools(config, deps = {}) {
|
|
|
869
1088
|
start: seg.start + chunk.offsetSeconds,
|
|
870
1089
|
end: seg.end + chunk.offsetSeconds,
|
|
871
1090
|
text: seg.text,
|
|
1091
|
+
...(seg.speaker !== undefined ? { speaker: seg.speaker } : {}),
|
|
872
1092
|
});
|
|
873
1093
|
}
|
|
874
1094
|
}
|
|
@@ -877,41 +1097,130 @@ export function buildVoiceTools(config, deps = {}) {
|
|
|
877
1097
|
duration = totalDuration;
|
|
878
1098
|
if (mergedSegments.length > 0)
|
|
879
1099
|
segments = mergedSegments;
|
|
1100
|
+
if (diarize) {
|
|
1101
|
+
if (mergedSegments.some((s) => typeof s.speaker === 'string' && s.speaker.trim() !== '')) {
|
|
1102
|
+
const diarized = formatDiarizedTranscript(mergedSegments);
|
|
1103
|
+
if (diarized !== '')
|
|
1104
|
+
text = diarized;
|
|
1105
|
+
}
|
|
1106
|
+
speakers = collectSpeakers(mergedSegments, chunkResults.flatMap((item) => item.res.speakers ?? []));
|
|
1107
|
+
}
|
|
880
1108
|
}
|
|
881
1109
|
finally {
|
|
882
1110
|
prep.cleanup();
|
|
883
1111
|
}
|
|
884
1112
|
}
|
|
885
1113
|
else {
|
|
886
|
-
let
|
|
887
|
-
if (deps.
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
1114
|
+
let handledStereo = false;
|
|
1115
|
+
if (diarize && (deps.splitStereo !== undefined || (deps.stt === undefined && hasFfmpeg()))) {
|
|
1116
|
+
const stereo = await raceAbort((deps.splitStereo ?? splitStereoChannelsForDiarization)(audioPath, signal), signal);
|
|
1117
|
+
if (stereo !== null && stereo.channels.length > 0) {
|
|
1118
|
+
handledStereo = true;
|
|
1119
|
+
try {
|
|
1120
|
+
const channelResults = await mapLimit(stereo.channels, 2, signal, async (ch) => {
|
|
1121
|
+
const res = await raceAbort(sttFn(baseUrl, cfg.asrApiKey, {
|
|
1122
|
+
audio: ch.audio,
|
|
1123
|
+
filename: ch.filename,
|
|
1124
|
+
model,
|
|
1125
|
+
language,
|
|
1126
|
+
prompt,
|
|
1127
|
+
timestamps: true,
|
|
1128
|
+
allowEmptyApiKey,
|
|
1129
|
+
}, fetchImpl, cfg.timeoutMs, signal), signal);
|
|
1130
|
+
return { ch, res };
|
|
1131
|
+
});
|
|
1132
|
+
throwIfAborted(signal);
|
|
1133
|
+
const mergedSegments = [];
|
|
1134
|
+
let totalDuration = 0;
|
|
1135
|
+
let hasDuration = false;
|
|
1136
|
+
for (const { ch, res } of channelResults) {
|
|
1137
|
+
if (detectedLanguage === '' && typeof res.language === 'string' && res.language !== '') {
|
|
1138
|
+
detectedLanguage = res.language;
|
|
1139
|
+
}
|
|
1140
|
+
if (typeof res.duration === 'number' && Number.isFinite(res.duration)) {
|
|
1141
|
+
totalDuration = Math.max(totalDuration, res.duration);
|
|
1142
|
+
hasDuration = true;
|
|
1143
|
+
}
|
|
1144
|
+
if (Array.isArray(res.segments) && res.segments.length > 0) {
|
|
1145
|
+
for (const seg of res.segments) {
|
|
1146
|
+
mergedSegments.push({
|
|
1147
|
+
id: seg.id,
|
|
1148
|
+
start: seg.start,
|
|
1149
|
+
end: seg.end,
|
|
1150
|
+
text: seg.text,
|
|
1151
|
+
speaker: ch.speaker,
|
|
1152
|
+
});
|
|
1153
|
+
}
|
|
1154
|
+
}
|
|
1155
|
+
else if (res.text.trim() !== '') {
|
|
1156
|
+
mergedSegments.push({
|
|
1157
|
+
id: 0,
|
|
1158
|
+
start: 0,
|
|
1159
|
+
end: typeof res.duration === 'number' && Number.isFinite(res.duration) ? res.duration : 0,
|
|
1160
|
+
text: res.text.trim(),
|
|
1161
|
+
speaker: ch.speaker,
|
|
1162
|
+
});
|
|
1163
|
+
}
|
|
1164
|
+
}
|
|
1165
|
+
mergedSegments.sort((a, b) => a.start - b.start);
|
|
1166
|
+
for (let i = 0; i < mergedSegments.length; i++) {
|
|
1167
|
+
mergedSegments[i] = { ...mergedSegments[i], id: i + 1 };
|
|
1168
|
+
}
|
|
1169
|
+
if (hasDuration)
|
|
1170
|
+
duration = totalDuration;
|
|
1171
|
+
if (mergedSegments.length > 0)
|
|
1172
|
+
segments = mergedSegments;
|
|
1173
|
+
const rawFallback = channelResults.map((item) => item.res.text.trim()).filter(Boolean).join('\n');
|
|
1174
|
+
const diarized = formatDiarizedTranscript(mergedSegments);
|
|
1175
|
+
text = diarized !== '' ? diarized : rawFallback;
|
|
1176
|
+
speakers = collectSpeakers(mergedSegments);
|
|
1177
|
+
}
|
|
1178
|
+
finally {
|
|
1179
|
+
stereo.cleanup();
|
|
1180
|
+
}
|
|
1181
|
+
}
|
|
892
1182
|
}
|
|
893
|
-
|
|
894
|
-
let
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
1183
|
+
if (!handledStereo) {
|
|
1184
|
+
let audioInput;
|
|
1185
|
+
if (deps.stt !== undefined) {
|
|
1186
|
+
// 保持单元测试兼容性:注入假 stt 时传 Buffer(部分测试会对 captured.audio 做扩展运算符迭代)。
|
|
1187
|
+
const buf = readFileSync(audioPath);
|
|
1188
|
+
assertAudioSize(buf.length);
|
|
1189
|
+
audioInput = buf;
|
|
1190
|
+
}
|
|
1191
|
+
else {
|
|
1192
|
+
let blob = await openAsBlob(audioPath, { type: mimeFromPath(audioPath) });
|
|
1193
|
+
// 伪文件(如 procfs)size 报 0 字节,回退到 readFileSync 再判一次。
|
|
1194
|
+
if (blob.size === 0)
|
|
1195
|
+
blob = readFileSync(audioPath);
|
|
1196
|
+
assertAudioSize(Buffer.isBuffer(blob) ? blob.length : blob.size);
|
|
1197
|
+
audioInput = blob;
|
|
1198
|
+
}
|
|
1199
|
+
const res = await raceAbort(sttFn(baseUrl, cfg.asrApiKey, {
|
|
1200
|
+
audio: audioInput,
|
|
1201
|
+
filename: basename(audioPath),
|
|
1202
|
+
model,
|
|
1203
|
+
language,
|
|
1204
|
+
prompt,
|
|
1205
|
+
timestamps: needTimestamps,
|
|
1206
|
+
...(diarize ? { diarize: true } : {}),
|
|
1207
|
+
allowEmptyApiKey,
|
|
1208
|
+
}, fetchImpl, cfg.timeoutMs, signal), signal);
|
|
1209
|
+
throwIfAborted(signal);
|
|
1210
|
+
const rawText = res.text;
|
|
1211
|
+
text = rawText;
|
|
1212
|
+
detectedLanguage = res.language ?? '';
|
|
1213
|
+
duration = res.duration;
|
|
1214
|
+
segments = res.segments;
|
|
1215
|
+
if (diarize) {
|
|
1216
|
+
if (Array.isArray(res.segments) && res.segments.some((s) => typeof s.speaker === 'string' && s.speaker.trim() !== '')) {
|
|
1217
|
+
const diarized = formatDiarizedTranscript(res.segments);
|
|
1218
|
+
if (diarized !== '')
|
|
1219
|
+
text = diarized;
|
|
1220
|
+
}
|
|
1221
|
+
speakers = collectSpeakers(res.segments, res.speakers);
|
|
1222
|
+
}
|
|
900
1223
|
}
|
|
901
|
-
const res = await raceAbort(sttFn(baseUrl, cfg.asrApiKey, {
|
|
902
|
-
audio: audioInput,
|
|
903
|
-
filename: basename(audioPath),
|
|
904
|
-
model,
|
|
905
|
-
language,
|
|
906
|
-
prompt,
|
|
907
|
-
timestamps: needTimestamps,
|
|
908
|
-
allowEmptyApiKey,
|
|
909
|
-
}, fetchImpl, cfg.timeoutMs, signal), signal);
|
|
910
|
-
throwIfAborted(signal);
|
|
911
|
-
text = res.text;
|
|
912
|
-
detectedLanguage = res.language ?? '';
|
|
913
|
-
duration = res.duration;
|
|
914
|
-
segments = res.segments;
|
|
915
1224
|
}
|
|
916
1225
|
let transcriptFile = '';
|
|
917
1226
|
if (output !== undefined) {
|
|
@@ -937,6 +1246,7 @@ export function buildVoiceTools(config, deps = {}) {
|
|
|
937
1246
|
subtitleFile,
|
|
938
1247
|
...(duration !== undefined ? { duration } : {}),
|
|
939
1248
|
...(segments !== undefined ? { segments } : {}),
|
|
1249
|
+
...(speakers !== undefined && speakers.length > 0 ? { speakers } : {}),
|
|
940
1250
|
};
|
|
941
1251
|
},
|
|
942
1252
|
get timeoutMs() { return getCfg().timeoutMs + 10000; },
|