dsh-voice 0.8.1 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/tools.js CHANGED
@@ -10,8 +10,8 @@ import { fetchEdgeVoices, synthesizeSpeech, synthesizeSpeechWithMetadata } from
10
10
  import { rateToSpeed, synthesizeOpenAiSpeech } from './openai-tts.js';
11
11
  import { assertAudioFile, assertWorkspaceFile, ensureWorkspaceDirectory, resolveOutputPath, writeWorkspaceFile } from './paths.js';
12
12
  import { createProxyFetch } from './proxy-fetch.js';
13
- import { assertAudioSize, formatSttSubtitles, hasFfmpeg, prepareAudioForAsr, transcribe } from './stt.js';
14
- import { cleanMarkdownForSpeech, concatMp3Buffers, createSilenceMp3, estimateMp3DurationMs, formatSubtitles, normalizeAudioLoudness, offsetCues, renderPreviewHtml, splitTextIntoChunks, } from './tts-text.js';
13
+ import { assertAudioSize, formatDiarizedTranscript, formatSttSubtitles, hasFfmpeg, prepareAudioForAsr, splitStereoChannelsForDiarization, transcribe } from './stt.js';
14
+ import { applyPronunciationsPlain, cleanMarkdownForSpeech, concatMp3Buffers, createSilenceMp3, formatSubtitles, inspectMp3Buffer, mixAudioWithBgm, normalizeAudioLoudness, offsetCues, parseSubtitleFile, renderPreviewHtml, scaleRatePercent, splitTextIntoChunks, } from './tts-text.js';
15
15
  import { filterVoices, isValidVoiceId, resolvePersona, VOICE_PERSONAS, VOICES } from './voices.js';
16
16
  function compileParameters(spec) {
17
17
  const properties = {};
@@ -134,11 +134,57 @@ const listSchema = {
134
134
  },
135
135
  additionalProperties: true,
136
136
  };
137
+ function parseCallPronunciations(raw) {
138
+ if (Array.isArray(raw)) {
139
+ const out = [];
140
+ for (const item of raw) {
141
+ if (typeof item !== 'object' || item === null || Array.isArray(item))
142
+ continue;
143
+ const rec = item;
144
+ const word = typeof rec.word === 'string' ? rec.word.trim() : '';
145
+ const alias = typeof rec.alias === 'string' ? rec.alias.trim() : '';
146
+ if (word !== '' && alias !== '')
147
+ out.push({ word, alias });
148
+ }
149
+ return out;
150
+ }
151
+ if (typeof raw === 'object' && raw !== null) {
152
+ const out = [];
153
+ for (const [key, val] of Object.entries(raw)) {
154
+ if (typeof val !== 'string')
155
+ continue;
156
+ const word = key.trim();
157
+ const alias = val.trim();
158
+ if (word !== '' && alias !== '')
159
+ out.push({ word, alias });
160
+ }
161
+ return out;
162
+ }
163
+ return [];
164
+ }
165
+ function mergePronunciations(base, overrides) {
166
+ const merged = new Map();
167
+ for (const rule of base ?? []) {
168
+ const word = rule.word.trim();
169
+ const alias = rule.alias.trim();
170
+ if (word !== '' && alias !== '')
171
+ merged.set(word, { word, alias });
172
+ }
173
+ for (const rule of overrides) {
174
+ const word = rule.word.trim();
175
+ const alias = rule.alias.trim();
176
+ if (word !== '' && alias !== '')
177
+ merged.set(word, { word, alias });
178
+ }
179
+ return Array.from(merged.values());
180
+ }
137
181
  const ttsSchema = {
138
182
  type: 'object',
139
183
  properties: {
140
184
  output: { type: 'string' },
141
185
  bytes: { type: 'integer' },
186
+ durationMs: { type: 'integer' },
187
+ durationSec: { type: 'number' },
142
188
  voice: { type: 'string' },
143
189
  rate: { type: 'string' },
144
190
  pitch: { type: 'string' },
@@ -157,6 +203,7 @@ const sttSchema = {
157
203
  audio: { type: 'string' },
158
204
  transcriptFile: { type: 'string' },
159
205
  subtitleFile: { type: 'string' },
206
+ speakers: { type: 'array', items: { type: 'string' } },
160
207
  },
161
208
  additionalProperties: true,
162
209
  };
@@ -177,6 +224,8 @@ const FALLBACK_ASR_MODEL = 'whisper-1';
177
224
  const MAX_TTS_CHARS = 100000;
178
225
  /** 多角色对话/播客分段数上限。 */
179
226
  const MAX_TTS_SEGMENTS = 50;
227
+ /** 反向字幕卡点配音条目上限。 */
228
+ const MAX_SUBTITLE_INPUT_CUES = 200;
180
229
  /** OpenAI 兼容 TTS 默认模型。 */
181
230
  const FALLBACK_TTS_MODEL = 'gpt-4o-mini-tts';
182
231
  /** ASR 直传上限(25MB)。 */
@@ -238,7 +287,7 @@ export function buildVoiceTools(config, deps = {}) {
238
287
  return false;
239
288
  return cfg.ttsEngine === 'edge';
240
289
  };
241
- const resolveEffectiveVoiceSpec = (cfg, cwd, input) => {
290
+ const resolveEffectiveVoiceSpec = (cfg, cwd, input, refAudioCache) => {
242
291
  let customAlias;
243
292
  let voiceConsumedAsAlias = false;
244
293
  let personaConsumedAsAlias = false;
@@ -282,22 +331,28 @@ export function buildVoiceTools(config, deps = {}) {
282
331
  const hasReferenceAudio = effectiveReferenceAudio !== undefined && effectiveReferenceAudio.trim() !== '';
283
332
  if (hasReferenceAudio) {
284
333
  const refPath = assertAudioFile(effectiveReferenceAudio, cwd);
285
- const size = statSync(refPath).size;
286
- if (size <= 0) {
287
- throw new Error('参考音频文件为空,无法用于音色克隆:' + effectiveReferenceAudio);
288
- }
289
- if (size > MAX_REFERENCE_AUDIO_BYTES) {
290
- throw new Error('参考音频超过 10MB 上限(当前 ' + size + ' 字节),请裁剪为 5–10 秒人声片段:' + effectiveReferenceAudio);
291
- }
292
- const buf = readFileSync(refPath);
293
- if (buf.length === 0 || buf.length > MAX_REFERENCE_AUDIO_BYTES) {
294
- throw new Error('参考音频大小异常(必须为 1 字节至 10MB):' + effectiveReferenceAudio);
334
+ let buf = refAudioCache?.get(refPath);
335
+ if (buf === undefined) {
336
+ const size = statSync(refPath).size;
337
+ if (size <= 0) {
338
+ throw new Error('参考音频文件为空,无法用于音色克隆:' + effectiveReferenceAudio);
339
+ }
340
+ if (size > MAX_REFERENCE_AUDIO_BYTES) {
341
+ throw new Error('参考音频超过 10MB 上限(当前 ' + size + ' 字节),请裁剪为 5–10 秒人声片段:' + effectiveReferenceAudio);
342
+ }
343
+ buf = readFileSync(refPath);
344
+ if (buf.length === 0 || buf.length > MAX_REFERENCE_AUDIO_BYTES) {
345
+ throw new Error('参考音频大小异常(必须为 1 字节至 10MB):' + effectiveReferenceAudio);
346
+ }
347
+ refAudioCache?.set(refPath, buf);
295
348
  }
296
349
  if (resolveUseEdge(cfg, true)) {
297
350
  throw new Error('零样本参考音频克隆(referenceAudio)需要兼容克隆的 TTS 引擎:请在 DSH「设置 → 插件 → 语音」将 ttsEngine 切换为 siliconflow(CosyVoice2-0.5B)或 custom(如本地 GPT-SoVITS / IndexTTS)。');
298
351
  }
299
352
  references = [{
300
353
  audioDataUri: 'data:' + mimeFromPath(refPath) + ';base64,' + buf.toString('base64'),
354
+ rawAudio: buf,
355
+ filename: basename(refPath),
301
356
  ...(effectiveReferenceText !== undefined && effectiveReferenceText.trim() !== '' ? { text: effectiveReferenceText.trim() } : {}),
302
357
  }];
303
358
  }
@@ -327,32 +382,52 @@ export function buildVoiceTools(config, deps = {}) {
327
382
  throwIfAborted(signal);
328
383
  return normalized;
329
384
  };
330
- const synthesizeChunk = async (cfg, useEdge, chunkText, voice, rate, pitch, needSubtitles, signal, instructions, references) => {
385
+ const finalizeVoiceAudio = async (rawAudio, bgmPath, bgmVolume, bgmDucking, shouldNormalize, signal) => {
386
+ throwIfAborted(signal);
387
+ if (bgmPath !== undefined) {
388
+ if (deps.mixBgm === undefined && !hasFfmpeg()) {
389
+ throw new Error('使用 bgmFile 背景音乐自动闪避混音需要本机安装 ffmpeg(或使用 dsh-ffmpeg)。');
390
+ }
391
+ const mixed = await raceAbort((deps.mixBgm ?? mixAudioWithBgm)(rawAudio, {
392
+ bgmPath,
393
+ bgmVolume,
394
+ ducking: bgmDucking,
395
+ normalizeLoudness: shouldNormalize,
396
+ }, signal), signal);
397
+ throwIfAborted(signal);
398
+ return mixed;
399
+ }
400
+ return await maybeNormalizeAudio(shouldNormalize, rawAudio, signal);
401
+ };
402
+ const synthesizeChunk = async (cfg, useEdge, chunkText, voice, rate, pitch, needSubtitles, signal, instructions, references, pronunciations) => {
403
+ const hasPronunciations = pronunciations !== undefined && pronunciations.length > 0;
331
404
  if (useEdge) {
405
+ const pronOpt = hasPronunciations ? { pronunciations } : {};
332
406
  if (!needSubtitles) {
333
407
  const ttsFn = deps.tts ?? (deps.ttsWithMeta !== undefined
334
408
  ? async (opts, ttsDeps, timeoutMs, abortSig) => (await deps.ttsWithMeta(opts, ttsDeps, timeoutMs, abortSig)).audio
335
409
  : synthesizeSpeech);
336
- const audio = await raceAbort(ttsFn({ text: chunkText, voice, rate, pitch }, { proxyUrl: cfg.proxyUrl }, cfg.timeoutMs, signal), signal);
410
+ const audio = await raceAbort(ttsFn({ text: chunkText, voice, rate, pitch, ...pronOpt }, { proxyUrl: cfg.proxyUrl }, cfg.timeoutMs, signal), signal);
337
411
  return { audio, cues: [] };
338
412
  }
339
413
  if (deps.ttsWithMeta !== undefined || deps.tts === undefined) {
340
414
  const withMetaFn = deps.ttsWithMeta ?? synthesizeSpeechWithMetadata;
341
- const res = await raceAbort(withMetaFn({ text: chunkText, voice, rate, pitch, boundary: 'sentence' }, { proxyUrl: cfg.proxyUrl }, cfg.timeoutMs, signal), signal);
415
+ const res = await raceAbort(withMetaFn({ text: chunkText, voice, rate, pitch, boundary: 'sentence', ...pronOpt }, { proxyUrl: cfg.proxyUrl }, cfg.timeoutMs, signal), signal);
342
416
  const cues = res.cues.length > 0
343
417
  ? res.cues
344
- : [{ index: 1, startMs: 0, endMs: Math.max(1, estimateMp3DurationMs(res.audio)), text: chunkText }];
418
+ : [{ index: 1, startMs: 0, endMs: Math.max(1, inspectMp3Buffer(res.audio).durationMs), text: chunkText }];
345
419
  return { audio: res.audio, cues };
346
420
  }
347
- const audio = await raceAbort(deps.tts({ text: chunkText, voice, rate, pitch }, { proxyUrl: cfg.proxyUrl }, cfg.timeoutMs, signal), signal);
421
+ const audio = await raceAbort(deps.tts({ text: chunkText, voice, rate, pitch, ...pronOpt }, { proxyUrl: cfg.proxyUrl }, cfg.timeoutMs, signal), signal);
348
422
  return {
349
423
  audio,
350
- cues: [{ index: 1, startMs: 0, endMs: Math.max(1, estimateMp3DurationMs(audio)), text: chunkText }],
424
+ cues: [{ index: 1, startMs: 0, endMs: Math.max(1, inspectMp3Buffer(audio).durationMs), text: chunkText }],
351
425
  };
352
426
  }
427
+ const spokenText = hasPronunciations ? applyPronunciationsPlain(chunkText, pronunciations) : chunkText;
353
428
  const openaiFn = deps.openaiTts ?? synthesizeOpenAiSpeech;
354
429
  const audio = await raceAbort(openaiFn(cfg.ttsBaseUrl, cfg.ttsApiKey, {
355
- text: chunkText,
430
+ text: spokenText,
356
431
  voice,
357
432
  model: cfg.ttsModel !== '' ? cfg.ttsModel : FALLBACK_TTS_MODEL,
358
433
  speed: rateToSpeed(rate),
@@ -363,7 +438,7 @@ export function buildVoiceTools(config, deps = {}) {
363
438
  return {
364
439
  audio,
365
440
  cues: needSubtitles
366
- ? [{ index: 1, startMs: 0, endMs: Math.max(1, estimateMp3DurationMs(audio)), text: chunkText }]
441
+ ? [{ index: 1, startMs: 0, endMs: Math.max(1, inspectMp3Buffer(audio).durationMs), text: chunkText }]
367
442
  : [],
368
443
  };
369
444
  };
@@ -472,10 +547,11 @@ export function buildVoiceTools(config, deps = {}) {
472
547
  const voiceTts = {
473
548
  name: 'voice_tts',
474
549
  isConcurrencySafe: () => !getCfg().overwrite,
475
- description: '文字转语音:默认调用 edge-tts(微软 Edge 朗读服务,免费)合成 MP3,也可在设置中切换 OpenAI/SiliconFlow/自定义 TTS 引擎。支持内置角色人设 persona、自然语言情绪指令 instructions、5–10 秒参考音频零样本克隆 referenceAudio、多角色播客 segments、读取工作区 inputFile(自动清理 Markdown)、长文自动分段(≤100000 字符)、导出 .srt/.vtt 字幕与 -16 LUFS 响度均衡 normalizeLoudness;输出默认 voice_output.mp3。',
550
+ description: '文字转语音:默认调用 edge-tts(微软 Edge 朗读服务,免费)合成 MP3,也可在设置中切换 OpenAI/SiliconFlow/自定义 TTS 引擎。支持内置角色人设 persona、自然语言情绪指令 instructions、5–10 秒参考音频零样本克隆 referenceAudio、自定义发音词典 pronunciations、多角色播客 segments、读取工作区 inputFile(自动清理 Markdown)、按 .srt/.vtt 字幕时间轴反向卡点配音 subtitleInput、背景音乐自动闪避混音 bgmFile、长文自动分段(≤100000 字符)、导出 .srt/.vtt 字幕与 -16 LUFS 响度均衡 normalizeLoudness,并返回精确时长 durationMs/durationSec;输出默认 voice_output.mp3。',
476
551
  parameters: compileParameters({
477
- text: { type: 'string', description: '要合成的文本(若提供 inputFile 或 segments 则可选;支持长文本自动分段,≤100000 字符)。' },
552
+ text: { type: 'string', description: '要合成的文本(若提供 inputFile、segments 或 subtitleInput 则可选;支持长文本自动分段,≤100000 字符)。' },
478
553
  inputFile: { type: 'string', description: '工作区内文稿文件路径(可选,如 article.md;指定时直接从文件读取)。' },
554
+ subtitleInput: { type: 'string', description: '按现有 .srt/.vtt 字幕时间轴反向卡点配音(可选,工作区内字幕路径;每条字幕超出时间窗时自动微调语速加速,句间自动补齐精确静音以对齐原视频画面)。' },
479
555
  cleanMarkdown: { type: 'boolean', description: '是否清理 Markdown 语法以便朗读(可选;inputFile 为 .md/.markdown 时默认 true)。' },
480
556
  segments: {
481
557
  type: 'array',
@@ -503,6 +579,21 @@ export function buildVoiceTools(config, deps = {}) {
503
579
  instructions: { type: 'string', description: '自然语言情绪/语气指令(可选,OpenAI gpt-4o-mini-tts 与 SiliconFlow CosyVoice2 可用,如“用开心轻快的语气说”,支持 [laughter] / [breath])。' },
504
580
  referenceAudio: { type: 'string', description: '零样本克隆参考音频路径(可选,会话工作区内 5–10 秒干净人声,≤10MB;需 siliconflow 或 custom 引擎)。' },
505
581
  referenceText: { type: 'string', description: '参考音频对应的原始台词文本(可选,零样本克隆推荐填写)。' },
582
+ pronunciations: {
583
+ type: 'array',
584
+ description: '发音替读规则数组(可选,如 [{ word: "Nginx", alias: "Engine X" }],与设置中的 pronunciations 合并,同名以本次参数优先;字幕保留原词,朗读按 alias 发音)。',
585
+ items: {
586
+ type: 'object',
587
+ properties: {
588
+ word: { type: 'string' },
589
+ alias: { type: 'string' },
590
+ },
591
+ required: ['word', 'alias'],
592
+ },
593
+ },
594
+ bgmFile: { type: 'string', description: '背景音乐文件路径(可选,工作区内音频文件;需本机安装 ffmpeg,将自动循环铺底并在人声出现时自动闪避降音)。' },
595
+ bgmVolume: { type: 'number', minimum: 0.02, maximum: 1, default: 0.18, description: '背景音乐相对音量(可选,0.02–1.0,默认 0.18)。' },
596
+ bgmDucking: { type: 'boolean', default: true, description: '人声说话时是否自动压低背景音乐音量(Auto-Ducking,可选,默认 true)。' },
506
597
  normalizeLoudness: { type: 'boolean', default: false, description: '是否在合成后通过 ffmpeg 执行 -16 LUFS 播客响度均衡(可选,默认 false)。' },
507
598
  output: { type: 'string', description: '输出 MP3 路径(可选,相对路径按会话工作目录解析)。' },
508
599
  subtitleOutput: { type: 'string', description: '导出同步字幕文件路径(可选,.srt 或 .vtt,相对路径按会话工作目录解析)。' },
@@ -513,7 +604,8 @@ export function buildVoiceTools(config, deps = {}) {
513
604
  const rec = asRecord(value);
514
605
  const chunksInfo = typeof rec.chunks === 'number' && rec.chunks > 1 ? ',共 ' + rec.chunks + ' 段' : '';
515
606
  const voiceLabel = typeof rec.voice === 'string' && rec.voice !== '' ? rec.voice : '参考音频克隆';
516
- const lines = ['语音合成完成:' + rec.output + '(' + rec.bytes + ' 字节,音色 ' + voiceLabel + chunksInfo + ')'];
607
+ const durationInfo = typeof rec.durationSec === 'number' ? ',时长 ' + rec.durationSec + ' 秒' : '';
608
+ const lines = ['语音合成完成:' + rec.output + '(' + rec.bytes + ' 字节,音色 ' + voiceLabel + chunksInfo + durationInfo + ')'];
517
609
  const subtitleFile = typeof rec.subtitleFile === 'string' ? rec.subtitleFile : '';
518
610
  if (subtitleFile !== '')
519
611
  lines.push('字幕文件已写入:' + subtitleFile);
@@ -537,6 +629,21 @@ export function buildVoiceTools(config, deps = {}) {
537
629
  const requested = optionalString(args, 'output');
538
630
  const subtitleOutput = optionalString(args, 'subtitleOutput');
539
631
  const needSubtitles = subtitleOutput !== undefined;
632
+ const mergedPronunciations = mergePronunciations(cfg.pronunciations, parseCallPronunciations(args.pronunciations));
633
+ const bgmFile = optionalString(args, 'bgmFile');
634
+ let bgmPath;
635
+ let bgmVolume = 0.18;
636
+ if (bgmFile !== undefined) {
637
+ bgmPath = assertAudioFile(bgmFile, cwd);
638
+ if (deps.mixBgm === undefined && !hasFfmpeg()) {
639
+ throw new Error('使用 bgmFile 背景音乐自动闪避混音需要本机安装 ffmpeg(或使用 dsh-ffmpeg)。');
640
+ }
641
+ if (typeof args.bgmVolume === 'number' && Number.isFinite(args.bgmVolume)) {
642
+ bgmVolume = Math.min(1.0, Math.max(0.02, args.bgmVolume));
643
+ }
644
+ }
645
+ const bgmDucking = args.bgmDucking !== false;
646
+ const refAudioCache = new Map();
540
647
  const rawSegments = Array.isArray(args.segments) && args.segments.length > 0 ? args.segments : undefined;
541
648
  if (rawSegments !== undefined) {
542
649
  if (rawSegments.length > MAX_TTS_SEGMENTS) {
@@ -550,7 +657,7 @@ export function buildVoiceTools(config, deps = {}) {
550
657
  instructions: topInstructions,
551
658
  referenceAudio: topReferenceAudio,
552
659
  referenceText: topReferenceText,
553
- });
660
+ }, refAudioCache);
554
661
  const parsedSegments = [];
555
662
  let totalChars = 0;
556
663
  for (let i = 0; i < rawSegments.length; i++) {
@@ -583,7 +690,7 @@ export function buildVoiceTools(config, deps = {}) {
583
690
  instructions: segInstructions ?? topInstructions,
584
691
  referenceAudio: segReferenceAudio ?? topReferenceAudio,
585
692
  referenceText: segReferenceText ?? topReferenceText,
586
- });
693
+ }, refAudioCache);
587
694
  }
588
695
  else if (segReferenceAudio !== undefined) {
589
696
  segSpec = resolveEffectiveVoiceSpec(cfg, cwd, {
@@ -595,11 +702,11 @@ export function buildVoiceTools(config, deps = {}) {
595
702
  instructions: segInstructions ?? topInstructions,
596
703
  referenceAudio: segReferenceAudio,
597
704
  referenceText: segReferenceText ?? topReferenceText,
598
- });
705
+ }, refAudioCache);
599
706
  }
600
707
  else {
601
708
  const reusedRefs = topSpec.references !== undefined && segReferenceText !== undefined
602
- ? topSpec.references.map((r) => ({ audioDataUri: r.audioDataUri, text: segReferenceText }))
709
+ ? topSpec.references.map((r) => ({ ...r, text: segReferenceText }))
603
710
  : topSpec.references;
604
711
  segSpec = {
605
712
  voice: topSpec.voice,
@@ -651,19 +758,23 @@ export function buildVoiceTools(config, deps = {}) {
651
758
  });
652
759
  }
653
760
  }
654
- const chunkResults = await mapLimit(tasks, TTS_CHUNK_CONCURRENCY, signal, (task) => synthesizeChunk(cfg, task.useEdge, task.text, task.voice, task.rate, task.pitch, needSubtitles, signal, task.instructions, task.references));
761
+ const chunkResults = await mapLimit(tasks, TTS_CHUNK_CONCURRENCY, signal, (task) => synthesizeChunk(cfg, task.useEdge, task.text, task.voice, task.rate, task.pitch, needSubtitles, signal, task.instructions, task.references, mergedPronunciations));
655
762
  throwIfAborted(signal);
656
763
  const buffers = [];
657
764
  const allCues = [];
658
765
  let offsetMs = 0;
659
766
  let cueIndex = 1;
660
767
  let resultIdx = 0;
768
+ let detectedSampleRate = 24000;
661
769
  for (let s = 0; s < parsedSegments.length; s++) {
662
770
  const seg = parsedSegments[s];
663
771
  for (let c = 0; c < seg.chunks.length; c++) {
664
772
  const res = chunkResults[resultIdx++];
665
773
  buffers.push(res.audio);
666
- const chunkDurationMs = estimateMp3DurationMs(res.audio);
774
+ const chunkInfo = inspectMp3Buffer(res.audio);
775
+ if (chunkInfo.frameCount > 0)
776
+ detectedSampleRate = chunkInfo.sampleRate;
777
+ const chunkDurationMs = chunkInfo.durationMs;
667
778
  if (needSubtitles) {
668
779
  const shifted = offsetCues(res.cues, offsetMs, cueIndex);
669
780
  allCues.push(...shifted);
@@ -673,22 +784,110 @@ export function buildVoiceTools(config, deps = {}) {
673
784
  }
674
785
  }
675
786
  if (s < parsedSegments.length - 1 && seg.pauseMs > 0) {
676
- const silence = createSilenceMp3(seg.pauseMs);
787
+ const silence = createSilenceMp3(seg.pauseMs, detectedSampleRate);
677
788
  if (silence.length > 0) {
678
789
  buffers.push(silence);
679
- offsetMs += estimateMp3DurationMs(silence);
790
+ offsetMs += inspectMp3Buffer(silence).durationMs;
680
791
  }
681
792
  }
682
793
  }
683
794
  let audio = buffers.length === 1 ? buffers[0] : concatMp3Buffers(buffers);
684
- audio = await maybeNormalizeAudio(normalizeLoudness, audio, signal);
795
+ audio = await finalizeVoiceAudio(audio, bgmPath, bgmVolume, bgmDucking, normalizeLoudness, signal);
796
+ const finalInfo = inspectMp3Buffer(audio);
797
+ const durationMs = finalInfo.durationMs;
798
+ const durationSec = Math.round(durationMs / 100) / 10;
685
799
  const output = writeWorkspaceFile(resolveOutputPath(requested, 'voice_output.mp3', cfg.overwrite, cwd), audio, cwd, undefined, cfg.overwrite);
686
800
  let subtitleFile = '';
687
801
  if (subtitleOutput !== undefined) {
688
802
  const formatted = formatSubtitles(allCues, subtitleFormatOf(subtitleOutput));
689
803
  subtitleFile = writeWorkspaceFile(resolveOutputPath(subtitleOutput, '', cfg.overwrite, cwd), formatted, cwd, undefined, cfg.overwrite);
690
804
  }
691
- return { output, bytes: audio.length, voice: topSpec.voice, rate: topSpec.rate, pitch: topSpec.pitch, textLength: totalChars, chunks: tasks.length, subtitleFile };
805
+ return { output, bytes: audio.length, durationMs, durationSec, voice: topSpec.voice, rate: topSpec.rate, pitch: topSpec.pitch, textLength: totalChars, chunks: tasks.length, subtitleFile };
806
+ }
807
+ const subtitleInput = optionalString(args, 'subtitleInput');
808
+ if (subtitleInput !== undefined) {
809
+ const resolvedSub = assertWorkspaceFile(subtitleInput, cwd, '字幕文件');
810
+ const rawSub = readFileSync(resolvedSub, 'utf8');
811
+ const cues = parseSubtitleFile(rawSub);
812
+ if (cues.length === 0) {
813
+ throw new Error('字幕文件未解析出任何有效字幕条目:' + subtitleInput);
814
+ }
815
+ if (cues.length > MAX_SUBTITLE_INPUT_CUES) {
816
+ throw new Error('subtitleInput 最多支持 ' + MAX_SUBTITLE_INPUT_CUES + ' 条字幕(当前 ' + cues.length + ' 条),请拆分为多段处理。');
817
+ }
818
+ const spec = resolveEffectiveVoiceSpec(cfg, cwd, {
819
+ voice: topVoice,
820
+ persona: topPersona,
821
+ rate: topRate,
822
+ pitch: topPitch,
823
+ instructions: topInstructions,
824
+ referenceAudio: topReferenceAudio,
825
+ referenceText: topReferenceText,
826
+ }, refAudioCache);
827
+ const useEdge = resolveUseEdge(cfg, spec.hasReferenceAudio);
828
+ assertTtsEngineReady(cfg, useEdge);
829
+ assertTtsVoice(spec.voice, useEdge, spec.references !== undefined);
830
+ assertProsody(spec.rate, spec.pitch);
831
+ resolveOutputPath(requested, 'voice_output.mp3', cfg.overwrite, cwd);
832
+ if (subtitleOutput !== undefined)
833
+ resolveOutputPath(subtitleOutput, '', cfg.overwrite, cwd);
834
+ const cueResults = await mapLimit(cues, TTS_CHUNK_CONCURRENCY, signal, async (cue) => {
835
+ const cueText = args.cleanMarkdown === true
836
+ ? (cleanMarkdownForSpeech(cue.text).trim() || cue.text.trim())
837
+ : cue.text.trim();
838
+ let res = await synthesizeChunk(cfg, useEdge, cueText, spec.voice, spec.rate, spec.pitch, false, signal, spec.instructions, spec.references, mergedPronunciations);
839
+ const actualMs = inspectMp3Buffer(res.audio).durationMs;
840
+ const windowMs = Math.max(200, cue.endMs - cue.startMs);
841
+ if (actualMs > windowMs * 1.08) {
842
+ const fastRate = scaleRatePercent(spec.rate, actualMs / windowMs);
843
+ if (fastRate !== spec.rate) {
844
+ res = await synthesizeChunk(cfg, useEdge, cueText, spec.voice, fastRate, spec.pitch, false, signal, spec.instructions, spec.references, mergedPronunciations);
845
+ }
846
+ }
847
+ return { audio: res.audio, textLength: cueText.length };
848
+ });
849
+ throwIfAborted(signal);
850
+ const buffers = [];
851
+ const outCues = [];
852
+ let cursorMs = 0;
853
+ let detectedSampleRate = 24000;
854
+ let totalChars = 0;
855
+ for (let i = 0; i < cues.length; i++) {
856
+ const cue = cues[i];
857
+ const cueAudio = cueResults[i].audio;
858
+ totalChars += cueResults[i].textLength;
859
+ const cueInfo = inspectMp3Buffer(cueAudio);
860
+ if (cueInfo.frameCount > 0)
861
+ detectedSampleRate = cueInfo.sampleRate;
862
+ if (cue.startMs > cursorMs) {
863
+ const pad = createSilenceMp3(cue.startMs - cursorMs, detectedSampleRate);
864
+ if (pad.length > 0) {
865
+ buffers.push(pad);
866
+ cursorMs += inspectMp3Buffer(pad).durationMs;
867
+ }
868
+ }
869
+ const cueStartMs = cursorMs;
870
+ buffers.push(cueAudio);
871
+ cursorMs += cueInfo.durationMs;
872
+ outCues.push({
873
+ index: i + 1,
874
+ startMs: cueStartMs,
875
+ endMs: Math.max(cueStartMs + 1, cursorMs),
876
+ text: cue.text,
877
+ });
878
+ }
879
+ let audio = buffers.length === 1 ? buffers[0] : concatMp3Buffers(buffers);
880
+ audio = await finalizeVoiceAudio(audio, bgmPath, bgmVolume, bgmDucking, normalizeLoudness, signal);
881
+ const finalInfo = inspectMp3Buffer(audio);
882
+ const durationMs = finalInfo.durationMs;
883
+ const durationSec = Math.round(durationMs / 100) / 10;
884
+ const output = writeWorkspaceFile(resolveOutputPath(requested, 'voice_output.mp3', cfg.overwrite, cwd), audio, cwd, undefined, cfg.overwrite);
885
+ let subtitleFile = '';
886
+ if (subtitleOutput !== undefined) {
887
+ const formatted = formatSubtitles(outCues, subtitleFormatOf(subtitleOutput));
888
+ subtitleFile = writeWorkspaceFile(resolveOutputPath(subtitleOutput, '', cfg.overwrite, cwd), formatted, cwd, undefined, cfg.overwrite);
889
+ }
890
+ return { output, bytes: audio.length, durationMs, durationSec, voice: spec.voice, rate: spec.rate, pitch: spec.pitch, textLength: totalChars, chunks: cues.length, subtitleFile };
692
891
  }
693
892
  const inputFile = optionalString(args, 'inputFile');
694
893
  let finalText;
@@ -720,7 +919,7 @@ export function buildVoiceTools(config, deps = {}) {
720
919
  instructions: topInstructions,
721
920
  referenceAudio: topReferenceAudio,
722
921
  referenceText: topReferenceText,
723
- });
922
+ }, refAudioCache);
724
923
  const useEdge = resolveUseEdge(cfg, spec.hasReferenceAudio);
725
924
  assertTtsEngineReady(cfg, useEdge);
726
925
  assertTtsVoice(spec.voice, useEdge, spec.references !== undefined);
@@ -732,13 +931,13 @@ export function buildVoiceTools(config, deps = {}) {
732
931
  let chunkCount = 1;
733
932
  let subtitleFile = '';
734
933
  if (finalText.length <= 5000 && !needSubtitles && useEdge) {
735
- const res = await synthesizeChunk(cfg, true, finalText, spec.voice, spec.rate, spec.pitch, false, signal, spec.instructions, spec.references);
934
+ const res = await synthesizeChunk(cfg, true, finalText, spec.voice, spec.rate, spec.pitch, false, signal, spec.instructions, spec.references, mergedPronunciations);
736
935
  audio = res.audio;
737
936
  }
738
937
  else {
739
938
  const chunks = splitTextIntoChunks(finalText, 2800);
740
939
  chunkCount = chunks.length;
741
- const results = await mapLimit(chunks, TTS_CHUNK_CONCURRENCY, signal, (chunk) => synthesizeChunk(cfg, useEdge, chunk, spec.voice, spec.rate, spec.pitch, needSubtitles, signal, spec.instructions, spec.references));
940
+ const results = await mapLimit(chunks, TTS_CHUNK_CONCURRENCY, signal, (chunk) => synthesizeChunk(cfg, useEdge, chunk, spec.voice, spec.rate, spec.pitch, needSubtitles, signal, spec.instructions, spec.references, mergedPronunciations));
742
941
  throwIfAborted(signal);
743
942
  audio = results.length === 1 ? results[0].audio : concatMp3Buffers(results.map((r) => r.audio));
744
943
  if (subtitleOutput !== undefined) {
@@ -749,7 +948,7 @@ export function buildVoiceTools(config, deps = {}) {
749
948
  const shifted = offsetCues(res.cues, offsetMs, cueIndex);
750
949
  allCues.push(...shifted);
751
950
  cueIndex += shifted.length;
752
- const chunkDurationMs = estimateMp3DurationMs(res.audio);
951
+ const chunkDurationMs = inspectMp3Buffer(res.audio).durationMs;
753
952
  const cueSpan = shifted.length > 0 ? Math.max(0, shifted[shifted.length - 1].endMs - offsetMs) : 0;
754
953
  offsetMs += Math.max(chunkDurationMs, cueSpan);
755
954
  }
@@ -758,16 +957,19 @@ export function buildVoiceTools(config, deps = {}) {
758
957
  }
759
958
  }
760
959
  throwIfAborted(signal);
761
- audio = await maybeNormalizeAudio(normalizeLoudness, audio, signal);
960
+ audio = await finalizeVoiceAudio(audio, bgmPath, bgmVolume, bgmDucking, normalizeLoudness, signal);
961
+ const finalInfo = inspectMp3Buffer(audio);
962
+ const durationMs = finalInfo.durationMs;
963
+ const durationSec = Math.round(durationMs / 100) / 10;
762
964
  const output = writeWorkspaceFile(resolveOutputPath(requested, 'voice_output.mp3', cfg.overwrite, cwd), audio, cwd, undefined, cfg.overwrite);
763
- return { output, bytes: audio.length, voice: spec.voice, rate: spec.rate, pitch: spec.pitch, textLength: finalText.length, chunks: chunkCount, subtitleFile };
965
+ return { output, bytes: audio.length, durationMs, durationSec, voice: spec.voice, rate: spec.rate, pitch: spec.pitch, textLength: finalText.length, chunks: chunkCount, subtitleFile };
764
966
  },
765
967
  get timeoutMs() { return getCfg().timeoutMs + 10000; },
766
968
  };
767
969
  const voiceStt = {
768
970
  name: 'voice_stt',
769
971
  isConcurrencySafe: () => !getCfg().overwrite,
770
- description: '语音转文字:调用 OpenAI 兼容 ASR 接口(默认 Groq whisper-large-v3-turbo;可切 openai/siliconflow/custom)。audio 为音频文件路径(mp3/wav/m4a/ogg/flac,≤25MB;超过 25MB 可设 autoCompress=true 用 ffmpeg 自动压缩分段);language/prompt 可选;output 可写 .txt,subtitleOutput 可写 .srt/.vtt 字幕。',
972
+ description: '语音转文字:调用 OpenAI 兼容 ASR 接口(默认 Groq whisper-large-v3-turbo;可切 openai/siliconflow/custom)。audio 为音频文件路径(mp3/wav/m4a/ogg/flac,≤25MB;超过 25MB 可设 autoCompress=true 用 ffmpeg 自动压缩分段);支持多人说话人区分 diarize=true(双声道按声道分离或推断说话人切换);language/prompt 可选;output 可写 .txt,subtitleOutput 可写 .srt/.vtt 字幕。',
771
973
  parameters: compileParameters({
772
974
  audio: { type: 'string', required: true, description: '音频文件路径(必填)。' },
773
975
  engine: { type: 'string', enum: ['groq', 'openai', 'custom'], description: '引擎(可选):只接受与配置 asrEngine 相同的值,用于显式确认;接口地址与密钥始终取配置。' },
@@ -777,6 +979,7 @@ export function buildVoiceTools(config, deps = {}) {
777
979
  output: { type: 'string', description: '把转写文本写成 .txt 的路径(可选,相对路径按会话工作目录解析)。' },
778
980
  subtitleOutput: { type: 'string', description: '导出字幕路径(可选,.srt 或 .vtt,相对路径按会话工作目录解析;指定时自动开启时间戳)。' },
779
981
  timestamps: { type: 'boolean', description: '是否请求分段时间戳(可选,默认 false)。' },
982
+ diarize: { type: 'boolean', description: '是否启用多人说话人区分(可选,默认 false;双声道录音在有 ffmpeg 时可按声道分离,或解析/推断说话人切换并输出 [说话人 A] 标记)。' },
780
983
  autoCompress: { type: 'boolean', description: '超过 25MB 时是否自动调用本地 ffmpeg 压缩与分段转写(可选,默认 false)。' },
781
984
  }),
782
985
  output: {
@@ -821,15 +1024,29 @@ export function buildVoiceTools(config, deps = {}) {
821
1024
  const model = optionalString(args, 'model') ?? (cfg.asrModel !== '' ? cfg.asrModel : FALLBACK_ASR_MODEL);
822
1025
  const language = optionalString(args, 'language');
823
1026
  const prompt = optionalString(args, 'prompt');
824
- const needTimestamps = args.timestamps === true || subtitleOutput !== undefined;
1027
+ const diarize = args.diarize === true;
1028
+ const needTimestamps = args.timestamps === true || subtitleOutput !== undefined || diarize;
825
1029
  const allowEmptyApiKey = cfg.asrEngine === 'custom' && isLoopbackBaseUrl(baseUrl);
826
1030
  const fetchImpl = getProxyFetch(cfg.proxyUrl) ?? globalThis.fetch;
827
1031
  const sttFn = deps.stt ?? transcribe;
1032
+ const collectSpeakers = (segs, explicit) => {
1033
+ if (Array.isArray(explicit) && explicit.length > 0)
1034
+ return [...explicit];
1035
+ if (!Array.isArray(segs))
1036
+ return undefined;
1037
+ const uniq = new Set();
1038
+ for (const s of segs) {
1039
+ if (typeof s.speaker === 'string' && s.speaker.trim() !== '')
1040
+ uniq.add(s.speaker.trim());
1041
+ }
1042
+ return uniq.size > 0 ? Array.from(uniq) : undefined;
1043
+ };
828
1044
  const fileSize = statSync(audioPath).size;
829
1045
  let text = '';
830
1046
  let detectedLanguage = '';
831
1047
  let duration;
832
1048
  let segments;
1049
+ let speakers;
833
1050
  if (fileSize > MAX_ASR_BYTES) {
834
1051
  if (args.autoCompress !== true)
835
1052
  assertAudioSize(fileSize);
@@ -846,12 +1063,14 @@ export function buildVoiceTools(config, deps = {}) {
846
1063
  language,
847
1064
  prompt,
848
1065
  timestamps: needTimestamps,
1066
+ ...(diarize ? { diarize: true } : {}),
849
1067
  allowEmptyApiKey,
850
1068
  }, fetchImpl, cfg.timeoutMs, signal), signal);
851
1069
  return { chunk, res };
852
1070
  });
853
1071
  throwIfAborted(signal);
854
- text = chunkResults.map((item) => item.res.text.trim()).filter(Boolean).join('\n');
1072
+ const rawText = chunkResults.map((item) => item.res.text.trim()).filter(Boolean).join('\n');
1073
+ text = rawText;
855
1074
  detectedLanguage = chunkResults.find((item) => typeof item.res.language === 'string' && item.res.language !== '')?.res.language ?? '';
856
1075
  let totalDuration = 0;
857
1076
  let hasDuration = false;
@@ -869,6 +1088,7 @@ export function buildVoiceTools(config, deps = {}) {
869
1088
  start: seg.start + chunk.offsetSeconds,
870
1089
  end: seg.end + chunk.offsetSeconds,
871
1090
  text: seg.text,
1091
+ ...(seg.speaker !== undefined ? { speaker: seg.speaker } : {}),
872
1092
  });
873
1093
  }
874
1094
  }
@@ -877,41 +1097,130 @@ export function buildVoiceTools(config, deps = {}) {
877
1097
  duration = totalDuration;
878
1098
  if (mergedSegments.length > 0)
879
1099
  segments = mergedSegments;
1100
+ if (diarize) {
1101
+ if (mergedSegments.some((s) => typeof s.speaker === 'string' && s.speaker.trim() !== '')) {
1102
+ const diarized = formatDiarizedTranscript(mergedSegments);
1103
+ if (diarized !== '')
1104
+ text = diarized;
1105
+ }
1106
+ speakers = collectSpeakers(mergedSegments, chunkResults.flatMap((item) => item.res.speakers ?? []));
1107
+ }
880
1108
  }
881
1109
  finally {
882
1110
  prep.cleanup();
883
1111
  }
884
1112
  }
885
1113
  else {
886
- let audioInput;
887
- if (deps.stt !== undefined) {
888
- // 保持单元测试兼容性:注入假 stt 时传 Buffer(部分测试会对 captured.audio 做扩展运算符迭代)。
889
- const buf = readFileSync(audioPath);
890
- assertAudioSize(buf.length);
891
- audioInput = buf;
1114
+ let handledStereo = false;
1115
+ if (diarize && (deps.splitStereo !== undefined || (deps.stt === undefined && hasFfmpeg()))) {
1116
+ const stereo = await raceAbort((deps.splitStereo ?? splitStereoChannelsForDiarization)(audioPath, signal), signal);
1117
+ if (stereo !== null && stereo.channels.length > 0) {
1118
+ handledStereo = true;
1119
+ try {
1120
+ const channelResults = await mapLimit(stereo.channels, 2, signal, async (ch) => {
1121
+ const res = await raceAbort(sttFn(baseUrl, cfg.asrApiKey, {
1122
+ audio: ch.audio,
1123
+ filename: ch.filename,
1124
+ model,
1125
+ language,
1126
+ prompt,
1127
+ timestamps: true,
1128
+ allowEmptyApiKey,
1129
+ }, fetchImpl, cfg.timeoutMs, signal), signal);
1130
+ return { ch, res };
1131
+ });
1132
+ throwIfAborted(signal);
1133
+ const mergedSegments = [];
1134
+ let totalDuration = 0;
1135
+ let hasDuration = false;
1136
+ for (const { ch, res } of channelResults) {
1137
+ if (detectedLanguage === '' && typeof res.language === 'string' && res.language !== '') {
1138
+ detectedLanguage = res.language;
1139
+ }
1140
+ if (typeof res.duration === 'number' && Number.isFinite(res.duration)) {
1141
+ totalDuration = Math.max(totalDuration, res.duration);
1142
+ hasDuration = true;
1143
+ }
1144
+ if (Array.isArray(res.segments) && res.segments.length > 0) {
1145
+ for (const seg of res.segments) {
1146
+ mergedSegments.push({
1147
+ id: seg.id,
1148
+ start: seg.start,
1149
+ end: seg.end,
1150
+ text: seg.text,
1151
+ speaker: ch.speaker,
1152
+ });
1153
+ }
1154
+ }
1155
+ else if (res.text.trim() !== '') {
1156
+ mergedSegments.push({
1157
+ id: 0,
1158
+ start: 0,
1159
+ end: typeof res.duration === 'number' && Number.isFinite(res.duration) ? res.duration : 0,
1160
+ text: res.text.trim(),
1161
+ speaker: ch.speaker,
1162
+ });
1163
+ }
1164
+ }
1165
+ mergedSegments.sort((a, b) => a.start - b.start);
1166
+ for (let i = 0; i < mergedSegments.length; i++) {
1167
+ mergedSegments[i] = { ...mergedSegments[i], id: i + 1 };
1168
+ }
1169
+ if (hasDuration)
1170
+ duration = totalDuration;
1171
+ if (mergedSegments.length > 0)
1172
+ segments = mergedSegments;
1173
+ const rawFallback = channelResults.map((item) => item.res.text.trim()).filter(Boolean).join('\n');
1174
+ const diarized = formatDiarizedTranscript(mergedSegments);
1175
+ text = diarized !== '' ? diarized : rawFallback;
1176
+ speakers = collectSpeakers(mergedSegments);
1177
+ }
1178
+ finally {
1179
+ stereo.cleanup();
1180
+ }
1181
+ }
892
1182
  }
893
- else {
894
- let blob = await openAsBlob(audioPath, { type: mimeFromPath(audioPath) });
895
- // 伪文件(如 procfs)size 报 0 字节,回退到 readFileSync 再判一次。
896
- if (blob.size === 0)
897
- blob = readFileSync(audioPath);
898
- assertAudioSize(Buffer.isBuffer(blob) ? blob.length : blob.size);
899
- audioInput = blob;
1183
+ if (!handledStereo) {
1184
+ let audioInput;
1185
+ if (deps.stt !== undefined) {
1186
+ // 保持单元测试兼容性:注入假 stt 时传 Buffer(部分测试会对 captured.audio 做扩展运算符迭代)。
1187
+ const buf = readFileSync(audioPath);
1188
+ assertAudioSize(buf.length);
1189
+ audioInput = buf;
1190
+ }
1191
+ else {
1192
+ let blob = await openAsBlob(audioPath, { type: mimeFromPath(audioPath) });
1193
+ // 伪文件(如 procfs)size 报 0 字节,回退到 readFileSync 再判一次。
1194
+ if (blob.size === 0)
1195
+ blob = readFileSync(audioPath);
1196
+ assertAudioSize(Buffer.isBuffer(blob) ? blob.length : blob.size);
1197
+ audioInput = blob;
1198
+ }
1199
+ const res = await raceAbort(sttFn(baseUrl, cfg.asrApiKey, {
1200
+ audio: audioInput,
1201
+ filename: basename(audioPath),
1202
+ model,
1203
+ language,
1204
+ prompt,
1205
+ timestamps: needTimestamps,
1206
+ ...(diarize ? { diarize: true } : {}),
1207
+ allowEmptyApiKey,
1208
+ }, fetchImpl, cfg.timeoutMs, signal), signal);
1209
+ throwIfAborted(signal);
1210
+ const rawText = res.text;
1211
+ text = rawText;
1212
+ detectedLanguage = res.language ?? '';
1213
+ duration = res.duration;
1214
+ segments = res.segments;
1215
+ if (diarize) {
1216
+ if (Array.isArray(res.segments) && res.segments.some((s) => typeof s.speaker === 'string' && s.speaker.trim() !== '')) {
1217
+ const diarized = formatDiarizedTranscript(res.segments);
1218
+ if (diarized !== '')
1219
+ text = diarized;
1220
+ }
1221
+ speakers = collectSpeakers(res.segments, res.speakers);
1222
+ }
900
1223
  }
901
- const res = await raceAbort(sttFn(baseUrl, cfg.asrApiKey, {
902
- audio: audioInput,
903
- filename: basename(audioPath),
904
- model,
905
- language,
906
- prompt,
907
- timestamps: needTimestamps,
908
- allowEmptyApiKey,
909
- }, fetchImpl, cfg.timeoutMs, signal), signal);
910
- throwIfAborted(signal);
911
- text = res.text;
912
- detectedLanguage = res.language ?? '';
913
- duration = res.duration;
914
- segments = res.segments;
915
1224
  }
916
1225
  let transcriptFile = '';
917
1226
  if (output !== undefined) {
@@ -937,6 +1246,7 @@ export function buildVoiceTools(config, deps = {}) {
937
1246
  subtitleFile,
938
1247
  ...(duration !== undefined ? { duration } : {}),
939
1248
  ...(segments !== undefined ? { segments } : {}),
1249
+ ...(speakers !== undefined && speakers.length > 0 ? { speakers } : {}),
940
1250
  };
941
1251
  },
942
1252
  get timeoutMs() { return getCfg().timeoutMs + 10000; },