dsh-voice 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/tools.js CHANGED
@@ -10,8 +10,8 @@ import { fetchEdgeVoices, synthesizeSpeech, synthesizeSpeechWithMetadata } from
10
10
  import { rateToSpeed, synthesizeOpenAiSpeech } from './openai-tts.js';
11
11
  import { assertAudioFile, assertWorkspaceFile, ensureWorkspaceDirectory, resolveOutputPath, writeWorkspaceFile } from './paths.js';
12
12
  import { createProxyFetch } from './proxy-fetch.js';
13
- import { assertAudioSize, formatDiarizedTranscript, formatSttSubtitles, hasFfmpeg, prepareAudioForAsr, splitStereoChannelsForDiarization, transcribe } from './stt.js';
14
- import { applyPronunciationsPlain, cleanMarkdownForSpeech, concatMp3Buffers, createSilenceMp3, formatSubtitles, inspectMp3Buffer, mixAudioWithBgm, normalizeAudioLoudness, offsetCues, parseSubtitleFile, renderPreviewHtml, scaleRatePercent, splitTextIntoChunks, } from './tts-text.js';
13
+ import { assertAudioSize, formatDiarizedTranscript, formatSttSubtitles, hasFfmpeg, isVideoContainerFile, prepareAudioForAsr, splitStereoChannelsForDiarization, transcribe } from './stt.js';
14
+ import { applyPronunciationsPlain, cleanMarkdownForSpeech, concatMp3Buffers, createSilenceMp3, embedMp3Chapters, extractMarkdownSections, fitAudioToWindow, formatChaptersMarkdown, formatSubtitles, groupWordCues, inspectMp3Buffer, mixAudioWithBgm, normalizeAudioLoudness, offsetCues, parseSubtitleFile, renderPreviewHtml, scaleRatePercent, splitInlinePauses, splitTextIntoChunks, stripInlinePauses, } from './tts-text.js';
15
15
  import { filterVoices, isValidVoiceId, resolvePersona, VOICE_PERSONAS, VOICES } from './voices.js';
16
16
  function compileParameters(spec) {
17
17
  const properties = {};
@@ -115,7 +115,12 @@ function mimeFromPath(filePath) {
115
115
  return table[ext] ?? 'application/octet-stream';
116
116
  }
117
117
  function subtitleFormatOf(filePath) {
118
- return extname(filePath).toLowerCase() === '.vtt' ? 'vtt' : 'srt';
118
+ const ext = extname(filePath).toLowerCase();
119
+ if (ext === '.vtt')
120
+ return 'vtt';
121
+ if (ext === '.ass')
122
+ return 'ass';
123
+ return 'srt';
119
124
  }
120
125
  const voiceItemSchema = {
121
126
  type: 'object',
@@ -191,6 +196,8 @@ const ttsSchema = {
191
196
  textLength: { type: 'integer' },
192
197
  chunks: { type: 'integer' },
193
198
  subtitleFile: { type: 'string' },
199
+ chapters: { type: 'array', items: { type: 'object', additionalProperties: true } },
200
+ chaptersFile: { type: 'string' },
194
201
  },
195
202
  additionalProperties: true,
196
203
  };
@@ -399,7 +406,7 @@ export function buildVoiceTools(config, deps = {}) {
399
406
  }
400
407
  return await maybeNormalizeAudio(shouldNormalize, rawAudio, signal);
401
408
  };
402
- const synthesizeChunk = async (cfg, useEdge, chunkText, voice, rate, pitch, needSubtitles, signal, instructions, references, pronunciations) => {
409
+ const synthesizeChunk = async (cfg, useEdge, chunkText, voice, rate, pitch, needSubtitles, signal, instructions, references, pronunciations, subtitleOpts) => {
403
410
  const hasPronunciations = pronunciations !== undefined && pronunciations.length > 0;
404
411
  if (useEdge) {
405
412
  const pronOpt = hasPronunciations ? { pronunciations } : {};
@@ -412,33 +419,91 @@ export function buildVoiceTools(config, deps = {}) {
412
419
  }
413
420
  if (deps.ttsWithMeta !== undefined || deps.tts === undefined) {
414
421
  const withMetaFn = deps.ttsWithMeta ?? synthesizeSpeechWithMetadata;
415
- const res = await raceAbort(withMetaFn({ text: chunkText, voice, rate, pitch, boundary: 'sentence', ...pronOpt }, { proxyUrl: cfg.proxyUrl }, cfg.timeoutMs, signal), signal);
416
- const cues = res.cues.length > 0
417
- ? res.cues
418
- : [{ index: 1, startMs: 0, endMs: Math.max(1, inspectMp3Buffer(res.audio).durationMs), text: chunkText }];
419
- return { audio: res.audio, cues };
422
+ const wantWordBoundary = subtitleOpts !== undefined
423
+ && (subtitleOpts.granularity === 'clause' || subtitleOpts.granularity === 'word' || subtitleOpts.isAss);
424
+ if (!wantWordBoundary) {
425
+ const res = await raceAbort(withMetaFn({ text: chunkText, voice, rate, pitch, boundary: 'sentence', ...pronOpt }, { proxyUrl: cfg.proxyUrl }, cfg.timeoutMs, signal), signal);
426
+ const cues = res.cues.length > 0
427
+ ? res.cues
428
+ : [{ index: 1, startMs: 0, endMs: Math.max(1, inspectMp3Buffer(res.audio).durationMs), text: stripInlinePauses(chunkText) }];
429
+ return { audio: res.audio, cues };
430
+ }
431
+ const res = await raceAbort(withMetaFn({ text: chunkText, voice, rate, pitch, boundary: 'word', ...pronOpt }, { proxyUrl: cfg.proxyUrl }, cfg.timeoutMs, signal), signal);
432
+ const rawWords = res.cues;
433
+ const effectiveGranularity = subtitleOpts.granularity === 'sentence' && subtitleOpts.isAss
434
+ ? 'clause'
435
+ : subtitleOpts.granularity;
436
+ const grouped = rawWords.length > 0
437
+ ? groupWordCues(rawWords, effectiveGranularity, subtitleOpts.maxCharsPerCue)
438
+ : [{ index: 1, startMs: 0, endMs: Math.max(1, inspectMp3Buffer(res.audio).durationMs), text: stripInlinePauses(chunkText) }];
439
+ return { audio: res.audio, cues: grouped, wordCues: rawWords };
420
440
  }
421
441
  const audio = await raceAbort(deps.tts({ text: chunkText, voice, rate, pitch, ...pronOpt }, { proxyUrl: cfg.proxyUrl }, cfg.timeoutMs, signal), signal);
422
442
  return {
423
443
  audio,
424
- cues: [{ index: 1, startMs: 0, endMs: Math.max(1, inspectMp3Buffer(audio).durationMs), text: chunkText }],
444
+ cues: [{ index: 1, startMs: 0, endMs: Math.max(1, inspectMp3Buffer(audio).durationMs), text: stripInlinePauses(chunkText) }],
425
445
  };
426
446
  }
427
- const spokenText = hasPronunciations ? applyPronunciationsPlain(chunkText, pronunciations) : chunkText;
428
447
  const openaiFn = deps.openaiTts ?? synthesizeOpenAiSpeech;
429
- const audio = await raceAbort(openaiFn(cfg.ttsBaseUrl, cfg.ttsApiKey, {
430
- text: spokenText,
431
- voice,
432
- model: cfg.ttsModel !== '' ? cfg.ttsModel : FALLBACK_TTS_MODEL,
433
- speed: rateToSpeed(rate),
434
- ...(instructions !== undefined ? { instructions } : {}),
435
- ...(references !== undefined && references.length > 0 ? { references } : {}),
436
- signal,
437
- }, getProxyFetch(cfg.proxyUrl) ?? globalThis.fetch, cfg.timeoutMs, signal), signal);
448
+ const runOpenAiPiece = (rawPieceText) => {
449
+ const spokenText = hasPronunciations ? applyPronunciationsPlain(rawPieceText, pronunciations) : rawPieceText;
450
+ return raceAbort(openaiFn(cfg.ttsBaseUrl, cfg.ttsApiKey, {
451
+ text: spokenText,
452
+ voice,
453
+ model: cfg.ttsModel !== '' ? cfg.ttsModel : FALLBACK_TTS_MODEL,
454
+ speed: rateToSpeed(rate),
455
+ ...(instructions !== undefined ? { instructions } : {}),
456
+ ...(references !== undefined && references.length > 0 ? { references } : {}),
457
+ signal,
458
+ }, getProxyFetch(cfg.proxyUrl) ?? globalThis.fetch, cfg.timeoutMs, signal), signal);
459
+ };
460
+ const pieces = splitInlinePauses(chunkText);
461
+ let audio;
462
+ if (pieces.some((p) => p.kind === 'pause')) {
463
+ const textPieces = pieces.filter((p) => p.kind === 'text' && p.text.trim() !== '');
464
+ if (textPieces.length === 0) {
465
+ const totalPauseMs = pieces.reduce((sum, p) => sum + (p.kind === 'pause' ? p.durationMs : 0), 0);
466
+ audio = createSilenceMp3(Math.max(50, totalPauseMs), 24000);
467
+ }
468
+ else {
469
+ const synthesizedByIndex = new Map();
470
+ let detectedSampleRate = 24000;
471
+ for (let i = 0; i < pieces.length; i++) {
472
+ const piece = pieces[i];
473
+ if (piece.kind !== 'text' || piece.text.trim() === '')
474
+ continue;
475
+ throwIfAborted(signal);
476
+ const partAudio = await runOpenAiPiece(piece.text);
477
+ throwIfAborted(signal);
478
+ const info = inspectMp3Buffer(partAudio);
479
+ if (info.frameCount > 0)
480
+ detectedSampleRate = info.sampleRate;
481
+ synthesizedByIndex.set(i, partAudio);
482
+ }
483
+ const partBuffers = [];
484
+ for (let i = 0; i < pieces.length; i++) {
485
+ const piece = pieces[i];
486
+ if (piece.kind === 'pause') {
487
+ const sil = createSilenceMp3(piece.durationMs, detectedSampleRate);
488
+ if (sil.length > 0)
489
+ partBuffers.push(sil);
490
+ }
491
+ else {
492
+ const partAudio = synthesizedByIndex.get(i);
493
+ if (partAudio !== undefined && partAudio.length > 0)
494
+ partBuffers.push(partAudio);
495
+ }
496
+ }
497
+ audio = partBuffers.length === 1 ? partBuffers[0] : concatMp3Buffers(partBuffers);
498
+ }
499
+ }
500
+ else {
501
+ audio = await runOpenAiPiece(chunkText);
502
+ }
438
503
  return {
439
504
  audio,
440
505
  cues: needSubtitles
441
- ? [{ index: 1, startMs: 0, endMs: Math.max(1, inspectMp3Buffer(audio).durationMs), text: chunkText }]
506
+ ? [{ index: 1, startMs: 0, endMs: Math.max(1, inspectMp3Buffer(audio).durationMs), text: stripInlinePauses(chunkText) }]
442
507
  : [],
443
508
  };
444
509
  };
@@ -547,18 +612,19 @@ export function buildVoiceTools(config, deps = {}) {
547
612
  const voiceTts = {
548
613
  name: 'voice_tts',
549
614
  isConcurrencySafe: () => !getCfg().overwrite,
550
- description: '文字转语音:默认调用 edge-tts(微软 Edge 朗读服务,免费)合成 MP3,也可在设置中切换 OpenAI/SiliconFlow/自定义 TTS 引擎。支持内置角色人设 persona、自然语言情绪指令 instructions、5–10 秒参考音频零样本克隆 referenceAudio、自定义发音词典 pronunciations、多角色播客 segments、读取工作区 inputFile(自动清理 Markdown)、按 .srt/.vtt 字幕时间轴反向卡点配音 subtitleInput、背景音乐自动闪避混音 bgmFile、长文自动分段(≤100000 字符)、导出 .srt/.vtt 字幕与 -16 LUFS 响度均衡 normalizeLoudness,并返回精确时长 durationMs/durationSec;输出默认 voice_output.mp3。',
615
+ description: '文字转语音:默认调用 edge-tts(微软 Edge 朗读服务,免费)合成 MP3,也可在设置中切换 OpenAI/SiliconFlow/自定义 TTS 引擎。支持内置角色人设 persona、自然语言情绪指令 instructions、5–10 秒参考音频零样本克隆 referenceAudio、自定义发音词典 pronunciations、行内停顿标签 [pause:500ms]、多角色播客 segments(支持章节书签 title)、读取工作区 inputFile(Markdown 多标题自动生成 MP3 CHAP 章节书签与 chaptersOutput 时间轴)、按 .srt/.vtt 字幕时间轴反向卡点配音 subtitleInput、背景音乐自动闪避混音 bgmFile、长文自动分段(≤100000 字符)、导出 .srt/.vtt/.ass 字幕(支持 subtitleGranularity 短句/逐词与卡拉 OK 高亮)及 -16 LUFS 响度均衡 normalizeLoudness,并返回精确时长 durationMs/durationSec;输出默认 voice_output.mp3。',
551
616
  parameters: compileParameters({
552
- text: { type: 'string', description: '要合成的文本(若提供 inputFile、segments 或 subtitleInput 则可选;支持长文本自动分段,≤100000 字符)。' },
553
- inputFile: { type: 'string', description: '工作区内文稿文件路径(可选,如 article.md;指定时直接从文件读取)。' },
554
- subtitleInput: { type: 'string', description: '按现有 .srt/.vtt 字幕时间轴反向卡点配音(可选,工作区内字幕路径;每条字幕超出时间窗时自动微调语速加速,句间自动补齐精确静音以对齐原视频画面)。' },
617
+ text: { type: 'string', description: '要合成的文本(若提供 inputFile、segments 或 subtitleInput 则可选;支持长文本自动分段与 [pause:500ms] 行内停顿标签,≤100000 字符)。' },
618
+ inputFile: { type: 'string', description: '工作区内文稿文件路径(可选,如 article.md;指定时直接从文件读取,多级标题 Markdown 自动写入 MP3 章节书签)。' },
619
+ subtitleInput: { type: 'string', description: '按现有 .srt/.vtt 字幕时间轴反向卡点配音(可选,工作区内字幕路径;每条字幕超出时间窗时自动微调语速加速,必要时用 ffmpeg atempo 精确贴合时间窗,句间自动补齐精确静音以对齐原视频画面)。' },
555
620
  cleanMarkdown: { type: 'boolean', description: '是否清理 Markdown 语法以便朗读(可选;inputFile 为 .md/.markdown 时默认 true)。' },
556
621
  segments: {
557
622
  type: 'array',
558
- description: '多角色对话/播客分段(可选,最多 50 段):每段可独立指定 voice/persona/text/rate/pitch/pauseMs/instructions/referenceAudio/referenceText。',
623
+ description: '多角色对话/播客分段(可选,最多 50 段):每段可独立指定 title/voice/persona/text/rate/pitch/pauseMs/instructions/referenceAudio/referenceText。',
559
624
  items: {
560
625
  type: 'object',
561
626
  properties: {
627
+ title: { type: 'string', description: '本段章节标题(可选,用于写入 MP3 章节书签与章节时间轴)。' },
562
628
  voice: { type: 'string' },
563
629
  persona: { type: 'string' },
564
630
  text: { type: 'string' },
@@ -596,19 +662,27 @@ export function buildVoiceTools(config, deps = {}) {
596
662
  bgmDucking: { type: 'boolean', default: true, description: '人声说话时是否自动压低背景音乐音量(Auto-Ducking,可选,默认 true)。' },
597
663
  normalizeLoudness: { type: 'boolean', default: false, description: '是否在合成后通过 ffmpeg 执行 -16 LUFS 播客响度均衡(可选,默认 false)。' },
598
664
  output: { type: 'string', description: '输出 MP3 路径(可选,相对路径按会话工作目录解析)。' },
599
- subtitleOutput: { type: 'string', description: '导出同步字幕文件路径(可选,.srt 或 .vtt,相对路径按会话工作目录解析)。' },
665
+ subtitleOutput: { type: 'string', description: '导出同步字幕文件路径(可选,支持 .srt、.vtt 或带卡拉 OK 高亮的 .ass,相对路径按会话工作目录解析)。' },
666
+ subtitleGranularity: { type: 'string', enum: ['sentence', 'clause', 'word'], description: '字幕粒度(可选:sentence 整句默认、clause 短视频短句切分、word 逐词时间轴;导出 .ass 字幕时自动获取词级时间轴生成卡拉 OK 高亮)。' },
667
+ maxCharsPerCue: { type: 'integer', minimum: 4, maximum: 80, description: '短句字幕每屏最大字数(可选,默认 16,配合 subtitleGranularity=clause 或 .ass 字幕生效)。' },
668
+ chaptersOutput: { type: 'string', description: '导出章节时间轴路径(可选,支持 .md/.txt 节目时间轴或 .json;Markdown 多标题文稿或带 title 的 segments 播客同时自动写入 MP3 ID3v2 CHAP 章节书签)。' },
600
669
  }),
601
670
  output: {
602
671
  schema: ttsSchema,
603
672
  render: (_args, value) => {
604
673
  const rec = asRecord(value);
605
674
  const chunksInfo = typeof rec.chunks === 'number' && rec.chunks > 1 ? ',共 ' + rec.chunks + ' 段' : '';
675
+ const chaptersList = Array.isArray(rec.chapters) ? rec.chapters : [];
676
+ const chaptersInfo = chaptersList.length > 0 ? ',含 ' + chaptersList.length + ' 个章节' : '';
606
677
  const voiceLabel = typeof rec.voice === 'string' && rec.voice !== '' ? rec.voice : '参考音频克隆';
607
678
  const durationInfo = typeof rec.durationSec === 'number' ? ',时长 ' + rec.durationSec + ' 秒' : '';
608
- const lines = ['语音合成完成:' + rec.output + '(' + rec.bytes + ' 字节,音色 ' + voiceLabel + chunksInfo + durationInfo + ')'];
679
+ const lines = ['语音合成完成:' + rec.output + '(' + rec.bytes + ' 字节,音色 ' + voiceLabel + chunksInfo + chaptersInfo + durationInfo + ')'];
609
680
  const subtitleFile = typeof rec.subtitleFile === 'string' ? rec.subtitleFile : '';
610
681
  if (subtitleFile !== '')
611
682
  lines.push('字幕文件已写入:' + subtitleFile);
683
+ const chaptersFile = typeof rec.chaptersFile === 'string' ? rec.chaptersFile : '';
684
+ if (chaptersFile !== '')
685
+ lines.push('章节文件已写入:' + chaptersFile);
612
686
  return [{ type: 'text', text: lines.join('\n') }];
613
687
  },
614
688
  },
@@ -628,8 +702,32 @@ export function buildVoiceTools(config, deps = {}) {
628
702
  const normalizeLoudness = args.normalizeLoudness === true;
629
703
  const requested = optionalString(args, 'output');
630
704
  const subtitleOutput = optionalString(args, 'subtitleOutput');
705
+ const chaptersOutput = optionalString(args, 'chaptersOutput');
631
706
  const needSubtitles = subtitleOutput !== undefined;
707
+ const rawGranularity = optionalString(args, 'subtitleGranularity');
708
+ if (rawGranularity !== undefined && rawGranularity !== 'sentence' && rawGranularity !== 'clause' && rawGranularity !== 'word') {
709
+ throw new Error('subtitleGranularity 必须是 sentence / clause / word 之一(当前:' + rawGranularity + ')。');
710
+ }
711
+ const subtitleGranularity = rawGranularity ?? 'sentence';
712
+ const maxCharsPerCue = typeof args.maxCharsPerCue === 'number' && Number.isFinite(args.maxCharsPerCue)
713
+ ? Math.min(80, Math.max(4, Math.round(args.maxCharsPerCue)))
714
+ : 16;
715
+ const subtitleOpts = needSubtitles
716
+ ? {
717
+ granularity: subtitleGranularity,
718
+ maxCharsPerCue,
719
+ isAss: subtitleOutput !== undefined && subtitleFormatOf(subtitleOutput) === 'ass',
720
+ }
721
+ : undefined;
632
722
  const mergedPronunciations = mergePronunciations(cfg.pronunciations, parseCallPronunciations(args.pronunciations));
723
+ const writeChaptersOutput = (chapters) => {
724
+ if (chaptersOutput === undefined || chapters.length === 0)
725
+ return '';
726
+ const content = extname(chaptersOutput).toLowerCase() === '.json'
727
+ ? JSON.stringify(chapters, null, 2) + '\n'
728
+ : formatChaptersMarkdown(chapters);
729
+ return writeWorkspaceFile(resolveOutputPath(chaptersOutput, '', cfg.overwrite, cwd), content, cwd, undefined, cfg.overwrite);
730
+ };
633
731
  const bgmFile = optionalString(args, 'bgmFile');
634
732
  let bgmPath;
635
733
  let bgmVolume = 0.18;
@@ -669,6 +767,7 @@ export function buildVoiceTools(config, deps = {}) {
669
767
  if (segText === '')
670
768
  throw new Error('第 ' + (i + 1) + ' 段清理 Markdown 后文本为空。');
671
769
  totalChars += segText.length;
770
+ const segTitle = optionalString(seg, 'title');
672
771
  const segVoice = optionalString(seg, 'voice');
673
772
  const segPersona = optionalString(seg, 'persona');
674
773
  const segRate = optionalString(seg, 'rate');
@@ -726,6 +825,7 @@ export function buildVoiceTools(config, deps = {}) {
726
825
  assertProsody(segSpec.rate, segSpec.pitch);
727
826
  const chunks = segText.length > 4000 ? splitTextIntoChunks(segText, 2800) : [segText];
728
827
  parsedSegments.push({
828
+ ...(segTitle !== undefined ? { title: segTitle } : {}),
729
829
  chunks,
730
830
  voice: segSpec.voice,
731
831
  rate: segSpec.rate,
@@ -742,6 +842,8 @@ export function buildVoiceTools(config, deps = {}) {
742
842
  resolveOutputPath(requested, 'voice_output.mp3', cfg.overwrite, cwd);
743
843
  if (subtitleOutput !== undefined)
744
844
  resolveOutputPath(subtitleOutput, '', cfg.overwrite, cwd);
845
+ if (chaptersOutput !== undefined)
846
+ resolveOutputPath(chaptersOutput, '', cfg.overwrite, cwd);
745
847
  const tasks = [];
746
848
  for (let s = 0; s < parsedSegments.length; s++) {
747
849
  const seg = parsedSegments[s];
@@ -758,16 +860,22 @@ export function buildVoiceTools(config, deps = {}) {
758
860
  });
759
861
  }
760
862
  }
761
- const chunkResults = await mapLimit(tasks, TTS_CHUNK_CONCURRENCY, signal, (task) => synthesizeChunk(cfg, task.useEdge, task.text, task.voice, task.rate, task.pitch, needSubtitles, signal, task.instructions, task.references, mergedPronunciations));
863
+ const chunkResults = await mapLimit(tasks, TTS_CHUNK_CONCURRENCY, signal, (task) => synthesizeChunk(cfg, task.useEdge, task.text, task.voice, task.rate, task.pitch, needSubtitles, signal, task.instructions, task.references, mergedPronunciations, subtitleOpts));
762
864
  throwIfAborted(signal);
763
865
  const buffers = [];
764
866
  const allCues = [];
867
+ const allWordCues = [];
868
+ const chapters = [];
869
+ const shouldRecordChapters = parsedSegments.some((s) => s.title !== undefined) || chaptersOutput !== undefined;
765
870
  let offsetMs = 0;
871
+ let audioOffsetMs = 0;
766
872
  let cueIndex = 1;
873
+ let wordCueIndex = 1;
767
874
  let resultIdx = 0;
768
875
  let detectedSampleRate = 24000;
769
876
  for (let s = 0; s < parsedSegments.length; s++) {
770
877
  const seg = parsedSegments[s];
878
+ const segStartMs = audioOffsetMs;
771
879
  for (let c = 0; c < seg.chunks.length; c++) {
772
880
  const res = chunkResults[resultIdx++];
773
881
  buffers.push(res.audio);
@@ -775,34 +883,67 @@ export function buildVoiceTools(config, deps = {}) {
775
883
  if (chunkInfo.frameCount > 0)
776
884
  detectedSampleRate = chunkInfo.sampleRate;
777
885
  const chunkDurationMs = chunkInfo.durationMs;
886
+ audioOffsetMs += chunkDurationMs;
778
887
  if (needSubtitles) {
779
888
  const shifted = offsetCues(res.cues, offsetMs, cueIndex);
780
889
  allCues.push(...shifted);
781
890
  cueIndex += shifted.length;
891
+ if (res.wordCues !== undefined && res.wordCues.length > 0) {
892
+ const shiftedWords = offsetCues(res.wordCues, offsetMs, wordCueIndex);
893
+ allWordCues.push(...shiftedWords);
894
+ wordCueIndex += shiftedWords.length;
895
+ }
782
896
  const cueSpan = shifted.length > 0 ? Math.max(0, shifted[shifted.length - 1].endMs - offsetMs) : 0;
783
897
  offsetMs += Math.max(chunkDurationMs, cueSpan);
784
898
  }
785
899
  }
900
+ const segEndMs = Math.max(segStartMs, audioOffsetMs);
901
+ if (shouldRecordChapters) {
902
+ chapters.push({
903
+ title: seg.title ?? ('第 ' + (s + 1) + ' 段'),
904
+ startMs: segStartMs,
905
+ endMs: segEndMs,
906
+ });
907
+ }
786
908
  if (s < parsedSegments.length - 1 && seg.pauseMs > 0) {
787
909
  const silence = createSilenceMp3(seg.pauseMs, detectedSampleRate);
788
910
  if (silence.length > 0) {
789
911
  buffers.push(silence);
790
- offsetMs += inspectMp3Buffer(silence).durationMs;
912
+ const silenceMs = inspectMp3Buffer(silence).durationMs;
913
+ audioOffsetMs += silenceMs;
914
+ offsetMs += silenceMs;
791
915
  }
792
916
  }
793
917
  }
794
918
  let audio = buffers.length === 1 ? buffers[0] : concatMp3Buffers(buffers);
795
919
  audio = await finalizeVoiceAudio(audio, bgmPath, bgmVolume, bgmDucking, normalizeLoudness, signal);
920
+ if (chapters.length > 0) {
921
+ audio = embedMp3Chapters(audio, chapters);
922
+ }
796
923
  const finalInfo = inspectMp3Buffer(audio);
797
924
  const durationMs = finalInfo.durationMs;
798
925
  const durationSec = Math.round(durationMs / 100) / 10;
799
926
  const output = writeWorkspaceFile(resolveOutputPath(requested, 'voice_output.mp3', cfg.overwrite, cwd), audio, cwd, undefined, cfg.overwrite);
800
927
  let subtitleFile = '';
801
928
  if (subtitleOutput !== undefined) {
802
- const formatted = formatSubtitles(allCues, subtitleFormatOf(subtitleOutput));
929
+ const formatted = formatSubtitles(allCues, subtitleFormatOf(subtitleOutput), allWordCues);
803
930
  subtitleFile = writeWorkspaceFile(resolveOutputPath(subtitleOutput, '', cfg.overwrite, cwd), formatted, cwd, undefined, cfg.overwrite);
804
931
  }
805
- return { output, bytes: audio.length, durationMs, durationSec, voice: topSpec.voice, rate: topSpec.rate, pitch: topSpec.pitch, textLength: totalChars, chunks: tasks.length, subtitleFile };
932
+ const chaptersFile = writeChaptersOutput(chapters);
933
+ return {
934
+ output,
935
+ bytes: audio.length,
936
+ durationMs,
937
+ durationSec,
938
+ voice: topSpec.voice,
939
+ rate: topSpec.rate,
940
+ pitch: topSpec.pitch,
941
+ textLength: totalChars,
942
+ chunks: tasks.length,
943
+ subtitleFile,
944
+ ...(chapters.length > 0 ? { chapters } : {}),
945
+ ...(chaptersFile !== '' ? { chaptersFile } : {}),
946
+ };
806
947
  }
807
948
  const subtitleInput = optionalString(args, 'subtitleInput');
808
949
  if (subtitleInput !== undefined) {
@@ -831,6 +972,8 @@ export function buildVoiceTools(config, deps = {}) {
831
972
  resolveOutputPath(requested, 'voice_output.mp3', cfg.overwrite, cwd);
832
973
  if (subtitleOutput !== undefined)
833
974
  resolveOutputPath(subtitleOutput, '', cfg.overwrite, cwd);
975
+ if (chaptersOutput !== undefined)
976
+ resolveOutputPath(chaptersOutput, '', cfg.overwrite, cwd);
834
977
  const cueResults = await mapLimit(cues, TTS_CHUNK_CONCURRENCY, signal, async (cue) => {
835
978
  const cueText = args.cleanMarkdown === true
836
979
  ? (cleanMarkdownForSpeech(cue.text).trim() || cue.text.trim())
@@ -844,6 +987,13 @@ export function buildVoiceTools(config, deps = {}) {
844
987
  res = await synthesizeChunk(cfg, useEdge, cueText, spec.voice, fastRate, spec.pitch, false, signal, spec.instructions, spec.references, mergedPronunciations);
845
988
  }
846
989
  }
990
+ const postInfo = inspectMp3Buffer(res.audio);
991
+ const postMs = postInfo.durationMs;
992
+ if (postMs > windowMs * 1.03 && (deps.fitWindow !== undefined || (postInfo.frameCount > 0 && hasFfmpeg()))) {
993
+ const fitted = await raceAbort((deps.fitWindow ?? fitAudioToWindow)(res.audio, windowMs, signal), signal);
994
+ throwIfAborted(signal);
995
+ res = { ...res, audio: fitted };
996
+ }
847
997
  return { audio: res.audio, textLength: cueText.length };
848
998
  });
849
999
  throwIfAborted(signal);
@@ -873,7 +1023,7 @@ export function buildVoiceTools(config, deps = {}) {
873
1023
  index: i + 1,
874
1024
  startMs: cueStartMs,
875
1025
  endMs: Math.max(cueStartMs + 1, cursorMs),
876
- text: cue.text,
1026
+ text: stripInlinePauses(cue.text),
877
1027
  });
878
1028
  }
879
1029
  let audio = buffers.length === 1 ? buffers[0] : concatMp3Buffers(buffers);
@@ -890,21 +1040,24 @@ export function buildVoiceTools(config, deps = {}) {
890
1040
  return { output, bytes: audio.length, durationMs, durationSec, voice: spec.voice, rate: spec.rate, pitch: spec.pitch, textLength: totalChars, chunks: cues.length, subtitleFile };
891
1041
  }
892
1042
  const inputFile = optionalString(args, 'inputFile');
1043
+ let rawSourceMarkdown = '';
1044
+ let shouldClean = false;
893
1045
  let finalText;
894
1046
  if (inputFile !== undefined) {
895
1047
  const resolvedInput = assertWorkspaceFile(inputFile, cwd, '文稿文件');
896
- const rawText = readFileSync(resolvedInput, 'utf8');
897
- const shouldClean = typeof args.cleanMarkdown === 'boolean'
1048
+ rawSourceMarkdown = readFileSync(resolvedInput, 'utf8');
1049
+ shouldClean = typeof args.cleanMarkdown === 'boolean'
898
1050
  ? args.cleanMarkdown
899
1051
  : /\.(md|markdown)$/i.test(resolvedInput);
900
- finalText = (shouldClean ? cleanMarkdownForSpeech(rawText) : rawText).trim();
1052
+ finalText = (shouldClean ? cleanMarkdownForSpeech(rawSourceMarkdown) : rawSourceMarkdown).trim();
901
1053
  if (finalText === '') {
902
1054
  throw new Error(shouldClean ? '文稿文件在清理 Markdown 后内容为空。' : '文稿文件内容为空。');
903
1055
  }
904
1056
  }
905
1057
  else {
906
- const rawText = requiredString(args, 'text', '要合成的文本');
907
- finalText = args.cleanMarkdown === true ? cleanMarkdownForSpeech(rawText).trim() : rawText;
1058
+ rawSourceMarkdown = requiredString(args, 'text', '要合成的文本');
1059
+ shouldClean = typeof args.cleanMarkdown === 'boolean' ? args.cleanMarkdown : true;
1060
+ finalText = args.cleanMarkdown === true ? cleanMarkdownForSpeech(rawSourceMarkdown).trim() : rawSourceMarkdown;
908
1061
  if (finalText === '')
909
1062
  throw new Error('要合成的文本在清理 Markdown 后为空。');
910
1063
  }
@@ -927,6 +1080,108 @@ export function buildVoiceTools(config, deps = {}) {
927
1080
  resolveOutputPath(requested, 'voice_output.mp3', cfg.overwrite, cwd);
928
1081
  if (subtitleOutput !== undefined)
929
1082
  resolveOutputPath(subtitleOutput, '', cfg.overwrite, cwd);
1083
+ if (chaptersOutput !== undefined)
1084
+ resolveOutputPath(chaptersOutput, '', cfg.overwrite, cwd);
1085
+ const checkMarkdownChapters = (inputFile !== undefined && /\.(md|markdown)$/i.test(inputFile)) || chaptersOutput !== undefined;
1086
+ const mdSections = checkMarkdownChapters ? extractMarkdownSections(rawSourceMarkdown, shouldClean) : [];
1087
+ if (mdSections.length >= 2 || (mdSections.length >= 1 && chaptersOutput !== undefined)) {
1088
+ const activeSections = mdSections
1089
+ .map((sec) => ({ title: sec.title, text: sec.text, chunks: splitTextIntoChunks(sec.text, 2800) }))
1090
+ .filter((sec) => sec.chunks.length > 0);
1091
+ if (activeSections.length > 0) {
1092
+ const secTasks = [];
1093
+ for (let s = 0; s < activeSections.length; s++) {
1094
+ for (const chunkText of activeSections[s].chunks) {
1095
+ secTasks.push({ secIndex: s, text: chunkText });
1096
+ }
1097
+ }
1098
+ const secResults = await mapLimit(secTasks, TTS_CHUNK_CONCURRENCY, signal, (task) => synthesizeChunk(cfg, useEdge, task.text, spec.voice, spec.rate, spec.pitch, needSubtitles, signal, spec.instructions, spec.references, mergedPronunciations, subtitleOpts));
1099
+ throwIfAborted(signal);
1100
+ const sectionBuffers = [];
1101
+ const chapters = [];
1102
+ const allCues = [];
1103
+ const allWordCues = [];
1104
+ let audioOffsetMs = 0;
1105
+ let cueOffsetMs = 0;
1106
+ let cueIndex = 1;
1107
+ let wordCueIndex = 1;
1108
+ let taskCursor = 0;
1109
+ let detectedSampleRate = 24000;
1110
+ for (let s = 0; s < activeSections.length; s++) {
1111
+ const sec = activeSections[s];
1112
+ const chunkBuffers = [];
1113
+ const secStartMs = audioOffsetMs;
1114
+ for (let c = 0; c < sec.chunks.length; c++) {
1115
+ const res = secResults[taskCursor++];
1116
+ chunkBuffers.push(res.audio);
1117
+ const chunkInfo = inspectMp3Buffer(res.audio);
1118
+ if (chunkInfo.frameCount > 0)
1119
+ detectedSampleRate = chunkInfo.sampleRate;
1120
+ if (needSubtitles) {
1121
+ const shifted = offsetCues(res.cues, cueOffsetMs, cueIndex);
1122
+ allCues.push(...shifted);
1123
+ cueIndex += shifted.length;
1124
+ if (res.wordCues !== undefined && res.wordCues.length > 0) {
1125
+ const shiftedWords = offsetCues(res.wordCues, cueOffsetMs, wordCueIndex);
1126
+ allWordCues.push(...shiftedWords);
1127
+ wordCueIndex += shiftedWords.length;
1128
+ }
1129
+ const cueSpan = shifted.length > 0 ? Math.max(0, shifted[shifted.length - 1].endMs - cueOffsetMs) : 0;
1130
+ cueOffsetMs += Math.max(chunkInfo.durationMs, cueSpan);
1131
+ }
1132
+ }
1133
+ const secAudio = chunkBuffers.length === 1 ? chunkBuffers[0] : concatMp3Buffers(chunkBuffers);
1134
+ sectionBuffers.push(secAudio);
1135
+ const secDurationMs = inspectMp3Buffer(secAudio).durationMs;
1136
+ const secEndMs = secStartMs + secDurationMs;
1137
+ chapters.push({
1138
+ title: sec.title,
1139
+ startMs: secStartMs,
1140
+ endMs: secEndMs,
1141
+ });
1142
+ audioOffsetMs = secEndMs;
1143
+ if (needSubtitles)
1144
+ cueOffsetMs = Math.max(cueOffsetMs, audioOffsetMs);
1145
+ if (s < activeSections.length - 1) {
1146
+ const gap = createSilenceMp3(350, detectedSampleRate);
1147
+ if (gap.length > 0) {
1148
+ sectionBuffers.push(gap);
1149
+ const gapMs = inspectMp3Buffer(gap).durationMs;
1150
+ audioOffsetMs += gapMs;
1151
+ cueOffsetMs += gapMs;
1152
+ }
1153
+ }
1154
+ }
1155
+ let audio = sectionBuffers.length === 1 ? sectionBuffers[0] : concatMp3Buffers(sectionBuffers);
1156
+ throwIfAborted(signal);
1157
+ audio = await finalizeVoiceAudio(audio, bgmPath, bgmVolume, bgmDucking, normalizeLoudness, signal);
1158
+ audio = embedMp3Chapters(audio, chapters, { title: activeSections[0].title });
1159
+ const finalInfo = inspectMp3Buffer(audio);
1160
+ const durationMs = finalInfo.durationMs;
1161
+ const durationSec = Math.round(durationMs / 100) / 10;
1162
+ const output = writeWorkspaceFile(resolveOutputPath(requested, 'voice_output.mp3', cfg.overwrite, cwd), audio, cwd, undefined, cfg.overwrite);
1163
+ let subtitleFile = '';
1164
+ if (subtitleOutput !== undefined) {
1165
+ const formatted = formatSubtitles(allCues, subtitleFormatOf(subtitleOutput), allWordCues);
1166
+ subtitleFile = writeWorkspaceFile(resolveOutputPath(subtitleOutput, '', cfg.overwrite, cwd), formatted, cwd, undefined, cfg.overwrite);
1167
+ }
1168
+ const chaptersFile = writeChaptersOutput(chapters);
1169
+ return {
1170
+ output,
1171
+ bytes: audio.length,
1172
+ durationMs,
1173
+ durationSec,
1174
+ voice: spec.voice,
1175
+ rate: spec.rate,
1176
+ pitch: spec.pitch,
1177
+ textLength: finalText.length,
1178
+ chunks: secTasks.length,
1179
+ subtitleFile,
1180
+ chapters,
1181
+ ...(chaptersFile !== '' ? { chaptersFile } : {}),
1182
+ };
1183
+ }
1184
+ }
930
1185
  let audio;
931
1186
  let chunkCount = 1;
932
1187
  let subtitleFile = '';
@@ -937,22 +1192,29 @@ export function buildVoiceTools(config, deps = {}) {
937
1192
  else {
938
1193
  const chunks = splitTextIntoChunks(finalText, 2800);
939
1194
  chunkCount = chunks.length;
940
- const results = await mapLimit(chunks, TTS_CHUNK_CONCURRENCY, signal, (chunk) => synthesizeChunk(cfg, useEdge, chunk, spec.voice, spec.rate, spec.pitch, needSubtitles, signal, spec.instructions, spec.references, mergedPronunciations));
1195
+ const results = await mapLimit(chunks, TTS_CHUNK_CONCURRENCY, signal, (chunk) => synthesizeChunk(cfg, useEdge, chunk, spec.voice, spec.rate, spec.pitch, needSubtitles, signal, spec.instructions, spec.references, mergedPronunciations, subtitleOpts));
941
1196
  throwIfAborted(signal);
942
1197
  audio = results.length === 1 ? results[0].audio : concatMp3Buffers(results.map((r) => r.audio));
943
1198
  if (subtitleOutput !== undefined) {
944
1199
  const allCues = [];
1200
+ const allWordCues = [];
945
1201
  let offsetMs = 0;
946
1202
  let cueIndex = 1;
1203
+ let wordCueIndex = 1;
947
1204
  for (const res of results) {
948
1205
  const shifted = offsetCues(res.cues, offsetMs, cueIndex);
949
1206
  allCues.push(...shifted);
950
1207
  cueIndex += shifted.length;
1208
+ if (res.wordCues !== undefined && res.wordCues.length > 0) {
1209
+ const shiftedWords = offsetCues(res.wordCues, offsetMs, wordCueIndex);
1210
+ allWordCues.push(...shiftedWords);
1211
+ wordCueIndex += shiftedWords.length;
1212
+ }
951
1213
  const chunkDurationMs = inspectMp3Buffer(res.audio).durationMs;
952
1214
  const cueSpan = shifted.length > 0 ? Math.max(0, shifted[shifted.length - 1].endMs - offsetMs) : 0;
953
1215
  offsetMs += Math.max(chunkDurationMs, cueSpan);
954
1216
  }
955
- const formatted = formatSubtitles(allCues, subtitleFormatOf(subtitleOutput));
1217
+ const formatted = formatSubtitles(allCues, subtitleFormatOf(subtitleOutput), allWordCues);
956
1218
  subtitleFile = writeWorkspaceFile(resolveOutputPath(subtitleOutput, '', cfg.overwrite, cwd), formatted, cwd, undefined, cfg.overwrite);
957
1219
  }
958
1220
  }
@@ -969,15 +1231,15 @@ export function buildVoiceTools(config, deps = {}) {
969
1231
  const voiceStt = {
970
1232
  name: 'voice_stt',
971
1233
  isConcurrencySafe: () => !getCfg().overwrite,
972
- description: '语音转文字:调用 OpenAI 兼容 ASR 接口(默认 Groq whisper-large-v3-turbo;可切 openai/siliconflow/custom)。audio 为音频文件路径(mp3/wav/m4a/ogg/flac,≤25MB;超过 25MB 可设 autoCompress=true 用 ffmpeg 自动压缩分段);支持多人说话人区分 diarize=true(双声道按声道分离或推断说话人切换);language/prompt 可选;output 可写 .txt,subtitleOutput 可写 .srt/.vtt 字幕。',
1234
+ description: '语音转文字:调用 OpenAI 兼容 ASR 接口(默认 Groq whisper-large-v3-turbo;可切 openai/siliconflow/custom)。audio 支持音频文件(mp3/wav/m4a/ogg/flac/webm,≤25MB;超过 25MB 可设 autoCompress=true 用 ffmpeg 自动压缩分段)及视频容器(mp4/mkv/mov/avi,在有 ffmpeg 时自动剥离视频轨提取轻量单声道音频);支持多人说话人区分 diarize=true(双声道按声道分离或推断说话人切换);language/prompt 可选;output 可写 .txt,subtitleOutput 可写 .srt/.vtt/.ass 字幕。',
973
1235
  parameters: compileParameters({
974
- audio: { type: 'string', required: true, description: '音频文件路径(必填)。' },
975
- engine: { type: 'string', enum: ['groq', 'openai', 'custom'], description: '引擎(可选):只接受与配置 asrEngine 相同的值,用于显式确认;接口地址与密钥始终取配置。' },
1236
+ audio: { type: 'string', required: true, description: '音频或视频文件路径(必填,支持 mp3/wav/m4a/ogg/flac/webm 及 mp4/mkv/mov/avi 视频自动提取音轨)。' },
1237
+ engine: { type: 'string', enum: ['groq', 'openai', 'siliconflow', 'custom'], description: '引擎(可选):只接受与配置 asrEngine 相同的值,用于显式确认;接口地址与密钥始终取配置。' },
976
1238
  model: { type: 'string', description: '模型名(可选,覆盖配置)。' },
977
1239
  language: { type: 'string', description: '语言提示,如 zh(可选)。' },
978
1240
  prompt: { type: 'string', description: '提示词(专有名词/术语纠偏,可选)。' },
979
1241
  output: { type: 'string', description: '把转写文本写成 .txt 的路径(可选,相对路径按会话工作目录解析)。' },
980
- subtitleOutput: { type: 'string', description: '导出字幕路径(可选,.srt 或 .vtt,相对路径按会话工作目录解析;指定时自动开启时间戳)。' },
1242
+ subtitleOutput: { type: 'string', description: '导出字幕路径(可选,.srt、.vtt 或 .ass,相对路径按会话工作目录解析;指定时自动开启时间戳)。' },
981
1243
  timestamps: { type: 'boolean', description: '是否请求分段时间戳(可选,默认 false)。' },
982
1244
  diarize: { type: 'boolean', description: '是否启用多人说话人区分(可选,默认 false;双声道录音在有 ffmpeg 时可按声道分离,或解析/推断说话人切换并输出 [说话人 A] 标记)。' },
983
1245
  autoCompress: { type: 'boolean', description: '超过 25MB 时是否自动调用本地 ffmpeg 压缩与分段转写(可选,默认 false)。' },
@@ -1006,7 +1268,7 @@ export function buildVoiceTools(config, deps = {}) {
1006
1268
  const audioPath = assertAudioFile(requiredString(args, 'audio', '音频文件'), cwd);
1007
1269
  const requestedEngine = optionalString(args, 'engine');
1008
1270
  if (requestedEngine !== undefined && !['groq', 'openai', 'siliconflow', 'custom'].includes(requestedEngine)) {
1009
- throw new Error('engine 必须是 groq / openai / custom 之一(当前:' + requestedEngine + ')。');
1271
+ throw new Error('engine 必须是 groq / openai / siliconflow / custom 之一(当前:' + requestedEngine + ')。');
1010
1272
  }
1011
1273
  if (requestedEngine !== undefined && requestedEngine !== cfg.asrEngine) {
1012
1274
  throw new Error('voice_stt 不支持用 engine 参数临时换引擎(配置 asrEngine=' + cfg.asrEngine + ',本次传入 ' + requestedEngine + ')。接口地址与密钥是一组配置,混着用会把密钥发到非预期的服务;如需切换请在 DSH「设置 → 插件 → 语音」修改 asrEngine / asrBaseUrl(保存即生效)。');
@@ -1042,15 +1304,20 @@ export function buildVoiceTools(config, deps = {}) {
1042
1304
  return uniq.size > 0 ? Array.from(uniq) : undefined;
1043
1305
  };
1044
1306
  const fileSize = statSync(audioPath).size;
1307
+ const isVideo = isVideoContainerFile(audioPath);
1045
1308
  let text = '';
1046
1309
  let detectedLanguage = '';
1047
1310
  let duration;
1048
1311
  let segments;
1049
1312
  let speakers;
1050
- if (fileSize > MAX_ASR_BYTES) {
1051
- if (args.autoCompress !== true)
1313
+ if (fileSize > MAX_ASR_BYTES || (isVideo && (deps.prepareAudio !== undefined || (deps.stt === undefined && hasFfmpeg())))) {
1314
+ if (fileSize > MAX_ASR_BYTES && !isVideo && args.autoCompress !== true) {
1052
1315
  assertAudioSize(fileSize);
1316
+ }
1053
1317
  if (deps.prepareAudio === undefined && !hasFfmpeg()) {
1318
+ if (isVideo) {
1319
+ throw new Error('视频文件超过 25MB 且未检测到本机 ffmpeg;请先安装 ffmpeg(或使用 dsh-ffmpeg 提取音轨)以便自动剥离视频流后转写。');
1320
+ }
1054
1321
  throw new Error('音频超过 25MB,且未检测到 ffmpeg;请先安装 ffmpeg(或使用 dsh-ffmpeg 压缩),再启用 autoCompress=true。');
1055
1322
  }
1056
1323
  const prep = await raceAbort((deps.prepareAudio ?? prepareAudioForAsr)(audioPath, signal), signal);
@@ -1233,7 +1500,21 @@ export function buildVoiceTools(config, deps = {}) {
1233
1500
  const cueSegments = segments !== undefined && segments.length > 0
1234
1501
  ? segments
1235
1502
  : [{ id: 1, start: 0, end: duration !== undefined && duration > 0 ? duration : 5, text }];
1236
- const subText = formatSttSubtitles(cueSegments, subtitleFormatOf(subtitleOutput));
1503
+ const sttSubFmt = subtitleFormatOf(subtitleOutput);
1504
+ const subText = sttSubFmt === 'ass'
1505
+ ? formatSubtitles(cueSegments
1506
+ .filter((seg) => seg.text.trim() !== '')
1507
+ .map((seg, idx) => {
1508
+ const clean = seg.text.trim();
1509
+ const speaker = typeof seg.speaker === 'string' ? seg.speaker.trim() : '';
1510
+ return {
1511
+ index: idx + 1,
1512
+ startMs: Math.max(0, Math.round(seg.start * 1000)),
1513
+ endMs: Math.max(1, Math.round(seg.end * 1000)),
1514
+ text: speaker !== '' ? `[${speaker}] ${clean}` : clean,
1515
+ };
1516
+ }), 'ass')
1517
+ : formatSttSubtitles(cueSegments, sttSubFmt);
1237
1518
  subtitleFile = writeWorkspaceFile(resolveOutputPath(subtitleOutput, '', cfg.overwrite, cwd), subText, cwd, 0o600, cfg.overwrite);
1238
1519
  }
1239
1520
  // The output schema declares string fields — never return null, or dsh rejects the whole tool result.