dsh-vidi 0.9.5 → 0.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/index.mjs CHANGED
@@ -360,6 +360,286 @@ var Glossary = class {
360
360
  }
361
361
  }
362
362
  };
363
+ // src/align.ts
364
+ /**
365
+ * Does each translation sit on its own line? A model translating several lines at once sometimes
366
+ * joins two and writes every later translation one line off. Numbers and Latin-script words
367
+ * (names, products) survive translation as they are, so they tell which line a translation
368
+ * belongs to.
369
+ */
370
+ const CJK_NUM = /[〇零一二两三四五六七八九十百千万亿萬億]{2}/;
371
+ const LATIN_STOP = /* @__PURE__ */ new Set([
372
+ "the",
373
+ "and",
374
+ "for",
375
+ "that",
376
+ "this",
377
+ "with",
378
+ "you",
379
+ "are",
380
+ "was",
381
+ "not",
382
+ "but",
383
+ "have",
384
+ "from",
385
+ "they",
386
+ "what",
387
+ "ok",
388
+ "okay"
389
+ ]);
390
+ /** The numbers in a text ("1,200" = "1200"; "5.5" stays one number). */
391
+ function numbers(text) {
392
+ return new Set((text.match(/\d+(?:[.,]\d+)*/g) ?? []).map((n) => n.replace(/,(?=\d{3}\b)/g, "").replace(/,/g, ".")));
393
+ }
394
+ /** Latin-script words of a text, lowercased (short and everyday ones left out). */
395
+ function words(text) {
396
+ return new Set((text.toLowerCase().match(/[a-z][a-z'’-]*[a-z]/g) ?? []).filter((w) => w.length > 2 && !LATIN_STOP.has(w)));
397
+ }
398
+ const NON_LATIN = /* @__PURE__ */ new Set([
399
+ "zh",
400
+ "zh-Hant",
401
+ "ja",
402
+ "ko",
403
+ "ru",
404
+ "ar",
405
+ "hi",
406
+ "th"
407
+ ]);
408
+ /** Is this translation of line `i` more likely a neighbour's? */
409
+ function shifted(lines, i, lang) {
410
+ const me = lines[i], tr = me.tr?.[lang];
411
+ if (!tr) return false;
412
+ const near = [lines[i - 1], lines[i + 1]].filter((x) => !!x);
413
+ const srcN = numbers(me.src), trN = numbers(tr);
414
+ for (const n of trN) if (!srcN.has(n) && near.some((x) => numbers(x.src).has(n))) return true;
415
+ for (const n of srcN) if (!trN.has(n) && near.some((x) => !numbers(x.src).has(n) && numbers(x.tr?.[lang] ?? "").has(n))) return true;
416
+ if (srcN.size && !trN.size && !(NON_LATIN.has(lang) && CJK_NUM.test(tr)) && [...srcN].some((n) => n.length > 1)) return true;
417
+ if (NON_LATIN.has(lang)) {
418
+ const srcW = words(me.src);
419
+ for (const w of words(tr)) if (!srcW.has(w) && near.some((x) => words(x.src).has(w))) return true;
420
+ }
421
+ if (near.some((x) => x.tr?.[lang] === tr && x.src.trim() !== me.src.trim()) && tr.length > 4) return true;
422
+ return false;
423
+ }
424
+ /**
425
+ * The lines whose translation cannot be trusted: missing, or looking like a neighbour's. A shift
426
+ * runs on over the following lines, so everything between the first and the last sign of one
427
+ * (and a line either side) is included.
428
+ */
429
+ function suspects(lines, targets) {
430
+ const bad = /* @__PURE__ */ new Set();
431
+ let first = -1, last = -1;
432
+ lines.forEach((l, i) => {
433
+ if (targets.some((t) => !l.tr?.[t]?.trim())) {
434
+ bad.add(l.id);
435
+ return;
436
+ }
437
+ if (targets.some((t) => shifted(lines, i, t))) {
438
+ if (first < 0) first = i;
439
+ last = i;
440
+ }
441
+ });
442
+ if (first >= 0) for (let i = Math.max(0, first - 1); i <= Math.min(lines.length - 1, last + 1); i++) bad.add(lines[i].id);
443
+ return bad;
444
+ }
445
+ /** A reply's ids against the lines asked for: unknown, repeated or out of order means the numbering cannot be trusted. */
446
+ function misnumbered(asked, got) {
447
+ let at = -1;
448
+ const seen = /* @__PURE__ */ new Set();
449
+ for (const id of got) {
450
+ const i = asked.indexOf(id);
451
+ if (i < 0 || seen.has(id) || i < at) return true;
452
+ seen.add(id);
453
+ at = i;
454
+ }
455
+ return false;
456
+ }
457
+ /** How alike two versions of a line are (0–1): shared words, or shared characters where there are no spaces. */
458
+ function alike(a, b) {
459
+ const units = (s) => {
460
+ const w = s.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [];
461
+ return new Set(w.length > 2 && !/[぀-鿿가-힯]/.test(s) ? w : [...s.replace(/[\s\p{P}]/gu, "")]);
462
+ };
463
+ const x = units(a), y = units(b);
464
+ if (!x.size || !y.size) return x.size === y.size ? 1 : 0;
465
+ let both = 0;
466
+ for (const u of x) if (y.has(u)) both++;
467
+ return both / Math.min(x.size, y.size);
468
+ }
469
+ // src/ask.ts
470
+ /**
471
+ * How long one request may take (ms). A request that sends nothing for `first` (before its first
472
+ * piece) or `idle` (between pieces) hangs: it is given up and asked again. One that keeps sending
473
+ * is alive (a model thinking at length), and only given up after `total`, or `perToken` for each
474
+ * token of room it was given if that is longer.
475
+ */
476
+ const LIMITS = {
477
+ first: 6e4,
478
+ idle: 4e4,
479
+ total: 18e4,
480
+ perToken: 40,
481
+ pause: 1500
482
+ };
483
+ /** The most a reply may be given, whatever the model. */
484
+ const MAX_BUDGET = 32e3;
485
+ /** Models seen thinking before they answer (their thinking counts against the reply's size). */
486
+ const thinkers = /* @__PURE__ */ new Set();
487
+ /** Models that refused a reply size: the largest they may be asked for. */
488
+ const caps = /* @__PURE__ */ new Map();
489
+ var AskError = class extends Error {
490
+ empty;
491
+ /** `empty`: the model answered, but with nothing usable (smaller pieces may do); otherwise it could not be reached. */
492
+ constructor(message, empty = false) {
493
+ super(message);
494
+ this.empty = empty;
495
+ }
496
+ };
497
+ /** Room for a reply: the answer, plus the thinking of a model that thinks first. */
498
+ function budgetFor(key, answer, grown = 1) {
499
+ const want = (thinkers.has(key) ? answer * 2 + 6e3 : answer) * grown;
500
+ return Math.round(Math.min(want, caps.get(key) ?? MAX_BUDGET, MAX_BUDGET));
501
+ }
502
+ /**
503
+ * One question to the subtitle model, answered as text. A request that hangs is cut off and asked
504
+ * again; one that fails is asked again after a pause; an answer that came back empty or cut short
505
+ * (a thinking model spent the room on its thoughts) is asked again with more room. Throws when no
506
+ * try gave an answer, or at once when `signal` says to stop.
507
+ */
508
+ async function ask(handle, o) {
509
+ const { provider, model } = handle.selection();
510
+ const key = `${provider}/${model}`;
511
+ const tries = o.tries ?? 3;
512
+ let grown = 1;
513
+ let last = "no answer", empty = false;
514
+ for (let n = 0; n < tries; n++) {
515
+ if (o.signal?.aborted) throw new AskError("stopped");
516
+ const maxTokens = budgetFor(key, o.maxTokens, grown);
517
+ let r;
518
+ try {
519
+ r = await once(handle, {
520
+ ...o,
521
+ provider,
522
+ model,
523
+ maxTokens
524
+ });
525
+ } catch (err) {
526
+ if (o.signal?.aborted) throw new AskError("stopped");
527
+ last = err.message || String(err);
528
+ empty = false;
529
+ if (/max[_ ]?(completion[_ ])?tokens|maxTokens/i.test(last) && maxTokens > 1024) caps.set(key, Math.floor(maxTokens / 2));
530
+ else if (/^4\d\d\b/.test(last) && !/^4(08|09|25|29)\b/.test(last)) throw new AskError(last);
531
+ o.log?.(`${o.purpose}: ${last}${n + 1 < tries ? ", asking again" : ""}`);
532
+ if (n + 1 < tries) await sleep(LIMITS.pause * (n + 1), o.signal);
533
+ continue;
534
+ }
535
+ if (r.thought) thinkers.add(key);
536
+ if (r.text.trim() && !r.cut) return r.text;
537
+ if (r.cut || r.thought) thinkers.add(key);
538
+ last = r.cut ? `answer cut short at ${maxTokens} tokens` : "empty answer";
539
+ empty = true;
540
+ const more = budgetFor(key, o.maxTokens, grown * 2);
541
+ o.log?.(`${o.purpose}: ${last}${r.thought ? " (the model was thinking)" : ""}${n + 1 < tries ? `, asking again with room for ${more}` : ""}`);
542
+ if (more <= maxTokens && r.text.trim()) return r.text;
543
+ grown *= 2;
544
+ }
545
+ throw new AskError(last, empty);
546
+ }
547
+ async function once(handle, o) {
548
+ const ac = new AbortController();
549
+ const outer = () => ac.abort();
550
+ o.signal?.addEventListener("abort", outer, { once: true });
551
+ let giveUp;
552
+ const gaveUp = new Promise((_, reject) => {
553
+ giveUp = reject;
554
+ });
555
+ gaveUp.catch(() => {});
556
+ ac.signal.addEventListener("abort", () => giveUp(new AskError("stopped")), { once: true });
557
+ const started = Date.now();
558
+ const total = Math.max(LIMITS.total, o.maxTokens * LIMITS.perToken);
559
+ let timer;
560
+ const arm = (ms, what) => {
561
+ clearTimeout(timer);
562
+ const left = Math.min(ms, total - (Date.now() - started));
563
+ timer = setTimeout(() => {
564
+ giveUp(new AskError(left < ms ? `no complete answer in ${Math.round(total / 1e3)} s` : what));
565
+ ac.abort();
566
+ }, Math.max(0, left));
567
+ };
568
+ const it = handle.stream({
569
+ provider: o.provider,
570
+ model: o.model,
571
+ system: o.system,
572
+ maxTokens: o.maxTokens,
573
+ purpose: o.purpose,
574
+ signal: ac.signal,
575
+ messages: [createUserMessage({
576
+ content: [...o.image ? [o.image] : [], {
577
+ type: "text",
578
+ text: o.text
579
+ }],
580
+ source: {
581
+ kind: "plugin",
582
+ plugin: "dsh-vidi"
583
+ }
584
+ })]
585
+ })[Symbol.asyncIterator]();
586
+ const asm = new BlockAssembler();
587
+ let cut = false, thought = false;
588
+ try {
589
+ arm(LIMITS.first, `no answer in ${Math.round(LIMITS.first / 1e3)} s`);
590
+ for (;;) {
591
+ const step = await Promise.race([it.next(), gaveUp]);
592
+ if (step.done) break;
593
+ const c = step.value;
594
+ arm(LIMITS.idle, `answer stalled for ${Math.round(LIMITS.idle / 1e3)} s`);
595
+ if (c.type === "reasoning-delta") thought = true;
596
+ else if (c.type === "usage") {
597
+ if ((c.usage?.reasoningTokens ?? 0) > 0) thought = true;
598
+ } else if (c.type === "finish") {
599
+ const kind = c.reason?.kind;
600
+ if (kind === "max-tokens") cut = true;
601
+ else if (kind === "error" || kind === "aborted") throw new AskError(c.reason?.failure?.message || `request ${kind}`);
602
+ continue;
603
+ }
604
+ asm.push(c);
605
+ }
606
+ } finally {
607
+ clearTimeout(timer);
608
+ o.signal?.removeEventListener("abort", outer);
609
+ Promise.resolve(it.return?.()).catch(() => {});
610
+ }
611
+ const blocks = asm.blocks();
612
+ if (blocks.some((b) => b.type === "reasoning")) thought = true;
613
+ return {
614
+ text: blocks.filter((b) => b.type === "text").map((b) => b.text ?? "").join(""),
615
+ cut,
616
+ thought
617
+ };
618
+ }
619
+ function sleep(ms, signal) {
620
+ return new Promise((resolve) => {
621
+ const t = setTimeout(done, ms);
622
+ function done() {
623
+ clearTimeout(t);
624
+ signal?.removeEventListener("abort", done);
625
+ resolve();
626
+ }
627
+ signal?.addEventListener("abort", done, { once: true });
628
+ });
629
+ }
630
+ /** The first complete JSON value of a kind in a reply (models wrap it in prose or code fences). */
631
+ function jsonIn(out, open) {
632
+ const close = open === "{" ? "}" : "]";
633
+ for (let from = out.indexOf(open); from >= 0; from = out.indexOf(open, from + 1)) {
634
+ for (let to = out.lastIndexOf(close); to > from; to = out.lastIndexOf(close, to - 1)) {
635
+ try {
636
+ return JSON.parse(out.slice(from, to + 1));
637
+ } catch {}
638
+ if (out.length - to > 2e3) break;
639
+ }
640
+ if (from > 2e3) break;
641
+ }
642
+ }
363
643
  // src/translate.ts
364
644
  /** Display names used in prompts (the model understands English names best). */
365
645
  const LANGS = {
@@ -398,54 +678,106 @@ var Translator = class {
398
678
  get available() {
399
679
  return this.llm() !== void 0;
400
680
  }
401
- async translate(targets, line, context, glossary = "", signal, reference = "") {
681
+ async translate(targets, line, context, glossary = "", signal, reference = "", log) {
402
682
  const handle = this.llm();
403
- if (!handle || !line.trim() || targets.length === 0) return void 0;
404
- const system = [
405
- `You translate live speech subtitles. Target languages: ${targets.map((t) => `"${t}" (${LANGS[t] ?? t})`).join(", ")}.`,
406
- "Translate only the final line. Be faithful, natural and concise, like professional subtitles, with normal punctuation.",
407
- "Follow the glossary when it covers a term. Otherwise keep brand, product and model names in their original form, and give well-known people, places and organisations their standard name in the target language.",
408
- "If the line is a fragment, translate the fragment as-is; do not complete it.",
409
- `Reply with one JSON object mapping each language code to its translation, e.g. {${targets.map((t) => `"${t}": "..."`).join(", ")}} — nothing else.`
410
- ].join(" ");
411
- const ctx = context.slice(-6).map((c) => `- ${c.src}${c.tr ? `\n => ${JSON.stringify(c.tr)}` : ""}`).join("\n");
683
+ if (!handle) return void 0;
684
+ return translateLine(handle, targets, line, context, glossary, reference, {
685
+ signal,
686
+ log
687
+ });
688
+ }
689
+ };
690
+ /**
691
+ * One line translated on its own (the lines before only as context). A reply that is really the
692
+ * line before it (its numbers or names, not this line's) is asked for again without the context.
693
+ */
694
+ async function translateLine(handle, targets, line, context, glossary = "", reference = "", o = {}) {
695
+ if (!line.trim() || targets.length === 0) return void 0;
696
+ const system = [
697
+ `You translate live speech subtitles. Target languages: ${targets.map((t) => `"${t}" (${LANGS[t] ?? t})`).join(", ")}.`,
698
+ "Translate only the final line. Be faithful, natural and concise, like professional subtitles, with normal punctuation.",
699
+ "Follow the glossary when it covers a term. Otherwise keep brand, product and model names in their original form, and give well-known people, places and organisations their standard name in the target language.",
700
+ "If the line is a fragment, translate the fragment as-is; do not complete it.",
701
+ `Reply with one JSON object mapping each language code to its translation, e.g. {${targets.map((t) => `"${t}": "..."`).join(", ")}} — nothing else.`
702
+ ].join(" ");
703
+ const once = async (ctxLines) => {
704
+ const ctx = ctxLines.slice(-6).map((c) => `- ${c.src}${c.tr ? `\n => ${JSON.stringify(c.tr)}` : ""}`).join("\n");
412
705
  const text = [
413
706
  glossary ? `Glossary:\n${glossary}\n` : "",
414
- ctx ? `Earlier lines (context, already translated):\n${ctx}\n` : "",
707
+ ctx ? `Earlier lines (context, already translated — do not translate these):\n${ctx}\n` : "",
415
708
  reference ? `The video's own subtitles around this moment (from the site; may be inaccurate or misaligned — use them to fix names, terms and misheard words, not as the answer):\n${reference}\n` : "",
416
709
  `Line to translate:\n${line}`
417
710
  ].join("\n");
418
- const { provider, model } = handle.selection();
419
- const assembler = new BlockAssembler();
420
- for await (const chunk of handle.stream({
421
- provider,
422
- model,
711
+ return parseTranslations((await ask(handle, {
423
712
  system,
713
+ text,
424
714
  maxTokens: 300 + 300 * targets.length,
425
715
  purpose: "live-subtitle-translate",
426
- signal,
427
- messages: [createUserMessage({
428
- content: [{
429
- type: "text",
430
- text
431
- }],
432
- source: {
433
- kind: "plugin",
434
- plugin: "dsh-vidi"
435
- }
436
- })]
437
- })) assembler.push(chunk);
438
- return parseTranslations(assembler.blocks().filter((b) => b.type === "text").map((b) => b.text ?? "").join("").trim(), targets);
716
+ signal: o.signal,
717
+ log: o.log
718
+ })).trim(), targets);
719
+ };
720
+ const tr = await once(context);
721
+ const before = context.at(-1);
722
+ if (tr && before && targets.some((t) => borrowed(tr[t] ?? "", line, before.src))) {
723
+ o.log?.("translate: the answer belongs to the line before, asking again without context");
724
+ return await once([]) ?? tr;
439
725
  }
440
- };
441
- /** Pull the JSON object out of the reply; fall back to plain text for a single target. */
726
+ return tr;
727
+ }
728
+ /** Does a translation carry a number or name that `other` has and its own line does not? */
729
+ function borrowed(tr, own, other) {
730
+ const n = numbers(own), on = numbers(other);
731
+ for (const x of numbers(tr)) if (!n.has(x) && on.has(x)) return true;
732
+ if (/[^\x00-ɏ]/.test(tr)) {
733
+ const w = words(own), ow = words(other);
734
+ for (const x of words(tr)) if (!w.has(x) && ow.has(x)) return true;
735
+ }
736
+ return false;
737
+ }
442
738
  /**
443
739
  * Translate many finished lines at once (adding a language to a video that
444
740
  * already has subtitles). Returns id → { lang: text }.
741
+ *
742
+ * Every line is checked against its id. A reply that cannot be read, or whose numbering is off, is
743
+ * asked for again in two halves (then line by line); lines left out, or whose translation looks like a neighbour's,
744
+ * are translated one at a time. A line no try could translate is simply absent from the result.
445
745
  */
446
- async function translateBatch(handle, targets, lines, glossary = "") {
746
+ async function translateBatch(handle, targets, lines, glossary = "", o = {}, depth = 0) {
747
+ const res = /* @__PURE__ */ new Map();
748
+ if (!lines.length || !targets.length) return res;
749
+ const alone = async (l) => {
750
+ const i = lines.indexOf(l);
751
+ const context = lines.slice(Math.max(0, i - 3), i).map((c) => ({
752
+ src: c.src,
753
+ tr: res.get(c.id)
754
+ }));
755
+ try {
756
+ const tr = await translateLine(handle, targets, l.src, context, glossary, "", o);
757
+ if (tr) res.set(l.id, tr);
758
+ else res.delete(l.id);
759
+ } catch (err) {
760
+ if (o.signal?.aborted) throw err;
761
+ res.delete(l.id);
762
+ o.log?.(`translate: line ${l.id} failed (${err.message})`);
763
+ }
764
+ };
765
+ if (lines.length === 1) {
766
+ await alone(lines[0]);
767
+ return res;
768
+ }
769
+ const halves = async () => {
770
+ if (depth > 0) {
771
+ for (const l of lines) await alone(l);
772
+ return res;
773
+ }
774
+ const mid = Math.ceil(lines.length / 2);
775
+ for (const part of [lines.slice(0, mid), lines.slice(mid)]) for (const [id, tr] of await translateBatch(handle, targets, part, glossary, o, depth + 1)) res.set(id, tr);
776
+ return res;
777
+ };
447
778
  const system = [
448
779
  `You translate subtitles into: ${targets.map((t) => `"${t}" (${LANGS[t] ?? t})`).join(", ")}. Keep each line separate (same ids), faithful, natural and concise, with normal punctuation.`,
780
+ "One translation per id, in the same order: never join two lines into one or move part of a line to the next, even when a sentence runs across lines.",
449
781
  "Use the earlier and later lines as context. Keep brand, product and model names as they are; follow the glossary.",
450
782
  `Reply with JSON only: [{"id":1,${targets.map((t) => `"${t}":"…"`).join(",")}}]`
451
783
  ].join(" ");
@@ -457,47 +789,58 @@ async function translateBatch(handle, targets, lines, glossary = "") {
457
789
  src: l.src
458
790
  })))
459
791
  ].filter(Boolean).join("\n");
460
- const { provider, model } = handle.selection();
461
- const asm = new BlockAssembler();
462
- for await (const chunk of handle.stream({
463
- provider,
464
- model,
465
- system,
466
- maxTokens: 400 + lines.length * 90 * targets.length,
467
- purpose: "live-translate-batch",
468
- messages: [createUserMessage({
469
- content: [{
470
- type: "text",
471
- text
472
- }],
473
- source: {
474
- kind: "plugin",
475
- plugin: "dsh-vidi"
476
- }
477
- })]
478
- })) asm.push(chunk);
479
- const out = asm.blocks().filter((b) => b.type === "text").map((b) => b.text ?? "").join("");
480
- const res = /* @__PURE__ */ new Map();
481
- const m = out.match(/\[[\s\S]*\]/);
482
- if (!m) return res;
792
+ let out;
483
793
  try {
484
- for (const row of JSON.parse(m[0])) {
485
- const id = Number(row.id);
486
- const tr = {};
487
- for (const t of targets) if (typeof row[t] === "string" && row[t].trim()) tr[t] = row[t].trim();
488
- if (Object.keys(tr).length) res.set(id, tr);
794
+ out = await ask(handle, {
795
+ system,
796
+ text,
797
+ maxTokens: 400 + lines.length * 90 * targets.length,
798
+ purpose: "live-translate-batch",
799
+ signal: o.signal,
800
+ log: o.log
801
+ });
802
+ } catch (err) {
803
+ if (o.signal?.aborted) throw err;
804
+ if (!(err instanceof AskError && err.empty)) {
805
+ o.log?.(`translate: ${lines.length} lines failed (${err.message})`);
806
+ return res;
489
807
  }
490
- } catch {}
808
+ o.log?.(`translate: ${lines.length} lines failed (${err.message}), trying smaller`);
809
+ return halves();
810
+ }
811
+ const rows = jsonIn(out, "[");
812
+ if (!Array.isArray(rows)) {
813
+ o.log?.(`translate: unreadable answer for ${lines.length} lines (${out.slice(0, 80)}), trying smaller`);
814
+ return halves();
815
+ }
816
+ const ids = rows.map((r) => Number(r?.id));
817
+ if (misnumbered(lines.map((l) => l.id), ids)) {
818
+ o.log?.(`translate: answer numbered wrongly for ${lines.length} lines, trying smaller`);
819
+ return halves();
820
+ }
821
+ for (const row of rows) {
822
+ const tr = {};
823
+ for (const t of targets) if (typeof row[t] === "string" && row[t].trim()) tr[t] = row[t].trim();
824
+ if (Object.keys(tr).length) res.set(Number(row.id), tr);
825
+ }
826
+ const bad = suspects(lines.map((l) => ({
827
+ ...l,
828
+ tr: res.get(l.id)
829
+ })), targets);
830
+ if (bad.size) {
831
+ o.log?.(`translate: ${bad.size} of ${lines.length} lines missing or on the wrong line (ids ${[...bad].join(", ")}), translating them one by one`);
832
+ for (const l of lines) if (bad.has(l.id)) await alone(l);
833
+ }
491
834
  return res;
492
835
  }
836
+ /** Pull the JSON object out of the reply; fall back to plain text for a single target. */
493
837
  function parseTranslations(out, targets) {
494
- const m = out.match(/\{[\s\S]*\}/);
495
- if (m) try {
496
- const obj = JSON.parse(m[0]);
838
+ const obj = jsonIn(out, "{");
839
+ if (obj && typeof obj === "object") {
497
840
  const res = {};
498
841
  for (const t of targets) if (typeof obj[t] === "string" && obj[t]) res[t] = obj[t].trim();
499
842
  if (Object.keys(res).length) return res;
500
- } catch {}
843
+ }
501
844
  if (targets.length === 1 && out && !out.startsWith("{")) return { [targets[0]]: out };
502
845
  }
503
846
  // src/polish.ts
@@ -529,7 +872,7 @@ var Polisher = class {
529
872
  return this.pending.length;
530
873
  }
531
874
  add(cue) {
532
- if (!cue.src.trim()) return;
875
+ if (!cue.src.trim() || this.deps.signal?.aborted) return;
533
876
  this.pending.push(cue);
534
877
  if (this.pending.length === 1) this.firstWaitAt = Date.now();
535
878
  this.kick();
@@ -547,7 +890,7 @@ var Polisher = class {
547
890
  }
548
891
  /** Polish whatever is left (end of a run). */
549
892
  async flush() {
550
- while (this.busy || this.pending.length) if (this.busy) await this.busy;
893
+ while ((this.busy || this.pending.length) && !this.deps.signal?.aborted) if (this.busy) await this.busy;
551
894
  else await this.runBatch();
552
895
  }
553
896
  runBatch() {
@@ -562,13 +905,19 @@ var Polisher = class {
562
905
  async polish(batch) {
563
906
  const handle = this.deps.llm();
564
907
  if (!handle || !batch.length) return;
908
+ await this.part(handle, batch, 0);
909
+ this.done.push(...batch);
910
+ if (this.done.length > 40) this.done.splice(0, this.done.length - 40);
911
+ }
912
+ /** Polish some lines in one request; what comes back is checked line by line against its id. */
913
+ async part(handle, batch, depth) {
565
914
  const targets = this.deps.targets();
566
915
  const names = targets.map((t) => `"${t}" (${LANGS[t] ?? t})`).join(", ");
567
916
  const system = [
568
917
  "You polish live speech-recognition subtitles and maintain the glossary for this video.",
569
918
  "A frame from the video in this stretch may be attached: use what is on screen (names, titles, slides, captions, products) to fix words and to confirm glossary entries.",
570
919
  "Fix misheard words (names, terms, homophones) using the video background, the glossary, the site's own subtitles and the surrounding lines. Otherwise keep the speaker's words: do not rewrite, summarise, merge or split lines; keep every id.",
571
- targets.length ? `Translate each fixed line into: ${names}, as natural, concise subtitles.` : "No translation is needed.",
920
+ targets.length ? `Translate each fixed line into: ${names}, as natural, concise subtitles. Each translation covers exactly the words of its own id: never join two lines into one translation or move part of a line to the next, even when a sentence runs across lines.` : "No translation is needed.",
572
921
  "Speaker: say who says each line — a name when the background or context makes it clear (host, guest, a named person), otherwise \"Speaker 1\", \"Speaker 2\"… consistent with the earlier lines; if only one person talks, use the same label throughout.",
573
922
  "Glossary: keep names and terms that must stay consistent — people, groups, places, products, titles, jargon, and words recognition keeps getting wrong. Merge spelling variants under one correct form and list the wrong forms as aliases. Only add what the text or background confirms; correct or remove earlier entries when new evidence shows they were wrong. Skip ordinary words, and skip entries whose translation equals the source unless they have aliases. Return only changes.",
574
923
  `Reply with JSON only: {"lines":[{"id":1,"speaker":"…","src":"…"${targets.length ? `,"tr":{${targets.map((t) => `"${t}":"…"`).join(",")}}` : ""}}],"glossary":{"set":[{"src":"correct form","aliases":["misheard form"],"tr":{"<lang>":"…"},"note":"who/what, short"}],"remove":["src"]}}`
@@ -594,50 +943,93 @@ var Polisher = class {
594
943
  this.deps.log("polish: no picture to show the model");
595
944
  }
596
945
  const image = jpeg ? await handle.image(jpeg).catch(() => void 0) : void 0;
597
- const { provider, model } = handle.selection();
598
- const asm = new BlockAssembler();
599
- for await (const chunk of handle.stream({
600
- provider,
601
- model,
602
- system,
603
- maxTokens: 800 + batch.length * 120 * Math.max(1, targets.length + 1),
604
- purpose: "live-polish",
605
- messages: [createUserMessage({
606
- content: [...image ? [image] : [], {
607
- type: "text",
608
- text
609
- }],
610
- source: {
611
- kind: "plugin",
612
- plugin: "dsh-vidi"
613
- }
614
- })]
615
- })) asm.push(chunk);
616
- const out = asm.blocks().filter((b) => b.type === "text").map((b) => b.text ?? "").join("");
617
- const m = out.match(/\{[\s\S]*\}/);
618
- if (!m) {
619
- this.deps.log(`polish: no JSON (${out.slice(0, 120)})`);
620
- this.done.push(...batch);
621
- return;
622
- }
623
- let res;
946
+ let out;
624
947
  try {
625
- res = JSON.parse(m[0]);
626
- } catch {
627
- this.deps.log("polish: bad JSON");
628
- this.done.push(...batch);
948
+ out = await ask(handle, {
949
+ system,
950
+ text,
951
+ image,
952
+ maxTokens: 800 + batch.length * 120 * Math.max(1, targets.length + 1),
953
+ purpose: "live-polish",
954
+ signal: this.deps.signal,
955
+ log: this.deps.log
956
+ });
957
+ } catch (err) {
958
+ if (this.deps.signal?.aborted) return;
959
+ if (err instanceof AskError && err.empty) return this.smaller(handle, batch, depth, err.message);
960
+ this.deps.log(`polish: ${batch.length} lines not polished (${err.message})`);
629
961
  return;
630
962
  }
631
- for (const line of res.lines ?? []) {
963
+ const res = jsonIn(out, "{");
964
+ if (!res || !Array.isArray(res.lines)) return this.smaller(handle, batch, depth, `unreadable answer (${out.slice(0, 80)})`);
965
+ if (misnumbered(batch.map((c) => c.id), res.lines.map((l) => Number(l?.id)))) return this.smaller(handle, batch, depth, "answer numbered wrongly");
966
+ const got = /* @__PURE__ */ new Map();
967
+ const strangers = /* @__PURE__ */ new Set();
968
+ for (const line of res.lines) {
632
969
  const cue = batch.find((c) => c.id === line.id);
633
970
  if (!cue || typeof line.src !== "string" || !line.src.trim()) continue;
971
+ const own = alike(line.src, cue.src) >= .4;
972
+ if (!own) strangers.add(cue.id);
634
973
  const tr = {};
635
- for (const t of targets) if (typeof line.tr?.[t] === "string") tr[t] = line.tr[t];
636
- this.deps.onLine(cue, line.src.trim(), tr, typeof line.speaker === "string" && line.speaker.trim() ? line.speaker.trim().slice(0, 40) : void 0);
974
+ for (const t of targets) if (typeof line.tr?.[t] === "string" && line.tr[t].trim()) tr[t] = line.tr[t].trim();
975
+ got.set(cue.id, {
976
+ src: own ? line.src.trim() : cue.src,
977
+ tr,
978
+ speaker: typeof line.speaker === "string" && line.speaker.trim() ? line.speaker.trim().slice(0, 40) : void 0
979
+ });
980
+ }
981
+ const bad = targets.length ? suspects(batch.map((c) => ({
982
+ id: c.id,
983
+ src: got.get(c.id)?.src ?? c.src,
984
+ tr: got.get(c.id)?.tr
985
+ })), targets) : /* @__PURE__ */ new Set();
986
+ for (const id of strangers) bad.add(id);
987
+ if (bad.size) this.deps.log(`polish: ${bad.size} of ${batch.length} lines missing or on the wrong line (ids ${[...bad].join(", ")}), translating them one by one`);
988
+ for (const cue of batch) {
989
+ const g = got.get(cue.id);
990
+ if (g && !bad.has(cue.id)) {
991
+ this.deps.onLine(cue, g.src, g.tr, g.speaker);
992
+ continue;
993
+ }
994
+ if (!targets.length) continue;
995
+ const src = g?.src ?? cue.src;
996
+ const tr = await this.alone(handle, batch, cue, src);
997
+ if (tr || g) this.deps.onLine(cue, src, tr ?? {}, g?.speaker);
637
998
  }
638
999
  if (res.glossary) this.deps.glossary.apply(res.glossary);
639
- this.done.push(...batch);
640
- if (this.done.length > 40) this.done.splice(0, this.done.length - 40);
1000
+ }
1001
+ /** A request that gave nothing usable: once more in two halves, then each line translated on its own. */
1002
+ async smaller(handle, batch, depth, why) {
1003
+ if (batch.length > 1 && depth === 0) {
1004
+ this.deps.log(`polish: ${why} for ${batch.length} lines, trying smaller`);
1005
+ const mid = Math.ceil(batch.length / 2);
1006
+ await this.part(handle, batch.slice(0, mid), 1);
1007
+ await this.part(handle, batch.slice(mid), 1);
1008
+ return;
1009
+ }
1010
+ this.deps.log(`polish: ${why} for ${batch.length} lines, translating them one by one`);
1011
+ if (!this.deps.targets().length) return;
1012
+ for (const cue of batch) {
1013
+ const tr = await this.alone(handle, batch, cue, cue.src);
1014
+ if (tr) this.deps.onLine(cue, cue.src, tr);
1015
+ }
1016
+ }
1017
+ /** One line translated on its own, with the lines before it as context. */
1018
+ async alone(handle, batch, cue, src) {
1019
+ if (this.deps.signal?.aborted) return void 0;
1020
+ const before = [...this.done.slice(-3), ...batch.slice(0, batch.indexOf(cue))].slice(-4);
1021
+ try {
1022
+ return await translateLine(handle, this.deps.targets(), src, before.map((c) => ({
1023
+ src: c.src,
1024
+ tr: c.tr
1025
+ })), this.deps.glossary.forPrompt(), "", {
1026
+ signal: this.deps.signal,
1027
+ log: this.deps.log
1028
+ });
1029
+ } catch (err) {
1030
+ if (!this.deps.signal?.aborted) this.deps.log(`polish: line ${cue.id} not translated (${err.message})`);
1031
+ return;
1032
+ }
641
1033
  }
642
1034
  };
643
1035
  // src/xiaoyuzhou.ts
@@ -1149,19 +1541,760 @@ function downloadAudio(url, dir, browser, onProgress) {
1149
1541
  p.stderr.on("data", (d) => {
1150
1542
  err += d.toString();
1151
1543
  });
1152
- p.on("error", reject);
1153
- p.on("exit", (code) => {
1154
- const file = out.split("\n").map((l) => l.trim()).filter((l) => l.startsWith("/")).at(-1);
1155
- if (code === 0 && file && existsSync(file)) resolve(file);
1156
- else reject(new Error(ytdlpError({ stderr: err })));
1544
+ p.on("error", reject);
1545
+ p.on("exit", (code) => {
1546
+ const file = out.split("\n").map((l) => l.trim()).filter((l) => l.startsWith("/")).at(-1);
1547
+ if (code === 0 && file && existsSync(file)) resolve(file);
1548
+ else reject(new Error(ytdlpError({ stderr: err })));
1549
+ });
1550
+ });
1551
+ }
1552
+ /** Keep the most recent downloaded original of a job dir, if any. */
1553
+ function findOriginal(dir) {
1554
+ if (!existsSync(dir)) return void 0;
1555
+ const f = readdirSync(dir).find((n) => /^original\.(mp4|mkv|webm|m4a|mp3)$/.test(n));
1556
+ return f ? join(dir, f) : void 0;
1557
+ }
1558
+ // src/dub.ts
1559
+ const VOICE_VENDORS = [
1560
+ "openai",
1561
+ "siliconflow",
1562
+ "gemini",
1563
+ "elevenlabs"
1564
+ ];
1565
+ /** Services that can learn a voice from a recording. */
1566
+ const CLONE_VENDORS = ["siliconflow", "elevenlabs"];
1567
+ /** A handful of voices per service, described the way people choose them. */
1568
+ const PRESETS = [
1569
+ {
1570
+ vendor: "openai",
1571
+ id: "marin",
1572
+ zh: "Marin · 女声,自然",
1573
+ en: "Marin · female, natural"
1574
+ },
1575
+ {
1576
+ vendor: "openai",
1577
+ id: "cedar",
1578
+ zh: "Cedar · 男声,自然",
1579
+ en: "Cedar · male, natural"
1580
+ },
1581
+ {
1582
+ vendor: "openai",
1583
+ id: "coral",
1584
+ zh: "Coral · 女声,明亮",
1585
+ en: "Coral · female, bright"
1586
+ },
1587
+ {
1588
+ vendor: "openai",
1589
+ id: "ash",
1590
+ zh: "Ash · 男声,沉稳",
1591
+ en: "Ash · male, steady"
1592
+ },
1593
+ {
1594
+ vendor: "openai",
1595
+ id: "sage",
1596
+ zh: "Sage · 女声,平和",
1597
+ en: "Sage · female, calm"
1598
+ },
1599
+ {
1600
+ vendor: "openai",
1601
+ id: "echo",
1602
+ zh: "Echo · 男声,清亮",
1603
+ en: "Echo · male, clear"
1604
+ },
1605
+ {
1606
+ vendor: "siliconflow",
1607
+ id: "anna",
1608
+ zh: "Anna · 女声,沉稳",
1609
+ en: "Anna · female, steady"
1610
+ },
1611
+ {
1612
+ vendor: "siliconflow",
1613
+ id: "bella",
1614
+ zh: "Bella · 女声,热情",
1615
+ en: "Bella · female, warm"
1616
+ },
1617
+ {
1618
+ vendor: "siliconflow",
1619
+ id: "claire",
1620
+ zh: "Claire · 女声,温柔",
1621
+ en: "Claire · female, gentle"
1622
+ },
1623
+ {
1624
+ vendor: "siliconflow",
1625
+ id: "alex",
1626
+ zh: "Alex · 男声,沉稳",
1627
+ en: "Alex · male, steady"
1628
+ },
1629
+ {
1630
+ vendor: "siliconflow",
1631
+ id: "benjamin",
1632
+ zh: "Benjamin · 男声,低沉",
1633
+ en: "Benjamin · male, deep"
1634
+ },
1635
+ {
1636
+ vendor: "siliconflow",
1637
+ id: "david",
1638
+ zh: "David · 男声,欢快",
1639
+ en: "David · male, cheerful"
1640
+ },
1641
+ {
1642
+ vendor: "gemini",
1643
+ id: "Kore",
1644
+ zh: "Kore · 女声,坚定",
1645
+ en: "Kore · female, firm"
1646
+ },
1647
+ {
1648
+ vendor: "gemini",
1649
+ id: "Aoede",
1650
+ zh: "Aoede · 女声,轻快",
1651
+ en: "Aoede · female, breezy"
1652
+ },
1653
+ {
1654
+ vendor: "gemini",
1655
+ id: "Charon",
1656
+ zh: "Charon · 男声,讲解",
1657
+ en: "Charon · male, informative"
1658
+ },
1659
+ {
1660
+ vendor: "gemini",
1661
+ id: "Puck",
1662
+ zh: "Puck · 男声,活泼",
1663
+ en: "Puck · male, upbeat"
1664
+ },
1665
+ {
1666
+ vendor: "elevenlabs",
1667
+ id: "EXAVITQu4vr4xnSDxMaL",
1668
+ zh: "Sarah · 女声",
1669
+ en: "Sarah · female"
1670
+ },
1671
+ {
1672
+ vendor: "elevenlabs",
1673
+ id: "JBFqnCBsd6RMkjVDRZzb",
1674
+ zh: "George · 男声",
1675
+ en: "George · male"
1676
+ }
1677
+ ];
1678
+ const SAMPLE_RATE = 24e3;
1679
+ const BYTES_PER_SEC$1 = SAMPLE_RATE * 2;
1680
+ /** WAV file around 16-bit mono PCM. */
1681
+ function wav(pcm, rate = SAMPLE_RATE) {
1682
+ const h = Buffer.alloc(44);
1683
+ h.write("RIFF", 0);
1684
+ h.writeUInt32LE(36 + pcm.length, 4);
1685
+ h.write("WAVE", 8);
1686
+ h.write("fmt ", 12);
1687
+ h.writeUInt32LE(16, 16);
1688
+ h.writeUInt16LE(1, 20);
1689
+ h.writeUInt16LE(1, 22);
1690
+ h.writeUInt32LE(rate, 24);
1691
+ h.writeUInt32LE(rate * 2, 28);
1692
+ h.writeUInt16LE(2, 32);
1693
+ h.writeUInt16LE(16, 34);
1694
+ h.write("data", 36);
1695
+ h.writeUInt32LE(pcm.length, 40);
1696
+ return Buffer.concat([h, pcm]);
1697
+ }
1698
+ /** Raw PCM of a WAV file written by `wav` (or any plain 44-byte-header WAV). */
1699
+ const pcmOf = (file) => file.subarray(44);
1700
+ /** Run ffmpeg over a buffer (stdin to stdout). */
1701
+ function ffmpegBuffer(args, input) {
1702
+ return new Promise((resolve, reject) => {
1703
+ const p = spawn("ffmpeg", [
1704
+ "-hide_banner",
1705
+ "-loglevel",
1706
+ "error",
1707
+ ...args
1708
+ ], { stdio: [
1709
+ "pipe",
1710
+ "pipe",
1711
+ "pipe"
1712
+ ] });
1713
+ const out = [];
1714
+ let err = "";
1715
+ p.stdout.on("data", (d) => out.push(d));
1716
+ p.stderr.on("data", (d) => {
1717
+ err = (err + d).slice(-400);
1718
+ });
1719
+ p.on("error", reject);
1720
+ p.on("exit", (code) => code === 0 ? resolve(Buffer.concat(out)) : reject(new Error(err.trim() || `ffmpeg exited (${code})`)));
1721
+ p.stdin.on("error", () => {});
1722
+ p.stdin.end(input);
1723
+ });
1724
+ }
1725
+ /** Faster speech, same pitch (for a line that would run into the next). */
1726
+ const speedUp = (pcm, tempo) => ffmpegBuffer([
1727
+ "-f",
1728
+ "s16le",
1729
+ "-ar",
1730
+ String(SAMPLE_RATE),
1731
+ "-ac",
1732
+ "1",
1733
+ "-i",
1734
+ "pipe:0",
1735
+ "-af",
1736
+ `atempo=${tempo.toFixed(3)}`,
1737
+ "-f",
1738
+ "s16le",
1739
+ "-ar",
1740
+ String(SAMPLE_RATE),
1741
+ "-ac",
1742
+ "1",
1743
+ "pipe:1"
1744
+ ], pcm);
1745
+ /** Cut the silence a service leaves before and after the words (so lines start on time). */
1746
+ function trimSilence(pcm) {
1747
+ const n = Math.floor(pcm.length / 2);
1748
+ const loud = (i) => Math.abs(pcm.readInt16LE(i * 2)) > 400;
1749
+ let a = 0, b = n - 1;
1750
+ while (a < n && !loud(a)) a++;
1751
+ while (b > a && !loud(b)) b--;
1752
+ const pad = Math.round(SAMPLE_RATE * .04);
1753
+ return pcm.subarray(Math.max(0, a - pad) * 2, Math.min(n, b + pad) * 2);
1754
+ }
1755
+ const LANG_NAMES = {
1756
+ zh: "Mandarin Chinese",
1757
+ en: "English",
1758
+ ja: "Japanese",
1759
+ ko: "Korean",
1760
+ fr: "French",
1761
+ de: "German",
1762
+ es: "Spanish",
1763
+ ru: "Russian",
1764
+ pt: "Portuguese",
1765
+ it: "Italian"
1766
+ };
1767
+ let geminiTtsModel;
1768
+ async function geminiModel(key) {
1769
+ if (geminiTtsModel) return geminiTtsModel;
1770
+ try {
1771
+ const names = ((await (await fetch("https://generativelanguage.googleapis.com/v1beta/models?pageSize=200", {
1772
+ headers: { "x-goog-api-key": key },
1773
+ signal: AbortSignal.timeout(1e4)
1774
+ })).json()).models ?? []).map((m) => m.name.replace(/^models\//, "")).filter((n) => /tts/.test(n) && /flash/.test(n));
1775
+ names.sort((a, b) => b.localeCompare(a, "en", { numeric: true }));
1776
+ geminiTtsModel = names[0] ?? "gemini-2.5-flash-preview-tts";
1777
+ } catch {
1778
+ geminiTtsModel = "gemini-2.5-flash-preview-tts";
1779
+ }
1780
+ return geminiTtsModel;
1781
+ }
1782
+ /**
1783
+ * Speak `text` (in language `lang`) with a voice: 16-bit mono PCM at 24 kHz,
1784
+ * silence at the ends trimmed.
1785
+ */
1786
+ async function speak(deps, voice, text, lang) {
1787
+ const [kind, id = ""] = voice.split(/:(.*)/s);
1788
+ let vendor, name = id;
1789
+ let mine;
1790
+ if (kind === "mine") {
1791
+ mine = deps.mine(id);
1792
+ if (!mine) throw new Error("that voice is gone");
1793
+ vendor = mine.vendor;
1794
+ } else vendor = kind;
1795
+ const key = deps.key(vendor);
1796
+ if (!key) throw new Error(`no ${vendor} key`);
1797
+ const timeout = AbortSignal.timeout(45e3);
1798
+ const fail = async (r) => {
1799
+ throw new Error(`${vendor} ${r.status}: ${(await r.text()).slice(0, 200)}`);
1800
+ };
1801
+ let pcm;
1802
+ if (vendor === "openai") {
1803
+ const r = await fetch("https://api.openai.com/v1/audio/speech", {
1804
+ method: "POST",
1805
+ signal: timeout,
1806
+ headers: {
1807
+ Authorization: `Bearer ${key}`,
1808
+ "content-type": "application/json"
1809
+ },
1810
+ body: JSON.stringify({
1811
+ model: "gpt-4o-mini-tts",
1812
+ voice: name,
1813
+ input: text,
1814
+ response_format: "pcm",
1815
+ instructions: `Speak ${LANG_NAMES[lang.split("-")[0]] ?? "the language of the text"} like a live interpreter: natural, clear, at an even, brisk pace.`
1816
+ })
1817
+ });
1818
+ if (!r.ok) await fail(r);
1819
+ pcm = Buffer.from(await r.arrayBuffer());
1820
+ } else if (vendor === "siliconflow") {
1821
+ const base = deps.base("siliconflow") ?? "https://api.siliconflow.cn/v1";
1822
+ const model = "FunAudioLLM/CosyVoice2-0.5B";
1823
+ const body = {
1824
+ model,
1825
+ input: text,
1826
+ response_format: "pcm",
1827
+ sample_rate: SAMPLE_RATE
1828
+ };
1829
+ if (mine?.remote) body.voice = mine.remote;
1830
+ else if (mine) {
1831
+ if (!mine.sample) throw new Error("the recording of that voice is missing");
1832
+ body.voice = "";
1833
+ body.references = [{
1834
+ audio: `data:audio/wav;base64,${mine.sample.toString("base64")}`,
1835
+ text: mine.text
1836
+ }];
1837
+ } else body.voice = `${model}:${name}`;
1838
+ const r = await fetch(`${base}/audio/speech`, {
1839
+ method: "POST",
1840
+ signal: timeout,
1841
+ headers: {
1842
+ Authorization: `Bearer ${key}`,
1843
+ "content-type": "application/json"
1844
+ },
1845
+ body: JSON.stringify(body)
1846
+ });
1847
+ if (!r.ok) await fail(r);
1848
+ pcm = Buffer.from(await r.arrayBuffer());
1849
+ } else if (vendor === "gemini") {
1850
+ const model = await geminiModel(key);
1851
+ const r = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(model)}:generateContent`, {
1852
+ method: "POST",
1853
+ signal: timeout,
1854
+ headers: {
1855
+ "x-goog-api-key": key,
1856
+ "content-type": "application/json"
1857
+ },
1858
+ body: JSON.stringify({
1859
+ contents: [{ parts: [{ text: `Read aloud, naturally and briskly, like an interpreter:\n${text}` }] }],
1860
+ generationConfig: {
1861
+ responseModalities: ["AUDIO"],
1862
+ speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: name } } }
1863
+ }
1864
+ })
1865
+ });
1866
+ if (!r.ok) await fail(r);
1867
+ const part = (await r.json()).candidates?.[0]?.content?.parts?.find((p) => p.inlineData?.data)?.inlineData;
1868
+ if (!part?.data) throw new Error("gemini returned no audio");
1869
+ pcm = Buffer.from(part.data, "base64");
1870
+ const rate = Number(/rate=(\d+)/.exec(part.mimeType ?? "")?.[1] ?? 24e3);
1871
+ if (rate !== 24e3) pcm = await ffmpegBuffer([
1872
+ "-f",
1873
+ "s16le",
1874
+ "-ar",
1875
+ String(rate),
1876
+ "-ac",
1877
+ "1",
1878
+ "-i",
1879
+ "pipe:0",
1880
+ "-f",
1881
+ "s16le",
1882
+ "-ar",
1883
+ String(SAMPLE_RATE),
1884
+ "-ac",
1885
+ "1",
1886
+ "pipe:1"
1887
+ ], pcm);
1888
+ } else {
1889
+ const voiceId = mine?.remote ?? name;
1890
+ const r = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${encodeURIComponent(voiceId)}?output_format=pcm_24000`, {
1891
+ method: "POST",
1892
+ signal: timeout,
1893
+ headers: {
1894
+ "xi-api-key": key,
1895
+ "content-type": "application/json"
1896
+ },
1897
+ body: JSON.stringify({
1898
+ text,
1899
+ model_id: "eleven_flash_v2_5",
1900
+ language_code: lang.split("-")[0]
1901
+ })
1902
+ });
1903
+ if (!r.ok) await fail(r);
1904
+ pcm = Buffer.from(await r.arrayBuffer());
1905
+ }
1906
+ if (pcm.length % 2) pcm = pcm.subarray(0, pcm.length - 1);
1907
+ return trimSilence(pcm);
1908
+ }
1909
+ /** Teach a service a voice from a recording; returns its id there (undefined: it goes along with each request). */
1910
+ async function cloneVoice(deps, vendor, name, sample, text) {
1911
+ const key = deps.key(vendor);
1912
+ if (!key) throw new Error(`no ${vendor} key`);
1913
+ if (vendor === "siliconflow") {
1914
+ const base = deps.base("siliconflow") ?? "https://api.siliconflow.cn/v1";
1915
+ const r = await fetch(`${base}/uploads/audio/voice`, {
1916
+ method: "POST",
1917
+ signal: AbortSignal.timeout(6e4),
1918
+ headers: {
1919
+ Authorization: `Bearer ${key}`,
1920
+ "content-type": "application/json"
1921
+ },
1922
+ body: JSON.stringify({
1923
+ model: "FunAudioLLM/CosyVoice2-0.5B",
1924
+ customName: name.replace(/[^\w-]+/g, "").slice(0, 40) || `vidi${Date.now()}`,
1925
+ audio: `data:audio/wav;base64,${sample.toString("base64")}`,
1926
+ text
1927
+ })
1928
+ });
1929
+ if (!r.ok) {
1930
+ deps.log(`siliconflow: voice upload refused (${r.status}: ${(await r.text()).slice(0, 160)}); sending the sample with each line`);
1931
+ return;
1932
+ }
1933
+ return (await r.json()).uri;
1934
+ }
1935
+ if (vendor === "elevenlabs") {
1936
+ const form = new FormData();
1937
+ form.append("name", name || "Vidi voice");
1938
+ form.append("files", new Blob([new Uint8Array(sample)], { type: "audio/wav" }), "sample.wav");
1939
+ const r = await fetch("https://api.elevenlabs.io/v1/voices/add", {
1940
+ method: "POST",
1941
+ signal: AbortSignal.timeout(6e4),
1942
+ headers: { "xi-api-key": key },
1943
+ body: form
1944
+ });
1945
+ if (!r.ok) throw new Error(`elevenlabs ${r.status}: ${(await r.text()).slice(0, 200)}`);
1946
+ return (await r.json()).voice_id;
1947
+ }
1948
+ throw new Error(`${vendor} cannot learn voices`);
1949
+ }
1950
+ /** Voices made from recordings, kept in one folder for every video. */
1951
+ var VoiceLibrary = class {
1952
+ dir;
1953
+ constructor(dir) {
1954
+ this.dir = dir;
1955
+ mkdirSync(dir, { recursive: true });
1956
+ }
1957
+ get file() {
1958
+ return join(this.dir, "voices.json");
1959
+ }
1960
+ list() {
1961
+ try {
1962
+ return JSON.parse(readFileSync(this.file, "utf8"));
1963
+ } catch {
1964
+ return [];
1965
+ }
1966
+ }
1967
+ get(id) {
1968
+ const v = this.list().find((x) => x.id === id);
1969
+ if (!v) return void 0;
1970
+ const f = this.samplePath(id);
1971
+ return {
1972
+ ...v,
1973
+ sample: existsSync(f) ? readFileSync(f) : void 0
1974
+ };
1975
+ }
1976
+ samplePath(id) {
1977
+ return join(this.dir, `${id.replace(/[^\w-]/g, "")}.wav`);
1978
+ }
1979
+ add(v, sample) {
1980
+ writeFileSync(this.samplePath(v.id), sample);
1981
+ writeFileSync(this.file, JSON.stringify([...this.list(), v], null, 2));
1982
+ }
1983
+ rename(id, name) {
1984
+ writeFileSync(this.file, JSON.stringify(this.list().map((v) => v.id === id ? {
1985
+ ...v,
1986
+ name
1987
+ } : v), null, 2));
1988
+ }
1989
+ remove(id) {
1990
+ writeFileSync(this.file, JSON.stringify(this.list().filter((v) => v.id !== id), null, 2));
1991
+ rmSync(this.samplePath(id), { force: true });
1992
+ }
1993
+ };
1994
+ /** How long a line may run into its successor before it is spoken faster (at most this much). */
1995
+ const MAX_TEMPO = 1.35;
1996
+ /** Lines this far ahead of what is being watched or pushed get their voice made in advance (s). */
1997
+ const AHEAD = 90;
1998
+ /**
1999
+ * The voiceover of one video: each subtitle line in the chosen language is
2000
+ * spoken with the chosen voice, kept as a small WAV, and placed at the
2001
+ * line's start (after the previous voice, if that one is still talking).
2002
+ * Voices are made only around what is being watched or pushed out, so a
2003
+ * long video costs only what is played. Events: 'change' (DubView).
2004
+ */
2005
+ var Dubber = class extends EventEmitter {
2006
+ dir;
2007
+ cues;
2008
+ spokenLang;
2009
+ polishedMode;
2010
+ speakFn;
2011
+ onSpoken;
2012
+ log;
2013
+ settings;
2014
+ stored = /* @__PURE__ */ new Map();
2015
+ storeKey = "";
2016
+ busy = /* @__PURE__ */ new Set();
2017
+ failed = /* @__PURE__ */ new Map();
2018
+ failures = /* @__PURE__ */ new Map();
2019
+ queueTimer;
2020
+ changeTimer;
2021
+ /** Media times being watched (the page) and pushed out (restream). */
2022
+ focus = /* @__PURE__ */ new Map();
2023
+ cache = /* @__PURE__ */ new Map();
2024
+ error;
2025
+ waiters = [];
2026
+ /** When each line was first seen finished (polished mode waits a while for its polished text). */
2027
+ seen = /* @__PURE__ */ new Map();
2028
+ recheck;
2029
+ constructor(dir, settings, cues, spokenLang, polishedMode, speakFn, onSpoken, log) {
2030
+ super();
2031
+ this.dir = dir;
2032
+ this.cues = cues;
2033
+ this.spokenLang = spokenLang;
2034
+ this.polishedMode = polishedMode;
2035
+ this.speakFn = speakFn;
2036
+ this.onSpoken = onSpoken;
2037
+ this.log = log;
2038
+ this.settings = settings;
2039
+ this.load();
2040
+ }
2041
+ get folder() {
2042
+ return join(this.dir, "dub", this.storeKey);
2043
+ }
2044
+ /** Clips are kept per language and voice: switching back finds them again. */
2045
+ load() {
2046
+ this.storeKey = `${this.settings.lang}.${this.settings.voice}`.replace(/[^\w.-]+/g, "_");
2047
+ this.stored.clear();
2048
+ this.cache.clear();
2049
+ this.failed.clear();
2050
+ this.failures.clear();
2051
+ try {
2052
+ for (const [id, s] of Object.entries(JSON.parse(readFileSync(join(this.folder, "clips.json"), "utf8")))) this.stored.set(Number(id), s);
2053
+ } catch {}
2054
+ }
2055
+ saveIndex() {
2056
+ mkdirSync(this.folder, { recursive: true });
2057
+ writeFileSync(join(this.folder, "clips.json"), JSON.stringify(Object.fromEntries(this.stored)));
2058
+ }
2059
+ configure(next) {
2060
+ const before = `${this.settings.lang}.${this.settings.voice}`;
2061
+ this.settings = {
2062
+ ...this.settings,
2063
+ ...next
2064
+ };
2065
+ if (`${this.settings.lang}.${this.settings.voice}` !== before) this.load();
2066
+ this.error = void 0;
2067
+ this.failed.clear();
2068
+ this.failures.clear();
2069
+ this.changed(true);
2070
+ this.kick();
2071
+ }
2072
+ /** Where someone is watching ('page') or the push-out is reading ('push'); undefined: gone. */
2073
+ setFocus(who, at) {
2074
+ if (at === void 0) this.focus.delete(who);
2075
+ else this.focus.set(who, at);
2076
+ this.kick();
2077
+ }
2078
+ /** The text a line speaks: its translation, or the line itself when it is already in that language. */
2079
+ textFor(cue) {
2080
+ if (!cue.final) return void 0;
2081
+ const lang = this.settings.lang;
2082
+ const tr = cue.tr?.[lang]?.trim();
2083
+ if (tr) return tr;
2084
+ const spoken = (cue.lang ?? this.spokenLang() ?? "").split("-")[0];
2085
+ if (spoken && spoken === lang.split("-")[0] && cue.src.trim()) return cue.src.trim();
2086
+ }
2087
+ /** A line changed (or came in): its voice may need making again. */
2088
+ onCue() {
2089
+ this.kick();
2090
+ }
2091
+ kick() {
2092
+ if (!this.settings.on) return;
2093
+ clearTimeout(this.queueTimer);
2094
+ this.queueTimer = setTimeout(() => void this.fill(), 150);
2095
+ }
2096
+ /** Is media time t within reach of someone watching or pushing (or a clip being cut)? */
2097
+ wanted(cue) {
2098
+ for (const w of this.waiters) if (cue.end > w.from && cue.start < w.to) return true;
2099
+ for (const at of this.focus.values()) if (cue.end > at - 5 && cue.start < at + AHEAD) return true;
2100
+ return false;
2101
+ }
2102
+ async fill() {
2103
+ if (!this.settings.on) return;
2104
+ const cues = this.cues().filter((c) => c.final);
2105
+ const now = Date.now();
2106
+ const todo = [];
2107
+ let waiting = false;
2108
+ cues.forEach((cue, i) => {
2109
+ if (this.busy.has(cue.id) || (this.failed.get(cue.id) ?? 0) > now) return;
2110
+ const text = this.textFor(cue);
2111
+ if (!text || this.stored.get(cue.id)?.text === text) return;
2112
+ if (!this.seen.has(cue.id)) this.seen.set(cue.id, now);
2113
+ if (this.polishedMode() && !cue.polished && !cue.edited && now - this.seen.get(cue.id) < 2e4) {
2114
+ waiting = true;
2115
+ return;
2116
+ }
2117
+ if (!this.wanted(cue)) return;
2118
+ todo.push({
2119
+ cue,
2120
+ text,
2121
+ i
2122
+ });
2123
+ });
2124
+ const focus = [...this.focus.values(), ...this.waiters.map((w) => w.from)];
2125
+ const dist = (c) => Math.min(...focus.map((f) => c.start >= f - 2 ? c.start - f : 1e3 + f - c.start), 1e9);
2126
+ todo.sort((a, b) => dist(a.cue) - dist(b.cue));
2127
+ for (const job of todo) {
2128
+ if (this.busy.size >= 3) break;
2129
+ this.busy.add(job.cue.id);
2130
+ this.make(job.cue, job.text, cues[job.i + 1]).finally(() => {
2131
+ this.busy.delete(job.cue.id);
2132
+ this.kick();
2133
+ this.checkWaiters();
2134
+ });
2135
+ }
2136
+ if (waiting) {
2137
+ clearTimeout(this.recheck);
2138
+ this.recheck = setTimeout(() => this.kick(), 5e3);
2139
+ }
2140
+ this.checkWaiters();
2141
+ }
2142
+ async make(cue, text, next) {
2143
+ const key = this.storeKey, voice = this.settings.voice;
2144
+ try {
2145
+ let pcm = await this.speakFn(voice, text, this.settings.lang);
2146
+ if (key !== this.storeKey) return;
2147
+ this.onSpoken(voice, pcm.length / BYTES_PER_SEC$1);
2148
+ const room = next && next.start - cue.end <= 4 ? next.start - cue.start : cue.end - cue.start + .8;
2149
+ const dur = pcm.length / BYTES_PER_SEC$1;
2150
+ if (room > .5 && dur > room * 1.04) pcm = await speedUp(pcm, Math.min(MAX_TEMPO, dur / room));
2151
+ mkdirSync(this.folder, { recursive: true });
2152
+ writeFileSync(join(this.folder, `${cue.id}.wav`), wav(pcm));
2153
+ this.stored.set(cue.id, {
2154
+ text,
2155
+ dur: pcm.length / BYTES_PER_SEC$1
2156
+ });
2157
+ this.cache.set(cue.id, pcm);
2158
+ this.error = void 0;
2159
+ this.saveIndex();
2160
+ this.changed();
2161
+ } catch (err) {
2162
+ const msg = err.message;
2163
+ this.log(`dub: line ${cue.id}: ${msg}`);
2164
+ this.failed.set(cue.id, Date.now() + 15e3);
2165
+ this.failures.set(cue.id, (this.failures.get(cue.id) ?? 0) + 1);
2166
+ if (this.error !== msg) {
2167
+ this.error = msg;
2168
+ this.changed(true);
2169
+ }
2170
+ }
2171
+ }
2172
+ /**
2173
+ * Start of the earliest line from `after` on that is still waiting for its voice (a live picture
2174
+ * waits for it, as it waits for subtitles); lines that keep failing are given up on.
2175
+ */
2176
+ waitingFrom(after) {
2177
+ if (!this.settings.on) return void 0;
2178
+ for (const c of this.cues()) {
2179
+ if (!c.final || c.end < after) continue;
2180
+ const text = this.textFor(c);
2181
+ if (!text || this.stored.get(c.id)?.text === text || (this.failures.get(c.id) ?? 0) >= 3) continue;
2182
+ return c.start;
2183
+ }
2184
+ }
2185
+ /** Clips in play order: each at its line's start, or right after the voice before it. */
2186
+ clips() {
2187
+ const out = [];
2188
+ let end = -Infinity;
2189
+ for (const cue of this.cues()) {
2190
+ if (!cue.final) continue;
2191
+ const s = this.stored.get(cue.id);
2192
+ const text = this.textFor(cue);
2193
+ if (!s || !text || s.text !== text) continue;
2194
+ const at = Math.max(cue.start, end + .05);
2195
+ end = at + s.dur;
2196
+ out.push({
2197
+ id: cue.id,
2198
+ at: Math.round(at * 1e3) / 1e3,
2199
+ dur: Math.round(s.dur * 1e3) / 1e3,
2200
+ v: hash(s.text)
2201
+ });
2202
+ }
2203
+ return out;
2204
+ }
2205
+ view() {
2206
+ return {
2207
+ ...this.settings,
2208
+ clips: this.settings.on ? this.clips() : [],
2209
+ error: this.error
2210
+ };
2211
+ }
2212
+ changed(now = false) {
2213
+ clearTimeout(this.changeTimer);
2214
+ if (now) {
2215
+ this.emit("change", this.view());
2216
+ return;
2217
+ }
2218
+ this.changeTimer = setTimeout(() => this.emit("change", this.view()), 250);
2219
+ }
2220
+ /** A clip's WAV file (for the page), if made. */
2221
+ clipFile(id) {
2222
+ const f = join(this.folder, `${id}.wav`);
2223
+ return existsSync(f) ? f : void 0;
2224
+ }
2225
+ pcm(id) {
2226
+ let b = this.cache.get(id);
2227
+ if (!b) {
2228
+ const f = this.clipFile(id);
2229
+ if (!f) return void 0;
2230
+ b = pcmOf(readFileSync(f));
2231
+ this.cache.set(id, b);
2232
+ if (this.cache.size > 300) this.cache.delete(this.cache.keys().next().value);
2233
+ }
2234
+ return b;
2235
+ }
2236
+ /**
2237
+ * The voices mixed for media time [from, to) as 16-bit PCM at 24 kHz.
2238
+ * `late`: clips that came in after their time (live) still play from
2239
+ * `from` on, once, if not more than LATE_MAX late; pass the same Set on
2240
+ * every call of one continuous run.
2241
+ */
2242
+ mix(from, to, late) {
2243
+ const n = Math.max(0, Math.round((to - from) * SAMPLE_RATE));
2244
+ const acc = new Int32Array(n);
2245
+ for (const c of this.clips()) {
2246
+ let at = c.at;
2247
+ if (late) {
2248
+ if (c.at < from && c.at + c.dur > from && !late.has(c.id) && from - c.at <= 4) at = from;
2249
+ if (at < to && at + c.dur > from) late.add(c.id);
2250
+ }
2251
+ if (at >= to || at + c.dur <= from) continue;
2252
+ const pcm = this.pcm(c.id);
2253
+ if (!pcm) continue;
2254
+ const start = Math.round((at - from) * SAMPLE_RATE);
2255
+ const total = pcm.length / 2;
2256
+ for (let i = Math.max(0, -start); i < total && start + i < n; i++) acc[start + i] += pcm.readInt16LE(i * 2);
2257
+ }
2258
+ const out = Buffer.alloc(n * 2);
2259
+ for (let i = 0; i < n; i++) out.writeInt16LE(Math.max(-32768, Math.min(32767, acc[i])), i * 2);
2260
+ return out;
2261
+ }
2262
+ /** Make the voices for [from, to) now (cutting a clip), waiting up to `ms`. */
2263
+ ensure(from, to, ms = 12e4) {
2264
+ if (!this.settings.on) return Promise.resolve();
2265
+ return new Promise((resolve) => {
2266
+ const w = {
2267
+ from,
2268
+ to,
2269
+ done: () => {
2270
+ clearTimeout(timer);
2271
+ this.waiters = this.waiters.filter((x) => x !== w);
2272
+ resolve();
2273
+ }
2274
+ };
2275
+ const timer = setTimeout(w.done, ms);
2276
+ this.waiters.push(w);
2277
+ this.kick();
1157
2278
  });
1158
- });
2279
+ }
2280
+ checkWaiters() {
2281
+ for (const w of [...this.waiters]) if (!this.cues().some((c) => c.final && c.end > w.from && c.start < w.to && (this.busy.has(c.id) || !!this.textFor(c) && this.stored.get(c.id)?.text !== this.textFor(c) && (this.failed.get(c.id) ?? 0) <= Date.now()))) w.done();
2282
+ }
2283
+ };
2284
+ function hash(s) {
2285
+ let h = 5381;
2286
+ for (let i = 0; i < s.length; i++) h = (h << 5) + h + s.charCodeAt(i) | 0;
2287
+ return (h >>> 0).toString(36);
1159
2288
  }
1160
- /** Keep the most recent downloaded original of a job dir, if any. */
1161
- function findOriginal(dir) {
1162
- if (!existsSync(dir)) return void 0;
1163
- const f = readdirSync(dir).find((n) => /^original\.(mp4|mkv|webm|m4a|mp3)$/.test(n));
1164
- return f ? join(dir, f) : void 0;
2289
+ /**
2290
+ * ffmpeg filter that puts the voiceover (input `dub`, 24 kHz mono) over the
2291
+ * original sound (input `orig`): the original is pressed down whenever the
2292
+ * voice speaks, or left out. Ends at `out`.
2293
+ */
2294
+ function dubFilter(orig, dub, original, out) {
2295
+ const voice = `[${dub}]asetpts=PTS-STARTPTS,aresample=44100,aformat=sample_fmts=fltp:channel_layouts=stereo`;
2296
+ if (original === "off") return `${voice}[${out}]`;
2297
+ return `${voice},asplit=2[dk][dm];[${orig}]asetpts=PTS-STARTPTS,aresample=44100,aformat=sample_fmts=fltp:channel_layouts=stereo[og];[og][dk]sidechaincompress=threshold=0.015:ratio=14:attack=8:release=450:makeup=1[od];[od][dm]amix=inputs=2:normalize=0:duration=first[${out}]`;
1165
2298
  }
1166
2299
  // src/wrap.ts
1167
2300
  /**
@@ -1433,6 +2566,8 @@ var Restream = class extends EventEmitter {
1433
2566
  stuckSince = Date.now();
1434
2567
  /** The encoder's outputs, in order (first on stdout, the next on pipes 4 and 5). */
1435
2568
  outs = [];
2569
+ dubTimer;
2570
+ dubSent = 0;
1436
2571
  constructor(source, opts, targets, log) {
1437
2572
  super();
1438
2573
  this.source = source;
@@ -1518,9 +2653,14 @@ var Restream = class extends EventEmitter {
1518
2653
  "-sc_threshold",
1519
2654
  "0"
1520
2655
  ];
1521
- const audio = [
1522
- "-map",
1523
- "0:a:0?",
2656
+ const dub = this.source.dub;
2657
+ const sounded = this.outs.filter((o) => o !== "subs");
2658
+ const dubIn = subs ? 2 : 1;
2659
+ if (dub && sounded.length) {
2660
+ parts.push(dubFilter("0:a:0", `${dubIn}:a`, dub.original, "amixed"));
2661
+ parts.push(`[amixed]${sounded.length > 1 ? `asplit=${sounded.length}${sounded.map((o) => `[a_${o}]`).join("")}` : `anull[a_${sounded[0]}]`}`);
2662
+ }
2663
+ const audioCodec = [
1524
2664
  "-c:a",
1525
2665
  "aac",
1526
2666
  "-b:a",
@@ -1530,6 +2670,15 @@ var Restream = class extends EventEmitter {
1530
2670
  "-ac",
1531
2671
  "2"
1532
2672
  ];
2673
+ const audioOf = (o) => dub && sounded.length ? [
2674
+ "-map",
2675
+ `[a_${o}]`,
2676
+ ...audioCodec
2677
+ ] : [
2678
+ "-map",
2679
+ "0:a:0?",
2680
+ ...audioCodec
2681
+ ];
1533
2682
  const fd = (i) => i === 0 ? 1 : i + 3;
1534
2683
  const args = [
1535
2684
  "-hide_banner",
@@ -1551,6 +2700,16 @@ var Restream = class extends EventEmitter {
1551
2700
  "-i",
1552
2701
  "pipe:0"
1553
2702
  ] : [],
2703
+ ...dub && sounded.length ? [
2704
+ "-f",
2705
+ "s16le",
2706
+ "-ar",
2707
+ String(SAMPLE_RATE),
2708
+ "-ac",
2709
+ "1",
2710
+ "-i",
2711
+ "pipe:6"
2712
+ ] : [],
1554
2713
  "-filter_complex",
1555
2714
  parts.join(";"),
1556
2715
  "-progress",
@@ -1559,7 +2718,8 @@ var Restream = class extends EventEmitter {
1559
2718
  "-map",
1560
2719
  `[o_${o}]`,
1561
2720
  ...video,
1562
- ...o === "subs" ? ["-an"] : audio,
2721
+ ...o === "subs" ? ["-an"] : audioOf(o),
2722
+ ...dub ? ["-shortest"] : [],
1563
2723
  "-f",
1564
2724
  "mpegts",
1565
2725
  `pipe:${fd(i)}`
@@ -1571,6 +2731,7 @@ var Restream = class extends EventEmitter {
1571
2731
  "pipe",
1572
2732
  "pipe",
1573
2733
  "pipe",
2734
+ "pipe",
1574
2735
  "pipe"
1575
2736
  ] });
1576
2737
  this.encoder = enc;
@@ -1606,6 +2767,7 @@ var Restream = class extends EventEmitter {
1606
2767
  this.stop(reason);
1607
2768
  });
1608
2769
  if (subs) this.feedSubtitles(w, h);
2770
+ if (dub && sounded.length) this.feedVoice();
1609
2771
  for (const p of this.pushes.values()) this.connect(p.target.id);
1610
2772
  this.watchdog = setInterval(() => {
1611
2773
  if (this.stopped) return;
@@ -1653,6 +2815,28 @@ var Restream = class extends EventEmitter {
1653
2815
  });
1654
2816
  }, 100);
1655
2817
  }
2818
+ /** The voiceover for media times from the start on, kept a couple of seconds ahead of the encoder (like the subtitles). */
2819
+ feedVoice() {
2820
+ const sin = this.encoder.stdio[6];
2821
+ sin.on("error", () => {});
2822
+ const late = /* @__PURE__ */ new Set();
2823
+ const step = .2;
2824
+ this.dubTimer = setInterval(() => {
2825
+ if (this.stopped) return;
2826
+ const ahead = Math.max(4, (Date.now() - this.spawnedAt) / 1e3 + 2.5);
2827
+ let guard = 0;
2828
+ while (this.dubSent <= ahead && sin.writableLength < 1 << 20 && guard++ < 100) {
2829
+ const from = this.startAt + this.dubSent;
2830
+ try {
2831
+ sin.write(this.source.dub.mix(from, from + step, late));
2832
+ } catch (err) {
2833
+ this.log(`restream: voiceover: ${err.message}`);
2834
+ sin.write(Buffer.alloc(Math.round(step * SAMPLE_RATE) * 2));
2835
+ }
2836
+ this.dubSent += step;
2837
+ }
2838
+ }, 100);
2839
+ }
1656
2840
  /** Start (or restart) one destination. */
1657
2841
  connect(id) {
1658
2842
  const p = this.pushes.get(id);
@@ -1776,6 +2960,8 @@ var Restream = class extends EventEmitter {
1776
2960
  clearInterval(this.watchdog);
1777
2961
  process.off("exit", this.killAll);
1778
2962
  clearInterval(this.subsTimer);
2963
+ clearInterval(this.dubTimer);
2964
+ this.source.dub?.done();
1779
2965
  for (const p of this.pushes.values()) {
1780
2966
  clearTimeout(p.timer);
1781
2967
  const proc = p.proc;
@@ -1978,6 +3164,8 @@ const FILE_LIKE = /\.(mp4|m4v|mov|mkv|webm|mp3|m4a|aac|wav|flac|ogg)(\?|$)/i;
1978
3164
  /** Pace for subtitles: stop reading ahead once recognition or translation is this far behind (seconds / lines). */
1979
3165
  const AHEAD_MAX_SEC = 30;
1980
3166
  const AHEAD_MAX_LINES = 10;
3167
+ /** How long pausing waits for lines still being translated or polished before leaving them to finish behind it (ms). */
3168
+ const SETTLE = { wait: 3e3 };
1981
3169
  /**
1982
3170
  * One video or live stream, in two independent parts:
1983
3171
  *
@@ -2082,6 +3270,8 @@ var LiveJob = class LiveJob extends EventEmitter {
2082
3270
  /** The video's language as the site states it (before any recognition). */
2083
3271
  siteLang;
2084
3272
  polisher;
3273
+ /** Set off when the video is closed: model requests under way are given up. */
3274
+ closing = new AbortController();
2085
3275
  nextId = 1;
2086
3276
  drafts = /* @__PURE__ */ new Map();
2087
3277
  ingest;
@@ -2096,6 +3286,8 @@ var LiveJob = class LiveJob extends EventEmitter {
2096
3286
  this.opts = opts;
2097
3287
  this.deps = deps;
2098
3288
  this.id = id;
3289
+ this.on("cue", () => this.dubber?.onCue());
3290
+ this.on("cues", () => this.dubber?.onCue());
2099
3291
  mkdirSync(join(deps.dir, "hls"), { recursive: true });
2100
3292
  this.polisher = new Polisher({
2101
3293
  llm: deps.llm,
@@ -2105,6 +3297,7 @@ var LiveJob = class LiveJob extends EventEmitter {
2105
3297
  siteText: (a, b) => this.siteText(a, b),
2106
3298
  targets: () => this.targets(),
2107
3299
  still: (at) => this.still(at),
3300
+ signal: this.closing.signal,
2108
3301
  onLine: (cue, src, tr, speaker) => {
2109
3302
  if (!this.cues.includes(cue) || cue.edited) return;
2110
3303
  this.deps.glossary.notice(src);
@@ -2774,39 +3967,62 @@ var LiveJob = class LiveJob extends EventEmitter {
2774
3967
  this.deps.glossary.setTargets(tgtLangs);
2775
3968
  this.persist();
2776
3969
  this.emit("state", this.summary());
3970
+ await this.fill();
3971
+ }
3972
+ filling = false;
3973
+ /**
3974
+ * Give every finished line the translations it lacks (a language just added, or lines a failed
3975
+ * request left behind), in batches. A line that cannot be translated is passed over, so the
3976
+ * lines after it still get theirs; it stops only when the model gives nothing twice in a row.
3977
+ */
3978
+ async fill() {
2777
3979
  if (this.filling) return;
2778
3980
  this.filling = true;
3981
+ const log = (m) => this.deps.log(`[${this.id}] ${m}`);
3982
+ const failed = /* @__PURE__ */ new Set();
3983
+ let empty = 0, filled = 0;
2779
3984
  try {
2780
- for (;;) {
3985
+ while (!this.closing.signal.aborted) {
2781
3986
  const targets = this.targets();
2782
3987
  const handle = this.deps.llm();
2783
3988
  if (!handle || !targets.length) break;
2784
- const todo = this.cues.filter((c) => c.final && c.src && targets.some((t) => !c.tr?.[t])).slice(0, 15);
3989
+ const todo = this.cues.filter((c) => c.final && c.src.trim() && !failed.has(c.id) && targets.some((t) => !c.tr?.[t])).slice(0, 15);
2785
3990
  if (!todo.length) break;
2786
3991
  const got = await translateBatch(handle, targets.filter((t) => todo.some((c) => !c.tr?.[t])), todo.map((c) => ({
2787
3992
  id: c.id,
2788
3993
  src: c.src
2789
- })), this.deps.glossary.forPrompt());
2790
- if (!got.size) break;
3994
+ })), this.deps.glossary.forPrompt(), {
3995
+ signal: this.closing.signal,
3996
+ log
3997
+ });
2791
3998
  for (const c of todo) {
2792
3999
  const tr = got.get(c.id);
2793
- if (!tr) continue;
2794
- c.tr = {
2795
- ...c.tr,
2796
- ...tr
2797
- };
2798
- this.emit("cue", c);
2799
- this.record(c);
4000
+ const add = Object.fromEntries(Object.entries(tr ?? {}).filter(([t]) => !c.tr?.[t]));
4001
+ if (Object.keys(add).length) {
4002
+ c.tr = {
4003
+ ...c.tr,
4004
+ ...add
4005
+ };
4006
+ filled++;
4007
+ this.emit("cue", c);
4008
+ this.record(c);
4009
+ }
4010
+ if (targets.some((t) => !c.tr?.[t])) failed.add(c.id);
4011
+ }
4012
+ empty = got.size ? 0 : empty + 1;
4013
+ if (empty >= 2) {
4014
+ log(`fill translations: the model gives nothing, stopping (${this.cues.filter((c) => c.final && c.src.trim() && targets.some((t) => !c.tr?.[t])).length} lines left)`);
4015
+ break;
2800
4016
  }
2801
4017
  }
4018
+ if (failed.size) log(`fill translations: ${failed.size} lines still without a translation (ids ${[...failed].slice(0, 20).join(", ")}${failed.size > 20 ? "…" : ""})`);
2802
4019
  } catch (err) {
2803
- this.deps.log(`[${this.id}] fill translations: ${err.message}`);
4020
+ if (!this.closing.signal.aborted) log(`fill translations: ${err.message}`);
2804
4021
  } finally {
2805
4022
  this.filling = false;
2806
- this.save();
4023
+ if (filled) this.save();
2807
4024
  }
2808
4025
  }
2809
- filling = false;
2810
4026
  /** Pause subtitles; the next start resumes from here. */
2811
4027
  async pauseSubs() {
2812
4028
  const rd = this.reader;
@@ -2825,11 +4041,15 @@ var LiveJob = class LiveJob extends EventEmitter {
2825
4041
  };
2826
4042
  this.persist();
2827
4043
  this.emit("state", this.summary());
2828
- await this.translating;
2829
- await this.polisher.flush();
2830
- this.save();
2831
- this.persist();
2832
- this.emit("state", this.summary());
4044
+ const settled = (async () => {
4045
+ await this.translating;
4046
+ await this.polisher.flush();
4047
+ if (!this.reader) await this.fill();
4048
+ this.save();
4049
+ this.persist();
4050
+ this.emit("state", this.summary());
4051
+ })().catch((err) => this.deps.log(`[${this.id}] finishing subtitles: ${err.message}`));
4052
+ await Promise.race([settled, new Promise((r) => setTimeout(r, SETTLE.wait))]);
2833
4053
  }
2834
4054
  /**
2835
4055
  * How far the subtitles are complete (media s): audio read minus what the
@@ -2843,6 +4063,8 @@ var LiveJob = class LiveJob extends EventEmitter {
2843
4063
  const waiting = this.cues.find((c) => c.final && targets.some((t) => !c.tr?.[t]) && c.start > ready - 60);
2844
4064
  if (waiting) ready = Math.min(ready, waiting.start);
2845
4065
  }
4066
+ const voice = this.dubber?.waitingFrom(ready - 60);
4067
+ if (voice !== void 0) ready = Math.min(ready, voice);
2846
4068
  return Math.max(0, ready);
2847
4069
  }
2848
4070
  /** Is recognition or translation too far behind the media read so far? (`slack` < 1 waits until it is well caught up.) */
@@ -2916,7 +4138,7 @@ var LiveJob = class LiveJob extends EventEmitter {
2916
4138
  this.translating = this.translating.then(async () => {
2917
4139
  try {
2918
4140
  const ref = this.siteText(cue.start, cue.end).map((x) => `[${x.lang}${x.auto ? ", auto" : ""}] ${x.text}`).join("\n");
2919
- const tr = await this.deps.translator.translate(this.targets(), text, context, this.deps.glossary.forPrompt(), void 0, ref);
4141
+ const tr = await this.deps.translator.translate(this.targets(), text, context, this.deps.glossary.forPrompt(), this.closing.signal, ref, (m) => this.deps.log(`[${this.id}] ${m}`));
2920
4142
  if (tr) {
2921
4143
  cue.tr = {
2922
4144
  ...cue.tr,
@@ -2925,7 +4147,7 @@ var LiveJob = class LiveJob extends EventEmitter {
2925
4147
  this.emit("cue", cue);
2926
4148
  }
2927
4149
  } catch (err) {
2928
- this.deps.log(`[${this.id}] translate: ${err.message}`);
4150
+ if (!this.closing.signal.aborted) this.deps.log(`[${this.id}] translate: ${err.message}`);
2929
4151
  }
2930
4152
  this.pendingTranslations--;
2931
4153
  this.record(cue);
@@ -3069,8 +4291,48 @@ var LiveJob = class LiveJob extends EventEmitter {
3069
4291
  await this.restreamer?.stop();
3070
4292
  await this.stop(reason);
3071
4293
  await this.pauseSubs();
4294
+ this.closing.abort();
4295
+ }
4296
+ dubber;
4297
+ /** The voiceover (made on first use; its clips stay in the video's folder). */
4298
+ get dub() {
4299
+ if (!this.dubber) {
4300
+ const d = new Dubber(this.deps.dir, this.opts.dub ?? this.deps.defaultDub(this.targets()), () => this.cues, () => this.spokenLang(), () => this.opts.mode === "polished", (v, t, l) => this.deps.speak(v, t, l), (v, sec) => this.deps.usage.addAudio(`voice-${this.deps.voiceVendor(v)}`, sec), (m) => this.deps.log(`[${this.id}] ${m}`));
4301
+ d.on("change", (v) => this.emit("dub", v));
4302
+ this.dubber = d;
4303
+ }
4304
+ return this.dubber;
4305
+ }
4306
+ /** Change the voiceover; a push that is running picks it up (restarted where it is when the sound changes shape). */
4307
+ async setDub(patch) {
4308
+ const before = { ...this.dub.settings };
4309
+ this.dub.configure(patch);
4310
+ this.opts.dub = { ...this.dub.settings };
4311
+ this.persist();
4312
+ const r = this.restreamer;
4313
+ if (r?.summary.running && this.restreamOpts && (before.on !== this.dub.settings.on || this.dub.settings.on && before.original !== this.dub.settings.original)) {
4314
+ const targets = r.targets, at = r.summary.at;
4315
+ await r.stop();
4316
+ await this.startRestream(targets, {
4317
+ ...this.restreamOpts,
4318
+ from: at
4319
+ });
4320
+ }
4321
+ return this.dub.view();
4322
+ }
4323
+ /** ffmpeg input and filter that put the voiceover on a cut that decodes from media time `start` (undefined: voiceover off). */
4324
+ async dubForCut(start, to, file, input) {
4325
+ if (!this.opts.dub?.on) return void 0;
4326
+ const d = this.dub;
4327
+ await d.ensure(start, to);
4328
+ writeFileSync(file, wav(d.mix(start, to)));
4329
+ return {
4330
+ args: ["-i", file],
4331
+ filter: dubFilter("0:a:0", `${input}:a`, d.settings.original, "aout")
4332
+ };
3072
4333
  }
3073
4334
  restreamer;
4335
+ restreamOpts;
3074
4336
  get restream() {
3075
4337
  return this.restreamer?.summary;
3076
4338
  }
@@ -3093,9 +4355,19 @@ var LiveJob = class LiveJob extends EventEmitter {
3093
4355
  ...opts,
3094
4356
  from: 0
3095
4357
  };
4358
+ this.restreamOpts = opts;
4359
+ const dub = this.opts.dub?.on ? this.dub : void 0;
3096
4360
  const r = new Restream({
3097
4361
  input: (from) => this.restreamInput(from),
3098
- rowsAt: (t) => this.rowsAt(t, opts.style)
4362
+ rowsAt: (t) => this.rowsAt(t, opts.style),
4363
+ dub: dub && {
4364
+ mix: (a, b, late) => {
4365
+ dub.setFocus("push", b);
4366
+ return dub.mix(a, b, late);
4367
+ },
4368
+ original: dub.settings.original,
4369
+ done: () => dub.setFocus("push", void 0)
4370
+ }
3099
4371
  }, {
3100
4372
  ...opts,
3101
4373
  bitrateK: opts.height === 1080 ? 4500 : 2500
@@ -3258,6 +4530,31 @@ var LiveJob = class LiveJob extends EventEmitter {
3258
4530
  return;
3259
4531
  }
3260
4532
  }
4533
+ /** The original sound of [from, to] as a WAV (24 kHz mono). */
4534
+ async audio(from, to) {
4535
+ const { args, skip } = this.mediaFor(from, to);
4536
+ const { stdout } = await run$1("ffmpeg", [
4537
+ "-v",
4538
+ "error",
4539
+ ...args,
4540
+ ...seekAfter(skip),
4541
+ "-t",
4542
+ String(to - from),
4543
+ "-vn",
4544
+ "-ac",
4545
+ "1",
4546
+ "-ar",
4547
+ "24000",
4548
+ "-f",
4549
+ "wav",
4550
+ "pipe:1"
4551
+ ], {
4552
+ timeout: 12e4,
4553
+ encoding: "buffer",
4554
+ maxBuffer: 64 << 20
4555
+ });
4556
+ return stdout;
4557
+ }
3261
4558
  /** Save one frame at media time `at` (seconds) as JPEG; returns the file path. */
3262
4559
  async frame(at) {
3263
4560
  const dir = join(this.deps.dir, "frames");
@@ -3307,19 +4604,22 @@ var LiveJob = class LiveJob extends EventEmitter {
3307
4604
  maps.push("-map", `${i + 1}:0`);
3308
4605
  });
3309
4606
  }
4607
+ const voice = await this.dubForCut(from - skip, to, join(dir, `.${name}.dub.wav`), 1 + maps.length / 2);
3310
4608
  await run$1("ffmpeg", [
3311
4609
  "-v",
3312
4610
  "error",
3313
4611
  "-y",
3314
4612
  ...args,
3315
4613
  ...subArgs,
4614
+ ...voice?.args ?? [],
4615
+ ...voice ? ["-filter_complex", voice.filter] : [],
3316
4616
  ...seekAfter(skip),
3317
4617
  "-t",
3318
4618
  String(to - from),
3319
4619
  "-map",
3320
4620
  "0:v?",
3321
4621
  "-map",
3322
- "0:a?",
4622
+ voice ? "[aout]" : "0:a?",
3323
4623
  ...maps,
3324
4624
  "-c:v",
3325
4625
  "libx264",
@@ -3391,6 +4691,7 @@ var LiveJob = class LiveJob extends EventEmitter {
3391
4691
  const list = join(tmp, "subs.ffconcat");
3392
4692
  writeFileSync(list, lines.join("\n") + "\n");
3393
4693
  try {
4694
+ const voice = await this.dubForCut(from - skip, to, join(tmp, "dub.wav"), 2);
3394
4695
  await run$1("ffmpeg", [
3395
4696
  "-v",
3396
4697
  "error",
@@ -3402,15 +4703,16 @@ var LiveJob = class LiveJob extends EventEmitter {
3402
4703
  "0",
3403
4704
  "-i",
3404
4705
  list,
4706
+ ...voice?.args ?? [],
3405
4707
  ...seekAfter(skip),
3406
4708
  "-t",
3407
4709
  String(to - from),
3408
4710
  "-filter_complex",
3409
- `[0:v]scale=${w}:${h}:force_original_aspect_ratio=decrease,pad=${w}:${h}:(ow-iw)/2:(oh-ih)/2,setsar=1,setpts=PTS-STARTPTS[v];[1:v]format=rgba,setpts=PTS-STARTPTS[s];[v][s]overlay=0:0:eof_action=repeat,format=yuv420p[o]`,
4711
+ `[0:v]scale=${w}:${h}:force_original_aspect_ratio=decrease,pad=${w}:${h}:(ow-iw)/2:(oh-ih)/2,setsar=1,setpts=PTS-STARTPTS[v];[1:v]format=rgba,setpts=PTS-STARTPTS[s];[v][s]overlay=0:0:eof_action=repeat,format=yuv420p[o]${voice ? `;${voice.filter}` : ""}`,
3410
4712
  "-map",
3411
4713
  "[o]",
3412
4714
  "-map",
3413
- "0:a?",
4715
+ voice ? "[aout]" : "0:a?",
3414
4716
  "-c:v",
3415
4717
  "libx264",
3416
4718
  "-preset",
@@ -6209,6 +7511,388 @@ function updateChecker(log, manager) {
6209
7511
  stop: () => clearInterval(timer)
6210
7512
  };
6211
7513
  }
7514
+ // src/voiceroutes.ts
7515
+ /** What to read aloud for a voice sample (about ten seconds, the way people talk). */
7516
+ const READ_ALOUD = {
7517
+ zh: "今天想跟你聊聊最近发生的几件事。有些挺有意思,有些让人意外。我会慢慢说,你有什么想法,随时打断我。",
7518
+ en: "Today I'd like to talk about a few things that happened recently. Some are interesting, some are surprising. I'll take it slowly, so feel free to jump in any time."
7519
+ };
7520
+ /** Sentences to try a voice with, per language. */
7521
+ const TRY_TEXT = {
7522
+ zh: "大家好,这是视频的中文配音,听起来怎么样?",
7523
+ en: "Hi everyone, this is the English voiceover. How does it sound?",
7524
+ ja: "こんにちは、これは日本語の吹き替えです。いかがでしょうか。",
7525
+ ko: "안녕하세요, 한국어 더빙입니다. 어떻게 들리나요?",
7526
+ fr: "Bonjour à tous, voici le doublage en français. Qu'en pensez-vous ?",
7527
+ de: "Hallo zusammen, das ist die deutsche Synchronisation. Wie klingt sie?",
7528
+ es: "Hola a todos, este es el doblaje en español. ¿Qué tal suena?"
7529
+ };
7530
+ /** The service a voice belongs to. */
7531
+ function vendorOf(library, voice) {
7532
+ const [kind, id = ""] = voice.split(/:(.*)/s);
7533
+ if (kind === "mine") return library.get(id)?.vendor;
7534
+ return VOICE_VENDORS.includes(kind) ? kind : void 0;
7535
+ }
7536
+ /**
7537
+ * Voiceover and voice routes (under /live/api/): the video's voiceover
7538
+ * settings and clips, the voices on offer, trying one, and making one from
7539
+ * a recording (the microphone, a file, or someone in the video).
7540
+ */
7541
+ function voiceRoutes(d) {
7542
+ const tmpDir = join(d.dataDir, "voices", "tmp");
7543
+ mkdirSync(tmpDir, { recursive: true });
7544
+ const tmpPath = (id) => join(tmpDir, `${id.replace(/[^\w-]/g, "")}.wav`);
7545
+ const ready = (v) => !!d.speakDeps.key(v);
7546
+ let rec = void 0;
7547
+ /** Turn any audio into a sample: 24 kHz mono WAV, at most 30 s, loudness evened out. */
7548
+ const toSample = async (input, id) => {
7549
+ const pcm = await ffmpegBuffer([
7550
+ "-i",
7551
+ "pipe:0",
7552
+ "-t",
7553
+ "30",
7554
+ "-af",
7555
+ "highpass=f=70,loudnorm=I=-18:TP=-2",
7556
+ "-f",
7557
+ "s16le",
7558
+ "-ar",
7559
+ String(SAMPLE_RATE),
7560
+ "-ac",
7561
+ "1",
7562
+ "pipe:1"
7563
+ ], input);
7564
+ writeFileSync(tmpPath(id), wav(pcm));
7565
+ return pcm.length / (SAMPLE_RATE * 2);
7566
+ };
7567
+ const newId = () => `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 6)}`;
7568
+ return async (route, req, res, url) => {
7569
+ const { send, readJson } = d;
7570
+ if (route === "dub" && req.method === "POST") {
7571
+ const b = await readJson(req);
7572
+ const job = d.loadJob(String(b.jobId));
7573
+ if (!job) {
7574
+ send(res, 404, { error: "no such job" });
7575
+ return true;
7576
+ }
7577
+ const patch = {};
7578
+ if (typeof b.on === "boolean") patch.on = b.on;
7579
+ if (typeof b.lang === "string") patch.lang = b.lang;
7580
+ if (typeof b.voice === "string" && vendorOf(d.library, b.voice)) patch.voice = b.voice;
7581
+ if (b.original === "duck" || b.original === "off") patch.original = b.original;
7582
+ const view = await job.setDub(patch);
7583
+ d.saveDubPrefs({
7584
+ voice: view.voice,
7585
+ original: view.original,
7586
+ lang: view.lang
7587
+ });
7588
+ if (typeof b.at === "number") job.dub.setFocus("page", b.at);
7589
+ send(res, 200, view);
7590
+ return true;
7591
+ }
7592
+ if (route === "dub/focus" && req.method === "POST") {
7593
+ const b = await readJson(req);
7594
+ const job = d.loadJob(String(b.jobId));
7595
+ if (job && job.opts.dub?.on) job.dub.setFocus("page", typeof b.at === "number" ? b.at : void 0);
7596
+ send(res, 200, { ok: true });
7597
+ return true;
7598
+ }
7599
+ if (route === "dub/clip") {
7600
+ const file = d.loadJob(url.searchParams.get("jobId") ?? "")?.dub.clipFile(Number(url.searchParams.get("id")));
7601
+ if (!file) {
7602
+ res.writeHead(404);
7603
+ res.end();
7604
+ return true;
7605
+ }
7606
+ res.writeHead(200, {
7607
+ "content-type": "audio/wav",
7608
+ "cache-control": "max-age=31536000, immutable",
7609
+ "content-length": String(statSync(file).size)
7610
+ });
7611
+ createReadStream(file).pipe(res);
7612
+ return true;
7613
+ }
7614
+ if (route === "voices" && req.method !== "POST") {
7615
+ const ui = url.searchParams.get("ui") === "en" ? "en" : "zh";
7616
+ send(res, 200, {
7617
+ vendors: VOICE_VENDORS.filter(ready),
7618
+ presets: PRESETS.filter((p) => ready(p.vendor)).map((p) => ({
7619
+ id: `${p.vendor}:${p.id}`,
7620
+ vendor: p.vendor,
7621
+ label: p[ui]
7622
+ })),
7623
+ mine: d.library.list().map((v) => ({
7624
+ id: `mine:${v.id}`,
7625
+ vendor: v.vendor,
7626
+ label: v.name,
7627
+ ready: ready(v.vendor),
7628
+ createdAt: v.createdAt
7629
+ })),
7630
+ clone: CLONE_VENDORS.filter(ready),
7631
+ readAloud: READ_ALOUD[ui] ?? READ_ALOUD.en,
7632
+ defaults: d.dubPrefs()
7633
+ });
7634
+ return true;
7635
+ }
7636
+ if (route === "voices/try" && req.method === "POST") {
7637
+ const b = await readJson(req);
7638
+ const lang = String(b.lang ?? "zh");
7639
+ const text = typeof b.text === "string" && b.text.trim() ? b.text.slice(0, 200) : TRY_TEXT[lang.split("-")[0]] ?? TRY_TEXT.en;
7640
+ try {
7641
+ const pcm = await speak(d.speakDeps, String(b.voice), text, lang);
7642
+ res.writeHead(200, {
7643
+ "content-type": "audio/wav",
7644
+ "cache-control": "no-store"
7645
+ });
7646
+ res.end(wav(pcm));
7647
+ } catch (err) {
7648
+ send(res, 502, { error: err.message });
7649
+ }
7650
+ return true;
7651
+ }
7652
+ if (route === "voices/record") {
7653
+ if (req.method !== "POST") {
7654
+ send(res, 200, rec ? {
7655
+ state: rec.done ? "stopping" : "recording",
7656
+ id: rec.id,
7657
+ sec: (Date.now() - rec.startedAt) / 1e3,
7658
+ level: rec.level,
7659
+ error: rec.error
7660
+ } : { state: "idle" });
7661
+ return true;
7662
+ }
7663
+ const b = await readJson(req);
7664
+ if (b.action === "start") {
7665
+ if (rec) {
7666
+ rec.proc.kill("SIGKILL");
7667
+ rec = void 0;
7668
+ }
7669
+ const mics = (await listDevices()).filter((x) => x.kind === "mic");
7670
+ const mic = mics.find((m) => m.id === b.mic) ?? mics.find((m) => m.preferred) ?? mics[0];
7671
+ if (!mic) {
7672
+ send(res, 400, { error: "no microphone" });
7673
+ return true;
7674
+ }
7675
+ const id = newId();
7676
+ const input = deviceInput({ audio: mic.id }, []);
7677
+ const proc = spawn("ffmpeg", [
7678
+ "-hide_banner",
7679
+ "-loglevel",
7680
+ "error",
7681
+ "-y",
7682
+ ...input.args,
7683
+ "-t",
7684
+ "30",
7685
+ "-map",
7686
+ input.audio,
7687
+ "-ac",
7688
+ "1",
7689
+ "-ar",
7690
+ String(SAMPLE_RATE),
7691
+ "-c:a",
7692
+ "pcm_s16le",
7693
+ join(tmpDir, `${id}.raw.wav`),
7694
+ "-map",
7695
+ input.audio,
7696
+ "-ac",
7697
+ "1",
7698
+ "-ar",
7699
+ "8000",
7700
+ "-f",
7701
+ "s16le",
7702
+ "pipe:1"
7703
+ ], { stdio: [
7704
+ "pipe",
7705
+ "pipe",
7706
+ "pipe"
7707
+ ] });
7708
+ const r = {
7709
+ proc,
7710
+ id,
7711
+ startedAt: Date.now(),
7712
+ level: 0
7713
+ };
7714
+ rec = r;
7715
+ let err = "";
7716
+ proc.stderr.on("data", (x) => {
7717
+ err = (err + x).slice(-300);
7718
+ });
7719
+ proc.stdout.on("data", (pcm) => {
7720
+ let peak = 0;
7721
+ for (let i = 0; i + 1 < pcm.length; i += 2) peak = Math.max(peak, Math.abs(pcm.readInt16LE(i)));
7722
+ r.level = Math.max(peak / 32768, r.level * .7);
7723
+ });
7724
+ proc.on("exit", (code) => {
7725
+ if (code && !r.done) r.error = err.trim().split("\n").pop() || "the microphone could not be opened";
7726
+ });
7727
+ send(res, 200, {
7728
+ state: "recording",
7729
+ id,
7730
+ mic: mic.name
7731
+ });
7732
+ return true;
7733
+ }
7734
+ if (b.action === "stop" || b.action === "cancel") {
7735
+ const r = rec;
7736
+ if (!r) {
7737
+ send(res, 200, { state: "idle" });
7738
+ return true;
7739
+ }
7740
+ r.done = new Promise((ok) => {
7741
+ r.proc.once("exit", () => ok());
7742
+ setTimeout(ok, 3e3);
7743
+ });
7744
+ try {
7745
+ r.proc.stdin.write("q");
7746
+ } catch {}
7747
+ await r.done;
7748
+ rec = void 0;
7749
+ const raw = join(tmpDir, `${r.id}.raw.wav`);
7750
+ if (b.action === "cancel" || !existsSync(raw)) {
7751
+ rmSync(raw, { force: true });
7752
+ send(res, 200, {
7753
+ state: "idle",
7754
+ error: r.error
7755
+ });
7756
+ return true;
7757
+ }
7758
+ try {
7759
+ const dur = await toSample(readFileSync(raw), r.id);
7760
+ rmSync(raw, { force: true });
7761
+ send(res, 200, {
7762
+ state: "done",
7763
+ sample: r.id,
7764
+ dur
7765
+ });
7766
+ } catch (err) {
7767
+ send(res, 500, { error: err.message });
7768
+ }
7769
+ return true;
7770
+ }
7771
+ send(res, 400, { error: "unknown action" });
7772
+ return true;
7773
+ }
7774
+ if (route === "voices/upload" && req.method === "POST") {
7775
+ const chunks = [];
7776
+ for await (const c of req) chunks.push(c);
7777
+ const id = newId();
7778
+ try {
7779
+ send(res, 200, {
7780
+ sample: id,
7781
+ dur: await toSample(Buffer.concat(chunks), id)
7782
+ });
7783
+ } catch (err) {
7784
+ send(res, 400, { error: err.message });
7785
+ }
7786
+ return true;
7787
+ }
7788
+ if (route === "voices/fromVideo" && req.method === "POST") {
7789
+ const b = await readJson(req);
7790
+ const job = d.loadJob(String(b.jobId));
7791
+ if (!job) {
7792
+ send(res, 404, { error: "no such job" });
7793
+ return true;
7794
+ }
7795
+ const at = typeof b.at === "number" ? b.at : 0;
7796
+ const lines = job.cues.filter((c) => c.final && c.src.trim());
7797
+ let best;
7798
+ for (let i = 0; i < lines.length; i++) {
7799
+ const run = [lines[i]];
7800
+ for (let k = i + 1; k < lines.length && lines[k].end - run[0].start <= 20; k++) {
7801
+ const c = lines[k];
7802
+ if (c.start - run.at(-1).end > .8 || c.speaker && run[0].speaker && c.speaker !== run[0].speaker) break;
7803
+ run.push(c);
7804
+ }
7805
+ const from = run[0].start, to = run.at(-1).end;
7806
+ if (to - from < 7) continue;
7807
+ const score = Math.abs(from - at) + Math.abs(to - from - 12);
7808
+ if (!best || score < best.score) best = {
7809
+ from,
7810
+ to,
7811
+ text: run.map((c) => c.src.trim()).join(" "),
7812
+ score
7813
+ };
7814
+ }
7815
+ if (!best) {
7816
+ send(res, 400, { error: "no stretch of talk long enough yet (subtitles needed)" });
7817
+ return true;
7818
+ }
7819
+ const id = newId();
7820
+ try {
7821
+ send(res, 200, {
7822
+ sample: id,
7823
+ dur: await toSample(await job.audio(best.from, best.to), id),
7824
+ text: best.text,
7825
+ from: best.from,
7826
+ to: best.to
7827
+ });
7828
+ } catch (err) {
7829
+ send(res, 500, { error: err.message });
7830
+ }
7831
+ return true;
7832
+ }
7833
+ if (route === "voices/sample") {
7834
+ const s = url.searchParams.get("sample"), id = url.searchParams.get("id");
7835
+ const file = s ? tmpPath(s) : id ? d.library.samplePath(id.replace(/^mine:/, "")) : "";
7836
+ if (!file || !existsSync(file)) {
7837
+ res.writeHead(404);
7838
+ res.end();
7839
+ return true;
7840
+ }
7841
+ res.writeHead(200, {
7842
+ "content-type": "audio/wav",
7843
+ "cache-control": "no-store"
7844
+ });
7845
+ createReadStream(file).pipe(res);
7846
+ return true;
7847
+ }
7848
+ if (route === "voices/save" && req.method === "POST") {
7849
+ const b = await readJson(req);
7850
+ const file = tmpPath(String(b.sample ?? ""));
7851
+ if (!existsSync(file)) {
7852
+ send(res, 404, { error: "the recording is gone, record again" });
7853
+ return true;
7854
+ }
7855
+ const vendor = CLONE_VENDORS.includes(String(b.vendor)) ? b.vendor : CLONE_VENDORS.find(ready);
7856
+ if (!vendor || !ready(vendor)) {
7857
+ send(res, 400, { error: "no service that can learn a voice (SiliconFlow or ElevenLabs key needed)" });
7858
+ return true;
7859
+ }
7860
+ const sample = readFileSync(file);
7861
+ const name = String(b.name ?? "").trim().slice(0, 40) || "My voice";
7862
+ const text = String(b.text ?? "").trim();
7863
+ try {
7864
+ const remote = await cloneVoice(d.speakDeps, vendor, name, sample, text);
7865
+ const id = newId();
7866
+ d.library.add({
7867
+ id,
7868
+ name,
7869
+ vendor,
7870
+ remote,
7871
+ text,
7872
+ createdAt: Date.now()
7873
+ }, sample);
7874
+ rmSync(file, { force: true });
7875
+ send(res, 200, { id: `mine:${id}` });
7876
+ } catch (err) {
7877
+ send(res, 502, { error: err.message });
7878
+ }
7879
+ return true;
7880
+ }
7881
+ if (route === "voices/remove" && req.method === "POST") {
7882
+ const b = await readJson(req);
7883
+ d.library.remove(String(b.id ?? "").replace(/^mine:/, ""));
7884
+ send(res, 200, { ok: true });
7885
+ return true;
7886
+ }
7887
+ if (route === "voices/rename" && req.method === "POST") {
7888
+ const b = await readJson(req);
7889
+ d.library.rename(String(b.id ?? "").replace(/^mine:/, ""), String(b.name ?? "").trim().slice(0, 40));
7890
+ send(res, 200, { ok: true });
7891
+ return true;
7892
+ }
7893
+ return false;
7894
+ };
7895
+ }
6212
7896
  // src/index.ts
6213
7897
  /**
6214
7898
  * dsh-vidi host half: live stream jobs, local/online speech providers,
@@ -6433,6 +8117,25 @@ function apply(ctx, config) {
6433
8117
  provider: defaultEngine(),
6434
8118
  mode: "polished"
6435
8119
  };
8120
+ const library = new VoiceLibrary(join(config.dataDir, "voices"));
8121
+ const speakDeps = {
8122
+ key: (v) => keys.get(v),
8123
+ base: (v) => keys.bases.get(v),
8124
+ mine: (id) => library.get(id),
8125
+ log
8126
+ };
8127
+ /** A voice to start with: the last one used, else one from the first service with a key. */
8128
+ const defaultDub = (langs) => {
8129
+ const p = prefs().dub ?? {};
8130
+ const fallback = keys.get("openai") ? "openai:marin" : keys.get("siliconflow") ? "siliconflow:anna" : keys.get("gemini") ? "gemini:Kore" : keys.get("elevenlabs") ? "elevenlabs:EXAVITQu4vr4xnSDxMaL" : "openai:marin";
8131
+ const voice = p.voice && vendorOf(library, p.voice) && keys.get(vendorOf(library, p.voice)) ? p.voice : fallback;
8132
+ return {
8133
+ on: false,
8134
+ lang: p.lang && langs.includes(p.lang) ? p.lang : langs[0] ?? p.lang ?? config.defaultTargetLang,
8135
+ voice,
8136
+ original: p.original ?? "duck"
8137
+ };
8138
+ };
6436
8139
  /** Browser notifications: "show this video" for a session (tools, /video). */
6437
8140
  const bus = new EventEmitter();
6438
8141
  bus.setMaxListeners(100);
@@ -6592,7 +8295,10 @@ function apply(ctx, config) {
6592
8295
  srtPort: config.srtPort,
6593
8296
  key: waiting ? pushKey() : void 0
6594
8297
  },
6595
- cookies: () => prefs().cookies
8298
+ cookies: () => prefs().cookies,
8299
+ speak: (voice, text, lang) => speak(speakDeps, voice, text, lang),
8300
+ voiceVendor: (voice) => vendorOf(library, voice) ?? "voice",
8301
+ defaultDub
6596
8302
  };
6597
8303
  }
6598
8304
  /** The conversation's video: in memory, or the latest one on disk (after a restart). */
@@ -6786,11 +8492,26 @@ function apply(ctx, config) {
6786
8492
  job.start();
6787
8493
  return job;
6788
8494
  }
8495
+ const voices = voiceRoutes({
8496
+ dataDir: config.dataDir,
8497
+ speakDeps,
8498
+ library,
8499
+ loadJob,
8500
+ readJson,
8501
+ send,
8502
+ log,
8503
+ dubPrefs: () => prefs().dub ?? {},
8504
+ saveDubPrefs: (p) => savePrefs({ dub: {
8505
+ ...prefs().dub,
8506
+ ...p
8507
+ } })
8508
+ });
6789
8509
  const api = async (req, res) => {
6790
8510
  if (!trusted(req)) return send(res, 403, { error: "forbidden host" });
6791
8511
  const url = new URL(req.url ?? "/", "http://127.0.0.1");
6792
8512
  const route = url.pathname.replace(/^\/live\/api\/?/, "");
6793
8513
  try {
8514
+ if ((route === "dub" || route.startsWith("dub/") || route === "voices" || route.startsWith("voices/")) && await voices(route, req, res, url)) return;
6794
8515
  if (route === "overlay") return send(res, 200, {
6795
8516
  url: `http://127.0.0.1:${config.overlayPort}/overlay`,
6796
8517
  screen: `http://127.0.0.1:${config.overlayPort}/screen`,
@@ -7309,6 +9030,9 @@ function apply(ctx, config) {
7309
9030
  write("state", job.summary());
7310
9031
  write("cues", job.cues);
7311
9032
  write("glossary", job.terms);
9033
+ write("dub", job.dub.view());
9034
+ const onDub = (v) => write("dub", v);
9035
+ job.on("dub", onDub);
7312
9036
  const onState = (s) => write("state", s);
7313
9037
  const onCue = (c) => write("cue", c);
7314
9038
  const onTerms = (t) => write("glossary", t);
@@ -7320,6 +9044,7 @@ function apply(ctx, config) {
7320
9044
  const ping = setInterval(() => res.write(": ping\n\n"), 15e3);
7321
9045
  req.on("close", () => {
7322
9046
  clearInterval(ping);
9047
+ job.off("dub", onDub);
7323
9048
  job.off("state", onState);
7324
9049
  job.off("cue", onCue);
7325
9050
  job.off("glossary", onTerms);