@softure-ai/marketing-kit 0.15.0 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. package/CHANGELOG.md +91 -0
  2. package/README.md +138 -16
  3. package/dist/config/schema.d.ts +2 -1
  4. package/dist/config/schema.d.ts.map +1 -1
  5. package/dist/config/schema.js +7 -0
  6. package/dist/config/schema.js.map +1 -1
  7. package/dist/footage/clip-sound.d.ts +58 -0
  8. package/dist/footage/clip-sound.d.ts.map +1 -0
  9. package/dist/footage/clip-sound.js +103 -0
  10. package/dist/footage/clip-sound.js.map +1 -0
  11. package/dist/footage/declick.d.ts +1 -1
  12. package/dist/footage/declick.d.ts.map +1 -1
  13. package/dist/footage/declick.js +3 -4
  14. package/dist/footage/declick.js.map +1 -1
  15. package/dist/footage/draft.d.ts +19 -5
  16. package/dist/footage/draft.d.ts.map +1 -1
  17. package/dist/footage/draft.js +92 -22
  18. package/dist/footage/draft.js.map +1 -1
  19. package/dist/footage/elevenlabs.d.ts +5 -0
  20. package/dist/footage/elevenlabs.d.ts.map +1 -1
  21. package/dist/footage/elevenlabs.js +10 -3
  22. package/dist/footage/elevenlabs.js.map +1 -1
  23. package/dist/footage/failure.d.ts +12 -1
  24. package/dist/footage/failure.d.ts.map +1 -1
  25. package/dist/footage/failure.js +10 -1
  26. package/dist/footage/failure.js.map +1 -1
  27. package/dist/footage/fake.d.ts +10 -1
  28. package/dist/footage/fake.d.ts.map +1 -1
  29. package/dist/footage/fake.js +19 -4
  30. package/dist/footage/fake.js.map +1 -1
  31. package/dist/footage/hyperframes-capture.d.ts +30 -0
  32. package/dist/footage/hyperframes-capture.d.ts.map +1 -0
  33. package/dist/footage/hyperframes-capture.js +80 -0
  34. package/dist/footage/hyperframes-capture.js.map +1 -0
  35. package/dist/footage/isolate-ranges.d.ts +34 -6
  36. package/dist/footage/isolate-ranges.d.ts.map +1 -1
  37. package/dist/footage/isolate-ranges.js +90 -23
  38. package/dist/footage/isolate-ranges.js.map +1 -1
  39. package/dist/footage/loudness.d.ts +16 -2
  40. package/dist/footage/loudness.d.ts.map +1 -1
  41. package/dist/footage/loudness.js +10 -7
  42. package/dist/footage/loudness.js.map +1 -1
  43. package/dist/footage/paid.d.ts +12 -2
  44. package/dist/footage/paid.d.ts.map +1 -1
  45. package/dist/footage/paid.js +8 -3
  46. package/dist/footage/paid.js.map +1 -1
  47. package/dist/footage/pipeline.d.ts +14 -2
  48. package/dist/footage/pipeline.d.ts.map +1 -1
  49. package/dist/footage/pipeline.js +55 -46
  50. package/dist/footage/pipeline.js.map +1 -1
  51. package/dist/footage/providers.d.ts +5 -0
  52. package/dist/footage/providers.d.ts.map +1 -1
  53. package/dist/footage/providers.js.map +1 -1
  54. package/dist/footage/render.d.ts +1 -1
  55. package/dist/footage/render.d.ts.map +1 -1
  56. package/dist/footage/render.js +4 -3
  57. package/dist/footage/render.js.map +1 -1
  58. package/dist/footage/speech-ranges.d.ts +79 -11
  59. package/dist/footage/speech-ranges.d.ts.map +1 -1
  60. package/dist/footage/speech-ranges.js +137 -17
  61. package/dist/footage/speech-ranges.js.map +1 -1
  62. package/dist/index.d.ts +9 -6
  63. package/dist/index.d.ts.map +1 -1
  64. package/dist/index.js +8 -5
  65. package/dist/index.js.map +1 -1
  66. package/dist/og/index.d.ts +2 -0
  67. package/dist/og/index.d.ts.map +1 -1
  68. package/dist/og/index.js +2 -0
  69. package/dist/og/index.js.map +1 -1
  70. package/dist/og/photo-cover.d.ts +74 -0
  71. package/dist/og/photo-cover.d.ts.map +1 -0
  72. package/dist/og/photo-cover.js +357 -0
  73. package/dist/og/photo-cover.js.map +1 -0
  74. package/dist/og/photo.d.ts +51 -0
  75. package/dist/og/photo.d.ts.map +1 -0
  76. package/dist/og/photo.js +154 -0
  77. package/dist/og/photo.js.map +1 -0
  78. package/dist/voice/provider-failure.d.ts +41 -0
  79. package/dist/voice/provider-failure.d.ts.map +1 -0
  80. package/dist/voice/provider-failure.js +68 -0
  81. package/dist/voice/provider-failure.js.map +1 -0
  82. package/dist/voice/provider.d.ts +7 -1
  83. package/dist/voice/provider.d.ts.map +1 -1
  84. package/dist/voice/provider.js +1 -1
  85. package/dist/voice/provider.js.map +1 -1
  86. package/package.json +2 -1
  87. package/schema/marketing.schema.json +6 -0
  88. package/src/config/schema.ts +7 -0
  89. package/src/footage/clip-sound.ts +124 -0
  90. package/src/footage/declick.ts +3 -4
  91. package/src/footage/draft.ts +123 -22
  92. package/src/footage/elevenlabs.ts +11 -3
  93. package/src/footage/failure.ts +14 -1
  94. package/src/footage/fake.ts +28 -4
  95. package/src/footage/hyperframes-capture.ts +100 -0
  96. package/src/footage/isolate-ranges.ts +124 -29
  97. package/src/footage/loudness.ts +17 -8
  98. package/src/footage/paid.ts +25 -3
  99. package/src/footage/pipeline.ts +68 -47
  100. package/src/footage/providers.ts +5 -0
  101. package/src/footage/render.ts +4 -3
  102. package/src/footage/speech-ranges.ts +184 -19
  103. package/src/index.ts +49 -1
  104. package/src/og/index.ts +28 -0
  105. package/src/og/photo-cover.ts +442 -0
  106. package/src/og/photo.ts +177 -0
  107. package/src/voice/provider-failure.ts +98 -0
  108. package/src/voice/provider.ts +7 -2
@@ -12,18 +12,35 @@ import { chooseFootageDraftFont } from "./draft-font.js";
12
12
  import { getEdlDuration, getSegmentStarts, remapWords, type Edl } from "./edl.js";
13
13
  import { declickSpeech, describeDeclick, type DeclickReport } from "./declick.js";
14
14
  import { getDeclickLevel, getFootageEdit, type FootageFilm } from "./film.js";
15
+ import { buildFilmMixArgs } from "./film-mix.js";
16
+ import { DEFAULT_LOUDNESS, formatDb, getLoudnessVerdict, measureLoudness, normalizeLoudness, type MixLoudnessTarget } from "./loudness.js";
15
17
  import { duckingKeyframes, getDuckingOptions } from "./music.js";
16
18
  import { FOOTAGE_FPS, getThreadArgs, type Frame } from "./normalize.js";
17
- import { findFootageMusic, getFfmpegReporting, getFrame, type FootageBrand, type FootageResult, type FootageRunOptions, type PreparedClip, type TranscribeOutcome } from "./pipeline.js";
19
+ import {
20
+ findFootageMusic,
21
+ getFfmpegReporting,
22
+ getFrame,
23
+ getSpeechChainFiles,
24
+ mixFootage,
25
+ type FinishedLoudness,
26
+ type FinishedMix,
27
+ type FootageBrand,
28
+ type FootageResult,
29
+ type FootageRunOptions,
30
+ type PreparedClip,
31
+ type PreparedMix,
32
+ type TranscribeOutcome,
33
+ } from "./pipeline.js";
18
34
  import { runFfmpeg, withFfmpegFailureReporting } from "./process.js";
19
35
  import { buildMusicArgs, buildSpeechArgs, getSpeechCutOptions } from "./render.js";
20
36
 
21
37
  /**
22
38
  * A fast draft of a footage film, ffmpeg only (no hyperframes, no browser): the cut at a small width with the film's
23
39
  * captions burned in (its caption style, in the brand's body font and caption colours), its zoom cuts (eased as in the
24
- * render), cross-fades and Ken Burns b-roll drawn as the full render draws them, the speech track and the ducked music
25
- * when it costs nothing. No loudness pass, end card, cover or sound effects. What an app shows the owner before the paid
26
- * render.
40
+ * render), cross-fades and Ken Burns b-roll drawn as the full render draws them, and the render's sound chain: the
41
+ * speech track's stages (cut, declick, loudness pass), the ducked music when it costs nothing, and the mix brought to
42
+ * `mixLoudness` as the render brings it (issue #1143). No end card, cover or sound effects. What an app shows the owner
43
+ * before the paid render.
27
44
  */
28
45
 
29
46
  /** The draft's default width: 540 makes a 540×960 draft of a 9:16 film. */
@@ -241,6 +258,11 @@ export interface FootageDraftArgsInput {
241
258
  speech: string;
242
259
  /** The ducked music track (`buildMusicArgs`, as long as the cut), mixed under the speech; null or absent for none. */
243
260
  music?: string | null;
261
+ /**
262
+ * The film's mix, already brought to its target and encoded to AAC (`mixFootage`): the draft copies it as its sound
263
+ * and reads neither `speech` nor `music`. Null or absent: the draft mixes `speech` and `music` and encodes them.
264
+ */
265
+ mix?: string | null;
244
266
  /** The film's full frame (the normalized clips' size). */
245
267
  frame: Frame;
246
268
  /** The draft's width; default 540. */
@@ -330,7 +352,8 @@ function buildPictureFilters(plan: DraftSegmentPlan, draft: Frame): string[] {
330
352
  /**
331
353
  * The ffmpeg arguments of the draft: each segment read from its own seek into its clip, zoomed and scaled to the
332
354
  * draft's width, joined by `xfade` at `fade` cuts and `concat` at the others, the captions burned in (`ass`), the
333
- * speech track under it; H.264 yuv420p and AAC with fast start, exactly the cut's length.
355
+ * film's mix copied under it (or the speech and music tracks mixed and encoded to AAC); H.264 yuv420p with fast start,
356
+ * exactly the cut's length.
334
357
  */
335
358
  export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
336
359
  const { edl } = input;
@@ -378,10 +401,13 @@ export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
378
401
  } else {
379
402
  parts.push(`[${joined}]null[draft]`);
380
403
  }
381
- const speechInput = edl.segments.length;
382
- const music = input.music ?? null;
383
- if (music !== null) parts.push(`[${speechInput}:a][${speechInput + 1}:a]amix=inputs=2:duration=first:normalize=0[sound]`);
404
+ const soundInput = edl.segments.length;
405
+ const mix = input.mix ?? null;
406
+ const music = mix === null ? (input.music ?? null) : null;
407
+ if (music !== null) parts.push(`[${soundInput}:a][${soundInput + 1}:a]amix=inputs=2:duration=first:normalize=0[sound]`);
384
408
  const seconds = getEdlDuration(edl);
409
+ // A prepared mix is copied: it was measured as encoded, and a second encode would move its loudness and true peak.
410
+ const audioCodec = mix === null ? ["-c:a", "aac", "-b:a", DRAFT_AUDIO_BITRATE] : ["-c:a", "copy"];
385
411
  return [
386
412
  "-v",
387
413
  "error",
@@ -389,14 +415,14 @@ export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
389
415
  ...threads.global,
390
416
  ...inputs,
391
417
  "-i",
392
- input.speech,
418
+ mix ?? input.speech,
393
419
  ...(music === null ? [] : ["-i", music]),
394
420
  "-filter_complex",
395
421
  parts.join(";"),
396
422
  "-map",
397
423
  "[draft]",
398
424
  "-map",
399
- music === null ? `${speechInput}:a` : "[sound]",
425
+ music === null ? `${soundInput}:a` : "[sound]",
400
426
  "-t",
401
427
  String(seconds),
402
428
  "-c:v",
@@ -407,10 +433,7 @@ export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
407
433
  String(DRAFT_CRF),
408
434
  "-pix_fmt",
409
435
  "yuv420p",
410
- "-c:a",
411
- "aac",
412
- "-b:a",
413
- DRAFT_AUDIO_BITRATE,
436
+ ...audioCodec,
414
437
  "-movflags",
415
438
  "+faststart",
416
439
  ...threads.output,
@@ -453,6 +476,13 @@ export interface FootageDraft {
453
476
  bytes: number;
454
477
  /** What the speech track's declick stage did, as the render does it; null when the film's declick is off. */
455
478
  declick: DeclickReport | null;
479
+ /**
480
+ * The draft's sound as measured, as the render measures its finished film (`warning` on a near miss inside
481
+ * `mixLoudness.warnTolerance`); null for a silent draft.
482
+ */
483
+ loudness: FinishedLoudness | null;
484
+ /** What the mix stage did, as in the render; null when it did not run (`mixLoudness: null`, a silent mix). */
485
+ mix: FinishedMix | null;
456
486
  }
457
487
 
458
488
  /** The draft's music track: a file to duck, or why there is none. */
@@ -505,15 +535,24 @@ async function runRenderFootageDraft(
505
535
  rmSync(draftDir, { recursive: true, force: true });
506
536
  mkdirSync(draftDir, { recursive: true });
507
537
  const declickLevel = getDeclickLevel(film);
508
- const items = ["speech", ...(declickLevel === "off" ? [] : ["declick"]), ...(music.value.kind === "file" ? ["music"] : []), "film"];
538
+ const loudness = film.loudness === undefined ? DEFAULT_LOUDNESS : film.loudness;
539
+ const mixLoudness = film.mixLoudness === undefined ? DEFAULT_LOUDNESS : film.mixLoudness;
540
+ const hasMusic = music.value.kind === "file";
541
+ const items = [
542
+ "speech",
543
+ ...(declickLevel === "off" ? [] : ["declick"]),
544
+ ...(loudness === null ? [] : ["loudness"]),
545
+ ...(hasMusic ? ["music"] : []),
546
+ ...(mixLoudness === null ? [] : ["mix"]),
547
+ "film",
548
+ ];
509
549
  const report = (item: string, done: number): void => {
510
550
  const itemIndex = items.indexOf(item) + 1;
511
551
  options.onProgress?.({ step: "draft", item, itemIndex, itemCount: items.length, done: Math.min(seconds * items.length, (itemIndex - 1) * seconds + done), total: seconds * items.length });
512
552
  };
513
553
  const clipFiles = Object.fromEntries(clips.map((clip) => [clip.clip.id, resolve(clip.file)]));
514
- const cutSpeech = join(draftDir, "speech.mp3");
515
- // The declicked track stays WAV: the draft's one encode is its only lossy step after the cut.
516
- const speech = declickLevel === "off" ? cutSpeech : join(draftDir, "speech.declicked.wav");
554
+ // The render's speech chain and its file names, so the draft's sound is made as the render's is.
555
+ const { rawSpeechFile, declickedFile, speechFile } = getSpeechChainFiles(draftDir, { isDeclick: declickLevel !== "off", isLoudness: loudness !== null });
517
556
  const words = remapWords(edl, transcript.words, film.cut.fillers);
518
557
  const frame = getFrame(film);
519
558
  const brandFont = chooseFootageDraftFont(options.brand?.fonts.body ?? null);
@@ -538,13 +577,22 @@ async function runRenderFootageDraft(
538
577
  const output = resolve(options.output);
539
578
  mkdirSync(dirname(output), { recursive: true });
540
579
  report("speech", 0);
541
- await runFfmpeg("The draft's speech track", buildSpeechArgs(edl, transcript.speech, cutSpeech, getSpeechCutOptions(getFootageEdit(film))), { ...aborting, output: cutSpeech, onSeconds: (done) => report("speech", done) });
580
+ await runFfmpeg("The draft's speech track", buildSpeechArgs(edl, transcript.speech, rawSpeechFile, getSpeechCutOptions(getFootageEdit(film))), { ...aborting, output: rawSpeechFile, onSeconds: (done) => report("speech", done) });
542
581
  let declick: DeclickReport | null = null;
543
582
  if (declickLevel !== "off") {
544
583
  report("declick", 0);
545
- declick = await declickSpeech({ input: cutSpeech, output: speech, words, level: declickLevel, seconds, ...aborting, onSeconds: (done) => report("declick", done) });
584
+ declick = await declickSpeech({ input: rawSpeechFile, output: declickedFile, words, level: declickLevel, seconds, ...aborting, onSeconds: (done) => report("declick", done) });
546
585
  log(describeDeclick(declick));
547
586
  }
587
+ if (loudness !== null) {
588
+ report("loudness", 0);
589
+ const outcome = await normalizeLoudness(declickedFile, speechFile, loudness, { ...aborting, onSeconds: (done) => report("loudness", done) });
590
+ log(
591
+ outcome.kind === "silent"
592
+ ? "loudness: the speech track is silent, left as it is."
593
+ : `loudness: the speech track measured ${outcome.measured.toFixed(1)} LUFS, normalized to ${loudness.target} LUFS (true peak ${loudness.truePeak} dBTP).`,
594
+ );
595
+ }
548
596
  let musicFile: string | null = null;
549
597
  if (music.value.kind === "file") {
550
598
  musicFile = join(draftDir, "music.mp3");
@@ -555,14 +603,27 @@ async function runRenderFootageDraft(
555
603
  } else if (music.value.kind === "not_cached") {
556
604
  log(`draft: no music, ${music.value.reason}.`);
557
605
  }
606
+ let prepared: PreparedMix | null = null;
607
+ if (mixLoudness !== null) {
608
+ report("mix", 0);
609
+ // The render's mix stage over the draft's tracks (the draft has no sound effects): the same target, ceiling and attempts.
610
+ const mixSource = join(draftDir, "mix.wav");
611
+ const args = buildFilmMixArgs({ speech: { file: speechFile, seconds }, music: musicFile === null ? null : { file: musicFile, seconds }, sfx: [], seconds, output: mixSource });
612
+ await runFfmpeg("The draft's mix", args, { ...aborting, output: mixSource, onSeconds: (done) => report("mix", done) });
613
+ const mixed = await mixFootage(mixLoudness, { mixSource, output: join(draftDir, "mix.m4a") }, { log, ...aborting });
614
+ rmSync(mixSource, { force: true });
615
+ if (!mixed.ok) return mixed;
616
+ prepared = mixed.value;
617
+ }
558
618
  report("film", 0);
559
619
  // Run in the draft folder: the captions file and the font folder are relative names, so no path needs escaping in the filter graph.
560
620
  const args = buildFootageDraftArgs({
561
621
  film,
562
622
  edl,
563
623
  clips: clipFiles,
564
- speech,
624
+ speech: speechFile,
565
625
  music: musicFile,
626
+ mix: prepared?.audioFile ?? null,
566
627
  frame,
567
628
  ...(options.width === undefined ? {} : { width: options.width }),
568
629
  ...(options.threads === undefined ? {} : { threads: options.threads }),
@@ -571,11 +632,51 @@ async function runRenderFootageDraft(
571
632
  output,
572
633
  });
573
634
  await runFfmpeg("The footage draft", args, { ...aborting, output, cwd: draftDir, onSeconds: (done) => report("film", done) });
635
+ const measured = await measureDraftLoudness(output, mixLoudness, prepared !== null, aborting);
636
+ if (!measured.ok) return measured;
637
+ const draftLoudness = measured.value;
638
+ if (draftLoudness !== null) {
639
+ log(`loudness: the draft measures ${formatDb(draftLoudness.integrated)} LUFS, true peak ${formatDb(draftLoudness.truePeak)} dBTP.`);
640
+ if (draftLoudness.warning !== null) log(`loudness warning: ${draftLoudness.warning}`);
641
+ }
574
642
  report("film", seconds);
575
643
  const draft = getFootageDraftFrame(frame, options.width);
576
644
  const bytes = statSync(output).size;
577
645
  const draftMusic: FootageDraftMusic = music.value.kind === "file" ? "played" : music.value.kind;
578
646
  const notes = [captions === null ? "no captions" : `captions in ${font.family}`, draftMusic === "played" ? "music" : draftMusic === "none" ? "no music" : "music not cached"];
579
647
  log(`draft: ${output}, ${draft.width}×${draft.height}, ${seconds.toFixed(1)} s, ${bytes} bytes, ${notes.join(", ")}.`);
580
- return ok({ output, music: draftMusic, captionFont: captions === null ? null : font.family, seconds, width: draft.width, height: draft.height, bytes, declick });
648
+ return ok({
649
+ output,
650
+ music: draftMusic,
651
+ captionFont: captions === null ? null : font.family,
652
+ seconds,
653
+ width: draft.width,
654
+ height: draft.height,
655
+ bytes,
656
+ declick,
657
+ loudness: draftLoudness,
658
+ mix: prepared?.mix ?? null,
659
+ });
660
+ }
661
+
662
+ /**
663
+ * The written draft's sound measured as `finishFootage` measures the finished film: held to `mixLoudness` when the mix
664
+ * stage made it (a miss fails, a near miss inside `warnTolerance` carries `warning`), reported otherwise; null when
665
+ * the draft is silent.
666
+ */
667
+ async function measureDraftLoudness(
668
+ output: string,
669
+ mixLoudness: MixLoudnessTarget | null,
670
+ isMixed: boolean,
671
+ aborting: { signal?: AbortSignal },
672
+ ): Promise<FootageResult<FinishedLoudness | null>> {
673
+ const measured = await measureLoudness(output, mixLoudness ?? DEFAULT_LOUDNESS, { label: "the footage draft", ...aborting });
674
+ if (measured.kind === "silent") return ok(null);
675
+ const sound = { integrated: measured.measurement.inputI, truePeak: measured.measurement.inputTp };
676
+ const verdict = mixLoudness === null || !isMixed ? { kind: "pass" as const } : getLoudnessVerdict(sound, mixLoudness);
677
+ if (verdict.kind === "fail") {
678
+ return err(`the footage draft ${output} measures ${formatDb(sound.integrated)} LUFS integrated, ${formatDb(sound.truePeak)} dBTP true peak: ${verdict.problems.join("; ")}.`, { kind: "loudness", loudness: sound });
679
+ }
680
+ return ok({ ...sound, warning: verdict.kind === "warn" ? `the footage draft ${output}: ${verdict.warning}` : null });
581
681
  }
682
+
@@ -5,6 +5,7 @@ import { z } from "zod";
5
5
 
6
6
  import { ELEVENLABS_API_KEY_ENV } from "../voice/elevenlabs.js";
7
7
  import { err, ok, type TtsResult } from "../voice/provider.js";
8
+ import { getConnectionFailure, getLocalRefusal, getProviderFailure } from "../voice/provider-failure.js";
8
9
  import { ELEVENLABS_API_URL, type TimedWord } from "../voice/voiceover.js";
9
10
  import type { IsolationProvider, MusicProvider, PaidAudio, SttProvider, SttTranscript } from "./providers.js";
10
11
 
@@ -57,17 +58,17 @@ function fileBlob(path: string): Blob {
57
58
  }
58
59
 
59
60
  async function send(options: ElevenLabsFootageOptions, label: string, path: string, init: RequestInit): Promise<TtsResult<Response>> {
60
- if (options.apiKey === undefined || options.apiKey.length === 0) return err(`no ${ELEVENLABS_API_KEY_ENV} in the environment.`);
61
+ if (options.apiKey === undefined || options.apiKey.length === 0) return err(`no ${ELEVENLABS_API_KEY_ENV} in the environment.`, getLocalRefusal("missing_api_key"));
61
62
  const request = options.fetch ?? globalThis.fetch;
62
63
  let response: Response;
63
64
  try {
64
65
  response = await request(`${ELEVENLABS_API_URL}${path}`, { ...init, headers: { ...init.headers, "xi-api-key": options.apiKey } });
65
66
  } catch (error) {
66
- return err(`ElevenLabs ${label}: the connection failed (${String(error)}).`);
67
+ return err(`ElevenLabs ${label}: the connection failed (${String(error)}).`, getConnectionFailure(error));
67
68
  }
68
69
  if (!response.ok) {
69
70
  const detail = await response.text().catch((error: unknown) => `(unreadable body: ${String(error)})`);
70
- return err(`ElevenLabs ${label} answered ${response.status}: ${detail.slice(0, ERROR_DETAIL_LENGTH)}`);
71
+ return err(`ElevenLabs ${label} answered ${response.status}: ${detail.slice(0, ERROR_DETAIL_LENGTH)}`, getProviderFailure(response, detail));
71
72
  }
72
73
  return ok(response);
73
74
  }
@@ -118,9 +119,16 @@ export function createElevenLabsSttProvider(options: ElevenLabsFootageOptions):
118
119
  };
119
120
  }
120
121
 
122
+ /**
123
+ * The shortest input the ElevenLabs Voice Isolator accepts, in seconds: production renders saw every range of 4.18 s
124
+ * or less refused on every attempt (`audio_too_short`) and every range of 4.64 s or more accepted (issue #1140).
125
+ */
126
+ export const ELEVENLABS_ISOLATION_MIN_SECONDS = 4.6;
127
+
121
128
  export function createElevenLabsIsolationProvider(options: ElevenLabsFootageOptions): IsolationProvider {
122
129
  return {
123
130
  id: "elevenlabs",
131
+ minSeconds: ELEVENLABS_ISOLATION_MIN_SECONDS,
124
132
  async isolate(input) {
125
133
  const form = new FormData();
126
134
  form.append("audio", fileBlob(input.audio), basename(input.audio));
@@ -1,3 +1,4 @@
1
+ import type { ProviderFailure } from "../voice/provider-failure.js";
1
2
  import type { MixLoudness } from "./loudness.js";
2
3
 
3
4
  /**
@@ -6,6 +7,8 @@ import type { MixLoudness } from "./loudness.js";
6
7
  * measures silent after its gain and limiter). The same inputs fail the same way: no retry.
7
8
  * - `publishable`: the finished file fails the container, codec profile or fast start checks (the problems).
8
9
  * - `input`: the film, its EDL, fonts, files or settings cannot be used as given. A retry fails the same way.
10
+ * - `provider`: a paid provider (speech to text, voice isolation, music) failed with the structured details it gave
11
+ * (issue #1141); a retry may succeed only when `provider.isRetryable`.
9
12
  * - `process`: anything else (a process that exited, the network, the disk). A retry may succeed.
10
13
  * A failed ffmpeg step still throws and an abort still rejects with the signal's reason; neither is a result.
11
14
  */
@@ -13,6 +16,7 @@ export type FootageFailure =
13
16
  | { kind: "loudness"; loudness: MixLoudness | null }
14
17
  | { kind: "publishable"; problems: string[] }
15
18
  | { kind: "input" }
19
+ | { kind: "provider"; provider: ProviderFailure }
16
20
  | { kind: "process" };
17
21
 
18
22
  export type FootageFailureKind = FootageFailure["kind"];
@@ -38,7 +42,16 @@ export function err(error: string, failure: FootageFailure = PROCESS_FAILURE): F
38
42
  return { ok: false, error, failure };
39
43
  }
40
44
 
41
- /** True when running the same step again may succeed (`process`); a gate miss or a bad input fails again. */
45
+ /** A provider's failure as a footage failure: `provider` with its details, or `process` when it gave none. */
46
+ export function getProviderFootageFailure(providerFailure: ProviderFailure | undefined): FootageFailure {
47
+ return providerFailure === undefined ? PROCESS_FAILURE : { kind: "provider", provider: providerFailure };
48
+ }
49
+
50
+ /**
51
+ * True when running the same step again may succeed: `process`, or `provider` with `isRetryable`; a gate miss, a bad
52
+ * input or a provider's refusal fails again.
53
+ */
42
54
  export function isRetryableFootageFailure(failure: FootageFailure): boolean {
55
+ if (failure.kind === "provider") return failure.provider.isRetryable;
43
56
  return failure.kind === "process";
44
57
  }
@@ -1,8 +1,9 @@
1
1
  import { readFileSync } from "node:fs";
2
2
  import { basename } from "node:path";
3
3
 
4
- import { ok, type TtsResult } from "../voice/provider.js";
4
+ import { err, ok, type TtsResult } from "../voice/provider.js";
5
5
  import type { TimedWord } from "../voice/voiceover.js";
6
+ import { probeAudio } from "./audio.js";
6
7
  import type { IsolationProvider, MusicProvider, SttInput, SttProvider, SttTranscript } from "./providers.js";
7
8
 
8
9
  /** Providers that spend nothing and reach no network, for tests of the footage pipeline and of a project's own flows. */
@@ -36,19 +37,42 @@ export function createFakeSttProvider(options: FakeSttOptions = {}): FakeSttProv
36
37
  };
37
38
  }
38
39
 
40
+ export interface FakeIsolationOptions {
41
+ /**
42
+ * The shortest input in seconds the fake accepts, declared as its `minSeconds`: a shorter input (probed with
43
+ * ffprobe) is refused with `audio_too_short`, as the ElevenLabs Voice Isolator refuses it.
44
+ */
45
+ minSeconds?: number;
46
+ }
47
+
39
48
  export interface FakeIsolationProvider extends IsolationProvider {
40
49
  readonly calls: string[];
50
+ /** The inputs refused for being shorter than `minSeconds`. */
51
+ readonly refused: string[];
41
52
  }
42
53
 
43
54
  /** Returns the input file's bytes unchanged: a valid audio file, so a pipeline with it still renders. */
44
- export function createFakeIsolationProvider(): FakeIsolationProvider {
55
+ export function createFakeIsolationProvider(options: FakeIsolationOptions = {}): FakeIsolationProvider {
45
56
  const calls: string[] = [];
57
+ const refused: string[] = [];
58
+ const { minSeconds } = options;
46
59
  return {
47
60
  id: "fake",
61
+ ...(minSeconds === undefined ? {} : { minSeconds }),
48
62
  calls,
49
- isolate(input) {
63
+ refused,
64
+ async isolate(input) {
50
65
  calls.push(input.audio);
51
- return Promise.resolve(ok(readFileSync(input.audio)));
66
+ if (minSeconds !== undefined) {
67
+ const probed = await probeAudio(input.audio, input.signal === undefined ? {} : { signal: input.signal });
68
+ if (!probed.ok) return err(`fake voice isolation: ${probed.error}`);
69
+ if (probed.probe.duration < minSeconds) {
70
+ refused.push(input.audio);
71
+ const message = `${probed.probe.duration.toFixed(3)} s, the minimum is ${String(minSeconds)} s`;
72
+ return err(`fake voice isolation: 400 audio_too_short (${message}).`, { kind: "refused", status: 400, code: "audio_too_short", message, requestId: null, isRetryable: false });
73
+ }
74
+ }
75
+ return ok(readFileSync(input.audio));
52
76
  },
53
77
  };
54
78
  }
@@ -0,0 +1,100 @@
1
+ /**
2
+ * How hyperframes captured a render, read from the summary it prints at the end (issue #1145). hyperframes 0.9.6 prints
3
+ * `beginframe capture · software gpu · compile 0.8s · extract 8.7s · … · assemble 0.6s` and, after a screenshot
4
+ * capture it did not ask for, a line `Screenshot capture (slower): BeginFrame did not run. …`.
5
+ */
6
+
7
+ /** hyperframes' capture modes: `beginframe` (fast), `screenshot` (slower), `drawelement`. */
8
+ export type HyperframesCapturePath = "beginframe" | "screenshot" | "drawelement";
9
+
10
+ /** The browser's GPU mode for the render. */
11
+ export type HyperframesGpuMode = "software" | "hardware";
12
+
13
+ /** The pipeline stages hyperframes times, in the order it prints them. */
14
+ export const HYPERFRAMES_CAPTURE_STAGES = ["compile", "extract", "audio", "probe", "setup", "capture", "encode", "assemble"] as const;
15
+
16
+ export type HyperframesCaptureStage = (typeof HYPERFRAMES_CAPTURE_STAGES)[number];
17
+
18
+ export interface HyperframesCapture {
19
+ /** The capture path; null when hyperframes printed none. */
20
+ path: HyperframesCapturePath | null;
21
+ /** The browser's GPU mode; null when hyperframes printed none. */
22
+ gpu: HyperframesGpuMode | null;
23
+ /** Seconds per stage, only the stages hyperframes printed. */
24
+ seconds: Partial<Record<HyperframesCaptureStage, number>>;
25
+ /** The encode ran during the capture (`encode (during capture)`), so its seconds overlap the capture's. */
26
+ isEncodeStreamed: boolean;
27
+ /** hyperframes' hint after a screenshot capture (`Screenshot capture (slower): …`); null when it printed none. */
28
+ fallbackHint: string | null;
29
+ }
30
+
31
+ const SEPARATOR = " · ";
32
+ const CAPTURE_PATHS: readonly string[] = ["beginframe", "screenshot", "drawelement"];
33
+ const GPU_MODES: readonly string[] = ["software", "hardware"];
34
+ const STAGES: readonly string[] = HYPERFRAMES_CAPTURE_STAGES;
35
+ const FALLBACK_HINT_PREFIX = "Screenshot capture";
36
+ // SGR colour codes (`\x1b[2m…\x1b[22m`) that hyperframes' dim style may add.
37
+ // eslint-disable-next-line no-control-regex -- the escape character is what the pattern matches
38
+ const ANSI_PATTERN = /\u001B\[[0-9;]*m/g;
39
+
40
+ /** hyperframes' `formatDuration`: `8.7s` under a minute, `1m 31.4s` from a minute on. */
41
+ function readDurationSeconds(text: string): number | null {
42
+ const match = /^(?:(\d+)m )?(\d+(?:\.\d+)?)s$/.exec(text);
43
+ if (match === null) return null;
44
+ const minutes = match[1] === undefined ? 0 : Number(match[1]);
45
+ return Math.round((minutes * 60 + Number(match[2])) * 1000) / 1000;
46
+ }
47
+
48
+ type SummaryPart =
49
+ | { kind: "path"; path: HyperframesCapturePath }
50
+ | { kind: "gpu"; gpu: HyperframesGpuMode }
51
+ | { kind: "stage"; stage: HyperframesCaptureStage; seconds: number; isStreamed: boolean };
52
+
53
+ function readSummaryPart(part: string): SummaryPart | null {
54
+ const path = /^(\w+) capture$/.exec(part)?.[1];
55
+ if (path !== undefined) return CAPTURE_PATHS.includes(path) ? { kind: "path", path: path as HyperframesCapturePath } : null;
56
+ const gpu = /^(\w+) gpu$/.exec(part)?.[1];
57
+ if (gpu !== undefined) return GPU_MODES.includes(gpu) ? { kind: "gpu", gpu: gpu as HyperframesGpuMode } : null;
58
+ const stage = /^(\w+)( \(during capture\))? (.+)$/.exec(part);
59
+ if (stage === null) return null;
60
+ const [, name = "", streamed, duration = ""] = stage;
61
+ const seconds = readDurationSeconds(duration);
62
+ if (!STAGES.includes(name) || seconds === null) return null;
63
+ return { kind: "stage", stage: name as HyperframesCaptureStage, seconds, isStreamed: streamed !== undefined };
64
+ }
65
+
66
+ /** A summary line: every ` · ` part is a capture path, a GPU mode or a timed stage. Null for any other line. */
67
+ function readSummaryLine(line: string): Omit<HyperframesCapture, "fallbackHint"> | null {
68
+ const parts = line.split(SEPARATOR).map(readSummaryPart);
69
+ if (parts.length === 0 || parts.some((part) => part === null)) return null;
70
+ const capture: Omit<HyperframesCapture, "fallbackHint"> = { path: null, gpu: null, seconds: {}, isEncodeStreamed: false };
71
+ for (const part of parts) {
72
+ if (part === null) continue;
73
+ if (part.kind === "path") capture.path = part.path;
74
+ else if (part.kind === "gpu") capture.gpu = part.gpu;
75
+ else {
76
+ capture.seconds[part.stage] = part.seconds;
77
+ if (part.stage === "encode" && part.isStreamed) capture.isEncodeStreamed = true;
78
+ }
79
+ }
80
+ return capture;
81
+ }
82
+
83
+ /**
84
+ * The capture report from hyperframes' output lines (stdout and stderr, in order): the last summary line, plus the
85
+ * screenshot hint when one was printed. Null when no line is a summary (an older or quiet hyperframes).
86
+ */
87
+ export function readHyperframesCapture(lines: readonly string[]): HyperframesCapture | null {
88
+ let summary: Omit<HyperframesCapture, "fallbackHint"> | null = null;
89
+ let fallbackHint: string | null = null;
90
+ for (const raw of lines) {
91
+ const line = raw.replace(ANSI_PATTERN, "").trim();
92
+ if (line.length === 0) continue;
93
+ if (line.startsWith(FALLBACK_HINT_PREFIX)) {
94
+ fallbackHint = line;
95
+ continue;
96
+ }
97
+ summary = readSummaryLine(line) ?? summary;
98
+ }
99
+ return summary === null ? null : { ...summary, fallbackHint };
100
+ }