@softure-ai/marketing-kit 0.11.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/CHANGELOG.md +85 -0
  2. package/README.md +67 -9
  3. package/dist/cli/footage.js +1 -1
  4. package/dist/cli/footage.js.map +1 -1
  5. package/dist/compose/timeline.d.ts +23 -3
  6. package/dist/compose/timeline.d.ts.map +1 -1
  7. package/dist/compose/timeline.js +31 -11
  8. package/dist/compose/timeline.js.map +1 -1
  9. package/dist/config/actions-schema.d.ts +14 -14
  10. package/dist/config/config.d.ts.map +1 -1
  11. package/dist/config/config.js +2 -0
  12. package/dist/config/config.js.map +1 -1
  13. package/dist/config/schema.d.ts +83 -70
  14. package/dist/config/schema.d.ts.map +1 -1
  15. package/dist/config/schema.js +21 -1
  16. package/dist/config/schema.js.map +1 -1
  17. package/dist/config/shot-steps.d.ts +16 -16
  18. package/dist/footage/captions.d.ts +45 -3
  19. package/dist/footage/captions.d.ts.map +1 -1
  20. package/dist/footage/captions.js +65 -5
  21. package/dist/footage/captions.js.map +1 -1
  22. package/dist/footage/compose.d.ts.map +1 -1
  23. package/dist/footage/compose.js +8 -4
  24. package/dist/footage/compose.js.map +1 -1
  25. package/dist/footage/declick.d.ts +106 -0
  26. package/dist/footage/declick.d.ts.map +1 -0
  27. package/dist/footage/declick.js +296 -0
  28. package/dist/footage/declick.js.map +1 -0
  29. package/dist/footage/draft.d.ts +8 -3
  30. package/dist/footage/draft.d.ts.map +1 -1
  31. package/dist/footage/draft.js +55 -22
  32. package/dist/footage/draft.js.map +1 -1
  33. package/dist/footage/edl.d.ts +28 -2
  34. package/dist/footage/edl.d.ts.map +1 -1
  35. package/dist/footage/edl.js +53 -2
  36. package/dist/footage/edl.js.map +1 -1
  37. package/dist/footage/film.d.ts +29 -3
  38. package/dist/footage/film.d.ts.map +1 -1
  39. package/dist/footage/film.js +12 -0
  40. package/dist/footage/film.js.map +1 -1
  41. package/dist/footage/isolate-ranges.d.ts +37 -0
  42. package/dist/footage/isolate-ranges.d.ts.map +1 -0
  43. package/dist/footage/isolate-ranges.js +113 -0
  44. package/dist/footage/isolate-ranges.js.map +1 -0
  45. package/dist/footage/loudness.d.ts +76 -0
  46. package/dist/footage/loudness.d.ts.map +1 -1
  47. package/dist/footage/loudness.js +134 -0
  48. package/dist/footage/loudness.js.map +1 -1
  49. package/dist/footage/paid.d.ts +18 -0
  50. package/dist/footage/paid.d.ts.map +1 -1
  51. package/dist/footage/paid.js +32 -2
  52. package/dist/footage/paid.js.map +1 -1
  53. package/dist/footage/pipeline.d.ts +43 -5
  54. package/dist/footage/pipeline.d.ts.map +1 -1
  55. package/dist/footage/pipeline.js +105 -20
  56. package/dist/footage/pipeline.js.map +1 -1
  57. package/dist/footage/providers.d.ts +5 -0
  58. package/dist/footage/providers.d.ts.map +1 -1
  59. package/dist/footage/publish.d.ts +11 -2
  60. package/dist/footage/publish.d.ts.map +1 -1
  61. package/dist/footage/publish.js +14 -10
  62. package/dist/footage/publish.js.map +1 -1
  63. package/dist/footage/speech-ranges.d.ts +65 -0
  64. package/dist/footage/speech-ranges.d.ts.map +1 -0
  65. package/dist/footage/speech-ranges.js +98 -0
  66. package/dist/footage/speech-ranges.js.map +1 -0
  67. package/dist/index.d.ts +10 -7
  68. package/dist/index.d.ts.map +1 -1
  69. package/dist/index.js +10 -7
  70. package/dist/index.js.map +1 -1
  71. package/package.json +1 -1
  72. package/schema/edl.schema.json +17 -0
  73. package/schema/marketing.schema.json +43 -1
  74. package/src/cli/footage.ts +1 -1
  75. package/src/compose/timeline.ts +41 -10
  76. package/src/config/config.ts +2 -0
  77. package/src/config/schema.ts +23 -1
  78. package/src/footage/captions.ts +81 -6
  79. package/src/footage/compose.ts +8 -4
  80. package/src/footage/declick.ts +353 -0
  81. package/src/footage/draft.ts +58 -23
  82. package/src/footage/edl.ts +69 -2
  83. package/src/footage/film.ts +37 -3
  84. package/src/footage/isolate-ranges.ts +161 -0
  85. package/src/footage/loudness.ts +170 -0
  86. package/src/footage/paid.ts +40 -2
  87. package/src/footage/pipeline.ts +158 -22
  88. package/src/footage/providers.ts +2 -0
  89. package/src/footage/publish.ts +20 -11
  90. package/src/footage/speech-ranges.ts +131 -0
  91. package/src/index.ts +75 -0
@@ -13,14 +13,32 @@ import type { TimedWord } from "../voice/voiceover.js";
13
13
  import { checkMusicFileType, describeAudioProbe, probeAudio } from "./audio.js";
14
14
  import { readCachedTranscript, writeCacheFile, writeTranscript } from "./cache.js";
15
15
  import { composeFootage, END_CARD_SECONDS, getFootageFilmSeconds } from "./compose.js";
16
- import { findMissingCaptionGlyphs, getFootageCaptionLayout } from "./captions.js";
16
+ import { findMissingCaptionGlyphs, getCaptionBounds, getFootageCaptionLayout } from "./captions.js";
17
17
  import { findLowDiskProblems, getFreeBytes, getRenderTempDir, type FreeBytesReader } from "./disk.js";
18
18
  import { buildEdl, getEdlDuration, readEdl, remapWords, validateEdl, type Edl } from "./edl.js";
19
19
  import { makePlaceholderTranscript } from "./fake.js";
20
20
  import { chooseCutTransitions, type CutMeasure } from "./cuts.js";
21
- import { DEFAULT_AUTO_THRESHOLD, getFootageEdit, type FootageClip, type FootageFilm, type MusicSource } from "./film.js";
21
+ import { declickSpeech, describeDeclick, type DeclickReport } from "./declick.js";
22
+ import { DEFAULT_AUTO_THRESHOLD, getDeclickLevel, getFootageEdit, type FootageClip, type FootageFilm, type MusicSource } from "./film.js";
22
23
  import { getFilmKeyTerms } from "./keyterms.js";
23
- import { DEFAULT_LOUDNESS, getLoudnessFilter, measureLoudness, normalizeLoudness } from "./loudness.js";
24
+ import {
25
+ buildMixArgs,
26
+ DEFAULT_LOUDNESS,
27
+ findLoudnessProblems,
28
+ formatDb,
29
+ getFirstMixAttempt,
30
+ getMixTolerance,
31
+ getMixFilter,
32
+ getNextMixAttempt,
33
+ measureLoudness,
34
+ MIX_ATTEMPTS,
35
+ normalizeLoudness,
36
+ type LoudnessMeasurement,
37
+ type MixAttempt,
38
+ type MixLoudness,
39
+ type MixLoudnessTarget,
40
+ type MixTry,
41
+ } from "./loudness.js";
24
42
  import { duckingKeyframes, getDuckingOptions } from "./music.js";
25
43
  import { FOOTAGE_FPS, normalizeFootage, type Frame, type NormalizeEncoder } from "./normalize.js";
26
44
  import { buildPictureConcatArgs, buildPictureList, buildPicturePieceArgs, getPictureFrames, planFootagePicture } from "./picture.js";
@@ -473,8 +491,11 @@ export function describeCutMeasures(cuts: CutMeasure[]): string[] {
473
491
  return cuts.map((cut) => `${cut.segment}: ${cut.transition} (difference ${cut.difference.toFixed(3)})`);
474
492
  }
475
493
 
476
- /** The EDL file as written by `cutFootage` or edited by hand, checked against the clips. */
477
- export function readEdlFile(film: FootageFilm, clips: PreparedClip[], edlPath: string): FootageResult<Edl> {
494
+ /**
495
+ * The EDL file as written by `cutFootage` or edited by hand, checked against the clips and (a segment's own caption
496
+ * box) against the film's frame, with the caption size the project's `layout` gives.
497
+ */
498
+ export function readEdlFile(film: FootageFilm, clips: PreparedClip[], edlPath: string, layout: LayoutOverride = {}): FootageResult<Edl> {
478
499
  let json: unknown;
479
500
  try {
480
501
  json = JSON.parse(readFileSync(edlPath, "utf8")) as unknown;
@@ -483,7 +504,8 @@ export function readEdlFile(film: FootageFilm, clips: PreparedClip[], edlPath: s
483
504
  }
484
505
  const read = readEdl(json);
485
506
  if (!read.ok) return err(`${edlPath}: ${read.error}`);
486
- const problems = validateEdl(read.edl, clips.map((clip) => ({ id: clip.clip.id, duration: clip.duration })), film.id);
507
+ const bounds = getCaptionBounds(getFrame(film).height, getFootageCaptionLayout(film, layout));
508
+ const problems = validateEdl(read.edl, clips.map((clip) => ({ id: clip.clip.id, duration: clip.duration })), film.id, bounds);
487
509
  return problems.length > 0 ? err(`${edlPath}: ${problems.join("; ")}`) : ok(read.edl);
488
510
  }
489
511
 
@@ -590,6 +612,17 @@ async function getMusic(film: FootageFilm, edl: Edl, clips: PreparedClip[], seco
590
612
  return ok(path);
591
613
  }
592
614
 
615
+ /**
616
+ * The speech chain's files: the cut speech, the declicked track and the final `speech.mp3`. The last stage that runs
617
+ * writes `speech.mp3`; the declicked track is float WAV when the loudness pass follows (one lossy encode fewer).
618
+ */
619
+ export function getSpeechChainFiles(assetsDir: string, stages: { isDeclick: boolean; isLoudness: boolean }): { rawSpeechFile: string; declickedFile: string; speechFile: string } {
620
+ const speechFile = join(assetsDir, "speech.mp3");
621
+ const rawSpeechFile = stages.isDeclick || stages.isLoudness ? join(assetsDir, "speech.raw.mp3") : speechFile;
622
+ const declickedFile = !stages.isDeclick ? rawSpeechFile : stages.isLoudness ? join(assetsDir, "speech.declicked.wav") : speechFile;
623
+ return { rawSpeechFile, declickedFile, speechFile };
624
+ }
625
+
593
626
  export interface RenderFootageOptions extends FootageRunOptions {
594
627
  brand: FootageBrand;
595
628
  sfx: Partial<Record<SfxEvent, string>>;
@@ -630,7 +663,16 @@ export interface RenderFootageOptions extends FootageRunOptions {
630
663
  finishVideo?: FinishVideo;
631
664
  }
632
665
 
633
- export type RenderedFootage = { kind: "dry-run"; steps: PaidStep[] } | ({ kind: "rendered"; output: string; seconds: number; edlPath: string } & FinishedFootage);
666
+ export type RenderedFootage =
667
+ | { kind: "dry-run"; steps: PaidStep[] }
668
+ | ({
669
+ kind: "rendered";
670
+ output: string;
671
+ seconds: number;
672
+ edlPath: string;
673
+ /** What the speech track's declick stage did (`FootageFilm.speech.declick`); null when it is off. */
674
+ declick: DeclickReport | null;
675
+ } & FinishedFootage);
634
676
 
635
677
  /**
636
678
  * Composes the cut and renders it: speech track (loudness-normalized), ducked music, end card backdrop, assets,
@@ -670,18 +712,29 @@ async function runRenderFootage(
670
712
  const musicFile = typeof music.value === "string" ? music.value : null;
671
713
  const aborting = signal === undefined ? {} : { signal };
672
714
  const loudness = film.loudness === undefined ? DEFAULT_LOUDNESS : film.loudness;
673
- const speechFile = join(assetsDir, "speech.mp3");
674
- const rawSpeechFile = loudness === null ? speechFile : join(assetsDir, "speech.raw.mp3");
715
+ const declickLevel = getDeclickLevel(film);
716
+ const { speechFile, rawSpeechFile, declickedFile } = getSpeechChainFiles(assetsDir, { isDeclick: declickLevel !== "off", isLoudness: loudness !== null });
675
717
  const ffmpegTrack = (label: string, output: string, args: () => string[]) => (onSeconds: (done: number) => void) => runFfmpeg(label, args(), { ...aborting, output, onSeconds });
676
718
  const tracks: { item: string; seconds: number; run: (onSeconds: (done: number) => void) => Promise<void> }[] = [
677
719
  { item: "speech", seconds, run: ffmpegTrack("The speech track", rawSpeechFile, () => buildSpeechArgs(edl, transcript.speech, rawSpeechFile, getSpeechCutOptions(getFootageEdit(film)))) },
678
720
  ];
721
+ let declick: DeclickReport | null = null;
722
+ if (declickLevel !== "off") {
723
+ tracks.push({
724
+ item: "declick",
725
+ seconds,
726
+ run: async (onSeconds) => {
727
+ declick = await declickSpeech({ input: rawSpeechFile, output: declickedFile, words, level: declickLevel, seconds, ...aborting, onSeconds });
728
+ log(describeDeclick(declick));
729
+ },
730
+ });
731
+ }
679
732
  if (loudness !== null) {
680
733
  tracks.push({
681
734
  item: "loudness",
682
735
  seconds,
683
736
  run: async (onSeconds) => {
684
- const outcome = await normalizeLoudness(rawSpeechFile, speechFile, loudness, { ...aborting, onSeconds });
737
+ const outcome = await normalizeLoudness(declickedFile, speechFile, loudness, { ...aborting, onSeconds });
685
738
  log(
686
739
  outcome.kind === "silent"
687
740
  ? "loudness: the speech track is silent, left as it is."
@@ -765,7 +818,7 @@ async function runRenderFootage(
765
818
  const roles = Object.fromEntries(clips.map((clip) => [clip.clip.id, clip.clip.role]));
766
819
  const finished = await finishFootage(film, { renderFile, seconds, coverAt: film.cover?.at ?? getDefaultCoverSeconds(edl, roles) }, options);
767
820
  if (!finished.ok) return finished;
768
- return ok({ kind: "rendered", output: options.output, seconds, edlPath: getEdlPath(buildDir, options.isPlaceholder), ...finished.value });
821
+ return ok({ kind: "rendered", output: options.output, seconds, edlPath: getEdlPath(buildDir, options.isPlaceholder), declick, ...finished.value });
769
822
  }
770
823
 
771
824
  /** What `renderFootage` reads from the written film. */
@@ -777,12 +830,32 @@ export interface FinishedFootage {
777
830
  durationSeconds: number;
778
831
  /** The cover JPG, or null when the film has none. */
779
832
  cover: string | null;
833
+ /** The finished file's sound as measured (EBU R128, true peak oversampled); null for a film without sound or a silent mix. */
834
+ loudness: MixLoudness | null;
835
+ /**
836
+ * What the mix's last stage did: the render's mix as measured, the gain applied (0 when `keep: "speech"` found the
837
+ * mix inside its window) and the limiter's ceiling; null when no stage ran (no sound, a silent mix, `mixLoudness: null`).
838
+ */
839
+ mix: FinishedMix | null;
840
+ }
841
+
842
+ /** The mix's last stage in `FinishedFootage`. */
843
+ export interface FinishedMix {
844
+ /** The render's mix before the stage. */
845
+ measured: MixLoudness;
846
+ gainDb: number;
847
+ ceilingDbtp: number;
848
+ /** What the gain kept (`MixLoudnessTarget.keep`). */
849
+ keep: "loudness" | "speech";
780
850
  }
781
851
 
782
852
  /**
783
- * The final pass over hyperframes' render: the mix normalized to `film.mixLoudness` (measured on the render, one
784
- * linear gain; the picture still copied), publishable codecs and fast start (stream copy where the render already
785
- * matches; `finishVideo` re-encodes the picture to H.264 High at its preset and CRF), checked with ffprobe, then the cover and the file's facts. Exported for tests only.
853
+ * The final pass over hyperframes' render: the mix brought to `film.mixLoudness` (measured on the render; one linear
854
+ * gain and a true peak limiter, encoded to AAC on its own and measured, the limiter lowered and encoded again while
855
+ * the codec's overshoot breaks the limit), muxed with the picture (copied where the render already matches;
856
+ * `finishVideo` re-encodes it to H.264 High at its preset and CRF), publishable codecs and fast start, checked with
857
+ * ffprobe, the finished file's sound measured and held to the target, then the cover and the file's facts. Exported
858
+ * for tests only.
786
859
  */
787
860
  export async function finishFootage(
788
861
  film: Pick<FootageFilm, "mixLoudness" | "cover">,
@@ -795,27 +868,52 @@ export async function finishFootage(
795
868
  progress(0, 0);
796
869
  const rendered = await probeFilmFacts(input.renderFile, aborting);
797
870
  if (!rendered.ok) return err(`the rendered film: ${rendered.error}`);
798
- const mixLoudness = film.mixLoudness === undefined ? DEFAULT_LOUDNESS : film.mixLoudness;
799
- let audioFilter: string | undefined;
871
+ const mixLoudness: MixLoudnessTarget | null = film.mixLoudness === undefined ? DEFAULT_LOUDNESS : film.mixLoudness;
872
+ let audioFile: string | undefined;
873
+ let mix: FinishedMix | null = null;
800
874
  if (mixLoudness !== null && rendered.facts.audio !== null) {
801
875
  const measured = await measureLoudness(input.renderFile, mixLoudness, { label: "the film's mix", ...aborting });
802
876
  if (measured.kind === "silent") {
803
877
  log("loudness: the film's mix is silent, left as it is.");
804
878
  } else {
805
- audioFilter = getLoudnessFilter(measured.measurement, mixLoudness);
806
- log(`loudness: the film's mix measured ${measured.measurement.inputI.toFixed(1)} LUFS, normalized to ${mixLoudness.target} LUFS (true peak ${mixLoudness.truePeak} dBTP).`);
879
+ audioFile = join(dirname(input.renderFile), "mix.m4a");
880
+ const mixed = await finishMix({ input: input.renderFile, output: audioFile, measurement: measured.measurement, target: mixLoudness, ...aborting });
881
+ if (!mixed.ok) {
882
+ rmSync(audioFile, { force: true });
883
+ return mixed;
884
+ }
885
+ const { attempt, tries } = mixed.value;
886
+ const keep = mixLoudness.keep ?? "loudness";
887
+ mix = { measured: { integrated: measured.measurement.inputI, truePeak: measured.measurement.inputTp }, gainDb: attempt.gainDb, ceilingDbtp: attempt.ceilingDbtp, keep };
888
+ const kept = keep === "speech" ? ` (speech kept: ${formatDb(mixLoudness.target)} ± ${getMixTolerance(mixLoudness)} LUFS)` : "";
889
+ log(
890
+ `loudness: the film's mix measured ${formatDb(measured.measurement.inputI)} LUFS, true peak ${formatDb(measured.measurement.inputTp)} dBTP; ` +
891
+ `gain ${formatDb(attempt.gainDb)} dB${kept}, limiter at ${formatDb(attempt.ceilingDbtp)} dBTP (encode ${tries} of ${MIX_ATTEMPTS}).`,
892
+ );
807
893
  }
808
894
  }
809
895
  const { finishVideo } = options;
810
896
  const isVideoCopied = finishVideo === undefined && isPublishableVideo(rendered.facts.video);
811
- const isAudioCopied = audioFilter === undefined && (rendered.facts.audio === null || isPublishableAudio(rendered.facts.audio));
812
- const finishArgs = buildFinishArgs(input.renderFile, rendered.facts, output, { ...(audioFilter === undefined ? {} : { audioFilter }), ...(finishVideo === undefined ? {} : { video: finishVideo }) });
897
+ const isAudioCopied = audioFile === undefined && (rendered.facts.audio === null || isPublishableAudio(rendered.facts.audio));
898
+ const finishArgs = buildFinishArgs(input.renderFile, rendered.facts, output, { ...(audioFile === undefined ? {} : { audioFile }), ...(finishVideo === undefined ? {} : { video: finishVideo }) });
813
899
  await runFfmpeg("The film's final pass", finishArgs, { ...aborting, output, onSeconds: (done) => progress(0, done) });
814
900
  rmSync(input.renderFile, { force: true });
901
+ if (audioFile !== undefined) rmSync(audioFile, { force: true });
815
902
  const finished = await probeFilmFacts(output, aborting);
816
903
  if (!finished.ok) return err(`the finished film: ${finished.error}`);
817
904
  const problems = findFinishProblems(finished.facts, readTopLevelBoxes(output));
818
905
  if (problems.length > 0) return err(`the finished film ${output} is not publishable: ${problems.join("; ")}.`);
906
+ let loudness: MixLoudness | null = null;
907
+ if (finished.facts.audio !== null) {
908
+ const measured = await measureLoudness(output, mixLoudness ?? DEFAULT_LOUDNESS, { label: "the finished film", ...aborting });
909
+ if (measured.kind === "measured") {
910
+ loudness = { integrated: measured.measurement.inputI, truePeak: measured.measurement.inputTp };
911
+ // A mix left as rendered (`mixLoudness: null`) or silent is reported, not held to a target it was not brought to.
912
+ const misses = mixLoudness === null || audioFile === undefined ? [] : findLoudnessProblems(loudness, mixLoudness);
913
+ if (misses.length > 0) return err(`the finished film ${output} measures ${describeMixLoudness(loudness)}: ${misses.join("; ")}.`);
914
+ log(`loudness: the finished film measures ${formatDb(loudness.integrated)} LUFS, true peak ${formatDb(loudness.truePeak)} dBTP.`);
915
+ }
916
+ }
819
917
  progress(0, input.seconds);
820
918
  const { bytes, width, height, durationSeconds } = finished.facts;
821
919
  const streams = describeFinishStreams({ isVideoCopied, isAudioCopied, hasAudio: rendered.facts.audio !== null, ...(finishVideo === undefined ? {} : { finishVideo }) });
@@ -828,7 +926,45 @@ export async function finishFootage(
828
926
  await runFfmpeg("The cover frame", buildCoverArgs(output, at, cover), { ...aborting, output: cover });
829
927
  log(`cover: ${cover} (at ${at} s).`);
830
928
  }
831
- return ok({ bytes, width, height, durationSeconds, cover });
929
+ return ok({ bytes, width, height, durationSeconds, cover, loudness, mix });
930
+ }
931
+
932
+ /**
933
+ * The mix's sound brought to `target` on its own: gain and limiter, encoded to AAC, measured; while the encoded file
934
+ * misses the target the gain and the ceiling are corrected and it is encoded again, at most `MIX_ATTEMPTS` times. An
935
+ * error names the last measurement.
936
+ */
937
+ async function finishMix(input: {
938
+ input: string;
939
+ output: string;
940
+ measurement: LoudnessMeasurement;
941
+ target: MixLoudnessTarget;
942
+ signal?: AbortSignal;
943
+ }): Promise<FootageResult<{ attempt: MixAttempt; tries: number; measured: MixLoudness }>> {
944
+ const aborting = input.signal === undefined ? {} : { signal: input.signal };
945
+ let attempt = getFirstMixAttempt(input.measurement, input.target);
946
+ let misses: string[] = [];
947
+ let measured: MixLoudness = { integrated: input.measurement.inputI, truePeak: input.measurement.inputTp };
948
+ const done: MixTry[] = [];
949
+ for (let tries = 1; tries <= MIX_ATTEMPTS; tries += 1) {
950
+ await runFfmpeg("The film's mix", buildMixArgs({ input: input.input, filter: getMixFilter(attempt), output: input.output }), { ...aborting, output: input.output });
951
+ const encoded = await measureLoudness(input.output, input.target, { label: "the film's encoded mix", ...aborting });
952
+ if (encoded.kind === "silent") return err("the film's mix measures silent after its gain and limiter.");
953
+ measured = { integrated: encoded.measurement.inputI, truePeak: encoded.measurement.inputTp };
954
+ misses = findLoudnessProblems(measured, input.target);
955
+ if (misses.length === 0) return ok({ attempt, tries, measured });
956
+ done.push({ attempt, measured });
957
+ if (tries < MIX_ATTEMPTS) attempt = getNextMixAttempt(done, input.measurement.inputI, input.target);
958
+ }
959
+ return err(
960
+ `the film's mix still measures ${describeMixLoudness(measured)} after ${MIX_ATTEMPTS} encodes (last: gain ${formatDb(attempt.gainDb)} dB, ` +
961
+ `limiter at ${formatDb(attempt.ceilingDbtp)} dBTP): ${misses.join("; ")}.`,
962
+ );
963
+ }
964
+
965
+ /** Both measurements of a mix, as an error names them. */
966
+ function describeMixLoudness(measured: MixLoudness): string {
967
+ return `${formatDb(measured.integrated)} LUFS integrated, ${formatDb(measured.truePeak)} dBTP true peak`;
832
968
  }
833
969
 
834
970
  /** What the final pass did to each stream, for its log line. */
@@ -969,7 +1105,7 @@ async function runRenderFootageFilm(film: FootageFilm, options: RenderFootageFil
969
1105
  const edlPath = getEdlPath(options.buildDir, options.isPlaceholder);
970
1106
  let edl: Edl;
971
1107
  if (existsSync(edlPath) && options.isRecut !== true) {
972
- const read = readEdlFile(film, clips.value, edlPath);
1108
+ const read = readEdlFile(film, clips.value, edlPath, options.layout);
973
1109
  if (!read.ok) return read;
974
1110
  edl = read.value;
975
1111
  options.log(`cut: ${edlPath} (${edl.segments.length} segments), as it is; softure-marketing footage --step=cut builds it again.`);
@@ -109,6 +109,8 @@ export interface FootageCharge {
109
109
  keyTerms?: number;
110
110
  /** The paid cache's key of the result (the file name's first part under `<cacheDir>/<film id>/`). */
111
111
  cacheKey: string;
112
+ /** The part of the clip a range isolation cleaned (`isolateSpeechRanges`), in seconds of the trimmed clip; absent: the whole clip. */
113
+ range?: { start: number; end: number };
112
114
  }
113
115
 
114
116
  /** A stage of a footage run that reports progress. */
@@ -15,7 +15,7 @@ import { runProcess } from "./process.js";
15
15
  */
16
16
 
17
17
  /** The audio bitrate of a re-encoded track. */
18
- const AUDIO_BITRATE = "192k";
18
+ export const FINISHED_AUDIO_BITRATE = "192k";
19
19
  /** The quality of a re-encoded picture (x264's constant rate factor; 18 is visually lossless). */
20
20
  const VIDEO_CRF = 18;
21
21
  /** How far before the film's end the latest cover frame is taken, so a frame still exists there. */
@@ -145,26 +145,35 @@ function getVideoEncodeArgs(video: FinishVideo): string[] {
145
145
 
146
146
  /**
147
147
  * The final pass: each stream that is already publishable is copied, the other re-encoded (the picture to H.264 High),
148
- * and the `moov` box moved to the front. A film without sound keeps none. `audioFilter` (the mix's loudness pass)
149
- * re-encodes the sound through it. `video` re-encodes the picture whatever it is, at its preset and CRF (issue #1068).
148
+ * and the `moov` box moved to the front. A film without sound keeps none. `audioFile` (the mix's finished sound, AAC
149
+ * encoded on its own) replaces the render's sound, copied as it is; `audioFilter` re-encodes the sound through it.
150
+ * `video` re-encodes the picture whatever it is, at its preset and CRF (issue #1068).
150
151
  */
151
- export function buildFinishArgs(input: string, facts: Pick<FilmFacts, "video" | "audio">, output: string, options: { audioFilter?: string; video?: FinishVideo } = {}): string[] {
152
+ export function buildFinishArgs(
153
+ input: string,
154
+ facts: Pick<FilmFacts, "video" | "audio">,
155
+ output: string,
156
+ options: { audioFilter?: string; audioFile?: string; video?: FinishVideo } = {},
157
+ ): string[] {
152
158
  const video =
153
159
  options.video !== undefined
154
160
  ? getVideoEncodeArgs(options.video)
155
161
  : isPublishableVideo(facts.video)
156
162
  ? ["-c:v", "copy"]
157
163
  : getVideoEncodeArgs({ preset: "medium", crf: VIDEO_CRF });
158
- const encodeAudio = ["-c:a", "aac", "-b:a", AUDIO_BITRATE, "-ar", String(FOOTAGE_SAMPLE_RATE), "-ac", "2"];
164
+ const encodeAudio = ["-c:a", "aac", "-b:a", FINISHED_AUDIO_BITRATE, "-ar", String(FOOTAGE_SAMPLE_RATE), "-ac", "2"];
165
+ const inputs = options.audioFile === undefined ? ["-i", input] : ["-i", input, "-i", options.audioFile];
159
166
  const audio =
160
167
  facts.audio === null
161
168
  ? []
162
- : options.audioFilter !== undefined
163
- ? ["-map", "0:a:0", "-af", options.audioFilter, ...encodeAudio]
164
- : isPublishableAudio(facts.audio)
165
- ? ["-map", "0:a:0", "-c:a", "copy"]
166
- : ["-map", "0:a:0", ...encodeAudio];
167
- return ["-v", "error", "-y", "-i", input, "-map", "0:v:0", ...video, ...audio, "-movflags", "+faststart", output];
169
+ : options.audioFile !== undefined
170
+ ? ["-map", "1:a:0", "-c:a", "copy"]
171
+ : options.audioFilter !== undefined
172
+ ? ["-map", "0:a:0", "-af", options.audioFilter, ...encodeAudio]
173
+ : isPublishableAudio(facts.audio)
174
+ ? ["-map", "0:a:0", "-c:a", "copy"]
175
+ : ["-map", "0:a:0", ...encodeAudio];
176
+ return ["-v", "error", "-y", ...inputs, "-map", "0:v:0", ...video, ...audio, "-movflags", "+faststart", output];
168
177
  }
169
178
 
170
179
  /** The types of an MP4's top-level boxes, in file order (`ftyp`, `moov`, `mdat`, …). */
@@ -0,0 +1,131 @@
1
+ import { footageCacheKey, getFootageCachePath } from "./cache.js";
2
+ import { getSoundSpan, type Edl } from "./edl.js";
3
+ import type { ClipRole } from "./film.js";
4
+ import { FOOTAGE_SAMPLE_RATE } from "./normalize.js";
5
+
6
+ /**
7
+ * The parts of each talking clip a cut hears, as ranges in whole milliseconds, and how their cleaned voice is laid
8
+ * back over the clip's own sound: pure planning and ffmpeg argument builders for `isolateSpeechRanges` and the
9
+ * per-range estimate, tested without ffmpeg.
10
+ */
11
+
12
+ /** Seconds added on each side of a sound span, so a cut's cross-fade and fades read cleaned sound. */
13
+ export const DEFAULT_SPEECH_RANGE_MARGIN = 0.25;
14
+
15
+ /** A range of a clip's trimmed sound, in whole milliseconds (the cache key's unit). */
16
+ export interface SpeechRange {
17
+ clip: string;
18
+ startMs: number;
19
+ endMs: number;
20
+ }
21
+
22
+ /** Where a range's cleaned sound is laid in its clip, in milliseconds; inside the range. */
23
+ export interface SpeechRangeWindow {
24
+ range: SpeechRange;
25
+ startMs: number;
26
+ endMs: number;
27
+ }
28
+
29
+ export interface SpeechRangeClip {
30
+ id: string;
31
+ role: ClipRole;
32
+ /** Seconds after the trim. */
33
+ duration: number;
34
+ }
35
+
36
+ /**
37
+ * The ranges the cut hears of each talking clip: every segment's sound span (its own clip's or `audio`'s), and the
38
+ * picture clip's own span when `audio.ownVolume` keeps it under, widened by `margin` seconds on each side, clamped to
39
+ * the clip and rounded out to whole milliseconds. Identical ranges and ranges inside another are dropped (the larger
40
+ * one is laid there anyway); the rest are not merged, so a moved segment changes only its own range. Sorted by clip
41
+ * order, then start.
42
+ */
43
+ export function planSpeechRanges(edl: Edl, clips: readonly SpeechRangeClip[], options: { margin?: number } = {}): SpeechRange[] {
44
+ const margin = Math.max(0, options.margin ?? DEFAULT_SPEECH_RANGE_MARGIN);
45
+ const byId = new Map(clips.map((clip) => [clip.id, clip]));
46
+ const found = new Map<string, SpeechRange[]>();
47
+ const add = (span: { clip: string; start: number; end: number }): void => {
48
+ const clip = byId.get(span.clip);
49
+ if (clip === undefined || clip.role !== "talk") return;
50
+ const clipMs = Math.floor(clip.duration * 1000);
51
+ const startMs = Math.max(0, Math.floor((span.start - margin) * 1000));
52
+ const endMs = Math.min(clipMs, Math.ceil((span.end + margin) * 1000));
53
+ if (endMs <= startMs) return;
54
+ const list = found.get(clip.id) ?? [];
55
+ list.push({ clip: clip.id, startMs, endMs });
56
+ found.set(clip.id, list);
57
+ };
58
+ for (const segment of edl.segments) {
59
+ add(getSoundSpan(segment));
60
+ if (segment.audio !== undefined && segment.audio.ownVolume > 0) add(segment);
61
+ }
62
+ const ranges: SpeechRange[] = [];
63
+ for (const clip of clips) {
64
+ const list = (found.get(clip.id) ?? []).sort((a, b) => a.startMs - b.startMs || b.endMs - a.endMs);
65
+ for (const range of list) {
66
+ const isInside = ranges.some((kept) => kept.clip === range.clip && kept.startMs <= range.startMs && kept.endMs >= range.endMs);
67
+ if (!isInside) ranges.push(range);
68
+ }
69
+ }
70
+ return ranges;
71
+ }
72
+
73
+ /**
74
+ * Where each of one clip's ranges is laid: the whole range, except that two overlapping neighbours switch at the
75
+ * middle of their overlap. Takes ranges of one clip, none inside another (as `planSpeechRanges` answers them).
76
+ */
77
+ export function getSpeechRangeWindows(ranges: readonly SpeechRange[]): SpeechRangeWindow[] {
78
+ const sorted = [...ranges].sort((a, b) => a.startMs - b.startMs);
79
+ return sorted.map((range, index) => {
80
+ const before = sorted[index - 1];
81
+ const after = sorted[index + 1];
82
+ const startMs = before !== undefined && before.endMs > range.startMs ? Math.round((range.startMs + before.endMs) / 2) : range.startMs;
83
+ const endMs = after !== undefined && after.startMs < range.endMs ? Math.round((after.startMs + range.endMs) / 2) : range.endMs;
84
+ return { range, startMs, endMs };
85
+ });
86
+ }
87
+
88
+ /** The paid cache of one range's cleaned voice: keyed by the clip's identity, the range in milliseconds and the provider. */
89
+ export function getSpeechRangeCache(filmId: string, clipKey: string, range: SpeechRange, options: { cacheDir: string; provider: string }): { key: string; path: string } {
90
+ const key = footageCacheKey("isolated", { clip: clipKey, range: [range.startMs, range.endMs], provider: options.provider });
91
+ return { key, path: getFootageCachePath(options.cacheDir, filmId, "isolated", key) };
92
+ }
93
+
94
+ const seconds = (ms: number): string => (ms / 1000).toFixed(3);
95
+
96
+ /** One range of a clip's sound (`audio`, read from its start) as mp3, for the isolation provider. */
97
+ export function buildRangeAudioArgs(audio: string, range: Pick<SpeechRange, "startMs" | "endMs">, output: string): string[] {
98
+ return ["-v", "error", "-y", "-ss", seconds(range.startMs), "-t", seconds(range.endMs - range.startMs), "-i", audio, "-vn", "-c:a", "libmp3lame", "-q:a", "2", output];
99
+ }
100
+
101
+ /** A cleaned range laid back: its file, and whether it is mono (panned to both channels at full level). */
102
+ export interface LaidRange {
103
+ window: SpeechRangeWindow;
104
+ file: string;
105
+ isMono: boolean;
106
+ }
107
+
108
+ /**
109
+ * The clip's own sound with each window replaced by its cleaned range: the base muted inside every window, each
110
+ * window cut from its cleaned file and delayed into place, summed without level change. As long as the base; 48 kHz
111
+ * stereo like the speech track; a mono answer goes to both channels as it is, not up-mixed at -3 dB.
112
+ */
113
+ export function buildRangeLayArgs(base: string, laid: readonly LaidRange[], output: string): string[] {
114
+ const format = `aformat=sample_rates=${FOOTAGE_SAMPLE_RATE}:channel_layouts=stereo`;
115
+ if (laid.length === 0) return ["-v", "error", "-y", "-i", base, "-vn", "-af", format, "-c:a", "libmp3lame", "-q:a", "2", output];
116
+ const inputs = ["-i", base];
117
+ const mute = laid.map(({ window }) => `between(t,${seconds(window.startMs)},${seconds(window.endMs)})`).join("+");
118
+ const parts = [`[0:a]${format},volume=0:enable='${mute}'[base]`];
119
+ laid.forEach(({ window, file, isMono }, index) => {
120
+ inputs.push("-i", file);
121
+ const offset = window.startMs - window.range.startMs;
122
+ const length = window.endMs - window.startMs;
123
+ const channels = isMono ? "pan=stereo|c0=c0|c1=c0," : "";
124
+ parts.push(
125
+ `[${index + 1}:a]atrim=start=${seconds(offset)}:end=${seconds(offset + length)},asetpts=PTS-STARTPTS,${channels}${format},adelay=${window.startMs}:all=1[w${index}]`,
126
+ );
127
+ });
128
+ const mixed = laid.map((_, index) => `[w${index}]`).join("");
129
+ parts.push(`[base]${mixed}amix=inputs=${laid.length + 1}:duration=first:normalize=0[speech]`);
130
+ return ["-v", "error", "-y", ...inputs, "-filter_complex", parts.join(";"), "-map", "[speech]", "-c:a", "libmp3lame", "-q:a", "2", output];
131
+ }
package/src/index.ts CHANGED
@@ -104,6 +104,7 @@ export {
104
104
  VIDEO_FORMATS,
105
105
  cameraPose,
106
106
  captionChunks,
107
+ splitAfterKept,
107
108
  crossfadeArgs,
108
109
  fitScale,
109
110
  fitsFrame,
@@ -119,6 +120,7 @@ export {
119
120
  widePose,
120
121
  type CameraPose,
121
122
  type CaptionChunk,
123
+ type CaptionChunkOptions,
122
124
  type CaptionLayout,
123
125
  type CrossfadeInput,
124
126
  type DeviceKind,
@@ -183,6 +185,7 @@ export {
183
185
  DEFAULT_AUTO_THRESHOLD,
184
186
  JUMP_TRANSITIONS,
185
187
  DEFAULT_CUT,
188
+ DEFAULT_CAPTION_KEEP_WITH_NEXT,
186
189
  DEFAULT_FILLERS,
187
190
  DEFAULT_MUSIC_VOLUME,
188
191
  DEFAULT_TRANSITIONS,
@@ -191,6 +194,7 @@ export {
191
194
  CAPTION_STYLES,
192
195
  DEFAULT_EDIT,
193
196
  getDefaultFillers,
197
+ getDeclickLevel,
194
198
  getFootageEdit,
195
199
  type CaptionPreset,
196
200
  type CaptionStyle,
@@ -203,9 +207,33 @@ export {
203
207
  type FootageClip,
204
208
  type FootageFilm,
205
209
  type FootageMusic,
210
+ type FootageSpeech,
206
211
  type FootageTransitions,
207
212
  type MusicSource,
208
213
  } from "./footage/film.js";
214
+ export {
215
+ DECLICK_GATE_FADE,
216
+ DECLICK_IMPULSE_DB,
217
+ DECLICK_LEVELS,
218
+ DECLICK_MAX_IMPULSE,
219
+ DECLICK_SETTINGS,
220
+ DECLICK_WORD_WINDOW,
221
+ buildDeclickDecodeArgs,
222
+ buildDeclickEncodeArgs,
223
+ declickSamples,
224
+ declickSpeech,
225
+ describeDeclick,
226
+ findImpulses,
227
+ getDeclickGain,
228
+ getGateWindows,
229
+ getRunningMedian,
230
+ type ActiveDeclickLevel,
231
+ type DeclickLevel,
232
+ type DeclickReport,
233
+ type DeclickSettings,
234
+ type DeclickSpeechOptions,
235
+ type TimeSpan as DeclickSpan,
236
+ } from "./footage/declick.js";
209
237
  export {
210
238
  MIN_CLIP_SECONDS,
211
239
  describeProbe,
@@ -295,6 +323,7 @@ export {
295
323
  diffEdl,
296
324
  edlSchema,
297
325
  findEdlProblems,
326
+ findSegmentAt,
298
327
  getEdlDuration,
299
328
  getEdlJsonSchema,
300
329
  getSegmentStarts,
@@ -307,6 +336,8 @@ export {
307
336
  type BuildEdlOptions,
308
337
  type Edl,
309
338
  type EdlAudio,
339
+ type EdlCaption,
340
+ type EdlCaptionBounds,
310
341
  type EdlClip,
311
342
  type EdlSegment,
312
343
  type SegmentMotion,
@@ -314,14 +345,23 @@ export {
314
345
  export { findRetakes, splitFilmSentences, splitSentences, type RetakeKind, type RetakeSuggestion, type Sentence } from "./footage/sentences.js";
315
346
  export {
316
347
  CAPTION_SAFE_ZONES,
348
+ applySegmentCaption,
317
349
  findMissingCaptionGlyphs,
350
+ getCaptionBounds,
351
+ getCaptionChunkLayouts,
352
+ getCaptionKeepsWithNext,
353
+ getFootageCaptionChunks,
318
354
  getCaptionPresetLayout,
319
355
  getCaptionSpans,
320
356
  getFootageCaptionLayout,
357
+ getSegmentCaptionLayout,
358
+ isSameCaptionPlace,
321
359
  phraseChunks,
322
360
  type CaptionSpan,
361
+ type CaptionWordsFilm,
323
362
  type MissingCaptionGlyphs,
324
363
  type PhraseChunk,
364
+ type PhraseChunkOptions,
325
365
  type SafeZone,
326
366
  } from "./footage/captions.js";
327
367
  export {
@@ -371,15 +411,34 @@ export {
371
411
  DEFAULT_LOUDNESS,
372
412
  buildLoudnessArgs,
373
413
  buildLoudnessMeasureArgs,
414
+ buildMixArgs,
415
+ findLoudnessProblems,
416
+ formatDb,
417
+ getFirstMixAttempt,
418
+ getLimiterSlope,
374
419
  getLoudnessFilter,
420
+ getMixFilter,
421
+ getMixGain,
422
+ getMixTolerance,
423
+ getNextMixAttempt,
375
424
  isSilent,
376
425
  measureLoudness,
426
+ MIX_ATTEMPTS,
427
+ MIX_ENCODE_HEADROOM,
428
+ MIX_LIMITER_SLOPE,
429
+ MIX_LIMITER_SLOPE_MAX,
430
+ MIX_LOUDNESS_TOLERANCE,
431
+ MIX_RETRY_EDGE_MARGIN,
377
432
  normalizeLoudness,
378
433
  readLoudnessMeasurement,
379
434
  type LoudnessMeasureOutcome,
380
435
  type LoudnessMeasurement,
381
436
  type LoudnessOutcome,
382
437
  type LoudnessTarget,
438
+ type MixAttempt,
439
+ type MixLoudness,
440
+ type MixLoudnessTarget,
441
+ type MixTry,
383
442
  } from "./footage/loudness.js";
384
443
  export {
385
444
  buildCoverArgs,
@@ -414,6 +473,7 @@ export { FFMPEG_FAILURE_STDERR_LINES, formatFfmpegCommand, splitFilterGraph, typ
414
473
  export { buildFfmpegFailureScript, FFMPEG_FAILURE_FILES, writeFfmpegFailure, type FfmpegFailureInput } from "./footage/ffmpeg-failure.js";
415
474
  export {
416
475
  estimateFootageFilm,
476
+ estimateSpeechRanges,
417
477
  findCachedStep,
418
478
  getDownloadPath,
419
479
  type EstimateFootageOptions,
@@ -421,6 +481,19 @@ export {
421
481
  type FootageEstimate,
422
482
  type FootageEstimateItem,
423
483
  } from "./footage/paid.js";
484
+ export {
485
+ DEFAULT_SPEECH_RANGE_MARGIN,
486
+ buildRangeAudioArgs,
487
+ buildRangeLayArgs,
488
+ getSpeechRangeCache,
489
+ getSpeechRangeWindows,
490
+ planSpeechRanges,
491
+ type LaidRange,
492
+ type SpeechRange,
493
+ type SpeechRangeClip,
494
+ type SpeechRangeWindow,
495
+ } from "./footage/speech-ranges.js";
496
+ export { isolateSpeechRanges, type IsolateSpeechRangesOptions, type IsolatedSpeech, type IsolatedSpeechRange } from "./footage/isolate-ranges.js";
424
497
  export { RENDER_MIN_FREE_BYTES, findLowDiskProblems, getFreeBytes, getRenderTempDir, type FreeBytesReader } from "./footage/disk.js";
425
498
  export { DRAFT_FILTERS, REQUIRED_DECODERS, REQUIRED_FILTERS, checkFootageTools, findBrowserProblem, readFfmpegListing, type FootageToolsOptions } from "./footage/tools.js";
426
499
  export {
@@ -431,6 +504,7 @@ export {
431
504
  findFootageMusic,
432
505
  getEdlPath,
433
506
  getFrame,
507
+ getSpeechChainFiles,
434
508
  prepareFootage,
435
509
  getFootageSentences,
436
510
  readEdlFile,
@@ -439,6 +513,7 @@ export {
439
513
  renderFootageFilm,
440
514
  transcribeFootage,
441
515
  type FinishedFootage,
516
+ type FinishedMix,
442
517
  type FootageBrand,
443
518
  type FootageMusicLookup,
444
519
  type FootageResult,