@softure-ai/marketing-kit 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/CHANGELOG.md +75 -0
  2. package/README.md +90 -11
  3. package/dist/cli/footage.d.ts.map +1 -1
  4. package/dist/cli/footage.js +5 -2
  5. package/dist/cli/footage.js.map +1 -1
  6. package/dist/compose/compose.d.ts.map +1 -1
  7. package/dist/compose/compose.js +3 -3
  8. package/dist/compose/compose.js.map +1 -1
  9. package/dist/compose/timeline.d.ts +16 -0
  10. package/dist/compose/timeline.d.ts.map +1 -1
  11. package/dist/compose/timeline.js +17 -1
  12. package/dist/compose/timeline.js.map +1 -1
  13. package/dist/config/config.d.ts.map +1 -1
  14. package/dist/config/config.js +18 -2
  15. package/dist/config/config.js.map +1 -1
  16. package/dist/config/schema.d.ts +16 -0
  17. package/dist/config/schema.d.ts.map +1 -1
  18. package/dist/config/schema.js +50 -3
  19. package/dist/config/schema.js.map +1 -1
  20. package/dist/footage/captions.d.ts +5 -1
  21. package/dist/footage/captions.d.ts.map +1 -1
  22. package/dist/footage/captions.js +6 -2
  23. package/dist/footage/captions.js.map +1 -1
  24. package/dist/footage/compose.d.ts +13 -2
  25. package/dist/footage/compose.d.ts.map +1 -1
  26. package/dist/footage/compose.js +75 -23
  27. package/dist/footage/compose.js.map +1 -1
  28. package/dist/footage/cuts.d.ts +44 -0
  29. package/dist/footage/cuts.d.ts.map +1 -0
  30. package/dist/footage/cuts.js +83 -0
  31. package/dist/footage/cuts.js.map +1 -0
  32. package/dist/footage/draft.d.ts +17 -3
  33. package/dist/footage/draft.d.ts.map +1 -1
  34. package/dist/footage/draft.js +28 -14
  35. package/dist/footage/draft.js.map +1 -1
  36. package/dist/footage/edl.d.ts +2 -1
  37. package/dist/footage/edl.d.ts.map +1 -1
  38. package/dist/footage/edl.js +5 -2
  39. package/dist/footage/edl.js.map +1 -1
  40. package/dist/footage/film.d.ts +42 -3
  41. package/dist/footage/film.d.ts.map +1 -1
  42. package/dist/footage/film.js +7 -1
  43. package/dist/footage/film.js.map +1 -1
  44. package/dist/footage/loudness.d.ts +20 -3
  45. package/dist/footage/loudness.d.ts.map +1 -1
  46. package/dist/footage/loudness.js +36 -18
  47. package/dist/footage/loudness.js.map +1 -1
  48. package/dist/footage/music.d.ts +3 -0
  49. package/dist/footage/music.d.ts.map +1 -1
  50. package/dist/footage/music.js +5 -0
  51. package/dist/footage/music.js.map +1 -1
  52. package/dist/footage/normalize.d.ts +27 -7
  53. package/dist/footage/normalize.d.ts.map +1 -1
  54. package/dist/footage/normalize.js +39 -19
  55. package/dist/footage/normalize.js.map +1 -1
  56. package/dist/footage/picture.d.ts +42 -0
  57. package/dist/footage/picture.d.ts.map +1 -0
  58. package/dist/footage/picture.js +141 -0
  59. package/dist/footage/picture.js.map +1 -0
  60. package/dist/footage/pipeline.d.ts +55 -1
  61. package/dist/footage/pipeline.d.ts.map +1 -1
  62. package/dist/footage/pipeline.js +136 -19
  63. package/dist/footage/pipeline.js.map +1 -1
  64. package/dist/footage/publish.d.ts +4 -2
  65. package/dist/footage/publish.d.ts.map +1 -1
  66. package/dist/footage/publish.js +8 -5
  67. package/dist/footage/publish.js.map +1 -1
  68. package/dist/footage/render.d.ts +17 -2
  69. package/dist/footage/render.d.ts.map +1 -1
  70. package/dist/footage/render.js +62 -13
  71. package/dist/footage/render.js.map +1 -1
  72. package/dist/footage/tonemap-lut.d.ts +45 -0
  73. package/dist/footage/tonemap-lut.d.ts.map +1 -0
  74. package/dist/footage/tonemap-lut.js +100 -0
  75. package/dist/footage/tonemap-lut.js.map +1 -0
  76. package/dist/footage/tools.d.ts +4 -4
  77. package/dist/footage/tools.d.ts.map +1 -1
  78. package/dist/footage/tools.js +4 -4
  79. package/dist/footage/tools.js.map +1 -1
  80. package/dist/index.d.ts +9 -6
  81. package/dist/index.d.ts.map +1 -1
  82. package/dist/index.js +9 -6
  83. package/dist/index.js.map +1 -1
  84. package/docker/footage-worker.Dockerfile +1 -0
  85. package/package.json +1 -1
  86. package/schema/marketing.schema.json +105 -4
  87. package/src/cli/footage.ts +4 -2
  88. package/src/compose/compose.ts +13 -3
  89. package/src/compose/timeline.ts +27 -1
  90. package/src/config/config.ts +18 -2
  91. package/src/config/schema.ts +52 -3
  92. package/src/footage/captions.ts +6 -2
  93. package/src/footage/compose.ts +86 -23
  94. package/src/footage/cuts.ts +104 -0
  95. package/src/footage/draft.ts +43 -14
  96. package/src/footage/edl.ts +5 -2
  97. package/src/footage/film.ts +47 -4
  98. package/src/footage/loudness.ts +38 -16
  99. package/src/footage/music.ts +7 -0
  100. package/src/footage/normalize.ts +47 -22
  101. package/src/footage/picture.ts +179 -0
  102. package/src/footage/pipeline.ts +179 -22
  103. package/src/footage/publish.ts +8 -5
  104. package/src/footage/render.ts +72 -13
  105. package/src/footage/tonemap-lut.ts +109 -0
  106. package/src/footage/tools.ts +4 -4
  107. package/src/index.ts +37 -2
@@ -15,13 +15,15 @@ import { readCachedTranscript, writeCacheFile, writeTranscript } from "./cache.j
15
15
  import { composeFootage, END_CARD_SECONDS, getFootageFilmSeconds } from "./compose.js";
16
16
  import { findMissingCaptionGlyphs, getFootageCaptionLayout } from "./captions.js";
17
17
  import { findLowDiskProblems, getFreeBytes, getRenderTempDir, type FreeBytesReader } from "./disk.js";
18
- import { buildEdl, readEdl, remapWords, validateEdl, type Edl } from "./edl.js";
18
+ import { buildEdl, getEdlDuration, readEdl, remapWords, validateEdl, type Edl } from "./edl.js";
19
19
  import { makePlaceholderTranscript } from "./fake.js";
20
- import { getFootageEdit, type FootageClip, type FootageFilm, type MusicSource } from "./film.js";
20
+ import { chooseCutTransitions, type CutMeasure } from "./cuts.js";
21
+ import { DEFAULT_AUTO_THRESHOLD, getFootageEdit, type FootageClip, type FootageFilm, type MusicSource } from "./film.js";
21
22
  import { getFilmKeyTerms } from "./keyterms.js";
22
- import { DEFAULT_LOUDNESS, normalizeLoudness } from "./loudness.js";
23
- import { duckingKeyframes } from "./music.js";
24
- import { normalizeFootage, type Frame, type NormalizeEncoder } from "./normalize.js";
23
+ import { DEFAULT_LOUDNESS, getLoudnessFilter, measureLoudness, normalizeLoudness } from "./loudness.js";
24
+ import { duckingKeyframes, getDuckingOptions } from "./music.js";
25
+ import { FOOTAGE_FPS, normalizeFootage, type Frame, type NormalizeEncoder } from "./normalize.js";
26
+ import { buildPictureConcatArgs, buildPictureList, buildPicturePieceArgs, getPictureFrames, planFootagePicture } from "./picture.js";
25
27
  import { describeProbe, getTrimmedDuration, probeFootage, type FootageProbe } from "./probe.js";
26
28
  import {
27
29
  getClipCacheKey,
@@ -49,7 +51,7 @@ import {
49
51
  readTopLevelBoxes,
50
52
  } from "./publish.js";
51
53
  import { describePaidSteps, toPaidAudio, type FootageCharge, type FootageProgress, type FootageProviders, type FootageStage, type PaidAudio, type PaidStep } from "./providers.js";
52
- import { buildAudioArgs, buildCutPreviewArgs, buildLastFrameArgs, buildMusicArgs, buildSpeechArgs } from "./render.js";
54
+ import { buildAudioArgs, buildCutPreviewArgs, buildLastFrameArgs, buildMusicArgs, buildSpeechArgs, getSpeechCutOptions } from "./render.js";
53
55
 
54
56
  export { getEdlPath } from "./paid.js";
55
57
 
@@ -369,20 +371,77 @@ export function getFootageSentences(clips: PreparedClip[], words: Record<string,
369
371
  return splitFilmSentences(clips.map((clip) => ({ id: clip.clip.id, role: clip.clip.role, words: words[clip.clip.id] ?? [] })));
370
372
  }
371
373
 
372
- /** The cut from the transcripts, written to the EDL file (replacing what is there); `keepSentences` keeps only those sentences. */
373
- export function cutFootage(film: FootageFilm, clips: PreparedClip[], words: Record<string, TimedWord[]>, edlPath: string, options: { keepSentences?: readonly string[] } = {}): Edl {
374
- const edl = buildEdl(
374
+ function buildFilmEdl(film: FootageFilm, clips: PreparedClip[], words: Record<string, TimedWord[]>, keepSentences: readonly string[] | undefined): Edl {
375
+ return buildEdl(
375
376
  film.id,
376
377
  clips.map((clip) => ({ id: clip.clip.id, role: clip.clip.role, duration: clip.duration, words: words[clip.clip.id] ?? [] })),
377
378
  film.cut,
378
379
  film.transitions,
379
- { isKenBurns: getFootageEdit(film).kenBurnsPercent > 0, ...(options.keepSentences === undefined ? {} : { keepSentences: options.keepSentences }) },
380
+ { isKenBurns: getFootageEdit(film).kenBurnsPercent > 0, ...(keepSentences === undefined ? {} : { keepSentences }) },
380
381
  );
382
+ }
383
+
384
+ function writeEdl(edl: Edl, edlPath: string): void {
381
385
  mkdirSync(dirname(edlPath), { recursive: true });
382
386
  writeFileSync(edlPath, `${JSON.stringify(edl, null, 2)}\n`);
387
+ }
388
+
389
+ /**
390
+ * The cut from the transcripts, written to the EDL file (replacing what is there); `keepSentences` keeps only those
391
+ * sentences. Sync, so it never looks at the frames: with `transitions.jump: "auto"` the jump cuts stay plain cuts, and
392
+ * `cutFootageFilm` is the step that chooses them.
393
+ */
394
+ export function cutFootage(film: FootageFilm, clips: PreparedClip[], words: Record<string, TimedWord[]>, edlPath: string, options: { keepSentences?: readonly string[] } = {}): Edl {
395
+ const edl = buildFilmEdl(film, clips, words, options.keepSentences);
396
+ writeEdl(edl, edlPath);
383
397
  return edl;
384
398
  }
385
399
 
400
+ export interface CutFootageFilmOptions {
401
+ /** The sentence ids to keep; absent keeps all. */
402
+ keepSentences?: readonly string[];
403
+ signal?: AbortSignal;
404
+ }
405
+
406
+ /** The file next to the EDL that records what the auto transitions measured. */
407
+ export const CUTS_FILE = "cuts.json";
408
+
409
+ /**
410
+ * The cut as `cutFootage` builds it, with `transitions.jump: "auto"` resolved from the frames of the normalized clips
411
+ * (`chooseCutTransitions`): each jump cut gets `zoom` or `fade`, and what it measured is answered and written to
412
+ * `cuts.json` next to the EDL, so an app or an agent sees why. Otherwise the same as `cutFootage`, and `cuts` is empty.
413
+ */
414
+ export async function cutFootageFilm(
415
+ film: FootageFilm,
416
+ clips: PreparedClip[],
417
+ words: Record<string, TimedWord[]>,
418
+ edlPath: string,
419
+ options: CutFootageFilmOptions = {},
420
+ ): Promise<{ edl: Edl; cuts: CutMeasure[] }> {
421
+ let edl = buildFilmEdl(film, clips, words, options.keepSentences);
422
+ let cuts: CutMeasure[] = [];
423
+ const cutsPath = join(dirname(edlPath), CUTS_FILE);
424
+ rmSync(cutsPath, { force: true });
425
+ if (film.transitions.jump === "auto") {
426
+ const chosen = await chooseCutTransitions(edl, Object.fromEntries(clips.map((clip) => [clip.clip.id, clip.file])), {
427
+ workDir: join(dirname(edlPath), "cut-frames"),
428
+ ...(film.transitions.autoThreshold === undefined ? {} : { threshold: film.transitions.autoThreshold }),
429
+ ...(options.signal === undefined ? {} : { signal: options.signal }),
430
+ });
431
+ edl = chosen.edl;
432
+ cuts = chosen.cuts;
433
+ mkdirSync(dirname(cutsPath), { recursive: true });
434
+ writeFileSync(cutsPath, `${JSON.stringify({ threshold: film.transitions.autoThreshold ?? DEFAULT_AUTO_THRESHOLD, cuts }, null, 2)}\n`);
435
+ }
436
+ writeEdl(edl, edlPath);
437
+ return { edl, cuts };
438
+ }
439
+
440
+ /** One line per auto-chosen cut, for a log: `s4: zoom (difference 0.031)`. */
441
+ export function describeCutMeasures(cuts: CutMeasure[]): string[] {
442
+ return cuts.map((cut) => `${cut.segment}: ${cut.transition} (difference ${cut.difference.toFixed(3)})`);
443
+ }
444
+
386
445
  /** The EDL file as written by `cutFootage` or edited by hand, checked against the clips. */
387
446
  export function readEdlFile(film: FootageFilm, clips: PreparedClip[], edlPath: string): FootageResult<Edl> {
388
447
  let json: unknown;
@@ -486,6 +545,22 @@ export interface RenderFootageOptions extends FootageRunOptions {
486
545
  tempDir?: string;
487
546
  /** Reads a folder's free bytes for the disk check; injected in tests. */
488
547
  readFreeBytes?: FreeBytesReader;
548
+ /**
549
+ * hyperframes' capture workers (`--workers`), each a Chrome of its own; by default hyperframes' choice (one in its
550
+ * low-memory profile). A positive whole number.
551
+ */
552
+ workers?: number;
553
+ /**
554
+ * hyperframes' low-memory profile (`--low-memory-mode`: one worker, screenshot capture): true forces it on, false
555
+ * off; by default hyperframes turns it on by itself on a machine (or cgroup) of 8 GB or less.
556
+ */
557
+ lowMemory?: boolean;
558
+ /**
559
+ * The encoder's constant rate factor (hyperframes' `--crf`, a whole number from 0 to 51), overriding the quality
560
+ * tier's CRF (`draft` 28, `standard` 18, `high` 15); the x264 preset stays the tier's (`ultrafast`, `medium`,
561
+ * `slow`). `quality: "draft"` with `crf: 18` encodes fast at the published quality. By default the tier's CRF.
562
+ */
563
+ crf?: number;
489
564
  }
490
565
 
491
566
  export type RenderedFootage = { kind: "dry-run"; steps: PaidStep[] } | ({ kind: "rendered"; output: string; seconds: number; edlPath: string } & FinishedFootage);
@@ -525,7 +600,7 @@ export async function renderFootage(
525
600
  const rawSpeechFile = loudness === null ? speechFile : join(assetsDir, "speech.raw.mp3");
526
601
  const ffmpegTrack = (label: string, output: string, args: () => string[]) => (onSeconds: (done: number) => void) => runFfmpeg(label, args(), { ...aborting, output, onSeconds });
527
602
  const tracks: { item: string; seconds: number; run: (onSeconds: (done: number) => void) => Promise<void> }[] = [
528
- { item: "speech", seconds, run: ffmpegTrack("The speech track", rawSpeechFile, () => buildSpeechArgs(edl, transcript.speech, rawSpeechFile)) },
603
+ { item: "speech", seconds, run: ffmpegTrack("The speech track", rawSpeechFile, () => buildSpeechArgs(edl, transcript.speech, rawSpeechFile, getSpeechCutOptions(getFootageEdit(film)))) },
529
604
  ];
530
605
  if (loudness !== null) {
531
606
  tracks.push({
@@ -542,13 +617,21 @@ export async function renderFootage(
542
617
  });
543
618
  }
544
619
  if (musicFile !== null) {
545
- const keyframes = duckingKeyframes(words, { volume: film.music.volume, duckedVolume: film.music.duckedVolume });
620
+ const keyframes = duckingKeyframes(words, getDuckingOptions(film.music));
546
621
  const output = join(assetsDir, "music.mp3");
547
622
  tracks.push({ item: "music", seconds, run: ffmpegTrack("The music track", output, () => buildMusicArgs({ input: musicFile, seconds, keyframes, output })) });
548
623
  }
549
624
  // A single frame: counted as a moment of work so the stage's bar still ends at its total.
550
625
  const lastFrame = join(assetsDir, "last.jpg");
551
626
  tracks.push({ item: "last frame", seconds: 0.1, run: ffmpegTrack("The end card's backdrop", lastFrame, () => buildLastFrameArgs(edl, clipFiles, lastFrame)) });
627
+ // The cut's picture in film time, one segment after the other: the composition plays it as one <video> (#1030).
628
+ const pictureFile = join(assetsDir, "picture.mp4");
629
+ const pictureSeconds = getEdlDuration(edl);
630
+ tracks.push({
631
+ item: "picture",
632
+ seconds: pictureSeconds,
633
+ run: (onSeconds) => buildPicture({ film, edl, clips: clipFiles, output: pictureFile, ...(options.normalize === undefined ? {} : { encoder: options.normalize }), ...aborting, onSeconds }),
634
+ });
552
635
  const assetProgress = createStageProgress(options.onProgress, "assets", tracks);
553
636
  for (const [index, track] of tracks.entries()) {
554
637
  assetProgress(index, 0);
@@ -589,6 +672,7 @@ export async function renderFootage(
589
672
  fonts,
590
673
  assets: {
591
674
  clips: Object.fromEntries(clips.map((clip) => [clip.clip.id, `clips/${clip.clip.id}.mp4`])),
675
+ picture: "assets/picture.mp4",
592
676
  speech: "assets/speech.mp3",
593
677
  music: musicAsset,
594
678
  lastFrame: "assets/last.jpg",
@@ -622,13 +706,14 @@ export interface FinishedFootage {
622
706
  }
623
707
 
624
708
  /**
625
- * The final pass over hyperframes' render: publishable codecs and fast start (stream copy where the render already
626
- * matches), checked with ffprobe, then the cover and the file's facts.
709
+ * The final pass over hyperframes' render: the mix normalized to `film.mixLoudness` (measured on the render, one
710
+ * linear gain; the picture still copied), publishable codecs and fast start (stream copy where the render already
711
+ * matches), checked with ffprobe, then the cover and the file's facts. Exported for tests only.
627
712
  */
628
- async function finishFootage(
629
- film: FootageFilm,
713
+ export async function finishFootage(
714
+ film: Pick<FootageFilm, "mixLoudness" | "cover">,
630
715
  input: { renderFile: string; seconds: number; coverAt: number },
631
- options: RenderFootageOptions,
716
+ options: Pick<RenderFootageOptions, "log" | "signal" | "output" | "onProgress">,
632
717
  ): Promise<FootageResult<FinishedFootage>> {
633
718
  const { log, signal, output } = options;
634
719
  const aborting = signal === undefined ? {} : { signal };
@@ -636,8 +721,21 @@ async function finishFootage(
636
721
  progress(0, 0);
637
722
  const rendered = await probeFilmFacts(input.renderFile, aborting);
638
723
  if (!rendered.ok) return err(`the rendered film: ${rendered.error}`);
639
- const isCopied = isPublishableVideo(rendered.facts.video) && (rendered.facts.audio === null || isPublishableAudio(rendered.facts.audio));
640
- await runFfmpeg("The film's final pass", buildFinishArgs(input.renderFile, rendered.facts, output), { ...aborting, output, onSeconds: (done) => progress(0, done) });
724
+ const mixLoudness = film.mixLoudness === undefined ? DEFAULT_LOUDNESS : film.mixLoudness;
725
+ let audioFilter: string | undefined;
726
+ if (mixLoudness !== null && rendered.facts.audio !== null) {
727
+ const measured = await measureLoudness(input.renderFile, mixLoudness, { label: "the film's mix", ...aborting });
728
+ if (measured.kind === "silent") {
729
+ log("loudness: the film's mix is silent, left as it is.");
730
+ } else {
731
+ audioFilter = getLoudnessFilter(measured.measurement, mixLoudness);
732
+ log(`loudness: the film's mix measured ${measured.measurement.inputI.toFixed(1)} LUFS, normalized to ${mixLoudness.target} LUFS (true peak ${mixLoudness.truePeak} dBTP).`);
733
+ }
734
+ }
735
+ const isVideoCopied = isPublishableVideo(rendered.facts.video);
736
+ const isAudioCopied = audioFilter === undefined && (rendered.facts.audio === null || isPublishableAudio(rendered.facts.audio));
737
+ const finishArgs = buildFinishArgs(input.renderFile, rendered.facts, output, audioFilter === undefined ? {} : { audioFilter });
738
+ await runFfmpeg("The film's final pass", finishArgs, { ...aborting, output, onSeconds: (done) => progress(0, done) });
641
739
  rmSync(input.renderFile, { force: true });
642
740
  const finished = await probeFilmFacts(output, aborting);
643
741
  if (!finished.ok) return err(`the finished film: ${finished.error}`);
@@ -645,7 +743,8 @@ async function finishFootage(
645
743
  if (problems.length > 0) return err(`the finished film ${output} is not publishable: ${problems.join("; ")}.`);
646
744
  progress(0, input.seconds);
647
745
  const { bytes, width, height, durationSeconds } = finished.facts;
648
- log(`finish: ${isCopied ? "streams copied" : "re-encoded"} to H.264 High, AAC 48 kHz stereo, fast start; ${width}×${height}, ${durationSeconds.toFixed(2)} s, ${bytes} bytes.`);
746
+ const streams = isVideoCopied && isAudioCopied ? "streams copied" : isVideoCopied ? "picture copied, sound re-encoded" : "re-encoded";
747
+ log(`finish: ${streams} to H.264 High, AAC 48 kHz stereo, fast start; ${width}×${height}, ${durationSeconds.toFixed(2)} s, ${bytes} bytes.`);
649
748
  let cover: string | null = null;
650
749
  if (film.cover !== null) {
651
750
  cover = getCoverPath(output);
@@ -666,6 +765,59 @@ export function readRenderPercent(line: string): number | null {
666
765
  return percent >= 0 && percent <= 100 ? percent : null;
667
766
  }
668
767
 
768
+ /**
769
+ * hyperframes' render arguments: the quality, the encoder's CRF, the output and the memory settings the caller set.
770
+ * Throws on invalid `workers` or `crf`.
771
+ */
772
+ export function getHyperframesRenderArgs(options: Pick<RenderFootageOptions, "quality" | "workers" | "lowMemory" | "crf">, output: string): string[] {
773
+ const args = ["render", "--quality", options.quality];
774
+ if (options.crf !== undefined) {
775
+ if (!Number.isInteger(options.crf) || options.crf < 0 || options.crf > 51) throw new Error(`crf must be a whole number from 0 to 51, got ${options.crf}.`);
776
+ args.push("--crf", String(options.crf));
777
+ }
778
+ if (options.workers !== undefined) {
779
+ if (!Number.isInteger(options.workers) || options.workers < 1) throw new Error(`workers must be a positive whole number, got ${options.workers}.`);
780
+ args.push("--workers", String(options.workers));
781
+ }
782
+ if (options.lowMemory !== undefined) args.push(options.lowMemory ? "--low-memory-mode" : "--no-low-memory-mode");
783
+ return [...args, "-o", output];
784
+ }
785
+
786
+ interface BuildPictureInput {
787
+ film: FootageFilm;
788
+ edl: Edl;
789
+ clips: Record<string, string>;
790
+ output: string;
791
+ encoder?: NormalizeEncoder;
792
+ signal?: AbortSignal;
793
+ onSeconds: (done: number) => void;
794
+ }
795
+
796
+ /** Renders the picture piece by piece (one ffmpeg at a time) and joins the pieces; the pieces are removed after the join. */
797
+ async function buildPicture(input: BuildPictureInput): Promise<void> {
798
+ const pieces = planFootagePicture(input.film, input.edl);
799
+ const dir = join(dirname(input.output), "picture");
800
+ rmSync(dir, { recursive: true, force: true });
801
+ mkdirSync(dir, { recursive: true });
802
+ const aborting = input.signal === undefined ? {} : { signal: input.signal };
803
+ const files: string[] = [];
804
+ for (const [index, piece] of pieces.entries()) {
805
+ const file = join(dir, `${String(index).padStart(3, "0")}.mp4`);
806
+ const done = piece.frame / FOOTAGE_FPS;
807
+ await runFfmpeg(`The picture of segment ${index + 1}`, buildPicturePieceArgs({ piece, clips: input.clips, output: file, ...(input.encoder === undefined ? {} : { encoder: input.encoder }) }), {
808
+ ...aborting,
809
+ output: file,
810
+ onSeconds: (seconds) => input.onSeconds(done + seconds),
811
+ });
812
+ files.push(file);
813
+ }
814
+ const list = join(dir, "pieces.txt");
815
+ writeFileSync(list, buildPictureList(files));
816
+ await runFfmpeg("The picture's join", buildPictureConcatArgs(list, input.output), { ...aborting, output: input.output });
817
+ rmSync(dir, { recursive: true, force: true });
818
+ input.onSeconds(getPictureFrames(pieces) / FOOTAGE_FPS);
819
+ }
820
+
669
821
  /** The hyperframes render as a child process: its lines go to the log (progress every 10 %), its percentages to `onProgress`. */
670
822
  async function runHyperframesRender(options: RenderFootageOptions, output: string, seconds: number): Promise<FootageResult<null>> {
671
823
  const progress = createStageProgress(options.onProgress, "render", [{ item: "film", seconds }]);
@@ -694,7 +846,7 @@ async function runHyperframesRender(options: RenderFootageOptions, output: strin
694
846
  }
695
847
  const result = await runProcess({
696
848
  command: process.execPath,
697
- args: [getHyperframesBin(), "render", "--quality", options.quality, "-o", output],
849
+ args: [getHyperframesBin(), ...getHyperframesRenderArgs(options, output)],
698
850
  cwd: options.buildDir,
699
851
  env,
700
852
  ...(options.signal === undefined ? {} : { signal: options.signal }),
@@ -733,8 +885,13 @@ export async function renderFootageFilm(film: FootageFilm, options: RenderFootag
733
885
  edl = read.value;
734
886
  options.log(`cut: ${edlPath} (${edl.segments.length} segments), as it is; softure-marketing footage --step=cut builds it again.`);
735
887
  } else {
736
- edl = cutFootage(film, clips.value, transcript.value.words, edlPath, options.keepSentences === undefined ? {} : { keepSentences: options.keepSentences });
888
+ const cut = await cutFootageFilm(film, clips.value, transcript.value.words, edlPath, {
889
+ ...(options.keepSentences === undefined ? {} : { keepSentences: options.keepSentences }),
890
+ ...(options.signal === undefined ? {} : { signal: options.signal }),
891
+ });
892
+ edl = cut.edl;
737
893
  options.log(`cut: ${edl.segments.length} segments written to ${edlPath}.`);
894
+ for (const line of describeCutMeasures(cut.cuts)) options.log(` auto transition ${line}`);
738
895
  }
739
896
  return renderFootage(film, clips.value, transcript.value, edl, options);
740
897
  }
@@ -98,18 +98,21 @@ export function isPublishableAudio(audio: NonNullable<FilmFacts["audio"]>): bool
98
98
 
99
99
  /**
100
100
  * The final pass: each stream that is already publishable is copied, the other re-encoded, and the `moov` box moved
101
- * to the front. A film without sound keeps none.
101
+ * to the front. A film without sound keeps none. `audioFilter` (the mix's loudness pass) re-encodes the sound through it.
102
102
  */
103
- export function buildFinishArgs(input: string, facts: Pick<FilmFacts, "video" | "audio">, output: string): string[] {
103
+ export function buildFinishArgs(input: string, facts: Pick<FilmFacts, "video" | "audio">, output: string, options: { audioFilter?: string } = {}): string[] {
104
104
  const video = isPublishableVideo(facts.video)
105
105
  ? ["-c:v", "copy"]
106
106
  : ["-c:v", "libx264", "-profile:v", "high", "-pix_fmt", "yuv420p", "-preset", "medium", "-crf", String(VIDEO_CRF)];
107
+ const encodeAudio = ["-c:a", "aac", "-b:a", AUDIO_BITRATE, "-ar", String(FOOTAGE_SAMPLE_RATE), "-ac", "2"];
107
108
  const audio =
108
109
  facts.audio === null
109
110
  ? []
110
- : isPublishableAudio(facts.audio)
111
- ? ["-map", "0:a:0", "-c:a", "copy"]
112
- : ["-map", "0:a:0", "-c:a", "aac", "-b:a", AUDIO_BITRATE, "-ar", String(FOOTAGE_SAMPLE_RATE), "-ac", "2"];
111
+ : options.audioFilter !== undefined
112
+ ? ["-map", "0:a:0", "-af", options.audioFilter, ...encodeAudio]
113
+ : isPublishableAudio(facts.audio)
114
+ ? ["-map", "0:a:0", "-c:a", "copy"]
115
+ : ["-map", "0:a:0", ...encodeAudio];
113
116
  return ["-v", "error", "-y", "-i", input, "-map", "0:v:0", ...video, ...audio, "-movflags", "+faststart", output];
114
117
  }
115
118
 
@@ -1,21 +1,42 @@
1
1
  import { getSoundSpan, type Edl } from "./edl.js";
2
+ import { DEFAULT_EDIT, type FootageEdit } from "./film.js";
2
3
  import { FOOTAGE_SAMPLE_RATE } from "./normalize.js";
3
4
  import { volumeExpression, type VolumeKeyframe } from "./music.js";
4
5
 
5
6
  /** ffmpeg steps of the footage render: pure argument builders, tested without ffmpeg (`runFfmpeg` in `process.ts` runs them). */
6
7
 
7
- /** A short fade at every cut of the speech track, so a cut never clicks. */
8
- const CUT_FADE_SECONDS = 0.02;
9
8
  const MUSIC_FADE_IN = 1;
10
9
  const MUSIC_FADE_OUT = 1.5;
11
10
 
12
11
  const r3 = (value: number): number => Math.round(value * 1000) / 1000 || 0;
12
+ const r4 = (value: number): number => Math.round(value * 10000) / 10000 || 0;
13
+
14
+ /** How the speech track is joined at the cuts. */
15
+ export interface SpeechCutOptions {
16
+ /** The fade at every cut, in seconds; default 0.02 (`DEFAULT_EDIT.cutAudioFadeMs`), so a cut never clicks. */
17
+ fadeSeconds?: number;
18
+ /**
19
+ * Cross-fade the two sides of every cut over `fadeSeconds`, centred on the cut: each side runs on into its source
20
+ * for half the fade (silence where the source ends). By default each side fades out and in (a short dip).
21
+ */
22
+ isCrossfade?: boolean;
23
+ }
24
+
25
+ /** The film's cut sound settings (`edit.cutAudioFadeMs`, `edit.isCutAudioCrossfade`) as `buildSpeechArgs` takes them. */
26
+ export function getSpeechCutOptions(edit: Pick<FootageEdit, "cutAudioFadeMs" | "isCutAudioCrossfade">): Required<SpeechCutOptions> {
27
+ return { fadeSeconds: r4(edit.cutAudioFadeMs / 1000), isCrossfade: edit.isCutAudioCrossfade };
28
+ }
13
29
 
14
30
  /**
15
31
  * The speech track: every segment's sound cut from its sound span (its own clip's, or `audio`'s: a J/L-cut or b-roll
16
- * over speech) and joined in order; `audio.ownVolume` keeps the picture clip's own sound under it.
32
+ * over speech) and joined in order; `audio.ownVolume` keeps the picture clip's own sound under it. Each cut fades over
33
+ * `fadeSeconds`: a short dip, or with `isCrossfade` a cross-fade centred on the cut. Either way the track is exactly
34
+ * the cut's length and every word stays under its picture.
17
35
  */
18
- export function buildSpeechArgs(edl: Edl, sources: Record<string, string>, output: string): string[] {
36
+ export function buildSpeechArgs(edl: Edl, sources: Record<string, string>, output: string, options: SpeechCutOptions = {}): string[] {
37
+ const fadeSeconds = options.fadeSeconds ?? r4(DEFAULT_EDIT.cutAudioFadeMs / 1000);
38
+ if (!(fadeSeconds > 0)) throw new Error(`The speech track's cut fade must be above 0 seconds, got ${String(fadeSeconds)}.`);
39
+ const isCrossfade = options.isCrossfade === true && edl.segments.length > 1;
19
40
  const inputs: string[] = [];
20
41
  const parts: string[] = [];
21
42
  const getSource = (clip: string): string => {
@@ -23,30 +44,68 @@ export function buildSpeechArgs(edl: Edl, sources: Record<string, string>, outpu
23
44
  if (source === undefined) throw new Error(`No audio for clip "${clip}" of the EDL.`);
24
45
  return source;
25
46
  };
47
+ const addInput = (clip: string): number => {
48
+ const input = inputs.length / 2;
49
+ inputs.push("-i", getSource(clip));
50
+ return input;
51
+ };
52
+ const format = `asetpts=PTS-STARTPTS,aformat=sample_rates=${FOOTAGE_SAMPLE_RATE}:channel_layouts=stereo`;
26
53
  const cut = (input: number, span: { start: number; end: number }, duration: number): string =>
27
- `[${input}:a]atrim=start=${span.start}:end=${span.end},asetpts=PTS-STARTPTS,aformat=sample_rates=${FOOTAGE_SAMPLE_RATE}:channel_layouts=stereo,apad=whole_dur=${duration}`;
54
+ `[${input}:a]atrim=start=${span.start}:end=${span.end},${format},apad=whole_dur=${duration}`;
55
+ const last = edl.segments.length - 1;
56
+ const half = r4(fadeSeconds / 2);
28
57
  edl.segments.forEach((segment, index) => {
29
58
  const sound = getSoundSpan(segment);
30
59
  const duration = r3(segment.end - segment.start);
31
- const fades = `afade=t=in:d=${CUT_FADE_SECONDS},afade=t=out:st=${r3(Math.max(0, duration - CUT_FADE_SECONDS))}:d=${CUT_FADE_SECONDS}`;
32
- const input = inputs.length / 2;
33
- inputs.push("-i", getSource(sound.clip));
34
60
  const own = segment.audio?.ownVolume ?? 0;
61
+ if (isCrossfade) {
62
+ // Half the fade on each side of the cut, read on into the source; where the source starts too late, silence.
63
+ const head = index === 0 ? 0 : half;
64
+ const tail = index === last ? 0 : half;
65
+ const length = r4(duration + head + tail);
66
+ const extend = (input: number, span: { start: number; end: number }): string => {
67
+ const lead = Math.min(head, span.start);
68
+ const silence = Math.round((head - lead) * 1000);
69
+ const delay = silence > 0 ? `,adelay=${silence}:all=1` : "";
70
+ return `[${input}:a]atrim=start=${r4(span.start - lead)}:end=${r4(span.start + duration + tail)},${format}${delay},apad=whole_dur=${length},atrim=end=${length}`;
71
+ };
72
+ const input = addInput(sound.clip);
73
+ if (own <= 0) {
74
+ parts.push(`${extend(input, sound)}[a${index}]`);
75
+ return;
76
+ }
77
+ const ownInput = addInput(segment.clip);
78
+ parts.push(`${extend(input, sound)}[s${index}]`);
79
+ parts.push(`${extend(ownInput, segment)},volume=${own}[o${index}]`);
80
+ parts.push(`[s${index}][o${index}]amix=inputs=2:duration=first:normalize=0[a${index}]`);
81
+ return;
82
+ }
83
+ const fades = `afade=t=in:d=${fadeSeconds},afade=t=out:st=${r3(Math.max(0, duration - fadeSeconds))}:d=${fadeSeconds}`;
84
+ const input = addInput(sound.clip);
35
85
  // A borrowed span may be a few milliseconds longer than the picture: it is cut to the picture's length.
36
86
  const fit = segment.audio === undefined ? "" : `,atrim=end=${duration}`;
37
87
  if (own <= 0) {
38
88
  parts.push(`${cut(input, sound, duration)}${fit},${fades}[a${index}]`);
39
89
  return;
40
90
  }
41
- const ownInput = inputs.length / 2;
42
- inputs.push("-i", getSource(segment.clip));
91
+ const ownInput = addInput(segment.clip);
43
92
  parts.push(`${cut(input, sound, duration)},atrim=end=${duration}[s${index}]`);
44
93
  parts.push(`${cut(ownInput, segment, duration)},atrim=end=${duration},volume=${own}[o${index}]`);
45
94
  parts.push(`[s${index}][o${index}]amix=inputs=2:duration=first:normalize=0,${fades}[a${index}]`);
46
95
  });
47
- const joined = edl.segments.map((_, index) => `[a${index}]`).join("");
48
- const graph = `${parts.join(";")};${joined}concat=n=${edl.segments.length}:v=0:a=1[speech]`;
49
- return ["-v", "error", "-y", ...inputs, "-filter_complex", graph, "-map", "[speech]", "-c:a", "libmp3lame", "-q:a", "2", output];
96
+ if (isCrossfade) {
97
+ let joined = "a0";
98
+ for (let index = 1; index <= last; index++) {
99
+ parts.push(`[${joined}][a${index}]acrossfade=d=${fadeSeconds}:c1=tri:c2=tri[x${index}]`);
100
+ joined = `x${index}`;
101
+ }
102
+ const total = r3(edl.segments.reduce((sum, segment) => sum + r3(segment.end - segment.start), 0));
103
+ parts.push(`[${joined}]afade=t=in:d=${fadeSeconds},afade=t=out:st=${r3(Math.max(0, total - fadeSeconds))}:d=${fadeSeconds}[speech]`);
104
+ } else {
105
+ const joined = edl.segments.map((_, index) => `[a${index}]`).join("");
106
+ parts.push(`${joined}concat=n=${edl.segments.length}:v=0:a=1[speech]`);
107
+ }
108
+ return ["-v", "error", "-y", ...inputs, "-filter_complex", parts.join(";"), "-map", "[speech]", "-c:a", "libmp3lame", "-q:a", "2", output];
50
109
  }
51
110
 
52
111
  /** The music track: looped to the film's length, ducked by the keyframes, faded in and out. */
@@ -0,0 +1,109 @@
1
+ import { existsSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
2
+ import { join } from "node:path";
3
+
4
+ import type { HdrKind } from "./probe.js";
5
+ import { runFfmpeg } from "./process.js";
6
+
7
+ /**
8
+ * The HDR to SDR tone map as a 3D LUT. The float chain (zscale to linear light, Hable curve, zscale back to BT.709)
9
+ * costs about a second per footage second at 1080x1920 on one CPU; the same chain is run once over an identity
10
+ * lattice of HDR R'G'B' values and written as a `.cube` file, which `lut3d` then applies per frame in 16-bit at about
11
+ * half the cost. A 3D LUT keeps what a per-channel curve would lose: HLG's system gamma and Hable's scaling both
12
+ * depend on all three channels.
13
+ *
14
+ * The LUT ends in linear BT.709 light, unclamped (negative where a colour lies outside BT.709), and the BT.709
15
+ * transfer is applied after it: the mapping up to there is smooth, so the lattice interpolates it closely, while the
16
+ * transfer's steep start and the gamut edge would not interpolate.
17
+ */
18
+
19
+ /** Points per axis: 65 is as fast as 33 in `lut3d` and closer to the float chain. */
20
+ export const TONE_MAP_LUT_SIZE = 65;
21
+ /** Bumped when the curve or the lattice changes, so a build folder's older LUT is not reused. */
22
+ const TONE_MAP_LUT_VERSION = 1;
23
+
24
+ /** The HDR transfers the tone map reads, by ffmpeg's names. */
25
+ export type ToneMapTransfer = "smpte2084" | "arib-std-b67";
26
+
27
+ const TRANSFER_BY_KIND: Record<HdrKind, ToneMapTransfer> = { pq: "smpte2084", hlg: "arib-std-b67", "dolby-vision": "arib-std-b67" };
28
+
29
+ /** The transfer to tone-map from: the stream's own when it is PQ or HLG, else the kind's (Dolby Vision through its HLG base layer). */
30
+ export function getToneMapTransfer(hdr: HdrKind, transfer: string | null): ToneMapTransfer {
31
+ return transfer === "smpte2084" || transfer === "arib-std-b67" ? transfer : TRANSFER_BY_KIND[hdr];
32
+ }
33
+
34
+ /** The float stages between linear BT.2020 light and linear BT.709 light: the Hable curve, no desaturation. */
35
+ export const HABLE_STAGES = ["format=gbrpf32le", "zscale=p=bt709", "tonemap=tonemap=hable:desat=0"] as const;
36
+
37
+ /** The LUT's file name; the file sits in the folder ffmpeg runs in, so the filter graph needs no path escaping. */
38
+ export function getToneMapLutName(transfer: ToneMapTransfer): string {
39
+ return `tonemap-v${TONE_MAP_LUT_VERSION}-${transfer}.cube`;
40
+ }
41
+
42
+ /** Every lattice point as 16-bit R, G, B (`rgb48le`), red changing fastest, then green, then blue, as `.cube` lists them. */
43
+ export function buildIdentityLattice(size: number): Buffer {
44
+ if (!Number.isInteger(size) || size < 2) throw new Error(`A LUT needs at least 2 whole points per axis, got ${size}.`);
45
+ const levels = Array.from({ length: size }, (_, i) => Math.round((i / (size - 1)) * 65_535));
46
+ const lattice = Buffer.alloc(size ** 3 * 6);
47
+ let offset = 0;
48
+ for (const blue of levels) {
49
+ for (const green of levels) {
50
+ for (const red of levels) {
51
+ offset = lattice.writeUInt16LE(red, offset);
52
+ offset = lattice.writeUInt16LE(green, offset);
53
+ offset = lattice.writeUInt16LE(blue, offset);
54
+ }
55
+ }
56
+ }
57
+ return lattice;
58
+ }
59
+
60
+ /**
61
+ * ffmpeg's arguments to run the float chain over the lattice: HDR R'G'B' (BT.2020, raw `rgb48le`) in, tone-mapped
62
+ * linear BT.709 light out (raw planar float `gbrpf32le`, unclamped).
63
+ */
64
+ export function buildToneMapLutArgs(options: { transfer: ToneMapTransfer; size: number; lattice: string; output: string }): string[] {
65
+ const { transfer, size } = options;
66
+ const chain = [`zscale=tin=${transfer}:pin=bt2020:t=linear:p=bt2020:npl=100`, ...HABLE_STAGES].join(",");
67
+ return ["-v", "error", "-y", "-f", "rawvideo", "-pix_fmt", "rgb48le", "-s", `${size}x${size * size}`, "-i", options.lattice, "-vf", chain, "-f", "rawvideo", "-pix_fmt", "gbrpf32le", options.output];
68
+ }
69
+
70
+ /** The `.cube` text for the mapped lattice (planar float: the green, blue, red planes). Throws when `mapped` does not hold `size`³ points. */
71
+ export function formatCube(size: number, mapped: Buffer): string {
72
+ const points = size ** 3;
73
+ if (mapped.length !== points * 12) throw new Error(`A ${size}-point LUT needs ${points * 12} bytes of gbrpf32le, got ${mapped.length}.`);
74
+ const lines = ["# HDR to linear BT.709 light, Hable curve (marketing-kit footage normalize)", `LUT_3D_SIZE ${size}`];
75
+ const plane = points * 4;
76
+ for (let offset = 0; offset < plane; offset += 4) {
77
+ const green = mapped.readFloatLE(offset);
78
+ const blue = mapped.readFloatLE(plane + offset);
79
+ const red = mapped.readFloatLE(2 * plane + offset);
80
+ lines.push(`${red.toFixed(6)} ${green.toFixed(6)} ${blue.toFixed(6)}`);
81
+ }
82
+ return `${lines.join("\n")}\n`;
83
+ }
84
+
85
+ /**
86
+ * Makes the transfer's LUT in `dir` unless it is there, and returns its file name. The lattice and ffmpeg's raw output
87
+ * are removed afterwards; the `.cube` is written under a temporary name and renamed, so a stopped run never leaves half
88
+ * a LUT. Throws (naming the step) when ffmpeg fails; an abort rejects with the signal's reason.
89
+ */
90
+ export async function ensureToneMapLut(transfer: ToneMapTransfer, dir: string, options: { signal?: AbortSignal } = {}): Promise<string> {
91
+ const name = getToneMapLutName(transfer);
92
+ const path = join(dir, name);
93
+ if (existsSync(path)) return name;
94
+ const lattice = `${path}.lattice.rgb48`;
95
+ const mapped = `${path}.mapped.gbrpf32`;
96
+ const partial = `${path}.partial`;
97
+ try {
98
+ writeFileSync(lattice, buildIdentityLattice(TONE_MAP_LUT_SIZE));
99
+ await runFfmpeg(`Making the ${transfer} tone map LUT (${path})`, buildToneMapLutArgs({ transfer, size: TONE_MAP_LUT_SIZE, lattice, output: mapped }), {
100
+ ...(options.signal === undefined ? {} : { signal: options.signal }),
101
+ output: mapped,
102
+ });
103
+ writeFileSync(partial, formatCube(TONE_MAP_LUT_SIZE, readFileSync(mapped)));
104
+ renameSync(partial, path);
105
+ } finally {
106
+ for (const file of [lattice, mapped, partial]) rmSync(file, { force: true });
107
+ }
108
+ return name;
109
+ }
@@ -34,7 +34,7 @@ export interface FootageToolsOptions {
34
34
 
35
35
  /** What the footage blocks need from ffmpeg: decoders and filters, by the name ffmpeg lists them under. */
36
36
  export const REQUIRED_DECODERS = ["hevc", "h264"] as const;
37
- export const REQUIRED_FILTERS = ["zscale", "tonemap"] as const;
37
+ export const REQUIRED_FILTERS = ["zscale", "tonemap", "lut3d"] as const;
38
38
  /** What the fast draft needs on top: libass (with libfreetype) for the captions, `xfade` for fades, `zoompan` for Ken Burns. */
39
39
  export const DRAFT_FILTERS = ["ass", "xfade", "zoompan"] as const;
40
40
 
@@ -94,9 +94,9 @@ export function findBrowserProblem(env: NodeJS.ProcessEnv): string | null {
94
94
  }
95
95
 
96
96
  /**
97
- * Checks the machine for a footage run: ffmpeg and ffprobe run, ffmpeg decodes HEVC and H.264 and has `zscale` and
98
- * `tonemap` (HDR clips), for a draft (`needsDraft`) `ass`, `xfade` and `zoompan`, and, for a render, hyperframes runs
99
- * and has its browser, and its temp folder (and the build folder, when given) has the 1 GiB free hyperframes requires.
97
+ * Checks the machine for a footage run: ffmpeg and ffprobe run, ffmpeg decodes HEVC and H.264 and has `zscale`,
98
+ * `tonemap` and `lut3d` (HDR clips), for a draft (`needsDraft`) `ass`, `xfade` and `zoompan`, and, for a render,
99
+ * hyperframes runs and has its browser, and its temp folder (and the build folder, when given) has the 1 GiB free hyperframes requires.
100
100
  * Returns every problem found; an empty list means the machine is ready.
101
101
  */
102
102
  export async function checkFootageTools(options: FootageToolsOptions = {}): Promise<string[]> {