@softure-ai/marketing-kit 0.7.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/CHANGELOG.md +35 -0
  2. package/README.md +51 -13
  3. package/dist/footage/draft-font.d.ts +18 -0
  4. package/dist/footage/draft-font.d.ts.map +1 -0
  5. package/dist/footage/draft-font.js +105 -0
  6. package/dist/footage/draft-font.js.map +1 -0
  7. package/dist/footage/draft.d.ts +49 -12
  8. package/dist/footage/draft.d.ts.map +1 -1
  9. package/dist/footage/draft.js +165 -35
  10. package/dist/footage/draft.js.map +1 -1
  11. package/dist/footage/normalize.d.ts +18 -7
  12. package/dist/footage/normalize.d.ts.map +1 -1
  13. package/dist/footage/normalize.js +31 -15
  14. package/dist/footage/normalize.js.map +1 -1
  15. package/dist/footage/pipeline.d.ts +31 -2
  16. package/dist/footage/pipeline.d.ts.map +1 -1
  17. package/dist/footage/pipeline.js +41 -12
  18. package/dist/footage/pipeline.js.map +1 -1
  19. package/dist/footage/tonemap-lut.d.ts +45 -0
  20. package/dist/footage/tonemap-lut.d.ts.map +1 -0
  21. package/dist/footage/tonemap-lut.js +100 -0
  22. package/dist/footage/tonemap-lut.js.map +1 -0
  23. package/dist/footage/tools.d.ts +4 -4
  24. package/dist/footage/tools.d.ts.map +1 -1
  25. package/dist/footage/tools.js +4 -4
  26. package/dist/footage/tools.js.map +1 -1
  27. package/dist/index.d.ts +4 -2
  28. package/dist/index.d.ts.map +1 -1
  29. package/dist/index.js +4 -2
  30. package/dist/index.js.map +1 -1
  31. package/package.json +1 -1
  32. package/src/footage/draft-font.ts +105 -0
  33. package/src/footage/draft.ts +184 -40
  34. package/src/footage/normalize.ts +39 -18
  35. package/src/footage/pipeline.ts +55 -12
  36. package/src/footage/tonemap-lut.ts +109 -0
  37. package/src/footage/tools.ts +4 -4
  38. package/src/index.ts +8 -0
@@ -1,23 +1,28 @@
1
- import { copyFileSync, mkdirSync, rmSync, statSync, writeFileSync } from "node:fs";
1
+ import { copyFileSync, existsSync, mkdirSync, rmSync, statSync, writeFileSync } from "node:fs";
2
2
  import { basename, dirname, join, resolve } from "node:path";
3
3
  import { fileURLToPath } from "node:url";
4
4
 
5
- import type { CaptionLayout, LayoutOverride } from "../compose/timeline.js";
6
- import { ok } from "../voice/provider.js";
5
+ import { captionChunks, type CaptionLayout, type LayoutOverride } from "../compose/timeline.js";
6
+ import type { BrandColors } from "../config/colors.js";
7
+ import { err, ok } from "../voice/provider.js";
7
8
  import type { TimedWord } from "../voice/voiceover.js";
8
9
  import { getCaptionSpans, getFootageCaptionLayout, phraseChunks } from "./captions.js";
9
10
  import { getSegmentScales, getSegmentZoomEases } from "./compose.js";
11
+ import { chooseFootageDraftFont } from "./draft-font.js";
10
12
  import { getEdlDuration, getSegmentStarts, remapWords, type Edl } from "./edl.js";
11
13
  import { getFootageEdit, type FootageFilm } from "./film.js";
14
+ import { duckingKeyframes, getDuckingOptions } from "./music.js";
12
15
  import { FOOTAGE_FPS, getThreadArgs, type Frame } from "./normalize.js";
13
- import { getFrame, type FootageResult, type FootageRunOptions, type PreparedClip, type TranscribeOutcome } from "./pipeline.js";
16
+ import { findFootageMusic, getFrame, type FootageBrand, type FootageResult, type FootageRunOptions, type PreparedClip, type TranscribeOutcome } from "./pipeline.js";
14
17
  import { runFfmpeg } from "./process.js";
15
- import { buildSpeechArgs, getSpeechCutOptions } from "./render.js";
18
+ import { buildMusicArgs, buildSpeechArgs, getSpeechCutOptions } from "./render.js";
16
19
 
17
20
  /**
18
21
  * A fast draft of a footage film, ffmpeg only (no hyperframes, no browser): the cut at a small width with the film's
19
- * phrase captions burned in, its zoom cuts (eased as in the render), cross-fades and Ken Burns b-roll drawn as the full render draws them, and
20
- * the speech track. No loudness pass, music, end card or cover. What an app shows the owner before the paid render.
22
+ * captions burned in (its caption style, in the brand's body font and caption colours), its zoom cuts (eased as in the
23
+ * render), cross-fades and Ken Burns b-roll drawn as the full render draws them, the speech track and the ducked music
24
+ * when it costs nothing. No loudness pass, end card, cover or sound effects. What an app shows the owner before the paid
25
+ * render.
21
26
  */
22
27
 
23
28
  /** The draft's default width: 540 makes a 540×960 draft of a 9:16 film. */
@@ -36,6 +41,13 @@ const ZOOM_ORIGIN = { x: 0.5, y: 0.42 } as const;
36
41
  const DRAFT_CRF = 28;
37
42
  const DRAFT_AUDIO_BITRATE = "128k";
38
43
 
44
+ /** The caption colours of a `highlight` draft made without a brand. */
45
+ export const FOOTAGE_DRAFT_CAPTION_COLORS = { captionBackground: "#111111", captionText: "#ffffff", captionHighlight: "#ffd400" } as const;
46
+ /** How long the render keeps a spoken word lit in the `highlight` pill. */
47
+ const HIGHLIGHT_SECONDS = 0.32;
48
+ /** The pill's padding around its text (`CAPTION_PILL_PADDING.y`): the opaque box's border in the film's frame. */
49
+ const HIGHLIGHT_BOX_PADDING = 16;
50
+
39
51
  const r3 = (value: number): number => Math.round(value * 1000) / 1000 || 0;
40
52
  const r4 = (value: number): number => Math.round(value * 10000) / 10000 || 0;
41
53
 
@@ -55,13 +67,29 @@ function toAssTime(seconds: number): string {
55
67
  return `${hours}:${pad(minutes)}:${pad(secs)}.${pad(centiseconds % 100)}`;
56
68
  }
57
69
 
70
+ /**
71
+ * A hex colour (`#rgb`, `#rgba`, `#rrggbb`, `#rrggbbaa`) as an ASS colour, `&HAABBGGRR&`-style without the closing
72
+ * `&`: ASS writes blue first and counts alpha the other way round (00 is opaque).
73
+ */
74
+ export function toAssColor(hex: string): string {
75
+ const digits = hex.replace(/^#/, "");
76
+ const full = digits.length <= 4 ? [...digits].map((digit) => digit + digit).join("") : digits;
77
+ if (!/^(?:[0-9a-f]{6}|[0-9a-f]{8})$/i.test(full)) throw new Error(`Not a hex colour: "${hex}".`);
78
+ const pair = (index: number): string => full.slice(index, index + 2).toUpperCase();
79
+ const alpha = full.length === 8 ? (255 - parseInt(full.slice(6, 8), 16)).toString(16).padStart(2, "0").toUpperCase() : "00";
80
+ return `&H${alpha}${pair(4)}${pair(2)}${pair(0)}`;
81
+ }
82
+
58
83
  /** Caption text as ASS dialogue: override braces and backslashes would be read as tags, line breaks end the line. */
59
84
  function toAssText(text: string): string {
60
85
  return text.replace(/\\/g, "/").replace(/\{/g, "(").replace(/\}/g, ")").replace(/\s+/g, " ").trim();
61
86
  }
62
87
 
88
+ /** The caption colours the draft paints with (`BrandColors`). */
89
+ export type FootageDraftCaptionColors = Pick<BrandColors, "captionBackground" | "captionText" | "captionHighlight">;
90
+
63
91
  export interface FootageDraftCaptionsInput {
64
- film: Pick<FootageFilm, "captions">;
92
+ film: Pick<FootageFilm, "captions" | "captionStyle">;
65
93
  edl: Edl;
66
94
  /** The film's words in film time (`remapWords`). */
67
95
  words: TimedWord[];
@@ -69,32 +97,73 @@ export interface FootageDraftCaptionsInput {
69
97
  caption: CaptionLayout;
70
98
  /** The film's full frame; the ASS file is laid out in it and libass scales it to the draft. */
71
99
  frame: Frame;
72
- /** The font family the captions are drawn in; default `FOOTAGE_DRAFT_FONT.family`. */
100
+ /** The font family the captions are drawn in, as the font file names it; default `FOOTAGE_DRAFT_FONT.family`. */
73
101
  fontFamily?: string;
102
+ /** The `highlight` pill's colours; default `FOOTAGE_DRAFT_CAPTION_COLORS`. A `phrase` caption is white, as in the render. */
103
+ colors?: FootageDraftCaptionColors;
74
104
  }
75
105
 
76
106
  /**
77
- * The draft's captions as an ASS subtitle file: the film's phrases (`phraseChunks`) shown for the render's spans
78
- * (`getCaptionSpans`), white bold text with a dark outline and shadow, centred in the caption box from its top (or
79
- * from its foot, with `bottom`). Null when the film has no captions or nothing is said.
107
+ * A `highlight` chunk as ASS events: its span cut wherever a word lights up or goes out, each piece showing the chunk
108
+ * with the words lit then in `highlight` (a word stays lit for the render's 0.32 s from its start).
109
+ */
110
+ function buildHighlightEvents(chunk: { words: TimedWord[] }, span: { start: number; end: number }, colors: { text: string; highlight: string }): string[] {
111
+ const cs = (seconds: number): number => Math.round(seconds * 100);
112
+ const [from, to] = [cs(span.start), cs(span.end)];
113
+ const lights = chunk.words.map((word) => ({ on: cs(word.start), off: cs(word.start + HIGHLIGHT_SECONDS) }));
114
+ const cuts = [...new Set([from, to, ...lights.flatMap((light) => [light.on, light.off])])].filter((at) => at >= from && at <= to).sort((a, b) => a - b);
115
+ const events: string[] = [];
116
+ cuts.slice(0, -1).forEach((at, index) => {
117
+ const next = cuts[index + 1] as number;
118
+ const text = chunk.words
119
+ .map((word, wordIndex) => {
120
+ const light = lights[wordIndex] as { on: number; off: number };
121
+ const shown = toAssText(word.text);
122
+ return light.on <= at && at < light.off ? `{\\1c${colors.highlight}&}${shown}{\\1c${colors.text}&}` : shown;
123
+ })
124
+ .join(" ");
125
+ events.push(`Dialogue: 0,${toAssTime(at / 100)},${toAssTime(next / 100)},Caption,,0,0,0,,${text}`);
126
+ });
127
+ return events;
128
+ }
129
+
130
+ /**
131
+ * The draft's captions as an ASS subtitle file, shown for the render's spans (`getCaptionSpans`) and centred in the
132
+ * caption box from its top (or from its foot, with `bottom`). `phrase` (the default): the film's phrases
133
+ * (`phraseChunks`), white bold text with a dark outline and shadow. `highlight`: the render's chunks of up to four
134
+ * words (`captionChunks`) on an opaque box of `captionBackground`, in `captionText`, the word being said in
135
+ * `captionHighlight`. Null when the film has no captions or nothing is said.
80
136
  */
81
137
  export function buildFootageDraftCaptions(input: FootageDraftCaptionsInput): string | null {
82
138
  if (!input.film.captions) return null;
83
- const chunks = phraseChunks(input.words);
139
+ const isHighlight = input.film.captionStyle === "highlight";
140
+ const chunks = isHighlight ? captionChunks(input.words) : phraseChunks(input.words);
84
141
  if (chunks.length === 0) return null;
85
142
  const spans = getCaptionSpans(chunks, getEdlDuration(input.edl));
86
143
  const { caption, frame } = input;
87
144
  const family = (input.fontFamily ?? FOOTAGE_DRAFT_FONT.family).replace(/,/g, " ");
88
- const outline = Math.max(1, Math.round(caption.fontSize / 16));
89
- const shadow = Math.max(1, Math.round(caption.fontSize / 25));
145
+ const colors = input.colors ?? FOOTAGE_DRAFT_CAPTION_COLORS;
146
+ const text = toAssColor(colors.captionText);
147
+ const box = toAssColor(colors.captionBackground);
148
+ // libass draws BorderStyle 3's box in the outline colour, grown by the outline width; the back colour is its shadow.
149
+ const look = isHighlight
150
+ ? { primary: text, outlineColor: box, backColor: box, borderStyle: 3, outline: HIGHLIGHT_BOX_PADDING, shadow: 0 }
151
+ : {
152
+ primary: "&H00FFFFFF",
153
+ outlineColor: "&H8C000000",
154
+ backColor: "&H8C000000",
155
+ borderStyle: 1,
156
+ outline: Math.max(1, Math.round(caption.fontSize / 16)),
157
+ shadow: Math.max(1, Math.round(caption.fontSize / 25)),
158
+ };
90
159
  const style = [
91
160
  "Caption",
92
161
  family,
93
162
  caption.fontSize,
94
- "&H00FFFFFF",
95
- "&H00FFFFFF",
96
- "&H8C000000",
97
- "&H8C000000",
163
+ look.primary,
164
+ look.primary,
165
+ look.outlineColor,
166
+ look.backColor,
98
167
  -1,
99
168
  0,
100
169
  0,
@@ -103,9 +172,9 @@ export function buildFootageDraftCaptions(input: FootageDraftCaptionsInput): str
103
172
  100,
104
173
  0,
105
174
  0,
106
- 1,
107
- outline,
108
- shadow,
175
+ look.borderStyle,
176
+ look.outline,
177
+ look.shadow,
109
178
  // ASS alignment 2 hangs the text from MarginV above the bottom, 8 from MarginV below the top.
110
179
  caption.bottom === undefined ? 8 : 2,
111
180
  Math.round(caption.left),
@@ -113,10 +182,12 @@ export function buildFootageDraftCaptions(input: FootageDraftCaptionsInput): str
113
182
  Math.round(caption.bottom ?? caption.top),
114
183
  1,
115
184
  ].join(",");
116
- const events = chunks.map((chunk, index) => {
185
+ const highlight = { text: text.replace(/^&H../, "&H"), highlight: toAssColor(colors.captionHighlight).replace(/^&H../, "&H") };
186
+ const events = chunks.flatMap((chunk, index) => {
117
187
  const span = spans[index] as { start: number; end: number };
118
- const text = toAssText(chunk.words.map((word) => word.text).join(" "));
119
- return `Dialogue: 0,${toAssTime(span.start)},${toAssTime(span.end)},Caption,,0,0,0,,${text}`;
188
+ if (isHighlight) return buildHighlightEvents(chunk, span, highlight);
189
+ const line = toAssText(chunk.words.map((word) => word.text).join(" "));
190
+ return [`Dialogue: 0,${toAssTime(span.start)},${toAssTime(span.end)},Caption,,0,0,0,,${line}`];
120
191
  });
121
192
  return [
122
193
  "[Script Info]",
@@ -144,6 +215,8 @@ export interface FootageDraftArgsInput {
144
215
  clips: Record<string, string>;
145
216
  /** The cut speech track (`buildSpeechArgs`). */
146
217
  speech: string;
218
+ /** The ducked music track (`buildMusicArgs`, as long as the cut), mixed under the speech; null or absent for none. */
219
+ music?: string | null;
147
220
  /** The film's full frame (the normalized clips' size). */
148
221
  frame: Frame;
149
222
  /** The draft's width; default 540. */
@@ -280,6 +353,8 @@ export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
280
353
  parts.push(`[${joined}]null[draft]`);
281
354
  }
282
355
  const speechInput = edl.segments.length;
356
+ const music = input.music ?? null;
357
+ if (music !== null) parts.push(`[${speechInput}:a][${speechInput + 1}:a]amix=inputs=2:duration=first:normalize=0[sound]`);
283
358
  const seconds = getEdlDuration(edl);
284
359
  return [
285
360
  "-v",
@@ -289,12 +364,13 @@ export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
289
364
  ...inputs,
290
365
  "-i",
291
366
  input.speech,
367
+ ...(music === null ? [] : ["-i", music]),
292
368
  "-filter_complex",
293
369
  parts.join(";"),
294
370
  "-map",
295
371
  "[draft]",
296
372
  "-map",
297
- `${speechInput}:a`,
373
+ music === null ? `${speechInput}:a` : "[sound]",
298
374
  "-t",
299
375
  String(seconds),
300
376
  "-c:v",
@@ -316,13 +392,19 @@ export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
316
392
  ];
317
393
  }
318
394
 
319
- export interface RenderFootageDraftOptions extends Pick<FootageRunOptions, "buildDir" | "log" | "signal" | "onProgress"> {
395
+ export interface RenderFootageDraftOptions extends Pick<FootageRunOptions, "buildDir" | "log" | "signal" | "onProgress">, Partial<Pick<FootageRunOptions, "cacheDir" | "providers">> {
320
396
  /** The draft MP4. */
321
397
  output: string;
322
398
  /** The draft's width; default 540 (540×960 for a 9:16 film). */
323
399
  width?: number;
324
400
  /** A project's override of the format's caption box (`marketing.json` `layout`), as the render takes it. */
325
401
  layout?: LayoutOverride;
402
+ /**
403
+ * The brand the render takes (`renderFootage`'s `brand`): the captions are drawn in its body font (a TrueType or
404
+ * OpenType file; DejaVu Sans when it has none) and its caption colours. Without it, DejaVu Sans and
405
+ * `FOOTAGE_DRAFT_CAPTION_COLORS`.
406
+ */
407
+ brand?: { colors: FootageDraftCaptionColors; fonts: Pick<FootageBrand["fonts"], "body"> };
326
408
  /**
327
409
  * ffmpeg's threads for the draft's decoders, filter graph and encoder (`buildFootageDraftArgs`); by default ffmpeg's
328
410
  * own choice, which follows the host's cores. Set it from a CPU-limited container's quota.
@@ -330,8 +412,14 @@ export interface RenderFootageDraftOptions extends Pick<FootageRunOptions, "buil
330
412
  threads?: number;
331
413
  }
332
414
 
415
+ /** The draft's music: the film has none, it played (a file, or generated music from the cache), or it is not cached yet. */
416
+ export type FootageDraftMusic = "none" | "played" | "not_cached";
417
+
333
418
  export interface FootageDraft {
334
419
  output: string;
420
+ music: FootageDraftMusic;
421
+ /** The family the captions are drawn in; null without captions. */
422
+ captionFont: string | null;
335
423
  /** The draft's length: the cut's. */
336
424
  seconds: number;
337
425
  width: number;
@@ -339,9 +427,34 @@ export interface FootageDraft {
339
427
  bytes: number;
340
428
  }
341
429
 
430
+ /** The draft's music track: a file to duck, or why there is none. */
431
+ type DraftMusicSource = { kind: "file"; path: string } | { kind: "none" } | { kind: "not_cached"; reason: string };
432
+
342
433
  /**
343
- * Makes the draft of a cut film: the speech track, the captions file and the font in `<buildDir>/draft/`, then one
344
- * ffmpeg run (`buildFootageDraftArgs`). Spends nothing; takes the same clips, transcript and EDL as `renderFootage`.
434
+ * The music the draft can play without spending: the film's own file, or generated music already in the paid cache
435
+ * (looked up as the render looks it up, `findFootageMusic`). Never calls a provider.
436
+ */
437
+ async function findDraftMusic(film: FootageFilm, edl: Edl, clips: PreparedClip[], options: RenderFootageDraftOptions): Promise<FootageResult<DraftMusicSource>> {
438
+ const source = film.music.source;
439
+ if (source.kind === "none") return ok({ kind: "none" });
440
+ if (source.kind === "file") return existsSync(source.path) ? ok({ kind: "file", path: source.path }) : err(`music: no file at ${source.path}`);
441
+ const { cacheDir, providers } = options;
442
+ if (cacheDir === undefined || providers === undefined) {
443
+ return ok({ kind: "not_cached", reason: "the film's music is generated and the draft was given no cacheDir and providers to find it in the cache" });
444
+ }
445
+ const found = await findFootageMusic(film, edl, clips, { cacheDir, providers });
446
+ if (!found.ok) return found;
447
+ const music = found.value;
448
+ if (music.kind === "none") return ok({ kind: "none" });
449
+ if (music.kind === "file") return ok({ kind: "file", path: music.path });
450
+ if (music.kind === "cached") return ok({ kind: "file", path: music.cache.path });
451
+ return ok({ kind: "not_cached", reason: `the film's generated music is not in the cache yet (${music.cache.path}); the render buys it` });
452
+ }
453
+
454
+ /**
455
+ * Makes the draft of a cut film: the speech track, the ducked music track (when it costs nothing), the captions file
456
+ * and its font in `<buildDir>/draft/`, then one ffmpeg run (`buildFootageDraftArgs`). Spends nothing: no provider is
457
+ * called; takes the same clips, transcript and EDL as `renderFootage`.
345
458
  */
346
459
  export async function renderFootageDraft(
347
460
  film: FootageFilm,
@@ -353,32 +466,61 @@ export async function renderFootageDraft(
353
466
  const { log, signal } = options;
354
467
  const aborting = signal === undefined ? {} : { signal };
355
468
  const seconds = getEdlDuration(edl);
469
+ const music = await findDraftMusic(film, edl, clips, options);
470
+ if (!music.ok) return music;
356
471
  const draftDir = resolve(options.buildDir, "draft");
357
472
  rmSync(draftDir, { recursive: true, force: true });
358
473
  mkdirSync(draftDir, { recursive: true });
359
- const report = (item: string, itemIndex: number, done: number): void =>
360
- options.onProgress?.({ step: "draft", item, itemIndex, itemCount: 2, done: Math.min(seconds * 2, (itemIndex - 1) * seconds + done), total: seconds * 2 });
474
+ const items = music.value.kind === "file" ? ["speech", "music", "film"] : ["speech", "film"];
475
+ const report = (item: string, done: number): void => {
476
+ const itemIndex = items.indexOf(item) + 1;
477
+ options.onProgress?.({ step: "draft", item, itemIndex, itemCount: items.length, done: Math.min(seconds * items.length, (itemIndex - 1) * seconds + done), total: seconds * items.length });
478
+ };
361
479
  const clipFiles = Object.fromEntries(clips.map((clip) => [clip.clip.id, resolve(clip.file)]));
362
480
  const speech = join(draftDir, "speech.mp3");
363
481
  const words = remapWords(edl, transcript.words, film.cut.fillers);
364
482
  const frame = getFrame(film);
365
- const captions = buildFootageDraftCaptions({ film, edl, words, caption: getFootageCaptionLayout(film, options.layout ?? {}), frame });
483
+ const brandFont = chooseFootageDraftFont(options.brand?.fonts.body ?? null);
484
+ const font = brandFont ?? FOOTAGE_DRAFT_FONT;
485
+ const captions = buildFootageDraftCaptions({
486
+ film,
487
+ edl,
488
+ words,
489
+ caption: getFootageCaptionLayout(film, options.layout ?? {}),
490
+ frame,
491
+ fontFamily: font.family,
492
+ ...(options.brand === undefined ? {} : { colors: options.brand.colors }),
493
+ });
366
494
  if (captions !== null) {
367
495
  writeFileSync(join(draftDir, "captions.ass"), captions);
368
496
  mkdirSync(join(draftDir, "fonts"));
369
- copyFileSync(FOOTAGE_DRAFT_FONT.file, join(draftDir, "fonts", basename(FOOTAGE_DRAFT_FONT.file)));
497
+ copyFileSync(font.file, join(draftDir, "fonts", basename(font.file)));
498
+ if (brandFont === null && options.brand?.fonts.body != null) {
499
+ log(`draft: the brand's body font ${options.brand.fonts.body.family} has no TrueType or OpenType file ffmpeg can read; the captions use ${FOOTAGE_DRAFT_FONT.family}.`);
500
+ }
370
501
  }
371
502
  const output = resolve(options.output);
372
503
  mkdirSync(dirname(output), { recursive: true });
373
- report("speech", 1, 0);
374
- await runFfmpeg("The draft's speech track", buildSpeechArgs(edl, transcript.speech, speech, getSpeechCutOptions(getFootageEdit(film))), { ...aborting, output: speech, onSeconds: (done) => report("speech", 1, done) });
375
- report("film", 2, 0);
504
+ report("speech", 0);
505
+ await runFfmpeg("The draft's speech track", buildSpeechArgs(edl, transcript.speech, speech, getSpeechCutOptions(getFootageEdit(film))), { ...aborting, output: speech, onSeconds: (done) => report("speech", done) });
506
+ let musicFile: string | null = null;
507
+ if (music.value.kind === "file") {
508
+ musicFile = join(draftDir, "music.mp3");
509
+ const keyframes = duckingKeyframes(words, getDuckingOptions(film.music));
510
+ report("music", 0);
511
+ const args = buildMusicArgs({ input: music.value.path, seconds, keyframes, output: musicFile });
512
+ await runFfmpeg("The draft's music track", args, { ...aborting, output: musicFile, onSeconds: (done) => report("music", done) });
513
+ } else if (music.value.kind === "not_cached") {
514
+ log(`draft: no music, ${music.value.reason}.`);
515
+ }
516
+ report("film", 0);
376
517
  // Run in the draft folder: the captions file and the font folder are relative names, so no path needs escaping in the filter graph.
377
518
  const args = buildFootageDraftArgs({
378
519
  film,
379
520
  edl,
380
521
  clips: clipFiles,
381
522
  speech,
523
+ music: musicFile,
382
524
  frame,
383
525
  ...(options.width === undefined ? {} : { width: options.width }),
384
526
  ...(options.threads === undefined ? {} : { threads: options.threads }),
@@ -386,10 +528,12 @@ export async function renderFootageDraft(
386
528
  fontsDir: captions === null ? null : "fonts",
387
529
  output,
388
530
  });
389
- await runFfmpeg("The footage draft", args, { ...aborting, output, cwd: draftDir, onSeconds: (done) => report("film", 2, done) });
390
- report("film", 2, seconds);
531
+ await runFfmpeg("The footage draft", args, { ...aborting, output, cwd: draftDir, onSeconds: (done) => report("film", done) });
532
+ report("film", seconds);
391
533
  const draft = getFootageDraftFrame(frame, options.width);
392
534
  const bytes = statSync(output).size;
393
- log(`draft: ${output}, ${draft.width}×${draft.height}, ${seconds.toFixed(1)} s, ${bytes} bytes${captions === null ? ", no captions" : ""}.`);
394
- return ok({ output, seconds, width: draft.width, height: draft.height, bytes });
535
+ const draftMusic: FootageDraftMusic = music.value.kind === "file" ? "played" : music.value.kind;
536
+ const notes = [captions === null ? "no captions" : `captions in ${font.family}`, draftMusic === "played" ? "music" : draftMusic === "none" ? "no music" : "music not cached"];
537
+ log(`draft: ${output}, ${draft.width}×${draft.height}, ${seconds.toFixed(1)} s, ${bytes} bytes, ${notes.join(", ")}.`);
538
+ return ok({ output, music: draftMusic, captionFont: captions === null ? null : font.family, seconds, width: draft.width, height: draft.height, bytes });
395
539
  }
@@ -1,6 +1,9 @@
1
+ import { dirname, resolve } from "node:path";
2
+
1
3
  import type { ClipFit, FootageClip } from "./film.js";
2
4
  import { getTrimmedDuration, type FootageProbe, type HdrKind } from "./probe.js";
3
5
  import { runFfmpeg, type FfmpegRunOptions } from "./process.js";
6
+ import { ensureToneMapLut, getToneMapTransfer, HABLE_STAGES } from "./tonemap-lut.js";
4
7
 
5
8
  /**
6
9
  * One intermediate format for every clip, so the composition can cut between them freely: the film's frame at
@@ -27,20 +30,21 @@ export function resolveFit(fit: ClipFit, probe: Pick<FootageProbe, "displayWidth
27
30
  return Math.abs(clipAspect - frameAspect) / frameAspect <= CROP_ASPECT_TOLERANCE ? "crop" : "blur";
28
31
  }
29
32
 
30
- const TRANSFER_INPUT: Record<HdrKind, string> = { pq: "smpte2084", hlg: "arib-std-b67", "dolby-vision": "arib-std-b67" };
31
-
32
33
  /**
33
- * HDR to SDR BT.709 on the CPU: to linear light (zscale), Hable tone curve, back to BT.709. A Dolby Vision phone
34
- * clip is read through its HLG base layer.
34
+ * HDR to SDR BT.709 on the CPU. A Dolby Vision phone clip is read through its HLG base layer. With `lut` (the file
35
+ * name of the transfer's tone map LUT, `ensureToneMapLut`, in the folder ffmpeg runs in): to 16-bit R'G'B', the LUT
36
+ * to linear BT.709 light, the BT.709 transfer and matrix, about half the cost; colours outside BT.709 are clipped per
37
+ * RGB channel. Without it: to linear light (zscale), the Hable curve in float, back to BT.709, the same curve the LUT
38
+ * is made from; colours outside BT.709 are clamped in YUV.
35
39
  */
36
- export function toneMapFilter(hdr: HdrKind | null, transfer: string | null): string[] {
40
+ export function toneMapFilter(hdr: HdrKind | null, transfer: string | null, lut?: string): string[] {
37
41
  if (hdr === null) return [];
38
- const input = transfer === "smpte2084" || transfer === "arib-std-b67" ? transfer : TRANSFER_INPUT[hdr];
42
+ if (lut !== undefined) {
43
+ return ["zscale=min=bt2020nc:m=gbr:r=full", "format=gbrp16le", `lut3d=file=${lut}:interp=tetrahedral`, "zscale=tin=linear:pin=bt709:t=bt709:p=bt709:m=bt709:r=tv", "format=yuv420p"];
44
+ }
39
45
  return [
40
- `zscale=tin=${input}:pin=bt2020:min=bt2020nc:t=linear:npl=100`,
41
- "format=gbrpf32le",
42
- "zscale=p=bt709",
43
- "tonemap=tonemap=hable:desat=0",
46
+ `zscale=tin=${getToneMapTransfer(hdr, transfer)}:pin=bt2020:min=bt2020nc:t=linear:npl=100`,
47
+ ...HABLE_STAGES,
44
48
  "zscale=t=bt709:m=bt709:r=tv",
45
49
  // Back to 8-bit before scaling: the blur fill on float frames crashed ffmpeg 6.1.
46
50
  "format=yuv420p",
@@ -50,13 +54,13 @@ export function toneMapFilter(hdr: HdrKind | null, transfer: string | null): str
50
54
  /**
51
55
  * The video filter graph from input 0's picture to `[v]`. Every stage runs on as little as it can: frames are
52
56
  * dropped to 30 fps first, then the picture is scaled to the frame (crop: covered and cropped to it; blur: fitted
53
- * inside it), and only then tone-mapped, so the float stages of an HDR clip work on frame-sized pictures instead of
57
+ * inside it), and only then tone-mapped, so the tone map of an HDR clip works on frame-sized pictures instead of
54
58
  * the source's (a 4K phone clip has 4x the pixels of a 1080x1920 frame). The blur's background is made from the
55
- * fitted picture.
59
+ * fitted picture. `toneMapLut` as in `toneMapFilter`.
56
60
  */
57
- export function buildVideoFilter(options: { fit: "crop" | "blur"; frame: Frame; hdr: HdrKind | null; transfer: string | null }): string {
61
+ export function buildVideoFilter(options: { fit: "crop" | "blur"; frame: Frame; hdr: HdrKind | null; transfer: string | null; toneMapLut?: string }): string {
58
62
  const { width, height } = options.frame;
59
- const toneMap = toneMapFilter(options.hdr, options.transfer);
63
+ const toneMap = toneMapFilter(options.hdr, options.transfer, options.toneMapLut);
60
64
  const tail = "setsar=1,format=yuv420p";
61
65
  if (options.fit === "crop") {
62
66
  const cover = `scale=${width}:${height}:force_original_aspect_ratio=increase,crop=${width}:${height}`;
@@ -96,6 +100,11 @@ export interface NormalizeInput {
96
100
  frame: Frame;
97
101
  output: string;
98
102
  encoder?: NormalizeEncoder;
103
+ /**
104
+ * The tone map LUT's file name for an HDR clip, resolved in the folder ffmpeg runs in (`toneMapFilter`). Without it
105
+ * `buildNormalizeArgs` uses the float chain; `normalizeFootage` makes the LUT next to the output and uses it.
106
+ */
107
+ toneMapLut?: string;
99
108
  }
100
109
 
101
110
  /**
@@ -129,7 +138,7 @@ export function buildNormalizeArgs(input: NormalizeInput): string[] {
129
138
  "-t",
130
139
  duration.toFixed(3),
131
140
  "-filter_complex",
132
- buildVideoFilter({ fit, frame, hdr: probe.hdr, transfer: probe.colorTransfer }),
141
+ buildVideoFilter({ fit, frame, hdr: probe.hdr, transfer: probe.colorTransfer, ...(input.toneMapLut === undefined ? {} : { toneMapLut: input.toneMapLut }) }),
133
142
  "-map",
134
143
  "[v]",
135
144
  "-map",
@@ -165,8 +174,20 @@ export function buildNormalizeArgs(input: NormalizeInput): string[] {
165
174
 
166
175
  /**
167
176
  * Runs ffmpeg for one clip without blocking; throws with the clip's id when it fails (a broken file is not an expected
168
- * outcome here). An abort removes the partial output and rejects with the signal's reason.
177
+ * outcome here). An HDR clip without `toneMapLut` gets its transfer's LUT made once in the output's folder, and ffmpeg
178
+ * runs there so the graph names the LUT without a path. An abort removes the partial output and rejects with the
179
+ * signal's reason.
169
180
  */
170
- export async function normalizeFootage(input: NormalizeInput, options: Omit<FfmpegRunOptions, "output"> = {}): Promise<void> {
171
- await runFfmpeg(`Normalizing clip ${input.clip.id} (${input.file})`, buildNormalizeArgs(input), { ...options, output: input.output });
181
+ export async function normalizeFootage(input: NormalizeInput, options: Omit<FfmpegRunOptions, "output" | "cwd"> = {}): Promise<void> {
182
+ const step = `Normalizing clip ${input.clip.id} (${input.file})`;
183
+ const output = resolve(input.output);
184
+ const { hdr, colorTransfer } = input.probe;
185
+ if (hdr === null || input.toneMapLut !== undefined) {
186
+ await runFfmpeg(step, buildNormalizeArgs(input), { ...options, output });
187
+ return;
188
+ }
189
+ const dir = dirname(output);
190
+ const toneMapLut = await ensureToneMapLut(getToneMapTransfer(hdr, colorTransfer), dir, options.signal === undefined ? {} : { signal: options.signal });
191
+ const args = buildNormalizeArgs({ ...input, file: resolve(input.file), output, toneMapLut });
192
+ await runFfmpeg(step, args, { ...options, output, cwd: dir });
172
193
  }
@@ -477,17 +477,27 @@ async function checkMusicFile(source: Extract<MusicSource, { kind: "file" }>, op
477
477
  return ok(path);
478
478
  }
479
479
 
480
+ /** Where a film's music comes from, found without spending: none, a local file, or the paid cache's entry (made or not). */
481
+ export type FootageMusicLookup =
482
+ | { kind: "none" }
483
+ | { kind: "file"; path: string }
484
+ | { kind: "cached"; cache: CacheEntry }
485
+ | { kind: "missing"; cache: CacheEntry };
486
+
480
487
  /**
481
- * The music file for the cut: a local file as it is, generated music from the cache, or (with `isCommit`) bought
482
- * from the music provider and cached. Null when the film has none or the rehearsal skips generated music; a paid
483
- * step when it must be bought and `isCommit` is false.
488
+ * The film's music as the render reads it, without a paid call: a `file` source as it is (an error when the file is
489
+ * gone), a `prompt` or `video` source's cache entry and whether it exists. The render and the draft share it, so they
490
+ * look up the same cache key.
484
491
  */
485
- async function getMusic(film: FootageFilm, edl: Edl, clips: PreparedClip[], seconds: number, options: FootageRunOptions): Promise<FootageResult<string | PaidStep | null>> {
492
+ export async function findFootageMusic(
493
+ film: FootageFilm,
494
+ edl: Edl,
495
+ clips: PreparedClip[],
496
+ options: Pick<FootageRunOptions, "cacheDir" | "providers">,
497
+ ): Promise<FootageResult<FootageMusicLookup>> {
486
498
  const source = film.music.source;
487
- if (source.kind === "none") return ok(null);
488
- if (source.kind === "file") return existsSync(source.path) ? ok(source.path) : err(`music: no file at ${source.path}`);
489
- if (options.isPlaceholder) return ok(null);
490
- const { providers, log, signal } = options;
499
+ if (source.kind === "none") return ok({ kind: "none" });
500
+ if (source.kind === "file") return existsSync(source.path) ? ok({ kind: "file", path: source.path }) : err(`music: no file at ${source.path}`);
491
501
  let cache: CacheEntry;
492
502
  if (source.kind === "prompt") {
493
503
  cache = getPromptMusicCache(film, options);
@@ -496,8 +506,26 @@ async function getMusic(film: FootageFilm, edl: Edl, clips: PreparedClip[], seco
496
506
  for (const clip of clips) clipKeys[clip.clip.id] = await getClipKey(clip);
497
507
  cache = getVideoMusicCache(film, edl, clipKeys, options);
498
508
  }
509
+ return ok(existsSync(cache.path) ? { kind: "cached", cache } : { kind: "missing", cache });
510
+ }
511
+
512
+ /**
513
+ * The music file for the cut: a local file as it is, generated music from the cache, or (with `isCommit`) bought
514
+ * from the music provider and cached. Null when the film has none or the rehearsal skips generated music; a paid
515
+ * step when it must be bought and `isCommit` is false.
516
+ */
517
+ async function getMusic(film: FootageFilm, edl: Edl, clips: PreparedClip[], seconds: number, options: FootageRunOptions): Promise<FootageResult<string | PaidStep | null>> {
518
+ const source = film.music.source;
519
+ if (options.isPlaceholder && (source.kind === "prompt" || source.kind === "video")) return ok(null);
520
+ const found = await findFootageMusic(film, edl, clips, options);
521
+ if (!found.ok) return found;
522
+ const music = found.value;
523
+ if (music.kind === "none") return ok(null);
524
+ if (music.kind === "file") return ok(music.path);
525
+ const { providers, log, signal } = options;
526
+ const { cache } = music;
499
527
  const { path } = cache;
500
- if (existsSync(path)) {
528
+ if (music.kind === "cached") {
501
529
  log(`music: from the cache ${path}, nothing spent.`);
502
530
  return ok(path);
503
531
  }
@@ -511,13 +539,15 @@ async function getMusic(film: FootageFilm, edl: Edl, clips: PreparedClip[], seco
511
539
  throwIfAborted(signal);
512
540
  log(`music (${providers.music.id}): ${seconds.toFixed(1)} s.`);
513
541
  bought = await providers.music.compose({ prompt: source.prompt, seconds, ...aborting });
514
- } else {
542
+ } else if (source.kind === "video") {
515
543
  const preview = join(options.buildDir, "cut-preview.mp4");
516
544
  const clipFiles = Object.fromEntries(clips.map((clip) => [clip.clip.id, clip.file]));
517
545
  await runFfmpeg("The cut preview for Video to Music", buildCutPreviewArgs(edl, clipFiles, preview), { ...aborting, output: preview });
518
546
  throwIfAborted(signal);
519
547
  log(`music (${providers.music.id}): ${seconds.toFixed(1)} s.`);
520
548
  bought = await providers.music.scoreVideo({ video: preview, description: source.description, tags: source.tags, ...aborting });
549
+ } else {
550
+ return ok(null);
521
551
  }
522
552
  throwIfAborted(signal);
523
553
  if (!bought.ok) return err(`music: ${bought.error}`);
@@ -555,6 +585,12 @@ export interface RenderFootageOptions extends FootageRunOptions {
555
585
  * off; by default hyperframes turns it on by itself on a machine (or cgroup) of 8 GB or less.
556
586
  */
557
587
  lowMemory?: boolean;
588
+ /**
589
+ * The encoder's constant rate factor (hyperframes' `--crf`, a whole number from 0 to 51), overriding the quality
590
+ * tier's CRF (`draft` 28, `standard` 18, `high` 15); the x264 preset stays the tier's (`ultrafast`, `medium`,
591
+ * `slow`). `quality: "draft"` with `crf: 18` encodes fast at the published quality. By default the tier's CRF.
592
+ */
593
+ crf?: number;
558
594
  }
559
595
 
560
596
  export type RenderedFootage = { kind: "dry-run"; steps: PaidStep[] } | ({ kind: "rendered"; output: string; seconds: number; edlPath: string } & FinishedFootage);
@@ -759,9 +795,16 @@ export function readRenderPercent(line: string): number | null {
759
795
  return percent >= 0 && percent <= 100 ? percent : null;
760
796
  }
761
797
 
762
- /** hyperframes' render arguments: the quality, the output and the memory settings the caller set. Throws on invalid `workers`. */
763
- export function getHyperframesRenderArgs(options: Pick<RenderFootageOptions, "quality" | "workers" | "lowMemory">, output: string): string[] {
798
+ /**
799
+ * hyperframes' render arguments: the quality, the encoder's CRF, the output and the memory settings the caller set.
800
+ * Throws on invalid `workers` or `crf`.
801
+ */
802
+ export function getHyperframesRenderArgs(options: Pick<RenderFootageOptions, "quality" | "workers" | "lowMemory" | "crf">, output: string): string[] {
764
803
  const args = ["render", "--quality", options.quality];
804
+ if (options.crf !== undefined) {
805
+ if (!Number.isInteger(options.crf) || options.crf < 0 || options.crf > 51) throw new Error(`crf must be a whole number from 0 to 51, got ${options.crf}.`);
806
+ args.push("--crf", String(options.crf));
807
+ }
765
808
  if (options.workers !== undefined) {
766
809
  if (!Number.isInteger(options.workers) || options.workers < 1) throw new Error(`workers must be a positive whole number, got ${options.workers}.`);
767
810
  args.push("--workers", String(options.workers));