@softure-ai/marketing-kit 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/CHANGELOG.md +55 -0
  2. package/README.md +64 -8
  3. package/dist/cli/footage.d.ts.map +1 -1
  4. package/dist/cli/footage.js +5 -2
  5. package/dist/cli/footage.js.map +1 -1
  6. package/dist/compose/compose.d.ts.map +1 -1
  7. package/dist/compose/compose.js +3 -3
  8. package/dist/compose/compose.js.map +1 -1
  9. package/dist/compose/timeline.d.ts +16 -0
  10. package/dist/compose/timeline.d.ts.map +1 -1
  11. package/dist/compose/timeline.js +17 -1
  12. package/dist/compose/timeline.js.map +1 -1
  13. package/dist/config/config.d.ts.map +1 -1
  14. package/dist/config/config.js +18 -2
  15. package/dist/config/config.js.map +1 -1
  16. package/dist/config/schema.d.ts +16 -0
  17. package/dist/config/schema.d.ts.map +1 -1
  18. package/dist/config/schema.js +50 -3
  19. package/dist/config/schema.js.map +1 -1
  20. package/dist/footage/captions.d.ts +5 -1
  21. package/dist/footage/captions.d.ts.map +1 -1
  22. package/dist/footage/captions.js +6 -2
  23. package/dist/footage/captions.js.map +1 -1
  24. package/dist/footage/compose.d.ts +13 -2
  25. package/dist/footage/compose.d.ts.map +1 -1
  26. package/dist/footage/compose.js +75 -23
  27. package/dist/footage/compose.js.map +1 -1
  28. package/dist/footage/cuts.d.ts +44 -0
  29. package/dist/footage/cuts.d.ts.map +1 -0
  30. package/dist/footage/cuts.js +83 -0
  31. package/dist/footage/cuts.js.map +1 -0
  32. package/dist/footage/draft.d.ts +17 -3
  33. package/dist/footage/draft.d.ts.map +1 -1
  34. package/dist/footage/draft.js +28 -14
  35. package/dist/footage/draft.js.map +1 -1
  36. package/dist/footage/edl.d.ts +2 -1
  37. package/dist/footage/edl.d.ts.map +1 -1
  38. package/dist/footage/edl.js +5 -2
  39. package/dist/footage/edl.js.map +1 -1
  40. package/dist/footage/film.d.ts +42 -3
  41. package/dist/footage/film.d.ts.map +1 -1
  42. package/dist/footage/film.js +7 -1
  43. package/dist/footage/film.js.map +1 -1
  44. package/dist/footage/loudness.d.ts +20 -3
  45. package/dist/footage/loudness.d.ts.map +1 -1
  46. package/dist/footage/loudness.js +36 -18
  47. package/dist/footage/loudness.js.map +1 -1
  48. package/dist/footage/music.d.ts +3 -0
  49. package/dist/footage/music.d.ts.map +1 -1
  50. package/dist/footage/music.js +5 -0
  51. package/dist/footage/music.js.map +1 -1
  52. package/dist/footage/normalize.d.ts +9 -0
  53. package/dist/footage/normalize.d.ts.map +1 -1
  54. package/dist/footage/normalize.js +8 -4
  55. package/dist/footage/normalize.js.map +1 -1
  56. package/dist/footage/picture.d.ts +42 -0
  57. package/dist/footage/picture.d.ts.map +1 -0
  58. package/dist/footage/picture.js +141 -0
  59. package/dist/footage/picture.js.map +1 -0
  60. package/dist/footage/pipeline.d.ts +46 -1
  61. package/dist/footage/pipeline.d.ts.map +1 -1
  62. package/dist/footage/pipeline.js +128 -19
  63. package/dist/footage/pipeline.js.map +1 -1
  64. package/dist/footage/publish.d.ts +4 -2
  65. package/dist/footage/publish.d.ts.map +1 -1
  66. package/dist/footage/publish.js +8 -5
  67. package/dist/footage/publish.js.map +1 -1
  68. package/dist/footage/render.d.ts +17 -2
  69. package/dist/footage/render.d.ts.map +1 -1
  70. package/dist/footage/render.js +62 -13
  71. package/dist/footage/render.js.map +1 -1
  72. package/dist/index.d.ts +8 -6
  73. package/dist/index.d.ts.map +1 -1
  74. package/dist/index.js +8 -6
  75. package/dist/index.js.map +1 -1
  76. package/docker/footage-worker.Dockerfile +1 -0
  77. package/package.json +1 -1
  78. package/schema/marketing.schema.json +105 -4
  79. package/src/cli/footage.ts +4 -2
  80. package/src/compose/compose.ts +13 -3
  81. package/src/compose/timeline.ts +27 -1
  82. package/src/config/config.ts +18 -2
  83. package/src/config/schema.ts +52 -3
  84. package/src/footage/captions.ts +6 -2
  85. package/src/footage/compose.ts +86 -23
  86. package/src/footage/cuts.ts +104 -0
  87. package/src/footage/draft.ts +43 -14
  88. package/src/footage/edl.ts +5 -2
  89. package/src/footage/film.ts +47 -4
  90. package/src/footage/loudness.ts +38 -16
  91. package/src/footage/music.ts +7 -0
  92. package/src/footage/normalize.ts +8 -4
  93. package/src/footage/picture.ts +179 -0
  94. package/src/footage/pipeline.ts +166 -22
  95. package/src/footage/publish.ts +8 -5
  96. package/src/footage/render.ts +72 -13
  97. package/src/index.ts +36 -2
@@ -8,7 +8,8 @@ import { runFfmpeg, runProcess } from "./process.js";
8
8
  /**
9
9
  * The speech track's loudness, measured and normalized in two passes of ffmpeg's `loudnorm` (EBU R128): the first
10
10
  * pass measures, the second applies one linear gain from that measurement, so a quiet phone clip is not left quiet
11
- * and a loud one does not clip. It runs before the music is mixed under the speech.
11
+ * and a loud one does not clip. It runs on the speech track before the music is mixed under it, and on the finished
12
+ * film's mix (`FootageFilm.mixLoudness`), so the published file plays at the target whatever the music adds.
12
13
  */
13
14
 
14
15
  export interface LoudnessTarget {
@@ -48,9 +49,9 @@ function getLoudnormFilter(target: LoudnessTarget): string {
48
49
  return `loudnorm=I=${target.target}:TP=${target.truePeak}:LRA=${LOUDNESS_RANGE}`;
49
50
  }
50
51
 
51
- /** The first pass: decodes the track, prints the measurement on stderr, writes nothing. */
52
+ /** The first pass: decodes the sound only (a film's picture is skipped), prints the measurement on stderr, writes nothing. */
52
53
  export function buildLoudnessMeasureArgs(input: string, target: LoudnessTarget): string[] {
53
- return ["-hide_banner", "-nostats", "-i", input, "-af", `${getLoudnormFilter(target)}:print_format=json`, "-f", "null", "-"];
54
+ return ["-hide_banner", "-nostats", "-i", input, "-vn", "-af", `${getLoudnormFilter(target)}:print_format=json`, "-f", "null", "-"];
54
55
  }
55
56
 
56
57
  /** The measurement from the first pass's stderr (its last JSON block); null when there is none or it is incomplete. */
@@ -70,8 +71,12 @@ export function readLoudnessMeasurement(stderr: string): LoudnessMeasurement | n
70
71
  const parsed = measurementSchema.safeParse(json);
71
72
  if (!parsed.success) return null;
72
73
  const value = parsed.data;
73
- // `-inf` (silence) reads as -Infinity, which `isSilent` tells apart.
74
- const read = (text: string): number => (text.trim() === "-inf" ? -Infinity : Number(text));
74
+ // Silence prints `-inf` loudness and an `inf` offset; both read as infinities, which `isSilent` tells apart.
75
+ const read = (text: string): number => {
76
+ const trimmed = text.trim();
77
+ if (trimmed === "-inf") return -Infinity;
78
+ return trimmed === "inf" || trimmed === "+inf" ? Infinity : Number(trimmed);
79
+ };
75
80
  const measurement = {
76
81
  inputI: read(value.input_i),
77
82
  inputTp: read(value.input_tp),
@@ -87,11 +92,10 @@ export function isSilent(measurement: LoudnessMeasurement): boolean {
87
92
  return !Number.isFinite(measurement.inputI) || measurement.inputI < SILENCE_LUFS || !Number.isFinite(measurement.inputThresh);
88
93
  }
89
94
 
90
- /** The second pass: one linear gain from the measurement, resampled back to the film's rate (`loudnorm` runs at 192 kHz). */
91
- export function buildLoudnessArgs(options: { input: string; measurement: LoudnessMeasurement; target: LoudnessTarget; output: string }): string[] {
92
- const { measurement } = options;
95
+ /** The second pass's filter: one linear gain from the measurement, resampled back to the film's rate (`loudnorm` runs at 192 kHz). */
96
+ export function getLoudnessFilter(measurement: LoudnessMeasurement, target: LoudnessTarget): string {
93
97
  const filter = [
94
- getLoudnormFilter(options.target),
98
+ getLoudnormFilter(target),
95
99
  `measured_I=${measurement.inputI}`,
96
100
  `measured_TP=${measurement.inputTp}`,
97
101
  `measured_LRA=${measurement.inputLra}`,
@@ -99,6 +103,11 @@ export function buildLoudnessArgs(options: { input: string; measurement: Loudnes
99
103
  `offset=${measurement.targetOffset}`,
100
104
  "linear=true",
101
105
  ].join(":");
106
+ return `${filter},aresample=${FOOTAGE_SAMPLE_RATE}`;
107
+ }
108
+
109
+ /** The speech track's second pass, written as an mp3 at the film's rate. */
110
+ export function buildLoudnessArgs(options: { input: string; measurement: LoudnessMeasurement; target: LoudnessTarget; output: string }): string[] {
102
111
  return [
103
112
  "-v",
104
113
  "error",
@@ -106,7 +115,7 @@ export function buildLoudnessArgs(options: { input: string; measurement: Loudnes
106
115
  "-i",
107
116
  options.input,
108
117
  "-af",
109
- `${filter},aresample=${FOOTAGE_SAMPLE_RATE}`,
118
+ getLoudnessFilter(options.measurement, options.target),
110
119
  "-ar",
111
120
  String(FOOTAGE_SAMPLE_RATE),
112
121
  "-c:a",
@@ -117,6 +126,22 @@ export function buildLoudnessArgs(options: { input: string; measurement: Loudnes
117
126
  ];
118
127
  }
119
128
 
129
+ export type LoudnessMeasureOutcome = { kind: "measured"; measurement: LoudnessMeasurement } | { kind: "silent" };
130
+
131
+ /**
132
+ * The first pass over `input`: its measurement, or silent when no gain can bring it to the target. Throws naming
133
+ * `label` (what is measured) when ffmpeg fails, rejects with the signal's reason on abort.
134
+ */
135
+ export async function measureLoudness(input: string, target: LoudnessTarget, options: { label: string; signal?: AbortSignal }): Promise<LoudnessMeasureOutcome> {
136
+ const step = `Measuring ${options.label}'s loudness`;
137
+ const measured = await runProcess({ command: "ffmpeg", args: buildLoudnessMeasureArgs(input, target), ...(options.signal === undefined ? {} : { signal: options.signal }) });
138
+ if (measured.kind === "not-started") throw new Error(`${step}: ffmpeg did not start (${measured.error}).`);
139
+ if (measured.code !== 0) throw new Error(`${step}: ffmpeg ended with code ${String(measured.code)}: ${measured.stderr.trim().split("\n").slice(-3).join(" | ")}.`);
140
+ const measurement = readLoudnessMeasurement(measured.stderr);
141
+ if (measurement === null) throw new Error(`${step}: ffmpeg printed no loudnorm measurement for ${input}.`);
142
+ return isSilent(measurement) ? { kind: "silent" } : { kind: "measured", measurement };
143
+ }
144
+
120
145
  export type LoudnessOutcome = { kind: "normalized"; measured: number } | { kind: "silent" };
121
146
 
122
147
  /**
@@ -125,15 +150,12 @@ export type LoudnessOutcome = { kind: "normalized"; measured: number } | { kind:
125
150
  */
126
151
  export async function normalizeLoudness(input: string, output: string, target: LoudnessTarget, options: { signal?: AbortSignal; onSeconds?: (seconds: number) => void } = {}): Promise<LoudnessOutcome> {
127
152
  const signal = options.signal === undefined ? {} : { signal: options.signal };
128
- const measured = await runProcess({ command: "ffmpeg", args: buildLoudnessMeasureArgs(input, target), ...signal });
129
- if (measured.kind === "not-started") throw new Error(`Measuring the speech track's loudness: ffmpeg did not start (${measured.error}).`);
130
- if (measured.code !== 0) throw new Error(`Measuring the speech track's loudness: ffmpeg ended with code ${String(measured.code)}: ${measured.stderr.trim().split("\n").slice(-3).join(" | ")}.`);
131
- const measurement = readLoudnessMeasurement(measured.stderr);
132
- if (measurement === null) throw new Error(`Measuring the speech track's loudness: ffmpeg printed no loudnorm measurement for ${input}.`);
133
- if (isSilent(measurement)) {
153
+ const measured = await measureLoudness(input, target, { label: "the speech track", ...signal });
154
+ if (measured.kind === "silent") {
134
155
  copyFileSync(input, output);
135
156
  return { kind: "silent" };
136
157
  }
158
+ const { measurement } = measured;
137
159
  await runFfmpeg("Normalizing the speech track's loudness", buildLoudnessArgs({ input, measurement, target, output }), {
138
160
  ...signal,
139
161
  output,
@@ -1,4 +1,5 @@
1
1
  import type { TimedWord } from "../voice/voiceover.js";
2
+ import type { FootageMusic } from "./film.js";
2
3
 
3
4
  /**
4
5
  * Music under the voice: loud between sentences, ducked while someone speaks. The envelope is a list of keyframes
@@ -22,6 +23,12 @@ export interface DuckingOptions {
22
23
  gap?: number;
23
24
  }
24
25
 
26
+ /** A film's music settings as ducking options: its volumes, and its attack and release when it sets them. */
27
+ export function getDuckingOptions(music: Pick<FootageMusic, "volume" | "duckedVolume" | "attack" | "release">): DuckingOptions {
28
+ const { volume, duckedVolume, attack, release } = music;
29
+ return { volume, duckedVolume, ...(attack === undefined ? {} : { attack }), ...(release === undefined ? {} : { release }) };
30
+ }
31
+
25
32
  const r3 = (value: number): number => Math.round(value * 1000) / 1000 || 0;
26
33
 
27
34
  /** Stretches of speech: the words joined when less than `gap` seconds apart. */
@@ -98,10 +98,14 @@ export interface NormalizeInput {
98
98
  encoder?: NormalizeEncoder;
99
99
  }
100
100
 
101
- /** The thread arguments: global filter threads (before the inputs) and encoder threads (before the output). */
102
- function getThreadArgs(threads: number | undefined): { global: string[]; output: string[] } {
101
+ /**
102
+ * The thread arguments: global filter threads (before the inputs) and encoder threads (before the output; the same
103
+ * flag before an input sets that input's decoder threads). Throws, naming `option`, when `threads` is not a positive
104
+ * whole number.
105
+ */
106
+ export function getThreadArgs(threads: number | undefined, option: string): { global: string[]; output: string[] } {
103
107
  if (threads === undefined) return { global: [], output: [] };
104
- if (!Number.isInteger(threads) || threads < 1) throw new Error(`normalize.threads must be a positive whole number, got ${threads}.`);
108
+ if (!Number.isInteger(threads) || threads < 1) throw new Error(`${option} must be a positive whole number, got ${threads}.`);
105
109
  return { global: ["-filter_complex_threads", String(threads)], output: ["-threads", String(threads)] };
106
110
  }
107
111
 
@@ -111,7 +115,7 @@ export function buildNormalizeArgs(input: NormalizeInput): string[] {
111
115
  const duration = getTrimmedDuration(clip, probe);
112
116
  const fit = resolveFit(clip.fit, probe, frame);
113
117
  const seek = clip.trim.start > 0 ? ["-ss", String(clip.trim.start)] : [];
114
- const threads = getThreadArgs(input.encoder?.threads);
118
+ const threads = getThreadArgs(input.encoder?.threads, "normalize.threads");
115
119
  const silence = probe.hasAudio ? [] : ["-f", "lavfi", "-i", `anullsrc=r=${FOOTAGE_SAMPLE_RATE}:cl=stereo`];
116
120
  return [
117
121
  "-v",
@@ -0,0 +1,179 @@
1
+ import { getSegmentScales } from "./compose.js";
2
+ import { getEdlDuration, getSegmentStarts, type Edl } from "./edl.js";
3
+ import { getFootageEdit, type FootageFilm } from "./film.js";
4
+ import { FOOTAGE_FPS, getThreadArgs, type NormalizeEncoder } from "./normalize.js";
5
+
6
+ /**
7
+ * The cut's picture in film time, which the render's composition plays as one `<video>`: one short ffmpeg per segment
8
+ * (run one after the other, so one decoder at a time) and a stream-copy join. hyperframes then extracts the whole
9
+ * film's frames with one ffmpeg, and its Chrome holds one video instead of one per segment (issue #1030: the
10
+ * extraction peak falls from about 1 GB to 0.2 GB, the capture's from 1.7 GB to 1.1 GB on a 45 s 1080x1920 film).
11
+ *
12
+ * Every piece starts on a film frame and runs to the next segment's frame, so rounding never adds up along the cut.
13
+ * A cross-fade is drawn into the incoming piece: the outgoing segment's tail, held at the zoom it ends on, dissolves
14
+ * into the incoming clip (`xfade`), as the per-segment composition plays it. Zoom cuts and Ken Burns stay tweens on
15
+ * the composition's video.
16
+ */
17
+
18
+ /** The point a zoom keeps still, as the composition's `transform-origin: 50% 42%`. */
19
+ const ZOOM_ORIGIN = { x: 0.5, y: 0.42 } as const;
20
+ /** How far a segment marked `kenburns` zooms when the film's `kenBurnsPercent` is 0, as in the composition. */
21
+ const HAND_KEN_BURNS_PERCENT = 4;
22
+ const PICTURE_CRF = 18;
23
+ const DEFAULT_PICTURE_PRESET = "veryfast";
24
+
25
+ const r3 = (value: number): number => Math.round(value * 1000) / 1000 || 0;
26
+
27
+ /** The outgoing segment's tail under a cross-fade. */
28
+ export interface PictureFadeTail {
29
+ clip: string;
30
+ /** Seconds in the clip: where the outgoing segment ends. */
31
+ start: number;
32
+ frames: number;
33
+ /** The zoom the outgoing segment ends on, held through the fade. */
34
+ scale: number;
35
+ }
36
+
37
+ /** One segment's piece of the picture. */
38
+ export interface PicturePiece {
39
+ clip: string;
40
+ /** Seconds in the clip where the piece starts. */
41
+ start: number;
42
+ /** The piece's first film frame. */
43
+ frame: number;
44
+ frames: number;
45
+ /** For a `fade` cut, the outgoing tail the piece dissolves from; null for any other cut. */
46
+ fade: PictureFadeTail | null;
47
+ }
48
+
49
+ /** The film frame a film second falls on. */
50
+ function toFrame(seconds: number): number {
51
+ return Math.round(seconds * FOOTAGE_FPS);
52
+ }
53
+
54
+ /** The picture's pieces, one per segment, covering the cut's frames without a gap. Throws on an EDL without segments. */
55
+ export function planFootagePicture(film: Pick<FootageFilm, "edit">, edl: Edl): PicturePiece[] {
56
+ if (edl.segments.length === 0) throw new Error("The EDL has no segments.");
57
+ const edit = getFootageEdit(film);
58
+ const kenBurnsScale = 1 + (edit.kenBurnsPercent > 0 ? edit.kenBurnsPercent : HAND_KEN_BURNS_PERCENT) / 100;
59
+ const scales = getSegmentScales(edl, edit.zoomPercent);
60
+ const starts = getSegmentStarts(edl);
61
+ const lastFrame = toFrame(getEdlDuration(edl));
62
+ return edl.segments.map((segment, index) => {
63
+ const frame = toFrame(starts[index] as number);
64
+ const nextStart = starts[index + 1];
65
+ const frames = Math.max(1, (nextStart === undefined ? lastFrame : toFrame(nextStart)) - frame);
66
+ const previous = edl.segments[index - 1];
67
+ const fadeFrames = segment.transition === "fade" ? Math.min(frames, edit.fadeFrames) : 0;
68
+ let fade: PictureFadeTail | null = null;
69
+ if (previous !== undefined && fadeFrames > 0) {
70
+ const previousScale = scales[index - 1] as number;
71
+ // The outgoing segment's Ken Burns tween ends at its end; its tail holds the zoom it reached.
72
+ const scale = previous.motion === "kenburns" ? Math.round(previousScale * kenBurnsScale * 10000) / 10000 : previousScale;
73
+ fade = { clip: previous.clip, start: r3(previous.end), frames: fadeFrames, scale };
74
+ }
75
+ return { clip: segment.clip, start: r3(segment.start), frame, frames, fade };
76
+ });
77
+ }
78
+
79
+ /** The number of frames the picture has: the cut's length in film frames. */
80
+ export function getPictureFrames(pieces: readonly PicturePiece[]): number {
81
+ return pieces.reduce((sum, piece) => sum + piece.frames, 0);
82
+ }
83
+
84
+ /** Filters that bring a stream to the film's rate and pixel format and exactly `frames` long (a short clip holds its last frame). */
85
+ function fitStream(frames: number): string[] {
86
+ const seconds = r3(frames / FOOTAGE_FPS);
87
+ return [
88
+ "setpts=PTS-STARTPTS",
89
+ `fps=${FOOTAGE_FPS}`,
90
+ "format=yuv420p",
91
+ "setsar=1",
92
+ `tpad=stop_mode=clone:stop_duration=${seconds}`,
93
+ `trim=end_frame=${frames}`,
94
+ "setpts=PTS-STARTPTS",
95
+ `settb=1/${FOOTAGE_FPS}`,
96
+ ];
97
+ }
98
+
99
+ /** A crop to `scale` about the zoom origin and back to the clip's size; nothing at scale 1. */
100
+ function zoomFilters(scale: number): string[] {
101
+ if (scale === 1) return [];
102
+ return [`crop=iw/${scale}:ih/${scale}:(iw-iw/${scale})*${ZOOM_ORIGIN.x}:(ih-ih/${scale})*${ZOOM_ORIGIN.y}`, `scale=iw*${scale}:ih*${scale}`];
103
+ }
104
+
105
+ export interface PicturePieceArgsInput {
106
+ piece: PicturePiece;
107
+ /** The normalized clips by id. */
108
+ clips: Record<string, string>;
109
+ output: string;
110
+ /** The intermediate clips' encoder (`RenderFootageOptions.normalize`): its preset and threads encode the piece too. */
111
+ encoder?: NormalizeEncoder;
112
+ }
113
+
114
+ /** One piece's ffmpeg arguments: its clip (and a fade's outgoing tail), exactly `frames` frames, H.264 without sound. */
115
+ export function buildPicturePieceArgs(input: PicturePieceArgsInput): string[] {
116
+ const { piece, clips } = input;
117
+ const getClip = (id: string): string => {
118
+ const file = clips[id];
119
+ if (file === undefined) throw new Error(`No file for clip "${id}" of the EDL.`);
120
+ return file;
121
+ };
122
+ const threads = getThreadArgs(input.encoder?.threads, "normalize.threads");
123
+ const seconds = (frames: number): string => String(r3(frames / FOOTAGE_FPS + 1 / FOOTAGE_FPS));
124
+ const inputs = [...threads.output, "-ss", String(piece.start), "-t", seconds(piece.frames), "-i", getClip(piece.clip)];
125
+ let graph = `[0:v]${fitStream(piece.frames).join(",")}[picture]`;
126
+ if (piece.fade !== null) {
127
+ const { fade } = piece;
128
+ inputs.push(...threads.output, "-ss", String(fade.start), "-t", seconds(fade.frames), "-i", getClip(fade.clip));
129
+ const tail = [...zoomFilters(fade.scale), ...fitStream(fade.frames)];
130
+ graph = [
131
+ `[0:v]${fitStream(piece.frames).join(",")}[incoming]`,
132
+ `[1:v]${tail.join(",")}[outgoing]`,
133
+ // xfade's output is the outgoing tail plus the incoming piece minus the fade: the piece's own length.
134
+ `[outgoing][incoming]xfade=transition=fade:duration=${r3(fade.frames / FOOTAGE_FPS)}:offset=0,trim=end_frame=${piece.frames}[picture]`,
135
+ ].join(";");
136
+ }
137
+ return [
138
+ "-v",
139
+ "error",
140
+ "-y",
141
+ ...threads.global,
142
+ ...inputs,
143
+ "-filter_complex",
144
+ graph,
145
+ "-map",
146
+ "[picture]",
147
+ "-an",
148
+ "-frames:v",
149
+ String(piece.frames),
150
+ "-c:v",
151
+ "libx264",
152
+ "-preset",
153
+ input.encoder?.preset ?? DEFAULT_PICTURE_PRESET,
154
+ "-crf",
155
+ String(PICTURE_CRF),
156
+ "-g",
157
+ String(FOOTAGE_FPS / 2),
158
+ "-pix_fmt",
159
+ "yuv420p",
160
+ "-color_primaries",
161
+ "bt709",
162
+ "-color_trc",
163
+ "bt709",
164
+ "-colorspace",
165
+ "bt709",
166
+ ...threads.output,
167
+ input.output,
168
+ ];
169
+ }
170
+
171
+ /** The concat demuxer's list: one `file` line per piece, quotes escaped as the demuxer reads them. */
172
+ export function buildPictureList(files: readonly string[]): string {
173
+ return `${files.map((file) => `file '${file.replace(/'/g, "'\\''")}'`).join("\n")}\n`;
174
+ }
175
+
176
+ /** Joins the pieces by stream copy (they share one encoder's settings) into the picture, fast start. */
177
+ export function buildPictureConcatArgs(list: string, output: string): string[] {
178
+ return ["-v", "error", "-y", "-f", "concat", "-safe", "0", "-i", list, "-c", "copy", "-movflags", "+faststart", output];
179
+ }
@@ -15,13 +15,15 @@ import { readCachedTranscript, writeCacheFile, writeTranscript } from "./cache.j
15
15
  import { composeFootage, END_CARD_SECONDS, getFootageFilmSeconds } from "./compose.js";
16
16
  import { findMissingCaptionGlyphs, getFootageCaptionLayout } from "./captions.js";
17
17
  import { findLowDiskProblems, getFreeBytes, getRenderTempDir, type FreeBytesReader } from "./disk.js";
18
- import { buildEdl, readEdl, remapWords, validateEdl, type Edl } from "./edl.js";
18
+ import { buildEdl, getEdlDuration, readEdl, remapWords, validateEdl, type Edl } from "./edl.js";
19
19
  import { makePlaceholderTranscript } from "./fake.js";
20
- import { getFootageEdit, type FootageClip, type FootageFilm, type MusicSource } from "./film.js";
20
+ import { chooseCutTransitions, type CutMeasure } from "./cuts.js";
21
+ import { DEFAULT_AUTO_THRESHOLD, getFootageEdit, type FootageClip, type FootageFilm, type MusicSource } from "./film.js";
21
22
  import { getFilmKeyTerms } from "./keyterms.js";
22
- import { DEFAULT_LOUDNESS, normalizeLoudness } from "./loudness.js";
23
- import { duckingKeyframes } from "./music.js";
24
- import { normalizeFootage, type Frame, type NormalizeEncoder } from "./normalize.js";
23
+ import { DEFAULT_LOUDNESS, getLoudnessFilter, measureLoudness, normalizeLoudness } from "./loudness.js";
24
+ import { duckingKeyframes, getDuckingOptions } from "./music.js";
25
+ import { FOOTAGE_FPS, normalizeFootage, type Frame, type NormalizeEncoder } from "./normalize.js";
26
+ import { buildPictureConcatArgs, buildPictureList, buildPicturePieceArgs, getPictureFrames, planFootagePicture } from "./picture.js";
25
27
  import { describeProbe, getTrimmedDuration, probeFootage, type FootageProbe } from "./probe.js";
26
28
  import {
27
29
  getClipCacheKey,
@@ -49,7 +51,7 @@ import {
49
51
  readTopLevelBoxes,
50
52
  } from "./publish.js";
51
53
  import { describePaidSteps, toPaidAudio, type FootageCharge, type FootageProgress, type FootageProviders, type FootageStage, type PaidAudio, type PaidStep } from "./providers.js";
52
- import { buildAudioArgs, buildCutPreviewArgs, buildLastFrameArgs, buildMusicArgs, buildSpeechArgs } from "./render.js";
54
+ import { buildAudioArgs, buildCutPreviewArgs, buildLastFrameArgs, buildMusicArgs, buildSpeechArgs, getSpeechCutOptions } from "./render.js";
53
55
 
54
56
  export { getEdlPath } from "./paid.js";
55
57
 
@@ -369,20 +371,77 @@ export function getFootageSentences(clips: PreparedClip[], words: Record<string,
369
371
  return splitFilmSentences(clips.map((clip) => ({ id: clip.clip.id, role: clip.clip.role, words: words[clip.clip.id] ?? [] })));
370
372
  }
371
373
 
372
- /** The cut from the transcripts, written to the EDL file (replacing what is there); `keepSentences` keeps only those sentences. */
373
- export function cutFootage(film: FootageFilm, clips: PreparedClip[], words: Record<string, TimedWord[]>, edlPath: string, options: { keepSentences?: readonly string[] } = {}): Edl {
374
- const edl = buildEdl(
374
+ function buildFilmEdl(film: FootageFilm, clips: PreparedClip[], words: Record<string, TimedWord[]>, keepSentences: readonly string[] | undefined): Edl {
375
+ return buildEdl(
375
376
  film.id,
376
377
  clips.map((clip) => ({ id: clip.clip.id, role: clip.clip.role, duration: clip.duration, words: words[clip.clip.id] ?? [] })),
377
378
  film.cut,
378
379
  film.transitions,
379
- { isKenBurns: getFootageEdit(film).kenBurnsPercent > 0, ...(options.keepSentences === undefined ? {} : { keepSentences: options.keepSentences }) },
380
+ { isKenBurns: getFootageEdit(film).kenBurnsPercent > 0, ...(keepSentences === undefined ? {} : { keepSentences }) },
380
381
  );
382
+ }
383
+
384
+ function writeEdl(edl: Edl, edlPath: string): void {
381
385
  mkdirSync(dirname(edlPath), { recursive: true });
382
386
  writeFileSync(edlPath, `${JSON.stringify(edl, null, 2)}\n`);
387
+ }
388
+
389
+ /**
390
+ * The cut from the transcripts, written to the EDL file (replacing what is there); `keepSentences` keeps only those
391
+ * sentences. Sync, so it never looks at the frames: with `transitions.jump: "auto"` the jump cuts stay plain cuts, and
392
+ * `cutFootageFilm` is the step that chooses them.
393
+ */
394
+ export function cutFootage(film: FootageFilm, clips: PreparedClip[], words: Record<string, TimedWord[]>, edlPath: string, options: { keepSentences?: readonly string[] } = {}): Edl {
395
+ const edl = buildFilmEdl(film, clips, words, options.keepSentences);
396
+ writeEdl(edl, edlPath);
383
397
  return edl;
384
398
  }
385
399
 
400
+ export interface CutFootageFilmOptions {
401
+ /** The sentence ids to keep; absent keeps all. */
402
+ keepSentences?: readonly string[];
403
+ signal?: AbortSignal;
404
+ }
405
+
406
+ /** The file next to the EDL that records what the auto transitions measured. */
407
+ export const CUTS_FILE = "cuts.json";
408
+
409
+ /**
410
+ * The cut as `cutFootage` builds it, with `transitions.jump: "auto"` resolved from the frames of the normalized clips
411
+ * (`chooseCutTransitions`): each jump cut gets `zoom` or `fade`, and what it measured is answered and written to
412
+ * `cuts.json` next to the EDL, so an app or an agent sees why. Otherwise the same as `cutFootage`, and `cuts` is empty.
413
+ */
414
+ export async function cutFootageFilm(
415
+ film: FootageFilm,
416
+ clips: PreparedClip[],
417
+ words: Record<string, TimedWord[]>,
418
+ edlPath: string,
419
+ options: CutFootageFilmOptions = {},
420
+ ): Promise<{ edl: Edl; cuts: CutMeasure[] }> {
421
+ let edl = buildFilmEdl(film, clips, words, options.keepSentences);
422
+ let cuts: CutMeasure[] = [];
423
+ const cutsPath = join(dirname(edlPath), CUTS_FILE);
424
+ rmSync(cutsPath, { force: true });
425
+ if (film.transitions.jump === "auto") {
426
+ const chosen = await chooseCutTransitions(edl, Object.fromEntries(clips.map((clip) => [clip.clip.id, clip.file])), {
427
+ workDir: join(dirname(edlPath), "cut-frames"),
428
+ ...(film.transitions.autoThreshold === undefined ? {} : { threshold: film.transitions.autoThreshold }),
429
+ ...(options.signal === undefined ? {} : { signal: options.signal }),
430
+ });
431
+ edl = chosen.edl;
432
+ cuts = chosen.cuts;
433
+ mkdirSync(dirname(cutsPath), { recursive: true });
434
+ writeFileSync(cutsPath, `${JSON.stringify({ threshold: film.transitions.autoThreshold ?? DEFAULT_AUTO_THRESHOLD, cuts }, null, 2)}\n`);
435
+ }
436
+ writeEdl(edl, edlPath);
437
+ return { edl, cuts };
438
+ }
439
+
440
+ /** One line per auto-chosen cut, for a log: `s4: zoom (difference 0.031)`. */
441
+ export function describeCutMeasures(cuts: CutMeasure[]): string[] {
442
+ return cuts.map((cut) => `${cut.segment}: ${cut.transition} (difference ${cut.difference.toFixed(3)})`);
443
+ }
444
+
386
445
  /** The EDL file as written by `cutFootage` or edited by hand, checked against the clips. */
387
446
  export function readEdlFile(film: FootageFilm, clips: PreparedClip[], edlPath: string): FootageResult<Edl> {
388
447
  let json: unknown;
@@ -486,6 +545,16 @@ export interface RenderFootageOptions extends FootageRunOptions {
486
545
  tempDir?: string;
487
546
  /** Reads a folder's free bytes for the disk check; injected in tests. */
488
547
  readFreeBytes?: FreeBytesReader;
548
+ /**
549
+ * hyperframes' capture workers (`--workers`), each a Chrome of its own; by default hyperframes' choice (one in its
550
+ * low-memory profile). A positive whole number.
551
+ */
552
+ workers?: number;
553
+ /**
554
+ * hyperframes' low-memory profile (`--low-memory-mode`: one worker, screenshot capture): true forces it on, false
555
+ * off; by default hyperframes turns it on by itself on a machine (or cgroup) of 8 GB or less.
556
+ */
557
+ lowMemory?: boolean;
489
558
  }
490
559
 
491
560
  export type RenderedFootage = { kind: "dry-run"; steps: PaidStep[] } | ({ kind: "rendered"; output: string; seconds: number; edlPath: string } & FinishedFootage);
@@ -525,7 +594,7 @@ export async function renderFootage(
525
594
  const rawSpeechFile = loudness === null ? speechFile : join(assetsDir, "speech.raw.mp3");
526
595
  const ffmpegTrack = (label: string, output: string, args: () => string[]) => (onSeconds: (done: number) => void) => runFfmpeg(label, args(), { ...aborting, output, onSeconds });
527
596
  const tracks: { item: string; seconds: number; run: (onSeconds: (done: number) => void) => Promise<void> }[] = [
528
- { item: "speech", seconds, run: ffmpegTrack("The speech track", rawSpeechFile, () => buildSpeechArgs(edl, transcript.speech, rawSpeechFile)) },
597
+ { item: "speech", seconds, run: ffmpegTrack("The speech track", rawSpeechFile, () => buildSpeechArgs(edl, transcript.speech, rawSpeechFile, getSpeechCutOptions(getFootageEdit(film)))) },
529
598
  ];
530
599
  if (loudness !== null) {
531
600
  tracks.push({
@@ -542,13 +611,21 @@ export async function renderFootage(
542
611
  });
543
612
  }
544
613
  if (musicFile !== null) {
545
- const keyframes = duckingKeyframes(words, { volume: film.music.volume, duckedVolume: film.music.duckedVolume });
614
+ const keyframes = duckingKeyframes(words, getDuckingOptions(film.music));
546
615
  const output = join(assetsDir, "music.mp3");
547
616
  tracks.push({ item: "music", seconds, run: ffmpegTrack("The music track", output, () => buildMusicArgs({ input: musicFile, seconds, keyframes, output })) });
548
617
  }
549
618
  // A single frame: counted as a moment of work so the stage's bar still ends at its total.
550
619
  const lastFrame = join(assetsDir, "last.jpg");
551
620
  tracks.push({ item: "last frame", seconds: 0.1, run: ffmpegTrack("The end card's backdrop", lastFrame, () => buildLastFrameArgs(edl, clipFiles, lastFrame)) });
621
+ // The cut's picture in film time, one segment after the other: the composition plays it as one <video> (#1030).
622
+ const pictureFile = join(assetsDir, "picture.mp4");
623
+ const pictureSeconds = getEdlDuration(edl);
624
+ tracks.push({
625
+ item: "picture",
626
+ seconds: pictureSeconds,
627
+ run: (onSeconds) => buildPicture({ film, edl, clips: clipFiles, output: pictureFile, ...(options.normalize === undefined ? {} : { encoder: options.normalize }), ...aborting, onSeconds }),
628
+ });
552
629
  const assetProgress = createStageProgress(options.onProgress, "assets", tracks);
553
630
  for (const [index, track] of tracks.entries()) {
554
631
  assetProgress(index, 0);
@@ -589,6 +666,7 @@ export async function renderFootage(
589
666
  fonts,
590
667
  assets: {
591
668
  clips: Object.fromEntries(clips.map((clip) => [clip.clip.id, `clips/${clip.clip.id}.mp4`])),
669
+ picture: "assets/picture.mp4",
592
670
  speech: "assets/speech.mp3",
593
671
  music: musicAsset,
594
672
  lastFrame: "assets/last.jpg",
@@ -622,13 +700,14 @@ export interface FinishedFootage {
622
700
  }
623
701
 
624
702
  /**
625
- * The final pass over hyperframes' render: publishable codecs and fast start (stream copy where the render already
626
- * matches), checked with ffprobe, then the cover and the file's facts.
703
+ * The final pass over hyperframes' render: the mix normalized to `film.mixLoudness` (measured on the render, one
704
+ * linear gain; the picture still copied), publishable codecs and fast start (stream copy where the render already
705
+ * matches), checked with ffprobe, then the cover and the file's facts. Exported for tests only.
627
706
  */
628
- async function finishFootage(
629
- film: FootageFilm,
707
+ export async function finishFootage(
708
+ film: Pick<FootageFilm, "mixLoudness" | "cover">,
630
709
  input: { renderFile: string; seconds: number; coverAt: number },
631
- options: RenderFootageOptions,
710
+ options: Pick<RenderFootageOptions, "log" | "signal" | "output" | "onProgress">,
632
711
  ): Promise<FootageResult<FinishedFootage>> {
633
712
  const { log, signal, output } = options;
634
713
  const aborting = signal === undefined ? {} : { signal };
@@ -636,8 +715,21 @@ async function finishFootage(
636
715
  progress(0, 0);
637
716
  const rendered = await probeFilmFacts(input.renderFile, aborting);
638
717
  if (!rendered.ok) return err(`the rendered film: ${rendered.error}`);
639
- const isCopied = isPublishableVideo(rendered.facts.video) && (rendered.facts.audio === null || isPublishableAudio(rendered.facts.audio));
640
- await runFfmpeg("The film's final pass", buildFinishArgs(input.renderFile, rendered.facts, output), { ...aborting, output, onSeconds: (done) => progress(0, done) });
718
+ const mixLoudness = film.mixLoudness === undefined ? DEFAULT_LOUDNESS : film.mixLoudness;
719
+ let audioFilter: string | undefined;
720
+ if (mixLoudness !== null && rendered.facts.audio !== null) {
721
+ const measured = await measureLoudness(input.renderFile, mixLoudness, { label: "the film's mix", ...aborting });
722
+ if (measured.kind === "silent") {
723
+ log("loudness: the film's mix is silent, left as it is.");
724
+ } else {
725
+ audioFilter = getLoudnessFilter(measured.measurement, mixLoudness);
726
+ log(`loudness: the film's mix measured ${measured.measurement.inputI.toFixed(1)} LUFS, normalized to ${mixLoudness.target} LUFS (true peak ${mixLoudness.truePeak} dBTP).`);
727
+ }
728
+ }
729
+ const isVideoCopied = isPublishableVideo(rendered.facts.video);
730
+ const isAudioCopied = audioFilter === undefined && (rendered.facts.audio === null || isPublishableAudio(rendered.facts.audio));
731
+ const finishArgs = buildFinishArgs(input.renderFile, rendered.facts, output, audioFilter === undefined ? {} : { audioFilter });
732
+ await runFfmpeg("The film's final pass", finishArgs, { ...aborting, output, onSeconds: (done) => progress(0, done) });
641
733
  rmSync(input.renderFile, { force: true });
642
734
  const finished = await probeFilmFacts(output, aborting);
643
735
  if (!finished.ok) return err(`the finished film: ${finished.error}`);
@@ -645,7 +737,8 @@ async function finishFootage(
645
737
  if (problems.length > 0) return err(`the finished film ${output} is not publishable: ${problems.join("; ")}.`);
646
738
  progress(0, input.seconds);
647
739
  const { bytes, width, height, durationSeconds } = finished.facts;
648
- log(`finish: ${isCopied ? "streams copied" : "re-encoded"} to H.264 High, AAC 48 kHz stereo, fast start; ${width}×${height}, ${durationSeconds.toFixed(2)} s, ${bytes} bytes.`);
740
+ const streams = isVideoCopied && isAudioCopied ? "streams copied" : isVideoCopied ? "picture copied, sound re-encoded" : "re-encoded";
741
+ log(`finish: ${streams} to H.264 High, AAC 48 kHz stereo, fast start; ${width}×${height}, ${durationSeconds.toFixed(2)} s, ${bytes} bytes.`);
649
742
  let cover: string | null = null;
650
743
  if (film.cover !== null) {
651
744
  cover = getCoverPath(output);
@@ -666,6 +759,52 @@ export function readRenderPercent(line: string): number | null {
666
759
  return percent >= 0 && percent <= 100 ? percent : null;
667
760
  }
668
761
 
762
+ /** hyperframes' render arguments: the quality, the output and the memory settings the caller set. Throws on invalid `workers`. */
763
+ export function getHyperframesRenderArgs(options: Pick<RenderFootageOptions, "quality" | "workers" | "lowMemory">, output: string): string[] {
764
+ const args = ["render", "--quality", options.quality];
765
+ if (options.workers !== undefined) {
766
+ if (!Number.isInteger(options.workers) || options.workers < 1) throw new Error(`workers must be a positive whole number, got ${options.workers}.`);
767
+ args.push("--workers", String(options.workers));
768
+ }
769
+ if (options.lowMemory !== undefined) args.push(options.lowMemory ? "--low-memory-mode" : "--no-low-memory-mode");
770
+ return [...args, "-o", output];
771
+ }
772
+
773
+ interface BuildPictureInput {
774
+ film: FootageFilm;
775
+ edl: Edl;
776
+ clips: Record<string, string>;
777
+ output: string;
778
+ encoder?: NormalizeEncoder;
779
+ signal?: AbortSignal;
780
+ onSeconds: (done: number) => void;
781
+ }
782
+
783
+ /** Renders the picture piece by piece (one ffmpeg at a time) and joins the pieces; the pieces are removed after the join. */
784
+ async function buildPicture(input: BuildPictureInput): Promise<void> {
785
+ const pieces = planFootagePicture(input.film, input.edl);
786
+ const dir = join(dirname(input.output), "picture");
787
+ rmSync(dir, { recursive: true, force: true });
788
+ mkdirSync(dir, { recursive: true });
789
+ const aborting = input.signal === undefined ? {} : { signal: input.signal };
790
+ const files: string[] = [];
791
+ for (const [index, piece] of pieces.entries()) {
792
+ const file = join(dir, `${String(index).padStart(3, "0")}.mp4`);
793
+ const done = piece.frame / FOOTAGE_FPS;
794
+ await runFfmpeg(`The picture of segment ${index + 1}`, buildPicturePieceArgs({ piece, clips: input.clips, output: file, ...(input.encoder === undefined ? {} : { encoder: input.encoder }) }), {
795
+ ...aborting,
796
+ output: file,
797
+ onSeconds: (seconds) => input.onSeconds(done + seconds),
798
+ });
799
+ files.push(file);
800
+ }
801
+ const list = join(dir, "pieces.txt");
802
+ writeFileSync(list, buildPictureList(files));
803
+ await runFfmpeg("The picture's join", buildPictureConcatArgs(list, input.output), { ...aborting, output: input.output });
804
+ rmSync(dir, { recursive: true, force: true });
805
+ input.onSeconds(getPictureFrames(pieces) / FOOTAGE_FPS);
806
+ }
807
+
669
808
  /** The hyperframes render as a child process: its lines go to the log (progress every 10 %), its percentages to `onProgress`. */
670
809
  async function runHyperframesRender(options: RenderFootageOptions, output: string, seconds: number): Promise<FootageResult<null>> {
671
810
  const progress = createStageProgress(options.onProgress, "render", [{ item: "film", seconds }]);
@@ -694,7 +833,7 @@ async function runHyperframesRender(options: RenderFootageOptions, output: strin
694
833
  }
695
834
  const result = await runProcess({
696
835
  command: process.execPath,
697
- args: [getHyperframesBin(), "render", "--quality", options.quality, "-o", output],
836
+ args: [getHyperframesBin(), ...getHyperframesRenderArgs(options, output)],
698
837
  cwd: options.buildDir,
699
838
  env,
700
839
  ...(options.signal === undefined ? {} : { signal: options.signal }),
@@ -733,8 +872,13 @@ export async function renderFootageFilm(film: FootageFilm, options: RenderFootag
733
872
  edl = read.value;
734
873
  options.log(`cut: ${edlPath} (${edl.segments.length} segments), as it is; softure-marketing footage --step=cut builds it again.`);
735
874
  } else {
736
- edl = cutFootage(film, clips.value, transcript.value.words, edlPath, options.keepSentences === undefined ? {} : { keepSentences: options.keepSentences });
875
+ const cut = await cutFootageFilm(film, clips.value, transcript.value.words, edlPath, {
876
+ ...(options.keepSentences === undefined ? {} : { keepSentences: options.keepSentences }),
877
+ ...(options.signal === undefined ? {} : { signal: options.signal }),
878
+ });
879
+ edl = cut.edl;
737
880
  options.log(`cut: ${edl.segments.length} segments written to ${edlPath}.`);
881
+ for (const line of describeCutMeasures(cut.cuts)) options.log(` auto transition ${line}`);
738
882
  }
739
883
  return renderFootage(film, clips.value, transcript.value, edl, options);
740
884
  }