@softure-ai/marketing-kit 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. package/CHANGELOG.md +98 -0
  2. package/README.md +130 -17
  3. package/dist/cli/footage.d.ts.map +1 -1
  4. package/dist/cli/footage.js +5 -2
  5. package/dist/cli/footage.js.map +1 -1
  6. package/dist/compose/compose.d.ts.map +1 -1
  7. package/dist/compose/compose.js +3 -3
  8. package/dist/compose/compose.js.map +1 -1
  9. package/dist/compose/timeline.d.ts +16 -0
  10. package/dist/compose/timeline.d.ts.map +1 -1
  11. package/dist/compose/timeline.js +17 -1
  12. package/dist/compose/timeline.js.map +1 -1
  13. package/dist/config/config.d.ts.map +1 -1
  14. package/dist/config/config.js +25 -2
  15. package/dist/config/config.js.map +1 -1
  16. package/dist/config/schema.d.ts +17 -0
  17. package/dist/config/schema.d.ts.map +1 -1
  18. package/dist/config/schema.js +61 -3
  19. package/dist/config/schema.js.map +1 -1
  20. package/dist/footage/audio.d.ts +6 -2
  21. package/dist/footage/audio.d.ts.map +1 -1
  22. package/dist/footage/audio.js +15 -3
  23. package/dist/footage/audio.js.map +1 -1
  24. package/dist/footage/captions.d.ts +26 -3
  25. package/dist/footage/captions.d.ts.map +1 -1
  26. package/dist/footage/captions.js +35 -2
  27. package/dist/footage/captions.js.map +1 -1
  28. package/dist/footage/compose.d.ts +13 -2
  29. package/dist/footage/compose.d.ts.map +1 -1
  30. package/dist/footage/compose.js +79 -28
  31. package/dist/footage/compose.js.map +1 -1
  32. package/dist/footage/cuts.d.ts +44 -0
  33. package/dist/footage/cuts.d.ts.map +1 -0
  34. package/dist/footage/cuts.js +83 -0
  35. package/dist/footage/cuts.js.map +1 -0
  36. package/dist/footage/disk.d.ts +25 -0
  37. package/dist/footage/disk.d.ts.map +1 -0
  38. package/dist/footage/disk.js +72 -0
  39. package/dist/footage/disk.js.map +1 -0
  40. package/dist/footage/draft.d.ts +119 -0
  41. package/dist/footage/draft.d.ts.map +1 -0
  42. package/dist/footage/draft.js +296 -0
  43. package/dist/footage/draft.js.map +1 -0
  44. package/dist/footage/edl.d.ts +2 -1
  45. package/dist/footage/edl.d.ts.map +1 -1
  46. package/dist/footage/edl.js +5 -2
  47. package/dist/footage/edl.js.map +1 -1
  48. package/dist/footage/elevenlabs.d.ts.map +1 -1
  49. package/dist/footage/elevenlabs.js +2 -0
  50. package/dist/footage/elevenlabs.js.map +1 -1
  51. package/dist/footage/film.d.ts +54 -3
  52. package/dist/footage/film.d.ts.map +1 -1
  53. package/dist/footage/film.js +7 -1
  54. package/dist/footage/film.js.map +1 -1
  55. package/dist/footage/keyterms.d.ts +26 -0
  56. package/dist/footage/keyterms.d.ts.map +1 -0
  57. package/dist/footage/keyterms.js +36 -0
  58. package/dist/footage/keyterms.js.map +1 -0
  59. package/dist/footage/loudness.d.ts +20 -3
  60. package/dist/footage/loudness.d.ts.map +1 -1
  61. package/dist/footage/loudness.js +36 -18
  62. package/dist/footage/loudness.js.map +1 -1
  63. package/dist/footage/music.d.ts +3 -0
  64. package/dist/footage/music.d.ts.map +1 -1
  65. package/dist/footage/music.js +5 -0
  66. package/dist/footage/music.js.map +1 -1
  67. package/dist/footage/normalize.d.ts +30 -2
  68. package/dist/footage/normalize.d.ts.map +1 -1
  69. package/dist/footage/normalize.js +39 -11
  70. package/dist/footage/normalize.js.map +1 -1
  71. package/dist/footage/paid.d.ts +2 -1
  72. package/dist/footage/paid.d.ts.map +1 -1
  73. package/dist/footage/paid.js +13 -3
  74. package/dist/footage/paid.js.map +1 -1
  75. package/dist/footage/picture.d.ts +42 -0
  76. package/dist/footage/picture.d.ts.map +1 -0
  77. package/dist/footage/picture.js +141 -0
  78. package/dist/footage/picture.js.map +1 -0
  79. package/dist/footage/pipeline.d.ts +66 -4
  80. package/dist/footage/pipeline.d.ts.map +1 -1
  81. package/dist/footage/pipeline.js +172 -34
  82. package/dist/footage/pipeline.js.map +1 -1
  83. package/dist/footage/process.d.ts +2 -0
  84. package/dist/footage/process.d.ts.map +1 -1
  85. package/dist/footage/process.js +1 -0
  86. package/dist/footage/process.js.map +1 -1
  87. package/dist/footage/providers.d.ts +12 -3
  88. package/dist/footage/providers.d.ts.map +1 -1
  89. package/dist/footage/providers.js.map +1 -1
  90. package/dist/footage/publish.d.ts +4 -2
  91. package/dist/footage/publish.d.ts.map +1 -1
  92. package/dist/footage/publish.js +8 -5
  93. package/dist/footage/publish.js.map +1 -1
  94. package/dist/footage/render.d.ts +18 -3
  95. package/dist/footage/render.d.ts.map +1 -1
  96. package/dist/footage/render.js +67 -16
  97. package/dist/footage/render.js.map +1 -1
  98. package/dist/footage/tools.d.ts +15 -3
  99. package/dist/footage/tools.d.ts.map +1 -1
  100. package/dist/footage/tools.js +15 -2
  101. package/dist/footage/tools.js.map +1 -1
  102. package/dist/index.d.ts +14 -9
  103. package/dist/index.d.ts.map +1 -1
  104. package/dist/index.js +14 -9
  105. package/dist/index.js.map +1 -1
  106. package/docker/footage-worker.Dockerfile +7 -0
  107. package/fonts/DejaVuSans-Bold-Latin.ttf +0 -0
  108. package/fonts/LICENSE-DejaVu.txt +46 -0
  109. package/fonts/README.md +12 -0
  110. package/package.json +2 -1
  111. package/schema/marketing.schema.json +112 -4
  112. package/src/cli/footage.ts +4 -2
  113. package/src/compose/compose.ts +13 -3
  114. package/src/compose/timeline.ts +27 -1
  115. package/src/config/config.ts +26 -2
  116. package/src/config/schema.ts +63 -3
  117. package/src/footage/audio.ts +16 -3
  118. package/src/footage/captions.ts +45 -4
  119. package/src/footage/compose.ts +90 -28
  120. package/src/footage/cuts.ts +104 -0
  121. package/src/footage/disk.ts +75 -0
  122. package/src/footage/draft.ts +395 -0
  123. package/src/footage/edl.ts +5 -2
  124. package/src/footage/elevenlabs.ts +1 -0
  125. package/src/footage/film.ts +63 -5
  126. package/src/footage/keyterms.ts +38 -0
  127. package/src/footage/loudness.ts +38 -16
  128. package/src/footage/music.ts +7 -0
  129. package/src/footage/normalize.ts +51 -10
  130. package/src/footage/paid.ts +12 -3
  131. package/src/footage/picture.ts +179 -0
  132. package/src/footage/pipeline.ts +219 -37
  133. package/src/footage/process.ts +3 -0
  134. package/src/footage/providers.ts +10 -3
  135. package/src/footage/publish.ts +8 -5
  136. package/src/footage/render.ts +77 -16
  137. package/src/footage/tools.ts +23 -3
  138. package/src/index.ts +70 -4
@@ -0,0 +1,395 @@
1
+ import { copyFileSync, mkdirSync, rmSync, statSync, writeFileSync } from "node:fs";
2
+ import { basename, dirname, join, resolve } from "node:path";
3
+ import { fileURLToPath } from "node:url";
4
+
5
+ import type { CaptionLayout, LayoutOverride } from "../compose/timeline.js";
6
+ import { ok } from "../voice/provider.js";
7
+ import type { TimedWord } from "../voice/voiceover.js";
8
+ import { getCaptionSpans, getFootageCaptionLayout, phraseChunks } from "./captions.js";
9
+ import { getSegmentScales, getSegmentZoomEases } from "./compose.js";
10
+ import { getEdlDuration, getSegmentStarts, remapWords, type Edl } from "./edl.js";
11
+ import { getFootageEdit, type FootageFilm } from "./film.js";
12
+ import { FOOTAGE_FPS, getThreadArgs, type Frame } from "./normalize.js";
13
+ import { getFrame, type FootageResult, type FootageRunOptions, type PreparedClip, type TranscribeOutcome } from "./pipeline.js";
14
+ import { runFfmpeg } from "./process.js";
15
+ import { buildSpeechArgs, getSpeechCutOptions } from "./render.js";
16
+
17
+ /**
18
+ * A fast draft of a footage film, ffmpeg only (no hyperframes, no browser): the cut at a small width with the film's
19
+ * phrase captions burned in, its zoom cuts (eased as in the render), cross-fades and Ken Burns b-roll drawn as the full render draws them, and
20
+ * the speech track. No loudness pass, music, end card or cover. What an app shows the owner before the paid render.
21
+ */
22
+
23
+ /** The draft's default width: 540 makes a 540×960 draft of a 9:16 film. */
24
+ export const FOOTAGE_DRAFT_WIDTH = 540;
25
+
26
+ /** The font the draft's captions are drawn with: DejaVu Sans Bold, subset to Latin with every Polish letter, shipped in `fonts/`. */
27
+ export const FOOTAGE_DRAFT_FONT = {
28
+ family: "DejaVu Sans",
29
+ file: fileURLToPath(new URL("../../fonts/DejaVuSans-Bold-Latin.ttf", import.meta.url)),
30
+ } as const;
31
+
32
+ /** How far a segment marked `kenburns` zooms when the film's `kenBurnsPercent` is 0, as in the render. */
33
+ const HAND_KEN_BURNS_PERCENT = 4;
34
+ /** The point a zoom keeps still, as the render's `transform-origin: 50% 42%`. */
35
+ const ZOOM_ORIGIN = { x: 0.5, y: 0.42 } as const;
36
+ const DRAFT_CRF = 28;
37
+ const DRAFT_AUDIO_BITRATE = "128k";
38
+
39
+ const r3 = (value: number): number => Math.round(value * 1000) / 1000 || 0;
40
+ const r4 = (value: number): number => Math.round(value * 10000) / 10000 || 0;
41
+
42
+ /** The draft's frame: the film's frame scaled to `width`, both sides even (H.264 yuv420p needs them even). */
43
+ export function getFootageDraftFrame(frame: Frame, width: number = FOOTAGE_DRAFT_WIDTH): Frame {
44
+ const even = (value: number): number => Math.max(2, Math.round(value / 2) * 2);
45
+ return { width: even(width), height: even((width * frame.height) / frame.width) };
46
+ }
47
+
48
+ /** An ASS time: `H:MM:SS.cc`. */
49
+ function toAssTime(seconds: number): string {
50
+ const centiseconds = Math.max(0, Math.round(seconds * 100));
51
+ const hours = Math.floor(centiseconds / 360000);
52
+ const minutes = Math.floor((centiseconds % 360000) / 6000);
53
+ const secs = Math.floor((centiseconds % 6000) / 100);
54
+ const pad = (value: number): string => String(value).padStart(2, "0");
55
+ return `${hours}:${pad(minutes)}:${pad(secs)}.${pad(centiseconds % 100)}`;
56
+ }
57
+
58
+ /** Caption text as ASS dialogue: override braces and backslashes would be read as tags, line breaks end the line. */
59
+ function toAssText(text: string): string {
60
+ return text.replace(/\\/g, "/").replace(/\{/g, "(").replace(/\}/g, ")").replace(/\s+/g, " ").trim();
61
+ }
62
+
63
+ export interface FootageDraftCaptionsInput {
64
+ film: Pick<FootageFilm, "captions">;
65
+ edl: Edl;
66
+ /** The film's words in film time (`remapWords`). */
67
+ words: TimedWord[];
68
+ /** The caption box in the film's frame (`getFootageCaptionLayout`). */
69
+ caption: CaptionLayout;
70
+ /** The film's full frame; the ASS file is laid out in it and libass scales it to the draft. */
71
+ frame: Frame;
72
+ /** The font family the captions are drawn in; default `FOOTAGE_DRAFT_FONT.family`. */
73
+ fontFamily?: string;
74
+ }
75
+
76
+ /**
77
+ * The draft's captions as an ASS subtitle file: the film's phrases (`phraseChunks`) shown for the render's spans
78
+ * (`getCaptionSpans`), white bold text with a dark outline and shadow, centred in the caption box from its top (or
79
+ * from its foot, with `bottom`). Null when the film has no captions or nothing is said.
80
+ */
81
+ export function buildFootageDraftCaptions(input: FootageDraftCaptionsInput): string | null {
82
+ if (!input.film.captions) return null;
83
+ const chunks = phraseChunks(input.words);
84
+ if (chunks.length === 0) return null;
85
+ const spans = getCaptionSpans(chunks, getEdlDuration(input.edl));
86
+ const { caption, frame } = input;
87
+ const family = (input.fontFamily ?? FOOTAGE_DRAFT_FONT.family).replace(/,/g, " ");
88
+ const outline = Math.max(1, Math.round(caption.fontSize / 16));
89
+ const shadow = Math.max(1, Math.round(caption.fontSize / 25));
90
+ const style = [
91
+ "Caption",
92
+ family,
93
+ caption.fontSize,
94
+ "&H00FFFFFF",
95
+ "&H00FFFFFF",
96
+ "&H8C000000",
97
+ "&H8C000000",
98
+ -1,
99
+ 0,
100
+ 0,
101
+ 0,
102
+ 100,
103
+ 100,
104
+ 0,
105
+ 0,
106
+ 1,
107
+ outline,
108
+ shadow,
109
+ // ASS alignment 2 hangs the text from MarginV above the bottom, 8 from MarginV below the top.
110
+ caption.bottom === undefined ? 8 : 2,
111
+ Math.round(caption.left),
112
+ Math.round(caption.right),
113
+ Math.round(caption.bottom ?? caption.top),
114
+ 1,
115
+ ].join(",");
116
+ const events = chunks.map((chunk, index) => {
117
+ const span = spans[index] as { start: number; end: number };
118
+ const text = toAssText(chunk.words.map((word) => word.text).join(" "));
119
+ return `Dialogue: 0,${toAssTime(span.start)},${toAssTime(span.end)},Caption,,0,0,0,,${text}`;
120
+ });
121
+ return [
122
+ "[Script Info]",
123
+ "ScriptType: v4.00+",
124
+ `PlayResX: ${frame.width}`,
125
+ `PlayResY: ${frame.height}`,
126
+ "WrapStyle: 0",
127
+ "ScaledBorderAndShadow: yes",
128
+ "",
129
+ "[V4+ Styles]",
130
+ "Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding",
131
+ `Style: ${style}`,
132
+ "",
133
+ "[Events]",
134
+ "Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text",
135
+ ...events,
136
+ "",
137
+ ].join("\n");
138
+ }
139
+
140
+ export interface FootageDraftArgsInput {
141
+ film: Pick<FootageFilm, "edit">;
142
+ edl: Edl;
143
+ /** The normalized clips by id. */
144
+ clips: Record<string, string>;
145
+ /** The cut speech track (`buildSpeechArgs`). */
146
+ speech: string;
147
+ /** The film's full frame (the normalized clips' size). */
148
+ frame: Frame;
149
+ /** The draft's width; default 540. */
150
+ width?: number;
151
+ /** The ASS captions file (`buildFootageDraftCaptions`), or null for none. */
152
+ captions: string | null;
153
+ /** The folder libass loads the captions' font from, or null for the system's fonts. */
154
+ fontsDir: string | null;
155
+ output: string;
156
+ /**
157
+ * ffmpeg's threads for every segment's decoder, the filter graph and the encoder; by default ffmpeg's own choice,
158
+ * which follows the host's cores, not a container's CPU quota. A positive whole number.
159
+ */
160
+ threads?: number;
161
+ }
162
+
163
+ /** A segment's picture as the draft plays it: how long its stream runs and the fade it starts with. */
164
+ export interface DraftSegmentPlan {
165
+ /** The segment's own length in the cut. */
166
+ length: number;
167
+ /** Seconds the segment's stream runs: its length plus the fade into the next segment, which plays over its end. */
168
+ seconds: number;
169
+ /** The film second the segment starts at. */
170
+ start: number;
171
+ /** The cross-fade from the segment before, in seconds; 0 for a plain cut. */
172
+ fade: number;
173
+ /** The zoom the segment starts at (`getSegmentScales`). */
174
+ scale: number;
175
+ /** The zoom it ends at: above `scale` for a `kenburns` segment, `scale` otherwise. */
176
+ endScale: number;
177
+ /** The zoom on screen before an eased `zoom` cut, which the segment eases from to `scale`; `scale` otherwise. */
178
+ startScale: number;
179
+ /** The frames the zoom eases over (`getSegmentZoomEases`); 0: the segment starts at `scale`. */
180
+ easeFrames: number;
181
+ }
182
+
183
+ /**
184
+ * How each segment plays in the draft. A `fade` cut lasts the film's `fadeFrames` (clamped to the incoming segment, so
185
+ * the cross-fade never outlasts it) and the outgoing segment runs on under it, as in the render.
186
+ */
187
+ export function planFootageDraft(film: Pick<FootageFilm, "edit">, edl: Edl): DraftSegmentPlan[] {
188
+ const edit = getFootageEdit(film);
189
+ const fadeSeconds = r3(edit.fadeFrames / FOOTAGE_FPS);
190
+ const kenBurnsScale = 1 + (edit.kenBurnsPercent > 0 ? edit.kenBurnsPercent : HAND_KEN_BURNS_PERCENT) / 100;
191
+ const scales = getSegmentScales(edl, edit.zoomPercent);
192
+ const eases = getSegmentZoomEases(edl, edit);
193
+ const starts = getSegmentStarts(edl);
194
+ const fades = edl.segments.map((segment, index) => (segment.transition === "fade" && index > 0 ? r3(Math.min(fadeSeconds, segment.end - segment.start)) : 0));
195
+ return edl.segments.map((segment, index) => {
196
+ const scale = scales[index] as number;
197
+ return {
198
+ length: r3(segment.end - segment.start),
199
+ seconds: r3(segment.end - segment.start + (fades[index + 1] ?? 0)),
200
+ start: starts[index] as number,
201
+ fade: fades[index] as number,
202
+ scale,
203
+ endScale: segment.motion === "kenburns" ? r4(scale * kenBurnsScale) : scale,
204
+ startScale: (eases[index] as number) > 0 ? (scales[index - 1] as number) : scale,
205
+ easeFrames: eases[index] as number,
206
+ };
207
+ });
208
+ }
209
+
210
+ /**
211
+ * The filters that zoom a segment's picture and scale it to the draft: a crop for a still zoom, `zoompan` for Ken Burns
212
+ * and for an eased zoom cut (the render's `power1.out`: `1 - (1 - t)^2`).
213
+ */
214
+ function buildPictureFilters(plan: DraftSegmentPlan, draft: Frame): string[] {
215
+ const size = `${draft.width}:${draft.height}`;
216
+ const zoompan = (zoom: string): string =>
217
+ `zoompan=z='${zoom}':x='(iw-iw/zoom)*${ZOOM_ORIGIN.x}':y='(ih-ih/zoom)*${ZOOM_ORIGIN.y}':d=1:s=${draft.width}x${draft.height}:fps=${FOOTAGE_FPS}`;
218
+ if (plan.easeFrames > 0) {
219
+ return [zoompan(`${plan.startScale}+(${r4(plan.scale - plan.startScale)})*(1-pow(1-min(on/${plan.easeFrames},1),2))`)];
220
+ }
221
+ if (plan.endScale !== plan.scale) {
222
+ // The zoom runs over the segment's own length, as the render's tween does, and holds through a fade's tail.
223
+ const frames = Math.max(1, Math.round(plan.length * FOOTAGE_FPS));
224
+ const zoom = `${plan.scale}+${r4(plan.endScale - plan.scale)}*min(on/${frames},1)`;
225
+ return [zoompan(zoom)];
226
+ }
227
+ if (plan.scale !== 1) {
228
+ return [`crop=iw/${plan.scale}:ih/${plan.scale}:(iw-iw/${plan.scale})*${ZOOM_ORIGIN.x}:(ih-ih/${plan.scale})*${ZOOM_ORIGIN.y}`, `scale=${size}`];
229
+ }
230
+ return [`scale=${size}`];
231
+ }
232
+
233
+ /**
234
+ * The ffmpeg arguments of the draft: each segment read from its own seek into its clip, zoomed and scaled to the
235
+ * draft's width, joined by `xfade` at `fade` cuts and `concat` at the others, the captions burned in (`ass`), the
236
+ * speech track under it; H.264 yuv420p and AAC with fast start, exactly the cut's length.
237
+ */
238
+ export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
239
+ const { edl } = input;
240
+ const draft = getFootageDraftFrame(input.frame, input.width);
241
+ const plans = planFootageDraft(input.film, edl);
242
+ const threads = getThreadArgs(input.threads, "draft.threads");
243
+ const inputs: string[] = [];
244
+ const parts: string[] = [];
245
+ edl.segments.forEach((segment, index) => {
246
+ const clip = input.clips[segment.clip];
247
+ if (clip === undefined) throw new Error(`No file for clip "${segment.clip}" of the EDL.`);
248
+ const plan = plans[index] as DraftSegmentPlan;
249
+ // Before an input, -threads sizes that input's decoder: one decoder runs per segment.
250
+ inputs.push(...threads.output, "-ss", String(r3(segment.start)), "-t", String(plan.seconds), "-i", clip);
251
+ const filters = [
252
+ "setpts=PTS-STARTPTS",
253
+ ...buildPictureFilters(plan, draft),
254
+ "setsar=1",
255
+ `fps=${FOOTAGE_FPS}`,
256
+ "format=yuv420p",
257
+ // A clip that ends before the segment's end (and its fade's tail) holds its last frame, so every stream runs its full length.
258
+ `tpad=stop_mode=clone:stop_duration=${plan.seconds}`,
259
+ `trim=duration=${plan.seconds}`,
260
+ "setpts=PTS-STARTPTS",
261
+ `settb=1/${FOOTAGE_FPS}`,
262
+ ];
263
+ parts.push(`[${index}:v]${filters.join(",")}[v${index}]`);
264
+ });
265
+ let joined = "v0";
266
+ plans.forEach((plan, index) => {
267
+ if (index === 0) return;
268
+ const label = `j${index}`;
269
+ parts.push(
270
+ plan.fade > 0
271
+ ? `[${joined}][v${index}]xfade=transition=fade:duration=${plan.fade}:offset=${plan.start}[${label}]`
272
+ : // concat changes the time base; xfade needs both of its inputs on the segments' own.
273
+ `[${joined}][v${index}]concat=n=2:v=1:a=0,settb=1/${FOOTAGE_FPS}[${label}]`,
274
+ );
275
+ joined = label;
276
+ });
277
+ if (input.captions !== null) {
278
+ parts.push(`[${joined}]ass=filename=${input.captions}${input.fontsDir === null ? "" : `:fontsdir=${input.fontsDir}`}[draft]`);
279
+ } else {
280
+ parts.push(`[${joined}]null[draft]`);
281
+ }
282
+ const speechInput = edl.segments.length;
283
+ const seconds = getEdlDuration(edl);
284
+ return [
285
+ "-v",
286
+ "error",
287
+ "-y",
288
+ ...threads.global,
289
+ ...inputs,
290
+ "-i",
291
+ input.speech,
292
+ "-filter_complex",
293
+ parts.join(";"),
294
+ "-map",
295
+ "[draft]",
296
+ "-map",
297
+ `${speechInput}:a`,
298
+ "-t",
299
+ String(seconds),
300
+ "-c:v",
301
+ "libx264",
302
+ "-preset",
303
+ "veryfast",
304
+ "-crf",
305
+ String(DRAFT_CRF),
306
+ "-pix_fmt",
307
+ "yuv420p",
308
+ "-c:a",
309
+ "aac",
310
+ "-b:a",
311
+ DRAFT_AUDIO_BITRATE,
312
+ "-movflags",
313
+ "+faststart",
314
+ ...threads.output,
315
+ input.output,
316
+ ];
317
+ }
318
+
319
+ export interface RenderFootageDraftOptions extends Pick<FootageRunOptions, "buildDir" | "log" | "signal" | "onProgress"> {
320
+ /** The draft MP4. */
321
+ output: string;
322
+ /** The draft's width; default 540 (540×960 for a 9:16 film). */
323
+ width?: number;
324
+ /** A project's override of the format's caption box (`marketing.json` `layout`), as the render takes it. */
325
+ layout?: LayoutOverride;
326
+ /**
327
+ * ffmpeg's threads for the draft's decoders, filter graph and encoder (`buildFootageDraftArgs`); by default ffmpeg's
328
+ * own choice, which follows the host's cores. Set it from a CPU-limited container's quota.
329
+ */
330
+ threads?: number;
331
+ }
332
+
333
+ export interface FootageDraft {
334
+ output: string;
335
+ /** The draft's length: the cut's. */
336
+ seconds: number;
337
+ width: number;
338
+ height: number;
339
+ bytes: number;
340
+ }
341
+
342
+ /**
343
+ * Makes the draft of a cut film: the speech track, the captions file and the font in `<buildDir>/draft/`, then one
344
+ * ffmpeg run (`buildFootageDraftArgs`). Spends nothing; takes the same clips, transcript and EDL as `renderFootage`.
345
+ */
346
+ export async function renderFootageDraft(
347
+ film: FootageFilm,
348
+ clips: PreparedClip[],
349
+ transcript: Extract<TranscribeOutcome, { kind: "ready" }>,
350
+ edl: Edl,
351
+ options: RenderFootageDraftOptions,
352
+ ): Promise<FootageResult<FootageDraft>> {
353
+ const { log, signal } = options;
354
+ const aborting = signal === undefined ? {} : { signal };
355
+ const seconds = getEdlDuration(edl);
356
+ const draftDir = resolve(options.buildDir, "draft");
357
+ rmSync(draftDir, { recursive: true, force: true });
358
+ mkdirSync(draftDir, { recursive: true });
359
+ const report = (item: string, itemIndex: number, done: number): void =>
360
+ options.onProgress?.({ step: "draft", item, itemIndex, itemCount: 2, done: Math.min(seconds * 2, (itemIndex - 1) * seconds + done), total: seconds * 2 });
361
+ const clipFiles = Object.fromEntries(clips.map((clip) => [clip.clip.id, resolve(clip.file)]));
362
+ const speech = join(draftDir, "speech.mp3");
363
+ const words = remapWords(edl, transcript.words, film.cut.fillers);
364
+ const frame = getFrame(film);
365
+ const captions = buildFootageDraftCaptions({ film, edl, words, caption: getFootageCaptionLayout(film, options.layout ?? {}), frame });
366
+ if (captions !== null) {
367
+ writeFileSync(join(draftDir, "captions.ass"), captions);
368
+ mkdirSync(join(draftDir, "fonts"));
369
+ copyFileSync(FOOTAGE_DRAFT_FONT.file, join(draftDir, "fonts", basename(FOOTAGE_DRAFT_FONT.file)));
370
+ }
371
+ const output = resolve(options.output);
372
+ mkdirSync(dirname(output), { recursive: true });
373
+ report("speech", 1, 0);
374
+ await runFfmpeg("The draft's speech track", buildSpeechArgs(edl, transcript.speech, speech, getSpeechCutOptions(getFootageEdit(film))), { ...aborting, output: speech, onSeconds: (done) => report("speech", 1, done) });
375
+ report("film", 2, 0);
376
+ // Run in the draft folder: the captions file and the font folder are relative names, so no path needs escaping in the filter graph.
377
+ const args = buildFootageDraftArgs({
378
+ film,
379
+ edl,
380
+ clips: clipFiles,
381
+ speech,
382
+ frame,
383
+ ...(options.width === undefined ? {} : { width: options.width }),
384
+ ...(options.threads === undefined ? {} : { threads: options.threads }),
385
+ captions: captions === null ? null : "captions.ass",
386
+ fontsDir: captions === null ? null : "fonts",
387
+ output,
388
+ });
389
+ await runFfmpeg("The footage draft", args, { ...aborting, output, cwd: draftDir, onSeconds: (done) => report("film", 2, done) });
390
+ report("film", 2, seconds);
391
+ const draft = getFootageDraftFrame(frame, options.width);
392
+ const bytes = statSync(output).size;
393
+ log(`draft: ${output}, ${draft.width}×${draft.height}, ${seconds.toFixed(1)} s, ${bytes} bytes${captions === null ? ", no captions" : ""}.`);
394
+ return ok({ output, seconds, width: draft.width, height: draft.height, bytes });
395
+ }
@@ -185,7 +185,8 @@ function claimId(base: string, used: Set<string>): string {
185
185
  * The cut from the transcripts. A talking clip keeps its runs of words: fillers are cut out, a pause longer than
186
186
  * `pauseSeconds` is cut, `padSeconds` stays around each cut, and with `keepSentences` only those sentences stay. A
187
187
  * talking clip without words, and every b-roll clip, is kept whole. The first segment starts with a cut; later ones
188
- * with `transitions.jump` inside one clip and `transitions.clip` between clips. Each segment's id names the sentence
188
+ * with `transitions.jump` inside one clip (a plain cut for `auto`, chosen later from the frames by
189
+ * `chooseCutTransitions`) and `transitions.clip` between clips. Each segment's id names the sentence
189
190
  * it starts with (`s4`), or the b-roll (`r1`) or wordless clip (`w1`) it shows.
190
191
  */
191
192
  export function buildEdl(filmId: string, clips: EdlClip[], options: CutOptions, transitions: FootageTransitions, edit: BuildEdlOptions = {}): Edl {
@@ -204,7 +205,9 @@ export function buildEdl(filmId: string, clips: EdlClip[], options: CutOptions,
204
205
  // A talking clip whose sentences were all dropped (or cut to nothing) leaves the film.
205
206
  for (const span of spans) {
206
207
  const previous = segments.at(-1);
207
- const transition = previous === undefined ? "cut" : previous.clip === clip.id ? transitions.jump : transitions.clip;
208
+ // An auto jump cut starts as a plain cut; `chooseCutTransitions` sets it from the frames.
209
+ const jump = transitions.jump === "auto" ? "cut" : transitions.jump;
210
+ const transition = previous === undefined ? "cut" : previous.clip === clip.id ? jump : transitions.clip;
208
211
  const base = span.sentence ?? (clip.role === "broll" ? `r${++broll}` : `w${++wordless}`);
209
212
  const motion = clip.role === "broll" && edit.isKenBurns === true ? "kenburns" : "none";
210
213
  segments.push({ id: claimId(base, used), clip: clip.id, start: r3(span.start), end: r3(span.end), transition, motion });
@@ -103,6 +103,7 @@ export function createElevenLabsSttProvider(options: ElevenLabsFootageOptions):
103
103
  form.append("language_code", input.language);
104
104
  form.append("timestamps_granularity", "word");
105
105
  form.append("tag_audio_events", "false");
106
+ for (const term of input.keyTerms ?? []) form.append("keyterms", term);
106
107
  const response = await send(options, "speech to text", "/v1/speech-to-text", { method: "POST", body: form, signal: input.signal ?? null });
107
108
  if (!response.ok) return response;
108
109
  let body: unknown;
@@ -44,7 +44,17 @@ export interface FootageClip {
44
44
  /** Where the music under the film comes from. `prompt` and `video` are paid (ElevenLabs Music, Video to Music). */
45
45
  export type MusicSource =
46
46
  | { kind: "none" }
47
- | { kind: "file"; path: string }
47
+ | {
48
+ kind: "file";
49
+ /** A local audio file, or a video whose sound is the bed. */
50
+ path: string;
51
+ /**
52
+ * The file's original name and content type, for the type check when the path has no extension (a storage key
53
+ * such as `@softure-ai/files`' `getStorageKey`). Without them and without an extension, only the probe checks it.
54
+ */
55
+ fileName?: string;
56
+ contentType?: string;
57
+ }
48
58
  | { kind: "prompt"; prompt: string }
49
59
  | { kind: "video"; description: string; tags: string[] };
50
60
 
@@ -54,6 +64,10 @@ export interface FootageMusic {
54
64
  volume: number;
55
65
  /** The music's volume under speech (0-1). */
56
66
  duckedVolume: number;
67
+ /** Seconds the music takes to go down before a stretch of speech. Absent: 0.25. */
68
+ attack?: number;
69
+ /** Seconds the music takes to come back up after a stretch of speech. Absent: 0.5. */
70
+ release?: number;
57
71
  }
58
72
 
59
73
  /** How the transcript turns into the cut (`buildEdl`). */
@@ -68,14 +82,27 @@ export interface CutOptions {
68
82
  maxSeconds: number | null;
69
83
  }
70
84
 
85
+ /** A jump cut's default: a transition, or `auto` to choose `zoom` or `fade` from the frames around each cut. */
86
+ export const JUMP_TRANSITIONS = [...CUT_TRANSITIONS, "auto"] as const;
87
+
88
+ export type JumpTransition = (typeof JUMP_TRANSITIONS)[number];
89
+
71
90
  /** Default transitions by kind of cut; a segment of the EDL may name its own. */
72
91
  export interface FootageTransitions {
73
- /** A cut inside one talking clip (a removed filler or pause). */
74
- jump: CutTransition;
92
+ /**
93
+ * A cut inside one talking clip (a removed filler or pause). `auto`: when the cut is built, the frames on both sides
94
+ * are compared (`chooseCutTransitions`): a small difference gets a `zoom`, a large one (the camera moved) a `fade`.
95
+ */
96
+ jump: JumpTransition;
75
97
  /** A cut from one clip to another. */
76
98
  clip: CutTransition;
99
+ /** With `jump: "auto"`, the frame difference (0-1) from which a jump cut fades instead of zooming; default 0.08. */
100
+ autoThreshold?: number;
77
101
  }
78
102
 
103
+ /** The frame difference (0-1) from which an `auto` jump cut fades instead of zooming. */
104
+ export const DEFAULT_AUTO_THRESHOLD = 0.08;
105
+
79
106
  export interface FootageFilm {
80
107
  id: string;
81
108
  title: string;
@@ -85,6 +112,11 @@ export interface FootageFilm {
85
112
  clips: FootageClip[];
86
113
  /** Clean the talking clips' voice (ElevenLabs Voice Isolator, paid) before transcribing them. */
87
114
  isolateVoice: boolean;
115
+ /**
116
+ * Names and brand words the transcript should spell as given (Scribe keyterm prompting, billed on top of the
117
+ * transcription). Checked against `SCRIBE_KEYTERM_LIMITS`; absent or empty: none.
118
+ */
119
+ keyTerms?: string[];
88
120
  cut: CutOptions;
89
121
  transitions: FootageTransitions;
90
122
  music: FootageMusic;
@@ -94,12 +126,23 @@ export interface FootageFilm {
94
126
  captionStyle?: CaptionStyle;
95
127
  /** Places the 9:16 caption box inside a platform's safe zone (`CAPTION_SAFE_ZONES`); absent: the layout as it is. */
96
128
  captionPreset?: CaptionPreset;
129
+ /**
130
+ * The most lines a caption card may take: the render shrinks a longer card's font (to at most
131
+ * `CAPTION_MIN_FONT_SCALE` of the box's size) until it fits. Absent: no limit. The draft keeps its font size.
132
+ */
133
+ captionMaxLines?: number;
97
134
  /** The cut's look: zoom size, fade length, Ken Burns, the end card's whoosh. Absent fields take `DEFAULT_EDIT`. */
98
135
  edit?: Partial<FootageEdit>;
99
136
  /** The closing card after the last segment; null: the film ends with its last segment. */
100
137
  endCard: EndCard | null;
101
138
  /** The speech track's loudness, normalized before the music is mixed under it. Absent: −14 LUFS, −1 dBTP; null: left as filmed. */
102
139
  loudness?: LoudnessTarget | null;
140
+ /**
141
+ * The finished film's mix (speech, music, sound effects), measured after the render and normalized in two passes of
142
+ * `loudnorm` (one linear gain; the picture is copied, only the sound re-encoded), so the published file plays at the
143
+ * target. A silent mix is left as it is. Absent: −14 LUFS, −1 dBTP; null: the mix as rendered.
144
+ */
145
+ mixLoudness?: LoudnessTarget | null;
103
146
  /**
104
147
  * The cover JPG written next to the film. Absent or without `at`: the middle of the first talking segment; `at`:
105
148
  * that second of the film; null: no cover.
@@ -127,22 +170,37 @@ export const CAPTION_STYLES = ["phrase", "highlight"] as const;
127
170
 
128
171
  export type CaptionStyle = (typeof CAPTION_STYLES)[number];
129
172
 
173
+ /** The smallest a caption line limit shrinks a card's font to, as a share of the caption box's size. */
174
+ export const CAPTION_MIN_FONT_SCALE = 0.7;
175
+
130
176
  export const CAPTION_PRESETS = ["reels", "shorts", "tiktok", "all"] as const;
131
177
 
132
178
  export type CaptionPreset = (typeof CAPTION_PRESETS)[number];
133
179
 
134
180
  export interface FootageEdit {
135
- /** How far a `zoom` cut punches in, in percent of the frame (105-110). */
181
+ /** How far a `zoom` cut punches in, in percent of the frame (105-112). */
136
182
  zoomPercent: number;
183
+ /**
184
+ * Frames at 30 fps a `zoom` cut eases into its scale over, from the scale on screen before the cut (0-15); 0 steps
185
+ * straight to it. A `kenburns` segment keeps its own slow zoom and is not eased.
186
+ */
187
+ zoomEaseFrames: number;
137
188
  /** A `fade` cut's length in frames at 30 fps (1-30). */
138
189
  fadeFrames: number;
139
190
  /** How far b-roll zooms over its length (0-5 %); 0 keeps b-roll still and `buildEdl` marks none `kenburns`. */
140
191
  kenBurnsPercent: number;
141
192
  /** Play the whoosh (and pop) as the end card comes in. */
142
193
  isEndCardWhoosh: boolean;
194
+ /** The speech track's fade at every cut, in milliseconds (10-100). */
195
+ cutAudioFadeMs: number;
196
+ /**
197
+ * Cross-fade the two sides of every cut over `cutAudioFadeMs`, centred on the cut, instead of fading each side out
198
+ * and in (a short dip). The speech stays where the picture is.
199
+ */
200
+ isCutAudioCrossfade: boolean;
143
201
  }
144
202
 
145
- export const DEFAULT_EDIT: FootageEdit = { zoomPercent: 108, fadeFrames: 10, kenBurnsPercent: 0, isEndCardWhoosh: false };
203
+ export const DEFAULT_EDIT: FootageEdit = { zoomPercent: 108, zoomEaseFrames: 0, fadeFrames: 10, kenBurnsPercent: 0, isEndCardWhoosh: false, cutAudioFadeMs: 20, isCutAudioCrossfade: false };
146
204
 
147
205
  /** The film's edit settings with the defaults filled in. */
148
206
  export function getFootageEdit(film: Pick<FootageFilm, "edit">): FootageEdit {
@@ -0,0 +1,38 @@
1
+ import type { FootageFilm } from "./film.js";
2
+
3
+ /**
4
+ * Key terms (Scribe keyterm prompting): names and brand words the transcript should spell as given. Scribe v2 batch
5
+ * limits, checked 2026-11-03 (elevenlabs.io keyterm prompting guide): up to 1000 terms of at most 50 characters and
6
+ * 5 words each. Keyterm prompting is billed on top of the transcription.
7
+ */
8
+ export const SCRIBE_KEYTERM_LIMITS = { maxTerms: 1000, maxCharacters: 50, maxWords: 5 } as const;
9
+
10
+ export type KeyTermsResult = { ok: true; terms: string[] } | { ok: false; errors: string[] };
11
+
12
+ const compareCodePoints = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0);
13
+
14
+ /**
15
+ * Key terms trimmed (inner whitespace collapsed to one space), without blanks and exact duplicates, sorted by code
16
+ * point (the same order on every host, so the transcript's cache key is stable), or every limit a term breaks.
17
+ */
18
+ export function normalizeKeyTerms(terms: readonly string[]): KeyTermsResult {
19
+ const unique = new Set<string>();
20
+ for (const term of terms) {
21
+ const cleaned = term.trim().replace(/\s+/g, " ");
22
+ if (cleaned.length > 0) unique.add(cleaned);
23
+ }
24
+ const { maxTerms, maxCharacters, maxWords } = SCRIBE_KEYTERM_LIMITS;
25
+ const errors: string[] = [];
26
+ if (unique.size > maxTerms) errors.push(`key terms: ${unique.size} terms, at most ${maxTerms}.`);
27
+ for (const term of unique) {
28
+ if (term.length > maxCharacters) errors.push(`key term "${term}": ${term.length} characters, at most ${maxCharacters}.`);
29
+ const words = term.split(" ").length;
30
+ if (words > maxWords) errors.push(`key term "${term}": ${words} words, at most ${maxWords}.`);
31
+ }
32
+ return errors.length > 0 ? { ok: false, errors } : { ok: true, terms: [...unique].sort(compareCodePoints) };
33
+ }
34
+
35
+ /** The film's key terms, normalized; none when the film has none. */
36
+ export function getFilmKeyTerms(film: Pick<FootageFilm, "keyTerms">): KeyTermsResult {
37
+ return normalizeKeyTerms(film.keyTerms ?? []);
38
+ }