@softure-ai/marketing-kit 0.4.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/CHANGELOG.md +57 -0
  2. package/README.md +87 -6
  3. package/dist/config/config.d.ts.map +1 -1
  4. package/dist/config/config.js +7 -0
  5. package/dist/config/config.js.map +1 -1
  6. package/dist/config/schema.d.ts +1 -0
  7. package/dist/config/schema.d.ts.map +1 -1
  8. package/dist/config/schema.js +11 -0
  9. package/dist/config/schema.js.map +1 -1
  10. package/dist/footage/audio.d.ts +56 -0
  11. package/dist/footage/audio.d.ts.map +1 -0
  12. package/dist/footage/audio.js +112 -0
  13. package/dist/footage/audio.js.map +1 -0
  14. package/dist/footage/captions.d.ts +21 -2
  15. package/dist/footage/captions.d.ts.map +1 -1
  16. package/dist/footage/captions.js +29 -0
  17. package/dist/footage/captions.js.map +1 -1
  18. package/dist/footage/compose.d.ts.map +1 -1
  19. package/dist/footage/compose.js +4 -5
  20. package/dist/footage/compose.js.map +1 -1
  21. package/dist/footage/disk.d.ts +25 -0
  22. package/dist/footage/disk.d.ts.map +1 -0
  23. package/dist/footage/disk.js +72 -0
  24. package/dist/footage/disk.js.map +1 -0
  25. package/dist/footage/draft.d.ts +105 -0
  26. package/dist/footage/draft.d.ts.map +1 -0
  27. package/dist/footage/draft.js +282 -0
  28. package/dist/footage/draft.js.map +1 -0
  29. package/dist/footage/elevenlabs.d.ts.map +1 -1
  30. package/dist/footage/elevenlabs.js +2 -0
  31. package/dist/footage/elevenlabs.js.map +1 -1
  32. package/dist/footage/film.d.ts +12 -0
  33. package/dist/footage/film.d.ts.map +1 -1
  34. package/dist/footage/film.js.map +1 -1
  35. package/dist/footage/keyterms.d.ts +26 -0
  36. package/dist/footage/keyterms.d.ts.map +1 -0
  37. package/dist/footage/keyterms.js +36 -0
  38. package/dist/footage/keyterms.js.map +1 -0
  39. package/dist/footage/normalize.d.ts +21 -2
  40. package/dist/footage/normalize.d.ts.map +1 -1
  41. package/dist/footage/normalize.js +35 -11
  42. package/dist/footage/normalize.js.map +1 -1
  43. package/dist/footage/paid.d.ts +2 -1
  44. package/dist/footage/paid.d.ts.map +1 -1
  45. package/dist/footage/paid.js +13 -3
  46. package/dist/footage/paid.js.map +1 -1
  47. package/dist/footage/pipeline.d.ts +21 -4
  48. package/dist/footage/pipeline.d.ts.map +1 -1
  49. package/dist/footage/pipeline.js +58 -10
  50. package/dist/footage/pipeline.js.map +1 -1
  51. package/dist/footage/probe.d.ts +10 -0
  52. package/dist/footage/probe.d.ts.map +1 -1
  53. package/dist/footage/probe.js +10 -5
  54. package/dist/footage/probe.js.map +1 -1
  55. package/dist/footage/process.d.ts +2 -0
  56. package/dist/footage/process.d.ts.map +1 -1
  57. package/dist/footage/process.js +1 -0
  58. package/dist/footage/process.js.map +1 -1
  59. package/dist/footage/providers.d.ts +12 -3
  60. package/dist/footage/providers.d.ts.map +1 -1
  61. package/dist/footage/providers.js.map +1 -1
  62. package/dist/footage/render.d.ts +1 -1
  63. package/dist/footage/render.d.ts.map +1 -1
  64. package/dist/footage/render.js +5 -3
  65. package/dist/footage/render.js.map +1 -1
  66. package/dist/footage/tools.d.ts +15 -3
  67. package/dist/footage/tools.d.ts.map +1 -1
  68. package/dist/footage/tools.js +15 -2
  69. package/dist/footage/tools.js.map +1 -1
  70. package/dist/index.d.ts +8 -4
  71. package/dist/index.d.ts.map +1 -1
  72. package/dist/index.js +8 -4
  73. package/dist/index.js.map +1 -1
  74. package/docker/footage-worker.Dockerfile +6 -0
  75. package/fonts/DejaVuSans-Bold-Latin.ttf +0 -0
  76. package/fonts/LICENSE-DejaVu.txt +46 -0
  77. package/fonts/README.md +12 -0
  78. package/package.json +2 -1
  79. package/schema/marketing.schema.json +7 -0
  80. package/src/config/config.ts +8 -0
  81. package/src/config/schema.ts +11 -0
  82. package/src/footage/audio.ts +136 -0
  83. package/src/footage/captions.ts +39 -2
  84. package/src/footage/compose.ts +4 -5
  85. package/src/footage/disk.ts +75 -0
  86. package/src/footage/draft.ts +366 -0
  87. package/src/footage/elevenlabs.ts +1 -0
  88. package/src/footage/film.ts +16 -1
  89. package/src/footage/keyterms.ts +38 -0
  90. package/src/footage/normalize.ts +47 -10
  91. package/src/footage/paid.ts +12 -3
  92. package/src/footage/pipeline.ts +67 -13
  93. package/src/footage/probe.ts +10 -5
  94. package/src/footage/process.ts +3 -0
  95. package/src/footage/providers.ts +10 -3
  96. package/src/footage/render.ts +5 -3
  97. package/src/footage/tools.ts +23 -3
  98. package/src/index.ts +46 -2
@@ -0,0 +1,366 @@
1
+ import { copyFileSync, mkdirSync, rmSync, statSync, writeFileSync } from "node:fs";
2
+ import { basename, dirname, join, resolve } from "node:path";
3
+ import { fileURLToPath } from "node:url";
4
+
5
+ import type { CaptionLayout, LayoutOverride } from "../compose/timeline.js";
6
+ import { ok } from "../voice/provider.js";
7
+ import type { TimedWord } from "../voice/voiceover.js";
8
+ import { getCaptionSpans, getFootageCaptionLayout, phraseChunks } from "./captions.js";
9
+ import { getSegmentScales } from "./compose.js";
10
+ import { getEdlDuration, getSegmentStarts, remapWords, type Edl } from "./edl.js";
11
+ import { getFootageEdit, type FootageFilm } from "./film.js";
12
+ import { FOOTAGE_FPS, type Frame } from "./normalize.js";
13
+ import { getFrame, type FootageResult, type FootageRunOptions, type PreparedClip, type TranscribeOutcome } from "./pipeline.js";
14
+ import { runFfmpeg } from "./process.js";
15
+ import { buildSpeechArgs } from "./render.js";
16
+
17
+ /**
18
+ * A fast draft of a footage film, ffmpeg only (no hyperframes, no browser): the cut at a small width with the film's
19
+ * phrase captions burned in, its zoom cuts, cross-fades and Ken Burns b-roll drawn as the full render draws them, and
20
+ * the speech track. No loudness pass, music, end card or cover. What an app shows the owner before the paid render.
21
+ */
22
+
23
+ /** The draft's default width: 540 makes a 540×960 draft of a 9:16 film. */
24
+ export const FOOTAGE_DRAFT_WIDTH = 540;
25
+
26
+ /** The font the draft's captions are drawn with: DejaVu Sans Bold, subset to Latin with every Polish letter, shipped in `fonts/`. */
27
+ export const FOOTAGE_DRAFT_FONT = {
28
+ family: "DejaVu Sans",
29
+ file: fileURLToPath(new URL("../../fonts/DejaVuSans-Bold-Latin.ttf", import.meta.url)),
30
+ } as const;
31
+
32
+ /** How far a segment marked `kenburns` zooms when the film's `kenBurnsPercent` is 0, as in the render. */
33
+ const HAND_KEN_BURNS_PERCENT = 4;
34
+ /** The point a zoom keeps still, as the render's `transform-origin: 50% 42%`. */
35
+ const ZOOM_ORIGIN = { x: 0.5, y: 0.42 } as const;
36
+ const DRAFT_CRF = 28;
37
+ const DRAFT_AUDIO_BITRATE = "128k";
38
+
39
+ const r3 = (value: number): number => Math.round(value * 1000) / 1000 || 0;
40
+ const r4 = (value: number): number => Math.round(value * 10000) / 10000 || 0;
41
+
42
+ /** The draft's frame: the film's frame scaled to `width`, both sides even (H.264 yuv420p needs them even). */
43
+ export function getFootageDraftFrame(frame: Frame, width: number = FOOTAGE_DRAFT_WIDTH): Frame {
44
+ const even = (value: number): number => Math.max(2, Math.round(value / 2) * 2);
45
+ return { width: even(width), height: even((width * frame.height) / frame.width) };
46
+ }
47
+
48
+ /** An ASS time: `H:MM:SS.cc`. */
49
+ function toAssTime(seconds: number): string {
50
+ const centiseconds = Math.max(0, Math.round(seconds * 100));
51
+ const hours = Math.floor(centiseconds / 360000);
52
+ const minutes = Math.floor((centiseconds % 360000) / 6000);
53
+ const secs = Math.floor((centiseconds % 6000) / 100);
54
+ const pad = (value: number): string => String(value).padStart(2, "0");
55
+ return `${hours}:${pad(minutes)}:${pad(secs)}.${pad(centiseconds % 100)}`;
56
+ }
57
+
58
+ /** Caption text as ASS dialogue: override braces and backslashes would be read as tags, line breaks end the line. */
59
+ function toAssText(text: string): string {
60
+ return text.replace(/\\/g, "/").replace(/\{/g, "(").replace(/\}/g, ")").replace(/\s+/g, " ").trim();
61
+ }
62
+
63
+ export interface FootageDraftCaptionsInput {
64
+ film: Pick<FootageFilm, "captions">;
65
+ edl: Edl;
66
+ /** The film's words in film time (`remapWords`). */
67
+ words: TimedWord[];
68
+ /** The caption box in the film's frame (`getFootageCaptionLayout`). */
69
+ caption: CaptionLayout;
70
+ /** The film's full frame; the ASS file is laid out in it and libass scales it to the draft. */
71
+ frame: Frame;
72
+ /** The font family the captions are drawn in; default `FOOTAGE_DRAFT_FONT.family`. */
73
+ fontFamily?: string;
74
+ }
75
+
76
+ /**
77
+ * The draft's captions as an ASS subtitle file: the film's phrases (`phraseChunks`) shown for the render's spans
78
+ * (`getCaptionSpans`), white bold text with a dark outline and shadow, centred in the caption box from its top. Null
79
+ * when the film has no captions or nothing is said.
80
+ */
81
+ export function buildFootageDraftCaptions(input: FootageDraftCaptionsInput): string | null {
82
+ if (!input.film.captions) return null;
83
+ const chunks = phraseChunks(input.words);
84
+ if (chunks.length === 0) return null;
85
+ const spans = getCaptionSpans(chunks, getEdlDuration(input.edl));
86
+ const { caption, frame } = input;
87
+ const family = (input.fontFamily ?? FOOTAGE_DRAFT_FONT.family).replace(/,/g, " ");
88
+ const outline = Math.max(1, Math.round(caption.fontSize / 16));
89
+ const shadow = Math.max(1, Math.round(caption.fontSize / 25));
90
+ const style = [
91
+ "Caption",
92
+ family,
93
+ caption.fontSize,
94
+ "&H00FFFFFF",
95
+ "&H00FFFFFF",
96
+ "&H8C000000",
97
+ "&H8C000000",
98
+ -1,
99
+ 0,
100
+ 0,
101
+ 0,
102
+ 100,
103
+ 100,
104
+ 0,
105
+ 0,
106
+ 1,
107
+ outline,
108
+ shadow,
109
+ 8,
110
+ Math.round(caption.left),
111
+ Math.round(caption.right),
112
+ Math.round(caption.top),
113
+ 1,
114
+ ].join(",");
115
+ const events = chunks.map((chunk, index) => {
116
+ const span = spans[index] as { start: number; end: number };
117
+ const text = toAssText(chunk.words.map((word) => word.text).join(" "));
118
+ return `Dialogue: 0,${toAssTime(span.start)},${toAssTime(span.end)},Caption,,0,0,0,,${text}`;
119
+ });
120
+ return [
121
+ "[Script Info]",
122
+ "ScriptType: v4.00+",
123
+ `PlayResX: ${frame.width}`,
124
+ `PlayResY: ${frame.height}`,
125
+ "WrapStyle: 0",
126
+ "ScaledBorderAndShadow: yes",
127
+ "",
128
+ "[V4+ Styles]",
129
+ "Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding",
130
+ `Style: ${style}`,
131
+ "",
132
+ "[Events]",
133
+ "Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text",
134
+ ...events,
135
+ "",
136
+ ].join("\n");
137
+ }
138
+
139
+ export interface FootageDraftArgsInput {
140
+ film: Pick<FootageFilm, "edit">;
141
+ edl: Edl;
142
+ /** The normalized clips by id. */
143
+ clips: Record<string, string>;
144
+ /** The cut speech track (`buildSpeechArgs`). */
145
+ speech: string;
146
+ /** The film's full frame (the normalized clips' size). */
147
+ frame: Frame;
148
+ /** The draft's width; default 540. */
149
+ width?: number;
150
+ /** The ASS captions file (`buildFootageDraftCaptions`), or null for none. */
151
+ captions: string | null;
152
+ /** The folder libass loads the captions' font from, or null for the system's fonts. */
153
+ fontsDir: string | null;
154
+ output: string;
155
+ }
156
+
157
+ /** A segment's picture as the draft plays it: how long its stream runs and the fade it starts with. */
158
+ export interface DraftSegmentPlan {
159
+ /** The segment's own length in the cut. */
160
+ length: number;
161
+ /** Seconds the segment's stream runs: its length plus the fade into the next segment, which plays over its end. */
162
+ seconds: number;
163
+ /** The film second the segment starts at. */
164
+ start: number;
165
+ /** The cross-fade from the segment before, in seconds; 0 for a plain cut. */
166
+ fade: number;
167
+ /** The zoom the segment starts at (`getSegmentScales`). */
168
+ scale: number;
169
+ /** The zoom it ends at: above `scale` for a `kenburns` segment, `scale` otherwise. */
170
+ endScale: number;
171
+ }
172
+
173
+ /**
174
+ * How each segment plays in the draft. A `fade` cut lasts the film's `fadeFrames` (clamped to the incoming segment, so
175
+ * the cross-fade never outlasts it) and the outgoing segment runs on under it, as in the render.
176
+ */
177
+ export function planFootageDraft(film: Pick<FootageFilm, "edit">, edl: Edl): DraftSegmentPlan[] {
178
+ const edit = getFootageEdit(film);
179
+ const fadeSeconds = r3(edit.fadeFrames / FOOTAGE_FPS);
180
+ const kenBurnsScale = 1 + (edit.kenBurnsPercent > 0 ? edit.kenBurnsPercent : HAND_KEN_BURNS_PERCENT) / 100;
181
+ const scales = getSegmentScales(edl, edit.zoomPercent);
182
+ const starts = getSegmentStarts(edl);
183
+ const fades = edl.segments.map((segment, index) => (segment.transition === "fade" && index > 0 ? r3(Math.min(fadeSeconds, segment.end - segment.start)) : 0));
184
+ return edl.segments.map((segment, index) => {
185
+ const scale = scales[index] as number;
186
+ return {
187
+ length: r3(segment.end - segment.start),
188
+ seconds: r3(segment.end - segment.start + (fades[index + 1] ?? 0)),
189
+ start: starts[index] as number,
190
+ fade: fades[index] as number,
191
+ scale,
192
+ endScale: segment.motion === "kenburns" ? r4(scale * kenBurnsScale) : scale,
193
+ };
194
+ });
195
+ }
196
+
197
+ /** The filters that zoom a segment's picture and scale it to the draft: a crop for a still zoom, `zoompan` for Ken Burns. */
198
+ function buildPictureFilters(plan: DraftSegmentPlan, draft: Frame): string[] {
199
+ const size = `${draft.width}:${draft.height}`;
200
+ if (plan.endScale !== plan.scale) {
201
+ // The zoom runs over the segment's own length, as the render's tween does, and holds through a fade's tail.
202
+ const frames = Math.max(1, Math.round(plan.length * FOOTAGE_FPS));
203
+ const zoom = `${plan.scale}+${r4(plan.endScale - plan.scale)}*min(on/${frames},1)`;
204
+ return [
205
+ `zoompan=z='${zoom}':x='(iw-iw/zoom)*${ZOOM_ORIGIN.x}':y='(ih-ih/zoom)*${ZOOM_ORIGIN.y}':d=1:s=${draft.width}x${draft.height}:fps=${FOOTAGE_FPS}`,
206
+ ];
207
+ }
208
+ if (plan.scale !== 1) {
209
+ return [`crop=iw/${plan.scale}:ih/${plan.scale}:(iw-iw/${plan.scale})*${ZOOM_ORIGIN.x}:(ih-ih/${plan.scale})*${ZOOM_ORIGIN.y}`, `scale=${size}`];
210
+ }
211
+ return [`scale=${size}`];
212
+ }
213
+
214
+ /**
215
+ * The ffmpeg arguments of the draft: each segment read from its own seek into its clip, zoomed and scaled to the
216
+ * draft's width, joined by `xfade` at `fade` cuts and `concat` at the others, the captions burned in (`ass`), the
217
+ * speech track under it; H.264 yuv420p and AAC with fast start, exactly the cut's length.
218
+ */
219
+ export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
220
+ const { edl } = input;
221
+ const draft = getFootageDraftFrame(input.frame, input.width);
222
+ const plans = planFootageDraft(input.film, edl);
223
+ const inputs: string[] = [];
224
+ const parts: string[] = [];
225
+ edl.segments.forEach((segment, index) => {
226
+ const clip = input.clips[segment.clip];
227
+ if (clip === undefined) throw new Error(`No file for clip "${segment.clip}" of the EDL.`);
228
+ const plan = plans[index] as DraftSegmentPlan;
229
+ inputs.push("-ss", String(r3(segment.start)), "-t", String(plan.seconds), "-i", clip);
230
+ const filters = [
231
+ "setpts=PTS-STARTPTS",
232
+ ...buildPictureFilters(plan, draft),
233
+ "setsar=1",
234
+ `fps=${FOOTAGE_FPS}`,
235
+ "format=yuv420p",
236
+ // A clip that ends before the segment's end (and its fade's tail) holds its last frame, so every stream runs its full length.
237
+ `tpad=stop_mode=clone:stop_duration=${plan.seconds}`,
238
+ `trim=duration=${plan.seconds}`,
239
+ "setpts=PTS-STARTPTS",
240
+ `settb=1/${FOOTAGE_FPS}`,
241
+ ];
242
+ parts.push(`[${index}:v]${filters.join(",")}[v${index}]`);
243
+ });
244
+ let joined = "v0";
245
+ plans.forEach((plan, index) => {
246
+ if (index === 0) return;
247
+ const label = `j${index}`;
248
+ parts.push(
249
+ plan.fade > 0
250
+ ? `[${joined}][v${index}]xfade=transition=fade:duration=${plan.fade}:offset=${plan.start}[${label}]`
251
+ : // concat changes the time base; xfade needs both of its inputs on the segments' own.
252
+ `[${joined}][v${index}]concat=n=2:v=1:a=0,settb=1/${FOOTAGE_FPS}[${label}]`,
253
+ );
254
+ joined = label;
255
+ });
256
+ if (input.captions !== null) {
257
+ parts.push(`[${joined}]ass=filename=${input.captions}${input.fontsDir === null ? "" : `:fontsdir=${input.fontsDir}`}[draft]`);
258
+ } else {
259
+ parts.push(`[${joined}]null[draft]`);
260
+ }
261
+ const speechInput = edl.segments.length;
262
+ const seconds = getEdlDuration(edl);
263
+ return [
264
+ "-v",
265
+ "error",
266
+ "-y",
267
+ ...inputs,
268
+ "-i",
269
+ input.speech,
270
+ "-filter_complex",
271
+ parts.join(";"),
272
+ "-map",
273
+ "[draft]",
274
+ "-map",
275
+ `${speechInput}:a`,
276
+ "-t",
277
+ String(seconds),
278
+ "-c:v",
279
+ "libx264",
280
+ "-preset",
281
+ "veryfast",
282
+ "-crf",
283
+ String(DRAFT_CRF),
284
+ "-pix_fmt",
285
+ "yuv420p",
286
+ "-c:a",
287
+ "aac",
288
+ "-b:a",
289
+ DRAFT_AUDIO_BITRATE,
290
+ "-movflags",
291
+ "+faststart",
292
+ input.output,
293
+ ];
294
+ }
295
+
296
+ export interface RenderFootageDraftOptions extends Pick<FootageRunOptions, "buildDir" | "log" | "signal" | "onProgress"> {
297
+ /** The draft MP4. */
298
+ output: string;
299
+ /** The draft's width; default 540 (540×960 for a 9:16 film). */
300
+ width?: number;
301
+ /** A project's override of the format's caption box (`marketing.json` `layout`), as the render takes it. */
302
+ layout?: LayoutOverride;
303
+ }
304
+
305
+ export interface FootageDraft {
306
+ output: string;
307
+ /** The draft's length: the cut's. */
308
+ seconds: number;
309
+ width: number;
310
+ height: number;
311
+ bytes: number;
312
+ }
313
+
314
+ /**
315
+ * Makes the draft of a cut film: the speech track, the captions file and the font in `<buildDir>/draft/`, then one
316
+ * ffmpeg run (`buildFootageDraftArgs`). Spends nothing; takes the same clips, transcript and EDL as `renderFootage`.
317
+ */
318
+ export async function renderFootageDraft(
319
+ film: FootageFilm,
320
+ clips: PreparedClip[],
321
+ transcript: Extract<TranscribeOutcome, { kind: "ready" }>,
322
+ edl: Edl,
323
+ options: RenderFootageDraftOptions,
324
+ ): Promise<FootageResult<FootageDraft>> {
325
+ const { log, signal } = options;
326
+ const aborting = signal === undefined ? {} : { signal };
327
+ const seconds = getEdlDuration(edl);
328
+ const draftDir = resolve(options.buildDir, "draft");
329
+ rmSync(draftDir, { recursive: true, force: true });
330
+ mkdirSync(draftDir, { recursive: true });
331
+ const report = (item: string, itemIndex: number, done: number): void =>
332
+ options.onProgress?.({ step: "draft", item, itemIndex, itemCount: 2, done: Math.min(seconds * 2, (itemIndex - 1) * seconds + done), total: seconds * 2 });
333
+ const clipFiles = Object.fromEntries(clips.map((clip) => [clip.clip.id, resolve(clip.file)]));
334
+ const speech = join(draftDir, "speech.mp3");
335
+ const words = remapWords(edl, transcript.words, film.cut.fillers);
336
+ const frame = getFrame(film);
337
+ const captions = buildFootageDraftCaptions({ film, edl, words, caption: getFootageCaptionLayout(film, options.layout ?? {}), frame });
338
+ if (captions !== null) {
339
+ writeFileSync(join(draftDir, "captions.ass"), captions);
340
+ mkdirSync(join(draftDir, "fonts"));
341
+ copyFileSync(FOOTAGE_DRAFT_FONT.file, join(draftDir, "fonts", basename(FOOTAGE_DRAFT_FONT.file)));
342
+ }
343
+ const output = resolve(options.output);
344
+ mkdirSync(dirname(output), { recursive: true });
345
+ report("speech", 1, 0);
346
+ await runFfmpeg("The draft's speech track", buildSpeechArgs(edl, transcript.speech, speech), { ...aborting, output: speech, onSeconds: (done) => report("speech", 1, done) });
347
+ report("film", 2, 0);
348
+ // Run in the draft folder: the captions file and the font folder are relative names, so no path needs escaping in the filter graph.
349
+ const args = buildFootageDraftArgs({
350
+ film,
351
+ edl,
352
+ clips: clipFiles,
353
+ speech,
354
+ frame,
355
+ ...(options.width === undefined ? {} : { width: options.width }),
356
+ captions: captions === null ? null : "captions.ass",
357
+ fontsDir: captions === null ? null : "fonts",
358
+ output,
359
+ });
360
+ await runFfmpeg("The footage draft", args, { ...aborting, output, cwd: draftDir, onSeconds: (done) => report("film", 2, done) });
361
+ report("film", 2, seconds);
362
+ const draft = getFootageDraftFrame(frame, options.width);
363
+ const bytes = statSync(output).size;
364
+ log(`draft: ${output}, ${draft.width}×${draft.height}, ${seconds.toFixed(1)} s, ${bytes} bytes${captions === null ? ", no captions" : ""}.`);
365
+ return ok({ output, seconds, width: draft.width, height: draft.height, bytes });
366
+ }
@@ -103,6 +103,7 @@ export function createElevenLabsSttProvider(options: ElevenLabsFootageOptions):
103
103
  form.append("language_code", input.language);
104
104
  form.append("timestamps_granularity", "word");
105
105
  form.append("tag_audio_events", "false");
106
+ for (const term of input.keyTerms ?? []) form.append("keyterms", term);
106
107
  const response = await send(options, "speech to text", "/v1/speech-to-text", { method: "POST", body: form, signal: input.signal ?? null });
107
108
  if (!response.ok) return response;
108
109
  let body: unknown;
@@ -44,7 +44,17 @@ export interface FootageClip {
44
44
  /** Where the music under the film comes from. `prompt` and `video` are paid (ElevenLabs Music, Video to Music). */
45
45
  export type MusicSource =
46
46
  | { kind: "none" }
47
- | { kind: "file"; path: string }
47
+ | {
48
+ kind: "file";
49
+ /** A local audio file, or a video whose sound is the bed. */
50
+ path: string;
51
+ /**
52
+ * The file's original name and content type, for the type check when the path has no extension (a storage key
53
+ * such as `@softure-ai/files`' `getStorageKey`). Without them and without an extension, only the probe checks it.
54
+ */
55
+ fileName?: string;
56
+ contentType?: string;
57
+ }
48
58
  | { kind: "prompt"; prompt: string }
49
59
  | { kind: "video"; description: string; tags: string[] };
50
60
 
@@ -85,6 +95,11 @@ export interface FootageFilm {
85
95
  clips: FootageClip[];
86
96
  /** Clean the talking clips' voice (ElevenLabs Voice Isolator, paid) before transcribing them. */
87
97
  isolateVoice: boolean;
98
+ /**
99
+ * Names and brand words the transcript should spell as given (Scribe keyterm prompting, billed on top of the
100
+ * transcription). Checked against `SCRIBE_KEYTERM_LIMITS`; absent or empty: none.
101
+ */
102
+ keyTerms?: string[];
88
103
  cut: CutOptions;
89
104
  transitions: FootageTransitions;
90
105
  music: FootageMusic;
@@ -0,0 +1,38 @@
1
+ import type { FootageFilm } from "./film.js";
2
+
3
+ /**
4
+ * Key terms (Scribe keyterm prompting): names and brand words the transcript should spell as given. Scribe v2 batch
5
+ * limits, checked 2026-11-03 (elevenlabs.io keyterm prompting guide): up to 1000 terms of at most 50 characters and
6
+ * 5 words each. Keyterm prompting is billed on top of the transcription.
7
+ */
8
+ export const SCRIBE_KEYTERM_LIMITS = { maxTerms: 1000, maxCharacters: 50, maxWords: 5 } as const;
9
+
10
+ export type KeyTermsResult = { ok: true; terms: string[] } | { ok: false; errors: string[] };
11
+
12
+ const compareCodePoints = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0);
13
+
14
+ /**
15
+ * Key terms trimmed (inner whitespace collapsed to one space), without blanks and exact duplicates, sorted by code
16
+ * point (the same order on every host, so the transcript's cache key is stable), or every limit a term breaks.
17
+ */
18
+ export function normalizeKeyTerms(terms: readonly string[]): KeyTermsResult {
19
+ const unique = new Set<string>();
20
+ for (const term of terms) {
21
+ const cleaned = term.trim().replace(/\s+/g, " ");
22
+ if (cleaned.length > 0) unique.add(cleaned);
23
+ }
24
+ const { maxTerms, maxCharacters, maxWords } = SCRIBE_KEYTERM_LIMITS;
25
+ const errors: string[] = [];
26
+ if (unique.size > maxTerms) errors.push(`key terms: ${unique.size} terms, at most ${maxTerms}.`);
27
+ for (const term of unique) {
28
+ if (term.length > maxCharacters) errors.push(`key term "${term}": ${term.length} characters, at most ${maxCharacters}.`);
29
+ const words = term.split(" ").length;
30
+ if (words > maxWords) errors.push(`key term "${term}": ${words} words, at most ${maxWords}.`);
31
+ }
32
+ return errors.length > 0 ? { ok: false, errors } : { ok: true, terms: [...unique].sort(compareCodePoints) };
33
+ }
34
+
35
+ /** The film's key terms, normalized; none when the film has none. */
36
+ export function getFilmKeyTerms(film: Pick<FootageFilm, "keyTerms">): KeyTermsResult {
37
+ return normalizeKeyTerms(film.keyTerms ?? []);
38
+ }
@@ -47,21 +47,47 @@ export function toneMapFilter(hdr: HdrKind | null, transfer: string | null): str
47
47
  ];
48
48
  }
49
49
 
50
- /** The video filter graph from input 0's picture to `[v]`. */
50
+ /**
51
+ * The video filter graph from input 0's picture to `[v]`. Every stage runs on as little as it can: frames are
52
+ * dropped to 30 fps first, then the picture is scaled to the frame (crop: covered and cropped to it; blur: fitted
53
+ * inside it), and only then tone-mapped, so the float stages of an HDR clip work on frame-sized pictures instead of
54
+ * the source's (a 4K phone clip has 4x the pixels of a 1080x1920 frame). The blur's background is made from the
55
+ * fitted picture.
56
+ */
51
57
  export function buildVideoFilter(options: { fit: "crop" | "blur"; frame: Frame; hdr: HdrKind | null; transfer: string | null }): string {
52
58
  const { width, height } = options.frame;
53
- const head = [...toneMapFilter(options.hdr, options.transfer), `fps=${FOOTAGE_FPS}`].join(",");
54
- const cover = `scale=${width}:${height}:force_original_aspect_ratio=increase,crop=${width}:${height}`;
59
+ const toneMap = toneMapFilter(options.hdr, options.transfer);
55
60
  const tail = "setsar=1,format=yuv420p";
56
- if (options.fit === "crop") return `[0:v]${head},${cover},${tail}[v]`;
61
+ if (options.fit === "crop") {
62
+ const cover = `scale=${width}:${height}:force_original_aspect_ratio=increase,crop=${width}:${height}`;
63
+ return `[0:v]${[`fps=${FOOTAGE_FPS}`, cover, ...toneMap, tail].join(",")}[v]`;
64
+ }
65
+ // Even sizes: zscale refuses odd ones in 4:2:0.
66
+ const fitted = `scale=${width}:${height}:force_original_aspect_ratio=decrease:force_divisible_by=2`;
67
+ const backWidth = Math.round(width / 4);
68
+ const backHeight = Math.round(height / 4);
57
69
  return [
58
- `[0:v]${head},split=2[bg][fg]`,
59
- `[bg]scale=${Math.round(width / 4)}:${Math.round(height / 4)}:force_original_aspect_ratio=increase,crop=${Math.round(width / 4)}:${Math.round(height / 4)},boxblur=20:2,scale=${width}:${height},eq=brightness=-0.06[back]`,
60
- `[fg]scale=${width}:${height}:force_original_aspect_ratio=decrease[front]`,
61
- `[back][front]overlay=(W-w)/2:(H-h)/2,${tail}[v]`,
70
+ `[0:v]${[`fps=${FOOTAGE_FPS}`, fitted, ...toneMap, "split=2[bg][fg]"].join(",")}`,
71
+ `[bg]scale=${backWidth}:${backHeight}:force_original_aspect_ratio=increase,crop=${backWidth}:${backHeight},boxblur=20:2,scale=${width}:${height},eq=brightness=-0.06[back]`,
72
+ `[back][fg]overlay=(W-w)/2:(H-h)/2,${tail}[v]`,
62
73
  ].join(";");
63
74
  }
64
75
 
76
+ /** x264's presets, fastest first. */
77
+ export const X264_PRESETS = ["ultrafast", "superfast", "veryfast", "faster", "fast", "medium", "slow", "slower", "veryslow"] as const;
78
+ export type X264Preset = (typeof X264_PRESETS)[number];
79
+ const DEFAULT_NORMALIZE_PRESET: X264Preset = "veryfast";
80
+
81
+ /**
82
+ * How the intermediate clips are encoded. A small worker can trade file size for speed: `preset` (default
83
+ * `veryfast`; `superfast` or `ultrafast` encode faster into bigger files), `threads` (ffmpeg's encoder and filter
84
+ * threads; by default ffmpeg's own choice, which follows the host's cores, not a container's CPU quota).
85
+ */
86
+ export interface NormalizeEncoder {
87
+ preset?: X264Preset;
88
+ threads?: number;
89
+ }
90
+
65
91
  export interface NormalizeInput {
66
92
  clip: FootageClip;
67
93
  /** The local file (a URL's downloaded copy). */
@@ -69,19 +95,29 @@ export interface NormalizeInput {
69
95
  probe: FootageProbe;
70
96
  frame: Frame;
71
97
  output: string;
98
+ encoder?: NormalizeEncoder;
99
+ }
100
+
101
+ /** The thread arguments: global filter threads (before the inputs) and encoder threads (before the output). */
102
+ function getThreadArgs(threads: number | undefined): { global: string[]; output: string[] } {
103
+ if (threads === undefined) return { global: [], output: [] };
104
+ if (!Number.isInteger(threads) || threads < 1) throw new Error(`normalize.threads must be a positive whole number, got ${threads}.`);
105
+ return { global: ["-filter_complex_threads", String(threads)], output: ["-threads", String(threads)] };
72
106
  }
73
107
 
74
- /** ffmpeg's arguments for one clip; pure, so the graph is tested without ffmpeg. */
108
+ /** ffmpeg's arguments for one clip; pure, so the graph is tested without ffmpeg. Throws on an invalid `encoder.threads`. */
75
109
  export function buildNormalizeArgs(input: NormalizeInput): string[] {
76
110
  const { clip, file, probe, frame, output } = input;
77
111
  const duration = getTrimmedDuration(clip, probe);
78
112
  const fit = resolveFit(clip.fit, probe, frame);
79
113
  const seek = clip.trim.start > 0 ? ["-ss", String(clip.trim.start)] : [];
114
+ const threads = getThreadArgs(input.encoder?.threads);
80
115
  const silence = probe.hasAudio ? [] : ["-f", "lavfi", "-i", `anullsrc=r=${FOOTAGE_SAMPLE_RATE}:cl=stereo`];
81
116
  return [
82
117
  "-v",
83
118
  "error",
84
119
  "-y",
120
+ ...threads.global,
85
121
  ...seek,
86
122
  "-i",
87
123
  file,
@@ -97,7 +133,7 @@ export function buildNormalizeArgs(input: NormalizeInput): string[] {
97
133
  "-c:v",
98
134
  "libx264",
99
135
  "-preset",
100
- "veryfast",
136
+ input.encoder?.preset ?? DEFAULT_NORMALIZE_PRESET,
101
137
  "-crf",
102
138
  "18",
103
139
  "-g",
@@ -118,6 +154,7 @@ export function buildNormalizeArgs(input: NormalizeInput): string[] {
118
154
  "2",
119
155
  "-movflags",
120
156
  "+faststart",
157
+ ...threads.output,
121
158
  output,
122
159
  ];
123
160
  }
@@ -6,6 +6,7 @@ import { err, ok, type TtsResult } from "../voice/provider.js";
6
6
  import { footageCacheKey, getFootageCachePath, hashFile } from "./cache.js";
7
7
  import { END_CARD_SECONDS, getFootageFilmSeconds } from "./compose.js";
8
8
  import { readEdl, type Edl } from "./edl.js";
9
+ import { getFilmKeyTerms } from "./keyterms.js";
9
10
  import type { FootageClip, FootageFilm } from "./film.js";
10
11
  import { getTrimmedDuration, probeFootage } from "./probe.js";
11
12
  import type { FootageProviders, FootageRates, PaidStepKind } from "./providers.js";
@@ -66,7 +67,10 @@ export function getIsolatedCache(film: FootageFilm, clipKey: string, options: Pi
66
67
 
67
68
  export function getTranscriptCache(film: FootageFilm, clipKey: string, options: Pick<FootageCacheOptions, "cacheDir" | "providers">): CacheEntry {
68
69
  const { stt } = options.providers;
69
- const key = footageCacheKey("transcript", { clip: clipKey, isolated: film.isolateVoice, provider: stt.id, model: stt.model, language: film.language });
70
+ const keyTerms = getFilmKeyTerms(film);
71
+ // Key terms join the key only when there are any, so a transcript bought without them keeps its key.
72
+ const withKeyTerms = keyTerms.ok && keyTerms.terms.length > 0 ? { keyTerms: keyTerms.terms } : {};
73
+ const key = footageCacheKey("transcript", { clip: clipKey, isolated: film.isolateVoice, provider: stt.id, model: stt.model, language: film.language, ...withKeyTerms });
70
74
  return { key, path: getFootageCachePath(options.cacheDir, film.id, "transcript", key) };
71
75
  }
72
76
 
@@ -175,10 +179,15 @@ const round = (value: number, digits: number): number => {
175
179
  * What a footage film will cost in the caller's credits, before anything runs: every clip probed (a URL clip not yet
176
180
  * downloaded is probed at its URL), nothing normalized, no provider called. Talking clips need speech to text (and
177
181
  * isolation when the film cleans the voice); generated music lasts the cut's length when the cut is known, else the
178
- * clips' length plus the end card. A cached step costs 0.
182
+ * clips' length plus the end card. A cached step costs 0. With key terms, transcription costs `rates.transcribe` plus
183
+ * the keyterm surcharge `rates.keyTerms`; invalid key terms fail the estimate.
179
184
  */
180
185
  export async function estimateFootageFilm(film: FootageFilm, options: EstimateFootageOptions): Promise<TtsResult<FootageEstimate>> {
181
186
  if (film.clips.length === 0) return err(`footage film ${film.id} has no clips.`);
187
+ const keyTerms = getFilmKeyTerms(film);
188
+ if (!keyTerms.ok) return err(`footage film ${film.id}: ${keyTerms.errors.join(" ")}`);
189
+ const surcharge = keyTerms.terms.length === 0 ? 0 : (options.rates.keyTerms ?? 0);
190
+ const rates: Record<PaidStepKind, number> = { ...options.rates, transcribe: options.rates.transcribe + surcharge };
182
191
  const files: Record<string, string> = {};
183
192
  for (const clip of film.clips) {
184
193
  const local = findLocalSource(clip, options.buildDir);
@@ -189,7 +198,7 @@ export async function estimateFootageFilm(film: FootageFilm, options: EstimateFo
189
198
  if (!probed.ok) return err(probed.errors.join("\n"));
190
199
  const items: FootageEstimateItem[] = [];
191
200
  const add = (step: PaidStepKind, item: string, provider: string, units: number, cached: boolean): void => {
192
- items.push({ step, item, provider, units: round(units, 3), unit: "seconds", credits: cached ? 0 : round(units * options.rates[step], 2), cached });
201
+ items.push({ step, item, provider, units: round(units, 3), unit: "seconds", credits: cached ? 0 : round(units * rates[step], 2), cached });
193
202
  };
194
203
  let clipSeconds = 0;
195
204
  for (const { clip, probe } of probed.clips) {