@softure-ai/marketing-kit 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +57 -0
- package/README.md +87 -6
- package/dist/config/config.d.ts.map +1 -1
- package/dist/config/config.js +7 -0
- package/dist/config/config.js.map +1 -1
- package/dist/config/schema.d.ts +1 -0
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +11 -0
- package/dist/config/schema.js.map +1 -1
- package/dist/footage/audio.d.ts +56 -0
- package/dist/footage/audio.d.ts.map +1 -0
- package/dist/footage/audio.js +112 -0
- package/dist/footage/audio.js.map +1 -0
- package/dist/footage/captions.d.ts +21 -2
- package/dist/footage/captions.d.ts.map +1 -1
- package/dist/footage/captions.js +29 -0
- package/dist/footage/captions.js.map +1 -1
- package/dist/footage/compose.d.ts.map +1 -1
- package/dist/footage/compose.js +4 -5
- package/dist/footage/compose.js.map +1 -1
- package/dist/footage/disk.d.ts +25 -0
- package/dist/footage/disk.d.ts.map +1 -0
- package/dist/footage/disk.js +72 -0
- package/dist/footage/disk.js.map +1 -0
- package/dist/footage/draft.d.ts +105 -0
- package/dist/footage/draft.d.ts.map +1 -0
- package/dist/footage/draft.js +282 -0
- package/dist/footage/draft.js.map +1 -0
- package/dist/footage/elevenlabs.d.ts.map +1 -1
- package/dist/footage/elevenlabs.js +2 -0
- package/dist/footage/elevenlabs.js.map +1 -1
- package/dist/footage/film.d.ts +12 -0
- package/dist/footage/film.d.ts.map +1 -1
- package/dist/footage/film.js.map +1 -1
- package/dist/footage/keyterms.d.ts +26 -0
- package/dist/footage/keyterms.d.ts.map +1 -0
- package/dist/footage/keyterms.js +36 -0
- package/dist/footage/keyterms.js.map +1 -0
- package/dist/footage/normalize.d.ts +21 -2
- package/dist/footage/normalize.d.ts.map +1 -1
- package/dist/footage/normalize.js +35 -11
- package/dist/footage/normalize.js.map +1 -1
- package/dist/footage/paid.d.ts +2 -1
- package/dist/footage/paid.d.ts.map +1 -1
- package/dist/footage/paid.js +13 -3
- package/dist/footage/paid.js.map +1 -1
- package/dist/footage/pipeline.d.ts +21 -4
- package/dist/footage/pipeline.d.ts.map +1 -1
- package/dist/footage/pipeline.js +58 -10
- package/dist/footage/pipeline.js.map +1 -1
- package/dist/footage/probe.d.ts +10 -0
- package/dist/footage/probe.d.ts.map +1 -1
- package/dist/footage/probe.js +10 -5
- package/dist/footage/probe.js.map +1 -1
- package/dist/footage/process.d.ts +2 -0
- package/dist/footage/process.d.ts.map +1 -1
- package/dist/footage/process.js +1 -0
- package/dist/footage/process.js.map +1 -1
- package/dist/footage/providers.d.ts +12 -3
- package/dist/footage/providers.d.ts.map +1 -1
- package/dist/footage/providers.js.map +1 -1
- package/dist/footage/render.d.ts +1 -1
- package/dist/footage/render.d.ts.map +1 -1
- package/dist/footage/render.js +5 -3
- package/dist/footage/render.js.map +1 -1
- package/dist/footage/tools.d.ts +15 -3
- package/dist/footage/tools.d.ts.map +1 -1
- package/dist/footage/tools.js +15 -2
- package/dist/footage/tools.js.map +1 -1
- package/dist/index.d.ts +8 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -4
- package/dist/index.js.map +1 -1
- package/docker/footage-worker.Dockerfile +6 -0
- package/fonts/DejaVuSans-Bold-Latin.ttf +0 -0
- package/fonts/LICENSE-DejaVu.txt +46 -0
- package/fonts/README.md +12 -0
- package/package.json +2 -1
- package/schema/marketing.schema.json +7 -0
- package/src/config/config.ts +8 -0
- package/src/config/schema.ts +11 -0
- package/src/footage/audio.ts +136 -0
- package/src/footage/captions.ts +39 -2
- package/src/footage/compose.ts +4 -5
- package/src/footage/disk.ts +75 -0
- package/src/footage/draft.ts +366 -0
- package/src/footage/elevenlabs.ts +1 -0
- package/src/footage/film.ts +16 -1
- package/src/footage/keyterms.ts +38 -0
- package/src/footage/normalize.ts +47 -10
- package/src/footage/paid.ts +12 -3
- package/src/footage/pipeline.ts +67 -13
- package/src/footage/probe.ts +10 -5
- package/src/footage/process.ts +3 -0
- package/src/footage/providers.ts +10 -3
- package/src/footage/render.ts +5 -3
- package/src/footage/tools.ts +23 -3
- package/src/index.ts +46 -2
|
@@ -0,0 +1,366 @@
|
|
|
1
|
+
import { copyFileSync, mkdirSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { basename, dirname, join, resolve } from "node:path";
|
|
3
|
+
import { fileURLToPath } from "node:url";
|
|
4
|
+
|
|
5
|
+
import type { CaptionLayout, LayoutOverride } from "../compose/timeline.js";
|
|
6
|
+
import { ok } from "../voice/provider.js";
|
|
7
|
+
import type { TimedWord } from "../voice/voiceover.js";
|
|
8
|
+
import { getCaptionSpans, getFootageCaptionLayout, phraseChunks } from "./captions.js";
|
|
9
|
+
import { getSegmentScales } from "./compose.js";
|
|
10
|
+
import { getEdlDuration, getSegmentStarts, remapWords, type Edl } from "./edl.js";
|
|
11
|
+
import { getFootageEdit, type FootageFilm } from "./film.js";
|
|
12
|
+
import { FOOTAGE_FPS, type Frame } from "./normalize.js";
|
|
13
|
+
import { getFrame, type FootageResult, type FootageRunOptions, type PreparedClip, type TranscribeOutcome } from "./pipeline.js";
|
|
14
|
+
import { runFfmpeg } from "./process.js";
|
|
15
|
+
import { buildSpeechArgs } from "./render.js";
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* A fast draft of a footage film, ffmpeg only (no hyperframes, no browser): the cut at a small width with the film's
|
|
19
|
+
* phrase captions burned in, its zoom cuts, cross-fades and Ken Burns b-roll drawn as the full render draws them, and
|
|
20
|
+
* the speech track. No loudness pass, music, end card or cover. What an app shows the owner before the paid render.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
/** The draft's default width: 540 makes a 540×960 draft of a 9:16 film. */
|
|
24
|
+
export const FOOTAGE_DRAFT_WIDTH = 540;
|
|
25
|
+
|
|
26
|
+
/** The font the draft's captions are drawn with: DejaVu Sans Bold, subset to Latin with every Polish letter, shipped in `fonts/`. */
|
|
27
|
+
export const FOOTAGE_DRAFT_FONT = {
|
|
28
|
+
family: "DejaVu Sans",
|
|
29
|
+
file: fileURLToPath(new URL("../../fonts/DejaVuSans-Bold-Latin.ttf", import.meta.url)),
|
|
30
|
+
} as const;
|
|
31
|
+
|
|
32
|
+
/** How far a segment marked `kenburns` zooms when the film's `kenBurnsPercent` is 0, as in the render. */
|
|
33
|
+
const HAND_KEN_BURNS_PERCENT = 4;
|
|
34
|
+
/** The point a zoom keeps still, as the render's `transform-origin: 50% 42%`. */
|
|
35
|
+
const ZOOM_ORIGIN = { x: 0.5, y: 0.42 } as const;
|
|
36
|
+
const DRAFT_CRF = 28;
|
|
37
|
+
const DRAFT_AUDIO_BITRATE = "128k";
|
|
38
|
+
|
|
39
|
+
const r3 = (value: number): number => Math.round(value * 1000) / 1000 || 0;
|
|
40
|
+
const r4 = (value: number): number => Math.round(value * 10000) / 10000 || 0;
|
|
41
|
+
|
|
42
|
+
/** The draft's frame: the film's frame scaled to `width`, both sides even (H.264 yuv420p needs them even). */
|
|
43
|
+
export function getFootageDraftFrame(frame: Frame, width: number = FOOTAGE_DRAFT_WIDTH): Frame {
|
|
44
|
+
const even = (value: number): number => Math.max(2, Math.round(value / 2) * 2);
|
|
45
|
+
return { width: even(width), height: even((width * frame.height) / frame.width) };
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** An ASS time: `H:MM:SS.cc`. */
|
|
49
|
+
function toAssTime(seconds: number): string {
|
|
50
|
+
const centiseconds = Math.max(0, Math.round(seconds * 100));
|
|
51
|
+
const hours = Math.floor(centiseconds / 360000);
|
|
52
|
+
const minutes = Math.floor((centiseconds % 360000) / 6000);
|
|
53
|
+
const secs = Math.floor((centiseconds % 6000) / 100);
|
|
54
|
+
const pad = (value: number): string => String(value).padStart(2, "0");
|
|
55
|
+
return `${hours}:${pad(minutes)}:${pad(secs)}.${pad(centiseconds % 100)}`;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Caption text as ASS dialogue: override braces and backslashes would be read as tags, line breaks end the line. */
|
|
59
|
+
function toAssText(text: string): string {
|
|
60
|
+
return text.replace(/\\/g, "/").replace(/\{/g, "(").replace(/\}/g, ")").replace(/\s+/g, " ").trim();
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export interface FootageDraftCaptionsInput {
|
|
64
|
+
film: Pick<FootageFilm, "captions">;
|
|
65
|
+
edl: Edl;
|
|
66
|
+
/** The film's words in film time (`remapWords`). */
|
|
67
|
+
words: TimedWord[];
|
|
68
|
+
/** The caption box in the film's frame (`getFootageCaptionLayout`). */
|
|
69
|
+
caption: CaptionLayout;
|
|
70
|
+
/** The film's full frame; the ASS file is laid out in it and libass scales it to the draft. */
|
|
71
|
+
frame: Frame;
|
|
72
|
+
/** The font family the captions are drawn in; default `FOOTAGE_DRAFT_FONT.family`. */
|
|
73
|
+
fontFamily?: string;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* The draft's captions as an ASS subtitle file: the film's phrases (`phraseChunks`) shown for the render's spans
|
|
78
|
+
* (`getCaptionSpans`), white bold text with a dark outline and shadow, centred in the caption box from its top. Null
|
|
79
|
+
* when the film has no captions or nothing is said.
|
|
80
|
+
*/
|
|
81
|
+
export function buildFootageDraftCaptions(input: FootageDraftCaptionsInput): string | null {
|
|
82
|
+
if (!input.film.captions) return null;
|
|
83
|
+
const chunks = phraseChunks(input.words);
|
|
84
|
+
if (chunks.length === 0) return null;
|
|
85
|
+
const spans = getCaptionSpans(chunks, getEdlDuration(input.edl));
|
|
86
|
+
const { caption, frame } = input;
|
|
87
|
+
const family = (input.fontFamily ?? FOOTAGE_DRAFT_FONT.family).replace(/,/g, " ");
|
|
88
|
+
const outline = Math.max(1, Math.round(caption.fontSize / 16));
|
|
89
|
+
const shadow = Math.max(1, Math.round(caption.fontSize / 25));
|
|
90
|
+
const style = [
|
|
91
|
+
"Caption",
|
|
92
|
+
family,
|
|
93
|
+
caption.fontSize,
|
|
94
|
+
"&H00FFFFFF",
|
|
95
|
+
"&H00FFFFFF",
|
|
96
|
+
"&H8C000000",
|
|
97
|
+
"&H8C000000",
|
|
98
|
+
-1,
|
|
99
|
+
0,
|
|
100
|
+
0,
|
|
101
|
+
0,
|
|
102
|
+
100,
|
|
103
|
+
100,
|
|
104
|
+
0,
|
|
105
|
+
0,
|
|
106
|
+
1,
|
|
107
|
+
outline,
|
|
108
|
+
shadow,
|
|
109
|
+
8,
|
|
110
|
+
Math.round(caption.left),
|
|
111
|
+
Math.round(caption.right),
|
|
112
|
+
Math.round(caption.top),
|
|
113
|
+
1,
|
|
114
|
+
].join(",");
|
|
115
|
+
const events = chunks.map((chunk, index) => {
|
|
116
|
+
const span = spans[index] as { start: number; end: number };
|
|
117
|
+
const text = toAssText(chunk.words.map((word) => word.text).join(" "));
|
|
118
|
+
return `Dialogue: 0,${toAssTime(span.start)},${toAssTime(span.end)},Caption,,0,0,0,,${text}`;
|
|
119
|
+
});
|
|
120
|
+
return [
|
|
121
|
+
"[Script Info]",
|
|
122
|
+
"ScriptType: v4.00+",
|
|
123
|
+
`PlayResX: ${frame.width}`,
|
|
124
|
+
`PlayResY: ${frame.height}`,
|
|
125
|
+
"WrapStyle: 0",
|
|
126
|
+
"ScaledBorderAndShadow: yes",
|
|
127
|
+
"",
|
|
128
|
+
"[V4+ Styles]",
|
|
129
|
+
"Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding",
|
|
130
|
+
`Style: ${style}`,
|
|
131
|
+
"",
|
|
132
|
+
"[Events]",
|
|
133
|
+
"Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text",
|
|
134
|
+
...events,
|
|
135
|
+
"",
|
|
136
|
+
].join("\n");
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
export interface FootageDraftArgsInput {
|
|
140
|
+
film: Pick<FootageFilm, "edit">;
|
|
141
|
+
edl: Edl;
|
|
142
|
+
/** The normalized clips by id. */
|
|
143
|
+
clips: Record<string, string>;
|
|
144
|
+
/** The cut speech track (`buildSpeechArgs`). */
|
|
145
|
+
speech: string;
|
|
146
|
+
/** The film's full frame (the normalized clips' size). */
|
|
147
|
+
frame: Frame;
|
|
148
|
+
/** The draft's width; default 540. */
|
|
149
|
+
width?: number;
|
|
150
|
+
/** The ASS captions file (`buildFootageDraftCaptions`), or null for none. */
|
|
151
|
+
captions: string | null;
|
|
152
|
+
/** The folder libass loads the captions' font from, or null for the system's fonts. */
|
|
153
|
+
fontsDir: string | null;
|
|
154
|
+
output: string;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** A segment's picture as the draft plays it: how long its stream runs and the fade it starts with. */
|
|
158
|
+
export interface DraftSegmentPlan {
|
|
159
|
+
/** The segment's own length in the cut. */
|
|
160
|
+
length: number;
|
|
161
|
+
/** Seconds the segment's stream runs: its length plus the fade into the next segment, which plays over its end. */
|
|
162
|
+
seconds: number;
|
|
163
|
+
/** The film second the segment starts at. */
|
|
164
|
+
start: number;
|
|
165
|
+
/** The cross-fade from the segment before, in seconds; 0 for a plain cut. */
|
|
166
|
+
fade: number;
|
|
167
|
+
/** The zoom the segment starts at (`getSegmentScales`). */
|
|
168
|
+
scale: number;
|
|
169
|
+
/** The zoom it ends at: above `scale` for a `kenburns` segment, `scale` otherwise. */
|
|
170
|
+
endScale: number;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* How each segment plays in the draft. A `fade` cut lasts the film's `fadeFrames` (clamped to the incoming segment, so
|
|
175
|
+
* the cross-fade never outlasts it) and the outgoing segment runs on under it, as in the render.
|
|
176
|
+
*/
|
|
177
|
+
export function planFootageDraft(film: Pick<FootageFilm, "edit">, edl: Edl): DraftSegmentPlan[] {
|
|
178
|
+
const edit = getFootageEdit(film);
|
|
179
|
+
const fadeSeconds = r3(edit.fadeFrames / FOOTAGE_FPS);
|
|
180
|
+
const kenBurnsScale = 1 + (edit.kenBurnsPercent > 0 ? edit.kenBurnsPercent : HAND_KEN_BURNS_PERCENT) / 100;
|
|
181
|
+
const scales = getSegmentScales(edl, edit.zoomPercent);
|
|
182
|
+
const starts = getSegmentStarts(edl);
|
|
183
|
+
const fades = edl.segments.map((segment, index) => (segment.transition === "fade" && index > 0 ? r3(Math.min(fadeSeconds, segment.end - segment.start)) : 0));
|
|
184
|
+
return edl.segments.map((segment, index) => {
|
|
185
|
+
const scale = scales[index] as number;
|
|
186
|
+
return {
|
|
187
|
+
length: r3(segment.end - segment.start),
|
|
188
|
+
seconds: r3(segment.end - segment.start + (fades[index + 1] ?? 0)),
|
|
189
|
+
start: starts[index] as number,
|
|
190
|
+
fade: fades[index] as number,
|
|
191
|
+
scale,
|
|
192
|
+
endScale: segment.motion === "kenburns" ? r4(scale * kenBurnsScale) : scale,
|
|
193
|
+
};
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** The filters that zoom a segment's picture and scale it to the draft: a crop for a still zoom, `zoompan` for Ken Burns. */
|
|
198
|
+
function buildPictureFilters(plan: DraftSegmentPlan, draft: Frame): string[] {
|
|
199
|
+
const size = `${draft.width}:${draft.height}`;
|
|
200
|
+
if (plan.endScale !== plan.scale) {
|
|
201
|
+
// The zoom runs over the segment's own length, as the render's tween does, and holds through a fade's tail.
|
|
202
|
+
const frames = Math.max(1, Math.round(plan.length * FOOTAGE_FPS));
|
|
203
|
+
const zoom = `${plan.scale}+${r4(plan.endScale - plan.scale)}*min(on/${frames},1)`;
|
|
204
|
+
return [
|
|
205
|
+
`zoompan=z='${zoom}':x='(iw-iw/zoom)*${ZOOM_ORIGIN.x}':y='(ih-ih/zoom)*${ZOOM_ORIGIN.y}':d=1:s=${draft.width}x${draft.height}:fps=${FOOTAGE_FPS}`,
|
|
206
|
+
];
|
|
207
|
+
}
|
|
208
|
+
if (plan.scale !== 1) {
|
|
209
|
+
return [`crop=iw/${plan.scale}:ih/${plan.scale}:(iw-iw/${plan.scale})*${ZOOM_ORIGIN.x}:(ih-ih/${plan.scale})*${ZOOM_ORIGIN.y}`, `scale=${size}`];
|
|
210
|
+
}
|
|
211
|
+
return [`scale=${size}`];
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* The ffmpeg arguments of the draft: each segment read from its own seek into its clip, zoomed and scaled to the
|
|
216
|
+
* draft's width, joined by `xfade` at `fade` cuts and `concat` at the others, the captions burned in (`ass`), the
|
|
217
|
+
* speech track under it; H.264 yuv420p and AAC with fast start, exactly the cut's length.
|
|
218
|
+
*/
|
|
219
|
+
export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
|
|
220
|
+
const { edl } = input;
|
|
221
|
+
const draft = getFootageDraftFrame(input.frame, input.width);
|
|
222
|
+
const plans = planFootageDraft(input.film, edl);
|
|
223
|
+
const inputs: string[] = [];
|
|
224
|
+
const parts: string[] = [];
|
|
225
|
+
edl.segments.forEach((segment, index) => {
|
|
226
|
+
const clip = input.clips[segment.clip];
|
|
227
|
+
if (clip === undefined) throw new Error(`No file for clip "${segment.clip}" of the EDL.`);
|
|
228
|
+
const plan = plans[index] as DraftSegmentPlan;
|
|
229
|
+
inputs.push("-ss", String(r3(segment.start)), "-t", String(plan.seconds), "-i", clip);
|
|
230
|
+
const filters = [
|
|
231
|
+
"setpts=PTS-STARTPTS",
|
|
232
|
+
...buildPictureFilters(plan, draft),
|
|
233
|
+
"setsar=1",
|
|
234
|
+
`fps=${FOOTAGE_FPS}`,
|
|
235
|
+
"format=yuv420p",
|
|
236
|
+
// A clip that ends before the segment's end (and its fade's tail) holds its last frame, so every stream runs its full length.
|
|
237
|
+
`tpad=stop_mode=clone:stop_duration=${plan.seconds}`,
|
|
238
|
+
`trim=duration=${plan.seconds}`,
|
|
239
|
+
"setpts=PTS-STARTPTS",
|
|
240
|
+
`settb=1/${FOOTAGE_FPS}`,
|
|
241
|
+
];
|
|
242
|
+
parts.push(`[${index}:v]${filters.join(",")}[v${index}]`);
|
|
243
|
+
});
|
|
244
|
+
let joined = "v0";
|
|
245
|
+
plans.forEach((plan, index) => {
|
|
246
|
+
if (index === 0) return;
|
|
247
|
+
const label = `j${index}`;
|
|
248
|
+
parts.push(
|
|
249
|
+
plan.fade > 0
|
|
250
|
+
? `[${joined}][v${index}]xfade=transition=fade:duration=${plan.fade}:offset=${plan.start}[${label}]`
|
|
251
|
+
: // concat changes the time base; xfade needs both of its inputs on the segments' own.
|
|
252
|
+
`[${joined}][v${index}]concat=n=2:v=1:a=0,settb=1/${FOOTAGE_FPS}[${label}]`,
|
|
253
|
+
);
|
|
254
|
+
joined = label;
|
|
255
|
+
});
|
|
256
|
+
if (input.captions !== null) {
|
|
257
|
+
parts.push(`[${joined}]ass=filename=${input.captions}${input.fontsDir === null ? "" : `:fontsdir=${input.fontsDir}`}[draft]`);
|
|
258
|
+
} else {
|
|
259
|
+
parts.push(`[${joined}]null[draft]`);
|
|
260
|
+
}
|
|
261
|
+
const speechInput = edl.segments.length;
|
|
262
|
+
const seconds = getEdlDuration(edl);
|
|
263
|
+
return [
|
|
264
|
+
"-v",
|
|
265
|
+
"error",
|
|
266
|
+
"-y",
|
|
267
|
+
...inputs,
|
|
268
|
+
"-i",
|
|
269
|
+
input.speech,
|
|
270
|
+
"-filter_complex",
|
|
271
|
+
parts.join(";"),
|
|
272
|
+
"-map",
|
|
273
|
+
"[draft]",
|
|
274
|
+
"-map",
|
|
275
|
+
`${speechInput}:a`,
|
|
276
|
+
"-t",
|
|
277
|
+
String(seconds),
|
|
278
|
+
"-c:v",
|
|
279
|
+
"libx264",
|
|
280
|
+
"-preset",
|
|
281
|
+
"veryfast",
|
|
282
|
+
"-crf",
|
|
283
|
+
String(DRAFT_CRF),
|
|
284
|
+
"-pix_fmt",
|
|
285
|
+
"yuv420p",
|
|
286
|
+
"-c:a",
|
|
287
|
+
"aac",
|
|
288
|
+
"-b:a",
|
|
289
|
+
DRAFT_AUDIO_BITRATE,
|
|
290
|
+
"-movflags",
|
|
291
|
+
"+faststart",
|
|
292
|
+
input.output,
|
|
293
|
+
];
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
export interface RenderFootageDraftOptions extends Pick<FootageRunOptions, "buildDir" | "log" | "signal" | "onProgress"> {
|
|
297
|
+
/** The draft MP4. */
|
|
298
|
+
output: string;
|
|
299
|
+
/** The draft's width; default 540 (540×960 for a 9:16 film). */
|
|
300
|
+
width?: number;
|
|
301
|
+
/** A project's override of the format's caption box (`marketing.json` `layout`), as the render takes it. */
|
|
302
|
+
layout?: LayoutOverride;
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
export interface FootageDraft {
|
|
306
|
+
output: string;
|
|
307
|
+
/** The draft's length: the cut's. */
|
|
308
|
+
seconds: number;
|
|
309
|
+
width: number;
|
|
310
|
+
height: number;
|
|
311
|
+
bytes: number;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
/**
|
|
315
|
+
* Makes the draft of a cut film: the speech track, the captions file and the font in `<buildDir>/draft/`, then one
|
|
316
|
+
* ffmpeg run (`buildFootageDraftArgs`). Spends nothing; takes the same clips, transcript and EDL as `renderFootage`.
|
|
317
|
+
*/
|
|
318
|
+
export async function renderFootageDraft(
|
|
319
|
+
film: FootageFilm,
|
|
320
|
+
clips: PreparedClip[],
|
|
321
|
+
transcript: Extract<TranscribeOutcome, { kind: "ready" }>,
|
|
322
|
+
edl: Edl,
|
|
323
|
+
options: RenderFootageDraftOptions,
|
|
324
|
+
): Promise<FootageResult<FootageDraft>> {
|
|
325
|
+
const { log, signal } = options;
|
|
326
|
+
const aborting = signal === undefined ? {} : { signal };
|
|
327
|
+
const seconds = getEdlDuration(edl);
|
|
328
|
+
const draftDir = resolve(options.buildDir, "draft");
|
|
329
|
+
rmSync(draftDir, { recursive: true, force: true });
|
|
330
|
+
mkdirSync(draftDir, { recursive: true });
|
|
331
|
+
const report = (item: string, itemIndex: number, done: number): void =>
|
|
332
|
+
options.onProgress?.({ step: "draft", item, itemIndex, itemCount: 2, done: Math.min(seconds * 2, (itemIndex - 1) * seconds + done), total: seconds * 2 });
|
|
333
|
+
const clipFiles = Object.fromEntries(clips.map((clip) => [clip.clip.id, resolve(clip.file)]));
|
|
334
|
+
const speech = join(draftDir, "speech.mp3");
|
|
335
|
+
const words = remapWords(edl, transcript.words, film.cut.fillers);
|
|
336
|
+
const frame = getFrame(film);
|
|
337
|
+
const captions = buildFootageDraftCaptions({ film, edl, words, caption: getFootageCaptionLayout(film, options.layout ?? {}), frame });
|
|
338
|
+
if (captions !== null) {
|
|
339
|
+
writeFileSync(join(draftDir, "captions.ass"), captions);
|
|
340
|
+
mkdirSync(join(draftDir, "fonts"));
|
|
341
|
+
copyFileSync(FOOTAGE_DRAFT_FONT.file, join(draftDir, "fonts", basename(FOOTAGE_DRAFT_FONT.file)));
|
|
342
|
+
}
|
|
343
|
+
const output = resolve(options.output);
|
|
344
|
+
mkdirSync(dirname(output), { recursive: true });
|
|
345
|
+
report("speech", 1, 0);
|
|
346
|
+
await runFfmpeg("The draft's speech track", buildSpeechArgs(edl, transcript.speech, speech), { ...aborting, output: speech, onSeconds: (done) => report("speech", 1, done) });
|
|
347
|
+
report("film", 2, 0);
|
|
348
|
+
// Run in the draft folder: the captions file and the font folder are relative names, so no path needs escaping in the filter graph.
|
|
349
|
+
const args = buildFootageDraftArgs({
|
|
350
|
+
film,
|
|
351
|
+
edl,
|
|
352
|
+
clips: clipFiles,
|
|
353
|
+
speech,
|
|
354
|
+
frame,
|
|
355
|
+
...(options.width === undefined ? {} : { width: options.width }),
|
|
356
|
+
captions: captions === null ? null : "captions.ass",
|
|
357
|
+
fontsDir: captions === null ? null : "fonts",
|
|
358
|
+
output,
|
|
359
|
+
});
|
|
360
|
+
await runFfmpeg("The footage draft", args, { ...aborting, output, cwd: draftDir, onSeconds: (done) => report("film", 2, done) });
|
|
361
|
+
report("film", 2, seconds);
|
|
362
|
+
const draft = getFootageDraftFrame(frame, options.width);
|
|
363
|
+
const bytes = statSync(output).size;
|
|
364
|
+
log(`draft: ${output}, ${draft.width}×${draft.height}, ${seconds.toFixed(1)} s, ${bytes} bytes${captions === null ? ", no captions" : ""}.`);
|
|
365
|
+
return ok({ output, seconds, width: draft.width, height: draft.height, bytes });
|
|
366
|
+
}
|
|
@@ -103,6 +103,7 @@ export function createElevenLabsSttProvider(options: ElevenLabsFootageOptions):
|
|
|
103
103
|
form.append("language_code", input.language);
|
|
104
104
|
form.append("timestamps_granularity", "word");
|
|
105
105
|
form.append("tag_audio_events", "false");
|
|
106
|
+
for (const term of input.keyTerms ?? []) form.append("keyterms", term);
|
|
106
107
|
const response = await send(options, "speech to text", "/v1/speech-to-text", { method: "POST", body: form, signal: input.signal ?? null });
|
|
107
108
|
if (!response.ok) return response;
|
|
108
109
|
let body: unknown;
|
package/src/footage/film.ts
CHANGED
|
@@ -44,7 +44,17 @@ export interface FootageClip {
|
|
|
44
44
|
/** Where the music under the film comes from. `prompt` and `video` are paid (ElevenLabs Music, Video to Music). */
|
|
45
45
|
export type MusicSource =
|
|
46
46
|
| { kind: "none" }
|
|
47
|
-
| {
|
|
47
|
+
| {
|
|
48
|
+
kind: "file";
|
|
49
|
+
/** A local audio file, or a video whose sound is the bed. */
|
|
50
|
+
path: string;
|
|
51
|
+
/**
|
|
52
|
+
* The file's original name and content type, for the type check when the path has no extension (a storage key
|
|
53
|
+
* such as `@softure-ai/files`' `getStorageKey`). Without them and without an extension, only the probe checks it.
|
|
54
|
+
*/
|
|
55
|
+
fileName?: string;
|
|
56
|
+
contentType?: string;
|
|
57
|
+
}
|
|
48
58
|
| { kind: "prompt"; prompt: string }
|
|
49
59
|
| { kind: "video"; description: string; tags: string[] };
|
|
50
60
|
|
|
@@ -85,6 +95,11 @@ export interface FootageFilm {
|
|
|
85
95
|
clips: FootageClip[];
|
|
86
96
|
/** Clean the talking clips' voice (ElevenLabs Voice Isolator, paid) before transcribing them. */
|
|
87
97
|
isolateVoice: boolean;
|
|
98
|
+
/**
|
|
99
|
+
* Names and brand words the transcript should spell as given (Scribe keyterm prompting, billed on top of the
|
|
100
|
+
* transcription). Checked against `SCRIBE_KEYTERM_LIMITS`; absent or empty: none.
|
|
101
|
+
*/
|
|
102
|
+
keyTerms?: string[];
|
|
88
103
|
cut: CutOptions;
|
|
89
104
|
transitions: FootageTransitions;
|
|
90
105
|
music: FootageMusic;
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import type { FootageFilm } from "./film.js";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Key terms (Scribe keyterm prompting): names and brand words the transcript should spell as given. Scribe v2 batch
|
|
5
|
+
* limits, checked 2026-11-03 (elevenlabs.io keyterm prompting guide): up to 1000 terms of at most 50 characters and
|
|
6
|
+
* 5 words each. Keyterm prompting is billed on top of the transcription.
|
|
7
|
+
*/
|
|
8
|
+
export const SCRIBE_KEYTERM_LIMITS = { maxTerms: 1000, maxCharacters: 50, maxWords: 5 } as const;
|
|
9
|
+
|
|
10
|
+
export type KeyTermsResult = { ok: true; terms: string[] } | { ok: false; errors: string[] };
|
|
11
|
+
|
|
12
|
+
const compareCodePoints = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0);
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Key terms trimmed (inner whitespace collapsed to one space), without blanks and exact duplicates, sorted by code
|
|
16
|
+
* point (the same order on every host, so the transcript's cache key is stable), or every limit a term breaks.
|
|
17
|
+
*/
|
|
18
|
+
export function normalizeKeyTerms(terms: readonly string[]): KeyTermsResult {
|
|
19
|
+
const unique = new Set<string>();
|
|
20
|
+
for (const term of terms) {
|
|
21
|
+
const cleaned = term.trim().replace(/\s+/g, " ");
|
|
22
|
+
if (cleaned.length > 0) unique.add(cleaned);
|
|
23
|
+
}
|
|
24
|
+
const { maxTerms, maxCharacters, maxWords } = SCRIBE_KEYTERM_LIMITS;
|
|
25
|
+
const errors: string[] = [];
|
|
26
|
+
if (unique.size > maxTerms) errors.push(`key terms: ${unique.size} terms, at most ${maxTerms}.`);
|
|
27
|
+
for (const term of unique) {
|
|
28
|
+
if (term.length > maxCharacters) errors.push(`key term "${term}": ${term.length} characters, at most ${maxCharacters}.`);
|
|
29
|
+
const words = term.split(" ").length;
|
|
30
|
+
if (words > maxWords) errors.push(`key term "${term}": ${words} words, at most ${maxWords}.`);
|
|
31
|
+
}
|
|
32
|
+
return errors.length > 0 ? { ok: false, errors } : { ok: true, terms: [...unique].sort(compareCodePoints) };
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** The film's key terms, normalized; none when the film has none. */
|
|
36
|
+
export function getFilmKeyTerms(film: Pick<FootageFilm, "keyTerms">): KeyTermsResult {
|
|
37
|
+
return normalizeKeyTerms(film.keyTerms ?? []);
|
|
38
|
+
}
|
package/src/footage/normalize.ts
CHANGED
|
@@ -47,21 +47,47 @@ export function toneMapFilter(hdr: HdrKind | null, transfer: string | null): str
|
|
|
47
47
|
];
|
|
48
48
|
}
|
|
49
49
|
|
|
50
|
-
/**
|
|
50
|
+
/**
|
|
51
|
+
* The video filter graph from input 0's picture to `[v]`. Every stage runs on as little as it can: frames are
|
|
52
|
+
* dropped to 30 fps first, then the picture is scaled to the frame (crop: covered and cropped to it; blur: fitted
|
|
53
|
+
* inside it), and only then tone-mapped, so the float stages of an HDR clip work on frame-sized pictures instead of
|
|
54
|
+
* the source's (a 4K phone clip has 4x the pixels of a 1080x1920 frame). The blur's background is made from the
|
|
55
|
+
* fitted picture.
|
|
56
|
+
*/
|
|
51
57
|
export function buildVideoFilter(options: { fit: "crop" | "blur"; frame: Frame; hdr: HdrKind | null; transfer: string | null }): string {
|
|
52
58
|
const { width, height } = options.frame;
|
|
53
|
-
const
|
|
54
|
-
const cover = `scale=${width}:${height}:force_original_aspect_ratio=increase,crop=${width}:${height}`;
|
|
59
|
+
const toneMap = toneMapFilter(options.hdr, options.transfer);
|
|
55
60
|
const tail = "setsar=1,format=yuv420p";
|
|
56
|
-
if (options.fit === "crop")
|
|
61
|
+
if (options.fit === "crop") {
|
|
62
|
+
const cover = `scale=${width}:${height}:force_original_aspect_ratio=increase,crop=${width}:${height}`;
|
|
63
|
+
return `[0:v]${[`fps=${FOOTAGE_FPS}`, cover, ...toneMap, tail].join(",")}[v]`;
|
|
64
|
+
}
|
|
65
|
+
// Even sizes: zscale refuses odd ones in 4:2:0.
|
|
66
|
+
const fitted = `scale=${width}:${height}:force_original_aspect_ratio=decrease:force_divisible_by=2`;
|
|
67
|
+
const backWidth = Math.round(width / 4);
|
|
68
|
+
const backHeight = Math.round(height / 4);
|
|
57
69
|
return [
|
|
58
|
-
`[0:v]${
|
|
59
|
-
`[bg]scale=${
|
|
60
|
-
`[fg]
|
|
61
|
-
`[back][front]overlay=(W-w)/2:(H-h)/2,${tail}[v]`,
|
|
70
|
+
`[0:v]${[`fps=${FOOTAGE_FPS}`, fitted, ...toneMap, "split=2[bg][fg]"].join(",")}`,
|
|
71
|
+
`[bg]scale=${backWidth}:${backHeight}:force_original_aspect_ratio=increase,crop=${backWidth}:${backHeight},boxblur=20:2,scale=${width}:${height},eq=brightness=-0.06[back]`,
|
|
72
|
+
`[back][fg]overlay=(W-w)/2:(H-h)/2,${tail}[v]`,
|
|
62
73
|
].join(";");
|
|
63
74
|
}
|
|
64
75
|
|
|
76
|
+
/** x264's presets, fastest first. */
|
|
77
|
+
export const X264_PRESETS = ["ultrafast", "superfast", "veryfast", "faster", "fast", "medium", "slow", "slower", "veryslow"] as const;
|
|
78
|
+
export type X264Preset = (typeof X264_PRESETS)[number];
|
|
79
|
+
const DEFAULT_NORMALIZE_PRESET: X264Preset = "veryfast";
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* How the intermediate clips are encoded. A small worker can trade file size for speed: `preset` (default
|
|
83
|
+
* `veryfast`; `superfast` or `ultrafast` encode faster into bigger files), `threads` (ffmpeg's encoder and filter
|
|
84
|
+
* threads; by default ffmpeg's own choice, which follows the host's cores, not a container's CPU quota).
|
|
85
|
+
*/
|
|
86
|
+
export interface NormalizeEncoder {
|
|
87
|
+
preset?: X264Preset;
|
|
88
|
+
threads?: number;
|
|
89
|
+
}
|
|
90
|
+
|
|
65
91
|
export interface NormalizeInput {
|
|
66
92
|
clip: FootageClip;
|
|
67
93
|
/** The local file (a URL's downloaded copy). */
|
|
@@ -69,19 +95,29 @@ export interface NormalizeInput {
|
|
|
69
95
|
probe: FootageProbe;
|
|
70
96
|
frame: Frame;
|
|
71
97
|
output: string;
|
|
98
|
+
encoder?: NormalizeEncoder;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** The thread arguments: global filter threads (before the inputs) and encoder threads (before the output). */
|
|
102
|
+
function getThreadArgs(threads: number | undefined): { global: string[]; output: string[] } {
|
|
103
|
+
if (threads === undefined) return { global: [], output: [] };
|
|
104
|
+
if (!Number.isInteger(threads) || threads < 1) throw new Error(`normalize.threads must be a positive whole number, got ${threads}.`);
|
|
105
|
+
return { global: ["-filter_complex_threads", String(threads)], output: ["-threads", String(threads)] };
|
|
72
106
|
}
|
|
73
107
|
|
|
74
|
-
/** ffmpeg's arguments for one clip; pure, so the graph is tested without ffmpeg. */
|
|
108
|
+
/** ffmpeg's arguments for one clip; pure, so the graph is tested without ffmpeg. Throws on an invalid `encoder.threads`. */
|
|
75
109
|
export function buildNormalizeArgs(input: NormalizeInput): string[] {
|
|
76
110
|
const { clip, file, probe, frame, output } = input;
|
|
77
111
|
const duration = getTrimmedDuration(clip, probe);
|
|
78
112
|
const fit = resolveFit(clip.fit, probe, frame);
|
|
79
113
|
const seek = clip.trim.start > 0 ? ["-ss", String(clip.trim.start)] : [];
|
|
114
|
+
const threads = getThreadArgs(input.encoder?.threads);
|
|
80
115
|
const silence = probe.hasAudio ? [] : ["-f", "lavfi", "-i", `anullsrc=r=${FOOTAGE_SAMPLE_RATE}:cl=stereo`];
|
|
81
116
|
return [
|
|
82
117
|
"-v",
|
|
83
118
|
"error",
|
|
84
119
|
"-y",
|
|
120
|
+
...threads.global,
|
|
85
121
|
...seek,
|
|
86
122
|
"-i",
|
|
87
123
|
file,
|
|
@@ -97,7 +133,7 @@ export function buildNormalizeArgs(input: NormalizeInput): string[] {
|
|
|
97
133
|
"-c:v",
|
|
98
134
|
"libx264",
|
|
99
135
|
"-preset",
|
|
100
|
-
|
|
136
|
+
input.encoder?.preset ?? DEFAULT_NORMALIZE_PRESET,
|
|
101
137
|
"-crf",
|
|
102
138
|
"18",
|
|
103
139
|
"-g",
|
|
@@ -118,6 +154,7 @@ export function buildNormalizeArgs(input: NormalizeInput): string[] {
|
|
|
118
154
|
"2",
|
|
119
155
|
"-movflags",
|
|
120
156
|
"+faststart",
|
|
157
|
+
...threads.output,
|
|
121
158
|
output,
|
|
122
159
|
];
|
|
123
160
|
}
|
package/src/footage/paid.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { err, ok, type TtsResult } from "../voice/provider.js";
|
|
|
6
6
|
import { footageCacheKey, getFootageCachePath, hashFile } from "./cache.js";
|
|
7
7
|
import { END_CARD_SECONDS, getFootageFilmSeconds } from "./compose.js";
|
|
8
8
|
import { readEdl, type Edl } from "./edl.js";
|
|
9
|
+
import { getFilmKeyTerms } from "./keyterms.js";
|
|
9
10
|
import type { FootageClip, FootageFilm } from "./film.js";
|
|
10
11
|
import { getTrimmedDuration, probeFootage } from "./probe.js";
|
|
11
12
|
import type { FootageProviders, FootageRates, PaidStepKind } from "./providers.js";
|
|
@@ -66,7 +67,10 @@ export function getIsolatedCache(film: FootageFilm, clipKey: string, options: Pi
|
|
|
66
67
|
|
|
67
68
|
export function getTranscriptCache(film: FootageFilm, clipKey: string, options: Pick<FootageCacheOptions, "cacheDir" | "providers">): CacheEntry {
|
|
68
69
|
const { stt } = options.providers;
|
|
69
|
-
const
|
|
70
|
+
const keyTerms = getFilmKeyTerms(film);
|
|
71
|
+
// Key terms join the key only when there are any, so a transcript bought without them keeps its key.
|
|
72
|
+
const withKeyTerms = keyTerms.ok && keyTerms.terms.length > 0 ? { keyTerms: keyTerms.terms } : {};
|
|
73
|
+
const key = footageCacheKey("transcript", { clip: clipKey, isolated: film.isolateVoice, provider: stt.id, model: stt.model, language: film.language, ...withKeyTerms });
|
|
70
74
|
return { key, path: getFootageCachePath(options.cacheDir, film.id, "transcript", key) };
|
|
71
75
|
}
|
|
72
76
|
|
|
@@ -175,10 +179,15 @@ const round = (value: number, digits: number): number => {
|
|
|
175
179
|
* What a footage film will cost in the caller's credits, before anything runs: every clip probed (a URL clip not yet
|
|
176
180
|
* downloaded is probed at its URL), nothing normalized, no provider called. Talking clips need speech to text (and
|
|
177
181
|
* isolation when the film cleans the voice); generated music lasts the cut's length when the cut is known, else the
|
|
178
|
-
* clips' length plus the end card. A cached step costs 0.
|
|
182
|
+
* clips' length plus the end card. A cached step costs 0. With key terms, transcription costs `rates.transcribe` plus
|
|
183
|
+
* the keyterm surcharge `rates.keyTerms`; invalid key terms fail the estimate.
|
|
179
184
|
*/
|
|
180
185
|
export async function estimateFootageFilm(film: FootageFilm, options: EstimateFootageOptions): Promise<TtsResult<FootageEstimate>> {
|
|
181
186
|
if (film.clips.length === 0) return err(`footage film ${film.id} has no clips.`);
|
|
187
|
+
const keyTerms = getFilmKeyTerms(film);
|
|
188
|
+
if (!keyTerms.ok) return err(`footage film ${film.id}: ${keyTerms.errors.join(" ")}`);
|
|
189
|
+
const surcharge = keyTerms.terms.length === 0 ? 0 : (options.rates.keyTerms ?? 0);
|
|
190
|
+
const rates: Record<PaidStepKind, number> = { ...options.rates, transcribe: options.rates.transcribe + surcharge };
|
|
182
191
|
const files: Record<string, string> = {};
|
|
183
192
|
for (const clip of film.clips) {
|
|
184
193
|
const local = findLocalSource(clip, options.buildDir);
|
|
@@ -189,7 +198,7 @@ export async function estimateFootageFilm(film: FootageFilm, options: EstimateFo
|
|
|
189
198
|
if (!probed.ok) return err(probed.errors.join("\n"));
|
|
190
199
|
const items: FootageEstimateItem[] = [];
|
|
191
200
|
const add = (step: PaidStepKind, item: string, provider: string, units: number, cached: boolean): void => {
|
|
192
|
-
items.push({ step, item, provider, units: round(units, 3), unit: "seconds", credits: cached ? 0 : round(units *
|
|
201
|
+
items.push({ step, item, provider, units: round(units, 3), unit: "seconds", credits: cached ? 0 : round(units * rates[step], 2), cached });
|
|
193
202
|
};
|
|
194
203
|
let clipSeconds = 0;
|
|
195
204
|
for (const { clip, probe } of probed.clips) {
|