@softure-ai/marketing-kit 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +25 -10
- package/dist/footage/draft-font.d.ts +18 -0
- package/dist/footage/draft-font.d.ts.map +1 -0
- package/dist/footage/draft-font.js +105 -0
- package/dist/footage/draft-font.js.map +1 -0
- package/dist/footage/draft.d.ts +49 -12
- package/dist/footage/draft.d.ts.map +1 -1
- package/dist/footage/draft.js +165 -35
- package/dist/footage/draft.js.map +1 -1
- package/dist/footage/pipeline.d.ts +20 -0
- package/dist/footage/pipeline.d.ts.map +1 -1
- package/dist/footage/pipeline.js +32 -11
- package/dist/footage/pipeline.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -2
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/src/footage/draft-font.ts +105 -0
- package/src/footage/draft.ts +184 -40
- package/src/footage/pipeline.ts +40 -10
- package/src/index.ts +7 -0
package/src/footage/draft.ts
CHANGED
|
@@ -1,23 +1,28 @@
|
|
|
1
|
-
import { copyFileSync, mkdirSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
1
|
+
import { copyFileSync, existsSync, mkdirSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
2
2
|
import { basename, dirname, join, resolve } from "node:path";
|
|
3
3
|
import { fileURLToPath } from "node:url";
|
|
4
4
|
|
|
5
|
-
import type
|
|
6
|
-
import {
|
|
5
|
+
import { captionChunks, type CaptionLayout, type LayoutOverride } from "../compose/timeline.js";
|
|
6
|
+
import type { BrandColors } from "../config/colors.js";
|
|
7
|
+
import { err, ok } from "../voice/provider.js";
|
|
7
8
|
import type { TimedWord } from "../voice/voiceover.js";
|
|
8
9
|
import { getCaptionSpans, getFootageCaptionLayout, phraseChunks } from "./captions.js";
|
|
9
10
|
import { getSegmentScales, getSegmentZoomEases } from "./compose.js";
|
|
11
|
+
import { chooseFootageDraftFont } from "./draft-font.js";
|
|
10
12
|
import { getEdlDuration, getSegmentStarts, remapWords, type Edl } from "./edl.js";
|
|
11
13
|
import { getFootageEdit, type FootageFilm } from "./film.js";
|
|
14
|
+
import { duckingKeyframes, getDuckingOptions } from "./music.js";
|
|
12
15
|
import { FOOTAGE_FPS, getThreadArgs, type Frame } from "./normalize.js";
|
|
13
|
-
import { getFrame, type FootageResult, type FootageRunOptions, type PreparedClip, type TranscribeOutcome } from "./pipeline.js";
|
|
16
|
+
import { findFootageMusic, getFrame, type FootageBrand, type FootageResult, type FootageRunOptions, type PreparedClip, type TranscribeOutcome } from "./pipeline.js";
|
|
14
17
|
import { runFfmpeg } from "./process.js";
|
|
15
|
-
import { buildSpeechArgs, getSpeechCutOptions } from "./render.js";
|
|
18
|
+
import { buildMusicArgs, buildSpeechArgs, getSpeechCutOptions } from "./render.js";
|
|
16
19
|
|
|
17
20
|
/**
|
|
18
21
|
* A fast draft of a footage film, ffmpeg only (no hyperframes, no browser): the cut at a small width with the film's
|
|
19
|
-
*
|
|
20
|
-
*
|
|
22
|
+
* captions burned in (its caption style, in the brand's body font and caption colours), its zoom cuts (eased as in the
|
|
23
|
+
* render), cross-fades and Ken Burns b-roll drawn as the full render draws them, the speech track and the ducked music
|
|
24
|
+
* when it costs nothing. No loudness pass, end card, cover or sound effects. What an app shows the owner before the paid
|
|
25
|
+
* render.
|
|
21
26
|
*/
|
|
22
27
|
|
|
23
28
|
/** The draft's default width: 540 makes a 540×960 draft of a 9:16 film. */
|
|
@@ -36,6 +41,13 @@ const ZOOM_ORIGIN = { x: 0.5, y: 0.42 } as const;
|
|
|
36
41
|
const DRAFT_CRF = 28;
|
|
37
42
|
const DRAFT_AUDIO_BITRATE = "128k";
|
|
38
43
|
|
|
44
|
+
/** The caption colours of a `highlight` draft made without a brand. */
|
|
45
|
+
export const FOOTAGE_DRAFT_CAPTION_COLORS = { captionBackground: "#111111", captionText: "#ffffff", captionHighlight: "#ffd400" } as const;
|
|
46
|
+
/** How long the render keeps a spoken word lit in the `highlight` pill. */
|
|
47
|
+
const HIGHLIGHT_SECONDS = 0.32;
|
|
48
|
+
/** The pill's padding around its text (`CAPTION_PILL_PADDING.y`): the opaque box's border in the film's frame. */
|
|
49
|
+
const HIGHLIGHT_BOX_PADDING = 16;
|
|
50
|
+
|
|
39
51
|
const r3 = (value: number): number => Math.round(value * 1000) / 1000 || 0;
|
|
40
52
|
const r4 = (value: number): number => Math.round(value * 10000) / 10000 || 0;
|
|
41
53
|
|
|
@@ -55,13 +67,29 @@ function toAssTime(seconds: number): string {
|
|
|
55
67
|
return `${hours}:${pad(minutes)}:${pad(secs)}.${pad(centiseconds % 100)}`;
|
|
56
68
|
}
|
|
57
69
|
|
|
70
|
+
/**
|
|
71
|
+
* A hex colour (`#rgb`, `#rgba`, `#rrggbb`, `#rrggbbaa`) as an ASS colour, `&HAABBGGRR&`-style without the closing
|
|
72
|
+
* `&`: ASS writes blue first and counts alpha the other way round (00 is opaque).
|
|
73
|
+
*/
|
|
74
|
+
export function toAssColor(hex: string): string {
|
|
75
|
+
const digits = hex.replace(/^#/, "");
|
|
76
|
+
const full = digits.length <= 4 ? [...digits].map((digit) => digit + digit).join("") : digits;
|
|
77
|
+
if (!/^(?:[0-9a-f]{6}|[0-9a-f]{8})$/i.test(full)) throw new Error(`Not a hex colour: "${hex}".`);
|
|
78
|
+
const pair = (index: number): string => full.slice(index, index + 2).toUpperCase();
|
|
79
|
+
const alpha = full.length === 8 ? (255 - parseInt(full.slice(6, 8), 16)).toString(16).padStart(2, "0").toUpperCase() : "00";
|
|
80
|
+
return `&H${alpha}${pair(4)}${pair(2)}${pair(0)}`;
|
|
81
|
+
}
|
|
82
|
+
|
|
58
83
|
/** Caption text as ASS dialogue: override braces and backslashes would be read as tags, line breaks end the line. */
|
|
59
84
|
function toAssText(text: string): string {
|
|
60
85
|
return text.replace(/\\/g, "/").replace(/\{/g, "(").replace(/\}/g, ")").replace(/\s+/g, " ").trim();
|
|
61
86
|
}
|
|
62
87
|
|
|
88
|
+
/** The caption colours the draft paints with (`BrandColors`). */
|
|
89
|
+
export type FootageDraftCaptionColors = Pick<BrandColors, "captionBackground" | "captionText" | "captionHighlight">;
|
|
90
|
+
|
|
63
91
|
export interface FootageDraftCaptionsInput {
|
|
64
|
-
film: Pick<FootageFilm, "captions">;
|
|
92
|
+
film: Pick<FootageFilm, "captions" | "captionStyle">;
|
|
65
93
|
edl: Edl;
|
|
66
94
|
/** The film's words in film time (`remapWords`). */
|
|
67
95
|
words: TimedWord[];
|
|
@@ -69,32 +97,73 @@ export interface FootageDraftCaptionsInput {
|
|
|
69
97
|
caption: CaptionLayout;
|
|
70
98
|
/** The film's full frame; the ASS file is laid out in it and libass scales it to the draft. */
|
|
71
99
|
frame: Frame;
|
|
72
|
-
/** The font family the captions are drawn in; default `FOOTAGE_DRAFT_FONT.family`. */
|
|
100
|
+
/** The font family the captions are drawn in, as the font file names it; default `FOOTAGE_DRAFT_FONT.family`. */
|
|
73
101
|
fontFamily?: string;
|
|
102
|
+
/** The `highlight` pill's colours; default `FOOTAGE_DRAFT_CAPTION_COLORS`. A `phrase` caption is white, as in the render. */
|
|
103
|
+
colors?: FootageDraftCaptionColors;
|
|
74
104
|
}
|
|
75
105
|
|
|
76
106
|
/**
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
-
|
|
107
|
+
* A `highlight` chunk as ASS events: its span cut wherever a word lights up or goes out, each piece showing the chunk
|
|
108
|
+
* with the words lit then in `highlight` (a word stays lit for the render's 0.32 s from its start).
|
|
109
|
+
*/
|
|
110
|
+
function buildHighlightEvents(chunk: { words: TimedWord[] }, span: { start: number; end: number }, colors: { text: string; highlight: string }): string[] {
|
|
111
|
+
const cs = (seconds: number): number => Math.round(seconds * 100);
|
|
112
|
+
const [from, to] = [cs(span.start), cs(span.end)];
|
|
113
|
+
const lights = chunk.words.map((word) => ({ on: cs(word.start), off: cs(word.start + HIGHLIGHT_SECONDS) }));
|
|
114
|
+
const cuts = [...new Set([from, to, ...lights.flatMap((light) => [light.on, light.off])])].filter((at) => at >= from && at <= to).sort((a, b) => a - b);
|
|
115
|
+
const events: string[] = [];
|
|
116
|
+
cuts.slice(0, -1).forEach((at, index) => {
|
|
117
|
+
const next = cuts[index + 1] as number;
|
|
118
|
+
const text = chunk.words
|
|
119
|
+
.map((word, wordIndex) => {
|
|
120
|
+
const light = lights[wordIndex] as { on: number; off: number };
|
|
121
|
+
const shown = toAssText(word.text);
|
|
122
|
+
return light.on <= at && at < light.off ? `{\\1c${colors.highlight}&}${shown}{\\1c${colors.text}&}` : shown;
|
|
123
|
+
})
|
|
124
|
+
.join(" ");
|
|
125
|
+
events.push(`Dialogue: 0,${toAssTime(at / 100)},${toAssTime(next / 100)},Caption,,0,0,0,,${text}`);
|
|
126
|
+
});
|
|
127
|
+
return events;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* The draft's captions as an ASS subtitle file, shown for the render's spans (`getCaptionSpans`) and centred in the
|
|
132
|
+
* caption box from its top (or from its foot, with `bottom`). `phrase` (the default): the film's phrases
|
|
133
|
+
* (`phraseChunks`), white bold text with a dark outline and shadow. `highlight`: the render's chunks of up to four
|
|
134
|
+
* words (`captionChunks`) on an opaque box of `captionBackground`, in `captionText`, the word being said in
|
|
135
|
+
* `captionHighlight`. Null when the film has no captions or nothing is said.
|
|
80
136
|
*/
|
|
81
137
|
export function buildFootageDraftCaptions(input: FootageDraftCaptionsInput): string | null {
|
|
82
138
|
if (!input.film.captions) return null;
|
|
83
|
-
const
|
|
139
|
+
const isHighlight = input.film.captionStyle === "highlight";
|
|
140
|
+
const chunks = isHighlight ? captionChunks(input.words) : phraseChunks(input.words);
|
|
84
141
|
if (chunks.length === 0) return null;
|
|
85
142
|
const spans = getCaptionSpans(chunks, getEdlDuration(input.edl));
|
|
86
143
|
const { caption, frame } = input;
|
|
87
144
|
const family = (input.fontFamily ?? FOOTAGE_DRAFT_FONT.family).replace(/,/g, " ");
|
|
88
|
-
const
|
|
89
|
-
const
|
|
145
|
+
const colors = input.colors ?? FOOTAGE_DRAFT_CAPTION_COLORS;
|
|
146
|
+
const text = toAssColor(colors.captionText);
|
|
147
|
+
const box = toAssColor(colors.captionBackground);
|
|
148
|
+
// libass draws BorderStyle 3's box in the outline colour, grown by the outline width; the back colour is its shadow.
|
|
149
|
+
const look = isHighlight
|
|
150
|
+
? { primary: text, outlineColor: box, backColor: box, borderStyle: 3, outline: HIGHLIGHT_BOX_PADDING, shadow: 0 }
|
|
151
|
+
: {
|
|
152
|
+
primary: "&H00FFFFFF",
|
|
153
|
+
outlineColor: "&H8C000000",
|
|
154
|
+
backColor: "&H8C000000",
|
|
155
|
+
borderStyle: 1,
|
|
156
|
+
outline: Math.max(1, Math.round(caption.fontSize / 16)),
|
|
157
|
+
shadow: Math.max(1, Math.round(caption.fontSize / 25)),
|
|
158
|
+
};
|
|
90
159
|
const style = [
|
|
91
160
|
"Caption",
|
|
92
161
|
family,
|
|
93
162
|
caption.fontSize,
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
163
|
+
look.primary,
|
|
164
|
+
look.primary,
|
|
165
|
+
look.outlineColor,
|
|
166
|
+
look.backColor,
|
|
98
167
|
-1,
|
|
99
168
|
0,
|
|
100
169
|
0,
|
|
@@ -103,9 +172,9 @@ export function buildFootageDraftCaptions(input: FootageDraftCaptionsInput): str
|
|
|
103
172
|
100,
|
|
104
173
|
0,
|
|
105
174
|
0,
|
|
106
|
-
|
|
107
|
-
outline,
|
|
108
|
-
shadow,
|
|
175
|
+
look.borderStyle,
|
|
176
|
+
look.outline,
|
|
177
|
+
look.shadow,
|
|
109
178
|
// ASS alignment 2 hangs the text from MarginV above the bottom, 8 from MarginV below the top.
|
|
110
179
|
caption.bottom === undefined ? 8 : 2,
|
|
111
180
|
Math.round(caption.left),
|
|
@@ -113,10 +182,12 @@ export function buildFootageDraftCaptions(input: FootageDraftCaptionsInput): str
|
|
|
113
182
|
Math.round(caption.bottom ?? caption.top),
|
|
114
183
|
1,
|
|
115
184
|
].join(",");
|
|
116
|
-
const
|
|
185
|
+
const highlight = { text: text.replace(/^&H../, "&H"), highlight: toAssColor(colors.captionHighlight).replace(/^&H../, "&H") };
|
|
186
|
+
const events = chunks.flatMap((chunk, index) => {
|
|
117
187
|
const span = spans[index] as { start: number; end: number };
|
|
118
|
-
|
|
119
|
-
|
|
188
|
+
if (isHighlight) return buildHighlightEvents(chunk, span, highlight);
|
|
189
|
+
const line = toAssText(chunk.words.map((word) => word.text).join(" "));
|
|
190
|
+
return [`Dialogue: 0,${toAssTime(span.start)},${toAssTime(span.end)},Caption,,0,0,0,,${line}`];
|
|
120
191
|
});
|
|
121
192
|
return [
|
|
122
193
|
"[Script Info]",
|
|
@@ -144,6 +215,8 @@ export interface FootageDraftArgsInput {
|
|
|
144
215
|
clips: Record<string, string>;
|
|
145
216
|
/** The cut speech track (`buildSpeechArgs`). */
|
|
146
217
|
speech: string;
|
|
218
|
+
/** The ducked music track (`buildMusicArgs`, as long as the cut), mixed under the speech; null or absent for none. */
|
|
219
|
+
music?: string | null;
|
|
147
220
|
/** The film's full frame (the normalized clips' size). */
|
|
148
221
|
frame: Frame;
|
|
149
222
|
/** The draft's width; default 540. */
|
|
@@ -280,6 +353,8 @@ export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
|
|
|
280
353
|
parts.push(`[${joined}]null[draft]`);
|
|
281
354
|
}
|
|
282
355
|
const speechInput = edl.segments.length;
|
|
356
|
+
const music = input.music ?? null;
|
|
357
|
+
if (music !== null) parts.push(`[${speechInput}:a][${speechInput + 1}:a]amix=inputs=2:duration=first:normalize=0[sound]`);
|
|
283
358
|
const seconds = getEdlDuration(edl);
|
|
284
359
|
return [
|
|
285
360
|
"-v",
|
|
@@ -289,12 +364,13 @@ export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
|
|
|
289
364
|
...inputs,
|
|
290
365
|
"-i",
|
|
291
366
|
input.speech,
|
|
367
|
+
...(music === null ? [] : ["-i", music]),
|
|
292
368
|
"-filter_complex",
|
|
293
369
|
parts.join(";"),
|
|
294
370
|
"-map",
|
|
295
371
|
"[draft]",
|
|
296
372
|
"-map",
|
|
297
|
-
`${speechInput}:a
|
|
373
|
+
music === null ? `${speechInput}:a` : "[sound]",
|
|
298
374
|
"-t",
|
|
299
375
|
String(seconds),
|
|
300
376
|
"-c:v",
|
|
@@ -316,13 +392,19 @@ export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
|
|
|
316
392
|
];
|
|
317
393
|
}
|
|
318
394
|
|
|
319
|
-
export interface RenderFootageDraftOptions extends Pick<FootageRunOptions, "buildDir" | "log" | "signal" | "onProgress"
|
|
395
|
+
export interface RenderFootageDraftOptions extends Pick<FootageRunOptions, "buildDir" | "log" | "signal" | "onProgress">, Partial<Pick<FootageRunOptions, "cacheDir" | "providers">> {
|
|
320
396
|
/** The draft MP4. */
|
|
321
397
|
output: string;
|
|
322
398
|
/** The draft's width; default 540 (540×960 for a 9:16 film). */
|
|
323
399
|
width?: number;
|
|
324
400
|
/** A project's override of the format's caption box (`marketing.json` `layout`), as the render takes it. */
|
|
325
401
|
layout?: LayoutOverride;
|
|
402
|
+
/**
|
|
403
|
+
* The brand the render takes (`renderFootage`'s `brand`): the captions are drawn in its body font (a TrueType or
|
|
404
|
+
* OpenType file; DejaVu Sans when it has none) and its caption colours. Without it, DejaVu Sans and
|
|
405
|
+
* `FOOTAGE_DRAFT_CAPTION_COLORS`.
|
|
406
|
+
*/
|
|
407
|
+
brand?: { colors: FootageDraftCaptionColors; fonts: Pick<FootageBrand["fonts"], "body"> };
|
|
326
408
|
/**
|
|
327
409
|
* ffmpeg's threads for the draft's decoders, filter graph and encoder (`buildFootageDraftArgs`); by default ffmpeg's
|
|
328
410
|
* own choice, which follows the host's cores. Set it from a CPU-limited container's quota.
|
|
@@ -330,8 +412,14 @@ export interface RenderFootageDraftOptions extends Pick<FootageRunOptions, "buil
|
|
|
330
412
|
threads?: number;
|
|
331
413
|
}
|
|
332
414
|
|
|
415
|
+
/** The draft's music: the film has none, it played (a file, or generated music from the cache), or it is not cached yet. */
|
|
416
|
+
export type FootageDraftMusic = "none" | "played" | "not_cached";
|
|
417
|
+
|
|
333
418
|
export interface FootageDraft {
|
|
334
419
|
output: string;
|
|
420
|
+
music: FootageDraftMusic;
|
|
421
|
+
/** The family the captions are drawn in; null without captions. */
|
|
422
|
+
captionFont: string | null;
|
|
335
423
|
/** The draft's length: the cut's. */
|
|
336
424
|
seconds: number;
|
|
337
425
|
width: number;
|
|
@@ -339,9 +427,34 @@ export interface FootageDraft {
|
|
|
339
427
|
bytes: number;
|
|
340
428
|
}
|
|
341
429
|
|
|
430
|
+
/** The draft's music track: a file to duck, or why there is none. */
|
|
431
|
+
type DraftMusicSource = { kind: "file"; path: string } | { kind: "none" } | { kind: "not_cached"; reason: string };
|
|
432
|
+
|
|
342
433
|
/**
|
|
343
|
-
*
|
|
344
|
-
*
|
|
434
|
+
* The music the draft can play without spending: the film's own file, or generated music already in the paid cache
|
|
435
|
+
* (looked up as the render looks it up, `findFootageMusic`). Never calls a provider.
|
|
436
|
+
*/
|
|
437
|
+
async function findDraftMusic(film: FootageFilm, edl: Edl, clips: PreparedClip[], options: RenderFootageDraftOptions): Promise<FootageResult<DraftMusicSource>> {
|
|
438
|
+
const source = film.music.source;
|
|
439
|
+
if (source.kind === "none") return ok({ kind: "none" });
|
|
440
|
+
if (source.kind === "file") return existsSync(source.path) ? ok({ kind: "file", path: source.path }) : err(`music: no file at ${source.path}`);
|
|
441
|
+
const { cacheDir, providers } = options;
|
|
442
|
+
if (cacheDir === undefined || providers === undefined) {
|
|
443
|
+
return ok({ kind: "not_cached", reason: "the film's music is generated and the draft was given no cacheDir and providers to find it in the cache" });
|
|
444
|
+
}
|
|
445
|
+
const found = await findFootageMusic(film, edl, clips, { cacheDir, providers });
|
|
446
|
+
if (!found.ok) return found;
|
|
447
|
+
const music = found.value;
|
|
448
|
+
if (music.kind === "none") return ok({ kind: "none" });
|
|
449
|
+
if (music.kind === "file") return ok({ kind: "file", path: music.path });
|
|
450
|
+
if (music.kind === "cached") return ok({ kind: "file", path: music.cache.path });
|
|
451
|
+
return ok({ kind: "not_cached", reason: `the film's generated music is not in the cache yet (${music.cache.path}); the render buys it` });
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
/**
|
|
455
|
+
* Makes the draft of a cut film: the speech track, the ducked music track (when it costs nothing), the captions file
|
|
456
|
+
* and its font in `<buildDir>/draft/`, then one ffmpeg run (`buildFootageDraftArgs`). Spends nothing: no provider is
|
|
457
|
+
* called; takes the same clips, transcript and EDL as `renderFootage`.
|
|
345
458
|
*/
|
|
346
459
|
export async function renderFootageDraft(
|
|
347
460
|
film: FootageFilm,
|
|
@@ -353,32 +466,61 @@ export async function renderFootageDraft(
|
|
|
353
466
|
const { log, signal } = options;
|
|
354
467
|
const aborting = signal === undefined ? {} : { signal };
|
|
355
468
|
const seconds = getEdlDuration(edl);
|
|
469
|
+
const music = await findDraftMusic(film, edl, clips, options);
|
|
470
|
+
if (!music.ok) return music;
|
|
356
471
|
const draftDir = resolve(options.buildDir, "draft");
|
|
357
472
|
rmSync(draftDir, { recursive: true, force: true });
|
|
358
473
|
mkdirSync(draftDir, { recursive: true });
|
|
359
|
-
const
|
|
360
|
-
|
|
474
|
+
const items = music.value.kind === "file" ? ["speech", "music", "film"] : ["speech", "film"];
|
|
475
|
+
const report = (item: string, done: number): void => {
|
|
476
|
+
const itemIndex = items.indexOf(item) + 1;
|
|
477
|
+
options.onProgress?.({ step: "draft", item, itemIndex, itemCount: items.length, done: Math.min(seconds * items.length, (itemIndex - 1) * seconds + done), total: seconds * items.length });
|
|
478
|
+
};
|
|
361
479
|
const clipFiles = Object.fromEntries(clips.map((clip) => [clip.clip.id, resolve(clip.file)]));
|
|
362
480
|
const speech = join(draftDir, "speech.mp3");
|
|
363
481
|
const words = remapWords(edl, transcript.words, film.cut.fillers);
|
|
364
482
|
const frame = getFrame(film);
|
|
365
|
-
const
|
|
483
|
+
const brandFont = chooseFootageDraftFont(options.brand?.fonts.body ?? null);
|
|
484
|
+
const font = brandFont ?? FOOTAGE_DRAFT_FONT;
|
|
485
|
+
const captions = buildFootageDraftCaptions({
|
|
486
|
+
film,
|
|
487
|
+
edl,
|
|
488
|
+
words,
|
|
489
|
+
caption: getFootageCaptionLayout(film, options.layout ?? {}),
|
|
490
|
+
frame,
|
|
491
|
+
fontFamily: font.family,
|
|
492
|
+
...(options.brand === undefined ? {} : { colors: options.brand.colors }),
|
|
493
|
+
});
|
|
366
494
|
if (captions !== null) {
|
|
367
495
|
writeFileSync(join(draftDir, "captions.ass"), captions);
|
|
368
496
|
mkdirSync(join(draftDir, "fonts"));
|
|
369
|
-
copyFileSync(
|
|
497
|
+
copyFileSync(font.file, join(draftDir, "fonts", basename(font.file)));
|
|
498
|
+
if (brandFont === null && options.brand?.fonts.body != null) {
|
|
499
|
+
log(`draft: the brand's body font ${options.brand.fonts.body.family} has no TrueType or OpenType file ffmpeg can read; the captions use ${FOOTAGE_DRAFT_FONT.family}.`);
|
|
500
|
+
}
|
|
370
501
|
}
|
|
371
502
|
const output = resolve(options.output);
|
|
372
503
|
mkdirSync(dirname(output), { recursive: true });
|
|
373
|
-
report("speech",
|
|
374
|
-
await runFfmpeg("The draft's speech track", buildSpeechArgs(edl, transcript.speech, speech, getSpeechCutOptions(getFootageEdit(film))), { ...aborting, output: speech, onSeconds: (done) => report("speech",
|
|
375
|
-
|
|
504
|
+
report("speech", 0);
|
|
505
|
+
await runFfmpeg("The draft's speech track", buildSpeechArgs(edl, transcript.speech, speech, getSpeechCutOptions(getFootageEdit(film))), { ...aborting, output: speech, onSeconds: (done) => report("speech", done) });
|
|
506
|
+
let musicFile: string | null = null;
|
|
507
|
+
if (music.value.kind === "file") {
|
|
508
|
+
musicFile = join(draftDir, "music.mp3");
|
|
509
|
+
const keyframes = duckingKeyframes(words, getDuckingOptions(film.music));
|
|
510
|
+
report("music", 0);
|
|
511
|
+
const args = buildMusicArgs({ input: music.value.path, seconds, keyframes, output: musicFile });
|
|
512
|
+
await runFfmpeg("The draft's music track", args, { ...aborting, output: musicFile, onSeconds: (done) => report("music", done) });
|
|
513
|
+
} else if (music.value.kind === "not_cached") {
|
|
514
|
+
log(`draft: no music, ${music.value.reason}.`);
|
|
515
|
+
}
|
|
516
|
+
report("film", 0);
|
|
376
517
|
// Run in the draft folder: the captions file and the font folder are relative names, so no path needs escaping in the filter graph.
|
|
377
518
|
const args = buildFootageDraftArgs({
|
|
378
519
|
film,
|
|
379
520
|
edl,
|
|
380
521
|
clips: clipFiles,
|
|
381
522
|
speech,
|
|
523
|
+
music: musicFile,
|
|
382
524
|
frame,
|
|
383
525
|
...(options.width === undefined ? {} : { width: options.width }),
|
|
384
526
|
...(options.threads === undefined ? {} : { threads: options.threads }),
|
|
@@ -386,10 +528,12 @@ export async function renderFootageDraft(
|
|
|
386
528
|
fontsDir: captions === null ? null : "fonts",
|
|
387
529
|
output,
|
|
388
530
|
});
|
|
389
|
-
await runFfmpeg("The footage draft", args, { ...aborting, output, cwd: draftDir, onSeconds: (done) => report("film",
|
|
390
|
-
report("film",
|
|
531
|
+
await runFfmpeg("The footage draft", args, { ...aborting, output, cwd: draftDir, onSeconds: (done) => report("film", done) });
|
|
532
|
+
report("film", seconds);
|
|
391
533
|
const draft = getFootageDraftFrame(frame, options.width);
|
|
392
534
|
const bytes = statSync(output).size;
|
|
393
|
-
|
|
394
|
-
|
|
535
|
+
const draftMusic: FootageDraftMusic = music.value.kind === "file" ? "played" : music.value.kind;
|
|
536
|
+
const notes = [captions === null ? "no captions" : `captions in ${font.family}`, draftMusic === "played" ? "music" : draftMusic === "none" ? "no music" : "music not cached"];
|
|
537
|
+
log(`draft: ${output}, ${draft.width}×${draft.height}, ${seconds.toFixed(1)} s, ${bytes} bytes, ${notes.join(", ")}.`);
|
|
538
|
+
return ok({ output, music: draftMusic, captionFont: captions === null ? null : font.family, seconds, width: draft.width, height: draft.height, bytes });
|
|
395
539
|
}
|
package/src/footage/pipeline.ts
CHANGED
|
@@ -477,17 +477,27 @@ async function checkMusicFile(source: Extract<MusicSource, { kind: "file" }>, op
|
|
|
477
477
|
return ok(path);
|
|
478
478
|
}
|
|
479
479
|
|
|
480
|
+
/** Where a film's music comes from, found without spending: none, a local file, or the paid cache's entry (made or not). */
|
|
481
|
+
export type FootageMusicLookup =
|
|
482
|
+
| { kind: "none" }
|
|
483
|
+
| { kind: "file"; path: string }
|
|
484
|
+
| { kind: "cached"; cache: CacheEntry }
|
|
485
|
+
| { kind: "missing"; cache: CacheEntry };
|
|
486
|
+
|
|
480
487
|
/**
|
|
481
|
-
* The music
|
|
482
|
-
*
|
|
483
|
-
*
|
|
488
|
+
* The film's music as the render reads it, without a paid call: a `file` source as it is (an error when the file is
|
|
489
|
+
* gone), a `prompt` or `video` source's cache entry and whether it exists. The render and the draft share it, so they
|
|
490
|
+
* look up the same cache key.
|
|
484
491
|
*/
|
|
485
|
-
async function
|
|
492
|
+
export async function findFootageMusic(
|
|
493
|
+
film: FootageFilm,
|
|
494
|
+
edl: Edl,
|
|
495
|
+
clips: PreparedClip[],
|
|
496
|
+
options: Pick<FootageRunOptions, "cacheDir" | "providers">,
|
|
497
|
+
): Promise<FootageResult<FootageMusicLookup>> {
|
|
486
498
|
const source = film.music.source;
|
|
487
|
-
if (source.kind === "none") return ok(
|
|
488
|
-
if (source.kind === "file") return existsSync(source.path) ? ok(source.path) : err(`music: no file at ${source.path}`);
|
|
489
|
-
if (options.isPlaceholder) return ok(null);
|
|
490
|
-
const { providers, log, signal } = options;
|
|
499
|
+
if (source.kind === "none") return ok({ kind: "none" });
|
|
500
|
+
if (source.kind === "file") return existsSync(source.path) ? ok({ kind: "file", path: source.path }) : err(`music: no file at ${source.path}`);
|
|
491
501
|
let cache: CacheEntry;
|
|
492
502
|
if (source.kind === "prompt") {
|
|
493
503
|
cache = getPromptMusicCache(film, options);
|
|
@@ -496,8 +506,26 @@ async function getMusic(film: FootageFilm, edl: Edl, clips: PreparedClip[], seco
|
|
|
496
506
|
for (const clip of clips) clipKeys[clip.clip.id] = await getClipKey(clip);
|
|
497
507
|
cache = getVideoMusicCache(film, edl, clipKeys, options);
|
|
498
508
|
}
|
|
509
|
+
return ok(existsSync(cache.path) ? { kind: "cached", cache } : { kind: "missing", cache });
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
/**
|
|
513
|
+
* The music file for the cut: a local file as it is, generated music from the cache, or (with `isCommit`) bought
|
|
514
|
+
* from the music provider and cached. Null when the film has none or the rehearsal skips generated music; a paid
|
|
515
|
+
* step when it must be bought and `isCommit` is false.
|
|
516
|
+
*/
|
|
517
|
+
async function getMusic(film: FootageFilm, edl: Edl, clips: PreparedClip[], seconds: number, options: FootageRunOptions): Promise<FootageResult<string | PaidStep | null>> {
|
|
518
|
+
const source = film.music.source;
|
|
519
|
+
if (options.isPlaceholder && (source.kind === "prompt" || source.kind === "video")) return ok(null);
|
|
520
|
+
const found = await findFootageMusic(film, edl, clips, options);
|
|
521
|
+
if (!found.ok) return found;
|
|
522
|
+
const music = found.value;
|
|
523
|
+
if (music.kind === "none") return ok(null);
|
|
524
|
+
if (music.kind === "file") return ok(music.path);
|
|
525
|
+
const { providers, log, signal } = options;
|
|
526
|
+
const { cache } = music;
|
|
499
527
|
const { path } = cache;
|
|
500
|
-
if (
|
|
528
|
+
if (music.kind === "cached") {
|
|
501
529
|
log(`music: from the cache ${path}, nothing spent.`);
|
|
502
530
|
return ok(path);
|
|
503
531
|
}
|
|
@@ -511,13 +539,15 @@ async function getMusic(film: FootageFilm, edl: Edl, clips: PreparedClip[], seco
|
|
|
511
539
|
throwIfAborted(signal);
|
|
512
540
|
log(`music (${providers.music.id}): ${seconds.toFixed(1)} s.`);
|
|
513
541
|
bought = await providers.music.compose({ prompt: source.prompt, seconds, ...aborting });
|
|
514
|
-
} else {
|
|
542
|
+
} else if (source.kind === "video") {
|
|
515
543
|
const preview = join(options.buildDir, "cut-preview.mp4");
|
|
516
544
|
const clipFiles = Object.fromEntries(clips.map((clip) => [clip.clip.id, clip.file]));
|
|
517
545
|
await runFfmpeg("The cut preview for Video to Music", buildCutPreviewArgs(edl, clipFiles, preview), { ...aborting, output: preview });
|
|
518
546
|
throwIfAborted(signal);
|
|
519
547
|
log(`music (${providers.music.id}): ${seconds.toFixed(1)} s.`);
|
|
520
548
|
bought = await providers.music.scoreVideo({ video: preview, description: source.description, tags: source.tags, ...aborting });
|
|
549
|
+
} else {
|
|
550
|
+
return ok(null);
|
|
521
551
|
}
|
|
522
552
|
throwIfAborted(signal);
|
|
523
553
|
if (!bought.ok) return err(`music: ${bought.error}`);
|
package/src/index.ts
CHANGED
|
@@ -339,6 +339,7 @@ export { duckingKeyframes, getDuckingOptions, getSpeechSpans, volumeExpression,
|
|
|
339
339
|
export { END_CARD_SECONDS, FADE_SECONDS, composeFootage, getFootageFilmSeconds, getSegmentScales, type FootageComposeAssets, type FootageComposeInput } from "./footage/compose.js";
|
|
340
340
|
export { buildCutPreviewArgs, buildLastFrameArgs, buildMusicArgs, buildSpeechArgs, getSpeechCutOptions, type SpeechCutOptions } from "./footage/render.js";
|
|
341
341
|
export {
|
|
342
|
+
FOOTAGE_DRAFT_CAPTION_COLORS,
|
|
342
343
|
FOOTAGE_DRAFT_FONT,
|
|
343
344
|
FOOTAGE_DRAFT_WIDTH,
|
|
344
345
|
buildFootageDraftArgs,
|
|
@@ -346,12 +347,16 @@ export {
|
|
|
346
347
|
getFootageDraftFrame,
|
|
347
348
|
planFootageDraft,
|
|
348
349
|
renderFootageDraft,
|
|
350
|
+
toAssColor,
|
|
349
351
|
type DraftSegmentPlan,
|
|
350
352
|
type FootageDraft,
|
|
351
353
|
type FootageDraftArgsInput,
|
|
354
|
+
type FootageDraftCaptionColors,
|
|
352
355
|
type FootageDraftCaptionsInput,
|
|
356
|
+
type FootageDraftMusic,
|
|
353
357
|
type RenderFootageDraftOptions,
|
|
354
358
|
} from "./footage/draft.js";
|
|
359
|
+
export { chooseFootageDraftFont, readFontFamilyName, type FootageDraftFontChoice } from "./footage/draft-font.js";
|
|
355
360
|
export {
|
|
356
361
|
buildPictureConcatArgs,
|
|
357
362
|
buildPictureList,
|
|
@@ -408,6 +413,7 @@ export {
|
|
|
408
413
|
cutFootage,
|
|
409
414
|
cutFootageFilm,
|
|
410
415
|
describeCutMeasures,
|
|
416
|
+
findFootageMusic,
|
|
411
417
|
getEdlPath,
|
|
412
418
|
getFrame,
|
|
413
419
|
prepareFootage,
|
|
@@ -419,6 +425,7 @@ export {
|
|
|
419
425
|
transcribeFootage,
|
|
420
426
|
type FinishedFootage,
|
|
421
427
|
type FootageBrand,
|
|
428
|
+
type FootageMusicLookup,
|
|
422
429
|
type FootageResult,
|
|
423
430
|
type FootageRunOptions,
|
|
424
431
|
type PreparedClip,
|