@softure-ai/marketing-kit 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +43 -0
- package/README.md +66 -9
- package/dist/config/config.d.ts.map +1 -1
- package/dist/config/config.js +7 -0
- package/dist/config/config.js.map +1 -1
- package/dist/config/schema.d.ts +1 -0
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +11 -0
- package/dist/config/schema.js.map +1 -1
- package/dist/footage/audio.d.ts +6 -2
- package/dist/footage/audio.d.ts.map +1 -1
- package/dist/footage/audio.js +15 -3
- package/dist/footage/audio.js.map +1 -1
- package/dist/footage/captions.d.ts +21 -2
- package/dist/footage/captions.d.ts.map +1 -1
- package/dist/footage/captions.js +29 -0
- package/dist/footage/captions.js.map +1 -1
- package/dist/footage/compose.d.ts.map +1 -1
- package/dist/footage/compose.js +4 -5
- package/dist/footage/compose.js.map +1 -1
- package/dist/footage/disk.d.ts +25 -0
- package/dist/footage/disk.d.ts.map +1 -0
- package/dist/footage/disk.js +72 -0
- package/dist/footage/disk.js.map +1 -0
- package/dist/footage/draft.d.ts +105 -0
- package/dist/footage/draft.d.ts.map +1 -0
- package/dist/footage/draft.js +282 -0
- package/dist/footage/draft.js.map +1 -0
- package/dist/footage/elevenlabs.d.ts.map +1 -1
- package/dist/footage/elevenlabs.js +2 -0
- package/dist/footage/elevenlabs.js.map +1 -1
- package/dist/footage/film.d.ts +12 -0
- package/dist/footage/film.d.ts.map +1 -1
- package/dist/footage/film.js.map +1 -1
- package/dist/footage/keyterms.d.ts +26 -0
- package/dist/footage/keyterms.d.ts.map +1 -0
- package/dist/footage/keyterms.js +36 -0
- package/dist/footage/keyterms.js.map +1 -0
- package/dist/footage/normalize.d.ts +21 -2
- package/dist/footage/normalize.d.ts.map +1 -1
- package/dist/footage/normalize.js +35 -11
- package/dist/footage/normalize.js.map +1 -1
- package/dist/footage/paid.d.ts +2 -1
- package/dist/footage/paid.d.ts.map +1 -1
- package/dist/footage/paid.js +13 -3
- package/dist/footage/paid.js.map +1 -1
- package/dist/footage/pipeline.d.ts +20 -3
- package/dist/footage/pipeline.d.ts.map +1 -1
- package/dist/footage/pipeline.js +44 -15
- package/dist/footage/pipeline.js.map +1 -1
- package/dist/footage/process.d.ts +2 -0
- package/dist/footage/process.d.ts.map +1 -1
- package/dist/footage/process.js +1 -0
- package/dist/footage/process.js.map +1 -1
- package/dist/footage/providers.d.ts +12 -3
- package/dist/footage/providers.d.ts.map +1 -1
- package/dist/footage/providers.js.map +1 -1
- package/dist/footage/render.d.ts +1 -1
- package/dist/footage/render.d.ts.map +1 -1
- package/dist/footage/render.js +5 -3
- package/dist/footage/render.js.map +1 -1
- package/dist/footage/tools.d.ts +15 -3
- package/dist/footage/tools.d.ts.map +1 -1
- package/dist/footage/tools.js +15 -2
- package/dist/footage/tools.js.map +1 -1
- package/dist/index.d.ts +6 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -3
- package/dist/index.js.map +1 -1
- package/docker/footage-worker.Dockerfile +6 -0
- package/fonts/DejaVuSans-Bold-Latin.ttf +0 -0
- package/fonts/LICENSE-DejaVu.txt +46 -0
- package/fonts/README.md +12 -0
- package/package.json +2 -1
- package/schema/marketing.schema.json +7 -0
- package/src/config/config.ts +8 -0
- package/src/config/schema.ts +11 -0
- package/src/footage/audio.ts +16 -3
- package/src/footage/captions.ts +39 -2
- package/src/footage/compose.ts +4 -5
- package/src/footage/disk.ts +75 -0
- package/src/footage/draft.ts +366 -0
- package/src/footage/elevenlabs.ts +1 -0
- package/src/footage/film.ts +16 -1
- package/src/footage/keyterms.ts +38 -0
- package/src/footage/normalize.ts +47 -10
- package/src/footage/paid.ts +12 -3
- package/src/footage/pipeline.ts +55 -17
- package/src/footage/process.ts +3 -0
- package/src/footage/providers.ts +10 -3
- package/src/footage/render.ts +5 -3
- package/src/footage/tools.ts +23 -3
- package/src/index.ts +34 -2
package/src/footage/captions.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { readFileSync } from "node:fs";
|
|
2
2
|
|
|
3
|
-
import type
|
|
3
|
+
import { resolveLayout, type CaptionLayout, type LayoutOverride } from "../compose/timeline.js";
|
|
4
4
|
import type { BrandFont } from "../config/config.js";
|
|
5
5
|
import { readCharacterMap, type CharacterMap } from "../og/character-map.js";
|
|
6
6
|
import type { TimedWord } from "../voice/voiceover.js";
|
|
7
|
-
import type { CaptionPreset } from "./film.js";
|
|
7
|
+
import type { CaptionPreset, FootageFilm } from "./film.js";
|
|
8
8
|
|
|
9
9
|
/**
|
|
10
10
|
* The footage film's captions: plain phrases (the default style), the 9:16 caption box inside a platform's safe zone,
|
|
@@ -55,6 +55,32 @@ export function phraseChunks(words: TimedWord[]): PhraseChunk[] {
|
|
|
55
55
|
return merged.map((group) => ({ words: group, start: (group[0] as TimedWord).start, end: (group[group.length - 1] as TimedWord).end }));
|
|
56
56
|
}
|
|
57
57
|
|
|
58
|
+
/** When a caption chunk is on screen, in film seconds. */
|
|
59
|
+
export interface CaptionSpan {
|
|
60
|
+
start: number;
|
|
61
|
+
end: number;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
const CAPTION_LEAD_SECONDS = 0.05;
|
|
65
|
+
const CAPTION_HOLD_SECONDS = 0.35;
|
|
66
|
+
const CAPTION_GAP_SECONDS = 0.06;
|
|
67
|
+
const MIN_CAPTION_SECONDS = 0.1;
|
|
68
|
+
|
|
69
|
+
const r3 = (value: number): number => Math.round(value * 1000) / 1000 || 0;
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* The span each caption chunk is shown for: from just before its first word to a moment after its last, never into
|
|
73
|
+
* the next chunk, the last one never past the cut's end; at least 0.1 s. The render and the draft time captions alike.
|
|
74
|
+
*/
|
|
75
|
+
export function getCaptionSpans(chunks: readonly { start: number; end: number }[], cutEnd: number): CaptionSpan[] {
|
|
76
|
+
return chunks.map((chunk, index) => {
|
|
77
|
+
const next = chunks[index + 1];
|
|
78
|
+
const start = r3(chunk.start - CAPTION_LEAD_SECONDS);
|
|
79
|
+
const stop = Math.min(chunk.end + CAPTION_HOLD_SECONDS, next === undefined ? cutEnd : next.start - CAPTION_GAP_SECONDS);
|
|
80
|
+
return { start, end: r3(start + r3(Math.max(MIN_CAPTION_SECONDS, stop - start))) };
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
|
|
58
84
|
/** Margins of a 1080×1920 frame that a platform's own buttons and text cover. */
|
|
59
85
|
export interface SafeZone {
|
|
60
86
|
top: number;
|
|
@@ -87,6 +113,17 @@ export function getCaptionPresetLayout(preset: CaptionPreset, fontSize: number):
|
|
|
87
113
|
return { top, left: side, right: side, fontSize };
|
|
88
114
|
}
|
|
89
115
|
|
|
116
|
+
/**
|
|
117
|
+
* The caption box a footage film's captions are drawn in: the format's layout, with a 9:16 film's box placed by its
|
|
118
|
+
* caption preset unless the project overrides the caption box (`marketing.json` `layout`). The render and the draft
|
|
119
|
+
* share it.
|
|
120
|
+
*/
|
|
121
|
+
export function getFootageCaptionLayout(film: Pick<FootageFilm, "format" | "captionPreset">, override: LayoutOverride = {}): CaptionLayout {
|
|
122
|
+
const base = resolveLayout(film.format, override);
|
|
123
|
+
if (film.captionPreset === undefined || film.format !== "9:16" || override.caption !== undefined) return base.caption;
|
|
124
|
+
return getCaptionPresetLayout(film.captionPreset, base.caption.fontSize);
|
|
125
|
+
}
|
|
126
|
+
|
|
90
127
|
/** A character a font is asked to draw and does not have. */
|
|
91
128
|
export interface MissingCaptionGlyphs {
|
|
92
129
|
/** The family, as `brand.fonts` names it. */
|
package/src/footage/compose.ts
CHANGED
|
@@ -4,7 +4,7 @@ import type { ComposeFont } from "../compose/compose.js";
|
|
|
4
4
|
import { escapeHtml, fontFaces, fontStack } from "../compose/html.js";
|
|
5
5
|
import { captionChunks, type CaptionLayout } from "../compose/timeline.js";
|
|
6
6
|
import type { TimedWord } from "../voice/voiceover.js";
|
|
7
|
-
import { phraseChunks } from "./captions.js";
|
|
7
|
+
import { getCaptionSpans, phraseChunks, type CaptionSpan } from "./captions.js";
|
|
8
8
|
import { getEdlDuration, getSegmentStarts, type Edl } from "./edl.js";
|
|
9
9
|
import { DEFAULT_EDIT, getFootageEdit, type FootageFilm } from "./film.js";
|
|
10
10
|
import { FOOTAGE_FPS } from "./normalize.js";
|
|
@@ -110,15 +110,14 @@ export function composeFootage(input: FootageComposeInput): string {
|
|
|
110
110
|
// --- Captions: plain phrases, or the pill that lights each word ---
|
|
111
111
|
const isHighlight = film.captionStyle === "highlight";
|
|
112
112
|
const caption = !film.captions ? [] : isHighlight ? captionChunks(words) : phraseChunks(words);
|
|
113
|
+
const spans = getCaptionSpans(caption, cutEnd);
|
|
113
114
|
const captionHtml = caption
|
|
114
115
|
.map((chunk, i) => {
|
|
115
|
-
const
|
|
116
|
-
const start = r3(chunk.start - 0.05);
|
|
117
|
-
const stop = Math.min(chunk.end + 0.35, next === undefined ? cutEnd : next.start - 0.06);
|
|
116
|
+
const { start, end: stop } = spans[i] as CaptionSpan;
|
|
118
117
|
const box = isHighlight
|
|
119
118
|
? `<p class="pill">${chunk.words.map((w, j) => `<span id="cw${i}_${j}">${escapeHtml(w.text)}</span>`).join(" ")}</p>`
|
|
120
119
|
: `<p class="phrase">${escapeHtml(chunk.words.map((w) => w.text).join(" "))}</p>`;
|
|
121
|
-
return `<div id="cap${i}" class="clip caption" data-start="${start}" data-duration="${r3(
|
|
120
|
+
return `<div id="cap${i}" class="clip caption" data-start="${start}" data-duration="${r3(stop - start)}">${box}</div>`;
|
|
122
121
|
})
|
|
123
122
|
.join("\n ");
|
|
124
123
|
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { existsSync, statfsSync } from "node:fs";
|
|
2
|
+
import { tmpdir } from "node:os";
|
|
3
|
+
import { dirname, resolve } from "node:path";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* The free space a hyperframes render needs. hyperframes 0.8.85 refuses to render (exit code 1, "Low disk space")
|
|
7
|
+
* when its temp folder (`os.tmpdir()`, where it also extracts the clips' frames) or its output's folder has under
|
|
8
|
+
* 1 GiB free, measured as `statfs` `bsize * bavail`. The kit measures the same way, so it never refuses a render
|
|
9
|
+
* hyperframes would run.
|
|
10
|
+
*/
|
|
11
|
+
export const RENDER_MIN_FREE_BYTES = 1024 ** 3;
|
|
12
|
+
|
|
13
|
+
/** Reads a folder's free bytes; null when it cannot be read. Injected in tests. */
|
|
14
|
+
export type FreeBytesReader = (folder: string) => number | null;
|
|
15
|
+
|
|
16
|
+
/** The nearest existing folder of a path: a folder the run creates later is measured on the disk it will live on. */
|
|
17
|
+
function findExistingFolder(path: string): string {
|
|
18
|
+
let current = resolve(path);
|
|
19
|
+
while (!existsSync(current)) {
|
|
20
|
+
const parent = dirname(current);
|
|
21
|
+
if (parent === current) return current;
|
|
22
|
+
current = parent;
|
|
23
|
+
}
|
|
24
|
+
return current;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** Free bytes for an unprivileged writer at `folder` (or its nearest existing parent); null when `statfs` fails. */
|
|
28
|
+
export function getFreeBytes(folder: string): number | null {
|
|
29
|
+
try {
|
|
30
|
+
const stats = statfsSync(findExistingFolder(folder));
|
|
31
|
+
return stats.bsize * stats.bavail;
|
|
32
|
+
} catch {
|
|
33
|
+
// An unreadable disk is not reported: hyperframes passes it as "Unable to check" too.
|
|
34
|
+
return null;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* The temp folder the hyperframes process will use: `tempDir` when given (the kit passes it as `TMPDIR`), else the
|
|
40
|
+
* `TMPDIR`, `TMP` or `TEMP` of the environment it runs with, else this process's `os.tmpdir()`.
|
|
41
|
+
*/
|
|
42
|
+
export function getRenderTempDir(env: NodeJS.ProcessEnv, tempDir?: string): string {
|
|
43
|
+
if (tempDir !== undefined && tempDir.length > 0) return resolve(tempDir);
|
|
44
|
+
for (const name of ["TMPDIR", "TMP", "TEMP"]) {
|
|
45
|
+
const value = env[name];
|
|
46
|
+
if (value !== undefined && value.length > 0) return resolve(value);
|
|
47
|
+
}
|
|
48
|
+
return tmpdir();
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const toGiB = (bytes: number): string => (bytes / 1024 ** 3).toFixed(1);
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* One sentence per folder with less free space than a render needs, naming the folder and how to fix it. Folders
|
|
55
|
+
* are checked once each; an unreadable one is skipped.
|
|
56
|
+
*/
|
|
57
|
+
export function findLowDiskProblems(folders: { temp: string; build?: string }, readFreeBytes: FreeBytesReader = getFreeBytes): string[] {
|
|
58
|
+
const entries: { folder: string; role: string; hint: string }[] = [
|
|
59
|
+
{
|
|
60
|
+
folder: resolve(folders.temp),
|
|
61
|
+
role: "the temp folder",
|
|
62
|
+
hint: "hyperframes extracts the clips' frames there (hundreds of MB per minute of film): point TMPDIR (or renderFootage's tempDir) at a data volume, not a small tmpfs",
|
|
63
|
+
},
|
|
64
|
+
];
|
|
65
|
+
if (folders.build !== undefined && resolve(folders.build) !== entries[0]?.folder) {
|
|
66
|
+
entries.push({ folder: resolve(folders.build), role: "the build folder", hint: "hyperframes writes the render there: free space or move buildDir to a larger disk" });
|
|
67
|
+
}
|
|
68
|
+
const problems: string[] = [];
|
|
69
|
+
for (const entry of entries) {
|
|
70
|
+
const free = readFreeBytes(entry.folder);
|
|
71
|
+
if (free === null || free >= RENDER_MIN_FREE_BYTES) continue;
|
|
72
|
+
problems.push(`${entry.role} ${entry.folder} has ${toGiB(free)} GiB free and a hyperframes render needs ${toGiB(RENDER_MIN_FREE_BYTES)} GiB; ${entry.hint}.`);
|
|
73
|
+
}
|
|
74
|
+
return problems;
|
|
75
|
+
}
|
|
@@ -0,0 +1,366 @@
|
|
|
1
|
+
import { copyFileSync, mkdirSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { basename, dirname, join, resolve } from "node:path";
|
|
3
|
+
import { fileURLToPath } from "node:url";
|
|
4
|
+
|
|
5
|
+
import type { CaptionLayout, LayoutOverride } from "../compose/timeline.js";
|
|
6
|
+
import { ok } from "../voice/provider.js";
|
|
7
|
+
import type { TimedWord } from "../voice/voiceover.js";
|
|
8
|
+
import { getCaptionSpans, getFootageCaptionLayout, phraseChunks } from "./captions.js";
|
|
9
|
+
import { getSegmentScales } from "./compose.js";
|
|
10
|
+
import { getEdlDuration, getSegmentStarts, remapWords, type Edl } from "./edl.js";
|
|
11
|
+
import { getFootageEdit, type FootageFilm } from "./film.js";
|
|
12
|
+
import { FOOTAGE_FPS, type Frame } from "./normalize.js";
|
|
13
|
+
import { getFrame, type FootageResult, type FootageRunOptions, type PreparedClip, type TranscribeOutcome } from "./pipeline.js";
|
|
14
|
+
import { runFfmpeg } from "./process.js";
|
|
15
|
+
import { buildSpeechArgs } from "./render.js";
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* A fast draft of a footage film, ffmpeg only (no hyperframes, no browser): the cut at a small width with the film's
|
|
19
|
+
* phrase captions burned in, its zoom cuts, cross-fades and Ken Burns b-roll drawn as the full render draws them, and
|
|
20
|
+
* the speech track. No loudness pass, music, end card or cover. What an app shows the owner before the paid render.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
/** The draft's default width: 540 makes a 540×960 draft of a 9:16 film. */
|
|
24
|
+
export const FOOTAGE_DRAFT_WIDTH = 540;
|
|
25
|
+
|
|
26
|
+
/** The font the draft's captions are drawn with: DejaVu Sans Bold, subset to Latin with every Polish letter, shipped in `fonts/`. */
|
|
27
|
+
export const FOOTAGE_DRAFT_FONT = {
|
|
28
|
+
family: "DejaVu Sans",
|
|
29
|
+
file: fileURLToPath(new URL("../../fonts/DejaVuSans-Bold-Latin.ttf", import.meta.url)),
|
|
30
|
+
} as const;
|
|
31
|
+
|
|
32
|
+
/** How far a segment marked `kenburns` zooms when the film's `kenBurnsPercent` is 0, as in the render. */
|
|
33
|
+
const HAND_KEN_BURNS_PERCENT = 4;
|
|
34
|
+
/** The point a zoom keeps still, as the render's `transform-origin: 50% 42%`. */
|
|
35
|
+
const ZOOM_ORIGIN = { x: 0.5, y: 0.42 } as const;
|
|
36
|
+
const DRAFT_CRF = 28;
|
|
37
|
+
const DRAFT_AUDIO_BITRATE = "128k";
|
|
38
|
+
|
|
39
|
+
const r3 = (value: number): number => Math.round(value * 1000) / 1000 || 0;
|
|
40
|
+
const r4 = (value: number): number => Math.round(value * 10000) / 10000 || 0;
|
|
41
|
+
|
|
42
|
+
/** The draft's frame: the film's frame scaled to `width`, both sides even (H.264 yuv420p needs them even). */
|
|
43
|
+
export function getFootageDraftFrame(frame: Frame, width: number = FOOTAGE_DRAFT_WIDTH): Frame {
|
|
44
|
+
const even = (value: number): number => Math.max(2, Math.round(value / 2) * 2);
|
|
45
|
+
return { width: even(width), height: even((width * frame.height) / frame.width) };
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** An ASS time: `H:MM:SS.cc`. */
|
|
49
|
+
function toAssTime(seconds: number): string {
|
|
50
|
+
const centiseconds = Math.max(0, Math.round(seconds * 100));
|
|
51
|
+
const hours = Math.floor(centiseconds / 360000);
|
|
52
|
+
const minutes = Math.floor((centiseconds % 360000) / 6000);
|
|
53
|
+
const secs = Math.floor((centiseconds % 6000) / 100);
|
|
54
|
+
const pad = (value: number): string => String(value).padStart(2, "0");
|
|
55
|
+
return `${hours}:${pad(minutes)}:${pad(secs)}.${pad(centiseconds % 100)}`;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Caption text as ASS dialogue: override braces and backslashes would be read as tags, line breaks end the line. */
|
|
59
|
+
function toAssText(text: string): string {
|
|
60
|
+
return text.replace(/\\/g, "/").replace(/\{/g, "(").replace(/\}/g, ")").replace(/\s+/g, " ").trim();
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export interface FootageDraftCaptionsInput {
|
|
64
|
+
film: Pick<FootageFilm, "captions">;
|
|
65
|
+
edl: Edl;
|
|
66
|
+
/** The film's words in film time (`remapWords`). */
|
|
67
|
+
words: TimedWord[];
|
|
68
|
+
/** The caption box in the film's frame (`getFootageCaptionLayout`). */
|
|
69
|
+
caption: CaptionLayout;
|
|
70
|
+
/** The film's full frame; the ASS file is laid out in it and libass scales it to the draft. */
|
|
71
|
+
frame: Frame;
|
|
72
|
+
/** The font family the captions are drawn in; default `FOOTAGE_DRAFT_FONT.family`. */
|
|
73
|
+
fontFamily?: string;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* The draft's captions as an ASS subtitle file: the film's phrases (`phraseChunks`) shown for the render's spans
|
|
78
|
+
* (`getCaptionSpans`), white bold text with a dark outline and shadow, centred in the caption box from its top. Null
|
|
79
|
+
* when the film has no captions or nothing is said.
|
|
80
|
+
*/
|
|
81
|
+
export function buildFootageDraftCaptions(input: FootageDraftCaptionsInput): string | null {
|
|
82
|
+
if (!input.film.captions) return null;
|
|
83
|
+
const chunks = phraseChunks(input.words);
|
|
84
|
+
if (chunks.length === 0) return null;
|
|
85
|
+
const spans = getCaptionSpans(chunks, getEdlDuration(input.edl));
|
|
86
|
+
const { caption, frame } = input;
|
|
87
|
+
const family = (input.fontFamily ?? FOOTAGE_DRAFT_FONT.family).replace(/,/g, " ");
|
|
88
|
+
const outline = Math.max(1, Math.round(caption.fontSize / 16));
|
|
89
|
+
const shadow = Math.max(1, Math.round(caption.fontSize / 25));
|
|
90
|
+
const style = [
|
|
91
|
+
"Caption",
|
|
92
|
+
family,
|
|
93
|
+
caption.fontSize,
|
|
94
|
+
"&H00FFFFFF",
|
|
95
|
+
"&H00FFFFFF",
|
|
96
|
+
"&H8C000000",
|
|
97
|
+
"&H8C000000",
|
|
98
|
+
-1,
|
|
99
|
+
0,
|
|
100
|
+
0,
|
|
101
|
+
0,
|
|
102
|
+
100,
|
|
103
|
+
100,
|
|
104
|
+
0,
|
|
105
|
+
0,
|
|
106
|
+
1,
|
|
107
|
+
outline,
|
|
108
|
+
shadow,
|
|
109
|
+
8,
|
|
110
|
+
Math.round(caption.left),
|
|
111
|
+
Math.round(caption.right),
|
|
112
|
+
Math.round(caption.top),
|
|
113
|
+
1,
|
|
114
|
+
].join(",");
|
|
115
|
+
const events = chunks.map((chunk, index) => {
|
|
116
|
+
const span = spans[index] as { start: number; end: number };
|
|
117
|
+
const text = toAssText(chunk.words.map((word) => word.text).join(" "));
|
|
118
|
+
return `Dialogue: 0,${toAssTime(span.start)},${toAssTime(span.end)},Caption,,0,0,0,,${text}`;
|
|
119
|
+
});
|
|
120
|
+
return [
|
|
121
|
+
"[Script Info]",
|
|
122
|
+
"ScriptType: v4.00+",
|
|
123
|
+
`PlayResX: ${frame.width}`,
|
|
124
|
+
`PlayResY: ${frame.height}`,
|
|
125
|
+
"WrapStyle: 0",
|
|
126
|
+
"ScaledBorderAndShadow: yes",
|
|
127
|
+
"",
|
|
128
|
+
"[V4+ Styles]",
|
|
129
|
+
"Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding",
|
|
130
|
+
`Style: ${style}`,
|
|
131
|
+
"",
|
|
132
|
+
"[Events]",
|
|
133
|
+
"Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text",
|
|
134
|
+
...events,
|
|
135
|
+
"",
|
|
136
|
+
].join("\n");
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
export interface FootageDraftArgsInput {
|
|
140
|
+
film: Pick<FootageFilm, "edit">;
|
|
141
|
+
edl: Edl;
|
|
142
|
+
/** The normalized clips by id. */
|
|
143
|
+
clips: Record<string, string>;
|
|
144
|
+
/** The cut speech track (`buildSpeechArgs`). */
|
|
145
|
+
speech: string;
|
|
146
|
+
/** The film's full frame (the normalized clips' size). */
|
|
147
|
+
frame: Frame;
|
|
148
|
+
/** The draft's width; default 540. */
|
|
149
|
+
width?: number;
|
|
150
|
+
/** The ASS captions file (`buildFootageDraftCaptions`), or null for none. */
|
|
151
|
+
captions: string | null;
|
|
152
|
+
/** The folder libass loads the captions' font from, or null for the system's fonts. */
|
|
153
|
+
fontsDir: string | null;
|
|
154
|
+
output: string;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** A segment's picture as the draft plays it: how long its stream runs and the fade it starts with. */
|
|
158
|
+
export interface DraftSegmentPlan {
|
|
159
|
+
/** The segment's own length in the cut. */
|
|
160
|
+
length: number;
|
|
161
|
+
/** Seconds the segment's stream runs: its length plus the fade into the next segment, which plays over its end. */
|
|
162
|
+
seconds: number;
|
|
163
|
+
/** The film second the segment starts at. */
|
|
164
|
+
start: number;
|
|
165
|
+
/** The cross-fade from the segment before, in seconds; 0 for a plain cut. */
|
|
166
|
+
fade: number;
|
|
167
|
+
/** The zoom the segment starts at (`getSegmentScales`). */
|
|
168
|
+
scale: number;
|
|
169
|
+
/** The zoom it ends at: above `scale` for a `kenburns` segment, `scale` otherwise. */
|
|
170
|
+
endScale: number;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* How each segment plays in the draft. A `fade` cut lasts the film's `fadeFrames` (clamped to the incoming segment, so
|
|
175
|
+
* the cross-fade never outlasts it) and the outgoing segment runs on under it, as in the render.
|
|
176
|
+
*/
|
|
177
|
+
export function planFootageDraft(film: Pick<FootageFilm, "edit">, edl: Edl): DraftSegmentPlan[] {
|
|
178
|
+
const edit = getFootageEdit(film);
|
|
179
|
+
const fadeSeconds = r3(edit.fadeFrames / FOOTAGE_FPS);
|
|
180
|
+
const kenBurnsScale = 1 + (edit.kenBurnsPercent > 0 ? edit.kenBurnsPercent : HAND_KEN_BURNS_PERCENT) / 100;
|
|
181
|
+
const scales = getSegmentScales(edl, edit.zoomPercent);
|
|
182
|
+
const starts = getSegmentStarts(edl);
|
|
183
|
+
const fades = edl.segments.map((segment, index) => (segment.transition === "fade" && index > 0 ? r3(Math.min(fadeSeconds, segment.end - segment.start)) : 0));
|
|
184
|
+
return edl.segments.map((segment, index) => {
|
|
185
|
+
const scale = scales[index] as number;
|
|
186
|
+
return {
|
|
187
|
+
length: r3(segment.end - segment.start),
|
|
188
|
+
seconds: r3(segment.end - segment.start + (fades[index + 1] ?? 0)),
|
|
189
|
+
start: starts[index] as number,
|
|
190
|
+
fade: fades[index] as number,
|
|
191
|
+
scale,
|
|
192
|
+
endScale: segment.motion === "kenburns" ? r4(scale * kenBurnsScale) : scale,
|
|
193
|
+
};
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** The filters that zoom a segment's picture and scale it to the draft: a crop for a still zoom, `zoompan` for Ken Burns. */
|
|
198
|
+
function buildPictureFilters(plan: DraftSegmentPlan, draft: Frame): string[] {
|
|
199
|
+
const size = `${draft.width}:${draft.height}`;
|
|
200
|
+
if (plan.endScale !== plan.scale) {
|
|
201
|
+
// The zoom runs over the segment's own length, as the render's tween does, and holds through a fade's tail.
|
|
202
|
+
const frames = Math.max(1, Math.round(plan.length * FOOTAGE_FPS));
|
|
203
|
+
const zoom = `${plan.scale}+${r4(plan.endScale - plan.scale)}*min(on/${frames},1)`;
|
|
204
|
+
return [
|
|
205
|
+
`zoompan=z='${zoom}':x='(iw-iw/zoom)*${ZOOM_ORIGIN.x}':y='(ih-ih/zoom)*${ZOOM_ORIGIN.y}':d=1:s=${draft.width}x${draft.height}:fps=${FOOTAGE_FPS}`,
|
|
206
|
+
];
|
|
207
|
+
}
|
|
208
|
+
if (plan.scale !== 1) {
|
|
209
|
+
return [`crop=iw/${plan.scale}:ih/${plan.scale}:(iw-iw/${plan.scale})*${ZOOM_ORIGIN.x}:(ih-ih/${plan.scale})*${ZOOM_ORIGIN.y}`, `scale=${size}`];
|
|
210
|
+
}
|
|
211
|
+
return [`scale=${size}`];
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* The ffmpeg arguments of the draft: each segment read from its own seek into its clip, zoomed and scaled to the
|
|
216
|
+
* draft's width, joined by `xfade` at `fade` cuts and `concat` at the others, the captions burned in (`ass`), the
|
|
217
|
+
* speech track under it; H.264 yuv420p and AAC with fast start, exactly the cut's length.
|
|
218
|
+
*/
|
|
219
|
+
export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
|
|
220
|
+
const { edl } = input;
|
|
221
|
+
const draft = getFootageDraftFrame(input.frame, input.width);
|
|
222
|
+
const plans = planFootageDraft(input.film, edl);
|
|
223
|
+
const inputs: string[] = [];
|
|
224
|
+
const parts: string[] = [];
|
|
225
|
+
edl.segments.forEach((segment, index) => {
|
|
226
|
+
const clip = input.clips[segment.clip];
|
|
227
|
+
if (clip === undefined) throw new Error(`No file for clip "${segment.clip}" of the EDL.`);
|
|
228
|
+
const plan = plans[index] as DraftSegmentPlan;
|
|
229
|
+
inputs.push("-ss", String(r3(segment.start)), "-t", String(plan.seconds), "-i", clip);
|
|
230
|
+
const filters = [
|
|
231
|
+
"setpts=PTS-STARTPTS",
|
|
232
|
+
...buildPictureFilters(plan, draft),
|
|
233
|
+
"setsar=1",
|
|
234
|
+
`fps=${FOOTAGE_FPS}`,
|
|
235
|
+
"format=yuv420p",
|
|
236
|
+
// A clip that ends before the segment's end (and its fade's tail) holds its last frame, so every stream runs its full length.
|
|
237
|
+
`tpad=stop_mode=clone:stop_duration=${plan.seconds}`,
|
|
238
|
+
`trim=duration=${plan.seconds}`,
|
|
239
|
+
"setpts=PTS-STARTPTS",
|
|
240
|
+
`settb=1/${FOOTAGE_FPS}`,
|
|
241
|
+
];
|
|
242
|
+
parts.push(`[${index}:v]${filters.join(",")}[v${index}]`);
|
|
243
|
+
});
|
|
244
|
+
let joined = "v0";
|
|
245
|
+
plans.forEach((plan, index) => {
|
|
246
|
+
if (index === 0) return;
|
|
247
|
+
const label = `j${index}`;
|
|
248
|
+
parts.push(
|
|
249
|
+
plan.fade > 0
|
|
250
|
+
? `[${joined}][v${index}]xfade=transition=fade:duration=${plan.fade}:offset=${plan.start}[${label}]`
|
|
251
|
+
: // concat changes the time base; xfade needs both of its inputs on the segments' own.
|
|
252
|
+
`[${joined}][v${index}]concat=n=2:v=1:a=0,settb=1/${FOOTAGE_FPS}[${label}]`,
|
|
253
|
+
);
|
|
254
|
+
joined = label;
|
|
255
|
+
});
|
|
256
|
+
if (input.captions !== null) {
|
|
257
|
+
parts.push(`[${joined}]ass=filename=${input.captions}${input.fontsDir === null ? "" : `:fontsdir=${input.fontsDir}`}[draft]`);
|
|
258
|
+
} else {
|
|
259
|
+
parts.push(`[${joined}]null[draft]`);
|
|
260
|
+
}
|
|
261
|
+
const speechInput = edl.segments.length;
|
|
262
|
+
const seconds = getEdlDuration(edl);
|
|
263
|
+
return [
|
|
264
|
+
"-v",
|
|
265
|
+
"error",
|
|
266
|
+
"-y",
|
|
267
|
+
...inputs,
|
|
268
|
+
"-i",
|
|
269
|
+
input.speech,
|
|
270
|
+
"-filter_complex",
|
|
271
|
+
parts.join(";"),
|
|
272
|
+
"-map",
|
|
273
|
+
"[draft]",
|
|
274
|
+
"-map",
|
|
275
|
+
`${speechInput}:a`,
|
|
276
|
+
"-t",
|
|
277
|
+
String(seconds),
|
|
278
|
+
"-c:v",
|
|
279
|
+
"libx264",
|
|
280
|
+
"-preset",
|
|
281
|
+
"veryfast",
|
|
282
|
+
"-crf",
|
|
283
|
+
String(DRAFT_CRF),
|
|
284
|
+
"-pix_fmt",
|
|
285
|
+
"yuv420p",
|
|
286
|
+
"-c:a",
|
|
287
|
+
"aac",
|
|
288
|
+
"-b:a",
|
|
289
|
+
DRAFT_AUDIO_BITRATE,
|
|
290
|
+
"-movflags",
|
|
291
|
+
"+faststart",
|
|
292
|
+
input.output,
|
|
293
|
+
];
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
export interface RenderFootageDraftOptions extends Pick<FootageRunOptions, "buildDir" | "log" | "signal" | "onProgress"> {
|
|
297
|
+
/** The draft MP4. */
|
|
298
|
+
output: string;
|
|
299
|
+
/** The draft's width; default 540 (540×960 for a 9:16 film). */
|
|
300
|
+
width?: number;
|
|
301
|
+
/** A project's override of the format's caption box (`marketing.json` `layout`), as the render takes it. */
|
|
302
|
+
layout?: LayoutOverride;
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
export interface FootageDraft {
|
|
306
|
+
output: string;
|
|
307
|
+
/** The draft's length: the cut's. */
|
|
308
|
+
seconds: number;
|
|
309
|
+
width: number;
|
|
310
|
+
height: number;
|
|
311
|
+
bytes: number;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
/**
|
|
315
|
+
* Makes the draft of a cut film: the speech track, the captions file and the font in `<buildDir>/draft/`, then one
|
|
316
|
+
* ffmpeg run (`buildFootageDraftArgs`). Spends nothing; takes the same clips, transcript and EDL as `renderFootage`.
|
|
317
|
+
*/
|
|
318
|
+
export async function renderFootageDraft(
|
|
319
|
+
film: FootageFilm,
|
|
320
|
+
clips: PreparedClip[],
|
|
321
|
+
transcript: Extract<TranscribeOutcome, { kind: "ready" }>,
|
|
322
|
+
edl: Edl,
|
|
323
|
+
options: RenderFootageDraftOptions,
|
|
324
|
+
): Promise<FootageResult<FootageDraft>> {
|
|
325
|
+
const { log, signal } = options;
|
|
326
|
+
const aborting = signal === undefined ? {} : { signal };
|
|
327
|
+
const seconds = getEdlDuration(edl);
|
|
328
|
+
const draftDir = resolve(options.buildDir, "draft");
|
|
329
|
+
rmSync(draftDir, { recursive: true, force: true });
|
|
330
|
+
mkdirSync(draftDir, { recursive: true });
|
|
331
|
+
const report = (item: string, itemIndex: number, done: number): void =>
|
|
332
|
+
options.onProgress?.({ step: "draft", item, itemIndex, itemCount: 2, done: Math.min(seconds * 2, (itemIndex - 1) * seconds + done), total: seconds * 2 });
|
|
333
|
+
const clipFiles = Object.fromEntries(clips.map((clip) => [clip.clip.id, resolve(clip.file)]));
|
|
334
|
+
const speech = join(draftDir, "speech.mp3");
|
|
335
|
+
const words = remapWords(edl, transcript.words, film.cut.fillers);
|
|
336
|
+
const frame = getFrame(film);
|
|
337
|
+
const captions = buildFootageDraftCaptions({ film, edl, words, caption: getFootageCaptionLayout(film, options.layout ?? {}), frame });
|
|
338
|
+
if (captions !== null) {
|
|
339
|
+
writeFileSync(join(draftDir, "captions.ass"), captions);
|
|
340
|
+
mkdirSync(join(draftDir, "fonts"));
|
|
341
|
+
copyFileSync(FOOTAGE_DRAFT_FONT.file, join(draftDir, "fonts", basename(FOOTAGE_DRAFT_FONT.file)));
|
|
342
|
+
}
|
|
343
|
+
const output = resolve(options.output);
|
|
344
|
+
mkdirSync(dirname(output), { recursive: true });
|
|
345
|
+
report("speech", 1, 0);
|
|
346
|
+
await runFfmpeg("The draft's speech track", buildSpeechArgs(edl, transcript.speech, speech), { ...aborting, output: speech, onSeconds: (done) => report("speech", 1, done) });
|
|
347
|
+
report("film", 2, 0);
|
|
348
|
+
// Run in the draft folder: the captions file and the font folder are relative names, so no path needs escaping in the filter graph.
|
|
349
|
+
const args = buildFootageDraftArgs({
|
|
350
|
+
film,
|
|
351
|
+
edl,
|
|
352
|
+
clips: clipFiles,
|
|
353
|
+
speech,
|
|
354
|
+
frame,
|
|
355
|
+
...(options.width === undefined ? {} : { width: options.width }),
|
|
356
|
+
captions: captions === null ? null : "captions.ass",
|
|
357
|
+
fontsDir: captions === null ? null : "fonts",
|
|
358
|
+
output,
|
|
359
|
+
});
|
|
360
|
+
await runFfmpeg("The footage draft", args, { ...aborting, output, cwd: draftDir, onSeconds: (done) => report("film", 2, done) });
|
|
361
|
+
report("film", 2, seconds);
|
|
362
|
+
const draft = getFootageDraftFrame(frame, options.width);
|
|
363
|
+
const bytes = statSync(output).size;
|
|
364
|
+
log(`draft: ${output}, ${draft.width}×${draft.height}, ${seconds.toFixed(1)} s, ${bytes} bytes${captions === null ? ", no captions" : ""}.`);
|
|
365
|
+
return ok({ output, seconds, width: draft.width, height: draft.height, bytes });
|
|
366
|
+
}
|
|
@@ -103,6 +103,7 @@ export function createElevenLabsSttProvider(options: ElevenLabsFootageOptions):
|
|
|
103
103
|
form.append("language_code", input.language);
|
|
104
104
|
form.append("timestamps_granularity", "word");
|
|
105
105
|
form.append("tag_audio_events", "false");
|
|
106
|
+
for (const term of input.keyTerms ?? []) form.append("keyterms", term);
|
|
106
107
|
const response = await send(options, "speech to text", "/v1/speech-to-text", { method: "POST", body: form, signal: input.signal ?? null });
|
|
107
108
|
if (!response.ok) return response;
|
|
108
109
|
let body: unknown;
|
package/src/footage/film.ts
CHANGED
|
@@ -44,7 +44,17 @@ export interface FootageClip {
|
|
|
44
44
|
/** Where the music under the film comes from. `prompt` and `video` are paid (ElevenLabs Music, Video to Music). */
|
|
45
45
|
export type MusicSource =
|
|
46
46
|
| { kind: "none" }
|
|
47
|
-
| {
|
|
47
|
+
| {
|
|
48
|
+
kind: "file";
|
|
49
|
+
/** A local audio file, or a video whose sound is the bed. */
|
|
50
|
+
path: string;
|
|
51
|
+
/**
|
|
52
|
+
* The file's original name and content type, for the type check when the path has no extension (a storage key
|
|
53
|
+
* such as `@softure-ai/files`' `getStorageKey`). Without them and without an extension, only the probe checks it.
|
|
54
|
+
*/
|
|
55
|
+
fileName?: string;
|
|
56
|
+
contentType?: string;
|
|
57
|
+
}
|
|
48
58
|
| { kind: "prompt"; prompt: string }
|
|
49
59
|
| { kind: "video"; description: string; tags: string[] };
|
|
50
60
|
|
|
@@ -85,6 +95,11 @@ export interface FootageFilm {
|
|
|
85
95
|
clips: FootageClip[];
|
|
86
96
|
/** Clean the talking clips' voice (ElevenLabs Voice Isolator, paid) before transcribing them. */
|
|
87
97
|
isolateVoice: boolean;
|
|
98
|
+
/**
|
|
99
|
+
* Names and brand words the transcript should spell as given (Scribe keyterm prompting, billed on top of the
|
|
100
|
+
* transcription). Checked against `SCRIBE_KEYTERM_LIMITS`; absent or empty: none.
|
|
101
|
+
*/
|
|
102
|
+
keyTerms?: string[];
|
|
88
103
|
cut: CutOptions;
|
|
89
104
|
transitions: FootageTransitions;
|
|
90
105
|
music: FootageMusic;
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import type { FootageFilm } from "./film.js";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Key terms (Scribe keyterm prompting): names and brand words the transcript should spell as given. Scribe v2 batch
|
|
5
|
+
* limits, checked 2026-11-03 (elevenlabs.io keyterm prompting guide): up to 1000 terms of at most 50 characters and
|
|
6
|
+
* 5 words each. Keyterm prompting is billed on top of the transcription.
|
|
7
|
+
*/
|
|
8
|
+
export const SCRIBE_KEYTERM_LIMITS = { maxTerms: 1000, maxCharacters: 50, maxWords: 5 } as const;
|
|
9
|
+
|
|
10
|
+
export type KeyTermsResult = { ok: true; terms: string[] } | { ok: false; errors: string[] };
|
|
11
|
+
|
|
12
|
+
const compareCodePoints = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0);
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Key terms trimmed (inner whitespace collapsed to one space), without blanks and exact duplicates, sorted by code
|
|
16
|
+
* point (the same order on every host, so the transcript's cache key is stable), or every limit a term breaks.
|
|
17
|
+
*/
|
|
18
|
+
export function normalizeKeyTerms(terms: readonly string[]): KeyTermsResult {
|
|
19
|
+
const unique = new Set<string>();
|
|
20
|
+
for (const term of terms) {
|
|
21
|
+
const cleaned = term.trim().replace(/\s+/g, " ");
|
|
22
|
+
if (cleaned.length > 0) unique.add(cleaned);
|
|
23
|
+
}
|
|
24
|
+
const { maxTerms, maxCharacters, maxWords } = SCRIBE_KEYTERM_LIMITS;
|
|
25
|
+
const errors: string[] = [];
|
|
26
|
+
if (unique.size > maxTerms) errors.push(`key terms: ${unique.size} terms, at most ${maxTerms}.`);
|
|
27
|
+
for (const term of unique) {
|
|
28
|
+
if (term.length > maxCharacters) errors.push(`key term "${term}": ${term.length} characters, at most ${maxCharacters}.`);
|
|
29
|
+
const words = term.split(" ").length;
|
|
30
|
+
if (words > maxWords) errors.push(`key term "${term}": ${words} words, at most ${maxWords}.`);
|
|
31
|
+
}
|
|
32
|
+
return errors.length > 0 ? { ok: false, errors } : { ok: true, terms: [...unique].sort(compareCodePoints) };
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** The film's key terms, normalized; none when the film has none. */
|
|
36
|
+
export function getFilmKeyTerms(film: Pick<FootageFilm, "keyTerms">): KeyTermsResult {
|
|
37
|
+
return normalizeKeyTerms(film.keyTerms ?? []);
|
|
38
|
+
}
|