@softure-ai/marketing-kit 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +43 -0
- package/README.md +66 -9
- package/dist/config/config.d.ts.map +1 -1
- package/dist/config/config.js +7 -0
- package/dist/config/config.js.map +1 -1
- package/dist/config/schema.d.ts +1 -0
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +11 -0
- package/dist/config/schema.js.map +1 -1
- package/dist/footage/audio.d.ts +6 -2
- package/dist/footage/audio.d.ts.map +1 -1
- package/dist/footage/audio.js +15 -3
- package/dist/footage/audio.js.map +1 -1
- package/dist/footage/captions.d.ts +21 -2
- package/dist/footage/captions.d.ts.map +1 -1
- package/dist/footage/captions.js +29 -0
- package/dist/footage/captions.js.map +1 -1
- package/dist/footage/compose.d.ts.map +1 -1
- package/dist/footage/compose.js +4 -5
- package/dist/footage/compose.js.map +1 -1
- package/dist/footage/disk.d.ts +25 -0
- package/dist/footage/disk.d.ts.map +1 -0
- package/dist/footage/disk.js +72 -0
- package/dist/footage/disk.js.map +1 -0
- package/dist/footage/draft.d.ts +105 -0
- package/dist/footage/draft.d.ts.map +1 -0
- package/dist/footage/draft.js +282 -0
- package/dist/footage/draft.js.map +1 -0
- package/dist/footage/elevenlabs.d.ts.map +1 -1
- package/dist/footage/elevenlabs.js +2 -0
- package/dist/footage/elevenlabs.js.map +1 -1
- package/dist/footage/film.d.ts +12 -0
- package/dist/footage/film.d.ts.map +1 -1
- package/dist/footage/film.js.map +1 -1
- package/dist/footage/keyterms.d.ts +26 -0
- package/dist/footage/keyterms.d.ts.map +1 -0
- package/dist/footage/keyterms.js +36 -0
- package/dist/footage/keyterms.js.map +1 -0
- package/dist/footage/normalize.d.ts +21 -2
- package/dist/footage/normalize.d.ts.map +1 -1
- package/dist/footage/normalize.js +35 -11
- package/dist/footage/normalize.js.map +1 -1
- package/dist/footage/paid.d.ts +2 -1
- package/dist/footage/paid.d.ts.map +1 -1
- package/dist/footage/paid.js +13 -3
- package/dist/footage/paid.js.map +1 -1
- package/dist/footage/pipeline.d.ts +20 -3
- package/dist/footage/pipeline.d.ts.map +1 -1
- package/dist/footage/pipeline.js +44 -15
- package/dist/footage/pipeline.js.map +1 -1
- package/dist/footage/process.d.ts +2 -0
- package/dist/footage/process.d.ts.map +1 -1
- package/dist/footage/process.js +1 -0
- package/dist/footage/process.js.map +1 -1
- package/dist/footage/providers.d.ts +12 -3
- package/dist/footage/providers.d.ts.map +1 -1
- package/dist/footage/providers.js.map +1 -1
- package/dist/footage/render.d.ts +1 -1
- package/dist/footage/render.d.ts.map +1 -1
- package/dist/footage/render.js +5 -3
- package/dist/footage/render.js.map +1 -1
- package/dist/footage/tools.d.ts +15 -3
- package/dist/footage/tools.d.ts.map +1 -1
- package/dist/footage/tools.js +15 -2
- package/dist/footage/tools.js.map +1 -1
- package/dist/index.d.ts +6 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -3
- package/dist/index.js.map +1 -1
- package/docker/footage-worker.Dockerfile +6 -0
- package/fonts/DejaVuSans-Bold-Latin.ttf +0 -0
- package/fonts/LICENSE-DejaVu.txt +46 -0
- package/fonts/README.md +12 -0
- package/package.json +2 -1
- package/schema/marketing.schema.json +7 -0
- package/src/config/config.ts +8 -0
- package/src/config/schema.ts +11 -0
- package/src/footage/audio.ts +16 -3
- package/src/footage/captions.ts +39 -2
- package/src/footage/compose.ts +4 -5
- package/src/footage/disk.ts +75 -0
- package/src/footage/draft.ts +366 -0
- package/src/footage/elevenlabs.ts +1 -0
- package/src/footage/film.ts +16 -1
- package/src/footage/keyterms.ts +38 -0
- package/src/footage/normalize.ts +47 -10
- package/src/footage/paid.ts +12 -3
- package/src/footage/pipeline.ts +55 -17
- package/src/footage/process.ts +3 -0
- package/src/footage/providers.ts +10 -3
- package/src/footage/render.ts +5 -3
- package/src/footage/tools.ts +23 -3
- package/src/index.ts +34 -2
package/src/footage/normalize.ts
CHANGED
|
@@ -47,21 +47,47 @@ export function toneMapFilter(hdr: HdrKind | null, transfer: string | null): str
|
|
|
47
47
|
];
|
|
48
48
|
}
|
|
49
49
|
|
|
50
|
-
/**
|
|
50
|
+
/**
|
|
51
|
+
* The video filter graph from input 0's picture to `[v]`. Every stage runs on as little as it can: frames are
|
|
52
|
+
* dropped to 30 fps first, then the picture is scaled to the frame (crop: covered and cropped to it; blur: fitted
|
|
53
|
+
* inside it), and only then tone-mapped, so the float stages of an HDR clip work on frame-sized pictures instead of
|
|
54
|
+
* the source's (a 4K phone clip has 4x the pixels of a 1080x1920 frame). The blur's background is made from the
|
|
55
|
+
* fitted picture.
|
|
56
|
+
*/
|
|
51
57
|
export function buildVideoFilter(options: { fit: "crop" | "blur"; frame: Frame; hdr: HdrKind | null; transfer: string | null }): string {
|
|
52
58
|
const { width, height } = options.frame;
|
|
53
|
-
const
|
|
54
|
-
const cover = `scale=${width}:${height}:force_original_aspect_ratio=increase,crop=${width}:${height}`;
|
|
59
|
+
const toneMap = toneMapFilter(options.hdr, options.transfer);
|
|
55
60
|
const tail = "setsar=1,format=yuv420p";
|
|
56
|
-
if (options.fit === "crop")
|
|
61
|
+
if (options.fit === "crop") {
|
|
62
|
+
const cover = `scale=${width}:${height}:force_original_aspect_ratio=increase,crop=${width}:${height}`;
|
|
63
|
+
return `[0:v]${[`fps=${FOOTAGE_FPS}`, cover, ...toneMap, tail].join(",")}[v]`;
|
|
64
|
+
}
|
|
65
|
+
// Even sizes: zscale refuses odd ones in 4:2:0.
|
|
66
|
+
const fitted = `scale=${width}:${height}:force_original_aspect_ratio=decrease:force_divisible_by=2`;
|
|
67
|
+
const backWidth = Math.round(width / 4);
|
|
68
|
+
const backHeight = Math.round(height / 4);
|
|
57
69
|
return [
|
|
58
|
-
`[0:v]${
|
|
59
|
-
`[bg]scale=${
|
|
60
|
-
`[fg]
|
|
61
|
-
`[back][front]overlay=(W-w)/2:(H-h)/2,${tail}[v]`,
|
|
70
|
+
`[0:v]${[`fps=${FOOTAGE_FPS}`, fitted, ...toneMap, "split=2[bg][fg]"].join(",")}`,
|
|
71
|
+
`[bg]scale=${backWidth}:${backHeight}:force_original_aspect_ratio=increase,crop=${backWidth}:${backHeight},boxblur=20:2,scale=${width}:${height},eq=brightness=-0.06[back]`,
|
|
72
|
+
`[back][fg]overlay=(W-w)/2:(H-h)/2,${tail}[v]`,
|
|
62
73
|
].join(";");
|
|
63
74
|
}
|
|
64
75
|
|
|
76
|
+
/** x264's presets, fastest first. */
|
|
77
|
+
export const X264_PRESETS = ["ultrafast", "superfast", "veryfast", "faster", "fast", "medium", "slow", "slower", "veryslow"] as const;
|
|
78
|
+
export type X264Preset = (typeof X264_PRESETS)[number];
|
|
79
|
+
const DEFAULT_NORMALIZE_PRESET: X264Preset = "veryfast";
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* How the intermediate clips are encoded. A small worker can trade file size for speed: `preset` (default
|
|
83
|
+
* `veryfast`; `superfast` or `ultrafast` encode faster into bigger files), `threads` (ffmpeg's encoder and filter
|
|
84
|
+
* threads; by default ffmpeg's own choice, which follows the host's cores, not a container's CPU quota).
|
|
85
|
+
*/
|
|
86
|
+
export interface NormalizeEncoder {
|
|
87
|
+
preset?: X264Preset;
|
|
88
|
+
threads?: number;
|
|
89
|
+
}
|
|
90
|
+
|
|
65
91
|
export interface NormalizeInput {
|
|
66
92
|
clip: FootageClip;
|
|
67
93
|
/** The local file (a URL's downloaded copy). */
|
|
@@ -69,19 +95,29 @@ export interface NormalizeInput {
|
|
|
69
95
|
probe: FootageProbe;
|
|
70
96
|
frame: Frame;
|
|
71
97
|
output: string;
|
|
98
|
+
encoder?: NormalizeEncoder;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** The thread arguments: global filter threads (before the inputs) and encoder threads (before the output). */
|
|
102
|
+
function getThreadArgs(threads: number | undefined): { global: string[]; output: string[] } {
|
|
103
|
+
if (threads === undefined) return { global: [], output: [] };
|
|
104
|
+
if (!Number.isInteger(threads) || threads < 1) throw new Error(`normalize.threads must be a positive whole number, got ${threads}.`);
|
|
105
|
+
return { global: ["-filter_complex_threads", String(threads)], output: ["-threads", String(threads)] };
|
|
72
106
|
}
|
|
73
107
|
|
|
74
|
-
/** ffmpeg's arguments for one clip; pure, so the graph is tested without ffmpeg. */
|
|
108
|
+
/** ffmpeg's arguments for one clip; pure, so the graph is tested without ffmpeg. Throws on an invalid `encoder.threads`. */
|
|
75
109
|
export function buildNormalizeArgs(input: NormalizeInput): string[] {
|
|
76
110
|
const { clip, file, probe, frame, output } = input;
|
|
77
111
|
const duration = getTrimmedDuration(clip, probe);
|
|
78
112
|
const fit = resolveFit(clip.fit, probe, frame);
|
|
79
113
|
const seek = clip.trim.start > 0 ? ["-ss", String(clip.trim.start)] : [];
|
|
114
|
+
const threads = getThreadArgs(input.encoder?.threads);
|
|
80
115
|
const silence = probe.hasAudio ? [] : ["-f", "lavfi", "-i", `anullsrc=r=${FOOTAGE_SAMPLE_RATE}:cl=stereo`];
|
|
81
116
|
return [
|
|
82
117
|
"-v",
|
|
83
118
|
"error",
|
|
84
119
|
"-y",
|
|
120
|
+
...threads.global,
|
|
85
121
|
...seek,
|
|
86
122
|
"-i",
|
|
87
123
|
file,
|
|
@@ -97,7 +133,7 @@ export function buildNormalizeArgs(input: NormalizeInput): string[] {
|
|
|
97
133
|
"-c:v",
|
|
98
134
|
"libx264",
|
|
99
135
|
"-preset",
|
|
100
|
-
|
|
136
|
+
input.encoder?.preset ?? DEFAULT_NORMALIZE_PRESET,
|
|
101
137
|
"-crf",
|
|
102
138
|
"18",
|
|
103
139
|
"-g",
|
|
@@ -118,6 +154,7 @@ export function buildNormalizeArgs(input: NormalizeInput): string[] {
|
|
|
118
154
|
"2",
|
|
119
155
|
"-movflags",
|
|
120
156
|
"+faststart",
|
|
157
|
+
...threads.output,
|
|
121
158
|
output,
|
|
122
159
|
];
|
|
123
160
|
}
|
package/src/footage/paid.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { err, ok, type TtsResult } from "../voice/provider.js";
|
|
|
6
6
|
import { footageCacheKey, getFootageCachePath, hashFile } from "./cache.js";
|
|
7
7
|
import { END_CARD_SECONDS, getFootageFilmSeconds } from "./compose.js";
|
|
8
8
|
import { readEdl, type Edl } from "./edl.js";
|
|
9
|
+
import { getFilmKeyTerms } from "./keyterms.js";
|
|
9
10
|
import type { FootageClip, FootageFilm } from "./film.js";
|
|
10
11
|
import { getTrimmedDuration, probeFootage } from "./probe.js";
|
|
11
12
|
import type { FootageProviders, FootageRates, PaidStepKind } from "./providers.js";
|
|
@@ -66,7 +67,10 @@ export function getIsolatedCache(film: FootageFilm, clipKey: string, options: Pi
|
|
|
66
67
|
|
|
67
68
|
export function getTranscriptCache(film: FootageFilm, clipKey: string, options: Pick<FootageCacheOptions, "cacheDir" | "providers">): CacheEntry {
|
|
68
69
|
const { stt } = options.providers;
|
|
69
|
-
const
|
|
70
|
+
const keyTerms = getFilmKeyTerms(film);
|
|
71
|
+
// Key terms join the key only when there are any, so a transcript bought without them keeps its key.
|
|
72
|
+
const withKeyTerms = keyTerms.ok && keyTerms.terms.length > 0 ? { keyTerms: keyTerms.terms } : {};
|
|
73
|
+
const key = footageCacheKey("transcript", { clip: clipKey, isolated: film.isolateVoice, provider: stt.id, model: stt.model, language: film.language, ...withKeyTerms });
|
|
70
74
|
return { key, path: getFootageCachePath(options.cacheDir, film.id, "transcript", key) };
|
|
71
75
|
}
|
|
72
76
|
|
|
@@ -175,10 +179,15 @@ const round = (value: number, digits: number): number => {
|
|
|
175
179
|
* What a footage film will cost in the caller's credits, before anything runs: every clip probed (a URL clip not yet
|
|
176
180
|
* downloaded is probed at its URL), nothing normalized, no provider called. Talking clips need speech to text (and
|
|
177
181
|
* isolation when the film cleans the voice); generated music lasts the cut's length when the cut is known, else the
|
|
178
|
-
* clips' length plus the end card. A cached step costs 0.
|
|
182
|
+
* clips' length plus the end card. A cached step costs 0. With key terms, transcription costs `rates.transcribe` plus
|
|
183
|
+
* the keyterm surcharge `rates.keyTerms`; invalid key terms fail the estimate.
|
|
179
184
|
*/
|
|
180
185
|
export async function estimateFootageFilm(film: FootageFilm, options: EstimateFootageOptions): Promise<TtsResult<FootageEstimate>> {
|
|
181
186
|
if (film.clips.length === 0) return err(`footage film ${film.id} has no clips.`);
|
|
187
|
+
const keyTerms = getFilmKeyTerms(film);
|
|
188
|
+
if (!keyTerms.ok) return err(`footage film ${film.id}: ${keyTerms.errors.join(" ")}`);
|
|
189
|
+
const surcharge = keyTerms.terms.length === 0 ? 0 : (options.rates.keyTerms ?? 0);
|
|
190
|
+
const rates: Record<PaidStepKind, number> = { ...options.rates, transcribe: options.rates.transcribe + surcharge };
|
|
182
191
|
const files: Record<string, string> = {};
|
|
183
192
|
for (const clip of film.clips) {
|
|
184
193
|
const local = findLocalSource(clip, options.buildDir);
|
|
@@ -189,7 +198,7 @@ export async function estimateFootageFilm(film: FootageFilm, options: EstimateFo
|
|
|
189
198
|
if (!probed.ok) return err(probed.errors.join("\n"));
|
|
190
199
|
const items: FootageEstimateItem[] = [];
|
|
191
200
|
const add = (step: PaidStepKind, item: string, provider: string, units: number, cached: boolean): void => {
|
|
192
|
-
items.push({ step, item, provider, units: round(units, 3), unit: "seconds", credits: cached ? 0 : round(units *
|
|
201
|
+
items.push({ step, item, provider, units: round(units, 3), unit: "seconds", credits: cached ? 0 : round(units * rates[step], 2), cached });
|
|
193
202
|
};
|
|
194
203
|
let clipSeconds = 0;
|
|
195
204
|
for (const { clip, probe } of probed.clips) {
|
package/src/footage/pipeline.ts
CHANGED
|
@@ -13,13 +13,15 @@ import type { TimedWord } from "../voice/voiceover.js";
|
|
|
13
13
|
import { checkMusicFileType, describeAudioProbe, probeAudio } from "./audio.js";
|
|
14
14
|
import { readCachedTranscript, writeCacheFile, writeTranscript } from "./cache.js";
|
|
15
15
|
import { composeFootage, END_CARD_SECONDS, getFootageFilmSeconds } from "./compose.js";
|
|
16
|
-
import { findMissingCaptionGlyphs,
|
|
16
|
+
import { findMissingCaptionGlyphs, getFootageCaptionLayout } from "./captions.js";
|
|
17
|
+
import { findLowDiskProblems, getFreeBytes, getRenderTempDir, type FreeBytesReader } from "./disk.js";
|
|
17
18
|
import { buildEdl, readEdl, remapWords, validateEdl, type Edl } from "./edl.js";
|
|
18
19
|
import { makePlaceholderTranscript } from "./fake.js";
|
|
19
|
-
import { getFootageEdit, type FootageClip, type FootageFilm } from "./film.js";
|
|
20
|
+
import { getFootageEdit, type FootageClip, type FootageFilm, type MusicSource } from "./film.js";
|
|
21
|
+
import { getFilmKeyTerms } from "./keyterms.js";
|
|
20
22
|
import { DEFAULT_LOUDNESS, normalizeLoudness } from "./loudness.js";
|
|
21
23
|
import { duckingKeyframes } from "./music.js";
|
|
22
|
-
import { normalizeFootage, type Frame } from "./normalize.js";
|
|
24
|
+
import { normalizeFootage, type Frame, type NormalizeEncoder } from "./normalize.js";
|
|
23
25
|
import { describeProbe, getTrimmedDuration, probeFootage, type FootageProbe } from "./probe.js";
|
|
24
26
|
import {
|
|
25
27
|
getClipCacheKey,
|
|
@@ -72,7 +74,7 @@ export { getEdlPath } from "./paid.js";
|
|
|
72
74
|
|
|
73
75
|
const require = createRequire(import.meta.url);
|
|
74
76
|
/** Bumped when the normalized format changes, so older normalized clips are made again. */
|
|
75
|
-
const NORMALIZE_VERSION =
|
|
77
|
+
const NORMALIZE_VERSION = 2;
|
|
76
78
|
|
|
77
79
|
export interface FootageBrand {
|
|
78
80
|
name: string;
|
|
@@ -100,6 +102,11 @@ export interface FootageRunOptions {
|
|
|
100
102
|
onProgress?: (progress: FootageProgress) => void;
|
|
101
103
|
/** Called after each paid call, for the app's own spend ledger. */
|
|
102
104
|
onCharge?: (charge: FootageCharge) => void;
|
|
105
|
+
/**
|
|
106
|
+
* How the normalized clips are encoded (`preset`, `threads`): a small worker trades file size for speed. Not part of
|
|
107
|
+
* the normalize stamp, so changing it does not normalize clips already made again.
|
|
108
|
+
*/
|
|
109
|
+
normalize?: NormalizeEncoder;
|
|
103
110
|
}
|
|
104
111
|
|
|
105
112
|
export interface PreparedClip {
|
|
@@ -178,7 +185,7 @@ function getNormalizeStamp(clip: FootageClip, source: string, frame: Frame): str
|
|
|
178
185
|
/** Downloads URL clips, probes every clip (all problems at once) and a local music file, normalizes each clip that changed. */
|
|
179
186
|
export async function prepareFootage(
|
|
180
187
|
film: FootageFilm,
|
|
181
|
-
options: Pick<FootageRunOptions, "buildDir" | "log" | "fetch" | "signal" | "onProgress">,
|
|
188
|
+
options: Pick<FootageRunOptions, "buildDir" | "log" | "fetch" | "signal" | "onProgress" | "normalize">,
|
|
182
189
|
): Promise<FootageResult<PreparedClip[]>> {
|
|
183
190
|
const { buildDir, log, signal } = options;
|
|
184
191
|
if (film.clips.length === 0) return err(`footage film ${film.id} has no clips.`);
|
|
@@ -196,7 +203,7 @@ export async function prepareFootage(
|
|
|
196
203
|
const probed = await probeFootage(film.clips, sources, signal === undefined ? {} : { signal });
|
|
197
204
|
if (!probed.ok) return err(probed.errors.join("\n"));
|
|
198
205
|
if (film.music.source.kind === "file") {
|
|
199
|
-
const music = await checkMusicFile(film.music.source
|
|
206
|
+
const music = await checkMusicFile(film.music.source, options);
|
|
200
207
|
if (!music.ok) return err(music.error);
|
|
201
208
|
}
|
|
202
209
|
const frame = getFrame(film);
|
|
@@ -219,7 +226,7 @@ export async function prepareFootage(
|
|
|
219
226
|
// The stamp goes first, so a normalize stopped half-way never leaves a stamp that vouches for a partial clip.
|
|
220
227
|
rmSync(stampPath, { force: true });
|
|
221
228
|
await normalizeFootage(
|
|
222
|
-
{ clip: clip.clip, file: clip.source, probe: clip.probe, frame, output: clip.file },
|
|
229
|
+
{ clip: clip.clip, file: clip.source, probe: clip.probe, frame, output: clip.file, ...(options.normalize === undefined ? {} : { encoder: options.normalize }) },
|
|
223
230
|
{ ...(signal === undefined ? {} : { signal }), onSeconds: (seconds) => progress(index, seconds) },
|
|
224
231
|
);
|
|
225
232
|
writeFileSync(stampPath, stamp);
|
|
@@ -268,6 +275,9 @@ export function getMusicPaidStep(film: FootageFilm, options: Pick<FootageRunOpti
|
|
|
268
275
|
export async function transcribeFootage(film: FootageFilm, clips: PreparedClip[], options: FootageRunOptions): Promise<FootageResult<TranscribeOutcome>> {
|
|
269
276
|
const { buildDir, providers, log, signal } = options;
|
|
270
277
|
if (options.isPlaceholder) return ok(transcribePlaceholder(film, clips, buildDir));
|
|
278
|
+
const keyTerms = getFilmKeyTerms(film);
|
|
279
|
+
if (!keyTerms.ok) return err(`footage film ${film.id}: ${keyTerms.errors.join(" ")}`);
|
|
280
|
+
const sttKeyTerms = keyTerms.terms.length === 0 ? {} : { keyTerms: keyTerms.terms };
|
|
271
281
|
const plan: { clip: PreparedClip; isolated: CacheEntry | null; transcript: CacheEntry }[] = [];
|
|
272
282
|
const steps: PaidStep[] = [];
|
|
273
283
|
for (const clip of clips) {
|
|
@@ -341,12 +351,12 @@ export async function transcribeFootage(film: FootageFilm, clips: PreparedClip[]
|
|
|
341
351
|
const input = sttInput ?? (await getAudio());
|
|
342
352
|
throwIfAborted(signal);
|
|
343
353
|
log(`speech to text (${providers.stt.id}): clip ${id}, ${clip.duration.toFixed(1)} s.`);
|
|
344
|
-
const result = await providers.stt.transcribe({ audio: input, language: film.language, ...(signal === undefined ? {} : { signal }) });
|
|
354
|
+
const result = await providers.stt.transcribe({ audio: input, language: film.language, ...sttKeyTerms, ...(signal === undefined ? {} : { signal }) });
|
|
345
355
|
throwIfAborted(signal);
|
|
346
356
|
if (!result.ok) return err(`clip "${id}": ${result.error}`);
|
|
347
357
|
writeTranscript(transcript.path, result.value.words);
|
|
348
358
|
const { credits } = result.value;
|
|
349
|
-
options.onCharge?.({ step: "transcribe", item: id, provider: providers.stt.id, units: clip.duration, ...(credits === undefined ? {} : { credits }), cacheKey: transcript.key });
|
|
359
|
+
options.onCharge?.({ step: "transcribe", item: id, provider: providers.stt.id, units: clip.duration, ...(credits === undefined ? {} : { credits }), ...(keyTerms.terms.length === 0 ? {} : { keyTerms: keyTerms.terms.length }), cacheKey: transcript.key });
|
|
350
360
|
log(`transcript: clip ${id}, ${result.value.words.length} words, saved ${transcript.path} (commit it).`);
|
|
351
361
|
transcribeProgress(index, clip.duration);
|
|
352
362
|
words[id] = result.value.words;
|
|
@@ -387,11 +397,21 @@ export function readEdlFile(film: FootageFilm, clips: PreparedClip[], edlPath: s
|
|
|
387
397
|
return problems.length > 0 ? err(`${edlPath}: ${problems.join("; ")}`) : ok(read.edl);
|
|
388
398
|
}
|
|
389
399
|
|
|
390
|
-
/**
|
|
391
|
-
|
|
400
|
+
/**
|
|
401
|
+
* A local music file checked like an app's upload: an accepted type, then readable audio (`probeAudio`). The type comes
|
|
402
|
+
* from the source's declared name (else the path) and content type; with no extension and no content type (a storage
|
|
403
|
+
* key) the type check is skipped and the probe alone decides.
|
|
404
|
+
*/
|
|
405
|
+
async function checkMusicFile(source: Extract<MusicSource, { kind: "file" }>, options: Pick<FootageRunOptions, "log" | "signal">): Promise<FootageResult<string>> {
|
|
406
|
+
const { path, fileName, contentType } = source;
|
|
392
407
|
if (!existsSync(path)) return err(`music: no file at ${path}`);
|
|
393
|
-
const
|
|
394
|
-
if (
|
|
408
|
+
const name = fileName ?? path;
|
|
409
|
+
if (extname(name) === "" && contentType === undefined) {
|
|
410
|
+
options.log(`music: ${path} has no extension and no declared name or content type; the probe checks it.`);
|
|
411
|
+
} else {
|
|
412
|
+
const type = checkMusicFileType(contentType === undefined ? { fileName: name } : { fileName: name, contentType });
|
|
413
|
+
if (!type.ok) return err(`music: ${type.error}`);
|
|
414
|
+
}
|
|
395
415
|
const probe = await probeAudio(path, options.signal === undefined ? {} : { signal: options.signal });
|
|
396
416
|
if (!probe.ok) return err(`music: ${probe.error}`);
|
|
397
417
|
options.log(`music: ${path} (${describeAudioProbe(probe.probe)}).`);
|
|
@@ -459,11 +479,22 @@ export interface RenderFootageOptions extends FootageRunOptions {
|
|
|
459
479
|
output: string;
|
|
460
480
|
/** A project's override of the format's caption box (`marketing.json` `layout`). */
|
|
461
481
|
layout?: LayoutOverride;
|
|
482
|
+
/**
|
|
483
|
+
* hyperframes' temp folder, passed to it as `TMPDIR` (created when missing): the clips' frames are extracted there,
|
|
484
|
+
* so it needs room (a data volume, not a small tmpfs). By default the `TMPDIR` the process already has.
|
|
485
|
+
*/
|
|
486
|
+
tempDir?: string;
|
|
487
|
+
/** Reads a folder's free bytes for the disk check; injected in tests. */
|
|
488
|
+
readFreeBytes?: FreeBytesReader;
|
|
462
489
|
}
|
|
463
490
|
|
|
464
491
|
export type RenderedFootage = { kind: "dry-run"; steps: PaidStep[] } | ({ kind: "rendered"; output: string; seconds: number; edlPath: string } & FinishedFootage);
|
|
465
492
|
|
|
466
|
-
/**
|
|
493
|
+
/**
|
|
494
|
+
* Composes the cut and renders it: speech track (loudness-normalized), ducked music, end card backdrop, assets,
|
|
495
|
+
* hyperframes, final pass, cover. First of all it checks that hyperframes' temp folder and the build folder have the
|
|
496
|
+
* 1 GiB free hyperframes requires, so a full disk ends the run with a sentence before anything is bought or built.
|
|
497
|
+
*/
|
|
467
498
|
export async function renderFootage(
|
|
468
499
|
film: FootageFilm,
|
|
469
500
|
clips: PreparedClip[],
|
|
@@ -472,6 +503,8 @@ export async function renderFootage(
|
|
|
472
503
|
options: RenderFootageOptions,
|
|
473
504
|
): Promise<FootageResult<RenderedFootage>> {
|
|
474
505
|
const { buildDir, log, signal } = options;
|
|
506
|
+
const lowDisk = findLowDiskProblems({ temp: getRenderTempDir(process.env, options.tempDir), build: buildDir }, options.readFreeBytes ?? getFreeBytes);
|
|
507
|
+
if (lowDisk.length > 0) return err(`not enough disk space to render: ${lowDisk.join(" ")}`);
|
|
475
508
|
const seconds = getFootageFilmSeconds(film, edl);
|
|
476
509
|
const words = remapWords(edl, transcript.words, film.cut.fillers);
|
|
477
510
|
const music = await getMusic(film, edl, clips, seconds, options);
|
|
@@ -535,8 +568,7 @@ export async function renderFootage(
|
|
|
535
568
|
cpSync(require.resolve("gsap/dist/gsap.min.js"), join(assetsDir, "gsap.min.js"));
|
|
536
569
|
const base = resolveLayout(film.format, options.layout ?? {});
|
|
537
570
|
// A caption preset places the 9:16 box in a platform's safe zone; the project's own caption override still wins.
|
|
538
|
-
const
|
|
539
|
-
const layout = preset === null || options.layout?.caption !== undefined ? base : { ...base, caption: preset };
|
|
571
|
+
const layout = { ...base, caption: getFootageCaptionLayout(film, options.layout ?? {}) };
|
|
540
572
|
const missing = findMissingCaptionGlyphs({
|
|
541
573
|
fonts: options.brand.fonts,
|
|
542
574
|
captions: film.captions ? words.map((word) => word.text) : [],
|
|
@@ -654,11 +686,17 @@ async function runHyperframesRender(options: RenderFootageOptions, output: strin
|
|
|
654
686
|
options.log(line.trim());
|
|
655
687
|
}
|
|
656
688
|
};
|
|
689
|
+
const env = getHyperframesEnv();
|
|
690
|
+
if (options.tempDir !== undefined && options.tempDir.length > 0) {
|
|
691
|
+
const tempDir = resolve(options.tempDir);
|
|
692
|
+
mkdirSync(tempDir, { recursive: true });
|
|
693
|
+
env.TMPDIR = tempDir;
|
|
694
|
+
}
|
|
657
695
|
const result = await runProcess({
|
|
658
696
|
command: process.execPath,
|
|
659
697
|
args: [getHyperframesBin(), "render", "--quality", options.quality, "-o", output],
|
|
660
698
|
cwd: options.buildDir,
|
|
661
|
-
env
|
|
699
|
+
env,
|
|
662
700
|
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
663
701
|
removeOnAbort: [output],
|
|
664
702
|
onStdoutLine: onLine,
|
package/src/footage/process.ts
CHANGED
|
@@ -109,6 +109,8 @@ export interface FfmpegRunOptions {
|
|
|
109
109
|
output?: string;
|
|
110
110
|
/** Seconds of output written so far, as ffmpeg reports them. */
|
|
111
111
|
onSeconds?: (seconds: number) => void;
|
|
112
|
+
/** The folder ffmpeg runs in, so relative names in the arguments and the filter graph resolve there. */
|
|
113
|
+
cwd?: string;
|
|
112
114
|
}
|
|
113
115
|
|
|
114
116
|
/** Runs ffmpeg with its progress on stdout; throws naming the step when it fails (a broken input or machine). */
|
|
@@ -118,6 +120,7 @@ export async function runFfmpeg(step: string, args: string[], options: FfmpegRun
|
|
|
118
120
|
command: "ffmpeg",
|
|
119
121
|
args: ["-progress", "pipe:1", "-nostats", ...args],
|
|
120
122
|
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
123
|
+
...(options.cwd === undefined ? {} : { cwd: options.cwd }),
|
|
121
124
|
removeOnAbort: options.output === undefined ? [] : [options.output],
|
|
122
125
|
onStdoutLine: (line) => {
|
|
123
126
|
const seconds = readFfmpegProgressSeconds(line);
|
package/src/footage/providers.ts
CHANGED
|
@@ -11,6 +11,8 @@ export interface SttInput {
|
|
|
11
11
|
audio: string;
|
|
12
12
|
/** ISO 639 code the speech is in. */
|
|
13
13
|
language: string;
|
|
14
|
+
/** Key terms the transcript should spell as given (normalized by `normalizeKeyTerms`); absent: none. */
|
|
15
|
+
keyTerms?: string[];
|
|
14
16
|
/** Aborts the request. */
|
|
15
17
|
signal?: AbortSignal;
|
|
16
18
|
}
|
|
@@ -87,8 +89,11 @@ export function describePaidSteps(steps: PaidStep[]): string[] {
|
|
|
87
89
|
return lines;
|
|
88
90
|
}
|
|
89
91
|
|
|
90
|
-
/**
|
|
91
|
-
|
|
92
|
+
/**
|
|
93
|
+
* Credits per second billed, for each paid step; the caller's prices (the kit hard-codes none). `keyTerms` is the
|
|
94
|
+
* surcharge per second of speech to text with key terms (keyterm prompting), added to `transcribe`; absent: 0.
|
|
95
|
+
*/
|
|
96
|
+
export type FootageRates = Record<PaidStepKind, number> & { keyTerms?: number };
|
|
92
97
|
|
|
93
98
|
/** One paid call made, as `onCharge` reports it: what for, how much was billed and where its result is cached. */
|
|
94
99
|
export interface FootageCharge {
|
|
@@ -100,12 +105,14 @@ export interface FootageCharge {
|
|
|
100
105
|
units: number;
|
|
101
106
|
/** The provider's reported charge, when its API returns one. */
|
|
102
107
|
credits?: number;
|
|
108
|
+
/** How many key terms a speech to text call sent (billed with keyterm prompting's surcharge); absent: none. */
|
|
109
|
+
keyTerms?: number;
|
|
103
110
|
/** The paid cache's key of the result (the file name's first part under `<cacheDir>/<film id>/`). */
|
|
104
111
|
cacheKey: string;
|
|
105
112
|
}
|
|
106
113
|
|
|
107
114
|
/** A stage of a footage run that reports progress. */
|
|
108
|
-
export type FootageStage = "normalize" | PaidStepKind | "assets" | "render" | "finish";
|
|
115
|
+
export type FootageStage = "normalize" | PaidStepKind | "assets" | "render" | "finish" | "draft";
|
|
109
116
|
|
|
110
117
|
/**
|
|
111
118
|
* Progress of a footage run. `done` and `total` are seconds of media across the whole stage (a bar per stage);
|
package/src/footage/render.ts
CHANGED
|
@@ -61,15 +61,17 @@ export function buildMusicArgs(options: { input: string; seconds: number; keyfra
|
|
|
61
61
|
return ["-v", "error", "-y", "-stream_loop", "-1", "-i", options.input, "-t", String(seconds), "-af", filters.join(","), "-c:a", "libmp3lame", "-q:a", "2", options.output];
|
|
62
62
|
}
|
|
63
63
|
|
|
64
|
-
/** A small silent copy of the cut, what Video to Music scores. */
|
|
64
|
+
/** A small silent copy of the cut, what Video to Music scores: each segment read from its own seek into its clip. */
|
|
65
65
|
export function buildCutPreviewArgs(edl: Edl, clips: Record<string, string>, output: string): string[] {
|
|
66
66
|
const inputs: string[] = [];
|
|
67
67
|
const parts: string[] = [];
|
|
68
68
|
edl.segments.forEach((segment, index) => {
|
|
69
69
|
const clip = clips[segment.clip];
|
|
70
70
|
if (clip === undefined) throw new Error(`No file for clip "${segment.clip}" of the EDL.`);
|
|
71
|
-
|
|
72
|
-
|
|
71
|
+
// Seeking the input (accurate while transcoding; normalized clips have a keyframe every half second) instead of
|
|
72
|
+
// trim, which decodes the clip from its start for every segment.
|
|
73
|
+
inputs.push("-ss", String(r3(segment.start)), "-t", String(r3(segment.end - segment.start)), "-i", clip);
|
|
74
|
+
parts.push(`[${index}:v]setpts=PTS-STARTPTS,scale=540:-2,setsar=1[v${index}]`);
|
|
73
75
|
});
|
|
74
76
|
const joined = edl.segments.map((_, index) => `[v${index}]`).join("");
|
|
75
77
|
const graph = `${parts.join(";")};${joined}concat=n=${edl.segments.length}:v=1:a=0[cut]`;
|
package/src/footage/tools.ts
CHANGED
|
@@ -3,6 +3,7 @@ import { homedir } from "node:os";
|
|
|
3
3
|
import { join } from "node:path";
|
|
4
4
|
|
|
5
5
|
import { getHyperframesBin, getHyperframesEnv, HYPERFRAMES_VERSION } from "../render/hyperframes.js";
|
|
6
|
+
import { findLowDiskProblems, getFreeBytes, getRenderTempDir, type FreeBytesReader } from "./disk.js";
|
|
6
7
|
import { runProcess } from "./process.js";
|
|
7
8
|
|
|
8
9
|
/**
|
|
@@ -18,15 +19,26 @@ export interface FootageToolsOptions {
|
|
|
18
19
|
* worker image ships the browser, so the default is false.
|
|
19
20
|
*/
|
|
20
21
|
isBrowserDownloadAllowed?: boolean;
|
|
21
|
-
/**
|
|
22
|
+
/** Check the filters the fast draft draws with (`renderFootageDraft`): `ass` (libass and libfreetype), `xfade`, `zoompan`. */
|
|
23
|
+
needsDraft?: boolean;
|
|
24
|
+
/** The environment the run sees (`HYPERFRAMES_BROWSER_PATH`, `PRODUCER_HEADLESS_SHELL_PATH`, `HOME`, `TMPDIR`). */
|
|
22
25
|
env?: NodeJS.ProcessEnv;
|
|
26
|
+
/** The render's temp folder, as passed to `renderFootage`; by default `TMPDIR` of `env`, else `os.tmpdir()`. */
|
|
27
|
+
tempDir?: string;
|
|
28
|
+
/** The film's build folder, where hyperframes writes its render; checked for free space when given. */
|
|
29
|
+
buildDir?: string;
|
|
23
30
|
signal?: AbortSignal;
|
|
31
|
+
/** Reads a folder's free bytes; injected in tests. */
|
|
32
|
+
readFreeBytes?: FreeBytesReader;
|
|
24
33
|
}
|
|
25
34
|
|
|
26
35
|
/** What the footage blocks need from ffmpeg: decoders and filters, by the name ffmpeg lists them under. */
|
|
27
36
|
export const REQUIRED_DECODERS = ["hevc", "h264"] as const;
|
|
28
37
|
export const REQUIRED_FILTERS = ["zscale", "tonemap"] as const;
|
|
38
|
+
/** What the fast draft needs on top: libass (with libfreetype) for the captions, `xfade` for fades, `zoompan` for Ken Burns. */
|
|
39
|
+
export const DRAFT_FILTERS = ["ass", "xfade", "zoompan"] as const;
|
|
29
40
|
|
|
41
|
+
const DRAFT_INSTALL_HINT = "install an ffmpeg built with libass and libfreetype (macOS: brew install ffmpeg, Debian/Ubuntu: apt install ffmpeg)";
|
|
30
42
|
const INSTALL_HINT = "install an ffmpeg built with libzimg (macOS: brew install ffmpeg, Debian/Ubuntu: apt install ffmpeg)";
|
|
31
43
|
|
|
32
44
|
/** The names in `ffmpeg -decoders` or `-filters` output: the second column of each listing line. */
|
|
@@ -83,8 +95,9 @@ export function findBrowserProblem(env: NodeJS.ProcessEnv): string | null {
|
|
|
83
95
|
|
|
84
96
|
/**
|
|
85
97
|
* Checks the machine for a footage run: ffmpeg and ffprobe run, ffmpeg decodes HEVC and H.264 and has `zscale` and
|
|
86
|
-
* `tonemap` (HDR clips),
|
|
87
|
-
*
|
|
98
|
+
* `tonemap` (HDR clips), for a draft (`needsDraft`) `ass`, `xfade` and `zoompan`, and, for a render, hyperframes runs
|
|
99
|
+
* and has its browser, and its temp folder (and the build folder, when given) has the 1 GiB free hyperframes requires.
|
|
100
|
+
* Returns every problem found; an empty list means the machine is ready.
|
|
88
101
|
*/
|
|
89
102
|
export async function checkFootageTools(options: FootageToolsOptions = {}): Promise<string[]> {
|
|
90
103
|
const { signal } = options;
|
|
@@ -100,6 +113,11 @@ export async function checkFootageTools(options: FootageToolsOptions = {}): Prom
|
|
|
100
113
|
for (const filter of REQUIRED_FILTERS) {
|
|
101
114
|
if (filters?.has(filter) !== true) problems.push(`ffmpeg has no ${filter} filter (HDR clips need it): ${INSTALL_HINT}.`);
|
|
102
115
|
}
|
|
116
|
+
if (options.needsDraft === true) {
|
|
117
|
+
for (const filter of DRAFT_FILTERS) {
|
|
118
|
+
if (filters?.has(filter) !== true) problems.push(`ffmpeg has no ${filter} filter (the draft needs it): ${DRAFT_INSTALL_HINT}.`);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
103
121
|
}
|
|
104
122
|
if (!(await isRunnable("ffprobe", ["-version"], aborting))) problems.push(`no ffprobe in PATH: it comes with ffmpeg; ${INSTALL_HINT}.`);
|
|
105
123
|
if (options.needsRender === false) return problems;
|
|
@@ -115,5 +133,7 @@ export async function checkFootageTools(options: FootageToolsOptions = {}): Prom
|
|
|
115
133
|
const browser = findBrowserProblem(env);
|
|
116
134
|
if (browser !== null) problems.push(browser);
|
|
117
135
|
}
|
|
136
|
+
const temp = getRenderTempDir(env, options.tempDir);
|
|
137
|
+
problems.push(...findLowDiskProblems(options.buildDir === undefined ? { temp } : { temp, build: options.buildDir }, options.readFreeBytes ?? getFreeBytes));
|
|
118
138
|
return problems;
|
|
119
139
|
}
|
package/src/index.ts
CHANGED
|
@@ -225,7 +225,20 @@ export {
|
|
|
225
225
|
type AudioProbeResult,
|
|
226
226
|
type MusicFileTypeCheck,
|
|
227
227
|
} from "./footage/audio.js";
|
|
228
|
-
export {
|
|
228
|
+
export {
|
|
229
|
+
FOOTAGE_FPS,
|
|
230
|
+
FOOTAGE_SAMPLE_RATE,
|
|
231
|
+
X264_PRESETS,
|
|
232
|
+
buildNormalizeArgs,
|
|
233
|
+
buildVideoFilter,
|
|
234
|
+
normalizeFootage,
|
|
235
|
+
resolveFit,
|
|
236
|
+
toneMapFilter,
|
|
237
|
+
type Frame,
|
|
238
|
+
type NormalizeEncoder,
|
|
239
|
+
type NormalizeInput,
|
|
240
|
+
type X264Preset,
|
|
241
|
+
} from "./footage/normalize.js";
|
|
229
242
|
export {
|
|
230
243
|
describePaidSteps,
|
|
231
244
|
toPaidAudio,
|
|
@@ -265,6 +278,7 @@ export {
|
|
|
265
278
|
type FakeSttOptions,
|
|
266
279
|
type FakeSttProvider,
|
|
267
280
|
} from "./footage/fake.js";
|
|
281
|
+
export { SCRIBE_KEYTERM_LIMITS, getFilmKeyTerms, normalizeKeyTerms, type KeyTermsResult } from "./footage/keyterms.js";
|
|
268
282
|
export { footageCacheKey, getFootageCachePath, hashFile, readCachedTranscript, type FootageCacheKind } from "./footage/cache.js";
|
|
269
283
|
export {
|
|
270
284
|
MIN_SEGMENT_SECONDS,
|
|
@@ -294,7 +308,10 @@ export {
|
|
|
294
308
|
CAPTION_SAFE_ZONES,
|
|
295
309
|
findMissingCaptionGlyphs,
|
|
296
310
|
getCaptionPresetLayout,
|
|
311
|
+
getCaptionSpans,
|
|
312
|
+
getFootageCaptionLayout,
|
|
297
313
|
phraseChunks,
|
|
314
|
+
type CaptionSpan,
|
|
298
315
|
type MissingCaptionGlyphs,
|
|
299
316
|
type PhraseChunk,
|
|
300
317
|
type SafeZone,
|
|
@@ -303,6 +320,20 @@ export { MAX_FRAMES, buildFrameArgs, extractFrames, getFrameTimes, type ExtractF
|
|
|
303
320
|
export { duckingKeyframes, getSpeechSpans, volumeExpression, type DuckingOptions, type VolumeKeyframe } from "./footage/music.js";
|
|
304
321
|
export { END_CARD_SECONDS, FADE_SECONDS, composeFootage, getFootageFilmSeconds, getSegmentScales, type FootageComposeAssets, type FootageComposeInput } from "./footage/compose.js";
|
|
305
322
|
export { buildCutPreviewArgs, buildLastFrameArgs, buildMusicArgs, buildSpeechArgs } from "./footage/render.js";
|
|
323
|
+
export {
|
|
324
|
+
FOOTAGE_DRAFT_FONT,
|
|
325
|
+
FOOTAGE_DRAFT_WIDTH,
|
|
326
|
+
buildFootageDraftArgs,
|
|
327
|
+
buildFootageDraftCaptions,
|
|
328
|
+
getFootageDraftFrame,
|
|
329
|
+
planFootageDraft,
|
|
330
|
+
renderFootageDraft,
|
|
331
|
+
type DraftSegmentPlan,
|
|
332
|
+
type FootageDraft,
|
|
333
|
+
type FootageDraftArgsInput,
|
|
334
|
+
type FootageDraftCaptionsInput,
|
|
335
|
+
type RenderFootageDraftOptions,
|
|
336
|
+
} from "./footage/draft.js";
|
|
306
337
|
export {
|
|
307
338
|
DEFAULT_LOUDNESS,
|
|
308
339
|
buildLoudnessArgs,
|
|
@@ -339,7 +370,8 @@ export {
|
|
|
339
370
|
type FootageEstimate,
|
|
340
371
|
type FootageEstimateItem,
|
|
341
372
|
} from "./footage/paid.js";
|
|
342
|
-
export {
|
|
373
|
+
export { RENDER_MIN_FREE_BYTES, findLowDiskProblems, getFreeBytes, getRenderTempDir, type FreeBytesReader } from "./footage/disk.js";
|
|
374
|
+
export { DRAFT_FILTERS, REQUIRED_DECODERS, REQUIRED_FILTERS, checkFootageTools, findBrowserProblem, readFfmpegListing, type FootageToolsOptions } from "./footage/tools.js";
|
|
343
375
|
export {
|
|
344
376
|
cutFootage,
|
|
345
377
|
getEdlPath,
|