@softure-ai/marketing-kit 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +98 -0
- package/README.md +130 -17
- package/dist/cli/footage.d.ts.map +1 -1
- package/dist/cli/footage.js +5 -2
- package/dist/cli/footage.js.map +1 -1
- package/dist/compose/compose.d.ts.map +1 -1
- package/dist/compose/compose.js +3 -3
- package/dist/compose/compose.js.map +1 -1
- package/dist/compose/timeline.d.ts +16 -0
- package/dist/compose/timeline.d.ts.map +1 -1
- package/dist/compose/timeline.js +17 -1
- package/dist/compose/timeline.js.map +1 -1
- package/dist/config/config.d.ts.map +1 -1
- package/dist/config/config.js +25 -2
- package/dist/config/config.js.map +1 -1
- package/dist/config/schema.d.ts +17 -0
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +61 -3
- package/dist/config/schema.js.map +1 -1
- package/dist/footage/audio.d.ts +6 -2
- package/dist/footage/audio.d.ts.map +1 -1
- package/dist/footage/audio.js +15 -3
- package/dist/footage/audio.js.map +1 -1
- package/dist/footage/captions.d.ts +26 -3
- package/dist/footage/captions.d.ts.map +1 -1
- package/dist/footage/captions.js +35 -2
- package/dist/footage/captions.js.map +1 -1
- package/dist/footage/compose.d.ts +13 -2
- package/dist/footage/compose.d.ts.map +1 -1
- package/dist/footage/compose.js +79 -28
- package/dist/footage/compose.js.map +1 -1
- package/dist/footage/cuts.d.ts +44 -0
- package/dist/footage/cuts.d.ts.map +1 -0
- package/dist/footage/cuts.js +83 -0
- package/dist/footage/cuts.js.map +1 -0
- package/dist/footage/disk.d.ts +25 -0
- package/dist/footage/disk.d.ts.map +1 -0
- package/dist/footage/disk.js +72 -0
- package/dist/footage/disk.js.map +1 -0
- package/dist/footage/draft.d.ts +119 -0
- package/dist/footage/draft.d.ts.map +1 -0
- package/dist/footage/draft.js +296 -0
- package/dist/footage/draft.js.map +1 -0
- package/dist/footage/edl.d.ts +2 -1
- package/dist/footage/edl.d.ts.map +1 -1
- package/dist/footage/edl.js +5 -2
- package/dist/footage/edl.js.map +1 -1
- package/dist/footage/elevenlabs.d.ts.map +1 -1
- package/dist/footage/elevenlabs.js +2 -0
- package/dist/footage/elevenlabs.js.map +1 -1
- package/dist/footage/film.d.ts +54 -3
- package/dist/footage/film.d.ts.map +1 -1
- package/dist/footage/film.js +7 -1
- package/dist/footage/film.js.map +1 -1
- package/dist/footage/keyterms.d.ts +26 -0
- package/dist/footage/keyterms.d.ts.map +1 -0
- package/dist/footage/keyterms.js +36 -0
- package/dist/footage/keyterms.js.map +1 -0
- package/dist/footage/loudness.d.ts +20 -3
- package/dist/footage/loudness.d.ts.map +1 -1
- package/dist/footage/loudness.js +36 -18
- package/dist/footage/loudness.js.map +1 -1
- package/dist/footage/music.d.ts +3 -0
- package/dist/footage/music.d.ts.map +1 -1
- package/dist/footage/music.js +5 -0
- package/dist/footage/music.js.map +1 -1
- package/dist/footage/normalize.d.ts +30 -2
- package/dist/footage/normalize.d.ts.map +1 -1
- package/dist/footage/normalize.js +39 -11
- package/dist/footage/normalize.js.map +1 -1
- package/dist/footage/paid.d.ts +2 -1
- package/dist/footage/paid.d.ts.map +1 -1
- package/dist/footage/paid.js +13 -3
- package/dist/footage/paid.js.map +1 -1
- package/dist/footage/picture.d.ts +42 -0
- package/dist/footage/picture.d.ts.map +1 -0
- package/dist/footage/picture.js +141 -0
- package/dist/footage/picture.js.map +1 -0
- package/dist/footage/pipeline.d.ts +66 -4
- package/dist/footage/pipeline.d.ts.map +1 -1
- package/dist/footage/pipeline.js +172 -34
- package/dist/footage/pipeline.js.map +1 -1
- package/dist/footage/process.d.ts +2 -0
- package/dist/footage/process.d.ts.map +1 -1
- package/dist/footage/process.js +1 -0
- package/dist/footage/process.js.map +1 -1
- package/dist/footage/providers.d.ts +12 -3
- package/dist/footage/providers.d.ts.map +1 -1
- package/dist/footage/providers.js.map +1 -1
- package/dist/footage/publish.d.ts +4 -2
- package/dist/footage/publish.d.ts.map +1 -1
- package/dist/footage/publish.js +8 -5
- package/dist/footage/publish.js.map +1 -1
- package/dist/footage/render.d.ts +18 -3
- package/dist/footage/render.d.ts.map +1 -1
- package/dist/footage/render.js +67 -16
- package/dist/footage/render.js.map +1 -1
- package/dist/footage/tools.d.ts +15 -3
- package/dist/footage/tools.d.ts.map +1 -1
- package/dist/footage/tools.js +15 -2
- package/dist/footage/tools.js.map +1 -1
- package/dist/index.d.ts +14 -9
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +14 -9
- package/dist/index.js.map +1 -1
- package/docker/footage-worker.Dockerfile +7 -0
- package/fonts/DejaVuSans-Bold-Latin.ttf +0 -0
- package/fonts/LICENSE-DejaVu.txt +46 -0
- package/fonts/README.md +12 -0
- package/package.json +2 -1
- package/schema/marketing.schema.json +112 -4
- package/src/cli/footage.ts +4 -2
- package/src/compose/compose.ts +13 -3
- package/src/compose/timeline.ts +27 -1
- package/src/config/config.ts +26 -2
- package/src/config/schema.ts +63 -3
- package/src/footage/audio.ts +16 -3
- package/src/footage/captions.ts +45 -4
- package/src/footage/compose.ts +90 -28
- package/src/footage/cuts.ts +104 -0
- package/src/footage/disk.ts +75 -0
- package/src/footage/draft.ts +395 -0
- package/src/footage/edl.ts +5 -2
- package/src/footage/elevenlabs.ts +1 -0
- package/src/footage/film.ts +63 -5
- package/src/footage/keyterms.ts +38 -0
- package/src/footage/loudness.ts +38 -16
- package/src/footage/music.ts +7 -0
- package/src/footage/normalize.ts +51 -10
- package/src/footage/paid.ts +12 -3
- package/src/footage/picture.ts +179 -0
- package/src/footage/pipeline.ts +219 -37
- package/src/footage/process.ts +3 -0
- package/src/footage/providers.ts +10 -3
- package/src/footage/publish.ts +8 -5
- package/src/footage/render.ts +77 -16
- package/src/footage/tools.ts +23 -3
- package/src/index.ts +70 -4
package/src/footage/pipeline.ts
CHANGED
|
@@ -13,13 +13,17 @@ import type { TimedWord } from "../voice/voiceover.js";
|
|
|
13
13
|
import { checkMusicFileType, describeAudioProbe, probeAudio } from "./audio.js";
|
|
14
14
|
import { readCachedTranscript, writeCacheFile, writeTranscript } from "./cache.js";
|
|
15
15
|
import { composeFootage, END_CARD_SECONDS, getFootageFilmSeconds } from "./compose.js";
|
|
16
|
-
import { findMissingCaptionGlyphs,
|
|
17
|
-
import {
|
|
16
|
+
import { findMissingCaptionGlyphs, getFootageCaptionLayout } from "./captions.js";
|
|
17
|
+
import { findLowDiskProblems, getFreeBytes, getRenderTempDir, type FreeBytesReader } from "./disk.js";
|
|
18
|
+
import { buildEdl, getEdlDuration, readEdl, remapWords, validateEdl, type Edl } from "./edl.js";
|
|
18
19
|
import { makePlaceholderTranscript } from "./fake.js";
|
|
19
|
-
import {
|
|
20
|
-
import {
|
|
21
|
-
import {
|
|
22
|
-
import {
|
|
20
|
+
import { chooseCutTransitions, type CutMeasure } from "./cuts.js";
|
|
21
|
+
import { DEFAULT_AUTO_THRESHOLD, getFootageEdit, type FootageClip, type FootageFilm, type MusicSource } from "./film.js";
|
|
22
|
+
import { getFilmKeyTerms } from "./keyterms.js";
|
|
23
|
+
import { DEFAULT_LOUDNESS, getLoudnessFilter, measureLoudness, normalizeLoudness } from "./loudness.js";
|
|
24
|
+
import { duckingKeyframes, getDuckingOptions } from "./music.js";
|
|
25
|
+
import { FOOTAGE_FPS, normalizeFootage, type Frame, type NormalizeEncoder } from "./normalize.js";
|
|
26
|
+
import { buildPictureConcatArgs, buildPictureList, buildPicturePieceArgs, getPictureFrames, planFootagePicture } from "./picture.js";
|
|
23
27
|
import { describeProbe, getTrimmedDuration, probeFootage, type FootageProbe } from "./probe.js";
|
|
24
28
|
import {
|
|
25
29
|
getClipCacheKey,
|
|
@@ -47,7 +51,7 @@ import {
|
|
|
47
51
|
readTopLevelBoxes,
|
|
48
52
|
} from "./publish.js";
|
|
49
53
|
import { describePaidSteps, toPaidAudio, type FootageCharge, type FootageProgress, type FootageProviders, type FootageStage, type PaidAudio, type PaidStep } from "./providers.js";
|
|
50
|
-
import { buildAudioArgs, buildCutPreviewArgs, buildLastFrameArgs, buildMusicArgs, buildSpeechArgs } from "./render.js";
|
|
54
|
+
import { buildAudioArgs, buildCutPreviewArgs, buildLastFrameArgs, buildMusicArgs, buildSpeechArgs, getSpeechCutOptions } from "./render.js";
|
|
51
55
|
|
|
52
56
|
export { getEdlPath } from "./paid.js";
|
|
53
57
|
|
|
@@ -72,7 +76,7 @@ export { getEdlPath } from "./paid.js";
|
|
|
72
76
|
|
|
73
77
|
const require = createRequire(import.meta.url);
|
|
74
78
|
/** Bumped when the normalized format changes, so older normalized clips are made again. */
|
|
75
|
-
const NORMALIZE_VERSION =
|
|
79
|
+
const NORMALIZE_VERSION = 2;
|
|
76
80
|
|
|
77
81
|
export interface FootageBrand {
|
|
78
82
|
name: string;
|
|
@@ -100,6 +104,11 @@ export interface FootageRunOptions {
|
|
|
100
104
|
onProgress?: (progress: FootageProgress) => void;
|
|
101
105
|
/** Called after each paid call, for the app's own spend ledger. */
|
|
102
106
|
onCharge?: (charge: FootageCharge) => void;
|
|
107
|
+
/**
|
|
108
|
+
* How the normalized clips are encoded (`preset`, `threads`): a small worker trades file size for speed. Not part of
|
|
109
|
+
* the normalize stamp, so changing it does not normalize clips already made again.
|
|
110
|
+
*/
|
|
111
|
+
normalize?: NormalizeEncoder;
|
|
103
112
|
}
|
|
104
113
|
|
|
105
114
|
export interface PreparedClip {
|
|
@@ -178,7 +187,7 @@ function getNormalizeStamp(clip: FootageClip, source: string, frame: Frame): str
|
|
|
178
187
|
/** Downloads URL clips, probes every clip (all problems at once) and a local music file, normalizes each clip that changed. */
|
|
179
188
|
export async function prepareFootage(
|
|
180
189
|
film: FootageFilm,
|
|
181
|
-
options: Pick<FootageRunOptions, "buildDir" | "log" | "fetch" | "signal" | "onProgress">,
|
|
190
|
+
options: Pick<FootageRunOptions, "buildDir" | "log" | "fetch" | "signal" | "onProgress" | "normalize">,
|
|
182
191
|
): Promise<FootageResult<PreparedClip[]>> {
|
|
183
192
|
const { buildDir, log, signal } = options;
|
|
184
193
|
if (film.clips.length === 0) return err(`footage film ${film.id} has no clips.`);
|
|
@@ -196,7 +205,7 @@ export async function prepareFootage(
|
|
|
196
205
|
const probed = await probeFootage(film.clips, sources, signal === undefined ? {} : { signal });
|
|
197
206
|
if (!probed.ok) return err(probed.errors.join("\n"));
|
|
198
207
|
if (film.music.source.kind === "file") {
|
|
199
|
-
const music = await checkMusicFile(film.music.source
|
|
208
|
+
const music = await checkMusicFile(film.music.source, options);
|
|
200
209
|
if (!music.ok) return err(music.error);
|
|
201
210
|
}
|
|
202
211
|
const frame = getFrame(film);
|
|
@@ -219,7 +228,7 @@ export async function prepareFootage(
|
|
|
219
228
|
// The stamp goes first, so a normalize stopped half-way never leaves a stamp that vouches for a partial clip.
|
|
220
229
|
rmSync(stampPath, { force: true });
|
|
221
230
|
await normalizeFootage(
|
|
222
|
-
{ clip: clip.clip, file: clip.source, probe: clip.probe, frame, output: clip.file },
|
|
231
|
+
{ clip: clip.clip, file: clip.source, probe: clip.probe, frame, output: clip.file, ...(options.normalize === undefined ? {} : { encoder: options.normalize }) },
|
|
223
232
|
{ ...(signal === undefined ? {} : { signal }), onSeconds: (seconds) => progress(index, seconds) },
|
|
224
233
|
);
|
|
225
234
|
writeFileSync(stampPath, stamp);
|
|
@@ -268,6 +277,9 @@ export function getMusicPaidStep(film: FootageFilm, options: Pick<FootageRunOpti
|
|
|
268
277
|
export async function transcribeFootage(film: FootageFilm, clips: PreparedClip[], options: FootageRunOptions): Promise<FootageResult<TranscribeOutcome>> {
|
|
269
278
|
const { buildDir, providers, log, signal } = options;
|
|
270
279
|
if (options.isPlaceholder) return ok(transcribePlaceholder(film, clips, buildDir));
|
|
280
|
+
const keyTerms = getFilmKeyTerms(film);
|
|
281
|
+
if (!keyTerms.ok) return err(`footage film ${film.id}: ${keyTerms.errors.join(" ")}`);
|
|
282
|
+
const sttKeyTerms = keyTerms.terms.length === 0 ? {} : { keyTerms: keyTerms.terms };
|
|
271
283
|
const plan: { clip: PreparedClip; isolated: CacheEntry | null; transcript: CacheEntry }[] = [];
|
|
272
284
|
const steps: PaidStep[] = [];
|
|
273
285
|
for (const clip of clips) {
|
|
@@ -341,12 +353,12 @@ export async function transcribeFootage(film: FootageFilm, clips: PreparedClip[]
|
|
|
341
353
|
const input = sttInput ?? (await getAudio());
|
|
342
354
|
throwIfAborted(signal);
|
|
343
355
|
log(`speech to text (${providers.stt.id}): clip ${id}, ${clip.duration.toFixed(1)} s.`);
|
|
344
|
-
const result = await providers.stt.transcribe({ audio: input, language: film.language, ...(signal === undefined ? {} : { signal }) });
|
|
356
|
+
const result = await providers.stt.transcribe({ audio: input, language: film.language, ...sttKeyTerms, ...(signal === undefined ? {} : { signal }) });
|
|
345
357
|
throwIfAborted(signal);
|
|
346
358
|
if (!result.ok) return err(`clip "${id}": ${result.error}`);
|
|
347
359
|
writeTranscript(transcript.path, result.value.words);
|
|
348
360
|
const { credits } = result.value;
|
|
349
|
-
options.onCharge?.({ step: "transcribe", item: id, provider: providers.stt.id, units: clip.duration, ...(credits === undefined ? {} : { credits }), cacheKey: transcript.key });
|
|
361
|
+
options.onCharge?.({ step: "transcribe", item: id, provider: providers.stt.id, units: clip.duration, ...(credits === undefined ? {} : { credits }), ...(keyTerms.terms.length === 0 ? {} : { keyTerms: keyTerms.terms.length }), cacheKey: transcript.key });
|
|
350
362
|
log(`transcript: clip ${id}, ${result.value.words.length} words, saved ${transcript.path} (commit it).`);
|
|
351
363
|
transcribeProgress(index, clip.duration);
|
|
352
364
|
words[id] = result.value.words;
|
|
@@ -359,20 +371,77 @@ export function getFootageSentences(clips: PreparedClip[], words: Record<string,
|
|
|
359
371
|
return splitFilmSentences(clips.map((clip) => ({ id: clip.clip.id, role: clip.clip.role, words: words[clip.clip.id] ?? [] })));
|
|
360
372
|
}
|
|
361
373
|
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
const edl = buildEdl(
|
|
374
|
+
function buildFilmEdl(film: FootageFilm, clips: PreparedClip[], words: Record<string, TimedWord[]>, keepSentences: readonly string[] | undefined): Edl {
|
|
375
|
+
return buildEdl(
|
|
365
376
|
film.id,
|
|
366
377
|
clips.map((clip) => ({ id: clip.clip.id, role: clip.clip.role, duration: clip.duration, words: words[clip.clip.id] ?? [] })),
|
|
367
378
|
film.cut,
|
|
368
379
|
film.transitions,
|
|
369
|
-
{ isKenBurns: getFootageEdit(film).kenBurnsPercent > 0, ...(
|
|
380
|
+
{ isKenBurns: getFootageEdit(film).kenBurnsPercent > 0, ...(keepSentences === undefined ? {} : { keepSentences }) },
|
|
370
381
|
);
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
function writeEdl(edl: Edl, edlPath: string): void {
|
|
371
385
|
mkdirSync(dirname(edlPath), { recursive: true });
|
|
372
386
|
writeFileSync(edlPath, `${JSON.stringify(edl, null, 2)}\n`);
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
/**
|
|
390
|
+
* The cut from the transcripts, written to the EDL file (replacing what is there); `keepSentences` keeps only those
|
|
391
|
+
* sentences. Sync, so it never looks at the frames: with `transitions.jump: "auto"` the jump cuts stay plain cuts, and
|
|
392
|
+
* `cutFootageFilm` is the step that chooses them.
|
|
393
|
+
*/
|
|
394
|
+
export function cutFootage(film: FootageFilm, clips: PreparedClip[], words: Record<string, TimedWord[]>, edlPath: string, options: { keepSentences?: readonly string[] } = {}): Edl {
|
|
395
|
+
const edl = buildFilmEdl(film, clips, words, options.keepSentences);
|
|
396
|
+
writeEdl(edl, edlPath);
|
|
373
397
|
return edl;
|
|
374
398
|
}
|
|
375
399
|
|
|
400
|
+
export interface CutFootageFilmOptions {
|
|
401
|
+
/** The sentence ids to keep; absent keeps all. */
|
|
402
|
+
keepSentences?: readonly string[];
|
|
403
|
+
signal?: AbortSignal;
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
/** The file next to the EDL that records what the auto transitions measured. */
|
|
407
|
+
export const CUTS_FILE = "cuts.json";
|
|
408
|
+
|
|
409
|
+
/**
|
|
410
|
+
* The cut as `cutFootage` builds it, with `transitions.jump: "auto"` resolved from the frames of the normalized clips
|
|
411
|
+
* (`chooseCutTransitions`): each jump cut gets `zoom` or `fade`, and what it measured is answered and written to
|
|
412
|
+
* `cuts.json` next to the EDL, so an app or an agent sees why. Otherwise the same as `cutFootage`, and `cuts` is empty.
|
|
413
|
+
*/
|
|
414
|
+
export async function cutFootageFilm(
|
|
415
|
+
film: FootageFilm,
|
|
416
|
+
clips: PreparedClip[],
|
|
417
|
+
words: Record<string, TimedWord[]>,
|
|
418
|
+
edlPath: string,
|
|
419
|
+
options: CutFootageFilmOptions = {},
|
|
420
|
+
): Promise<{ edl: Edl; cuts: CutMeasure[] }> {
|
|
421
|
+
let edl = buildFilmEdl(film, clips, words, options.keepSentences);
|
|
422
|
+
let cuts: CutMeasure[] = [];
|
|
423
|
+
const cutsPath = join(dirname(edlPath), CUTS_FILE);
|
|
424
|
+
rmSync(cutsPath, { force: true });
|
|
425
|
+
if (film.transitions.jump === "auto") {
|
|
426
|
+
const chosen = await chooseCutTransitions(edl, Object.fromEntries(clips.map((clip) => [clip.clip.id, clip.file])), {
|
|
427
|
+
workDir: join(dirname(edlPath), "cut-frames"),
|
|
428
|
+
...(film.transitions.autoThreshold === undefined ? {} : { threshold: film.transitions.autoThreshold }),
|
|
429
|
+
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
430
|
+
});
|
|
431
|
+
edl = chosen.edl;
|
|
432
|
+
cuts = chosen.cuts;
|
|
433
|
+
mkdirSync(dirname(cutsPath), { recursive: true });
|
|
434
|
+
writeFileSync(cutsPath, `${JSON.stringify({ threshold: film.transitions.autoThreshold ?? DEFAULT_AUTO_THRESHOLD, cuts }, null, 2)}\n`);
|
|
435
|
+
}
|
|
436
|
+
writeEdl(edl, edlPath);
|
|
437
|
+
return { edl, cuts };
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
/** One line per auto-chosen cut, for a log: `s4: zoom (difference 0.031)`. */
|
|
441
|
+
export function describeCutMeasures(cuts: CutMeasure[]): string[] {
|
|
442
|
+
return cuts.map((cut) => `${cut.segment}: ${cut.transition} (difference ${cut.difference.toFixed(3)})`);
|
|
443
|
+
}
|
|
444
|
+
|
|
376
445
|
/** The EDL file as written by `cutFootage` or edited by hand, checked against the clips. */
|
|
377
446
|
export function readEdlFile(film: FootageFilm, clips: PreparedClip[], edlPath: string): FootageResult<Edl> {
|
|
378
447
|
let json: unknown;
|
|
@@ -387,11 +456,21 @@ export function readEdlFile(film: FootageFilm, clips: PreparedClip[], edlPath: s
|
|
|
387
456
|
return problems.length > 0 ? err(`${edlPath}: ${problems.join("; ")}`) : ok(read.edl);
|
|
388
457
|
}
|
|
389
458
|
|
|
390
|
-
/**
|
|
391
|
-
|
|
459
|
+
/**
|
|
460
|
+
* A local music file checked like an app's upload: an accepted type, then readable audio (`probeAudio`). The type comes
|
|
461
|
+
* from the source's declared name (else the path) and content type; with no extension and no content type (a storage
|
|
462
|
+
* key) the type check is skipped and the probe alone decides.
|
|
463
|
+
*/
|
|
464
|
+
async function checkMusicFile(source: Extract<MusicSource, { kind: "file" }>, options: Pick<FootageRunOptions, "log" | "signal">): Promise<FootageResult<string>> {
|
|
465
|
+
const { path, fileName, contentType } = source;
|
|
392
466
|
if (!existsSync(path)) return err(`music: no file at ${path}`);
|
|
393
|
-
const
|
|
394
|
-
if (
|
|
467
|
+
const name = fileName ?? path;
|
|
468
|
+
if (extname(name) === "" && contentType === undefined) {
|
|
469
|
+
options.log(`music: ${path} has no extension and no declared name or content type; the probe checks it.`);
|
|
470
|
+
} else {
|
|
471
|
+
const type = checkMusicFileType(contentType === undefined ? { fileName: name } : { fileName: name, contentType });
|
|
472
|
+
if (!type.ok) return err(`music: ${type.error}`);
|
|
473
|
+
}
|
|
395
474
|
const probe = await probeAudio(path, options.signal === undefined ? {} : { signal: options.signal });
|
|
396
475
|
if (!probe.ok) return err(`music: ${probe.error}`);
|
|
397
476
|
options.log(`music: ${path} (${describeAudioProbe(probe.probe)}).`);
|
|
@@ -459,11 +538,32 @@ export interface RenderFootageOptions extends FootageRunOptions {
|
|
|
459
538
|
output: string;
|
|
460
539
|
/** A project's override of the format's caption box (`marketing.json` `layout`). */
|
|
461
540
|
layout?: LayoutOverride;
|
|
541
|
+
/**
|
|
542
|
+
* hyperframes' temp folder, passed to it as `TMPDIR` (created when missing): the clips' frames are extracted there,
|
|
543
|
+
* so it needs room (a data volume, not a small tmpfs). By default the `TMPDIR` the process already has.
|
|
544
|
+
*/
|
|
545
|
+
tempDir?: string;
|
|
546
|
+
/** Reads a folder's free bytes for the disk check; injected in tests. */
|
|
547
|
+
readFreeBytes?: FreeBytesReader;
|
|
548
|
+
/**
|
|
549
|
+
* hyperframes' capture workers (`--workers`), each a Chrome of its own; by default hyperframes' choice (one in its
|
|
550
|
+
* low-memory profile). A positive whole number.
|
|
551
|
+
*/
|
|
552
|
+
workers?: number;
|
|
553
|
+
/**
|
|
554
|
+
* hyperframes' low-memory profile (`--low-memory-mode`: one worker, screenshot capture): true forces it on, false
|
|
555
|
+
* off; by default hyperframes turns it on by itself on a machine (or cgroup) of 8 GB or less.
|
|
556
|
+
*/
|
|
557
|
+
lowMemory?: boolean;
|
|
462
558
|
}
|
|
463
559
|
|
|
464
560
|
export type RenderedFootage = { kind: "dry-run"; steps: PaidStep[] } | ({ kind: "rendered"; output: string; seconds: number; edlPath: string } & FinishedFootage);
|
|
465
561
|
|
|
466
|
-
/**
|
|
562
|
+
/**
|
|
563
|
+
* Composes the cut and renders it: speech track (loudness-normalized), ducked music, end card backdrop, assets,
|
|
564
|
+
* hyperframes, final pass, cover. First of all it checks that hyperframes' temp folder and the build folder have the
|
|
565
|
+
* 1 GiB free hyperframes requires, so a full disk ends the run with a sentence before anything is bought or built.
|
|
566
|
+
*/
|
|
467
567
|
export async function renderFootage(
|
|
468
568
|
film: FootageFilm,
|
|
469
569
|
clips: PreparedClip[],
|
|
@@ -472,6 +572,8 @@ export async function renderFootage(
|
|
|
472
572
|
options: RenderFootageOptions,
|
|
473
573
|
): Promise<FootageResult<RenderedFootage>> {
|
|
474
574
|
const { buildDir, log, signal } = options;
|
|
575
|
+
const lowDisk = findLowDiskProblems({ temp: getRenderTempDir(process.env, options.tempDir), build: buildDir }, options.readFreeBytes ?? getFreeBytes);
|
|
576
|
+
if (lowDisk.length > 0) return err(`not enough disk space to render: ${lowDisk.join(" ")}`);
|
|
475
577
|
const seconds = getFootageFilmSeconds(film, edl);
|
|
476
578
|
const words = remapWords(edl, transcript.words, film.cut.fillers);
|
|
477
579
|
const music = await getMusic(film, edl, clips, seconds, options);
|
|
@@ -492,7 +594,7 @@ export async function renderFootage(
|
|
|
492
594
|
const rawSpeechFile = loudness === null ? speechFile : join(assetsDir, "speech.raw.mp3");
|
|
493
595
|
const ffmpegTrack = (label: string, output: string, args: () => string[]) => (onSeconds: (done: number) => void) => runFfmpeg(label, args(), { ...aborting, output, onSeconds });
|
|
494
596
|
const tracks: { item: string; seconds: number; run: (onSeconds: (done: number) => void) => Promise<void> }[] = [
|
|
495
|
-
{ item: "speech", seconds, run: ffmpegTrack("The speech track", rawSpeechFile, () => buildSpeechArgs(edl, transcript.speech, rawSpeechFile)) },
|
|
597
|
+
{ item: "speech", seconds, run: ffmpegTrack("The speech track", rawSpeechFile, () => buildSpeechArgs(edl, transcript.speech, rawSpeechFile, getSpeechCutOptions(getFootageEdit(film)))) },
|
|
496
598
|
];
|
|
497
599
|
if (loudness !== null) {
|
|
498
600
|
tracks.push({
|
|
@@ -509,13 +611,21 @@ export async function renderFootage(
|
|
|
509
611
|
});
|
|
510
612
|
}
|
|
511
613
|
if (musicFile !== null) {
|
|
512
|
-
const keyframes = duckingKeyframes(words,
|
|
614
|
+
const keyframes = duckingKeyframes(words, getDuckingOptions(film.music));
|
|
513
615
|
const output = join(assetsDir, "music.mp3");
|
|
514
616
|
tracks.push({ item: "music", seconds, run: ffmpegTrack("The music track", output, () => buildMusicArgs({ input: musicFile, seconds, keyframes, output })) });
|
|
515
617
|
}
|
|
516
618
|
// A single frame: counted as a moment of work so the stage's bar still ends at its total.
|
|
517
619
|
const lastFrame = join(assetsDir, "last.jpg");
|
|
518
620
|
tracks.push({ item: "last frame", seconds: 0.1, run: ffmpegTrack("The end card's backdrop", lastFrame, () => buildLastFrameArgs(edl, clipFiles, lastFrame)) });
|
|
621
|
+
// The cut's picture in film time, one segment after the other: the composition plays it as one <video> (#1030).
|
|
622
|
+
const pictureFile = join(assetsDir, "picture.mp4");
|
|
623
|
+
const pictureSeconds = getEdlDuration(edl);
|
|
624
|
+
tracks.push({
|
|
625
|
+
item: "picture",
|
|
626
|
+
seconds: pictureSeconds,
|
|
627
|
+
run: (onSeconds) => buildPicture({ film, edl, clips: clipFiles, output: pictureFile, ...(options.normalize === undefined ? {} : { encoder: options.normalize }), ...aborting, onSeconds }),
|
|
628
|
+
});
|
|
519
629
|
const assetProgress = createStageProgress(options.onProgress, "assets", tracks);
|
|
520
630
|
for (const [index, track] of tracks.entries()) {
|
|
521
631
|
assetProgress(index, 0);
|
|
@@ -535,8 +645,7 @@ export async function renderFootage(
|
|
|
535
645
|
cpSync(require.resolve("gsap/dist/gsap.min.js"), join(assetsDir, "gsap.min.js"));
|
|
536
646
|
const base = resolveLayout(film.format, options.layout ?? {});
|
|
537
647
|
// A caption preset places the 9:16 box in a platform's safe zone; the project's own caption override still wins.
|
|
538
|
-
const
|
|
539
|
-
const layout = preset === null || options.layout?.caption !== undefined ? base : { ...base, caption: preset };
|
|
648
|
+
const layout = { ...base, caption: getFootageCaptionLayout(film, options.layout ?? {}) };
|
|
540
649
|
const missing = findMissingCaptionGlyphs({
|
|
541
650
|
fonts: options.brand.fonts,
|
|
542
651
|
captions: film.captions ? words.map((word) => word.text) : [],
|
|
@@ -557,6 +666,7 @@ export async function renderFootage(
|
|
|
557
666
|
fonts,
|
|
558
667
|
assets: {
|
|
559
668
|
clips: Object.fromEntries(clips.map((clip) => [clip.clip.id, `clips/${clip.clip.id}.mp4`])),
|
|
669
|
+
picture: "assets/picture.mp4",
|
|
560
670
|
speech: "assets/speech.mp3",
|
|
561
671
|
music: musicAsset,
|
|
562
672
|
lastFrame: "assets/last.jpg",
|
|
@@ -590,13 +700,14 @@ export interface FinishedFootage {
|
|
|
590
700
|
}
|
|
591
701
|
|
|
592
702
|
/**
|
|
593
|
-
* The final pass over hyperframes' render:
|
|
594
|
-
*
|
|
703
|
+
* The final pass over hyperframes' render: the mix normalized to `film.mixLoudness` (measured on the render, one
|
|
704
|
+
* linear gain; the picture still copied), publishable codecs and fast start (stream copy where the render already
|
|
705
|
+
* matches), checked with ffprobe, then the cover and the file's facts. Exported for tests only.
|
|
595
706
|
*/
|
|
596
|
-
async function finishFootage(
|
|
597
|
-
film: FootageFilm,
|
|
707
|
+
export async function finishFootage(
|
|
708
|
+
film: Pick<FootageFilm, "mixLoudness" | "cover">,
|
|
598
709
|
input: { renderFile: string; seconds: number; coverAt: number },
|
|
599
|
-
options: RenderFootageOptions,
|
|
710
|
+
options: Pick<RenderFootageOptions, "log" | "signal" | "output" | "onProgress">,
|
|
600
711
|
): Promise<FootageResult<FinishedFootage>> {
|
|
601
712
|
const { log, signal, output } = options;
|
|
602
713
|
const aborting = signal === undefined ? {} : { signal };
|
|
@@ -604,8 +715,21 @@ async function finishFootage(
|
|
|
604
715
|
progress(0, 0);
|
|
605
716
|
const rendered = await probeFilmFacts(input.renderFile, aborting);
|
|
606
717
|
if (!rendered.ok) return err(`the rendered film: ${rendered.error}`);
|
|
607
|
-
const
|
|
608
|
-
|
|
718
|
+
const mixLoudness = film.mixLoudness === undefined ? DEFAULT_LOUDNESS : film.mixLoudness;
|
|
719
|
+
let audioFilter: string | undefined;
|
|
720
|
+
if (mixLoudness !== null && rendered.facts.audio !== null) {
|
|
721
|
+
const measured = await measureLoudness(input.renderFile, mixLoudness, { label: "the film's mix", ...aborting });
|
|
722
|
+
if (measured.kind === "silent") {
|
|
723
|
+
log("loudness: the film's mix is silent, left as it is.");
|
|
724
|
+
} else {
|
|
725
|
+
audioFilter = getLoudnessFilter(measured.measurement, mixLoudness);
|
|
726
|
+
log(`loudness: the film's mix measured ${measured.measurement.inputI.toFixed(1)} LUFS, normalized to ${mixLoudness.target} LUFS (true peak ${mixLoudness.truePeak} dBTP).`);
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
const isVideoCopied = isPublishableVideo(rendered.facts.video);
|
|
730
|
+
const isAudioCopied = audioFilter === undefined && (rendered.facts.audio === null || isPublishableAudio(rendered.facts.audio));
|
|
731
|
+
const finishArgs = buildFinishArgs(input.renderFile, rendered.facts, output, audioFilter === undefined ? {} : { audioFilter });
|
|
732
|
+
await runFfmpeg("The film's final pass", finishArgs, { ...aborting, output, onSeconds: (done) => progress(0, done) });
|
|
609
733
|
rmSync(input.renderFile, { force: true });
|
|
610
734
|
const finished = await probeFilmFacts(output, aborting);
|
|
611
735
|
if (!finished.ok) return err(`the finished film: ${finished.error}`);
|
|
@@ -613,7 +737,8 @@ async function finishFootage(
|
|
|
613
737
|
if (problems.length > 0) return err(`the finished film ${output} is not publishable: ${problems.join("; ")}.`);
|
|
614
738
|
progress(0, input.seconds);
|
|
615
739
|
const { bytes, width, height, durationSeconds } = finished.facts;
|
|
616
|
-
|
|
740
|
+
const streams = isVideoCopied && isAudioCopied ? "streams copied" : isVideoCopied ? "picture copied, sound re-encoded" : "re-encoded";
|
|
741
|
+
log(`finish: ${streams} to H.264 High, AAC 48 kHz stereo, fast start; ${width}×${height}, ${durationSeconds.toFixed(2)} s, ${bytes} bytes.`);
|
|
617
742
|
let cover: string | null = null;
|
|
618
743
|
if (film.cover !== null) {
|
|
619
744
|
cover = getCoverPath(output);
|
|
@@ -634,6 +759,52 @@ export function readRenderPercent(line: string): number | null {
|
|
|
634
759
|
return percent >= 0 && percent <= 100 ? percent : null;
|
|
635
760
|
}
|
|
636
761
|
|
|
762
|
+
/** hyperframes' render arguments: the quality, the output and the memory settings the caller set. Throws on invalid `workers`. */
|
|
763
|
+
export function getHyperframesRenderArgs(options: Pick<RenderFootageOptions, "quality" | "workers" | "lowMemory">, output: string): string[] {
|
|
764
|
+
const args = ["render", "--quality", options.quality];
|
|
765
|
+
if (options.workers !== undefined) {
|
|
766
|
+
if (!Number.isInteger(options.workers) || options.workers < 1) throw new Error(`workers must be a positive whole number, got ${options.workers}.`);
|
|
767
|
+
args.push("--workers", String(options.workers));
|
|
768
|
+
}
|
|
769
|
+
if (options.lowMemory !== undefined) args.push(options.lowMemory ? "--low-memory-mode" : "--no-low-memory-mode");
|
|
770
|
+
return [...args, "-o", output];
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
interface BuildPictureInput {
|
|
774
|
+
film: FootageFilm;
|
|
775
|
+
edl: Edl;
|
|
776
|
+
clips: Record<string, string>;
|
|
777
|
+
output: string;
|
|
778
|
+
encoder?: NormalizeEncoder;
|
|
779
|
+
signal?: AbortSignal;
|
|
780
|
+
onSeconds: (done: number) => void;
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
/** Renders the picture piece by piece (one ffmpeg at a time) and joins the pieces; the pieces are removed after the join. */
|
|
784
|
+
async function buildPicture(input: BuildPictureInput): Promise<void> {
|
|
785
|
+
const pieces = planFootagePicture(input.film, input.edl);
|
|
786
|
+
const dir = join(dirname(input.output), "picture");
|
|
787
|
+
rmSync(dir, { recursive: true, force: true });
|
|
788
|
+
mkdirSync(dir, { recursive: true });
|
|
789
|
+
const aborting = input.signal === undefined ? {} : { signal: input.signal };
|
|
790
|
+
const files: string[] = [];
|
|
791
|
+
for (const [index, piece] of pieces.entries()) {
|
|
792
|
+
const file = join(dir, `${String(index).padStart(3, "0")}.mp4`);
|
|
793
|
+
const done = piece.frame / FOOTAGE_FPS;
|
|
794
|
+
await runFfmpeg(`The picture of segment ${index + 1}`, buildPicturePieceArgs({ piece, clips: input.clips, output: file, ...(input.encoder === undefined ? {} : { encoder: input.encoder }) }), {
|
|
795
|
+
...aborting,
|
|
796
|
+
output: file,
|
|
797
|
+
onSeconds: (seconds) => input.onSeconds(done + seconds),
|
|
798
|
+
});
|
|
799
|
+
files.push(file);
|
|
800
|
+
}
|
|
801
|
+
const list = join(dir, "pieces.txt");
|
|
802
|
+
writeFileSync(list, buildPictureList(files));
|
|
803
|
+
await runFfmpeg("The picture's join", buildPictureConcatArgs(list, input.output), { ...aborting, output: input.output });
|
|
804
|
+
rmSync(dir, { recursive: true, force: true });
|
|
805
|
+
input.onSeconds(getPictureFrames(pieces) / FOOTAGE_FPS);
|
|
806
|
+
}
|
|
807
|
+
|
|
637
808
|
/** The hyperframes render as a child process: its lines go to the log (progress every 10 %), its percentages to `onProgress`. */
|
|
638
809
|
async function runHyperframesRender(options: RenderFootageOptions, output: string, seconds: number): Promise<FootageResult<null>> {
|
|
639
810
|
const progress = createStageProgress(options.onProgress, "render", [{ item: "film", seconds }]);
|
|
@@ -654,11 +825,17 @@ async function runHyperframesRender(options: RenderFootageOptions, output: strin
|
|
|
654
825
|
options.log(line.trim());
|
|
655
826
|
}
|
|
656
827
|
};
|
|
828
|
+
const env = getHyperframesEnv();
|
|
829
|
+
if (options.tempDir !== undefined && options.tempDir.length > 0) {
|
|
830
|
+
const tempDir = resolve(options.tempDir);
|
|
831
|
+
mkdirSync(tempDir, { recursive: true });
|
|
832
|
+
env.TMPDIR = tempDir;
|
|
833
|
+
}
|
|
657
834
|
const result = await runProcess({
|
|
658
835
|
command: process.execPath,
|
|
659
|
-
args: [getHyperframesBin(),
|
|
836
|
+
args: [getHyperframesBin(), ...getHyperframesRenderArgs(options, output)],
|
|
660
837
|
cwd: options.buildDir,
|
|
661
|
-
env
|
|
838
|
+
env,
|
|
662
839
|
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
663
840
|
removeOnAbort: [output],
|
|
664
841
|
onStdoutLine: onLine,
|
|
@@ -695,8 +872,13 @@ export async function renderFootageFilm(film: FootageFilm, options: RenderFootag
|
|
|
695
872
|
edl = read.value;
|
|
696
873
|
options.log(`cut: ${edlPath} (${edl.segments.length} segments), as it is; softure-marketing footage --step=cut builds it again.`);
|
|
697
874
|
} else {
|
|
698
|
-
|
|
875
|
+
const cut = await cutFootageFilm(film, clips.value, transcript.value.words, edlPath, {
|
|
876
|
+
...(options.keepSentences === undefined ? {} : { keepSentences: options.keepSentences }),
|
|
877
|
+
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
878
|
+
});
|
|
879
|
+
edl = cut.edl;
|
|
699
880
|
options.log(`cut: ${edl.segments.length} segments written to ${edlPath}.`);
|
|
881
|
+
for (const line of describeCutMeasures(cut.cuts)) options.log(` auto transition ${line}`);
|
|
700
882
|
}
|
|
701
883
|
return renderFootage(film, clips.value, transcript.value, edl, options);
|
|
702
884
|
}
|
package/src/footage/process.ts
CHANGED
|
@@ -109,6 +109,8 @@ export interface FfmpegRunOptions {
|
|
|
109
109
|
output?: string;
|
|
110
110
|
/** Seconds of output written so far, as ffmpeg reports them. */
|
|
111
111
|
onSeconds?: (seconds: number) => void;
|
|
112
|
+
/** The folder ffmpeg runs in, so relative names in the arguments and the filter graph resolve there. */
|
|
113
|
+
cwd?: string;
|
|
112
114
|
}
|
|
113
115
|
|
|
114
116
|
/** Runs ffmpeg with its progress on stdout; throws naming the step when it fails (a broken input or machine). */
|
|
@@ -118,6 +120,7 @@ export async function runFfmpeg(step: string, args: string[], options: FfmpegRun
|
|
|
118
120
|
command: "ffmpeg",
|
|
119
121
|
args: ["-progress", "pipe:1", "-nostats", ...args],
|
|
120
122
|
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
123
|
+
...(options.cwd === undefined ? {} : { cwd: options.cwd }),
|
|
121
124
|
removeOnAbort: options.output === undefined ? [] : [options.output],
|
|
122
125
|
onStdoutLine: (line) => {
|
|
123
126
|
const seconds = readFfmpegProgressSeconds(line);
|
package/src/footage/providers.ts
CHANGED
|
@@ -11,6 +11,8 @@ export interface SttInput {
|
|
|
11
11
|
audio: string;
|
|
12
12
|
/** ISO 639 code the speech is in. */
|
|
13
13
|
language: string;
|
|
14
|
+
/** Key terms the transcript should spell as given (normalized by `normalizeKeyTerms`); absent: none. */
|
|
15
|
+
keyTerms?: string[];
|
|
14
16
|
/** Aborts the request. */
|
|
15
17
|
signal?: AbortSignal;
|
|
16
18
|
}
|
|
@@ -87,8 +89,11 @@ export function describePaidSteps(steps: PaidStep[]): string[] {
|
|
|
87
89
|
return lines;
|
|
88
90
|
}
|
|
89
91
|
|
|
90
|
-
/**
|
|
91
|
-
|
|
92
|
+
/**
|
|
93
|
+
* Credits per second billed, for each paid step; the caller's prices (the kit hard-codes none). `keyTerms` is the
|
|
94
|
+
* surcharge per second of speech to text with key terms (keyterm prompting), added to `transcribe`; absent: 0.
|
|
95
|
+
*/
|
|
96
|
+
export type FootageRates = Record<PaidStepKind, number> & { keyTerms?: number };
|
|
92
97
|
|
|
93
98
|
/** One paid call made, as `onCharge` reports it: what for, how much was billed and where its result is cached. */
|
|
94
99
|
export interface FootageCharge {
|
|
@@ -100,12 +105,14 @@ export interface FootageCharge {
|
|
|
100
105
|
units: number;
|
|
101
106
|
/** The provider's reported charge, when its API returns one. */
|
|
102
107
|
credits?: number;
|
|
108
|
+
/** How many key terms a speech to text call sent (billed with keyterm prompting's surcharge); absent: none. */
|
|
109
|
+
keyTerms?: number;
|
|
103
110
|
/** The paid cache's key of the result (the file name's first part under `<cacheDir>/<film id>/`). */
|
|
104
111
|
cacheKey: string;
|
|
105
112
|
}
|
|
106
113
|
|
|
107
114
|
/** A stage of a footage run that reports progress. */
|
|
108
|
-
export type FootageStage = "normalize" | PaidStepKind | "assets" | "render" | "finish";
|
|
115
|
+
export type FootageStage = "normalize" | PaidStepKind | "assets" | "render" | "finish" | "draft";
|
|
109
116
|
|
|
110
117
|
/**
|
|
111
118
|
* Progress of a footage run. `done` and `total` are seconds of media across the whole stage (a bar per stage);
|
package/src/footage/publish.ts
CHANGED
|
@@ -98,18 +98,21 @@ export function isPublishableAudio(audio: NonNullable<FilmFacts["audio"]>): bool
|
|
|
98
98
|
|
|
99
99
|
/**
|
|
100
100
|
* The final pass: each stream that is already publishable is copied, the other re-encoded, and the `moov` box moved
|
|
101
|
-
* to the front. A film without sound keeps none.
|
|
101
|
+
* to the front. A film without sound keeps none. `audioFilter` (the mix's loudness pass) re-encodes the sound through it.
|
|
102
102
|
*/
|
|
103
|
-
export function buildFinishArgs(input: string, facts: Pick<FilmFacts, "video" | "audio">, output: string): string[] {
|
|
103
|
+
export function buildFinishArgs(input: string, facts: Pick<FilmFacts, "video" | "audio">, output: string, options: { audioFilter?: string } = {}): string[] {
|
|
104
104
|
const video = isPublishableVideo(facts.video)
|
|
105
105
|
? ["-c:v", "copy"]
|
|
106
106
|
: ["-c:v", "libx264", "-profile:v", "high", "-pix_fmt", "yuv420p", "-preset", "medium", "-crf", String(VIDEO_CRF)];
|
|
107
|
+
const encodeAudio = ["-c:a", "aac", "-b:a", AUDIO_BITRATE, "-ar", String(FOOTAGE_SAMPLE_RATE), "-ac", "2"];
|
|
107
108
|
const audio =
|
|
108
109
|
facts.audio === null
|
|
109
110
|
? []
|
|
110
|
-
:
|
|
111
|
-
? ["-map", "0:a:0", "-
|
|
112
|
-
:
|
|
111
|
+
: options.audioFilter !== undefined
|
|
112
|
+
? ["-map", "0:a:0", "-af", options.audioFilter, ...encodeAudio]
|
|
113
|
+
: isPublishableAudio(facts.audio)
|
|
114
|
+
? ["-map", "0:a:0", "-c:a", "copy"]
|
|
115
|
+
: ["-map", "0:a:0", ...encodeAudio];
|
|
113
116
|
return ["-v", "error", "-y", "-i", input, "-map", "0:v:0", ...video, ...audio, "-movflags", "+faststart", output];
|
|
114
117
|
}
|
|
115
118
|
|