@softure-ai/marketing-kit 0.14.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +95 -0
- package/README.md +136 -12
- package/dist/config/schema.d.ts +2 -1
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +7 -0
- package/dist/config/schema.js.map +1 -1
- package/dist/footage/clip-sound.d.ts +58 -0
- package/dist/footage/clip-sound.d.ts.map +1 -0
- package/dist/footage/clip-sound.js +103 -0
- package/dist/footage/clip-sound.js.map +1 -0
- package/dist/footage/declick.d.ts +1 -1
- package/dist/footage/declick.d.ts.map +1 -1
- package/dist/footage/declick.js +3 -4
- package/dist/footage/declick.js.map +1 -1
- package/dist/footage/draft.d.ts +19 -5
- package/dist/footage/draft.d.ts.map +1 -1
- package/dist/footage/draft.js +94 -24
- package/dist/footage/draft.js.map +1 -1
- package/dist/footage/elevenlabs.d.ts +5 -0
- package/dist/footage/elevenlabs.d.ts.map +1 -1
- package/dist/footage/elevenlabs.js +10 -3
- package/dist/footage/elevenlabs.js.map +1 -1
- package/dist/footage/failure.d.ts +53 -0
- package/dist/footage/failure.d.ts.map +1 -0
- package/dist/footage/failure.js +23 -0
- package/dist/footage/failure.js.map +1 -0
- package/dist/footage/fake.d.ts +10 -1
- package/dist/footage/fake.d.ts.map +1 -1
- package/dist/footage/fake.js +19 -4
- package/dist/footage/fake.js.map +1 -1
- package/dist/footage/film-mix.d.ts +5 -1
- package/dist/footage/film-mix.d.ts.map +1 -1
- package/dist/footage/film-mix.js +11 -2
- package/dist/footage/film-mix.js.map +1 -1
- package/dist/footage/hyperframes-capture.d.ts +30 -0
- package/dist/footage/hyperframes-capture.d.ts.map +1 -0
- package/dist/footage/hyperframes-capture.js +80 -0
- package/dist/footage/hyperframes-capture.js.map +1 -0
- package/dist/footage/isolate-ranges.d.ts +67 -5
- package/dist/footage/isolate-ranges.d.ts.map +1 -1
- package/dist/footage/isolate-ranges.js +92 -21
- package/dist/footage/isolate-ranges.js.map +1 -1
- package/dist/footage/loudness.d.ts +16 -2
- package/dist/footage/loudness.d.ts.map +1 -1
- package/dist/footage/loudness.js +10 -7
- package/dist/footage/loudness.js.map +1 -1
- package/dist/footage/paid.d.ts +14 -1
- package/dist/footage/paid.d.ts.map +1 -1
- package/dist/footage/paid.js +10 -4
- package/dist/footage/paid.js.map +1 -1
- package/dist/footage/pipeline.d.ts +10 -4
- package/dist/footage/pipeline.d.ts.map +1 -1
- package/dist/footage/pipeline.js +74 -64
- package/dist/footage/pipeline.js.map +1 -1
- package/dist/footage/providers.d.ts +5 -0
- package/dist/footage/providers.d.ts.map +1 -1
- package/dist/footage/providers.js.map +1 -1
- package/dist/footage/render.d.ts +1 -1
- package/dist/footage/render.d.ts.map +1 -1
- package/dist/footage/render.js +4 -3
- package/dist/footage/render.js.map +1 -1
- package/dist/footage/speech-ranges.d.ts +80 -14
- package/dist/footage/speech-ranges.d.ts.map +1 -1
- package/dist/footage/speech-ranges.js +146 -26
- package/dist/footage/speech-ranges.js.map +1 -1
- package/dist/index.d.ts +10 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -5
- package/dist/index.js.map +1 -1
- package/dist/og/index.d.ts +2 -0
- package/dist/og/index.d.ts.map +1 -1
- package/dist/og/index.js +2 -0
- package/dist/og/index.js.map +1 -1
- package/dist/og/photo-cover.d.ts +55 -0
- package/dist/og/photo-cover.d.ts.map +1 -0
- package/dist/og/photo-cover.js +215 -0
- package/dist/og/photo-cover.js.map +1 -0
- package/dist/og/photo.d.ts +51 -0
- package/dist/og/photo.d.ts.map +1 -0
- package/dist/og/photo.js +154 -0
- package/dist/og/photo.js.map +1 -0
- package/dist/voice/provider-failure.d.ts +41 -0
- package/dist/voice/provider-failure.d.ts.map +1 -0
- package/dist/voice/provider-failure.js +68 -0
- package/dist/voice/provider-failure.js.map +1 -0
- package/dist/voice/provider.d.ts +7 -1
- package/dist/voice/provider.d.ts.map +1 -1
- package/dist/voice/provider.js +1 -1
- package/dist/voice/provider.js.map +1 -1
- package/package.json +2 -1
- package/schema/marketing.schema.json +6 -0
- package/src/config/schema.ts +7 -0
- package/src/footage/clip-sound.ts +124 -0
- package/src/footage/declick.ts +3 -4
- package/src/footage/draft.ts +125 -24
- package/src/footage/elevenlabs.ts +11 -3
- package/src/footage/failure.ts +57 -0
- package/src/footage/fake.ts +28 -4
- package/src/footage/film-mix.ts +10 -2
- package/src/footage/hyperframes-capture.ts +100 -0
- package/src/footage/isolate-ranges.ts +158 -22
- package/src/footage/loudness.ts +17 -8
- package/src/footage/paid.ts +29 -4
- package/src/footage/pipeline.ts +85 -66
- package/src/footage/providers.ts +5 -0
- package/src/footage/render.ts +4 -3
- package/src/footage/speech-ranges.ts +197 -27
- package/src/index.ts +55 -1
- package/src/og/index.ts +23 -0
- package/src/og/photo-cover.ts +280 -0
- package/src/og/photo.ts +177 -0
- package/src/voice/provider-failure.ts +98 -0
- package/src/voice/provider.ts +7 -2
package/src/footage/pipeline.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { cpSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
2
2
|
import { createRequire } from "node:module";
|
|
3
3
|
import { dirname, extname, join, resolve } from "node:path";
|
|
4
4
|
|
|
@@ -7,13 +7,16 @@ import type { BrandColors } from "../config/colors.js";
|
|
|
7
7
|
import type { BrandFont } from "../config/config.js";
|
|
8
8
|
import type { SfxEvent } from "../config/schema.js";
|
|
9
9
|
import { getHyperframesBin, getHyperframesEnv } from "../render/hyperframes.js";
|
|
10
|
+
import { readHyperframesCapture, type HyperframesCapture } from "./hyperframes-capture.js";
|
|
10
11
|
import { copyFonts } from "../render/render.js";
|
|
11
|
-
import {
|
|
12
|
+
import type { TtsResult } from "../voice/provider.js";
|
|
12
13
|
import type { TimedWord } from "../voice/voiceover.js";
|
|
13
14
|
import { checkMusicFileType, describeAudioProbe, probeAudio } from "./audio.js";
|
|
15
|
+
import { prepareClipSounds } from "./clip-sound.js";
|
|
14
16
|
import { readCachedTranscript, writeCacheFile, writeTranscript } from "./cache.js";
|
|
15
17
|
import { composeFootage, END_CARD_SECONDS, getFootageFilmSeconds, getFootageSfxCues } from "./compose.js";
|
|
16
18
|
import { findMissingCaptionGlyphs, getCaptionBounds, getFootageCaptionLayout } from "./captions.js";
|
|
19
|
+
import { err, getProviderFootageFailure, ok, type FootageResult } from "./failure.js";
|
|
17
20
|
import { findLowDiskProblems, getFreeBytes, getRenderTempDir, type FreeBytesReader } from "./disk.js";
|
|
18
21
|
import { buildEdl, getEdlDuration, readEdl, remapWords, validateEdl, type Edl } from "./edl.js";
|
|
19
22
|
import { makePlaceholderTranscript } from "./fake.js";
|
|
@@ -23,6 +26,7 @@ import { DEFAULT_AUTO_THRESHOLD, getDeclickLevel, getFootageEdit, type FootageCl
|
|
|
23
26
|
import { getFilmKeyTerms } from "./keyterms.js";
|
|
24
27
|
import { buildFilmMixArgs } from "./film-mix.js";
|
|
25
28
|
import {
|
|
29
|
+
AUDIO_BITRATE_RANGE_KBPS,
|
|
26
30
|
buildMixArgs,
|
|
27
31
|
DEFAULT_LOUDNESS,
|
|
28
32
|
formatDb,
|
|
@@ -84,7 +88,8 @@ export { getEdlPath } from "./paid.js";
|
|
|
84
88
|
* (a transcript from each clip's `script`, no isolation, no generated music) and keeps its files apart from the real
|
|
85
89
|
* film's (`placeholder/` in the build folder, its own EDL).
|
|
86
90
|
*
|
|
87
|
-
* Build folder layout: `sources/` (downloaded URLs), `clips/<id>.mp4` (normalized), `audio/<id>.
|
|
91
|
+
* Build folder layout: `sources/` (downloaded URLs), `clips/<id>.mp4` (normalized), `audio/<id>.wav` (the clip's
|
|
92
|
+
* lossless, pre-gained sound; `audio/<id>.isolated.wav` its cleaned voice), `edl.json`
|
|
88
93
|
* (`edl.placeholder.json`), `assets/`, `index.html`, `render.mp4` (hyperframes' output until the final pass).
|
|
89
94
|
*
|
|
90
95
|
* The finished film is publishable: the speech track normalized to a loudness target before the music goes under it,
|
|
@@ -101,6 +106,8 @@ const require = createRequire(import.meta.url);
|
|
|
101
106
|
/** Bumped when the normalized format changes, so older normalized clips are made again. */
|
|
102
107
|
const NORMALIZE_VERSION = 2;
|
|
103
108
|
|
|
109
|
+
export type { FootageResult } from "./failure.js";
|
|
110
|
+
|
|
104
111
|
export interface FootageBrand {
|
|
105
112
|
name: string;
|
|
106
113
|
/** BCP 47, e.g. `en-US`. */
|
|
@@ -151,7 +158,6 @@ export interface PreparedClip {
|
|
|
151
158
|
duration: number;
|
|
152
159
|
}
|
|
153
160
|
|
|
154
|
-
export type FootageResult<T> = TtsResult<T>;
|
|
155
161
|
|
|
156
162
|
export type TranscribeOutcome =
|
|
157
163
|
| { kind: "dry-run"; steps: PaidStep[] }
|
|
@@ -229,23 +235,23 @@ async function runPrepareFootage(
|
|
|
229
235
|
options: Pick<FootageRunOptions, "buildDir" | "log" | "fetch" | "signal" | "onProgress" | "normalize">,
|
|
230
236
|
): Promise<FootageResult<PreparedClip[]>> {
|
|
231
237
|
const { buildDir, log, signal } = options;
|
|
232
|
-
if (film.clips.length === 0) return err(`footage film ${film.id} has no clips
|
|
238
|
+
if (film.clips.length === 0) return err(`footage film ${film.id} has no clips.`, { kind: "input" });
|
|
233
239
|
const sources: Record<string, string> = {};
|
|
234
240
|
for (const clip of film.clips) {
|
|
235
241
|
if (!isUrl(clip.source)) {
|
|
236
|
-
if (!existsSync(clip.source)) return err(`clip "${clip.id}": no file at ${clip.source}
|
|
242
|
+
if (!existsSync(clip.source)) return err(`clip "${clip.id}": no file at ${clip.source}`, { kind: "input" });
|
|
237
243
|
sources[clip.id] = clip.source;
|
|
238
244
|
continue;
|
|
239
245
|
}
|
|
240
246
|
const downloaded = await download(clip.source, join(buildDir, "sources"), options);
|
|
241
|
-
if (!downloaded.ok) return err(`clip "${clip.id}": ${downloaded.error}
|
|
247
|
+
if (!downloaded.ok) return err(`clip "${clip.id}": ${downloaded.error}`, downloaded.failure);
|
|
242
248
|
sources[clip.id] = downloaded.value;
|
|
243
249
|
}
|
|
244
250
|
const probed = await probeFootage(film.clips, sources, signal === undefined ? {} : { signal });
|
|
245
251
|
if (!probed.ok) return err(probed.errors.join("\n"));
|
|
246
252
|
if (film.music.source.kind === "file") {
|
|
247
253
|
const music = await checkMusicFile(film.music.source, options);
|
|
248
|
-
if (!music.ok) return
|
|
254
|
+
if (!music.ok) return music;
|
|
249
255
|
}
|
|
250
256
|
const frame = getFrame(film);
|
|
251
257
|
const clipsDir = join(buildDir, "clips");
|
|
@@ -281,23 +287,28 @@ function getClipKey(clip: PreparedClip): Promise<string> {
|
|
|
281
287
|
return getClipCacheKey(clip.source, clip.clip);
|
|
282
288
|
}
|
|
283
289
|
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
290
|
+
/** The cached cleaned voice (the provider's file) decoded into the build folder as lossless WAV, named by the clip's id. */
|
|
291
|
+
async function decodeIsolatedVoice(cached: string, clipId: string, buildDir: string, signal: AbortSignal | undefined): Promise<string> {
|
|
292
|
+
const output = join(buildDir, "audio", `${clipId}.isolated.wav`);
|
|
293
|
+
mkdirSync(dirname(output), { recursive: true });
|
|
294
|
+
await runFfmpeg(`Decoding the cleaned voice of clip ${clipId}`, buildAudioArgs(cached, output), { ...(signal === undefined ? {} : { signal }), output });
|
|
295
|
+
return output;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
/** Each clip's lossless, pre-gained sound (`prepareClipSounds`), by clip id. */
|
|
299
|
+
async function getClipSoundFiles(clips: PreparedClip[], buildDir: string, signal: AbortSignal | undefined): Promise<Record<string, string>> {
|
|
300
|
+
const sounds = await prepareClipSounds(clips, { buildDir, ...(signal === undefined ? {} : { signal }) });
|
|
301
|
+
return Object.fromEntries(Object.entries(sounds).map(([id, sound]) => [id, sound.file]));
|
|
289
302
|
}
|
|
290
303
|
|
|
291
|
-
/** The free transcripts of `--placeholder`, written under `placeholder/` for a look. */
|
|
292
|
-
function transcribePlaceholder(
|
|
304
|
+
/** The free transcripts of `--placeholder`, written under `placeholder/` for a look; the speech is each clip's lossless sound. */
|
|
305
|
+
async function transcribePlaceholder(clips: PreparedClip[], buildDir: string, signal: AbortSignal | undefined): Promise<TranscribeOutcome> {
|
|
293
306
|
const words: Record<string, TimedWord[]> = {};
|
|
294
|
-
const speech: Record<string, string> = {};
|
|
295
307
|
for (const clip of clips) {
|
|
296
308
|
words[clip.clip.id] = clip.clip.role === "talk" ? makePlaceholderTranscript(clip.clip.script, clip.duration) : [];
|
|
297
309
|
writeTranscript(join(buildDir, "placeholder", `${clip.clip.id}.transcript.json`), words[clip.clip.id] ?? []);
|
|
298
|
-
speech[clip.clip.id] = clip.file;
|
|
299
310
|
}
|
|
300
|
-
return { kind: "ready", words, speech };
|
|
311
|
+
return { kind: "ready", words, speech: await getClipSoundFiles(clips, buildDir, signal) };
|
|
301
312
|
}
|
|
302
313
|
|
|
303
314
|
/** The paid music step, when the film's music still has to be bought; null when it is cached or free. */
|
|
@@ -320,9 +331,9 @@ export function transcribeFootage(film: FootageFilm, clips: PreparedClip[], opti
|
|
|
320
331
|
/** The body of `transcribeFootage`, inside its ffmpeg failure reporting. */
|
|
321
332
|
async function runTranscribeFootage(film: FootageFilm, clips: PreparedClip[], options: FootageRunOptions): Promise<FootageResult<TranscribeOutcome>> {
|
|
322
333
|
const { buildDir, providers, log, signal } = options;
|
|
323
|
-
if (options.isPlaceholder) return ok(transcribePlaceholder(
|
|
334
|
+
if (options.isPlaceholder) return ok(await transcribePlaceholder(clips, buildDir, signal));
|
|
324
335
|
const keyTerms = getFilmKeyTerms(film);
|
|
325
|
-
if (!keyTerms.ok) return err(`footage film ${film.id}: ${keyTerms.errors.join(" ")}
|
|
336
|
+
if (!keyTerms.ok) return err(`footage film ${film.id}: ${keyTerms.errors.join(" ")}`, { kind: "input" });
|
|
326
337
|
const sttKeyTerms = keyTerms.terms.length === 0 ? {} : { keyTerms: keyTerms.terms };
|
|
327
338
|
const plan: { clip: PreparedClip; isolated: CacheEntry | null; transcript: CacheEntry }[] = [];
|
|
328
339
|
const steps: PaidStep[] = [];
|
|
@@ -344,11 +355,9 @@ async function runTranscribeFootage(film: FootageFilm, clips: PreparedClip[], op
|
|
|
344
355
|
return ok({ kind: "dry-run", steps: all });
|
|
345
356
|
}
|
|
346
357
|
const words: Record<string, TimedWord[]> = {};
|
|
347
|
-
const
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
speech[clip.clip.id] = clip.file;
|
|
351
|
-
}
|
|
358
|
+
for (const clip of clips) words[clip.clip.id] = [];
|
|
359
|
+
// Every clip's sound, lossless and pre-gained: what the isolator, the speech to text and the speech track hear (#1139).
|
|
360
|
+
const speech = await getClipSoundFiles(clips, buildDir, signal);
|
|
352
361
|
const isolating = plan.filter(({ isolated }) => isolated !== null && !existsSync(isolated.path));
|
|
353
362
|
const transcribing = plan.filter(({ transcript }) => !existsSync(transcript.path));
|
|
354
363
|
const isolateProgress = createStageProgress(options.onProgress, "isolate", isolating.map(({ clip }) => ({ item: clip.clip.id, seconds: clip.duration })));
|
|
@@ -356,27 +365,27 @@ async function runTranscribeFootage(film: FootageFilm, clips: PreparedClip[], op
|
|
|
356
365
|
for (const entry of plan) {
|
|
357
366
|
const { clip, isolated, transcript } = entry;
|
|
358
367
|
const id = clip.clip.id;
|
|
359
|
-
|
|
360
|
-
const getAudio = (): Promise<string> => (audio ??= extractAudio(clip, buildDir, signal));
|
|
368
|
+
const sound = speech[id] ?? clip.file;
|
|
361
369
|
let sttInput: string | null = null;
|
|
362
370
|
if (isolated !== null) {
|
|
363
371
|
if (!existsSync(isolated.path)) {
|
|
364
372
|
const index = isolating.indexOf(entry);
|
|
365
373
|
isolateProgress(index, 0);
|
|
366
|
-
const input = await getAudio();
|
|
367
374
|
throwIfAborted(signal);
|
|
368
375
|
log(`voice isolation (${providers.isolation.id}): clip ${id}, ${clip.duration.toFixed(1)} s.`);
|
|
369
|
-
const cleaned = await providers.isolation.isolate({ audio:
|
|
376
|
+
const cleaned = await providers.isolation.isolate({ audio: sound, ...(signal === undefined ? {} : { signal }) });
|
|
370
377
|
throwIfAborted(signal);
|
|
371
|
-
if (!cleaned.ok) return err(`clip "${id}": ${cleaned.error}
|
|
378
|
+
if (!cleaned.ok) return err(`clip "${id}": ${cleaned.error}`, getProviderFootageFailure(cleaned.providerFailure));
|
|
372
379
|
const paid = toPaidAudio(cleaned.value);
|
|
373
380
|
writeCacheFile(isolated.path, paid.audio);
|
|
374
381
|
options.onCharge?.({ step: "isolate", item: id, provider: providers.isolation.id, units: clip.duration, ...(paid.credits === undefined ? {} : { credits: paid.credits }), cacheKey: isolated.key });
|
|
375
382
|
log(`voice isolation: saved ${isolated.path} (commit it).`);
|
|
376
383
|
isolateProgress(index, clip.duration);
|
|
377
384
|
}
|
|
378
|
-
speech
|
|
379
|
-
|
|
385
|
+
// The provider's file decoded once: the speech track and the speech to text read it lossless, named by the clip.
|
|
386
|
+
const decoded = await decodeIsolatedVoice(isolated.path, id, buildDir, signal);
|
|
387
|
+
speech[id] = decoded;
|
|
388
|
+
sttInput = decoded;
|
|
380
389
|
}
|
|
381
390
|
const cached = readCachedTranscript(transcript.path);
|
|
382
391
|
if (!cached.ok) return err(cached.error);
|
|
@@ -385,21 +394,15 @@ async function runTranscribeFootage(film: FootageFilm, clips: PreparedClip[], op
|
|
|
385
394
|
words[id] = cached.words;
|
|
386
395
|
continue;
|
|
387
396
|
}
|
|
388
|
-
if (sttInput !== null) {
|
|
389
|
-
// The provider sees the clip's id as the file name, as it does for the plain sound.
|
|
390
|
-
const cleaned = sttInput;
|
|
391
|
-
sttInput = join(buildDir, "audio", `${id}.isolated.mp3`);
|
|
392
|
-
mkdirSync(dirname(sttInput), { recursive: true });
|
|
393
|
-
copyFileSync(cleaned, sttInput);
|
|
394
|
-
}
|
|
395
397
|
const index = transcribing.indexOf(entry);
|
|
396
398
|
transcribeProgress(index, 0);
|
|
397
|
-
|
|
399
|
+
// The provider sees the clip's id as the file name, for the cleaned voice as for the plain sound.
|
|
400
|
+
const input = sttInput ?? sound;
|
|
398
401
|
throwIfAborted(signal);
|
|
399
402
|
log(`speech to text (${providers.stt.id}): clip ${id}, ${clip.duration.toFixed(1)} s.`);
|
|
400
403
|
const result = await providers.stt.transcribe({ audio: input, language: film.language, ...sttKeyTerms, ...(signal === undefined ? {} : { signal }) });
|
|
401
404
|
throwIfAborted(signal);
|
|
402
|
-
if (!result.ok) return err(`clip "${id}": ${result.error}
|
|
405
|
+
if (!result.ok) return err(`clip "${id}": ${result.error}`, getProviderFootageFailure(result.providerFailure));
|
|
403
406
|
writeTranscript(transcript.path, result.value.words);
|
|
404
407
|
const { credits } = result.value;
|
|
405
408
|
options.onCharge?.({ step: "transcribe", item: id, provider: providers.stt.id, units: clip.duration, ...(credits === undefined ? {} : { credits }), ...(keyTerms.terms.length === 0 ? {} : { keyTerms: keyTerms.terms.length }), cacheKey: transcript.key });
|
|
@@ -502,13 +505,13 @@ export function readEdlFile(film: FootageFilm, clips: PreparedClip[], edlPath: s
|
|
|
502
505
|
try {
|
|
503
506
|
json = JSON.parse(readFileSync(edlPath, "utf8")) as unknown;
|
|
504
507
|
} catch (error) {
|
|
505
|
-
return err(`${edlPath}: not JSON (${String(error)})
|
|
508
|
+
return err(`${edlPath}: not JSON (${String(error)})`, { kind: "input" });
|
|
506
509
|
}
|
|
507
510
|
const read = readEdl(json);
|
|
508
|
-
if (!read.ok) return err(`${edlPath}: ${read.error}
|
|
511
|
+
if (!read.ok) return err(`${edlPath}: ${read.error}`, { kind: "input" });
|
|
509
512
|
const bounds = getCaptionBounds(getFrame(film).height, getFootageCaptionLayout(film, layout));
|
|
510
513
|
const problems = validateEdl(read.edl, clips.map((clip) => ({ id: clip.clip.id, duration: clip.duration })), film.id, bounds);
|
|
511
|
-
return problems.length > 0 ? err(`${edlPath}: ${problems.join("; ")}
|
|
514
|
+
return problems.length > 0 ? err(`${edlPath}: ${problems.join("; ")}`, { kind: "input" }) : ok(read.edl);
|
|
512
515
|
}
|
|
513
516
|
|
|
514
517
|
/**
|
|
@@ -518,16 +521,16 @@ export function readEdlFile(film: FootageFilm, clips: PreparedClip[], edlPath: s
|
|
|
518
521
|
*/
|
|
519
522
|
async function checkMusicFile(source: Extract<MusicSource, { kind: "file" }>, options: Pick<FootageRunOptions, "log" | "signal">): Promise<FootageResult<string>> {
|
|
520
523
|
const { path, fileName, contentType } = source;
|
|
521
|
-
if (!existsSync(path)) return err(`music: no file at ${path}
|
|
524
|
+
if (!existsSync(path)) return err(`music: no file at ${path}`, { kind: "input" });
|
|
522
525
|
const name = fileName ?? path;
|
|
523
526
|
if (extname(name) === "" && contentType === undefined) {
|
|
524
527
|
options.log(`music: ${path} has no extension and no declared name or content type; the probe checks it.`);
|
|
525
528
|
} else {
|
|
526
529
|
const type = checkMusicFileType(contentType === undefined ? { fileName: name } : { fileName: name, contentType });
|
|
527
|
-
if (!type.ok) return err(`music: ${type.error}
|
|
530
|
+
if (!type.ok) return err(`music: ${type.error}`, { kind: "input" });
|
|
528
531
|
}
|
|
529
532
|
const probe = await probeAudio(path, options.signal === undefined ? {} : { signal: options.signal });
|
|
530
|
-
if (!probe.ok) return err(`music: ${probe.error}
|
|
533
|
+
if (!probe.ok) return err(`music: ${probe.error}`, { kind: "input" });
|
|
531
534
|
options.log(`music: ${path} (${describeAudioProbe(probe.probe)}).`);
|
|
532
535
|
return ok(path);
|
|
533
536
|
}
|
|
@@ -552,7 +555,7 @@ export async function findFootageMusic(
|
|
|
552
555
|
): Promise<FootageResult<FootageMusicLookup>> {
|
|
553
556
|
const source = film.music.source;
|
|
554
557
|
if (source.kind === "none") return ok({ kind: "none" });
|
|
555
|
-
if (source.kind === "file") return existsSync(source.path) ? ok({ kind: "file", path: source.path }) : err(`music: no file at ${source.path}
|
|
558
|
+
if (source.kind === "file") return existsSync(source.path) ? ok({ kind: "file", path: source.path }) : err(`music: no file at ${source.path}`, { kind: "input" });
|
|
556
559
|
let cache: CacheEntry;
|
|
557
560
|
if (source.kind === "prompt") {
|
|
558
561
|
cache = getPromptMusicCache(film, options);
|
|
@@ -605,7 +608,7 @@ async function getMusic(film: FootageFilm, edl: Edl, clips: PreparedClip[], seco
|
|
|
605
608
|
return ok(null);
|
|
606
609
|
}
|
|
607
610
|
throwIfAborted(signal);
|
|
608
|
-
if (!bought.ok) return err(`music: ${bought.error}
|
|
611
|
+
if (!bought.ok) return err(`music: ${bought.error}`, getProviderFootageFailure(bought.providerFailure));
|
|
609
612
|
const paid = toPaidAudio(bought.value);
|
|
610
613
|
writeCacheFile(path, paid.audio);
|
|
611
614
|
options.onCharge?.({ step: "music", item: "film", provider: providers.music.id, units: seconds, ...(paid.credits === undefined ? {} : { credits: paid.credits }), cacheKey: cache.key });
|
|
@@ -615,12 +618,12 @@ async function getMusic(film: FootageFilm, edl: Edl, clips: PreparedClip[], seco
|
|
|
615
618
|
}
|
|
616
619
|
|
|
617
620
|
/**
|
|
618
|
-
* The speech chain's files: the cut speech, the declicked track and the final
|
|
619
|
-
*
|
|
621
|
+
* The speech chain's files, all lossless float WAV (issue #1139): the cut speech, the declicked track and the final
|
|
622
|
+
* `speech.wav`, which the last stage that runs writes. The film's final AAC is the chain's only lossy encode.
|
|
620
623
|
*/
|
|
621
624
|
export function getSpeechChainFiles(assetsDir: string, stages: { isDeclick: boolean; isLoudness: boolean }): { rawSpeechFile: string; declickedFile: string; speechFile: string } {
|
|
622
|
-
const speechFile = join(assetsDir, "speech.
|
|
623
|
-
const rawSpeechFile = stages.isDeclick || stages.isLoudness ? join(assetsDir, "speech.raw.
|
|
625
|
+
const speechFile = join(assetsDir, "speech.wav");
|
|
626
|
+
const rawSpeechFile = stages.isDeclick || stages.isLoudness ? join(assetsDir, "speech.raw.wav") : speechFile;
|
|
624
627
|
const declickedFile = !stages.isDeclick ? rawSpeechFile : stages.isLoudness ? join(assetsDir, "speech.declicked.wav") : speechFile;
|
|
625
628
|
return { rawSpeechFile, declickedFile, speechFile };
|
|
626
629
|
}
|
|
@@ -674,6 +677,11 @@ export type RenderedFootage =
|
|
|
674
677
|
edlPath: string;
|
|
675
678
|
/** What the speech track's declick stage did (`FootageFilm.speech.declick`); null when it is off. */
|
|
676
679
|
declick: DeclickReport | null;
|
|
680
|
+
/**
|
|
681
|
+
* How hyperframes captured the frames (issue #1145): the capture path (`screenshot` is the slower one), the GPU
|
|
682
|
+
* mode, each stage's seconds and the screenshot hint, read from hyperframes' summary; null when it printed none.
|
|
683
|
+
*/
|
|
684
|
+
capture: HyperframesCapture | null;
|
|
677
685
|
} & FinishedFootage);
|
|
678
686
|
|
|
679
687
|
/**
|
|
@@ -695,7 +703,7 @@ async function runRenderFootage(
|
|
|
695
703
|
): Promise<FootageResult<RenderedFootage>> {
|
|
696
704
|
const { buildDir, log, signal } = options;
|
|
697
705
|
const finishVideoProblems = options.finishVideo === undefined ? [] : findFinishVideoProblems(options.finishVideo);
|
|
698
|
-
if (finishVideoProblems.length > 0) return err(finishVideoProblems.join(" "));
|
|
706
|
+
if (finishVideoProblems.length > 0) return err(finishVideoProblems.join(" "), { kind: "input" });
|
|
699
707
|
const lowDisk = findLowDiskProblems({ temp: getRenderTempDir(process.env, options.tempDir), build: buildDir }, options.readFreeBytes ?? getFreeBytes);
|
|
700
708
|
if (lowDisk.length > 0) return err(`not enough disk space to render: ${lowDisk.join(" ")}`);
|
|
701
709
|
const seconds = getFootageFilmSeconds(film, edl);
|
|
@@ -759,7 +767,8 @@ async function runRenderFootage(
|
|
|
759
767
|
}
|
|
760
768
|
// The film's mix, brought to `mixLoudness` and gated before hyperframes captures a frame (issue #1124).
|
|
761
769
|
const mixLoudness = getMixLoudnessTarget(film);
|
|
762
|
-
|
|
770
|
+
// Widened: the mix track assigns it inside a closure, which narrowing does not see.
|
|
771
|
+
let mixed = ok(null) as FootageResult<PreparedMix | null>;
|
|
763
772
|
if (mixLoudness !== null) {
|
|
764
773
|
const mixSource = join(buildDir, "mix.wav");
|
|
765
774
|
const musicTrack = musicFile === null ? null : { file: join(assetsDir, "music.mp3"), seconds };
|
|
@@ -809,7 +818,7 @@ async function runRenderFootage(
|
|
|
809
818
|
brandName: options.brand.name,
|
|
810
819
|
});
|
|
811
820
|
if (missing.length > 0) {
|
|
812
|
-
return err(`the brand fonts cannot draw every letter of the film: ${missing.map((entry) => `${entry.font} lacks ${entry.characters.join(" ")}`).join("; ")}. Add a font file that covers them (brand.fonts)
|
|
821
|
+
return err(`the brand fonts cannot draw every letter of the film: ${missing.map((entry) => `${entry.font} lacks ${entry.characters.join(" ")}`).join("; ")}. Add a font file that covers them (brand.fonts).`, { kind: "input" });
|
|
813
822
|
}
|
|
814
823
|
const html = composeFootage({
|
|
815
824
|
film,
|
|
@@ -823,7 +832,7 @@ async function runRenderFootage(
|
|
|
823
832
|
assets: {
|
|
824
833
|
clips: Object.fromEntries(clips.map((clip) => [clip.clip.id, `clips/${clip.clip.id}.mp4`])),
|
|
825
834
|
picture: "assets/picture.mp4",
|
|
826
|
-
speech: "assets/speech.
|
|
835
|
+
speech: "assets/speech.wav",
|
|
827
836
|
music: musicAsset,
|
|
828
837
|
lastFrame: "assets/last.jpg",
|
|
829
838
|
logo: options.brand.logo === null ? null : "assets/logo.svg",
|
|
@@ -841,7 +850,7 @@ async function runRenderFootage(
|
|
|
841
850
|
const roles = Object.fromEntries(clips.map((clip) => [clip.clip.id, clip.clip.role]));
|
|
842
851
|
const finished = await finishFootage(film, { renderFile, seconds, coverAt: film.cover?.at ?? getDefaultCoverSeconds(edl, roles), mix: prepared.value }, options);
|
|
843
852
|
if (!finished.ok) return finished;
|
|
844
|
-
return ok({ kind: "rendered", output: options.output, seconds, edlPath: getEdlPath(buildDir, options.isPlaceholder), declick, ...finished.value });
|
|
853
|
+
return ok({ kind: "rendered", output: options.output, seconds, edlPath: getEdlPath(buildDir, options.isPlaceholder), declick, capture: rendered.value, ...finished.value });
|
|
845
854
|
}
|
|
846
855
|
|
|
847
856
|
/** What `renderFootage` reads from the written film. */
|
|
@@ -964,7 +973,7 @@ export async function finishFootage(
|
|
|
964
973
|
const finished = await probeFilmFacts(output, aborting);
|
|
965
974
|
if (!finished.ok) return err(`the finished film: ${finished.error}`);
|
|
966
975
|
const problems = findFinishProblems(finished.facts, readTopLevelBoxes(output));
|
|
967
|
-
if (problems.length > 0) return err(`the finished film ${output} is not publishable: ${problems.join("; ")}
|
|
976
|
+
if (problems.length > 0) return err(`the finished film ${output} is not publishable: ${problems.join("; ")}.`, { kind: "publishable", problems });
|
|
968
977
|
let loudness: FinishedLoudness | null = null;
|
|
969
978
|
if (finished.facts.audio !== null) {
|
|
970
979
|
const measured = await measureLoudness(output, mixLoudness ?? DEFAULT_LOUDNESS, { label: "the finished film", ...aborting });
|
|
@@ -972,7 +981,7 @@ export async function finishFootage(
|
|
|
972
981
|
const sound = { integrated: measured.measurement.inputI, truePeak: measured.measurement.inputTp };
|
|
973
982
|
// A mix left as rendered (`mixLoudness: null`) or silent is reported, not held to a target it was not brought to.
|
|
974
983
|
const verdict = mixLoudness === null || audioFile === undefined ? { kind: "pass" as const } : getLoudnessVerdict(sound, mixLoudness);
|
|
975
|
-
if (verdict.kind === "fail") return err(`the finished film ${output} measures ${describeMixLoudness(sound)}: ${verdict.problems.join("; ")}
|
|
984
|
+
if (verdict.kind === "fail") return err(`the finished film ${output} measures ${describeMixLoudness(sound)}: ${verdict.problems.join("; ")}.`, { kind: "loudness", loudness: sound });
|
|
976
985
|
loudness = { ...sound, warning: verdict.kind === "warn" ? `the finished film ${output}: ${verdict.warning}` : null };
|
|
977
986
|
log(`loudness: the finished film measures ${formatDb(sound.integrated)} LUFS, true peak ${formatDb(sound.truePeak)} dBTP.`);
|
|
978
987
|
if (loudness.warning !== null) log(`loudness warning: ${loudness.warning}`);
|
|
@@ -1008,6 +1017,10 @@ async function finishMix(input: {
|
|
|
1008
1017
|
signal?: AbortSignal;
|
|
1009
1018
|
}): Promise<FootageResult<{ attempt: MixAttempt; tries: number; probes: number; measured: MixLoudness }>> {
|
|
1010
1019
|
const aborting = input.signal === undefined ? {} : { signal: input.signal };
|
|
1020
|
+
const bitrate = input.target.audioBitrateKbps;
|
|
1021
|
+
if (bitrate !== undefined && !(Number.isInteger(bitrate) && bitrate >= AUDIO_BITRATE_RANGE_KBPS.min && bitrate <= AUDIO_BITRATE_RANGE_KBPS.max)) {
|
|
1022
|
+
return err(`mixLoudness.audioBitrateKbps must be a whole number from ${String(AUDIO_BITRATE_RANGE_KBPS.min)} to ${String(AUDIO_BITRATE_RANGE_KBPS.max)}, got ${String(bitrate)}.`, { kind: "input" });
|
|
1023
|
+
}
|
|
1011
1024
|
const attempts = getMixAttempts(input.target);
|
|
1012
1025
|
const probe = async (attempt: MixAttempt): Promise<MixLoudness> => {
|
|
1013
1026
|
const probed = await measureLoudness(input.input, input.target, { label: "the film's mix through its limiter", filter: getMixFilter(attempt), ...aborting });
|
|
@@ -1022,9 +1035,9 @@ async function finishMix(input: {
|
|
|
1022
1035
|
const solved = await solveMixGain({ ceilingDbtp, startGainDb, target: input.target, shift, measure: probe });
|
|
1023
1036
|
probes += solved.probes.length;
|
|
1024
1037
|
const attempt = solved.best.attempt;
|
|
1025
|
-
await runFfmpeg("The film's mix", buildMixArgs({ input: input.input, filter: getMixFilter(attempt), output: input.output }), { ...aborting, output: input.output });
|
|
1038
|
+
await runFfmpeg("The film's mix", buildMixArgs({ input: input.input, filter: getMixFilter(attempt), output: input.output, ...(input.target.audioBitrateKbps === undefined ? {} : { bitrateKbps: input.target.audioBitrateKbps }) }), { ...aborting, output: input.output });
|
|
1026
1039
|
const encoded = await measureLoudness(input.output, input.target, { label: "the film's encoded mix", ...aborting });
|
|
1027
|
-
if (encoded.kind === "silent") return err("the film's mix measures silent after its gain and limiter.");
|
|
1040
|
+
if (encoded.kind === "silent") return err("the film's mix measures silent after its gain and limiter.", { kind: "loudness", loudness: null });
|
|
1028
1041
|
const measured = { integrated: encoded.measurement.inputI, truePeak: encoded.measurement.inputTp };
|
|
1029
1042
|
const verdict = getLoudnessVerdict(measured, input.target);
|
|
1030
1043
|
if (verdict.kind === "pass" || (verdict.kind === "warn" && tries === attempts)) return ok({ attempt, tries, probes, measured });
|
|
@@ -1032,13 +1045,14 @@ async function finishMix(input: {
|
|
|
1032
1045
|
return err(
|
|
1033
1046
|
`the film's mix still measures ${describeMixLoudness(measured)} after ${attempts} encodes (last: gain ${formatDb(attempt.gainDb)} dB, ` +
|
|
1034
1047
|
`limiter at ${formatDb(attempt.ceilingDbtp)} dBTP): ${verdict.kind === "fail" ? verdict.problems.join("; ") : ""}.`,
|
|
1048
|
+
{ kind: "loudness", loudness: measured },
|
|
1035
1049
|
);
|
|
1036
1050
|
}
|
|
1037
1051
|
shift = Number.isFinite(solved.best.measured.integrated) ? measured.integrated - solved.best.measured.integrated : 0;
|
|
1038
1052
|
ceilingDbtp = getNextMixCeiling(attempt.ceilingDbtp, measured, input.target);
|
|
1039
1053
|
startGainDb = attempt.gainDb;
|
|
1040
1054
|
}
|
|
1041
|
-
return err(`the film's mix was not encoded: mixLoudness.attempts is ${attempts}
|
|
1055
|
+
return err(`the film's mix was not encoded: mixLoudness.attempts is ${attempts}.`, { kind: "input" });
|
|
1042
1056
|
}
|
|
1043
1057
|
|
|
1044
1058
|
/** Both measurements of a mix, as an error names them. */
|
|
@@ -1117,15 +1131,20 @@ async function buildPicture(input: BuildPictureInput): Promise<void> {
|
|
|
1117
1131
|
input.onSeconds(getPictureFrames(pieces) / FOOTAGE_FPS);
|
|
1118
1132
|
}
|
|
1119
1133
|
|
|
1120
|
-
/**
|
|
1121
|
-
|
|
1134
|
+
/**
|
|
1135
|
+
* The hyperframes render as a child process: its lines go to the log (progress every 10 %), its percentages to
|
|
1136
|
+
* `onProgress`; it answers the capture report read from its summary (null when it printed none).
|
|
1137
|
+
*/
|
|
1138
|
+
async function runHyperframesRender(options: RenderFootageOptions, output: string, seconds: number): Promise<FootageResult<HyperframesCapture | null>> {
|
|
1122
1139
|
const progress = createStageProgress(options.onProgress, "render", [{ item: "film", seconds }]);
|
|
1123
1140
|
progress(0, 0);
|
|
1124
1141
|
let loggedTenth = -1;
|
|
1142
|
+
const lines: string[] = [];
|
|
1125
1143
|
const onLine = (line: string): void => {
|
|
1126
1144
|
if (line.trim().length === 0) return;
|
|
1127
1145
|
const percent = readRenderPercent(line);
|
|
1128
1146
|
if (percent === null) {
|
|
1147
|
+
lines.push(line);
|
|
1129
1148
|
options.log(line.trim());
|
|
1130
1149
|
return;
|
|
1131
1150
|
}
|
|
@@ -1156,7 +1175,7 @@ async function runHyperframesRender(options: RenderFootageOptions, output: strin
|
|
|
1156
1175
|
if (result.kind === "not-started") return err(`hyperframes render did not start: ${result.error}.`);
|
|
1157
1176
|
if (result.code !== 0) return err(`hyperframes render ended with code ${String(result.code)} (signal ${result.signal ?? "-"}).`);
|
|
1158
1177
|
progress(0, seconds);
|
|
1159
|
-
return ok(
|
|
1178
|
+
return ok(readHyperframesCapture(lines));
|
|
1160
1179
|
}
|
|
1161
1180
|
|
|
1162
1181
|
export interface RenderFootageFilmOptions extends RenderFootageOptions {
|
package/src/footage/providers.ts
CHANGED
|
@@ -42,6 +42,11 @@ export interface SttProvider {
|
|
|
42
42
|
/** Removes background noise from speech; returns the cleaned audio (mp3), alone or with the reported charge. */
|
|
43
43
|
export interface IsolationProvider {
|
|
44
44
|
readonly id: string;
|
|
45
|
+
/**
|
|
46
|
+
* The shortest input in seconds the provider accepts; absent: no minimum. `planSpeechRanges` merges and widens the
|
|
47
|
+
* cut's ranges to reach it and never sends a range still shorter.
|
|
48
|
+
*/
|
|
49
|
+
readonly minSeconds?: number;
|
|
45
50
|
isolate(input: { audio: string; signal?: AbortSignal }): Promise<TtsResult<Buffer | PaidAudio>>;
|
|
46
51
|
}
|
|
47
52
|
|
package/src/footage/render.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { getAudioCodecArgs } from "./clip-sound.js";
|
|
1
2
|
import { getSoundSpan, type Edl } from "./edl.js";
|
|
2
3
|
import { DEFAULT_EDIT, type FootageEdit } from "./film.js";
|
|
3
4
|
import { FOOTAGE_SAMPLE_RATE } from "./normalize.js";
|
|
@@ -105,7 +106,7 @@ export function buildSpeechArgs(edl: Edl, sources: Record<string, string>, outpu
|
|
|
105
106
|
const joined = edl.segments.map((_, index) => `[a${index}]`).join("");
|
|
106
107
|
parts.push(`${joined}concat=n=${edl.segments.length}:v=0:a=1[speech]`);
|
|
107
108
|
}
|
|
108
|
-
return ["-v", "error", "-y", ...inputs, "-filter_complex", parts.join(";"), "-map", "[speech]",
|
|
109
|
+
return ["-v", "error", "-y", ...inputs, "-filter_complex", parts.join(";"), "-map", "[speech]", ...getAudioCodecArgs(output), output];
|
|
109
110
|
}
|
|
110
111
|
|
|
111
112
|
/** The music track: looped to the film's length, ducked by the keyframes, faded in and out. */
|
|
@@ -147,7 +148,7 @@ export function buildLastFrameArgs(edl: Edl, clips: Record<string, string>, outp
|
|
|
147
148
|
return ["-v", "error", "-y", "-ss", String(at), "-i", clip, "-frames:v", "1", "-q:v", "2", output];
|
|
148
149
|
}
|
|
149
150
|
|
|
150
|
-
/**
|
|
151
|
+
/** A file's sound alone, encoded by the output's extension (`getAudioCodecArgs`: `.wav` lossless). */
|
|
151
152
|
export function buildAudioArgs(clip: string, output: string): string[] {
|
|
152
|
-
return ["-v", "error", "-y", "-i", clip, "-vn",
|
|
153
|
+
return ["-v", "error", "-y", "-i", clip, "-vn", ...getAudioCodecArgs(output), output];
|
|
153
154
|
}
|