@softure-ai/marketing-kit 0.15.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +91 -0
- package/README.md +138 -16
- package/dist/config/schema.d.ts +2 -1
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +7 -0
- package/dist/config/schema.js.map +1 -1
- package/dist/footage/clip-sound.d.ts +58 -0
- package/dist/footage/clip-sound.d.ts.map +1 -0
- package/dist/footage/clip-sound.js +103 -0
- package/dist/footage/clip-sound.js.map +1 -0
- package/dist/footage/declick.d.ts +1 -1
- package/dist/footage/declick.d.ts.map +1 -1
- package/dist/footage/declick.js +3 -4
- package/dist/footage/declick.js.map +1 -1
- package/dist/footage/draft.d.ts +19 -5
- package/dist/footage/draft.d.ts.map +1 -1
- package/dist/footage/draft.js +92 -22
- package/dist/footage/draft.js.map +1 -1
- package/dist/footage/elevenlabs.d.ts +5 -0
- package/dist/footage/elevenlabs.d.ts.map +1 -1
- package/dist/footage/elevenlabs.js +10 -3
- package/dist/footage/elevenlabs.js.map +1 -1
- package/dist/footage/failure.d.ts +12 -1
- package/dist/footage/failure.d.ts.map +1 -1
- package/dist/footage/failure.js +10 -1
- package/dist/footage/failure.js.map +1 -1
- package/dist/footage/fake.d.ts +10 -1
- package/dist/footage/fake.d.ts.map +1 -1
- package/dist/footage/fake.js +19 -4
- package/dist/footage/fake.js.map +1 -1
- package/dist/footage/hyperframes-capture.d.ts +30 -0
- package/dist/footage/hyperframes-capture.d.ts.map +1 -0
- package/dist/footage/hyperframes-capture.js +80 -0
- package/dist/footage/hyperframes-capture.js.map +1 -0
- package/dist/footage/isolate-ranges.d.ts +34 -6
- package/dist/footage/isolate-ranges.d.ts.map +1 -1
- package/dist/footage/isolate-ranges.js +90 -23
- package/dist/footage/isolate-ranges.js.map +1 -1
- package/dist/footage/loudness.d.ts +16 -2
- package/dist/footage/loudness.d.ts.map +1 -1
- package/dist/footage/loudness.js +10 -7
- package/dist/footage/loudness.js.map +1 -1
- package/dist/footage/paid.d.ts +12 -2
- package/dist/footage/paid.d.ts.map +1 -1
- package/dist/footage/paid.js +8 -3
- package/dist/footage/paid.js.map +1 -1
- package/dist/footage/pipeline.d.ts +14 -2
- package/dist/footage/pipeline.d.ts.map +1 -1
- package/dist/footage/pipeline.js +55 -46
- package/dist/footage/pipeline.js.map +1 -1
- package/dist/footage/providers.d.ts +5 -0
- package/dist/footage/providers.d.ts.map +1 -1
- package/dist/footage/providers.js.map +1 -1
- package/dist/footage/render.d.ts +1 -1
- package/dist/footage/render.d.ts.map +1 -1
- package/dist/footage/render.js +4 -3
- package/dist/footage/render.js.map +1 -1
- package/dist/footage/speech-ranges.d.ts +79 -11
- package/dist/footage/speech-ranges.d.ts.map +1 -1
- package/dist/footage/speech-ranges.js +137 -17
- package/dist/footage/speech-ranges.js.map +1 -1
- package/dist/index.d.ts +9 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -5
- package/dist/index.js.map +1 -1
- package/dist/og/index.d.ts +2 -0
- package/dist/og/index.d.ts.map +1 -1
- package/dist/og/index.js +2 -0
- package/dist/og/index.js.map +1 -1
- package/dist/og/photo-cover.d.ts +74 -0
- package/dist/og/photo-cover.d.ts.map +1 -0
- package/dist/og/photo-cover.js +357 -0
- package/dist/og/photo-cover.js.map +1 -0
- package/dist/og/photo.d.ts +51 -0
- package/dist/og/photo.d.ts.map +1 -0
- package/dist/og/photo.js +154 -0
- package/dist/og/photo.js.map +1 -0
- package/dist/voice/provider-failure.d.ts +41 -0
- package/dist/voice/provider-failure.d.ts.map +1 -0
- package/dist/voice/provider-failure.js +68 -0
- package/dist/voice/provider-failure.js.map +1 -0
- package/dist/voice/provider.d.ts +7 -1
- package/dist/voice/provider.d.ts.map +1 -1
- package/dist/voice/provider.js +1 -1
- package/dist/voice/provider.js.map +1 -1
- package/package.json +2 -1
- package/schema/marketing.schema.json +6 -0
- package/src/config/schema.ts +7 -0
- package/src/footage/clip-sound.ts +124 -0
- package/src/footage/declick.ts +3 -4
- package/src/footage/draft.ts +123 -22
- package/src/footage/elevenlabs.ts +11 -3
- package/src/footage/failure.ts +14 -1
- package/src/footage/fake.ts +28 -4
- package/src/footage/hyperframes-capture.ts +100 -0
- package/src/footage/isolate-ranges.ts +124 -29
- package/src/footage/loudness.ts +17 -8
- package/src/footage/paid.ts +25 -3
- package/src/footage/pipeline.ts +68 -47
- package/src/footage/providers.ts +5 -0
- package/src/footage/render.ts +4 -3
- package/src/footage/speech-ranges.ts +184 -19
- package/src/index.ts +49 -1
- package/src/og/index.ts +28 -0
- package/src/og/photo-cover.ts +442 -0
- package/src/og/photo.ts +177 -0
- package/src/voice/provider-failure.ts +98 -0
- package/src/voice/provider.ts +7 -2
package/src/footage/pipeline.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { cpSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
2
2
|
import { createRequire } from "node:module";
|
|
3
3
|
import { dirname, extname, join, resolve } from "node:path";
|
|
4
4
|
|
|
@@ -7,14 +7,16 @@ import type { BrandColors } from "../config/colors.js";
|
|
|
7
7
|
import type { BrandFont } from "../config/config.js";
|
|
8
8
|
import type { SfxEvent } from "../config/schema.js";
|
|
9
9
|
import { getHyperframesBin, getHyperframesEnv } from "../render/hyperframes.js";
|
|
10
|
+
import { readHyperframesCapture, type HyperframesCapture } from "./hyperframes-capture.js";
|
|
10
11
|
import { copyFonts } from "../render/render.js";
|
|
11
12
|
import type { TtsResult } from "../voice/provider.js";
|
|
12
13
|
import type { TimedWord } from "../voice/voiceover.js";
|
|
13
14
|
import { checkMusicFileType, describeAudioProbe, probeAudio } from "./audio.js";
|
|
15
|
+
import { prepareClipSounds } from "./clip-sound.js";
|
|
14
16
|
import { readCachedTranscript, writeCacheFile, writeTranscript } from "./cache.js";
|
|
15
17
|
import { composeFootage, END_CARD_SECONDS, getFootageFilmSeconds, getFootageSfxCues } from "./compose.js";
|
|
16
18
|
import { findMissingCaptionGlyphs, getCaptionBounds, getFootageCaptionLayout } from "./captions.js";
|
|
17
|
-
import { err, ok, type FootageResult } from "./failure.js";
|
|
19
|
+
import { err, getProviderFootageFailure, ok, type FootageResult } from "./failure.js";
|
|
18
20
|
import { findLowDiskProblems, getFreeBytes, getRenderTempDir, type FreeBytesReader } from "./disk.js";
|
|
19
21
|
import { buildEdl, getEdlDuration, readEdl, remapWords, validateEdl, type Edl } from "./edl.js";
|
|
20
22
|
import { makePlaceholderTranscript } from "./fake.js";
|
|
@@ -24,6 +26,7 @@ import { DEFAULT_AUTO_THRESHOLD, getDeclickLevel, getFootageEdit, type FootageCl
|
|
|
24
26
|
import { getFilmKeyTerms } from "./keyterms.js";
|
|
25
27
|
import { buildFilmMixArgs } from "./film-mix.js";
|
|
26
28
|
import {
|
|
29
|
+
AUDIO_BITRATE_RANGE_KBPS,
|
|
27
30
|
buildMixArgs,
|
|
28
31
|
DEFAULT_LOUDNESS,
|
|
29
32
|
formatDb,
|
|
@@ -85,7 +88,8 @@ export { getEdlPath } from "./paid.js";
|
|
|
85
88
|
* (a transcript from each clip's `script`, no isolation, no generated music) and keeps its files apart from the real
|
|
86
89
|
* film's (`placeholder/` in the build folder, its own EDL).
|
|
87
90
|
*
|
|
88
|
-
* Build folder layout: `sources/` (downloaded URLs), `clips/<id>.mp4` (normalized), `audio/<id>.
|
|
91
|
+
* Build folder layout: `sources/` (downloaded URLs), `clips/<id>.mp4` (normalized), `audio/<id>.wav` (the clip's
|
|
92
|
+
* lossless, pre-gained sound; `audio/<id>.isolated.wav` its cleaned voice), `edl.json`
|
|
89
93
|
* (`edl.placeholder.json`), `assets/`, `index.html`, `render.mp4` (hyperframes' output until the final pass).
|
|
90
94
|
*
|
|
91
95
|
* The finished film is publishable: the speech track normalized to a loudness target before the music goes under it,
|
|
@@ -163,6 +167,12 @@ export type TranscribeOutcome =
|
|
|
163
167
|
words: Record<string, TimedWord[]>;
|
|
164
168
|
/** The audio each clip's speech is cut from (the cleaned voice when isolated). */
|
|
165
169
|
speech: Record<string, string>;
|
|
170
|
+
/**
|
|
171
|
+
* Each clip's pre-gain in dB (`prepareClipSounds`, issue #1159): the gain of the sound `speech` is made from. An
|
|
172
|
+
* app that lays its own cached ranges raises one cut from the clip's un-gained sound by it. Always answered;
|
|
173
|
+
* optional so an app's own `ready` outcome still compiles.
|
|
174
|
+
*/
|
|
175
|
+
speechGainDb?: Record<string, number>;
|
|
166
176
|
};
|
|
167
177
|
|
|
168
178
|
export function getFrame(film: Pick<FootageFilm, "format">): Frame {
|
|
@@ -283,23 +293,28 @@ function getClipKey(clip: PreparedClip): Promise<string> {
|
|
|
283
293
|
return getClipCacheKey(clip.source, clip.clip);
|
|
284
294
|
}
|
|
285
295
|
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
296
|
+
/** The cached cleaned voice (the provider's file) decoded into the build folder as lossless WAV, named by the clip's id. */
|
|
297
|
+
async function decodeIsolatedVoice(cached: string, clipId: string, buildDir: string, signal: AbortSignal | undefined): Promise<string> {
|
|
298
|
+
const output = join(buildDir, "audio", `${clipId}.isolated.wav`);
|
|
299
|
+
mkdirSync(dirname(output), { recursive: true });
|
|
300
|
+
await runFfmpeg(`Decoding the cleaned voice of clip ${clipId}`, buildAudioArgs(cached, output), { ...(signal === undefined ? {} : { signal }), output });
|
|
301
|
+
return output;
|
|
291
302
|
}
|
|
292
303
|
|
|
293
|
-
/**
|
|
294
|
-
function
|
|
304
|
+
/** Each clip's lossless, pre-gained sound (`prepareClipSounds`) and its pre-gain in dB, by clip id. */
|
|
305
|
+
async function getClipSoundFiles(clips: PreparedClip[], buildDir: string, signal: AbortSignal | undefined): Promise<{ speech: Record<string, string>; speechGainDb: Record<string, number> }> {
|
|
306
|
+
const sounds = Object.entries(await prepareClipSounds(clips, { buildDir, ...(signal === undefined ? {} : { signal }) }));
|
|
307
|
+
return { speech: Object.fromEntries(sounds.map(([id, sound]) => [id, sound.file])), speechGainDb: Object.fromEntries(sounds.map(([id, sound]) => [id, sound.gainDb])) };
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/** The free transcripts of `--placeholder`, written under `placeholder/` for a look; the speech is each clip's lossless sound. */
|
|
311
|
+
async function transcribePlaceholder(clips: PreparedClip[], buildDir: string, signal: AbortSignal | undefined): Promise<TranscribeOutcome> {
|
|
295
312
|
const words: Record<string, TimedWord[]> = {};
|
|
296
|
-
const speech: Record<string, string> = {};
|
|
297
313
|
for (const clip of clips) {
|
|
298
314
|
words[clip.clip.id] = clip.clip.role === "talk" ? makePlaceholderTranscript(clip.clip.script, clip.duration) : [];
|
|
299
315
|
writeTranscript(join(buildDir, "placeholder", `${clip.clip.id}.transcript.json`), words[clip.clip.id] ?? []);
|
|
300
|
-
speech[clip.clip.id] = clip.file;
|
|
301
316
|
}
|
|
302
|
-
return { kind: "ready", words,
|
|
317
|
+
return { kind: "ready", words, ...(await getClipSoundFiles(clips, buildDir, signal)) };
|
|
303
318
|
}
|
|
304
319
|
|
|
305
320
|
/** The paid music step, when the film's music still has to be bought; null when it is cached or free. */
|
|
@@ -322,7 +337,7 @@ export function transcribeFootage(film: FootageFilm, clips: PreparedClip[], opti
|
|
|
322
337
|
/** The body of `transcribeFootage`, inside its ffmpeg failure reporting. */
|
|
323
338
|
async function runTranscribeFootage(film: FootageFilm, clips: PreparedClip[], options: FootageRunOptions): Promise<FootageResult<TranscribeOutcome>> {
|
|
324
339
|
const { buildDir, providers, log, signal } = options;
|
|
325
|
-
if (options.isPlaceholder) return ok(transcribePlaceholder(
|
|
340
|
+
if (options.isPlaceholder) return ok(await transcribePlaceholder(clips, buildDir, signal));
|
|
326
341
|
const keyTerms = getFilmKeyTerms(film);
|
|
327
342
|
if (!keyTerms.ok) return err(`footage film ${film.id}: ${keyTerms.errors.join(" ")}`, { kind: "input" });
|
|
328
343
|
const sttKeyTerms = keyTerms.terms.length === 0 ? {} : { keyTerms: keyTerms.terms };
|
|
@@ -346,11 +361,9 @@ async function runTranscribeFootage(film: FootageFilm, clips: PreparedClip[], op
|
|
|
346
361
|
return ok({ kind: "dry-run", steps: all });
|
|
347
362
|
}
|
|
348
363
|
const words: Record<string, TimedWord[]> = {};
|
|
349
|
-
const
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
speech[clip.clip.id] = clip.file;
|
|
353
|
-
}
|
|
364
|
+
for (const clip of clips) words[clip.clip.id] = [];
|
|
365
|
+
// Every clip's sound, lossless and pre-gained: what the isolator, the speech to text and the speech track hear (#1139).
|
|
366
|
+
const { speech, speechGainDb } = await getClipSoundFiles(clips, buildDir, signal);
|
|
354
367
|
const isolating = plan.filter(({ isolated }) => isolated !== null && !existsSync(isolated.path));
|
|
355
368
|
const transcribing = plan.filter(({ transcript }) => !existsSync(transcript.path));
|
|
356
369
|
const isolateProgress = createStageProgress(options.onProgress, "isolate", isolating.map(({ clip }) => ({ item: clip.clip.id, seconds: clip.duration })));
|
|
@@ -358,27 +371,27 @@ async function runTranscribeFootage(film: FootageFilm, clips: PreparedClip[], op
|
|
|
358
371
|
for (const entry of plan) {
|
|
359
372
|
const { clip, isolated, transcript } = entry;
|
|
360
373
|
const id = clip.clip.id;
|
|
361
|
-
|
|
362
|
-
const getAudio = (): Promise<string> => (audio ??= extractAudio(clip, buildDir, signal));
|
|
374
|
+
const sound = speech[id] ?? clip.file;
|
|
363
375
|
let sttInput: string | null = null;
|
|
364
376
|
if (isolated !== null) {
|
|
365
377
|
if (!existsSync(isolated.path)) {
|
|
366
378
|
const index = isolating.indexOf(entry);
|
|
367
379
|
isolateProgress(index, 0);
|
|
368
|
-
const input = await getAudio();
|
|
369
380
|
throwIfAborted(signal);
|
|
370
381
|
log(`voice isolation (${providers.isolation.id}): clip ${id}, ${clip.duration.toFixed(1)} s.`);
|
|
371
|
-
const cleaned = await providers.isolation.isolate({ audio:
|
|
382
|
+
const cleaned = await providers.isolation.isolate({ audio: sound, ...(signal === undefined ? {} : { signal }) });
|
|
372
383
|
throwIfAborted(signal);
|
|
373
|
-
if (!cleaned.ok) return err(`clip "${id}": ${cleaned.error}
|
|
384
|
+
if (!cleaned.ok) return err(`clip "${id}": ${cleaned.error}`, getProviderFootageFailure(cleaned.providerFailure));
|
|
374
385
|
const paid = toPaidAudio(cleaned.value);
|
|
375
386
|
writeCacheFile(isolated.path, paid.audio);
|
|
376
387
|
options.onCharge?.({ step: "isolate", item: id, provider: providers.isolation.id, units: clip.duration, ...(paid.credits === undefined ? {} : { credits: paid.credits }), cacheKey: isolated.key });
|
|
377
388
|
log(`voice isolation: saved ${isolated.path} (commit it).`);
|
|
378
389
|
isolateProgress(index, clip.duration);
|
|
379
390
|
}
|
|
380
|
-
speech
|
|
381
|
-
|
|
391
|
+
// The provider's file decoded once: the speech track and the speech to text read it lossless, named by the clip.
|
|
392
|
+
const decoded = await decodeIsolatedVoice(isolated.path, id, buildDir, signal);
|
|
393
|
+
speech[id] = decoded;
|
|
394
|
+
sttInput = decoded;
|
|
382
395
|
}
|
|
383
396
|
const cached = readCachedTranscript(transcript.path);
|
|
384
397
|
if (!cached.ok) return err(cached.error);
|
|
@@ -387,21 +400,15 @@ async function runTranscribeFootage(film: FootageFilm, clips: PreparedClip[], op
|
|
|
387
400
|
words[id] = cached.words;
|
|
388
401
|
continue;
|
|
389
402
|
}
|
|
390
|
-
if (sttInput !== null) {
|
|
391
|
-
// The provider sees the clip's id as the file name, as it does for the plain sound.
|
|
392
|
-
const cleaned = sttInput;
|
|
393
|
-
sttInput = join(buildDir, "audio", `${id}.isolated.mp3`);
|
|
394
|
-
mkdirSync(dirname(sttInput), { recursive: true });
|
|
395
|
-
copyFileSync(cleaned, sttInput);
|
|
396
|
-
}
|
|
397
403
|
const index = transcribing.indexOf(entry);
|
|
398
404
|
transcribeProgress(index, 0);
|
|
399
|
-
|
|
405
|
+
// The provider sees the clip's id as the file name, for the cleaned voice as for the plain sound.
|
|
406
|
+
const input = sttInput ?? sound;
|
|
400
407
|
throwIfAborted(signal);
|
|
401
408
|
log(`speech to text (${providers.stt.id}): clip ${id}, ${clip.duration.toFixed(1)} s.`);
|
|
402
409
|
const result = await providers.stt.transcribe({ audio: input, language: film.language, ...sttKeyTerms, ...(signal === undefined ? {} : { signal }) });
|
|
403
410
|
throwIfAborted(signal);
|
|
404
|
-
if (!result.ok) return err(`clip "${id}": ${result.error}
|
|
411
|
+
if (!result.ok) return err(`clip "${id}": ${result.error}`, getProviderFootageFailure(result.providerFailure));
|
|
405
412
|
writeTranscript(transcript.path, result.value.words);
|
|
406
413
|
const { credits } = result.value;
|
|
407
414
|
options.onCharge?.({ step: "transcribe", item: id, provider: providers.stt.id, units: clip.duration, ...(credits === undefined ? {} : { credits }), ...(keyTerms.terms.length === 0 ? {} : { keyTerms: keyTerms.terms.length }), cacheKey: transcript.key });
|
|
@@ -409,7 +416,7 @@ async function runTranscribeFootage(film: FootageFilm, clips: PreparedClip[], op
|
|
|
409
416
|
transcribeProgress(index, clip.duration);
|
|
410
417
|
words[id] = result.value.words;
|
|
411
418
|
}
|
|
412
|
-
return ok({ kind: "ready", words, speech });
|
|
419
|
+
return ok({ kind: "ready", words, speech, speechGainDb });
|
|
413
420
|
}
|
|
414
421
|
|
|
415
422
|
/** The talking clips' sentences with their ids (`s1`…), as `buildEdl` names them, for choosing what to keep. */
|
|
@@ -607,7 +614,7 @@ async function getMusic(film: FootageFilm, edl: Edl, clips: PreparedClip[], seco
|
|
|
607
614
|
return ok(null);
|
|
608
615
|
}
|
|
609
616
|
throwIfAborted(signal);
|
|
610
|
-
if (!bought.ok) return err(`music: ${bought.error}
|
|
617
|
+
if (!bought.ok) return err(`music: ${bought.error}`, getProviderFootageFailure(bought.providerFailure));
|
|
611
618
|
const paid = toPaidAudio(bought.value);
|
|
612
619
|
writeCacheFile(path, paid.audio);
|
|
613
620
|
options.onCharge?.({ step: "music", item: "film", provider: providers.music.id, units: seconds, ...(paid.credits === undefined ? {} : { credits: paid.credits }), cacheKey: cache.key });
|
|
@@ -617,12 +624,12 @@ async function getMusic(film: FootageFilm, edl: Edl, clips: PreparedClip[], seco
|
|
|
617
624
|
}
|
|
618
625
|
|
|
619
626
|
/**
|
|
620
|
-
* The speech chain's files: the cut speech, the declicked track and the final
|
|
621
|
-
*
|
|
627
|
+
* The speech chain's files, all lossless float WAV (issue #1139): the cut speech, the declicked track and the final
|
|
628
|
+
* `speech.wav`, which the last stage that runs writes. The film's final AAC is the chain's only lossy encode.
|
|
622
629
|
*/
|
|
623
630
|
export function getSpeechChainFiles(assetsDir: string, stages: { isDeclick: boolean; isLoudness: boolean }): { rawSpeechFile: string; declickedFile: string; speechFile: string } {
|
|
624
|
-
const speechFile = join(assetsDir, "speech.
|
|
625
|
-
const rawSpeechFile = stages.isDeclick || stages.isLoudness ? join(assetsDir, "speech.raw.
|
|
631
|
+
const speechFile = join(assetsDir, "speech.wav");
|
|
632
|
+
const rawSpeechFile = stages.isDeclick || stages.isLoudness ? join(assetsDir, "speech.raw.wav") : speechFile;
|
|
626
633
|
const declickedFile = !stages.isDeclick ? rawSpeechFile : stages.isLoudness ? join(assetsDir, "speech.declicked.wav") : speechFile;
|
|
627
634
|
return { rawSpeechFile, declickedFile, speechFile };
|
|
628
635
|
}
|
|
@@ -676,6 +683,11 @@ export type RenderedFootage =
|
|
|
676
683
|
edlPath: string;
|
|
677
684
|
/** What the speech track's declick stage did (`FootageFilm.speech.declick`); null when it is off. */
|
|
678
685
|
declick: DeclickReport | null;
|
|
686
|
+
/**
|
|
687
|
+
* How hyperframes captured the frames (issue #1145): the capture path (`screenshot` is the slower one), the GPU
|
|
688
|
+
* mode, each stage's seconds and the screenshot hint, read from hyperframes' summary; null when it printed none.
|
|
689
|
+
*/
|
|
690
|
+
capture: HyperframesCapture | null;
|
|
679
691
|
} & FinishedFootage);
|
|
680
692
|
|
|
681
693
|
/**
|
|
@@ -826,7 +838,7 @@ async function runRenderFootage(
|
|
|
826
838
|
assets: {
|
|
827
839
|
clips: Object.fromEntries(clips.map((clip) => [clip.clip.id, `clips/${clip.clip.id}.mp4`])),
|
|
828
840
|
picture: "assets/picture.mp4",
|
|
829
|
-
speech: "assets/speech.
|
|
841
|
+
speech: "assets/speech.wav",
|
|
830
842
|
music: musicAsset,
|
|
831
843
|
lastFrame: "assets/last.jpg",
|
|
832
844
|
logo: options.brand.logo === null ? null : "assets/logo.svg",
|
|
@@ -844,7 +856,7 @@ async function runRenderFootage(
|
|
|
844
856
|
const roles = Object.fromEntries(clips.map((clip) => [clip.clip.id, clip.clip.role]));
|
|
845
857
|
const finished = await finishFootage(film, { renderFile, seconds, coverAt: film.cover?.at ?? getDefaultCoverSeconds(edl, roles), mix: prepared.value }, options);
|
|
846
858
|
if (!finished.ok) return finished;
|
|
847
|
-
return ok({ kind: "rendered", output: options.output, seconds, edlPath: getEdlPath(buildDir, options.isPlaceholder), declick, ...finished.value });
|
|
859
|
+
return ok({ kind: "rendered", output: options.output, seconds, edlPath: getEdlPath(buildDir, options.isPlaceholder), declick, capture: rendered.value, ...finished.value });
|
|
848
860
|
}
|
|
849
861
|
|
|
850
862
|
/** What `renderFootage` reads from the written film. */
|
|
@@ -1011,6 +1023,10 @@ async function finishMix(input: {
|
|
|
1011
1023
|
signal?: AbortSignal;
|
|
1012
1024
|
}): Promise<FootageResult<{ attempt: MixAttempt; tries: number; probes: number; measured: MixLoudness }>> {
|
|
1013
1025
|
const aborting = input.signal === undefined ? {} : { signal: input.signal };
|
|
1026
|
+
const bitrate = input.target.audioBitrateKbps;
|
|
1027
|
+
if (bitrate !== undefined && !(Number.isInteger(bitrate) && bitrate >= AUDIO_BITRATE_RANGE_KBPS.min && bitrate <= AUDIO_BITRATE_RANGE_KBPS.max)) {
|
|
1028
|
+
return err(`mixLoudness.audioBitrateKbps must be a whole number from ${String(AUDIO_BITRATE_RANGE_KBPS.min)} to ${String(AUDIO_BITRATE_RANGE_KBPS.max)}, got ${String(bitrate)}.`, { kind: "input" });
|
|
1029
|
+
}
|
|
1014
1030
|
const attempts = getMixAttempts(input.target);
|
|
1015
1031
|
const probe = async (attempt: MixAttempt): Promise<MixLoudness> => {
|
|
1016
1032
|
const probed = await measureLoudness(input.input, input.target, { label: "the film's mix through its limiter", filter: getMixFilter(attempt), ...aborting });
|
|
@@ -1025,7 +1041,7 @@ async function finishMix(input: {
|
|
|
1025
1041
|
const solved = await solveMixGain({ ceilingDbtp, startGainDb, target: input.target, shift, measure: probe });
|
|
1026
1042
|
probes += solved.probes.length;
|
|
1027
1043
|
const attempt = solved.best.attempt;
|
|
1028
|
-
await runFfmpeg("The film's mix", buildMixArgs({ input: input.input, filter: getMixFilter(attempt), output: input.output }), { ...aborting, output: input.output });
|
|
1044
|
+
await runFfmpeg("The film's mix", buildMixArgs({ input: input.input, filter: getMixFilter(attempt), output: input.output, ...(input.target.audioBitrateKbps === undefined ? {} : { bitrateKbps: input.target.audioBitrateKbps }) }), { ...aborting, output: input.output });
|
|
1029
1045
|
const encoded = await measureLoudness(input.output, input.target, { label: "the film's encoded mix", ...aborting });
|
|
1030
1046
|
if (encoded.kind === "silent") return err("the film's mix measures silent after its gain and limiter.", { kind: "loudness", loudness: null });
|
|
1031
1047
|
const measured = { integrated: encoded.measurement.inputI, truePeak: encoded.measurement.inputTp };
|
|
@@ -1121,15 +1137,20 @@ async function buildPicture(input: BuildPictureInput): Promise<void> {
|
|
|
1121
1137
|
input.onSeconds(getPictureFrames(pieces) / FOOTAGE_FPS);
|
|
1122
1138
|
}
|
|
1123
1139
|
|
|
1124
|
-
/**
|
|
1125
|
-
|
|
1140
|
+
/**
|
|
1141
|
+
* The hyperframes render as a child process: its lines go to the log (progress every 10 %), its percentages to
|
|
1142
|
+
* `onProgress`; it answers the capture report read from its summary (null when it printed none).
|
|
1143
|
+
*/
|
|
1144
|
+
async function runHyperframesRender(options: RenderFootageOptions, output: string, seconds: number): Promise<FootageResult<HyperframesCapture | null>> {
|
|
1126
1145
|
const progress = createStageProgress(options.onProgress, "render", [{ item: "film", seconds }]);
|
|
1127
1146
|
progress(0, 0);
|
|
1128
1147
|
let loggedTenth = -1;
|
|
1148
|
+
const lines: string[] = [];
|
|
1129
1149
|
const onLine = (line: string): void => {
|
|
1130
1150
|
if (line.trim().length === 0) return;
|
|
1131
1151
|
const percent = readRenderPercent(line);
|
|
1132
1152
|
if (percent === null) {
|
|
1153
|
+
lines.push(line);
|
|
1133
1154
|
options.log(line.trim());
|
|
1134
1155
|
return;
|
|
1135
1156
|
}
|
|
@@ -1160,7 +1181,7 @@ async function runHyperframesRender(options: RenderFootageOptions, output: strin
|
|
|
1160
1181
|
if (result.kind === "not-started") return err(`hyperframes render did not start: ${result.error}.`);
|
|
1161
1182
|
if (result.code !== 0) return err(`hyperframes render ended with code ${String(result.code)} (signal ${result.signal ?? "-"}).`);
|
|
1162
1183
|
progress(0, seconds);
|
|
1163
|
-
return ok(
|
|
1184
|
+
return ok(readHyperframesCapture(lines));
|
|
1164
1185
|
}
|
|
1165
1186
|
|
|
1166
1187
|
export interface RenderFootageFilmOptions extends RenderFootageOptions {
|
package/src/footage/providers.ts
CHANGED
|
@@ -42,6 +42,11 @@ export interface SttProvider {
|
|
|
42
42
|
/** Removes background noise from speech; returns the cleaned audio (mp3), alone or with the reported charge. */
|
|
43
43
|
export interface IsolationProvider {
|
|
44
44
|
readonly id: string;
|
|
45
|
+
/**
|
|
46
|
+
* The shortest input in seconds the provider accepts; absent: no minimum. `planSpeechRanges` merges and widens the
|
|
47
|
+
* cut's ranges to reach it and never sends a range still shorter.
|
|
48
|
+
*/
|
|
49
|
+
readonly minSeconds?: number;
|
|
45
50
|
isolate(input: { audio: string; signal?: AbortSignal }): Promise<TtsResult<Buffer | PaidAudio>>;
|
|
46
51
|
}
|
|
47
52
|
|
package/src/footage/render.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { getAudioCodecArgs } from "./clip-sound.js";
|
|
1
2
|
import { getSoundSpan, type Edl } from "./edl.js";
|
|
2
3
|
import { DEFAULT_EDIT, type FootageEdit } from "./film.js";
|
|
3
4
|
import { FOOTAGE_SAMPLE_RATE } from "./normalize.js";
|
|
@@ -105,7 +106,7 @@ export function buildSpeechArgs(edl: Edl, sources: Record<string, string>, outpu
|
|
|
105
106
|
const joined = edl.segments.map((_, index) => `[a${index}]`).join("");
|
|
106
107
|
parts.push(`${joined}concat=n=${edl.segments.length}:v=0:a=1[speech]`);
|
|
107
108
|
}
|
|
108
|
-
return ["-v", "error", "-y", ...inputs, "-filter_complex", parts.join(";"), "-map", "[speech]",
|
|
109
|
+
return ["-v", "error", "-y", ...inputs, "-filter_complex", parts.join(";"), "-map", "[speech]", ...getAudioCodecArgs(output), output];
|
|
109
110
|
}
|
|
110
111
|
|
|
111
112
|
/** The music track: looped to the film's length, ducked by the keyframes, faded in and out. */
|
|
@@ -147,7 +148,7 @@ export function buildLastFrameArgs(edl: Edl, clips: Record<string, string>, outp
|
|
|
147
148
|
return ["-v", "error", "-y", "-ss", String(at), "-i", clip, "-frames:v", "1", "-q:v", "2", output];
|
|
148
149
|
}
|
|
149
150
|
|
|
150
|
-
/**
|
|
151
|
+
/** A file's sound alone, encoded by the output's extension (`getAudioCodecArgs`: `.wav` lossless). */
|
|
151
152
|
export function buildAudioArgs(clip: string, output: string): string[] {
|
|
152
|
-
return ["-v", "error", "-y", "-i", clip, "-vn",
|
|
153
|
+
return ["-v", "error", "-y", "-i", clip, "-vn", ...getAudioCodecArgs(output), output];
|
|
153
154
|
}
|
|
@@ -1,7 +1,11 @@
|
|
|
1
|
+
import { extname } from "node:path";
|
|
2
|
+
|
|
1
3
|
import { footageCacheKey, getFootageCachePath } from "./cache.js";
|
|
4
|
+
import { getAudioCodecArgs } from "./clip-sound.js";
|
|
2
5
|
import { getSoundSpan, type Edl } from "./edl.js";
|
|
3
6
|
import type { ClipRole } from "./film.js";
|
|
4
7
|
import { FOOTAGE_SAMPLE_RATE } from "./normalize.js";
|
|
8
|
+
import type { IsolationProvider } from "./providers.js";
|
|
5
9
|
|
|
6
10
|
/**
|
|
7
11
|
* The parts of each talking clip a cut hears, as ranges in whole milliseconds, and how their cleaned voice is laid
|
|
@@ -12,11 +16,31 @@ import { FOOTAGE_SAMPLE_RATE } from "./normalize.js";
|
|
|
12
16
|
/** Seconds added on each side of a sound span, so a cut's cross-fade and fades read cleaned sound. */
|
|
13
17
|
export const DEFAULT_SPEECH_RANGE_MARGIN = 0.25;
|
|
14
18
|
|
|
19
|
+
/** Seconds of the largest gap between two ranges of a clip that are isolated as one range (`planSpeechRanges`). */
|
|
20
|
+
export const DEFAULT_SPEECH_RANGE_MERGE_GAP = 2;
|
|
21
|
+
|
|
22
|
+
/** Seconds a range shorter than the provider's minimum is widened past it, so the cut file is safely long enough. */
|
|
23
|
+
export const DEFAULT_SPEECH_RANGE_MIN_PAD = 0.4;
|
|
24
|
+
|
|
25
|
+
/** Why a planned range is not sent: shorter than the provider's minimum even after merging and widening. */
|
|
26
|
+
export interface SpeechRangeSkip {
|
|
27
|
+
reason: "too_short";
|
|
28
|
+
/** The provider's minimum, in seconds. */
|
|
29
|
+
minSeconds: number;
|
|
30
|
+
}
|
|
31
|
+
|
|
15
32
|
/** A range of a clip's trimmed sound, in whole milliseconds (the cache key's unit). */
|
|
16
33
|
export interface SpeechRange {
|
|
17
34
|
clip: string;
|
|
18
35
|
startMs: number;
|
|
19
36
|
endMs: number;
|
|
37
|
+
/**
|
|
38
|
+
* The part of the range the cut hears, when the range was widened with the clip's own context to reach the
|
|
39
|
+
* provider's minimum; absent: the whole range. Only this part of the cleaned voice is laid.
|
|
40
|
+
*/
|
|
41
|
+
heard?: { startMs: number; endMs: number };
|
|
42
|
+
/** Set when the range cannot pass the provider (the clip itself is shorter than its minimum): it is not sent. */
|
|
43
|
+
skip?: SpeechRangeSkip;
|
|
20
44
|
}
|
|
21
45
|
|
|
22
46
|
/** Where a range's cleaned sound is laid in its clip, in milliseconds; inside the range. */
|
|
@@ -33,15 +57,35 @@ export interface SpeechRangeClip {
|
|
|
33
57
|
duration: number;
|
|
34
58
|
}
|
|
35
59
|
|
|
60
|
+
export interface PlanSpeechRangesOptions {
|
|
61
|
+
/** Seconds added on each side of a sound span; default `DEFAULT_SPEECH_RANGE_MARGIN` (0.25). */
|
|
62
|
+
margin?: number;
|
|
63
|
+
/** The isolation provider's minimum input in seconds (`IsolationProvider.minSeconds`); absent: no minimum. */
|
|
64
|
+
minSeconds?: number;
|
|
65
|
+
/**
|
|
66
|
+
* Ranges of a clip at most this many seconds apart are isolated as one range; default
|
|
67
|
+
* `DEFAULT_SPEECH_RANGE_MERGE_GAP` (2). `null` keeps every range apart (only identical ranges and ranges inside
|
|
68
|
+
* another are dropped); `Infinity` isolates each clip's whole used span once.
|
|
69
|
+
*/
|
|
70
|
+
mergeGap?: number | null;
|
|
71
|
+
/** Seconds a range shorter than `minSeconds` is widened past it; default `DEFAULT_SPEECH_RANGE_MIN_PAD` (0.4). */
|
|
72
|
+
minPad?: number;
|
|
73
|
+
}
|
|
74
|
+
|
|
36
75
|
/**
|
|
37
|
-
* The ranges
|
|
76
|
+
* The source ranges to isolate of each talking clip: every segment's sound span (its own clip's or `audio`'s), and the
|
|
38
77
|
* picture clip's own span when `audio.ownVolume` keeps it under, widened by `margin` seconds on each side, clamped to
|
|
39
|
-
* the clip and rounded out to whole milliseconds. Identical ranges and ranges inside another are dropped
|
|
40
|
-
*
|
|
41
|
-
*
|
|
78
|
+
* the clip and rounded out to whole milliseconds. Identical ranges and ranges inside another are dropped; ranges of a
|
|
79
|
+
* clip at most `mergeGap` seconds apart become one. With `minSeconds`, a range still shorter is widened symmetrically
|
|
80
|
+
* with the clip's own context to `minSeconds + minPad` (shifted where the clip's edge stops it; `heard` keeps the part
|
|
81
|
+
* the cut hears), ranges the widening brought within `mergeGap` are merged, and a range still shorter (a clip shorter
|
|
82
|
+
* than the minimum) is marked `skip`. Sorted by clip order, then start.
|
|
42
83
|
*/
|
|
43
|
-
export function planSpeechRanges(edl: Edl, clips: readonly SpeechRangeClip[], options:
|
|
84
|
+
export function planSpeechRanges(edl: Edl, clips: readonly SpeechRangeClip[], options: PlanSpeechRangesOptions = {}): SpeechRange[] {
|
|
44
85
|
const margin = Math.max(0, options.margin ?? DEFAULT_SPEECH_RANGE_MARGIN);
|
|
86
|
+
const mergeGap = options.mergeGap === undefined ? DEFAULT_SPEECH_RANGE_MERGE_GAP : options.mergeGap;
|
|
87
|
+
const gapMs = mergeGap === null ? null : Math.max(0, mergeGap) * 1000;
|
|
88
|
+
const minSeconds = options.minSeconds !== undefined && options.minSeconds > 0 ? options.minSeconds : null;
|
|
45
89
|
const byId = new Map(clips.map((clip) => [clip.id, clip]));
|
|
46
90
|
const found = new Map<string, SpeechRange[]>();
|
|
47
91
|
const add = (span: { clip: string; start: number; end: number }): void => {
|
|
@@ -62,25 +106,90 @@ export function planSpeechRanges(edl: Edl, clips: readonly SpeechRangeClip[], op
|
|
|
62
106
|
const ranges: SpeechRange[] = [];
|
|
63
107
|
for (const clip of clips) {
|
|
64
108
|
const list = (found.get(clip.id) ?? []).sort((a, b) => a.startMs - b.startMs || b.endMs - a.endMs);
|
|
109
|
+
let kept: SpeechRange[] = [];
|
|
65
110
|
for (const range of list) {
|
|
66
|
-
const isInside =
|
|
67
|
-
if (!isInside)
|
|
111
|
+
const isInside = kept.some((other) => other.startMs <= range.startMs && other.endMs >= range.endMs);
|
|
112
|
+
if (!isInside) kept.push(range);
|
|
113
|
+
}
|
|
114
|
+
if (gapMs !== null) kept = mergeSpeechRanges(kept, gapMs);
|
|
115
|
+
if (minSeconds !== null) {
|
|
116
|
+
const minMs = Math.ceil(minSeconds * 1000);
|
|
117
|
+
const targetMs = Math.ceil((minSeconds + Math.max(0, options.minPad ?? DEFAULT_SPEECH_RANGE_MIN_PAD)) * 1000);
|
|
118
|
+
const clipMs = Math.floor(clip.duration * 1000);
|
|
119
|
+
kept = kept.map((range) => widenSpeechRange(range, { minMs, targetMs, clipMs }));
|
|
120
|
+
if (gapMs !== null) kept = mergeSpeechRanges(kept, gapMs);
|
|
121
|
+
kept = kept.map((range) => (range.endMs - range.startMs < minMs ? { ...range, skip: { reason: "too_short", minSeconds } } : range));
|
|
68
122
|
}
|
|
123
|
+
ranges.push(...kept);
|
|
69
124
|
}
|
|
70
125
|
return ranges;
|
|
71
126
|
}
|
|
72
127
|
|
|
128
|
+
/** How `isolateSpeechRanges` and the estimate plan the ranges: the caller's margin and merge gap, the provider's minimum. */
|
|
129
|
+
export function getPlanOptions(options: Pick<PlanSpeechRangesOptions, "margin" | "mergeGap" | "minPad">, isolation: IsolationProvider): PlanSpeechRangesOptions {
|
|
130
|
+
return {
|
|
131
|
+
...(options.margin === undefined ? {} : { margin: options.margin }),
|
|
132
|
+
...(options.mergeGap === undefined ? {} : { mergeGap: options.mergeGap }),
|
|
133
|
+
...(options.minPad === undefined ? {} : { minPad: options.minPad }),
|
|
134
|
+
...(isolation.minSeconds === undefined ? {} : { minSeconds: isolation.minSeconds }),
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** The part of a range the cut hears. */
|
|
139
|
+
function getHeard(range: SpeechRange): { startMs: number; endMs: number } {
|
|
140
|
+
return range.heard ?? { startMs: range.startMs, endMs: range.endMs };
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** A range with `heard` set only when it is narrower than the range. */
|
|
144
|
+
function withHeard(range: Pick<SpeechRange, "clip" | "startMs" | "endMs">, heard: { startMs: number; endMs: number }): SpeechRange {
|
|
145
|
+
const isWhole = heard.startMs <= range.startMs && heard.endMs >= range.endMs;
|
|
146
|
+
return isWhole ? { clip: range.clip, startMs: range.startMs, endMs: range.endMs } : { clip: range.clip, startMs: range.startMs, endMs: range.endMs, heard };
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** One clip's ranges (sorted by start) with every two at most `gapMs` apart merged into one; the heard parts joined. */
|
|
150
|
+
function mergeSpeechRanges(ranges: readonly SpeechRange[], gapMs: number): SpeechRange[] {
|
|
151
|
+
const merged: SpeechRange[] = [];
|
|
152
|
+
for (const range of [...ranges].sort((a, b) => a.startMs - b.startMs)) {
|
|
153
|
+
const last = merged.at(-1);
|
|
154
|
+
if (last === undefined || range.startMs - last.endMs > gapMs) {
|
|
155
|
+
merged.push(range);
|
|
156
|
+
continue;
|
|
157
|
+
}
|
|
158
|
+
const lastHeard = getHeard(last);
|
|
159
|
+
const heard = getHeard(range);
|
|
160
|
+
merged[merged.length - 1] = withHeard(
|
|
161
|
+
{ clip: last.clip, startMs: last.startMs, endMs: Math.max(last.endMs, range.endMs) },
|
|
162
|
+
{ startMs: Math.min(lastHeard.startMs, heard.startMs), endMs: Math.max(lastHeard.endMs, heard.endMs) },
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
return merged;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/** A range shorter than `minMs` widened symmetrically to `targetMs` with the clip's own context, inside the clip. */
|
|
169
|
+
function widenSpeechRange(range: SpeechRange, limits: { minMs: number; targetMs: number; clipMs: number }): SpeechRange {
|
|
170
|
+
const length = range.endMs - range.startMs;
|
|
171
|
+
if (length >= limits.minMs) return range;
|
|
172
|
+
const target = Math.min(limits.targetMs, limits.clipMs);
|
|
173
|
+
const endMs = Math.min(limits.clipMs, Math.max(0, range.startMs - Math.floor((target - length) / 2)) + target);
|
|
174
|
+
const startMs = Math.max(0, endMs - target);
|
|
175
|
+
return withHeard({ clip: range.clip, startMs, endMs }, getHeard(range));
|
|
176
|
+
}
|
|
177
|
+
|
|
73
178
|
/**
|
|
74
|
-
* Where each of one clip's ranges is laid: the whole range, except that two
|
|
75
|
-
* middle of their overlap. Takes ranges of one clip, none inside another (as
|
|
179
|
+
* Where each of one clip's ranges is laid: the part the cut hears (`heard`, else the whole range), except that two
|
|
180
|
+
* overlapping neighbours switch at the middle of their overlap. Takes ranges of one clip, none inside another (as
|
|
181
|
+
* `planSpeechRanges` answers them).
|
|
76
182
|
*/
|
|
77
183
|
export function getSpeechRangeWindows(ranges: readonly SpeechRange[]): SpeechRangeWindow[] {
|
|
78
|
-
const sorted = [...ranges].sort((a, b) => a.startMs - b.startMs);
|
|
184
|
+
const sorted = [...ranges].sort((a, b) => getHeard(a).startMs - getHeard(b).startMs);
|
|
79
185
|
return sorted.map((range, index) => {
|
|
186
|
+
const heard = getHeard(range);
|
|
80
187
|
const before = sorted[index - 1];
|
|
81
188
|
const after = sorted[index + 1];
|
|
82
|
-
const
|
|
83
|
-
const
|
|
189
|
+
const beforeEnd = before === undefined ? null : getHeard(before).endMs;
|
|
190
|
+
const afterStart = after === undefined ? null : getHeard(after).startMs;
|
|
191
|
+
const startMs = beforeEnd !== null && beforeEnd > heard.startMs ? Math.round((heard.startMs + beforeEnd) / 2) : heard.startMs;
|
|
192
|
+
const endMs = afterStart !== null && afterStart < heard.endMs ? Math.round((afterStart + heard.endMs) / 2) : heard.endMs;
|
|
84
193
|
return { range, startMs, endMs };
|
|
85
194
|
});
|
|
86
195
|
}
|
|
@@ -96,16 +205,71 @@ export type SpeechRangeCacheResolver = (range: SpeechRange, clipKey: string) =>
|
|
|
96
205
|
|
|
97
206
|
const seconds = (ms: number): string => (ms / 1000).toFixed(3);
|
|
98
207
|
|
|
99
|
-
/**
|
|
208
|
+
/**
|
|
209
|
+
* One range of a clip's sound (`audio`, read from its start) for the isolation provider: 24-bit PCM for a `.wav`
|
|
210
|
+
* output (lossless, issue #1139), else as `getAudioCodecArgs` encodes it.
|
|
211
|
+
*/
|
|
100
212
|
export function buildRangeAudioArgs(audio: string, range: Pick<SpeechRange, "startMs" | "endMs">, output: string): string[] {
|
|
101
|
-
|
|
213
|
+
const codec = extname(output).toLowerCase() === ".wav" ? ["-c:a", "pcm_s24le"] : getAudioCodecArgs(output);
|
|
214
|
+
return ["-v", "error", "-y", "-ss", seconds(range.startMs), "-t", seconds(range.endMs - range.startMs), "-i", audio, "-vn", ...codec, output];
|
|
102
215
|
}
|
|
103
216
|
|
|
104
|
-
/**
|
|
217
|
+
/**
|
|
218
|
+
* A cleaned range laid back: its file, whether it is mono (panned to both channels at full level), and the dB it is
|
|
219
|
+
* raised by before its fades (`gainDb`, issue #1159: a range cut from the clip's un-gained sound is raised by the
|
|
220
|
+
* clip's pre-gain; absent or 0 leaves it as it is).
|
|
221
|
+
*/
|
|
105
222
|
export interface LaidRange {
|
|
106
223
|
window: SpeechRangeWindow;
|
|
107
224
|
file: string;
|
|
108
225
|
isMono: boolean;
|
|
226
|
+
gainDb?: number;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* Where the gain of the sound a range's cleaned voice was cut from is recorded (issue #1159): next to its cached file,
|
|
231
|
+
* as `<path>.gain.json` holding `{ "inputGainDb": <dB> }`. Commit it with the cached file.
|
|
232
|
+
*/
|
|
233
|
+
export function getSpeechRangeGainPath(path: string): string {
|
|
234
|
+
return `${path}.gain.json`;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/** The record of `getSpeechRangeGainPath`'s file. */
|
|
238
|
+
export function formatSpeechRangeGain(inputGainDb: number): string {
|
|
239
|
+
return `${JSON.stringify({ inputGainDb })}\n`;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/** The input gain in a record's text; null when the text is not such a record. */
|
|
243
|
+
export function readSpeechRangeGain(text: string): number | null {
|
|
244
|
+
let json: unknown;
|
|
245
|
+
try {
|
|
246
|
+
json = JSON.parse(text);
|
|
247
|
+
} catch {
|
|
248
|
+
return null;
|
|
249
|
+
}
|
|
250
|
+
if (typeof json !== "object" || json === null) return null;
|
|
251
|
+
const value = (json as Record<string, unknown>).inputGainDb;
|
|
252
|
+
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/** A measuring pass (`loudnorm`, nothing written) over `file`, or only over `range` of it, as 48 kHz stereo. */
|
|
256
|
+
export function buildRangeMeasureArgs(file: string, range?: Pick<SpeechRange, "startMs" | "endMs">): string[] {
|
|
257
|
+
const window = range === undefined ? [] : ["-ss", seconds(range.startMs), "-t", seconds(range.endMs - range.startMs)];
|
|
258
|
+
const format = `aformat=sample_rates=${String(FOOTAGE_SAMPLE_RATE)}:channel_layouts=stereo`;
|
|
259
|
+
return ["-hide_banner", "-nostats", ...window, "-i", file, "-map", "0:a:0", "-vn", "-af", `${format},loudnorm=I=-23:TP=-2:LRA=11:print_format=json`, "-f", "null", "-"];
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/**
|
|
263
|
+
* The gain of the sound a cached cleaned range was cut from, told from its loudness (issue #1159): a range cut from
|
|
264
|
+
* the clip's un-gained sound sits about `clipGainDb` under the same range of the pre-gained sound, one cut from the
|
|
265
|
+
* pre-gained sound about level with it (the cleaning takes off a few dB at most). The nearer case wins: 0 (un-gained)
|
|
266
|
+
* or `clipGainDb`. Without a loudness on either side (null: silence) it is `clipGainDb`, so nothing is changed.
|
|
267
|
+
*/
|
|
268
|
+
export function getRangeInputGainDb(options: { clipGainDb: number; cleanedLufs: number | null; baseLufs: number | null }): number {
|
|
269
|
+
const { clipGainDb, cleanedLufs, baseLufs } = options;
|
|
270
|
+
if (clipGainDb === 0 || cleanedLufs === null || baseLufs === null) return clipGainDb;
|
|
271
|
+
const difference = cleanedLufs - baseLufs;
|
|
272
|
+
return Math.abs(difference + clipGainDb) < Math.abs(difference) ? 0 : clipGainDb;
|
|
109
273
|
}
|
|
110
274
|
|
|
111
275
|
/** Milliseconds of the linear crossfade at every join between the clip's own sound and a cleaned window. */
|
|
@@ -161,28 +325,29 @@ function buildGainExpression(fade: SpeechRangeFade): string {
|
|
|
161
325
|
*/
|
|
162
326
|
export function buildRangeLayArgs(base: string, laid: readonly LaidRange[], output: string, options: { fadeMs?: number } = {}): string[] {
|
|
163
327
|
const format = `aformat=sample_rates=${FOOTAGE_SAMPLE_RATE}:channel_layouts=stereo`;
|
|
164
|
-
if (laid.length === 0) return ["-v", "error", "-y", "-i", base, "-vn", "-af", format,
|
|
328
|
+
if (laid.length === 0) return ["-v", "error", "-y", "-i", base, "-vn", "-af", format, ...getAudioCodecArgs(output), output];
|
|
165
329
|
const windows = laid.map(({ window }) => window);
|
|
166
330
|
const fades = getSpeechRangeFades(windows, options.fadeMs ?? DEFAULT_SPEECH_RANGE_FADE_MS);
|
|
167
331
|
const byWindow = new Map([...windows].sort((a, b) => a.startMs - b.startMs).map((window, index) => [window, fades[index]]));
|
|
168
332
|
const inputs = ["-i", base];
|
|
169
333
|
const gains: string[] = [];
|
|
170
334
|
const parts: string[] = [];
|
|
171
|
-
laid.forEach(({ window, file, isMono }, index) => {
|
|
335
|
+
laid.forEach(({ window, file, isMono, gainDb }, index) => {
|
|
172
336
|
const fade = byWindow.get(window);
|
|
173
337
|
if (fade === undefined) return;
|
|
174
338
|
inputs.push("-i", file);
|
|
175
339
|
gains.push(buildGainExpression(fade));
|
|
176
340
|
const channels = isMono ? "pan=stereo|c0=c0|c1=c0," : "";
|
|
341
|
+
const volume = gainDb === undefined || gainDb === 0 ? "" : `,volume=${String(gainDb)}dB`;
|
|
177
342
|
const fadeIn = fade.inEndMs > fade.inStartMs ? `,afade=t=in:st=0:d=${seconds(fade.inEndMs - fade.inStartMs)}` : "";
|
|
178
343
|
const fadeOut = fade.outEndMs > fade.outStartMs ? `,afade=t=out:st=${seconds(fade.outStartMs - fade.inStartMs)}:d=${seconds(fade.outEndMs - fade.outStartMs)}` : "";
|
|
179
344
|
const delay = Math.round((fade.inStartMs * FOOTAGE_SAMPLE_RATE) / 1000);
|
|
180
345
|
parts.push(
|
|
181
|
-
`[${index + 1}:a]atrim=start=${seconds(fade.inStartMs - window.range.startMs)}:end=${seconds(fade.outEndMs - window.range.startMs)},asetpts=PTS-STARTPTS,${channels}${format}${fadeIn}${fadeOut},adelay=${String(delay)}S:all=1[w${String(index)}]`,
|
|
346
|
+
`[${index + 1}:a]atrim=start=${seconds(fade.inStartMs - window.range.startMs)}:end=${seconds(fade.outEndMs - window.range.startMs)},asetpts=PTS-STARTPTS,${channels}${format}${volume}${fadeIn}${fadeOut},adelay=${String(delay)}S:all=1[w${String(index)}]`,
|
|
182
347
|
);
|
|
183
348
|
});
|
|
184
349
|
parts.unshift(`[0:a]${format},aeval='val(ch)*(1-(${gains.join("+")}))':c=same[base]`);
|
|
185
350
|
const mixed = laid.map((_, index) => `[w${String(index)}]`).join("");
|
|
186
351
|
parts.push(`[base]${mixed}amix=inputs=${String(laid.length + 1)}:duration=first:normalize=0[speech]`);
|
|
187
|
-
return ["-v", "error", "-y", ...inputs, "-filter_complex", parts.join(";"), "-map", "[speech]",
|
|
352
|
+
return ["-v", "error", "-y", ...inputs, "-filter_complex", parts.join(";"), "-map", "[speech]", ...getAudioCodecArgs(output), output];
|
|
188
353
|
}
|