@softure-ai/marketing-kit 0.15.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +91 -0
- package/README.md +138 -16
- package/dist/config/schema.d.ts +2 -1
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +7 -0
- package/dist/config/schema.js.map +1 -1
- package/dist/footage/clip-sound.d.ts +58 -0
- package/dist/footage/clip-sound.d.ts.map +1 -0
- package/dist/footage/clip-sound.js +103 -0
- package/dist/footage/clip-sound.js.map +1 -0
- package/dist/footage/declick.d.ts +1 -1
- package/dist/footage/declick.d.ts.map +1 -1
- package/dist/footage/declick.js +3 -4
- package/dist/footage/declick.js.map +1 -1
- package/dist/footage/draft.d.ts +19 -5
- package/dist/footage/draft.d.ts.map +1 -1
- package/dist/footage/draft.js +92 -22
- package/dist/footage/draft.js.map +1 -1
- package/dist/footage/elevenlabs.d.ts +5 -0
- package/dist/footage/elevenlabs.d.ts.map +1 -1
- package/dist/footage/elevenlabs.js +10 -3
- package/dist/footage/elevenlabs.js.map +1 -1
- package/dist/footage/failure.d.ts +12 -1
- package/dist/footage/failure.d.ts.map +1 -1
- package/dist/footage/failure.js +10 -1
- package/dist/footage/failure.js.map +1 -1
- package/dist/footage/fake.d.ts +10 -1
- package/dist/footage/fake.d.ts.map +1 -1
- package/dist/footage/fake.js +19 -4
- package/dist/footage/fake.js.map +1 -1
- package/dist/footage/hyperframes-capture.d.ts +30 -0
- package/dist/footage/hyperframes-capture.d.ts.map +1 -0
- package/dist/footage/hyperframes-capture.js +80 -0
- package/dist/footage/hyperframes-capture.js.map +1 -0
- package/dist/footage/isolate-ranges.d.ts +34 -6
- package/dist/footage/isolate-ranges.d.ts.map +1 -1
- package/dist/footage/isolate-ranges.js +90 -23
- package/dist/footage/isolate-ranges.js.map +1 -1
- package/dist/footage/loudness.d.ts +16 -2
- package/dist/footage/loudness.d.ts.map +1 -1
- package/dist/footage/loudness.js +10 -7
- package/dist/footage/loudness.js.map +1 -1
- package/dist/footage/paid.d.ts +12 -2
- package/dist/footage/paid.d.ts.map +1 -1
- package/dist/footage/paid.js +8 -3
- package/dist/footage/paid.js.map +1 -1
- package/dist/footage/pipeline.d.ts +14 -2
- package/dist/footage/pipeline.d.ts.map +1 -1
- package/dist/footage/pipeline.js +55 -46
- package/dist/footage/pipeline.js.map +1 -1
- package/dist/footage/providers.d.ts +5 -0
- package/dist/footage/providers.d.ts.map +1 -1
- package/dist/footage/providers.js.map +1 -1
- package/dist/footage/render.d.ts +1 -1
- package/dist/footage/render.d.ts.map +1 -1
- package/dist/footage/render.js +4 -3
- package/dist/footage/render.js.map +1 -1
- package/dist/footage/speech-ranges.d.ts +79 -11
- package/dist/footage/speech-ranges.d.ts.map +1 -1
- package/dist/footage/speech-ranges.js +137 -17
- package/dist/footage/speech-ranges.js.map +1 -1
- package/dist/index.d.ts +9 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -5
- package/dist/index.js.map +1 -1
- package/dist/og/index.d.ts +2 -0
- package/dist/og/index.d.ts.map +1 -1
- package/dist/og/index.js +2 -0
- package/dist/og/index.js.map +1 -1
- package/dist/og/photo-cover.d.ts +74 -0
- package/dist/og/photo-cover.d.ts.map +1 -0
- package/dist/og/photo-cover.js +357 -0
- package/dist/og/photo-cover.js.map +1 -0
- package/dist/og/photo.d.ts +51 -0
- package/dist/og/photo.d.ts.map +1 -0
- package/dist/og/photo.js +154 -0
- package/dist/og/photo.js.map +1 -0
- package/dist/voice/provider-failure.d.ts +41 -0
- package/dist/voice/provider-failure.d.ts.map +1 -0
- package/dist/voice/provider-failure.js +68 -0
- package/dist/voice/provider-failure.js.map +1 -0
- package/dist/voice/provider.d.ts +7 -1
- package/dist/voice/provider.d.ts.map +1 -1
- package/dist/voice/provider.js +1 -1
- package/dist/voice/provider.js.map +1 -1
- package/package.json +2 -1
- package/schema/marketing.schema.json +6 -0
- package/src/config/schema.ts +7 -0
- package/src/footage/clip-sound.ts +124 -0
- package/src/footage/declick.ts +3 -4
- package/src/footage/draft.ts +123 -22
- package/src/footage/elevenlabs.ts +11 -3
- package/src/footage/failure.ts +14 -1
- package/src/footage/fake.ts +28 -4
- package/src/footage/hyperframes-capture.ts +100 -0
- package/src/footage/isolate-ranges.ts +124 -29
- package/src/footage/loudness.ts +17 -8
- package/src/footage/paid.ts +25 -3
- package/src/footage/pipeline.ts +68 -47
- package/src/footage/providers.ts +5 -0
- package/src/footage/render.ts +4 -3
- package/src/footage/speech-ranges.ts +184 -19
- package/src/index.ts +49 -1
- package/src/og/index.ts +28 -0
- package/src/og/photo-cover.ts +442 -0
- package/src/og/photo.ts +177 -0
- package/src/voice/provider-failure.ts +98 -0
- package/src/voice/provider.ts +7 -2
package/src/footage/draft.ts
CHANGED
|
@@ -12,18 +12,35 @@ import { chooseFootageDraftFont } from "./draft-font.js";
|
|
|
12
12
|
import { getEdlDuration, getSegmentStarts, remapWords, type Edl } from "./edl.js";
|
|
13
13
|
import { declickSpeech, describeDeclick, type DeclickReport } from "./declick.js";
|
|
14
14
|
import { getDeclickLevel, getFootageEdit, type FootageFilm } from "./film.js";
|
|
15
|
+
import { buildFilmMixArgs } from "./film-mix.js";
|
|
16
|
+
import { DEFAULT_LOUDNESS, formatDb, getLoudnessVerdict, measureLoudness, normalizeLoudness, type MixLoudnessTarget } from "./loudness.js";
|
|
15
17
|
import { duckingKeyframes, getDuckingOptions } from "./music.js";
|
|
16
18
|
import { FOOTAGE_FPS, getThreadArgs, type Frame } from "./normalize.js";
|
|
17
|
-
import {
|
|
19
|
+
import {
|
|
20
|
+
findFootageMusic,
|
|
21
|
+
getFfmpegReporting,
|
|
22
|
+
getFrame,
|
|
23
|
+
getSpeechChainFiles,
|
|
24
|
+
mixFootage,
|
|
25
|
+
type FinishedLoudness,
|
|
26
|
+
type FinishedMix,
|
|
27
|
+
type FootageBrand,
|
|
28
|
+
type FootageResult,
|
|
29
|
+
type FootageRunOptions,
|
|
30
|
+
type PreparedClip,
|
|
31
|
+
type PreparedMix,
|
|
32
|
+
type TranscribeOutcome,
|
|
33
|
+
} from "./pipeline.js";
|
|
18
34
|
import { runFfmpeg, withFfmpegFailureReporting } from "./process.js";
|
|
19
35
|
import { buildMusicArgs, buildSpeechArgs, getSpeechCutOptions } from "./render.js";
|
|
20
36
|
|
|
21
37
|
/**
|
|
22
38
|
* A fast draft of a footage film, ffmpeg only (no hyperframes, no browser): the cut at a small width with the film's
|
|
23
39
|
* captions burned in (its caption style, in the brand's body font and caption colours), its zoom cuts (eased as in the
|
|
24
|
-
* render), cross-fades and Ken Burns b-roll drawn as the full render draws them, the
|
|
25
|
-
*
|
|
26
|
-
* render.
|
|
40
|
+
* render), cross-fades and Ken Burns b-roll drawn as the full render draws them, and the render's sound chain: the
|
|
41
|
+
* speech track's stages (cut, declick, loudness pass), the ducked music when it costs nothing, and the mix brought to
|
|
42
|
+
* `mixLoudness` as the render brings it (issue #1143). No end card, cover or sound effects. What an app shows the owner
|
|
43
|
+
* before the paid render.
|
|
27
44
|
*/
|
|
28
45
|
|
|
29
46
|
/** The draft's default width: 540 makes a 540×960 draft of a 9:16 film. */
|
|
@@ -241,6 +258,11 @@ export interface FootageDraftArgsInput {
|
|
|
241
258
|
speech: string;
|
|
242
259
|
/** The ducked music track (`buildMusicArgs`, as long as the cut), mixed under the speech; null or absent for none. */
|
|
243
260
|
music?: string | null;
|
|
261
|
+
/**
|
|
262
|
+
* The film's mix, already brought to its target and encoded to AAC (`mixFootage`): the draft copies it as its sound
|
|
263
|
+
* and reads neither `speech` nor `music`. Null or absent: the draft mixes `speech` and `music` and encodes them.
|
|
264
|
+
*/
|
|
265
|
+
mix?: string | null;
|
|
244
266
|
/** The film's full frame (the normalized clips' size). */
|
|
245
267
|
frame: Frame;
|
|
246
268
|
/** The draft's width; default 540. */
|
|
@@ -330,7 +352,8 @@ function buildPictureFilters(plan: DraftSegmentPlan, draft: Frame): string[] {
|
|
|
330
352
|
/**
|
|
331
353
|
* The ffmpeg arguments of the draft: each segment read from its own seek into its clip, zoomed and scaled to the
|
|
332
354
|
* draft's width, joined by `xfade` at `fade` cuts and `concat` at the others, the captions burned in (`ass`), the
|
|
333
|
-
*
|
|
355
|
+
* film's mix copied under it (or the speech and music tracks mixed and encoded to AAC); H.264 yuv420p with fast start,
|
|
356
|
+
* exactly the cut's length.
|
|
334
357
|
*/
|
|
335
358
|
export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
|
|
336
359
|
const { edl } = input;
|
|
@@ -378,10 +401,13 @@ export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
|
|
|
378
401
|
} else {
|
|
379
402
|
parts.push(`[${joined}]null[draft]`);
|
|
380
403
|
}
|
|
381
|
-
const
|
|
382
|
-
const
|
|
383
|
-
|
|
404
|
+
const soundInput = edl.segments.length;
|
|
405
|
+
const mix = input.mix ?? null;
|
|
406
|
+
const music = mix === null ? (input.music ?? null) : null;
|
|
407
|
+
if (music !== null) parts.push(`[${soundInput}:a][${soundInput + 1}:a]amix=inputs=2:duration=first:normalize=0[sound]`);
|
|
384
408
|
const seconds = getEdlDuration(edl);
|
|
409
|
+
// A prepared mix is copied: it was measured as encoded, and a second encode would move its loudness and true peak.
|
|
410
|
+
const audioCodec = mix === null ? ["-c:a", "aac", "-b:a", DRAFT_AUDIO_BITRATE] : ["-c:a", "copy"];
|
|
385
411
|
return [
|
|
386
412
|
"-v",
|
|
387
413
|
"error",
|
|
@@ -389,14 +415,14 @@ export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
|
|
|
389
415
|
...threads.global,
|
|
390
416
|
...inputs,
|
|
391
417
|
"-i",
|
|
392
|
-
input.speech,
|
|
418
|
+
mix ?? input.speech,
|
|
393
419
|
...(music === null ? [] : ["-i", music]),
|
|
394
420
|
"-filter_complex",
|
|
395
421
|
parts.join(";"),
|
|
396
422
|
"-map",
|
|
397
423
|
"[draft]",
|
|
398
424
|
"-map",
|
|
399
|
-
music === null ? `${
|
|
425
|
+
music === null ? `${soundInput}:a` : "[sound]",
|
|
400
426
|
"-t",
|
|
401
427
|
String(seconds),
|
|
402
428
|
"-c:v",
|
|
@@ -407,10 +433,7 @@ export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
|
|
|
407
433
|
String(DRAFT_CRF),
|
|
408
434
|
"-pix_fmt",
|
|
409
435
|
"yuv420p",
|
|
410
|
-
|
|
411
|
-
"aac",
|
|
412
|
-
"-b:a",
|
|
413
|
-
DRAFT_AUDIO_BITRATE,
|
|
436
|
+
...audioCodec,
|
|
414
437
|
"-movflags",
|
|
415
438
|
"+faststart",
|
|
416
439
|
...threads.output,
|
|
@@ -453,6 +476,13 @@ export interface FootageDraft {
|
|
|
453
476
|
bytes: number;
|
|
454
477
|
/** What the speech track's declick stage did, as the render does it; null when the film's declick is off. */
|
|
455
478
|
declick: DeclickReport | null;
|
|
479
|
+
/**
|
|
480
|
+
* The draft's sound as measured, as the render measures its finished film (`warning` on a near miss inside
|
|
481
|
+
* `mixLoudness.warnTolerance`); null for a silent draft.
|
|
482
|
+
*/
|
|
483
|
+
loudness: FinishedLoudness | null;
|
|
484
|
+
/** What the mix stage did, as in the render; null when it did not run (`mixLoudness: null`, a silent mix). */
|
|
485
|
+
mix: FinishedMix | null;
|
|
456
486
|
}
|
|
457
487
|
|
|
458
488
|
/** The draft's music track: a file to duck, or why there is none. */
|
|
@@ -505,15 +535,24 @@ async function runRenderFootageDraft(
|
|
|
505
535
|
rmSync(draftDir, { recursive: true, force: true });
|
|
506
536
|
mkdirSync(draftDir, { recursive: true });
|
|
507
537
|
const declickLevel = getDeclickLevel(film);
|
|
508
|
-
const
|
|
538
|
+
const loudness = film.loudness === undefined ? DEFAULT_LOUDNESS : film.loudness;
|
|
539
|
+
const mixLoudness = film.mixLoudness === undefined ? DEFAULT_LOUDNESS : film.mixLoudness;
|
|
540
|
+
const hasMusic = music.value.kind === "file";
|
|
541
|
+
const items = [
|
|
542
|
+
"speech",
|
|
543
|
+
...(declickLevel === "off" ? [] : ["declick"]),
|
|
544
|
+
...(loudness === null ? [] : ["loudness"]),
|
|
545
|
+
...(hasMusic ? ["music"] : []),
|
|
546
|
+
...(mixLoudness === null ? [] : ["mix"]),
|
|
547
|
+
"film",
|
|
548
|
+
];
|
|
509
549
|
const report = (item: string, done: number): void => {
|
|
510
550
|
const itemIndex = items.indexOf(item) + 1;
|
|
511
551
|
options.onProgress?.({ step: "draft", item, itemIndex, itemCount: items.length, done: Math.min(seconds * items.length, (itemIndex - 1) * seconds + done), total: seconds * items.length });
|
|
512
552
|
};
|
|
513
553
|
const clipFiles = Object.fromEntries(clips.map((clip) => [clip.clip.id, resolve(clip.file)]));
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
const speech = declickLevel === "off" ? cutSpeech : join(draftDir, "speech.declicked.wav");
|
|
554
|
+
// The render's speech chain and its file names, so the draft's sound is made as the render's is.
|
|
555
|
+
const { rawSpeechFile, declickedFile, speechFile } = getSpeechChainFiles(draftDir, { isDeclick: declickLevel !== "off", isLoudness: loudness !== null });
|
|
517
556
|
const words = remapWords(edl, transcript.words, film.cut.fillers);
|
|
518
557
|
const frame = getFrame(film);
|
|
519
558
|
const brandFont = chooseFootageDraftFont(options.brand?.fonts.body ?? null);
|
|
@@ -538,13 +577,22 @@ async function runRenderFootageDraft(
|
|
|
538
577
|
const output = resolve(options.output);
|
|
539
578
|
mkdirSync(dirname(output), { recursive: true });
|
|
540
579
|
report("speech", 0);
|
|
541
|
-
await runFfmpeg("The draft's speech track", buildSpeechArgs(edl, transcript.speech,
|
|
580
|
+
await runFfmpeg("The draft's speech track", buildSpeechArgs(edl, transcript.speech, rawSpeechFile, getSpeechCutOptions(getFootageEdit(film))), { ...aborting, output: rawSpeechFile, onSeconds: (done) => report("speech", done) });
|
|
542
581
|
let declick: DeclickReport | null = null;
|
|
543
582
|
if (declickLevel !== "off") {
|
|
544
583
|
report("declick", 0);
|
|
545
|
-
declick = await declickSpeech({ input:
|
|
584
|
+
declick = await declickSpeech({ input: rawSpeechFile, output: declickedFile, words, level: declickLevel, seconds, ...aborting, onSeconds: (done) => report("declick", done) });
|
|
546
585
|
log(describeDeclick(declick));
|
|
547
586
|
}
|
|
587
|
+
if (loudness !== null) {
|
|
588
|
+
report("loudness", 0);
|
|
589
|
+
const outcome = await normalizeLoudness(declickedFile, speechFile, loudness, { ...aborting, onSeconds: (done) => report("loudness", done) });
|
|
590
|
+
log(
|
|
591
|
+
outcome.kind === "silent"
|
|
592
|
+
? "loudness: the speech track is silent, left as it is."
|
|
593
|
+
: `loudness: the speech track measured ${outcome.measured.toFixed(1)} LUFS, normalized to ${loudness.target} LUFS (true peak ${loudness.truePeak} dBTP).`,
|
|
594
|
+
);
|
|
595
|
+
}
|
|
548
596
|
let musicFile: string | null = null;
|
|
549
597
|
if (music.value.kind === "file") {
|
|
550
598
|
musicFile = join(draftDir, "music.mp3");
|
|
@@ -555,14 +603,27 @@ async function runRenderFootageDraft(
|
|
|
555
603
|
} else if (music.value.kind === "not_cached") {
|
|
556
604
|
log(`draft: no music, ${music.value.reason}.`);
|
|
557
605
|
}
|
|
606
|
+
let prepared: PreparedMix | null = null;
|
|
607
|
+
if (mixLoudness !== null) {
|
|
608
|
+
report("mix", 0);
|
|
609
|
+
// The render's mix stage over the draft's tracks (the draft has no sound effects): the same target, ceiling and attempts.
|
|
610
|
+
const mixSource = join(draftDir, "mix.wav");
|
|
611
|
+
const args = buildFilmMixArgs({ speech: { file: speechFile, seconds }, music: musicFile === null ? null : { file: musicFile, seconds }, sfx: [], seconds, output: mixSource });
|
|
612
|
+
await runFfmpeg("The draft's mix", args, { ...aborting, output: mixSource, onSeconds: (done) => report("mix", done) });
|
|
613
|
+
const mixed = await mixFootage(mixLoudness, { mixSource, output: join(draftDir, "mix.m4a") }, { log, ...aborting });
|
|
614
|
+
rmSync(mixSource, { force: true });
|
|
615
|
+
if (!mixed.ok) return mixed;
|
|
616
|
+
prepared = mixed.value;
|
|
617
|
+
}
|
|
558
618
|
report("film", 0);
|
|
559
619
|
// Run in the draft folder: the captions file and the font folder are relative names, so no path needs escaping in the filter graph.
|
|
560
620
|
const args = buildFootageDraftArgs({
|
|
561
621
|
film,
|
|
562
622
|
edl,
|
|
563
623
|
clips: clipFiles,
|
|
564
|
-
speech,
|
|
624
|
+
speech: speechFile,
|
|
565
625
|
music: musicFile,
|
|
626
|
+
mix: prepared?.audioFile ?? null,
|
|
566
627
|
frame,
|
|
567
628
|
...(options.width === undefined ? {} : { width: options.width }),
|
|
568
629
|
...(options.threads === undefined ? {} : { threads: options.threads }),
|
|
@@ -571,11 +632,51 @@ async function runRenderFootageDraft(
|
|
|
571
632
|
output,
|
|
572
633
|
});
|
|
573
634
|
await runFfmpeg("The footage draft", args, { ...aborting, output, cwd: draftDir, onSeconds: (done) => report("film", done) });
|
|
635
|
+
const measured = await measureDraftLoudness(output, mixLoudness, prepared !== null, aborting);
|
|
636
|
+
if (!measured.ok) return measured;
|
|
637
|
+
const draftLoudness = measured.value;
|
|
638
|
+
if (draftLoudness !== null) {
|
|
639
|
+
log(`loudness: the draft measures ${formatDb(draftLoudness.integrated)} LUFS, true peak ${formatDb(draftLoudness.truePeak)} dBTP.`);
|
|
640
|
+
if (draftLoudness.warning !== null) log(`loudness warning: ${draftLoudness.warning}`);
|
|
641
|
+
}
|
|
574
642
|
report("film", seconds);
|
|
575
643
|
const draft = getFootageDraftFrame(frame, options.width);
|
|
576
644
|
const bytes = statSync(output).size;
|
|
577
645
|
const draftMusic: FootageDraftMusic = music.value.kind === "file" ? "played" : music.value.kind;
|
|
578
646
|
const notes = [captions === null ? "no captions" : `captions in ${font.family}`, draftMusic === "played" ? "music" : draftMusic === "none" ? "no music" : "music not cached"];
|
|
579
647
|
log(`draft: ${output}, ${draft.width}×${draft.height}, ${seconds.toFixed(1)} s, ${bytes} bytes, ${notes.join(", ")}.`);
|
|
580
|
-
return ok({
|
|
648
|
+
return ok({
|
|
649
|
+
output,
|
|
650
|
+
music: draftMusic,
|
|
651
|
+
captionFont: captions === null ? null : font.family,
|
|
652
|
+
seconds,
|
|
653
|
+
width: draft.width,
|
|
654
|
+
height: draft.height,
|
|
655
|
+
bytes,
|
|
656
|
+
declick,
|
|
657
|
+
loudness: draftLoudness,
|
|
658
|
+
mix: prepared?.mix ?? null,
|
|
659
|
+
});
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
/**
|
|
663
|
+
* The written draft's sound measured as `finishFootage` measures the finished film: held to `mixLoudness` when the mix
|
|
664
|
+
* stage made it (a miss fails, a near miss inside `warnTolerance` carries `warning`), reported otherwise; null when
|
|
665
|
+
* the draft is silent.
|
|
666
|
+
*/
|
|
667
|
+
async function measureDraftLoudness(
|
|
668
|
+
output: string,
|
|
669
|
+
mixLoudness: MixLoudnessTarget | null,
|
|
670
|
+
isMixed: boolean,
|
|
671
|
+
aborting: { signal?: AbortSignal },
|
|
672
|
+
): Promise<FootageResult<FinishedLoudness | null>> {
|
|
673
|
+
const measured = await measureLoudness(output, mixLoudness ?? DEFAULT_LOUDNESS, { label: "the footage draft", ...aborting });
|
|
674
|
+
if (measured.kind === "silent") return ok(null);
|
|
675
|
+
const sound = { integrated: measured.measurement.inputI, truePeak: measured.measurement.inputTp };
|
|
676
|
+
const verdict = mixLoudness === null || !isMixed ? { kind: "pass" as const } : getLoudnessVerdict(sound, mixLoudness);
|
|
677
|
+
if (verdict.kind === "fail") {
|
|
678
|
+
return err(`the footage draft ${output} measures ${formatDb(sound.integrated)} LUFS integrated, ${formatDb(sound.truePeak)} dBTP true peak: ${verdict.problems.join("; ")}.`, { kind: "loudness", loudness: sound });
|
|
679
|
+
}
|
|
680
|
+
return ok({ ...sound, warning: verdict.kind === "warn" ? `the footage draft ${output}: ${verdict.warning}` : null });
|
|
581
681
|
}
|
|
682
|
+
|
|
@@ -5,6 +5,7 @@ import { z } from "zod";
|
|
|
5
5
|
|
|
6
6
|
import { ELEVENLABS_API_KEY_ENV } from "../voice/elevenlabs.js";
|
|
7
7
|
import { err, ok, type TtsResult } from "../voice/provider.js";
|
|
8
|
+
import { getConnectionFailure, getLocalRefusal, getProviderFailure } from "../voice/provider-failure.js";
|
|
8
9
|
import { ELEVENLABS_API_URL, type TimedWord } from "../voice/voiceover.js";
|
|
9
10
|
import type { IsolationProvider, MusicProvider, PaidAudio, SttProvider, SttTranscript } from "./providers.js";
|
|
10
11
|
|
|
@@ -57,17 +58,17 @@ function fileBlob(path: string): Blob {
|
|
|
57
58
|
}
|
|
58
59
|
|
|
59
60
|
async function send(options: ElevenLabsFootageOptions, label: string, path: string, init: RequestInit): Promise<TtsResult<Response>> {
|
|
60
|
-
if (options.apiKey === undefined || options.apiKey.length === 0) return err(`no ${ELEVENLABS_API_KEY_ENV} in the environment
|
|
61
|
+
if (options.apiKey === undefined || options.apiKey.length === 0) return err(`no ${ELEVENLABS_API_KEY_ENV} in the environment.`, getLocalRefusal("missing_api_key"));
|
|
61
62
|
const request = options.fetch ?? globalThis.fetch;
|
|
62
63
|
let response: Response;
|
|
63
64
|
try {
|
|
64
65
|
response = await request(`${ELEVENLABS_API_URL}${path}`, { ...init, headers: { ...init.headers, "xi-api-key": options.apiKey } });
|
|
65
66
|
} catch (error) {
|
|
66
|
-
return err(`ElevenLabs ${label}: the connection failed (${String(error)})
|
|
67
|
+
return err(`ElevenLabs ${label}: the connection failed (${String(error)}).`, getConnectionFailure(error));
|
|
67
68
|
}
|
|
68
69
|
if (!response.ok) {
|
|
69
70
|
const detail = await response.text().catch((error: unknown) => `(unreadable body: ${String(error)})`);
|
|
70
|
-
return err(`ElevenLabs ${label} answered ${response.status}: ${detail.slice(0, ERROR_DETAIL_LENGTH)}
|
|
71
|
+
return err(`ElevenLabs ${label} answered ${response.status}: ${detail.slice(0, ERROR_DETAIL_LENGTH)}`, getProviderFailure(response, detail));
|
|
71
72
|
}
|
|
72
73
|
return ok(response);
|
|
73
74
|
}
|
|
@@ -118,9 +119,16 @@ export function createElevenLabsSttProvider(options: ElevenLabsFootageOptions):
|
|
|
118
119
|
};
|
|
119
120
|
}
|
|
120
121
|
|
|
122
|
+
/**
|
|
123
|
+
* The shortest input the ElevenLabs Voice Isolator accepts, in seconds: production renders saw every range of 4.18 s
|
|
124
|
+
* or less refused on every attempt (`audio_too_short`) and every range of 4.64 s or more accepted (issue #1140).
|
|
125
|
+
*/
|
|
126
|
+
export const ELEVENLABS_ISOLATION_MIN_SECONDS = 4.6;
|
|
127
|
+
|
|
121
128
|
export function createElevenLabsIsolationProvider(options: ElevenLabsFootageOptions): IsolationProvider {
|
|
122
129
|
return {
|
|
123
130
|
id: "elevenlabs",
|
|
131
|
+
minSeconds: ELEVENLABS_ISOLATION_MIN_SECONDS,
|
|
124
132
|
async isolate(input) {
|
|
125
133
|
const form = new FormData();
|
|
126
134
|
form.append("audio", fileBlob(input.audio), basename(input.audio));
|
package/src/footage/failure.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { ProviderFailure } from "../voice/provider-failure.js";
|
|
1
2
|
import type { MixLoudness } from "./loudness.js";
|
|
2
3
|
|
|
3
4
|
/**
|
|
@@ -6,6 +7,8 @@ import type { MixLoudness } from "./loudness.js";
|
|
|
6
7
|
* measures silent after its gain and limiter). The same inputs fail the same way: no retry.
|
|
7
8
|
* - `publishable`: the finished file fails the container, codec profile or fast start checks (the problems).
|
|
8
9
|
* - `input`: the film, its EDL, fonts, files or settings cannot be used as given. A retry fails the same way.
|
|
10
|
+
* - `provider`: a paid provider (speech to text, voice isolation, music) failed with the structured details it gave
|
|
11
|
+
* (issue #1141); a retry may succeed only when `provider.isRetryable`.
|
|
9
12
|
* - `process`: anything else (a process that exited, the network, the disk). A retry may succeed.
|
|
10
13
|
* A failed ffmpeg step still throws and an abort still rejects with the signal's reason; neither is a result.
|
|
11
14
|
*/
|
|
@@ -13,6 +16,7 @@ export type FootageFailure =
|
|
|
13
16
|
| { kind: "loudness"; loudness: MixLoudness | null }
|
|
14
17
|
| { kind: "publishable"; problems: string[] }
|
|
15
18
|
| { kind: "input" }
|
|
19
|
+
| { kind: "provider"; provider: ProviderFailure }
|
|
16
20
|
| { kind: "process" };
|
|
17
21
|
|
|
18
22
|
export type FootageFailureKind = FootageFailure["kind"];
|
|
@@ -38,7 +42,16 @@ export function err(error: string, failure: FootageFailure = PROCESS_FAILURE): F
|
|
|
38
42
|
return { ok: false, error, failure };
|
|
39
43
|
}
|
|
40
44
|
|
|
41
|
-
/**
|
|
45
|
+
/** A provider's failure as a footage failure: `provider` with its details, or `process` when it gave none. */
|
|
46
|
+
export function getProviderFootageFailure(providerFailure: ProviderFailure | undefined): FootageFailure {
|
|
47
|
+
return providerFailure === undefined ? PROCESS_FAILURE : { kind: "provider", provider: providerFailure };
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* True when running the same step again may succeed: `process`, or `provider` with `isRetryable`; a gate miss, a bad
|
|
52
|
+
* input or a provider's refusal fails again.
|
|
53
|
+
*/
|
|
42
54
|
export function isRetryableFootageFailure(failure: FootageFailure): boolean {
|
|
55
|
+
if (failure.kind === "provider") return failure.provider.isRetryable;
|
|
43
56
|
return failure.kind === "process";
|
|
44
57
|
}
|
package/src/footage/fake.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { readFileSync } from "node:fs";
|
|
2
2
|
import { basename } from "node:path";
|
|
3
3
|
|
|
4
|
-
import { ok, type TtsResult } from "../voice/provider.js";
|
|
4
|
+
import { err, ok, type TtsResult } from "../voice/provider.js";
|
|
5
5
|
import type { TimedWord } from "../voice/voiceover.js";
|
|
6
|
+
import { probeAudio } from "./audio.js";
|
|
6
7
|
import type { IsolationProvider, MusicProvider, SttInput, SttProvider, SttTranscript } from "./providers.js";
|
|
7
8
|
|
|
8
9
|
/** Providers that spend nothing and reach no network, for tests of the footage pipeline and of a project's own flows. */
|
|
@@ -36,19 +37,42 @@ export function createFakeSttProvider(options: FakeSttOptions = {}): FakeSttProv
|
|
|
36
37
|
};
|
|
37
38
|
}
|
|
38
39
|
|
|
40
|
+
export interface FakeIsolationOptions {
|
|
41
|
+
/**
|
|
42
|
+
* The shortest input in seconds the fake accepts, declared as its `minSeconds`: a shorter input (probed with
|
|
43
|
+
* ffprobe) is refused with `audio_too_short`, as the ElevenLabs Voice Isolator refuses it.
|
|
44
|
+
*/
|
|
45
|
+
minSeconds?: number;
|
|
46
|
+
}
|
|
47
|
+
|
|
39
48
|
export interface FakeIsolationProvider extends IsolationProvider {
|
|
40
49
|
readonly calls: string[];
|
|
50
|
+
/** The inputs refused for being shorter than `minSeconds`. */
|
|
51
|
+
readonly refused: string[];
|
|
41
52
|
}
|
|
42
53
|
|
|
43
54
|
/** Returns the input file's bytes unchanged: a valid audio file, so a pipeline with it still renders. */
|
|
44
|
-
export function createFakeIsolationProvider(): FakeIsolationProvider {
|
|
55
|
+
export function createFakeIsolationProvider(options: FakeIsolationOptions = {}): FakeIsolationProvider {
|
|
45
56
|
const calls: string[] = [];
|
|
57
|
+
const refused: string[] = [];
|
|
58
|
+
const { minSeconds } = options;
|
|
46
59
|
return {
|
|
47
60
|
id: "fake",
|
|
61
|
+
...(minSeconds === undefined ? {} : { minSeconds }),
|
|
48
62
|
calls,
|
|
49
|
-
|
|
63
|
+
refused,
|
|
64
|
+
async isolate(input) {
|
|
50
65
|
calls.push(input.audio);
|
|
51
|
-
|
|
66
|
+
if (minSeconds !== undefined) {
|
|
67
|
+
const probed = await probeAudio(input.audio, input.signal === undefined ? {} : { signal: input.signal });
|
|
68
|
+
if (!probed.ok) return err(`fake voice isolation: ${probed.error}`);
|
|
69
|
+
if (probed.probe.duration < minSeconds) {
|
|
70
|
+
refused.push(input.audio);
|
|
71
|
+
const message = `${probed.probe.duration.toFixed(3)} s, the minimum is ${String(minSeconds)} s`;
|
|
72
|
+
return err(`fake voice isolation: 400 audio_too_short (${message}).`, { kind: "refused", status: 400, code: "audio_too_short", message, requestId: null, isRetryable: false });
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return ok(readFileSync(input.audio));
|
|
52
76
|
},
|
|
53
77
|
};
|
|
54
78
|
}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* How hyperframes captured a render, read from the summary it prints at the end (issue #1145). hyperframes 0.9.6 prints
|
|
3
|
+
* `beginframe capture · software gpu · compile 0.8s · extract 8.7s · … · assemble 0.6s` and, after a screenshot
|
|
4
|
+
* capture it did not ask for, a line `Screenshot capture (slower): BeginFrame did not run. …`.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
/** hyperframes' capture modes: `beginframe` (fast), `screenshot` (slower), `drawelement`. */
|
|
8
|
+
export type HyperframesCapturePath = "beginframe" | "screenshot" | "drawelement";
|
|
9
|
+
|
|
10
|
+
/** The browser's GPU mode for the render. */
|
|
11
|
+
export type HyperframesGpuMode = "software" | "hardware";
|
|
12
|
+
|
|
13
|
+
/** The pipeline stages hyperframes times, in the order it prints them. */
|
|
14
|
+
export const HYPERFRAMES_CAPTURE_STAGES = ["compile", "extract", "audio", "probe", "setup", "capture", "encode", "assemble"] as const;
|
|
15
|
+
|
|
16
|
+
export type HyperframesCaptureStage = (typeof HYPERFRAMES_CAPTURE_STAGES)[number];
|
|
17
|
+
|
|
18
|
+
export interface HyperframesCapture {
|
|
19
|
+
/** The capture path; null when hyperframes printed none. */
|
|
20
|
+
path: HyperframesCapturePath | null;
|
|
21
|
+
/** The browser's GPU mode; null when hyperframes printed none. */
|
|
22
|
+
gpu: HyperframesGpuMode | null;
|
|
23
|
+
/** Seconds per stage, only the stages hyperframes printed. */
|
|
24
|
+
seconds: Partial<Record<HyperframesCaptureStage, number>>;
|
|
25
|
+
/** The encode ran during the capture (`encode (during capture)`), so its seconds overlap the capture's. */
|
|
26
|
+
isEncodeStreamed: boolean;
|
|
27
|
+
/** hyperframes' hint after a screenshot capture (`Screenshot capture (slower): …`); null when it printed none. */
|
|
28
|
+
fallbackHint: string | null;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const SEPARATOR = " · ";
|
|
32
|
+
const CAPTURE_PATHS: readonly string[] = ["beginframe", "screenshot", "drawelement"];
|
|
33
|
+
const GPU_MODES: readonly string[] = ["software", "hardware"];
|
|
34
|
+
const STAGES: readonly string[] = HYPERFRAMES_CAPTURE_STAGES;
|
|
35
|
+
const FALLBACK_HINT_PREFIX = "Screenshot capture";
|
|
36
|
+
// SGR colour codes (`\x1b[2m…\x1b[22m`) that hyperframes' dim style may add.
|
|
37
|
+
// eslint-disable-next-line no-control-regex -- the escape character is what the pattern matches
|
|
38
|
+
const ANSI_PATTERN = /\u001B\[[0-9;]*m/g;
|
|
39
|
+
|
|
40
|
+
/** hyperframes' `formatDuration`: `8.7s` under a minute, `1m 31.4s` from a minute on. */
|
|
41
|
+
function readDurationSeconds(text: string): number | null {
|
|
42
|
+
const match = /^(?:(\d+)m )?(\d+(?:\.\d+)?)s$/.exec(text);
|
|
43
|
+
if (match === null) return null;
|
|
44
|
+
const minutes = match[1] === undefined ? 0 : Number(match[1]);
|
|
45
|
+
return Math.round((minutes * 60 + Number(match[2])) * 1000) / 1000;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
type SummaryPart =
|
|
49
|
+
| { kind: "path"; path: HyperframesCapturePath }
|
|
50
|
+
| { kind: "gpu"; gpu: HyperframesGpuMode }
|
|
51
|
+
| { kind: "stage"; stage: HyperframesCaptureStage; seconds: number; isStreamed: boolean };
|
|
52
|
+
|
|
53
|
+
function readSummaryPart(part: string): SummaryPart | null {
|
|
54
|
+
const path = /^(\w+) capture$/.exec(part)?.[1];
|
|
55
|
+
if (path !== undefined) return CAPTURE_PATHS.includes(path) ? { kind: "path", path: path as HyperframesCapturePath } : null;
|
|
56
|
+
const gpu = /^(\w+) gpu$/.exec(part)?.[1];
|
|
57
|
+
if (gpu !== undefined) return GPU_MODES.includes(gpu) ? { kind: "gpu", gpu: gpu as HyperframesGpuMode } : null;
|
|
58
|
+
const stage = /^(\w+)( \(during capture\))? (.+)$/.exec(part);
|
|
59
|
+
if (stage === null) return null;
|
|
60
|
+
const [, name = "", streamed, duration = ""] = stage;
|
|
61
|
+
const seconds = readDurationSeconds(duration);
|
|
62
|
+
if (!STAGES.includes(name) || seconds === null) return null;
|
|
63
|
+
return { kind: "stage", stage: name as HyperframesCaptureStage, seconds, isStreamed: streamed !== undefined };
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** A summary line: every ` · ` part is a capture path, a GPU mode or a timed stage. Null for any other line. */
|
|
67
|
+
function readSummaryLine(line: string): Omit<HyperframesCapture, "fallbackHint"> | null {
|
|
68
|
+
const parts = line.split(SEPARATOR).map(readSummaryPart);
|
|
69
|
+
if (parts.length === 0 || parts.some((part) => part === null)) return null;
|
|
70
|
+
const capture: Omit<HyperframesCapture, "fallbackHint"> = { path: null, gpu: null, seconds: {}, isEncodeStreamed: false };
|
|
71
|
+
for (const part of parts) {
|
|
72
|
+
if (part === null) continue;
|
|
73
|
+
if (part.kind === "path") capture.path = part.path;
|
|
74
|
+
else if (part.kind === "gpu") capture.gpu = part.gpu;
|
|
75
|
+
else {
|
|
76
|
+
capture.seconds[part.stage] = part.seconds;
|
|
77
|
+
if (part.stage === "encode" && part.isStreamed) capture.isEncodeStreamed = true;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return capture;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* The capture report from hyperframes' output lines (stdout and stderr, in order): the last summary line, plus the
|
|
85
|
+
* screenshot hint when one was printed. Null when no line is a summary (an older or quiet hyperframes).
|
|
86
|
+
*/
|
|
87
|
+
export function readHyperframesCapture(lines: readonly string[]): HyperframesCapture | null {
|
|
88
|
+
let summary: Omit<HyperframesCapture, "fallbackHint"> | null = null;
|
|
89
|
+
let fallbackHint: string | null = null;
|
|
90
|
+
for (const raw of lines) {
|
|
91
|
+
const line = raw.replace(ANSI_PATTERN, "").trim();
|
|
92
|
+
if (line.length === 0) continue;
|
|
93
|
+
if (line.startsWith(FALLBACK_HINT_PREFIX)) {
|
|
94
|
+
fallbackHint = line;
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
97
|
+
summary = readSummaryLine(line) ?? summary;
|
|
98
|
+
}
|
|
99
|
+
return summary === null ? null : { ...summary, fallbackHint };
|
|
100
|
+
}
|