@softure-ai/marketing-kit 0.14.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +95 -0
- package/README.md +136 -12
- package/dist/config/schema.d.ts +2 -1
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +7 -0
- package/dist/config/schema.js.map +1 -1
- package/dist/footage/clip-sound.d.ts +58 -0
- package/dist/footage/clip-sound.d.ts.map +1 -0
- package/dist/footage/clip-sound.js +103 -0
- package/dist/footage/clip-sound.js.map +1 -0
- package/dist/footage/declick.d.ts +1 -1
- package/dist/footage/declick.d.ts.map +1 -1
- package/dist/footage/declick.js +3 -4
- package/dist/footage/declick.js.map +1 -1
- package/dist/footage/draft.d.ts +19 -5
- package/dist/footage/draft.d.ts.map +1 -1
- package/dist/footage/draft.js +94 -24
- package/dist/footage/draft.js.map +1 -1
- package/dist/footage/elevenlabs.d.ts +5 -0
- package/dist/footage/elevenlabs.d.ts.map +1 -1
- package/dist/footage/elevenlabs.js +10 -3
- package/dist/footage/elevenlabs.js.map +1 -1
- package/dist/footage/failure.d.ts +53 -0
- package/dist/footage/failure.d.ts.map +1 -0
- package/dist/footage/failure.js +23 -0
- package/dist/footage/failure.js.map +1 -0
- package/dist/footage/fake.d.ts +10 -1
- package/dist/footage/fake.d.ts.map +1 -1
- package/dist/footage/fake.js +19 -4
- package/dist/footage/fake.js.map +1 -1
- package/dist/footage/film-mix.d.ts +5 -1
- package/dist/footage/film-mix.d.ts.map +1 -1
- package/dist/footage/film-mix.js +11 -2
- package/dist/footage/film-mix.js.map +1 -1
- package/dist/footage/hyperframes-capture.d.ts +30 -0
- package/dist/footage/hyperframes-capture.d.ts.map +1 -0
- package/dist/footage/hyperframes-capture.js +80 -0
- package/dist/footage/hyperframes-capture.js.map +1 -0
- package/dist/footage/isolate-ranges.d.ts +67 -5
- package/dist/footage/isolate-ranges.d.ts.map +1 -1
- package/dist/footage/isolate-ranges.js +92 -21
- package/dist/footage/isolate-ranges.js.map +1 -1
- package/dist/footage/loudness.d.ts +16 -2
- package/dist/footage/loudness.d.ts.map +1 -1
- package/dist/footage/loudness.js +10 -7
- package/dist/footage/loudness.js.map +1 -1
- package/dist/footage/paid.d.ts +14 -1
- package/dist/footage/paid.d.ts.map +1 -1
- package/dist/footage/paid.js +10 -4
- package/dist/footage/paid.js.map +1 -1
- package/dist/footage/pipeline.d.ts +10 -4
- package/dist/footage/pipeline.d.ts.map +1 -1
- package/dist/footage/pipeline.js +74 -64
- package/dist/footage/pipeline.js.map +1 -1
- package/dist/footage/providers.d.ts +5 -0
- package/dist/footage/providers.d.ts.map +1 -1
- package/dist/footage/providers.js.map +1 -1
- package/dist/footage/render.d.ts +1 -1
- package/dist/footage/render.d.ts.map +1 -1
- package/dist/footage/render.js +4 -3
- package/dist/footage/render.js.map +1 -1
- package/dist/footage/speech-ranges.d.ts +80 -14
- package/dist/footage/speech-ranges.d.ts.map +1 -1
- package/dist/footage/speech-ranges.js +146 -26
- package/dist/footage/speech-ranges.js.map +1 -1
- package/dist/index.d.ts +10 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -5
- package/dist/index.js.map +1 -1
- package/dist/og/index.d.ts +2 -0
- package/dist/og/index.d.ts.map +1 -1
- package/dist/og/index.js +2 -0
- package/dist/og/index.js.map +1 -1
- package/dist/og/photo-cover.d.ts +55 -0
- package/dist/og/photo-cover.d.ts.map +1 -0
- package/dist/og/photo-cover.js +215 -0
- package/dist/og/photo-cover.js.map +1 -0
- package/dist/og/photo.d.ts +51 -0
- package/dist/og/photo.d.ts.map +1 -0
- package/dist/og/photo.js +154 -0
- package/dist/og/photo.js.map +1 -0
- package/dist/voice/provider-failure.d.ts +41 -0
- package/dist/voice/provider-failure.d.ts.map +1 -0
- package/dist/voice/provider-failure.js +68 -0
- package/dist/voice/provider-failure.js.map +1 -0
- package/dist/voice/provider.d.ts +7 -1
- package/dist/voice/provider.d.ts.map +1 -1
- package/dist/voice/provider.js +1 -1
- package/dist/voice/provider.js.map +1 -1
- package/package.json +2 -1
- package/schema/marketing.schema.json +6 -0
- package/src/config/schema.ts +7 -0
- package/src/footage/clip-sound.ts +124 -0
- package/src/footage/declick.ts +3 -4
- package/src/footage/draft.ts +125 -24
- package/src/footage/elevenlabs.ts +11 -3
- package/src/footage/failure.ts +57 -0
- package/src/footage/fake.ts +28 -4
- package/src/footage/film-mix.ts +10 -2
- package/src/footage/hyperframes-capture.ts +100 -0
- package/src/footage/isolate-ranges.ts +158 -22
- package/src/footage/loudness.ts +17 -8
- package/src/footage/paid.ts +29 -4
- package/src/footage/pipeline.ts +85 -66
- package/src/footage/providers.ts +5 -0
- package/src/footage/render.ts +4 -3
- package/src/footage/speech-ranges.ts +197 -27
- package/src/index.ts +55 -1
- package/src/og/index.ts +23 -0
- package/src/og/photo-cover.ts +280 -0
- package/src/og/photo.ts +177 -0
- package/src/voice/provider-failure.ts +98 -0
- package/src/voice/provider.ts +7 -2
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"provider.js","sourceRoot":"","sources":["../../src/voice/provider.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"provider.js","sourceRoot":"","sources":["../../src/voice/provider.ts"],"names":[],"mappings":"AA6CA,MAAM,CAAC,MAAM,EAAE,GAAG,CAAI,KAAQ,EAAgB,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;AACvE,MAAM,CAAC,MAAM,GAAG,GAAG,CAAI,KAAa,EAAE,eAAiC,EAAgB,EAAE,CAAC,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@softure-ai/marketing-kit",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.16.0",
|
|
4
4
|
"description": "SOFTURE AI marketing kit: a CLI that turns a project's real app into a vertical film (cached voiceover, frame-by-frame recording, HTML composition, hyperframes render) plus per-platform post copy and OG images.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -51,6 +51,7 @@
|
|
|
51
51
|
"@resvg/resvg-js": "^2.6.2",
|
|
52
52
|
"gsap": "^3.15.0",
|
|
53
53
|
"hyperframes": "0.9.6",
|
|
54
|
+
"jpeg-js": "^0.4.4",
|
|
54
55
|
"playwright": "^1.63.0",
|
|
55
56
|
"satori": "0.63.2",
|
|
56
57
|
"tsx": "^4.23.0",
|
|
@@ -3069,6 +3069,12 @@
|
|
|
3069
3069
|
"type": "number",
|
|
3070
3070
|
"minimum": 0.2,
|
|
3071
3071
|
"maximum": 3
|
|
3072
|
+
},
|
|
3073
|
+
"audioBitrateKbps": {
|
|
3074
|
+
"description": "The finished sound's AAC bitrate in kbps (192 to 512, default 192): the film's only lossy audio encode.",
|
|
3075
|
+
"type": "integer",
|
|
3076
|
+
"minimum": 192,
|
|
3077
|
+
"maximum": 512
|
|
3072
3078
|
}
|
|
3073
3079
|
},
|
|
3074
3080
|
"additionalProperties": false
|
package/src/config/schema.ts
CHANGED
|
@@ -643,6 +643,13 @@ const footageSchema = z
|
|
|
643
643
|
.describe(
|
|
644
644
|
"A near-miss band in LU (0.2 to 3, at least tolerance): a mix outside target ± tolerance but inside target ± warnTolerance, under the true peak limit, finishes with a warning instead of failing.",
|
|
645
645
|
),
|
|
646
|
+
audioBitrateKbps: z
|
|
647
|
+
.number()
|
|
648
|
+
.int()
|
|
649
|
+
.min(192)
|
|
650
|
+
.max(512)
|
|
651
|
+
.optional()
|
|
652
|
+
.describe("The finished sound's AAC bitrate in kbps (192 to 512, default 192): the film's only lossy audio encode."),
|
|
646
653
|
})
|
|
647
654
|
.refine((mix) => mix.warnTolerance === undefined || mix.warnTolerance >= (mix.tolerance ?? 1), { path: ["warnTolerance"], message: "must be at least tolerance (default 1)" }),
|
|
648
655
|
])
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import { mkdirSync } from "node:fs";
|
|
2
|
+
import { dirname, extname, join } from "node:path";
|
|
3
|
+
|
|
4
|
+
import type { FootageClip } from "./film.js";
|
|
5
|
+
import { isSilent, readLoudnessMeasurement, type LoudnessMeasurement } from "./loudness.js";
|
|
6
|
+
import { FOOTAGE_SAMPLE_RATE } from "./normalize.js";
|
|
7
|
+
import { runFfmpeg, runProcess } from "./process.js";
|
|
8
|
+
import type { FootageProbe } from "./probe.js";
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Each clip's sound for the speech chain (issue #1139): read from the clip's source with the normalize step's own seek
|
|
12
|
+
* and length (so the EDL's times hold), brought to 48 kHz stereo, raised by one linear pre-gain to about
|
|
13
|
+
* `SPEECH_PRE_GAIN.target` with its true peak kept under `SPEECH_PRE_GAIN.peakCeiling`, and written as 24-bit PCM WAV.
|
|
14
|
+
* Quiet phone speech is then never encoded lossily at its quiet level: the voice isolator, the speech to text, the cut,
|
|
15
|
+
* the declick and the loudness pass all work on lossless sound, and the film's final AAC is the only lossy encode.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
/** Where the pre-gain brings a talking clip's sound: integrated loudness in LUFS, and the true peak's ceiling in dBTP. */
|
|
19
|
+
export const SPEECH_PRE_GAIN = { target: -23, peakCeiling: -3 } as const;
|
|
20
|
+
|
|
21
|
+
/** The loudness range `loudnorm` measures with; the pre-gain itself is a plain linear `volume`. */
|
|
22
|
+
const MEASURE_RANGE = 11;
|
|
23
|
+
|
|
24
|
+
/** The clip as the normalize step reads it: its trim, its source file and what the probe found. */
|
|
25
|
+
export interface ClipSoundInput {
|
|
26
|
+
clip: Pick<FootageClip, "id" | "role" | "trim">;
|
|
27
|
+
/** The source as read (a URL's downloaded copy). */
|
|
28
|
+
source: string;
|
|
29
|
+
probe: Pick<FootageProbe, "hasAudio">;
|
|
30
|
+
/** Seconds after the trim. */
|
|
31
|
+
duration: number;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** A clip's lossless, pre-gained sound and the gain it got in dB. */
|
|
35
|
+
export interface ClipSound {
|
|
36
|
+
file: string;
|
|
37
|
+
gainDb: number;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const toFormat = `aformat=sample_rates=${String(FOOTAGE_SAMPLE_RATE)}:channel_layouts=stereo`;
|
|
41
|
+
|
|
42
|
+
/** The seek and the length of the trimmed clip, as `buildNormalizeArgs` reads its source. */
|
|
43
|
+
function getTrimArgs(input: ClipSoundInput): { seek: string[]; length: string[] } {
|
|
44
|
+
return { seek: input.clip.trim.start > 0 ? ["-ss", String(input.clip.trim.start)] : [], length: ["-t", input.duration.toFixed(3)] };
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** The measuring pass over the trimmed clip's sound, after the same format change the extraction makes; writes nothing. */
|
|
48
|
+
export function buildClipSoundMeasureArgs(input: ClipSoundInput): string[] {
|
|
49
|
+
const { seek, length } = getTrimArgs(input);
|
|
50
|
+
const measure = `loudnorm=I=${String(SPEECH_PRE_GAIN.target)}:TP=${String(SPEECH_PRE_GAIN.peakCeiling)}:LRA=${String(MEASURE_RANGE)}:print_format=json`;
|
|
51
|
+
return ["-hide_banner", "-nostats", ...seek, "-i", input.source, ...length, "-map", "0:a:0", "-vn", "-af", `${toFormat},${measure}`, "-f", "null", "-"];
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* The pre-gain in dB for a measured clip: up (or down) to `SPEECH_PRE_GAIN.target`, but never past the gain that puts
|
|
56
|
+
* its true peak at `SPEECH_PRE_GAIN.peakCeiling`; 0 for silence. Rounded to a hundredth of a dB.
|
|
57
|
+
*/
|
|
58
|
+
export function getPreGainDb(measurement: LoudnessMeasurement | null): number {
|
|
59
|
+
if (measurement === null || isSilent(measurement)) return 0;
|
|
60
|
+
const toTarget = SPEECH_PRE_GAIN.target - measurement.inputI;
|
|
61
|
+
const toCeiling = Number.isFinite(measurement.inputTp) ? SPEECH_PRE_GAIN.peakCeiling - measurement.inputTp : toTarget;
|
|
62
|
+
return Math.round(Math.min(toTarget, toCeiling) * 100) / 100 || 0;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** The trimmed clip's sound, gained by `gainDb`, as a 24-bit 48 kHz stereo WAV; silence as long as the clip when it has none. */
|
|
66
|
+
export function buildClipSoundArgs(input: ClipSoundInput & { gainDb: number; output: string }): string[] {
|
|
67
|
+
const codec = ["-ar", String(FOOTAGE_SAMPLE_RATE), "-ac", "2", "-c:a", "pcm_s24le", input.output];
|
|
68
|
+
const { seek, length } = getTrimArgs(input);
|
|
69
|
+
if (!input.probe.hasAudio) return ["-v", "error", "-y", "-f", "lavfi", "-i", `anullsrc=r=${String(FOOTAGE_SAMPLE_RATE)}:cl=stereo`, ...length, ...codec];
|
|
70
|
+
return ["-v", "error", "-y", ...seek, "-i", input.source, ...length, "-map", "0:a:0", "-vn", "-af", `${toFormat},volume=${String(input.gainDb)}dB`, ...codec];
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Where a clip's sound goes in the build folder. */
|
|
74
|
+
export function getClipSoundPath(buildDir: string, clipId: string): string {
|
|
75
|
+
return join(buildDir, "audio", `${clipId}.wav`);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** Measures one clip's sound; null when the clip has none. Throws naming the clip when ffmpeg fails. */
|
|
79
|
+
async function measureClipSound(input: ClipSoundInput, signal: AbortSignal | undefined): Promise<LoudnessMeasurement | null> {
|
|
80
|
+
if (!input.probe.hasAudio) return null;
|
|
81
|
+
const step = `Measuring the sound of clip ${input.clip.id}`;
|
|
82
|
+
const measured = await runProcess({ command: "ffmpeg", args: buildClipSoundMeasureArgs(input), ...(signal === undefined ? {} : { signal }) });
|
|
83
|
+
if (measured.kind === "not-started") throw new Error(`${step}: ffmpeg did not start (${measured.error}).`);
|
|
84
|
+
if (measured.code !== 0) throw new Error(`${step}: ffmpeg ended with code ${String(measured.code)}: ${measured.stderr.trim().split("\n").slice(-3).join(" | ")}.`);
|
|
85
|
+
const measurement = readLoudnessMeasurement(measured.stderr);
|
|
86
|
+
if (measurement === null) throw new Error(`${step}: ffmpeg printed no loudnorm measurement for ${input.source}.`);
|
|
87
|
+
return measurement;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Every clip's lossless, pre-gained sound under `<buildDir>/audio/<id>.wav`, by clip id. A talking clip is gained by its
|
|
92
|
+
* own measurement (`getPreGainDb`); any other clip by the mean of the talking clips' gains (0 dB without one), so a
|
|
93
|
+
* b-roll's own sound under the speech keeps its relation to it. Throws naming the clip when ffmpeg fails.
|
|
94
|
+
*/
|
|
95
|
+
export async function prepareClipSounds(clips: readonly ClipSoundInput[], options: { buildDir: string; signal?: AbortSignal }): Promise<Record<string, ClipSound>> {
|
|
96
|
+
const { signal } = options;
|
|
97
|
+
const gains = new Map<string, number>();
|
|
98
|
+
for (const clip of clips) {
|
|
99
|
+
if (clip.clip.role !== "talk") continue;
|
|
100
|
+
gains.set(clip.clip.id, getPreGainDb(await measureClipSound(clip, signal)));
|
|
101
|
+
}
|
|
102
|
+
const talking = [...gains.values()];
|
|
103
|
+
const sharedGain = talking.length === 0 ? 0 : Math.round((talking.reduce((sum, gain) => sum + gain, 0) / talking.length) * 100) / 100 || 0;
|
|
104
|
+
const sounds: Record<string, ClipSound> = {};
|
|
105
|
+
for (const clip of clips) {
|
|
106
|
+
const gainDb = gains.get(clip.clip.id) ?? sharedGain;
|
|
107
|
+
const output = getClipSoundPath(options.buildDir, clip.clip.id);
|
|
108
|
+
mkdirSync(dirname(output), { recursive: true });
|
|
109
|
+
await runFfmpeg(`Extracting the sound of clip ${clip.clip.id}`, buildClipSoundArgs({ ...clip, gainDb, output }), { ...(signal === undefined ? {} : { signal }), output });
|
|
110
|
+
sounds[clip.clip.id] = { file: output, gainDb };
|
|
111
|
+
}
|
|
112
|
+
return sounds;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* The audio codec for an intermediate file by its extension: `.wav` 32-bit float PCM (no stage clips between stages),
|
|
117
|
+
* `.flac` FLAC, anything else MP3 (an app's own `.mp3` names keep working).
|
|
118
|
+
*/
|
|
119
|
+
export function getAudioCodecArgs(output: string): string[] {
|
|
120
|
+
const extension = extname(output).toLowerCase();
|
|
121
|
+
if (extension === ".wav") return ["-c:a", "pcm_f32le"];
|
|
122
|
+
if (extension === ".flac") return ["-c:a", "flac"];
|
|
123
|
+
return ["-c:a", "libmp3lame", "-q:a", "2"];
|
|
124
|
+
}
|
package/src/footage/declick.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
2
|
-
import { extname } from "node:path";
|
|
3
2
|
|
|
4
3
|
import type { TimedWord } from "../voice/voiceover.js";
|
|
4
|
+
import { getAudioCodecArgs } from "./clip-sound.js";
|
|
5
5
|
import { FOOTAGE_SAMPLE_RATE } from "./normalize.js";
|
|
6
6
|
import { runFfmpeg } from "./process.js";
|
|
7
7
|
|
|
@@ -295,10 +295,9 @@ export function buildDeclickDecodeArgs(input: string, output: string): string[]
|
|
|
295
295
|
return ["-v", "error", "-y", "-i", input, "-vn", "-c:a", "pcm_f32le", ...PCM_FORMAT, output];
|
|
296
296
|
}
|
|
297
297
|
|
|
298
|
-
/** The declicked PCM encoded
|
|
298
|
+
/** The declicked PCM encoded by the output's extension (`getAudioCodecArgs`: `.wav` lossless float, as the chain writes it). */
|
|
299
299
|
export function buildDeclickEncodeArgs(input: string, output: string): string[] {
|
|
300
|
-
|
|
301
|
-
return ["-v", "error", "-y", ...PCM_FORMAT, "-i", input, ...codec, output];
|
|
300
|
+
return ["-v", "error", "-y", ...PCM_FORMAT, "-i", input, ...getAudioCodecArgs(output), output];
|
|
302
301
|
}
|
|
303
302
|
|
|
304
303
|
/** Raw little-endian float32 bytes as samples (copied, so the view is aligned). */
|
package/src/footage/draft.ts
CHANGED
|
@@ -4,7 +4,7 @@ import { fileURLToPath } from "node:url";
|
|
|
4
4
|
|
|
5
5
|
import type { CaptionLayout, LayoutOverride } from "../compose/timeline.js";
|
|
6
6
|
import type { BrandColors } from "../config/colors.js";
|
|
7
|
-
import { err, ok } from "
|
|
7
|
+
import { err, ok } from "./failure.js";
|
|
8
8
|
import type { TimedWord } from "../voice/voiceover.js";
|
|
9
9
|
import { getCaptionChunkLayouts, getCaptionSpans, getFootageCaptionChunks, getFootageCaptionLayout, isSameCaptionPlace, type CaptionWordsFilm } from "./captions.js";
|
|
10
10
|
import { getSegmentScales, getSegmentZoomEases } from "./compose.js";
|
|
@@ -12,18 +12,35 @@ import { chooseFootageDraftFont } from "./draft-font.js";
|
|
|
12
12
|
import { getEdlDuration, getSegmentStarts, remapWords, type Edl } from "./edl.js";
|
|
13
13
|
import { declickSpeech, describeDeclick, type DeclickReport } from "./declick.js";
|
|
14
14
|
import { getDeclickLevel, getFootageEdit, type FootageFilm } from "./film.js";
|
|
15
|
+
import { buildFilmMixArgs } from "./film-mix.js";
|
|
16
|
+
import { DEFAULT_LOUDNESS, formatDb, getLoudnessVerdict, measureLoudness, normalizeLoudness, type MixLoudnessTarget } from "./loudness.js";
|
|
15
17
|
import { duckingKeyframes, getDuckingOptions } from "./music.js";
|
|
16
18
|
import { FOOTAGE_FPS, getThreadArgs, type Frame } from "./normalize.js";
|
|
17
|
-
import {
|
|
19
|
+
import {
|
|
20
|
+
findFootageMusic,
|
|
21
|
+
getFfmpegReporting,
|
|
22
|
+
getFrame,
|
|
23
|
+
getSpeechChainFiles,
|
|
24
|
+
mixFootage,
|
|
25
|
+
type FinishedLoudness,
|
|
26
|
+
type FinishedMix,
|
|
27
|
+
type FootageBrand,
|
|
28
|
+
type FootageResult,
|
|
29
|
+
type FootageRunOptions,
|
|
30
|
+
type PreparedClip,
|
|
31
|
+
type PreparedMix,
|
|
32
|
+
type TranscribeOutcome,
|
|
33
|
+
} from "./pipeline.js";
|
|
18
34
|
import { runFfmpeg, withFfmpegFailureReporting } from "./process.js";
|
|
19
35
|
import { buildMusicArgs, buildSpeechArgs, getSpeechCutOptions } from "./render.js";
|
|
20
36
|
|
|
21
37
|
/**
|
|
22
38
|
* A fast draft of a footage film, ffmpeg only (no hyperframes, no browser): the cut at a small width with the film's
|
|
23
39
|
* captions burned in (its caption style, in the brand's body font and caption colours), its zoom cuts (eased as in the
|
|
24
|
-
* render), cross-fades and Ken Burns b-roll drawn as the full render draws them, the
|
|
25
|
-
*
|
|
26
|
-
* render.
|
|
40
|
+
* render), cross-fades and Ken Burns b-roll drawn as the full render draws them, and the render's sound chain: the
|
|
41
|
+
* speech track's stages (cut, declick, loudness pass), the ducked music when it costs nothing, and the mix brought to
|
|
42
|
+
* `mixLoudness` as the render brings it (issue #1143). No end card, cover or sound effects. What an app shows the owner
|
|
43
|
+
* before the paid render.
|
|
27
44
|
*/
|
|
28
45
|
|
|
29
46
|
/** The draft's default width: 540 makes a 540×960 draft of a 9:16 film. */
|
|
@@ -241,6 +258,11 @@ export interface FootageDraftArgsInput {
|
|
|
241
258
|
speech: string;
|
|
242
259
|
/** The ducked music track (`buildMusicArgs`, as long as the cut), mixed under the speech; null or absent for none. */
|
|
243
260
|
music?: string | null;
|
|
261
|
+
/**
|
|
262
|
+
* The film's mix, already brought to its target and encoded to AAC (`mixFootage`): the draft copies it as its sound
|
|
263
|
+
* and reads neither `speech` nor `music`. Null or absent: the draft mixes `speech` and `music` and encodes them.
|
|
264
|
+
*/
|
|
265
|
+
mix?: string | null;
|
|
244
266
|
/** The film's full frame (the normalized clips' size). */
|
|
245
267
|
frame: Frame;
|
|
246
268
|
/** The draft's width; default 540. */
|
|
@@ -330,7 +352,8 @@ function buildPictureFilters(plan: DraftSegmentPlan, draft: Frame): string[] {
|
|
|
330
352
|
/**
|
|
331
353
|
* The ffmpeg arguments of the draft: each segment read from its own seek into its clip, zoomed and scaled to the
|
|
332
354
|
* draft's width, joined by `xfade` at `fade` cuts and `concat` at the others, the captions burned in (`ass`), the
|
|
333
|
-
*
|
|
355
|
+
* film's mix copied under it (or the speech and music tracks mixed and encoded to AAC); H.264 yuv420p with fast start,
|
|
356
|
+
* exactly the cut's length.
|
|
334
357
|
*/
|
|
335
358
|
export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
|
|
336
359
|
const { edl } = input;
|
|
@@ -378,10 +401,13 @@ export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
|
|
|
378
401
|
} else {
|
|
379
402
|
parts.push(`[${joined}]null[draft]`);
|
|
380
403
|
}
|
|
381
|
-
const
|
|
382
|
-
const
|
|
383
|
-
|
|
404
|
+
const soundInput = edl.segments.length;
|
|
405
|
+
const mix = input.mix ?? null;
|
|
406
|
+
const music = mix === null ? (input.music ?? null) : null;
|
|
407
|
+
if (music !== null) parts.push(`[${soundInput}:a][${soundInput + 1}:a]amix=inputs=2:duration=first:normalize=0[sound]`);
|
|
384
408
|
const seconds = getEdlDuration(edl);
|
|
409
|
+
// A prepared mix is copied: it was measured as encoded, and a second encode would move its loudness and true peak.
|
|
410
|
+
const audioCodec = mix === null ? ["-c:a", "aac", "-b:a", DRAFT_AUDIO_BITRATE] : ["-c:a", "copy"];
|
|
385
411
|
return [
|
|
386
412
|
"-v",
|
|
387
413
|
"error",
|
|
@@ -389,14 +415,14 @@ export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
|
|
|
389
415
|
...threads.global,
|
|
390
416
|
...inputs,
|
|
391
417
|
"-i",
|
|
392
|
-
input.speech,
|
|
418
|
+
mix ?? input.speech,
|
|
393
419
|
...(music === null ? [] : ["-i", music]),
|
|
394
420
|
"-filter_complex",
|
|
395
421
|
parts.join(";"),
|
|
396
422
|
"-map",
|
|
397
423
|
"[draft]",
|
|
398
424
|
"-map",
|
|
399
|
-
music === null ? `${
|
|
425
|
+
music === null ? `${soundInput}:a` : "[sound]",
|
|
400
426
|
"-t",
|
|
401
427
|
String(seconds),
|
|
402
428
|
"-c:v",
|
|
@@ -407,10 +433,7 @@ export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
|
|
|
407
433
|
String(DRAFT_CRF),
|
|
408
434
|
"-pix_fmt",
|
|
409
435
|
"yuv420p",
|
|
410
|
-
|
|
411
|
-
"aac",
|
|
412
|
-
"-b:a",
|
|
413
|
-
DRAFT_AUDIO_BITRATE,
|
|
436
|
+
...audioCodec,
|
|
414
437
|
"-movflags",
|
|
415
438
|
"+faststart",
|
|
416
439
|
...threads.output,
|
|
@@ -453,6 +476,13 @@ export interface FootageDraft {
|
|
|
453
476
|
bytes: number;
|
|
454
477
|
/** What the speech track's declick stage did, as the render does it; null when the film's declick is off. */
|
|
455
478
|
declick: DeclickReport | null;
|
|
479
|
+
/**
|
|
480
|
+
* The draft's sound as measured, as the render measures its finished film (`warning` on a near miss inside
|
|
481
|
+
* `mixLoudness.warnTolerance`); null for a silent draft.
|
|
482
|
+
*/
|
|
483
|
+
loudness: FinishedLoudness | null;
|
|
484
|
+
/** What the mix stage did, as in the render; null when it did not run (`mixLoudness: null`, a silent mix). */
|
|
485
|
+
mix: FinishedMix | null;
|
|
456
486
|
}
|
|
457
487
|
|
|
458
488
|
/** The draft's music track: a file to duck, or why there is none. */
|
|
@@ -465,7 +495,7 @@ type DraftMusicSource = { kind: "file"; path: string } | { kind: "none" } | { ki
|
|
|
465
495
|
async function findDraftMusic(film: FootageFilm, edl: Edl, clips: PreparedClip[], options: RenderFootageDraftOptions): Promise<FootageResult<DraftMusicSource>> {
|
|
466
496
|
const source = film.music.source;
|
|
467
497
|
if (source.kind === "none") return ok({ kind: "none" });
|
|
468
|
-
if (source.kind === "file") return existsSync(source.path) ? ok({ kind: "file", path: source.path }) : err(`music: no file at ${source.path}
|
|
498
|
+
if (source.kind === "file") return existsSync(source.path) ? ok({ kind: "file", path: source.path }) : err(`music: no file at ${source.path}`, { kind: "input" });
|
|
469
499
|
const { cacheDir, providers } = options;
|
|
470
500
|
if (cacheDir === undefined || providers === undefined) {
|
|
471
501
|
return ok({ kind: "not_cached", reason: "the film's music is generated and the draft was given no cacheDir and providers to find it in the cache" });
|
|
@@ -505,15 +535,24 @@ async function runRenderFootageDraft(
|
|
|
505
535
|
rmSync(draftDir, { recursive: true, force: true });
|
|
506
536
|
mkdirSync(draftDir, { recursive: true });
|
|
507
537
|
const declickLevel = getDeclickLevel(film);
|
|
508
|
-
const
|
|
538
|
+
const loudness = film.loudness === undefined ? DEFAULT_LOUDNESS : film.loudness;
|
|
539
|
+
const mixLoudness = film.mixLoudness === undefined ? DEFAULT_LOUDNESS : film.mixLoudness;
|
|
540
|
+
const hasMusic = music.value.kind === "file";
|
|
541
|
+
const items = [
|
|
542
|
+
"speech",
|
|
543
|
+
...(declickLevel === "off" ? [] : ["declick"]),
|
|
544
|
+
...(loudness === null ? [] : ["loudness"]),
|
|
545
|
+
...(hasMusic ? ["music"] : []),
|
|
546
|
+
...(mixLoudness === null ? [] : ["mix"]),
|
|
547
|
+
"film",
|
|
548
|
+
];
|
|
509
549
|
const report = (item: string, done: number): void => {
|
|
510
550
|
const itemIndex = items.indexOf(item) + 1;
|
|
511
551
|
options.onProgress?.({ step: "draft", item, itemIndex, itemCount: items.length, done: Math.min(seconds * items.length, (itemIndex - 1) * seconds + done), total: seconds * items.length });
|
|
512
552
|
};
|
|
513
553
|
const clipFiles = Object.fromEntries(clips.map((clip) => [clip.clip.id, resolve(clip.file)]));
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
const speech = declickLevel === "off" ? cutSpeech : join(draftDir, "speech.declicked.wav");
|
|
554
|
+
// The render's speech chain and its file names, so the draft's sound is made as the render's is.
|
|
555
|
+
const { rawSpeechFile, declickedFile, speechFile } = getSpeechChainFiles(draftDir, { isDeclick: declickLevel !== "off", isLoudness: loudness !== null });
|
|
517
556
|
const words = remapWords(edl, transcript.words, film.cut.fillers);
|
|
518
557
|
const frame = getFrame(film);
|
|
519
558
|
const brandFont = chooseFootageDraftFont(options.brand?.fonts.body ?? null);
|
|
@@ -538,13 +577,22 @@ async function runRenderFootageDraft(
|
|
|
538
577
|
const output = resolve(options.output);
|
|
539
578
|
mkdirSync(dirname(output), { recursive: true });
|
|
540
579
|
report("speech", 0);
|
|
541
|
-
await runFfmpeg("The draft's speech track", buildSpeechArgs(edl, transcript.speech,
|
|
580
|
+
await runFfmpeg("The draft's speech track", buildSpeechArgs(edl, transcript.speech, rawSpeechFile, getSpeechCutOptions(getFootageEdit(film))), { ...aborting, output: rawSpeechFile, onSeconds: (done) => report("speech", done) });
|
|
542
581
|
let declick: DeclickReport | null = null;
|
|
543
582
|
if (declickLevel !== "off") {
|
|
544
583
|
report("declick", 0);
|
|
545
|
-
declick = await declickSpeech({ input:
|
|
584
|
+
declick = await declickSpeech({ input: rawSpeechFile, output: declickedFile, words, level: declickLevel, seconds, ...aborting, onSeconds: (done) => report("declick", done) });
|
|
546
585
|
log(describeDeclick(declick));
|
|
547
586
|
}
|
|
587
|
+
if (loudness !== null) {
|
|
588
|
+
report("loudness", 0);
|
|
589
|
+
const outcome = await normalizeLoudness(declickedFile, speechFile, loudness, { ...aborting, onSeconds: (done) => report("loudness", done) });
|
|
590
|
+
log(
|
|
591
|
+
outcome.kind === "silent"
|
|
592
|
+
? "loudness: the speech track is silent, left as it is."
|
|
593
|
+
: `loudness: the speech track measured ${outcome.measured.toFixed(1)} LUFS, normalized to ${loudness.target} LUFS (true peak ${loudness.truePeak} dBTP).`,
|
|
594
|
+
);
|
|
595
|
+
}
|
|
548
596
|
let musicFile: string | null = null;
|
|
549
597
|
if (music.value.kind === "file") {
|
|
550
598
|
musicFile = join(draftDir, "music.mp3");
|
|
@@ -555,14 +603,27 @@ async function runRenderFootageDraft(
|
|
|
555
603
|
} else if (music.value.kind === "not_cached") {
|
|
556
604
|
log(`draft: no music, ${music.value.reason}.`);
|
|
557
605
|
}
|
|
606
|
+
let prepared: PreparedMix | null = null;
|
|
607
|
+
if (mixLoudness !== null) {
|
|
608
|
+
report("mix", 0);
|
|
609
|
+
// The render's mix stage over the draft's tracks (the draft has no sound effects): the same target, ceiling and attempts.
|
|
610
|
+
const mixSource = join(draftDir, "mix.wav");
|
|
611
|
+
const args = buildFilmMixArgs({ speech: { file: speechFile, seconds }, music: musicFile === null ? null : { file: musicFile, seconds }, sfx: [], seconds, output: mixSource });
|
|
612
|
+
await runFfmpeg("The draft's mix", args, { ...aborting, output: mixSource, onSeconds: (done) => report("mix", done) });
|
|
613
|
+
const mixed = await mixFootage(mixLoudness, { mixSource, output: join(draftDir, "mix.m4a") }, { log, ...aborting });
|
|
614
|
+
rmSync(mixSource, { force: true });
|
|
615
|
+
if (!mixed.ok) return mixed;
|
|
616
|
+
prepared = mixed.value;
|
|
617
|
+
}
|
|
558
618
|
report("film", 0);
|
|
559
619
|
// Run in the draft folder: the captions file and the font folder are relative names, so no path needs escaping in the filter graph.
|
|
560
620
|
const args = buildFootageDraftArgs({
|
|
561
621
|
film,
|
|
562
622
|
edl,
|
|
563
623
|
clips: clipFiles,
|
|
564
|
-
speech,
|
|
624
|
+
speech: speechFile,
|
|
565
625
|
music: musicFile,
|
|
626
|
+
mix: prepared?.audioFile ?? null,
|
|
566
627
|
frame,
|
|
567
628
|
...(options.width === undefined ? {} : { width: options.width }),
|
|
568
629
|
...(options.threads === undefined ? {} : { threads: options.threads }),
|
|
@@ -571,11 +632,51 @@ async function runRenderFootageDraft(
|
|
|
571
632
|
output,
|
|
572
633
|
});
|
|
573
634
|
await runFfmpeg("The footage draft", args, { ...aborting, output, cwd: draftDir, onSeconds: (done) => report("film", done) });
|
|
635
|
+
const measured = await measureDraftLoudness(output, mixLoudness, prepared !== null, aborting);
|
|
636
|
+
if (!measured.ok) return measured;
|
|
637
|
+
const draftLoudness = measured.value;
|
|
638
|
+
if (draftLoudness !== null) {
|
|
639
|
+
log(`loudness: the draft measures ${formatDb(draftLoudness.integrated)} LUFS, true peak ${formatDb(draftLoudness.truePeak)} dBTP.`);
|
|
640
|
+
if (draftLoudness.warning !== null) log(`loudness warning: ${draftLoudness.warning}`);
|
|
641
|
+
}
|
|
574
642
|
report("film", seconds);
|
|
575
643
|
const draft = getFootageDraftFrame(frame, options.width);
|
|
576
644
|
const bytes = statSync(output).size;
|
|
577
645
|
const draftMusic: FootageDraftMusic = music.value.kind === "file" ? "played" : music.value.kind;
|
|
578
646
|
const notes = [captions === null ? "no captions" : `captions in ${font.family}`, draftMusic === "played" ? "music" : draftMusic === "none" ? "no music" : "music not cached"];
|
|
579
647
|
log(`draft: ${output}, ${draft.width}×${draft.height}, ${seconds.toFixed(1)} s, ${bytes} bytes, ${notes.join(", ")}.`);
|
|
580
|
-
return ok({
|
|
648
|
+
return ok({
|
|
649
|
+
output,
|
|
650
|
+
music: draftMusic,
|
|
651
|
+
captionFont: captions === null ? null : font.family,
|
|
652
|
+
seconds,
|
|
653
|
+
width: draft.width,
|
|
654
|
+
height: draft.height,
|
|
655
|
+
bytes,
|
|
656
|
+
declick,
|
|
657
|
+
loudness: draftLoudness,
|
|
658
|
+
mix: prepared?.mix ?? null,
|
|
659
|
+
});
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
/**
|
|
663
|
+
* The written draft's sound measured as `finishFootage` measures the finished film: held to `mixLoudness` when the mix
|
|
664
|
+
* stage made it (a miss fails, a near miss inside `warnTolerance` carries `warning`), reported otherwise; null when
|
|
665
|
+
* the draft is silent.
|
|
666
|
+
*/
|
|
667
|
+
async function measureDraftLoudness(
|
|
668
|
+
output: string,
|
|
669
|
+
mixLoudness: MixLoudnessTarget | null,
|
|
670
|
+
isMixed: boolean,
|
|
671
|
+
aborting: { signal?: AbortSignal },
|
|
672
|
+
): Promise<FootageResult<FinishedLoudness | null>> {
|
|
673
|
+
const measured = await measureLoudness(output, mixLoudness ?? DEFAULT_LOUDNESS, { label: "the footage draft", ...aborting });
|
|
674
|
+
if (measured.kind === "silent") return ok(null);
|
|
675
|
+
const sound = { integrated: measured.measurement.inputI, truePeak: measured.measurement.inputTp };
|
|
676
|
+
const verdict = mixLoudness === null || !isMixed ? { kind: "pass" as const } : getLoudnessVerdict(sound, mixLoudness);
|
|
677
|
+
if (verdict.kind === "fail") {
|
|
678
|
+
return err(`the footage draft ${output} measures ${formatDb(sound.integrated)} LUFS integrated, ${formatDb(sound.truePeak)} dBTP true peak: ${verdict.problems.join("; ")}.`, { kind: "loudness", loudness: sound });
|
|
679
|
+
}
|
|
680
|
+
return ok({ ...sound, warning: verdict.kind === "warn" ? `the footage draft ${output}: ${verdict.warning}` : null });
|
|
581
681
|
}
|
|
682
|
+
|
|
@@ -5,6 +5,7 @@ import { z } from "zod";
|
|
|
5
5
|
|
|
6
6
|
import { ELEVENLABS_API_KEY_ENV } from "../voice/elevenlabs.js";
|
|
7
7
|
import { err, ok, type TtsResult } from "../voice/provider.js";
|
|
8
|
+
import { getConnectionFailure, getLocalRefusal, getProviderFailure } from "../voice/provider-failure.js";
|
|
8
9
|
import { ELEVENLABS_API_URL, type TimedWord } from "../voice/voiceover.js";
|
|
9
10
|
import type { IsolationProvider, MusicProvider, PaidAudio, SttProvider, SttTranscript } from "./providers.js";
|
|
10
11
|
|
|
@@ -57,17 +58,17 @@ function fileBlob(path: string): Blob {
|
|
|
57
58
|
}
|
|
58
59
|
|
|
59
60
|
async function send(options: ElevenLabsFootageOptions, label: string, path: string, init: RequestInit): Promise<TtsResult<Response>> {
|
|
60
|
-
if (options.apiKey === undefined || options.apiKey.length === 0) return err(`no ${ELEVENLABS_API_KEY_ENV} in the environment
|
|
61
|
+
if (options.apiKey === undefined || options.apiKey.length === 0) return err(`no ${ELEVENLABS_API_KEY_ENV} in the environment.`, getLocalRefusal("missing_api_key"));
|
|
61
62
|
const request = options.fetch ?? globalThis.fetch;
|
|
62
63
|
let response: Response;
|
|
63
64
|
try {
|
|
64
65
|
response = await request(`${ELEVENLABS_API_URL}${path}`, { ...init, headers: { ...init.headers, "xi-api-key": options.apiKey } });
|
|
65
66
|
} catch (error) {
|
|
66
|
-
return err(`ElevenLabs ${label}: the connection failed (${String(error)})
|
|
67
|
+
return err(`ElevenLabs ${label}: the connection failed (${String(error)}).`, getConnectionFailure(error));
|
|
67
68
|
}
|
|
68
69
|
if (!response.ok) {
|
|
69
70
|
const detail = await response.text().catch((error: unknown) => `(unreadable body: ${String(error)})`);
|
|
70
|
-
return err(`ElevenLabs ${label} answered ${response.status}: ${detail.slice(0, ERROR_DETAIL_LENGTH)}
|
|
71
|
+
return err(`ElevenLabs ${label} answered ${response.status}: ${detail.slice(0, ERROR_DETAIL_LENGTH)}`, getProviderFailure(response, detail));
|
|
71
72
|
}
|
|
72
73
|
return ok(response);
|
|
73
74
|
}
|
|
@@ -118,9 +119,16 @@ export function createElevenLabsSttProvider(options: ElevenLabsFootageOptions):
|
|
|
118
119
|
};
|
|
119
120
|
}
|
|
120
121
|
|
|
122
|
+
/**
|
|
123
|
+
* The shortest input the ElevenLabs Voice Isolator accepts, in seconds: production renders saw every range of 4.18 s
|
|
124
|
+
* or less refused on every attempt (`audio_too_short`) and every range of 4.64 s or more accepted (issue #1140).
|
|
125
|
+
*/
|
|
126
|
+
export const ELEVENLABS_ISOLATION_MIN_SECONDS = 4.6;
|
|
127
|
+
|
|
121
128
|
export function createElevenLabsIsolationProvider(options: ElevenLabsFootageOptions): IsolationProvider {
|
|
122
129
|
return {
|
|
123
130
|
id: "elevenlabs",
|
|
131
|
+
minSeconds: ELEVENLABS_ISOLATION_MIN_SECONDS,
|
|
124
132
|
async isolate(input) {
|
|
125
133
|
const form = new FormData();
|
|
126
134
|
form.append("audio", fileBlob(input.audio), basename(input.audio));
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import type { ProviderFailure } from "../voice/provider-failure.js";
|
|
2
|
+
import type { MixLoudness } from "./loudness.js";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Why a footage step failed, for an app to act on without reading `error` (issue #1126):
|
|
6
|
+
* - `loudness`: the film's mix or the finished film misses `mixLoudness` (the last measurement; null when the mix
|
|
7
|
+
* measures silent after its gain and limiter). The same inputs fail the same way: no retry.
|
|
8
|
+
* - `publishable`: the finished file fails the container, codec profile or fast start checks (the problems).
|
|
9
|
+
* - `input`: the film, its EDL, fonts, files or settings cannot be used as given. A retry fails the same way.
|
|
10
|
+
* - `provider`: a paid provider (speech to text, voice isolation, music) failed with the structured details it gave
|
|
11
|
+
* (issue #1141); a retry may succeed only when `provider.isRetryable`.
|
|
12
|
+
* - `process`: anything else (a process that exited, the network, the disk). A retry may succeed.
|
|
13
|
+
* A failed ffmpeg step still throws and an abort still rejects with the signal's reason; neither is a result.
|
|
14
|
+
*/
|
|
15
|
+
export type FootageFailure =
|
|
16
|
+
| { kind: "loudness"; loudness: MixLoudness | null }
|
|
17
|
+
| { kind: "publishable"; problems: string[] }
|
|
18
|
+
| { kind: "input" }
|
|
19
|
+
| { kind: "provider"; provider: ProviderFailure }
|
|
20
|
+
| { kind: "process" };
|
|
21
|
+
|
|
22
|
+
export type FootageFailureKind = FootageFailure["kind"];
|
|
23
|
+
|
|
24
|
+
/** A footage failure: `error` is the sentence for logs, `failure` what the app reads. */
|
|
25
|
+
export interface FootageError {
|
|
26
|
+
ok: false;
|
|
27
|
+
error: string;
|
|
28
|
+
failure: FootageFailure;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export type FootageResult<T> = { ok: true; value: T } | FootageError;
|
|
32
|
+
|
|
33
|
+
/** A footage success. */
|
|
34
|
+
export function ok<T>(value: T): { ok: true; value: T } {
|
|
35
|
+
return { ok: true, value };
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const PROCESS_FAILURE: FootageFailure = { kind: "process" };
|
|
39
|
+
|
|
40
|
+
/** A footage failure with its sentence; without a `failure`, the transient `process` kind. */
|
|
41
|
+
export function err(error: string, failure: FootageFailure = PROCESS_FAILURE): FootageError {
|
|
42
|
+
return { ok: false, error, failure };
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** A provider's failure as a footage failure: `provider` with its details, or `process` when it gave none. */
|
|
46
|
+
export function getProviderFootageFailure(providerFailure: ProviderFailure | undefined): FootageFailure {
|
|
47
|
+
return providerFailure === undefined ? PROCESS_FAILURE : { kind: "provider", provider: providerFailure };
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* True when running the same step again may succeed: `process`, or `provider` with `isRetryable`; a gate miss, a bad
|
|
52
|
+
* input or a provider's refusal fails again.
|
|
53
|
+
*/
|
|
54
|
+
export function isRetryableFootageFailure(failure: FootageFailure): boolean {
|
|
55
|
+
if (failure.kind === "provider") return failure.provider.isRetryable;
|
|
56
|
+
return failure.kind === "process";
|
|
57
|
+
}
|
package/src/footage/fake.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { readFileSync } from "node:fs";
|
|
2
2
|
import { basename } from "node:path";
|
|
3
3
|
|
|
4
|
-
import { ok, type TtsResult } from "../voice/provider.js";
|
|
4
|
+
import { err, ok, type TtsResult } from "../voice/provider.js";
|
|
5
5
|
import type { TimedWord } from "../voice/voiceover.js";
|
|
6
|
+
import { probeAudio } from "./audio.js";
|
|
6
7
|
import type { IsolationProvider, MusicProvider, SttInput, SttProvider, SttTranscript } from "./providers.js";
|
|
7
8
|
|
|
8
9
|
/** Providers that spend nothing and reach no network, for tests of the footage pipeline and of a project's own flows. */
|
|
@@ -36,19 +37,42 @@ export function createFakeSttProvider(options: FakeSttOptions = {}): FakeSttProv
|
|
|
36
37
|
};
|
|
37
38
|
}
|
|
38
39
|
|
|
40
|
+
export interface FakeIsolationOptions {
|
|
41
|
+
/**
|
|
42
|
+
* The shortest input in seconds the fake accepts, declared as its `minSeconds`: a shorter input (probed with
|
|
43
|
+
* ffprobe) is refused with `audio_too_short`, as the ElevenLabs Voice Isolator refuses it.
|
|
44
|
+
*/
|
|
45
|
+
minSeconds?: number;
|
|
46
|
+
}
|
|
47
|
+
|
|
39
48
|
export interface FakeIsolationProvider extends IsolationProvider {
|
|
40
49
|
readonly calls: string[];
|
|
50
|
+
/** The inputs refused for being shorter than `minSeconds`. */
|
|
51
|
+
readonly refused: string[];
|
|
41
52
|
}
|
|
42
53
|
|
|
43
54
|
/** Returns the input file's bytes unchanged: a valid audio file, so a pipeline with it still renders. */
|
|
44
|
-
export function createFakeIsolationProvider(): FakeIsolationProvider {
|
|
55
|
+
export function createFakeIsolationProvider(options: FakeIsolationOptions = {}): FakeIsolationProvider {
|
|
45
56
|
const calls: string[] = [];
|
|
57
|
+
const refused: string[] = [];
|
|
58
|
+
const { minSeconds } = options;
|
|
46
59
|
return {
|
|
47
60
|
id: "fake",
|
|
61
|
+
...(minSeconds === undefined ? {} : { minSeconds }),
|
|
48
62
|
calls,
|
|
49
|
-
|
|
63
|
+
refused,
|
|
64
|
+
async isolate(input) {
|
|
50
65
|
calls.push(input.audio);
|
|
51
|
-
|
|
66
|
+
if (minSeconds !== undefined) {
|
|
67
|
+
const probed = await probeAudio(input.audio, input.signal === undefined ? {} : { signal: input.signal });
|
|
68
|
+
if (!probed.ok) return err(`fake voice isolation: ${probed.error}`);
|
|
69
|
+
if (probed.probe.duration < minSeconds) {
|
|
70
|
+
refused.push(input.audio);
|
|
71
|
+
const message = `${probed.probe.duration.toFixed(3)} s, the minimum is ${String(minSeconds)} s`;
|
|
72
|
+
return err(`fake voice isolation: 400 audio_too_short (${message}).`, { kind: "refused", status: 400, code: "audio_too_short", message, requestId: null, isRetryable: false });
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return ok(readFileSync(input.audio));
|
|
52
76
|
},
|
|
53
77
|
};
|
|
54
78
|
}
|