@softure-ai/marketing-kit 0.15.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +66 -0
- package/README.md +119 -16
- package/dist/config/schema.d.ts +2 -1
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +7 -0
- package/dist/config/schema.js.map +1 -1
- package/dist/footage/clip-sound.d.ts +58 -0
- package/dist/footage/clip-sound.d.ts.map +1 -0
- package/dist/footage/clip-sound.js +103 -0
- package/dist/footage/clip-sound.js.map +1 -0
- package/dist/footage/declick.d.ts +1 -1
- package/dist/footage/declick.d.ts.map +1 -1
- package/dist/footage/declick.js +3 -4
- package/dist/footage/declick.js.map +1 -1
- package/dist/footage/draft.d.ts +19 -5
- package/dist/footage/draft.d.ts.map +1 -1
- package/dist/footage/draft.js +92 -22
- package/dist/footage/draft.js.map +1 -1
- package/dist/footage/elevenlabs.d.ts +5 -0
- package/dist/footage/elevenlabs.d.ts.map +1 -1
- package/dist/footage/elevenlabs.js +10 -3
- package/dist/footage/elevenlabs.js.map +1 -1
- package/dist/footage/failure.d.ts +12 -1
- package/dist/footage/failure.d.ts.map +1 -1
- package/dist/footage/failure.js +10 -1
- package/dist/footage/failure.js.map +1 -1
- package/dist/footage/fake.d.ts +10 -1
- package/dist/footage/fake.d.ts.map +1 -1
- package/dist/footage/fake.js +19 -4
- package/dist/footage/fake.js.map +1 -1
- package/dist/footage/hyperframes-capture.d.ts +30 -0
- package/dist/footage/hyperframes-capture.d.ts.map +1 -0
- package/dist/footage/hyperframes-capture.js +80 -0
- package/dist/footage/hyperframes-capture.js.map +1 -0
- package/dist/footage/isolate-ranges.d.ts +25 -6
- package/dist/footage/isolate-ranges.d.ts.map +1 -1
- package/dist/footage/isolate-ranges.js +37 -16
- package/dist/footage/isolate-ranges.js.map +1 -1
- package/dist/footage/loudness.d.ts +16 -2
- package/dist/footage/loudness.d.ts.map +1 -1
- package/dist/footage/loudness.js +10 -7
- package/dist/footage/loudness.js.map +1 -1
- package/dist/footage/paid.d.ts +12 -2
- package/dist/footage/paid.d.ts.map +1 -1
- package/dist/footage/paid.js +8 -3
- package/dist/footage/paid.js.map +1 -1
- package/dist/footage/pipeline.d.ts +8 -2
- package/dist/footage/pipeline.d.ts.map +1 -1
- package/dist/footage/pipeline.js +54 -45
- package/dist/footage/pipeline.js.map +1 -1
- package/dist/footage/providers.d.ts +5 -0
- package/dist/footage/providers.d.ts.map +1 -1
- package/dist/footage/providers.js.map +1 -1
- package/dist/footage/render.d.ts +1 -1
- package/dist/footage/render.d.ts.map +1 -1
- package/dist/footage/render.js +4 -3
- package/dist/footage/render.js.map +1 -1
- package/dist/footage/speech-ranges.d.ts +51 -10
- package/dist/footage/speech-ranges.d.ts.map +1 -1
- package/dist/footage/speech-ranges.js +90 -15
- package/dist/footage/speech-ranges.js.map +1 -1
- package/dist/index.d.ts +9 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -5
- package/dist/index.js.map +1 -1
- package/dist/og/index.d.ts +2 -0
- package/dist/og/index.d.ts.map +1 -1
- package/dist/og/index.js +2 -0
- package/dist/og/index.js.map +1 -1
- package/dist/og/photo-cover.d.ts +55 -0
- package/dist/og/photo-cover.d.ts.map +1 -0
- package/dist/og/photo-cover.js +215 -0
- package/dist/og/photo-cover.js.map +1 -0
- package/dist/og/photo.d.ts +51 -0
- package/dist/og/photo.d.ts.map +1 -0
- package/dist/og/photo.js +154 -0
- package/dist/og/photo.js.map +1 -0
- package/dist/voice/provider-failure.d.ts +41 -0
- package/dist/voice/provider-failure.d.ts.map +1 -0
- package/dist/voice/provider-failure.js +68 -0
- package/dist/voice/provider-failure.js.map +1 -0
- package/dist/voice/provider.d.ts +7 -1
- package/dist/voice/provider.d.ts.map +1 -1
- package/dist/voice/provider.js +1 -1
- package/dist/voice/provider.js.map +1 -1
- package/package.json +2 -1
- package/schema/marketing.schema.json +6 -0
- package/src/config/schema.ts +7 -0
- package/src/footage/clip-sound.ts +124 -0
- package/src/footage/declick.ts +3 -4
- package/src/footage/draft.ts +123 -22
- package/src/footage/elevenlabs.ts +11 -3
- package/src/footage/failure.ts +14 -1
- package/src/footage/fake.ts +28 -4
- package/src/footage/hyperframes-capture.ts +100 -0
- package/src/footage/isolate-ranges.ts +62 -22
- package/src/footage/loudness.ts +17 -8
- package/src/footage/paid.ts +25 -3
- package/src/footage/pipeline.ts +61 -46
- package/src/footage/providers.ts +5 -0
- package/src/footage/render.ts +4 -3
- package/src/footage/speech-ranges.ts +129 -16
- package/src/index.ts +41 -1
- package/src/og/index.ts +23 -0
- package/src/og/photo-cover.ts +280 -0
- package/src/og/photo.ts +177 -0
- package/src/voice/provider-failure.ts +98 -0
- package/src/voice/provider.ts +7 -2
package/src/footage/failure.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { ProviderFailure } from "../voice/provider-failure.js";
|
|
1
2
|
import type { MixLoudness } from "./loudness.js";
|
|
2
3
|
|
|
3
4
|
/**
|
|
@@ -6,6 +7,8 @@ import type { MixLoudness } from "./loudness.js";
|
|
|
6
7
|
* measures silent after its gain and limiter). The same inputs fail the same way: no retry.
|
|
7
8
|
* - `publishable`: the finished file fails the container, codec profile or fast start checks (the problems).
|
|
8
9
|
* - `input`: the film, its EDL, fonts, files or settings cannot be used as given. A retry fails the same way.
|
|
10
|
+
* - `provider`: a paid provider (speech to text, voice isolation, music) failed with the structured details it gave
|
|
11
|
+
* (issue #1141); a retry may succeed only when `provider.isRetryable`.
|
|
9
12
|
* - `process`: anything else (a process that exited, the network, the disk). A retry may succeed.
|
|
10
13
|
* A failed ffmpeg step still throws and an abort still rejects with the signal's reason; neither is a result.
|
|
11
14
|
*/
|
|
@@ -13,6 +16,7 @@ export type FootageFailure =
|
|
|
13
16
|
| { kind: "loudness"; loudness: MixLoudness | null }
|
|
14
17
|
| { kind: "publishable"; problems: string[] }
|
|
15
18
|
| { kind: "input" }
|
|
19
|
+
| { kind: "provider"; provider: ProviderFailure }
|
|
16
20
|
| { kind: "process" };
|
|
17
21
|
|
|
18
22
|
export type FootageFailureKind = FootageFailure["kind"];
|
|
@@ -38,7 +42,16 @@ export function err(error: string, failure: FootageFailure = PROCESS_FAILURE): F
|
|
|
38
42
|
return { ok: false, error, failure };
|
|
39
43
|
}
|
|
40
44
|
|
|
41
|
-
/**
|
|
45
|
+
/** A provider's failure as a footage failure: `provider` with its details, or `process` when it gave none. */
|
|
46
|
+
export function getProviderFootageFailure(providerFailure: ProviderFailure | undefined): FootageFailure {
|
|
47
|
+
return providerFailure === undefined ? PROCESS_FAILURE : { kind: "provider", provider: providerFailure };
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* True when running the same step again may succeed: `process`, or `provider` with `isRetryable`; a gate miss, a bad
|
|
52
|
+
* input or a provider's refusal fails again.
|
|
53
|
+
*/
|
|
42
54
|
export function isRetryableFootageFailure(failure: FootageFailure): boolean {
|
|
55
|
+
if (failure.kind === "provider") return failure.provider.isRetryable;
|
|
43
56
|
return failure.kind === "process";
|
|
44
57
|
}
|
package/src/footage/fake.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { readFileSync } from "node:fs";
|
|
2
2
|
import { basename } from "node:path";
|
|
3
3
|
|
|
4
|
-
import { ok, type TtsResult } from "../voice/provider.js";
|
|
4
|
+
import { err, ok, type TtsResult } from "../voice/provider.js";
|
|
5
5
|
import type { TimedWord } from "../voice/voiceover.js";
|
|
6
|
+
import { probeAudio } from "./audio.js";
|
|
6
7
|
import type { IsolationProvider, MusicProvider, SttInput, SttProvider, SttTranscript } from "./providers.js";
|
|
7
8
|
|
|
8
9
|
/** Providers that spend nothing and reach no network, for tests of the footage pipeline and of a project's own flows. */
|
|
@@ -36,19 +37,42 @@ export function createFakeSttProvider(options: FakeSttOptions = {}): FakeSttProv
|
|
|
36
37
|
};
|
|
37
38
|
}
|
|
38
39
|
|
|
40
|
+
export interface FakeIsolationOptions {
|
|
41
|
+
/**
|
|
42
|
+
* The shortest input in seconds the fake accepts, declared as its `minSeconds`: a shorter input (probed with
|
|
43
|
+
* ffprobe) is refused with `audio_too_short`, as the ElevenLabs Voice Isolator refuses it.
|
|
44
|
+
*/
|
|
45
|
+
minSeconds?: number;
|
|
46
|
+
}
|
|
47
|
+
|
|
39
48
|
export interface FakeIsolationProvider extends IsolationProvider {
|
|
40
49
|
readonly calls: string[];
|
|
50
|
+
/** The inputs refused for being shorter than `minSeconds`. */
|
|
51
|
+
readonly refused: string[];
|
|
41
52
|
}
|
|
42
53
|
|
|
43
54
|
/** Returns the input file's bytes unchanged: a valid audio file, so a pipeline with it still renders. */
|
|
44
|
-
export function createFakeIsolationProvider(): FakeIsolationProvider {
|
|
55
|
+
export function createFakeIsolationProvider(options: FakeIsolationOptions = {}): FakeIsolationProvider {
|
|
45
56
|
const calls: string[] = [];
|
|
57
|
+
const refused: string[] = [];
|
|
58
|
+
const { minSeconds } = options;
|
|
46
59
|
return {
|
|
47
60
|
id: "fake",
|
|
61
|
+
...(minSeconds === undefined ? {} : { minSeconds }),
|
|
48
62
|
calls,
|
|
49
|
-
|
|
63
|
+
refused,
|
|
64
|
+
async isolate(input) {
|
|
50
65
|
calls.push(input.audio);
|
|
51
|
-
|
|
66
|
+
if (minSeconds !== undefined) {
|
|
67
|
+
const probed = await probeAudio(input.audio, input.signal === undefined ? {} : { signal: input.signal });
|
|
68
|
+
if (!probed.ok) return err(`fake voice isolation: ${probed.error}`);
|
|
69
|
+
if (probed.probe.duration < minSeconds) {
|
|
70
|
+
refused.push(input.audio);
|
|
71
|
+
const message = `${probed.probe.duration.toFixed(3)} s, the minimum is ${String(minSeconds)} s`;
|
|
72
|
+
return err(`fake voice isolation: 400 audio_too_short (${message}).`, { kind: "refused", status: 400, code: "audio_too_short", message, requestId: null, isRetryable: false });
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return ok(readFileSync(input.audio));
|
|
52
76
|
},
|
|
53
77
|
};
|
|
54
78
|
}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* How hyperframes captured a render, read from the summary it prints at the end (issue #1145). hyperframes 0.9.6 prints
|
|
3
|
+
* `beginframe capture · software gpu · compile 0.8s · extract 8.7s · … · assemble 0.6s` and, after a screenshot
|
|
4
|
+
* capture it did not ask for, a line `Screenshot capture (slower): BeginFrame did not run. …`.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
/** hyperframes' capture modes: `beginframe` (fast), `screenshot` (slower), `drawelement`. */
|
|
8
|
+
export type HyperframesCapturePath = "beginframe" | "screenshot" | "drawelement";
|
|
9
|
+
|
|
10
|
+
/** The browser's GPU mode for the render. */
|
|
11
|
+
export type HyperframesGpuMode = "software" | "hardware";
|
|
12
|
+
|
|
13
|
+
/** The pipeline stages hyperframes times, in the order it prints them. */
|
|
14
|
+
export const HYPERFRAMES_CAPTURE_STAGES = ["compile", "extract", "audio", "probe", "setup", "capture", "encode", "assemble"] as const;
|
|
15
|
+
|
|
16
|
+
export type HyperframesCaptureStage = (typeof HYPERFRAMES_CAPTURE_STAGES)[number];
|
|
17
|
+
|
|
18
|
+
export interface HyperframesCapture {
|
|
19
|
+
/** The capture path; null when hyperframes printed none. */
|
|
20
|
+
path: HyperframesCapturePath | null;
|
|
21
|
+
/** The browser's GPU mode; null when hyperframes printed none. */
|
|
22
|
+
gpu: HyperframesGpuMode | null;
|
|
23
|
+
/** Seconds per stage, only the stages hyperframes printed. */
|
|
24
|
+
seconds: Partial<Record<HyperframesCaptureStage, number>>;
|
|
25
|
+
/** The encode ran during the capture (`encode (during capture)`), so its seconds overlap the capture's. */
|
|
26
|
+
isEncodeStreamed: boolean;
|
|
27
|
+
/** hyperframes' hint after a screenshot capture (`Screenshot capture (slower): …`); null when it printed none. */
|
|
28
|
+
fallbackHint: string | null;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const SEPARATOR = " · ";
|
|
32
|
+
const CAPTURE_PATHS: readonly string[] = ["beginframe", "screenshot", "drawelement"];
|
|
33
|
+
const GPU_MODES: readonly string[] = ["software", "hardware"];
|
|
34
|
+
const STAGES: readonly string[] = HYPERFRAMES_CAPTURE_STAGES;
|
|
35
|
+
const FALLBACK_HINT_PREFIX = "Screenshot capture";
|
|
36
|
+
// SGR colour codes (`\x1b[2m…\x1b[22m`) that hyperframes' dim style may add.
|
|
37
|
+
// eslint-disable-next-line no-control-regex -- the escape character is what the pattern matches
|
|
38
|
+
const ANSI_PATTERN = /\u001B\[[0-9;]*m/g;
|
|
39
|
+
|
|
40
|
+
/** hyperframes' `formatDuration`: `8.7s` under a minute, `1m 31.4s` from a minute on. */
|
|
41
|
+
function readDurationSeconds(text: string): number | null {
|
|
42
|
+
const match = /^(?:(\d+)m )?(\d+(?:\.\d+)?)s$/.exec(text);
|
|
43
|
+
if (match === null) return null;
|
|
44
|
+
const minutes = match[1] === undefined ? 0 : Number(match[1]);
|
|
45
|
+
return Math.round((minutes * 60 + Number(match[2])) * 1000) / 1000;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
type SummaryPart =
|
|
49
|
+
| { kind: "path"; path: HyperframesCapturePath }
|
|
50
|
+
| { kind: "gpu"; gpu: HyperframesGpuMode }
|
|
51
|
+
| { kind: "stage"; stage: HyperframesCaptureStage; seconds: number; isStreamed: boolean };
|
|
52
|
+
|
|
53
|
+
function readSummaryPart(part: string): SummaryPart | null {
|
|
54
|
+
const path = /^(\w+) capture$/.exec(part)?.[1];
|
|
55
|
+
if (path !== undefined) return CAPTURE_PATHS.includes(path) ? { kind: "path", path: path as HyperframesCapturePath } : null;
|
|
56
|
+
const gpu = /^(\w+) gpu$/.exec(part)?.[1];
|
|
57
|
+
if (gpu !== undefined) return GPU_MODES.includes(gpu) ? { kind: "gpu", gpu: gpu as HyperframesGpuMode } : null;
|
|
58
|
+
const stage = /^(\w+)( \(during capture\))? (.+)$/.exec(part);
|
|
59
|
+
if (stage === null) return null;
|
|
60
|
+
const [, name = "", streamed, duration = ""] = stage;
|
|
61
|
+
const seconds = readDurationSeconds(duration);
|
|
62
|
+
if (!STAGES.includes(name) || seconds === null) return null;
|
|
63
|
+
return { kind: "stage", stage: name as HyperframesCaptureStage, seconds, isStreamed: streamed !== undefined };
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** A summary line: every ` · ` part is a capture path, a GPU mode or a timed stage. Null for any other line. */
|
|
67
|
+
function readSummaryLine(line: string): Omit<HyperframesCapture, "fallbackHint"> | null {
|
|
68
|
+
const parts = line.split(SEPARATOR).map(readSummaryPart);
|
|
69
|
+
if (parts.length === 0 || parts.some((part) => part === null)) return null;
|
|
70
|
+
const capture: Omit<HyperframesCapture, "fallbackHint"> = { path: null, gpu: null, seconds: {}, isEncodeStreamed: false };
|
|
71
|
+
for (const part of parts) {
|
|
72
|
+
if (part === null) continue;
|
|
73
|
+
if (part.kind === "path") capture.path = part.path;
|
|
74
|
+
else if (part.kind === "gpu") capture.gpu = part.gpu;
|
|
75
|
+
else {
|
|
76
|
+
capture.seconds[part.stage] = part.seconds;
|
|
77
|
+
if (part.stage === "encode" && part.isStreamed) capture.isEncodeStreamed = true;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return capture;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* The capture report from hyperframes' output lines (stdout and stderr, in order): the last summary line, plus the
|
|
85
|
+
* screenshot hint when one was printed. Null when no line is a summary (an older or quiet hyperframes).
|
|
86
|
+
*/
|
|
87
|
+
export function readHyperframesCapture(lines: readonly string[]): HyperframesCapture | null {
|
|
88
|
+
let summary: Omit<HyperframesCapture, "fallbackHint"> | null = null;
|
|
89
|
+
let fallbackHint: string | null = null;
|
|
90
|
+
for (const raw of lines) {
|
|
91
|
+
const line = raw.replace(ANSI_PATTERN, "").trim();
|
|
92
|
+
if (line.length === 0) continue;
|
|
93
|
+
if (line.startsWith(FALLBACK_HINT_PREFIX)) {
|
|
94
|
+
fallbackHint = line;
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
97
|
+
summary = readSummaryLine(line) ?? summary;
|
|
98
|
+
}
|
|
99
|
+
return summary === null ? null : { ...summary, fallbackHint };
|
|
100
|
+
}
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
import { existsSync, mkdirSync } from "node:fs";
|
|
1
|
+
import { existsSync, mkdirSync, rmSync } from "node:fs";
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
import { setTimeout as sleep } from "node:timers/promises";
|
|
4
4
|
|
|
5
|
+
import { describeProviderFailure, type ProviderFailure } from "../voice/provider-failure.js";
|
|
5
6
|
import { err, ok } from "./failure.js";
|
|
6
7
|
import { probeAudio } from "./audio.js";
|
|
8
|
+
import { prepareClipSounds } from "./clip-sound.js";
|
|
7
9
|
import { writeCacheFile } from "./cache.js";
|
|
8
10
|
import type { Edl } from "./edl.js";
|
|
9
11
|
import type { FootageFilm } from "./film.js";
|
|
@@ -14,11 +16,13 @@ import { toPaidAudio, type IsolationProvider, type PaidAudio } from "./providers
|
|
|
14
16
|
import {
|
|
15
17
|
buildRangeAudioArgs,
|
|
16
18
|
buildRangeLayArgs,
|
|
19
|
+
getPlanOptions,
|
|
17
20
|
getSpeechRangeCache,
|
|
18
21
|
getSpeechRangeWindows,
|
|
19
22
|
planSpeechRanges,
|
|
20
23
|
DEFAULT_SPEECH_RANGE_FADE_MS,
|
|
21
24
|
type LaidRange,
|
|
25
|
+
type PlanSpeechRangesOptions,
|
|
22
26
|
type SpeechRange,
|
|
23
27
|
type SpeechRangeCacheResolver,
|
|
24
28
|
} from "./speech-ranges.js";
|
|
@@ -29,10 +33,16 @@ import {
|
|
|
29
33
|
* segment is never paid twice and a changed cut pays only its new ranges. The cleaned ranges are laid back over the
|
|
30
34
|
* clip's own sound into one file as long as the clip: the speech map `renderFootage` takes. A range the provider refuses
|
|
31
35
|
* is asked again, and one refused on every attempt keeps the clip's own sound in its window while the others are laid
|
|
32
|
-
* (issue #1127): only a cancel stops the call.
|
|
36
|
+
* (issue #1127): only a cancel stops the call. The ranges reach the provider's minimum input (`minSeconds`, issue
|
|
37
|
+
* #1140): near ranges of a clip are merged, a short one is widened with the clip's own context and only the part the
|
|
38
|
+
* cut hears is laid; a range that still cannot pass is never sent. A refusal the provider marks not retryable
|
|
39
|
+
* (`isRetryable: false`, e.g. a 400 `audio_too_short`) is not asked again (issue #1141).
|
|
33
40
|
*/
|
|
34
41
|
|
|
35
|
-
/**
|
|
42
|
+
/**
|
|
43
|
+
* How often a refused range is asked again: `attempts` in all (default 3), waiting `delaysMs[n]` before attempt n + 2.
|
|
44
|
+
* A failure whose `providerFailure.isRetryable` is false ends the range at once.
|
|
45
|
+
*/
|
|
36
46
|
export interface SpeechRangeRetry {
|
|
37
47
|
/** Attempts per range, at least 1; default `DEFAULT_SPEECH_RANGE_ATTEMPTS` (3). */
|
|
38
48
|
attempts?: number;
|
|
@@ -48,6 +58,10 @@ export const DEFAULT_SPEECH_RANGE_RETRY_DELAYS_MS: readonly number[] = [1000, 20
|
|
|
48
58
|
export type IsolateSpeechRangesOptions = Pick<FootageRunOptions, "buildDir" | "cacheDir" | "providers" | "isCommit" | "signal" | "onCharge" | "onProgress" | "onFfmpegFailure"> & {
|
|
49
59
|
/** Seconds added on each side of a sound span; default `DEFAULT_SPEECH_RANGE_MARGIN` (0.25). */
|
|
50
60
|
margin?: number;
|
|
61
|
+
/** Ranges of a clip at most this many seconds apart are isolated as one; default 2, `null` keeps them apart. */
|
|
62
|
+
mergeGap?: PlanSpeechRangesOptions["mergeGap"];
|
|
63
|
+
/** Seconds a range shorter than the provider's minimum is widened past it; default `DEFAULT_SPEECH_RANGE_MIN_PAD` (0.4). */
|
|
64
|
+
minPad?: number;
|
|
51
65
|
/** Retries of a refused range. */
|
|
52
66
|
retry?: SpeechRangeRetry;
|
|
53
67
|
/** Milliseconds of the linear crossfade at every join; default `DEFAULT_SPEECH_RANGE_FADE_MS` (15), 0 switches at once. */
|
|
@@ -66,7 +80,14 @@ export type IsolateSpeechRangesOptions = Pick<FootageRunOptions, "buildDir" | "c
|
|
|
66
80
|
export type SpeechRangeOutcome =
|
|
67
81
|
| { kind: "cached" }
|
|
68
82
|
| { kind: "bought"; attempts: number }
|
|
69
|
-
| {
|
|
83
|
+
| {
|
|
84
|
+
kind: "failed";
|
|
85
|
+
error: string;
|
|
86
|
+
attempts: number;
|
|
87
|
+
/** The provider's structured failure of the last attempt (safe to log); null when it gave none. */
|
|
88
|
+
providerFailure: ProviderFailure | null;
|
|
89
|
+
}
|
|
90
|
+
| { kind: "skipped"; reason: "too_short"; minSeconds: number }
|
|
70
91
|
| { kind: "not_bought" };
|
|
71
92
|
|
|
72
93
|
export interface IsolatedSpeechRange extends SpeechRange {
|
|
@@ -79,13 +100,14 @@ export interface IsolatedSpeechRange extends SpeechRange {
|
|
|
79
100
|
laid: boolean;
|
|
80
101
|
/**
|
|
81
102
|
* `cached`; `bought` in this call; `failed` when the provider refused it on every attempt (its window keeps the
|
|
82
|
-
* clip's own sound); `
|
|
103
|
+
* clip's own sound); `skipped` when it is shorter than the provider's minimum even after merging and widening (never
|
|
104
|
+
* sent, its window keeps the clip's own sound); `not_bought` without `isCommit`.
|
|
83
105
|
*/
|
|
84
106
|
outcome: SpeechRangeOutcome;
|
|
85
107
|
}
|
|
86
108
|
|
|
87
109
|
export interface IsolatedSpeech {
|
|
88
|
-
/** Every clip's audio for the speech track: its laid file, or
|
|
110
|
+
/** Every clip's audio for the speech track: its laid file, or its lossless sound when none of its ranges is laid. */
|
|
89
111
|
speech: Record<string, string>;
|
|
90
112
|
ranges: IsolatedSpeechRange[];
|
|
91
113
|
}
|
|
@@ -95,7 +117,8 @@ export interface IsolatedSpeech {
|
|
|
95
117
|
* With `isCommit` every uncached range is bought (`onCharge` step `isolate`, item the clip id, with its `range`);
|
|
96
118
|
* without it only cached ranges are laid, a free draft, and `ranges` names what is still missing. A refused range is
|
|
97
119
|
* retried (`retry`); one refused on every attempt is answered `failed` and the call still succeeds. Each clip with a
|
|
98
|
-
* laid range gets `<buildDir>/audio/<id>.ranges.
|
|
120
|
+
* laid range gets `<buildDir>/audio/<id>.ranges.wav`, crossfaded at every join (`fadeMs`). The isolator hears, and the
|
|
121
|
+
* ranges are laid over, each clip's lossless, pre-gained sound (`prepareClipSounds`, issue #1139).
|
|
99
122
|
*/
|
|
100
123
|
export function isolateSpeechRanges(film: FootageFilm, clips: PreparedClip[], edl: Edl, options: IsolateSpeechRangesOptions): Promise<FootageResult<IsolatedSpeech>> {
|
|
101
124
|
const log = options.log ?? ((): void => undefined);
|
|
@@ -117,7 +140,7 @@ async function runIsolateSpeechRanges(
|
|
|
117
140
|
const planned = planSpeechRanges(
|
|
118
141
|
edl,
|
|
119
142
|
clips.map((clip) => ({ id: clip.clip.id, role: clip.clip.role, duration: clip.duration })),
|
|
120
|
-
options
|
|
143
|
+
getPlanOptions(options, providers.isolation),
|
|
121
144
|
);
|
|
122
145
|
const byClip = new Map(clips.map((clip) => [clip.clip.id, clip]));
|
|
123
146
|
const clipKeys = new Map<string, string>();
|
|
@@ -131,15 +154,27 @@ async function runIsolateSpeechRanges(
|
|
|
131
154
|
clipKeys.set(range.clip, clipKey);
|
|
132
155
|
}
|
|
133
156
|
const cache = options.rangeCache?.(range, clipKey) ?? getSpeechRangeCache(film.id, clipKey, range, { cacheDir: options.cacheDir, provider: providers.isolation.id });
|
|
157
|
+
if (range.skip !== undefined) {
|
|
158
|
+
ranges.push({ ...range, cacheKey: cache.key, path: cache.path, cached: false, laid: false, outcome: { kind: "skipped", ...range.skip } });
|
|
159
|
+
continue;
|
|
160
|
+
}
|
|
134
161
|
const cached = existsSync(cache.path);
|
|
135
162
|
ranges.push({ ...range, cacheKey: cache.key, path: cache.path, cached, laid: cached, outcome: cached ? { kind: "cached" } : { kind: "not_bought" } });
|
|
136
163
|
}
|
|
137
|
-
const
|
|
164
|
+
const skipped = ranges.filter((range) => range.outcome.kind === "skipped");
|
|
165
|
+
if (skipped.length > 0) {
|
|
166
|
+
const minimum = providers.isolation.minSeconds ?? 0;
|
|
167
|
+
log(`voice isolation: ${String(skipped.length)} range${skipped.length === 1 ? "" : "s"} shorter than ${providers.isolation.id}'s minimum of ${String(minimum)} s even with the clip's whole sound, not sent, laid with the clip's own sound: ${skipped.map(describeRange).join("; ")}.`);
|
|
168
|
+
}
|
|
169
|
+
const buying = options.isCommit ? ranges.filter((range) => range.outcome.kind === "not_bought") : [];
|
|
138
170
|
const total = buying.reduce((sum, range) => sum + (range.endMs - range.startMs) / 1000, 0);
|
|
139
171
|
let done = 0;
|
|
140
172
|
const report = (range: SpeechRange, index: number): void => {
|
|
141
173
|
options.onProgress?.({ step: "isolate", item: range.clip, itemIndex: index + 1, itemCount: buying.length, done: Math.round(done * 1000) / 1000, total: Math.round(total * 1000) / 1000 });
|
|
142
174
|
};
|
|
175
|
+
// Every clip's sound, lossless and pre-gained (#1139): what the isolator hears and what the cleaned ranges are laid over.
|
|
176
|
+
const sounds = await prepareClipSounds(clips, { buildDir, ...aborting });
|
|
177
|
+
const getSound = (clip: PreparedClip): string => sounds[clip.clip.id]?.file ?? clip.file;
|
|
143
178
|
const workDir = join(buildDir, "audio", "ranges");
|
|
144
179
|
for (const [index, range] of buying.entries()) {
|
|
145
180
|
const clip = byClip.get(range.clip);
|
|
@@ -148,8 +183,8 @@ async function runIsolateSpeechRanges(
|
|
|
148
183
|
report(range, index);
|
|
149
184
|
mkdirSync(workDir, { recursive: true });
|
|
150
185
|
// The provider sees the clip's id and the range as the file name.
|
|
151
|
-
const input = join(workDir, `${range.clip}.${String(range.startMs)}-${String(range.endMs)}.
|
|
152
|
-
await runFfmpeg(`Cutting ${describeRange(range)} for voice isolation`, buildRangeAudioArgs(clip
|
|
186
|
+
const input = join(workDir, `${range.clip}.${String(range.startMs)}-${String(range.endMs)}.wav`);
|
|
187
|
+
await runFfmpeg(`Cutting ${describeRange(range)} for voice isolation`, buildRangeAudioArgs(getSound(clip), range, input), { ...aborting, output: input });
|
|
153
188
|
signal?.throwIfAborted();
|
|
154
189
|
log(`voice isolation (${providers.isolation.id}): ${describeRange(range)}, ${units.toFixed(1)} s.`);
|
|
155
190
|
const answer = await isolateWithRetries({
|
|
@@ -157,12 +192,14 @@ async function runIsolateSpeechRanges(
|
|
|
157
192
|
isolation: providers.isolation,
|
|
158
193
|
retry: options.retry ?? {},
|
|
159
194
|
...aborting,
|
|
160
|
-
onRefused: (error, attempt) => {
|
|
161
|
-
|
|
195
|
+
onRefused: (error, attempt, providerFailure) => {
|
|
196
|
+
const reason = providerFailure === null ? error : describeProviderFailure(providerFailure);
|
|
197
|
+
log(`voice isolation: ${describeRange(range)} refused (attempt ${String(attempt)}): ${reason}`);
|
|
162
198
|
},
|
|
163
199
|
});
|
|
164
200
|
if (!answer.ok) {
|
|
165
|
-
|
|
201
|
+
rmSync(input, { force: true });
|
|
202
|
+
range.outcome = { kind: "failed", error: answer.error, attempts: answer.attempts, providerFailure: answer.providerFailure };
|
|
166
203
|
done += units;
|
|
167
204
|
report(range, index);
|
|
168
205
|
continue;
|
|
@@ -194,7 +231,7 @@ async function runIsolateSpeechRanges(
|
|
|
194
231
|
const speech: Record<string, string> = {};
|
|
195
232
|
for (const clip of clips) {
|
|
196
233
|
const id = clip.clip.id;
|
|
197
|
-
speech[id] = clip
|
|
234
|
+
speech[id] = getSound(clip);
|
|
198
235
|
const laidRanges = ranges.filter((range) => range.clip === id && range.laid);
|
|
199
236
|
if (laidRanges.length === 0) continue;
|
|
200
237
|
const laid: LaidRange[] = [];
|
|
@@ -205,29 +242,30 @@ async function runIsolateSpeechRanges(
|
|
|
205
242
|
if (!probed.ok) return err(`${describeRange(window.range)}: the cleaned voice at ${file} does not read (${probed.error}); delete it and run again.`);
|
|
206
243
|
laid.push({ window, file, isMono: probed.probe.channels === 1 });
|
|
207
244
|
}
|
|
208
|
-
const output = join(buildDir, "audio", `${id}.ranges.
|
|
245
|
+
const output = join(buildDir, "audio", `${id}.ranges.wav`);
|
|
209
246
|
mkdirSync(join(buildDir, "audio"), { recursive: true });
|
|
210
|
-
await runFfmpeg(`Laying the cleaned voice over clip ${id}`, buildRangeLayArgs(clip
|
|
247
|
+
await runFfmpeg(`Laying the cleaned voice over clip ${id}`, buildRangeLayArgs(getSound(clip), laid, output, { fadeMs: options.fadeMs ?? DEFAULT_SPEECH_RANGE_FADE_MS }), { ...aborting, output });
|
|
211
248
|
speech[id] = output;
|
|
212
249
|
}
|
|
213
250
|
return ok({ speech, ranges });
|
|
214
251
|
}
|
|
215
252
|
|
|
216
|
-
type IsolationAnswer = { ok: true; paid: PaidAudio; attempts: number } | { ok: false; error: string; attempts: number };
|
|
253
|
+
type IsolationAnswer = { ok: true; paid: PaidAudio; attempts: number } | { ok: false; error: string; attempts: number; providerFailure: ProviderFailure | null };
|
|
217
254
|
|
|
218
|
-
/** Asks the provider for one range's cleaned voice, again after each refusal up to `retry.attempts`; a cancel throws. */
|
|
255
|
+
/** Asks the provider for one range's cleaned voice, again after each retryable refusal up to `retry.attempts`; a cancel throws. */
|
|
219
256
|
async function isolateWithRetries(input: {
|
|
220
257
|
audio: string;
|
|
221
258
|
isolation: IsolationProvider;
|
|
222
259
|
retry: SpeechRangeRetry;
|
|
223
260
|
signal?: AbortSignal;
|
|
224
|
-
onRefused: (error: string, attempt: number) => void;
|
|
261
|
+
onRefused: (error: string, attempt: number, providerFailure: ProviderFailure | null) => void;
|
|
225
262
|
}): Promise<IsolationAnswer> {
|
|
226
263
|
const { audio, isolation, retry, signal, onRefused } = input;
|
|
227
264
|
const attempts = Math.max(1, Math.floor(retry.attempts ?? DEFAULT_SPEECH_RANGE_ATTEMPTS));
|
|
228
265
|
const delays = retry.delaysMs ?? DEFAULT_SPEECH_RANGE_RETRY_DELAYS_MS;
|
|
229
266
|
const wait = retry.wait ?? waitFor;
|
|
230
267
|
let error = "";
|
|
268
|
+
let providerFailure: ProviderFailure | null = null;
|
|
231
269
|
for (let attempt = 1; attempt <= attempts; attempt += 1) {
|
|
232
270
|
if (attempt > 1) {
|
|
233
271
|
await wait(delays[Math.min(attempt - 2, delays.length - 1)] ?? 0, signal);
|
|
@@ -237,9 +275,11 @@ async function isolateWithRetries(input: {
|
|
|
237
275
|
signal?.throwIfAborted();
|
|
238
276
|
if (cleaned.ok) return { ok: true, paid: toPaidAudio(cleaned.value), attempts: attempt };
|
|
239
277
|
error = cleaned.error;
|
|
240
|
-
|
|
278
|
+
providerFailure = cleaned.providerFailure ?? null;
|
|
279
|
+
onRefused(error, attempt, providerFailure);
|
|
280
|
+
if (providerFailure?.isRetryable === false) return { ok: false, error, attempts: attempt, providerFailure };
|
|
241
281
|
}
|
|
242
|
-
return { ok: false, error, attempts };
|
|
282
|
+
return { ok: false, error, attempts, providerFailure };
|
|
243
283
|
}
|
|
244
284
|
|
|
245
285
|
/** Resolves after `ms`, or rejects with the signal's reason when it aborts first. */
|
package/src/footage/loudness.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { copyFileSync } from "node:fs";
|
|
|
2
2
|
|
|
3
3
|
import { z } from "zod";
|
|
4
4
|
|
|
5
|
+
import { getAudioCodecArgs } from "./clip-sound.js";
|
|
5
6
|
import { FOOTAGE_SAMPLE_RATE } from "./normalize.js";
|
|
6
7
|
import { runFfmpeg, runProcess } from "./process.js";
|
|
7
8
|
import { FINISHED_AUDIO_BITRATE } from "./publish.js";
|
|
@@ -111,7 +112,7 @@ export function getLoudnessFilter(measurement: LoudnessMeasurement, target: Loud
|
|
|
111
112
|
return `${filter},aresample=${FOOTAGE_SAMPLE_RATE}`;
|
|
112
113
|
}
|
|
113
114
|
|
|
114
|
-
/** The speech track's second pass
|
|
115
|
+
/** The speech track's second pass at the film's rate, encoded by the output's extension (`.wav` lossless float). */
|
|
115
116
|
export function buildLoudnessArgs(options: { input: string; measurement: LoudnessMeasurement; target: LoudnessTarget; output: string }): string[] {
|
|
116
117
|
return [
|
|
117
118
|
"-v",
|
|
@@ -123,10 +124,7 @@ export function buildLoudnessArgs(options: { input: string; measurement: Loudnes
|
|
|
123
124
|
getLoudnessFilter(options.measurement, options.target),
|
|
124
125
|
"-ar",
|
|
125
126
|
String(FOOTAGE_SAMPLE_RATE),
|
|
126
|
-
|
|
127
|
-
"libmp3lame",
|
|
128
|
-
"-q:a",
|
|
129
|
-
"2",
|
|
127
|
+
...getAudioCodecArgs(options.output),
|
|
130
128
|
options.output,
|
|
131
129
|
];
|
|
132
130
|
}
|
|
@@ -196,8 +194,16 @@ export interface MixLoudnessTarget extends LoudnessTarget {
|
|
|
196
194
|
* `target ± warnTolerance`, under the true peak limit, finishes with a warning instead of failing. Absent: none.
|
|
197
195
|
*/
|
|
198
196
|
warnTolerance?: number;
|
|
197
|
+
/**
|
|
198
|
+
* The finished sound's AAC bitrate in kbps (192 to 512), the film's only lossy audio encode (issue #1139). Absent:
|
|
199
|
+
* `FINISHED_AUDIO_BITRATE` (192).
|
|
200
|
+
*/
|
|
201
|
+
audioBitrateKbps?: number;
|
|
199
202
|
}
|
|
200
203
|
|
|
204
|
+
/** The lowest and highest AAC bitrate `MixLoudnessTarget.audioBitrateKbps` takes, in kbps. */
|
|
205
|
+
export const AUDIO_BITRATE_RANGE_KBPS = { min: 192, max: 512 } as const;
|
|
206
|
+
|
|
201
207
|
/** A gain to the window's edge aims this far inside it (at most half the tolerance), so the encode's drift stays in. */
|
|
202
208
|
const MIX_EDGE_MARGIN = 0.2;
|
|
203
209
|
/** The limiter's ceiling sits this far under the true peak limit: AAC adds inter-sample overshoot after it. */
|
|
@@ -367,8 +373,11 @@ export function getMixFilter(attempt: MixAttempt): string {
|
|
|
367
373
|
return [`volume=${attempt.gainDb}dB`, `aresample=${LIMITER_RATE}`, `alimiter=limit=${limit}:level=false:latency=true`, `aresample=${FOOTAGE_SAMPLE_RATE}`, `lowpass=f=${MIX_LOWPASS_HZ}`].join(",");
|
|
368
374
|
}
|
|
369
375
|
|
|
370
|
-
/**
|
|
371
|
-
|
|
376
|
+
/**
|
|
377
|
+
* The mix's sound on its own through `filter`, encoded as the finished film carries it (AAC at `bitrateKbps`, default
|
|
378
|
+
* `FINISHED_AUDIO_BITRATE`, cut at `MIX_LOWPASS_HZ`, 48 kHz, stereo).
|
|
379
|
+
*/
|
|
380
|
+
export function buildMixArgs(options: { input: string; filter: string; output: string; bitrateKbps?: number }): string[] {
|
|
372
381
|
return [
|
|
373
382
|
"-v",
|
|
374
383
|
"error",
|
|
@@ -383,7 +392,7 @@ export function buildMixArgs(options: { input: string; filter: string; output: s
|
|
|
383
392
|
"-c:a",
|
|
384
393
|
"aac",
|
|
385
394
|
"-b:a",
|
|
386
|
-
FINISHED_AUDIO_BITRATE
|
|
395
|
+
options.bitrateKbps === undefined ? FINISHED_AUDIO_BITRATE : `${String(options.bitrateKbps)}k`,
|
|
387
396
|
"-cutoff",
|
|
388
397
|
String(MIX_LOWPASS_HZ),
|
|
389
398
|
"-ar",
|
package/src/footage/paid.ts
CHANGED
|
@@ -10,7 +10,15 @@ import { getFilmKeyTerms } from "./keyterms.js";
|
|
|
10
10
|
import type { FootageClip, FootageFilm } from "./film.js";
|
|
11
11
|
import { getTrimmedDuration, probeFootage } from "./probe.js";
|
|
12
12
|
import type { FootageProviders, FootageRates, PaidStepKind } from "./providers.js";
|
|
13
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
getPlanOptions,
|
|
15
|
+
getSpeechRangeCache,
|
|
16
|
+
planSpeechRanges,
|
|
17
|
+
type PlanSpeechRangesOptions,
|
|
18
|
+
type SpeechRangeCacheResolver,
|
|
19
|
+
type SpeechRangeClip,
|
|
20
|
+
type SpeechRangeSkip,
|
|
21
|
+
} from "./speech-ranges.js";
|
|
14
22
|
|
|
15
23
|
/**
|
|
16
24
|
* Where each paid result of a footage film is cached, shared by the pipeline and by the questions an app asks before
|
|
@@ -161,6 +169,11 @@ export interface FootageEstimateItem {
|
|
|
161
169
|
cached: boolean;
|
|
162
170
|
/** The range of the clip an `isolate: "segments"` item prices, in seconds of the trimmed clip; absent: the whole clip. */
|
|
163
171
|
range?: { start: number; end: number };
|
|
172
|
+
/**
|
|
173
|
+
* Set on an `isolate: "segments"` range that will not be sent: shorter than the provider's minimum even after
|
|
174
|
+
* merging and widening (`units` and `credits` 0).
|
|
175
|
+
*/
|
|
176
|
+
skipped?: SpeechRangeSkip;
|
|
164
177
|
}
|
|
165
178
|
|
|
166
179
|
export interface FootageEstimate {
|
|
@@ -179,6 +192,10 @@ export interface EstimateFootageOptions extends FootageCacheOptions {
|
|
|
179
192
|
isolate?: "clips" | "segments";
|
|
180
193
|
/** Seconds around each sound span with `isolate: "segments"`; default `DEFAULT_SPEECH_RANGE_MARGIN`. */
|
|
181
194
|
margin?: number;
|
|
195
|
+
/** Ranges of a clip at most this many seconds apart are priced as one with `isolate: "segments"`, as `isolateSpeechRanges` merges them. */
|
|
196
|
+
mergeGap?: PlanSpeechRangesOptions["mergeGap"];
|
|
197
|
+
/** Seconds a range shorter than the provider's minimum is widened past it with `isolate: "segments"`. */
|
|
198
|
+
minPad?: number;
|
|
182
199
|
/** Where a range's cleaned voice is cached with `isolate: "segments"`, as `isolateSpeechRanges` takes it; default `getSpeechRangeCache`. */
|
|
183
200
|
rangeCache?: SpeechRangeCacheResolver;
|
|
184
201
|
}
|
|
@@ -234,7 +251,11 @@ export async function estimateFootageFilm(film: FootageFilm, options: EstimateFo
|
|
|
234
251
|
}
|
|
235
252
|
if (isSegments && edl !== null) {
|
|
236
253
|
const provider = options.providers.isolation.id;
|
|
237
|
-
for (const range of planSpeechRanges(edl, rangeClips, options
|
|
254
|
+
for (const range of planSpeechRanges(edl, rangeClips, getPlanOptions(options, options.providers.isolation))) {
|
|
255
|
+
if (range.skip !== undefined) {
|
|
256
|
+
items.push({ step: "isolate", item: range.clip, provider, units: 0, unit: "seconds", credits: 0, cached: false, range: { start: range.startMs / 1000, end: range.endMs / 1000 }, skipped: range.skip });
|
|
257
|
+
continue;
|
|
258
|
+
}
|
|
238
259
|
const clipKey = clipKeys[range.clip] ?? null;
|
|
239
260
|
const cache = clipKey === null ? null : (options.rangeCache?.(range, clipKey) ?? getSpeechRangeCache(film.id, clipKey, range, { cacheDir: options.cacheDir, provider }));
|
|
240
261
|
const cached = cache !== null && existsSync(cache.path);
|
|
@@ -253,7 +274,8 @@ export async function estimateFootageFilm(film: FootageFilm, options: EstimateFo
|
|
|
253
274
|
|
|
254
275
|
/**
|
|
255
276
|
* What isolating only the cut's ranges will cost (`isolateSpeechRanges`), for a confirmation step: the `isolate` items
|
|
256
|
-
* of `estimateFootageFilm` with `isolate: "segments"`, one per range, a cached range at 0
|
|
277
|
+
* of `estimateFootageFilm` with `isolate: "segments"`, one per range as it will be sent, a cached range at 0, a range
|
|
278
|
+
* shorter than the provider's minimum marked `skipped` at 0.
|
|
257
279
|
*/
|
|
258
280
|
export async function estimateSpeechRanges(film: FootageFilm, options: Omit<EstimateFootageOptions, "isolate">): Promise<TtsResult<FootageEstimate>> {
|
|
259
281
|
const estimate = await estimateFootageFilm(film, { ...options, isolate: "segments" });
|