@softure-ai/marketing-kit 0.15.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +91 -0
- package/README.md +138 -16
- package/dist/config/schema.d.ts +2 -1
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +7 -0
- package/dist/config/schema.js.map +1 -1
- package/dist/footage/clip-sound.d.ts +58 -0
- package/dist/footage/clip-sound.d.ts.map +1 -0
- package/dist/footage/clip-sound.js +103 -0
- package/dist/footage/clip-sound.js.map +1 -0
- package/dist/footage/declick.d.ts +1 -1
- package/dist/footage/declick.d.ts.map +1 -1
- package/dist/footage/declick.js +3 -4
- package/dist/footage/declick.js.map +1 -1
- package/dist/footage/draft.d.ts +19 -5
- package/dist/footage/draft.d.ts.map +1 -1
- package/dist/footage/draft.js +92 -22
- package/dist/footage/draft.js.map +1 -1
- package/dist/footage/elevenlabs.d.ts +5 -0
- package/dist/footage/elevenlabs.d.ts.map +1 -1
- package/dist/footage/elevenlabs.js +10 -3
- package/dist/footage/elevenlabs.js.map +1 -1
- package/dist/footage/failure.d.ts +12 -1
- package/dist/footage/failure.d.ts.map +1 -1
- package/dist/footage/failure.js +10 -1
- package/dist/footage/failure.js.map +1 -1
- package/dist/footage/fake.d.ts +10 -1
- package/dist/footage/fake.d.ts.map +1 -1
- package/dist/footage/fake.js +19 -4
- package/dist/footage/fake.js.map +1 -1
- package/dist/footage/hyperframes-capture.d.ts +30 -0
- package/dist/footage/hyperframes-capture.d.ts.map +1 -0
- package/dist/footage/hyperframes-capture.js +80 -0
- package/dist/footage/hyperframes-capture.js.map +1 -0
- package/dist/footage/isolate-ranges.d.ts +34 -6
- package/dist/footage/isolate-ranges.d.ts.map +1 -1
- package/dist/footage/isolate-ranges.js +90 -23
- package/dist/footage/isolate-ranges.js.map +1 -1
- package/dist/footage/loudness.d.ts +16 -2
- package/dist/footage/loudness.d.ts.map +1 -1
- package/dist/footage/loudness.js +10 -7
- package/dist/footage/loudness.js.map +1 -1
- package/dist/footage/paid.d.ts +12 -2
- package/dist/footage/paid.d.ts.map +1 -1
- package/dist/footage/paid.js +8 -3
- package/dist/footage/paid.js.map +1 -1
- package/dist/footage/pipeline.d.ts +14 -2
- package/dist/footage/pipeline.d.ts.map +1 -1
- package/dist/footage/pipeline.js +55 -46
- package/dist/footage/pipeline.js.map +1 -1
- package/dist/footage/providers.d.ts +5 -0
- package/dist/footage/providers.d.ts.map +1 -1
- package/dist/footage/providers.js.map +1 -1
- package/dist/footage/render.d.ts +1 -1
- package/dist/footage/render.d.ts.map +1 -1
- package/dist/footage/render.js +4 -3
- package/dist/footage/render.js.map +1 -1
- package/dist/footage/speech-ranges.d.ts +79 -11
- package/dist/footage/speech-ranges.d.ts.map +1 -1
- package/dist/footage/speech-ranges.js +137 -17
- package/dist/footage/speech-ranges.js.map +1 -1
- package/dist/index.d.ts +9 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -5
- package/dist/index.js.map +1 -1
- package/dist/og/index.d.ts +2 -0
- package/dist/og/index.d.ts.map +1 -1
- package/dist/og/index.js +2 -0
- package/dist/og/index.js.map +1 -1
- package/dist/og/photo-cover.d.ts +74 -0
- package/dist/og/photo-cover.d.ts.map +1 -0
- package/dist/og/photo-cover.js +357 -0
- package/dist/og/photo-cover.js.map +1 -0
- package/dist/og/photo.d.ts +51 -0
- package/dist/og/photo.d.ts.map +1 -0
- package/dist/og/photo.js +154 -0
- package/dist/og/photo.js.map +1 -0
- package/dist/voice/provider-failure.d.ts +41 -0
- package/dist/voice/provider-failure.d.ts.map +1 -0
- package/dist/voice/provider-failure.js +68 -0
- package/dist/voice/provider-failure.js.map +1 -0
- package/dist/voice/provider.d.ts +7 -1
- package/dist/voice/provider.d.ts.map +1 -1
- package/dist/voice/provider.js +1 -1
- package/dist/voice/provider.js.map +1 -1
- package/package.json +2 -1
- package/schema/marketing.schema.json +6 -0
- package/src/config/schema.ts +7 -0
- package/src/footage/clip-sound.ts +124 -0
- package/src/footage/declick.ts +3 -4
- package/src/footage/draft.ts +123 -22
- package/src/footage/elevenlabs.ts +11 -3
- package/src/footage/failure.ts +14 -1
- package/src/footage/fake.ts +28 -4
- package/src/footage/hyperframes-capture.ts +100 -0
- package/src/footage/isolate-ranges.ts +124 -29
- package/src/footage/loudness.ts +17 -8
- package/src/footage/paid.ts +25 -3
- package/src/footage/pipeline.ts +68 -47
- package/src/footage/providers.ts +5 -0
- package/src/footage/render.ts +4 -3
- package/src/footage/speech-ranges.ts +184 -19
- package/src/index.ts +49 -1
- package/src/og/index.ts +28 -0
- package/src/og/photo-cover.ts +442 -0
- package/src/og/photo.ts +177 -0
- package/src/voice/provider-failure.ts +98 -0
- package/src/voice/provider.ts +7 -2
|
@@ -1,24 +1,34 @@
|
|
|
1
|
-
import { existsSync, mkdirSync } from "node:fs";
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, rmSync } from "node:fs";
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
import { setTimeout as sleep } from "node:timers/promises";
|
|
4
4
|
|
|
5
|
+
import { describeProviderFailure, type ProviderFailure } from "../voice/provider-failure.js";
|
|
5
6
|
import { err, ok } from "./failure.js";
|
|
6
7
|
import { probeAudio } from "./audio.js";
|
|
8
|
+
import { prepareClipSounds, type ClipSound } from "./clip-sound.js";
|
|
9
|
+
import { isSilent, readLoudnessMeasurement } from "./loudness.js";
|
|
7
10
|
import { writeCacheFile } from "./cache.js";
|
|
8
11
|
import type { Edl } from "./edl.js";
|
|
9
12
|
import type { FootageFilm } from "./film.js";
|
|
10
13
|
import { getClipCacheKey } from "./paid.js";
|
|
11
14
|
import { getFfmpegReporting, type FootageResult, type FootageRunOptions, type PreparedClip } from "./pipeline.js";
|
|
12
|
-
import { runFfmpeg, withFfmpegFailureReporting } from "./process.js";
|
|
15
|
+
import { runFfmpeg, runProcess, withFfmpegFailureReporting } from "./process.js";
|
|
13
16
|
import { toPaidAudio, type IsolationProvider, type PaidAudio } from "./providers.js";
|
|
14
17
|
import {
|
|
15
18
|
buildRangeAudioArgs,
|
|
16
19
|
buildRangeLayArgs,
|
|
20
|
+
buildRangeMeasureArgs,
|
|
21
|
+
formatSpeechRangeGain,
|
|
22
|
+
getPlanOptions,
|
|
23
|
+
getRangeInputGainDb,
|
|
24
|
+
getSpeechRangeGainPath,
|
|
17
25
|
getSpeechRangeCache,
|
|
18
26
|
getSpeechRangeWindows,
|
|
19
27
|
planSpeechRanges,
|
|
28
|
+
readSpeechRangeGain,
|
|
20
29
|
DEFAULT_SPEECH_RANGE_FADE_MS,
|
|
21
30
|
type LaidRange,
|
|
31
|
+
type PlanSpeechRangesOptions,
|
|
22
32
|
type SpeechRange,
|
|
23
33
|
type SpeechRangeCacheResolver,
|
|
24
34
|
} from "./speech-ranges.js";
|
|
@@ -29,10 +39,18 @@ import {
|
|
|
29
39
|
* segment is never paid twice and a changed cut pays only its new ranges. The cleaned ranges are laid back over the
|
|
30
40
|
* clip's own sound into one file as long as the clip: the speech map `renderFootage` takes. A range the provider refuses
|
|
31
41
|
* is asked again, and one refused on every attempt keeps the clip's own sound in its window while the others are laid
|
|
32
|
-
* (issue #1127): only a cancel stops the call.
|
|
42
|
+
* (issue #1127): only a cancel stops the call. The ranges reach the provider's minimum input (`minSeconds`, issue
|
|
43
|
+
* #1140): near ranges of a clip are merged, a short one is widened with the clip's own context and only the part the
|
|
44
|
+
* cut hears is laid; a range that still cannot pass is never sent. A refusal the provider marks not retryable
|
|
45
|
+
* (`isRetryable: false`, e.g. a 400 `audio_too_short`) is not asked again (issue #1141). Every window lands at the
|
|
46
|
+
* level of the clip's pre-gained sound (issue #1159): a bought range records the gain it was cut at next to its cached
|
|
47
|
+
* file, and a cached range cut from the un-gained sound (bought before the pre-gain) is raised by the clip's pre-gain.
|
|
33
48
|
*/
|
|
34
49
|
|
|
35
|
-
/**
|
|
50
|
+
/**
|
|
51
|
+
* How often a refused range is asked again: `attempts` in all (default 3), waiting `delaysMs[n]` before attempt n + 2.
|
|
52
|
+
* A failure whose `providerFailure.isRetryable` is false ends the range at once.
|
|
53
|
+
*/
|
|
36
54
|
export interface SpeechRangeRetry {
|
|
37
55
|
/** Attempts per range, at least 1; default `DEFAULT_SPEECH_RANGE_ATTEMPTS` (3). */
|
|
38
56
|
attempts?: number;
|
|
@@ -48,6 +66,10 @@ export const DEFAULT_SPEECH_RANGE_RETRY_DELAYS_MS: readonly number[] = [1000, 20
|
|
|
48
66
|
export type IsolateSpeechRangesOptions = Pick<FootageRunOptions, "buildDir" | "cacheDir" | "providers" | "isCommit" | "signal" | "onCharge" | "onProgress" | "onFfmpegFailure"> & {
|
|
49
67
|
/** Seconds added on each side of a sound span; default `DEFAULT_SPEECH_RANGE_MARGIN` (0.25). */
|
|
50
68
|
margin?: number;
|
|
69
|
+
/** Ranges of a clip at most this many seconds apart are isolated as one; default 2, `null` keeps them apart. */
|
|
70
|
+
mergeGap?: PlanSpeechRangesOptions["mergeGap"];
|
|
71
|
+
/** Seconds a range shorter than the provider's minimum is widened past it; default `DEFAULT_SPEECH_RANGE_MIN_PAD` (0.4). */
|
|
72
|
+
minPad?: number;
|
|
51
73
|
/** Retries of a refused range. */
|
|
52
74
|
retry?: SpeechRangeRetry;
|
|
53
75
|
/** Milliseconds of the linear crossfade at every join; default `DEFAULT_SPEECH_RANGE_FADE_MS` (15), 0 switches at once. */
|
|
@@ -66,7 +88,14 @@ export type IsolateSpeechRangesOptions = Pick<FootageRunOptions, "buildDir" | "c
|
|
|
66
88
|
export type SpeechRangeOutcome =
|
|
67
89
|
| { kind: "cached" }
|
|
68
90
|
| { kind: "bought"; attempts: number }
|
|
69
|
-
| {
|
|
91
|
+
| {
|
|
92
|
+
kind: "failed";
|
|
93
|
+
error: string;
|
|
94
|
+
attempts: number;
|
|
95
|
+
/** The provider's structured failure of the last attempt (safe to log); null when it gave none. */
|
|
96
|
+
providerFailure: ProviderFailure | null;
|
|
97
|
+
}
|
|
98
|
+
| { kind: "skipped"; reason: "too_short"; minSeconds: number }
|
|
70
99
|
| { kind: "not_bought" };
|
|
71
100
|
|
|
72
101
|
export interface IsolatedSpeechRange extends SpeechRange {
|
|
@@ -79,14 +108,22 @@ export interface IsolatedSpeechRange extends SpeechRange {
|
|
|
79
108
|
laid: boolean;
|
|
80
109
|
/**
|
|
81
110
|
* `cached`; `bought` in this call; `failed` when the provider refused it on every attempt (its window keeps the
|
|
82
|
-
* clip's own sound); `
|
|
111
|
+
* clip's own sound); `skipped` when it is shorter than the provider's minimum even after merging and widening (never
|
|
112
|
+
* sent, its window keeps the clip's own sound); `not_bought` without `isCommit`.
|
|
83
113
|
*/
|
|
84
114
|
outcome: SpeechRangeOutcome;
|
|
115
|
+
/**
|
|
116
|
+
* The dB its cleaned voice was raised by when laid (issue #1159): the clip's pre-gain for a range cut from the
|
|
117
|
+
* clip's un-gained sound, else 0; 0 when it is not laid.
|
|
118
|
+
*/
|
|
119
|
+
gainDb: number;
|
|
85
120
|
}
|
|
86
121
|
|
|
87
122
|
export interface IsolatedSpeech {
|
|
88
|
-
/** Every clip's audio for the speech track: its laid file, or
|
|
123
|
+
/** Every clip's audio for the speech track: its laid file, or its lossless sound when none of its ranges is laid. */
|
|
89
124
|
speech: Record<string, string>;
|
|
125
|
+
/** Each clip's pre-gain in dB (`prepareClipSounds`): the gain of the sound its speech file is made from (issue #1159). */
|
|
126
|
+
speechGainDb: Record<string, number>;
|
|
90
127
|
ranges: IsolatedSpeechRange[];
|
|
91
128
|
}
|
|
92
129
|
|
|
@@ -95,7 +132,8 @@ export interface IsolatedSpeech {
|
|
|
95
132
|
* With `isCommit` every uncached range is bought (`onCharge` step `isolate`, item the clip id, with its `range`);
|
|
96
133
|
* without it only cached ranges are laid, a free draft, and `ranges` names what is still missing. A refused range is
|
|
97
134
|
* retried (`retry`); one refused on every attempt is answered `failed` and the call still succeeds. Each clip with a
|
|
98
|
-
* laid range gets `<buildDir>/audio/<id>.ranges.
|
|
135
|
+
* laid range gets `<buildDir>/audio/<id>.ranges.wav`, crossfaded at every join (`fadeMs`). The isolator hears, and the
|
|
136
|
+
* ranges are laid over, each clip's lossless, pre-gained sound (`prepareClipSounds`, issue #1139).
|
|
99
137
|
*/
|
|
100
138
|
export function isolateSpeechRanges(film: FootageFilm, clips: PreparedClip[], edl: Edl, options: IsolateSpeechRangesOptions): Promise<FootageResult<IsolatedSpeech>> {
|
|
101
139
|
const log = options.log ?? ((): void => undefined);
|
|
@@ -117,7 +155,7 @@ async function runIsolateSpeechRanges(
|
|
|
117
155
|
const planned = planSpeechRanges(
|
|
118
156
|
edl,
|
|
119
157
|
clips.map((clip) => ({ id: clip.clip.id, role: clip.clip.role, duration: clip.duration })),
|
|
120
|
-
options
|
|
158
|
+
getPlanOptions(options, providers.isolation),
|
|
121
159
|
);
|
|
122
160
|
const byClip = new Map(clips.map((clip) => [clip.clip.id, clip]));
|
|
123
161
|
const clipKeys = new Map<string, string>();
|
|
@@ -131,15 +169,27 @@ async function runIsolateSpeechRanges(
|
|
|
131
169
|
clipKeys.set(range.clip, clipKey);
|
|
132
170
|
}
|
|
133
171
|
const cache = options.rangeCache?.(range, clipKey) ?? getSpeechRangeCache(film.id, clipKey, range, { cacheDir: options.cacheDir, provider: providers.isolation.id });
|
|
172
|
+
if (range.skip !== undefined) {
|
|
173
|
+
ranges.push({ ...range, cacheKey: cache.key, path: cache.path, cached: false, laid: false, outcome: { kind: "skipped", ...range.skip }, gainDb: 0 });
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
134
176
|
const cached = existsSync(cache.path);
|
|
135
|
-
ranges.push({ ...range, cacheKey: cache.key, path: cache.path, cached, laid: cached, outcome: cached ? { kind: "cached" } : { kind: "not_bought" } });
|
|
177
|
+
ranges.push({ ...range, cacheKey: cache.key, path: cache.path, cached, laid: cached, outcome: cached ? { kind: "cached" } : { kind: "not_bought" }, gainDb: 0 });
|
|
178
|
+
}
|
|
179
|
+
const skipped = ranges.filter((range) => range.outcome.kind === "skipped");
|
|
180
|
+
if (skipped.length > 0) {
|
|
181
|
+
const minimum = providers.isolation.minSeconds ?? 0;
|
|
182
|
+
log(`voice isolation: ${String(skipped.length)} range${skipped.length === 1 ? "" : "s"} shorter than ${providers.isolation.id}'s minimum of ${String(minimum)} s even with the clip's whole sound, not sent, laid with the clip's own sound: ${skipped.map(describeRange).join("; ")}.`);
|
|
136
183
|
}
|
|
137
|
-
const buying = options.isCommit ? ranges.filter((range) =>
|
|
184
|
+
const buying = options.isCommit ? ranges.filter((range) => range.outcome.kind === "not_bought") : [];
|
|
138
185
|
const total = buying.reduce((sum, range) => sum + (range.endMs - range.startMs) / 1000, 0);
|
|
139
186
|
let done = 0;
|
|
140
187
|
const report = (range: SpeechRange, index: number): void => {
|
|
141
188
|
options.onProgress?.({ step: "isolate", item: range.clip, itemIndex: index + 1, itemCount: buying.length, done: Math.round(done * 1000) / 1000, total: Math.round(total * 1000) / 1000 });
|
|
142
189
|
};
|
|
190
|
+
// Every clip's sound, lossless and pre-gained (#1139): what the isolator hears and what the cleaned ranges are laid over.
|
|
191
|
+
const sounds = await prepareClipSounds(clips, { buildDir, ...aborting });
|
|
192
|
+
const getSound = (clip: PreparedClip): string => sounds[clip.clip.id]?.file ?? clip.file;
|
|
143
193
|
const workDir = join(buildDir, "audio", "ranges");
|
|
144
194
|
for (const [index, range] of buying.entries()) {
|
|
145
195
|
const clip = byClip.get(range.clip);
|
|
@@ -148,8 +198,8 @@ async function runIsolateSpeechRanges(
|
|
|
148
198
|
report(range, index);
|
|
149
199
|
mkdirSync(workDir, { recursive: true });
|
|
150
200
|
// The provider sees the clip's id and the range as the file name.
|
|
151
|
-
const input = join(workDir, `${range.clip}.${String(range.startMs)}-${String(range.endMs)}.
|
|
152
|
-
await runFfmpeg(`Cutting ${describeRange(range)} for voice isolation`, buildRangeAudioArgs(clip
|
|
201
|
+
const input = join(workDir, `${range.clip}.${String(range.startMs)}-${String(range.endMs)}.wav`);
|
|
202
|
+
await runFfmpeg(`Cutting ${describeRange(range)} for voice isolation`, buildRangeAudioArgs(getSound(clip), range, input), { ...aborting, output: input });
|
|
153
203
|
signal?.throwIfAborted();
|
|
154
204
|
log(`voice isolation (${providers.isolation.id}): ${describeRange(range)}, ${units.toFixed(1)} s.`);
|
|
155
205
|
const answer = await isolateWithRetries({
|
|
@@ -157,18 +207,22 @@ async function runIsolateSpeechRanges(
|
|
|
157
207
|
isolation: providers.isolation,
|
|
158
208
|
retry: options.retry ?? {},
|
|
159
209
|
...aborting,
|
|
160
|
-
onRefused: (error, attempt) => {
|
|
161
|
-
|
|
210
|
+
onRefused: (error, attempt, providerFailure) => {
|
|
211
|
+
const reason = providerFailure === null ? error : describeProviderFailure(providerFailure);
|
|
212
|
+
log(`voice isolation: ${describeRange(range)} refused (attempt ${String(attempt)}): ${reason}`);
|
|
162
213
|
},
|
|
163
214
|
});
|
|
164
215
|
if (!answer.ok) {
|
|
165
|
-
|
|
216
|
+
rmSync(input, { force: true });
|
|
217
|
+
range.outcome = { kind: "failed", error: answer.error, attempts: answer.attempts, providerFailure: answer.providerFailure };
|
|
166
218
|
done += units;
|
|
167
219
|
report(range, index);
|
|
168
220
|
continue;
|
|
169
221
|
}
|
|
170
222
|
const paid = answer.paid;
|
|
171
223
|
writeCacheFile(range.path, paid.audio);
|
|
224
|
+
// The gain of the sound it was cut from, so a later call lays it at the level of the clip's sound then (#1159).
|
|
225
|
+
writeCacheFile(getSpeechRangeGainPath(range.path), formatSpeechRangeGain(sounds[clip.clip.id]?.gainDb ?? 0));
|
|
172
226
|
range.laid = true;
|
|
173
227
|
range.outcome = { kind: "bought", attempts: answer.attempts };
|
|
174
228
|
options.onCharge?.({
|
|
@@ -192,42 +246,81 @@ async function runIsolateSpeechRanges(
|
|
|
192
246
|
log(`voice isolation DRY RUN: ${String(missing.length)} range${missing.length === 1 ? "" : "s"} (${seconds.toFixed(1)} s) not cached, laid with the clip's own sound. Run with --commit to pay.`);
|
|
193
247
|
}
|
|
194
248
|
const speech: Record<string, string> = {};
|
|
249
|
+
const speechGainDb: Record<string, number> = {};
|
|
195
250
|
for (const clip of clips) {
|
|
196
251
|
const id = clip.clip.id;
|
|
197
|
-
speech[id] = clip
|
|
252
|
+
speech[id] = getSound(clip);
|
|
253
|
+
const clipGainDb = sounds[id]?.gainDb ?? 0;
|
|
254
|
+
speechGainDb[id] = clipGainDb;
|
|
198
255
|
const laidRanges = ranges.filter((range) => range.clip === id && range.laid);
|
|
199
256
|
if (laidRanges.length === 0) continue;
|
|
200
257
|
const laid: LaidRange[] = [];
|
|
201
258
|
for (const window of getSpeechRangeWindows(laidRanges)) {
|
|
202
|
-
const
|
|
203
|
-
if (
|
|
259
|
+
const range = laidRanges.find((candidate) => candidate === window.range);
|
|
260
|
+
if (range === undefined) continue;
|
|
261
|
+
const file = range.path;
|
|
204
262
|
const probed = await probeAudio(file, aborting);
|
|
205
|
-
if (!probed.ok) return err(`${describeRange(
|
|
206
|
-
|
|
263
|
+
if (!probed.ok) return err(`${describeRange(range)}: the cleaned voice at ${file} does not read (${probed.error}); delete it and run again.`);
|
|
264
|
+
const inputGainDb = await getCachedInputGainDb(range, { sound: sounds[id], file, ...aborting });
|
|
265
|
+
const gainDb = Math.round((clipGainDb - inputGainDb) * 100) / 100 || 0;
|
|
266
|
+
range.gainDb = gainDb;
|
|
267
|
+
if (gainDb !== 0) log(`voice isolation: ${describeRange(range)} was cut from the clip's sound at ${String(inputGainDb)} dB, laid over it at ${String(clipGainDb)} dB: raised by ${String(gainDb)} dB.`);
|
|
268
|
+
laid.push({ window, file, isMono: probed.probe.channels === 1, gainDb });
|
|
207
269
|
}
|
|
208
|
-
const output = join(buildDir, "audio", `${id}.ranges.
|
|
270
|
+
const output = join(buildDir, "audio", `${id}.ranges.wav`);
|
|
209
271
|
mkdirSync(join(buildDir, "audio"), { recursive: true });
|
|
210
|
-
await runFfmpeg(`Laying the cleaned voice over clip ${id}`, buildRangeLayArgs(clip
|
|
272
|
+
await runFfmpeg(`Laying the cleaned voice over clip ${id}`, buildRangeLayArgs(getSound(clip), laid, output, { fadeMs: options.fadeMs ?? DEFAULT_SPEECH_RANGE_FADE_MS }), { ...aborting, output });
|
|
211
273
|
speech[id] = output;
|
|
212
274
|
}
|
|
213
|
-
return ok({ speech, ranges });
|
|
275
|
+
return ok({ speech, speechGainDb, ranges });
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* The gain of the sound a laid range's cleaned voice was cut from (issue #1159): the clip's pre-gain when it was bought
|
|
280
|
+
* in this call; its record (`getSpeechRangeGainPath`) when there is one; else told from its loudness against the same
|
|
281
|
+
* range of the clip's pre-gained sound (`getRangeInputGainDb`), for a range cached before the record existed.
|
|
282
|
+
*/
|
|
283
|
+
async function getCachedInputGainDb(range: IsolatedSpeechRange, input: { sound: ClipSound | undefined; file: string; signal?: AbortSignal }): Promise<number> {
|
|
284
|
+
const clipGainDb = input.sound?.gainDb ?? 0;
|
|
285
|
+
if (range.outcome.kind === "bought" || input.sound === undefined) return clipGainDb;
|
|
286
|
+
const record = getSpeechRangeGainPath(input.file);
|
|
287
|
+
if (existsSync(record)) {
|
|
288
|
+
const recorded = readSpeechRangeGain(readFileSync(record, "utf8"));
|
|
289
|
+
if (recorded !== null) return recorded;
|
|
290
|
+
}
|
|
291
|
+
if (clipGainDb === 0) return 0;
|
|
292
|
+
const cleanedLufs = await measureRangeLoudness(`the cleaned voice of ${describeRange(range)}`, buildRangeMeasureArgs(input.file), input.signal);
|
|
293
|
+
const baseLufs = await measureRangeLoudness(`${describeRange(range)} of the clip's sound`, buildRangeMeasureArgs(input.sound.file, range), input.signal);
|
|
294
|
+
return getRangeInputGainDb({ clipGainDb, cleanedLufs, baseLufs });
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/** Integrated loudness in LUFS of a measuring pass (`buildRangeMeasureArgs`); null for silence. Throws naming `label` when ffmpeg fails. */
|
|
298
|
+
async function measureRangeLoudness(label: string, args: string[], signal: AbortSignal | undefined): Promise<number | null> {
|
|
299
|
+
const step = `Measuring ${label}`;
|
|
300
|
+
const measured = await runProcess({ command: "ffmpeg", args, ...(signal === undefined ? {} : { signal }) });
|
|
301
|
+
if (measured.kind === "not-started") throw new Error(`${step}: ffmpeg did not start (${measured.error}).`);
|
|
302
|
+
if (measured.code !== 0) throw new Error(`${step}: ffmpeg ended with code ${String(measured.code)}: ${measured.stderr.trim().split("\n").slice(-3).join(" | ")}.`);
|
|
303
|
+
const measurement = readLoudnessMeasurement(measured.stderr);
|
|
304
|
+
if (measurement === null) throw new Error(`${step}: ffmpeg printed no loudnorm measurement.`);
|
|
305
|
+
return isSilent(measurement) ? null : measurement.inputI;
|
|
214
306
|
}
|
|
215
307
|
|
|
216
|
-
type IsolationAnswer = { ok: true; paid: PaidAudio; attempts: number } | { ok: false; error: string; attempts: number };
|
|
308
|
+
type IsolationAnswer = { ok: true; paid: PaidAudio; attempts: number } | { ok: false; error: string; attempts: number; providerFailure: ProviderFailure | null };
|
|
217
309
|
|
|
218
|
-
/** Asks the provider for one range's cleaned voice, again after each refusal up to `retry.attempts`; a cancel throws. */
|
|
310
|
+
/** Asks the provider for one range's cleaned voice, again after each retryable refusal up to `retry.attempts`; a cancel throws. */
|
|
219
311
|
async function isolateWithRetries(input: {
|
|
220
312
|
audio: string;
|
|
221
313
|
isolation: IsolationProvider;
|
|
222
314
|
retry: SpeechRangeRetry;
|
|
223
315
|
signal?: AbortSignal;
|
|
224
|
-
onRefused: (error: string, attempt: number) => void;
|
|
316
|
+
onRefused: (error: string, attempt: number, providerFailure: ProviderFailure | null) => void;
|
|
225
317
|
}): Promise<IsolationAnswer> {
|
|
226
318
|
const { audio, isolation, retry, signal, onRefused } = input;
|
|
227
319
|
const attempts = Math.max(1, Math.floor(retry.attempts ?? DEFAULT_SPEECH_RANGE_ATTEMPTS));
|
|
228
320
|
const delays = retry.delaysMs ?? DEFAULT_SPEECH_RANGE_RETRY_DELAYS_MS;
|
|
229
321
|
const wait = retry.wait ?? waitFor;
|
|
230
322
|
let error = "";
|
|
323
|
+
let providerFailure: ProviderFailure | null = null;
|
|
231
324
|
for (let attempt = 1; attempt <= attempts; attempt += 1) {
|
|
232
325
|
if (attempt > 1) {
|
|
233
326
|
await wait(delays[Math.min(attempt - 2, delays.length - 1)] ?? 0, signal);
|
|
@@ -237,9 +330,11 @@ async function isolateWithRetries(input: {
|
|
|
237
330
|
signal?.throwIfAborted();
|
|
238
331
|
if (cleaned.ok) return { ok: true, paid: toPaidAudio(cleaned.value), attempts: attempt };
|
|
239
332
|
error = cleaned.error;
|
|
240
|
-
|
|
333
|
+
providerFailure = cleaned.providerFailure ?? null;
|
|
334
|
+
onRefused(error, attempt, providerFailure);
|
|
335
|
+
if (providerFailure?.isRetryable === false) return { ok: false, error, attempts: attempt, providerFailure };
|
|
241
336
|
}
|
|
242
|
-
return { ok: false, error, attempts };
|
|
337
|
+
return { ok: false, error, attempts, providerFailure };
|
|
243
338
|
}
|
|
244
339
|
|
|
245
340
|
/** Resolves after `ms`, or rejects with the signal's reason when it aborts first. */
|
package/src/footage/loudness.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { copyFileSync } from "node:fs";
|
|
|
2
2
|
|
|
3
3
|
import { z } from "zod";
|
|
4
4
|
|
|
5
|
+
import { getAudioCodecArgs } from "./clip-sound.js";
|
|
5
6
|
import { FOOTAGE_SAMPLE_RATE } from "./normalize.js";
|
|
6
7
|
import { runFfmpeg, runProcess } from "./process.js";
|
|
7
8
|
import { FINISHED_AUDIO_BITRATE } from "./publish.js";
|
|
@@ -111,7 +112,7 @@ export function getLoudnessFilter(measurement: LoudnessMeasurement, target: Loud
|
|
|
111
112
|
return `${filter},aresample=${FOOTAGE_SAMPLE_RATE}`;
|
|
112
113
|
}
|
|
113
114
|
|
|
114
|
-
/** The speech track's second pass
|
|
115
|
+
/** The speech track's second pass at the film's rate, encoded by the output's extension (`.wav` lossless float). */
|
|
115
116
|
export function buildLoudnessArgs(options: { input: string; measurement: LoudnessMeasurement; target: LoudnessTarget; output: string }): string[] {
|
|
116
117
|
return [
|
|
117
118
|
"-v",
|
|
@@ -123,10 +124,7 @@ export function buildLoudnessArgs(options: { input: string; measurement: Loudnes
|
|
|
123
124
|
getLoudnessFilter(options.measurement, options.target),
|
|
124
125
|
"-ar",
|
|
125
126
|
String(FOOTAGE_SAMPLE_RATE),
|
|
126
|
-
|
|
127
|
-
"libmp3lame",
|
|
128
|
-
"-q:a",
|
|
129
|
-
"2",
|
|
127
|
+
...getAudioCodecArgs(options.output),
|
|
130
128
|
options.output,
|
|
131
129
|
];
|
|
132
130
|
}
|
|
@@ -196,8 +194,16 @@ export interface MixLoudnessTarget extends LoudnessTarget {
|
|
|
196
194
|
* `target ± warnTolerance`, under the true peak limit, finishes with a warning instead of failing. Absent: none.
|
|
197
195
|
*/
|
|
198
196
|
warnTolerance?: number;
|
|
197
|
+
/**
|
|
198
|
+
* The finished sound's AAC bitrate in kbps (192 to 512), the film's only lossy audio encode (issue #1139). Absent:
|
|
199
|
+
* `FINISHED_AUDIO_BITRATE` (192).
|
|
200
|
+
*/
|
|
201
|
+
audioBitrateKbps?: number;
|
|
199
202
|
}
|
|
200
203
|
|
|
204
|
+
/** The lowest and highest AAC bitrate `MixLoudnessTarget.audioBitrateKbps` takes, in kbps. */
|
|
205
|
+
export const AUDIO_BITRATE_RANGE_KBPS = { min: 192, max: 512 } as const;
|
|
206
|
+
|
|
201
207
|
/** A gain to the window's edge aims this far inside it (at most half the tolerance), so the encode's drift stays in. */
|
|
202
208
|
const MIX_EDGE_MARGIN = 0.2;
|
|
203
209
|
/** The limiter's ceiling sits this far under the true peak limit: AAC adds inter-sample overshoot after it. */
|
|
@@ -367,8 +373,11 @@ export function getMixFilter(attempt: MixAttempt): string {
|
|
|
367
373
|
return [`volume=${attempt.gainDb}dB`, `aresample=${LIMITER_RATE}`, `alimiter=limit=${limit}:level=false:latency=true`, `aresample=${FOOTAGE_SAMPLE_RATE}`, `lowpass=f=${MIX_LOWPASS_HZ}`].join(",");
|
|
368
374
|
}
|
|
369
375
|
|
|
370
|
-
/**
|
|
371
|
-
|
|
376
|
+
/**
|
|
377
|
+
* The mix's sound on its own through `filter`, encoded as the finished film carries it (AAC at `bitrateKbps`, default
|
|
378
|
+
* `FINISHED_AUDIO_BITRATE`, cut at `MIX_LOWPASS_HZ`, 48 kHz, stereo).
|
|
379
|
+
*/
|
|
380
|
+
export function buildMixArgs(options: { input: string; filter: string; output: string; bitrateKbps?: number }): string[] {
|
|
372
381
|
return [
|
|
373
382
|
"-v",
|
|
374
383
|
"error",
|
|
@@ -383,7 +392,7 @@ export function buildMixArgs(options: { input: string; filter: string; output: s
|
|
|
383
392
|
"-c:a",
|
|
384
393
|
"aac",
|
|
385
394
|
"-b:a",
|
|
386
|
-
FINISHED_AUDIO_BITRATE
|
|
395
|
+
options.bitrateKbps === undefined ? FINISHED_AUDIO_BITRATE : `${String(options.bitrateKbps)}k`,
|
|
387
396
|
"-cutoff",
|
|
388
397
|
String(MIX_LOWPASS_HZ),
|
|
389
398
|
"-ar",
|
package/src/footage/paid.ts
CHANGED
|
@@ -10,7 +10,15 @@ import { getFilmKeyTerms } from "./keyterms.js";
|
|
|
10
10
|
import type { FootageClip, FootageFilm } from "./film.js";
|
|
11
11
|
import { getTrimmedDuration, probeFootage } from "./probe.js";
|
|
12
12
|
import type { FootageProviders, FootageRates, PaidStepKind } from "./providers.js";
|
|
13
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
getPlanOptions,
|
|
15
|
+
getSpeechRangeCache,
|
|
16
|
+
planSpeechRanges,
|
|
17
|
+
type PlanSpeechRangesOptions,
|
|
18
|
+
type SpeechRangeCacheResolver,
|
|
19
|
+
type SpeechRangeClip,
|
|
20
|
+
type SpeechRangeSkip,
|
|
21
|
+
} from "./speech-ranges.js";
|
|
14
22
|
|
|
15
23
|
/**
|
|
16
24
|
* Where each paid result of a footage film is cached, shared by the pipeline and by the questions an app asks before
|
|
@@ -161,6 +169,11 @@ export interface FootageEstimateItem {
|
|
|
161
169
|
cached: boolean;
|
|
162
170
|
/** The range of the clip an `isolate: "segments"` item prices, in seconds of the trimmed clip; absent: the whole clip. */
|
|
163
171
|
range?: { start: number; end: number };
|
|
172
|
+
/**
|
|
173
|
+
* Set on an `isolate: "segments"` range that will not be sent: shorter than the provider's minimum even after
|
|
174
|
+
* merging and widening (`units` and `credits` 0).
|
|
175
|
+
*/
|
|
176
|
+
skipped?: SpeechRangeSkip;
|
|
164
177
|
}
|
|
165
178
|
|
|
166
179
|
export interface FootageEstimate {
|
|
@@ -179,6 +192,10 @@ export interface EstimateFootageOptions extends FootageCacheOptions {
|
|
|
179
192
|
isolate?: "clips" | "segments";
|
|
180
193
|
/** Seconds around each sound span with `isolate: "segments"`; default `DEFAULT_SPEECH_RANGE_MARGIN`. */
|
|
181
194
|
margin?: number;
|
|
195
|
+
/** Ranges of a clip at most this many seconds apart are priced as one with `isolate: "segments"`, as `isolateSpeechRanges` merges them. */
|
|
196
|
+
mergeGap?: PlanSpeechRangesOptions["mergeGap"];
|
|
197
|
+
/** Seconds a range shorter than the provider's minimum is widened past it with `isolate: "segments"`. */
|
|
198
|
+
minPad?: number;
|
|
182
199
|
/** Where a range's cleaned voice is cached with `isolate: "segments"`, as `isolateSpeechRanges` takes it; default `getSpeechRangeCache`. */
|
|
183
200
|
rangeCache?: SpeechRangeCacheResolver;
|
|
184
201
|
}
|
|
@@ -234,7 +251,11 @@ export async function estimateFootageFilm(film: FootageFilm, options: EstimateFo
|
|
|
234
251
|
}
|
|
235
252
|
if (isSegments && edl !== null) {
|
|
236
253
|
const provider = options.providers.isolation.id;
|
|
237
|
-
for (const range of planSpeechRanges(edl, rangeClips, options
|
|
254
|
+
for (const range of planSpeechRanges(edl, rangeClips, getPlanOptions(options, options.providers.isolation))) {
|
|
255
|
+
if (range.skip !== undefined) {
|
|
256
|
+
items.push({ step: "isolate", item: range.clip, provider, units: 0, unit: "seconds", credits: 0, cached: false, range: { start: range.startMs / 1000, end: range.endMs / 1000 }, skipped: range.skip });
|
|
257
|
+
continue;
|
|
258
|
+
}
|
|
238
259
|
const clipKey = clipKeys[range.clip] ?? null;
|
|
239
260
|
const cache = clipKey === null ? null : (options.rangeCache?.(range, clipKey) ?? getSpeechRangeCache(film.id, clipKey, range, { cacheDir: options.cacheDir, provider }));
|
|
240
261
|
const cached = cache !== null && existsSync(cache.path);
|
|
@@ -253,7 +274,8 @@ export async function estimateFootageFilm(film: FootageFilm, options: EstimateFo
|
|
|
253
274
|
|
|
254
275
|
/**
|
|
255
276
|
* What isolating only the cut's ranges will cost (`isolateSpeechRanges`), for a confirmation step: the `isolate` items
|
|
256
|
-
* of `estimateFootageFilm` with `isolate: "segments"`, one per range, a cached range at 0
|
|
277
|
+
* of `estimateFootageFilm` with `isolate: "segments"`, one per range as it will be sent, a cached range at 0, a range
|
|
278
|
+
* shorter than the provider's minimum marked `skipped` at 0.
|
|
257
279
|
*/
|
|
258
280
|
export async function estimateSpeechRanges(film: FootageFilm, options: Omit<EstimateFootageOptions, "isolate">): Promise<TtsResult<FootageEstimate>> {
|
|
259
281
|
const estimate = await estimateFootageFilm(film, { ...options, isolate: "segments" });
|