@softure-ai/marketing-kit 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +71 -0
- package/README.md +66 -9
- package/dist/cli/footage.js +1 -1
- package/dist/cli/footage.js.map +1 -1
- package/dist/compose/timeline.d.ts +23 -3
- package/dist/compose/timeline.d.ts.map +1 -1
- package/dist/compose/timeline.js +31 -11
- package/dist/compose/timeline.js.map +1 -1
- package/dist/config/actions-schema.d.ts +14 -14
- package/dist/config/config.d.ts.map +1 -1
- package/dist/config/config.js +2 -0
- package/dist/config/config.js.map +1 -1
- package/dist/config/schema.d.ts +83 -70
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +21 -1
- package/dist/config/schema.js.map +1 -1
- package/dist/config/shot-steps.d.ts +16 -16
- package/dist/footage/captions.d.ts +45 -3
- package/dist/footage/captions.d.ts.map +1 -1
- package/dist/footage/captions.js +65 -5
- package/dist/footage/captions.js.map +1 -1
- package/dist/footage/compose.d.ts.map +1 -1
- package/dist/footage/compose.js +8 -4
- package/dist/footage/compose.js.map +1 -1
- package/dist/footage/declick.d.ts +106 -0
- package/dist/footage/declick.d.ts.map +1 -0
- package/dist/footage/declick.js +296 -0
- package/dist/footage/declick.js.map +1 -0
- package/dist/footage/draft.d.ts +8 -3
- package/dist/footage/draft.d.ts.map +1 -1
- package/dist/footage/draft.js +55 -22
- package/dist/footage/draft.js.map +1 -1
- package/dist/footage/edl.d.ts +28 -2
- package/dist/footage/edl.d.ts.map +1 -1
- package/dist/footage/edl.js +53 -2
- package/dist/footage/edl.js.map +1 -1
- package/dist/footage/film.d.ts +29 -3
- package/dist/footage/film.d.ts.map +1 -1
- package/dist/footage/film.js +12 -0
- package/dist/footage/film.js.map +1 -1
- package/dist/footage/isolate-ranges.d.ts +37 -0
- package/dist/footage/isolate-ranges.d.ts.map +1 -0
- package/dist/footage/isolate-ranges.js +113 -0
- package/dist/footage/isolate-ranges.js.map +1 -0
- package/dist/footage/loudness.d.ts +58 -0
- package/dist/footage/loudness.d.ts.map +1 -1
- package/dist/footage/loudness.js +80 -0
- package/dist/footage/loudness.js.map +1 -1
- package/dist/footage/paid.d.ts +18 -0
- package/dist/footage/paid.d.ts.map +1 -1
- package/dist/footage/paid.js +32 -2
- package/dist/footage/paid.js.map +1 -1
- package/dist/footage/pipeline.d.ts +43 -5
- package/dist/footage/pipeline.d.ts.map +1 -1
- package/dist/footage/pipeline.js +103 -20
- package/dist/footage/pipeline.js.map +1 -1
- package/dist/footage/providers.d.ts +5 -0
- package/dist/footage/providers.d.ts.map +1 -1
- package/dist/footage/publish.d.ts +11 -2
- package/dist/footage/publish.d.ts.map +1 -1
- package/dist/footage/publish.js +14 -10
- package/dist/footage/publish.js.map +1 -1
- package/dist/footage/speech-ranges.d.ts +65 -0
- package/dist/footage/speech-ranges.d.ts.map +1 -0
- package/dist/footage/speech-ranges.js +98 -0
- package/dist/footage/speech-ranges.js.map +1 -0
- package/dist/index.d.ts +10 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +10 -7
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/schema/edl.schema.json +17 -0
- package/schema/marketing.schema.json +43 -1
- package/src/cli/footage.ts +1 -1
- package/src/compose/timeline.ts +41 -10
- package/src/config/config.ts +2 -0
- package/src/config/schema.ts +23 -1
- package/src/footage/captions.ts +81 -6
- package/src/footage/compose.ts +8 -4
- package/src/footage/declick.ts +353 -0
- package/src/footage/draft.ts +58 -23
- package/src/footage/edl.ts +69 -2
- package/src/footage/film.ts +37 -3
- package/src/footage/isolate-ranges.ts +161 -0
- package/src/footage/loudness.ts +112 -0
- package/src/footage/paid.ts +40 -2
- package/src/footage/pipeline.ts +155 -22
- package/src/footage/providers.ts +2 -0
- package/src/footage/publish.ts +20 -11
- package/src/footage/speech-ranges.ts +131 -0
- package/src/index.ts +70 -0
package/src/footage/loudness.ts
CHANGED
|
@@ -4,6 +4,7 @@ import { z } from "zod";
|
|
|
4
4
|
|
|
5
5
|
import { FOOTAGE_SAMPLE_RATE } from "./normalize.js";
|
|
6
6
|
import { runFfmpeg, runProcess } from "./process.js";
|
|
7
|
+
import { FINISHED_AUDIO_BITRATE } from "./publish.js";
|
|
7
8
|
|
|
8
9
|
/**
|
|
9
10
|
* The speech track's loudness, measured and normalized in two passes of ffmpeg's `loudnorm` (EBU R128): the first
|
|
@@ -163,3 +164,114 @@ export async function normalizeLoudness(input: string, output: string, target: L
|
|
|
163
164
|
});
|
|
164
165
|
return { kind: "normalized", measured: measurement.inputI };
|
|
165
166
|
}
|
|
167
|
+
|
|
168
|
+
/** The film's mix as measured on an encoded file: integrated loudness in LUFS, true peak in dBTP. */
|
|
169
|
+
export interface MixLoudness {
|
|
170
|
+
integrated: number;
|
|
171
|
+
truePeak: number;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/** How far from `LoudnessTarget.target` the finished mix may land, in LU, unless `MixLoudnessTarget.tolerance` says. */
|
|
175
|
+
export const MIX_LOUDNESS_TOLERANCE = 1;
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* The finished film's mix target. `keep: "loudness"` (the default) brings the mix to `target` with one gain measured on
|
|
179
|
+
* it, so the speech moves with whatever the music adds; `keep: "speech"` applies no gain while the mix measures inside
|
|
180
|
+
* `target ± tolerance` and otherwise only the smallest gain to that window, so the speech (already at the film's
|
|
181
|
+
* `loudness`) keeps its level whatever the music volume. Either way a true peak limiter is the last step.
|
|
182
|
+
*/
|
|
183
|
+
export interface MixLoudnessTarget extends LoudnessTarget {
|
|
184
|
+
/** How far from `target` the mix may land, in LU. Absent: `MIX_LOUDNESS_TOLERANCE`. */
|
|
185
|
+
tolerance?: number;
|
|
186
|
+
/** What the mix's gain keeps: the mix at the target (absent, `"loudness"`) or the speech's level (`"speech"`). */
|
|
187
|
+
keep?: "loudness" | "speech";
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/** A gain to the window's edge aims this far inside it (at most half the tolerance), so the encode's drift stays in. */
|
|
191
|
+
const MIX_EDGE_MARGIN = 0.2;
|
|
192
|
+
/** The limiter's first ceiling sits this far under the true peak limit: AAC adds inter-sample overshoot after it. */
|
|
193
|
+
export const MIX_ENCODE_HEADROOM = 1;
|
|
194
|
+
/** How many times the mix is encoded and measured before the film fails. */
|
|
195
|
+
export const MIX_ATTEMPTS = 4;
|
|
196
|
+
/** What a retry lowers the ceiling by on top of the measured overshoot, in dB. */
|
|
197
|
+
const MIX_RETRY_MARGIN = 0.3;
|
|
198
|
+
/** `alimiter`'s lowest ceiling (its `limit` of 0.0625), in dB. */
|
|
199
|
+
const LIMITER_FLOOR = -24;
|
|
200
|
+
/** The limiter runs at 4x the film's rate, so it holds the true peak rather than the sample peak. */
|
|
201
|
+
const LIMITER_RATE = FOOTAGE_SAMPLE_RATE * 4;
|
|
202
|
+
|
|
203
|
+
/** One try at the mix's last stage: a linear gain, then a true peak limiter at the ceiling. */
|
|
204
|
+
export interface MixAttempt {
|
|
205
|
+
gainDb: number;
|
|
206
|
+
ceilingDbtp: number;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
function roundTo(value: number, digits: number): number {
|
|
210
|
+
const scale = 10 ** digits;
|
|
211
|
+
return Math.round(value * scale) / scale;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/** The window's half width around the mix target, in LU. */
|
|
215
|
+
export function getMixTolerance(target: MixLoudnessTarget): number {
|
|
216
|
+
return target.tolerance ?? MIX_LOUDNESS_TOLERANCE;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* The gain a mix measuring `integrated` LUFS needs: to the target in the default mode; with `keep: "speech"` none
|
|
221
|
+
* inside `target ± tolerance`, else to the nearest edge (aimed a little inside, see `MIX_EDGE_MARGIN`).
|
|
222
|
+
*/
|
|
223
|
+
export function getMixGain(integrated: number, target: MixLoudnessTarget): number {
|
|
224
|
+
if (target.keep !== "speech") return roundTo(target.target - integrated, 2);
|
|
225
|
+
const tolerance = getMixTolerance(target);
|
|
226
|
+
const inside = tolerance - Math.min(MIX_EDGE_MARGIN, tolerance / 2);
|
|
227
|
+
if (integrated < target.target - tolerance) return roundTo(target.target - inside - integrated, 2);
|
|
228
|
+
if (integrated > target.target + tolerance) return roundTo(target.target + inside - integrated, 2);
|
|
229
|
+
return 0;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/** The first try: the gain the measured mix needs (`getMixGain`), the ceiling one encode's headroom under the limit. */
|
|
233
|
+
export function getFirstMixAttempt(measurement: Pick<LoudnessMeasurement, "inputI">, target: MixLoudnessTarget): MixAttempt {
|
|
234
|
+
return { gainDb: getMixGain(measurement.inputI, target), ceilingDbtp: Math.max(LIMITER_FLOOR, roundTo(target.truePeak - MIX_ENCODE_HEADROOM, 2)) };
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* The next try after an encoded mix measured `measured`: the ceiling lowered by the overshoot plus a margin when the
|
|
239
|
+
* true peak is over (never under the limiter's floor), the gain corrected by what the integrated loudness missed (with
|
|
240
|
+
* `keep: "speech"`, only by what it missed the window by).
|
|
241
|
+
*/
|
|
242
|
+
export function getNextMixAttempt(attempt: MixAttempt, measured: MixLoudness, target: MixLoudnessTarget): MixAttempt {
|
|
243
|
+
const overshoot = measured.truePeak - target.truePeak;
|
|
244
|
+
const ceilingDbtp = overshoot > 0 ? Math.max(LIMITER_FLOOR, roundTo(attempt.ceilingDbtp - overshoot - MIX_RETRY_MARGIN, 2)) : attempt.ceilingDbtp;
|
|
245
|
+
return { gainDb: roundTo(attempt.gainDb + getMixGain(measured.integrated, target), 2), ceilingDbtp };
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* The mix's last stage: one linear gain, then a lookahead limiter at 4x the film's rate (its delay compensated, so
|
|
250
|
+
* the sound stays in sync with the picture; no auto level), back to the film's rate.
|
|
251
|
+
*/
|
|
252
|
+
export function getMixFilter(attempt: MixAttempt): string {
|
|
253
|
+
const limit = roundTo(10 ** (attempt.ceilingDbtp / 20), 5);
|
|
254
|
+
return [`volume=${attempt.gainDb}dB`, `aresample=${LIMITER_RATE}`, `alimiter=limit=${limit}:level=false:latency=true`, `aresample=${FOOTAGE_SAMPLE_RATE}`].join(",");
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/** The mix's sound on its own through `filter`, encoded as the finished film carries it (AAC 192k, 48 kHz, stereo). */
|
|
258
|
+
export function buildMixArgs(options: { input: string; filter: string; output: string }): string[] {
|
|
259
|
+
return ["-v", "error", "-y", "-i", options.input, "-vn", "-map", "0:a:0", "-af", options.filter, "-c:a", "aac", "-b:a", FINISHED_AUDIO_BITRATE, "-ar", String(FOOTAGE_SAMPLE_RATE), "-ac", "2", options.output];
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/** What keeps a measured mix outside the target, as sentences with both values; empty when it meets it. */
|
|
263
|
+
export function findLoudnessProblems(measured: MixLoudness, target: MixLoudnessTarget): string[] {
|
|
264
|
+
const problems: string[] = [];
|
|
265
|
+
const tolerance = getMixTolerance(target);
|
|
266
|
+
if (measured.truePeak > target.truePeak) problems.push(`true peak ${formatDb(measured.truePeak)} dBTP, over the ${formatDb(target.truePeak)} dBTP limit`);
|
|
267
|
+
if (Math.abs(measured.integrated - target.target) > tolerance) {
|
|
268
|
+
problems.push(`integrated ${formatDb(measured.integrated)} LUFS, outside ${formatDb(target.target)} ± ${tolerance} LUFS`);
|
|
269
|
+
}
|
|
270
|
+
return problems;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/** A level with one decimal and its sign (`+0.1`, `-14.0`). */
|
|
274
|
+
export function formatDb(value: number): string {
|
|
275
|
+
const text = value.toFixed(1);
|
|
276
|
+
return value > 0 && text !== "0.0" ? `+${text}` : text === "-0.0" ? "0.0" : text;
|
|
277
|
+
}
|
package/src/footage/paid.ts
CHANGED
|
@@ -10,6 +10,7 @@ import { getFilmKeyTerms } from "./keyterms.js";
|
|
|
10
10
|
import type { FootageClip, FootageFilm } from "./film.js";
|
|
11
11
|
import { getTrimmedDuration, probeFootage } from "./probe.js";
|
|
12
12
|
import type { FootageProviders, FootageRates, PaidStepKind } from "./providers.js";
|
|
13
|
+
import { getSpeechRangeCache, planSpeechRanges, type SpeechRangeClip } from "./speech-ranges.js";
|
|
13
14
|
|
|
14
15
|
/**
|
|
15
16
|
* Where each paid result of a footage film is cached, shared by the pipeline and by the questions an app asks before
|
|
@@ -158,6 +159,8 @@ export interface FootageEstimateItem {
|
|
|
158
159
|
/** `units` × the step's rate; 0 when cached. */
|
|
159
160
|
credits: number;
|
|
160
161
|
cached: boolean;
|
|
162
|
+
/** The range of the clip an `isolate: "segments"` item prices, in seconds of the trimmed clip; absent: the whole clip. */
|
|
163
|
+
range?: { start: number; end: number };
|
|
161
164
|
}
|
|
162
165
|
|
|
163
166
|
export interface FootageEstimate {
|
|
@@ -168,6 +171,14 @@ export interface FootageEstimate {
|
|
|
168
171
|
export interface EstimateFootageOptions extends FootageCacheOptions {
|
|
169
172
|
rates: FootageRates;
|
|
170
173
|
signal?: AbortSignal;
|
|
174
|
+
/**
|
|
175
|
+
* What voice isolation is priced on: `clips` (default) the whole talking clips when the film isolates the voice;
|
|
176
|
+
* `segments` only the ranges the cut hears (`isolateSpeechRanges`), one item per range, whatever `isolateVoice`
|
|
177
|
+
* says. `segments` needs the cut (`edl`, or `<buildDir>/edl.json`).
|
|
178
|
+
*/
|
|
179
|
+
isolate?: "clips" | "segments";
|
|
180
|
+
/** Seconds around each sound span with `isolate: "segments"`; default `DEFAULT_SPEECH_RANGE_MARGIN`. */
|
|
181
|
+
margin?: number;
|
|
171
182
|
}
|
|
172
183
|
|
|
173
184
|
const round = (value: number, digits: number): number => {
|
|
@@ -200,6 +211,11 @@ export async function estimateFootageFilm(film: FootageFilm, options: EstimateFo
|
|
|
200
211
|
const add = (step: PaidStepKind, item: string, provider: string, units: number, cached: boolean): void => {
|
|
201
212
|
items.push({ step, item, provider, units: round(units, 3), unit: "seconds", credits: cached ? 0 : round(units * rates[step], 2), cached });
|
|
202
213
|
};
|
|
214
|
+
const isSegments = options.isolate === "segments";
|
|
215
|
+
const edl = findEdl(options);
|
|
216
|
+
if (isSegments && edl === null) return err(`footage film ${film.id}: isolating segments needs the cut (edl, or edl.json in the build folder).`);
|
|
217
|
+
const clipKeys: Record<string, string | null> = {};
|
|
218
|
+
const rangeClips: SpeechRangeClip[] = [];
|
|
203
219
|
let clipSeconds = 0;
|
|
204
220
|
for (const { clip, probe } of probed.clips) {
|
|
205
221
|
const seconds = getTrimmedDuration(clip, probe);
|
|
@@ -207,16 +223,38 @@ export async function estimateFootageFilm(film: FootageFilm, options: EstimateFo
|
|
|
207
223
|
if (clip.role !== "talk") continue;
|
|
208
224
|
const source = findLocalSource(clip, options.buildDir);
|
|
209
225
|
const clipKey = source === null ? null : await getClipCacheKey(source, clip);
|
|
226
|
+
clipKeys[clip.id] = clipKey;
|
|
227
|
+
rangeClips.push({ id: clip.id, role: clip.role, duration: seconds });
|
|
210
228
|
const isolated = clipKey === null ? null : getIsolatedCache(film, clipKey, options);
|
|
211
|
-
if (film.isolateVoice) add("isolate", clip.id, options.providers.isolation.id, seconds, isolated !== null && existsSync(isolated.path));
|
|
229
|
+
if (film.isolateVoice && !isSegments) add("isolate", clip.id, options.providers.isolation.id, seconds, isolated !== null && existsSync(isolated.path));
|
|
212
230
|
const transcript = clipKey === null ? null : getTranscriptCache(film, clipKey, options);
|
|
213
231
|
add("transcribe", clip.id, options.providers.stt.id, seconds, transcript !== null && existsSync(transcript.path));
|
|
214
232
|
}
|
|
233
|
+
if (isSegments && edl !== null) {
|
|
234
|
+
const provider = options.providers.isolation.id;
|
|
235
|
+
for (const range of planSpeechRanges(edl, rangeClips, options.margin === undefined ? {} : { margin: options.margin })) {
|
|
236
|
+
const clipKey = clipKeys[range.clip] ?? null;
|
|
237
|
+
const cached = clipKey !== null && existsSync(getSpeechRangeCache(film.id, clipKey, range, { cacheDir: options.cacheDir, provider }).path);
|
|
238
|
+
add("isolate", range.clip, provider, (range.endMs - range.startMs) / 1000, cached);
|
|
239
|
+
const item = items.at(-1);
|
|
240
|
+
if (item !== undefined) item.range = { start: range.startMs / 1000, end: range.endMs / 1000 };
|
|
241
|
+
}
|
|
242
|
+
}
|
|
215
243
|
const music = film.music.source;
|
|
216
244
|
if (music.kind === "prompt" || music.kind === "video") {
|
|
217
|
-
const edl = findEdl(options);
|
|
218
245
|
const seconds = edl === null ? clipSeconds + (film.endCard === null ? 0 : END_CARD_SECONDS) : getFootageFilmSeconds(film, edl);
|
|
219
246
|
add("music", "film", options.providers.music.id, seconds, await isMusicCached(film, options));
|
|
220
247
|
}
|
|
221
248
|
return ok({ items, totalCredits: round(items.reduce((sum, item) => sum + item.credits, 0), 2) });
|
|
222
249
|
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* What isolating only the cut's ranges will cost (`isolateSpeechRanges`), for a confirmation step: the `isolate` items
|
|
253
|
+
* of `estimateFootageFilm` with `isolate: "segments"`, one per range, a cached range at 0.
|
|
254
|
+
*/
|
|
255
|
+
export async function estimateSpeechRanges(film: FootageFilm, options: Omit<EstimateFootageOptions, "isolate">): Promise<TtsResult<FootageEstimate>> {
|
|
256
|
+
const estimate = await estimateFootageFilm(film, { ...options, isolate: "segments" });
|
|
257
|
+
if (!estimate.ok) return estimate;
|
|
258
|
+
const items = estimate.value.items.filter((item) => item.step === "isolate");
|
|
259
|
+
return ok({ items, totalCredits: round(items.reduce((sum, item) => sum + item.credits, 0), 2) });
|
|
260
|
+
}
|
package/src/footage/pipeline.ts
CHANGED
|
@@ -13,14 +13,31 @@ import type { TimedWord } from "../voice/voiceover.js";
|
|
|
13
13
|
import { checkMusicFileType, describeAudioProbe, probeAudio } from "./audio.js";
|
|
14
14
|
import { readCachedTranscript, writeCacheFile, writeTranscript } from "./cache.js";
|
|
15
15
|
import { composeFootage, END_CARD_SECONDS, getFootageFilmSeconds } from "./compose.js";
|
|
16
|
-
import { findMissingCaptionGlyphs, getFootageCaptionLayout } from "./captions.js";
|
|
16
|
+
import { findMissingCaptionGlyphs, getCaptionBounds, getFootageCaptionLayout } from "./captions.js";
|
|
17
17
|
import { findLowDiskProblems, getFreeBytes, getRenderTempDir, type FreeBytesReader } from "./disk.js";
|
|
18
18
|
import { buildEdl, getEdlDuration, readEdl, remapWords, validateEdl, type Edl } from "./edl.js";
|
|
19
19
|
import { makePlaceholderTranscript } from "./fake.js";
|
|
20
20
|
import { chooseCutTransitions, type CutMeasure } from "./cuts.js";
|
|
21
|
-
import {
|
|
21
|
+
import { declickSpeech, describeDeclick, type DeclickReport } from "./declick.js";
|
|
22
|
+
import { DEFAULT_AUTO_THRESHOLD, getDeclickLevel, getFootageEdit, type FootageClip, type FootageFilm, type MusicSource } from "./film.js";
|
|
22
23
|
import { getFilmKeyTerms } from "./keyterms.js";
|
|
23
|
-
import {
|
|
24
|
+
import {
|
|
25
|
+
buildMixArgs,
|
|
26
|
+
DEFAULT_LOUDNESS,
|
|
27
|
+
findLoudnessProblems,
|
|
28
|
+
formatDb,
|
|
29
|
+
getFirstMixAttempt,
|
|
30
|
+
getMixTolerance,
|
|
31
|
+
getMixFilter,
|
|
32
|
+
getNextMixAttempt,
|
|
33
|
+
measureLoudness,
|
|
34
|
+
MIX_ATTEMPTS,
|
|
35
|
+
normalizeLoudness,
|
|
36
|
+
type LoudnessMeasurement,
|
|
37
|
+
type MixAttempt,
|
|
38
|
+
type MixLoudness,
|
|
39
|
+
type MixLoudnessTarget,
|
|
40
|
+
} from "./loudness.js";
|
|
24
41
|
import { duckingKeyframes, getDuckingOptions } from "./music.js";
|
|
25
42
|
import { FOOTAGE_FPS, normalizeFootage, type Frame, type NormalizeEncoder } from "./normalize.js";
|
|
26
43
|
import { buildPictureConcatArgs, buildPictureList, buildPicturePieceArgs, getPictureFrames, planFootagePicture } from "./picture.js";
|
|
@@ -473,8 +490,11 @@ export function describeCutMeasures(cuts: CutMeasure[]): string[] {
|
|
|
473
490
|
return cuts.map((cut) => `${cut.segment}: ${cut.transition} (difference ${cut.difference.toFixed(3)})`);
|
|
474
491
|
}
|
|
475
492
|
|
|
476
|
-
/**
|
|
477
|
-
|
|
493
|
+
/**
|
|
494
|
+
* The EDL file as written by `cutFootage` or edited by hand, checked against the clips and (a segment's own caption
|
|
495
|
+
* box) against the film's frame, with the caption size the project's `layout` gives.
|
|
496
|
+
*/
|
|
497
|
+
export function readEdlFile(film: FootageFilm, clips: PreparedClip[], edlPath: string, layout: LayoutOverride = {}): FootageResult<Edl> {
|
|
478
498
|
let json: unknown;
|
|
479
499
|
try {
|
|
480
500
|
json = JSON.parse(readFileSync(edlPath, "utf8")) as unknown;
|
|
@@ -483,7 +503,8 @@ export function readEdlFile(film: FootageFilm, clips: PreparedClip[], edlPath: s
|
|
|
483
503
|
}
|
|
484
504
|
const read = readEdl(json);
|
|
485
505
|
if (!read.ok) return err(`${edlPath}: ${read.error}`);
|
|
486
|
-
const
|
|
506
|
+
const bounds = getCaptionBounds(getFrame(film).height, getFootageCaptionLayout(film, layout));
|
|
507
|
+
const problems = validateEdl(read.edl, clips.map((clip) => ({ id: clip.clip.id, duration: clip.duration })), film.id, bounds);
|
|
487
508
|
return problems.length > 0 ? err(`${edlPath}: ${problems.join("; ")}`) : ok(read.edl);
|
|
488
509
|
}
|
|
489
510
|
|
|
@@ -590,6 +611,17 @@ async function getMusic(film: FootageFilm, edl: Edl, clips: PreparedClip[], seco
|
|
|
590
611
|
return ok(path);
|
|
591
612
|
}
|
|
592
613
|
|
|
614
|
+
/**
|
|
615
|
+
* The speech chain's files: the cut speech, the declicked track and the final `speech.mp3`. The last stage that runs
|
|
616
|
+
* writes `speech.mp3`; the declicked track is float WAV when the loudness pass follows (one lossy encode fewer).
|
|
617
|
+
*/
|
|
618
|
+
export function getSpeechChainFiles(assetsDir: string, stages: { isDeclick: boolean; isLoudness: boolean }): { rawSpeechFile: string; declickedFile: string; speechFile: string } {
|
|
619
|
+
const speechFile = join(assetsDir, "speech.mp3");
|
|
620
|
+
const rawSpeechFile = stages.isDeclick || stages.isLoudness ? join(assetsDir, "speech.raw.mp3") : speechFile;
|
|
621
|
+
const declickedFile = !stages.isDeclick ? rawSpeechFile : stages.isLoudness ? join(assetsDir, "speech.declicked.wav") : speechFile;
|
|
622
|
+
return { rawSpeechFile, declickedFile, speechFile };
|
|
623
|
+
}
|
|
624
|
+
|
|
593
625
|
export interface RenderFootageOptions extends FootageRunOptions {
|
|
594
626
|
brand: FootageBrand;
|
|
595
627
|
sfx: Partial<Record<SfxEvent, string>>;
|
|
@@ -630,7 +662,16 @@ export interface RenderFootageOptions extends FootageRunOptions {
|
|
|
630
662
|
finishVideo?: FinishVideo;
|
|
631
663
|
}
|
|
632
664
|
|
|
633
|
-
export type RenderedFootage =
|
|
665
|
+
export type RenderedFootage =
|
|
666
|
+
| { kind: "dry-run"; steps: PaidStep[] }
|
|
667
|
+
| ({
|
|
668
|
+
kind: "rendered";
|
|
669
|
+
output: string;
|
|
670
|
+
seconds: number;
|
|
671
|
+
edlPath: string;
|
|
672
|
+
/** What the speech track's declick stage did (`FootageFilm.speech.declick`); null when it is off. */
|
|
673
|
+
declick: DeclickReport | null;
|
|
674
|
+
} & FinishedFootage);
|
|
634
675
|
|
|
635
676
|
/**
|
|
636
677
|
* Composes the cut and renders it: speech track (loudness-normalized), ducked music, end card backdrop, assets,
|
|
@@ -670,18 +711,29 @@ async function runRenderFootage(
|
|
|
670
711
|
const musicFile = typeof music.value === "string" ? music.value : null;
|
|
671
712
|
const aborting = signal === undefined ? {} : { signal };
|
|
672
713
|
const loudness = film.loudness === undefined ? DEFAULT_LOUDNESS : film.loudness;
|
|
673
|
-
const
|
|
674
|
-
const rawSpeechFile =
|
|
714
|
+
const declickLevel = getDeclickLevel(film);
|
|
715
|
+
const { speechFile, rawSpeechFile, declickedFile } = getSpeechChainFiles(assetsDir, { isDeclick: declickLevel !== "off", isLoudness: loudness !== null });
|
|
675
716
|
const ffmpegTrack = (label: string, output: string, args: () => string[]) => (onSeconds: (done: number) => void) => runFfmpeg(label, args(), { ...aborting, output, onSeconds });
|
|
676
717
|
const tracks: { item: string; seconds: number; run: (onSeconds: (done: number) => void) => Promise<void> }[] = [
|
|
677
718
|
{ item: "speech", seconds, run: ffmpegTrack("The speech track", rawSpeechFile, () => buildSpeechArgs(edl, transcript.speech, rawSpeechFile, getSpeechCutOptions(getFootageEdit(film)))) },
|
|
678
719
|
];
|
|
720
|
+
let declick: DeclickReport | null = null;
|
|
721
|
+
if (declickLevel !== "off") {
|
|
722
|
+
tracks.push({
|
|
723
|
+
item: "declick",
|
|
724
|
+
seconds,
|
|
725
|
+
run: async (onSeconds) => {
|
|
726
|
+
declick = await declickSpeech({ input: rawSpeechFile, output: declickedFile, words, level: declickLevel, seconds, ...aborting, onSeconds });
|
|
727
|
+
log(describeDeclick(declick));
|
|
728
|
+
},
|
|
729
|
+
});
|
|
730
|
+
}
|
|
679
731
|
if (loudness !== null) {
|
|
680
732
|
tracks.push({
|
|
681
733
|
item: "loudness",
|
|
682
734
|
seconds,
|
|
683
735
|
run: async (onSeconds) => {
|
|
684
|
-
const outcome = await normalizeLoudness(
|
|
736
|
+
const outcome = await normalizeLoudness(declickedFile, speechFile, loudness, { ...aborting, onSeconds });
|
|
685
737
|
log(
|
|
686
738
|
outcome.kind === "silent"
|
|
687
739
|
? "loudness: the speech track is silent, left as it is."
|
|
@@ -765,7 +817,7 @@ async function runRenderFootage(
|
|
|
765
817
|
const roles = Object.fromEntries(clips.map((clip) => [clip.clip.id, clip.clip.role]));
|
|
766
818
|
const finished = await finishFootage(film, { renderFile, seconds, coverAt: film.cover?.at ?? getDefaultCoverSeconds(edl, roles) }, options);
|
|
767
819
|
if (!finished.ok) return finished;
|
|
768
|
-
return ok({ kind: "rendered", output: options.output, seconds, edlPath: getEdlPath(buildDir, options.isPlaceholder), ...finished.value });
|
|
820
|
+
return ok({ kind: "rendered", output: options.output, seconds, edlPath: getEdlPath(buildDir, options.isPlaceholder), declick, ...finished.value });
|
|
769
821
|
}
|
|
770
822
|
|
|
771
823
|
/** What `renderFootage` reads from the written film. */
|
|
@@ -777,12 +829,32 @@ export interface FinishedFootage {
|
|
|
777
829
|
durationSeconds: number;
|
|
778
830
|
/** The cover JPG, or null when the film has none. */
|
|
779
831
|
cover: string | null;
|
|
832
|
+
/** The finished file's sound as measured (EBU R128, true peak oversampled); null for a film without sound or a silent mix. */
|
|
833
|
+
loudness: MixLoudness | null;
|
|
834
|
+
/**
|
|
835
|
+
* What the mix's last stage did: the render's mix as measured, the gain applied (0 when `keep: "speech"` found the
|
|
836
|
+
* mix inside its window) and the limiter's ceiling; null when no stage ran (no sound, a silent mix, `mixLoudness: null`).
|
|
837
|
+
*/
|
|
838
|
+
mix: FinishedMix | null;
|
|
839
|
+
}
|
|
840
|
+
|
|
841
|
+
/** The mix's last stage in `FinishedFootage`. */
|
|
842
|
+
export interface FinishedMix {
|
|
843
|
+
/** The render's mix before the stage. */
|
|
844
|
+
measured: MixLoudness;
|
|
845
|
+
gainDb: number;
|
|
846
|
+
ceilingDbtp: number;
|
|
847
|
+
/** What the gain kept (`MixLoudnessTarget.keep`). */
|
|
848
|
+
keep: "loudness" | "speech";
|
|
780
849
|
}
|
|
781
850
|
|
|
782
851
|
/**
|
|
783
|
-
* The final pass over hyperframes' render: the mix
|
|
784
|
-
*
|
|
785
|
-
*
|
|
852
|
+
* The final pass over hyperframes' render: the mix brought to `film.mixLoudness` (measured on the render; one linear
|
|
853
|
+
* gain and a true peak limiter, encoded to AAC on its own and measured, the limiter lowered and encoded again while
|
|
854
|
+
* the codec's overshoot breaks the limit), muxed with the picture (copied where the render already matches;
|
|
855
|
+
* `finishVideo` re-encodes it to H.264 High at its preset and CRF), publishable codecs and fast start, checked with
|
|
856
|
+
* ffprobe, the finished file's sound measured and held to the target, then the cover and the file's facts. Exported
|
|
857
|
+
* for tests only.
|
|
786
858
|
*/
|
|
787
859
|
export async function finishFootage(
|
|
788
860
|
film: Pick<FootageFilm, "mixLoudness" | "cover">,
|
|
@@ -795,27 +867,52 @@ export async function finishFootage(
|
|
|
795
867
|
progress(0, 0);
|
|
796
868
|
const rendered = await probeFilmFacts(input.renderFile, aborting);
|
|
797
869
|
if (!rendered.ok) return err(`the rendered film: ${rendered.error}`);
|
|
798
|
-
const mixLoudness = film.mixLoudness === undefined ? DEFAULT_LOUDNESS : film.mixLoudness;
|
|
799
|
-
let
|
|
870
|
+
const mixLoudness: MixLoudnessTarget | null = film.mixLoudness === undefined ? DEFAULT_LOUDNESS : film.mixLoudness;
|
|
871
|
+
let audioFile: string | undefined;
|
|
872
|
+
let mix: FinishedMix | null = null;
|
|
800
873
|
if (mixLoudness !== null && rendered.facts.audio !== null) {
|
|
801
874
|
const measured = await measureLoudness(input.renderFile, mixLoudness, { label: "the film's mix", ...aborting });
|
|
802
875
|
if (measured.kind === "silent") {
|
|
803
876
|
log("loudness: the film's mix is silent, left as it is.");
|
|
804
877
|
} else {
|
|
805
|
-
|
|
806
|
-
|
|
878
|
+
audioFile = join(dirname(input.renderFile), "mix.m4a");
|
|
879
|
+
const mixed = await finishMix({ input: input.renderFile, output: audioFile, measurement: measured.measurement, target: mixLoudness, ...aborting });
|
|
880
|
+
if (!mixed.ok) {
|
|
881
|
+
rmSync(audioFile, { force: true });
|
|
882
|
+
return mixed;
|
|
883
|
+
}
|
|
884
|
+
const { attempt, tries } = mixed.value;
|
|
885
|
+
const keep = mixLoudness.keep ?? "loudness";
|
|
886
|
+
mix = { measured: { integrated: measured.measurement.inputI, truePeak: measured.measurement.inputTp }, gainDb: attempt.gainDb, ceilingDbtp: attempt.ceilingDbtp, keep };
|
|
887
|
+
const kept = keep === "speech" ? ` (speech kept: ${formatDb(mixLoudness.target)} ± ${getMixTolerance(mixLoudness)} LUFS)` : "";
|
|
888
|
+
log(
|
|
889
|
+
`loudness: the film's mix measured ${formatDb(measured.measurement.inputI)} LUFS, true peak ${formatDb(measured.measurement.inputTp)} dBTP; ` +
|
|
890
|
+
`gain ${formatDb(attempt.gainDb)} dB${kept}, limiter at ${formatDb(attempt.ceilingDbtp)} dBTP (encode ${tries} of ${MIX_ATTEMPTS}).`,
|
|
891
|
+
);
|
|
807
892
|
}
|
|
808
893
|
}
|
|
809
894
|
const { finishVideo } = options;
|
|
810
895
|
const isVideoCopied = finishVideo === undefined && isPublishableVideo(rendered.facts.video);
|
|
811
|
-
const isAudioCopied =
|
|
812
|
-
const finishArgs = buildFinishArgs(input.renderFile, rendered.facts, output, { ...(
|
|
896
|
+
const isAudioCopied = audioFile === undefined && (rendered.facts.audio === null || isPublishableAudio(rendered.facts.audio));
|
|
897
|
+
const finishArgs = buildFinishArgs(input.renderFile, rendered.facts, output, { ...(audioFile === undefined ? {} : { audioFile }), ...(finishVideo === undefined ? {} : { video: finishVideo }) });
|
|
813
898
|
await runFfmpeg("The film's final pass", finishArgs, { ...aborting, output, onSeconds: (done) => progress(0, done) });
|
|
814
899
|
rmSync(input.renderFile, { force: true });
|
|
900
|
+
if (audioFile !== undefined) rmSync(audioFile, { force: true });
|
|
815
901
|
const finished = await probeFilmFacts(output, aborting);
|
|
816
902
|
if (!finished.ok) return err(`the finished film: ${finished.error}`);
|
|
817
903
|
const problems = findFinishProblems(finished.facts, readTopLevelBoxes(output));
|
|
818
904
|
if (problems.length > 0) return err(`the finished film ${output} is not publishable: ${problems.join("; ")}.`);
|
|
905
|
+
let loudness: MixLoudness | null = null;
|
|
906
|
+
if (finished.facts.audio !== null) {
|
|
907
|
+
const measured = await measureLoudness(output, mixLoudness ?? DEFAULT_LOUDNESS, { label: "the finished film", ...aborting });
|
|
908
|
+
if (measured.kind === "measured") {
|
|
909
|
+
loudness = { integrated: measured.measurement.inputI, truePeak: measured.measurement.inputTp };
|
|
910
|
+
// A mix left as rendered (`mixLoudness: null`) or silent is reported, not held to a target it was not brought to.
|
|
911
|
+
const misses = mixLoudness === null || audioFile === undefined ? [] : findLoudnessProblems(loudness, mixLoudness);
|
|
912
|
+
if (misses.length > 0) return err(`the finished film ${output} measures ${describeMixLoudness(loudness)}: ${misses.join("; ")}.`);
|
|
913
|
+
log(`loudness: the finished film measures ${formatDb(loudness.integrated)} LUFS, true peak ${formatDb(loudness.truePeak)} dBTP.`);
|
|
914
|
+
}
|
|
915
|
+
}
|
|
819
916
|
progress(0, input.seconds);
|
|
820
917
|
const { bytes, width, height, durationSeconds } = finished.facts;
|
|
821
918
|
const streams = describeFinishStreams({ isVideoCopied, isAudioCopied, hasAudio: rendered.facts.audio !== null, ...(finishVideo === undefined ? {} : { finishVideo }) });
|
|
@@ -828,7 +925,43 @@ export async function finishFootage(
|
|
|
828
925
|
await runFfmpeg("The cover frame", buildCoverArgs(output, at, cover), { ...aborting, output: cover });
|
|
829
926
|
log(`cover: ${cover} (at ${at} s).`);
|
|
830
927
|
}
|
|
831
|
-
return ok({ bytes, width, height, durationSeconds, cover });
|
|
928
|
+
return ok({ bytes, width, height, durationSeconds, cover, loudness, mix });
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
/**
|
|
932
|
+
* The mix's sound brought to `target` on its own: gain and limiter, encoded to AAC, measured; while the encoded file
|
|
933
|
+
* misses the target the gain and the ceiling are corrected and it is encoded again, at most `MIX_ATTEMPTS` times. An
|
|
934
|
+
* error names the last measurement.
|
|
935
|
+
*/
|
|
936
|
+
async function finishMix(input: {
|
|
937
|
+
input: string;
|
|
938
|
+
output: string;
|
|
939
|
+
measurement: LoudnessMeasurement;
|
|
940
|
+
target: MixLoudnessTarget;
|
|
941
|
+
signal?: AbortSignal;
|
|
942
|
+
}): Promise<FootageResult<{ attempt: MixAttempt; tries: number; measured: MixLoudness }>> {
|
|
943
|
+
const aborting = input.signal === undefined ? {} : { signal: input.signal };
|
|
944
|
+
let attempt = getFirstMixAttempt(input.measurement, input.target);
|
|
945
|
+
let misses: string[] = [];
|
|
946
|
+
let measured: MixLoudness = { integrated: input.measurement.inputI, truePeak: input.measurement.inputTp };
|
|
947
|
+
for (let tries = 1; tries <= MIX_ATTEMPTS; tries += 1) {
|
|
948
|
+
await runFfmpeg("The film's mix", buildMixArgs({ input: input.input, filter: getMixFilter(attempt), output: input.output }), { ...aborting, output: input.output });
|
|
949
|
+
const encoded = await measureLoudness(input.output, input.target, { label: "the film's encoded mix", ...aborting });
|
|
950
|
+
if (encoded.kind === "silent") return err("the film's mix measures silent after its gain and limiter.");
|
|
951
|
+
measured = { integrated: encoded.measurement.inputI, truePeak: encoded.measurement.inputTp };
|
|
952
|
+
misses = findLoudnessProblems(measured, input.target);
|
|
953
|
+
if (misses.length === 0) return ok({ attempt, tries, measured });
|
|
954
|
+
if (tries < MIX_ATTEMPTS) attempt = getNextMixAttempt(attempt, measured, input.target);
|
|
955
|
+
}
|
|
956
|
+
return err(
|
|
957
|
+
`the film's mix still measures ${describeMixLoudness(measured)} after ${MIX_ATTEMPTS} encodes (last: gain ${formatDb(attempt.gainDb)} dB, ` +
|
|
958
|
+
`limiter at ${formatDb(attempt.ceilingDbtp)} dBTP): ${misses.join("; ")}.`,
|
|
959
|
+
);
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
/** Both measurements of a mix, as an error names them. */
|
|
963
|
+
function describeMixLoudness(measured: MixLoudness): string {
|
|
964
|
+
return `${formatDb(measured.integrated)} LUFS integrated, ${formatDb(measured.truePeak)} dBTP true peak`;
|
|
832
965
|
}
|
|
833
966
|
|
|
834
967
|
/** What the final pass did to each stream, for its log line. */
|
|
@@ -969,7 +1102,7 @@ async function runRenderFootageFilm(film: FootageFilm, options: RenderFootageFil
|
|
|
969
1102
|
const edlPath = getEdlPath(options.buildDir, options.isPlaceholder);
|
|
970
1103
|
let edl: Edl;
|
|
971
1104
|
if (existsSync(edlPath) && options.isRecut !== true) {
|
|
972
|
-
const read = readEdlFile(film, clips.value, edlPath);
|
|
1105
|
+
const read = readEdlFile(film, clips.value, edlPath, options.layout);
|
|
973
1106
|
if (!read.ok) return read;
|
|
974
1107
|
edl = read.value;
|
|
975
1108
|
options.log(`cut: ${edlPath} (${edl.segments.length} segments), as it is; softure-marketing footage --step=cut builds it again.`);
|
package/src/footage/providers.ts
CHANGED
|
@@ -109,6 +109,8 @@ export interface FootageCharge {
|
|
|
109
109
|
keyTerms?: number;
|
|
110
110
|
/** The paid cache's key of the result (the file name's first part under `<cacheDir>/<film id>/`). */
|
|
111
111
|
cacheKey: string;
|
|
112
|
+
/** The part of the clip a range isolation cleaned (`isolateSpeechRanges`), in seconds of the trimmed clip; absent: the whole clip. */
|
|
113
|
+
range?: { start: number; end: number };
|
|
112
114
|
}
|
|
113
115
|
|
|
114
116
|
/** A stage of a footage run that reports progress. */
|
package/src/footage/publish.ts
CHANGED
|
@@ -15,7 +15,7 @@ import { runProcess } from "./process.js";
|
|
|
15
15
|
*/
|
|
16
16
|
|
|
17
17
|
/** The audio bitrate of a re-encoded track. */
|
|
18
|
-
const
|
|
18
|
+
export const FINISHED_AUDIO_BITRATE = "192k";
|
|
19
19
|
/** The quality of a re-encoded picture (x264's constant rate factor; 18 is visually lossless). */
|
|
20
20
|
const VIDEO_CRF = 18;
|
|
21
21
|
/** How far before the film's end the latest cover frame is taken, so a frame still exists there. */
|
|
@@ -145,26 +145,35 @@ function getVideoEncodeArgs(video: FinishVideo): string[] {
|
|
|
145
145
|
|
|
146
146
|
/**
|
|
147
147
|
* The final pass: each stream that is already publishable is copied, the other re-encoded (the picture to H.264 High),
|
|
148
|
-
* and the `moov` box moved to the front. A film without sound keeps none. `
|
|
149
|
-
*
|
|
148
|
+
* and the `moov` box moved to the front. A film without sound keeps none. `audioFile` (the mix's finished sound, AAC
|
|
149
|
+
* encoded on its own) replaces the render's sound, copied as it is; `audioFilter` re-encodes the sound through it.
|
|
150
|
+
* `video` re-encodes the picture whatever it is, at its preset and CRF (issue #1068).
|
|
150
151
|
*/
|
|
151
|
-
export function buildFinishArgs(
|
|
152
|
+
export function buildFinishArgs(
|
|
153
|
+
input: string,
|
|
154
|
+
facts: Pick<FilmFacts, "video" | "audio">,
|
|
155
|
+
output: string,
|
|
156
|
+
options: { audioFilter?: string; audioFile?: string; video?: FinishVideo } = {},
|
|
157
|
+
): string[] {
|
|
152
158
|
const video =
|
|
153
159
|
options.video !== undefined
|
|
154
160
|
? getVideoEncodeArgs(options.video)
|
|
155
161
|
: isPublishableVideo(facts.video)
|
|
156
162
|
? ["-c:v", "copy"]
|
|
157
163
|
: getVideoEncodeArgs({ preset: "medium", crf: VIDEO_CRF });
|
|
158
|
-
const encodeAudio = ["-c:a", "aac", "-b:a",
|
|
164
|
+
const encodeAudio = ["-c:a", "aac", "-b:a", FINISHED_AUDIO_BITRATE, "-ar", String(FOOTAGE_SAMPLE_RATE), "-ac", "2"];
|
|
165
|
+
const inputs = options.audioFile === undefined ? ["-i", input] : ["-i", input, "-i", options.audioFile];
|
|
159
166
|
const audio =
|
|
160
167
|
facts.audio === null
|
|
161
168
|
? []
|
|
162
|
-
: options.
|
|
163
|
-
? ["-map", "
|
|
164
|
-
:
|
|
165
|
-
? ["-map", "0:a:0", "-
|
|
166
|
-
:
|
|
167
|
-
|
|
169
|
+
: options.audioFile !== undefined
|
|
170
|
+
? ["-map", "1:a:0", "-c:a", "copy"]
|
|
171
|
+
: options.audioFilter !== undefined
|
|
172
|
+
? ["-map", "0:a:0", "-af", options.audioFilter, ...encodeAudio]
|
|
173
|
+
: isPublishableAudio(facts.audio)
|
|
174
|
+
? ["-map", "0:a:0", "-c:a", "copy"]
|
|
175
|
+
: ["-map", "0:a:0", ...encodeAudio];
|
|
176
|
+
return ["-v", "error", "-y", ...inputs, "-map", "0:v:0", ...video, ...audio, "-movflags", "+faststart", output];
|
|
168
177
|
}
|
|
169
178
|
|
|
170
179
|
/** The types of an MP4's top-level boxes, in file order (`ftyp`, `moov`, `mdat`, …). */
|