@softure-ai/marketing-kit 0.11.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/CHANGELOG.md +85 -0
  2. package/README.md +67 -9
  3. package/dist/cli/footage.js +1 -1
  4. package/dist/cli/footage.js.map +1 -1
  5. package/dist/compose/timeline.d.ts +23 -3
  6. package/dist/compose/timeline.d.ts.map +1 -1
  7. package/dist/compose/timeline.js +31 -11
  8. package/dist/compose/timeline.js.map +1 -1
  9. package/dist/config/actions-schema.d.ts +14 -14
  10. package/dist/config/config.d.ts.map +1 -1
  11. package/dist/config/config.js +2 -0
  12. package/dist/config/config.js.map +1 -1
  13. package/dist/config/schema.d.ts +83 -70
  14. package/dist/config/schema.d.ts.map +1 -1
  15. package/dist/config/schema.js +21 -1
  16. package/dist/config/schema.js.map +1 -1
  17. package/dist/config/shot-steps.d.ts +16 -16
  18. package/dist/footage/captions.d.ts +45 -3
  19. package/dist/footage/captions.d.ts.map +1 -1
  20. package/dist/footage/captions.js +65 -5
  21. package/dist/footage/captions.js.map +1 -1
  22. package/dist/footage/compose.d.ts.map +1 -1
  23. package/dist/footage/compose.js +8 -4
  24. package/dist/footage/compose.js.map +1 -1
  25. package/dist/footage/declick.d.ts +106 -0
  26. package/dist/footage/declick.d.ts.map +1 -0
  27. package/dist/footage/declick.js +296 -0
  28. package/dist/footage/declick.js.map +1 -0
  29. package/dist/footage/draft.d.ts +8 -3
  30. package/dist/footage/draft.d.ts.map +1 -1
  31. package/dist/footage/draft.js +55 -22
  32. package/dist/footage/draft.js.map +1 -1
  33. package/dist/footage/edl.d.ts +28 -2
  34. package/dist/footage/edl.d.ts.map +1 -1
  35. package/dist/footage/edl.js +53 -2
  36. package/dist/footage/edl.js.map +1 -1
  37. package/dist/footage/film.d.ts +29 -3
  38. package/dist/footage/film.d.ts.map +1 -1
  39. package/dist/footage/film.js +12 -0
  40. package/dist/footage/film.js.map +1 -1
  41. package/dist/footage/isolate-ranges.d.ts +37 -0
  42. package/dist/footage/isolate-ranges.d.ts.map +1 -0
  43. package/dist/footage/isolate-ranges.js +113 -0
  44. package/dist/footage/isolate-ranges.js.map +1 -0
  45. package/dist/footage/loudness.d.ts +76 -0
  46. package/dist/footage/loudness.d.ts.map +1 -1
  47. package/dist/footage/loudness.js +134 -0
  48. package/dist/footage/loudness.js.map +1 -1
  49. package/dist/footage/paid.d.ts +18 -0
  50. package/dist/footage/paid.d.ts.map +1 -1
  51. package/dist/footage/paid.js +32 -2
  52. package/dist/footage/paid.js.map +1 -1
  53. package/dist/footage/pipeline.d.ts +43 -5
  54. package/dist/footage/pipeline.d.ts.map +1 -1
  55. package/dist/footage/pipeline.js +105 -20
  56. package/dist/footage/pipeline.js.map +1 -1
  57. package/dist/footage/providers.d.ts +5 -0
  58. package/dist/footage/providers.d.ts.map +1 -1
  59. package/dist/footage/publish.d.ts +11 -2
  60. package/dist/footage/publish.d.ts.map +1 -1
  61. package/dist/footage/publish.js +14 -10
  62. package/dist/footage/publish.js.map +1 -1
  63. package/dist/footage/speech-ranges.d.ts +65 -0
  64. package/dist/footage/speech-ranges.d.ts.map +1 -0
  65. package/dist/footage/speech-ranges.js +98 -0
  66. package/dist/footage/speech-ranges.js.map +1 -0
  67. package/dist/index.d.ts +10 -7
  68. package/dist/index.d.ts.map +1 -1
  69. package/dist/index.js +10 -7
  70. package/dist/index.js.map +1 -1
  71. package/package.json +1 -1
  72. package/schema/edl.schema.json +17 -0
  73. package/schema/marketing.schema.json +43 -1
  74. package/src/cli/footage.ts +1 -1
  75. package/src/compose/timeline.ts +41 -10
  76. package/src/config/config.ts +2 -0
  77. package/src/config/schema.ts +23 -1
  78. package/src/footage/captions.ts +81 -6
  79. package/src/footage/compose.ts +8 -4
  80. package/src/footage/declick.ts +353 -0
  81. package/src/footage/draft.ts +58 -23
  82. package/src/footage/edl.ts +69 -2
  83. package/src/footage/film.ts +37 -3
  84. package/src/footage/isolate-ranges.ts +161 -0
  85. package/src/footage/loudness.ts +170 -0
  86. package/src/footage/paid.ts +40 -2
  87. package/src/footage/pipeline.ts +158 -22
  88. package/src/footage/providers.ts +2 -0
  89. package/src/footage/publish.ts +20 -11
  90. package/src/footage/speech-ranges.ts +131 -0
  91. package/src/index.ts +75 -0
@@ -27,6 +27,16 @@ export interface EdlAudio {
27
27
  ownVolume: number;
28
28
  }
29
29
 
30
+ /**
31
+ * A segment's own caption box, merged over the film's (`getFootageCaptionLayout`) for the captions that start in it.
32
+ * Px of the film's frame, as `CaptionLayout`: `bottom` anchors the box by its foot (above the frame's foot) and wins;
33
+ * `top` alone hangs the box from that top.
34
+ */
35
+ export interface EdlCaption {
36
+ bottom?: number;
37
+ top?: number;
38
+ }
39
+
30
40
  export interface EdlSegment {
31
41
  /** Unique in the cut: `s<n>` (the sentence it starts with), `r<n>` (b-roll), `w<n>` (a talking clip without words). */
32
42
  id: string;
@@ -39,6 +49,8 @@ export interface EdlSegment {
39
49
  /** `kenburns`: a slow zoom over a segment without speech. */
40
50
  motion: SegmentMotion;
41
51
  audio?: EdlAudio;
52
+ /** Moves the caption box for the captions that start in this segment; absent: the film's box. */
53
+ caption?: EdlCaption;
42
54
  }
43
55
 
44
56
  export interface Edl {
@@ -75,6 +87,13 @@ export const edlSchema = z
75
87
  })
76
88
  .optional()
77
89
  .describe("Sound from another clip or span: a J- or L-cut, or b-roll over speech. Without it, the segment plays its own sound."),
90
+ caption: z
91
+ .strictObject({
92
+ bottom: z.number().min(0).optional().describe("The caption box's foot, in px above the frame's foot; anchors the box by its foot and wins over top."),
93
+ top: z.number().min(0).optional().describe("The caption box's top, in px below the frame's top; used when bottom is absent."),
94
+ })
95
+ .optional()
96
+ .describe("Moves the caption box for the captions that start in this segment (px of the film's frame). Without it, the film's caption box."),
78
97
  }),
79
98
  )
80
99
  .min(1, "the cut needs at least one segment")
@@ -221,11 +240,18 @@ export function getSoundSpan(segment: EdlSegment): { clip: string; start: number
221
240
  return segment.audio ?? { clip: segment.clip, start: segment.start, end: segment.end };
222
241
  }
223
242
 
243
+ /** The frame a segment's caption box must stay in: its height and the height of a one-line caption card, in px. */
244
+ export interface EdlCaptionBounds {
245
+ frameHeight: number;
246
+ cardHeight: number;
247
+ }
248
+
224
249
  /**
225
250
  * What makes an EDL (a hand-edited one above all) unusable with these clips; empty when it is fine: another film,
226
- * an unknown clip, a span past a clip's end, a too short segment, a repeated id, a sound span of another length.
251
+ * an unknown clip, a span past a clip's end, a too short segment, a repeated id, a sound span of another length, and
252
+ * (with `captionBounds`) a segment's caption box that would leave the frame.
227
253
  */
228
- export function validateEdl(edl: Edl, clips: Pick<EdlClip, "id" | "duration">[], filmId: string): string[] {
254
+ export function validateEdl(edl: Edl, clips: Pick<EdlClip, "id" | "duration">[], filmId: string, captionBounds?: EdlCaptionBounds): string[] {
229
255
  const problems: string[] = [];
230
256
  if (edl.film !== filmId) problems.push(`it belongs to film "${edl.film}", not "${filmId}"`);
231
257
  const durations = new Map(clips.map((clip) => [clip.id, clip.duration]));
@@ -241,6 +267,7 @@ export function validateEdl(edl: Edl, clips: Pick<EdlClip, "id" | "duration">[],
241
267
  seen.add(segment.id);
242
268
  checkSpan(at, segment);
243
269
  if (segment.end - segment.start < MIN_SEGMENT_SECONDS) problems.push(`${at}: shorter than ${MIN_SEGMENT_SECONDS} s (start ${segment.start}, end ${segment.end})`);
270
+ if (captionBounds !== undefined) problems.push(...findCaptionProblems(`${at}.caption`, segment.caption, captionBounds));
244
271
  const { audio } = segment;
245
272
  if (audio === undefined) return;
246
273
  checkSpan(`${at}.audio`, audio);
@@ -251,6 +278,19 @@ export function validateEdl(edl: Edl, clips: Pick<EdlClip, "id" | "duration">[],
251
278
  return problems;
252
279
  }
253
280
 
281
+ /** A caption box's edges past the frame: each edge at most the frame's height minus one card, so a card always fits. */
282
+ function findCaptionProblems(at: string, caption: EdlCaption | undefined, bounds: EdlCaptionBounds): string[] {
283
+ const limit = Math.round(bounds.frameHeight - bounds.cardHeight);
284
+ const problems: string[] = [];
285
+ for (const edge of ["bottom", "top"] as const) {
286
+ const value = caption?.[edge];
287
+ if (value === undefined) continue;
288
+ if (value < 0) problems.push(`${at}.${edge}: ${value} px is outside the frame`);
289
+ else if (value > limit) problems.push(`${at}.${edge}: ${value} px leaves no room for a caption card (at most ${limit} px in a ${bounds.frameHeight} px frame)`);
290
+ }
291
+ return problems;
292
+ }
293
+
254
294
  /** `validateEdl` under its first name. */
255
295
  export const findEdlProblems = validateEdl;
256
296
 
@@ -326,6 +366,11 @@ export function diffEdl(before: Edl, after: Edl): string[] {
326
366
  if (a.transition !== b.transition) lines.push(`${id}: ${a.transition} → ${b.transition}`);
327
367
  if (a.motion !== b.motion) lines.push(`${id}: motion ${a.motion} → ${b.motion}`);
328
368
  }
369
+ for (const segment of after.segments) {
370
+ const previous = old.get(segment.id);
371
+ if (previous === undefined || isSameCaption(previous.caption, segment.caption)) continue;
372
+ lines.push(isMovedCaption(segment.caption) ? `${segment.id} captions moved` : `${segment.id} captions back in the film's box`);
373
+ }
329
374
  for (const segment of after.segments) {
330
375
  const was = old.get(segment.id)?.audio;
331
376
  const { audio } = segment;
@@ -341,10 +386,32 @@ export function diffEdl(before: Edl, after: Edl): string[] {
341
386
  return lines;
342
387
  }
343
388
 
389
+ /** Whether a segment's caption field moves its box (an empty object does not). */
390
+ function isMovedCaption(caption: EdlCaption | undefined): boolean {
391
+ return caption?.bottom !== undefined || caption?.top !== undefined;
392
+ }
393
+
394
+ function isSameCaption(a: EdlCaption | undefined, b: EdlCaption | undefined): boolean {
395
+ return a?.bottom === b?.bottom && a?.top === b?.top;
396
+ }
397
+
344
398
  export function getEdlDuration(edl: Edl): number {
345
399
  return r3(edl.segments.reduce((sum, segment) => sum + (segment.end - segment.start), 0));
346
400
  }
347
401
 
402
+ /**
403
+ * The index of the segment that plays at `time` (film seconds): the one whose span holds it; a time before the film
404
+ * gives the first segment, one at or past the cut's end the last.
405
+ */
406
+ export function findSegmentAt(edl: Edl, time: number): number {
407
+ const starts = getSegmentStarts(edl);
408
+ let found = 0;
409
+ starts.forEach((start, index) => {
410
+ if (start <= time) found = index;
411
+ });
412
+ return found;
413
+ }
414
+
348
415
  /** Where each segment starts in the film, in seconds. */
349
416
  export function getSegmentStarts(edl: Edl): number[] {
350
417
  const starts: number[] = [];
@@ -1,6 +1,7 @@
1
1
  import type { VideoFormat } from "../compose/timeline.js";
2
2
  import type { EndCard } from "../film.js";
3
- import type { LoudnessTarget } from "./loudness.js";
3
+ import type { DeclickLevel } from "./declick.js";
4
+ import type { LoudnessTarget, MixLoudnessTarget } from "./loudness.js";
4
5
 
5
6
  /**
6
7
  * A footage film: filmed clips (a person talking to camera, b-roll) cut by their transcript into a captioned
@@ -131,18 +132,28 @@ export interface FootageFilm {
131
132
  * `CAPTION_MIN_FONT_SCALE` of the box's size) until it fits. Absent: no limit. The draft keeps its font size.
132
133
  */
133
134
  captionMaxLines?: number;
135
+ /**
136
+ * Words a caption card must not end on: the card hands them to the next one, so `w` stays with `domu`. Compared
137
+ * lower-case; a number always keeps with the word after it. Absent: the language's `DEFAULT_CAPTION_KEEP_WITH_NEXT`;
138
+ * empty: numbers only.
139
+ */
140
+ captionKeepWithNext?: string[];
134
141
  /** The cut's look: zoom size, fade length, Ken Burns, the end card's whoosh. Absent fields take `DEFAULT_EDIT`. */
135
142
  edit?: Partial<FootageEdit>;
136
143
  /** The closing card after the last segment; null: the film ends with its last segment. */
137
144
  endCard: EndCard | null;
145
+ /** The speech track's cleanup before its loudness normalization. Absent: none. */
146
+ speech?: FootageSpeech;
138
147
  /** The speech track's loudness, normalized before the music is mixed under it. Absent: −14 LUFS, −1 dBTP; null: left as filmed. */
139
148
  loudness?: LoudnessTarget | null;
140
149
  /**
141
150
  * The finished film's mix (speech, music, sound effects), measured after the render and normalized in two passes of
142
151
  * `loudnorm` (one linear gain; the picture is copied, only the sound re-encoded), so the published file plays at the
143
- * target. A silent mix is left as it is. Absent: −14 LUFS, −1 dBTP; null: the mix as rendered.
152
+ * target. A silent mix is left as it is. Absent: −14 LUFS, −1 dBTP; null: the mix as rendered. `keep: "speech"`
153
+ * applies no gain while the mix measures inside `target ± tolerance` (default 1 LU), so the speech keeps its level
154
+ * whatever the music volume (see `MixLoudnessTarget`).
144
155
  */
145
- mixLoudness?: LoudnessTarget | null;
156
+ mixLoudness?: MixLoudnessTarget | null;
146
157
  /**
147
158
  * The cover JPG written next to the film. Absent or without `at`: the middle of the first talking segment; `at`:
148
159
  * that second of the film; null: no cover.
@@ -150,12 +161,35 @@ export interface FootageFilm {
150
161
  cover?: { at?: number } | null;
151
162
  }
152
163
 
164
+ /** Settings of the speech track itself. */
165
+ export interface FootageSpeech {
166
+ /**
167
+ * Mouth clicks and lip smacks between words (issue #1084): `gentle` lowers the pauses by 15 dB and removes impulses
168
+ * that jump 15 dB above the band's median, `strong` by 20 dB and 9 dB. Absent or `off`: no pass, the track unchanged.
169
+ */
170
+ declick?: DeclickLevel;
171
+ }
172
+
173
+ /** The film's declick level; `off` when it sets none. */
174
+ export function getDeclickLevel(film: Pick<FootageFilm, "speech">): DeclickLevel {
175
+ return film.speech?.declick ?? "off";
176
+ }
177
+
153
178
  /** Filler words per language, written with repeated letters collapsed (`yyy` and `yyyy` are both `y`). */
154
179
  export const DEFAULT_FILLERS: Record<string, string[]> = {
155
180
  pl: ["y", "e", "yhm", "yh", "ym", "em", "hm", "mhm", "eh", "uh"],
156
181
  en: ["um", "uh", "uhm", "er", "erm", "hm", "mhm", "ah"],
157
182
  };
158
183
 
184
+ /**
185
+ * Words a caption card does not end on, per language: Polish one-letter words and short prepositions, English
186
+ * articles. A language without a list keeps numbers only.
187
+ */
188
+ export const DEFAULT_CAPTION_KEEP_WITH_NEXT: Record<string, string[]> = {
189
+ pl: ["a", "i", "o", "u", "w", "z", "we", "ze", "na", "do", "od", "po", "za", "ku"],
190
+ en: ["a", "an", "the"],
191
+ };
192
+
159
193
  /** The default fillers of a language; none for a language without a list. */
160
194
  export function getDefaultFillers(language: string): string[] {
161
195
  return DEFAULT_FILLERS[language] ?? [];
@@ -0,0 +1,161 @@
1
+ import { existsSync, mkdirSync } from "node:fs";
2
+ import { join } from "node:path";
3
+
4
+ import { err, ok } from "../voice/provider.js";
5
+ import { probeAudio } from "./audio.js";
6
+ import { writeCacheFile } from "./cache.js";
7
+ import type { Edl } from "./edl.js";
8
+ import type { FootageFilm } from "./film.js";
9
+ import { getClipCacheKey } from "./paid.js";
10
+ import { getFfmpegReporting, type FootageResult, type FootageRunOptions, type PreparedClip } from "./pipeline.js";
11
+ import { runFfmpeg, withFfmpegFailureReporting } from "./process.js";
12
+ import { toPaidAudio } from "./providers.js";
13
+ import {
14
+ buildRangeAudioArgs,
15
+ buildRangeLayArgs,
16
+ getSpeechRangeCache,
17
+ getSpeechRangeWindows,
18
+ planSpeechRanges,
19
+ type LaidRange,
20
+ type SpeechRange,
21
+ } from "./speech-ranges.js";
22
+
23
+ /**
24
+ * Voice isolation of only what the cut hears (issue #1087): each talking clip's sound ranges (`planSpeechRanges`) are
25
+ * cleaned one by one and cached by the clip's identity, the range in milliseconds and the provider, so an unchanged
26
+ * segment is never paid twice and a changed cut pays only its new ranges. The cleaned ranges are laid back over the
27
+ * clip's own sound into one file as long as the clip: the speech map `renderFootage` takes.
28
+ */
29
+
30
+ export type IsolateSpeechRangesOptions = Pick<FootageRunOptions, "buildDir" | "cacheDir" | "providers" | "isCommit" | "signal" | "onCharge" | "onProgress" | "onFfmpegFailure"> & {
31
+ /** Seconds added on each side of a sound span; default `DEFAULT_SPEECH_RANGE_MARGIN` (0.25). */
32
+ margin?: number;
33
+ log?: (line: string) => void;
34
+ };
35
+
36
+ export interface IsolatedSpeechRange extends SpeechRange {
37
+ /** The paid cache's key and file of the range's cleaned voice. */
38
+ cacheKey: string;
39
+ path: string;
40
+ /** Whether the cleaned voice was in the cache before this call. */
41
+ cached: boolean;
42
+ /** Whether it is laid in the clip's speech file (cached, or bought with `isCommit`). */
43
+ laid: boolean;
44
+ }
45
+
46
+ export interface IsolatedSpeech {
47
+ /** Every clip's audio for the speech track: its laid file, or the normalized clip when none of its ranges is laid. */
48
+ speech: Record<string, string>;
49
+ ranges: IsolatedSpeechRange[];
50
+ }
51
+
52
+ /**
53
+ * Cleans the voice of the ranges the cut hears and answers the speech map the render takes (`TranscribeOutcome.speech`).
54
+ * With `isCommit` every uncached range is bought (`onCharge` step `isolate`, item the clip id, with its `range`);
55
+ * without it only cached ranges are laid, a free draft, and `ranges` names what is still missing. Each clip with a laid
56
+ * range gets `<buildDir>/audio/<id>.ranges.mp3`.
57
+ */
58
+ export function isolateSpeechRanges(film: FootageFilm, clips: PreparedClip[], edl: Edl, options: IsolateSpeechRangesOptions): Promise<FootageResult<IsolatedSpeech>> {
59
+ const log = options.log ?? ((): void => undefined);
60
+ return withFfmpegFailureReporting(getFfmpegReporting({ log, ...(options.onFfmpegFailure === undefined ? {} : { onFfmpegFailure: options.onFfmpegFailure }) }), () =>
61
+ runIsolateSpeechRanges(film, clips, edl, options, log),
62
+ );
63
+ }
64
+
65
+ /** The body of `isolateSpeechRanges`, inside its ffmpeg failure reporting. */
66
+ async function runIsolateSpeechRanges(
67
+ film: FootageFilm,
68
+ clips: PreparedClip[],
69
+ edl: Edl,
70
+ options: IsolateSpeechRangesOptions,
71
+ log: (line: string) => void,
72
+ ): Promise<FootageResult<IsolatedSpeech>> {
73
+ const { buildDir, providers, signal } = options;
74
+ const aborting = signal === undefined ? {} : { signal };
75
+ const planned = planSpeechRanges(
76
+ edl,
77
+ clips.map((clip) => ({ id: clip.clip.id, role: clip.clip.role, duration: clip.duration })),
78
+ options.margin === undefined ? {} : { margin: options.margin },
79
+ );
80
+ const byClip = new Map(clips.map((clip) => [clip.clip.id, clip]));
81
+ const clipKeys = new Map<string, string>();
82
+ const ranges: IsolatedSpeechRange[] = [];
83
+ for (const range of planned) {
84
+ const clip = byClip.get(range.clip);
85
+ if (clip === undefined) continue;
86
+ let clipKey = clipKeys.get(range.clip);
87
+ if (clipKey === undefined) {
88
+ clipKey = await getClipCacheKey(clip.source, clip.clip);
89
+ clipKeys.set(range.clip, clipKey);
90
+ }
91
+ const cache = getSpeechRangeCache(film.id, clipKey, range, { cacheDir: options.cacheDir, provider: providers.isolation.id });
92
+ const cached = existsSync(cache.path);
93
+ ranges.push({ ...range, cacheKey: cache.key, path: cache.path, cached, laid: cached });
94
+ }
95
+ const buying = options.isCommit ? ranges.filter((range) => !range.cached) : [];
96
+ const total = buying.reduce((sum, range) => sum + (range.endMs - range.startMs) / 1000, 0);
97
+ let done = 0;
98
+ const report = (range: SpeechRange, index: number): void => {
99
+ options.onProgress?.({ step: "isolate", item: range.clip, itemIndex: index + 1, itemCount: buying.length, done: Math.round(done * 1000) / 1000, total: Math.round(total * 1000) / 1000 });
100
+ };
101
+ const workDir = join(buildDir, "audio", "ranges");
102
+ for (const [index, range] of buying.entries()) {
103
+ const clip = byClip.get(range.clip);
104
+ if (clip === undefined) continue;
105
+ const units = (range.endMs - range.startMs) / 1000;
106
+ report(range, index);
107
+ mkdirSync(workDir, { recursive: true });
108
+ // The provider sees the clip's id and the range as the file name.
109
+ const input = join(workDir, `${range.clip}.${String(range.startMs)}-${String(range.endMs)}.mp3`);
110
+ await runFfmpeg(`Cutting ${describeRange(range)} for voice isolation`, buildRangeAudioArgs(clip.file, range, input), { ...aborting, output: input });
111
+ signal?.throwIfAborted();
112
+ log(`voice isolation (${providers.isolation.id}): ${describeRange(range)}, ${units.toFixed(1)} s.`);
113
+ const cleaned = await providers.isolation.isolate({ audio: input, ...aborting });
114
+ signal?.throwIfAborted();
115
+ if (!cleaned.ok) return err(`${describeRange(range)}: ${cleaned.error}`);
116
+ const paid = toPaidAudio(cleaned.value);
117
+ writeCacheFile(range.path, paid.audio);
118
+ range.laid = true;
119
+ options.onCharge?.({
120
+ step: "isolate",
121
+ item: range.clip,
122
+ provider: providers.isolation.id,
123
+ units,
124
+ ...(paid.credits === undefined ? {} : { credits: paid.credits }),
125
+ cacheKey: range.cacheKey,
126
+ range: { start: range.startMs / 1000, end: range.endMs / 1000 },
127
+ });
128
+ log(`voice isolation: saved ${range.path} (commit it).`);
129
+ done += units;
130
+ report(range, index);
131
+ }
132
+ const missing = ranges.filter((range) => !range.laid);
133
+ if (missing.length > 0) {
134
+ const seconds = missing.reduce((sum, range) => sum + (range.endMs - range.startMs) / 1000, 0);
135
+ log(`voice isolation DRY RUN: ${String(missing.length)} range${missing.length === 1 ? "" : "s"} (${seconds.toFixed(1)} s) not cached, laid with the clip's own sound. Run with --commit to pay.`);
136
+ }
137
+ const speech: Record<string, string> = {};
138
+ for (const clip of clips) {
139
+ const id = clip.clip.id;
140
+ speech[id] = clip.file;
141
+ const laidRanges = ranges.filter((range) => range.clip === id && range.laid);
142
+ if (laidRanges.length === 0) continue;
143
+ const laid: LaidRange[] = [];
144
+ for (const window of getSpeechRangeWindows(laidRanges)) {
145
+ const file = laidRanges.find((range) => range === window.range)?.path;
146
+ if (file === undefined) continue;
147
+ const probed = await probeAudio(file, aborting);
148
+ if (!probed.ok) return err(`${describeRange(window.range)}: the cleaned voice at ${file} does not read (${probed.error}); delete it and run again.`);
149
+ laid.push({ window, file, isMono: probed.probe.channels === 1 });
150
+ }
151
+ const output = join(buildDir, "audio", `${id}.ranges.mp3`);
152
+ mkdirSync(join(buildDir, "audio"), { recursive: true });
153
+ await runFfmpeg(`Laying the cleaned voice over clip ${id}`, buildRangeLayArgs(clip.file, laid, output), { ...aborting, output });
154
+ speech[id] = output;
155
+ }
156
+ return ok({ speech, ranges });
157
+ }
158
+
159
+ function describeRange(range: SpeechRange): string {
160
+ return `clip ${range.clip} ${(range.startMs / 1000).toFixed(3)}-${(range.endMs / 1000).toFixed(3)} s`;
161
+ }
@@ -4,6 +4,7 @@ import { z } from "zod";
4
4
 
5
5
  import { FOOTAGE_SAMPLE_RATE } from "./normalize.js";
6
6
  import { runFfmpeg, runProcess } from "./process.js";
7
+ import { FINISHED_AUDIO_BITRATE } from "./publish.js";
7
8
 
8
9
  /**
9
10
  * The speech track's loudness, measured and normalized in two passes of ffmpeg's `loudnorm` (EBU R128): the first
@@ -163,3 +164,172 @@ export async function normalizeLoudness(input: string, output: string, target: L
163
164
  });
164
165
  return { kind: "normalized", measured: measurement.inputI };
165
166
  }
167
+
168
+ /** The film's mix as measured on an encoded file: integrated loudness in LUFS, true peak in dBTP. */
169
+ export interface MixLoudness {
170
+ integrated: number;
171
+ truePeak: number;
172
+ }
173
+
174
+ /** How far from `LoudnessTarget.target` the finished mix may land, in LU, unless `MixLoudnessTarget.tolerance` says. */
175
+ export const MIX_LOUDNESS_TOLERANCE = 1;
176
+
177
+ /**
178
+ * The finished film's mix target. `keep: "loudness"` (the default) brings the mix to `target` with one gain measured on
179
+ * it, so the speech moves with whatever the music adds; `keep: "speech"` applies no gain while the mix measures inside
180
+ * `target ± tolerance` and otherwise only the smallest gain to that window, so the speech (already at the film's
181
+ * `loudness`) keeps its level whatever the music volume. Either way a true peak limiter is the last step.
182
+ */
183
+ export interface MixLoudnessTarget extends LoudnessTarget {
184
+ /** How far from `target` the mix may land, in LU. Absent: `MIX_LOUDNESS_TOLERANCE`. */
185
+ tolerance?: number;
186
+ /** What the mix's gain keeps: the mix at the target (absent, `"loudness"`) or the speech's level (`"speech"`). */
187
+ keep?: "loudness" | "speech";
188
+ }
189
+
190
+ /** A gain to the window's edge aims this far inside it (at most half the tolerance), so the encode's drift stays in. */
191
+ const MIX_EDGE_MARGIN = 0.2;
192
+ /** The limiter's first ceiling sits this far under the true peak limit: AAC adds inter-sample overshoot after it. */
193
+ export const MIX_ENCODE_HEADROOM = 1;
194
+ /** How many times the mix is encoded and measured before the film fails. */
195
+ export const MIX_ATTEMPTS = 4;
196
+ /** What a retry lowers the ceiling by on top of the measured overshoot, in dB. */
197
+ const MIX_RETRY_MARGIN = 0.3;
198
+ /** With `keep: "speech"`, a retry aims this far inside the window's edge (at most half the tolerance), in LU. */
199
+ export const MIX_RETRY_EDGE_MARGIN = 0.3;
200
+ /** The limiter's loudness cost per dB of drive (gain minus ceiling) until the tries measure it, in LU per dB. */
201
+ export const MIX_LIMITER_SLOPE = 0.5;
202
+ /** The steepest limiter cost a retry believes: near 1 a gain step would barely move the mix and the gain would run away. */
203
+ export const MIX_LIMITER_SLOPE_MAX = 0.7;
204
+ /** How far apart the tries' drives must spread (their variance, in dB²) before the slope is fitted from them. */
205
+ const MIX_SLOPE_SPREAD = 0.09;
206
+ /** `alimiter`'s lowest ceiling (its `limit` of 0.0625), in dB. */
207
+ const LIMITER_FLOOR = -24;
208
+ /** The limiter runs at 4x the film's rate, so it holds the true peak rather than the sample peak. */
209
+ const LIMITER_RATE = FOOTAGE_SAMPLE_RATE * 4;
210
+
211
+ /** One try at the mix's last stage: a linear gain, then a true peak limiter at the ceiling. */
212
+ export interface MixAttempt {
213
+ gainDb: number;
214
+ ceilingDbtp: number;
215
+ }
216
+
217
+ /** One encode of the mix: the attempt and what the encoded file measured. */
218
+ export interface MixTry {
219
+ attempt: MixAttempt;
220
+ measured: MixLoudness;
221
+ }
222
+
223
+ function roundTo(value: number, digits: number): number {
224
+ const scale = 10 ** digits;
225
+ return Math.round(value * scale) / scale;
226
+ }
227
+
228
+ /** The window's half width around the mix target, in LU. */
229
+ export function getMixTolerance(target: MixLoudnessTarget): number {
230
+ return target.tolerance ?? MIX_LOUDNESS_TOLERANCE;
231
+ }
232
+
233
+ /**
234
+ * The gain a mix measuring `integrated` LUFS needs: to the target in the default mode; with `keep: "speech"` none
235
+ * inside `target ± tolerance`, else to the nearest edge (aimed a little inside, see `MIX_EDGE_MARGIN`).
236
+ */
237
+ export function getMixGain(integrated: number, target: MixLoudnessTarget): number {
238
+ if (target.keep !== "speech") return roundTo(target.target - integrated, 2);
239
+ const tolerance = getMixTolerance(target);
240
+ const inside = tolerance - Math.min(MIX_EDGE_MARGIN, tolerance / 2);
241
+ if (integrated < target.target - tolerance) return roundTo(target.target - inside - integrated, 2);
242
+ if (integrated > target.target + tolerance) return roundTo(target.target + inside - integrated, 2);
243
+ return 0;
244
+ }
245
+
246
+ /** The first try: the gain the measured mix needs (`getMixGain`), the ceiling one encode's headroom under the limit. */
247
+ export function getFirstMixAttempt(measurement: Pick<LoudnessMeasurement, "inputI">, target: MixLoudnessTarget): MixAttempt {
248
+ return { gainDb: getMixGain(measurement.inputI, target), ceilingDbtp: Math.max(LIMITER_FLOOR, roundTo(target.truePeak - MIX_ENCODE_HEADROOM, 2)) };
249
+ }
250
+
251
+ /** One try as the limiter model sees it: how hard the limiter was driven (gain minus ceiling) and the loudness it took. */
252
+ function getLimiterPoint(entry: MixTry, inputI: number): { drive: number; cost: number } {
253
+ return { drive: entry.attempt.gainDb - entry.attempt.ceilingDbtp, cost: inputI + entry.attempt.gainDb - entry.measured.integrated };
254
+ }
255
+
256
+ /**
257
+ * How much loudness the limiter takes per dB it is driven harder, fitted (least squares) from the tries once their
258
+ * drives spread, else `MIX_LIMITER_SLOPE`; clamped to 0..`MIX_LIMITER_SLOPE_MAX`. `inputI` is the render's mix as measured.
259
+ */
260
+ export function getLimiterSlope(tries: readonly MixTry[], inputI: number): number {
261
+ const points = tries.map((entry) => getLimiterPoint(entry, inputI));
262
+ const meanDrive = points.reduce((sum, point) => sum + point.drive, 0) / Math.max(1, points.length);
263
+ const meanCost = points.reduce((sum, point) => sum + point.cost, 0) / Math.max(1, points.length);
264
+ const spread = points.reduce((sum, point) => sum + (point.drive - meanDrive) ** 2, 0);
265
+ const slope = points.length >= 2 && spread >= MIX_SLOPE_SPREAD ? points.reduce((sum, point) => sum + (point.drive - meanDrive) * (point.cost - meanCost), 0) / spread : MIX_LIMITER_SLOPE;
266
+ return roundTo(Math.min(MIX_LIMITER_SLOPE_MAX, Math.max(0, slope)), 3);
267
+ }
268
+
269
+ /**
270
+ * Where a retry aims the mix's integrated loudness when it would land at `predicted` LUFS with the gain kept: the
271
+ * target in the default mode; with `keep: "speech"` nowhere (the gain stays) while `predicted` is inside the window
272
+ * shrunk by `MIX_RETRY_EDGE_MARGIN`, else that margin inside the edge it would miss.
273
+ */
274
+ function getRetryAim(predicted: number, target: MixLoudnessTarget): number | null {
275
+ if (target.keep !== "speech") return target.target;
276
+ const tolerance = getMixTolerance(target);
277
+ const inside = tolerance - Math.min(MIX_RETRY_EDGE_MARGIN, tolerance / 2);
278
+ if (predicted < target.target - inside) return target.target - inside;
279
+ if (predicted > target.target + inside) return target.target + inside;
280
+ return null;
281
+ }
282
+
283
+ /**
284
+ * The next try after the encodes in `tries` (oldest first; the last one missed `target`). The ceiling is lowered by
285
+ * the last overshoot plus a margin when its true peak is over (never under the limiter's floor). The gain comes from
286
+ * a model of the limiter: the loudness it takes grows with how hard it is driven (gain minus ceiling) at the slope
287
+ * `getLimiterSlope` measures, so the gain is solved for the loudness the mix will have at the new ceiling, not the
288
+ * one it had at the old one. `inputI` is the render's mix as measured before any try.
289
+ */
290
+ export function getNextMixAttempt(tries: readonly MixTry[], inputI: number, target: MixLoudnessTarget): MixAttempt {
291
+ const last = tries.at(-1);
292
+ if (last === undefined) throw new Error("getNextMixAttempt: no encode of the mix to correct.");
293
+ const overshoot = last.measured.truePeak - target.truePeak;
294
+ const ceilingDbtp = overshoot > 0 ? Math.max(LIMITER_FLOOR, roundTo(last.attempt.ceilingDbtp - overshoot - MIX_RETRY_MARGIN, 2)) : last.attempt.ceilingDbtp;
295
+ const slope = getLimiterSlope(tries, inputI);
296
+ const { drive, cost } = getLimiterPoint(last, inputI);
297
+ // The limiter's cost at a gain, at the new ceiling: never below none.
298
+ const getCost = (gainDb: number): number => Math.max(0, cost + slope * (gainDb - ceilingDbtp - drive));
299
+ const aim = getRetryAim(inputI + last.attempt.gainDb - getCost(last.attempt.gainDb), target);
300
+ if (aim === null) return { gainDb: last.attempt.gainDb, ceilingDbtp };
301
+ const gainDb = (aim - inputI + cost - slope * (ceilingDbtp + drive)) / (1 - slope);
302
+ // A gain so low that the limiter would take nothing: the mix moves by the gain alone.
303
+ return { gainDb: roundTo(getCost(gainDb) > 0 ? gainDb : aim - inputI, 2), ceilingDbtp };
304
+ }
305
+
306
+ /**
307
+ * The mix's last stage: one linear gain, then a lookahead limiter at 4x the film's rate (its delay compensated, so
308
+ * the sound stays in sync with the picture; no auto level), back to the film's rate.
309
+ */
310
+ export function getMixFilter(attempt: MixAttempt): string {
311
+ const limit = roundTo(10 ** (attempt.ceilingDbtp / 20), 5);
312
+ return [`volume=${attempt.gainDb}dB`, `aresample=${LIMITER_RATE}`, `alimiter=limit=${limit}:level=false:latency=true`, `aresample=${FOOTAGE_SAMPLE_RATE}`].join(",");
313
+ }
314
+
315
+ /** The mix's sound on its own through `filter`, encoded as the finished film carries it (AAC 192k, 48 kHz, stereo). */
316
+ export function buildMixArgs(options: { input: string; filter: string; output: string }): string[] {
317
+ return ["-v", "error", "-y", "-i", options.input, "-vn", "-map", "0:a:0", "-af", options.filter, "-c:a", "aac", "-b:a", FINISHED_AUDIO_BITRATE, "-ar", String(FOOTAGE_SAMPLE_RATE), "-ac", "2", options.output];
318
+ }
319
+
320
+ /** What keeps a measured mix outside the target, as sentences with both values; empty when it meets it. */
321
+ export function findLoudnessProblems(measured: MixLoudness, target: MixLoudnessTarget): string[] {
322
+ const problems: string[] = [];
323
+ const tolerance = getMixTolerance(target);
324
+ if (measured.truePeak > target.truePeak) problems.push(`true peak ${formatDb(measured.truePeak)} dBTP, over the ${formatDb(target.truePeak)} dBTP limit`);
325
+ if (Math.abs(measured.integrated - target.target) > tolerance) {
326
+ problems.push(`integrated ${formatDb(measured.integrated)} LUFS, outside ${formatDb(target.target)} ± ${tolerance} LUFS`);
327
+ }
328
+ return problems;
329
+ }
330
+
331
+ /** A level with one decimal and its sign (`+0.1`, `-14.0`). */
332
+ export function formatDb(value: number): string {
333
+ const text = value.toFixed(1);
334
+ return value > 0 && text !== "0.0" ? `+${text}` : text === "-0.0" ? "0.0" : text;
335
+ }
@@ -10,6 +10,7 @@ import { getFilmKeyTerms } from "./keyterms.js";
10
10
  import type { FootageClip, FootageFilm } from "./film.js";
11
11
  import { getTrimmedDuration, probeFootage } from "./probe.js";
12
12
  import type { FootageProviders, FootageRates, PaidStepKind } from "./providers.js";
13
+ import { getSpeechRangeCache, planSpeechRanges, type SpeechRangeClip } from "./speech-ranges.js";
13
14
 
14
15
  /**
15
16
  * Where each paid result of a footage film is cached, shared by the pipeline and by the questions an app asks before
@@ -158,6 +159,8 @@ export interface FootageEstimateItem {
158
159
  /** `units` × the step's rate; 0 when cached. */
159
160
  credits: number;
160
161
  cached: boolean;
162
+ /** The range of the clip an `isolate: "segments"` item prices, in seconds of the trimmed clip; absent: the whole clip. */
163
+ range?: { start: number; end: number };
161
164
  }
162
165
 
163
166
  export interface FootageEstimate {
@@ -168,6 +171,14 @@ export interface FootageEstimate {
168
171
  export interface EstimateFootageOptions extends FootageCacheOptions {
169
172
  rates: FootageRates;
170
173
  signal?: AbortSignal;
174
+ /**
175
+ * What voice isolation is priced on: `clips` (default) the whole talking clips when the film isolates the voice;
176
+ * `segments` only the ranges the cut hears (`isolateSpeechRanges`), one item per range, whatever `isolateVoice`
177
+ * says. `segments` needs the cut (`edl`, or `<buildDir>/edl.json`).
178
+ */
179
+ isolate?: "clips" | "segments";
180
+ /** Seconds around each sound span with `isolate: "segments"`; default `DEFAULT_SPEECH_RANGE_MARGIN`. */
181
+ margin?: number;
171
182
  }
172
183
 
173
184
  const round = (value: number, digits: number): number => {
@@ -200,6 +211,11 @@ export async function estimateFootageFilm(film: FootageFilm, options: EstimateFo
200
211
  const add = (step: PaidStepKind, item: string, provider: string, units: number, cached: boolean): void => {
201
212
  items.push({ step, item, provider, units: round(units, 3), unit: "seconds", credits: cached ? 0 : round(units * rates[step], 2), cached });
202
213
  };
214
+ const isSegments = options.isolate === "segments";
215
+ const edl = findEdl(options);
216
+ if (isSegments && edl === null) return err(`footage film ${film.id}: isolating segments needs the cut (edl, or edl.json in the build folder).`);
217
+ const clipKeys: Record<string, string | null> = {};
218
+ const rangeClips: SpeechRangeClip[] = [];
203
219
  let clipSeconds = 0;
204
220
  for (const { clip, probe } of probed.clips) {
205
221
  const seconds = getTrimmedDuration(clip, probe);
@@ -207,16 +223,38 @@ export async function estimateFootageFilm(film: FootageFilm, options: EstimateFo
207
223
  if (clip.role !== "talk") continue;
208
224
  const source = findLocalSource(clip, options.buildDir);
209
225
  const clipKey = source === null ? null : await getClipCacheKey(source, clip);
226
+ clipKeys[clip.id] = clipKey;
227
+ rangeClips.push({ id: clip.id, role: clip.role, duration: seconds });
210
228
  const isolated = clipKey === null ? null : getIsolatedCache(film, clipKey, options);
211
- if (film.isolateVoice) add("isolate", clip.id, options.providers.isolation.id, seconds, isolated !== null && existsSync(isolated.path));
229
+ if (film.isolateVoice && !isSegments) add("isolate", clip.id, options.providers.isolation.id, seconds, isolated !== null && existsSync(isolated.path));
212
230
  const transcript = clipKey === null ? null : getTranscriptCache(film, clipKey, options);
213
231
  add("transcribe", clip.id, options.providers.stt.id, seconds, transcript !== null && existsSync(transcript.path));
214
232
  }
233
+ if (isSegments && edl !== null) {
234
+ const provider = options.providers.isolation.id;
235
+ for (const range of planSpeechRanges(edl, rangeClips, options.margin === undefined ? {} : { margin: options.margin })) {
236
+ const clipKey = clipKeys[range.clip] ?? null;
237
+ const cached = clipKey !== null && existsSync(getSpeechRangeCache(film.id, clipKey, range, { cacheDir: options.cacheDir, provider }).path);
238
+ add("isolate", range.clip, provider, (range.endMs - range.startMs) / 1000, cached);
239
+ const item = items.at(-1);
240
+ if (item !== undefined) item.range = { start: range.startMs / 1000, end: range.endMs / 1000 };
241
+ }
242
+ }
215
243
  const music = film.music.source;
216
244
  if (music.kind === "prompt" || music.kind === "video") {
217
- const edl = findEdl(options);
218
245
  const seconds = edl === null ? clipSeconds + (film.endCard === null ? 0 : END_CARD_SECONDS) : getFootageFilmSeconds(film, edl);
219
246
  add("music", "film", options.providers.music.id, seconds, await isMusicCached(film, options));
220
247
  }
221
248
  return ok({ items, totalCredits: round(items.reduce((sum, item) => sum + item.credits, 0), 2) });
222
249
  }
250
+
251
+ /**
252
+ * What isolating only the cut's ranges will cost (`isolateSpeechRanges`), for a confirmation step: the `isolate` items
253
+ * of `estimateFootageFilm` with `isolate: "segments"`, one per range, a cached range at 0.
254
+ */
255
+ export async function estimateSpeechRanges(film: FootageFilm, options: Omit<EstimateFootageOptions, "isolate">): Promise<TtsResult<FootageEstimate>> {
256
+ const estimate = await estimateFootageFilm(film, { ...options, isolate: "segments" });
257
+ if (!estimate.ok) return estimate;
258
+ const items = estimate.value.items.filter((item) => item.step === "isolate");
259
+ return ok({ items, totalCredits: round(items.reduce((sum, item) => sum + item.credits, 0), 2) });
260
+ }