@softure-ai/marketing-kit 0.11.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/CHANGELOG.md +85 -0
  2. package/README.md +67 -9
  3. package/dist/cli/footage.js +1 -1
  4. package/dist/cli/footage.js.map +1 -1
  5. package/dist/compose/timeline.d.ts +23 -3
  6. package/dist/compose/timeline.d.ts.map +1 -1
  7. package/dist/compose/timeline.js +31 -11
  8. package/dist/compose/timeline.js.map +1 -1
  9. package/dist/config/actions-schema.d.ts +14 -14
  10. package/dist/config/config.d.ts.map +1 -1
  11. package/dist/config/config.js +2 -0
  12. package/dist/config/config.js.map +1 -1
  13. package/dist/config/schema.d.ts +83 -70
  14. package/dist/config/schema.d.ts.map +1 -1
  15. package/dist/config/schema.js +21 -1
  16. package/dist/config/schema.js.map +1 -1
  17. package/dist/config/shot-steps.d.ts +16 -16
  18. package/dist/footage/captions.d.ts +45 -3
  19. package/dist/footage/captions.d.ts.map +1 -1
  20. package/dist/footage/captions.js +65 -5
  21. package/dist/footage/captions.js.map +1 -1
  22. package/dist/footage/compose.d.ts.map +1 -1
  23. package/dist/footage/compose.js +8 -4
  24. package/dist/footage/compose.js.map +1 -1
  25. package/dist/footage/declick.d.ts +106 -0
  26. package/dist/footage/declick.d.ts.map +1 -0
  27. package/dist/footage/declick.js +296 -0
  28. package/dist/footage/declick.js.map +1 -0
  29. package/dist/footage/draft.d.ts +8 -3
  30. package/dist/footage/draft.d.ts.map +1 -1
  31. package/dist/footage/draft.js +55 -22
  32. package/dist/footage/draft.js.map +1 -1
  33. package/dist/footage/edl.d.ts +28 -2
  34. package/dist/footage/edl.d.ts.map +1 -1
  35. package/dist/footage/edl.js +53 -2
  36. package/dist/footage/edl.js.map +1 -1
  37. package/dist/footage/film.d.ts +29 -3
  38. package/dist/footage/film.d.ts.map +1 -1
  39. package/dist/footage/film.js +12 -0
  40. package/dist/footage/film.js.map +1 -1
  41. package/dist/footage/isolate-ranges.d.ts +37 -0
  42. package/dist/footage/isolate-ranges.d.ts.map +1 -0
  43. package/dist/footage/isolate-ranges.js +113 -0
  44. package/dist/footage/isolate-ranges.js.map +1 -0
  45. package/dist/footage/loudness.d.ts +76 -0
  46. package/dist/footage/loudness.d.ts.map +1 -1
  47. package/dist/footage/loudness.js +134 -0
  48. package/dist/footage/loudness.js.map +1 -1
  49. package/dist/footage/paid.d.ts +18 -0
  50. package/dist/footage/paid.d.ts.map +1 -1
  51. package/dist/footage/paid.js +32 -2
  52. package/dist/footage/paid.js.map +1 -1
  53. package/dist/footage/pipeline.d.ts +43 -5
  54. package/dist/footage/pipeline.d.ts.map +1 -1
  55. package/dist/footage/pipeline.js +105 -20
  56. package/dist/footage/pipeline.js.map +1 -1
  57. package/dist/footage/providers.d.ts +5 -0
  58. package/dist/footage/providers.d.ts.map +1 -1
  59. package/dist/footage/publish.d.ts +11 -2
  60. package/dist/footage/publish.d.ts.map +1 -1
  61. package/dist/footage/publish.js +14 -10
  62. package/dist/footage/publish.js.map +1 -1
  63. package/dist/footage/speech-ranges.d.ts +65 -0
  64. package/dist/footage/speech-ranges.d.ts.map +1 -0
  65. package/dist/footage/speech-ranges.js +98 -0
  66. package/dist/footage/speech-ranges.js.map +1 -0
  67. package/dist/index.d.ts +10 -7
  68. package/dist/index.d.ts.map +1 -1
  69. package/dist/index.js +10 -7
  70. package/dist/index.js.map +1 -1
  71. package/package.json +1 -1
  72. package/schema/edl.schema.json +17 -0
  73. package/schema/marketing.schema.json +43 -1
  74. package/src/cli/footage.ts +1 -1
  75. package/src/compose/timeline.ts +41 -10
  76. package/src/config/config.ts +2 -0
  77. package/src/config/schema.ts +23 -1
  78. package/src/footage/captions.ts +81 -6
  79. package/src/footage/compose.ts +8 -4
  80. package/src/footage/declick.ts +353 -0
  81. package/src/footage/draft.ts +58 -23
  82. package/src/footage/edl.ts +69 -2
  83. package/src/footage/film.ts +37 -3
  84. package/src/footage/isolate-ranges.ts +161 -0
  85. package/src/footage/loudness.ts +170 -0
  86. package/src/footage/paid.ts +40 -2
  87. package/src/footage/pipeline.ts +158 -22
  88. package/src/footage/providers.ts +2 -0
  89. package/src/footage/publish.ts +20 -11
  90. package/src/footage/speech-ranges.ts +131 -0
  91. package/src/index.ts +75 -0
@@ -0,0 +1,353 @@
1
+ import { readFileSync, rmSync, writeFileSync } from "node:fs";
2
+ import { extname } from "node:path";
3
+
4
+ import type { TimedWord } from "../voice/voiceover.js";
5
+ import { FOOTAGE_SAMPLE_RATE } from "./normalize.js";
6
+ import { runFfmpeg } from "./process.js";
7
+
8
+ /**
9
+ * The speech track's declick stage (issue #1084), between the cut speech track and its loudness normalization: mouth
10
+ * clicks and lip smacks between words are short broadband bursts well below the speech peaks, which ffmpeg's
11
+ * `adeclick` does not see. Two steps over the cut speech, both from the words' film times:
12
+ *
13
+ * - a gate: outside each word's window (start and end ± 40 ms) the speech is lowered, with a 20 ms linear fade that
14
+ * sits outside the window, so no word onset or tail is touched;
15
+ * - impulse removal: outside the words' own spans, bursts under 30 ms that jump above the 2-8 kHz band's running median
16
+ * are lowered by 30 dB with a short crossfade on both sides.
17
+ *
18
+ * The gain is applied per sample to the decoded PCM (a `volume` expression is evaluated only once per audio frame,
19
+ * about 21 ms, coarser than the crossfades).
20
+ */
21
+
22
+ export const DECLICK_LEVELS = ["off", "gentle", "strong"] as const;
23
+
24
+ export type DeclickLevel = (typeof DECLICK_LEVELS)[number];
25
+
26
+ /** A level that runs the stage. */
27
+ export type ActiveDeclickLevel = Exclude<DeclickLevel, "off">;
28
+
29
+ export interface DeclickSettings {
30
+ /** How far the speech is lowered outside the words' windows, in dB (negative). */
31
+ gateDb: number;
32
+ /** How far a frame's 2-8 kHz energy must rise above the band's running median to count as an impulse, in dB. */
33
+ jumpDb: number;
34
+ /** The crossfade on both sides of a lowered impulse, in seconds. */
35
+ crossfadeSeconds: number;
36
+ }
37
+
38
+ export const DECLICK_SETTINGS: Record<ActiveDeclickLevel, DeclickSettings> = {
39
+ gentle: { gateDb: -15, jumpDb: 15, crossfadeSeconds: 0.005 },
40
+ strong: { gateDb: -20, jumpDb: 9, crossfadeSeconds: 0.008 },
41
+ };
42
+
43
+ /** Seconds a word's window reaches past its start and end; the gate leaves the window untouched. */
44
+ export const DECLICK_WORD_WINDOW = 0.04;
45
+ /** The gate's fade, outside the window. */
46
+ export const DECLICK_GATE_FADE = 0.02;
47
+ /** The longest burst that counts as an impulse. */
48
+ export const DECLICK_MAX_IMPULSE = 0.03;
49
+ /** How far an impulse is lowered, in dB. */
50
+ export const DECLICK_IMPULSE_DB = -30;
51
+
52
+ const BAND_LOW_HZ = 2000;
53
+ const BAND_HIGH_HZ = 8000;
54
+ const FRAME_SECONDS = 0.002;
55
+ const HOP_SECONDS = 0.001;
56
+ /** The running median's reach on each side of a frame. */
57
+ const MEDIAN_SECONDS = 0.1;
58
+ /** A lowered impulse reaches this far past its detected frames, for the band filter's delay and the frames' edges. */
59
+ const IMPULSE_MARGIN = 0.001;
60
+ /** A frame quieter than this (mean square, dB) is never an impulse: digital silence with a stray bit is not a click. */
61
+ const BAND_FLOOR_DB = -90;
62
+
63
+ export interface TimeSpan {
64
+ start: number;
65
+ end: number;
66
+ }
67
+
68
+ /** What the stage did, in the render's and the draft's ok value. */
69
+ export interface DeclickReport {
70
+ level: ActiveDeclickLevel;
71
+ /** Impulses lowered. */
72
+ impulses: number;
73
+ /** The gate between words in dB; null when the track has no words to tell speech from pauses (no gate). */
74
+ gateDb: number | null;
75
+ }
76
+
77
+ const fromDb = (db: number): number => 10 ** (db / 20);
78
+
79
+ const sortSpans = (words: readonly TimeSpan[]): TimeSpan[] => [...words].filter((word) => word.end > word.start).sort((a, b) => a.start - b.start);
80
+
81
+ /**
82
+ * The windows the gate leaves open: each word's start and end ± `DECLICK_WORD_WINDOW`, merged when two windows (with
83
+ * their fades) would touch, so the gain never dips between two close words.
84
+ */
85
+ export function getGateWindows(words: readonly TimeSpan[]): TimeSpan[] {
86
+ const windows: TimeSpan[] = [];
87
+ for (const word of sortSpans(words)) {
88
+ const start = Math.max(0, word.start - DECLICK_WORD_WINDOW);
89
+ const end = word.end + DECLICK_WORD_WINDOW;
90
+ const last = windows.at(-1);
91
+ if (last !== undefined && start - last.end < 2 * DECLICK_GATE_FADE) last.end = Math.max(last.end, end);
92
+ else windows.push({ start, end });
93
+ }
94
+ return windows;
95
+ }
96
+
97
+ /** One second-order section (RBJ cookbook), run in place over the samples. */
98
+ function runBiquad(samples: Float64Array, kind: "highpass" | "lowpass", frequency: number, sampleRate: number): void {
99
+ const w0 = (2 * Math.PI * Math.min(frequency, sampleRate * 0.45)) / sampleRate;
100
+ const cos = Math.cos(w0);
101
+ const alpha = Math.sin(w0) / (2 * Math.SQRT1_2);
102
+ const a0 = 1 + alpha;
103
+ const b0 = (kind === "lowpass" ? (1 - cos) / 2 : (1 + cos) / 2) / a0;
104
+ const b1 = (kind === "lowpass" ? 1 - cos : -(1 + cos)) / a0;
105
+ const b2 = b0;
106
+ const a1 = (-2 * cos) / a0;
107
+ const a2 = (1 - alpha) / a0;
108
+ let x1 = 0;
109
+ let x2 = 0;
110
+ let y1 = 0;
111
+ let y2 = 0;
112
+ for (let index = 0; index < samples.length; index++) {
113
+ const x = samples[index] ?? 0;
114
+ const y = b0 * x + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2;
115
+ x2 = x1;
116
+ x1 = x;
117
+ y2 = y1;
118
+ y1 = y;
119
+ samples[index] = y;
120
+ }
121
+ }
122
+
123
+ /** The 2-8 kHz band of the signal: two high-pass and two low-pass sections (fourth order on each side). */
124
+ function getBand(mono: Float32Array, sampleRate: number): Float64Array {
125
+ const band = Float64Array.from(mono);
126
+ runBiquad(band, "highpass", BAND_LOW_HZ, sampleRate);
127
+ runBiquad(band, "highpass", BAND_LOW_HZ, sampleRate);
128
+ runBiquad(band, "lowpass", BAND_HIGH_HZ, sampleRate);
129
+ runBiquad(band, "lowpass", BAND_HIGH_HZ, sampleRate);
130
+ return band;
131
+ }
132
+
133
+ /** Each frame's mean square in dB, frames of `frameLength` samples every `hop` samples. */
134
+ function getFrameEnergies(band: Float64Array, frameLength: number, hop: number): Float64Array {
135
+ const sums = new Float64Array(band.length + 1);
136
+ for (let index = 0; index < band.length; index++) {
137
+ const value = band[index] ?? 0;
138
+ sums[index + 1] = (sums[index] ?? 0) + value * value;
139
+ }
140
+ const count = band.length < frameLength ? 0 : Math.floor((band.length - frameLength) / hop) + 1;
141
+ const energies = new Float64Array(count);
142
+ for (let frame = 0; frame < count; frame++) {
143
+ const from = frame * hop;
144
+ const power = ((sums[from + frameLength] ?? 0) - (sums[from] ?? 0)) / frameLength;
145
+ energies[frame] = 10 * Math.log10(Math.max(power, 1e-20));
146
+ }
147
+ return energies;
148
+ }
149
+
150
+ /** The index of the first entry of the sorted list not below `value`. */
151
+ function findInsertIndex(sorted: number[], value: number): number {
152
+ let low = 0;
153
+ let high = sorted.length;
154
+ while (low < high) {
155
+ const middle = (low + high) >> 1;
156
+ if ((sorted[middle] ?? 0) < value) low = middle + 1;
157
+ else high = middle;
158
+ }
159
+ return low;
160
+ }
161
+
162
+ /** The running median of the values over `reach` entries on each side (fewer at the ends). */
163
+ export function getRunningMedian(values: Float64Array, reach: number): Float64Array {
164
+ const medians = new Float64Array(values.length);
165
+ const window: number[] = [];
166
+ let from = 0;
167
+ let to = 0;
168
+ for (let index = 0; index < values.length; index++) {
169
+ const wantedTo = Math.min(values.length, index + reach + 1);
170
+ for (; to < wantedTo; to++) {
171
+ const value = values[to] ?? 0;
172
+ window.splice(findInsertIndex(window, value), 0, value);
173
+ }
174
+ const wantedFrom = Math.max(0, index - reach);
175
+ for (; from < wantedFrom; from++) window.splice(findInsertIndex(window, values[from] ?? 0), 1);
176
+ const middle = window.length >> 1;
177
+ medians[index] = window.length % 2 === 1 ? (window[middle] ?? 0) : ((window[middle - 1] ?? 0) + (window[middle] ?? 0)) / 2;
178
+ }
179
+ return medians;
180
+ }
181
+
182
+ /** True when the span overlaps a word's own span. */
183
+ const isInsideWord = (span: TimeSpan, words: readonly TimeSpan[]): boolean => words.some((word) => span.start < word.end && span.end > word.start);
184
+
185
+ /**
186
+ * The impulses of a mono signal: runs of 2 ms frames (1 ms hop) whose 2-8 kHz energy jumps more than the level's
187
+ * `jumpDb` above the band's running median (±100 ms), shorter than `DECLICK_MAX_IMPULSE`, overlapping no word's own
188
+ * span. Times in seconds, widened by a millisecond on each side.
189
+ */
190
+ export function findImpulses(mono: Float32Array, sampleRate: number, words: readonly TimeSpan[], level: ActiveDeclickLevel): TimeSpan[] {
191
+ const { jumpDb } = DECLICK_SETTINGS[level];
192
+ const frameLength = Math.max(1, Math.round(FRAME_SECONDS * sampleRate));
193
+ const hop = Math.max(1, Math.round(HOP_SECONDS * sampleRate));
194
+ const energies = getFrameEnergies(getBand(mono, sampleRate), frameLength, hop);
195
+ const medians = getRunningMedian(energies, Math.round(MEDIAN_SECONDS / HOP_SECONDS));
196
+ const spoken = sortSpans(words);
197
+ const seconds = mono.length / sampleRate;
198
+ const impulses: TimeSpan[] = [];
199
+ let runStart = -1;
200
+ const closeRun = (lastFrame: number): void => {
201
+ const start = (runStart * hop) / sampleRate;
202
+ const end = (lastFrame * hop + frameLength) / sampleRate;
203
+ runStart = -1;
204
+ if (end - start >= DECLICK_MAX_IMPULSE) return;
205
+ const span = { start: Math.max(0, start - IMPULSE_MARGIN), end: Math.min(seconds, end + IMPULSE_MARGIN) };
206
+ if (!isInsideWord(span, spoken)) impulses.push(span);
207
+ };
208
+ for (let frame = 0; frame < energies.length; frame++) {
209
+ const energy = energies[frame] ?? 0;
210
+ const isJump = energy > BAND_FLOOR_DB && energy - (medians[frame] ?? 0) > jumpDb;
211
+ if (isJump && runStart === -1) runStart = frame;
212
+ else if (!isJump && runStart !== -1) closeRun(frame - 1);
213
+ }
214
+ if (runStart !== -1) closeRun(energies.length - 1);
215
+ return impulses;
216
+ }
217
+
218
+ /** Sets the gain to `value` from sample `from` up to (not including) `to`, multiplying what is there. */
219
+ function scaleRange(gain: Float32Array, from: number, to: number, value: (index: number) => number): void {
220
+ for (let index = Math.max(0, from); index < Math.min(gain.length, to); index++) gain[index] = (gain[index] ?? 1) * value(index);
221
+ }
222
+
223
+ /** A linear ramp from `fromValue` at sample `from` to `toValue` at sample `to`. */
224
+ const ramp = (from: number, to: number, fromValue: number, toValue: number) => (index: number) => (to <= from ? toValue : fromValue + ((toValue - fromValue) * (index - from)) / (to - from));
225
+
226
+ /**
227
+ * The stage's gain per sample: the gate (1 inside the windows, a linear fade outside them, the gate's level beyond)
228
+ * times a dip for every impulse (its crossfades clamped to the neighbouring words, so no word is touched). With no
229
+ * words, no gate.
230
+ */
231
+ export function getDeclickGain(options: { length: number; sampleRate: number; words: readonly TimeSpan[]; impulses: readonly TimeSpan[]; level: ActiveDeclickLevel }): Float32Array {
232
+ const { length, sampleRate, level } = options;
233
+ const settings = DECLICK_SETTINGS[level];
234
+ const at = (seconds: number): number => Math.round(seconds * sampleRate);
235
+ const windows = getGateWindows(options.words);
236
+ const gate = windows.length === 0 ? 1 : fromDb(settings.gateDb);
237
+ const gain = new Float32Array(length).fill(1);
238
+ if (gate < 1) {
239
+ const fade = at(DECLICK_GATE_FADE);
240
+ const open = new Float32Array(length).fill(gate);
241
+ for (const window of windows) {
242
+ const start = at(window.start);
243
+ const end = at(window.end);
244
+ for (let index = Math.max(0, start - fade); index < Math.min(length, end + fade); index++) {
245
+ const value = index < start ? gate + ((1 - gate) * (index - (start - fade))) / fade : index >= end ? 1 - ((1 - gate) * (index - end)) / fade : 1;
246
+ open[index] = Math.max(open[index] ?? gate, value);
247
+ }
248
+ }
249
+ gain.set(open);
250
+ }
251
+ const spoken = sortSpans(options.words);
252
+ const dip = fromDb(DECLICK_IMPULSE_DB);
253
+ const crossfade = settings.crossfadeSeconds;
254
+ for (const impulse of options.impulses) {
255
+ const before = spoken.filter((word) => word.end <= impulse.start).reduce((latest, word) => Math.max(latest, word.end), 0);
256
+ const after = spoken.filter((word) => word.start >= impulse.end).reduce((earliest, word) => Math.min(earliest, word.start), Infinity);
257
+ const start = at(impulse.start);
258
+ const end = at(impulse.end);
259
+ const fadeIn = at(Math.max(before, impulse.start - crossfade));
260
+ const fadeOut = at(Math.min(after, impulse.end + crossfade));
261
+ scaleRange(gain, fadeIn, start, ramp(fadeIn, start, 1, dip));
262
+ scaleRange(gain, start, end, () => dip);
263
+ scaleRange(gain, end, fadeOut, ramp(end, fadeOut, dip, 1));
264
+ }
265
+ return gain;
266
+ }
267
+
268
+ /** The stage over interleaved samples: the impulses found on the channels' average, then the gain on every channel. */
269
+ export function declickSamples(options: { samples: Float32Array; channels: number; sampleRate: number; words: readonly TimeSpan[]; level: ActiveDeclickLevel }): {
270
+ samples: Float32Array;
271
+ impulses: TimeSpan[];
272
+ report: DeclickReport;
273
+ } {
274
+ const { samples, channels, sampleRate, words, level } = options;
275
+ const length = Math.floor(samples.length / channels);
276
+ const mono = new Float32Array(length);
277
+ for (let index = 0; index < length; index++) {
278
+ let sum = 0;
279
+ for (let channel = 0; channel < channels; channel++) sum += samples[index * channels + channel] ?? 0;
280
+ mono[index] = sum / channels;
281
+ }
282
+ const impulses = findImpulses(mono, sampleRate, words, level);
283
+ const gain = getDeclickGain({ length, sampleRate, words, impulses, level });
284
+ const output = new Float32Array(samples.length);
285
+ for (let index = 0; index < samples.length; index++) output[index] = (samples[index] ?? 0) * (gain[Math.floor(index / channels)] ?? 1);
286
+ const hasGate = getGateWindows(words).length > 0;
287
+ return { samples: output, impulses, report: { level, impulses: impulses.length, gateDb: hasGate ? DECLICK_SETTINGS[level].gateDb : null } };
288
+ }
289
+
290
+ const CHANNELS = 2;
291
+ const PCM_FORMAT = ["-f", "f32le", "-ar", String(FOOTAGE_SAMPLE_RATE), "-ac", String(CHANNELS)];
292
+
293
+ /** The cut speech decoded to raw 32-bit float PCM at the film's rate, stereo. */
294
+ export function buildDeclickDecodeArgs(input: string, output: string): string[] {
295
+ return ["-v", "error", "-y", "-i", input, "-vn", "-c:a", "pcm_f32le", ...PCM_FORMAT, output];
296
+ }
297
+
298
+ /** The declicked PCM encoded: WAV (float) when a loudness pass follows, else mp3 as the speech track is. */
299
+ export function buildDeclickEncodeArgs(input: string, output: string): string[] {
300
+ const codec = extname(output).toLowerCase() === ".wav" ? ["-c:a", "pcm_f32le"] : ["-c:a", "libmp3lame", "-q:a", "2"];
301
+ return ["-v", "error", "-y", ...PCM_FORMAT, "-i", input, ...codec, output];
302
+ }
303
+
304
+ /** Raw little-endian float32 bytes as samples (copied, so the view is aligned). */
305
+ function readFloats(file: string): Float32Array {
306
+ const bytes = readFileSync(file);
307
+ const samples = new Float32Array(Math.floor(bytes.length / 4));
308
+ for (let index = 0; index < samples.length; index++) samples[index] = bytes.readFloatLE(index * 4);
309
+ return samples;
310
+ }
311
+
312
+ function writeFloats(file: string, samples: Float32Array): void {
313
+ const bytes = Buffer.alloc(samples.length * 4);
314
+ for (let index = 0; index < samples.length; index++) bytes.writeFloatLE(samples[index] ?? 0, index * 4);
315
+ writeFileSync(file, bytes);
316
+ }
317
+
318
+ export interface DeclickSpeechOptions {
319
+ /** The cut speech track. */
320
+ input: string;
321
+ /** The declicked track: `.wav` (float) or `.mp3`. */
322
+ output: string;
323
+ /** The words in film time (the speech track's time). */
324
+ words: readonly TimedWord[];
325
+ level: ActiveDeclickLevel;
326
+ /** The track's seconds, for the progress: decoding is its first half, encoding its second. */
327
+ seconds: number;
328
+ signal?: AbortSignal;
329
+ onSeconds?: (done: number) => void;
330
+ }
331
+
332
+ /** Runs the stage on a speech track file: decode, find and lower, encode. The raw PCM next to the output is removed. */
333
+ export async function declickSpeech(options: DeclickSpeechOptions): Promise<DeclickReport> {
334
+ const { input, output, seconds } = options;
335
+ const aborting = options.signal === undefined ? {} : { signal: options.signal };
336
+ const raw = `${output}.f32`;
337
+ const report = (done: number): void => options.onSeconds?.(Math.min(seconds, Math.max(0, done)));
338
+ try {
339
+ await runFfmpeg("The speech declick's decode", buildDeclickDecodeArgs(input, raw), { ...aborting, output: raw, onSeconds: (done) => report(done / 2) });
340
+ const declicked = declickSamples({ samples: readFloats(raw), channels: CHANNELS, sampleRate: FOOTAGE_SAMPLE_RATE, words: options.words, level: options.level });
341
+ writeFloats(raw, declicked.samples);
342
+ await runFfmpeg("The speech declick's encode", buildDeclickEncodeArgs(raw, output), { ...aborting, output, onSeconds: (done) => report(seconds / 2 + done / 2) });
343
+ return declicked.report;
344
+ } finally {
345
+ rmSync(raw, { force: true });
346
+ }
347
+ }
348
+
349
+ /** The stage's log line. */
350
+ export function describeDeclick(report: DeclickReport): string {
351
+ const gate = report.gateDb === null ? "no words, so no gate" : `the pauses lowered by ${String(-report.gateDb)} dB`;
352
+ return `declick (${report.level}): ${String(report.impulses)} impulse${report.impulses === 1 ? "" : "s"} removed, ${gate}.`;
353
+ }
@@ -2,15 +2,16 @@ import { copyFileSync, existsSync, mkdirSync, rmSync, statSync, writeFileSync }
2
2
  import { basename, dirname, join, resolve } from "node:path";
3
3
  import { fileURLToPath } from "node:url";
4
4
 
5
- import { captionChunks, type CaptionLayout, type LayoutOverride } from "../compose/timeline.js";
5
+ import type { CaptionLayout, LayoutOverride } from "../compose/timeline.js";
6
6
  import type { BrandColors } from "../config/colors.js";
7
7
  import { err, ok } from "../voice/provider.js";
8
8
  import type { TimedWord } from "../voice/voiceover.js";
9
- import { getCaptionSpans, getFootageCaptionLayout, phraseChunks } from "./captions.js";
9
+ import { getCaptionChunkLayouts, getCaptionSpans, getFootageCaptionChunks, getFootageCaptionLayout, isSameCaptionPlace, type CaptionWordsFilm } from "./captions.js";
10
10
  import { getSegmentScales, getSegmentZoomEases } from "./compose.js";
11
11
  import { chooseFootageDraftFont } from "./draft-font.js";
12
12
  import { getEdlDuration, getSegmentStarts, remapWords, type Edl } from "./edl.js";
13
- import { getFootageEdit, type FootageFilm } from "./film.js";
13
+ import { declickSpeech, describeDeclick, type DeclickReport } from "./declick.js";
14
+ import { getDeclickLevel, getFootageEdit, type FootageFilm } from "./film.js";
14
15
  import { duckingKeyframes, getDuckingOptions } from "./music.js";
15
16
  import { FOOTAGE_FPS, getThreadArgs, type Frame } from "./normalize.js";
16
17
  import { findFootageMusic, getFfmpegReporting, getFrame, type FootageBrand, type FootageResult, type FootageRunOptions, type PreparedClip, type TranscribeOutcome } from "./pipeline.js";
@@ -80,16 +81,25 @@ export function toAssColor(hex: string): string {
80
81
  return `&H${alpha}${pair(4)}${pair(2)}${pair(0)}`;
81
82
  }
82
83
 
83
- /** Caption text as ASS dialogue: override braces and backslashes would be read as tags, line breaks end the line. */
84
+ /**
85
+ * Caption text as ASS dialogue: override braces and backslashes would be read as tags, line breaks end the line. A
86
+ * no-break space (U+00A0) becomes ASS's hard space `\h`, which libass never breaks a line at, as the render's HTML.
87
+ */
84
88
  function toAssText(text: string): string {
85
- return text.replace(/\\/g, "/").replace(/\{/g, "(").replace(/\}/g, ")").replace(/\s+/g, " ").trim();
89
+ return text
90
+ .replace(/\\/g, "/")
91
+ .replace(/\{/g, "(")
92
+ .replace(/\}/g, ")")
93
+ .replace(/[^\S\u00a0]+/g, " ")
94
+ .replace(/^ +| +$/g, "")
95
+ .replace(/\u00a0/g, "\\h");
86
96
  }
87
97
 
88
98
  /** The caption colours the draft paints with (`BrandColors`). */
89
99
  export type FootageDraftCaptionColors = Pick<BrandColors, "captionBackground" | "captionText" | "captionHighlight">;
90
100
 
91
101
  export interface FootageDraftCaptionsInput {
92
- film: Pick<FootageFilm, "captions" | "captionStyle">;
102
+ film: Pick<FootageFilm, "captions" | "captionStyle"> & CaptionWordsFilm;
93
103
  edl: Edl;
94
104
  /** The film's words in film time (`remapWords`). */
95
105
  words: TimedWord[];
@@ -107,7 +117,7 @@ export interface FootageDraftCaptionsInput {
107
117
  * A `highlight` chunk as ASS events: its span cut wherever a word lights up or goes out, each piece showing the chunk
108
118
  * with the words lit then in `highlight` (a word stays lit for the render's 0.32 s from its start).
109
119
  */
110
- function buildHighlightEvents(chunk: { words: TimedWord[] }, span: { start: number; end: number }, colors: { text: string; highlight: string }): string[] {
120
+ function buildHighlightEvents(chunk: { words: TimedWord[] }, span: { start: number; end: number }, colors: { text: string; highlight: string }, styleName: string): string[] {
111
121
  const cs = (seconds: number): number => Math.round(seconds * 100);
112
122
  const [from, to] = [cs(span.start), cs(span.end)];
113
123
  const lights = chunk.words.map((word) => ({ on: cs(word.start), off: cs(word.start + HIGHLIGHT_SECONDS) }));
@@ -122,7 +132,7 @@ function buildHighlightEvents(chunk: { words: TimedWord[] }, span: { start: numb
122
132
  return light.on <= at && at < light.off ? `{\\1c${colors.highlight}&}${shown}{\\1c${colors.text}&}` : shown;
123
133
  })
124
134
  .join(" ");
125
- events.push(`Dialogue: 0,${toAssTime(at / 100)},${toAssTime(next / 100)},Caption,,0,0,0,,${text}`);
135
+ events.push(`Dialogue: 0,${toAssTime(at / 100)},${toAssTime(next / 100)},${styleName},,0,0,0,,${text}`);
126
136
  });
127
137
  return events;
128
138
  }
@@ -132,12 +142,13 @@ function buildHighlightEvents(chunk: { words: TimedWord[] }, span: { start: numb
132
142
  * caption box from its top (or from its foot, with `bottom`). `phrase` (the default): the film's phrases
133
143
  * (`phraseChunks`), white bold text with a dark outline and shadow. `highlight`: the render's chunks of up to four
134
144
  * words (`captionChunks`) on an opaque box of `captionBackground`, in `captionText`, the word being said in
135
- * `captionHighlight`. Null when the film has no captions or nothing is said.
145
+ * `captionHighlight`. Both keep the film's kept words with the next, as the render does. Null when the film has no
146
+ * captions or nothing is said.
136
147
  */
137
148
  export function buildFootageDraftCaptions(input: FootageDraftCaptionsInput): string | null {
138
149
  if (!input.film.captions) return null;
139
150
  const isHighlight = input.film.captionStyle === "highlight";
140
- const chunks = isHighlight ? captionChunks(input.words) : phraseChunks(input.words);
151
+ const chunks = getFootageCaptionChunks(input.film, input.words);
141
152
  if (chunks.length === 0) return null;
142
153
  const spans = getCaptionSpans(chunks, getEdlDuration(input.edl));
143
154
  const { caption, frame } = input;
@@ -156,8 +167,8 @@ export function buildFootageDraftCaptions(input: FootageDraftCaptionsInput): str
156
167
  outline: Math.max(1, Math.round(caption.fontSize / 16)),
157
168
  shadow: Math.max(1, Math.round(caption.fontSize / 25)),
158
169
  };
159
- const style = [
160
- "Caption",
170
+ const toStyle = (name: string, place: CaptionLayout): string => [
171
+ name,
161
172
  family,
162
173
  caption.fontSize,
163
174
  look.primary,
@@ -176,18 +187,31 @@ export function buildFootageDraftCaptions(input: FootageDraftCaptionsInput): str
176
187
  look.outline,
177
188
  look.shadow,
178
189
  // ASS alignment 2 hangs the text from MarginV above the bottom, 8 from MarginV below the top.
179
- caption.bottom === undefined ? 8 : 2,
180
- Math.round(caption.left),
181
- Math.round(caption.right),
182
- Math.round(caption.bottom ?? caption.top),
190
+ place.bottom === undefined ? 8 : 2,
191
+ Math.round(place.left),
192
+ Math.round(place.right),
193
+ Math.round(place.bottom ?? place.top),
183
194
  1,
184
195
  ].join(",");
196
+ // A chunk in a segment's own box gets a style of that box (Caption2, Caption3, ...); the film's box stays `Caption`.
197
+ const styles = [toStyle("Caption", caption)];
198
+ const moved: CaptionLayout[] = [];
199
+ const styleNames = getCaptionChunkLayouts(chunks, input.edl, caption).map((place) => {
200
+ if (isSameCaptionPlace(place, caption)) return "Caption";
201
+ let index = moved.findIndex((other) => isSameCaptionPlace(other, place));
202
+ if (index === -1) {
203
+ index = moved.push(place) - 1;
204
+ styles.push(toStyle(`Caption${index + 2}`, place));
205
+ }
206
+ return `Caption${index + 2}`;
207
+ });
185
208
  const highlight = { text: text.replace(/^&H../, "&H"), highlight: toAssColor(colors.captionHighlight).replace(/^&H../, "&H") };
186
209
  const events = chunks.flatMap((chunk, index) => {
187
210
  const span = spans[index] as { start: number; end: number };
188
- if (isHighlight) return buildHighlightEvents(chunk, span, highlight);
211
+ const styleName = styleNames[index] ?? "Caption";
212
+ if (isHighlight) return buildHighlightEvents(chunk, span, highlight, styleName);
189
213
  const line = toAssText(chunk.words.map((word) => word.text).join(" "));
190
- return [`Dialogue: 0,${toAssTime(span.start)},${toAssTime(span.end)},Caption,,0,0,0,,${line}`];
214
+ return [`Dialogue: 0,${toAssTime(span.start)},${toAssTime(span.end)},${styleName},,0,0,0,,${line}`];
191
215
  });
192
216
  return [
193
217
  "[Script Info]",
@@ -199,7 +223,7 @@ export function buildFootageDraftCaptions(input: FootageDraftCaptionsInput): str
199
223
  "",
200
224
  "[V4+ Styles]",
201
225
  "Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding",
202
- `Style: ${style}`,
226
+ ...styles.map((style) => `Style: ${style}`),
203
227
  "",
204
228
  "[Events]",
205
229
  "Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text",
@@ -427,6 +451,8 @@ export interface FootageDraft {
427
451
  width: number;
428
452
  height: number;
429
453
  bytes: number;
454
+ /** What the speech track's declick stage did, as the render does it; null when the film's declick is off. */
455
+ declick: DeclickReport | null;
430
456
  }
431
457
 
432
458
  /** The draft's music track: a file to duck, or why there is none. */
@@ -478,13 +504,16 @@ async function runRenderFootageDraft(
478
504
  const draftDir = resolve(options.buildDir, "draft");
479
505
  rmSync(draftDir, { recursive: true, force: true });
480
506
  mkdirSync(draftDir, { recursive: true });
481
- const items = music.value.kind === "file" ? ["speech", "music", "film"] : ["speech", "film"];
507
+ const declickLevel = getDeclickLevel(film);
508
+ const items = ["speech", ...(declickLevel === "off" ? [] : ["declick"]), ...(music.value.kind === "file" ? ["music"] : []), "film"];
482
509
  const report = (item: string, done: number): void => {
483
510
  const itemIndex = items.indexOf(item) + 1;
484
511
  options.onProgress?.({ step: "draft", item, itemIndex, itemCount: items.length, done: Math.min(seconds * items.length, (itemIndex - 1) * seconds + done), total: seconds * items.length });
485
512
  };
486
513
  const clipFiles = Object.fromEntries(clips.map((clip) => [clip.clip.id, resolve(clip.file)]));
487
- const speech = join(draftDir, "speech.mp3");
514
+ const cutSpeech = join(draftDir, "speech.mp3");
515
+ // The declicked track stays WAV: the draft's one encode is its only lossy step after the cut.
516
+ const speech = declickLevel === "off" ? cutSpeech : join(draftDir, "speech.declicked.wav");
488
517
  const words = remapWords(edl, transcript.words, film.cut.fillers);
489
518
  const frame = getFrame(film);
490
519
  const brandFont = chooseFootageDraftFont(options.brand?.fonts.body ?? null);
@@ -509,7 +538,13 @@ async function runRenderFootageDraft(
509
538
  const output = resolve(options.output);
510
539
  mkdirSync(dirname(output), { recursive: true });
511
540
  report("speech", 0);
512
- await runFfmpeg("The draft's speech track", buildSpeechArgs(edl, transcript.speech, speech, getSpeechCutOptions(getFootageEdit(film))), { ...aborting, output: speech, onSeconds: (done) => report("speech", done) });
541
+ await runFfmpeg("The draft's speech track", buildSpeechArgs(edl, transcript.speech, cutSpeech, getSpeechCutOptions(getFootageEdit(film))), { ...aborting, output: cutSpeech, onSeconds: (done) => report("speech", done) });
542
+ let declick: DeclickReport | null = null;
543
+ if (declickLevel !== "off") {
544
+ report("declick", 0);
545
+ declick = await declickSpeech({ input: cutSpeech, output: speech, words, level: declickLevel, seconds, ...aborting, onSeconds: (done) => report("declick", done) });
546
+ log(describeDeclick(declick));
547
+ }
513
548
  let musicFile: string | null = null;
514
549
  if (music.value.kind === "file") {
515
550
  musicFile = join(draftDir, "music.mp3");
@@ -542,5 +577,5 @@ async function runRenderFootageDraft(
542
577
  const draftMusic: FootageDraftMusic = music.value.kind === "file" ? "played" : music.value.kind;
543
578
  const notes = [captions === null ? "no captions" : `captions in ${font.family}`, draftMusic === "played" ? "music" : draftMusic === "none" ? "no music" : "music not cached"];
544
579
  log(`draft: ${output}, ${draft.width}×${draft.height}, ${seconds.toFixed(1)} s, ${bytes} bytes, ${notes.join(", ")}.`);
545
- return ok({ output, music: draftMusic, captionFont: captions === null ? null : font.family, seconds, width: draft.width, height: draft.height, bytes });
580
+ return ok({ output, music: draftMusic, captionFont: captions === null ? null : font.family, seconds, width: draft.width, height: draft.height, bytes, declick });
546
581
  }