@softure-ai/marketing-kit 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +85 -0
- package/README.md +67 -9
- package/dist/cli/footage.js +1 -1
- package/dist/cli/footage.js.map +1 -1
- package/dist/compose/timeline.d.ts +23 -3
- package/dist/compose/timeline.d.ts.map +1 -1
- package/dist/compose/timeline.js +31 -11
- package/dist/compose/timeline.js.map +1 -1
- package/dist/config/actions-schema.d.ts +14 -14
- package/dist/config/config.d.ts.map +1 -1
- package/dist/config/config.js +2 -0
- package/dist/config/config.js.map +1 -1
- package/dist/config/schema.d.ts +83 -70
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +21 -1
- package/dist/config/schema.js.map +1 -1
- package/dist/config/shot-steps.d.ts +16 -16
- package/dist/footage/captions.d.ts +45 -3
- package/dist/footage/captions.d.ts.map +1 -1
- package/dist/footage/captions.js +65 -5
- package/dist/footage/captions.js.map +1 -1
- package/dist/footage/compose.d.ts.map +1 -1
- package/dist/footage/compose.js +8 -4
- package/dist/footage/compose.js.map +1 -1
- package/dist/footage/declick.d.ts +106 -0
- package/dist/footage/declick.d.ts.map +1 -0
- package/dist/footage/declick.js +296 -0
- package/dist/footage/declick.js.map +1 -0
- package/dist/footage/draft.d.ts +8 -3
- package/dist/footage/draft.d.ts.map +1 -1
- package/dist/footage/draft.js +55 -22
- package/dist/footage/draft.js.map +1 -1
- package/dist/footage/edl.d.ts +28 -2
- package/dist/footage/edl.d.ts.map +1 -1
- package/dist/footage/edl.js +53 -2
- package/dist/footage/edl.js.map +1 -1
- package/dist/footage/film.d.ts +29 -3
- package/dist/footage/film.d.ts.map +1 -1
- package/dist/footage/film.js +12 -0
- package/dist/footage/film.js.map +1 -1
- package/dist/footage/isolate-ranges.d.ts +37 -0
- package/dist/footage/isolate-ranges.d.ts.map +1 -0
- package/dist/footage/isolate-ranges.js +113 -0
- package/dist/footage/isolate-ranges.js.map +1 -0
- package/dist/footage/loudness.d.ts +76 -0
- package/dist/footage/loudness.d.ts.map +1 -1
- package/dist/footage/loudness.js +134 -0
- package/dist/footage/loudness.js.map +1 -1
- package/dist/footage/paid.d.ts +18 -0
- package/dist/footage/paid.d.ts.map +1 -1
- package/dist/footage/paid.js +32 -2
- package/dist/footage/paid.js.map +1 -1
- package/dist/footage/pipeline.d.ts +43 -5
- package/dist/footage/pipeline.d.ts.map +1 -1
- package/dist/footage/pipeline.js +105 -20
- package/dist/footage/pipeline.js.map +1 -1
- package/dist/footage/providers.d.ts +5 -0
- package/dist/footage/providers.d.ts.map +1 -1
- package/dist/footage/publish.d.ts +11 -2
- package/dist/footage/publish.d.ts.map +1 -1
- package/dist/footage/publish.js +14 -10
- package/dist/footage/publish.js.map +1 -1
- package/dist/footage/speech-ranges.d.ts +65 -0
- package/dist/footage/speech-ranges.d.ts.map +1 -0
- package/dist/footage/speech-ranges.js +98 -0
- package/dist/footage/speech-ranges.js.map +1 -0
- package/dist/index.d.ts +10 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +10 -7
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/schema/edl.schema.json +17 -0
- package/schema/marketing.schema.json +43 -1
- package/src/cli/footage.ts +1 -1
- package/src/compose/timeline.ts +41 -10
- package/src/config/config.ts +2 -0
- package/src/config/schema.ts +23 -1
- package/src/footage/captions.ts +81 -6
- package/src/footage/compose.ts +8 -4
- package/src/footage/declick.ts +353 -0
- package/src/footage/draft.ts +58 -23
- package/src/footage/edl.ts +69 -2
- package/src/footage/film.ts +37 -3
- package/src/footage/isolate-ranges.ts +161 -0
- package/src/footage/loudness.ts +170 -0
- package/src/footage/paid.ts +40 -2
- package/src/footage/pipeline.ts +158 -22
- package/src/footage/providers.ts +2 -0
- package/src/footage/publish.ts +20 -11
- package/src/footage/speech-ranges.ts +131 -0
- package/src/index.ts +75 -0
|
@@ -0,0 +1,353 @@
|
|
|
1
|
+
import { readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { extname } from "node:path";
|
|
3
|
+
|
|
4
|
+
import type { TimedWord } from "../voice/voiceover.js";
|
|
5
|
+
import { FOOTAGE_SAMPLE_RATE } from "./normalize.js";
|
|
6
|
+
import { runFfmpeg } from "./process.js";
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* The speech track's declick stage (issue #1084), between the cut speech track and its loudness normalization: mouth
|
|
10
|
+
* clicks and lip smacks between words are short broadband bursts well below the speech peaks, which ffmpeg's
|
|
11
|
+
* `adeclick` does not see. Two steps over the cut speech, both from the words' film times:
|
|
12
|
+
*
|
|
13
|
+
* - a gate: outside each word's window (start and end ± 40 ms) the speech is lowered, with a 20 ms linear fade that
|
|
14
|
+
* sits outside the window, so no word onset or tail is touched;
|
|
15
|
+
* - impulse removal: outside the words' own spans, bursts under 30 ms that jump above the 2-8 kHz band's running median
|
|
16
|
+
* are lowered by 30 dB with a short crossfade on both sides.
|
|
17
|
+
*
|
|
18
|
+
* The gain is applied per sample to the decoded PCM (a `volume` expression is evaluated only once per audio frame,
|
|
19
|
+
* about 21 ms, coarser than the crossfades).
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
export const DECLICK_LEVELS = ["off", "gentle", "strong"] as const;
|
|
23
|
+
|
|
24
|
+
export type DeclickLevel = (typeof DECLICK_LEVELS)[number];
|
|
25
|
+
|
|
26
|
+
/** A level that runs the stage. */
|
|
27
|
+
export type ActiveDeclickLevel = Exclude<DeclickLevel, "off">;
|
|
28
|
+
|
|
29
|
+
export interface DeclickSettings {
|
|
30
|
+
/** How far the speech is lowered outside the words' windows, in dB (negative). */
|
|
31
|
+
gateDb: number;
|
|
32
|
+
/** How far a frame's 2-8 kHz energy must rise above the band's running median to count as an impulse, in dB. */
|
|
33
|
+
jumpDb: number;
|
|
34
|
+
/** The crossfade on both sides of a lowered impulse, in seconds. */
|
|
35
|
+
crossfadeSeconds: number;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export const DECLICK_SETTINGS: Record<ActiveDeclickLevel, DeclickSettings> = {
|
|
39
|
+
gentle: { gateDb: -15, jumpDb: 15, crossfadeSeconds: 0.005 },
|
|
40
|
+
strong: { gateDb: -20, jumpDb: 9, crossfadeSeconds: 0.008 },
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
/** Seconds a word's window reaches past its start and end; the gate leaves the window untouched. */
|
|
44
|
+
export const DECLICK_WORD_WINDOW = 0.04;
|
|
45
|
+
/** The gate's fade, outside the window. */
|
|
46
|
+
export const DECLICK_GATE_FADE = 0.02;
|
|
47
|
+
/** The longest burst that counts as an impulse. */
|
|
48
|
+
export const DECLICK_MAX_IMPULSE = 0.03;
|
|
49
|
+
/** How far an impulse is lowered, in dB. */
|
|
50
|
+
export const DECLICK_IMPULSE_DB = -30;
|
|
51
|
+
|
|
52
|
+
const BAND_LOW_HZ = 2000;
|
|
53
|
+
const BAND_HIGH_HZ = 8000;
|
|
54
|
+
const FRAME_SECONDS = 0.002;
|
|
55
|
+
const HOP_SECONDS = 0.001;
|
|
56
|
+
/** The running median's reach on each side of a frame. */
|
|
57
|
+
const MEDIAN_SECONDS = 0.1;
|
|
58
|
+
/** A lowered impulse reaches this far past its detected frames, for the band filter's delay and the frames' edges. */
|
|
59
|
+
const IMPULSE_MARGIN = 0.001;
|
|
60
|
+
/** A frame quieter than this (mean square, dB) is never an impulse: digital silence with a stray bit is not a click. */
|
|
61
|
+
const BAND_FLOOR_DB = -90;
|
|
62
|
+
|
|
63
|
+
export interface TimeSpan {
|
|
64
|
+
start: number;
|
|
65
|
+
end: number;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** What the stage did, in the render's and the draft's ok value. */
|
|
69
|
+
export interface DeclickReport {
|
|
70
|
+
level: ActiveDeclickLevel;
|
|
71
|
+
/** Impulses lowered. */
|
|
72
|
+
impulses: number;
|
|
73
|
+
/** The gate between words in dB; null when the track has no words to tell speech from pauses (no gate). */
|
|
74
|
+
gateDb: number | null;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const fromDb = (db: number): number => 10 ** (db / 20);
|
|
78
|
+
|
|
79
|
+
const sortSpans = (words: readonly TimeSpan[]): TimeSpan[] => [...words].filter((word) => word.end > word.start).sort((a, b) => a.start - b.start);
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* The windows the gate leaves open: each word's start and end ± `DECLICK_WORD_WINDOW`, merged when two windows (with
|
|
83
|
+
* their fades) would touch, so the gain never dips between two close words.
|
|
84
|
+
*/
|
|
85
|
+
export function getGateWindows(words: readonly TimeSpan[]): TimeSpan[] {
|
|
86
|
+
const windows: TimeSpan[] = [];
|
|
87
|
+
for (const word of sortSpans(words)) {
|
|
88
|
+
const start = Math.max(0, word.start - DECLICK_WORD_WINDOW);
|
|
89
|
+
const end = word.end + DECLICK_WORD_WINDOW;
|
|
90
|
+
const last = windows.at(-1);
|
|
91
|
+
if (last !== undefined && start - last.end < 2 * DECLICK_GATE_FADE) last.end = Math.max(last.end, end);
|
|
92
|
+
else windows.push({ start, end });
|
|
93
|
+
}
|
|
94
|
+
return windows;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** One second-order section (RBJ cookbook), run in place over the samples. */
|
|
98
|
+
function runBiquad(samples: Float64Array, kind: "highpass" | "lowpass", frequency: number, sampleRate: number): void {
|
|
99
|
+
const w0 = (2 * Math.PI * Math.min(frequency, sampleRate * 0.45)) / sampleRate;
|
|
100
|
+
const cos = Math.cos(w0);
|
|
101
|
+
const alpha = Math.sin(w0) / (2 * Math.SQRT1_2);
|
|
102
|
+
const a0 = 1 + alpha;
|
|
103
|
+
const b0 = (kind === "lowpass" ? (1 - cos) / 2 : (1 + cos) / 2) / a0;
|
|
104
|
+
const b1 = (kind === "lowpass" ? 1 - cos : -(1 + cos)) / a0;
|
|
105
|
+
const b2 = b0;
|
|
106
|
+
const a1 = (-2 * cos) / a0;
|
|
107
|
+
const a2 = (1 - alpha) / a0;
|
|
108
|
+
let x1 = 0;
|
|
109
|
+
let x2 = 0;
|
|
110
|
+
let y1 = 0;
|
|
111
|
+
let y2 = 0;
|
|
112
|
+
for (let index = 0; index < samples.length; index++) {
|
|
113
|
+
const x = samples[index] ?? 0;
|
|
114
|
+
const y = b0 * x + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2;
|
|
115
|
+
x2 = x1;
|
|
116
|
+
x1 = x;
|
|
117
|
+
y2 = y1;
|
|
118
|
+
y1 = y;
|
|
119
|
+
samples[index] = y;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** The 2-8 kHz band of the signal: two high-pass and two low-pass sections (fourth order on each side). */
|
|
124
|
+
function getBand(mono: Float32Array, sampleRate: number): Float64Array {
|
|
125
|
+
const band = Float64Array.from(mono);
|
|
126
|
+
runBiquad(band, "highpass", BAND_LOW_HZ, sampleRate);
|
|
127
|
+
runBiquad(band, "highpass", BAND_LOW_HZ, sampleRate);
|
|
128
|
+
runBiquad(band, "lowpass", BAND_HIGH_HZ, sampleRate);
|
|
129
|
+
runBiquad(band, "lowpass", BAND_HIGH_HZ, sampleRate);
|
|
130
|
+
return band;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** Each frame's mean square in dB, frames of `frameLength` samples every `hop` samples. */
|
|
134
|
+
function getFrameEnergies(band: Float64Array, frameLength: number, hop: number): Float64Array {
|
|
135
|
+
const sums = new Float64Array(band.length + 1);
|
|
136
|
+
for (let index = 0; index < band.length; index++) {
|
|
137
|
+
const value = band[index] ?? 0;
|
|
138
|
+
sums[index + 1] = (sums[index] ?? 0) + value * value;
|
|
139
|
+
}
|
|
140
|
+
const count = band.length < frameLength ? 0 : Math.floor((band.length - frameLength) / hop) + 1;
|
|
141
|
+
const energies = new Float64Array(count);
|
|
142
|
+
for (let frame = 0; frame < count; frame++) {
|
|
143
|
+
const from = frame * hop;
|
|
144
|
+
const power = ((sums[from + frameLength] ?? 0) - (sums[from] ?? 0)) / frameLength;
|
|
145
|
+
energies[frame] = 10 * Math.log10(Math.max(power, 1e-20));
|
|
146
|
+
}
|
|
147
|
+
return energies;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** The index of the first entry of the sorted list not below `value`. */
|
|
151
|
+
function findInsertIndex(sorted: number[], value: number): number {
|
|
152
|
+
let low = 0;
|
|
153
|
+
let high = sorted.length;
|
|
154
|
+
while (low < high) {
|
|
155
|
+
const middle = (low + high) >> 1;
|
|
156
|
+
if ((sorted[middle] ?? 0) < value) low = middle + 1;
|
|
157
|
+
else high = middle;
|
|
158
|
+
}
|
|
159
|
+
return low;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** The running median of the values over `reach` entries on each side (fewer at the ends). */
|
|
163
|
+
export function getRunningMedian(values: Float64Array, reach: number): Float64Array {
|
|
164
|
+
const medians = new Float64Array(values.length);
|
|
165
|
+
const window: number[] = [];
|
|
166
|
+
let from = 0;
|
|
167
|
+
let to = 0;
|
|
168
|
+
for (let index = 0; index < values.length; index++) {
|
|
169
|
+
const wantedTo = Math.min(values.length, index + reach + 1);
|
|
170
|
+
for (; to < wantedTo; to++) {
|
|
171
|
+
const value = values[to] ?? 0;
|
|
172
|
+
window.splice(findInsertIndex(window, value), 0, value);
|
|
173
|
+
}
|
|
174
|
+
const wantedFrom = Math.max(0, index - reach);
|
|
175
|
+
for (; from < wantedFrom; from++) window.splice(findInsertIndex(window, values[from] ?? 0), 1);
|
|
176
|
+
const middle = window.length >> 1;
|
|
177
|
+
medians[index] = window.length % 2 === 1 ? (window[middle] ?? 0) : ((window[middle - 1] ?? 0) + (window[middle] ?? 0)) / 2;
|
|
178
|
+
}
|
|
179
|
+
return medians;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/** True when the span overlaps a word's own span. */
|
|
183
|
+
const isInsideWord = (span: TimeSpan, words: readonly TimeSpan[]): boolean => words.some((word) => span.start < word.end && span.end > word.start);
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* The impulses of a mono signal: runs of 2 ms frames (1 ms hop) whose 2-8 kHz energy jumps more than the level's
|
|
187
|
+
* `jumpDb` above the band's running median (±100 ms), shorter than `DECLICK_MAX_IMPULSE`, overlapping no word's own
|
|
188
|
+
* span. Times in seconds, widened by a millisecond on each side.
|
|
189
|
+
*/
|
|
190
|
+
export function findImpulses(mono: Float32Array, sampleRate: number, words: readonly TimeSpan[], level: ActiveDeclickLevel): TimeSpan[] {
|
|
191
|
+
const { jumpDb } = DECLICK_SETTINGS[level];
|
|
192
|
+
const frameLength = Math.max(1, Math.round(FRAME_SECONDS * sampleRate));
|
|
193
|
+
const hop = Math.max(1, Math.round(HOP_SECONDS * sampleRate));
|
|
194
|
+
const energies = getFrameEnergies(getBand(mono, sampleRate), frameLength, hop);
|
|
195
|
+
const medians = getRunningMedian(energies, Math.round(MEDIAN_SECONDS / HOP_SECONDS));
|
|
196
|
+
const spoken = sortSpans(words);
|
|
197
|
+
const seconds = mono.length / sampleRate;
|
|
198
|
+
const impulses: TimeSpan[] = [];
|
|
199
|
+
let runStart = -1;
|
|
200
|
+
const closeRun = (lastFrame: number): void => {
|
|
201
|
+
const start = (runStart * hop) / sampleRate;
|
|
202
|
+
const end = (lastFrame * hop + frameLength) / sampleRate;
|
|
203
|
+
runStart = -1;
|
|
204
|
+
if (end - start >= DECLICK_MAX_IMPULSE) return;
|
|
205
|
+
const span = { start: Math.max(0, start - IMPULSE_MARGIN), end: Math.min(seconds, end + IMPULSE_MARGIN) };
|
|
206
|
+
if (!isInsideWord(span, spoken)) impulses.push(span);
|
|
207
|
+
};
|
|
208
|
+
for (let frame = 0; frame < energies.length; frame++) {
|
|
209
|
+
const energy = energies[frame] ?? 0;
|
|
210
|
+
const isJump = energy > BAND_FLOOR_DB && energy - (medians[frame] ?? 0) > jumpDb;
|
|
211
|
+
if (isJump && runStart === -1) runStart = frame;
|
|
212
|
+
else if (!isJump && runStart !== -1) closeRun(frame - 1);
|
|
213
|
+
}
|
|
214
|
+
if (runStart !== -1) closeRun(energies.length - 1);
|
|
215
|
+
return impulses;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/** Sets the gain to `value` from sample `from` up to (not including) `to`, multiplying what is there. */
|
|
219
|
+
function scaleRange(gain: Float32Array, from: number, to: number, value: (index: number) => number): void {
|
|
220
|
+
for (let index = Math.max(0, from); index < Math.min(gain.length, to); index++) gain[index] = (gain[index] ?? 1) * value(index);
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/** A linear ramp from `fromValue` at sample `from` to `toValue` at sample `to`. */
|
|
224
|
+
const ramp = (from: number, to: number, fromValue: number, toValue: number) => (index: number) => (to <= from ? toValue : fromValue + ((toValue - fromValue) * (index - from)) / (to - from));
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* The stage's gain per sample: the gate (1 inside the windows, a linear fade outside them, the gate's level beyond)
|
|
228
|
+
* times a dip for every impulse (its crossfades clamped to the neighbouring words, so no word is touched). With no
|
|
229
|
+
* words, no gate.
|
|
230
|
+
*/
|
|
231
|
+
export function getDeclickGain(options: { length: number; sampleRate: number; words: readonly TimeSpan[]; impulses: readonly TimeSpan[]; level: ActiveDeclickLevel }): Float32Array {
|
|
232
|
+
const { length, sampleRate, level } = options;
|
|
233
|
+
const settings = DECLICK_SETTINGS[level];
|
|
234
|
+
const at = (seconds: number): number => Math.round(seconds * sampleRate);
|
|
235
|
+
const windows = getGateWindows(options.words);
|
|
236
|
+
const gate = windows.length === 0 ? 1 : fromDb(settings.gateDb);
|
|
237
|
+
const gain = new Float32Array(length).fill(1);
|
|
238
|
+
if (gate < 1) {
|
|
239
|
+
const fade = at(DECLICK_GATE_FADE);
|
|
240
|
+
const open = new Float32Array(length).fill(gate);
|
|
241
|
+
for (const window of windows) {
|
|
242
|
+
const start = at(window.start);
|
|
243
|
+
const end = at(window.end);
|
|
244
|
+
for (let index = Math.max(0, start - fade); index < Math.min(length, end + fade); index++) {
|
|
245
|
+
const value = index < start ? gate + ((1 - gate) * (index - (start - fade))) / fade : index >= end ? 1 - ((1 - gate) * (index - end)) / fade : 1;
|
|
246
|
+
open[index] = Math.max(open[index] ?? gate, value);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
gain.set(open);
|
|
250
|
+
}
|
|
251
|
+
const spoken = sortSpans(options.words);
|
|
252
|
+
const dip = fromDb(DECLICK_IMPULSE_DB);
|
|
253
|
+
const crossfade = settings.crossfadeSeconds;
|
|
254
|
+
for (const impulse of options.impulses) {
|
|
255
|
+
const before = spoken.filter((word) => word.end <= impulse.start).reduce((latest, word) => Math.max(latest, word.end), 0);
|
|
256
|
+
const after = spoken.filter((word) => word.start >= impulse.end).reduce((earliest, word) => Math.min(earliest, word.start), Infinity);
|
|
257
|
+
const start = at(impulse.start);
|
|
258
|
+
const end = at(impulse.end);
|
|
259
|
+
const fadeIn = at(Math.max(before, impulse.start - crossfade));
|
|
260
|
+
const fadeOut = at(Math.min(after, impulse.end + crossfade));
|
|
261
|
+
scaleRange(gain, fadeIn, start, ramp(fadeIn, start, 1, dip));
|
|
262
|
+
scaleRange(gain, start, end, () => dip);
|
|
263
|
+
scaleRange(gain, end, fadeOut, ramp(end, fadeOut, dip, 1));
|
|
264
|
+
}
|
|
265
|
+
return gain;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/** The stage over interleaved samples: the impulses found on the channels' average, then the gain on every channel. */
|
|
269
|
+
export function declickSamples(options: { samples: Float32Array; channels: number; sampleRate: number; words: readonly TimeSpan[]; level: ActiveDeclickLevel }): {
|
|
270
|
+
samples: Float32Array;
|
|
271
|
+
impulses: TimeSpan[];
|
|
272
|
+
report: DeclickReport;
|
|
273
|
+
} {
|
|
274
|
+
const { samples, channels, sampleRate, words, level } = options;
|
|
275
|
+
const length = Math.floor(samples.length / channels);
|
|
276
|
+
const mono = new Float32Array(length);
|
|
277
|
+
for (let index = 0; index < length; index++) {
|
|
278
|
+
let sum = 0;
|
|
279
|
+
for (let channel = 0; channel < channels; channel++) sum += samples[index * channels + channel] ?? 0;
|
|
280
|
+
mono[index] = sum / channels;
|
|
281
|
+
}
|
|
282
|
+
const impulses = findImpulses(mono, sampleRate, words, level);
|
|
283
|
+
const gain = getDeclickGain({ length, sampleRate, words, impulses, level });
|
|
284
|
+
const output = new Float32Array(samples.length);
|
|
285
|
+
for (let index = 0; index < samples.length; index++) output[index] = (samples[index] ?? 0) * (gain[Math.floor(index / channels)] ?? 1);
|
|
286
|
+
const hasGate = getGateWindows(words).length > 0;
|
|
287
|
+
return { samples: output, impulses, report: { level, impulses: impulses.length, gateDb: hasGate ? DECLICK_SETTINGS[level].gateDb : null } };
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
const CHANNELS = 2;
|
|
291
|
+
const PCM_FORMAT = ["-f", "f32le", "-ar", String(FOOTAGE_SAMPLE_RATE), "-ac", String(CHANNELS)];
|
|
292
|
+
|
|
293
|
+
/** The cut speech decoded to raw 32-bit float PCM at the film's rate, stereo. */
|
|
294
|
+
export function buildDeclickDecodeArgs(input: string, output: string): string[] {
|
|
295
|
+
return ["-v", "error", "-y", "-i", input, "-vn", "-c:a", "pcm_f32le", ...PCM_FORMAT, output];
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
/** The declicked PCM encoded: WAV (float) when a loudness pass follows, else mp3 as the speech track is. */
|
|
299
|
+
export function buildDeclickEncodeArgs(input: string, output: string): string[] {
|
|
300
|
+
const codec = extname(output).toLowerCase() === ".wav" ? ["-c:a", "pcm_f32le"] : ["-c:a", "libmp3lame", "-q:a", "2"];
|
|
301
|
+
return ["-v", "error", "-y", ...PCM_FORMAT, "-i", input, ...codec, output];
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/** Raw little-endian float32 bytes as samples (copied, so the view is aligned). */
|
|
305
|
+
function readFloats(file: string): Float32Array {
|
|
306
|
+
const bytes = readFileSync(file);
|
|
307
|
+
const samples = new Float32Array(Math.floor(bytes.length / 4));
|
|
308
|
+
for (let index = 0; index < samples.length; index++) samples[index] = bytes.readFloatLE(index * 4);
|
|
309
|
+
return samples;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
function writeFloats(file: string, samples: Float32Array): void {
|
|
313
|
+
const bytes = Buffer.alloc(samples.length * 4);
|
|
314
|
+
for (let index = 0; index < samples.length; index++) bytes.writeFloatLE(samples[index] ?? 0, index * 4);
|
|
315
|
+
writeFileSync(file, bytes);
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
export interface DeclickSpeechOptions {
|
|
319
|
+
/** The cut speech track. */
|
|
320
|
+
input: string;
|
|
321
|
+
/** The declicked track: `.wav` (float) or `.mp3`. */
|
|
322
|
+
output: string;
|
|
323
|
+
/** The words in film time (the speech track's time). */
|
|
324
|
+
words: readonly TimedWord[];
|
|
325
|
+
level: ActiveDeclickLevel;
|
|
326
|
+
/** The track's seconds, for the progress: decoding is its first half, encoding its second. */
|
|
327
|
+
seconds: number;
|
|
328
|
+
signal?: AbortSignal;
|
|
329
|
+
onSeconds?: (done: number) => void;
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/** Runs the stage on a speech track file: decode, find and lower, encode. The raw PCM next to the output is removed. */
|
|
333
|
+
export async function declickSpeech(options: DeclickSpeechOptions): Promise<DeclickReport> {
|
|
334
|
+
const { input, output, seconds } = options;
|
|
335
|
+
const aborting = options.signal === undefined ? {} : { signal: options.signal };
|
|
336
|
+
const raw = `${output}.f32`;
|
|
337
|
+
const report = (done: number): void => options.onSeconds?.(Math.min(seconds, Math.max(0, done)));
|
|
338
|
+
try {
|
|
339
|
+
await runFfmpeg("The speech declick's decode", buildDeclickDecodeArgs(input, raw), { ...aborting, output: raw, onSeconds: (done) => report(done / 2) });
|
|
340
|
+
const declicked = declickSamples({ samples: readFloats(raw), channels: CHANNELS, sampleRate: FOOTAGE_SAMPLE_RATE, words: options.words, level: options.level });
|
|
341
|
+
writeFloats(raw, declicked.samples);
|
|
342
|
+
await runFfmpeg("The speech declick's encode", buildDeclickEncodeArgs(raw, output), { ...aborting, output, onSeconds: (done) => report(seconds / 2 + done / 2) });
|
|
343
|
+
return declicked.report;
|
|
344
|
+
} finally {
|
|
345
|
+
rmSync(raw, { force: true });
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/** The stage's log line. */
|
|
350
|
+
export function describeDeclick(report: DeclickReport): string {
|
|
351
|
+
const gate = report.gateDb === null ? "no words, so no gate" : `the pauses lowered by ${String(-report.gateDb)} dB`;
|
|
352
|
+
return `declick (${report.level}): ${String(report.impulses)} impulse${report.impulses === 1 ? "" : "s"} removed, ${gate}.`;
|
|
353
|
+
}
|
package/src/footage/draft.ts
CHANGED
|
@@ -2,15 +2,16 @@ import { copyFileSync, existsSync, mkdirSync, rmSync, statSync, writeFileSync }
|
|
|
2
2
|
import { basename, dirname, join, resolve } from "node:path";
|
|
3
3
|
import { fileURLToPath } from "node:url";
|
|
4
4
|
|
|
5
|
-
import {
|
|
5
|
+
import type { CaptionLayout, LayoutOverride } from "../compose/timeline.js";
|
|
6
6
|
import type { BrandColors } from "../config/colors.js";
|
|
7
7
|
import { err, ok } from "../voice/provider.js";
|
|
8
8
|
import type { TimedWord } from "../voice/voiceover.js";
|
|
9
|
-
import { getCaptionSpans, getFootageCaptionLayout,
|
|
9
|
+
import { getCaptionChunkLayouts, getCaptionSpans, getFootageCaptionChunks, getFootageCaptionLayout, isSameCaptionPlace, type CaptionWordsFilm } from "./captions.js";
|
|
10
10
|
import { getSegmentScales, getSegmentZoomEases } from "./compose.js";
|
|
11
11
|
import { chooseFootageDraftFont } from "./draft-font.js";
|
|
12
12
|
import { getEdlDuration, getSegmentStarts, remapWords, type Edl } from "./edl.js";
|
|
13
|
-
import {
|
|
13
|
+
import { declickSpeech, describeDeclick, type DeclickReport } from "./declick.js";
|
|
14
|
+
import { getDeclickLevel, getFootageEdit, type FootageFilm } from "./film.js";
|
|
14
15
|
import { duckingKeyframes, getDuckingOptions } from "./music.js";
|
|
15
16
|
import { FOOTAGE_FPS, getThreadArgs, type Frame } from "./normalize.js";
|
|
16
17
|
import { findFootageMusic, getFfmpegReporting, getFrame, type FootageBrand, type FootageResult, type FootageRunOptions, type PreparedClip, type TranscribeOutcome } from "./pipeline.js";
|
|
@@ -80,16 +81,25 @@ export function toAssColor(hex: string): string {
|
|
|
80
81
|
return `&H${alpha}${pair(4)}${pair(2)}${pair(0)}`;
|
|
81
82
|
}
|
|
82
83
|
|
|
83
|
-
/**
|
|
84
|
+
/**
|
|
85
|
+
* Caption text as ASS dialogue: override braces and backslashes would be read as tags, line breaks end the line. A
|
|
86
|
+
* no-break space (U+00A0) becomes ASS's hard space `\h`, which libass never breaks a line at, as the render's HTML.
|
|
87
|
+
*/
|
|
84
88
|
function toAssText(text: string): string {
|
|
85
|
-
return text
|
|
89
|
+
return text
|
|
90
|
+
.replace(/\\/g, "/")
|
|
91
|
+
.replace(/\{/g, "(")
|
|
92
|
+
.replace(/\}/g, ")")
|
|
93
|
+
.replace(/[^\S\u00a0]+/g, " ")
|
|
94
|
+
.replace(/^ +| +$/g, "")
|
|
95
|
+
.replace(/\u00a0/g, "\\h");
|
|
86
96
|
}
|
|
87
97
|
|
|
88
98
|
/** The caption colours the draft paints with (`BrandColors`). */
|
|
89
99
|
export type FootageDraftCaptionColors = Pick<BrandColors, "captionBackground" | "captionText" | "captionHighlight">;
|
|
90
100
|
|
|
91
101
|
export interface FootageDraftCaptionsInput {
|
|
92
|
-
film: Pick<FootageFilm, "captions" | "captionStyle"
|
|
102
|
+
film: Pick<FootageFilm, "captions" | "captionStyle"> & CaptionWordsFilm;
|
|
93
103
|
edl: Edl;
|
|
94
104
|
/** The film's words in film time (`remapWords`). */
|
|
95
105
|
words: TimedWord[];
|
|
@@ -107,7 +117,7 @@ export interface FootageDraftCaptionsInput {
|
|
|
107
117
|
* A `highlight` chunk as ASS events: its span cut wherever a word lights up or goes out, each piece showing the chunk
|
|
108
118
|
* with the words lit then in `highlight` (a word stays lit for the render's 0.32 s from its start).
|
|
109
119
|
*/
|
|
110
|
-
function buildHighlightEvents(chunk: { words: TimedWord[] }, span: { start: number; end: number }, colors: { text: string; highlight: string }): string[] {
|
|
120
|
+
function buildHighlightEvents(chunk: { words: TimedWord[] }, span: { start: number; end: number }, colors: { text: string; highlight: string }, styleName: string): string[] {
|
|
111
121
|
const cs = (seconds: number): number => Math.round(seconds * 100);
|
|
112
122
|
const [from, to] = [cs(span.start), cs(span.end)];
|
|
113
123
|
const lights = chunk.words.map((word) => ({ on: cs(word.start), off: cs(word.start + HIGHLIGHT_SECONDS) }));
|
|
@@ -122,7 +132,7 @@ function buildHighlightEvents(chunk: { words: TimedWord[] }, span: { start: numb
|
|
|
122
132
|
return light.on <= at && at < light.off ? `{\\1c${colors.highlight}&}${shown}{\\1c${colors.text}&}` : shown;
|
|
123
133
|
})
|
|
124
134
|
.join(" ");
|
|
125
|
-
events.push(`Dialogue: 0,${toAssTime(at / 100)},${toAssTime(next / 100)}
|
|
135
|
+
events.push(`Dialogue: 0,${toAssTime(at / 100)},${toAssTime(next / 100)},${styleName},,0,0,0,,${text}`);
|
|
126
136
|
});
|
|
127
137
|
return events;
|
|
128
138
|
}
|
|
@@ -132,12 +142,13 @@ function buildHighlightEvents(chunk: { words: TimedWord[] }, span: { start: numb
|
|
|
132
142
|
* caption box from its top (or from its foot, with `bottom`). `phrase` (the default): the film's phrases
|
|
133
143
|
* (`phraseChunks`), white bold text with a dark outline and shadow. `highlight`: the render's chunks of up to four
|
|
134
144
|
* words (`captionChunks`) on an opaque box of `captionBackground`, in `captionText`, the word being said in
|
|
135
|
-
* `captionHighlight`.
|
|
145
|
+
* `captionHighlight`. Both keep the film's kept words with the next, as the render does. Null when the film has no
|
|
146
|
+
* captions or nothing is said.
|
|
136
147
|
*/
|
|
137
148
|
export function buildFootageDraftCaptions(input: FootageDraftCaptionsInput): string | null {
|
|
138
149
|
if (!input.film.captions) return null;
|
|
139
150
|
const isHighlight = input.film.captionStyle === "highlight";
|
|
140
|
-
const chunks =
|
|
151
|
+
const chunks = getFootageCaptionChunks(input.film, input.words);
|
|
141
152
|
if (chunks.length === 0) return null;
|
|
142
153
|
const spans = getCaptionSpans(chunks, getEdlDuration(input.edl));
|
|
143
154
|
const { caption, frame } = input;
|
|
@@ -156,8 +167,8 @@ export function buildFootageDraftCaptions(input: FootageDraftCaptionsInput): str
|
|
|
156
167
|
outline: Math.max(1, Math.round(caption.fontSize / 16)),
|
|
157
168
|
shadow: Math.max(1, Math.round(caption.fontSize / 25)),
|
|
158
169
|
};
|
|
159
|
-
const
|
|
160
|
-
|
|
170
|
+
const toStyle = (name: string, place: CaptionLayout): string => [
|
|
171
|
+
name,
|
|
161
172
|
family,
|
|
162
173
|
caption.fontSize,
|
|
163
174
|
look.primary,
|
|
@@ -176,18 +187,31 @@ export function buildFootageDraftCaptions(input: FootageDraftCaptionsInput): str
|
|
|
176
187
|
look.outline,
|
|
177
188
|
look.shadow,
|
|
178
189
|
// ASS alignment 2 hangs the text from MarginV above the bottom, 8 from MarginV below the top.
|
|
179
|
-
|
|
180
|
-
Math.round(
|
|
181
|
-
Math.round(
|
|
182
|
-
Math.round(
|
|
190
|
+
place.bottom === undefined ? 8 : 2,
|
|
191
|
+
Math.round(place.left),
|
|
192
|
+
Math.round(place.right),
|
|
193
|
+
Math.round(place.bottom ?? place.top),
|
|
183
194
|
1,
|
|
184
195
|
].join(",");
|
|
196
|
+
// A chunk in a segment's own box gets a style of that box (Caption2, Caption3, ...); the film's box stays `Caption`.
|
|
197
|
+
const styles = [toStyle("Caption", caption)];
|
|
198
|
+
const moved: CaptionLayout[] = [];
|
|
199
|
+
const styleNames = getCaptionChunkLayouts(chunks, input.edl, caption).map((place) => {
|
|
200
|
+
if (isSameCaptionPlace(place, caption)) return "Caption";
|
|
201
|
+
let index = moved.findIndex((other) => isSameCaptionPlace(other, place));
|
|
202
|
+
if (index === -1) {
|
|
203
|
+
index = moved.push(place) - 1;
|
|
204
|
+
styles.push(toStyle(`Caption${index + 2}`, place));
|
|
205
|
+
}
|
|
206
|
+
return `Caption${index + 2}`;
|
|
207
|
+
});
|
|
185
208
|
const highlight = { text: text.replace(/^&H../, "&H"), highlight: toAssColor(colors.captionHighlight).replace(/^&H../, "&H") };
|
|
186
209
|
const events = chunks.flatMap((chunk, index) => {
|
|
187
210
|
const span = spans[index] as { start: number; end: number };
|
|
188
|
-
|
|
211
|
+
const styleName = styleNames[index] ?? "Caption";
|
|
212
|
+
if (isHighlight) return buildHighlightEvents(chunk, span, highlight, styleName);
|
|
189
213
|
const line = toAssText(chunk.words.map((word) => word.text).join(" "));
|
|
190
|
-
return [`Dialogue: 0,${toAssTime(span.start)},${toAssTime(span.end)}
|
|
214
|
+
return [`Dialogue: 0,${toAssTime(span.start)},${toAssTime(span.end)},${styleName},,0,0,0,,${line}`];
|
|
191
215
|
});
|
|
192
216
|
return [
|
|
193
217
|
"[Script Info]",
|
|
@@ -199,7 +223,7 @@ export function buildFootageDraftCaptions(input: FootageDraftCaptionsInput): str
|
|
|
199
223
|
"",
|
|
200
224
|
"[V4+ Styles]",
|
|
201
225
|
"Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding",
|
|
202
|
-
`Style: ${style}
|
|
226
|
+
...styles.map((style) => `Style: ${style}`),
|
|
203
227
|
"",
|
|
204
228
|
"[Events]",
|
|
205
229
|
"Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text",
|
|
@@ -427,6 +451,8 @@ export interface FootageDraft {
|
|
|
427
451
|
width: number;
|
|
428
452
|
height: number;
|
|
429
453
|
bytes: number;
|
|
454
|
+
/** What the speech track's declick stage did, as the render does it; null when the film's declick is off. */
|
|
455
|
+
declick: DeclickReport | null;
|
|
430
456
|
}
|
|
431
457
|
|
|
432
458
|
/** The draft's music track: a file to duck, or why there is none. */
|
|
@@ -478,13 +504,16 @@ async function runRenderFootageDraft(
|
|
|
478
504
|
const draftDir = resolve(options.buildDir, "draft");
|
|
479
505
|
rmSync(draftDir, { recursive: true, force: true });
|
|
480
506
|
mkdirSync(draftDir, { recursive: true });
|
|
481
|
-
const
|
|
507
|
+
const declickLevel = getDeclickLevel(film);
|
|
508
|
+
const items = ["speech", ...(declickLevel === "off" ? [] : ["declick"]), ...(music.value.kind === "file" ? ["music"] : []), "film"];
|
|
482
509
|
const report = (item: string, done: number): void => {
|
|
483
510
|
const itemIndex = items.indexOf(item) + 1;
|
|
484
511
|
options.onProgress?.({ step: "draft", item, itemIndex, itemCount: items.length, done: Math.min(seconds * items.length, (itemIndex - 1) * seconds + done), total: seconds * items.length });
|
|
485
512
|
};
|
|
486
513
|
const clipFiles = Object.fromEntries(clips.map((clip) => [clip.clip.id, resolve(clip.file)]));
|
|
487
|
-
const
|
|
514
|
+
const cutSpeech = join(draftDir, "speech.mp3");
|
|
515
|
+
// The declicked track stays WAV: the draft's one encode is its only lossy step after the cut.
|
|
516
|
+
const speech = declickLevel === "off" ? cutSpeech : join(draftDir, "speech.declicked.wav");
|
|
488
517
|
const words = remapWords(edl, transcript.words, film.cut.fillers);
|
|
489
518
|
const frame = getFrame(film);
|
|
490
519
|
const brandFont = chooseFootageDraftFont(options.brand?.fonts.body ?? null);
|
|
@@ -509,7 +538,13 @@ async function runRenderFootageDraft(
|
|
|
509
538
|
const output = resolve(options.output);
|
|
510
539
|
mkdirSync(dirname(output), { recursive: true });
|
|
511
540
|
report("speech", 0);
|
|
512
|
-
await runFfmpeg("The draft's speech track", buildSpeechArgs(edl, transcript.speech,
|
|
541
|
+
await runFfmpeg("The draft's speech track", buildSpeechArgs(edl, transcript.speech, cutSpeech, getSpeechCutOptions(getFootageEdit(film))), { ...aborting, output: cutSpeech, onSeconds: (done) => report("speech", done) });
|
|
542
|
+
let declick: DeclickReport | null = null;
|
|
543
|
+
if (declickLevel !== "off") {
|
|
544
|
+
report("declick", 0);
|
|
545
|
+
declick = await declickSpeech({ input: cutSpeech, output: speech, words, level: declickLevel, seconds, ...aborting, onSeconds: (done) => report("declick", done) });
|
|
546
|
+
log(describeDeclick(declick));
|
|
547
|
+
}
|
|
513
548
|
let musicFile: string | null = null;
|
|
514
549
|
if (music.value.kind === "file") {
|
|
515
550
|
musicFile = join(draftDir, "music.mp3");
|
|
@@ -542,5 +577,5 @@ async function runRenderFootageDraft(
|
|
|
542
577
|
const draftMusic: FootageDraftMusic = music.value.kind === "file" ? "played" : music.value.kind;
|
|
543
578
|
const notes = [captions === null ? "no captions" : `captions in ${font.family}`, draftMusic === "played" ? "music" : draftMusic === "none" ? "no music" : "music not cached"];
|
|
544
579
|
log(`draft: ${output}, ${draft.width}×${draft.height}, ${seconds.toFixed(1)} s, ${bytes} bytes, ${notes.join(", ")}.`);
|
|
545
|
-
return ok({ output, music: draftMusic, captionFont: captions === null ? null : font.family, seconds, width: draft.width, height: draft.height, bytes });
|
|
580
|
+
return ok({ output, music: draftMusic, captionFont: captions === null ? null : font.family, seconds, width: draft.width, height: draft.height, bytes, declick });
|
|
546
581
|
}
|