@softure-ai/marketing-kit 0.19.0 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,8 +2,12 @@ import { readFileSync, rmSync, writeFileSync } from "node:fs";
2
2
 
3
3
  import type { TimedWord } from "../voice/voiceover.js";
4
4
  import { getAudioCodecArgs } from "./clip-sound.js";
5
+ import { MOUTH_CLICK_RULES, analyzeMouthClicks } from "./mouth-clicks.js";
5
6
  import { FOOTAGE_SAMPLE_RATE } from "./normalize.js";
6
7
  import { runFfmpeg } from "./process.js";
8
+ import { getBand, getFrameEnergies, getRunningMedian, type TimeSpan } from "./sound-analysis.js";
9
+
10
+ export { getRunningMedian, type TimeSpan };
7
11
 
8
12
  /**
9
13
  * The speech track's declick stage (issues #1084, #1170), between the cut speech track and its loudness normalization:
@@ -17,22 +21,30 @@ import { runFfmpeg } from "./process.js";
17
21
  * samples are replaced by AR (LPC) interpolation from the sound on both sides, joined with a raised-cosine crossfade
18
22
  * of `DECLICK_CROSSFADE` on each side. Every other sample leaves the stage as it came in.
19
23
  *
24
+ * The `repair` level (issue #1183) uses the mouth-click detector shared with moving a cut (`MOUTH_CLICK_RULES`,
25
+ * issue #1182) instead: impulses of 1 to 25 ms, outside the speech protection zone (word bounds on the real sound
26
+ * ±80 ms, nothing above -45 dBFS), only in pauses at least 150 ms long after the guard, and within the limits of
27
+ * `DECLICK_REPAIR_RULES` (the impulses over a limit are left and counted).
28
+ *
20
29
  * Without words the stage repairs nothing: it cannot tell a click from a consonant.
21
30
  */
22
31
 
23
- export const DECLICK_LEVELS = ["off", "gentle", "strong"] as const;
32
+ export const DECLICK_LEVELS = ["off", "gentle", "strong", "repair"] as const;
24
33
 
25
34
  export type DeclickLevel = (typeof DECLICK_LEVELS)[number];
26
35
 
27
36
  /** A level that runs the stage. */
28
37
  export type ActiveDeclickLevel = Exclude<DeclickLevel, "off">;
29
38
 
39
+ /** A level that runs the 0.18.0 detector (`findImpulses`, `DECLICK_SETTINGS`). */
40
+ export type DetectorDeclickLevel = Exclude<ActiveDeclickLevel, "repair">;
41
+
30
42
  export interface DeclickSettings {
31
43
  /** How far a frame's 2-8 kHz energy must rise above the band's running median to count as an impulse, in dB. */
32
44
  jumpDb: number;
33
45
  }
34
46
 
35
- export const DECLICK_SETTINGS: Record<ActiveDeclickLevel, DeclickSettings> = {
47
+ export const DECLICK_SETTINGS: Record<DetectorDeclickLevel, DeclickSettings> = {
36
48
  gentle: { jumpDb: 15 },
37
49
  strong: { jumpDb: 9 },
38
50
  };
@@ -43,6 +55,17 @@ export const DECLICK_WORD_GUARD = 0.08;
43
55
  export const DECLICK_MAX_IMPULSE = 0.012;
44
56
  /** The raised-cosine crossfade between the original and the interpolation on each side of a repair. */
45
57
  export const DECLICK_CROSSFADE = 0.003;
58
+ /** The `repair` level's rules (issue #1183); its impulses are `MOUTH_CLICK_RULES`'. */
59
+ export const DECLICK_REPAIR_RULES = {
60
+ /** The shortest pause that holds a repair once the word guard is taken off both sides, seconds. */
61
+ minPauseSeconds: 0.15,
62
+ /** The longest repair, its crossfades included, seconds. */
63
+ maxRepairSeconds: 0.025,
64
+ /** The most repairs in any one second. */
65
+ maxPerSecond: 3,
66
+ /** The share of the track's length all repairs (crossfades included) may take. */
67
+ maxShare: 0.01,
68
+ } as const;
46
69
  /** The AR model's order. */
47
70
  export const DECLICK_AR_ORDER = 24;
48
71
  /** Seconds of sound on each side of a repair the AR model is fitted on. */
@@ -61,11 +84,6 @@ const BAND_FLOOR_DB = -90;
61
84
  /** Added to Burg's denominator, so a silent context gives a zero model instead of a division by zero. */
62
85
  const AR_RIDGE = 1e-9;
63
86
 
64
- export interface TimeSpan {
65
- start: number;
66
- end: number;
67
- }
68
-
69
87
  /** What the stage did, in the render's and the draft's ok value. */
70
88
  export interface DeclickReport {
71
89
  level: ActiveDeclickLevel;
@@ -73,6 +91,8 @@ export interface DeclickReport {
73
91
  impulses: number;
74
92
  /** Each repaired impulse in the speech track's time, seconds (without its crossfades). */
75
93
  repairs: TimeSpan[];
94
+ /** Impulses the `repair` level left because a limit of `DECLICK_REPAIR_RULES` was reached; 0 for the other levels. */
95
+ skippedLimit: number;
76
96
  }
77
97
 
78
98
  const sortSpans = (words: readonly TimeSpan[]): TimeSpan[] => [...words].filter((word) => word.end > word.start).sort((a, b) => a.start - b.start);
@@ -99,91 +119,6 @@ export function getRepairZones(words: readonly TimeSpan[], seconds: number): Tim
99
119
  return zones;
100
120
  }
101
121
 
102
- /** One second-order section (RBJ cookbook), run in place over the samples. */
103
- function runBiquad(samples: Float64Array, kind: "highpass" | "lowpass", frequency: number, sampleRate: number): void {
104
- const w0 = (2 * Math.PI * Math.min(frequency, sampleRate * 0.45)) / sampleRate;
105
- const cos = Math.cos(w0);
106
- const alpha = Math.sin(w0) / (2 * Math.SQRT1_2);
107
- const a0 = 1 + alpha;
108
- const b0 = (kind === "lowpass" ? (1 - cos) / 2 : (1 + cos) / 2) / a0;
109
- const b1 = (kind === "lowpass" ? 1 - cos : -(1 + cos)) / a0;
110
- const b2 = b0;
111
- const a1 = (-2 * cos) / a0;
112
- const a2 = (1 - alpha) / a0;
113
- let x1 = 0;
114
- let x2 = 0;
115
- let y1 = 0;
116
- let y2 = 0;
117
- for (let index = 0; index < samples.length; index++) {
118
- const x = samples[index] ?? 0;
119
- const y = b0 * x + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2;
120
- x2 = x1;
121
- x1 = x;
122
- y2 = y1;
123
- y1 = y;
124
- samples[index] = y;
125
- }
126
- }
127
-
128
- /** The 2-8 kHz band of the signal: two high-pass and two low-pass sections (fourth order on each side). */
129
- function getBand(mono: Float32Array, sampleRate: number): Float64Array {
130
- const band = Float64Array.from(mono);
131
- runBiquad(band, "highpass", BAND_LOW_HZ, sampleRate);
132
- runBiquad(band, "highpass", BAND_LOW_HZ, sampleRate);
133
- runBiquad(band, "lowpass", BAND_HIGH_HZ, sampleRate);
134
- runBiquad(band, "lowpass", BAND_HIGH_HZ, sampleRate);
135
- return band;
136
- }
137
-
138
- /** Each frame's mean square in dB, frames of `frameLength` samples every `hop` samples. */
139
- function getFrameEnergies(band: Float64Array, frameLength: number, hop: number): Float64Array {
140
- const sums = new Float64Array(band.length + 1);
141
- for (let index = 0; index < band.length; index++) {
142
- const value = band[index] ?? 0;
143
- sums[index + 1] = (sums[index] ?? 0) + value * value;
144
- }
145
- const count = band.length < frameLength ? 0 : Math.floor((band.length - frameLength) / hop) + 1;
146
- const energies = new Float64Array(count);
147
- for (let frame = 0; frame < count; frame++) {
148
- const from = frame * hop;
149
- const power = ((sums[from + frameLength] ?? 0) - (sums[from] ?? 0)) / frameLength;
150
- energies[frame] = 10 * Math.log10(Math.max(power, 1e-20));
151
- }
152
- return energies;
153
- }
154
-
155
- /** The index of the first entry of the sorted list not below `value`. */
156
- function findInsertIndex(sorted: number[], value: number): number {
157
- let low = 0;
158
- let high = sorted.length;
159
- while (low < high) {
160
- const middle = (low + high) >> 1;
161
- if ((sorted[middle] ?? 0) < value) low = middle + 1;
162
- else high = middle;
163
- }
164
- return low;
165
- }
166
-
167
- /** The running median of the values over `reach` entries on each side (fewer at the ends). */
168
- export function getRunningMedian(values: Float64Array, reach: number): Float64Array {
169
- const medians = new Float64Array(values.length);
170
- const window: number[] = [];
171
- let from = 0;
172
- let to = 0;
173
- for (let index = 0; index < values.length; index++) {
174
- const wantedTo = Math.min(values.length, index + reach + 1);
175
- for (; to < wantedTo; to++) {
176
- const value = values[to] ?? 0;
177
- window.splice(findInsertIndex(window, value), 0, value);
178
- }
179
- const wantedFrom = Math.max(0, index - reach);
180
- for (; from < wantedFrom; from++) window.splice(findInsertIndex(window, values[from] ?? 0), 1);
181
- const middle = window.length >> 1;
182
- medians[index] = window.length % 2 === 1 ? (window[middle] ?? 0) : ((window[middle - 1] ?? 0) + (window[middle] ?? 0)) / 2;
183
- }
184
- return medians;
185
- }
186
-
187
122
  /** True when the impulse with its crossfades lies inside one zone. */
188
123
  const isRepairable = (span: TimeSpan, zones: readonly TimeSpan[]): boolean =>
189
124
  zones.some((zone) => span.start - DECLICK_CROSSFADE >= zone.start && span.end + DECLICK_CROSSFADE <= zone.end);
@@ -193,14 +128,14 @@ const isRepairable = (span: TimeSpan, zones: readonly TimeSpan[]): boolean =>
193
128
  * the level's `jumpDb` above the band's running median (±100 ms), shorter than `DECLICK_MAX_IMPULSE`, that lie with
194
129
  * their crossfades inside a repair zone (`getRepairZones`). Times in seconds, widened by a millisecond on each side.
195
130
  */
196
- export function findImpulses(mono: Float32Array, sampleRate: number, words: readonly TimeSpan[], level: ActiveDeclickLevel): TimeSpan[] {
131
+ export function findImpulses(mono: Float32Array, sampleRate: number, words: readonly TimeSpan[], level: DetectorDeclickLevel): TimeSpan[] {
197
132
  const seconds = mono.length / sampleRate;
198
133
  const zones = getRepairZones(words, seconds);
199
134
  if (zones.length === 0) return [];
200
135
  const { jumpDb } = DECLICK_SETTINGS[level];
201
136
  const frameLength = Math.max(1, Math.round(FRAME_SECONDS * sampleRate));
202
137
  const hop = Math.max(1, Math.round(HOP_SECONDS * sampleRate));
203
- const energies = getFrameEnergies(getBand(mono, sampleRate), frameLength, hop);
138
+ const energies = getFrameEnergies(getBand({ mono, sampleRate, lowHz: BAND_LOW_HZ, highHz: BAND_HIGH_HZ }), frameLength, hop);
204
139
  const medians = getRunningMedian(energies, Math.round(MEDIAN_SECONDS / HOP_SECONDS));
205
140
  const impulses: TimeSpan[] = [];
206
141
  let runStart = -1;
@@ -222,6 +157,35 @@ export function findImpulses(mono: Float32Array, sampleRate: number, words: read
222
157
  return impulses;
223
158
  }
224
159
 
160
+ /**
161
+ * The impulses the `repair` level repairs (issue #1183): `MOUTH_CLICK_RULES` impulses whose repair (with its
162
+ * crossfades) lies inside a pause at least `minPauseSeconds` long between the words' bounds on the real sound after
163
+ * the guard, outside the speech protection zone, taken in time order while the limits allow; the others in such a
164
+ * pause are counted as `skippedLimit`.
165
+ */
166
+ export function findRepairs(mono: Float32Array, sampleRate: number, words: readonly TimeSpan[]): { repairs: TimeSpan[]; skippedLimit: number } {
167
+ const seconds = mono.length / sampleRate;
168
+ const analysis = analyzeMouthClicks(mono, sampleRate, words);
169
+ const zones = getRepairZones(analysis.bounds, seconds).filter((zone) => zone.end - zone.start >= DECLICK_REPAIR_RULES.minPauseSeconds);
170
+ const budget = DECLICK_REPAIR_RULES.maxShare * seconds;
171
+ const repairs: TimeSpan[] = [];
172
+ let used = 0;
173
+ let skippedLimit = 0;
174
+ for (const candidate of analysis.candidates) {
175
+ const span = { start: candidate.start - DECLICK_CROSSFADE, end: candidate.end + DECLICK_CROSSFADE };
176
+ if (!zones.some((zone) => span.start >= zone.start && span.end <= zone.end) || analysis.isProtected(span)) continue;
177
+ const length = span.end - span.start;
178
+ const inLastSecond = repairs.filter((repair) => repair.start > candidate.start - 1).length;
179
+ if (length > DECLICK_REPAIR_RULES.maxRepairSeconds || inLastSecond >= DECLICK_REPAIR_RULES.maxPerSecond || used + length > budget) {
180
+ skippedLimit++;
181
+ continue;
182
+ }
183
+ repairs.push(candidate);
184
+ used += length;
185
+ }
186
+ return { repairs, skippedLimit };
187
+ }
188
+
225
189
  /**
226
190
  * The AR polynomial of the context segments (Burg's method over all of them at once): `a[0] = 1` and
227
191
  * `Σ a[k]·x[n-k] ≈ 0`. Fewer coefficients when the segments are short; a silent context gives all zeros after `a[0]`.
@@ -340,7 +304,8 @@ export function declickSamples(options: { samples: Float32Array; channels: numbe
340
304
  for (let channel = 0; channel < channels; channel++) sum += samples[index * channels + channel] ?? 0;
341
305
  mono[index] = sum / channels;
342
306
  }
343
- const impulses = findImpulses(mono, sampleRate, words, level);
307
+ const found = level === "repair" ? findRepairs(mono, sampleRate, words) : { repairs: findImpulses(mono, sampleRate, words, level), skippedLimit: 0 };
308
+ const impulses = found.repairs;
344
309
  const output = Float32Array.from(samples);
345
310
  const crossfade = Math.round(DECLICK_CROSSFADE * sampleRate);
346
311
  const context = Math.round(DECLICK_AR_CONTEXT * sampleRate);
@@ -354,7 +319,7 @@ export function declickSamples(options: { samples: Float32Array; channels: numbe
354
319
  for (let index = Math.max(0, from); index < Math.min(length, to); index++) output[index * channels + channel] = track[index] ?? 0;
355
320
  }
356
321
  }
357
- return { samples: output, impulses, report: { level, impulses: impulses.length, repairs: impulses.map((impulse) => ({ ...impulse })) } };
322
+ return { samples: output, impulses, report: { level, impulses: impulses.length, repairs: impulses.map((impulse) => ({ ...impulse })), skippedLimit: found.skippedLimit } };
358
323
  }
359
324
 
360
325
  const CHANNELS = 2;
@@ -423,5 +388,7 @@ export function describeDeclick(report: DeclickReport): string {
423
388
  const times = report.repairs.slice(0, LOGGED_REPAIRS).map((repair) => `${repair.start.toFixed(3)} s`);
424
389
  const more = report.repairs.length > LOGGED_REPAIRS ? `, +${String(report.repairs.length - LOGGED_REPAIRS)} more` : "";
425
390
  const where = times.length === 0 ? "" : ` at ${times.join(", ")}${more}`;
426
- return `declick (${report.level}): ${String(report.impulses)} impulse${report.impulses === 1 ? "" : "s"} repaired${where}; no gain change, nothing within ${String(DECLICK_WORD_GUARD * 1000)} ms of a word touched.`;
391
+ const count = `${String(report.impulses)} impulse${report.impulses === 1 ? "" : "s"} repaired${where}`;
392
+ if (report.level !== "repair") return `declick (${report.level}): ${count}; no gain change, nothing within ${String(DECLICK_WORD_GUARD * 1000)} ms of a word touched.`;
393
+ return `declick (repair): ${count}, ${String(report.skippedLimit)} left over the limits; no gain change, nothing within ${String(DECLICK_WORD_GUARD * 1000)} ms of a word's sound or above ${String(MOUTH_CLICK_RULES.loudDbfs)} dBFS touched.`;
427
394
  }
@@ -216,8 +216,11 @@ export function getEndCardTexts(card: FootageEndCard | null): { headline: string
216
216
  /** Settings of the speech track itself. */
217
217
  export interface FootageSpeech {
218
218
  /**
219
- * Mouth clicks and lip smacks between words (issue #1084): `gentle` lowers the pauses by 15 dB and removes impulses
220
- * that jump 15 dB above the band's median, `strong` by 20 dB and 9 dB. Absent or `off`: no pass, the track unchanged.
219
+ * Mouth clicks and lip smacks between words (issues #1084, #1170, #1183): each single impulse in a pause is repaired
220
+ * by AR interpolation and no gain changes. `gentle` and `strong` repair impulses that jump 15 or 9 dB above the band's
221
+ * median, nothing within 80 ms of a transcript word; `repair` uses the shared mouth-click rules (`MOUTH_CLICK_RULES`):
222
+ * word bounds on the real sound, nothing above -45 dBFS, pauses of 150 ms or more, at most 3 repairs a second and
223
+ * 1 % of the track. Absent or `off`: no pass, the track unchanged.
221
224
  */
222
225
  declick?: DeclickLevel;
223
226
  }
@@ -0,0 +1,224 @@
1
+ import { getBand, getFrameEnergies, getRunningMedian, getSquareSums, toPowerDb, type TimeSpan } from "./sound-analysis.js";
2
+
3
+ /**
4
+ * The mouth-click detector shared by moving a segment's cut past a click (stage A, issue #1182) and the declick
5
+ * stage's `repair` level (stage B, issue #1183), so a click that moves a cut and a click that is repaired are the same
6
+ * thing. Its input is the speech as the chain hears it: the clips' sound after the -23 LUFS pre-gain.
7
+ *
8
+ * - **An impulse** lasts 1 to 25 ms, its 2-10 kHz energy rises more than 12 dB above the band's ±50 ms running median,
9
+ * it is not periodic (the band's normalized autocorrelation stays under 0.3 at the pitch lags, 2.5 to 15 ms, over a
10
+ * 30 ms window centred on it) and it is over within its length (the 10 ms after it at least 12 dB under its peak, so
11
+ * the onset of a breath or a fricative is not one).
12
+ * - **The speech protection zone:** each word's bound on the real sound (the earlier start and the later end of the
13
+ * transcript's bound and the point where the 10 ms energy falls 20 dB under the word's peak), widened by 80 ms; and
14
+ * every stretch whose 10 ms energy is above -45 dBFS, measured with the impulses' own samples left out (a click's
15
+ * own energy must not protect it).
16
+ */
17
+ export const MOUTH_CLICK_RULES = {
18
+ /** The shortest impulse, seconds. */
19
+ minSeconds: 0.001,
20
+ /** The longest impulse, seconds. */
21
+ maxSeconds: 0.025,
22
+ /** The band the impulse is measured in. */
23
+ bandLowHz: 2000,
24
+ bandHighHz: 10_000,
25
+ /** How far an impulse's band energy rises above the band's running median, dB. */
26
+ jumpDb: 12,
27
+ /** The running median's reach on each side, seconds. */
28
+ medianSeconds: 0.05,
29
+ /** The highest normalized autocorrelation at the pitch lags an impulse may show. */
30
+ maxCorrelation: 0.3,
31
+ /** The pitch lags, seconds. */
32
+ minLagSeconds: 0.0025,
33
+ maxLagSeconds: 0.015,
34
+ /** The window the autocorrelation is measured over, centred on the impulse, seconds. */
35
+ correlationSeconds: 0.03,
36
+ /** The stretch after an impulse that must be quieter than its peak, seconds, and by how much, dB. */
37
+ afterSeconds: 0.01,
38
+ afterDropDb: 12,
39
+ /** The energy window of the word bounds and the loudness rule, seconds. */
40
+ energySeconds: 0.01,
41
+ /** A word's sound lasts while its 10 ms energy stays within this many dB of the word's peak. */
42
+ wordDropDb: 20,
43
+ /** The guard on each side of a word's bound, seconds. */
44
+ wordGuardSeconds: 0.08,
45
+ /** Nothing whose 10 ms energy is above this is a click to act on, dBFS (mean square). */
46
+ loudDbfs: -45,
47
+ } as const;
48
+
49
+ const FRAME_SECONDS = 0.002;
50
+ const HOP_SECONDS = 0.001;
51
+ /** The step the word bounds and the loudness rule are measured at. */
52
+ const STEP_SECONDS = 0.001;
53
+ /** An impulse reaches this far past its detected frames, for the band filter's delay and the frames' edges. */
54
+ const IMPULSE_MARGIN = 0.001;
55
+ /** A frame quieter than this (mean square, dB) is never an impulse: digital silence with a stray bit is not a click. */
56
+ const BAND_FLOOR_DB = -90;
57
+
58
+ /** The highest normalized autocorrelation of `band` at the pitch lags over the window centred on `center`. */
59
+ function getPeakCorrelation(band: Float64Array, center: number, sampleRate: number): number {
60
+ const half = Math.round((MOUTH_CLICK_RULES.correlationSeconds * sampleRate) / 2);
61
+ const from = Math.max(0, center - half);
62
+ const to = Math.min(band.length, center + half);
63
+ const minLag = Math.max(1, Math.round(MOUTH_CLICK_RULES.minLagSeconds * sampleRate));
64
+ const maxLag = Math.min(Math.round(MOUTH_CLICK_RULES.maxLagSeconds * sampleRate), to - from - 1);
65
+ let peak = 0;
66
+ for (let lag = minLag; lag <= maxLag; lag++) {
67
+ let product = 0;
68
+ let head = 0;
69
+ let tail = 0;
70
+ for (let index = from; index + lag < to; index++) {
71
+ const a = band[index] ?? 0;
72
+ const b = band[index + lag] ?? 0;
73
+ product += a * b;
74
+ head += a * a;
75
+ tail += b * b;
76
+ }
77
+ if (head > 0 && tail > 0) peak = Math.max(peak, product / Math.sqrt(head * tail));
78
+ }
79
+ return peak;
80
+ }
81
+
82
+ /**
83
+ * The impulses of a mono signal by `MOUTH_CLICK_RULES` (length, band jump, periodicity, end), anywhere in it. Times in
84
+ * seconds from sample 0, widened by a millisecond on each side.
85
+ */
86
+ export function findClickCandidates(mono: Float32Array, sampleRate: number): TimeSpan[] {
87
+ const seconds = mono.length / sampleRate;
88
+ const band = getBand({ mono, sampleRate, lowHz: MOUTH_CLICK_RULES.bandLowHz, highHz: MOUTH_CLICK_RULES.bandHighHz });
89
+ const sums = getSquareSums(band);
90
+ const frameLength = Math.max(1, Math.round(FRAME_SECONDS * sampleRate));
91
+ const hop = Math.max(1, Math.round(HOP_SECONDS * sampleRate));
92
+ const after = Math.round(MOUTH_CLICK_RULES.afterSeconds * sampleRate);
93
+ const energies = getFrameEnergies(band, frameLength, hop);
94
+ const medians = getRunningMedian(energies, Math.round(MOUTH_CLICK_RULES.medianSeconds / HOP_SECONDS));
95
+ const candidates: TimeSpan[] = [];
96
+ const checkRun = (first: number, last: number): void => {
97
+ const start = (first * hop) / sampleRate;
98
+ const end = (last * hop + frameLength) / sampleRate;
99
+ if (end - start < MOUTH_CLICK_RULES.minSeconds || end - start > MOUTH_CLICK_RULES.maxSeconds) return;
100
+ const span = { start: Math.max(0, start - IMPULSE_MARGIN), end: Math.min(seconds, end + IMPULSE_MARGIN) };
101
+ let peakDb = -Infinity;
102
+ for (let frame = first; frame <= last; frame++) peakDb = Math.max(peakDb, energies[frame] ?? -Infinity);
103
+ const afterFrom = Math.round(span.end * sampleRate);
104
+ const afterTo = Math.min(band.length, afterFrom + after);
105
+ if (afterTo > afterFrom && toPowerDb(((sums[afterTo] ?? 0) - (sums[afterFrom] ?? 0)) / (afterTo - afterFrom)) > peakDb - MOUTH_CLICK_RULES.afterDropDb) return;
106
+ const center = Math.round(((span.start + span.end) / 2) * sampleRate);
107
+ if (getPeakCorrelation(band, center, sampleRate) >= MOUTH_CLICK_RULES.maxCorrelation) return;
108
+ candidates.push(span);
109
+ };
110
+ let runStart = -1;
111
+ for (let frame = 0; frame < energies.length; frame++) {
112
+ const energy = energies[frame] ?? 0;
113
+ const isJump = energy > BAND_FLOOR_DB && energy - (medians[frame] ?? 0) > MOUTH_CLICK_RULES.jumpDb;
114
+ if (isJump && runStart === -1) runStart = frame;
115
+ else if (!isJump && runStart !== -1) {
116
+ checkRun(runStart, frame - 1);
117
+ runStart = -1;
118
+ }
119
+ }
120
+ if (runStart !== -1) checkRun(runStart, energies.length - 1);
121
+ return candidates;
122
+ }
123
+
124
+ /** The analysis of one mono stretch in its own time (sample 0 at 0 s). */
125
+ export interface MouthClickAnalysis {
126
+ /** Every impulse by the rule, protected or not. */
127
+ candidates: TimeSpan[];
128
+ /** Each word's bound on the real sound (before the guard). */
129
+ bounds: TimeSpan[];
130
+ /** True when the span touches a word's bound with its guard, or a 10 ms energy above the loudness rule. */
131
+ isProtected: (span: TimeSpan) => boolean;
132
+ /** True when the stretch holds no 10 ms window above the loudness rule, the impulses' samples left out. */
133
+ isQuiet: (stretch: TimeSpan) => boolean;
134
+ }
135
+
136
+ /** The detector over a mono stretch whose words are in the stretch's own time. */
137
+ export function analyzeMouthClicks(mono: Float32Array, sampleRate: number, words: readonly TimeSpan[]): MouthClickAnalysis {
138
+ const length = mono.length;
139
+ const candidates = findClickCandidates(mono, sampleRate);
140
+ const sums = getSquareSums(mono);
141
+ const masked = Float64Array.from(mono);
142
+ const counts = new Float64Array(length + 1);
143
+ const isMasked = new Uint8Array(length);
144
+ for (const candidate of candidates) {
145
+ for (let index = Math.max(0, Math.floor(candidate.start * sampleRate)); index < Math.min(length, Math.ceil(candidate.end * sampleRate)); index++) {
146
+ isMasked[index] = 1;
147
+ masked[index] = 0;
148
+ }
149
+ }
150
+ for (let index = 0; index < length; index++) counts[index + 1] = (counts[index] ?? 0) + (isMasked[index] === 1 ? 0 : 1);
151
+ const maskedSums = getSquareSums(masked);
152
+ const window = Math.max(1, Math.round(MOUTH_CLICK_RULES.energySeconds * sampleRate));
153
+ const step = Math.max(1, Math.round(STEP_SECONDS * sampleRate));
154
+ const clamp = (index: number): number => Math.min(length, Math.max(0, index));
155
+ const getDb = (from: number, to: number): number => {
156
+ const [a, b] = [clamp(from), clamp(to)];
157
+ return b > a ? toPowerDb(((sums[b] ?? 0) - (sums[a] ?? 0)) / (b - a)) : -Infinity;
158
+ };
159
+ const getMaskedDb = (from: number, to: number): number => {
160
+ const [a, b] = [clamp(from), clamp(to)];
161
+ const count = (counts[b] ?? 0) - (counts[a] ?? 0);
162
+ return count > 0 ? toPowerDb(((maskedSums[b] ?? 0) - (maskedSums[a] ?? 0)) / count) : -Infinity;
163
+ };
164
+ const getBound = (word: TimeSpan): TimeSpan => {
165
+ const from = clamp(Math.round(word.start * sampleRate));
166
+ const to = clamp(Math.round(word.end * sampleRate));
167
+ if (to <= from) return { ...word };
168
+ let peakDb = -Infinity;
169
+ for (let index = from; index <= Math.max(from, to - window); index += step) peakDb = Math.max(peakDb, getDb(index, Math.min(to, index + window)));
170
+ const threshold = peakDb - MOUTH_CLICK_RULES.wordDropDb;
171
+ // The start walks back while the 10 ms before it hold the word's sound, the end forward while the 10 ms after it do.
172
+ let start = from;
173
+ while (start > 0 && getDb(start - window, start) >= threshold) start = clamp(start - step);
174
+ let end = to;
175
+ while (end < length && getDb(end, end + window) >= threshold) end = clamp(end + step);
176
+ return { start: Math.min(word.start, start / sampleRate), end: Math.max(word.end, end / sampleRate) };
177
+ };
178
+ const bounds = [...words].filter((word) => word.end > word.start).sort((a, b) => a.start - b.start).map(getBound);
179
+ const isLoud = (span: TimeSpan): boolean => {
180
+ const from = Math.floor(span.start * sampleRate);
181
+ const to = Math.ceil(span.end * sampleRate);
182
+ for (let center = from; ; center = Math.min(to, center + step)) {
183
+ if (getMaskedDb(center - (window >> 1), center + window - (window >> 1)) > MOUTH_CLICK_RULES.loudDbfs) return true;
184
+ if (center >= to) return false;
185
+ }
186
+ };
187
+ const guard = MOUTH_CLICK_RULES.wordGuardSeconds;
188
+ return {
189
+ candidates,
190
+ bounds,
191
+ isProtected: (span) => bounds.some((bound) => span.start < bound.end + guard && span.end > bound.start - guard) || isLoud(span),
192
+ isQuiet: (stretch) => stretch.end <= 0 || stretch.start >= length / sampleRate || !isLoud({ start: Math.max(0, stretch.start), end: Math.min(length / sampleRate, stretch.end) }),
193
+ };
194
+ }
195
+
196
+ /** What `findMouthClicks` found, in the film's time (the stretch's `offset` added). */
197
+ export interface MouthClicks {
198
+ /** The impulses outside the speech protection zone, in time order. */
199
+ clicks: TimeSpan[];
200
+ /** Each word's bound on the real sound (before the ±80 ms guard), in time order. */
201
+ words: TimeSpan[];
202
+ /** True when the stretch holds no 10 ms window above -45 dBFS, the impulses' samples left out. */
203
+ isQuiet: (stretch: TimeSpan) => boolean;
204
+ }
205
+
206
+ /**
207
+ * Mouth clicks of a mono stretch whose sample 0 plays at `offset` seconds, outside the speech protection zone
208
+ * (`MOUTH_CLICK_RULES`); `words` are in the same time as `offset`. No audio is changed: an app can move a segment's
209
+ * cut past a click before its first word or after its last (issue #1182).
210
+ */
211
+ export function findMouthClicks(options: { mono: Float32Array; sampleRate: number; offset: number; words: readonly TimeSpan[] }): MouthClicks {
212
+ const { mono, sampleRate, offset } = options;
213
+ const shift = (span: TimeSpan, by: number): TimeSpan => ({ start: span.start + by, end: span.end + by });
214
+ const analysis = analyzeMouthClicks(
215
+ mono,
216
+ sampleRate,
217
+ options.words.map((word) => shift(word, -offset)),
218
+ );
219
+ return {
220
+ clicks: analysis.candidates.filter((candidate) => !analysis.isProtected(candidate)).map((candidate) => shift(candidate, offset)),
221
+ words: analysis.bounds.map((bound) => shift(bound, offset)),
222
+ isQuiet: (stretch) => analysis.isQuiet(shift(stretch, -offset)),
223
+ };
224
+ }
@@ -0,0 +1,104 @@
1
+ /**
2
+ * Sound analysis shared by the speech declick stage and the mouth-click detector (issues #1170, #1182, #1183): a
3
+ * fourth-order band pass, frame energies and a running median.
4
+ */
5
+
6
+ /** A stretch of time in seconds. */
7
+ export interface TimeSpan {
8
+ start: number;
9
+ end: number;
10
+ }
11
+
12
+ /** One second-order section (RBJ cookbook), run in place over the samples. */
13
+ function runBiquad(samples: Float64Array, kind: "highpass" | "lowpass", frequency: number, sampleRate: number): void {
14
+ const w0 = (2 * Math.PI * Math.min(frequency, sampleRate * 0.45)) / sampleRate;
15
+ const cos = Math.cos(w0);
16
+ const alpha = Math.sin(w0) / (2 * Math.SQRT1_2);
17
+ const a0 = 1 + alpha;
18
+ const b0 = (kind === "lowpass" ? (1 - cos) / 2 : (1 + cos) / 2) / a0;
19
+ const b1 = (kind === "lowpass" ? 1 - cos : -(1 + cos)) / a0;
20
+ const b2 = b0;
21
+ const a1 = (-2 * cos) / a0;
22
+ const a2 = (1 - alpha) / a0;
23
+ let x1 = 0;
24
+ let x2 = 0;
25
+ let y1 = 0;
26
+ let y2 = 0;
27
+ for (let index = 0; index < samples.length; index++) {
28
+ const x = samples[index] ?? 0;
29
+ const y = b0 * x + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2;
30
+ x2 = x1;
31
+ x1 = x;
32
+ y2 = y1;
33
+ y1 = y;
34
+ samples[index] = y;
35
+ }
36
+ }
37
+
38
+ /** The band of the signal between `lowHz` and `highHz`: two high-pass and two low-pass sections (fourth order on each side). */
39
+ export function getBand(options: { mono: Float32Array; sampleRate: number; lowHz: number; highHz: number }): Float64Array {
40
+ const { mono, sampleRate, lowHz, highHz } = options;
41
+ const band = Float64Array.from(mono);
42
+ runBiquad(band, "highpass", lowHz, sampleRate);
43
+ runBiquad(band, "highpass", lowHz, sampleRate);
44
+ runBiquad(band, "lowpass", highHz, sampleRate);
45
+ runBiquad(band, "lowpass", highHz, sampleRate);
46
+ return band;
47
+ }
48
+
49
+ /** The running sums of the squared samples: `sums[i]` is the energy of samples 0 to i - 1. */
50
+ export function getSquareSums(samples: ArrayLike<number>): Float64Array {
51
+ const sums = new Float64Array(samples.length + 1);
52
+ for (let index = 0; index < samples.length; index++) {
53
+ const value = samples[index] ?? 0;
54
+ sums[index + 1] = (sums[index] ?? 0) + value * value;
55
+ }
56
+ return sums;
57
+ }
58
+
59
+ /** A mean square in dB (10·log10), with a floor so silence stays finite. */
60
+ export const toPowerDb = (meanSquare: number): number => 10 * Math.log10(Math.max(meanSquare, 1e-20));
61
+
62
+ /** Each frame's mean square in dB, frames of `frameLength` samples every `hop` samples. */
63
+ export function getFrameEnergies(band: Float64Array, frameLength: number, hop: number): Float64Array {
64
+ const sums = getSquareSums(band);
65
+ const count = band.length < frameLength ? 0 : Math.floor((band.length - frameLength) / hop) + 1;
66
+ const energies = new Float64Array(count);
67
+ for (let frame = 0; frame < count; frame++) {
68
+ const from = frame * hop;
69
+ energies[frame] = toPowerDb(((sums[from + frameLength] ?? 0) - (sums[from] ?? 0)) / frameLength);
70
+ }
71
+ return energies;
72
+ }
73
+
74
+ /** The index of the first entry of the sorted list not below `value`. */
75
+ function findInsertIndex(sorted: number[], value: number): number {
76
+ let low = 0;
77
+ let high = sorted.length;
78
+ while (low < high) {
79
+ const middle = (low + high) >> 1;
80
+ if ((sorted[middle] ?? 0) < value) low = middle + 1;
81
+ else high = middle;
82
+ }
83
+ return low;
84
+ }
85
+
86
+ /** The running median of the values over `reach` entries on each side (fewer at the ends). */
87
+ export function getRunningMedian(values: Float64Array, reach: number): Float64Array {
88
+ const medians = new Float64Array(values.length);
89
+ const window: number[] = [];
90
+ let from = 0;
91
+ let to = 0;
92
+ for (let index = 0; index < values.length; index++) {
93
+ const wantedTo = Math.min(values.length, index + reach + 1);
94
+ for (; to < wantedTo; to++) {
95
+ const value = values[to] ?? 0;
96
+ window.splice(findInsertIndex(window, value), 0, value);
97
+ }
98
+ const wantedFrom = Math.max(0, index - reach);
99
+ for (; from < wantedFrom; from++) window.splice(findInsertIndex(window, values[from] ?? 0), 1);
100
+ const middle = window.length >> 1;
101
+ medians[index] = window.length % 2 === 1 ? (window[middle] ?? 0) : ((window[middle - 1] ?? 0) + (window[middle] ?? 0)) / 2;
102
+ }
103
+ return medians;
104
+ }
package/src/index.ts CHANGED
@@ -252,6 +252,7 @@ export {
252
252
  DECLICK_CROSSFADE,
253
253
  DECLICK_LEVELS,
254
254
  DECLICK_MAX_IMPULSE,
255
+ DECLICK_REPAIR_RULES,
255
256
  DECLICK_SETTINGS,
256
257
  DECLICK_WORD_GUARD,
257
258
  buildDeclickDecodeArgs,
@@ -260,6 +261,7 @@ export {
260
261
  declickSpeech,
261
262
  describeDeclick,
262
263
  findImpulses,
264
+ findRepairs,
263
265
  getArPolynomial,
264
266
  getRepairZones,
265
267
  getRunningMedian,
@@ -269,8 +271,10 @@ export {
269
271
  type DeclickReport,
270
272
  type DeclickSettings,
271
273
  type DeclickSpeechOptions,
274
+ type DetectorDeclickLevel,
272
275
  type TimeSpan as DeclickSpan,
273
276
  } from "./footage/declick.js";
277
+ export { MOUTH_CLICK_RULES, findMouthClicks, type MouthClicks } from "./footage/mouth-clicks.js";
274
278
  export {
275
279
  MIN_CLIP_SECONDS,
276
280
  describeProbe,