@softure-ai/marketing-kit 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +18 -0
- package/dist/config/schema.d.ts +1 -0
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +1 -1
- package/dist/config/schema.js.map +1 -1
- package/dist/footage/declick.d.ts +35 -9
- package/dist/footage/declick.d.ts.map +1 -1
- package/dist/footage/declick.js +57 -88
- package/dist/footage/declick.js.map +1 -1
- package/dist/footage/film.d.ts +5 -2
- package/dist/footage/film.d.ts.map +1 -1
- package/dist/footage/film.js.map +1 -1
- package/dist/footage/mouth-clicks.d.ts +85 -0
- package/dist/footage/mouth-clicks.d.ts.map +1 -0
- package/dist/footage/mouth-clicks.js +207 -0
- package/dist/footage/mouth-clicks.js.map +1 -0
- package/dist/footage/sound-analysis.d.ts +25 -0
- package/dist/footage/sound-analysis.d.ts.map +1 -0
- package/dist/footage/sound-analysis.js +95 -0
- package/dist/footage/sound-analysis.js.map +1 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/schema/marketing.schema.json +3 -2
- package/src/config/schema.ts +1 -1
- package/src/footage/declick.ts +64 -97
- package/src/footage/film.ts +5 -2
- package/src/footage/mouth-clicks.ts +224 -0
- package/src/footage/sound-analysis.ts +104 -0
- package/src/index.ts +4 -0
package/src/footage/declick.ts
CHANGED
|
@@ -2,8 +2,12 @@ import { readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
|
2
2
|
|
|
3
3
|
import type { TimedWord } from "../voice/voiceover.js";
|
|
4
4
|
import { getAudioCodecArgs } from "./clip-sound.js";
|
|
5
|
+
import { MOUTH_CLICK_RULES, analyzeMouthClicks } from "./mouth-clicks.js";
|
|
5
6
|
import { FOOTAGE_SAMPLE_RATE } from "./normalize.js";
|
|
6
7
|
import { runFfmpeg } from "./process.js";
|
|
8
|
+
import { getBand, getFrameEnergies, getRunningMedian, type TimeSpan } from "./sound-analysis.js";
|
|
9
|
+
|
|
10
|
+
export { getRunningMedian, type TimeSpan };
|
|
7
11
|
|
|
8
12
|
/**
|
|
9
13
|
* The speech track's declick stage (issues #1084, #1170), between the cut speech track and its loudness normalization:
|
|
@@ -17,22 +21,30 @@ import { runFfmpeg } from "./process.js";
|
|
|
17
21
|
* samples are replaced by AR (LPC) interpolation from the sound on both sides, joined with a raised-cosine crossfade
|
|
18
22
|
* of `DECLICK_CROSSFADE` on each side. Every other sample leaves the stage as it came in.
|
|
19
23
|
*
|
|
24
|
+
* The `repair` level (issue #1183) uses the mouth-click detector shared with moving a cut (`MOUTH_CLICK_RULES`,
|
|
25
|
+
* issue #1182) instead: impulses of 1 to 25 ms, outside the speech protection zone (word bounds on the real sound
|
|
26
|
+
* ±80 ms, nothing above -45 dBFS), only in pauses at least 150 ms long after the guard, and within the limits of
|
|
27
|
+
* `DECLICK_REPAIR_RULES` (the impulses over a limit are left and counted).
|
|
28
|
+
*
|
|
20
29
|
* Without words the stage repairs nothing: it cannot tell a click from a consonant.
|
|
21
30
|
*/
|
|
22
31
|
|
|
23
|
-
export const DECLICK_LEVELS = ["off", "gentle", "strong"] as const;
|
|
32
|
+
export const DECLICK_LEVELS = ["off", "gentle", "strong", "repair"] as const;
|
|
24
33
|
|
|
25
34
|
export type DeclickLevel = (typeof DECLICK_LEVELS)[number];
|
|
26
35
|
|
|
27
36
|
/** A level that runs the stage. */
|
|
28
37
|
export type ActiveDeclickLevel = Exclude<DeclickLevel, "off">;
|
|
29
38
|
|
|
39
|
+
/** A level that runs the 0.18.0 detector (`findImpulses`, `DECLICK_SETTINGS`). */
|
|
40
|
+
export type DetectorDeclickLevel = Exclude<ActiveDeclickLevel, "repair">;
|
|
41
|
+
|
|
30
42
|
export interface DeclickSettings {
|
|
31
43
|
/** How far a frame's 2-8 kHz energy must rise above the band's running median to count as an impulse, in dB. */
|
|
32
44
|
jumpDb: number;
|
|
33
45
|
}
|
|
34
46
|
|
|
35
|
-
export const DECLICK_SETTINGS: Record<
|
|
47
|
+
export const DECLICK_SETTINGS: Record<DetectorDeclickLevel, DeclickSettings> = {
|
|
36
48
|
gentle: { jumpDb: 15 },
|
|
37
49
|
strong: { jumpDb: 9 },
|
|
38
50
|
};
|
|
@@ -43,6 +55,17 @@ export const DECLICK_WORD_GUARD = 0.08;
|
|
|
43
55
|
export const DECLICK_MAX_IMPULSE = 0.012;
|
|
44
56
|
/** The raised-cosine crossfade between the original and the interpolation on each side of a repair. */
|
|
45
57
|
export const DECLICK_CROSSFADE = 0.003;
|
|
58
|
+
/** The `repair` level's rules (issue #1183); its impulses are `MOUTH_CLICK_RULES`'. */
|
|
59
|
+
export const DECLICK_REPAIR_RULES = {
|
|
60
|
+
/** The shortest pause that holds a repair once the word guard is taken off both sides, seconds. */
|
|
61
|
+
minPauseSeconds: 0.15,
|
|
62
|
+
/** The longest repair, its crossfades included, seconds. */
|
|
63
|
+
maxRepairSeconds: 0.025,
|
|
64
|
+
/** The most repairs in any one second. */
|
|
65
|
+
maxPerSecond: 3,
|
|
66
|
+
/** The share of the track's length all repairs (crossfades included) may take. */
|
|
67
|
+
maxShare: 0.01,
|
|
68
|
+
} as const;
|
|
46
69
|
/** The AR model's order. */
|
|
47
70
|
export const DECLICK_AR_ORDER = 24;
|
|
48
71
|
/** Seconds of sound on each side of a repair the AR model is fitted on. */
|
|
@@ -61,11 +84,6 @@ const BAND_FLOOR_DB = -90;
|
|
|
61
84
|
/** Added to Burg's denominator, so a silent context gives a zero model instead of a division by zero. */
|
|
62
85
|
const AR_RIDGE = 1e-9;
|
|
63
86
|
|
|
64
|
-
export interface TimeSpan {
|
|
65
|
-
start: number;
|
|
66
|
-
end: number;
|
|
67
|
-
}
|
|
68
|
-
|
|
69
87
|
/** What the stage did, in the render's and the draft's ok value. */
|
|
70
88
|
export interface DeclickReport {
|
|
71
89
|
level: ActiveDeclickLevel;
|
|
@@ -73,6 +91,8 @@ export interface DeclickReport {
|
|
|
73
91
|
impulses: number;
|
|
74
92
|
/** Each repaired impulse in the speech track's time, seconds (without its crossfades). */
|
|
75
93
|
repairs: TimeSpan[];
|
|
94
|
+
/** Impulses the `repair` level left because a limit of `DECLICK_REPAIR_RULES` was reached; 0 for the other levels. */
|
|
95
|
+
skippedLimit: number;
|
|
76
96
|
}
|
|
77
97
|
|
|
78
98
|
const sortSpans = (words: readonly TimeSpan[]): TimeSpan[] => [...words].filter((word) => word.end > word.start).sort((a, b) => a.start - b.start);
|
|
@@ -99,91 +119,6 @@ export function getRepairZones(words: readonly TimeSpan[], seconds: number): Tim
|
|
|
99
119
|
return zones;
|
|
100
120
|
}
|
|
101
121
|
|
|
102
|
-
/** One second-order section (RBJ cookbook), run in place over the samples. */
|
|
103
|
-
function runBiquad(samples: Float64Array, kind: "highpass" | "lowpass", frequency: number, sampleRate: number): void {
|
|
104
|
-
const w0 = (2 * Math.PI * Math.min(frequency, sampleRate * 0.45)) / sampleRate;
|
|
105
|
-
const cos = Math.cos(w0);
|
|
106
|
-
const alpha = Math.sin(w0) / (2 * Math.SQRT1_2);
|
|
107
|
-
const a0 = 1 + alpha;
|
|
108
|
-
const b0 = (kind === "lowpass" ? (1 - cos) / 2 : (1 + cos) / 2) / a0;
|
|
109
|
-
const b1 = (kind === "lowpass" ? 1 - cos : -(1 + cos)) / a0;
|
|
110
|
-
const b2 = b0;
|
|
111
|
-
const a1 = (-2 * cos) / a0;
|
|
112
|
-
const a2 = (1 - alpha) / a0;
|
|
113
|
-
let x1 = 0;
|
|
114
|
-
let x2 = 0;
|
|
115
|
-
let y1 = 0;
|
|
116
|
-
let y2 = 0;
|
|
117
|
-
for (let index = 0; index < samples.length; index++) {
|
|
118
|
-
const x = samples[index] ?? 0;
|
|
119
|
-
const y = b0 * x + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2;
|
|
120
|
-
x2 = x1;
|
|
121
|
-
x1 = x;
|
|
122
|
-
y2 = y1;
|
|
123
|
-
y1 = y;
|
|
124
|
-
samples[index] = y;
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
/** The 2-8 kHz band of the signal: two high-pass and two low-pass sections (fourth order on each side). */
|
|
129
|
-
function getBand(mono: Float32Array, sampleRate: number): Float64Array {
|
|
130
|
-
const band = Float64Array.from(mono);
|
|
131
|
-
runBiquad(band, "highpass", BAND_LOW_HZ, sampleRate);
|
|
132
|
-
runBiquad(band, "highpass", BAND_LOW_HZ, sampleRate);
|
|
133
|
-
runBiquad(band, "lowpass", BAND_HIGH_HZ, sampleRate);
|
|
134
|
-
runBiquad(band, "lowpass", BAND_HIGH_HZ, sampleRate);
|
|
135
|
-
return band;
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
/** Each frame's mean square in dB, frames of `frameLength` samples every `hop` samples. */
|
|
139
|
-
function getFrameEnergies(band: Float64Array, frameLength: number, hop: number): Float64Array {
|
|
140
|
-
const sums = new Float64Array(band.length + 1);
|
|
141
|
-
for (let index = 0; index < band.length; index++) {
|
|
142
|
-
const value = band[index] ?? 0;
|
|
143
|
-
sums[index + 1] = (sums[index] ?? 0) + value * value;
|
|
144
|
-
}
|
|
145
|
-
const count = band.length < frameLength ? 0 : Math.floor((band.length - frameLength) / hop) + 1;
|
|
146
|
-
const energies = new Float64Array(count);
|
|
147
|
-
for (let frame = 0; frame < count; frame++) {
|
|
148
|
-
const from = frame * hop;
|
|
149
|
-
const power = ((sums[from + frameLength] ?? 0) - (sums[from] ?? 0)) / frameLength;
|
|
150
|
-
energies[frame] = 10 * Math.log10(Math.max(power, 1e-20));
|
|
151
|
-
}
|
|
152
|
-
return energies;
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
/** The index of the first entry of the sorted list not below `value`. */
|
|
156
|
-
function findInsertIndex(sorted: number[], value: number): number {
|
|
157
|
-
let low = 0;
|
|
158
|
-
let high = sorted.length;
|
|
159
|
-
while (low < high) {
|
|
160
|
-
const middle = (low + high) >> 1;
|
|
161
|
-
if ((sorted[middle] ?? 0) < value) low = middle + 1;
|
|
162
|
-
else high = middle;
|
|
163
|
-
}
|
|
164
|
-
return low;
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
/** The running median of the values over `reach` entries on each side (fewer at the ends). */
|
|
168
|
-
export function getRunningMedian(values: Float64Array, reach: number): Float64Array {
|
|
169
|
-
const medians = new Float64Array(values.length);
|
|
170
|
-
const window: number[] = [];
|
|
171
|
-
let from = 0;
|
|
172
|
-
let to = 0;
|
|
173
|
-
for (let index = 0; index < values.length; index++) {
|
|
174
|
-
const wantedTo = Math.min(values.length, index + reach + 1);
|
|
175
|
-
for (; to < wantedTo; to++) {
|
|
176
|
-
const value = values[to] ?? 0;
|
|
177
|
-
window.splice(findInsertIndex(window, value), 0, value);
|
|
178
|
-
}
|
|
179
|
-
const wantedFrom = Math.max(0, index - reach);
|
|
180
|
-
for (; from < wantedFrom; from++) window.splice(findInsertIndex(window, values[from] ?? 0), 1);
|
|
181
|
-
const middle = window.length >> 1;
|
|
182
|
-
medians[index] = window.length % 2 === 1 ? (window[middle] ?? 0) : ((window[middle - 1] ?? 0) + (window[middle] ?? 0)) / 2;
|
|
183
|
-
}
|
|
184
|
-
return medians;
|
|
185
|
-
}
|
|
186
|
-
|
|
187
122
|
/** True when the impulse with its crossfades lies inside one zone. */
|
|
188
123
|
const isRepairable = (span: TimeSpan, zones: readonly TimeSpan[]): boolean =>
|
|
189
124
|
zones.some((zone) => span.start - DECLICK_CROSSFADE >= zone.start && span.end + DECLICK_CROSSFADE <= zone.end);
|
|
@@ -193,14 +128,14 @@ const isRepairable = (span: TimeSpan, zones: readonly TimeSpan[]): boolean =>
|
|
|
193
128
|
* the level's `jumpDb` above the band's running median (±100 ms), shorter than `DECLICK_MAX_IMPULSE`, that lie with
|
|
194
129
|
* their crossfades inside a repair zone (`getRepairZones`). Times in seconds, widened by a millisecond on each side.
|
|
195
130
|
*/
|
|
196
|
-
export function findImpulses(mono: Float32Array, sampleRate: number, words: readonly TimeSpan[], level:
|
|
131
|
+
export function findImpulses(mono: Float32Array, sampleRate: number, words: readonly TimeSpan[], level: DetectorDeclickLevel): TimeSpan[] {
|
|
197
132
|
const seconds = mono.length / sampleRate;
|
|
198
133
|
const zones = getRepairZones(words, seconds);
|
|
199
134
|
if (zones.length === 0) return [];
|
|
200
135
|
const { jumpDb } = DECLICK_SETTINGS[level];
|
|
201
136
|
const frameLength = Math.max(1, Math.round(FRAME_SECONDS * sampleRate));
|
|
202
137
|
const hop = Math.max(1, Math.round(HOP_SECONDS * sampleRate));
|
|
203
|
-
const energies = getFrameEnergies(getBand(mono, sampleRate), frameLength, hop);
|
|
138
|
+
const energies = getFrameEnergies(getBand({ mono, sampleRate, lowHz: BAND_LOW_HZ, highHz: BAND_HIGH_HZ }), frameLength, hop);
|
|
204
139
|
const medians = getRunningMedian(energies, Math.round(MEDIAN_SECONDS / HOP_SECONDS));
|
|
205
140
|
const impulses: TimeSpan[] = [];
|
|
206
141
|
let runStart = -1;
|
|
@@ -222,6 +157,35 @@ export function findImpulses(mono: Float32Array, sampleRate: number, words: read
|
|
|
222
157
|
return impulses;
|
|
223
158
|
}
|
|
224
159
|
|
|
160
|
+
/**
|
|
161
|
+
* The impulses the `repair` level repairs (issue #1183): `MOUTH_CLICK_RULES` impulses whose repair (with its
|
|
162
|
+
* crossfades) lies inside a pause at least `minPauseSeconds` long between the words' bounds on the real sound after
|
|
163
|
+
* the guard, outside the speech protection zone, taken in time order while the limits allow; the others in such a
|
|
164
|
+
* pause are counted as `skippedLimit`.
|
|
165
|
+
*/
|
|
166
|
+
export function findRepairs(mono: Float32Array, sampleRate: number, words: readonly TimeSpan[]): { repairs: TimeSpan[]; skippedLimit: number } {
|
|
167
|
+
const seconds = mono.length / sampleRate;
|
|
168
|
+
const analysis = analyzeMouthClicks(mono, sampleRate, words);
|
|
169
|
+
const zones = getRepairZones(analysis.bounds, seconds).filter((zone) => zone.end - zone.start >= DECLICK_REPAIR_RULES.minPauseSeconds);
|
|
170
|
+
const budget = DECLICK_REPAIR_RULES.maxShare * seconds;
|
|
171
|
+
const repairs: TimeSpan[] = [];
|
|
172
|
+
let used = 0;
|
|
173
|
+
let skippedLimit = 0;
|
|
174
|
+
for (const candidate of analysis.candidates) {
|
|
175
|
+
const span = { start: candidate.start - DECLICK_CROSSFADE, end: candidate.end + DECLICK_CROSSFADE };
|
|
176
|
+
if (!zones.some((zone) => span.start >= zone.start && span.end <= zone.end) || analysis.isProtected(span)) continue;
|
|
177
|
+
const length = span.end - span.start;
|
|
178
|
+
const inLastSecond = repairs.filter((repair) => repair.start > candidate.start - 1).length;
|
|
179
|
+
if (length > DECLICK_REPAIR_RULES.maxRepairSeconds || inLastSecond >= DECLICK_REPAIR_RULES.maxPerSecond || used + length > budget) {
|
|
180
|
+
skippedLimit++;
|
|
181
|
+
continue;
|
|
182
|
+
}
|
|
183
|
+
repairs.push(candidate);
|
|
184
|
+
used += length;
|
|
185
|
+
}
|
|
186
|
+
return { repairs, skippedLimit };
|
|
187
|
+
}
|
|
188
|
+
|
|
225
189
|
/**
|
|
226
190
|
* The AR polynomial of the context segments (Burg's method over all of them at once): `a[0] = 1` and
|
|
227
191
|
* `Σ a[k]·x[n-k] ≈ 0`. Fewer coefficients when the segments are short; a silent context gives all zeros after `a[0]`.
|
|
@@ -340,7 +304,8 @@ export function declickSamples(options: { samples: Float32Array; channels: numbe
|
|
|
340
304
|
for (let channel = 0; channel < channels; channel++) sum += samples[index * channels + channel] ?? 0;
|
|
341
305
|
mono[index] = sum / channels;
|
|
342
306
|
}
|
|
343
|
-
const
|
|
307
|
+
const found = level === "repair" ? findRepairs(mono, sampleRate, words) : { repairs: findImpulses(mono, sampleRate, words, level), skippedLimit: 0 };
|
|
308
|
+
const impulses = found.repairs;
|
|
344
309
|
const output = Float32Array.from(samples);
|
|
345
310
|
const crossfade = Math.round(DECLICK_CROSSFADE * sampleRate);
|
|
346
311
|
const context = Math.round(DECLICK_AR_CONTEXT * sampleRate);
|
|
@@ -354,7 +319,7 @@ export function declickSamples(options: { samples: Float32Array; channels: numbe
|
|
|
354
319
|
for (let index = Math.max(0, from); index < Math.min(length, to); index++) output[index * channels + channel] = track[index] ?? 0;
|
|
355
320
|
}
|
|
356
321
|
}
|
|
357
|
-
return { samples: output, impulses, report: { level, impulses: impulses.length, repairs: impulses.map((impulse) => ({ ...impulse })) } };
|
|
322
|
+
return { samples: output, impulses, report: { level, impulses: impulses.length, repairs: impulses.map((impulse) => ({ ...impulse })), skippedLimit: found.skippedLimit } };
|
|
358
323
|
}
|
|
359
324
|
|
|
360
325
|
const CHANNELS = 2;
|
|
@@ -423,5 +388,7 @@ export function describeDeclick(report: DeclickReport): string {
|
|
|
423
388
|
const times = report.repairs.slice(0, LOGGED_REPAIRS).map((repair) => `${repair.start.toFixed(3)} s`);
|
|
424
389
|
const more = report.repairs.length > LOGGED_REPAIRS ? `, +${String(report.repairs.length - LOGGED_REPAIRS)} more` : "";
|
|
425
390
|
const where = times.length === 0 ? "" : ` at ${times.join(", ")}${more}`;
|
|
426
|
-
|
|
391
|
+
const count = `${String(report.impulses)} impulse${report.impulses === 1 ? "" : "s"} repaired${where}`;
|
|
392
|
+
if (report.level !== "repair") return `declick (${report.level}): ${count}; no gain change, nothing within ${String(DECLICK_WORD_GUARD * 1000)} ms of a word touched.`;
|
|
393
|
+
return `declick (repair): ${count}, ${String(report.skippedLimit)} left over the limits; no gain change, nothing within ${String(DECLICK_WORD_GUARD * 1000)} ms of a word's sound or above ${String(MOUTH_CLICK_RULES.loudDbfs)} dBFS touched.`;
|
|
427
394
|
}
|
package/src/footage/film.ts
CHANGED
|
@@ -216,8 +216,11 @@ export function getEndCardTexts(card: FootageEndCard | null): { headline: string
|
|
|
216
216
|
/** Settings of the speech track itself. */
|
|
217
217
|
export interface FootageSpeech {
|
|
218
218
|
/**
|
|
219
|
-
* Mouth clicks and lip smacks between words (
|
|
220
|
-
*
|
|
219
|
+
* Mouth clicks and lip smacks between words (issues #1084, #1170, #1183): each single impulse in a pause is repaired
|
|
220
|
+
* by AR interpolation and no gain changes. `gentle` and `strong` repair impulses that jump 15 or 9 dB above the band's
|
|
221
|
+
* median, nothing within 80 ms of a transcript word; `repair` uses the shared mouth-click rules (`MOUTH_CLICK_RULES`):
|
|
222
|
+
* word bounds on the real sound, nothing above -45 dBFS, pauses of 150 ms or more, at most 3 repairs a second and
|
|
223
|
+
* 1 % of the track. Absent or `off`: no pass, the track unchanged.
|
|
221
224
|
*/
|
|
222
225
|
declick?: DeclickLevel;
|
|
223
226
|
}
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
import { getBand, getFrameEnergies, getRunningMedian, getSquareSums, toPowerDb, type TimeSpan } from "./sound-analysis.js";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The mouth-click detector shared by moving a segment's cut past a click (stage A, issue #1182) and the declick
|
|
5
|
+
* stage's `repair` level (stage B, issue #1183), so a click that moves a cut and a click that is repaired are the same
|
|
6
|
+
* thing. Its input is the speech as the chain hears it: the clips' sound after the -23 LUFS pre-gain.
|
|
7
|
+
*
|
|
8
|
+
* - **An impulse** lasts 1 to 25 ms, its 2-10 kHz energy rises more than 12 dB above the band's ±50 ms running median,
|
|
9
|
+
* it is not periodic (the band's normalized autocorrelation stays under 0.3 at the pitch lags, 2.5 to 15 ms, over a
|
|
10
|
+
* 30 ms window centred on it) and it is over within its length (the 10 ms after it at least 12 dB under its peak, so
|
|
11
|
+
* the onset of a breath or a fricative is not one).
|
|
12
|
+
* - **The speech protection zone:** each word's bound on the real sound (the earlier start and the later end of the
|
|
13
|
+
* transcript's bound and the point where the 10 ms energy falls 20 dB under the word's peak), widened by 80 ms; and
|
|
14
|
+
* every stretch whose 10 ms energy is above -45 dBFS, measured with the impulses' own samples left out (a click's
|
|
15
|
+
* own energy must not protect it).
|
|
16
|
+
*/
|
|
17
|
+
export const MOUTH_CLICK_RULES = {
|
|
18
|
+
/** The shortest impulse, seconds. */
|
|
19
|
+
minSeconds: 0.001,
|
|
20
|
+
/** The longest impulse, seconds. */
|
|
21
|
+
maxSeconds: 0.025,
|
|
22
|
+
/** The band the impulse is measured in. */
|
|
23
|
+
bandLowHz: 2000,
|
|
24
|
+
bandHighHz: 10_000,
|
|
25
|
+
/** How far an impulse's band energy rises above the band's running median, dB. */
|
|
26
|
+
jumpDb: 12,
|
|
27
|
+
/** The running median's reach on each side, seconds. */
|
|
28
|
+
medianSeconds: 0.05,
|
|
29
|
+
/** The highest normalized autocorrelation at the pitch lags an impulse may show. */
|
|
30
|
+
maxCorrelation: 0.3,
|
|
31
|
+
/** The pitch lags, seconds. */
|
|
32
|
+
minLagSeconds: 0.0025,
|
|
33
|
+
maxLagSeconds: 0.015,
|
|
34
|
+
/** The window the autocorrelation is measured over, centred on the impulse, seconds. */
|
|
35
|
+
correlationSeconds: 0.03,
|
|
36
|
+
/** The stretch after an impulse that must be quieter than its peak, seconds, and by how much, dB. */
|
|
37
|
+
afterSeconds: 0.01,
|
|
38
|
+
afterDropDb: 12,
|
|
39
|
+
/** The energy window of the word bounds and the loudness rule, seconds. */
|
|
40
|
+
energySeconds: 0.01,
|
|
41
|
+
/** A word's sound lasts while its 10 ms energy stays within this many dB of the word's peak. */
|
|
42
|
+
wordDropDb: 20,
|
|
43
|
+
/** The guard on each side of a word's bound, seconds. */
|
|
44
|
+
wordGuardSeconds: 0.08,
|
|
45
|
+
/** Nothing whose 10 ms energy is above this is a click to act on, dBFS (mean square). */
|
|
46
|
+
loudDbfs: -45,
|
|
47
|
+
} as const;
|
|
48
|
+
|
|
49
|
+
const FRAME_SECONDS = 0.002;
|
|
50
|
+
const HOP_SECONDS = 0.001;
|
|
51
|
+
/** The step the word bounds and the loudness rule are measured at. */
|
|
52
|
+
const STEP_SECONDS = 0.001;
|
|
53
|
+
/** An impulse reaches this far past its detected frames, for the band filter's delay and the frames' edges. */
|
|
54
|
+
const IMPULSE_MARGIN = 0.001;
|
|
55
|
+
/** A frame quieter than this (mean square, dB) is never an impulse: digital silence with a stray bit is not a click. */
|
|
56
|
+
const BAND_FLOOR_DB = -90;
|
|
57
|
+
|
|
58
|
+
/** The highest normalized autocorrelation of `band` at the pitch lags over the window centred on `center`. */
|
|
59
|
+
function getPeakCorrelation(band: Float64Array, center: number, sampleRate: number): number {
|
|
60
|
+
const half = Math.round((MOUTH_CLICK_RULES.correlationSeconds * sampleRate) / 2);
|
|
61
|
+
const from = Math.max(0, center - half);
|
|
62
|
+
const to = Math.min(band.length, center + half);
|
|
63
|
+
const minLag = Math.max(1, Math.round(MOUTH_CLICK_RULES.minLagSeconds * sampleRate));
|
|
64
|
+
const maxLag = Math.min(Math.round(MOUTH_CLICK_RULES.maxLagSeconds * sampleRate), to - from - 1);
|
|
65
|
+
let peak = 0;
|
|
66
|
+
for (let lag = minLag; lag <= maxLag; lag++) {
|
|
67
|
+
let product = 0;
|
|
68
|
+
let head = 0;
|
|
69
|
+
let tail = 0;
|
|
70
|
+
for (let index = from; index + lag < to; index++) {
|
|
71
|
+
const a = band[index] ?? 0;
|
|
72
|
+
const b = band[index + lag] ?? 0;
|
|
73
|
+
product += a * b;
|
|
74
|
+
head += a * a;
|
|
75
|
+
tail += b * b;
|
|
76
|
+
}
|
|
77
|
+
if (head > 0 && tail > 0) peak = Math.max(peak, product / Math.sqrt(head * tail));
|
|
78
|
+
}
|
|
79
|
+
return peak;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* The impulses of a mono signal by `MOUTH_CLICK_RULES` (length, band jump, periodicity, end), anywhere in it. Times in
|
|
84
|
+
* seconds from sample 0, widened by a millisecond on each side.
|
|
85
|
+
*/
|
|
86
|
+
export function findClickCandidates(mono: Float32Array, sampleRate: number): TimeSpan[] {
|
|
87
|
+
const seconds = mono.length / sampleRate;
|
|
88
|
+
const band = getBand({ mono, sampleRate, lowHz: MOUTH_CLICK_RULES.bandLowHz, highHz: MOUTH_CLICK_RULES.bandHighHz });
|
|
89
|
+
const sums = getSquareSums(band);
|
|
90
|
+
const frameLength = Math.max(1, Math.round(FRAME_SECONDS * sampleRate));
|
|
91
|
+
const hop = Math.max(1, Math.round(HOP_SECONDS * sampleRate));
|
|
92
|
+
const after = Math.round(MOUTH_CLICK_RULES.afterSeconds * sampleRate);
|
|
93
|
+
const energies = getFrameEnergies(band, frameLength, hop);
|
|
94
|
+
const medians = getRunningMedian(energies, Math.round(MOUTH_CLICK_RULES.medianSeconds / HOP_SECONDS));
|
|
95
|
+
const candidates: TimeSpan[] = [];
|
|
96
|
+
const checkRun = (first: number, last: number): void => {
|
|
97
|
+
const start = (first * hop) / sampleRate;
|
|
98
|
+
const end = (last * hop + frameLength) / sampleRate;
|
|
99
|
+
if (end - start < MOUTH_CLICK_RULES.minSeconds || end - start > MOUTH_CLICK_RULES.maxSeconds) return;
|
|
100
|
+
const span = { start: Math.max(0, start - IMPULSE_MARGIN), end: Math.min(seconds, end + IMPULSE_MARGIN) };
|
|
101
|
+
let peakDb = -Infinity;
|
|
102
|
+
for (let frame = first; frame <= last; frame++) peakDb = Math.max(peakDb, energies[frame] ?? -Infinity);
|
|
103
|
+
const afterFrom = Math.round(span.end * sampleRate);
|
|
104
|
+
const afterTo = Math.min(band.length, afterFrom + after);
|
|
105
|
+
if (afterTo > afterFrom && toPowerDb(((sums[afterTo] ?? 0) - (sums[afterFrom] ?? 0)) / (afterTo - afterFrom)) > peakDb - MOUTH_CLICK_RULES.afterDropDb) return;
|
|
106
|
+
const center = Math.round(((span.start + span.end) / 2) * sampleRate);
|
|
107
|
+
if (getPeakCorrelation(band, center, sampleRate) >= MOUTH_CLICK_RULES.maxCorrelation) return;
|
|
108
|
+
candidates.push(span);
|
|
109
|
+
};
|
|
110
|
+
let runStart = -1;
|
|
111
|
+
for (let frame = 0; frame < energies.length; frame++) {
|
|
112
|
+
const energy = energies[frame] ?? 0;
|
|
113
|
+
const isJump = energy > BAND_FLOOR_DB && energy - (medians[frame] ?? 0) > MOUTH_CLICK_RULES.jumpDb;
|
|
114
|
+
if (isJump && runStart === -1) runStart = frame;
|
|
115
|
+
else if (!isJump && runStart !== -1) {
|
|
116
|
+
checkRun(runStart, frame - 1);
|
|
117
|
+
runStart = -1;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
if (runStart !== -1) checkRun(runStart, energies.length - 1);
|
|
121
|
+
return candidates;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** The analysis of one mono stretch in its own time (sample 0 at 0 s). */
|
|
125
|
+
export interface MouthClickAnalysis {
|
|
126
|
+
/** Every impulse by the rule, protected or not. */
|
|
127
|
+
candidates: TimeSpan[];
|
|
128
|
+
/** Each word's bound on the real sound (before the guard). */
|
|
129
|
+
bounds: TimeSpan[];
|
|
130
|
+
/** True when the span touches a word's bound with its guard, or a 10 ms energy above the loudness rule. */
|
|
131
|
+
isProtected: (span: TimeSpan) => boolean;
|
|
132
|
+
/** True when the stretch holds no 10 ms window above the loudness rule, the impulses' samples left out. */
|
|
133
|
+
isQuiet: (stretch: TimeSpan) => boolean;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** The detector over a mono stretch whose words are in the stretch's own time. */
|
|
137
|
+
export function analyzeMouthClicks(mono: Float32Array, sampleRate: number, words: readonly TimeSpan[]): MouthClickAnalysis {
|
|
138
|
+
const length = mono.length;
|
|
139
|
+
const candidates = findClickCandidates(mono, sampleRate);
|
|
140
|
+
const sums = getSquareSums(mono);
|
|
141
|
+
const masked = Float64Array.from(mono);
|
|
142
|
+
const counts = new Float64Array(length + 1);
|
|
143
|
+
const isMasked = new Uint8Array(length);
|
|
144
|
+
for (const candidate of candidates) {
|
|
145
|
+
for (let index = Math.max(0, Math.floor(candidate.start * sampleRate)); index < Math.min(length, Math.ceil(candidate.end * sampleRate)); index++) {
|
|
146
|
+
isMasked[index] = 1;
|
|
147
|
+
masked[index] = 0;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
for (let index = 0; index < length; index++) counts[index + 1] = (counts[index] ?? 0) + (isMasked[index] === 1 ? 0 : 1);
|
|
151
|
+
const maskedSums = getSquareSums(masked);
|
|
152
|
+
const window = Math.max(1, Math.round(MOUTH_CLICK_RULES.energySeconds * sampleRate));
|
|
153
|
+
const step = Math.max(1, Math.round(STEP_SECONDS * sampleRate));
|
|
154
|
+
const clamp = (index: number): number => Math.min(length, Math.max(0, index));
|
|
155
|
+
const getDb = (from: number, to: number): number => {
|
|
156
|
+
const [a, b] = [clamp(from), clamp(to)];
|
|
157
|
+
return b > a ? toPowerDb(((sums[b] ?? 0) - (sums[a] ?? 0)) / (b - a)) : -Infinity;
|
|
158
|
+
};
|
|
159
|
+
const getMaskedDb = (from: number, to: number): number => {
|
|
160
|
+
const [a, b] = [clamp(from), clamp(to)];
|
|
161
|
+
const count = (counts[b] ?? 0) - (counts[a] ?? 0);
|
|
162
|
+
return count > 0 ? toPowerDb(((maskedSums[b] ?? 0) - (maskedSums[a] ?? 0)) / count) : -Infinity;
|
|
163
|
+
};
|
|
164
|
+
const getBound = (word: TimeSpan): TimeSpan => {
|
|
165
|
+
const from = clamp(Math.round(word.start * sampleRate));
|
|
166
|
+
const to = clamp(Math.round(word.end * sampleRate));
|
|
167
|
+
if (to <= from) return { ...word };
|
|
168
|
+
let peakDb = -Infinity;
|
|
169
|
+
for (let index = from; index <= Math.max(from, to - window); index += step) peakDb = Math.max(peakDb, getDb(index, Math.min(to, index + window)));
|
|
170
|
+
const threshold = peakDb - MOUTH_CLICK_RULES.wordDropDb;
|
|
171
|
+
// The start walks back while the 10 ms before it hold the word's sound, the end forward while the 10 ms after it do.
|
|
172
|
+
let start = from;
|
|
173
|
+
while (start > 0 && getDb(start - window, start) >= threshold) start = clamp(start - step);
|
|
174
|
+
let end = to;
|
|
175
|
+
while (end < length && getDb(end, end + window) >= threshold) end = clamp(end + step);
|
|
176
|
+
return { start: Math.min(word.start, start / sampleRate), end: Math.max(word.end, end / sampleRate) };
|
|
177
|
+
};
|
|
178
|
+
const bounds = [...words].filter((word) => word.end > word.start).sort((a, b) => a.start - b.start).map(getBound);
|
|
179
|
+
const isLoud = (span: TimeSpan): boolean => {
|
|
180
|
+
const from = Math.floor(span.start * sampleRate);
|
|
181
|
+
const to = Math.ceil(span.end * sampleRate);
|
|
182
|
+
for (let center = from; ; center = Math.min(to, center + step)) {
|
|
183
|
+
if (getMaskedDb(center - (window >> 1), center + window - (window >> 1)) > MOUTH_CLICK_RULES.loudDbfs) return true;
|
|
184
|
+
if (center >= to) return false;
|
|
185
|
+
}
|
|
186
|
+
};
|
|
187
|
+
const guard = MOUTH_CLICK_RULES.wordGuardSeconds;
|
|
188
|
+
return {
|
|
189
|
+
candidates,
|
|
190
|
+
bounds,
|
|
191
|
+
isProtected: (span) => bounds.some((bound) => span.start < bound.end + guard && span.end > bound.start - guard) || isLoud(span),
|
|
192
|
+
isQuiet: (stretch) => stretch.end <= 0 || stretch.start >= length / sampleRate || !isLoud({ start: Math.max(0, stretch.start), end: Math.min(length / sampleRate, stretch.end) }),
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/** What `findMouthClicks` found, in the film's time (the stretch's `offset` added). */
|
|
197
|
+
export interface MouthClicks {
|
|
198
|
+
/** The impulses outside the speech protection zone, in time order. */
|
|
199
|
+
clicks: TimeSpan[];
|
|
200
|
+
/** Each word's bound on the real sound (before the ±80 ms guard), in time order. */
|
|
201
|
+
words: TimeSpan[];
|
|
202
|
+
/** True when the stretch holds no 10 ms window above -45 dBFS, the impulses' samples left out. */
|
|
203
|
+
isQuiet: (stretch: TimeSpan) => boolean;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* Mouth clicks of a mono stretch whose sample 0 plays at `offset` seconds, outside the speech protection zone
|
|
208
|
+
* (`MOUTH_CLICK_RULES`); `words` are in the same time as `offset`. No audio is changed: an app can move a segment's
|
|
209
|
+
* cut past a click before its first word or after its last (issue #1182).
|
|
210
|
+
*/
|
|
211
|
+
export function findMouthClicks(options: { mono: Float32Array; sampleRate: number; offset: number; words: readonly TimeSpan[] }): MouthClicks {
|
|
212
|
+
const { mono, sampleRate, offset } = options;
|
|
213
|
+
const shift = (span: TimeSpan, by: number): TimeSpan => ({ start: span.start + by, end: span.end + by });
|
|
214
|
+
const analysis = analyzeMouthClicks(
|
|
215
|
+
mono,
|
|
216
|
+
sampleRate,
|
|
217
|
+
options.words.map((word) => shift(word, -offset)),
|
|
218
|
+
);
|
|
219
|
+
return {
|
|
220
|
+
clicks: analysis.candidates.filter((candidate) => !analysis.isProtected(candidate)).map((candidate) => shift(candidate, offset)),
|
|
221
|
+
words: analysis.bounds.map((bound) => shift(bound, offset)),
|
|
222
|
+
isQuiet: (stretch) => analysis.isQuiet(shift(stretch, -offset)),
|
|
223
|
+
};
|
|
224
|
+
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sound analysis shared by the speech declick stage and the mouth-click detector (issues #1170, #1182, #1183): a
|
|
3
|
+
* fourth-order band pass, frame energies and a running median.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
/** A stretch of time in seconds. */
|
|
7
|
+
export interface TimeSpan {
|
|
8
|
+
start: number;
|
|
9
|
+
end: number;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/** One second-order section (RBJ cookbook), run in place over the samples. */
|
|
13
|
+
function runBiquad(samples: Float64Array, kind: "highpass" | "lowpass", frequency: number, sampleRate: number): void {
|
|
14
|
+
const w0 = (2 * Math.PI * Math.min(frequency, sampleRate * 0.45)) / sampleRate;
|
|
15
|
+
const cos = Math.cos(w0);
|
|
16
|
+
const alpha = Math.sin(w0) / (2 * Math.SQRT1_2);
|
|
17
|
+
const a0 = 1 + alpha;
|
|
18
|
+
const b0 = (kind === "lowpass" ? (1 - cos) / 2 : (1 + cos) / 2) / a0;
|
|
19
|
+
const b1 = (kind === "lowpass" ? 1 - cos : -(1 + cos)) / a0;
|
|
20
|
+
const b2 = b0;
|
|
21
|
+
const a1 = (-2 * cos) / a0;
|
|
22
|
+
const a2 = (1 - alpha) / a0;
|
|
23
|
+
let x1 = 0;
|
|
24
|
+
let x2 = 0;
|
|
25
|
+
let y1 = 0;
|
|
26
|
+
let y2 = 0;
|
|
27
|
+
for (let index = 0; index < samples.length; index++) {
|
|
28
|
+
const x = samples[index] ?? 0;
|
|
29
|
+
const y = b0 * x + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2;
|
|
30
|
+
x2 = x1;
|
|
31
|
+
x1 = x;
|
|
32
|
+
y2 = y1;
|
|
33
|
+
y1 = y;
|
|
34
|
+
samples[index] = y;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** The band of the signal between `lowHz` and `highHz`: two high-pass and two low-pass sections (fourth order on each side). */
|
|
39
|
+
export function getBand(options: { mono: Float32Array; sampleRate: number; lowHz: number; highHz: number }): Float64Array {
|
|
40
|
+
const { mono, sampleRate, lowHz, highHz } = options;
|
|
41
|
+
const band = Float64Array.from(mono);
|
|
42
|
+
runBiquad(band, "highpass", lowHz, sampleRate);
|
|
43
|
+
runBiquad(band, "highpass", lowHz, sampleRate);
|
|
44
|
+
runBiquad(band, "lowpass", highHz, sampleRate);
|
|
45
|
+
runBiquad(band, "lowpass", highHz, sampleRate);
|
|
46
|
+
return band;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** The running sums of the squared samples: `sums[i]` is the energy of samples 0 to i - 1. */
|
|
50
|
+
export function getSquareSums(samples: ArrayLike<number>): Float64Array {
|
|
51
|
+
const sums = new Float64Array(samples.length + 1);
|
|
52
|
+
for (let index = 0; index < samples.length; index++) {
|
|
53
|
+
const value = samples[index] ?? 0;
|
|
54
|
+
sums[index + 1] = (sums[index] ?? 0) + value * value;
|
|
55
|
+
}
|
|
56
|
+
return sums;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** A mean square in dB (10·log10), with a floor so silence stays finite. */
|
|
60
|
+
export const toPowerDb = (meanSquare: number): number => 10 * Math.log10(Math.max(meanSquare, 1e-20));
|
|
61
|
+
|
|
62
|
+
/** Each frame's mean square in dB, frames of `frameLength` samples every `hop` samples. */
|
|
63
|
+
export function getFrameEnergies(band: Float64Array, frameLength: number, hop: number): Float64Array {
|
|
64
|
+
const sums = getSquareSums(band);
|
|
65
|
+
const count = band.length < frameLength ? 0 : Math.floor((band.length - frameLength) / hop) + 1;
|
|
66
|
+
const energies = new Float64Array(count);
|
|
67
|
+
for (let frame = 0; frame < count; frame++) {
|
|
68
|
+
const from = frame * hop;
|
|
69
|
+
energies[frame] = toPowerDb(((sums[from + frameLength] ?? 0) - (sums[from] ?? 0)) / frameLength);
|
|
70
|
+
}
|
|
71
|
+
return energies;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** The index of the first entry of the sorted list not below `value`. */
|
|
75
|
+
function findInsertIndex(sorted: number[], value: number): number {
|
|
76
|
+
let low = 0;
|
|
77
|
+
let high = sorted.length;
|
|
78
|
+
while (low < high) {
|
|
79
|
+
const middle = (low + high) >> 1;
|
|
80
|
+
if ((sorted[middle] ?? 0) < value) low = middle + 1;
|
|
81
|
+
else high = middle;
|
|
82
|
+
}
|
|
83
|
+
return low;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** The running median of the values over `reach` entries on each side (fewer at the ends). */
|
|
87
|
+
export function getRunningMedian(values: Float64Array, reach: number): Float64Array {
|
|
88
|
+
const medians = new Float64Array(values.length);
|
|
89
|
+
const window: number[] = [];
|
|
90
|
+
let from = 0;
|
|
91
|
+
let to = 0;
|
|
92
|
+
for (let index = 0; index < values.length; index++) {
|
|
93
|
+
const wantedTo = Math.min(values.length, index + reach + 1);
|
|
94
|
+
for (; to < wantedTo; to++) {
|
|
95
|
+
const value = values[to] ?? 0;
|
|
96
|
+
window.splice(findInsertIndex(window, value), 0, value);
|
|
97
|
+
}
|
|
98
|
+
const wantedFrom = Math.max(0, index - reach);
|
|
99
|
+
for (; from < wantedFrom; from++) window.splice(findInsertIndex(window, values[from] ?? 0), 1);
|
|
100
|
+
const middle = window.length >> 1;
|
|
101
|
+
medians[index] = window.length % 2 === 1 ? (window[middle] ?? 0) : ((window[middle - 1] ?? 0) + (window[middle] ?? 0)) / 2;
|
|
102
|
+
}
|
|
103
|
+
return medians;
|
|
104
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -252,6 +252,7 @@ export {
|
|
|
252
252
|
DECLICK_CROSSFADE,
|
|
253
253
|
DECLICK_LEVELS,
|
|
254
254
|
DECLICK_MAX_IMPULSE,
|
|
255
|
+
DECLICK_REPAIR_RULES,
|
|
255
256
|
DECLICK_SETTINGS,
|
|
256
257
|
DECLICK_WORD_GUARD,
|
|
257
258
|
buildDeclickDecodeArgs,
|
|
@@ -260,6 +261,7 @@ export {
|
|
|
260
261
|
declickSpeech,
|
|
261
262
|
describeDeclick,
|
|
262
263
|
findImpulses,
|
|
264
|
+
findRepairs,
|
|
263
265
|
getArPolynomial,
|
|
264
266
|
getRepairZones,
|
|
265
267
|
getRunningMedian,
|
|
@@ -269,8 +271,10 @@ export {
|
|
|
269
271
|
type DeclickReport,
|
|
270
272
|
type DeclickSettings,
|
|
271
273
|
type DeclickSpeechOptions,
|
|
274
|
+
type DetectorDeclickLevel,
|
|
272
275
|
type TimeSpan as DeclickSpan,
|
|
273
276
|
} from "./footage/declick.js";
|
|
277
|
+
export { MOUTH_CLICK_RULES, findMouthClicks, type MouthClicks } from "./footage/mouth-clicks.js";
|
|
274
278
|
export {
|
|
275
279
|
MIN_CLIP_SECONDS,
|
|
276
280
|
describeProbe,
|