@softure-ai/marketing-kit 0.18.0 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CHANGELOG.md +27 -0
  2. package/README.md +27 -0
  3. package/dist/config/schema.d.ts +1 -0
  4. package/dist/config/schema.d.ts.map +1 -1
  5. package/dist/config/schema.js +1 -1
  6. package/dist/config/schema.js.map +1 -1
  7. package/dist/footage/clip-sound.d.ts +59 -4
  8. package/dist/footage/clip-sound.d.ts.map +1 -1
  9. package/dist/footage/clip-sound.js +58 -12
  10. package/dist/footage/clip-sound.js.map +1 -1
  11. package/dist/footage/declick.d.ts +35 -9
  12. package/dist/footage/declick.d.ts.map +1 -1
  13. package/dist/footage/declick.js +57 -88
  14. package/dist/footage/declick.js.map +1 -1
  15. package/dist/footage/film.d.ts +5 -2
  16. package/dist/footage/film.d.ts.map +1 -1
  17. package/dist/footage/film.js.map +1 -1
  18. package/dist/footage/isolate-ranges.d.ts +5 -0
  19. package/dist/footage/isolate-ranges.d.ts.map +1 -1
  20. package/dist/footage/isolate-ranges.js +4 -2
  21. package/dist/footage/isolate-ranges.js.map +1 -1
  22. package/dist/footage/mouth-clicks.d.ts +85 -0
  23. package/dist/footage/mouth-clicks.d.ts.map +1 -0
  24. package/dist/footage/mouth-clicks.js +207 -0
  25. package/dist/footage/mouth-clicks.js.map +1 -0
  26. package/dist/footage/pipeline.d.ts +11 -0
  27. package/dist/footage/pipeline.d.ts.map +1 -1
  28. package/dist/footage/pipeline.js +15 -9
  29. package/dist/footage/pipeline.js.map +1 -1
  30. package/dist/footage/sound-analysis.d.ts +25 -0
  31. package/dist/footage/sound-analysis.d.ts.map +1 -0
  32. package/dist/footage/sound-analysis.js +95 -0
  33. package/dist/footage/sound-analysis.js.map +1 -0
  34. package/dist/index.d.ts +3 -2
  35. package/dist/index.d.ts.map +1 -1
  36. package/dist/index.js +3 -2
  37. package/dist/index.js.map +1 -1
  38. package/package.json +1 -1
  39. package/schema/marketing.schema.json +3 -2
  40. package/src/config/schema.ts +1 -1
  41. package/src/footage/clip-sound.ts +95 -15
  42. package/src/footage/declick.ts +64 -97
  43. package/src/footage/film.ts +5 -2
  44. package/src/footage/isolate-ranges.ts +9 -3
  45. package/src/footage/mouth-clicks.ts +224 -0
  46. package/src/footage/pipeline.ts +29 -10
  47. package/src/footage/sound-analysis.ts +104 -0
  48. package/src/index.ts +9 -0
@@ -0,0 +1,224 @@
1
+ import { getBand, getFrameEnergies, getRunningMedian, getSquareSums, toPowerDb, type TimeSpan } from "./sound-analysis.js";
2
+
3
+ /**
4
+ * The mouth-click detector shared by moving a segment's cut past a click (stage A, issue #1182) and the declick
5
+ * stage's `repair` level (stage B, issue #1183), so a click that moves a cut and a click that is repaired are the same
6
+ * thing. Its input is the speech as the chain hears it: the clips' sound after the -23 LUFS pre-gain.
7
+ *
8
+ * - **An impulse** lasts 1 to 25 ms, its 2-10 kHz energy rises more than 12 dB above the band's ±50 ms running median,
9
+ * it is not periodic (the band's normalized autocorrelation stays under 0.3 at the pitch lags, 2.5 to 15 ms, over a
10
+ * 30 ms window centred on it) and it is over within its length (the 10 ms after it at least 12 dB under its peak, so
11
+ * the onset of a breath or a fricative is not one).
12
+ * - **The speech protection zone:** each word's bound on the real sound (the earlier start and the later end of the
13
+ * transcript's bound and the point where the 10 ms energy falls 20 dB under the word's peak), widened by 80 ms; and
14
+ * every stretch whose 10 ms energy is above -45 dBFS, measured with the impulses' own samples left out (a click's
15
+ * own energy must not protect it).
16
+ */
17
+ export const MOUTH_CLICK_RULES = {
18
+ /** The shortest impulse, seconds. */
19
+ minSeconds: 0.001,
20
+ /** The longest impulse, seconds. */
21
+ maxSeconds: 0.025,
22
+ /** The band the impulse is measured in. */
23
+ bandLowHz: 2000,
24
+ bandHighHz: 10_000,
25
+ /** How far an impulse's band energy rises above the band's running median, dB. */
26
+ jumpDb: 12,
27
+ /** The running median's reach on each side, seconds. */
28
+ medianSeconds: 0.05,
29
+ /** The highest normalized autocorrelation at the pitch lags an impulse may show. */
30
+ maxCorrelation: 0.3,
31
+ /** The pitch lags, seconds. */
32
+ minLagSeconds: 0.0025,
33
+ maxLagSeconds: 0.015,
34
+ /** The window the autocorrelation is measured over, centred on the impulse, seconds. */
35
+ correlationSeconds: 0.03,
36
+ /** The stretch after an impulse that must be quieter than its peak, seconds, and by how much, dB. */
37
+ afterSeconds: 0.01,
38
+ afterDropDb: 12,
39
+ /** The energy window of the word bounds and the loudness rule, seconds. */
40
+ energySeconds: 0.01,
41
+ /** A word's sound lasts while its 10 ms energy stays within this many dB of the word's peak. */
42
+ wordDropDb: 20,
43
+ /** The guard on each side of a word's bound, seconds. */
44
+ wordGuardSeconds: 0.08,
45
+ /** Nothing whose 10 ms energy is above this is a click to act on, dBFS (mean square). */
46
+ loudDbfs: -45,
47
+ } as const;
48
+
49
+ const FRAME_SECONDS = 0.002;
50
+ const HOP_SECONDS = 0.001;
51
+ /** The step the word bounds and the loudness rule are measured at. */
52
+ const STEP_SECONDS = 0.001;
53
+ /** An impulse reaches this far past its detected frames, for the band filter's delay and the frames' edges. */
54
+ const IMPULSE_MARGIN = 0.001;
55
+ /** A frame quieter than this (mean square, dB) is never an impulse: digital silence with a stray bit is not a click. */
56
+ const BAND_FLOOR_DB = -90;
57
+
58
+ /** The highest normalized autocorrelation of `band` at the pitch lags over the window centred on `center`. */
59
+ function getPeakCorrelation(band: Float64Array, center: number, sampleRate: number): number {
60
+ const half = Math.round((MOUTH_CLICK_RULES.correlationSeconds * sampleRate) / 2);
61
+ const from = Math.max(0, center - half);
62
+ const to = Math.min(band.length, center + half);
63
+ const minLag = Math.max(1, Math.round(MOUTH_CLICK_RULES.minLagSeconds * sampleRate));
64
+ const maxLag = Math.min(Math.round(MOUTH_CLICK_RULES.maxLagSeconds * sampleRate), to - from - 1);
65
+ let peak = 0;
66
+ for (let lag = minLag; lag <= maxLag; lag++) {
67
+ let product = 0;
68
+ let head = 0;
69
+ let tail = 0;
70
+ for (let index = from; index + lag < to; index++) {
71
+ const a = band[index] ?? 0;
72
+ const b = band[index + lag] ?? 0;
73
+ product += a * b;
74
+ head += a * a;
75
+ tail += b * b;
76
+ }
77
+ if (head > 0 && tail > 0) peak = Math.max(peak, product / Math.sqrt(head * tail));
78
+ }
79
+ return peak;
80
+ }
81
+
82
+ /**
83
+ * The impulses of a mono signal by `MOUTH_CLICK_RULES` (length, band jump, periodicity, end), anywhere in it. Times in
84
+ * seconds from sample 0, widened by a millisecond on each side.
85
+ */
86
+ export function findClickCandidates(mono: Float32Array, sampleRate: number): TimeSpan[] {
87
+ const seconds = mono.length / sampleRate;
88
+ const band = getBand({ mono, sampleRate, lowHz: MOUTH_CLICK_RULES.bandLowHz, highHz: MOUTH_CLICK_RULES.bandHighHz });
89
+ const sums = getSquareSums(band);
90
+ const frameLength = Math.max(1, Math.round(FRAME_SECONDS * sampleRate));
91
+ const hop = Math.max(1, Math.round(HOP_SECONDS * sampleRate));
92
+ const after = Math.round(MOUTH_CLICK_RULES.afterSeconds * sampleRate);
93
+ const energies = getFrameEnergies(band, frameLength, hop);
94
+ const medians = getRunningMedian(energies, Math.round(MOUTH_CLICK_RULES.medianSeconds / HOP_SECONDS));
95
+ const candidates: TimeSpan[] = [];
96
+ const checkRun = (first: number, last: number): void => {
97
+ const start = (first * hop) / sampleRate;
98
+ const end = (last * hop + frameLength) / sampleRate;
99
+ if (end - start < MOUTH_CLICK_RULES.minSeconds || end - start > MOUTH_CLICK_RULES.maxSeconds) return;
100
+ const span = { start: Math.max(0, start - IMPULSE_MARGIN), end: Math.min(seconds, end + IMPULSE_MARGIN) };
101
+ let peakDb = -Infinity;
102
+ for (let frame = first; frame <= last; frame++) peakDb = Math.max(peakDb, energies[frame] ?? -Infinity);
103
+ const afterFrom = Math.round(span.end * sampleRate);
104
+ const afterTo = Math.min(band.length, afterFrom + after);
105
+ if (afterTo > afterFrom && toPowerDb(((sums[afterTo] ?? 0) - (sums[afterFrom] ?? 0)) / (afterTo - afterFrom)) > peakDb - MOUTH_CLICK_RULES.afterDropDb) return;
106
+ const center = Math.round(((span.start + span.end) / 2) * sampleRate);
107
+ if (getPeakCorrelation(band, center, sampleRate) >= MOUTH_CLICK_RULES.maxCorrelation) return;
108
+ candidates.push(span);
109
+ };
110
+ let runStart = -1;
111
+ for (let frame = 0; frame < energies.length; frame++) {
112
+ const energy = energies[frame] ?? 0;
113
+ const isJump = energy > BAND_FLOOR_DB && energy - (medians[frame] ?? 0) > MOUTH_CLICK_RULES.jumpDb;
114
+ if (isJump && runStart === -1) runStart = frame;
115
+ else if (!isJump && runStart !== -1) {
116
+ checkRun(runStart, frame - 1);
117
+ runStart = -1;
118
+ }
119
+ }
120
+ if (runStart !== -1) checkRun(runStart, energies.length - 1);
121
+ return candidates;
122
+ }
123
+
124
+ /** The analysis of one mono stretch in its own time (sample 0 at 0 s). */
125
+ export interface MouthClickAnalysis {
126
+ /** Every impulse by the rule, protected or not. */
127
+ candidates: TimeSpan[];
128
+ /** Each word's bound on the real sound (before the guard). */
129
+ bounds: TimeSpan[];
130
+ /** True when the span touches a word's bound with its guard, or a 10 ms energy above the loudness rule. */
131
+ isProtected: (span: TimeSpan) => boolean;
132
+ /** True when the stretch holds no 10 ms window above the loudness rule, the impulses' samples left out. */
133
+ isQuiet: (stretch: TimeSpan) => boolean;
134
+ }
135
+
136
+ /** The detector over a mono stretch whose words are in the stretch's own time. */
137
+ export function analyzeMouthClicks(mono: Float32Array, sampleRate: number, words: readonly TimeSpan[]): MouthClickAnalysis {
138
+ const length = mono.length;
139
+ const candidates = findClickCandidates(mono, sampleRate);
140
+ const sums = getSquareSums(mono);
141
+ const masked = Float64Array.from(mono);
142
+ const counts = new Float64Array(length + 1);
143
+ const isMasked = new Uint8Array(length);
144
+ for (const candidate of candidates) {
145
+ for (let index = Math.max(0, Math.floor(candidate.start * sampleRate)); index < Math.min(length, Math.ceil(candidate.end * sampleRate)); index++) {
146
+ isMasked[index] = 1;
147
+ masked[index] = 0;
148
+ }
149
+ }
150
+ for (let index = 0; index < length; index++) counts[index + 1] = (counts[index] ?? 0) + (isMasked[index] === 1 ? 0 : 1);
151
+ const maskedSums = getSquareSums(masked);
152
+ const window = Math.max(1, Math.round(MOUTH_CLICK_RULES.energySeconds * sampleRate));
153
+ const step = Math.max(1, Math.round(STEP_SECONDS * sampleRate));
154
+ const clamp = (index: number): number => Math.min(length, Math.max(0, index));
155
+ const getDb = (from: number, to: number): number => {
156
+ const [a, b] = [clamp(from), clamp(to)];
157
+ return b > a ? toPowerDb(((sums[b] ?? 0) - (sums[a] ?? 0)) / (b - a)) : -Infinity;
158
+ };
159
+ const getMaskedDb = (from: number, to: number): number => {
160
+ const [a, b] = [clamp(from), clamp(to)];
161
+ const count = (counts[b] ?? 0) - (counts[a] ?? 0);
162
+ return count > 0 ? toPowerDb(((maskedSums[b] ?? 0) - (maskedSums[a] ?? 0)) / count) : -Infinity;
163
+ };
164
+ const getBound = (word: TimeSpan): TimeSpan => {
165
+ const from = clamp(Math.round(word.start * sampleRate));
166
+ const to = clamp(Math.round(word.end * sampleRate));
167
+ if (to <= from) return { ...word };
168
+ let peakDb = -Infinity;
169
+ for (let index = from; index <= Math.max(from, to - window); index += step) peakDb = Math.max(peakDb, getDb(index, Math.min(to, index + window)));
170
+ const threshold = peakDb - MOUTH_CLICK_RULES.wordDropDb;
171
+ // The start walks back while the 10 ms before it hold the word's sound, the end forward while the 10 ms after it do.
172
+ let start = from;
173
+ while (start > 0 && getDb(start - window, start) >= threshold) start = clamp(start - step);
174
+ let end = to;
175
+ while (end < length && getDb(end, end + window) >= threshold) end = clamp(end + step);
176
+ return { start: Math.min(word.start, start / sampleRate), end: Math.max(word.end, end / sampleRate) };
177
+ };
178
+ const bounds = [...words].filter((word) => word.end > word.start).sort((a, b) => a.start - b.start).map(getBound);
179
+ const isLoud = (span: TimeSpan): boolean => {
180
+ const from = Math.floor(span.start * sampleRate);
181
+ const to = Math.ceil(span.end * sampleRate);
182
+ for (let center = from; ; center = Math.min(to, center + step)) {
183
+ if (getMaskedDb(center - (window >> 1), center + window - (window >> 1)) > MOUTH_CLICK_RULES.loudDbfs) return true;
184
+ if (center >= to) return false;
185
+ }
186
+ };
187
+ const guard = MOUTH_CLICK_RULES.wordGuardSeconds;
188
+ return {
189
+ candidates,
190
+ bounds,
191
+ isProtected: (span) => bounds.some((bound) => span.start < bound.end + guard && span.end > bound.start - guard) || isLoud(span),
192
+ isQuiet: (stretch) => stretch.end <= 0 || stretch.start >= length / sampleRate || !isLoud({ start: Math.max(0, stretch.start), end: Math.min(length / sampleRate, stretch.end) }),
193
+ };
194
+ }
195
+
196
+ /** What `findMouthClicks` found, in the film's time (the stretch's `offset` added). */
197
+ export interface MouthClicks {
198
+ /** The impulses outside the speech protection zone, in time order. */
199
+ clicks: TimeSpan[];
200
+ /** Each word's bound on the real sound (before the ±80 ms guard), in time order. */
201
+ words: TimeSpan[];
202
+ /** True when the stretch holds no 10 ms window above -45 dBFS, the impulses' samples left out. */
203
+ isQuiet: (stretch: TimeSpan) => boolean;
204
+ }
205
+
206
+ /**
207
+ * Mouth clicks of a mono stretch whose sample 0 plays at `offset` seconds, outside the speech protection zone
208
+ * (`MOUTH_CLICK_RULES`); `words` are in the same time as `offset`. No audio is changed: an app can move a segment's
209
+ * cut past a click before its first word or after its last (issue #1182).
210
+ */
211
+ export function findMouthClicks(options: { mono: Float32Array; sampleRate: number; offset: number; words: readonly TimeSpan[] }): MouthClicks {
212
+ const { mono, sampleRate, offset } = options;
213
+ const shift = (span: TimeSpan, by: number): TimeSpan => ({ start: span.start + by, end: span.end + by });
214
+ const analysis = analyzeMouthClicks(
215
+ mono,
216
+ sampleRate,
217
+ options.words.map((word) => shift(word, -offset)),
218
+ );
219
+ return {
220
+ clicks: analysis.candidates.filter((candidate) => !analysis.isProtected(candidate)).map((candidate) => shift(candidate, offset)),
221
+ words: analysis.bounds.map((bound) => shift(bound, offset)),
222
+ isQuiet: (stretch) => analysis.isQuiet(shift(stretch, -offset)),
223
+ };
224
+ }
@@ -12,7 +12,7 @@ import { copyFonts } from "../render/render.js";
12
12
  import type { TtsResult } from "../voice/provider.js";
13
13
  import type { TimedWord } from "../voice/voiceover.js";
14
14
  import { checkMusicFileType, describeAudioProbe, probeAudio } from "./audio.js";
15
- import { prepareClipSounds } from "./clip-sound.js";
15
+ import { prepareClipSounds, type SpeechPeaks, type SpeechPreGain } from "./clip-sound.js";
16
16
  import { readCachedTranscript, writeCacheFile, writeTranscript } from "./cache.js";
17
17
  import { composeFootage, getFootageFilmSeconds, getFootageSfxCues } from "./compose.js";
18
18
  import { findMissingCaptionGlyphs, getCaptionBounds, getFootageCaptionLayout } from "./captions.js";
@@ -145,6 +145,11 @@ export interface FootageRunOptions {
145
145
  * person to run again. Every failure also gets one `log` line with the step and its full command.
146
146
  */
147
147
  onFfmpegFailure?: FfmpegFailureHook;
148
+ /**
149
+ * How the clip sound's pre-gain treats a true peak above `SPEECH_PRE_GAIN.peakCeiling` (issue #1181): `ceiling`
150
+ * (default) stops the gain there; `float` reaches the target and writes the clip sound as 32-bit float.
151
+ */
152
+ speechPeaks?: SpeechPeaks;
148
153
  }
149
154
 
150
155
  export interface PreparedClip {
@@ -173,6 +178,11 @@ export type TranscribeOutcome =
173
178
  * optional so an app's own `ready` outcome still compiles.
174
179
  */
175
180
  speechGainDb?: Record<string, number>;
181
+ /**
182
+ * What each clip's pre-gain reached (`ClipSound.preGain`, issue #1181): `held_by_peak_ceiling` names a clip the
183
+ * speech chain hears under the target. Always answered; optional so an app's own `ready` outcome still compiles.
184
+ */
185
+ speechPreGain?: Record<string, SpeechPreGain>;
176
186
  };
177
187
 
178
188
  export function getFrame(film: Pick<FootageFilm, "format">): Frame {
@@ -301,20 +311,29 @@ async function decodeIsolatedVoice(cached: string, clipId: string, buildDir: str
301
311
  return output;
302
312
  }
303
313
 
304
- /** Each clip's lossless, pre-gained sound (`prepareClipSounds`) and its pre-gain in dB, by clip id. */
305
- async function getClipSoundFiles(clips: PreparedClip[], buildDir: string, signal: AbortSignal | undefined): Promise<{ speech: Record<string, string>; speechGainDb: Record<string, number> }> {
306
- const sounds = Object.entries(await prepareClipSounds(clips, { buildDir, ...(signal === undefined ? {} : { signal }) }));
307
- return { speech: Object.fromEntries(sounds.map(([id, sound]) => [id, sound.file])), speechGainDb: Object.fromEntries(sounds.map(([id, sound]) => [id, sound.gainDb])) };
314
+ /** Each clip's lossless, pre-gained sound (`prepareClipSounds`), its pre-gain in dB and what it reached, by clip id. */
315
+ async function getClipSoundFiles(
316
+ clips: PreparedClip[],
317
+ options: Pick<FootageRunOptions, "buildDir" | "signal" | "speechPeaks" | "log">,
318
+ ): Promise<{ speech: Record<string, string>; speechGainDb: Record<string, number>; speechPreGain: Record<string, SpeechPreGain> }> {
319
+ const { buildDir, signal, speechPeaks, log } = options;
320
+ const sounds = Object.entries(await prepareClipSounds(clips, { buildDir, log, ...(signal === undefined ? {} : { signal }), ...(speechPeaks === undefined ? {} : { peaks: speechPeaks }) }));
321
+ return {
322
+ speech: Object.fromEntries(sounds.map(([id, sound]) => [id, sound.file])),
323
+ speechGainDb: Object.fromEntries(sounds.map(([id, sound]) => [id, sound.gainDb])),
324
+ speechPreGain: Object.fromEntries(sounds.map(([id, sound]) => [id, sound.preGain])),
325
+ };
308
326
  }
309
327
 
310
328
  /** The free transcripts of `--placeholder`, written under `placeholder/` for a look; the speech is each clip's lossless sound. */
311
- async function transcribePlaceholder(clips: PreparedClip[], buildDir: string, signal: AbortSignal | undefined): Promise<TranscribeOutcome> {
329
+ async function transcribePlaceholder(clips: PreparedClip[], options: FootageRunOptions): Promise<TranscribeOutcome> {
330
+ const { buildDir } = options;
312
331
  const words: Record<string, TimedWord[]> = {};
313
332
  for (const clip of clips) {
314
333
  words[clip.clip.id] = clip.clip.role === "talk" ? makePlaceholderTranscript(clip.clip.script, clip.duration) : [];
315
334
  writeTranscript(join(buildDir, "placeholder", `${clip.clip.id}.transcript.json`), words[clip.clip.id] ?? []);
316
335
  }
317
- return { kind: "ready", words, ...(await getClipSoundFiles(clips, buildDir, signal)) };
336
+ return { kind: "ready", words, ...(await getClipSoundFiles(clips, options)) };
318
337
  }
319
338
 
320
339
  /** The paid music step, when the film's music still has to be bought; null when it is cached or free. */
@@ -337,7 +356,7 @@ export function transcribeFootage(film: FootageFilm, clips: PreparedClip[], opti
337
356
  /** The body of `transcribeFootage`, inside its ffmpeg failure reporting. */
338
357
  async function runTranscribeFootage(film: FootageFilm, clips: PreparedClip[], options: FootageRunOptions): Promise<FootageResult<TranscribeOutcome>> {
339
358
  const { buildDir, providers, log, signal } = options;
340
- if (options.isPlaceholder) return ok(await transcribePlaceholder(clips, buildDir, signal));
359
+ if (options.isPlaceholder) return ok(await transcribePlaceholder(clips, options));
341
360
  const keyTerms = getFilmKeyTerms(film);
342
361
  if (!keyTerms.ok) return err(`footage film ${film.id}: ${keyTerms.errors.join(" ")}`, { kind: "input" });
343
362
  const sttKeyTerms = keyTerms.terms.length === 0 ? {} : { keyTerms: keyTerms.terms };
@@ -363,7 +382,7 @@ async function runTranscribeFootage(film: FootageFilm, clips: PreparedClip[], op
363
382
  const words: Record<string, TimedWord[]> = {};
364
383
  for (const clip of clips) words[clip.clip.id] = [];
365
384
  // Every clip's sound, lossless and pre-gained: what the isolator, the speech to text and the speech track hear (#1139).
366
- const { speech, speechGainDb } = await getClipSoundFiles(clips, buildDir, signal);
385
+ const { speech, speechGainDb, speechPreGain } = await getClipSoundFiles(clips, options);
367
386
  const isolating = plan.filter(({ isolated }) => isolated !== null && !existsSync(isolated.path));
368
387
  const transcribing = plan.filter(({ transcript }) => !existsSync(transcript.path));
369
388
  const isolateProgress = createStageProgress(options.onProgress, "isolate", isolating.map(({ clip }) => ({ item: clip.clip.id, seconds: clip.duration })));
@@ -416,7 +435,7 @@ async function runTranscribeFootage(film: FootageFilm, clips: PreparedClip[], op
416
435
  transcribeProgress(index, clip.duration);
417
436
  words[id] = result.value.words;
418
437
  }
419
- return ok({ kind: "ready", words, speech, speechGainDb });
438
+ return ok({ kind: "ready", words, speech, speechGainDb, speechPreGain });
420
439
  }
421
440
 
422
441
  /** The talking clips' sentences with their ids (`s1`…), as `buildEdl` names them, for choosing what to keep. */
@@ -0,0 +1,104 @@
1
+ /**
2
+ * Sound analysis shared by the speech declick stage and the mouth-click detector (issues #1170, #1182, #1183): a
3
+ * fourth-order band pass, frame energies and a running median.
4
+ */
5
+
6
+ /** A stretch of time in seconds. */
7
+ export interface TimeSpan {
8
+ start: number;
9
+ end: number;
10
+ }
11
+
12
+ /** One second-order section (RBJ cookbook), run in place over the samples. */
13
+ function runBiquad(samples: Float64Array, kind: "highpass" | "lowpass", frequency: number, sampleRate: number): void {
14
+ const w0 = (2 * Math.PI * Math.min(frequency, sampleRate * 0.45)) / sampleRate;
15
+ const cos = Math.cos(w0);
16
+ const alpha = Math.sin(w0) / (2 * Math.SQRT1_2);
17
+ const a0 = 1 + alpha;
18
+ const b0 = (kind === "lowpass" ? (1 - cos) / 2 : (1 + cos) / 2) / a0;
19
+ const b1 = (kind === "lowpass" ? 1 - cos : -(1 + cos)) / a0;
20
+ const b2 = b0;
21
+ const a1 = (-2 * cos) / a0;
22
+ const a2 = (1 - alpha) / a0;
23
+ let x1 = 0;
24
+ let x2 = 0;
25
+ let y1 = 0;
26
+ let y2 = 0;
27
+ for (let index = 0; index < samples.length; index++) {
28
+ const x = samples[index] ?? 0;
29
+ const y = b0 * x + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2;
30
+ x2 = x1;
31
+ x1 = x;
32
+ y2 = y1;
33
+ y1 = y;
34
+ samples[index] = y;
35
+ }
36
+ }
37
+
38
+ /** The band of the signal between `lowHz` and `highHz`: two high-pass and two low-pass sections (fourth order on each side). */
39
+ export function getBand(options: { mono: Float32Array; sampleRate: number; lowHz: number; highHz: number }): Float64Array {
40
+ const { mono, sampleRate, lowHz, highHz } = options;
41
+ const band = Float64Array.from(mono);
42
+ runBiquad(band, "highpass", lowHz, sampleRate);
43
+ runBiquad(band, "highpass", lowHz, sampleRate);
44
+ runBiquad(band, "lowpass", highHz, sampleRate);
45
+ runBiquad(band, "lowpass", highHz, sampleRate);
46
+ return band;
47
+ }
48
+
49
+ /** The running sums of the squared samples: `sums[i]` is the energy of samples 0 to i - 1. */
50
+ export function getSquareSums(samples: ArrayLike<number>): Float64Array {
51
+ const sums = new Float64Array(samples.length + 1);
52
+ for (let index = 0; index < samples.length; index++) {
53
+ const value = samples[index] ?? 0;
54
+ sums[index + 1] = (sums[index] ?? 0) + value * value;
55
+ }
56
+ return sums;
57
+ }
58
+
59
+ /** A mean square in dB (10·log10), with a floor so silence stays finite. */
60
+ export const toPowerDb = (meanSquare: number): number => 10 * Math.log10(Math.max(meanSquare, 1e-20));
61
+
62
+ /** Each frame's mean square in dB, frames of `frameLength` samples every `hop` samples. */
63
+ export function getFrameEnergies(band: Float64Array, frameLength: number, hop: number): Float64Array {
64
+ const sums = getSquareSums(band);
65
+ const count = band.length < frameLength ? 0 : Math.floor((band.length - frameLength) / hop) + 1;
66
+ const energies = new Float64Array(count);
67
+ for (let frame = 0; frame < count; frame++) {
68
+ const from = frame * hop;
69
+ energies[frame] = toPowerDb(((sums[from + frameLength] ?? 0) - (sums[from] ?? 0)) / frameLength);
70
+ }
71
+ return energies;
72
+ }
73
+
74
+ /** The index of the first entry of the sorted list not below `value`. */
75
+ function findInsertIndex(sorted: number[], value: number): number {
76
+ let low = 0;
77
+ let high = sorted.length;
78
+ while (low < high) {
79
+ const middle = (low + high) >> 1;
80
+ if ((sorted[middle] ?? 0) < value) low = middle + 1;
81
+ else high = middle;
82
+ }
83
+ return low;
84
+ }
85
+
86
+ /** The running median of the values over `reach` entries on each side (fewer at the ends). */
87
+ export function getRunningMedian(values: Float64Array, reach: number): Float64Array {
88
+ const medians = new Float64Array(values.length);
89
+ const window: number[] = [];
90
+ let from = 0;
91
+ let to = 0;
92
+ for (let index = 0; index < values.length; index++) {
93
+ const wantedTo = Math.min(values.length, index + reach + 1);
94
+ for (; to < wantedTo; to++) {
95
+ const value = values[to] ?? 0;
96
+ window.splice(findInsertIndex(window, value), 0, value);
97
+ }
98
+ const wantedFrom = Math.max(0, index - reach);
99
+ for (; from < wantedFrom; from++) window.splice(findInsertIndex(window, values[from] ?? 0), 1);
100
+ const middle = window.length >> 1;
101
+ medians[index] = window.length % 2 === 1 ? (window[middle] ?? 0) : ((window[middle - 1] ?? 0) + (window[middle] ?? 0)) / 2;
102
+ }
103
+ return medians;
104
+ }
package/src/index.ts CHANGED
@@ -252,6 +252,7 @@ export {
252
252
  DECLICK_CROSSFADE,
253
253
  DECLICK_LEVELS,
254
254
  DECLICK_MAX_IMPULSE,
255
+ DECLICK_REPAIR_RULES,
255
256
  DECLICK_SETTINGS,
256
257
  DECLICK_WORD_GUARD,
257
258
  buildDeclickDecodeArgs,
@@ -260,6 +261,7 @@ export {
260
261
  declickSpeech,
261
262
  describeDeclick,
262
263
  findImpulses,
264
+ findRepairs,
263
265
  getArPolynomial,
264
266
  getRepairZones,
265
267
  getRunningMedian,
@@ -269,8 +271,10 @@ export {
269
271
  type DeclickReport,
270
272
  type DeclickSettings,
271
273
  type DeclickSpeechOptions,
274
+ type DetectorDeclickLevel,
272
275
  type TimeSpan as DeclickSpan,
273
276
  } from "./footage/declick.js";
277
+ export { MOUTH_CLICK_RULES, findMouthClicks, type MouthClicks } from "./footage/mouth-clicks.js";
274
278
  export {
275
279
  MIN_CLIP_SECONDS,
276
280
  describeProbe,
@@ -464,10 +468,15 @@ export {
464
468
  buildClipSoundMeasureArgs,
465
469
  getAudioCodecArgs,
466
470
  getClipSoundPath,
471
+ describeSpeechPreGain,
467
472
  getPreGainDb,
473
+ getSpeechPreGain,
468
474
  prepareClipSounds,
469
475
  type ClipSound,
470
476
  type ClipSoundInput,
477
+ type MeasuredPreGain,
478
+ type SpeechPeaks,
479
+ type SpeechPreGain,
471
480
  } from "./footage/clip-sound.js";
472
481
  export {
473
482
  AUDIO_BITRATE_RANGE_KBPS,