@audio/neural-denoise 0.6.1 → 0.7.0

package/README.md

@@ -121,6 +121,35 @@ Both models enhance speech, and to them everything else is noise: at their defau
 
 It runs in JS: a stream's `process()` can't await ONNX Runtime, and RNNoise needs no runtime. The three upper convolutions (through im2col) and the dense layers run in a 1,074-byte WebAssembly SIMD kernel (`gemm.wat`), the first convolution once per frame and shared by the overlapping patches (the max pooling after it commutes with each patch's standardization); 1.1 s of CPU per minute of audio, 11 s on the JS path where WebAssembly SIMD is missing (same bits).
 
+`segments(x, sampleRate)` gives the classifier's own reading of an input: what it is, stretch by stretch, `[{ kind, start, end }]` in seconds, `kind` one of `'speech'`, `'music'`, `'noise'`, in order and end to end over the input. It is step 2 without the prior for speech, which is the guard's: ina's decision as it stands, each 10 ms frame taking the patch nearest it. Digital silence takes the kind around it; an input of nothing but silence has no segments. A host names a recording by it (`audio`'s editor: speech, music, both).
+
+```js
+import { segments } from '@audio/neural-denoise'
+
+await segments(pcm, 44100)  // [{ kind: 'music', start: 0, end: 2.5 }, { kind: 'speech', start: 2.5, end: 11.27 }]
+```
+
+What it names the test sets below (`node scripts/accuracy.mjs kinds test`): the share of each set's active 10 ms frames named each kind, and its files by the kind most of their active frames have.
+
+| Test set | Speech | Music | Noise | Files, by their most frames |
+|---|---|---|---|---|
+| VoiceBank+DEMAND, clean / noisy (824 each) | 99.9% / 99.8% | 0.0% / 0.0% | 0.1% / 0.2% | 823 / 822 speech, 1 / 2 noise |
+| the same in white noise, 20 / 10 / 5 dB (275 each) | 99.6% / 99.6% / 98.9% | 0.0% | 0.4% / 0.4% / 1.1% | 274 / 274 / 273 speech, the rest noise |
+| the same in pink noise, 20 / 10 / 5 dB | 99.6% / 99.4% / 99.4% | 0.0% | 0.4% / 0.6% / 0.6% | 274 speech each, 1 noise |
+| over a music bed 20 / 15 / 10 dB down (275 each) | 99.6% / 99.3% / 98.6% | 0.4% / 0.7% / 1.4% | 0.0% | 274 / 274 / 272 speech, the rest music |
+| ten home narrations (Rooms) | 99.9% | 0.0% | 0.1% | 10 speech |
+| VocalSet, spoken excerpts (10) | 98.2% | 1.8% | 0.0% | 10 speech |
+| MUSDB18 test previews, songs (50) | 0.0% | 100% | 0.0% | 50 music |
+| the same, accompaniment only (50) | 0.0% | 100% | 0.0% | 50 music |
+| Slakh2100 mixes 11 to 20 | 0.0% | 100% | 0.0% | 10 music |
+| the four repair pieces | 0.0% | 95.7% | 4.3% | 3 music, 1 noise |
+| the programmes (8), speech and music in turn | 53.3% | 46.7% | 0.0% | |
+| MUSDB18 vocal stems (50) | 38.1% | 57.5% | 4.4% | 28 music, 19 speech, 3 noise |
+| VocalSet sung phrases (20) / long tones (10) | 26.8% / 11.5% | 66.4% / 67.8% | 6.8% / 20.7% | 14 music, 6 speech / 8 music, 1 speech, 1 noise |
+| DEMAND noise alone (noisy − clean, 824) | 13.6% | 7.8% | 78.6% | 637 noise, 120 speech, 67 music |
+
+Speech is named speech and music music wherever the input is one of them (98.2% of the frames or more). A voice singing alone is either. What is neither is called noise four frames in five, speech or music otherwise: of DEMAND's noises alone, 187 of 824 files are named for a kind they are not, so its `'noise'` is the weak answer of the three.
+
 **Accuracy**, on labelled sets (`python scripts/accuracy.py guard-sets`, `node scripts/accuracy.mjs guard train|test`): the share of active 10 ms frames passed (within 40 dB of the loudest). The constants (one patch per 100 ms, the streaming bound, prior and hysteresis) were chosen on the training sets: VoiceBank+DEMAND's training speakers, the MUSDB18 training previews, Slakh2100 mixes 1 to 10, VocalSet singers 1 to 5, the Spoken Wikipedia narrations not in Rooms; the test sets below were run once. Beds: clean utterances over a song's accompaniment (the MUSDB18 stems but the vocals) 20, 15 and 10 dB under the speech; hiss: white and pink Gaussian noise at 20, 10 and 5 dB SNR. Programmes: 8 sequences of six segments, speech (five utterances of one kind, or 15 s of narration) and music (up to 20 s of a song, its accompaniment or a Slakh mix) in turn, scored 0.5 s or more from each cut.
 
 | Test set | Frames | Passed, offline (DeepFilterNet3) | Passed, streaming (RNNoise) |

package/denoise.js

@@ -6,7 +6,7 @@
 import resample from '@audio/resample-sinc'
 import { model as parseRNNoise, create, FRAME, LIMIT as RNNOISE_LIMIT } from './rnnoise.js'
 import { loadDeepFilter, enhance, level, bandEdge, MODEL, LEVEL, LIMIT as DFN_LIMIT, FLOOR } from './deepfilter.js'
-import { guardNet, offline, online, gains, blend } from './guard.js'
+import { guardNet, states, patchOf, offline, online, gains, blend } from './guard.js'
 
 export { create as rnnoise, FRAME } from './rnnoise.js'
 export { MODEL } from './deepfilter.js'
@@ -115,6 +115,29 @@ export async function guard(x, sampleRate = RATE) {
 	return sampleRate === RATE ? g : fit(resample(g, { from: RATE, to: sampleRate }), x.length)
 }
 
+/**
+ * segments(x, sampleRate) → what the input is, stretch by stretch: [{ kind, start, end }], kind 'speech', 'music' or
+ * 'noise', in seconds, in order and end to end from 0 to the input's length. Ina's segments over the whole input, as
+ * the classifier itself decides them: guard.js states() with no prior for speech (that is the guard's, for a voice
+ * under a bed). What counts as what follows its training, French radio and TV: speech over a music bed is speech, a
+ * song is music, its singer included. A change falls on the 10 ms frame whose nearest patch changes, a patch every
+ * 100 ms. Digital silence takes the kind around it; an input of nothing but silence, or shorter than a frame, has no
+ * segments.
+ */
+export const KINDS = ['speech', 'music', 'noise']
+export async function segments(x, sampleRate = RATE) {
+	if (!x.some(v => v !== 0)) return []
+	let nn = await guardNet(), x48 = sampleRate === RATE ? x : resample(Float32Array.from(x), { from: sampleRate, to: RATE })
+	let st = states(x48, nn, 0), T = st.length, n = Math.ceil(x48.length / 480), out = []
+	for (let g = 0; T && g < n; g++) {
+		let kind = KINDS[st[patchOf(g, T)]], last = out.at(-1)
+		if (last?.kind === kind) continue
+		if (last) last.end = g / 100
+		out.push({ kind, start: g / 100, end: x.length / sampleRate })
+	}
+	return out
+}
+
 // the enhanced y where the gain g (at 48 kHz) is 1, the input x where it is 0: music passes untouched
 function mix(y, x, g, rate) {
 	if (rate !== RATE) g = fit(resample(g, { from: RATE, to: rate }), x.length)

package/guard.js

@@ -279,22 +279,33 @@ export function analyzer(nn) {
 
 const lp = (p, floor) => Math.log(Math.max(p, floor))
 
-/** offline(x, nn) → per 48 kHz frame of x (480 samples), 1 where it is music: ina's Viterbi over the whole input. */
-export function offline(x, nn) {
-	let a = analyzer(nn), P = [...a(x), ...a.end()], T = P.length, n = Math.ceil(x.length / 480), pass = new Uint8Array(n)
-	if (!T) return pass
+/**
+ * states(x, nn, prior) → per patch of x (48 kHz), its state on the best path, 0 speech, 1 music, 2 noise: ina's Viterbi
+ * over the whole input, each patch `prior` nats in favour of speech (the guard's PRIOR; 0 is ina's own decision).
+ */
+export function states(x, nn, prior = PRIOR) {
+	let a = analyzer(nn), P = [...a(x), ...a.end()], T = P.length, st = new Uint8Array(T)
+	if (!T) return st
 	// V: the best path's log-likelihood ending in each state (speech, music, noise); B: the state it came from. Every
-	// switch costs COST, speech gains PRIOR a patch; a patch of silence scores all states alike. Ties go to the lower
-	// state, as numpy's argmax.
+	// switch costs COST, speech gains the prior a patch; a patch of silence scores all states alike. Ties go to the
+	// lower state, as numpy's argmax.
 	let V = [0, 0, 0], B = new Uint8Array(T * 3)
 	for (let t = 0; t < T; t++) {
-		let e = P[t] ? P[t].map((p, s) => lp(p, 1e-10) + (s ? 0 : PRIOR)) : [0, 0, 0], j = V.indexOf(Math.max(...V))
+		let e = P[t] ? P[t].map((p, s) => lp(p, 1e-10) + (s ? 0 : prior)) : [0, 0, 0], j = V.indexOf(Math.max(...V))
 		V = V.map((v, s) => { let b = V[j] - COST > v ? j : s; B[t * 3 + s] = b; return (b === s ? v : V[j] - COST) + e[s] })
 	}
-	let music = new Uint8Array(T)
-	for (let t = T - 1, s = V.indexOf(Math.max(...V)); t >= 0; s = B[t * 3 + s], t--) music[t] = +(s === 1)
-	// each frame takes the patch centered nearest it
-	for (let g = 0; g < n; g++) pass[g] = music[Math.min(T - 1, Math.max(0, Math.round((g + .5 - 34.716) / STEP)))]
+	for (let t = T - 1, s = V.indexOf(Math.max(...V)); t >= 0; s = B[t * 3 + s], t--) st[t] = s
+	return st
+}
+
+/** patchOf(g, T) → the patch, of T, a 48 kHz frame g (480 samples) takes: the one centered nearest it. */
+export const patchOf = (g, T) => Math.min(T - 1, Math.max(0, Math.round((g + .5 - 34.716) / STEP)))
+
+/** offline(x, nn) → per 48 kHz frame of x (480 samples), 1 where it is music: ina's Viterbi over the whole input. */
+export function offline(x, nn) {
+	let st = states(x, nn), T = st.length, n = Math.ceil(x.length / 480), pass = new Uint8Array(n)
+	if (!T) return pass
+	for (let g = 0; g < n; g++) pass[g] = +(st[patchOf(g, T)] === 1)
 	return pass
 }
 

package/index.d.ts

@@ -51,6 +51,12 @@ export function mixback(x: Float32Array, y: Float32Array, opts?: { limit?: numbe
 /** Per sample of x, the gain an enhancement of it takes: 1 where it is speech or noise, 0 where it is music (inaSpeechSegmenter's segments over the whole input, as denoise() decides), ramped over 200 ms; blend g·y + (1 − g)·x. */
 export function guard(x: Float32Array, sampleRate?: number): Promise<Float32Array>
 
+/** What x is, stretch by stretch, as inaSpeechSegmenter's speech/music/noise CNN segments the whole input (no prior for speech, which is the guard's): in order, end to end from 0 to its length, seconds; none for an input of digital silence. */
+export function segments(x: Float32Array, sampleRate?: number): Promise<{ kind: 'speech' | 'music' | 'noise', start: number, end: number }[]>
+
+/** The kinds segments() names, as the classifier orders them. */
+export const KINDS: ['speech', 'music', 'noise']
+
 /** Load a model once to reuse it across denoise() calls. */
 export function load(model?: 'rnnoise' | 'deepfilternet3', opts?: LoadOptions): Promise<Model>
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@audio/neural-denoise",
-  "version": "0.6.1",
-  "description": "Neural speech enhancement: RNNoise (Valin 2018) in pure JS, bit-exact to its C, weights bundled; DeepFilterNet3 through ONNX Runtime, matching the Python original; music passes through untouched (inaSpeechSegmenter's speech/music/noise CNN)",
+  "version": "0.7.0",
+  "description": "Neural speech enhancement: RNNoise (Valin 2018) in pure JS, bit-exact to its C, weights bundled; DeepFilterNet3 through ONNX Runtime, matching the Python original; music passes through untouched (inaSpeechSegmenter's speech/music/noise CNN), and the same classifier names an input's stretches (segments)",
   "type": "module",
   "sideEffects": false,
   "main": "denoise.js",