@audio/neural-quality 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +41 -0
- package/NOTICE +63 -0
- package/README.md +215 -2
- package/audio.d.ts +15 -0
- package/audio.js +12 -0
- package/index.d.ts +75 -0
- package/package.json +71 -4
- package/quality.js +172 -0
- package/resampy.js +80 -0
- package/scripts/inputs.mjs +58 -0
- package/scripts/reference.py +127 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
Kṛṣnized
|
|
2
|
+
|
|
3
|
+
This is a reminder that, in the absolute sense, all results of work belong to the Supreme.
|
|
4
|
+
|
|
5
|
+
While software and intellectual property (IP) require a formal license to define authorship,
|
|
6
|
+
responsibilities, or distribution limitations, these are temporary material designations.
|
|
7
|
+
The origin and destination of all ideas, as well as the ultimate owner of all results,
|
|
8
|
+
is the Supreme. The most meaningful action is to dedicate these results to the Supreme.
|
|
9
|
+
This process is known as Karma-Yoga and is described in BG 3.9, BG 12.10 and BG 18.46.
|
|
10
|
+
|
|
11
|
+
"Work done as a sacrifice for the Supreme must be performed;
|
|
12
|
+
otherwise, work binds one to the material world."
|
|
13
|
+
|
|
14
|
+
This license does not override or alter the terms specified by the material-level license below.
|
|
15
|
+
|
|
16
|
+
ॐ
|
|
17
|
+
|
|
18
|
+
---
|
|
19
|
+
|
|
20
|
+
MIT License
|
|
21
|
+
|
|
22
|
+
Copyright (c) 2026 Dmitry Iv. and audiojs contributors
|
|
23
|
+
Copyright (c) Microsoft Corporation (the method of DNSMOS/dnsmos_local.py, translated in quality.js)
|
|
24
|
+
|
|
25
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
26
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
27
|
+
in the Software without restriction, including without limitation the rights
|
|
28
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
29
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
30
|
+
furnished to do so, subject to the following conditions:
|
|
31
|
+
|
|
32
|
+
The above copyright notice and this permission notice shall be included in all
|
|
33
|
+
copies or substantial portions of the Software.
|
|
34
|
+
|
|
35
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
36
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
37
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
38
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
39
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
40
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
41
|
+
SOFTWARE.
|
package/NOTICE
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
@audio/neural-quality
|
|
2
|
+
Copyright 2026 Dmitry Iv. and audiojs contributors
|
|
3
|
+
|
|
4
|
+
This product includes a JavaScript translation of the scoring method of DNSMOS/dnsmos_local.py from
|
|
5
|
+
https://github.com/microsoft/DNS-Challenge at commit 82f1b17e7776a43eee395d0f45bae8abb700ad00
|
|
6
|
+
(ComputeScore.__call__ and get_polyfit_val's non-personalized polynomials), into quality.js. The
|
|
7
|
+
repository licenses its code under the MIT License (its README, Legal Notices; LICENSE-CODE):
|
|
8
|
+
|
|
9
|
+
MIT License, Copyright (c) Microsoft Corporation.
|
|
10
|
+
|
|
11
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this
|
|
12
|
+
software and associated documentation files (the "Software"), to deal in the Software
|
|
13
|
+
without restriction, including without limitation the rights to use, copy, modify, merge,
|
|
14
|
+
publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons
|
|
15
|
+
to whom the Software is furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all copies or
|
|
18
|
+
substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
|
21
|
+
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
|
|
22
|
+
PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
|
|
23
|
+
FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
|
|
24
|
+
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
25
|
+
DEALINGS IN THE SOFTWARE.
|
|
26
|
+
|
|
27
|
+
The changes are listed in the header of quality.js.
|
|
28
|
+
|
|
29
|
+
This product includes a JavaScript translation of the resampling dnsmos_local.py ran with on the day
|
|
30
|
+
of that commit, into resampy.js: resampy 0.2.2 (https://github.com/bmcfee/resampy; core.py
|
|
31
|
+
resample(), interpn.py resample_f(), filters.py sinc_window() and the kaiser_best filter's stated
|
|
32
|
+
parameters) and librosa 0.9.1 (https://github.com/librosa/librosa; core/audio.py resample(), its
|
|
33
|
+
length fix). The changes are listed in the header of resampy.js. Both are under the ISC License:
|
|
34
|
+
|
|
35
|
+
ISC License, Copyright (c) 2016, Brian McFee (resampy)
|
|
36
|
+
ISC License, Copyright (c) 2013--2017, librosa development team (librosa)
|
|
37
|
+
|
|
38
|
+
Permission to use, copy, modify, and/or distribute this software for any purpose with or
|
|
39
|
+
without fee is hereby granted, provided that the above copyright notice and this permission
|
|
40
|
+
notice appear in all copies.
|
|
41
|
+
|
|
42
|
+
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS
|
|
43
|
+
SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
|
|
44
|
+
THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY
|
|
45
|
+
DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
|
|
46
|
+
CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE
|
|
47
|
+
OR PERFORMANCE OF THIS SOFTWARE.
|
|
48
|
+
|
|
49
|
+
The model is not included. quality.js reads DNSMOS P.835's network, DNSMOS/DNSMOS/sig_bak_ovr.onnx
|
|
50
|
+
(1,157,965 bytes, SHA-256 269fbebdb513aa23cddfbb593542ecc540284a91849ac50516870e1ac78f6edd), from
|
|
51
|
+
opts.weights, the neural cache, or the same repository at the same commit:
|
|
52
|
+
https://raw.githubusercontent.com/microsoft/DNS-Challenge/82f1b17e7776a43eee395d0f45bae8abb700ad00/DNSMOS/DNSMOS/sig_bak_ovr.onnx
|
|
53
|
+
|
|
54
|
+
"DNSMOS P.835" (sig_bak_ovr.onnx), by Chandan K. A. Reddy, Vishak Gopal and Ross Cutler,
|
|
55
|
+
Copyright (c) Microsoft Corporation, from https://github.com/microsoft/DNS-Challenge, licensed
|
|
56
|
+
under the Creative Commons Attribution 4.0 International Public License
|
|
57
|
+
(https://creativecommons.org/licenses/by/4.0/legalcode), which disclaims all warranties
|
|
58
|
+
(its Section 5). The repository grants it under that licence as "other content" (its README,
|
|
59
|
+
Legal Notices; LICENSE). Used unmodified: this package runs the file as published.
|
|
60
|
+
|
|
61
|
+
DNSMOS P.835 is described in C. K. A. Reddy, V. Gopal, R. Cutler, "DNSMOS P.835: A Non-Intrusive
|
|
62
|
+
Perceptual Objective Speech Quality Metric to Evaluate Noise Suppressors", ICASSP 2022,
|
|
63
|
+
arXiv:2110.01763.
|
package/README.md
CHANGED
|
@@ -1,3 +1,216 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @audio/neural-quality
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
> How listeners would rate a recording of speech, with no clean reference to compare it to: Microsoft's DNSMOS P.835, computed in JS as its own script computes it.
|
|
4
|
+
|
|
5
|
+
DNSMOS P.835 (Reddy, Gopal, Cutler, ICASSP 2022) is a network trained on listeners' ITU-T P.835 ratings of noisy and
|
|
6
|
+
noise-suppressed speech. It gives three numbers on P.835's scales, 1 (bad) to 5 (excellent):
|
|
7
|
+
|
|
8
|
+
- **SIG**, the speech: how natural and undistorted the voice itself sounds.
|
|
9
|
+
- **BAK**, the background: how little and how unobtrusive the noise behind it is.
|
|
10
|
+
- **OVRL**, the whole: the rating a listener gives the recording overall.
|
|
11
|
+
|
|
12
|
+
This package runs Microsoft's network (`sig_bak_ovr.onnx`, microsoft/DNS-Challenge at `82f1b17e77`) in Node and in a
|
|
13
|
+
browser through `@audio/neural-runtime`, with the method of Microsoft's `dnsmos_local.py` at the same commit around it:
|
|
14
|
+
the same resampling, windows, polynomials and averages. On 72 files at 8 to 48 kHz it gives the script's numbers to
|
|
15
|
+
1.8e-15 (Verification).
|
|
16
|
+
|
|
17
|
+
```
|
|
18
|
+
npm install @audio/neural-quality onnxruntime-node # onnxruntime-web in the browser
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
```js
|
|
22
|
+
import dnsmos, { load } from '@audio/neural-quality'
|
|
23
|
+
|
|
24
|
+
let { sig, bak, ovrl } = await dnsmos(samples, { sampleRate: 44100 }) // lena: { sig: 3.58, bak: 3.07, ovrl: 2.76, segments }
|
|
25
|
+
let model = await load() // held across calls
|
|
26
|
+
let before = await dnsmos([left, right], { sampleRate: 48000, model }) // channels mixed to their mean
|
|
27
|
+
let after = await dnsmos(edited, { sampleRate: 48000, model, windows: 'all' })
|
|
28
|
+
model.free()
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
`segments` holds each window the mean is taken over: `{ time, sig, bak, ovrl, raw }`, `time` its start in seconds,
|
|
32
|
+
`raw` the network's three outputs before the polynomials. The model file is fetched once from GitHub at the pinned
|
|
33
|
+
commit (1.1 MB), refused unless of its SHA-256, and in Node kept at `~/.cache/audiojs/neural/dnsmos/sig_bak_ovr.onnx`,
|
|
34
|
+
the path `audio`'s `bench/speech.py` reads it from too; `opts.weights` reads another copy (bytes, a URL, a path in Node).
|
|
35
|
+
|
|
36
|
+
## What it is not
|
|
37
|
+
|
|
38
|
+
- **Not a judge of music or other sound.** It is trained on speech in noise and reads anything else as bad speech:
|
|
39
|
+
piano, an orchestra, chiptune, a jazz combo, rain and birdsong all score OVRL 1.07 to 1.10 here, a 1 kHz tone 1.19,
|
|
40
|
+
digital silence 1.84 (with BAK 3.47).
|
|
41
|
+
- **Deaf above 8 kHz.** It hears 16 kHz audio. Speech low-passed at 4 kHz scores OVRL 3.43 against the full band's
|
|
42
|
+
3.45, and the same speech at 8 kHz 3.44: it barely tells a narrowband recording from a wideband one.
|
|
43
|
+
- **Not a codec meter.** Its training material is noise suppressors' output (the paper's section 3), not codecs; no
|
|
44
|
+
claim is made for MP3 or Opus artefacts, and none was measured here.
|
|
45
|
+
- **Level-dependent on purpose.** The paper avoids "any kind of feature normalization" because "Humans tend to give
|
|
46
|
+
lower ratings to clips with low amplitudes" (p. 3). The same speech 50 dB down scores OVRL 2.03 against 3.45:
|
|
47
|
+
compare an edit at the level it will be heard at, or at the input's loudness, as `audio`'s bench does. A gain of
|
|
48
|
+
6 dB alone moves BAK by 0.3 to 0.45 on noisy speech (four VoiceBank takes end to end: 2.57 at -6 dB, 2.27, 1.82 at
|
|
49
|
+
+6 dB): an edit that changes the level is scored for that too.
|
|
50
|
+
- **Unsteady on a clip shorter than its window.** A clip under 9.01 s is repeated to fill the window, so where it
|
|
51
|
+
ends changes what the network hears: 50 ms of silence added after a 1.6 s take moves SIG by 0.69, and 5 ms fades
|
|
52
|
+
on a 4 s clip by 0.16, though neither is heard. Compare a short edit with its input at the same length.
|
|
53
|
+
- **Rate-dependent at 8 kHz.** The same speech in white noise scores BAK 0.60 higher at 8 kHz than at 16 kHz: half
|
|
54
|
+
the noise band is missing (clean speech moves by 0.013). From 22.05 to 48 kHz it moves by 0.013 at most. Compare
|
|
55
|
+
an edit at the rate of its input.
|
|
56
|
+
- **Shaky on one clip.** The paper's correlation with listeners' P.835 ratings (arXiv:2110.01763v4, p. 4, Table 3, on
|
|
57
|
+
850 unseen clips from 17 noise suppressors): averaged per model, PCC 0.94 SIG, 0.98 BAK, 0.98 OVRL (SRCC 0.95, 0.99,
|
|
58
|
+
0.98); per clip, PCC 0.71, 0.83, 0.82 (SRCC 0.72, 0.82, 0.81), measured on two of the models with 30 ratings a clip.
|
|
59
|
+
It ranks systems over many clips far better than it rates one recording. The weights at `82f1b17e77` came in a
|
|
60
|
+
commit titled "Add support for personalized and improved models" (2022-05-24), after the paper's last version
|
|
61
|
+
(4 February 2022); the repository states no correlation for them.
|
|
62
|
+
- **Not yet checked against this project's own listening.** No verdict of a listener on these recordings has been
|
|
63
|
+
compared with its numbers.
|
|
64
|
+
- **Blind to part of a longer clip, as upstream computes it.** Below.
|
|
65
|
+
|
|
66
|
+
## The windows upstream drops
|
|
67
|
+
|
|
68
|
+
`dnsmos_local.py` cuts window `i` as `audio[int(i*fs) : int((i+INPUT_LENGTH)*fs)]` and skips it when shorter than
|
|
69
|
+
144160 samples (lines 83-85). `(i + 9.01) * 16000` is a float product, and for `i` = 7 to 23 and 119 to 122 (and in
|
|
70
|
+
longer runs past 4.5 hours) it truncates one sample short, so those windows are skipped. A clip from 17 s to under 34 s
|
|
71
|
+
is scored on its first 15.01 s alone, and in any longer clip the stretch from 15.01 to 24 s is never heard. Noise
|
|
72
|
+
added over 16 to 30 s of a 30 s clip leaves its scores unchanged to the last digit (Verification). torchmetrics 1.9.0
|
|
73
|
+
copies the same arithmetic (`functional/audio/dnsmos.py`, lines 254-265).
|
|
74
|
+
|
|
75
|
+
This package drops the same windows by default, so its numbers are upstream's. `windows: 'all'` keeps every 1 s hop:
|
|
76
|
+
on 60 s narrations OVRL moves by up to 0.06; on the 30 s clip above, from 3.43 to 2.79. Use it where what follows
|
|
77
|
+
15 s matters, for an edit there above all; its numbers are then this package's, not upstream's.
|
|
78
|
+
|
|
79
|
+
## How it works
|
|
80
|
+
|
|
81
|
+
As `DNSMOS/dnsmos_local.py` at `82f1b17e77` computes it (`ComputeScore.__call__`, lines 61-106):
|
|
82
|
+
|
|
83
|
+
- **Resampling** (lines 63-67): to 16 kHz by `librosa.resample(aud, input_fs, fs)`. That positional call runs on
|
|
84
|
+
librosa 0.9 and older (0.10 and later refuse it: `resample() takes 1 positional argument but 3 were given`), whose
|
|
85
|
+
default is resampy's `kaiser_best`; then librosa cuts or pads it to ceil(n · 16000 / rate). On the day of the
|
|
86
|
+
commit (2022-05-24) the releases were librosa 0.9.1 and resampy 0.2.2, whose `kaiser_best` is a Kaiser-windowed
|
|
87
|
+
sinc of 64 zero crossings with its rolloff at 0.948 of Nyquist. `resampy.js` translates both and gives their 16 kHz
|
|
88
|
+
signal sample for sample, as float32, on all 41 files at 8, 22.05, 44.1 and 48 kHz below. resampy 0.3.0
|
|
89
|
+
(2022-06-29) replaced the filter (50 zero crossings, rolloff 0.917), so the script run today on librosa 0.9.2 gives
|
|
90
|
+
other numbers at those rates: up to 0.056 BAK (Verification).
|
|
91
|
+
- **Channels**: the script passes a multichannel file's 2-D array on, and at a rate other than 16 kHz librosa
|
|
92
|
+
resamples it along its last axis, the channels. Above 32 kHz resampy refuses it (`Input signal length=2 is too
|
|
93
|
+
small`); at 32 kHz and below the network refuses a file of 144,160 frames or more (`Invalid rank for input: input_1
|
|
94
|
+
Got: 3 Expected: 2`), and a shorter one `np.append` flattens into one signal of its interleaved values, which the
|
|
95
|
+
script scores without an error (a narration in two channels at 8 to 24 kHz: OVRL 1.56 to 1.84, against 3.18 to 3.36
|
|
96
|
+
for the channels' mean). Here channels are mixed to their mean, as `audio`'s bench and this family's other packages
|
|
97
|
+
do; no path of the script gives that mean.
|
|
98
|
+
- **Short clips** (lines 70-71): doubled (`np.append(audio, audio)`) until they hold a window; 0.1 s becomes 12.8 s of
|
|
99
|
+
the same 0.1 s, scored over 3 windows (OVRL 1.65; 2 s of the same speech, 3.09).
|
|
100
|
+
- **Windows** (lines 73-85): 9.01 s (144160 samples) every 1 s, `int(floor(len / 16000) − 9.01) + 1` of them, less
|
|
101
|
+
those the float product drops (above).
|
|
102
|
+
- **The network**: the waveform in (`input_1`, [N, 144160] float32), the spectrogram computed inside (a 320-sample
|
|
103
|
+
DFT as two convolutions, 161 bins, log power), then 7 convolutions and 3 dense layers: three raw outputs.
|
|
104
|
+
- **Polynomials** (`get_polyfit_val`, lines 51-53, non-personalized), on each window's raw outputs s, b, o:
|
|
105
|
+
SIG = −0.08397278 s² + 1.22083953 s + 0.0052439, BAK = −0.13166888 b² + 1.60915514 b − 0.39604546,
|
|
106
|
+
OVRL = −0.06766283 o² + 1.11546468 o + 0.04602535; the scores are their means over the windows (lines 103-105).
|
|
107
|
+
|
|
108
|
+
Refused, where the script would not give a number: an empty input (its doubling loop never ends), a sample that is NaN
|
|
109
|
+
or infinite. The personalized variant (`pDNSMOS`) and P.808 (`model_v8.onnx`, not in the repository at this commit)
|
|
110
|
+
are left out.
|
|
111
|
+
|
|
112
|
+
## Verification
|
|
113
|
+
|
|
114
|
+
`node test.js`, against `fixtures/reference.json`, which `scripts/reference.py` writes from Microsoft's own script
|
|
115
|
+
(imported and called as its `main()` calls it; librosa 0.9.1, resampy 0.2.2, onnxruntime 1.30.0, numpy 1.26.4) on the
|
|
116
|
+
waveforms `scripts/inputs.mjs` makes, their SHA-256 checked first: the windows of ten ramps whose sample values name
|
|
117
|
+
their positions, `get_polyfit_val` on six inputs, 0.5 s of librosa's 16 kHz signal for lena at 44.1 and 8 kHz, every
|
|
118
|
+
window's raw outputs and the scores of six inputs. Every check is exact on this machine (raw outputs to the last bit).
|
|
119
|
+
|
|
120
|
+
Measured once, by scripts outside the package, on 73 files: VoiceBank+DEMAND clean and noisy takes (48 kHz), Spoken
|
|
121
|
+
Wikipedia narrations (60 s, 48 kHz), recordings of 1860 and 1890, speech, music, rain and birdsong from
|
|
122
|
+
audiojs.github.io/samples (22.05 to 48 kHz), lena; at 16 kHz the same speech in white and park noise at 20 to −5 dB SNR,
|
|
123
|
+
low-passed, telephone-band, 0.1 to 99 s long, 50 dB down, clipped, silence, a tone, white noise; and five recordings
|
|
124
|
+
each at 8, 16, 22.05, 44.1 and 48 kHz. Microsoft's script against this package, the model the same file, onnxruntime
|
|
125
|
+
1.30.0 on both sides:
|
|
126
|
+
|
|
127
|
+
| Files | SIG, BAK, OVRL: largest difference | Raw outputs |
|
|
128
|
+
|---|---|---|
|
|
129
|
+
| 31 at 16 kHz | 1.8e-15 | equal |
|
|
130
|
+
| 41 at 8, 22.05, 44.1, 48 kHz | 8.9e-16 | equal |
|
|
131
|
+
| 1 with two channels | the script refuses it; here the mean | |
|
|
132
|
+
|
|
133
|
+
What the other resamplers give, on the same 41 files, against the script as it ran in May 2022:
|
|
134
|
+
|
|
135
|
+
| Resampler | Largest difference | Over 0.01 |
|
|
136
|
+
|---|---|---|
|
|
137
|
+
| resampy 0.3 and later (`kaiser_best` redesigned; the script today on librosa 0.9.2) | 0.056 BAK, 0.044 OVRL | 6 of 41 |
|
|
138
|
+
| librosa 0.10's default `soxr_hq` (the script with its call made keyword, as torchmetrics 1.9.0 writes it) | 0.022 BAK | 3 of 41 |
|
|
139
|
+
| `@audio/resample-sinc` (this family's) | 0.137 BAK, 0.083 OVRL | 7 of 41 |
|
|
140
|
+
|
|
141
|
+
Each largest difference is speech in white noise 5 dB down, at 22.05 or 44.1 kHz. There the resamplers differ in how
|
|
142
|
+
much of the noise between 7.6 and 8 kHz they keep: after resample-sinc, which passes the band to 8 kHz, it stands 12
|
|
143
|
+
to 32 dB higher than after resampy 0.2.2's filter (measured on the 48 kHz copy).
|
|
144
|
+
|
|
145
|
+
## Speed
|
|
146
|
+
|
|
147
|
+
60 s of a narration at 48 kHz, onnxruntime-node 1.30, Node 25, an Apple M4 Max (14 cores) running other jobs (load
|
|
148
|
+
averages 155 to 184, so the wall times are long), the session made first, three runs each:
|
|
149
|
+
|
|
150
|
+
| Session | Windows | CPU per minute of audio | Wall |
|
|
151
|
+
|---|---|---|---|
|
|
152
|
+
| one thread (`sessionOptions: { intraOpNumThreads: 1 }`) | 34 | 5.2 to 5.4 s | 21 to 22 s |
|
|
153
|
+
| one thread, `windows: 'all'` | 51 | 7.7 to 7.8 s | 22 to 28 s |
|
|
154
|
+
| onnxruntime's default | 34 | 13.7 to 14.3 s | 6.3 to 6.7 s |
|
|
155
|
+
|
|
156
|
+
Of that, resampling to 16 kHz takes 0.5 s from 48 kHz, 0.6 s from 44.1 kHz; the rest is the network, 0.15 s of CPU a
|
|
157
|
+
window on one thread. Each second of audio is in nine windows, so the cost grows with the length, about one window a
|
|
158
|
+
second. The Python script took 12.3 to 13.5 s of CPU per minute under the same load (onnxruntime's default). Memory:
|
|
159
|
+
36 MB for the session; while scoring, a peak 255 MB over Node's own (arena on), 199 MB with `sessionOptions:
|
|
160
|
+
{ enableCpuMemArena: false, enableMemPattern: false }`. Untested in a browser.
|
|
161
|
+
|
|
162
|
+
## The model
|
|
163
|
+
|
|
164
|
+
| File | Size | SHA-256 | Source |
|
|
165
|
+
|---|---|---|---|
|
|
166
|
+
| `sig_bak_ovr.onnx` | 1,157,965 bytes | `269fbebdb513aa23cddfbb593542ecc540284a91849ac50516870e1ac78f6edd` | [microsoft/DNS-Challenge](https://github.com/microsoft/DNS-Challenge/tree/82f1b17e7776a43eee395d0f45bae8abb700ad00/DNSMOS) `DNSMOS/DNSMOS/` at `82f1b17e7776a43eee395d0f45bae8abb700ad00` |
|
|
167
|
+
|
|
168
|
+
keras2onnx 1.8.1, opset 12, graph `SIG_BAK_OVR_mse_cosine`: 184,227 trained values and 103,040 in the DFT kernels.
|
|
169
|
+
Fetched, never bundled; used as published.
|
|
170
|
+
|
|
171
|
+
## API
|
|
172
|
+
|
|
173
|
+
| Export | |
|
|
174
|
+
|---|---|
|
|
175
|
+
| `dnsmos(audio, opts)` (default) | → `Promise<{ sig, bak, ovrl, segments }>` |
|
|
176
|
+
| `load(opts?)` | → `Promise<{ session, free() }>`: the network, for `opts.model` |
|
|
177
|
+
| `polyfit({ sig, bak, ovrl })` | the polynomials alone: raw outputs to the P.835 scale |
|
|
178
|
+
| `windows(n, { windows }?)` | → `{ length, starts }`: the doubled length of a 16 kHz signal of `n` samples and each window's first sample |
|
|
179
|
+
| `prepare(audio, { sampleRate })` | → `Float32Array`: the channels' mean at 16 kHz, as librosa 0.9 resamples it |
|
|
180
|
+
| `SAMPLE_RATE`, `INPUT_LENGTH`, `WINDOW` | 16000, 9.01, 144160 |
|
|
181
|
+
| `MODEL`, `REVISION`, `SHA256` | the pinned file's URL, its commit, its SHA-256 |
|
|
182
|
+
|
|
183
|
+
`audio`: a `Float32Array` (mono), an array of channels, or `{ channelData, sampleRate }`. `opts`: `{ sampleRate,
|
|
184
|
+
model, windows = 'upstream', weights, backend, sessionOptions, progress, fetch }`; without `model`, the network is
|
|
185
|
+
loaded and freed within the call. `@audio/neural-quality/audio` is the `audio` stat manifest: `{ stat: 'dnsmos',
|
|
186
|
+
compute(channels, { sampleRate, ...opts }) }`, returning the same Promise. The stat scores every window (`windows: 'all'`), so a clip of 17 s or more is heard whole; pass `windows: 'upstream'` for the reference script's numbers.
|
|
187
|
+
|
|
188
|
+
## Licences
|
|
189
|
+
|
|
190
|
+
Checked 2026-11-18 against what each project states; a summary, not legal advice.
|
|
191
|
+
|
|
192
|
+
| Part | Licence | Source |
|
|
193
|
+
|---|---|---|
|
|
194
|
+
| `dnsmos_local.py`'s method, translated in `quality.js` | MIT, Copyright (c) Microsoft Corporation | the repository's README, Legal Notices: "grant you a license to any code in the repository under the MIT License, see the LICENSE-CODE file" |
|
|
195
|
+
| `sig_bak_ovr.onnx` | CC BY 4.0 | the same notice: "a license to the Microsoft documentation and other content in this repository under the Creative Commons Attribution 4.0 International Public License, see the LICENSE file". Attribution as its Section 3(a) asks is in [NOTICE](./NOTICE) |
|
|
196
|
+
| resampy 0.2.2's `kaiser_best` and librosa 0.9.1's `resample`, translated in `resampy.js` | ISC, Copyright (c) 2016 Brian McFee; ISC, Copyright (c) 2013--2017 the librosa development team | their LICENSE files |
|
|
197
|
+
|
|
198
|
+
The package is MIT; [NOTICE](./NOTICE) carries the attributions.
|
|
199
|
+
|
|
200
|
+
## Reference
|
|
201
|
+
|
|
202
|
+
C. K. A. Reddy, V. Gopal, R. Cutler, "DNSMOS P.835: A Non-Intrusive Perceptual Objective Speech Quality Metric to
|
|
203
|
+
Evaluate Noise Suppressors", ICASSP 2022, [arXiv:2110.01763](https://arxiv.org/abs/2110.01763). · ITU-T P.835 (2003),
|
|
204
|
+
Subjective test methodology for evaluating speech communication systems that include noise suppression algorithm. · [microsoft/DNS-Challenge](https://github.com/microsoft/DNS-Challenge/tree/82f1b17e7776a43eee395d0f45bae8abb700ad00/DNSMOS)
|
|
205
|
+
at `82f1b17e77`. · B. McFee, [resampy](https://github.com/bmcfee/resampy) 0.2.2. · [librosa](https://github.com/librosa/librosa) 0.9.1.
|
|
206
|
+
|
|
207
|
+
**Use when:** asking whether an edit made a voice cleaner, its background quieter, the whole better, on speech, in
|
|
208
|
+
Node or a browser, from a 1.1 MB model, the numbers Microsoft's script gives.<br>
|
|
209
|
+
**Not for:** music, band-limited or codec damage, a verdict on one recording without a listener's, or what lies past
|
|
210
|
+
15 s in a clip of 17 to 34 s (without `windows: 'all'`).
|
|
211
|
+
|
|
212
|
+
---
|
|
213
|
+
|
|
214
|
+
Part of the [@audio/neural](https://github.com/audiojs/neural) lane.
|
|
215
|
+
|
|
216
|
+
MIT © 2026 audiojs; DNSMOS P.835 © Microsoft Corporation, CC BY 4.0
|
package/audio.d.ts
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
// Generated from the audio.js manifest (params metadata is the source of truth).
|
|
2
|
+
// Regenerate: node tools/dts.js in @audio/compile. Do not edit by hand.
|
|
3
|
+
|
|
4
|
+
/** Automatable number — scalar, `t => value` fn, or breakpoint curve {t, v} */
|
|
5
|
+
type Auto = number | ((t: number) => number) | { t: number[], v: number[] }
|
|
6
|
+
/** Per-block param values as delivered by hosts (numbers arrive as 1-length Float32Array) */
|
|
7
|
+
type Live = Record<string, Float32Array | string | boolean>
|
|
8
|
+
type Ctx = { sampleRate: number, maxBlockSize: number, maxChannels: number, currentTime: number, duration?: number, events?: readonly any[], emit?: (name: string, ...args: any[]) => void, [k: string]: unknown }
|
|
9
|
+
type Process = (inputs: Float32Array[][], outputs: Float32Array[][], params: Live) => void
|
|
10
|
+
|
|
11
|
+
/** Stat plugin 'dnsmos' — whole-signal analysis, registers as a.stat('dnsmos') */
|
|
12
|
+
export declare const dnsmos: {
|
|
13
|
+
stat: 'dnsmos'
|
|
14
|
+
compute(channels: Float32Array[], opts: { sampleRate: number, [k: string]: unknown }): unknown
|
|
15
|
+
}
|
package/audio.js
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
// stat manifest: DNSMOS P.835 of a whole signal, { sig, bak, ovrl, segments } (quality.js). The host reads the PCM and
|
|
2
|
+
// calls compute once, the (channels, { sampleRate }) shape dnsmos() takes; compute returns a Promise, which the host's
|
|
3
|
+
// stat method awaits. Options pass through: model (a load() handle), windows, weights, backend. As a stat it hears the
|
|
4
|
+
// whole clip (windows 'all'): the reference script skips windows 7 to 23, so it never hears 15 to 24 s of a clip of
|
|
5
|
+
// 17 s or more, and a verdict on an edit must not rest on a part of it; `windows: 'upstream'` gives the script's numbers.
|
|
6
|
+
|
|
7
|
+
import score from './quality.js'
|
|
8
|
+
|
|
9
|
+
export const dnsmos = {
|
|
10
|
+
stat: 'dnsmos',
|
|
11
|
+
compute: (channels, { sampleRate, ...opts }) => score(channels, { sampleRate, windows: 'all', ...opts }),
|
|
12
|
+
}
|
package/index.d.ts
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Speech quality with no clean reference: Microsoft's DNSMOS P.835 (microsoft/DNS-Challenge at 82f1b17e77,
|
|
3
|
+
* sig_bak_ovr.onnx, CC BY 4.0) through @audio/neural-runtime, as its dnsmos_local.py computes it.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
/** a mono Float32Array, one array per channel (mixed down to their mean), or a decoded sound */
|
|
7
|
+
export type Audio = Float32Array | ArrayLike<number> | Float32Array[] | { channelData: Float32Array[], sampleRate: number }
|
|
8
|
+
|
|
9
|
+
export interface LoadOptions {
|
|
10
|
+
/** the model: bytes, a URL, or in Node a file or directory path holding sig_bak_ovr.onnx; default the neural cache's copy when of SHA256, else MODEL */
|
|
11
|
+
weights?: string | URL | Uint8Array | ArrayBuffer
|
|
12
|
+
/** @audio/neural-runtime's backend: 'auto' (default), 'node', 'wasm', 'webgpu', 'coreml' */
|
|
13
|
+
backend?: string
|
|
14
|
+
/** ONNX Runtime session options, e.g. { intraOpNumThreads: 1 } */
|
|
15
|
+
sessionOptions?: Record<string, unknown>
|
|
16
|
+
progress?: (p: { loaded: number, total: number | null }) => void
|
|
17
|
+
fetch?: typeof fetch
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export interface Model {
|
|
21
|
+
session: { inputs: { name: string }[], outputs: { name: string }[], run: (feeds: Record<string, unknown>) => Promise<Record<string, { data: Float32Array }>> }
|
|
22
|
+
free(): void
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface Options extends LoadOptions {
|
|
26
|
+
/** required unless `audio` carries it */
|
|
27
|
+
sampleRate?: number
|
|
28
|
+
/** a load() handle, kept by the caller across calls; without it the model is loaded and freed per call */
|
|
29
|
+
model?: Model
|
|
30
|
+
/** 'upstream' (default): the windows dnsmos_local.py keeps, which leaves 15.01 to 24 s of a longer clip unscored; 'all': every 1 s hop */
|
|
31
|
+
windows?: 'upstream' | 'all'
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** The three P.835 scales, 1 (bad) to 5 (excellent) */
|
|
35
|
+
export interface Scores {
|
|
36
|
+
/** the speech: how clean and undistorted the voice sounds */
|
|
37
|
+
sig: number
|
|
38
|
+
/** the background: how little and how unobtrusive the noise is */
|
|
39
|
+
bak: number
|
|
40
|
+
/** the whole: the quality a listener gives the recording overall */
|
|
41
|
+
ovrl: number
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export interface Segment extends Scores {
|
|
45
|
+
/** the window's start in seconds; in the doubled signal for a clip under 9.01 s */
|
|
46
|
+
time: number
|
|
47
|
+
/** the network's own outputs, before the polynomials: [sig, bak, ovrl] */
|
|
48
|
+
raw: [number, number, number]
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** SIG, BAK, OVRL: each the mean over the windows of the polynomial of the network's output */
|
|
52
|
+
export default function dnsmos(audio: Audio, opts?: Options): Promise<Scores & { segments: Segment[] }>
|
|
53
|
+
export function dnsmos(audio: Audio, opts?: Options): Promise<Scores & { segments: Segment[] }>
|
|
54
|
+
|
|
55
|
+
/** The network, loaded once, for dnsmos() to reuse through opts.model */
|
|
56
|
+
export function load(opts?: LoadOptions): Promise<Model>
|
|
57
|
+
|
|
58
|
+
/** get_polyfit_val's polynomials: the network's raw outputs to the P.835 scale */
|
|
59
|
+
export function polyfit(raw: Scores): Scores
|
|
60
|
+
|
|
61
|
+
/** The windows of a 16 kHz signal of n samples: the doubled length, and each window's first sample */
|
|
62
|
+
export function windows(n: number, opts?: { windows?: 'upstream' | 'all' }): { length: number, starts: number[] }
|
|
63
|
+
|
|
64
|
+
/** What the windows are cut from: the channels' mean at 16 kHz, librosa's length, ceil(n · 16000 / rate) */
|
|
65
|
+
export function prepare(audio: Audio, opts?: { sampleRate?: number }): Float32Array
|
|
66
|
+
|
|
67
|
+
/** 16000, 9.01 s, 144160 samples */
|
|
68
|
+
export const SAMPLE_RATE: 16000
|
|
69
|
+
export const INPUT_LENGTH: 9.01
|
|
70
|
+
export const WINDOW: 144160
|
|
71
|
+
|
|
72
|
+
/** microsoft/DNS-Challenge's commit, sig_bak_ovr.onnx's SHA-256 there, and its URL at that commit */
|
|
73
|
+
export const REVISION: string
|
|
74
|
+
export const SHA256: string
|
|
75
|
+
export const MODEL: URL
|
package/package.json
CHANGED
|
@@ -1,6 +1,73 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@audio/neural-quality",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Speech quality with no clean reference: Microsoft's DNSMOS P.835 (SIG, BAK, OVRL, 1 to 5) as its dnsmos_local.py computes it, the network through @audio/neural-runtime",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"sideEffects": false,
|
|
7
|
+
"main": "quality.js",
|
|
8
|
+
"types": "./index.d.ts",
|
|
9
|
+
"exports": {
|
|
10
|
+
".": {
|
|
11
|
+
"types": "./index.d.ts",
|
|
12
|
+
"default": "./quality.js"
|
|
13
|
+
},
|
|
14
|
+
"./audio": {
|
|
15
|
+
"types": "./audio.d.ts",
|
|
16
|
+
"default": "./audio.js"
|
|
17
|
+
},
|
|
18
|
+
"./package.json": "./package.json"
|
|
19
|
+
},
|
|
20
|
+
"files": [
|
|
21
|
+
"quality.js",
|
|
22
|
+
"resampy.js",
|
|
23
|
+
"audio.js",
|
|
24
|
+
"index.d.ts",
|
|
25
|
+
"audio.d.ts",
|
|
26
|
+
"scripts/*.mjs",
|
|
27
|
+
"scripts/*.py",
|
|
28
|
+
"LICENSE",
|
|
29
|
+
"NOTICE"
|
|
30
|
+
],
|
|
31
|
+
"dependencies": {
|
|
32
|
+
"@audio/neural-runtime": "^0.2.0"
|
|
33
|
+
},
|
|
34
|
+
"devDependencies": {
|
|
35
|
+
"audio-lena": "^3.0.1",
|
|
36
|
+
"onnxruntime-node": "^1.29.0",
|
|
37
|
+
"tst": "^9.4.0"
|
|
38
|
+
},
|
|
39
|
+
"scripts": {
|
|
40
|
+
"test": "node test.js",
|
|
41
|
+
"prepublishOnly": "node ../../../compile/tools/tested.js"
|
|
42
|
+
},
|
|
43
|
+
"keywords": [
|
|
44
|
+
"audio",
|
|
45
|
+
"ml",
|
|
46
|
+
"neural",
|
|
47
|
+
"speech-quality",
|
|
48
|
+
"mos",
|
|
49
|
+
"dnsmos",
|
|
50
|
+
"p835",
|
|
51
|
+
"non-intrusive",
|
|
52
|
+
"noise-suppression",
|
|
53
|
+
"onnx"
|
|
54
|
+
],
|
|
55
|
+
"license": "MIT",
|
|
56
|
+
"author": "Dmitry Iv. <dfcreative@gmail.com>",
|
|
57
|
+
"contributors": [
|
|
58
|
+
"Chandan K. A. Reddy, Vishak Gopal, Ross Cutler and Microsoft Corporation (DNSMOS P.835)"
|
|
59
|
+
],
|
|
60
|
+
"audio": "./audio.js",
|
|
61
|
+
"engines": {
|
|
62
|
+
"node": ">=18"
|
|
63
|
+
},
|
|
64
|
+
"repository": {
|
|
65
|
+
"type": "git",
|
|
66
|
+
"url": "git+https://github.com/audiojs/neural.git",
|
|
67
|
+
"directory": "packages/neural-quality"
|
|
68
|
+
},
|
|
69
|
+
"funding": "https://github.com/sponsors/audiojs",
|
|
70
|
+
"publishConfig": {
|
|
71
|
+
"access": "public"
|
|
72
|
+
}
|
|
73
|
+
}
|
package/quality.js
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
// How listeners would rate a recording of speech, with no clean reference: DNSMOS P.835 (C. K. A. Reddy, V. Gopal,
|
|
2
|
+
// R. Cutler, "DNSMOS P.835: A Non-Intrusive Perceptual Objective Speech Quality Metric to Evaluate Noise Suppressors",
|
|
3
|
+
// ICASSP 2022), ITU-T P.835's three scales from 1 to 5: the speech (SIG), the background (BAK), the whole (OVRL).
|
|
4
|
+
//
|
|
5
|
+
// Microsoft's network (DNSMOS/DNSMOS/sig_bak_ovr.onnx, CC BY 4.0) runs through @audio/neural-runtime. Around it, the
|
|
6
|
+
// method of DNSMOS/dnsmos_local.py (MIT License, Copyright (c) Microsoft Corporation), both at microsoft/DNS-Challenge
|
|
7
|
+
// commit 82f1b17e7776a43eee395d0f45bae8abb700ad00, translated to JavaScript: ComputeScore.__call__ (lines 61-106) and
|
|
8
|
+
// get_polyfit_val's non-personalized branch (lines 50-59). The network takes the waveform (input_1, [N, 144160]
|
|
9
|
+
// float32) and computes its spectrogram itself, so nothing of it is computed here.
|
|
10
|
+
//
|
|
11
|
+
// Changes, 2026, audiojs: librosa.resample translated as the script ran it on the day of its commit (resampy.js:
|
|
12
|
+
// librosa 0.9.1, resampy 0.2.2's kaiser_best; librosa 0.10 and later refuse its positional call). Channels mixed to
|
|
13
|
+
// their mean (the script passes soundfile's 2-D array of a multichannel file on, and away from 16 kHz librosa resamples
|
|
14
|
+
// it along the channels: above 32 kHz resampy refuses it; at 32 kHz and below the network refuses one of 144,160
|
|
15
|
+
// frames or more, and a shorter one np.append flattens into its interleaved values, which the script scores without an
|
|
16
|
+
// error). An empty input refused (the script's doubling loop never ends on it), and a sample that is NaN or infinite.
|
|
17
|
+
// opts.windows: 'all' scores the windows the script's float arithmetic drops (below). The personalized variant
|
|
18
|
+
// (pDNSMOS) is left out.
|
|
19
|
+
|
|
20
|
+
import { load as loadModel, tensor, fetchModel } from '@audio/neural-runtime'
|
|
21
|
+
import resampy from './resampy.js'
|
|
22
|
+
|
|
23
|
+
/** The network's rate and window: SAMPLING_RATE and INPUT_LENGTH, dnsmos_local.py lines 19-20 */
|
|
24
|
+
export const SAMPLE_RATE = 16000, INPUT_LENGTH = 9.01
|
|
25
|
+
/** A window in samples: int(INPUT_LENGTH * fs), line 69 */
|
|
26
|
+
export const WINDOW = Math.trunc(INPUT_LENGTH * SAMPLE_RATE)
|
|
27
|
+
|
|
28
|
+
// get_polyfit_val, non-personalized, lines 51-53: np.poly1d coefficients, highest power first
|
|
29
|
+
const P_SIG = [-0.08397278, 1.22083953, 0.0052439]
|
|
30
|
+
const P_BAK = [-0.13166888, 1.60915514, -0.39604546]
|
|
31
|
+
const P_OVR = [-0.06766283, 1.11546468, 0.04602535]
|
|
32
|
+
|
|
33
|
+
// The pinned file: its commit, its SHA-256 and size as fetched from there (1,157,965 bytes); in Node kept at the neural
|
|
34
|
+
// cache's dnsmos/sig_bak_ovr.onnx, the path audio's bench/speech.py reads it from too
|
|
35
|
+
export const REVISION = '82f1b17e7776a43eee395d0f45bae8abb700ad00', SHA256 = '269fbebdb513aa23cddfbb593542ecc540284a91849ac50516870e1ac78f6edd'
|
|
36
|
+
const FILE = 'sig_bak_ovr.onnx'
|
|
37
|
+
export const MODEL = new URL(`https://raw.githubusercontent.com/microsoft/DNS-Challenge/${REVISION}/DNSMOS/DNSMOS/${FILE}`)
|
|
38
|
+
|
|
39
|
+
const isNode = globalThis.process?.release?.name === 'node' // Node, Bun, Deno: not a process polyfill (esm.sh's, a bundler's)
|
|
40
|
+
const isUrl = s => /^[a-z][a-z0-9+.-]+:/i.test(s)
|
|
41
|
+
const horner = (p, x) => (p[0] * x + p[1]) * x + p[2]
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* The polynomials that map the network's raw outputs to the P.835 scale (get_polyfit_val, non-personalized):
|
|
45
|
+
* { sig, bak, ovrl } in, { sig, bak, ovrl } out. numpy evaluates them in float64 on the float32 outputs, as here.
|
|
46
|
+
*/
|
|
47
|
+
export function polyfit({ sig, bak, ovrl }) {
|
|
48
|
+
return { sig: horner(P_SIG, sig), bak: horner(P_BAK, bak), ovrl: horner(P_OVR, ovrl) }
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* The windows of a 16 kHz signal of `n` samples (n > 0), as ComputeScore lays them: the signal doubled until it holds a
|
|
53
|
+
* window (np.append(audio, audio), lines 70-71); int(floor(length / fs) − 9.01) + 1 hops of 1 s (line 73); each window
|
|
54
|
+
* audio[int(i · fs) : int((i + 9.01) · fs)], skipped when shorter than WINDOW (lines 83-85). That bound is a float
|
|
55
|
+
* product: for i = 7 to 23, 119 to 122 and longer runs past 4.5 hours it truncates a sample short, so the script drops
|
|
56
|
+
* those windows, and of a clip from 17 s to under 34 s it scores the first 15.01 s alone. opts.windows: 'upstream'
|
|
57
|
+
* (default) drops them as the script does; 'all' keeps every hop.
|
|
58
|
+
* → { length: the doubled length, starts: each window's first sample }
|
|
59
|
+
*/
|
|
60
|
+
export function windows(n, { windows: mode = 'upstream' } = {}) {
|
|
61
|
+
if (!(n > 0)) throw new RangeError(`neural-quality: a signal of ${n} samples has no windows`)
|
|
62
|
+
if (mode !== 'upstream' && mode !== 'all') throw new TypeError(`neural-quality: opts.windows is 'upstream' or 'all', not ${mode}`)
|
|
63
|
+
let length = n
|
|
64
|
+
while (length < WINDOW) length *= 2
|
|
65
|
+
let hops = Math.trunc(Math.floor(length / SAMPLE_RATE) - INPUT_LENGTH) + 1, starts = []
|
|
66
|
+
for (let i = 0; i < hops; i++) {
|
|
67
|
+
let a = Math.trunc(i * SAMPLE_RATE), b = Math.min(length, Math.trunc((i + INPUT_LENGTH) * SAMPLE_RATE))
|
|
68
|
+
if (mode === 'upstream' && b - a < WINDOW) continue
|
|
69
|
+
starts.push(a)
|
|
70
|
+
}
|
|
71
|
+
return { length, starts }
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* What the windows are cut from: `audio` (a Float32Array, an array of channels, or { channelData, sampleRate }) mixed
|
|
76
|
+
* to the mean of its channels and resampled to 16 kHz as librosa 0.9.1 and resampy 0.2.2 resample it: Float32Array,
|
|
77
|
+
* the float32 the script casts each window to.
|
|
78
|
+
*/
|
|
79
|
+
export function prepare(audio, opts = {}) {
|
|
80
|
+
let channels, rate = opts.sampleRate
|
|
81
|
+
if (ArrayBuffer.isView(audio)) channels = [audio]
|
|
82
|
+
else if (Array.isArray(audio)) channels = typeof audio[0] === 'number' ? [audio] : audio
|
|
83
|
+
else if (audio?.channelData) { channels = audio.channelData; rate = audio.sampleRate ?? rate }
|
|
84
|
+
else throw new TypeError('neural-quality: audio must be a Float32Array, Float32Array[] or { channelData, sampleRate }')
|
|
85
|
+
if (!(rate > 0 && rate < Infinity)) throw new TypeError(`neural-quality: sampleRate is required, a positive number (opts.sampleRate, or audio.sampleRate), got ${rate}`)
|
|
86
|
+
if (!channels.length) throw new TypeError('neural-quality: audio has no channels')
|
|
87
|
+
let n = channels[0].length, x = new Float32Array(n)
|
|
88
|
+
if (channels.some(c => c.length !== n)) throw new RangeError('neural-quality: channels of different lengths')
|
|
89
|
+
if (!n) throw new RangeError('neural-quality: no samples to score')
|
|
90
|
+
for (let c of channels) for (let i = 0; i < n; i++) x[i] += channels.length > 1 ? c[i] / channels.length : c[i]
|
|
91
|
+
for (let i = 0; i < n; i++) if (!Number.isFinite(x[i])) throw new RangeError(`neural-quality: sample ${i} is ${x[i]}: refused, the network would score it NaN`)
|
|
92
|
+
return rate === SAMPLE_RATE ? x : Float32Array.from(resampy(x, rate, SAMPLE_RATE))
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// The model's bytes: opts.weights (bytes, a URL or, in Node, a file or directory path holding sig_bak_ovr.onnx), else
|
|
96
|
+
// in Node the neural cache's copy when of SHA256, else the pinned file, refused unless of SHA256
|
|
97
|
+
async function bytesOf(opts) {
|
|
98
|
+
let w = opts.weights
|
|
99
|
+
if (w instanceof Uint8Array || w instanceof ArrayBuffer) return new Uint8Array(w)
|
|
100
|
+
if (w != null) {
|
|
101
|
+
let href = String(w)
|
|
102
|
+
if (!isUrl(href)) {
|
|
103
|
+
if (!isNode) throw new Error(`neural-quality: opts.weights must be a URL in the browser, got '${href}'`)
|
|
104
|
+
href = (await import('node:url')).pathToFileURL(href).href
|
|
105
|
+
}
|
|
106
|
+
if (!href.endsWith('.onnx')) href = (href.endsWith('/') ? href : href + '/') + FILE
|
|
107
|
+
return fetchModel(href, { progress: opts.progress, fetch: opts.fetch })
|
|
108
|
+
}
|
|
109
|
+
let store = isNode ? await cached() : null, bytes = store?.read()
|
|
110
|
+
if (bytes && await sha256(bytes) === SHA256) return bytes
|
|
111
|
+
try { bytes = await fetchModel(MODEL.href, { progress: opts.progress, fetch: opts.fetch, cache: !store }) }
|
|
112
|
+
catch (e) { throw new Error(`neural-quality: ${FILE} not read (${e.message}): opts.weights takes a URL${isNode ? ', a path' : ''} or the bytes of a copy`) }
|
|
113
|
+
let hex = await sha256(bytes)
|
|
114
|
+
if (hex !== SHA256) throw new Error(`neural-quality: ${MODEL} has SHA-256 ${hex}, expected ${SHA256}: a partial or altered download`)
|
|
115
|
+
store?.write(bytes)
|
|
116
|
+
return bytes
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const subtle = async () => globalThis.crypto?.subtle ?? (await import('node:crypto')).webcrypto.subtle
|
|
120
|
+
const sha256 = async bytes => Array.from(new Uint8Array(await (await subtle()).digest('SHA-256', bytes)), b => b.toString(16).padStart(2, '0')).join('')
|
|
121
|
+
|
|
122
|
+
// <neural cache>/dnsmos/sig_bak_ovr.onnx: { dest, read() → bytes or null, write(bytes) whole or not at all }
|
|
123
|
+
async function cached() {
|
|
124
|
+
let [fs, path, os] = await Promise.all([import('node:fs'), import('node:path'), import('node:os')])
|
|
125
|
+
let dest = path.join(process.env.AUDIO_NEURAL_CACHE || path.join(os.homedir(), '.cache', 'audiojs', 'neural'), 'dnsmos', FILE)
|
|
126
|
+
return {
|
|
127
|
+
dest,
|
|
128
|
+
read() { try { let b = fs.readFileSync(dest); return new Uint8Array(b.buffer, b.byteOffset, b.length) } catch { return null } },
|
|
129
|
+
write(bytes) {
|
|
130
|
+
let part = `${dest}.${process.pid}.${Math.random().toString(36).slice(2)}.part`
|
|
131
|
+
fs.mkdirSync(path.dirname(dest), { recursive: true })
|
|
132
|
+
fs.writeFileSync(part, bytes)
|
|
133
|
+
fs.renameSync(part, dest)
|
|
134
|
+
},
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* The network, loaded once, for dnsmos() to reuse through opts.model: { session, free() }.
|
|
140
|
+
* opts: { weights, backend, sessionOptions, progress, fetch }.
|
|
141
|
+
*/
|
|
142
|
+
export async function load(opts = {}) {
|
|
143
|
+
let session = await loadModel(await bytesOf(opts), { backend: opts.backend, sessionOptions: opts.sessionOptions })
|
|
144
|
+
return { session, free: () => session.free() }
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* SIG, BAK and OVRL of `audio` (a Float32Array, an array of channels or { channelData, sampleRate }), each the mean over
|
|
149
|
+
* its windows of the polynomial of the network's output, as dnsmos_local.py gives them: { sig, bak, ovrl, segments },
|
|
150
|
+
* segments the windows scored, { time: its start in seconds (in the doubled signal, for a clip under 9.01 s), sig,
|
|
151
|
+
* bak, ovrl, raw: the network's own three outputs }. opts: { sampleRate, model: a load() handle, windows, and load()'s
|
|
152
|
+
* options when no handle is given }.
|
|
153
|
+
*/
|
|
154
|
+
export default async function dnsmos(audio, opts = {}) {
|
|
155
|
+
let x = prepare(audio, opts), { starts } = windows(x.length, opts)
|
|
156
|
+
let handle = opts.model ?? await load(opts)
|
|
157
|
+
try {
|
|
158
|
+
let s = handle.session, input = s.inputs[0]?.name ?? 'input_1', output = s.outputs[0]?.name, seg = new Float32Array(WINDOW), segments = []
|
|
159
|
+
for (let a of starts) {
|
|
160
|
+
// the doubled signal, sample a + j of it x[(a + j) mod n]
|
|
161
|
+
for (let j = 0, k = a % x.length; j < WINDOW; j++, k = k + 1 === x.length ? 0 : k + 1) seg[j] = x[k]
|
|
162
|
+
let [sig, bak, ovrl] = (await s.run({ [input]: tensor(seg, [1, WINDOW]) }))[output].data
|
|
163
|
+
segments.push({ time: a / SAMPLE_RATE, ...polyfit({ sig, bak, ovrl }), raw: [sig, bak, ovrl] })
|
|
164
|
+
}
|
|
165
|
+
let mean = k => segments.reduce((m, r) => m + r[k], 0) / segments.length
|
|
166
|
+
return { sig: mean('sig'), bak: mean('bak'), ovrl: mean('ovrl'), segments }
|
|
167
|
+
} finally {
|
|
168
|
+
if (handle !== opts.model) handle.free()
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
export { dnsmos }
|
package/resampy.js
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
// The resampler dnsmos_local.py ran with when 82f1b17e77 was committed (2022-05-24): librosa.resample(aud, input_fs,
|
|
2
|
+
// fs) on librosa 0.9.1, the release of the day, whose default res_type 'kaiser_best' is resampy's precomputed filter,
|
|
3
|
+
// from resampy 0.2.2, the one release between August 2019 and resampy 0.3.0 (2022-06-29), which changed the filter; then
|
|
4
|
+
// librosa's fix_length to ceil(n · target / orig). Translated from resampy 0.2.2 (ISC License, Copyright (c) 2016
|
|
5
|
+
// Brian McFee): core.py resample(), interpn.py resample_f(), filters.py sinc_window(); and from librosa 0.9.1 (ISC
|
|
6
|
+
// License, Copyright (c) 2013--2017, librosa development team) core/audio.py resample(), the fix_length step.
|
|
7
|
+
//
|
|
8
|
+
// Changes, 2026, audiojs: kaiser_best's table (data/kaiser_best.npz, 32,769 values) is not shipped but built by
|
|
9
|
+
// sinc_window from the parameters filters.py states (64 zero crossings, a Kaiser window of beta 14.769656459379492, a
|
|
10
|
+
// rolloff of 0.9475937167399596 of Nyquist) and the file's 2^9 values per crossing; so built, 27,650 of its values
|
|
11
|
+
// differ from the file's, by up to 1.0e-15 (13 ulps; I0 is its power series, not Cephes' Chebyshev expansion), and the
|
|
12
|
+
// 16 kHz signal by 2.8e-17 at most: 15 of 9,968,612 samples as float32, on 54 files at 8 to 96 kHz (measured once, by
|
|
13
|
+
// scripts outside the package). One channel; the output float64, as resampy's for float64 input.
|
|
14
|
+
|
|
15
|
+
const ZEROS = 64, NUM_TABLE = 2 ** 9, ROLLOFF = 0.9475937167399596, BETA = 14.769656459379492
|
|
16
|
+
|
|
17
|
+
// I0(x) = Σ ((x/2)^k / k!)², summed until a term no longer changes the sum
|
|
18
|
+
function i0(x) {
|
|
19
|
+
let s = 1, t = 1, q = x * x / 4
|
|
20
|
+
for (let k = 1; k < 500; k++) { t *= q / (k * k); let s1 = s + t; if (s1 === s) break; s = s1 }
|
|
21
|
+
return s
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
// sinc_window(num_zeros=64, precision=9, window=kaiser(beta), rolloff): the filter's right wing,
|
|
25
|
+
// rolloff · sinc(rolloff · t) · kaiser(2n + 1, beta)[n:], t = linspace(0, 64, n + 1)
|
|
26
|
+
let table
|
|
27
|
+
function wing() {
|
|
28
|
+
if (table) return table
|
|
29
|
+
let n = NUM_TABLE * ZEROS, w = new Float64Array(n + 1), step = ZEROS / n, ib = i0(BETA)
|
|
30
|
+
for (let k = 0; k <= n; k++) {
|
|
31
|
+
let t = k === n ? ZEROS : k * step, x = Math.PI * (t === 0 ? 1e-20 : ROLLOFF * t) // np.sinc: π · where(x == 0, 1e-20, x)
|
|
32
|
+
let r = k / n, taper = i0(BETA * Math.sqrt(1 - r * r)) / ib // scipy.signal.windows.kaiser: (n − α) / α
|
|
33
|
+
w[k] = taper * (ROLLOFF * (Math.sin(x) / x))
|
|
34
|
+
}
|
|
35
|
+
return table = w
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// the wing scaled by the rate's ratio where it falls (core.py: interp_win *= sample_ratio), and its differences
|
|
39
|
+
// (interp_delta[:-1] = np.diff(interp_win), the last 0), kept for the last two ratios
|
|
40
|
+
const scaled = new Map()
|
|
41
|
+
function filters(ratio) {
|
|
42
|
+
let key = ratio < 1 ? ratio : 1, f = scaled.get(key)
|
|
43
|
+
if (f) return f
|
|
44
|
+
let w = wing()
|
|
45
|
+
if (ratio < 1) w = w.map(v => v * ratio)
|
|
46
|
+
let d = new Float64Array(w.length)
|
|
47
|
+
for (let i = 0; i < w.length - 1; i++) d[i] = w[i + 1] - w[i]
|
|
48
|
+
if (scaled.size >= 2) scaled.delete(scaled.keys().next().value)
|
|
49
|
+
scaled.set(key, f = { w, d })
|
|
50
|
+
return f
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* x (one channel at rate `from`) at rate `to`, as librosa 0.9.1's resample(x, from, to) gives it with resampy 0.2.2's
|
|
55
|
+
* kaiser_best: Float64Array of ceil(n · (to / from)) samples.
|
|
56
|
+
*/
|
|
57
|
+
export default function resampy(x, from, to) {
|
|
58
|
+
let n = x.length, ratio = to / from
|
|
59
|
+
let size = Math.ceil(n * ratio) // librosa: int(np.ceil(y.shape[-1] * ratio))
|
|
60
|
+
let nOut = Math.trunc(n * ratio) // resampy: int(shape[axis] * sample_ratio)
|
|
61
|
+
if (nOut < 1) throw new RangeError(`neural-quality: ${n} samples are too few to resample from ${from} to ${to} Hz`)
|
|
62
|
+
let { w, d } = filters(ratio), y = new Float64Array(size)
|
|
63
|
+
let scale = Math.min(1, ratio), step = Math.trunc(scale * NUM_TABLE), inc = 1 / ratio, nwin = w.length, time = 0
|
|
64
|
+
for (let t = 0; t < Math.min(nOut, size); t++, time += inc) { // the time register summed, as resample_f sums it
|
|
65
|
+
let m = Math.trunc(time), frac = scale * (time - m)
|
|
66
|
+
let index = frac * NUM_TABLE, offset = Math.trunc(index), eta = index - offset, acc = 0
|
|
67
|
+
for (let i = 0, e = Math.min(m + 1, Math.floor((nwin - offset) / step)); i < e; i++) {
|
|
68
|
+
let j = offset + i * step
|
|
69
|
+
acc += (w[j] + eta * d[j]) * x[m - i]
|
|
70
|
+
}
|
|
71
|
+
frac = scale - frac
|
|
72
|
+
index = frac * NUM_TABLE; offset = Math.trunc(index); eta = index - offset
|
|
73
|
+
for (let k = 0, e = Math.min(n - m - 1, Math.floor((nwin - offset) / step)); k < e; k++) {
|
|
74
|
+
let j = offset + k * step
|
|
75
|
+
acc += (w[j] + eta * d[j]) * x[m + k + 1]
|
|
76
|
+
}
|
|
77
|
+
y[t] = acc
|
|
78
|
+
}
|
|
79
|
+
return y
|
|
80
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
// The waveforms test.js and scripts/reference.py give DNSMOS, mono float32: lena (audio-lena: speech from a film scene,
|
|
2
|
+
// 44.1 kHz) as it is, and at 16 and 8 kHz through resampy.js (at 16 kHz the very samples the reference makes of it);
|
|
3
|
+
// lena under white noise (a seeded LCG) 5 dB down at 16 kHz; its first 0.1 s; 2 s of digital silence; and ramps
|
|
4
|
+
// (sample i = i · 2^-21, exact in float32) whose values tell reference.py which samples each of the reference's windows
|
|
5
|
+
// holds. reference.py runs this file and gives the reference those very bytes; test.js checks their SHA-256 against
|
|
6
|
+
// fixtures/reference.json first.
|
|
7
|
+
//
|
|
8
|
+
// node scripts/inputs.mjs <dir> writes <dir>/<name>.f32 for each, and <dir>/rates.json
|
|
9
|
+
|
|
10
|
+
import { writeFileSync, mkdirSync } from 'node:fs'
|
|
11
|
+
import path from 'node:path'
|
|
12
|
+
import { fileURLToPath } from 'node:url'
|
|
13
|
+
import lena from 'audio-lena/raw'
|
|
14
|
+
import resampy from '../resampy.js'
|
|
15
|
+
|
|
16
|
+
export const LENA_RATE = 44100
|
|
17
|
+
const lcg = (s = 1) => () => (s = (s * 1664525 + 1013904223) % 4294967296, s / 4294967296 - .5)
|
|
18
|
+
const rms = x => Math.sqrt(x.reduce((m, v) => m + v * v, 0) / x.length)
|
|
19
|
+
|
|
20
|
+
export const lena44 = () => new Float32Array(lena)
|
|
21
|
+
export const lena16 = () => Float32Array.from(resampy(lena44(), LENA_RATE, 16000))
|
|
22
|
+
|
|
23
|
+
// lena at 16 kHz plus white noise at `snr` dB under its RMS
|
|
24
|
+
export function noisy(snr = 5) {
|
|
25
|
+
let x = lena16(), rnd = lcg(5), w = Float64Array.from(x, () => rnd()), g = rms(x) / rms(w) * 10 ** (-snr / 20)
|
|
26
|
+
return Float32Array.from(x, (v, i) => v + g * w[i])
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// the lengths whose windows the fixtures hold (samples at 16 kHz): around the window (144160) and a doubling, and
|
|
30
|
+
// clips long enough to reach the hops the reference's float product drops (7 to 23, 119 to 122)
|
|
31
|
+
export const RAMPS = [1600, 32000, 144159, 144160, 160159, 160160, 16000 * 17, 16000 * 30, 16000 * 34, 16000 * 132]
|
|
32
|
+
export const ramp = n => Float32Array.from({ length: n }, (_, i) => i * 2 ** -21)
|
|
33
|
+
|
|
34
|
+
/** name → [samples, sample rate] */
|
|
35
|
+
export function inputs() {
|
|
36
|
+
let l16 = lena16()
|
|
37
|
+
return {
|
|
38
|
+
'lena@16k': [l16, 16000],
|
|
39
|
+
'lena-white5@16k': [noisy(5), 16000],
|
|
40
|
+
'lena@44.1k': [lena44(), LENA_RATE],
|
|
41
|
+
'lena@8k': [Float32Array.from(resampy(lena44(), LENA_RATE, 8000)), 8000],
|
|
42
|
+
'lena-0.1s@16k': [l16.slice(0, 1600), 16000],
|
|
43
|
+
'silence@16k': [new Float32Array(32000), 16000],
|
|
44
|
+
...Object.fromEntries(RAMPS.map(n => [`ramp-${n}`, [ramp(n), 16000]])),
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
|
49
|
+
let dir = process.argv[2]
|
|
50
|
+
if (!dir) { console.error('usage: node scripts/inputs.mjs <dir>'); process.exit(1) }
|
|
51
|
+
mkdirSync(dir, { recursive: true })
|
|
52
|
+
let rates = {}
|
|
53
|
+
for (let [name, [x, sr]] of Object.entries(inputs())) {
|
|
54
|
+
writeFileSync(path.join(dir, `${name}.f32`), new Uint8Array(x.buffer, x.byteOffset, x.byteLength))
|
|
55
|
+
rates[name] = sr
|
|
56
|
+
}
|
|
57
|
+
writeFileSync(path.join(dir, 'rates.json'), JSON.stringify(rates))
|
|
58
|
+
}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""DNSMOS P.835 as Microsoft's dnsmos_local.py computes it, on the waveforms scripts/inputs.mjs makes: fixtures/ for test.js.
|
|
3
|
+
|
|
4
|
+
From the package directory, after npm install at the repository's root (it runs node scripts/inputs.mjs):
|
|
5
|
+
|
|
6
|
+
uv run --python 3.11 --with librosa==0.9.1 --with resampy==0.2.2 --with numba==0.59.1 --with numpy==1.26.4 \\
|
|
7
|
+
--with onnxruntime==1.30.0 --with soundfile==0.14.0 --with pandas --with tqdm --with requests \\
|
|
8
|
+
--with 'setuptools<70' python -I scripts/reference.py
|
|
9
|
+
|
|
10
|
+
Reads DNSMOS/dnsmos_local.py and DNSMOS/DNSMOS/sig_bak_ovr.onnx of microsoft/DNS-Challenge at COMMIT from <dir>
|
|
11
|
+
($AUDIO_NEURAL_CACHE/dnsmos, else ~/.cache/audiojs/neural/dnsmos), fetched there first where missing, each refused
|
|
12
|
+
unless of its SHA-256. Imports the script and calls it as its main() does, ComputeScore(model)(wav, 16000, False), on
|
|
13
|
+
each input written as a float32 WAV (soundfile reads it back to the same values), recording every call of the network.
|
|
14
|
+
librosa 0.9.1 and resampy 0.2.2, the releases of the day the commit was made (2022-05-24): the script calls
|
|
15
|
+
librosa.resample(aud, input_fs, fs) positionally, which librosa 0.10 and later refuse; 0.9's default resampler is
|
|
16
|
+
resampy's 'kaiser_best', whose filter resampy 0.3.0 (2022-06-29) replaced. Writes fixtures/reference.json: the versions, the inputs' SHA-256,
|
|
17
|
+
and per input SIG, BAK, OVRL and num_hops as ComputeScore returns them, each window's raw outputs (the ramps: each
|
|
18
|
+
window's first sample, from its value, instead); get_polyfit_val on some raw values; and, of lena at 44.1 and 8 kHz,
|
|
19
|
+
the length of librosa's 16 kHz signal and 0.5 s of it (fixtures/<input>.librosa16.f32, samples SLICE).
|
|
20
|
+
"""
|
|
21
|
+
import hashlib
|
|
22
|
+
import importlib.util
|
|
23
|
+
import json
|
|
24
|
+
import os
|
|
25
|
+
import platform
|
|
26
|
+
import subprocess
|
|
27
|
+
import sys
|
|
28
|
+
import tempfile
|
|
29
|
+
import urllib.request
|
|
30
|
+
from pathlib import Path
|
|
31
|
+
|
|
32
|
+
import numpy as np
|
|
33
|
+
import soundfile as sf
|
|
34
|
+
|
|
35
|
+
COMMIT = "82f1b17e7776a43eee395d0f45bae8abb700ad00" # microsoft/DNS-Challenge, "Add support for personalized and improved models"
|
|
36
|
+
RAW = f"https://raw.githubusercontent.com/microsoft/DNS-Challenge/{COMMIT}/DNSMOS/"
|
|
37
|
+
SOURCES = {
|
|
38
|
+
"dnsmos_local.py": (RAW + "dnsmos_local.py", "7195de211df8b1943570ee2e2d920099fc1bcca1eb950758e8ca390b2463be55"),
|
|
39
|
+
"sig_bak_ovr.onnx": (RAW + "DNSMOS/sig_bak_ovr.onnx", "269fbebdb513aa23cddfbb593542ecc540284a91849ac50516870e1ac78f6edd"),
|
|
40
|
+
}
|
|
41
|
+
HERE = Path(__file__).resolve().parent.parent
|
|
42
|
+
SLICE = (80000, 88000) # 0.5 s of the 16 kHz signal, 5 s in
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def sha(b):
|
|
46
|
+
return hashlib.sha256(b).hexdigest()
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def fetch(dir):
|
|
50
|
+
dir.mkdir(parents=True, exist_ok=True)
|
|
51
|
+
for name, (url, want) in SOURCES.items():
|
|
52
|
+
p = dir / name
|
|
53
|
+
if not p.exists():
|
|
54
|
+
with urllib.request.urlopen(url) as r:
|
|
55
|
+
b = r.read()
|
|
56
|
+
if sha(b) != want:
|
|
57
|
+
sys.exit(f"{url}: SHA-256 {sha(b)}, expected {want}")
|
|
58
|
+
p.write_bytes(b)
|
|
59
|
+
if sha(p.read_bytes()) != want:
|
|
60
|
+
sys.exit(f"{p}: SHA-256 {sha(p.read_bytes())}, expected {want}: not the file at {COMMIT}")
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def main():
|
|
64
|
+
import librosa, onnxruntime, resampy
|
|
65
|
+
sys.dont_write_bytecode = True # importing dnsmos_local.py leaves no __pycache__ in the neural cache
|
|
66
|
+
cache = Path(os.environ.get("AUDIO_NEURAL_CACHE") or Path.home() / ".cache" / "audiojs" / "neural") / "dnsmos"
|
|
67
|
+
fetch(cache)
|
|
68
|
+
spec = importlib.util.spec_from_file_location("dnsmos_local", cache / "dnsmos_local.py")
|
|
69
|
+
dl = importlib.util.module_from_spec(spec)
|
|
70
|
+
spec.loader.exec_module(dl)
|
|
71
|
+
cs = dl.ComputeScore(str(cache / "sig_bak_ovr.onnx"))
|
|
72
|
+
calls, run = [], cs.onnx_sess.run
|
|
73
|
+
|
|
74
|
+
def recorded(*a):
|
|
75
|
+
r = run(*a)
|
|
76
|
+
x = a[1]["input_1"]
|
|
77
|
+
calls.append(dict(first=float(x[0, 0]), length=int(x.shape[1]), raw=[float(v) for v in r[0][0]]))
|
|
78
|
+
return r
|
|
79
|
+
cs.onnx_sess.run = recorded
|
|
80
|
+
|
|
81
|
+
seen = {}
|
|
82
|
+
resample = dl.librosa.resample
|
|
83
|
+
def kept(y, *a, **k):
|
|
84
|
+
seen["y16"] = z = resample(y, *a, **k)
|
|
85
|
+
return z
|
|
86
|
+
dl.librosa.resample = kept
|
|
87
|
+
|
|
88
|
+
out = dict(by="scripts/reference.py: dnsmos_local.py's ComputeScore, called as its main() calls it",
|
|
89
|
+
commit=COMMIT, script=SOURCES["dnsmos_local.py"][1], model=SOURCES["sig_bak_ovr.onnx"][1],
|
|
90
|
+
versions=dict(python=platform.python_version(), numpy=np.__version__, librosa=librosa.__version__,
|
|
91
|
+
resampy=resampy.__version__, onnxruntime=onnxruntime.__version__, soundfile=sf.__version__),
|
|
92
|
+
slice=list(SLICE), inputs={})
|
|
93
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
94
|
+
subprocess.run(["node", str(HERE / "scripts" / "inputs.mjs"), tmp], check=True, cwd=HERE)
|
|
95
|
+
rates = json.loads(Path(tmp, "rates.json").read_text())
|
|
96
|
+
for name, sr in rates.items():
|
|
97
|
+
b = Path(tmp, f"{name}.f32").read_bytes()
|
|
98
|
+
x = np.frombuffer(b, dtype=np.float32)
|
|
99
|
+
wav = Path(tmp, f"{name}.wav")
|
|
100
|
+
sf.write(wav, x, sr, subtype="FLOAT")
|
|
101
|
+
assert np.array_equal(sf.read(wav, dtype="float32")[0], x)
|
|
102
|
+
calls.clear(); seen.clear()
|
|
103
|
+
d = cs(str(wav), 16000, False)
|
|
104
|
+
r = dict(samples=len(x), rate=sr, sha256=sha(b), num_hops=int(d["num_hops"]),
|
|
105
|
+
SIG=float(d["SIG"]), BAK=float(d["BAK"]), OVRL=float(d["OVRL"]))
|
|
106
|
+
if name.startswith("ramp-"):
|
|
107
|
+
# sample i of a ramp is i · 2^-21: a window's first value is the input sample it starts on
|
|
108
|
+
r["windows"] = [round(c["first"] * 2 ** 21) for c in calls]
|
|
109
|
+
r["lengths"] = sorted({c["length"] for c in calls})
|
|
110
|
+
del r["SIG"], r["BAK"], r["OVRL"]
|
|
111
|
+
else:
|
|
112
|
+
r["raw"] = [c["raw"] for c in calls]
|
|
113
|
+
if "y16" in seen:
|
|
114
|
+
y = np.asarray(seen["y16"], dtype=np.float32)
|
|
115
|
+
r["length16"] = len(y)
|
|
116
|
+
(HERE / "fixtures" / f"{name}.librosa16.f32").write_bytes(y[SLICE[0]:SLICE[1]].tobytes())
|
|
117
|
+
out["inputs"][name] = r
|
|
118
|
+
print(name, {k: v for k, v in r.items() if k in ("num_hops", "SIG", "BAK", "OVRL")}, len(calls), "windows", flush=True)
|
|
119
|
+
|
|
120
|
+
# the polynomials alone, on lena's raw outputs and on 1, 3 and 5, through get_polyfit_val
|
|
121
|
+
raws = out["inputs"]["lena@16k"]["raw"] + [[1.0, 1.0, 1.0], [3.0, 3.0, 3.0], [5.0, 5.0, 5.0]]
|
|
122
|
+
out["polyfit"] = [dict(raw=r, mos=[float(v) for v in cs.get_polyfit_val(*map(np.float32, r), False)]) for r in raws]
|
|
123
|
+
(HERE / "fixtures" / "reference.json").write_text(json.dumps(out, indent=1) + "\n")
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
if __name__ == "__main__":
|
|
127
|
+
main()
|