@audio/neural-separate 0.1.0 → 0.4.0

package/separate.js

@@ -12,13 +12,22 @@
 //   of sevagh/demucs.onnx. 7.8 s segments overlap-added as demucs.apply.apply_model does; each
 //   runs as waveform + complex-as-channels spectrogram in, frequency- and time-branch estimates
 //   out, summed after the iSTFT.
+// 'complex': complex spectrogram models (SCNet: Tong, Zhu, Chen, Kang, Jiang, Li, Wu, Meng, "SCNet: Sparse Compression
+//   Network for Music Source Separation", ICASSP 2024; the Band-Split RoFormer family has the same
+//   contract) with their STFT and iSTFT out of the graph: each channel's STFT in, re and im as
+//   channels, each source's out. 11 s segments overlap-added under linear fades, as
+//   ZFTurbo/Music-Source-Separation-Training's demix() runs them.
+// 'multires': masking models over several STFT resolutions (MRX: Petermann, Wichern, Wang, Le Roux, "The Cocktail
+//   Fork Problem: Three-Stem Audio Separation for Real-World Soundtracks", ICASSP 2022), their STFTs and iSTFTs out of
+//   the graph: each channel's magnitudes in, a real mask per source and resolution out; a source is the sum of the
+//   iSTFTs of its masked spectrograms. The input at the loudness the model trained at, as upstream's separate.py sets it.
 // 'waveform': any [1, C, N] → [1, S, C, N] graph.
 //
 // Long inputs run in overlapping chunks, so memory stays bounded by the chunk, not the file.
 
 import { fft, ifft } from 'fourier-transform'
 import resampleSinc from '@audio/resample-sinc'
-import { load as neuralLoad, tensor } from '@audio/neural-runtime'
+import { load as neuralLoad, tensor, fetchModel } from '@audio/neural-runtime'
 
 const PI2 = Math.PI * 2
 const isNode = typeof process !== 'undefined' && !!process.versions?.node
@@ -342,12 +351,61 @@ function complexInverse(Are, Aim, outRe, outIm, n, re, im) {
 // Demucs source order (demucs/pretrained.py SOURCES), the S axis of its outputs.
 const DEMUCS = ['drums', 'bass', 'other', 'vocals']
 
-// Presets: files are <weights>/<name>/<target>.onnx (one graph per target) or
-// <weights>/<name>/<name>.onnx, as scripts/export-*.py write them.
+// SCNet's STFT (torch.stft's window=None: ones; normalized) and segment, 476 frames of 11 s
+const SCNET = { modelType: 'complex', sampleRate: 44100, targets: DEMUCS, n: 4096, hop: 1024, window: 'ones', normalized: true, segment: 485100, frames: 476 }
+
+// MRX's STFTs (mrx.py _stft: Hann, hop 256, centered, normalized) at three resolutions; its sources in its order
+// (music, speech, sfx), named as a soundtrack's stems are; channels encoded apart, so mono stays mono; the input at
+// -27 LUFS (separate.py's input_lufs, the stems scaled back); 20 s chunks (Node's peak 1.9 GB on 60 s, whole 4.4 GB)
+const MRX = {
+	modelType: 'multires', sampleRate: 44100, targets: ['dialogue', 'music', 'effects'], sources: ['music', 'dialogue', 'effects'],
+	windows: [1024, 2048, 8192], hop: 256, loudness: -27, mono: true, chunk: 20, script: 'export-mrx.py',
+}
+
+// TIGER's DnR model as scripts/export-tiger.py exports it (three band-split models, each its kept source, one graph): its
+// STFT (Hann, unnormalized, centered), 12 s segments of 1034 frames every 4 s, as TIGERDNR.wav_chunk_inference cuts
+// them; mono, channels apart
+const TIGER = {
+	modelType: 'complex', sampleRate: 44100, targets: ['dialogue', 'music', 'effects'], sources: ['dialogue', 'music', 'effects'],
+	n: 2048, hop: 512, window: 'hann', normalized: false, segment: 529200, frames: 1034, step: 1 / 3, fade: 0, pad: 'zero',
+	standardize: false, mono: true, script: 'export-tiger.py',
+}
+
+// The compact files (scripts/compact.py: int8 weights, a few float16, float32 compute), each within 0.05 dB of its
+// export's median SDR or SNR per stem (README, Size); their Hugging Face repositories, and SHA-256 checked when fetched
+const FILES = {
+	'scnet-large': { file: 'scnet-large.int8.onnx', repo: 'audiojs/scnet-large', sha256: 'b2dc586a1e0e6c0afe4915e9057ea29111397b7bbf589edc3d85de91de79cd72' },
+	scnet: { file: 'scnet.int8.onnx', repo: 'audiojs/scnet', sha256: '98228931494151762a1c4ab1ec7899a894b1f81fd4a509921a9d2f9bacc50845' },
+	mrx: { file: 'mrx.int8.onnx', repo: 'audiojs/mrx', sha256: 'd876c92d224f5d8cd2207d528b23369da895a6de71fb24132416fcb16f9f8424' },
+	tiger: { file: 'tiger.int8.onnx', repo: 'audiojs/tiger-dnr', sha256: '2189859ac7e89cf4d9d284c6f393e737aaa2b3233ed7ee424ad622012bce7bca' },
+}
+
+// The hosted revision of each compact file's repository (a commit), its URL https://huggingface.co/<repo>/resolve/<revision>/
+// <file>: what a compact preset reads without opts.weights, in the browser, and in Node fetched once into the neural cache
+// (unless the cache holds it or the export). Empty: not hosted.
+export const REVISIONS = {
+	'scnet-large': '95f22d9e68f6cab78aef358f1a0de248d4edf805',
+	scnet: 'e71bcdb42bce1d62b7fae25ba90fcbcf90f73f46',
+	mrx: 'dedc823a0bea2ed085d19e8b7f32589d35169ee8',
+	tiger: '1c959ab7ff8794dc5663b104ba301687ab95d2ab',
+}
+
+const hosted = name => models[name].repo && REVISIONS[name] ? `https://huggingface.co/${models[name].repo}/resolve/${REVISIONS[name]}/${models[name].file}` : null
+
+// Presets: files are <weights>/<name>/<target>.onnx (one graph per target) or <weights>/<name>/<name>.onnx, as
+// scripts/export-*.py write them; a compact preset's <weights>/<name>/<file> first, as scripts/compact.py writes it.
 export const models = {
 	umxhq: { modelType: 'openunmix', sampleRate: 44100, targets: ['vocals', 'drums', 'bass', 'other'], perTarget: true },
 	htdemucs: { modelType: 'hybrid', sampleRate: 44100, targets: DEMUCS },
 	htdemucs_ft: { modelType: 'hybrid', sampleRate: 44100, targets: DEMUCS, perTarget: true },
+	// SCNet-large and SCNet (starrytong/SCNet, MIT weights) as scripts/export-scnet.py exports them
+	'scnet-large': { ...SCNET, ...FILES['scnet-large'] },
+	scnet: { ...SCNET, ...FILES.scnet },
+	// MRX (merlresearch/cocktail-fork-separation, MIT weights) as scripts/export-mrx.py exports it: dialogue, music
+	// and effects of a soundtrack (Divide and Remaster)
+	mrx: { ...MRX, ...FILES.mrx },
+	// TIGER (JusperLee/TIGER, Apache-2.0 weights): the same three stems, three band-split models, about 50 times MRX's time
+	tiger: { ...TIGER, ...FILES.tiger },
 }
 
 async function cacheDir() {
@@ -368,18 +426,40 @@ async function weightsBase(opts, name) {
 	return pathToFileURL(w.endsWith('/') ? w : w + '/').href
 }
 
-// Weights produced locally must exist before a session loads them: name the missing file and
-// how to make it, rather than surfacing ENOENT from deep inside the runtime.
-async function checkLocal(graphs, name) {
-	let files = graphs.map(g => g.spec).filter(s => typeof s === 'string' && s.startsWith('file:'))
-	if (!files.length) return
-	let [{ existsSync }, { fileURLToPath }] = await Promise.all([import('node:fs'), import('node:url')])
-	let missing = files.map(f => fileURLToPath(f)).filter(f => !existsSync(f))
-	if (!missing.length) return
-	let script = models[name].modelType === 'hybrid' ? 'export-htdemucs.py' : 'export-openunmix.py'
-	throw new Error(`neural-separate: ${name} weights not found: ${missing.join(', ')}. ` +
-		`Produce them with python3 node_modules/@audio/neural-separate/scripts/${script} --model ${name}, ` +
-		`or pass opts.weights: a URL or directory holding ${name}/<file>.onnx`)
+// A preset's files in a local directory; a file missing named with how to make it, rather than ENOENT from the runtime.
+// A compact preset reads its compact file, else its export (float32). From the neural cache (no opts.weights) its compact
+// file only as hosted (its SHA-256), else the export, else the hosted file fetched into the cache, checked, written whole;
+// offline, a compact file of another SHA-256 (scripts/compact.py's own) as it is.
+async function local(graphs, name, opts) {
+	let [fs, { fileURLToPath }] = await Promise.all([import('node:fs'), import('node:url')])
+	let file = g => typeof g.spec === 'string' && g.spec.startsWith('file:') ? fileURLToPath(g.spec) : null
+	let here = g => !file(g) || fs.existsSync(file(g)), p = models[name], why = ''
+	let whole = g => ({ ...g, spec: g.spec.replace(/[^/]*$/, `${name}.onnx`), sha256: null })
+	if (p.file) graphs = await Promise.all(graphs.map(async g => {
+		if (!file(g)) return g
+		let cached = opts.weights == null
+		if (here(g) && (!cached || await sha256(fs.readFileSync(file(g))) === p.sha256)) return { ...g, sha256: null }
+		if (here(whole(g))) return whole(g)
+		if (cached && hosted(name)) try { return { ...g, spec: await fetched(name, file(g)), sha256: null } } catch (e) { why = ` (${e.message})` }
+		return { ...g, sha256: null }
+	}))
+	let missing = graphs.filter(g => !here(g)).map(file)
+	if (!missing.length) return graphs
+	let script = p.script ?? { hybrid: 'export-htdemucs.py', complex: 'export-scnet.py' }[p.modelType] ?? 'export-openunmix.py'
+	throw new Error(`neural-separate: ${name} weights not found: ${missing.join(', ')}${why}. ` +
+		`Produce them with python3 node_modules/@audio/neural-separate/scripts/${script} --model ${name}` +
+		(p.file ? ` (and scripts/compact.py --model ${name} for ${p.file})` : '') +
+		`, or pass opts.weights: a URL or directory holding ${name}/<file>.onnx`)
+}
+
+// The hosted compact file into the cache, checked against its SHA-256, written whole or not at all → its file URL
+async function fetched(name, dest) {
+	let [fs, path, { pathToFileURL }] = await Promise.all([import('node:fs'), import('node:path'), import('node:url')])
+	let bytes = await checked(hosted(name), models[name].sha256, { cache: false })
+	fs.mkdirSync(path.dirname(dest), { recursive: true })
+	fs.writeFileSync(dest + '.part', bytes)
+	fs.renameSync(dest + '.part', dest)
+	return pathToFileURL(dest).href
 }
 
 // opts.model → { graphs: [{ spec, sources, keep }], modelType, sampleRate, targets }. A graph's
@@ -392,15 +472,16 @@ async function resolveModel(opts) {
 	let pick = names => want ? names.filter(n => want.includes(n)) : names
 
 	if (typeof model === 'string' && Object.hasOwn(models, model)) {
-		let p = models[model], base = await weightsBase(opts, model)
+		// the browser without opts.weights: the hosted file
+		let p = models[model], url = !isNode && opts.weights == null && hosted(model), base = url ? '' : await weightsBase(opts, model)
 		if (want) for (let t of want) if (!p.targets.includes(t)) throw new Error(`neural-separate: ${model} has no target '${t}' (has ${p.targets.join(', ')})`)
 		let targets = pick(p.targets)
-		let sources = p.modelType === 'hybrid' ? DEMUCS : null
+		let sources = p.sources ?? (p.modelType === 'hybrid' || p.modelType === 'complex' ? DEMUCS : null)
 		let graphs = p.perTarget
 			? targets.map(t => ({ spec: `${base}${model}/${t}.onnx`, sources: sources ?? [t], keep: [t] }))
-			: [{ spec: `${base}${model}/${model}.onnx`, sources: sources ?? p.targets, keep: targets }]
-		if (isNode) await checkLocal(graphs, model)
-		return { graphs, modelType: p.modelType, sampleRate: p.sampleRate, targets }
+			: [{ spec: url || `${base}${model}/${p.file ?? `${model}.onnx`}`, sources: sources ?? p.targets, keep: targets, sha256: p.sha256 }]
+		if (isNode) graphs = await local(graphs, model, opts)
+		return { graphs, modelType: p.modelType, sampleRate: p.sampleRate, targets, preset: p }
 	}
 
 	let graphs
@@ -422,8 +503,25 @@ async function resolveModel(opts) {
 // No pooled arena: one htdemucs segment peaks at 2.7 GB RSS in onnxruntime-node instead of 3.2 GB.
 const sessionOptions = { enableCpuMemArena: false, enableMemPattern: false }
 
+// A graph's session; a fetched file with a SHA-256 checked against it first
 function loader(opts) {
-	return spec => opts.session ? opts.session(spec, opts) : neuralLoad(spec, { backend: opts.device, sessionOptions })
+	return async g => {
+		if (opts.session) return opts.session(g.spec, opts)
+		let spec = g.sha256 && !g.spec.startsWith('file:') ? await checked(g.spec, g.sha256) : g.spec
+		return neuralLoad(spec, { backend: opts.device, sessionOptions })
+	}
+}
+
+// A fetched file's bytes, refused unless of its SHA-256
+async function checked(url, hash, opts) {
+	let bytes = await fetchModel(url, opts), hex = await sha256(bytes)
+	if (hex !== hash) throw new Error(`neural-separate: ${url} has SHA-256 ${hex}, expected ${hash}: a partial or altered download (clear the 'audio-neural' cache), or another build of the file`)
+	return bytes
+}
+
+async function sha256(bytes) {
+	let subtle = globalThis.crypto?.subtle ?? (await import('node:crypto')).webcrypto.subtle
+	return Array.from(new Uint8Array(await subtle.digest('SHA-256', bytes)), b => b.toString(16).padStart(2, '0')).join('')
 }
 
 function outputOf(session, out, i = 0) {
@@ -512,7 +610,7 @@ async function separateSpectral(channels, rate, opts, graphs, modelType, load) {
 	let wopts = { iterations, softmask: opts.softmask ?? false, eps: opts.eps, residual }
 	let sessions = []
 	try {
-		for (let g of graphs) sessions.push(await load(g.spec))
+		for (let g of graphs) sessions.push(await load(g))
 		let C = channels.length, N = channels[0].length
 		let { starts, size, weight } = crossfadeChunks(N, rate, opts)
 		return await overlapAdd(N, C, starts, size, weight, async (start, len) => {
@@ -542,6 +640,23 @@ async function separateSpectral(channels, rate, opts, graphs, modelType, load) {
 
 // ------------------------------------------------------------------ hybrid
 
+// The input over its whole length: (x − mean) / deviation of the channels' mean (demucs.api, and demix's callers
+// with inference.normalize); back(ch) scales a source back and adds the mean, as both do to every source; scale(ch)
+// only scales it back, the mean (the mixture's DC) left to no source.
+function normalize(channels) {
+	let C = channels.length, N = channels[0].length, mono = new Float64Array(N), mean = 0, sq = 0
+	for (let x of channels) for (let i = 0; i < N; i++) mono[i] += x[i] / C
+	for (let i = 0; i < N; i++) mean += mono[i]
+	mean /= N
+	for (let i = 0; i < N; i++) sq += (mono[i] - mean) ** 2
+	let std = Math.sqrt(sq / Math.max(1, N - 1)) + 1e-8
+	return {
+		norm: channels.map(x => Float32Array.from(x, v => (v - mean) / std)),
+		back: ch => { for (let i = 0; i < ch.length; i++) ch[i] = ch[i] * std + mean; return ch },
+		scale: ch => { for (let i = 0; i < ch.length; i++) ch[i] *= std; return ch },
+	}
+}
+
 // One segment through a demucs.onnx-contract graph: STFT and iSTFT as HTDemucs._spec,
 // _magnitude, _mask and _ispec do them (demucs/htdemucs.py, spec.py: torch.stft with
 // normalized=True, n_fft 4096, hop 1024; the Nyquist bin dropped; reflect re-padding by 3/4
@@ -585,19 +700,11 @@ async function hybridSegment(session, mix, want, n, hop) {
 // triangular window normalized by its sum.
 async function separateHybrid(channels, rate, opts, graphs, load) {
 	let n = opts.n ?? 4096, hop = opts.hop ?? 1024
-	let C = channels.length, N = channels[0].length
-	let mono = new Float64Array(N)
-	for (let x of channels) for (let i = 0; i < N; i++) mono[i] += x[i] / C
-	let mean = 0, sq = 0
-	for (let i = 0; i < N; i++) mean += mono[i]
-	mean /= N
-	for (let i = 0; i < N; i++) sq += (mono[i] - mean) ** 2
-	let std = Math.sqrt(sq / Math.max(1, N - 1)) + 1e-8
-	let norm = channels.map(x => Float32Array.from(x, v => (v - mean) / std))
+	let C = channels.length, N = channels[0].length, { norm, back } = normalize(channels)
 
 	let result = {}
 	for (let g of graphs) {
-		let session = await load(g.spec)
+		let session = await load(g)
 		try {
 			let L = session.inputs?.[0]?.dims?.[2]
 			L = typeof L === 'number' && L > 0 ? L : opts.segment ?? 343980 // htdemucs: int(7.8 · 44100)
@@ -615,7 +722,92 @@ async function separateHybrid(channels, rate, opts, graphs, load) {
 				g.keep.forEach((name, i) => { res[name] = stems[i].map(ch => ch.subarray(off, off + len)) })
 				return res
 			}, opts.progress)
-			for (let name in out) result[name] = out[name].map(ch => { for (let i = 0; i < N; i++) ch[i] = ch[i] * std + mean; return ch })
+			for (let name in out) result[name] = out[name].map(back)
+		} finally {
+			session.free?.()
+		}
+	}
+	return result
+}
+
+// ------------------------------------------------------------------ complex
+
+// One segment through a complex-spectrogram graph, [1, 2C, F, T] in (channel c's STFT at 2c, re, and 2c + 1, im),
+// [1, 2·S·C, F, T] out (source s, channel c at 2(sC + c), re, and the next, im): the segment zero-padded to
+// (T − 1)·hop samples when shorter (SCNet.forward pads so), centered reflect-padded frames, × `scale` (normalized:
+// 1/√n) and back. seg: C channels of L samples; returns the sources at indices `want`, C channels of L samples each.
+async function complexSegment(session, seg, want, n, hop, win, scale, T) {
+	let C = seg.length, L = seg[0].length, F = (n >> 1) + 1, spec = new Float32Array(2 * C * F * T)
+	seg.forEach((x, c) => {
+		let padded = new Float64Array(Math.max(L, (T - 1) * hop))
+		padded.set(x)
+		let { re, im } = stftFlat(padded, n, hop, win, true)
+		let zr = spec.subarray(2 * c * F * T), zi = spec.subarray((2 * c + 1) * F * T)
+		for (let f = 0, k = 0; f < F; f++) for (let t = 0, i = f; t < T; t++, i += F, k++) { zr[k] = re[i] * scale; zi[k] = im[i] * scale }
+	})
+	let name = session.inputs?.[0]?.name ?? 'mix_spec'
+	let z = outputOf(session, await session.run({ [name]: tensor(spec, [1, 2 * C, F, T], 'float32') })).data
+	let env = envelope(T, n, hop, win)
+	return want.map(s => Array.from({ length: C }, (_, c) => {
+		let re = new Float64Array(T * F), im = new Float64Array(T * F), b = 2 * (s * C + c) * F * T
+		// the [F, T] planes read in order, frames written strided
+		for (let f = 0, k = b; f < F; f++) for (let t = 0, i = f; t < T; t++, i += F, k++) { re[i] = z[k] / scale; im[i] = z[k + F * T] / scale }
+		let out = new Float64Array((T - 1) * hop + n)
+		ola(rows(re, F), rows(im, F), out, 0, n, hop, win)
+		return olaFinish(out, env, n, true, L)
+	}))
+}
+
+// demix() of Music-Source-Separation-Training (utils/model_utils.py, generic mode): segments of L samples every
+// `step` of L, each weighted by a window fading in and out linearly over L/10 (the first not in, the last not out),
+// the sum divided by the summed weights. An input longer than two segments less a step is first reflected L − step
+// samples out on each side; a segment running past the end is reflected out when more than half of it is input,
+// zero-padded when not. The input is normalized by its mean and deviation, and the sources scaled back; demix's
+// callers add the mean back to every source, here to none: the stems sum to the input less its DC, which stays in
+// the residual. And two departures: the segments stop at the first to reach the end (demix runs on while one
+// starts before it: a 7 s song took three 11 s segments, now one, its SDR within 0.02 dB), and the first and last
+// fades are set per segment (demix sets them per batch of 8: its first segment fades in, every one of a last
+// batch holds).
+// TIGER's DnR model (its preset's `step` 1/3, `fade` 0, `pad` 'zero', `standardize` false, `mono`) runs the segments
+// TIGERDNR.wav_chunk_inference cuts: 12 s every 4 s, unweighted, the input zero-padded by 8 s at each end and the last
+// segments with zeros, each channel apart, nothing normalized; its sum divided by the three segments over every
+// sample, as the summed weights divide it here.
+async function separateComplex(channels, rate, opts, graphs, load, p) {
+	let n = opts.n ?? p?.n ?? 4096, hop = opts.hop ?? p?.hop ?? 1024
+	let win = (opts.window ?? p?.window) === 'ones' ? new Float64Array(n).fill(1) : hann(n), scale = (opts.normalized ?? p?.normalized ?? true) ? 1 / Math.sqrt(n) : 1
+	let C = channels.length, N = channels[0].length, zero = (opts.pad ?? p?.pad) === 'zero'
+	let { norm, scale: back } = opts.standardize ?? p?.standardize ?? true ? normalize(channels) : { norm: channels, scale: c => c }
+	let result = {}
+	for (let g of graphs) {
+		let session = await load(g)
+		try {
+			let T = session.inputs?.[0]?.dims?.[3]
+			T = typeof T === 'number' && T > 0 ? T : opts.frames ?? p?.frames
+			let L = opts.segment ?? p?.segment ?? (T - 1) * hop, step = Math.max(1, Math.floor(L * (opts.step ?? p?.step ?? 0.25)))
+			let border = L - step, wide = border > 0 && (zero || N > 2 * border)
+			let x = wide ? norm.map(c => Float32Array.from({ length: N + 2 * border }, (_, i) => zero ? (i < border || i >= border + N ? 0 : c[i - border]) : c[reflectIndex(i - border, N)])) : norm
+			let M = x[0].length, fade = Math.floor(L * (opts.fade ?? p?.fade ?? 0.1)), starts = []
+			for (let s = 0; ; s += step) { starts.push(s); if (s + L >= M) break }
+			let keep = g.keep.map(name => g.sources.indexOf(name))
+			let acc = g.keep.map(() => Array.from({ length: C }, () => new Float64Array(M))), wsum = new Float64Array(M)
+			for (let k = 0; k < starts.length; k++) {
+				let start = starts[k], len = Math.min(L, M - start), reflect = !zero && len > L / 2
+				opts.progress?.({ chunk: k + 1, totalChunks: starts.length })
+				let seg = x.map(c => { let s = new Float32Array(L); for (let i = 0; i < L; i++) s[i] = i < len ? c[start + i] : reflect ? c[start + reflectIndex(i, len)] : 0; return s })
+				// a mono model hears each channel apart
+				let stems = p?.mono ? [] : await complexSegment(session, seg, keep, n, hop, win, scale, T)
+				if (p?.mono) for (let c = 0; c < C; c++) (await complexSegment(session, [seg[c]], keep, n, hop, win, scale, T)).forEach((st, j) => (stems[j] ??= [])[c] = st[0])
+				for (let i = 0; i < len; i++) {
+					let w = 1
+					if (k > 0 && i < fade) w = i / (fade - 1)
+					if (k < starts.length - 1 && i >= L - fade) w = Math.min(w, (L - 1 - i) / (fade - 1))
+					wsum[start + i] += w
+					for (let j = 0; j < keep.length; j++) for (let c = 0; c < C; c++) acc[j][c][start + i] += stems[j][c][i] * w
+				}
+			}
+			g.keep.forEach((name, j) => {
+				result[name] = acc[j].map(a => back(Float32Array.from({ length: N }, (_, i) => wsum[i + (wide ? border : 0)] > 0 ? a[i + (wide ? border : 0)] / wsum[i + (wide ? border : 0)] : 0)))
+			})
 		} finally {
 			session.free?.()
 		}
@@ -623,6 +815,91 @@ async function separateHybrid(channels, rate, opts, graphs, load) {
 	return result
 }
 
+// ------------------------------------------------------------------ multires
+
+// Each channel apart (B = 1), chunk by chunk: its STFT at every resolution (Hann windows of `windows` samples, one
+// hop, centered), the magnitudes scaled 1/√n (torch.stft's normalized=True) in as mag_<n> [1, F, T], masks out as
+// mask_<n> [1, S, F, T] in the graph's source order; a source is the sum over resolutions of the iSTFT of its mask
+// times the spectrogram (the masks are real: the 1/√n and the iSTFT's √n cancel).
+async function separateMultires(channels, rate, opts, graphs, load, p) {
+	let windows = opts.windows ?? p?.windows, hop = opts.hop ?? p?.hop ?? 256
+	let C = channels.length, N = channels[0].length, result = {}
+	for (let g of graphs) {
+		let session = await load(g)
+		try {
+			let { starts, size, weight } = crossfadeChunks(N, rate, { ...opts, chunk: opts.chunk ?? p?.chunk })
+			Object.assign(result, await overlapAdd(N, C, starts, size, weight, async (start, len) => {
+				let res = Object.fromEntries(g.keep.map(name => [name, []]))
+				for (let c = 0; c < C; c++) {
+					let x = channels[c].subarray(start, start + len), feeds = {}
+					// the spectra kept in float32 (the 8192 window's alone 113 MB a 20 s chunk), one masked copy reused
+					let specs = windows.map((n, r) => {
+						let { re, im, T, bins: F } = stftFlat(x, n, hop, hann(n), true), mag = new Float32Array(F * T), k = 1 / Math.sqrt(n)
+						for (let t = 0, i = 0; t < T; t++) for (let f = 0; f < F; f++, i++) mag[f * T + t] = Math.hypot(re[i], im[i]) * k
+						feeds[session.inputs?.[r]?.name ?? `mag_${n}`] = tensor(mag, [1, F, T], 'float32')
+						return { re: Float32Array.from(re), im: Float32Array.from(im), T, F }
+					})
+					let out = await session.run(feeds), acc = g.keep.map(() => new Float64Array(len))
+					specs.forEach(({ re, im, T, F }, r) => {
+						let n = windows[r], m = outputOf(session, out, r).data, mr = new Float64Array(T * F), mi = new Float64Array(T * F)
+						g.keep.forEach((name, j) => {
+							let b = g.sources.indexOf(name) * F * T
+							for (let f = 0; f < F; f++) for (let t = 0, i = f, q = b + f * T; t < T; t++, i += F, q++) { mr[i] = re[i] * m[q]; mi[i] = im[i] * m[q] }
+							let o = new Float64Array((T - 1) * hop + n)
+							ola(rows(mr, F), rows(mi, F), o, 0, n, hop, hann(n))
+							let y = olaFinish(o, envelope(T, n, hop, hann(n)), n, true, len)
+							for (let i = 0; i < len; i++) acc[j][i] += y[i]
+						})
+					})
+					g.keep.forEach((name, j) => res[name].push(acc[j]))
+				}
+				return res
+			}, opts.progress))
+		} finally {
+			session.free?.()
+		}
+	}
+	return result
+}
+
+// ------------------------------------------------------------------ loudness
+
+// Integrated loudness, ITU-R BS.1770-4, as pyloudnorm 0.2 measures it (upstream MRX's separate.py sets its input with
+// it): K-weighting by pyloudnorm's two biquads at any rate (a 4 dB shelf over 1.5 kHz, a 38 Hz high-pass), 400 ms
+// blocks every 100 ms, gated at -70 LUFS, then 10 LU under the mean of the rest; -Infinity for silence.
+function lufs(channels, rate) {
+	let shelf = (G, Q, fc) => {
+		let A = 10 ** (G / 40), w = 2 * Math.PI * fc / rate, al = Math.sin(w) / (2 * Q), c = Math.cos(w), r = 2 * Math.sqrt(A) * al
+		return [A * ((A + 1) + (A - 1) * c + r), -2 * A * ((A - 1) + (A + 1) * c), A * ((A + 1) + (A - 1) * c - r), (A + 1) - (A - 1) * c + r, 2 * ((A - 1) - (A + 1) * c), (A + 1) - (A - 1) * c - r]
+	}
+	let pass = (Q, fc) => { let w = 2 * Math.PI * fc / rate, al = Math.sin(w) / (2 * Q), c = Math.cos(w); return [(1 + c) / 2, -(1 + c), (1 + c) / 2, 1 + al, -2 * c, 1 - al] }
+	let filters = [shelf(4, 1 / Math.sqrt(2), 1500), pass(0.5, 38)], N = channels[0].length, T = 0.4, step = 0.25
+	// numpy's round, half to even, as pyloudnorm counts its blocks
+	let even = v => { let f = Math.floor(v), d = v - f; return d > 0.5 || (d === 0.5 && f % 2) ? f + 1 : f }
+	let blocks = even((N / rate - T) / (T * step)) + 1
+	if (!(blocks > 0)) return -Infinity
+	let z = new Float64Array(blocks), G = [1, 1, 1, 1.41, 1.41]
+	channels.forEach((x, c) => {
+		let y = Float64Array.from(x)
+		for (let [b0, b1, b2, a0, a1, a2] of filters) {
+			// scipy.signal.lfilter from rest: direct form II transposed, coefficients over a0
+			b0 /= a0; b1 /= a0; b2 /= a0; a1 /= a0; a2 /= a0
+			let z1 = 0, z2 = 0
+			for (let i = 0; i < N; i++) { let v = y[i], o = b0 * v + z1; z1 = b1 * v - a1 * o + z2; z2 = b2 * v - a2 * o; y[i] = o }
+		}
+		for (let j = 0; j < blocks; j++) {
+			let lo = Math.trunc(T * (j * step) * rate), hi = Math.min(N, Math.trunc(T * (j * step + 1) * rate)), s = 0
+			for (let i = lo; i < hi; i++) s += y[i] * y[i]
+			z[j] += (G[c] ?? 1) * s / (T * rate)
+		}
+	})
+	let L = v => -0.691 + 10 * Math.log10(v), mean = v => v.reduce((a, b) => a + b, 0) / v.length
+	let abs = [...z].filter(v => L(v) >= -70)
+	if (!abs.length) return -Infinity
+	let rel = L(mean(abs)) - 10, gated = [...z].filter(v => L(v) > rel && L(v) > -70)
+	return gated.length ? L(mean(gated)) : -Infinity
+}
+
 // ------------------------------------------------------------------ waveform
 
 // [1, C, N] in, [1, S, C, N] out, contiguous per (source, channel).
@@ -631,7 +908,7 @@ async function separateWaveform(channels, rate, opts, graphs, load) {
 	let { starts, size, weight } = crossfadeChunks(N, rate, opts)
 	let result = {}
 	for (let g of graphs) {
-		let session = await load(g.spec)
+		let session = await load(g)
 		try {
 			let out = await overlapAdd(N, C, starts, size, weight, async (start, len) => {
 				let data = new Float32Array(C * len)
@@ -658,8 +935,6 @@ function normalizeAudio(audio, opts) {
 	else throw new Error('neural-separate: audio must be Float32Array[] or { channelData, sampleRate }')
 	if (!sampleRate) throw new Error('neural-separate: sampleRate is required (opts.sampleRate, or audio.sampleRate)')
 	if (!channelData.length) throw new Error('neural-separate: audio has no channels')
-	// mono → duplicated stereo, as openunmix.utils.preprocess does
-	if (channelData.length === 1) channelData = [channelData[0], channelData[0]]
 	return { channelData, sampleRate }
 }
 
@@ -677,20 +952,30 @@ export default async function separate(audio, opts = {}) {
 	if (opts.dtype && opts.dtype !== 'float32') throw new Error(`neural-separate: dtype '${opts.dtype}' not supported: only 'float32' tensor marshalling is implemented`)
 
 	let { channelData, sampleRate: rate } = normalizeAudio(audio, opts)
-	let { graphs, modelType, sampleRate: modelRate, targets } = await resolveModel(opts)
+	let { graphs, modelType, sampleRate: modelRate, targets, preset } = await resolveModel(opts)
+	// mono → duplicated stereo, as openunmix.utils.preprocess does; a model hearing channels apart takes it as it is
+	if (channelData.length === 1 && !preset?.mono) channelData = [channelData[0], channelData[0]]
 	let empty = () => channelData.map(() => new Float32Array(0))
 	if (!channelData[0].length) return { stems: Object.fromEntries(targets.map(t => [t, empty()])), sampleRate: rate, residual: empty() }
 	let targetRate = opts.targetRate ?? modelRate ?? rate
 	let proc = targetRate !== rate ? channelData.map(c => resampleSinc(c, { from: rate, to: targetRate })) : channelData
 	let load = loader(opts)
+	// the input at the loudness the model trained at, its stems scaled back (a silent input as it is)
+	let level = preset?.loudness == null ? -Infinity : lufs(proc, targetRate), gain = Number.isFinite(level) ? 10 ** ((preset.loudness - level) / 20) : 1
+	if (gain !== 1) proc = proc.map(c => c.map(v => v * gain))
 
 	let stemsAtRate = modelType === 'hybrid' ? await separateHybrid(proc, targetRate, opts, graphs, load)
+		: modelType === 'complex' ? await separateComplex(proc, targetRate, opts, graphs, load, preset)
+		: modelType === 'multires' ? await separateMultires(proc, targetRate, opts, graphs, load, preset)
 		: modelType === 'waveform' ? await separateWaveform(proc, targetRate, opts, graphs, load)
 		: await separateSpectral(proc, targetRate, opts, graphs, modelType, load)
 
 	let N = channelData[0].length
 	let stems = {}
-	for (let name of targets) stems[name] = stemsAtRate[name].map(c => fitLength(targetRate !== rate ? resampleSinc(c, { from: targetRate, to: rate }) : c, N))
+	for (let name of targets) stems[name] = stemsAtRate[name].map(c => {
+		if (gain !== 1) c = Float32Array.from(c, v => v / gain)
+		return fitLength(targetRate !== rate ? resampleSinc(c, { from: targetRate, to: rate }) : c, N)
+	})
 
 	let residual = channelData.map((c, ci) => {
 		let r = Float32Array.from(c)