@audio/neural-timbre 0.0.0-stage → 0.1.0

package/scripts/augment.js

@@ -0,0 +1,180 @@
+// Augmentation for training views and benchmark queries, all in audiojs: chorus, EQ, reverb,
+// another voice or drums underneath, noise, compression, gain and a low-bitrate codec, applied in
+// that order to a mono 44.1 kHz note. Every choice is drawn from a seeded generator, so a view or a
+// benchmark query is reproducible from its seed.
+import chorus from '@audio/effect-chorus'
+import eq from '@audio/eq-parametric'
+import freeverb from '@audio/reverb-freeverb'
+import dattorro from '@audio/reverb-dattorro'
+import compressor from '@audio/dynamics-compressor'
+import encodeMp3 from '@audio/encode-mp3'
+import decodeMp3 from '@audio/decode-mp3'
+import encodeOpus from '@audio/encode-opus'
+import decodeOpus from '@audio/decode-opus'
+import { membrane, metal, noiseDrum } from '@audio/synth-drum'
+import resample from '@audio/resample-sinc'
+
+export const FS = 44100
+
+// mulberry32 (Tommy Ettinger): 32-bit state, full period, adequate for choosing augmentations
+export function rng(seed) {
+	let a = seed >>> 0
+	let next = () => {
+		a = (a + 0x6d2b79f5) >>> 0
+		let t = Math.imul(a ^ (a >>> 15), 1 | a)
+		t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t
+		return ((t ^ (t >>> 14)) >>> 0) / 4294967296
+	}
+	next.uniform = (lo, hi) => lo + (hi - lo) * next()
+	next.int = (lo, hi) => lo + Math.floor(next() * (hi - lo + 1)) // inclusive
+	next.pick = list => list[Math.floor(next() * list.length)]
+	next.log = (lo, hi) => lo * (hi / lo) ** next()
+	return next
+}
+
+// A training chain: a third of views stay clean, the rest take each stage by its own chance.
+// `other` draws the note under a mix: { voice, note, velocity, hold } of another voice.
+export function chain(r, other) {
+	if (r() < 1 / 3) return {}
+	let c = {}
+	if (r() < 0.25) c.chorus = { rate: r.uniform(0.2, 2), depth: r.uniform(0.1, 0.6), delay: r.uniform(0.008, 0.03), voices: r.int(2, 4) }
+	if (r() < 0.45) c.eq = eqBands(r)
+	if (r() < 0.35) c.reverb = r() < 0.75
+		? { kind: 'freeverb', room: r.uniform(0.3, 0.95), damp: r.uniform(0.1, 0.7), mix: r.uniform(0.1, 0.5) }
+		: { kind: 'dattorro', decay: r.uniform(0.3, 0.9), damping: r.uniform(0.1, 0.7), mix: r.uniform(0.1, 0.4) }
+	if (r() < 0.25) c.mix = r() < 0.7 ? { ...other(r), snr: r.uniform(0, 20) } : { drums: r.int(0, 2 ** 31), snr: r.uniform(0, 20) }
+	if (r() < 0.25) c.noise = { color: r.pick(['white', 'pink']), snr: r.uniform(5, 35), seed: r.int(0, 2 ** 31) }
+	if (r() < 0.25) c.compressor = { threshold: r.uniform(-36, -10), ratio: r.uniform(2, 10), attack: r.uniform(1, 30), release: r.uniform(50, 300) }
+	c.gain = r.uniform(-24, 0)
+	if (r() < 0.15) c.codec = r() < 0.7 ? { kind: 'mp3', bitrate: r.pick([32, 48, 64]) } : { kind: 'opus', bitrate: r.pick([12, 16, 24]) }
+	return c
+}
+
+const eqBands = r => [
+	{ type: 'lowshelf', fc: r.uniform(80, 300), gain: r.uniform(-9, 9) },
+	{ fc: r.log(300, 5000), Q: r.uniform(0.5, 2), gain: r.uniform(-9, 9) },
+	{ type: 'highshelf', fc: r.uniform(3000, 8000), gain: r.uniform(-12, 6) },
+]
+
+// Benchmark conditions, one stage each at a fixed setting, plus the training chain drawn from the
+// query's own seed. Frozen with the benchmark: changing one changes every figure that cites it.
+export const CONDITIONS = {
+	clean: () => ({}),
+	reverb: () => ({ reverb: { kind: 'freeverb', room: 0.8, damp: 0.4, mix: 0.35 } }),
+	eq: () => ({ eq: [{ type: 'lowshelf', fc: 150, gain: 6 }, { fc: 1500, Q: 1, gain: -6 }, { type: 'highshelf', fc: 5000, gain: -9 }] }),
+	compressor: () => ({ compressor: { threshold: -30, ratio: 8, attack: 5, release: 100 } }),
+	chorus: () => ({ chorus: { rate: 0.8, depth: 0.4, delay: 0.015, voices: 3 } }),
+	gain: () => ({ gain: -30 }),
+	mp3: () => ({ codec: { kind: 'mp3', bitrate: 32 } }),
+	opus: () => ({ codec: { kind: 'opus', bitrate: 12 } }),
+	noise20: r => ({ noise: { color: 'pink', snr: 20, seed: r.int(0, 2 ** 31) } }),
+	noise10: r => ({ noise: { color: 'pink', snr: 10, seed: r.int(0, 2 ** 31) } }),
+	mix10: (r, other) => ({ mix: { ...other(r), snr: 10 } }),
+	mix0: (r, other) => ({ mix: { ...other(r), snr: 0 } }),
+	drums10: r => ({ mix: { drums: r.int(0, 2 ** 31), snr: 10 } }),
+	chain: (r, other) => { let c = {}; while (!Object.keys(c).length || (Object.keys(c).length === 1 && 'gain' in c)) c = chain(r, other); return c },
+}
+
+const rms = x => { let s = 0; for (let i = 0; i < x.length; i++) s += x[i] * x[i]; return Math.sqrt(s / Math.max(1, x.length)) }
+const db2lin = db => 10 ** (db / 20)
+
+// Mix `y` into `x` (in place) at `snr` dB below x's RMS over x's length.
+function addAt(x, y, snr) {
+	let gx = rms(x), gy = rms(y)
+	if (!(gy > 0) || !(gx > 0)) return x
+	let k = gx / gy * db2lin(-snr)
+	for (let i = 0; i < x.length && i < y.length; i++) x[i] += k * y[i]
+	return x
+}
+
+function noise({ color, seed }, n) {
+	let r = rng(seed), out = new Float32Array(n)
+	// pink: Paul Kellet's refined filter of white noise (musicdsp.org #76), -3 dB/octave within 0.05 dB above 9 Hz at 44.1 kHz
+	let b0 = 0, b1 = 0, b2 = 0, b3 = 0, b4 = 0, b5 = 0, b6 = 0
+	for (let i = 0; i < n; i++) {
+		let w = r() * 2 - 1
+		if (color === 'white') { out[i] = w; continue }
+		b0 = 0.99886 * b0 + w * 0.0555179; b1 = 0.99332 * b1 + w * 0.0750759; b2 = 0.969 * b2 + w * 0.153852
+		b3 = 0.8665 * b3 + w * 0.3104856; b4 = 0.55 * b4 + w * 0.5329522; b5 = -0.7616 * b5 - w * 0.016898
+		out[i] = b0 + b1 + b2 + b3 + b4 + b5 + b6 + w * 0.5362; b6 = w * 0.115926
+	}
+	return out
+}
+
+// A bar of drums at 120 BPM from a seed: kick, snare or hat and cymbal hits on eighth notes.
+function drums(seed, n) {
+	let r = rng(seed), out = new Float32Array(n), step = Math.round(FS * 0.25)
+	for (let at = Math.round(r() * step); at < n; at += step) {
+		let hit = r.pick(['kick', 'kick', 'snare', 'hat', 'cymbal', 'rest'])
+		if (hit === 'rest') continue
+		let s = hit === 'kick' ? membrane(r.uniform(45, 70), { duration: 0.4, fs: FS })
+			: hit === 'cymbal' ? metal(r.uniform(180, 260), { duration: 0.6, fs: FS, amp: 0.3 })
+			: noiseDrum({ duration: hit === 'hat' ? 0.06 : 0.2, fs: FS, seed: r.int(1, 1e6), amp: hit === 'hat' ? 0.3 : 0.7 })
+		for (let i = 0; i < s.length && at + i < n; i++) out[at + i] += s[i]
+	}
+	return out
+}
+
+const concat = parts => { let out = new Uint8Array(parts.reduce((n, p) => n + p.length, 0)), o = 0; for (let p of parts) { out.set(p, o); o += p.length } return out }
+
+// Halfband lowpass for 2x rate changes: 63-tap Blackman-windowed sinc cut at a quarter of the
+// rate (Oppenheim & Schafer, window method), every other tap but the centre zero.
+const HALF = (() => {
+	let n = 63, m = 31, h = new Float64Array(n)
+	for (let i = 0; i < n; i++) {
+		let k = i - m, w = 0.42 - 0.5 * Math.cos(2 * Math.PI * i / (n - 1)) + 0.08 * Math.cos(4 * Math.PI * i / (n - 1))
+		h[i] = (k === 0 ? 0.5 : Math.sin(Math.PI * k / 2) / (Math.PI * k)) * w
+	}
+	return h
+})()
+function decimate2(x) {
+	let out = new Float32Array(x.length >> 1), m = HALF.length >> 1
+	for (let o = 0; o < out.length; o++) {
+		let c = 2 * o, s = HALF[m] * x[c]
+		for (let k = 1; k <= m; k += 2) s += HALF[m + k] * ((x[c - k] ?? 0) + (x[c + k] ?? 0))
+		out[o] = s
+	}
+	return out
+}
+function interpolate2(y, n) {
+	let out = new Float32Array(n), m = HALF.length >> 1
+	for (let i = 0; i < n; i++) {
+		if (i % 2 === 0) { out[i] = y[i >> 1] ?? 0; continue } // the centre tap is 0.5, doubled by the zero-stuffing gain
+		let s = 0
+		for (let k = 1; k <= m; k += 2) s += HALF[m + k] * ((y[(i - k) >> 1] ?? 0) + (y[(i + k) >> 1] ?? 0))
+		out[i] = 2 * s
+	}
+	return out
+}
+
+// MP3 runs as MPEG-2 layer III at 22.05 kHz, the rate LAME picks itself for these bitrates at
+// 44.1 kHz input; Opus codes at 48 kHz and comes back through the resampler.
+async function codec(x, { kind, bitrate }) {
+	let mp3 = kind === 'mp3', rate = mp3 ? FS / 2 : FS
+	let enc = mp3 ? await encodeMp3({ sampleRate: rate, channels: 1, bitrate }) : await encodeOpus({ sampleRate: rate, channels: 1, bitrate })
+	let bytes = concat([enc.encode([mp3 ? decimate2(x) : x]), enc.flush()]); enc.free?.()
+	let { channelData, sampleRate } = mp3 ? await decodeMp3(bytes) : await decodeOpus(bytes)
+	let y = channelData[0]
+	if (sampleRate === FS / 2) return interpolate2(y, 2 * y.length)
+	return sampleRate === FS ? y : resample(y, { from: sampleRate, to: FS })
+}
+
+// apply(x, spec, render) → Float32Array: x is the dry note (not modified); render(mix) returns
+// the other voice's note for a mix stage.
+export async function apply(x, spec, render) {
+	let y = Float32Array.from(x)
+	let peak = 0
+	for (let i = 0; i < y.length; i++) peak = Math.max(peak, Math.abs(y[i]))
+	if (peak > 0) for (let i = 0; i < y.length; i++) y[i] *= 0.5 / peak // a nominal level before level-dependent stages
+	if (spec.chorus) y = chorus(y, { ...spec.chorus, fs: FS })
+	if (spec.eq) y = Float32Array.from(eq(Float64Array.from(y), { bands: spec.eq.map(b => ({ ...b })), fs: FS }))
+	if (spec.reverb) y = spec.reverb.kind === 'dattorro'
+		? dattorro(y, { decay: spec.reverb.decay, damping: spec.reverb.damping, mix: spec.reverb.mix, fs: FS })
+		: freeverb(y, { room: spec.reverb.room, damp: spec.reverb.damp, mix: spec.reverb.mix, fs: FS })
+	if (spec.mix) addAt(y, spec.mix.drums != null ? drums(spec.mix.drums, y.length) : render(spec.mix), spec.mix.snr)
+	if (spec.noise) addAt(y, noise(spec.noise, y.length), spec.noise.snr)
+	if (spec.compressor) y = compressor(y, { ...spec.compressor })
+	if (spec.gain) { let g = db2lin(spec.gain); for (let i = 0; i < y.length; i++) y[i] *= g }
+	if (spec.codec) y = await codec(y, spec.codec)
+	return y
+}

package/scripts/bench.js

@@ -0,0 +1,90 @@
+#!/usr/bin/env node
+// The frozen benchmark (scripts/freeze.js), read through the shipped path: the exported encoder
+// in @audio/neural-runtime and catalogs as scripts/catalogs.py writes them.
+//
+//   node scripts/bench.js [--model file] [--catalogs dir] [--out file]
+//
+// Queries: every audible held-out voice, one note at a note and velocity no reference uses, under
+// each condition of augment.js (features computed once by scripts/factory.js heldout, float16).
+// Each is ranked against the held-out catalog (2,925 voices) and against every voice (dx7-all),
+// with figures apart for novel voices (4 or more parameters from any training voice) and near
+// copies; the algorithm head is scored on the same queries.
+import { readFileSync, writeFileSync, openSync, readSync, closeSync, existsSync } from 'node:fs'
+import path from 'node:path'
+import os from 'node:os'
+import { readCatalog, unit, sha256, featuresSha256 } from '../timbre.js'
+import { load, tensor } from '@audio/neural-runtime'
+import { pins } from './freeze.js'
+
+const DATA = process.env.TIMBRE_DATA ?? path.join(os.homedir(), '.cache/audiojs/data/dx7')
+const CACHE = path.join(process.env.AUDIO_NEURAL_CACHE ?? path.join(os.homedir(), '.cache/audiojs/neural'), 'timbre')
+
+let o = {}
+for (let a = process.argv.slice(2), i = 0; i < a.length; i += 2) o[a[i].replace(/^--/, '')] = a[i + 1]
+let model = o.model ?? path.join(CACHE, 'encoder.int8.onnx'), dir = o.catalogs ?? path.join(CACHE, 'catalogs')
+
+let frozen = JSON.parse(readFileSync(path.join(DATA, 'bench.json'), 'utf8')), now = pins()
+for (let [k, v] of Object.entries(frozen.pins)) if (now[k] !== v) throw new Error(`bench: ${k} changed since the benchmark froze (${frozen.frozen})`)
+
+let meta = JSON.parse(readFileSync(path.join(DATA, 'features/heldout.json'), 'utf8'))
+let f = meta.features.cqt, size = f.bands * f.frames, rows = meta.rows.map(r => Object.fromEntries(meta.keys.map((k, i) => [k, r[i]])))
+let stats = new Float32Array(readFileSync(path.join(DATA, 'features/heldout.stats.f32')).buffer)
+let voices = JSON.parse(readFileSync(path.join(DATA, 'voices.json'), 'utf8')).voices
+let fd = openSync(path.join(DATA, 'features/heldout.cqt.f16'), 'r')
+let read = i => { let b = new Uint8Array(size * 2); readSync(fd, b, 0, b.length, i * size * 2); return Float32Array.from(new Float16Array(b.buffer)) }
+
+let bytes = new Uint8Array(readFileSync(model)), binding = { encoderSha256: await sha256(bytes), featuresSha256: await featuresSha256() }
+let session = await load(bytes)
+// dx7-heldout is required; dx7-all joins when its references exist
+let files = Object.fromEntries(['heldout', 'all'].filter(n => n === 'heldout' || existsSync(path.join(dir, `dx7-${n}.json`))).map(n => [n, JSON.parse(readFileSync(path.join(dir, `dx7-${n}.json`), 'utf8'))]))
+let catalogs = Object.fromEntries(Object.entries(files).map(([n, json]) => [n, readCatalog(json, binding)]))
+let position = Object.fromEntries(Object.entries(catalogs).map(([n, c]) => [n, new Map(c.entries.map((e, i) => [e.id, i]))]))
+
+// rank of the true entry: entries scoring higher, ties broken by index as rank() does
+function rankOf(e, { vectors, dimensions: d }, truth) {
+	let n = vectors.length / d, t = 0
+	for (let i = 0; i < d; i++) t += e[i] * vectors[truth * d + i]
+	let k = 0
+	for (let r = 0; r < n; r++) {
+		let v = 0
+		for (let i = 0; i < d; i++) v += e[i] * vectors[r * d + i]
+		if (v > t || (v === t && r < truth)) k++
+	}
+	return k
+}
+
+let queries = rows.map((r, i) => [r, i]).filter(([r, i]) => r.kind === 'query' && stats[2 * i] >= 2 ** -13)
+let results = {}, t0 = performance.now(), B = 64
+for (let b = 0; b < queries.length; b += B) {
+	let batch = queries.slice(b, b + B), x = new Float32Array(batch.length * size)
+	batch.forEach(([, i], k) => x.set(read(i), k * size))
+	let out = await session.run({ features: tensor(x, [batch.length, 1, f.bands, f.frames]) })
+	let e = out.embedding.data, d = e.length / batch.length, alg = out.algorithm.data
+	batch.forEach(([r], k) => {
+		let v = voices[r.voice], group = v.near >= 4 ? 'novel' : 'near', emb = unit(e.subarray(k * d, (k + 1) * d))
+		let a = Array.from(alg.subarray(k * 32, k * 32 + 32)), guess = a.indexOf(Math.max(...a)) + 1
+		for (let [n, c] of Object.entries(catalogs)) {
+			let rank = rankOf(emb, c, position[n].get(v.id))
+			for (let g of [group, 'all']) {
+				let s = results[`${n}|${r.cond}|${g}`] ??= { n: 0, top1: 0, top5: 0, algorithm: 0 }
+				s.n++; s.top1 += rank < 1; s.top5 += rank < 5; s.algorithm += guess === v.algorithm
+			}
+		}
+	})
+	if ((b / B) % 100 === 0) console.log(`${b + batch.length}/${queries.length} queries, ${((performance.now() - t0) / 1000).toFixed(0)} s`)
+}
+closeSync(fd); session.free()
+
+let table = Object.entries(results).map(([key, s]) => { let [catalog, cond, group] = key.split('|'); return { catalog, cond, group, n: s.n, top1: s.top1 / s.n, top5: s.top5 / s.n, algorithm: s.algorithm / s.n } })
+let out = o.out ?? path.join(CACHE, 'bench.json')
+writeFileSync(out, JSON.stringify({ model: path.basename(model), encoderSha256: binding.encoderSha256, frozen: frozen.frozen,
+	catalogs: Object.fromEntries(Object.entries(files).map(([n, json]) => [n, { voices: json.entries.length, encoding: json.vectors.encoding }])),
+	seconds: Math.round((performance.now() - t0) / 1000), results: table }, null, 1))
+let pct = x => (100 * x).toFixed(1)
+console.log(`\n| condition | held-out top-1 | top-5 | novel top-1 | novel top-5 | all voices top-1 | all top-5 | algorithm |\n|---|---:|---:|---:|---:|---:|---:|---:|`)
+for (let cond of frozen.conditions) {
+	let g = (c, grp) => table.find(t => t.catalog === c && t.cond === cond && t.group === grp)
+	let h = g('heldout', 'all'), nv = g('heldout', 'novel'), a = g('all', 'all'), p = x => x ? pct(x) : '-'
+	if (h) console.log(`| ${cond} | ${pct(h.top1)} | ${pct(h.top5)} | ${pct(nv.top1)} | ${pct(nv.top5)} | ${p(a?.top1)} | ${p(a?.top5)} | ${pct(h.algorithm)} |`)
+}
+console.log(`\n${out}`)

package/scripts/catalog.js

@@ -0,0 +1,95 @@
+#!/usr/bin/env node
+// Build a DX7 catalog: every chosen voice rendered at the reference notes (scripts/recipe.js refs),
+// each note embedded by the exported encoder through @audio/neural-runtime, the mean of the audible
+// ones kept as the voice's vector (a note below 4 int16 steps, which some key scalings give, is
+// skipped, as scripts/catalogs.py skips it). Names, cartridges and vectors only: no voice data
+// leaves the sysex.
+//
+//   node scripts/catalog.js <voices> [--model file] [--out file] [--encoding int8-base64|int4-base64]
+//
+//   voices   factory (Yamaha ROM1A-4B and VRC-101-112), heldout, dev, train or all (every audible voice)
+//   model    default ~/.cache/audiojs/neural/timbre/encoder.int8.onnx
+//   out      default ~/.cache/audiojs/neural/timbre/catalogs/dx7-<voices>.json
+//
+// Four worker threads render and prepare notes; the main thread runs the encoder in batches.
+import { Worker, isMainThread, parentPort, workerData } from 'node:worker_threads'
+import { readFileSync, writeFileSync, mkdirSync } from 'node:fs'
+import path from 'node:path'
+import os from 'node:os'
+import { cqt, onset } from '../features.js'
+import { refs, note } from './recipe.js'
+
+const DATA = process.env.TIMBRE_DATA ?? path.join(os.homedir(), '.cache/audiojs/data/dx7')
+const CACHE = path.join(process.env.AUDIO_NEURAL_CACHE ?? path.join(os.homedir(), '.cache/audiojs/neural'), 'timbre')
+const WORKERS = 4, CHUNK = 16
+
+const SOURCES = { factory: v => v.source === 'yamaha-factory' || v.source === 'yamaha-vrc' }
+const pick = which => SOURCES[which] ?? (which === 'all' ? () => true : v => v.split === which)
+
+// 'factory/rom1a.syx' → 'ROM1A'; other cartridges keep their path under the source folder
+const cartridge = file => file.startsWith('factory/') || file.startsWith('vrc/') ? path.basename(file, path.extname(file)).toUpperCase() : file.replace(/^[^/]+\//, '')
+
+async function main() {
+	let [which, ...rest] = process.argv.slice(2), o = {}
+	for (let i = 0; i < rest.length; i += 2) o[rest[i].replace(/^--/, '')] = rest[i + 1]
+	let { createEncoder, catalog } = await import('../timbre.js')
+	let voices = JSON.parse(readFileSync(path.join(DATA, 'voices.json'), 'utf8')).voices
+	let chosen = voices.map((v, i) => [v, i]).filter(([v]) => !v.silent && pick(which)(v))
+	let encoder = await createEncoder({ model: o.model ?? path.join(CACHE, 'encoder.int8.onnx') })
+	let vectors = new Array(chosen.length), t0 = performance.now(), done = 0
+	let todo = Array.from({ length: Math.ceil(chosen.length / CHUNK) }, (_, c) => c)
+	let pending = Promise.resolve()
+	await Promise.all(Array.from({ length: Math.min(WORKERS, todo.length) }, () => new Promise((resolve, reject) => {
+		let w = new Worker(new URL(import.meta.url), { workerData: { data: DATA } })
+		let next = () => todo.length ? (c => w.postMessage({ c, voices: chosen.slice(c * CHUNK, (c + 1) * CHUNK).map(([, i]) => i) }))(todo.shift()) : w.postMessage(null)
+		w.on('message', ({ c, patches, owners }) => {
+			// each voice's audible reference notes → one mean vector; runs are serialized on the session
+			pending = pending.then(async () => {
+				let e = patches.length ? await encoder.project(patches) : []
+				e.forEach((v, j) => {
+					let k = c * CHUNK + owners[j], mean = vectors[k] ??= new Float32Array(v.length)
+					for (let d = 0; d < v.length; d++) mean[d] += v[d]
+				})
+				done += Math.min(CHUNK, chosen.length - c * CHUNK)
+				if (done % 1024 < CHUNK) console.log(`${done}/${chosen.length} voices, ${(done / ((performance.now() - t0) / 1000)).toFixed(1)}/s`)
+			})
+			next()
+		})
+		w.on('error', reject); w.on('exit', resolve); next()
+	})))
+	await pending
+	let { unit } = await import('../timbre.js')
+	let kept = chosen.map((_, k) => k).filter(k => vectors[k])
+	let entries = kept.map(k => chosen[k][0]).map(v => ({ name: v.name, cartridge: cartridge(v.cartridge), index: v.index + 1, source: v.source, algorithm: v.algorithm, id: v.id }))
+	let json = catalog(entries, kept.map(k => unit(vectors[k])), encoder.binding, {
+		encoding: o.encoding ?? 'int8-base64',
+		name: `DX7 ${which}`, synth: 'Yamaha DX7 (@audio/synth-dx7, MSFA)',
+		references: `MIDI notes ${refs(0).map(r => r.note).filter((n, i, a) => a.indexOf(n) === i).join(', ')} at velocities 64 and 112, held 1 s, 44.1 kHz`,
+		sources: 'names, cartridges and vectors only; each source and its stated terms: the @audio/neural-timbre README, Catalogs',
+	})
+	let out = o.out ?? path.join(CACHE, 'catalogs', `dx7-${which}.json`)
+	mkdirSync(path.dirname(out), { recursive: true })
+	writeFileSync(out, JSON.stringify(json))
+	encoder.free()
+	console.log(`${out}: ${entries.length} voices in ${((performance.now() - t0) / 60000).toFixed(1)} min`)
+}
+
+function worker() {
+	let patches = new Uint8Array(readFileSync(path.join(workerData.data, 'patches.u8')))
+	parentPort.on('message', msg => {
+		if (!msg) return parentPort.close()
+		let out = [], owners = []
+		msg.voices.forEach((v, k) => {
+			for (let r of refs(v)) {
+				let x = note(patches.subarray(v * 155, v * 155 + 155), r), peak = 0
+				for (let s of x) peak = Math.max(peak, Math.abs(s))
+				if (peak < 2 ** -13) continue
+				out.push(cqt(x.subarray(onset(x)))); owners.push(k)
+			}
+		})
+		parentPort.postMessage({ c: msg.c, patches: out, owners }, out.map(p => p.buffer))
+	})
+}
+
+if (isMainThread) main().catch(e => { console.error(e); process.exit(1) })
+else worker()

package/scripts/catalogs.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+"""Build the DX7 catalogs from stored reference features, on MPS: the bulk path.
+
+    python scripts/catalogs.py --run NAME [--model encoder.int8.onnx] [--encoding int8-base64] [--dir DIR] [--only factory,all]
+
+Reads the reference rows of the refs (training voices), dev and heldout feature sets that
+scripts/factory.js wrote, embeds them with the run's encoder (weights rounded to int8 per row when
+--model is the int8 graph, as scripts/export.py rounds them), averages each voice's audible
+references (a note below 4 int16 steps, which some key scalings give, is skipped) and writes ~/.cache/audiojs/neural/timbre/catalogs/dx7-{factory,dev,heldout,all}.json in the format
+timbre.js catalog() writes and readCatalog() checks, bound to --model's SHA-256 and the features'.
+scripts/catalog.js is the same builder through @audio/neural-runtime, rendering on the fly, for
+small catalogs; on dx7-factory the two agree (README, Catalogs). The package ships dx7-factory and
+dx7-all at int4: --encoding int4-base64 --dir catalogs --only factory,all.
+"""
+import argparse, base64, hashlib, json, os, sys
+from pathlib import Path
+import numpy as np
+import torch
+from torch.nn import functional as F
+
+sys.path.insert(0, str(Path(__file__).parent))
+from train import Store, DATA, voices as load_voices  # noqa: E402
+from export import load, round_int8, OUT  # noqa: E402
+
+FACTORY = ('yamaha-factory', 'yamaha-vrc')
+
+
+def cartridge(f):
+    """'factory/rom1a.syx' -> 'ROM1A'; other cartridges keep their path under the source folder, as scripts/catalog.js."""
+    if f.startswith('factory/') or f.startswith('vrc/'): return Path(f).stem.upper()
+    return f.split('/', 1)[1]
+
+
+def pack(values, bits):
+    """Signed integers `bits` apiece in a little-endian bit stream, as timbre.js pack()."""
+    v = np.asarray(values, np.int64).ravel() & ((1 << bits) - 1)
+    return np.packbits(((v[:, None] >> np.arange(bits)) & 1).astype(np.uint8).ravel(), bitorder='little').tobytes()
+
+
+def quantize(vectors, bits=8):
+    """Per-row scale max|v| / (2^(bits-1) - 1), values rounded half away from zero as Math.round does for these magnitudes."""
+    top = 2 ** (bits - 1) - 1
+    scales = np.maximum(np.abs(vectors).max(1), 1e-30) / top
+    scales = np.array([float(f'{s:.7g}') for s in scales])  # toPrecision(7), as catalog() stores them
+    q = np.floor(vectors / scales[:, None] + 0.5).astype(np.int64)
+    return q, scales
+
+
+def write(path, entries, vectors, binding, info, encoding='int8-base64'):
+    bits = {'int8-base64': 8, 'int4-base64': 4}[encoding]
+    q, scales = quantize(vectors, bits)
+    data = {'version': 1, 'kind': 'timbre-catalog', **binding, 'dimensions': vectors.shape[1], **info, 'entries': entries,
+            'vectors': {'encoding': encoding, 'shape': list(vectors.shape), 'scales': scales.tolist(), 'data': base64.b64encode(pack(q, bits)).decode()}}
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(data, separators=(',', ':')))
+    print(f'{path}: {len(entries)} voices, {path.stat().st_size / 1e6:.2f} MB')
+
+
+@torch.no_grad()
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument('--run', required=True)
+    ap.add_argument('--model', default='encoder.int8.onnx', help='the graph the catalogs bind to, in --out')
+    ap.add_argument('--out', default=str(OUT))
+    ap.add_argument('--encoding', default='int8-base64')
+    ap.add_argument('--dir', help='where the catalogs go, default <out>/catalogs')
+    ap.add_argument('--only', default='factory,dev,heldout,all', help='the catalogs to write')
+    a = ap.parse_args()
+    torch.set_num_threads(4)
+    device = 'mps'
+    torch.mps.set_per_process_memory_fraction(min(1.0, 3e9 / torch.mps.recommended_max_memory()))
+    out = Path(a.out)
+    graph, ck = load(a.run)
+    if 'int8' in a.model: round_int8(graph)
+    graph = graph.to(device).eval()
+    feats = ck['features']
+    canonical = json.dumps(feats, sort_keys=True, separators=(',', ':'))  # JSON.stringify of timbre.js's canonical()
+    binding = {'encoderSha256': hashlib.sha256((out / a.model).read_bytes()).hexdigest(), 'featuresSha256': hashlib.sha256(canonical.encode()).hexdigest()}
+    meta = load_voices()
+    vec, loaded = {}, set()
+    for name, split in (('refs', 'train'), ('dev', 'dev'), ('heldout', 'heldout')):
+        if not (DATA / f'features/{name}.json').exists(): print(f'{name}: no feature set, its catalogs are skipped'); continue
+        loaded.add(split)
+        store = Store(name, ck['kind'])
+        rows = [i for i, r in enumerate(store.rows) if r['kind'] == 'ref' and store.audible[i]]
+        owners = np.array([store.rows[i]['voice'] for i in rows])
+        for b in range(0, len(rows), 256):
+            x = torch.from_numpy(store.read(rows[b:b + 256])).to(device).float()[:, None]
+            e, _ = graph(x)
+            e = e.cpu().numpy()
+            for o, v in zip(owners[b:b + 256], e): vec.setdefault(int(o), []).append(v)
+        print(f'{name}: {len(rows)} references', flush=True)
+    ids = sorted(vec)
+    print(f'{len(ids)} voices; {sum(len(v) < 10 for v in vec.values())} with fewer than ten audible references', flush=True)
+    mean = F.normalize(torch.tensor(np.stack([np.mean(vec[i], 0) for i in ids])), dim=1).numpy()
+    row = {v: k for k, v in enumerate(ids)}
+    info = lambda which: {'name': f'DX7 {which}', 'synth': 'Yamaha DX7 (@audio/synth-dx7, MSFA)', 'references': 'MIDI notes 36, 48, 60, 72, 84 at velocities 64 and 112, held 1 s, 44.1 kHz',
+                          'sources': 'names, cartridges and vectors only; each source and its stated terms: the @audio/neural-timbre README, Catalogs'}
+    entry = lambda i: {'name': meta[i]['name'], 'cartridge': cartridge(meta[i]['cartridge']), 'index': meta[i]['index'] + 1, 'source': meta[i]['source'], 'algorithm': meta[i]['algorithm'], 'id': meta[i]['id']}
+    for which, keep in (('factory', lambda v: v['source'] in FACTORY), ('dev', lambda v: v['split'] == 'dev'), ('heldout', lambda v: v['split'] == 'heldout'), ('all', lambda v: True)):
+        if which not in a.only.split(','): continue
+        chosen = [i for i in ids if keep(meta[i])]
+        if not {v['split'] for v in meta if keep(v)} <= loaded: print(f'dx7-{which}: references missing, skipped'); continue
+        write(Path(a.dir or out / 'catalogs') / f'dx7-{which}.json', [entry(i) for i in chosen], mean[[row[i] for i in chosen]], binding, info(which), a.encoding)
+
+
+if __name__ == '__main__':
+    main()

package/scripts/export.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+"""Export a trained encoder to ONNX for @audio/neural-runtime, with parity fixtures.
+
+    python scripts/export.py --run NAME [--out ~/.cache/audiojs/neural/timbre]
+
+Writes to --out:
+  encoder.onnx        float32: features [N, 1, bands, frames] -> embedding [N, dims] (unit),
+                      algorithm [N, 32] (logits of the DX7 algorithm head)
+  encoder.int8.onnx   the same graph, every conv and linear weight rounded per output row to int8
+                      (gpu-font's per-row weight rounding) and restored by DequantizeLinear
+  parity.json         fixture patches from the development set (float16, base64), their PyTorch
+                      embeddings for each graph, and the top-5 of each against a PyTorch catalog of
+                      development voices: test.js checks ONNX in JS against them
+Batch norm is folded into the convolutions first, so both graphs compute what training evaluated.
+"""
+import argparse, base64, json, os, sys
+from pathlib import Path
+import numpy as np
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+sys.path.insert(0, str(Path(__file__).parent))
+from train import Encoder, Store, RUNS, DATA  # noqa: E402
+
+OUT = Path(os.environ.get('AUDIO_NEURAL_CACHE', Path.home() / '.cache/audiojs/neural')) / 'timbre'
+
+
+class Graph(nn.Module):
+    """The deployed function: unit embedding and algorithm logits."""
+    def __init__(self, encoder, heads):
+        super().__init__(); self.encoder, self.heads = encoder, heads
+
+    def forward(self, features):
+        e = self.encoder(features)
+        return e, self.heads(e) * 10
+
+
+def fold(model):
+    """Conv + BatchNorm -> one conv; Linear + BatchNorm1d -> one linear. Eval-mode identities."""
+    model = model.eval()
+    seq, out, i = list(model.features), [], 0
+    while i < len(seq):
+        m = seq[i]
+        if isinstance(m, nn.Conv2d) and i + 1 < len(seq) and isinstance(seq[i + 1], nn.BatchNorm2d):
+            out.append(nn.utils.fuse_conv_bn_eval(m, seq[i + 1])); i += 2
+        else: out.append(m); i += 1
+    model.features = nn.Sequential(*out)
+    lin, bn, *rest = list(model.head)
+    model.head = nn.Sequential(nn.utils.fuse_linear_bn_eval(lin, bn), *rest)
+    return model
+
+
+def load(run):
+    ck = torch.load(RUNS / run / 'best.pt', map_location='cpu', weights_only=False)
+    c = ck['config']
+    enc = Encoder(c['bands'], c['dims'], tuple(c['widths']), c['reduce'], c['hidden']); enc.load_state_dict(ck['state'])
+    heads = nn.Linear(c['dims'], 32); heads.load_state_dict(ck['heads'])
+    return Graph(fold(enc), heads.eval()).eval(), ck
+
+
+@torch.no_grad()
+def round_int8(graph):
+    """Every conv and linear weight rounded per output row to int8 in place: the PyTorch twin of quantize()."""
+    for m in graph.modules():
+        if isinstance(m, (nn.Conv2d, nn.Linear)):
+            w = m.weight.data; rows = w.reshape(w.shape[0], -1); s = rows.abs().amax(1).clamp(min=1e-12) / 127
+            m.weight.data = (torch.round(rows / s[:, None]).clamp(-127, 127) * s[:, None]).reshape(w.shape)
+    return graph
+
+
+def quantize(path, out):
+    """Every float initializer of rank >= 2 feeding a Conv, Gemm or MatMul becomes int8 rows (axis 0 for
+    Conv and transB Gemm, the output axis for MatMul) plus a per-row scale, dequantized in-graph."""
+    import onnx
+    from onnx import helper, numpy_helper
+    m = onnx.load(path); g = m.graph
+    inits = {i.name: i for i in g.initializer}
+    users = {}
+    for n in g.node:
+        for k, name in enumerate(n.input): users.setdefault(name, []).append((n, k))
+    for name, init in list(inits.items()):
+        w = numpy_helper.to_array(init)
+        if w.ndim < 2 or w.dtype != np.float32: continue
+        uses = users.get(name, [])
+        if not uses or any(n.op_type not in ('Conv', 'Gemm', 'MatMul') for n, _ in uses): continue
+        n0 = uses[0][0]
+        axis = 1 if n0.op_type == 'MatMul' or (n0.op_type == 'Gemm' and not any(a.name == 'transB' and a.i for a in n0.attribute)) else 0
+        moved = np.moveaxis(w, axis, 0); rows = moved.reshape(moved.shape[0], -1)
+        scale = np.maximum(np.abs(rows).max(1), 1e-12) / 127
+        q = np.clip(np.round(rows / scale[:, None]), -127, 127).astype(np.int8).reshape(moved.shape)
+        q = np.moveaxis(q, 0, axis)
+        g.initializer.remove(init)
+        g.initializer.extend([numpy_helper.from_array(q, name + '_q'), numpy_helper.from_array(scale.astype(np.float32), name + '_scale'),
+                              numpy_helper.from_array(np.zeros_like(scale, np.int8), name + '_zero')])
+        g.node.insert(0, helper.make_node('DequantizeLinear', [name + '_q', name + '_scale', name + '_zero'], [name], axis=axis))
+    onnx.checker.check_model(m)
+    onnx.save(m, out)
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument('--run', required=True)
+    ap.add_argument('--out', default=str(OUT))
+    ap.add_argument('--fixtures', type=int, default=32)
+    a = ap.parse_args()
+    out = Path(a.out); out.mkdir(parents=True, exist_ok=True)
+    graph, ck = load(a.run)
+    bands, frames = ck['features']['bands'], ck['features']['frames']
+    x = torch.zeros(2, 1, bands, frames)
+    onnx_path = out / 'encoder.onnx'
+    torch.onnx.export(graph, (x,), str(onnx_path), input_names=['features'], output_names=['embedding', 'algorithm'],
+                      dynamic_axes={'features': {0: 'n'}, 'embedding': {0: 'n'}, 'algorithm': {0: 'n'}}, opset_version=17, dynamo=False)
+    import onnx
+    m = onnx.load(onnx_path)
+    for k, v in {'features': json.dumps(ck['features'], sort_keys=True), 'run': a.run, 'step': str(ck['step'])}.items():
+        m.metadata_props.append(onnx.StringStringEntryProto(key=k, value=v))
+    onnx.save(m, onnx_path)
+    quantize(str(onnx_path), str(out / 'encoder.int8.onnx'))
+
+    # fixtures: queries and references of development voices, and a catalog of those voices
+    dev = Store('dev', ck['kind']); rows = dev.rows
+    voices = sorted({r['voice'] for r in rows})[:64]
+    refs = [i for i, r in enumerate(rows) if r['kind'] == 'ref' and r['voice'] in voices and dev.audible[i]]
+    queries = [i for i, r in enumerate(rows) if r['kind'] == 'query' and r['voice'] in voices and dev.audible[i]][:a.fixtures]
+    import onnxruntime as ort
+    fixture = {'features': ck['features'], 'rows': queries, 'voices': voices, 'patches': base64.b64encode(dev.read(queries).tobytes()).decode()}
+    with torch.no_grad():
+        for name in ('encoder.onnx', 'encoder.int8.onnx'):
+            # the int8 graph's PyTorch twin: the same weights rounded per row
+            g = graph if name == 'encoder.onnx' else round_int8(load(a.run)[0])
+            run = lambda rows: torch.cat([g(torch.from_numpy(dev.read(rows[b:b + 64])).float()[:, None])[0] for b in range(0, len(rows), 64)])
+            qe, re_ = run(queries), run(refs)
+            owners = torch.tensor([voices.index(rows[i]['voice']) for i in refs])
+            cat = F.normalize(torch.zeros(len(voices), re_.shape[1]).index_add_(0, owners, re_), dim=1)
+            top5 = (qe @ cat.T).argsort(1, descending=True)[:, :5]
+            sess = ort.InferenceSession(str(out / name), providers=['CPUExecutionProvider'])
+            oe = sess.run(['embedding'], {'features': dev.read(queries).astype(np.float32)[:, None]})[0]
+            print(f'{name}: {(out / name).stat().st_size / 1e6:.2f} MB, python onnxruntime vs torch max |diff| {np.abs(oe - qe.numpy()).max():.2e}')
+            fixture[name] = {'embeddings': qe.numpy().round(7).tolist(), 'catalog': cat.numpy().round(7).tolist(), 'top5': top5.tolist()}
+    (out / 'parity.json').write_text(json.dumps(fixture))
+    print('wrote', onnx_path, out / 'encoder.int8.onnx', out / 'parity.json')
+
+
+if __name__ == '__main__':
+    main()