@audio/neural-timbre 0.0.0-stage → 0.1.1

package/scripts/factory.js

@@ -0,0 +1,125 @@
+#!/usr/bin/env node
+// Feature sets for training and the benchmark: every row of a set (scripts/recipe.js) rendered by
+// synth-dx7, augmented by scripts/augment.js and prepared by the package's own frontend, stored
+// as float16 [rows, bands, frames] on disk for memory mapping.
+//
+//   node scripts/factory.js <set> [--name NAME] [--limit N] [--views N] [--kinds logmel,cqt] [--workers N] [--data dir]
+//
+//   set      train (views of training voices), dev (references and clean + chain queries of
+//            development voices), heldout (references and every benchmark condition), refs
+//            (references of training voices, for the catalogs scripts/catalogs.py builds)
+//   writes   <data>/features/<name>.json (rows and feature settings; name defaults to the set),
+//            <name>.<kind>.f16 and <name>.stats.f32 (the dry note's peak and RMS a row); chunks done
+//            so far are kept in <name>.done, so a stopped run resumes
+//
+// Needs @audio/synth-dx7 and the augmentation packages resolvable from here (npm link or a local
+// node_modules); it runs at most --workers threads (default 4).
+import { Worker, isMainThread, parentPort, workerData } from 'node:worker_threads'
+import { readFileSync, writeFileSync, openSync, writeSync, closeSync, existsSync, mkdirSync, ftruncateSync } from 'node:fs'
+import path from 'node:path'
+import os from 'node:os'
+import { FEATURES, cqt, onset } from '../features.js'
+import { LOGMEL, logmel } from './logmel.js'
+import { views, refs, queries, specOf, note, mixNote, seedOf } from './recipe.js'
+import { apply, CONDITIONS } from './augment.js'
+
+const KINDS = { cqt: [FEATURES, cqt], logmel: [LOGMEL, logmel] } // logmel: the feature pilot only
+const CHUNK = 64
+
+function args(argv) {
+	let o = { set: argv[0], limit: Infinity, views: 8, kinds: ['cqt'], workers: 4, data: path.join(os.homedir(), '.cache/audiojs/data/dx7') }
+	for (let i = 1; i < argv.length; i += 2) {
+		let [k, v] = [argv[i].replace(/^--/, ''), argv[i + 1]]
+		o[k] = k === 'kinds' ? v.split(',') : k === 'data' || k === 'name' ? v : Number(v)
+	}
+	return o
+}
+
+function rowsOf(set, voices, o) {
+	let pick = split => voices.map((v, i) => [v, i]).filter(([v]) => v.split === split && !v.silent)
+	if (set === 'train') {
+		// a fixed order by id hash, so --limit takes the same voices every time
+		let train = pick('train').sort((a, b) => seedOf(a[0].id) - seedOf(b[0].id)).slice(0, o.limit)
+		return { rows: train.flatMap(([v, i]) => views(i, v.id, o.views)), pool: 'train' }
+	}
+	if (set === 'refs') return { rows: pick('train').slice(0, o.limit).flatMap(([, i]) => refs(i)), pool: 'train' }
+	let split = set === 'dev' ? 'dev' : 'heldout', conds = set === 'dev' ? ['clean', 'chain'] : Object.keys(CONDITIONS)
+	let list = pick(split).slice(0, o.limit)
+	return { rows: list.flatMap(([v, i]) => [...refs(i), ...queries(i, v.id, conds)]), pool: split }
+}
+
+async function main() {
+	let o = args(process.argv.slice(2))
+	if (!['train', 'dev', 'heldout', 'refs'].includes(o.set)) throw new Error('set: train, dev, heldout or refs')
+	let voices = JSON.parse(readFileSync(path.join(o.data, 'voices.json'), 'utf8')).voices
+	let { rows, pool } = rowsOf(o.set, voices, o)
+	let out = path.join(o.data, 'features'); mkdirSync(out, { recursive: true })
+	let base = path.join(out, o.name ?? o.set), meta = base + '.json'
+	let settings = Object.fromEntries(o.kinds.map(k => [k, KINDS[k][0]]))
+	let keys = ['voice', 'kind', 'note', 'velocity', 'hold', 'pre', 'cond', 'seed']
+	// full precision: a row re-renders exactly from its JSON
+	let table = { version: 1, set: o.set, pool, keys, features: settings, rows: rows.map(r => keys.map(k => r[k])) }
+	if (existsSync(meta)) {
+		let old = JSON.parse(readFileSync(meta, 'utf8'))
+		if (JSON.stringify(old.rows) !== JSON.stringify(table.rows) || JSON.stringify(old.features) !== JSON.stringify(settings)) throw new Error(`${meta} holds another set: remove it and its .f16/.done to rebuild`)
+		table = old
+	} else writeFileSync(meta, JSON.stringify(table))
+	let files = [...o.kinds.map(k => [`${base}.${k}.f16`, KINDS[k][0].bands * KINDS[k][0].frames * 2]), [base + '.stats.f32', 8]]
+	for (let [f, size] of files) if (!existsSync(f)) { let fd = openSync(f, 'w'); ftruncateSync(fd, rows.length * size); closeSync(fd) }
+	let doneFile = base + '.done', done = existsSync(doneFile) ? new Uint8Array(readFileSync(doneFile)) : new Uint8Array(Math.ceil(rows.length / CHUNK))
+	let todo = [...done.keys()].filter(c => !done[c])
+	console.log(`${o.name ?? o.set}: ${rows.length} rows, ${todo.length} of ${done.length} chunks to do, ${o.workers} workers, ${o.kinds}`)
+	let t0 = performance.now(), finished = 0
+	await Promise.all(Array.from({ length: Math.min(o.workers, todo.length) }, () => new Promise((resolve, reject) => {
+		let w = new Worker(new URL(import.meta.url), { workerData: { data: o.data, base, kinds: o.kinds, pool } })
+		let next = () => {
+			if (!todo.length) return w.postMessage(null)
+			let c = todo.shift()
+			w.postMessage({ chunk: c, rows: rows.slice(c * CHUNK, (c + 1) * CHUNK), start: c * CHUNK })
+		}
+		w.on('message', ({ chunk }) => {
+			done[chunk] = 1; finished++
+			if (finished % 20 === 0) {
+				writeFileSync(doneFile, done)
+				let rate = finished * CHUNK / ((performance.now() - t0) / 1000)
+				console.log(`${finished}/${done.length} chunks, ${rate.toFixed(0)} rows/s, ${((todo.length * CHUNK) / rate / 60).toFixed(1)} min left`)
+			}
+			next()
+		})
+		w.on('error', reject)
+		w.on('exit', resolve)
+		next()
+	})))
+	writeFileSync(doneFile, done)
+	let stats = new Float32Array(readFileSync(base + '.stats.f32').buffer), silent = 0
+	for (let i = 0; i < rows.length; i++) if (stats[2 * i] < 2 ** -13) silent++
+	console.log(`done in ${((performance.now() - t0) / 60000).toFixed(1)} min; ${silent} silent rows`)
+}
+
+async function worker() {
+	let { data, base, kinds, pool: split } = workerData
+	let patches = new Uint8Array(readFileSync(path.join(data, 'patches.u8')))
+	let voices = JSON.parse(readFileSync(path.join(data, 'voices.json'), 'utf8')).voices
+	let pool = voices.map((v, i) => [v, i]).filter(([v]) => v.split === split).map(([, i]) => i)
+	let patch = i => patches.subarray(i * 155, i * 155 + 155)
+	let fds = kinds.map(k => openSync(`${base}.${k}.f16`, 'r+')), statsFd = openSync(base + '.stats.f32', 'r+')
+	parentPort.on('message', async msg => {
+		if (!msg) { [...fds, statsFd].forEach(closeSync); return parentPort.close() }
+		for (let [j, row] of msg.rows.entries()) {
+			let dry = note(patch(row.voice), row), peak = 0, sum = 0
+			for (let x of dry) { peak = Math.max(peak, Math.abs(x)); sum += x * x }
+			let spec = specOf(row, pool)
+			let y = await apply(dry, spec, m => mixNote(patch(m.voice), m, row.pre))
+			let x = y.subarray(onset(y))
+			kinds.forEach((k, ki) => {
+				let [f, fn] = KINDS[k], v = new Float16Array(fn(x, f)), size = f.bands * f.frames * 2
+				writeSync(fds[ki], new Uint8Array(v.buffer), 0, size, (msg.start + j) * size)
+			})
+			writeSync(statsFd, new Uint8Array(Float32Array.of(peak, Math.sqrt(sum / dry.length)).buffer), 0, 8, (msg.start + j) * 8)
+		}
+		parentPort.postMessage({ chunk: msg.chunk })
+	})
+}
+
+if (isMainThread) main().catch(e => { console.error(e); process.exit(1) })
+else worker()

package/scripts/freeze.js

@@ -0,0 +1,46 @@
+#!/usr/bin/env node
+// Freeze the benchmark before any training result: pin the files that define it (the voice split,
+// the render and augmentation recipe, the frontend) by SHA-256 in <data>/bench.json. scripts/bench.js
+// refuses to report against a changed pin. Refuses to overwrite an existing manifest; a pin moves
+// only with --repin FILE --reason TEXT, recorded in the manifest's `repins`.
+//
+//   node scripts/freeze.js [--data dir]
+//   node scripts/freeze.js --repin features.js --reason '...'
+import { readFileSync, writeFileSync, existsSync } from 'node:fs'
+import { createHash } from 'node:crypto'
+import path from 'node:path'
+import os from 'node:os'
+import { fileURLToPath } from 'node:url'
+import { CONDITIONS } from './augment.js'
+import { REF_NOTES, REF_VELOCITIES, REF_HOLD } from './recipe.js'
+
+const arg = k => { let i = process.argv.indexOf(k); return i > 0 ? process.argv[i + 1] : undefined }
+const data = arg('--data') ?? path.join(os.homedir(), '.cache/audiojs/data/dx7')
+const here = path.dirname(fileURLToPath(import.meta.url))
+const sha = f => createHash('sha256').update(readFileSync(f)).digest('hex')
+export const PINS = { 'voices.json': path.join(data, 'voices.json'), 'patches.u8': path.join(data, 'patches.u8'),
+	'scripts/recipe.js': path.join(here, 'recipe.js'), 'scripts/augment.js': path.join(here, 'augment.js'), 'features.js': path.join(here, '../features.js') }
+export const pins = () => Object.fromEntries(Object.entries(PINS).map(([k, f]) => [k, sha(f)]))
+
+if (process.argv[1] === fileURLToPath(import.meta.url) && arg('--repin')) {
+	let out = path.join(data, 'bench.json'), bench = JSON.parse(readFileSync(out, 'utf8')), file = arg('--repin'), reason = arg('--reason')
+	if (!PINS[file] || !reason) throw new Error('--repin one of ' + Object.keys(PINS) + ' with --reason')
+	let now = sha(PINS[file])
+	;(bench.repins ??= []).push({ date: new Date().toISOString(), file, from: bench.pins[file], to: now, reason })
+	bench.pins[file] = now
+	writeFileSync(out, JSON.stringify(bench, null, 1))
+	console.log('repinned', file)
+} else if (process.argv[1] === fileURLToPath(import.meta.url)) {
+	let out = path.join(data, 'bench.json')
+	if (existsSync(out)) throw new Error(`${out} exists: the benchmark is frozen`)
+	let voices = JSON.parse(readFileSync(PINS['voices.json'], 'utf8')).voices, held = voices.filter(v => v.split === 'heldout' && !v.silent)
+	writeFileSync(out, JSON.stringify({
+		version: 1, frozen: new Date().toISOString(),
+		what: 'Held-out DX7 cartridges: every audible held-out voice is one catalog entry (references: C2-C6 at velocities 64 and 112, held 1 s) and one query note at a note and velocity no reference uses, under each condition. Near copies: held-out voices within 3 parameters of a training voice, reported apart.',
+		voices: held.length, novel: held.filter(v => v.near >= 4).length, nearCopies: held.filter(v => v.near <= 3).length,
+		references: { notes: REF_NOTES, velocities: REF_VELOCITIES, hold: REF_HOLD }, conditions: Object.keys(CONDITIONS),
+		metrics: ['top-1 and top-5 against the held-out catalog', 'top-1 and top-5 against the catalog of every voice', 'rebuild: specLoss of the retrieved preset before and after seeded match, against blind match at the same budget'],
+		pins: pins(),
+	}, null, 1))
+	console.log('froze', out)
+}

package/scripts/logmel.js

@@ -0,0 +1,45 @@
+// Log-mel spectrogram for the feature pilot (README, Features): the frontend the constant-Q one was
+// measured against, on the same framing, level normalization and frame count. As librosa computes
+// it: Slaney mel scale and area normalization (librosa.filters.mel), power spectrum of a
+// periodic-Hann STFT without centering (librosa.stft(center=False)), dB re the loudest cell with an
+// 80 dB floor (librosa.power_to_db(ref=np.max, top_db=80)); it matched librosa 1.0.0 to 3e-6 dB.
+import { fft } from 'fourier-transform'
+
+export const LOGMEL = Object.freeze({ kind: 'logmel', sampleRate: 44100, n: 2048, hop: 512, bands: 96, fmin: 20, fmax: 11025, frames: 128, top: 80 })
+
+const hann = n => Float64Array.from({ length: n }, (_, i) => 0.5 - 0.5 * Math.cos(2 * Math.PI * i / n))
+
+// Slaney's Auditory Toolbox mel scale, librosa.hz_to_mel(htk=False): linear below 1 kHz, log above.
+const F_SP = 200 / 3, MIN_LOG_HZ = 1000, MIN_LOG_MEL = MIN_LOG_HZ / F_SP, LOGSTEP = Math.log(6.4) / 27
+const hzToMel = f => f < MIN_LOG_HZ ? f / F_SP : MIN_LOG_MEL + Math.log(f / MIN_LOG_HZ) / LOGSTEP
+const melToHz = m => m < MIN_LOG_MEL ? m * F_SP : MIN_LOG_HZ * Math.exp(LOGSTEP * (m - MIN_LOG_MEL))
+
+// Triangular filters on the FFT bin frequencies, each scaled to unit area (norm='slaney').
+function melBank({ sampleRate: fs, n, bands, fmin, fmax }) {
+	let lo = hzToMel(fmin), hi = hzToMel(fmax)
+	let edges = Float64Array.from({ length: bands + 2 }, (_, i) => melToHz(lo + (hi - lo) * i / (bands + 1)))
+	return Array.from({ length: bands }, (_, b) => {
+		let [l, c, r] = [edges[b], edges[b + 1], edges[b + 2]], norm = 2 / (r - l)
+		return Float64Array.from({ length: n / 2 + 1 }, (_, k) => { let f = k * fs / n; return Math.max(0, Math.min((f - l) / (c - l), (r - f) / (r - c))) * norm })
+	})
+}
+
+let bank = null, window = null
+export function logmel(x, f = LOGMEL) {
+	let { n, hop, bands, frames, top } = f, frame = new Float64Array(n)
+	bank ??= melBank(f); window ??= hann(n)
+	let out = new Float32Array(bands * frames), max = 0
+	for (let t = 0; t < frames; t++) {
+		for (let i = 0; i < n; i++) frame[i] = (x[t * hop + i] || 0) * window[i]
+		let [re, im] = fft(frame)
+		for (let b = 0; b < bands; b++) {
+			let w = bank[b], s = 0
+			for (let k = 0; k <= n / 2; k++) if (w[k]) s += w[k] * (re[k] * re[k] + im[k] * im[k])
+			out[b * frames + t] = s
+			if (s > max) max = s
+		}
+	}
+	let ref = Math.max(max, 1e-10)
+	for (let i = 0; i < out.length; i++) { let db = 10 * Math.log10(Math.max(out[i], 1e-10) / ref); out[i] = Math.max(db, -top) / (top / 2) + 1 }
+	return out
+}

package/scripts/rebuild.js

@@ -0,0 +1,96 @@
+#!/usr/bin/env node
+// The rebuild half of the benchmark: for held-out queries, the spectral distance (neural-synth's
+// specLoss) between the query note and the best retrieved preset played at the same note, before
+// and after seeded refinement with @audio/neural-synth's match, against blind match at the same
+// budget. The query is the clean held-out note; retrieval ranks it against dx7-heldout.
+//
+//   node scripts/rebuild.js [--n 64] [--budget 1000] [--model file] [--out file] [--misses file]
+//
+//   misses   a previous run's output: rerun only its retrieval misses (at another budget, say)
+//
+// match searches the 145 sound parameters of the VCED voice (MAX bounds; the 10 name bytes, which
+// MAX also bounds, cannot change the sound and stay fixed), rounding each candidate to integers.
+// Each CMA-ES start is evaluated before it steps (cmaes.js), so a seeded result is never worse than
+// the retrieved preset (`kept`, the better of the two, equals `seeded`). `random` is another
+// held-out voice at the same note: the scale of a wrong answer.
+import { Worker, isMainThread, parentPort, workerData } from 'node:worker_threads'
+import { readFileSync, writeFileSync, openSync, readSync, closeSync } from 'node:fs'
+import path from 'node:path'
+import os from 'node:os'
+import { createEncoder, readCatalog, rank } from '../timbre.js'
+import { note, seedOf } from './recipe.js'
+
+const DATA = process.env.TIMBRE_DATA ?? path.join(os.homedir(), '.cache/audiojs/data/dx7')
+const CACHE = path.join(process.env.AUDIO_NEURAL_CACHE ?? path.join(os.homedir(), '.cache/audiojs/neural'), 'timbre')
+const WORKERS = 4
+
+async function main() {
+	let o = { n: 64, budget: 1000 }
+	for (let a = process.argv.slice(2), i = 0; i < a.length; i += 2) o[a[i].replace(/^--/, '')] = isNaN(a[i + 1]) ? a[i + 1] : Number(a[i + 1])
+	let meta = JSON.parse(readFileSync(path.join(DATA, 'features/heldout.json'), 'utf8'))
+	let f = meta.features.cqt, size = f.bands * f.frames
+	let rows = meta.rows.map(r => Object.fromEntries(meta.keys.map((k, i) => [k, r[i]])))
+	let stats = new Float32Array(readFileSync(path.join(DATA, 'features/heldout.stats.f32')).buffer)
+	let voices = JSON.parse(readFileSync(path.join(DATA, 'voices.json'), 'utf8')).voices
+	// a fixed sample: clean queries ordered by a hash of their voice id
+	let picked = rows.map((r, i) => [r, i]).filter(([r, i]) => r.kind === 'query' && r.cond === 'clean' && stats[2 * i] >= 2 ** -13)
+		.sort((a, b) => seedOf(voices[a[0].voice].id, 'rebuild') - seedOf(voices[b[0].voice].id, 'rebuild')).slice(0, o.n)
+	if (o.misses) {
+		let miss = new Set(JSON.parse(readFileSync(o.misses, 'utf8')).results.filter(r => !r.hit).map(r => r.voice))
+		picked = picked.filter(([r]) => miss.has(voices[r.voice].id))
+	}
+	let fd = openSync(path.join(DATA, 'features/heldout.cqt.f16'), 'r')
+	let read = i => { let b = new Uint8Array(size * 2); readSync(fd, b, 0, b.length, i * size * 2); return Float32Array.from(new Float16Array(b.buffer)) }
+	let encoder = await createEncoder({ model: o.model ?? path.join(CACHE, 'encoder.int8.onnx') }) // the graph the catalogs bind
+	let cat = readCatalog(JSON.parse(readFileSync(path.join(CACHE, 'catalogs/dx7-heldout.json'), 'utf8')), encoder.binding)
+	let byId = new Map(voices.map((v, i) => [v.id, i]))
+	let emb = await encoder.project(picked.map(([, i]) => read(i)))
+	closeSync(fd); encoder.free()
+	let pool = voices.map((v, i) => [v, i]).filter(([v]) => v.split === 'heldout' && !v.silent).map(([, i]) => i)
+	let jobs = picked.map(([r], k) => {
+		let [best] = rank(emb[k], cat, 1), other = r.voice
+		for (let s = 0; other === r.voice; s++) other = pool[seedOf(voices[r.voice].id, 'random', s) % pool.length]
+		return { row: r, truth: r.voice, retrieved: byId.get(cat.entries[best.index].id), random: other, score: best.score }
+	})
+	let results = [], t0 = performance.now()
+	await Promise.all(Array.from({ length: WORKERS }, () => new Promise((resolve, reject) => {
+		let w = new Worker(new URL(import.meta.url), { workerData: { data: DATA, budget: o.budget } })
+		let next = () => jobs.length ? w.postMessage(jobs.shift()) : w.postMessage(null)
+		w.on('message', r => { results.push(r); console.log(`${results.length}/${picked.length}: ${r.name} ${r.hit ? 'hit' : 'miss'} random ${r.random.toFixed(3)} retrieved ${r.before.toFixed(3)} seeded ${r.seeded.toFixed(3)} blind ${r.blind.toFixed(3)} (${((performance.now() - t0) / 60000).toFixed(1)} min)`); next() })
+		w.on('error', reject); w.on('exit', resolve); next()
+	})))
+	let summary = group => {
+		let s = results.filter(group), m = k => s.reduce((a, r) => a + r[k], 0) / s.length
+		if (!s.length) return { n: 0 }
+		let med = k => { let v = s.map(r => r[k]).sort((a, b) => a - b); return v[v.length >> 1] }
+		let both = k => ({ mean: m(k), median: med(k) })
+		return { n: s.length, random: both('random'), retrieved: both('before'), seeded: both('seeded'), kept: both('kept'), blind: both('blind'),
+			seededBeatsBlind: s.filter(r => r.seeded < r.blind).length / s.length, seededBeatsRetrieved: s.filter(r => r.seeded < r.before).length / s.length,
+			blindBeatsRetrieved: s.filter(r => r.blind < r.before).length / s.length }
+	}
+	let out = { budget: o.budget, n: results.length, all: summary(() => true), misses: summary(r => !r.hit), hits: summary(r => r.hit), results }
+	writeFileSync(o.out ?? path.join(CACHE, 'rebuild.json'), JSON.stringify(out, null, 1))
+	console.log(JSON.stringify({ all: out.all, misses: out.misses }, null, 1))
+}
+
+async function worker() {
+	let { default: match, specLoss } = await import('@audio/neural-synth')
+	let { MAX } = await import('@audio/synth-dx7')
+	let patches = new Uint8Array(readFileSync(path.join(workerData.data, 'patches.u8')))
+	let voices = JSON.parse(readFileSync(path.join(workerData.data, 'voices.json'), 'utf8')).voices
+	let patch = i => patches.subarray(i * 155, i * 155 + 155)
+	let bounds = Array.from(MAX.subarray(0, 145), m => [0, m])
+	parentPort.on('message', job => {
+		if (!job) return parentPort.close()
+		let spec = { ...job.row, pre: 0 }, target = note(patch(job.truth), spec), seed = patch(job.retrieved)
+		let render = p => { let q = Uint8Array.from(seed); for (let i = 0; i < 145; i++) q[i] = Math.min(MAX[i], Math.max(0, Math.round(p[i]))); return note(q, spec) }
+		let before = specLoss(target, note(seed, spec)), random = specLoss(target, note(patch(job.random), spec))
+		let seeded = match(render, target, { bounds, seeds: [Array.from(seed.subarray(0, 145))], budget: workerData.budget })
+		let blind = match(render, target, { bounds, budget: workerData.budget })
+		parentPort.postMessage({ voice: voices[job.truth].id, name: voices[job.truth].name, retrieved: voices[job.retrieved].name, hit: job.truth === job.retrieved,
+			score: job.score, note: spec.note, velocity: spec.velocity, random, before, seeded: seeded.loss, kept: Math.min(seeded.loss, before), blind: blind.loss, evals: [seeded.evals, blind.evals] })
+	})
+}
+
+if (isMainThread) main().catch(e => { console.error(e); process.exit(1) })
+else worker()

package/scripts/recipe.js

@@ -0,0 +1,68 @@
+// What gets rendered: every row of a feature set as { voice, kind, note, velocity, hold, pre, cond,
+// seed }, drawn from seeds derived from the voice id, so a set is the same wherever it is built.
+//
+//   view   training view: any note 33-99, velocity 20-127, held or staccato, a training chain
+//   ref    catalog reference: C2, C3, C4, C5, C6 (MIDI 36-84) at velocities 64 and 112, held 1 s, clean
+//   query  benchmark query: a note and velocity no reference uses (MIDI 38-94, velocity 30-127),
+//          held or staccato, under one condition of augment.js CONDITIONS
+import { createHash } from 'node:crypto'
+import dx7 from '@audio/synth-dx7'
+import { rng, chain, CONDITIONS, FS } from './augment.js'
+
+export const REF_NOTES = [36, 48, 60, 72, 84], REF_VELOCITIES = [64, 112], REF_HOLD = 1
+export const LENGTH = 1.6 // seconds rendered after the pre-roll: the 1.52 s a view spans, plus codec delay
+export const seedOf = (...parts) => parseInt(createHash('sha1').update(parts.join(':')).digest('hex').slice(0, 8), 16)
+
+const hold = r => r() < 0.35 ? r.uniform(0.06, 0.4) : r.uniform(0.4, 2.5)
+const queryHold = r => r() < 0.5 ? r.uniform(0.08, 0.3) : r.uniform(0.6, 2)
+
+// another voice of the same split under a mix: pool is a list of voice rows, self excluded
+const other = (pool, self) => r => {
+	let v = self
+	while (v === self) v = r.pick(pool)
+	return { voice: v, note: r.int(40, 84), velocity: r.int(60, 120), hold: r.uniform(0.3, 1.5), offset: r.uniform(-0.5, 0.5) }
+}
+
+export function views(voice, id, count) {
+	return Array.from({ length: count }, (_, i) => {
+		let seed = seedOf(id, 'view', i), r = rng(seed)
+		return { voice, kind: 'view', note: r.int(33, 99), velocity: r.int(20, 127), hold: hold(r), pre: r.uniform(0, 0.03), cond: 'chain', seed }
+	})
+}
+
+export const refs = voice => REF_NOTES.flatMap(note => REF_VELOCITIES.map(velocity => ({ voice, kind: 'ref', note, velocity, hold: REF_HOLD, pre: 0, cond: 'clean', seed: 0 })))
+
+// One query draw per voice (note, velocity, hold, pre-roll), repeated under each condition so the
+// conditions compare on the same notes.
+export function queries(voice, id, conds) {
+	let r = rng(seedOf(id, 'query')), note, velocity
+	do note = r.int(38, 94); while (REF_NOTES.includes(note))
+	do velocity = r.int(30, 127); while (REF_VELOCITIES.includes(velocity))
+	let base = { voice, kind: 'query', note, velocity, hold: queryHold(r), pre: r.uniform(0, 0.05) }
+	return conds.map(cond => ({ ...base, cond, seed: seedOf(id, 'query', cond) }))
+}
+
+// The augmentation a row carries, from its seed (a view's note was drawn from the same seed, so
+// its chain takes a derived one).
+export function specOf(row, pool) {
+	let r = rng(row.kind === 'view' ? seedOf(row.seed, 'chain') : row.seed), o = other(pool, row.voice)
+	if (row.cond === 'chain' && row.kind === 'view') return chain(r, o)
+	return CONDITIONS[row.cond](r, o)
+}
+
+// The dry note: pre-roll of silence, then `LENGTH` seconds with the key up after `hold`.
+export function note(patch, { note, velocity, hold, pre = 0 }) {
+	let freq = 440 * 2 ** ((note - 69) / 12), held = Math.min(hold, LENGTH)
+	let y = dx7(freq, { patch, velocity: velocity / 127, duration: held, release: LENGTH - held, fs: FS })
+	let x = new Float32Array(Math.round(pre * FS) + y.length)
+	x.set(y, x.length - y.length)
+	return x
+}
+
+// the other voice's note under a mix, shifted by its offset against the target's onset
+export function mixNote(patch, m, pre) {
+	let y = note(patch, { ...m, pre: 0 }), shift = Math.round((pre + m.offset) * FS)
+	let x = new Float32Array(Math.round(pre * FS) + Math.round(LENGTH * FS))
+	for (let i = 0; i < x.length; i++) { let j = i - shift; if (j >= 0 && j < y.length) x[i] = y[j] }
+	return x
+}

package/scripts/train.py

@@ -0,0 +1,252 @@
+#!/usr/bin/env python3
+"""Train the timbre encoder on DX7 voices: supervised contrastive, on MPS.
+
+    python scripts/train.py --run NAME [--kind cqt] [--steps 30000] [--limit VOICES]
+
+Reads the feature sets scripts/factory.js writes (<data>/features/{train,dev}.json and
+.<kind>.f16) and writes ~/.cache/audiojs/neural/timbre/runs/NAME/{best.pt,progress.json}.
+
+Each step holds 12 voices, each with one of its 8 nearest training voices by parameters (the
+near-copies retrieval must tell apart, gpu-font's pairing of each face with a near design), 4
+views apiece. Loss: supervised contrastive over the batch's views (Khosla et al., NeurIPS 2020,
+L_out, temperature 0.1), a margin softmax of each view against a proxy per training voice
+(CosFace: Wang et al., CVPR 2018; scale 30 and margin 0.15 as gpu-font's style encoder), and the
+voice's algorithm (1-32) read linearly from the unit embedding (weight 0.2), the DX7's operator
+topology. The checkpoint kept is the best development top-1, averaged over clean and augmented
+queries against a catalog of development voices.
+"""
+import argparse, json, math, os, threading, queue, time
+from pathlib import Path
+import numpy as np
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+DATA = Path(os.environ.get('TIMBRE_DATA', Path.home() / '.cache/audiojs/data/dx7'))
+RUNS = Path(os.environ.get('AUDIO_NEURAL_CACHE', Path.home() / '.cache/audiojs/neural')) / 'timbre/runs'
+TEMPERATURE, SCALE, MARGIN, HEAD = 0.1, 30.0, 0.15, 0.2
+SILENT = 2 ** -13  # dry peak below 4 int16 steps: nothing to recognize
+
+
+class Encoder(nn.Module):
+    """Log-feature patch [B, 1, bands, frames] -> unit embedding [B, dims]. Four conv stages with 2x2
+    max pooling, a 1x1 conv down to `reduce` channels, then mean and max over time of each
+    (channel, band) cell and two linear layers. Frequency keeps its position to the end: brightness
+    is identity, not noise."""
+    def __init__(self, bands=96, dims=128, widths=(32, 64, 128, 256), reduce=96, hidden=512):
+        super().__init__()
+        layers, c = [], 1
+        for i, w in enumerate(widths):
+            for _ in range(1 if i == 0 else 2):
+                layers += [nn.Conv2d(c, w, 3, padding=1, bias=False), nn.BatchNorm2d(w), nn.ReLU(inplace=True)]
+                c = w
+            layers.append(nn.MaxPool2d(2))
+        layers += [nn.Conv2d(c, reduce, 1, bias=False), nn.BatchNorm2d(reduce), nn.ReLU(inplace=True)]
+        self.features = nn.Sequential(*layers)
+        cells = reduce * (bands // 2 ** len(widths))
+        self.head = nn.Sequential(nn.Linear(2 * cells, hidden), nn.BatchNorm1d(hidden), nn.ReLU(inplace=True), nn.Linear(hidden, dims))
+        self.config = {'bands': bands, 'dims': dims, 'widths': list(widths), 'reduce': reduce, 'hidden': hidden}
+
+    def forward(self, x):
+        h = self.features(x)                       # [B, C, F', T']
+        h = h.flatten(1, 2)                        # [B, C*F', T']
+        h = torch.cat([h.mean(2), h.amax(2)], 1)   # time pooled
+        return F.normalize(self.head(h), dim=1)
+
+
+class Store:
+    """A feature set on disk, read row by row with pread: the float16 file stays out of memory."""
+    def __init__(self, name, kind):
+        meta = json.loads((DATA / f'features/{name}.json').read_text())
+        f = meta['features'][kind]
+        self.shape, self.size = (f['bands'], f['frames']), f['bands'] * f['frames'] * 2
+        self.rows = [dict(zip(meta['keys'], r)) for r in meta['rows']]
+        self.fd = os.open(DATA / f'features/{name}.{kind}.f16', os.O_RDONLY)
+        stats = np.fromfile(DATA / f'features/{name}.stats.f32', np.float32).reshape(-1, 2)
+        self.audible = stats[:, 0] >= SILENT
+
+    def read(self, index):
+        out = np.empty((len(index), *self.shape), np.float16)
+        for k, i in enumerate(index):
+            out[k] = np.frombuffer(os.pread(self.fd, self.size, int(i) * self.size), np.float16).reshape(self.shape)
+        return out
+
+
+def voices():
+    return json.loads((DATA / 'voices.json').read_text())['voices']
+
+
+def canonical(patches):
+    """The sound-key bytes of scripts/voices.js: engine-ignored parameters zeroed."""
+    p = patches[:, :145].copy()
+    for op in range(6): p[:, op * 21 + 14] = 0
+    p[:, 140] = 0; p[:, 136] = 0
+    lfo = (p[:, 139] == 0) | (p[:, 143] == 0); p[lfo, 137:144] = 0
+    flat = (p[:, 130] == p[:, 131]) & (p[:, 131] == p[:, 132]) & (p[:, 132] == p[:, 133]); p[flat, 126:130] = 0
+    return p
+
+
+def neighbours(ids, k=8, device='mps'):
+    """The k nearest training voices of each by differing parameters, computed on the GPU in blocks."""
+    patches = np.fromfile(DATA / 'patches.u8', np.uint8).reshape(-1, 155)
+    p = torch.from_numpy(canonical(patches[ids]).astype(np.int16)).to(device)
+    out = []
+    for i in range(0, len(p), 64):
+        d = (p[i:i + 64, None, :] != p[None, :, :]).sum(2).float()
+        d[torch.arange(len(d)), torch.arange(i, i + len(d))] = 1e9
+        out.append(d.topk(k, largest=False).indices.cpu())
+    return torch.cat(out).numpy()
+
+
+class Batches:
+    """Background reader: `anchors` voices a step, each with one parameter neighbour, `views` views each."""
+    def __init__(self, store, voice_rows, neigh, anchors, views, seed):
+        self.store, self.voice_rows, self.neigh, self.anchors, self.views = store, voice_rows, neigh, anchors, views
+        self.rng = np.random.default_rng(seed); self.q = queue.Queue(maxsize=6); self.stop = False
+        self.thread = threading.Thread(target=self.run, daemon=True); self.thread.start()
+
+    def one(self):
+        n = len(self.voice_rows); chosen = []
+        seen = set()
+        for a in self.rng.permutation(n):
+            if len(chosen) >= 2 * self.anchors: break
+            b = int(self.rng.choice(self.neigh[a]))
+            if a in seen or b in seen: continue
+            chosen += [a, b]; seen |= {a, b}
+        rows, labels = [], []
+        for v in chosen:
+            r = self.voice_rows[v]
+            rows += list(self.rng.choice(r, self.views, replace=len(r) < self.views)); labels += [v] * self.views
+        return torch.from_numpy(self.store.read(rows)), torch.tensor(labels)
+
+    def run(self):
+        while not self.stop: self.q.put(self.one())
+
+    def __next__(self): return self.q.get()
+
+
+def supcon(z, labels, t=TEMPERATURE):
+    logits = (z @ z.T) / t
+    self_mask = torch.eye(len(z), dtype=torch.bool, device=z.device)
+    logits = logits.masked_fill(self_mask, -1e4)
+    log_prob = logits - torch.logsumexp(logits, 1, keepdim=True)
+    pos = (labels[:, None] == labels[None, :]) & ~self_mask
+    return (-(log_prob * pos).sum(1) / pos.sum(1).clamp(min=1)).mean()
+
+
+@torch.no_grad()
+def embed(model, store, rows, device, batch=128):
+    model.eval(); out = []
+    for i in range(0, len(rows), batch):
+        x = torch.from_numpy(store.read(rows[i:i + batch])).to(device).float()[:, None]
+        out.append(model(x).float().cpu())
+    return torch.cat(out)
+
+
+@torch.no_grad()
+def evaluate(model, store, device):
+    """Catalog: each development voice's mean reference embedding. Queries: one clean and one
+    augmented note per voice at a note and velocity no reference uses."""
+    rows = store.rows; ok = store.audible
+    refs = [i for i, r in enumerate(rows) if r['kind'] == 'ref' and ok[i]]
+    e = embed(model, store, refs, device); owners = np.array([rows[i]['voice'] for i in refs])
+    ids = np.unique(owners); pos = {v: k for k, v in enumerate(ids)}
+    cat = torch.zeros(len(ids), e.shape[1]).index_add_(0, torch.tensor([pos[o] for o in owners]), e)
+    cat = F.normalize(cat, dim=1)
+    report = {}
+    for cond in ('clean', 'chain'):
+        q = [i for i, r in enumerate(rows) if r['kind'] == 'query' and r['cond'] == cond and ok[i] and r['voice'] in pos]
+        qe = embed(model, store, q, device); truth = torch.tensor([pos[rows[i]['voice']] for i in q])
+        rank = (qe @ cat.T).argsort(1, descending=True)
+        hit = rank == truth[:, None]
+        report[cond] = {'top1': hit[:, :1].any(1).float().mean().item(), 'top5': hit[:, :5].any(1).float().mean().item(), 'n': len(q)}
+    report['catalog'] = len(ids)
+    report['select'] = (report['clean']['top1'] + report['chain']['top1']) / 2
+    return report
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument('--run', required=True)
+    ap.add_argument('--kind', default='cqt')
+    ap.add_argument('--train', default='train', help='feature set to train on (scripts/factory.js --name)')
+    ap.add_argument('--steps', type=int, default=30000)
+    ap.add_argument('--limit', type=int, default=0, help='train on the first N training voices of the set')
+    ap.add_argument('--anchors', type=int, default=12, help='voices a step, each with a parameter neighbour: 12 x 2 x 4 views = 96, what 3 GB of MPS memory holds with room')
+    ap.add_argument('--views', type=int, default=4)
+    ap.add_argument('--lr', type=float, default=2e-3)
+    ap.add_argument('--proxy', type=float, default=1.0, help='weight of the proxy loss, 0 for supervised contrastive alone')
+    ap.add_argument('--check', type=int, default=2500)
+    ap.add_argument('--seed', type=int, default=1)
+    a = ap.parse_args()
+
+    torch.set_num_threads(4); torch.manual_seed(a.seed)
+    device = 'mps'
+    torch.mps.set_per_process_memory_fraction(min(1.0, 3e9 / torch.mps.recommended_max_memory()))
+    out = RUNS / a.run; out.mkdir(parents=True, exist_ok=True)
+
+    train, dev = Store(a.train, a.kind), Store('dev', a.kind)
+    meta = voices()
+    by_voice = {}
+    for i, r in enumerate(train.rows):
+        if train.audible[i]: by_voice.setdefault(r['voice'], []).append(i)
+    ids = sorted(v for v, rs in by_voice.items() if len(rs) >= 2)
+    if a.limit: ids = ids[:a.limit]
+    voice_rows = [np.array(by_voice[v]) for v in ids]
+    algorithm = torch.tensor([meta[v]['algorithm'] - 1 for v in ids])
+    neigh = neighbours(np.array(ids), device=device)
+    print(f'{len(ids)} training voices, {sum(map(len, voice_rows))} views; dev {len(dev.rows)} rows', flush=True)
+
+    bands = train.shape[0]
+    model = Encoder(bands).to(device)
+    heads = nn.Linear(128, 32).to(device)
+    proxies = nn.Parameter(torch.randn(len(ids), 128, device=device) * 0.01)
+    params = sum(p.numel() for p in model.parameters())
+    print(f'encoder {params} parameters', flush=True)
+    opt = torch.optim.AdamW([{'params': model.parameters(), 'lr': a.lr, 'weight_decay': 1e-4},
+                             {'params': heads.parameters(), 'lr': a.lr, 'weight_decay': 1e-4},
+                             {'params': [proxies], 'lr': 10 * a.lr, 'weight_decay': 0}])
+    sched = torch.optim.lr_scheduler.LambdaLR(opt, lambda s: min(1, (s + 1) / 500) * (0.02 + 0.98 * 0.5 * (1 + math.cos(math.pi * min(s, a.steps) / a.steps))))
+    batches = Batches(train, voice_rows, neigh, a.anchors, a.views, a.seed)
+    history, best, t0, running = [], -1, time.time(), {}
+
+    def check(step):
+        nonlocal best
+        torch.mps.empty_cache()
+        rep = evaluate(model, dev, device); rep['step'] = step; rep['seconds'] = round(time.time() - t0)
+        torch.mps.empty_cache()
+        history.append(rep)
+        if rep['select'] > best:
+            best = rep['select']
+            torch.save({'state': {k: v.cpu() for k, v in model.state_dict().items()}, 'config': model.config,
+                        'heads': {k: v.cpu() for k, v in heads.state_dict().items()}, 'step': step, 'kind': a.kind,
+                        'features': json.loads((DATA / f'features/{a.train}.json').read_text())['features'][a.kind]}, out / 'best.pt')
+        (out / 'progress.json').write_text(json.dumps({'args': vars(a), 'voices': len(ids), 'parameters': params, 'best': best, 'history': history}, indent=1))
+        print(f"check {step}: clean top1 {rep['clean']['top1']:.3f} top5 {rep['clean']['top5']:.3f} | chain top1 {rep['chain']['top1']:.3f} top5 {rep['chain']['top5']:.3f} "
+              f"({rep['catalog']} voices, {rep['seconds']}s, {torch.mps.driver_allocated_memory() / 2**30:.1f} GB)", flush=True)
+
+    try:
+        for step in range(1, a.steps + 1):
+            model.train(); heads.train()
+            x, labels = next(batches)
+            x = x.to(device, non_blocking=True).float()[:, None]; labels = labels.to(device)
+            with torch.autocast('mps', dtype=torch.bfloat16):
+                z = model(x).float()
+            con = supcon(z, labels)
+            cos = z @ F.normalize(proxies, dim=1).T
+            onehot = F.one_hot(labels, len(ids)).bool()
+            prox = F.cross_entropy(SCALE * (cos - MARGIN * onehot), labels) if a.proxy else z.sum() * 0
+            alg = F.cross_entropy(heads(z) * 10, algorithm.to(device)[labels])
+            loss = con + a.proxy * prox + HEAD * alg
+            opt.zero_grad(set_to_none=True); loss.backward(); opt.step(); sched.step()
+            for k, v in (('supcon', con), ('proxy', prox), ('algorithm', alg)): running[k] = running.get(k, 0) + v.detach()
+            if step % 250 == 0:
+                print(f'step {step}: ' + ' '.join(f'{k} {float(v) / 250:.3f}' for k, v in running.items()) + f' ({time.time() - t0:.0f}s)', flush=True); running = {}
+                torch.mps.empty_cache()
+            if step % a.check == 0 or step == a.steps: check(step)
+    finally:
+        batches.stop = True
+
+
+if __name__ == '__main__':
+    main()