@octoseq/mir 0.1.0-main.2e286ce

package/src/dsp/mel.ts

@@ -0,0 +1,219 @@
+import type { MirGPU } from "../gpu/context";
+import { gpuMelProjectFlat } from "../gpu/melProject";
+
+import type { Spectrogram } from "./spectrogram";
+
+export type MelConfig = {
+    nMels: number;
+    fMin?: number;
+    fMax?: number;
+};
+
+export type MelSpectrogram = {
+    times: Float32Array;
+    melBands: Float32Array[]; // [frame][mel]
+    /** Optional observability. Present when GPU path runs. */
+    gpuTimings?: {
+        gpuSubmitToReadbackMs: number;
+    };
+};
+
+function assertPositiveInt(name: string, value: number): void {
+    if (!Number.isFinite(value) || value <= 0 || (value | 0) !== value) {
+        throw new Error(`@octoseq/mir: ${name} must be a positive integer`);
+    }
+}
+
+function hzToMel(hz: number): number {
+    // Slaney-style mel approximation (HTK-like).
+    return 2595 * Math.log10(1 + hz / 700);
+}
+
+function melToHz(mel: number): number {
+    return 700 * (Math.pow(10, mel / 2595) - 1);
+}
+
+function buildMelFilterBank(
+    sampleRate: number,
+    fftSize: number,
+    nMels: number,
+    fMin: number,
+    fMax: number
+): Float32Array[] {
+    const nBins = (fftSize >>> 1) + 1;
+    const nyquist = sampleRate / 2;
+
+    const fMinClamped = Math.max(0, Math.min(fMin, nyquist));
+    const fMaxClamped = Math.max(0, Math.min(fMax, nyquist));
+    if (fMaxClamped <= fMinClamped) {
+        throw new Error("@octoseq/mir: mel fMax must be > fMin");
+    }
+
+    // We create nMels triangular filters defined by nMels+2 mel points.
+    const melMin = hzToMel(fMinClamped);
+    const melMax = hzToMel(fMaxClamped);
+
+    const melPoints = new Float32Array(nMels + 2);
+    for (let i = 0; i < melPoints.length; i++) {
+        melPoints[i] = melMin + (i * (melMax - melMin)) / (nMels + 1);
+    }
+
+    const hzPoints = new Float32Array(melPoints.length);
+    for (let i = 0; i < hzPoints.length; i++) hzPoints[i] = melToHz(melPoints[i] ?? 0);
+
+    const binHz = sampleRate / fftSize;
+    const binPoints = new Int32Array(hzPoints.length);
+    for (let i = 0; i < binPoints.length; i++) {
+        binPoints[i] = Math.max(0, Math.min(nBins - 1, Math.round((hzPoints[i] ?? 0) / binHz)));
+    }
+
+    const filters: Float32Array[] = new Array(nMels);
+    for (let m = 0; m < nMels; m++) {
+        const left = binPoints[m] ?? 0;
+        const center = binPoints[m + 1] ?? 0;
+        const right = binPoints[m + 2] ?? 0;
+
+        const w = new Float32Array(nBins);
+        if (center === left || right === center) {
+            filters[m] = w;
+            continue;
+        }
+
+        for (let k = left; k < center; k++) {
+            w[k] = (k - left) / (center - left);
+        }
+        for (let k = center; k < right; k++) {
+            w[k] = (right - k) / (right - center);
+        }
+
+        filters[m] = w;
+    }
+
+    return filters;
+}
+
+function cpuMelProject(
+    spec: Spectrogram,
+    filters: Float32Array[]
+): MelSpectrogram {
+    const nFrames = spec.times.length;
+    const nMels = filters.length;
+    const out: Float32Array[] = new Array(nFrames);
+
+    const eps = 1e-12;
+
+    for (let t = 0; t < nFrames; t++) {
+        const mags = spec.magnitudes[t];
+        if (!mags) {
+            out[t] = new Float32Array(nMels);
+            continue;
+        }
+
+        const bands = new Float32Array(nMels);
+        for (let m = 0; m < nMels; m++) {
+            const w = filters[m];
+            if (!w) continue;
+
+            let sum = 0;
+            // Project linear magnitudes onto mel filters.
+            for (let k = 0; k < mags.length; k++) {
+                sum += (mags[k] ?? 0) * (w[k] ?? 0);
+            }
+
+            // Log scaling for visualisation / downstream features.
+            bands[m] = Math.log10(eps + sum);
+        }
+        out[t] = bands;
+    }
+
+    return {
+        times: spec.times,
+        melBands: out
+    };
+}
+
+async function gpuMelProject(
+    spec: Spectrogram,
+    filters: Float32Array[],
+    gpu: MirGPU
+): Promise<MelSpectrogram> {
+    const nFrames = spec.times.length;
+    const nBins = (spec.fftSize >>> 1) + 1;
+    const nMels = filters.length;
+
+    const magsFlat = new Float32Array(nFrames * nBins);
+    for (let t = 0; t < nFrames; t++) {
+        const mags = spec.magnitudes[t];
+        if (!mags) continue;
+        magsFlat.set(mags, t * nBins);
+    }
+
+    const filterFlat = new Float32Array(nMels * nBins);
+    for (let m = 0; m < nMels; m++) {
+        filterFlat.set(filters[m] ?? new Float32Array(nBins), m * nBins);
+    }
+
+    // GPU stage timing (submission -> readback) is surfaced for validation/debug.
+    const { value, timing } = await gpuMelProjectFlat(gpu, {
+        nFrames,
+        nBins,
+        nMels,
+        magsFlat,
+        filterFlat,
+    });
+
+    const outFlat = value.outFlat;
+
+    const melBands: Float32Array[] = new Array(nFrames);
+    for (let t = 0; t < nFrames; t++) {
+        // Keep zero-copy views into the single flat buffer.
+        melBands[t] = outFlat.subarray(t * nMels, (t + 1) * nMels);
+    }
+
+    return {
+        times: spec.times,
+        melBands,
+        gpuTimings: {
+            gpuSubmitToReadbackMs: timing.gpuSubmitToReadbackMs,
+        },
+    };
+}
+
+/**
+ * Compute a (log) mel spectrogram by projecting an existing spectrogram.
+ *
+ * Design rule compliance:
+ * - The caller provides the spectrogram (we do not hide STFT internally).
+ * - Output is aligned to `spec.times`.
+ */
+export async function melSpectrogram(
+    spec: Spectrogram,
+    config: MelConfig,
+    gpu?: MirGPU
+): Promise<MelSpectrogram> {
+    assertPositiveInt("config.nMels", config.nMels);
+
+    const fMin = config.fMin ?? 0;
+    const fMax = config.fMax ?? spec.sampleRate / 2;
+
+    const filters = buildMelFilterBank(
+        spec.sampleRate,
+        spec.fftSize,
+        config.nMels,
+        fMin,
+        fMax
+    );
+
+    if (gpu) {
+        // Try GPU; if anything goes wrong, fall back to CPU.
+        try {
+            return await gpuMelProject(spec, filters, gpu);
+        } catch {
+            // GPU can fail due to missing features, adapter resets, etc.
+            // v0.1 prioritises correctness: we silently fall back.
+            return cpuMelProject(spec, filters);
+        }
+    }
+
+    return cpuMelProject(spec, filters);
+}

package/src/dsp/mfcc.ts

@@ -0,0 +1,119 @@
+import type { MelSpectrogram } from "./mel";
+
+export type MfccOptions = {
+    nCoeffs?: number;
+};
+
+export type MfccResult = {
+    times: Float32Array;
+    coeffs: Float32Array[]; // [frame][coeff]
+};
+
+function assertPositiveInt(name: string, v: number): void {
+    if (!Number.isFinite(v) || v <= 0 || (v | 0) !== v) {
+        throw new Error(`@octoseq/mir: ${name} must be a positive integer`);
+    }
+}
+
+function buildDctMatrix(nCoeffs: number, nMels: number): Float32Array {
+    // DCT-II (ortho-ish scaling). Many MFCC refs use a scaled DCT; for visualisation and
+    // relative features this is sufficient and stable.
+    // Shape: [nCoeffs][nMels]
+    const out = new Float32Array(nCoeffs * nMels);
+
+    const scale0 = Math.sqrt(1 / nMels);
+    const scale = Math.sqrt(2 / nMels);
+
+    for (let i = 0; i < nCoeffs; i++) {
+        for (let j = 0; j < nMels; j++) {
+            const c = Math.cos((Math.PI / nMels) * (j + 0.5) * i);
+            out[i * nMels + j] = (i === 0 ? scale0 : scale) * c;
+        }
+    }
+
+    return out;
+}
+
+export function mfcc(mel: MelSpectrogram, options: MfccOptions = {}): MfccResult {
+    const nFrames = mel.times.length;
+    const nMels = mel.melBands[0]?.length ?? 0;
+
+    const nCoeffs = options.nCoeffs ?? 13;
+    assertPositiveInt("options.nCoeffs", nCoeffs);
+    if (nMels <= 0) {
+        return { times: mel.times, coeffs: new Array(nFrames).fill(0).map(() => new Float32Array(nCoeffs)) };
+    }
+
+    const dct = buildDctMatrix(nCoeffs, nMels);
+
+    const out: Float32Array[] = new Array(nFrames);
+    for (let t = 0; t < nFrames; t++) {
+        const x = mel.melBands[t] ?? new Float32Array(nMels);
+
+        // melSpectrogram already returns log10 energies. For MFCC we typically use ln energies.
+        // We keep it simple here: treat the existing log-scaled values as log-energy features.
+        const c = new Float32Array(nCoeffs);
+
+        for (let i = 0; i < nCoeffs; i++) {
+            let sum = 0;
+            const rowOff = i * nMels;
+            for (let j = 0; j < nMels; j++) {
+                sum += (dct[rowOff + j] ?? 0) * (x[j] ?? 0);
+            }
+            c[i] = sum;
+        }
+
+        out[t] = c;
+    }
+
+    return { times: mel.times, coeffs: out };
+}
+
+export type DeltaOptions = {
+    /** Regression window size N (frames). Standard choice is 2. */
+    window?: number;
+};
+
+export type Features2D = {
+    times: Float32Array;
+    values: Float32Array[]; // [frame][feature]
+};
+
+export function delta(features: Features2D, options: DeltaOptions = {}): Features2D {
+    const N = options.window ?? 2;
+    assertPositiveInt("options.window", N);
+
+    const nFrames = features.times.length;
+    const nFeat = features.values[0]?.length ?? 0;
+
+    const out: Float32Array[] = new Array(nFrames);
+
+    // denom = 2 * sum_{n=1..N} n^2
+    let denom = 0;
+    for (let n = 1; n <= N; n++) denom += n * n;
+    denom *= 2;
+
+    for (let t = 0; t < nFrames; t++) {
+        const d = new Float32Array(nFeat);
+
+        for (let f = 0; f < nFeat; f++) {
+            let num = 0;
+            for (let n = 1; n <= N; n++) {
+                const tPlus = Math.min(nFrames - 1, t + n);
+                const tMinus = Math.max(0, t - n);
+                const a = features.values[tPlus]?.[f] ?? 0;
+                const b = features.values[tMinus]?.[f] ?? 0;
+                num += n * (a - b);
+            }
+            d[f] = denom > 0 ? num / denom : 0;
+        }
+
+        out[t] = d;
+    }
+
+    return { times: features.times, values: out };
+}
+
+export function deltaDelta(features: Features2D, options: DeltaOptions = {}): Features2D {
+    return delta(delta(features, options), options);
+}

package/src/dsp/onset.ts

@@ -0,0 +1,205 @@
+import type { MirGPU } from "../gpu/context";
+import { gpuOnsetEnvelopeFromMelFlat } from "../gpu/onsetEnvelope";
+
+import type { MelSpectrogram } from "./mel";
+import type { Spectrogram } from "./spectrogram";
+
+export type OnsetEnvelope = {
+    times: Float32Array;
+    values: Float32Array;
+};
+
+export type OnsetEnvelopeOptions = {
+    /** If true, log-compress magnitudes/energies before differencing. */
+    useLog?: boolean;
+    /** Moving-average smoothing window length in milliseconds. 0 disables smoothing. */
+    smoothMs?: number;
+    /** How to convert temporal differences into novelty. */
+    diffMethod?: "rectified" | "abs";
+};
+
+function movingAverage(values: Float32Array, windowFrames: number): Float32Array {
+    if (windowFrames <= 1) return values;
+
+    const n = values.length;
+    const out = new Float32Array(n);
+
+    // Centered window.
+    const half = Math.floor(windowFrames / 2);
+
+    // Prefix sums for stable, bug-free O(n) moving average.
+    const prefix = new Float64Array(n + 1);
+    prefix[0] = 0;
+    for (let i = 0; i < n; i++) {
+        prefix[i + 1] = (prefix[i] ?? 0) + (values[i] ?? 0);
+    }
+
+    for (let i = 0; i < n; i++) {
+        const start = Math.max(0, i - half);
+        const end = Math.min(n, i + half + 1);
+        const sum = (prefix[end] ?? 0) - (prefix[start] ?? 0);
+        const count = Math.max(1, end - start);
+        out[i] = sum / count;
+    }
+
+    return out;
+}
+
+function defaultOptions(opts?: OnsetEnvelopeOptions): Required<OnsetEnvelopeOptions> {
+    return {
+        useLog: opts?.useLog ?? false,
+        smoothMs: opts?.smoothMs ?? 30,
+        diffMethod: opts?.diffMethod ?? "rectified",
+    };
+}
+
+function logCompress(x: number): number {
+    // Stable compression without -Inf.
+    // We use ln(1+x) so it behaves well for both linear mags and log-mel (already log10).
+    return Math.log1p(Math.max(0, x));
+}
+
+export function onsetEnvelopeFromSpectrogram(spec: Spectrogram, options?: OnsetEnvelopeOptions): OnsetEnvelope {
+    const opts = defaultOptions(options);
+
+    const nFrames = spec.times.length;
+    const out = new Float32Array(nFrames);
+
+    const nBins = (spec.fftSize >>> 1) + 1;
+
+    // First frame has no previous frame.
+    out[0] = 0;
+
+    for (let t = 1; t < nFrames; t++) {
+        const cur = spec.magnitudes[t];
+        const prev = spec.magnitudes[t - 1];
+        if (!cur || !prev) {
+            out[t] = 0;
+            continue;
+        }
+
+        let sum = 0;
+        for (let k = 0; k < nBins; k++) {
+            let a = cur[k] ?? 0;
+            let b = prev[k] ?? 0;
+            if (opts.useLog) {
+                a = logCompress(a);
+                b = logCompress(b);
+            }
+            const d = a - b;
+            sum += opts.diffMethod === "abs" ? Math.abs(d) : Math.max(0, d);
+        }
+
+        // Use an average over frequency bins so the overall scale is not tied to FFT size.
+        out[t] = nBins > 0 ? sum / nBins : 0;
+    }
+
+    // Optional smoothing based on average frame spacing.
+    const smoothMs = opts.smoothMs;
+    if (smoothMs > 0 && nFrames >= 2) {
+        const dt = (spec.times[1] ?? 0) - (spec.times[0] ?? 0);
+        const windowFrames = Math.max(1, Math.round((smoothMs / 1000) / Math.max(1e-9, dt)));
+        return {
+            times: spec.times,
+            values: movingAverage(out, windowFrames | 1),
+        };
+    }
+
+    return { times: spec.times, values: out };
+}
+
+export function onsetEnvelopeFromMel(mel: MelSpectrogram, options?: OnsetEnvelopeOptions): OnsetEnvelope {
+    const opts = defaultOptions(options);
+
+    const nFrames = mel.times.length;
+    const out = new Float32Array(nFrames);
+
+    out[0] = 0;
+
+    for (let t = 1; t < nFrames; t++) {
+        const cur = mel.melBands[t];
+        const prev = mel.melBands[t - 1];
+        if (!cur || !prev) {
+            out[t] = 0;
+            continue;
+        }
+
+        const nBands = cur.length;
+
+        let sum = 0;
+        for (let m = 0; m < nBands; m++) {
+            let a = cur[m] ?? 0;
+            let b = prev[m] ?? 0;
+
+            // Note: melSpectrogram currently outputs log10(eps + energy).
+            // If useLog is requested, we apply an additional stable compression.
+            if (opts.useLog) {
+                a = logCompress(a);
+                b = logCompress(b);
+            }
+
+            const d = a - b;
+            sum += opts.diffMethod === "abs" ? Math.abs(d) : Math.max(0, d);
+        }
+
+        // Use an average over bands so the overall scale is not tied to nMels.
+        out[t] = nBands > 0 ? sum / nBands : 0;
+    }
+
+    const smoothMs = opts.smoothMs;
+    if (smoothMs > 0 && nFrames >= 2) {
+        const dt = (mel.times[1] ?? 0) - (mel.times[0] ?? 0);
+        const windowFrames = Math.max(1, Math.round((smoothMs / 1000) / Math.max(1e-9, dt)));
+        return {
+            times: mel.times,
+            values: movingAverage(out, windowFrames | 1),
+        };
+    }
+
+    return { times: mel.times, values: out };
+}
+
+export type OnsetEnvelopeGpuResult = {
+    times: Float32Array;
+    values: Float32Array;
+    gpuTimings: { gpuSubmitToReadbackMs: number };
+};
+
+/**
+ * GPU-accelerated onset envelope from mel spectrogram.
+ *
+ * Notes:
+ * - This bypasses JS loops for the diff+reduction step.
+ * - Smoothing/log options are intentionally limited for v0.1 (keeps WGSL simple).
+ * - Callers should fall back to CPU on errors.
+ */
+export async function onsetEnvelopeFromMelGpu(
+    mel: MelSpectrogram,
+    gpu: MirGPU,
+    options?: Pick<OnsetEnvelopeOptions, "diffMethod">
+): Promise<OnsetEnvelopeGpuResult> {
+    const nFrames = mel.times.length;
+    const nMels = mel.melBands[0]?.length ?? 0;
+
+    const melFlat = new Float32Array(nFrames * nMels);
+    for (let t = 0; t < nFrames; t++) {
+        const row = mel.melBands[t];
+        if (!row) continue;
+        melFlat.set(row, t * nMels);
+    }
+
+    const diffMethod = options?.diffMethod ?? "rectified";
+
+    const { value, timing } = await gpuOnsetEnvelopeFromMelFlat(gpu, {
+        nFrames,
+        nMels,
+        melFlat,
+        diffMethod,
+    });
+
+    return {
+        times: mel.times,
+        values: value.out,
+        gpuTimings: { gpuSubmitToReadbackMs: timing.gpuSubmitToReadbackMs },
+    };
+}

package/src/dsp/peakPick.ts

@@ -0,0 +1,112 @@
+export type PeakPickEvent = {
+    time: number;
+    strength: number;
+    index: number;
+};
+
+export type PeakPickOptions = {
+    /** Minimum peak height (absolute). */
+    threshold?: number;
+    /** Minimum inter-peak interval (seconds). */
+    minIntervalSec?: number;
+
+    /** If provided, use adaptive threshold: mean(values) + factor*std(values). */
+    adaptive?: {
+        method?: "meanStd" | "median";
+        factor?: number;
+    };
+
+    /** If true, prefer strict maxima (> neighbors); else allow flat plateaus. */
+    strict?: boolean;
+};
+
+function meanStd(values: Float32Array): { mean: number; std: number } {
+    const n = values.length;
+    if (n <= 0) return { mean: 0, std: 0 };
+
+    let mean = 0;
+    for (let i = 0; i < n; i++) mean += values[i] ?? 0;
+    mean /= n;
+
+    let varSum = 0;
+    for (let i = 0; i < n; i++) {
+        const d = (values[i] ?? 0) - mean;
+        varSum += d * d;
+    }
+
+    const std = Math.sqrt(varSum / n);
+    return { mean, std };
+}
+
+function median(values: Float32Array): number {
+    const arr = Array.from(values);
+    arr.sort((a, b) => a - b);
+    const n = arr.length;
+    if (n === 0) return 0;
+    const mid = n >>> 1;
+    if (n % 2 === 1) return arr[mid] ?? 0;
+    return ((arr[mid - 1] ?? 0) + (arr[mid] ?? 0)) / 2;
+}
+
+export function peakPick(
+    times: Float32Array,
+    values: Float32Array,
+    options: PeakPickOptions = {}
+): PeakPickEvent[] {
+    if (times.length !== values.length) {
+        throw new Error("@octoseq/mir: peakPick times/values length mismatch");
+    }
+
+    const n = values.length;
+    if (n === 0) return [];
+
+    const strict = options.strict ?? true;
+
+    let thr = options.threshold ?? 0;
+    if (options.adaptive) {
+        const method = options.adaptive.method ?? "meanStd";
+        const factor = options.adaptive.factor ?? 1;
+
+        if (method === "median") {
+            thr = median(values) * factor;
+        } else {
+            const { mean, std } = meanStd(values);
+            thr = mean + factor * std;
+        }
+    }
+
+    const minIntervalSec = options.minIntervalSec ?? 0;
+
+    const out: PeakPickEvent[] = [];
+
+    let lastPeakTime = -Infinity;
+
+    for (let i = 1; i < n - 1; i++) {
+        const v = values[i] ?? 0;
+        if (!(v >= thr)) continue;
+
+        const prev = values[i - 1] ?? 0;
+        const next = values[i + 1] ?? 0;
+
+        const isMax = strict ? v > prev && v > next : v >= prev && v >= next;
+        if (!isMax) continue;
+
+        const t = times[i] ?? 0;
+        if (t - lastPeakTime < minIntervalSec) {
+            // If we're within the minimum interval, keep the stronger peak.
+            const last = out[out.length - 1];
+            if (last && v > last.strength) {
+                last.time = t;
+                last.strength = v;
+                last.index = i;
+                lastPeakTime = t;
+            }
+            continue;
+        }
+
+        out.push({ time: t, strength: v, index: i });
+        lastPeakTime = t;
+    }
+
+    return out;
+}