@octoseq/mir 0.1.0-main.4ecb074 → 0.1.0-main.5fdb072

package/src/dsp/beatCandidates.ts

@@ -0,0 +1,299 @@
+import type { MelSpectrogram } from "./mel";
+import type { OnsetEnvelope } from "./onset";
+import { onsetEnvelopeFromMel } from "./onset";
+import type { Spectrogram } from "./spectrogram";
+import { spectralFlux } from "./spectral";
+import type { BeatCandidate, BeatCandidateSource } from "../types";
+
+/**
+ * Configuration for beat candidate detection.
+ */
+export type BeatCandidatesOptions = {
+    /** Minimum inter-candidate interval in seconds. Default: 0.1 (100ms). */
+    minIntervalSec?: number;
+    /** Threshold factor for adaptive peak detection. Lower = more candidates. Default: 0.5. */
+    thresholdFactor?: number;
+    /** Smoothing window for salience signal in ms. Default: 50. */
+    smoothMs?: number;
+};
+
+/**
+ * Result of beat candidate detection.
+ */
+export type BeatCandidatesOutput = {
+    candidates: BeatCandidate[];
+    /** The computed salience signal (for debugging/visualization). */
+    salience: {
+        times: Float32Array;
+        values: Float32Array;
+    };
+};
+
+/**
+ * Compute a beat-oriented salience signal from mel spectrogram.
+ *
+ * This combines:
+ * - Onset envelope (captures transients/attacks)
+ * - Spectral flux from the underlying spectrogram (captures spectral change)
+ *
+ * The signals are normalized and combined to produce a single salience curve
+ * suitable for peak picking.
+ *
+ * Key design choices:
+ * - Whole-track normalization (z-score) for consistent behavior
+ * - Gentle smoothing to suppress micro-transients while preserving beat structure
+ * - No BPM inference or grid assumptions
+ */
+export type BeatSalienceSignal = {
+    times: Float32Array;
+    values: Float32Array;
+};
+
+function movingAverage(values: Float32Array, windowFrames: number): Float32Array {
+    if (windowFrames <= 1) return values;
+
+    const n = values.length;
+    const out = new Float32Array(n);
+
+    const half = Math.floor(windowFrames / 2);
+
+    // Prefix sums for O(n) moving average.
+    const prefix = new Float64Array(n + 1);
+    prefix[0] = 0;
+    for (let i = 0; i < n; i++) {
+        prefix[i + 1] = (prefix[i] ?? 0) + (values[i] ?? 0);
+    }
+
+    for (let i = 0; i < n; i++) {
+        const start = Math.max(0, i - half);
+        const end = Math.min(n, i + half + 1);
+        const sum = (prefix[end] ?? 0) - (prefix[start] ?? 0);
+        const count = Math.max(1, end - start);
+        out[i] = sum / count;
+    }
+
+    return out;
+}
+
+function meanStd(values: Float32Array): { mean: number; std: number } {
+    const n = values.length;
+    if (n <= 0) return { mean: 0, std: 0 };
+
+    let mean = 0;
+    for (let i = 0; i < n; i++) mean += values[i] ?? 0;
+    mean /= n;
+
+    let varSum = 0;
+    for (let i = 0; i < n; i++) {
+        const d = (values[i] ?? 0) - mean;
+        varSum += d * d;
+    }
+
+    const std = Math.sqrt(varSum / n);
+    return { mean, std };
+}
+
+/**
+ * Z-score normalize a signal (whole-track normalization).
+ * Result has mean ~0 and std ~1.
+ */
+function zScoreNormalize(values: Float32Array): Float32Array {
+    const { mean, std } = meanStd(values);
+    const n = values.length;
+    const out = new Float32Array(n);
+
+    if (std === 0 || !Number.isFinite(std)) {
+        // Degenerate case: all values are the same
+        out.fill(0);
+        return out;
+    }
+
+    for (let i = 0; i < n; i++) {
+        out[i] = ((values[i] ?? 0) - mean) / std;
+    }
+
+    return out;
+}
+
+/**
+ * Min-max normalize to [0, 1] range.
+ */
+function minMaxNormalize(values: Float32Array): Float32Array {
+    const n = values.length;
+    if (n === 0) return new Float32Array(0);
+
+    let min = Infinity;
+    let max = -Infinity;
+    for (let i = 0; i < n; i++) {
+        const v = values[i] ?? 0;
+        if (v < min) min = v;
+        if (v > max) max = v;
+    }
+
+    const out = new Float32Array(n);
+    const range = max - min;
+
+    if (range === 0 || !Number.isFinite(range)) {
+        out.fill(0.5);
+        return out;
+    }
+
+    for (let i = 0; i < n; i++) {
+        out[i] = ((values[i] ?? 0) - min) / range;
+    }
+
+    return out;
+}
+
+/**
+ * Compute beat salience signal from mel spectrogram.
+ *
+ * This is an intermediate signal suitable for peak picking to extract
+ * beat candidates. It combines onset envelope with additional smoothing
+ * tuned for beat-like (rather than onset-like) detection.
+ */
+export function beatSalienceFromMel(
+    mel: MelSpectrogram,
+    spec: Spectrogram,
+    options?: { smoothMs?: number }
+): BeatSalienceSignal {
+    const smoothMs = options?.smoothMs ?? 50;
+
+    // Compute onset envelope with more smoothing than default onset detection.
+    // We want to capture the "attack envelope" of beats, not individual onsets.
+    const onset = onsetEnvelopeFromMel(mel, {
+        smoothMs: smoothMs,
+        diffMethod: "rectified",
+        useLog: false,
+    });
+
+    // Compute spectral flux from the spectrogram.
+    const flux = spectralFlux(spec);
+
+    // Ensure times align (they should, but be defensive).
+    const n = Math.min(onset.times.length, flux.length);
+
+    // Z-score normalize both signals for equal contribution.
+    const onsetNorm = zScoreNormalize(onset.values.subarray(0, n));
+    const fluxNorm = zScoreNormalize(flux.subarray(0, n));
+
+    // Combine: weighted sum favoring onset envelope (it's more beat-specific).
+    const combined = new Float32Array(n);
+    const onsetWeight = 0.7;
+    const fluxWeight = 0.3;
+
+    for (let i = 0; i < n; i++) {
+        combined[i] = onsetWeight * (onsetNorm[i] ?? 0) + fluxWeight * (fluxNorm[i] ?? 0);
+    }
+
+    // Apply final smoothing to reduce micro-peaks.
+    const dt = n >= 2 ? ((onset.times[1] ?? 0) - (onset.times[0] ?? 0)) : 0.01;
+    const windowFrames = Math.max(1, Math.round((smoothMs / 1000) / Math.max(1e-9, dt)));
+    const smoothed = movingAverage(combined, windowFrames | 1);
+
+    // Normalize to [0, 1] for consistent interpretation.
+    const normalized = minMaxNormalize(smoothed);
+
+    return {
+        times: onset.times.subarray(0, n),
+        values: normalized,
+    };
+}
+
+/**
+ * Pick peaks from the salience signal to extract beat candidates.
+ *
+ * Uses relaxed parameters to err on the side of too many candidates.
+ * The goal is coverage, not precision.
+ */
+function pickBeatCandidates(
+    salience: BeatSalienceSignal,
+    options: BeatCandidatesOptions,
+    source: BeatCandidateSource
+): BeatCandidate[] {
+    const minIntervalSec = options.minIntervalSec ?? 0.1;
+    const thresholdFactor = options.thresholdFactor ?? 0.5;
+
+    const { times, values } = salience;
+    const n = values.length;
+
+    if (n < 3) return [];
+
+    // Compute adaptive threshold based on signal statistics.
+    const { mean, std } = meanStd(values);
+    // Low threshold to get dense candidates.
+    // thresholdFactor of 0.5 means: mean + 0.5*std (quite low).
+    const threshold = mean + thresholdFactor * std;
+
+    const candidates: BeatCandidate[] = [];
+    let lastPeakTime = -Infinity;
+
+    for (let i = 1; i < n - 1; i++) {
+        const v = values[i] ?? 0;
+
+        // Must be above threshold.
+        if (v < threshold) continue;
+
+        // Must be a local maximum.
+        const prev = values[i - 1] ?? 0;
+        const next = values[i + 1] ?? 0;
+        if (!(v > prev && v > next)) continue;
+
+        const t = times[i] ?? 0;
+
+        // Enforce minimum interval.
+        if (t - lastPeakTime < minIntervalSec) {
+            // If within interval, keep the stronger peak.
+            const last = candidates[candidates.length - 1];
+            if (last && v > last.strength) {
+                last.time = t;
+                last.strength = v;
+            }
+            continue;
+        }
+
+        candidates.push({
+            time: t,
+            strength: v,
+            source,
+        });
+        lastPeakTime = t;
+    }
+
+    return candidates;
+}
+
+/**
+ * Detect beat candidates from mel spectrogram and spectrogram.
+ *
+ * This is the main entry point for beat candidate detection.
+ *
+ * Design principles:
+ * - Dense candidates (err on side of too many)
+ * - No BPM inference
+ * - No grid assumptions
+ * - Whole-track normalization for consistency
+ * - Deterministic (same input -> same output)
+ */
+export function detectBeatCandidates(
+    mel: MelSpectrogram,
+    spec: Spectrogram,
+    options?: BeatCandidatesOptions
+): BeatCandidatesOutput {
+    const opts: BeatCandidatesOptions = {
+        minIntervalSec: options?.minIntervalSec ?? 0.1,
+        thresholdFactor: options?.thresholdFactor ?? 0.5,
+        smoothMs: options?.smoothMs ?? 50,
+    };
+
+    // Compute beat salience signal.
+    const salience = beatSalienceFromMel(mel, spec, { smoothMs: opts.smoothMs });
+
+    // Pick peaks from salience.
+    const candidates = pickBeatCandidates(salience, opts, "combined");
+
+    return {
+        candidates,
+        salience,
+    };
+}

package/src/dsp/cqt.ts

@@ -0,0 +1,386 @@
+/**
+ * Constant-Q Transform (CQT) implementation for F5.
+ *
+ * CQT provides log-frequency resolution aligned to musical pitch ratios.
+ * This implementation builds on the existing STFT infrastructure by:
+ * 1. Computing an STFT using the existing spectrogram function
+ * 2. Applying a CQT filterbank in the frequency domain
+ *
+ * The CQT is an internal spectral view, not a user-facing spectrogram.
+ */
+
+import type { CqtConfig, CqtSpectrogram, MirRunMeta } from "../types";
+import { spectrogram, type AudioBufferLike } from "./spectrogram";
+
+// ----------------------------
+// Default Configuration
+// ----------------------------
+
+/** Default CQT configuration values */
+export const CQT_DEFAULTS = {
+    /** Quarter-tone resolution (24 bins per octave) */
+    binsPerOctave: 24,
+    /** C1 (lowest note on a standard piano) */
+    fMin: 32.7,
+    /** C9 (well above audible range for most content) */
+    fMax: 8372,
+} as const;
+
+// ----------------------------
+// Utility Functions
+// ----------------------------
+
+/**
+ * Convert CQT bin index to frequency in Hz.
+ */
+export function cqtBinToHz(bin: number, config: CqtConfig): number {
+    return config.fMin * Math.pow(2, bin / config.binsPerOctave);
+}
+
+/**
+ * Convert frequency in Hz to CQT bin index (may be fractional).
+ */
+export function hzToCqtBin(hz: number, config: CqtConfig): number {
+    if (hz <= 0) return -Infinity;
+    return config.binsPerOctave * Math.log2(hz / config.fMin);
+}
+
+/**
+ * Calculate number of octaves covered by the CQT config.
+ */
+export function getNumOctaves(config: CqtConfig): number {
+    return Math.log2(config.fMax / config.fMin);
+}
+
+/**
+ * Calculate total number of CQT bins.
+ */
+export function getNumBins(config: CqtConfig): number {
+    const nOctaves = getNumOctaves(config);
+    return Math.ceil(nOctaves * config.binsPerOctave);
+}
+
+/**
+ * Generate the center frequencies for all CQT bins.
+ */
+export function getCqtBinFrequencies(config: CqtConfig): Float32Array {
+    const nBins = getNumBins(config);
+    const freqs = new Float32Array(nBins);
+    for (let k = 0; k < nBins; k++) {
+        freqs[k] = cqtBinToHz(k, config);
+    }
+    return freqs;
+}
+
+// ----------------------------
+// CQT Kernel Bank
+// ----------------------------
+
+/**
+ * A sparse CQT kernel for a single frequency bin.
+ * Each kernel maps a range of STFT bins to a single CQT bin.
+ */
+type CqtKernel = {
+    /** Center frequency of this CQT bin in Hz */
+    centerFreq: number;
+    /** Starting STFT bin index (inclusive) */
+    startBin: number;
+    /** Ending STFT bin index (exclusive) */
+    endBin: number;
+    /** Weights for each STFT bin in the range [startBin, endBin) */
+    weights: Float32Array;
+};
+
+/**
+ * Pre-computed CQT kernel bank for efficient application to STFT frames.
+ */
+type CqtKernelBank = {
+    config: CqtConfig;
+    fftSize: number;
+    sampleRate: number;
+    kernels: CqtKernel[];
+};
+
+/** Cache for kernel banks to avoid recomputation */
+const kernelBankCache = new Map<string, CqtKernelBank>();
+
+/**
+ * Generate a cache key for the kernel bank.
+ */
+function kernelCacheKey(config: CqtConfig, fftSize: number, sampleRate: number): string {
+    return `${config.binsPerOctave}:${config.fMin}:${config.fMax}:${fftSize}:${sampleRate}`;
+}
+
+/**
+ * Create a triangular window centered at a frequency with log-spaced bandwidth.
+ * This creates a simple triangular filterbank similar to mel filterbanks.
+ */
+function createCqtKernel(
+    binIndex: number,
+    config: CqtConfig,
+    fftSize: number,
+    sampleRate: number
+): CqtKernel {
+    const centerFreq = cqtBinToHz(binIndex, config);
+    const freqResolution = sampleRate / fftSize;
+
+    // Q factor: ratio of center frequency to bandwidth
+    // For CQT, Q is constant, which gives logarithmic frequency resolution
+    // Q = f / Δf = 1 / (2^(1/binsPerOctave) - 1)
+    const Q = 1 / (Math.pow(2, 1 / config.binsPerOctave) - 1);
+
+    // Bandwidth for this bin
+    const bandwidth = centerFreq / Q;
+
+    // Lower and upper edge frequencies
+    const fLow = centerFreq - bandwidth / 2;
+    const fHigh = centerFreq + bandwidth / 2;
+
+    // Convert to STFT bin indices
+    const startBin = Math.max(0, Math.floor(fLow / freqResolution));
+    const endBin = Math.min(
+        Math.floor(fftSize / 2) + 1,
+        Math.ceil(fHigh / freqResolution) + 1
+    );
+
+    const numBins = Math.max(1, endBin - startBin);
+    const weights = new Float32Array(numBins);
+
+    // Create triangular window weights
+    for (let i = 0; i < numBins; i++) {
+        const binFreq = (startBin + i) * freqResolution;
+
+        if (binFreq <= centerFreq) {
+            // Rising edge
+            if (centerFreq > fLow) {
+                weights[i] = (binFreq - fLow) / (centerFreq - fLow);
+            } else {
+                weights[i] = 1;
+            }
+        } else {
+            // Falling edge
+            if (fHigh > centerFreq) {
+                weights[i] = (fHigh - binFreq) / (fHigh - centerFreq);
+            } else {
+                weights[i] = 1;
+            }
+        }
+
+        // Clamp to valid range
+        weights[i] = Math.max(0, Math.min(1, weights[i] ?? 0));
+    }
+
+    // Normalize weights so they sum to 1
+    let sum = 0;
+    for (let i = 0; i < numBins; i++) {
+        sum += weights[i] ?? 0;
+    }
+    if (sum > 0) {
+        for (let i = 0; i < numBins; i++) {
+            weights[i] = (weights[i] ?? 0) / sum;
+        }
+    }
+
+    return {
+        centerFreq,
+        startBin,
+        endBin,
+        weights,
+    };
+}
+
+/**
+ * Create or retrieve a cached CQT kernel bank.
+ */
+function getCqtKernelBank(
+    config: CqtConfig,
+    fftSize: number,
+    sampleRate: number
+): CqtKernelBank {
+    const key = kernelCacheKey(config, fftSize, sampleRate);
+    const cached = kernelBankCache.get(key);
+    if (cached) return cached;
+
+    const nBins = getNumBins(config);
+    const kernels: CqtKernel[] = new Array(nBins);
+
+    for (let k = 0; k < nBins; k++) {
+        kernels[k] = createCqtKernel(k, config, fftSize, sampleRate);
+    }
+
+    const bank: CqtKernelBank = {
+        config,
+        fftSize,
+        sampleRate,
+        kernels,
+    };
+
+    kernelBankCache.set(key, bank);
+    return bank;
+}
+
+/**
+ * Apply CQT kernel bank to an STFT magnitude frame.
+ */
+function applyCqtKernels(
+    stftMagnitudes: Float32Array,
+    kernelBank: CqtKernelBank
+): Float32Array {
+    const nCqtBins = kernelBank.kernels.length;
+    const cqtMagnitudes = new Float32Array(nCqtBins);
+
+    for (let k = 0; k < nCqtBins; k++) {
+        const kernel = kernelBank.kernels[k];
+        if (!kernel) continue;
+
+        let sum = 0;
+        for (let i = 0; i < kernel.weights.length; i++) {
+            const stftBin = kernel.startBin + i;
+            const stftMag = stftMagnitudes[stftBin] ?? 0;
+            const weight = kernel.weights[i] ?? 0;
+            sum += stftMag * weight;
+        }
+        cqtMagnitudes[k] = sum;
+    }
+
+    return cqtMagnitudes;
+}
+
+// ----------------------------
+// Main CQT Function
+// ----------------------------
+
+export type CqtOptions = {
+    /** Optional cancellation hook; checked periodically. */
+    isCancelled?: () => boolean;
+};
+
+/**
+ * Apply default values to a partial CQT config.
+ */
+export function withCqtDefaults(partial?: Partial<CqtConfig>): CqtConfig {
+    return {
+        binsPerOctave: partial?.binsPerOctave ?? CQT_DEFAULTS.binsPerOctave,
+        fMin: partial?.fMin ?? CQT_DEFAULTS.fMin,
+        fMax: partial?.fMax ?? CQT_DEFAULTS.fMax,
+        hopSize: partial?.hopSize,
+    };
+}
+
+/**
+ * Compute a CQT spectrogram from audio.
+ *
+ * This function:
+ * 1. Computes an STFT using the existing spectrogram infrastructure
+ * 2. Applies a CQT filterbank to each STFT frame
+ * 3. Returns a log-frequency representation
+ *
+ * @param audio - Audio buffer to analyze
+ * @param config - CQT configuration
+ * @param options - Optional processing options
+ * @returns CQT spectrogram with log-frequency resolution
+ */
+export async function cqtSpectrogram(
+    audio: AudioBufferLike,
+    config: CqtConfig,
+    options: CqtOptions = {}
+): Promise<CqtSpectrogram> {
+    const sampleRate = audio.sampleRate;
+
+    // Validate config
+    if (config.fMin <= 0) {
+        throw new Error("@octoseq/mir: CQT fMin must be positive");
+    }
+    if (config.fMax <= config.fMin) {
+        throw new Error("@octoseq/mir: CQT fMax must be greater than fMin");
+    }
+    if (config.binsPerOctave <= 0) {
+        throw new Error("@octoseq/mir: CQT binsPerOctave must be positive");
+    }
+
+    // Determine FFT size based on lowest frequency
+    // We need sufficient frequency resolution for the lowest CQT bins
+    // Rule of thumb: fftSize should give frequency resolution of fMin / Q
+    const Q = 1 / (Math.pow(2, 1 / config.binsPerOctave) - 1);
+    const minFreqResolution = config.fMin / Q / 2; // Nyquist for the lowest bin
+    const minFftSize = Math.ceil(sampleRate / minFreqResolution);
+
+    // Round up to next power of 2
+    let fftSize = 1;
+    while (fftSize < minFftSize) {
+        fftSize *= 2;
+    }
+
+    // Cap at reasonable maximum to avoid memory issues
+    fftSize = Math.min(fftSize, 16384);
+
+    // Hop size: default to 1/4 of FFT size for good time resolution
+    const hopSize = config.hopSize ?? Math.floor(fftSize / 4);
+
+    // Compute STFT
+    const stft = await spectrogram(
+        audio,
+        { fftSize, hopSize, window: "hann" },
+        undefined,
+        { isCancelled: options.isCancelled }
+    );
+
+    // Create or get cached kernel bank
+    const kernelBank = getCqtKernelBank(config, fftSize, sampleRate);
+
+    // Apply CQT kernels to each STFT frame
+    const nFrames = stft.magnitudes.length;
+    const cqtMagnitudes: Float32Array[] = new Array(nFrames);
+
+    for (let frame = 0; frame < nFrames; frame++) {
+        if (options.isCancelled?.()) {
+            throw new Error("@octoseq/mir: cancelled");
+        }
+
+        const stftFrame = stft.magnitudes[frame];
+        if (!stftFrame) continue;
+
+        cqtMagnitudes[frame] = applyCqtKernels(stftFrame, kernelBank);
+    }
+
+    const nOctaves = getNumOctaves(config);
+    const nBins = getNumBins(config);
+
+    return {
+        sampleRate,
+        config,
+        times: stft.times,
+        magnitudes: cqtMagnitudes,
+        nOctaves,
+        binsPerOctave: config.binsPerOctave,
+        binFrequencies: getCqtBinFrequencies(config),
+    };
+}
+
+/**
+ * Compute CQT and return with metadata for MIR pipeline.
+ */
+export async function computeCqt(
+    audio: AudioBufferLike,
+    config?: Partial<CqtConfig>,
+    options: CqtOptions = {}
+): Promise<{ cqt: CqtSpectrogram; meta: MirRunMeta }> {
+    const startTime = performance.now();
+
+    const fullConfig = withCqtDefaults(config);
+    const cqt = await cqtSpectrogram(audio, fullConfig, options);
+
+    const endTime = performance.now();
+
+    return {
+        cqt,
+        meta: {
+            backend: "cpu",
+            usedGpu: false,
+            timings: {
+                totalMs: endTime - startTime,
+                cpuMs: endTime - startTime,
+            },
+        },
+    };
+}