@octoseq/mir 0.1.0-main.4ecb074 → 0.1.0-main.5fdb072

package/src/runner/runMir.ts

@@ -1,13 +1,19 @@
+import { detectBeatCandidates } from "../dsp/beatCandidates";
+import { generateTempoHypotheses } from "../dsp/tempoHypotheses";
 import { melSpectrogram, type MelConfig, type MelSpectrogram } from "../dsp/mel";
 import { mfcc, delta, deltaDelta } from "../dsp/mfcc";
 import { onsetEnvelopeFromMel, onsetEnvelopeFromMelGpu } from "../dsp/onset";
 import { peakPick } from "../dsp/peakPick";
 import { hpss } from "../dsp/hpss";
 import { hpssGpu } from "../dsp/hpssGpu";
-import { spectralCentroid, spectralFlux } from "../dsp/spectral";
+import { amplitudeEnvelope, spectralCentroid, spectralFlux } from "../dsp/spectral";
 import { spectrogram, type AudioBufferLike, type Spectrogram, type SpectrogramConfig } from "../dsp/spectrogram";
+import { cqtSpectrogram, withCqtDefaults } from "../dsp/cqt";
+import { harmonicEnergy, bassPitchMotion, tonalStability } from "../dsp/cqtSignals";
+import { pitchF0, pitchConfidence } from "../dsp/pitch";
+import { computeActivityFromMel } from "../dsp/activity";
 import type { MirGPU } from "../gpu/context";
-import type { MirAudioPayload, MirBackend, MirResult, MirRunRequest } from "../types";
+import type { CqtConfig, MirAudioPayload, MirBackend, MirResult, MirRunRequest } from "../types";
 
 function nowMs(): number {
     return typeof performance !== "undefined" ? performance.now() : Date.now();
@@ -84,6 +90,29 @@ export async function runMir(
         window: "hann",
     };
 
+    // Amplitude envelope: compute directly from raw audio (no spectrogram needed)
+    if (request.fn === "amplitudeEnvelope") {
+        const cpuStart = nowMs();
+        const result = amplitudeEnvelope(audio.mono, audio.sampleRate, {
+            hopSize: specConfig.hopSize,
+            windowSize: specConfig.fftSize,
+        });
+        const cpuEnd = nowMs();
+        return {
+            kind: "1d",
+            times: result.times,
+            values: result.values,
+            meta: {
+                backend: "cpu",
+                usedGpu: false,
+                timings: {
+                    totalMs: cpuEnd - t0,
+                    cpuMs: cpuEnd - cpuStart,
+                },
+            },
+        };
+    }
+
     // CPU: spectrogram + centroid/flux are CPU-only today.
     const cpuStart = nowMs();
     const spec: Spectrogram = await spectrogram(asAudioBufferLike(audio), specConfig, undefined, {
@@ -285,6 +314,76 @@ export async function runMir(
         };
     }
 
+    if (request.fn === "beatCandidates") {
+        // Beat candidate detection requires both mel spectrogram and raw spectrogram.
+        const { mel, cpuExtraMs: melCpuMs } = await computeMel(false);
+
+        const beatOpts = request.beatCandidates ?? {};
+        const result = detectBeatCandidates(mel, spec, {
+            minIntervalSec: beatOpts.minIntervalSec,
+            thresholdFactor: beatOpts.thresholdFactor,
+            smoothMs: beatOpts.smoothMs,
+        });
+
+        const end = nowMs();
+        return {
+            kind: "beatCandidates",
+            times: result.salience.times,
+            candidates: result.candidates,
+            salience: beatOpts.includeSalience ? result.salience : undefined,
+            meta: {
+                backend: "cpu",
+                usedGpu: false,
+                timings: {
+                    totalMs: end - t0,
+                    cpuMs: cpuAfterSpec - cpuStart + melCpuMs,
+                },
+            },
+        };
+    }
+
+    if (request.fn === "tempoHypotheses") {
+        // Tempo hypothesis generation requires beat candidates.
+        // We compute them internally (could accept pre-computed in future).
+        const { mel, cpuExtraMs: melCpuMs } = await computeMel(false);
+
+        const beatOpts = request.beatCandidates ?? {};
+        const beatResult = detectBeatCandidates(mel, spec, {
+            minIntervalSec: beatOpts.minIntervalSec,
+            thresholdFactor: beatOpts.thresholdFactor,
+            smoothMs: beatOpts.smoothMs,
+        });
+
+        const tempoStart = nowMs();
+        const tempoOpts = request.tempoHypotheses ?? {};
+        const result = generateTempoHypotheses(beatResult.candidates, {
+            minBpm: tempoOpts.minBpm,
+            maxBpm: tempoOpts.maxBpm,
+            binSizeBpm: tempoOpts.binSizeBpm,
+            maxHypotheses: tempoOpts.maxHypotheses,
+            minConfidence: tempoOpts.minConfidence,
+            weightByStrength: tempoOpts.weightByStrength,
+            includeHistogram: tempoOpts.includeHistogram,
+        });
+
+        const end = nowMs();
+        return {
+            kind: "tempoHypotheses",
+            times: spec.times,
+            hypotheses: result.hypotheses,
+            inputCandidateCount: result.inputCandidateCount,
+            histogram: result.histogram,
+            meta: {
+                backend: "cpu",
+                usedGpu: false,
+                timings: {
+                    totalMs: end - t0,
+                    cpuMs: cpuAfterSpec - cpuStart + melCpuMs + (end - tempoStart),
+                },
+            },
+        };
+    }
+
     if (request.fn === "hpssHarmonic" || request.fn === "hpssPercussive") {
         // HPSS may use a custom spectrogram config
         const hpssSpecConfig = options.hpss?.spectrogram ?? specConfig;
@@ -410,6 +509,139 @@ export async function runMir(
         };
     }
 
+    // ----------------------------
+    // CQT-derived signals (F5)
+    // ----------------------------
+
+    if (request.fn === "cqtHarmonicEnergy" || request.fn === "cqtBassPitchMotion" || request.fn === "cqtTonalStability") {
+        // CQT signals bypass the STFT we computed above and compute their own CQT.
+        // This is intentional: CQT has different frequency resolution requirements.
+        const cqtStart = nowMs();
+
+        const cqtConfig: CqtConfig = withCqtDefaults(request.cqt);
+        const cqt = await cqtSpectrogram(asAudioBufferLike(audio), cqtConfig, {
+            isCancelled: options.isCancelled,
+        });
+
+        const cqtEnd = nowMs();
+
+        // Compute the requested signal
+        let signal;
+        if (request.fn === "cqtHarmonicEnergy") {
+            signal = harmonicEnergy(cqt);
+        } else if (request.fn === "cqtBassPitchMotion") {
+            signal = bassPitchMotion(cqt);
+        } else {
+            signal = tonalStability(cqt);
+        }
+
+        const end = nowMs();
+
+        return {
+            kind: "1d",
+            times: signal.times,
+            values: signal.values,
+            meta: {
+                backend: "cpu",
+                usedGpu: false,
+                timings: {
+                    totalMs: end - t0,
+                    cpuMs: (cqtEnd - cqtStart) + (end - cqtEnd),
+                },
+            },
+        };
+    }
+
+    // ----------------------------
+    // Pitch detection (P1)
+    // ----------------------------
+
+    if (request.fn === "pitchF0" || request.fn === "pitchConfidence") {
+        // Pitch detection works directly on raw audio samples (like amplitudeEnvelope).
+        // Uses YIN algorithm for monophonic pitch estimation.
+        const pitchStart = nowMs();
+
+        const pitchConfig = {
+            fMinHz: request.pitch?.fMinHz,
+            fMaxHz: request.pitch?.fMaxHz,
+            threshold: request.pitch?.threshold,
+            hopSize: specConfig.hopSize,
+            windowSize: specConfig.fftSize,
+        };
+
+        const result = request.fn === "pitchF0"
+            ? pitchF0(audio.mono, audio.sampleRate, pitchConfig)
+            : pitchConfidence(audio.mono, audio.sampleRate, pitchConfig);
+
+        const end = nowMs();
+
+        return {
+            kind: "1d",
+            times: result.times,
+            values: result.values,
+            meta: {
+                backend: "cpu",
+                usedGpu: false,
+                timings: {
+                    totalMs: end - t0,
+                    cpuMs: end - pitchStart,
+                },
+            },
+        };
+    }
+
+    // ----------------------------
+    // Activity detection
+    // ----------------------------
+
+    if (request.fn === "activity") {
+        // Activity detection requires mel spectrogram.
+        const { mel, cpuExtraMs: melCpuMs } = await computeMel(false);
+        const activityStart = nowMs();
+
+        const activityConfig = {
+            energyPercentile: request.activity?.energyPercentile,
+            enterMargin: request.activity?.enterMargin,
+            exitMargin: request.activity?.exitMargin,
+            hangoverMs: request.activity?.hangoverMs,
+            minActiveMs: request.activity?.minActiveMs,
+            smoothMs: request.activity?.smoothMs,
+        };
+
+        const activity = computeActivityFromMel(mel, activityConfig);
+        const end = nowMs();
+
+        // Compute diagnostics
+        let activeCount = 0;
+        for (let i = 0; i < activity.isActive.length; i++) {
+            if (activity.isActive[i]) activeCount++;
+        }
+
+        return {
+            kind: "activity",
+            times: activity.times,
+            activityLevel: activity.activityLevel,
+            isActive: activity.isActive,
+            suppressMask: activity.suppressMask,
+            meta: {
+                backend: "cpu",
+                usedGpu: false,
+                timings: {
+                    totalMs: end - t0,
+                    cpuMs: cpuAfterSpec - cpuStart + melCpuMs + (end - activityStart),
+                },
+            },
+            diagnostics: {
+                noiseFloor: activity.diagnostics.noiseFloor,
+                enterThreshold: activity.diagnostics.enterThreshold,
+                exitThreshold: activity.diagnostics.exitThreshold,
+                totalFrames: activity.times.length,
+                activeFrames: activeCount,
+                activeFraction: activity.times.length > 0 ? activeCount / activity.times.length : 0,
+            },
+        };
+    }
+
     // Fallback: keep old behaviour (melSpectrogram) if unknown fn.
     const { mel, usedGpu, gpuMs, cpuExtraMs } = await computeMel(backend === "gpu");
     const end = nowMs();

package/src/runner/workerProtocol.ts

@@ -1,4 +1,4 @@
-import type { MirAudioPayload, MirResult, MirRunRequest } from "../types";
+import type { BeatCandidate, MirAudioPayload, MirResult, MirRunRequest, TempoHypothesis } from "../types";
 
 export type MirWorkerInitMessage = {
     type: "INIT";
@@ -93,6 +93,14 @@ export type MirWorkerResultMessage = {
         values?: ArrayBufferLike;
         data2d?: ArrayBufferLike[];
         events?: Array<{ time: number; strength: number; index: number }>;
+        candidates?: BeatCandidate[];
+        // For tempoHypotheses
+        hypotheses?: TempoHypothesis[];
+        inputCandidateCount?: number;
+        histogram?: {
+            bpmBins: ArrayBufferLike;
+            counts: ArrayBufferLike;
+        };
         meta: MirResult["meta"];
     };
 };