@hum2music/audio-analysis 0.0.0-stage → 0.1.0

package/package.json

@@ -1,6 +1,19 @@
 {
   "name": "@hum2music/audio-analysis",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "description": "WAV decoding/encoding, YIN pitch detection, and melody transcription for Hum2Music.",
+  "license": "Apache-2.0",
+  "homepage": "https://github.com/hum2music/hum2music",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/hum2music/hum2music.git",
+    "directory": "packages/audio-analysis"
+  },
+  "type": "module",
+  "main": "./src/index.ts",
+  "types": "./src/index.ts",
+  "files": ["src"],
+  "dependencies": {
+    "@hum2music/core": "0.1.0"
+  }
+}

package/src/index.ts

@@ -0,0 +1,6 @@
+export * from "./wav";
+export * from "./pitch-to-midi";
+export * from "./yin";
+export * from "./smoothing";
+export * from "./segmentation";
+export * from "./transcribe";

package/src/pitch-to-midi.ts

@@ -0,0 +1,7 @@
+export function frequencyToMidiNote(frequencyHz: number): number {
+  return 69 + 12 * Math.log2(frequencyHz / 440);
+}
+
+export function midiNoteToFrequency(midiNote: number): number {
+  return 440 * Math.pow(2, (midiNote - 69) / 12);
+}

package/src/segmentation.ts

@@ -0,0 +1,62 @@
+import type { NoteCandidate, PitchFrame } from "@hum2music/core";
+import { frequencyToMidiNote } from "./pitch-to-midi";
+
+export interface SegmentationOptions {
+  hopMs: number;
+  pitchJumpCents?: number;
+  minVoicedFrames?: number;
+}
+
+/**
+ * Groups a cleaned pitch trajectory into note candidates. A run of voiced
+ * frames stays one note as long as it doesn't jump more than
+ * `pitchJumpCents` away from the running average of the current note;
+ * unvoiced frames (rests) always end the current note.
+ */
+export function segmentNotes(frames: PitchFrame[], options: SegmentationOptions): NoteCandidate[] {
+  const pitchJumpCents = options.pitchJumpCents ?? 50;
+  const minVoicedFrames = options.minVoicedFrames ?? 2;
+
+  const candidates: NoteCandidate[] = [];
+  let current: PitchFrame[] = [];
+  let midiSum = 0;
+
+  const flush = () => {
+    if (current.length >= minVoicedFrames) {
+      candidates.push(buildCandidate(current, options.hopMs));
+    }
+    current = [];
+    midiSum = 0;
+  };
+
+  for (const frame of frames) {
+    if (!frame.voiced || frame.f0Hz === null) {
+      flush();
+      continue;
+    }
+
+    const midi = frequencyToMidiNote(frame.f0Hz);
+
+    if (current.length > 0) {
+      const runningAverage = midiSum / current.length;
+      const centsDelta = (midi - runningAverage) * 100;
+      if (Math.abs(centsDelta) > pitchJumpCents) flush();
+    }
+
+    current.push(frame);
+    midiSum += midi;
+  }
+  flush();
+
+  return candidates;
+}
+
+function buildCandidate(frames: PitchFrame[], hopMs: number): NoteCandidate {
+  const startMs = frames[0].timeMs;
+  const durationMs = frames[frames.length - 1].timeMs - startMs + hopMs;
+  const midiValues = frames.map((frame) => frequencyToMidiNote(frame.f0Hz as number));
+  const rawPitch = midiValues.reduce((sum, value) => sum + value, 0) / midiValues.length;
+  const onsetConfidence = frames.reduce((sum, frame) => sum + frame.confidence, 0) / frames.length;
+
+  return { startMs, durationMs, rawPitch, pitchTrajectory: frames, onsetConfidence };
+}

package/src/smoothing.ts

@@ -0,0 +1,64 @@
+import type { PitchFrame } from "@hum2music/core";
+
+export interface OutlierOptions {
+  neighborRadius?: number;
+  centsThreshold?: number;
+}
+
+/** Nulls out isolated pitch spikes that deviate too far from their local median. */
+export function removePitchOutliers(frames: PitchFrame[], options: OutlierOptions = {}): PitchFrame[] {
+  const neighborRadius = options.neighborRadius ?? 3;
+  const centsThreshold = options.centsThreshold ?? 80;
+
+  const result = frames.map((frame) => ({ ...frame }));
+
+  for (let i = 0; i < result.length; i++) {
+    const frame = result[i];
+    if (!frame.voiced || frame.f0Hz === null) continue;
+
+    const neighborFrequencies: number[] = [];
+    for (let offset = -neighborRadius; offset <= neighborRadius; offset++) {
+      if (offset === 0) continue;
+      const neighbor = result[i + offset];
+      if (neighbor?.voiced && neighbor.f0Hz !== null) neighborFrequencies.push(neighbor.f0Hz);
+    }
+    if (neighborFrequencies.length === 0) continue;
+
+    const medianNeighbor = median(neighborFrequencies);
+    const cents = 1200 * Math.log2(frame.f0Hz / medianNeighbor);
+    if (Math.abs(cents) > centsThreshold) {
+      frame.f0Hz = null;
+      frame.voiced = false;
+      frame.confidence = 0;
+    }
+  }
+
+  return result;
+}
+
+/** Moving-average smoothing of the voiced pitch trajectory; gaps (rests) are left untouched. */
+export function smoothPitchTrajectory(frames: PitchFrame[], windowSize = 3): PitchFrame[] {
+  const half = Math.floor(windowSize / 2);
+  const result = frames.map((frame) => ({ ...frame }));
+
+  for (let i = 0; i < result.length; i++) {
+    if (!frames[i].voiced || frames[i].f0Hz === null) continue;
+
+    const windowValues: number[] = [];
+    for (let offset = -half; offset <= half; offset++) {
+      const neighbor = frames[i + offset];
+      if (neighbor?.voiced && neighbor.f0Hz !== null) windowValues.push(neighbor.f0Hz);
+    }
+    if (windowValues.length > 0) {
+      result[i].f0Hz = windowValues.reduce((sum, value) => sum + value, 0) / windowValues.length;
+    }
+  }
+
+  return result;
+}
+
+function median(values: number[]): number {
+  const sorted = [...values].sort((a, b) => a - b);
+  const mid = Math.floor(sorted.length / 2);
+  return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
+}

package/src/transcribe.ts

@@ -0,0 +1,58 @@
+import { DEFAULT_HOP_SIZE, detectPitchYin } from "./yin";
+import { removePitchOutliers, smoothPitchTrajectory } from "./smoothing";
+import { segmentNotes } from "./segmentation";
+
+export interface TranscribedNote {
+  midiPitch: number;
+  rawPitch: number;
+  startMs: number;
+  durationMs: number;
+  velocity: number;
+  confidence: number;
+}
+
+/**
+ * Audio-to-melody transcription pipeline: raw samples -> YIN pitch track ->
+ * outlier removal and smoothing -> note segmentation -> integer MIDI notes
+ * ready for export. There is no tempo/beat model yet, so timing stays in ms.
+ */
+export function transcribeMelody(samples: Float32Array, sampleRate: number): TranscribedNote[] {
+  const rawFrames = detectPitchYin(samples, sampleRate);
+  const cleaned = smoothPitchTrajectory(removePitchOutliers(rawFrames));
+  const hopMs = (DEFAULT_HOP_SIZE / sampleRate) * 1000;
+  const candidates = segmentNotes(cleaned, { hopMs });
+
+  return candidates.map((candidate) => ({
+    midiPitch: clampMidiPitch(Math.round(candidate.rawPitch)),
+    rawPitch: candidate.rawPitch,
+    startMs: candidate.startMs,
+    durationMs: candidate.durationMs,
+    velocity: estimateVelocity(samples, sampleRate, candidate.startMs, candidate.durationMs),
+    confidence: candidate.onsetConfidence,
+  }));
+}
+
+function clampMidiPitch(pitch: number): number {
+  return Math.min(127, Math.max(0, pitch));
+}
+
+// Scaling factor chosen so a full-scale sine (RMS ~0.707) maps near the top
+// of the MIDI velocity range; proposed value, to be revisited with real
+// humming recordings (see docs/specs/audio-analysis.md).
+const VELOCITY_RMS_SCALE = 170;
+
+function estimateVelocity(samples: Float32Array, sampleRate: number, startMs: number, durationMs: number): number {
+  const startSample = Math.max(0, Math.floor((startMs / 1000) * sampleRate));
+  const endSample = Math.min(samples.length, Math.floor(((startMs + durationMs) / 1000) * sampleRate));
+
+  let sumOfSquares = 0;
+  let count = 0;
+  for (let i = startSample; i < endSample; i++) {
+    sumOfSquares += samples[i] * samples[i];
+    count++;
+  }
+  if (count === 0) return 64;
+
+  const rms = Math.sqrt(sumOfSquares / count);
+  return Math.min(127, Math.max(1, Math.round(rms * VELOCITY_RMS_SCALE)));
+}

package/src/wav.ts

@@ -0,0 +1,103 @@
+export interface DecodedAudio {
+  sampleRate: number;
+  samples: Float32Array;
+}
+
+export function decodeWav(buffer: ArrayBuffer): DecodedAudio {
+  const view = new DataView(buffer);
+
+  if (readAscii(view, 0, 4) !== "RIFF") throw new Error("Not a RIFF file");
+  if (readAscii(view, 8, 4) !== "WAVE") throw new Error("Not a WAVE file");
+
+  let offset = 12;
+  let fmt: { audioFormat: number; numChannels: number; sampleRate: number; bitsPerSample: number } | null = null;
+  let dataOffset = -1;
+  let dataLength = 0;
+
+  while (offset + 8 <= view.byteLength) {
+    const chunkId = readAscii(view, offset, 4);
+    const chunkSize = view.getUint32(offset + 4, true);
+    const chunkStart = offset + 8;
+
+    if (chunkId === "fmt ") {
+      fmt = {
+        audioFormat: view.getUint16(chunkStart, true),
+        numChannels: view.getUint16(chunkStart + 2, true),
+        sampleRate: view.getUint32(chunkStart + 4, true),
+        bitsPerSample: view.getUint16(chunkStart + 14, true),
+      };
+    } else if (chunkId === "data") {
+      dataOffset = chunkStart;
+      dataLength = chunkSize;
+    }
+
+    offset = chunkStart + chunkSize + (chunkSize % 2);
+  }
+
+  if (!fmt) throw new Error("Malformed WAV file: missing fmt chunk");
+  if (dataOffset < 0) throw new Error("Malformed WAV file: missing data chunk");
+  if (fmt.audioFormat !== 1) {
+    throw new Error(`Unsupported WAV encoding (audioFormat=${fmt.audioFormat}); only PCM is supported`);
+  }
+  if (fmt.bitsPerSample !== 16) {
+    throw new Error(`Unsupported bit depth (${fmt.bitsPerSample}-bit); only 16-bit PCM is supported`);
+  }
+
+  const bytesPerSample = fmt.bitsPerSample / 8;
+  const frameCount = Math.floor(dataLength / (bytesPerSample * fmt.numChannels));
+  const mono = new Float32Array(frameCount);
+
+  for (let i = 0; i < frameCount; i++) {
+    let sum = 0;
+    for (let channel = 0; channel < fmt.numChannels; channel++) {
+      const sampleOffset = dataOffset + (i * fmt.numChannels + channel) * bytesPerSample;
+      sum += view.getInt16(sampleOffset, true) / 32768;
+    }
+    mono[i] = sum / fmt.numChannels;
+  }
+
+  return { sampleRate: fmt.sampleRate, samples: mono };
+}
+
+function readAscii(view: DataView, offset: number, length: number): string {
+  let text = "";
+  for (let i = 0; i < length; i++) text += String.fromCharCode(view.getUint8(offset + i));
+  return text;
+}
+
+/** Counterpart to decodeWav: encodes mono samples as a 16-bit PCM WAV file. */
+export function encodeWavPcm16(samples: Float32Array, sampleRate: number): ArrayBuffer {
+  const numChannels = 1;
+  const bitsPerSample = 16;
+  const blockAlign = (numChannels * bitsPerSample) / 8;
+  const byteRate = sampleRate * blockAlign;
+  const dataSize = samples.length * blockAlign;
+
+  const buffer = new ArrayBuffer(44 + dataSize);
+  const view = new DataView(buffer);
+
+  writeAscii(view, 0, "RIFF");
+  view.setUint32(4, 36 + dataSize, true);
+  writeAscii(view, 8, "WAVE");
+  writeAscii(view, 12, "fmt ");
+  view.setUint32(16, 16, true);
+  view.setUint16(20, 1, true);
+  view.setUint16(22, numChannels, true);
+  view.setUint32(24, sampleRate, true);
+  view.setUint32(28, byteRate, true);
+  view.setUint16(32, blockAlign, true);
+  view.setUint16(34, bitsPerSample, true);
+  writeAscii(view, 36, "data");
+  view.setUint32(40, dataSize, true);
+
+  for (let i = 0; i < samples.length; i++) {
+    const clamped = Math.max(-1, Math.min(1, samples[i]));
+    view.setInt16(44 + i * 2, Math.round(clamped * 32767), true);
+  }
+
+  return buffer;
+}
+
+function writeAscii(view: DataView, offset: number, text: string): void {
+  for (let i = 0; i < text.length; i++) view.setUint8(offset + i, text.charCodeAt(i));
+}

package/src/yin.ts

@@ -0,0 +1,84 @@
+import type { PitchFrame } from "@hum2music/core";
+
+export const DEFAULT_FRAME_SIZE = 1024;
+export const DEFAULT_HOP_SIZE = 512;
+
+export interface YinOptions {
+  frameSize?: number;
+  hopSize?: number;
+  threshold?: number;
+  minFrequencyHz?: number;
+  maxFrequencyHz?: number;
+}
+
+/**
+ * YIN pitch detection (de Cheveigne & Kawahara, 2002), evaluated per sliding
+ * frame over the whole signal. Frames where no period passes the absolute
+ * threshold are reported as unvoiced rather than forced to a guess.
+ */
+export function detectPitchYin(samples: Float32Array, sampleRate: number, options: YinOptions = {}): PitchFrame[] {
+  const frameSize = options.frameSize ?? DEFAULT_FRAME_SIZE;
+  const hopSize = options.hopSize ?? DEFAULT_HOP_SIZE;
+  const threshold = options.threshold ?? 0.12;
+  const minFrequencyHz = options.minFrequencyHz ?? 80;
+  const maxFrequencyHz = options.maxFrequencyHz ?? 1000;
+
+  const tauMin = Math.max(2, Math.floor(sampleRate / maxFrequencyHz));
+  const tauMax = Math.min(Math.floor(frameSize / 2) - 1, Math.floor(sampleRate / minFrequencyHz));
+
+  const frames: PitchFrame[] = [];
+  const diff = new Float64Array(tauMax + 1);
+  const cmndf = new Float64Array(tauMax + 1);
+
+  for (let start = 0; start + frameSize <= samples.length; start += hopSize) {
+    for (let tau = 1; tau <= tauMax; tau++) {
+      let sum = 0;
+      for (let j = 0; j < frameSize - tau; j++) {
+        const delta = samples[start + j] - samples[start + j + tau];
+        sum += delta * delta;
+      }
+      diff[tau] = sum;
+    }
+
+    cmndf[0] = 1;
+    let runningSum = 0;
+    for (let tau = 1; tau <= tauMax; tau++) {
+      runningSum += diff[tau];
+      cmndf[tau] = runningSum === 0 ? 1 : (diff[tau] * tau) / runningSum;
+    }
+
+    let tauEstimate = -1;
+    for (let tau = tauMin; tau <= tauMax; tau++) {
+      if (cmndf[tau] < threshold) {
+        while (tau + 1 <= tauMax && cmndf[tau + 1] < cmndf[tau]) tau++;
+        tauEstimate = tau;
+        break;
+      }
+    }
+
+    const timeMs = (start / sampleRate) * 1000;
+
+    if (tauEstimate === -1) {
+      frames.push({ timeMs, f0Hz: null, confidence: 0, voiced: false });
+      continue;
+    }
+
+    const refinedTau = parabolicInterpolate(cmndf, tauEstimate, tauMax);
+    const f0Hz = sampleRate / refinedTau;
+    const confidence = Math.max(0, 1 - cmndf[tauEstimate]);
+
+    frames.push({ timeMs, f0Hz, confidence, voiced: true });
+  }
+
+  return frames;
+}
+
+function parabolicInterpolate(cmndf: Float64Array, tau: number, tauMax: number): number {
+  if (tau <= 0 || tau >= tauMax) return tau;
+  const s0 = cmndf[tau - 1];
+  const s1 = cmndf[tau];
+  const s2 = cmndf[tau + 1];
+  const denominator = s0 + s2 - 2 * s1;
+  if (denominator === 0) return tau;
+  return tau + (0.5 * (s0 - s2)) / denominator;
+}

package/README.md

@@ -1,3 +0,0 @@
-# Temporary Holding Version
-
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.