@hraness/dawg 0.0.0-stage → 0.3.0

package/src/media/dsp.ts

@@ -0,0 +1,281 @@
+/**
+ * Pure-TypeScript analysis used when no StemDeck beat grid exists: an onset
+ * envelope → autocorrelation tempo estimate with beat phase, a pitch-class
+ * histogram for key estimation, and a 240-bucket RMS waveform. Everything
+ * works on a mono float signal and is O(n) in samples plus a bounded search.
+ */
+import { estimateKey } from "../../core/key.ts";
+import { MEDIA_LIMITS } from "./types.ts";
+import type { ParsedWav } from "./vendor/wav.ts";
+
+export const MIN_BPM = 60;
+export const MAX_BPM = 200;
+const HOP = 512;
+/** Analyse at most this much audio for tempo and key (the first minutes decide). */
+const MAX_ANALYSIS_SECONDS = 240;
+
+/** Downmixed mono samples, bounded to MAX_ANALYSIS_SECONDS. */
+export function monoSignal(wav: ParsedWav, maxSeconds = MAX_ANALYSIS_SECONDS) {
+  const count = Math.min(
+    wav.sampleCount,
+    Math.floor(maxSeconds * wav.sampleRate),
+  );
+  const mono = new Float32Array(count);
+  for (let frame = 0; frame < count; frame += 1) {
+    let sum = 0;
+    for (let channel = 0; channel < wav.channels; channel += 1)
+      sum += wav.sample(frame, channel);
+    mono[frame] = sum / wav.channels;
+  }
+  return mono;
+}
+
+/** Half-wave-rectified log-energy difference per hop: a cheap onset envelope. */
+export function onsetEnvelope(mono: Float32Array, sampleRate: number) {
+  const frames = Math.floor(mono.length / HOP);
+  const envelope = new Float32Array(frames);
+  let previous = 0;
+  for (let frame = 0; frame < frames; frame += 1) {
+    let energy = 0;
+    const start = frame * HOP;
+    for (let index = 0; index < HOP; index += 1) {
+      const value = mono[start + index]!;
+      energy += value * value;
+    }
+    const level = Math.log1p(energy * 1e3);
+    envelope[frame] = Math.max(0, level - previous);
+    previous = level;
+  }
+  // Remove the local mean so sustained loudness does not look like onsets.
+  const window = Math.max(1, Math.round((0.25 * sampleRate) / HOP));
+  const smoothed = new Float32Array(frames);
+  let running = 0;
+  for (let frame = 0; frame < frames; frame += 1) {
+    running += envelope[frame]!;
+    if (frame >= window) running -= envelope[frame - window]!;
+    const mean = running / Math.min(window, frame + 1);
+    smoothed[frame] = Math.max(0, envelope[frame]! - mean);
+  }
+  return { envelope: smoothed, hopSeconds: HOP / sampleRate };
+}
+
+export type TempoEstimate = Readonly<{
+  bpm: number;
+  /** 0..1, how far the winning lag stands above the autocorrelation floor. */
+  confidence: number;
+  /** Seconds of the first beat, from the phase that best matches onsets. */
+  offsetSeconds: number;
+}>;
+
+/**
+ * Autocorrelation over the 60–200 BPM lag range with a mild preference for
+ * ~120 BPM, so half/double-time ambiguities resolve toward the usual octave.
+ */
+export function estimateTempo(
+  envelope: Float32Array,
+  hopSeconds: number,
+): TempoEstimate | undefined {
+  const frames = envelope.length;
+  // Spread each onset over ±2 frames so a beat period that is not a whole
+  // number of hops still correlates with itself (else 2× the period wins).
+  envelope = smoothEnvelope(envelope);
+  const minLag = Math.max(1, Math.floor(60 / MAX_BPM / hopSeconds));
+  const maxLag = Math.ceil(60 / MIN_BPM / hopSeconds);
+  if (frames < maxLag * 4) return undefined;
+  let total = 0;
+  for (const value of envelope) total += value;
+  if (!(total > 0)) return undefined;
+  const scores = new Float64Array(maxLag + 1);
+  let best = -1;
+  let bestScore = 0;
+  let floor = 0;
+  for (let lag = minLag; lag <= maxLag; lag += 1) {
+    let sum = 0;
+    for (let frame = lag; frame < frames; frame += 1)
+      sum += envelope[frame]! * envelope[frame - lag]!;
+    const bpm = 60 / (lag * hopSeconds);
+    // Gaussian weight centred on 120 BPM in log tempo (Ellis-style prior).
+    const weight = Math.exp(-0.5 * (Math.log2(bpm / 120) / 1.0) ** 2);
+    const score = (sum / (frames - lag)) * weight;
+    scores[lag] = score;
+    floor += score;
+    if (score > bestScore) {
+      bestScore = score;
+      best = lag;
+    }
+  }
+  if (best < 0 || bestScore <= 0) return undefined;
+  floor /= maxLag - minLag + 1;
+  // Refine with a parabolic fit between neighbouring lags.
+  let lag = best;
+  if (best > minLag && best < maxLag) {
+    const left = scores[best - 1]!;
+    const right = scores[best + 1]!;
+    const denominator = left - 2 * bestScore + right;
+    if (denominator < 0) lag = best + (0.5 * (left - right)) / denominator;
+  }
+  const bpm = Math.round((60 / (lag * hopSeconds)) * 10) / 10;
+  const confidence = Math.max(0, Math.min(1, 1 - floor / bestScore));
+  // Phase: the offset whose comb of beats collects the most onset energy.
+  const period = lag;
+  const steps = Math.max(1, Math.round(period));
+  let bestOffset = 0;
+  let bestComb = -1;
+  for (let step = 0; step < steps; step += 1) {
+    let comb = 0;
+    for (let position = step; position < frames; position += period)
+      comb += envelope[Math.round(position)] ?? 0;
+    if (comb > bestComb) {
+      bestComb = comb;
+      bestOffset = step;
+    }
+  }
+  return { bpm, confidence, offsetSeconds: bestOffset * hopSeconds };
+}
+
+const SMOOTH_KERNEL = [1, 2, 3, 2, 1].map((weight) => weight / 9);
+
+function smoothEnvelope(envelope: Float32Array): Float32Array {
+  const out = new Float32Array(envelope.length);
+  const half = (SMOOTH_KERNEL.length - 1) / 2;
+  for (let frame = 0; frame < envelope.length; frame += 1) {
+    let sum = 0;
+    for (let offset = -half; offset <= half; offset += 1) {
+      const value = envelope[frame + offset];
+      if (value !== undefined) sum += value * SMOOTH_KERNEL[offset + half]!;
+    }
+    out[frame] = sum;
+  }
+  return out;
+}
+
+type Fft = {
+  size: number;
+  reverse: Uint32Array;
+  cos: Float64Array;
+  sin: Float64Array;
+};
+
+function createFft(size: number): Fft {
+  const reverse = new Uint32Array(size);
+  const bits = Math.log2(size);
+  for (let index = 0; index < size; index += 1) {
+    let value = 0;
+    for (let bit = 0; bit < bits; bit += 1)
+      value |= ((index >> bit) & 1) << (bits - 1 - bit);
+    reverse[index] = value;
+  }
+  const cos = new Float64Array(size / 2);
+  const sin = new Float64Array(size / 2);
+  for (let index = 0; index < size / 2; index += 1) {
+    cos[index] = Math.cos((2 * Math.PI * index) / size);
+    sin[index] = -Math.sin((2 * Math.PI * index) / size);
+  }
+  return { size, reverse, cos, sin };
+}
+
+function fftInPlace(fft: Fft, re: Float64Array, im: Float64Array): void {
+  const size = fft.size;
+  for (let index = 0; index < size; index += 1) {
+    const target = fft.reverse[index]!;
+    if (target > index) {
+      [re[index], re[target]] = [re[target]!, re[index]!];
+      [im[index], im[target]] = [im[target]!, im[index]!];
+    }
+  }
+  for (let width = 2; width <= size; width *= 2) {
+    const half = width / 2;
+    const step = size / width;
+    for (let start = 0; start < size; start += width) {
+      for (let offset = 0; offset < half; offset += 1) {
+        const twiddle = offset * step;
+        const cos = fft.cos[twiddle]!;
+        const sin = fft.sin[twiddle]!;
+        const even = start + offset;
+        const odd = even + half;
+        const oddRe = re[odd]! * cos - im[odd]! * sin;
+        const oddIm = re[odd]! * sin + im[odd]! * cos;
+        re[odd] = re[even]! - oddRe;
+        im[odd] = im[even]! - oddIm;
+        re[even] = re[even]! + oddRe;
+        im[even] = im[even]! + oddIm;
+      }
+    }
+  }
+}
+
+/**
+ * 12-bin pitch-class histogram from sparse 4096-point frames (one every
+ * half second) over 55–1760 Hz, weighted by log magnitude.
+ */
+export function pitchClassHistogram(mono: Float32Array, sampleRate: number) {
+  const size = 4096;
+  const histogram = new Array<number>(12).fill(0);
+  if (mono.length < size) return histogram;
+  const fft = createFft(size);
+  const window = new Float64Array(size);
+  for (let index = 0; index < size; index += 1)
+    window[index] = 0.5 - 0.5 * Math.cos((2 * Math.PI * index) / (size - 1));
+  const re = new Float64Array(size);
+  const im = new Float64Array(size);
+  const hop = Math.max(size, Math.round(sampleRate / 2));
+  const binHz = sampleRate / size;
+  const lowBin = Math.max(1, Math.floor(55 / binHz));
+  const highBin = Math.min(size / 2 - 1, Math.ceil(1760 / binHz));
+  for (let start = 0; start + size <= mono.length; start += hop) {
+    for (let index = 0; index < size; index += 1) {
+      re[index] = mono[start + index]! * window[index]!;
+      im[index] = 0;
+    }
+    fftInPlace(fft, re, im);
+    for (let bin = lowBin; bin <= highBin; bin += 1) {
+      const magnitude = Math.hypot(re[bin]!, im[bin]!);
+      if (magnitude < 1e-4) continue;
+      // Only spectral peaks vote, so harmonics spread less onto neighbours.
+      const left = Math.hypot(re[bin - 1]!, im[bin - 1]!);
+      const right = Math.hypot(re[bin + 1]!, im[bin + 1]!);
+      if (magnitude < left || magnitude < right) continue;
+      const midi = 69 + 12 * Math.log2((bin * binHz) / 440);
+      const pitchClass = ((Math.round(midi) % 12) + 12) % 12;
+      histogram[pitchClass] =
+        histogram[pitchClass]! + Math.log1p(magnitude * 100);
+    }
+  }
+  return histogram;
+}
+
+export function keyFromHistogram(histogram: readonly number[]): string | null {
+  return estimateKey(histogram);
+}
+
+/** 240 normalised RMS buckets over the whole file, rounded to 3 decimals. */
+export function waveformPeaks(
+  wav: ParsedWav,
+  buckets = MEDIA_LIMITS.waveformBuckets,
+): readonly number[] {
+  const count = Math.max(1, Math.min(buckets, wav.sampleCount));
+  const peaks = new Array<number>(count).fill(0);
+  const perBucket = wav.sampleCount / count;
+  let maximum = 0;
+  for (let bucket = 0; bucket < count; bucket += 1) {
+    const start = Math.floor(bucket * perBucket);
+    const end = Math.max(start + 1, Math.floor((bucket + 1) * perBucket));
+    // Stride long buckets so a 10-minute file costs the same as a short one.
+    const stride = Math.max(1, Math.floor((end - start) / 2_048));
+    let energy = 0;
+    let samples = 0;
+    for (let frame = start; frame < end; frame += stride) {
+      for (let channel = 0; channel < wav.channels; channel += 1) {
+        const value = wav.sample(frame, channel);
+        energy += value * value;
+        samples += 1;
+      }
+    }
+    const rms = samples > 0 ? Math.sqrt(energy / samples) : 0;
+    peaks[bucket] = rms;
+    if (rms > maximum) maximum = rms;
+  }
+  return peaks.map((value) =>
+    maximum > 0 ? Math.round((value / maximum) * 1000) / 1000 : 0,
+  );
+}

package/src/media/import.ts

@@ -0,0 +1,130 @@
+/**
+ * `import_sample(file, name, {begin?, end?, root?})`: ffmpeg converts any
+ * audio file to a 48 kHz PCM16 wav at `tracks/<slug>/samples/<name>.wav`
+ * and the result carries the `sampler({...})` snippet for `track.ts`. The
+ * sampler schema itself belongs to the project-format lane; this tool only
+ * copies the file and returns the snippet.
+ */
+import { SCORE_LIMITS } from "../../core/score.ts";
+import { join } from "node:path";
+import { missingTool } from "./backend.ts";
+import { MediaToolError, runHelper } from "./process.ts";
+import {
+  ensureDir,
+  formatBytes,
+  projectPath,
+  resolveInput,
+  samplesDir,
+  sha256File,
+} from "./paths.ts";
+import {
+  MEDIA_LIMITS,
+  type MediaResult,
+  type MediaRunContext,
+} from "./types.ts";
+import { parseWav } from "./vendor/wav.ts";
+
+export const SAMPLE_LIMITS = Object.freeze({
+  maxBytes: SCORE_LIMITS.maxSampleFileBytes,
+  maxSeconds: SCORE_LIMITS.maxSampleSeconds,
+  sampleRate: 48_000,
+});
+
+export type ImportSampleArgs = Readonly<{
+  file: string;
+  name: string;
+  /** Loop/playback window as fractions 0..1 of the file. */
+  begin?: number;
+  end?: number;
+  /** Root note name or MIDI number the sample is pitched at (default C4). */
+  root?: string | number;
+}>;
+
+export async function importSample(
+  args: ImportSampleArgs,
+  context: MediaRunContext,
+): Promise<MediaResult> {
+  if (!context.runner.which("ffmpeg"))
+    throw new MediaToolError(missingTool("ffmpeg"));
+  const input = await resolveInput(context, args.file);
+  const dir = await ensureDir(samplesDir(context));
+  const target = join(dir, `${args.name}.wav`);
+  context.progress(`ffmpeg → samples/${args.name}.wav`);
+  const temp = `${target}.part.wav`;
+  await runHelper(
+    context,
+    [
+      "ffmpeg",
+      "-hide_banner",
+      "-nostdin",
+      "-loglevel",
+      "error",
+      "-y",
+      "-i",
+      input.absolute,
+      "-vn",
+      "-t",
+      String(SAMPLE_LIMITS.maxSeconds),
+      "-ar",
+      String(SAMPLE_LIMITS.sampleRate),
+      "-ac",
+      "2",
+      "-c:a",
+      "pcm_s16le",
+      temp,
+    ],
+    { timeoutMs: MEDIA_LIMITS.sampleTimeoutMs },
+  );
+  const file = Bun.file(temp);
+  if (file.size > SAMPLE_LIMITS.maxBytes) {
+    await file.delete().catch(() => undefined);
+    throw new MediaToolError(
+      `${args.name}.wav would be ${formatBytes(file.size)}, over the ${formatBytes(SAMPLE_LIMITS.maxBytes)} sample cap; trim with begin/end on a shorter source`,
+    );
+  }
+  const wav = parseWav(new Uint8Array(await file.arrayBuffer()), {
+    maximumBytes: SAMPLE_LIMITS.maxBytes,
+    maximumDurationSeconds: SAMPLE_LIMITS.maxSeconds + 1,
+    maximumChannels: 2,
+  });
+  const { rename } = await import("node:fs/promises");
+  await rename(temp, target);
+  const sha256 = await sha256File(target);
+  const durationSeconds =
+    Math.round((wav.sampleCount / wav.sampleRate) * 1000) / 1000;
+  const src = `samples/${args.name}.wav`;
+  const ref: Record<string, unknown> = { src };
+  if (args.root !== undefined) ref.root = args.root;
+  if (args.begin !== undefined) ref.begin = args.begin;
+  if (args.end !== undefined) ref.end = args.end;
+  const inner =
+    Object.keys(ref).length === 1
+      ? JSON.stringify(src)
+      : `{ ${Object.entries(ref)
+          .map(([key, value]) => `${key}: ${JSON.stringify(value)}`)
+          .join(", ")} }`;
+  // A root means the sample is pitched: keyed mode resamples it across notes.
+  const snippet = `instrument: sampler({ ${args.name}: ${inner} }${args.root !== undefined ? ', { mode: "keyed" }' : ""})`;
+  const relativePath = projectPath(context.projectRoot, target);
+  return {
+    summary: `imported ${src} (${durationSeconds}s, ${formatBytes(file.size)})`,
+    outputs: [relativePath],
+    content: {
+      sample: {
+        name: args.name,
+        src,
+        path: relativePath,
+        sha256,
+        durationSeconds,
+        sampleRate: wav.sampleRate,
+        channels: wav.channels,
+        bytes: file.size,
+        ...(args.root !== undefined ? { root: args.root } : {}),
+        ...(args.begin !== undefined ? { begin: args.begin } : {}),
+        ...(args.end !== undefined ? { end: args.end } : {}),
+      },
+      snippet,
+      note: "Paste the snippet into the track's instrument in track.ts; the sampler schema (voices, root, begin, end, gain, loop, choke) is defined in DAWG.md.",
+    },
+  };
+}

package/src/media/lyrics.ts

@@ -0,0 +1,201 @@
+/**
+ * `transcribe_lyrics(file, {lang?})`: whisper.cpp (`whisper-cli`) on a 16 kHz
+ * mono copy of the file. The ggml model lives in `~/.cache/dawg/whisper/`;
+ * when it is missing a progress card names the size and destination before
+ * dawg fetches it from Hugging Face. Output: `<name>.lyrics.json` (segments
+ * with seconds) and `<name>.lyrics.txt`.
+ */
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { missingTool } from "./backend.ts";
+import {
+  ensureDir,
+  exists,
+  formatBytes,
+  projectPath,
+  resolveInput,
+  writeJsonAtomic,
+} from "./paths.ts";
+import {
+  MediaToolError,
+  downloadToFile,
+  runHelper,
+  throwIfAborted,
+  withTempDir,
+} from "./process.ts";
+import {
+  MEDIA_LIMITS,
+  type MediaResult,
+  type MediaRunContext,
+} from "./types.ts";
+import { finiteNumber, isRecord, optionalString } from "./vendor/util.ts";
+
+export const WHISPER_MODEL_BASE_URL =
+  "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/";
+const MODEL_MAX_BYTES = 200 * 1024 * 1024;
+const MAX_SEGMENTS = 4_000;
+const MAX_TEXT_CHARS = 4_000;
+const LANGUAGE_PATTERN = /^[a-z]{2,3}$|^auto$/;
+
+export type LyricsArgs = Readonly<{ file: string; lang?: string }>;
+
+export function whisperModelName(lang: string | undefined): string {
+  return lang === undefined || lang === "en"
+    ? "ggml-base.en.bin"
+    : "ggml-base.bin";
+}
+
+export function whisperModelDir(homeDir: string | undefined): string {
+  return join(homeDir ?? homedir(), ".cache", "dawg", "whisper");
+}
+
+export type LyricSegment = Readonly<{
+  start: number;
+  end: number;
+  text: string;
+}>;
+
+/** Parse whisper-cli's `-oj` JSON (`transcription[].offsets` are milliseconds). */
+export function parseWhisperJson(value: unknown): readonly LyricSegment[] {
+  if (!isRecord(value) || !Array.isArray(value.transcription)) return [];
+  const segments: LyricSegment[] = [];
+  for (const entry of value.transcription.slice(0, MAX_SEGMENTS)) {
+    if (!isRecord(entry)) continue;
+    const offsets = isRecord(entry.offsets) ? entry.offsets : {};
+    const from = finiteNumber(offsets.from);
+    const to = finiteNumber(offsets.to);
+    const text = optionalString(entry.text, 2_000);
+    if (from === undefined || to === undefined || !text) continue;
+    segments.push({ start: from / 1000, end: to / 1000, text });
+  }
+  return segments;
+}
+
+export async function transcribeLyrics(
+  args: LyricsArgs,
+  context: MediaRunContext,
+): Promise<MediaResult> {
+  if (args.lang !== undefined && !LANGUAGE_PATTERN.test(args.lang))
+    throw new MediaToolError(
+      "lang must be a two- or three-letter code or auto",
+    );
+  if (!context.runner.which("whisper-cli"))
+    throw new MediaToolError(missingTool("whisper-cli"));
+  if (!context.runner.which("ffmpeg"))
+    throw new MediaToolError(missingTool("ffmpeg"));
+  const input = await resolveInput(context, args.file);
+  const modelName = whisperModelName(args.lang);
+  const modelDir = whisperModelDir(context.homeDir ?? context.env?.HOME);
+  const modelPath = join(modelDir, modelName);
+  if (!(await exists(modelPath))) {
+    await ensureDir(modelDir);
+    context.progress(
+      `downloading ${modelName} (~${formatBytes(MEDIA_LIMITS.whisperModelBytes)}) to ${modelDir} from huggingface.co/ggerganov/whisper.cpp`,
+    );
+    await downloadToFile(
+      context.fetch ?? fetch,
+      `${WHISPER_MODEL_BASE_URL}${modelName}`,
+      modelPath,
+      {
+        maxBytes: MODEL_MAX_BYTES,
+        signal: context.signal,
+        progress: (received, total) =>
+          context.progress(
+            total
+              ? `model ${Math.round((received / total) * 100)}%`
+              : `model ${formatBytes(received)}`,
+          ),
+      },
+    );
+  }
+  throwIfAborted(context.signal);
+  return withTempDir("lyrics", async (temp) => {
+    const pcm = join(temp, "audio16k.wav");
+    context.progress("ffmpeg → 16 kHz mono");
+    await runHelper(
+      context,
+      [
+        "ffmpeg",
+        "-hide_banner",
+        "-nostdin",
+        "-loglevel",
+        "error",
+        "-y",
+        "-i",
+        input.absolute,
+        "-vn",
+        "-t",
+        "3600",
+        "-ar",
+        "16000",
+        "-ac",
+        "1",
+        "-c:a",
+        "pcm_s16le",
+        pcm,
+      ],
+      { timeoutMs: MEDIA_LIMITS.sampleTimeoutMs },
+    );
+    const outBase = join(temp, "transcript");
+    context.progress("whisper transcribing");
+    await runHelper(
+      context,
+      [
+        "whisper-cli",
+        "-m",
+        modelPath,
+        "-f",
+        pcm,
+        "-l",
+        args.lang ?? "en",
+        "-np",
+        "-oj",
+        "-of",
+        outBase,
+      ],
+      {
+        timeoutMs: MEDIA_LIMITS.lyricsTimeoutMs,
+        progress: (chunk) => {
+          const match = chunk.match(/progress\s*=\s*(\d{1,3})%/);
+          return match ? `whisper ${match[1]}%` : undefined;
+        },
+      },
+    );
+    const jsonFile = Bun.file(`${outBase}.json`);
+    if (jsonFile.size > 32 * 1024 * 1024)
+      throw new MediaToolError("whisper output too large");
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(await jsonFile.text()) as unknown;
+    } catch {
+      throw new MediaToolError("whisper-cli wrote no JSON transcript");
+    }
+    const segments = parseWhisperJson(parsed);
+    const base = input.absolute.replace(/\.[A-Za-z0-9]{1,5}$/, "");
+    const jsonOut = `${base}.lyrics.json`;
+    const textOut = `${base}.lyrics.txt`;
+    const text = segments.map((segment) => segment.text.trim()).join("\n");
+    await writeJsonAtomic(jsonOut, {
+      file: input.relative,
+      model: modelName,
+      language: args.lang ?? "en",
+      segments,
+    });
+    await Bun.write(textOut, `${text}\n`);
+    const outputs = [jsonOut, textOut].map((path) =>
+      projectPath(context.projectRoot, path),
+    );
+    return {
+      summary: `transcribed ${segments.length} segments from ${input.relative}`,
+      outputs,
+      content: {
+        file: input.relative,
+        lyrics: outputs[0],
+        text: outputs[1],
+        segments: segments.length,
+        preview: text.slice(0, MAX_TEXT_CHARS),
+        ...(text.length > MAX_TEXT_CHARS ? { truncated: true } : {}),
+      },
+    };
+  });
+}