@hraness/dawg 0.2.0 → 0.3.0

package/src/media/analyze.ts

@@ -0,0 +1,364 @@
+/**
+ * `analyze_audio(file)`: ffprobe for format and duration, a beat grid from
+ * StemDeck when the file came through it, otherwise a TypeScript onset
+ * autocorrelation tempo estimate; key from a pitch-class histogram (the same
+ * scale fit the session namer uses); and 240 waveform peaks. Everything is
+ * written to `<name>.analysis.json` next to the file and summarised for the
+ * model.
+ */
+import { join } from "node:path";
+import { detectStemDeck, missingTool } from "./backend.ts";
+import {
+  estimateTempo,
+  keyFromHistogram,
+  monoSignal,
+  onsetEnvelope,
+  pitchClassHistogram,
+  waveformPeaks,
+} from "./dsp.ts";
+import {
+  formatSeconds,
+  projectPath,
+  readJson,
+  resolveInput,
+  writeJsonAtomic,
+} from "./paths.ts";
+import {
+  MediaToolError,
+  runHelper,
+  throwIfAborted,
+  withTempDir,
+} from "./process.ts";
+import { readSidecar } from "./sidecar.ts";
+import { createStemDeck, type StemDeckOptions } from "./stemdeck.ts";
+import {
+  MEDIA_LIMITS,
+  type BeatGrid,
+  type MediaResult,
+  type MediaRunContext,
+} from "./types.ts";
+import { beatIntervalCv, fixedBeatGrid, gridMedianBpm } from "./vendor/grid.ts";
+import { finiteNumber, isRecord, optionalString } from "./vendor/util.ts";
+import { parseWav, type ParsedWav } from "./vendor/wav.ts";
+
+export type AudioAnalysis = Readonly<{
+  file: string;
+  durationSeconds: number;
+  sampleRate: number;
+  channels: number;
+  codec: string;
+  bytes: number;
+  bpm?: number;
+  bpmConfidence?: number;
+  key?: string;
+  grid?: BeatGrid;
+  peaks: readonly number[];
+  analyzedAt: string;
+}>;
+
+export type AnalyzeArgs = Readonly<{ file: string }>;
+export type AnalyzeOptions = Readonly<{
+  stemdeck?: StemDeckOptions;
+  now?: () => Date;
+}>;
+
+export function analysisPath(filePath: string): string {
+  return filePath.replace(/\.[A-Za-z0-9]{1,5}$/, "") + ".analysis.json";
+}
+
+type Probe = Readonly<{
+  durationSeconds: number;
+  sampleRate: number;
+  channels: number;
+  codec: string;
+  bytes: number;
+}>;
+
+export async function ffprobe(
+  context: MediaRunContext,
+  path: string,
+): Promise<Probe> {
+  if (!context.runner.which("ffprobe"))
+    throw new MediaToolError(missingTool("ffprobe"));
+  const result = await runHelper(
+    context,
+    [
+      "ffprobe",
+      "-v",
+      "error",
+      "-print_format",
+      "json",
+      "-show_format",
+      "-show_streams",
+      "-select_streams",
+      "a:0",
+      path,
+    ],
+    { timeoutMs: 60_000 },
+  );
+  let body: unknown;
+  try {
+    body = JSON.parse(result.stdout) as unknown;
+  } catch {
+    throw new MediaToolError("ffprobe did not return JSON");
+  }
+  if (!isRecord(body))
+    throw new MediaToolError("ffprobe output was not an object");
+  const format = isRecord(body.format) ? body.format : {};
+  const stream =
+    Array.isArray(body.streams) && isRecord(body.streams[0])
+      ? body.streams[0]
+      : {};
+  const duration = Number(
+    optionalString(format.duration ?? stream.duration, 40),
+  );
+  const sampleRate = Number(optionalString(stream.sample_rate, 20));
+  const channels = finiteNumber(stream.channels);
+  const bytes = Number(optionalString(format.size, 30));
+  if (!Number.isFinite(duration) || duration <= 0)
+    throw new MediaToolError(
+      "ffprobe found no audio duration (not an audio file?)",
+    );
+  return {
+    durationSeconds: Math.round(duration * 1000) / 1000,
+    sampleRate: Number.isFinite(sampleRate) && sampleRate > 0 ? sampleRate : 0,
+    channels: channels !== undefined && channels > 0 ? channels : 0,
+    codec: optionalString(stream.codec_name, 40) ?? "unknown",
+    bytes: Number.isFinite(bytes) ? bytes : Bun.file(path).size,
+  };
+}
+
+/** Parse a wav directly, or decode anything else to 22.05 kHz mono via ffmpeg. */
+export async function loadWav(
+  context: MediaRunContext,
+  path: string,
+): Promise<ParsedWav> {
+  const file = Bun.file(path);
+  if (file.size > MEDIA_LIMITS.maxWavBytes)
+    throw new MediaToolError(
+      `${path.split("/").at(-1)} is over the 256 MiB analysis cap`,
+    );
+  const head = new Uint8Array(await file.slice(0, 12).arrayBuffer());
+  const riff = String.fromCharCode(...head.subarray(0, 4)) === "RIFF";
+  if (riff) return parseWav(new Uint8Array(await file.arrayBuffer()));
+  if (!context.runner.which("ffmpeg"))
+    throw new MediaToolError(missingTool("ffmpeg"));
+  return withTempDir("decode", async (temp) => {
+    const out = join(temp, "decoded.wav");
+    context.progress("ffmpeg decoding");
+    await runHelper(
+      context,
+      [
+        "ffmpeg",
+        "-hide_banner",
+        "-nostdin",
+        "-loglevel",
+        "error",
+        "-y",
+        "-i",
+        path,
+        "-vn",
+        "-t",
+        "600",
+        "-ar",
+        "22050",
+        "-ac",
+        "1",
+        "-c:a",
+        "pcm_s16le",
+        out,
+      ],
+      { timeoutMs: MEDIA_LIMITS.analyzeTimeoutMs },
+    );
+    return parseWav(new Uint8Array(await Bun.file(out).arrayBuffer()));
+  });
+}
+
+export async function readAnalysis(
+  filePath: string,
+): Promise<AudioAnalysis | undefined> {
+  const value = await readJson(analysisPath(filePath));
+  if (!isRecord(value)) return undefined;
+  const duration = finiteNumber(value.durationSeconds);
+  if (duration === undefined) return undefined;
+  const grid =
+    isRecord(value.grid) && Array.isArray(value.grid.beats)
+      ? parseGrid(value.grid)
+      : undefined;
+  const bpm = finiteNumber(value.bpm);
+  const key = optionalString(value.key, 20);
+  return {
+    file: optionalString(value.file, 1024) ?? "",
+    durationSeconds: duration,
+    sampleRate: finiteNumber(value.sampleRate) ?? 0,
+    channels: finiteNumber(value.channels) ?? 0,
+    codec: optionalString(value.codec, 40) ?? "unknown",
+    bytes: finiteNumber(value.bytes) ?? 0,
+    ...(bpm !== undefined ? { bpm } : {}),
+    ...(key ? { key } : {}),
+    ...(grid ? { grid } : {}),
+    peaks: Array.isArray(value.peaks)
+      ? value.peaks.slice(0, 240).map((entry) => finiteNumber(entry) ?? 0)
+      : [],
+    analyzedAt: optionalString(value.analyzedAt, 40) ?? "",
+  };
+}
+
+function parseGrid(value: Record<string, unknown>): BeatGrid | undefined {
+  const beats = (value.beats as unknown[])
+    .slice(0, 20_000)
+    .map((entry) => finiteNumber(entry));
+  if (beats.length < 2 || beats.some((beat) => beat === undefined))
+    return undefined;
+  const clean = beats as number[];
+  for (let index = 1; index < clean.length; index += 1)
+    if (clean[index]! <= clean[index - 1]!) return undefined;
+  const bars = Array.isArray(value.bars)
+    ? value.bars.slice(0, 2_000).flatMap((entry) => {
+        if (!isRecord(entry)) return [];
+        const beat = finiteNumber(entry.beat);
+        const beatsPerBar = finiteNumber(entry.beatsPerBar);
+        return beat === undefined || beatsPerBar === undefined
+          ? []
+          : [{ beat, beatsPerBar }];
+      })
+    : [];
+  const bpm = finiteNumber(value.bpm);
+  const detector = optionalString(value.detector, 80);
+  return {
+    beats: clean,
+    bars,
+    durationSeconds: finiteNumber(value.durationSeconds) ?? clean.at(-1)!,
+    ...(bpm !== undefined ? { bpm } : {}),
+    ...(detector ? { detector } : {}),
+    intervalCv: beatIntervalCv(clean),
+  };
+}
+
+export async function analyzeAudio(
+  args: AnalyzeArgs,
+  context: MediaRunContext,
+  options: AnalyzeOptions = {},
+): Promise<MediaResult> {
+  const input = await resolveInput(context, args.file);
+  context.progress("ffprobe");
+  const probe = await ffprobe(context, input.absolute);
+  if (probe.durationSeconds > 7_200)
+    throw new MediaToolError("analysis is limited to files under two hours");
+  throwIfAborted(context.signal);
+
+  // Beat grid from StemDeck when this file (or the download it was split from) went through it.
+  let grid: BeatGrid | undefined;
+  let bpm: number | undefined;
+  let bpmConfidence: number | undefined;
+  let key: string | undefined;
+  const sidecar =
+    (await readSidecar(input.absolute)) ??
+    (await readSidecar(parentDownload(input.absolute)));
+  if (sidecar?.stemdeck) {
+    const health = await detectStemDeck(context);
+    if (health && health.url === sidecar.stemdeck.url) {
+      context.progress("stemdeck beat grid");
+      const client = createStemDeck(health, context, options.stemdeck);
+      grid = await client.beats(sidecar.stemdeck.jobId);
+      if (grid) {
+        bpm = gridMedianBpm(grid);
+        bpmConfidence =
+          grid.confidence !== undefined
+            ? Math.min(1, grid.confidence / 100)
+            : 1 - Math.min(1, grid.intervalCv ?? 0);
+      }
+    }
+    if (sidecar.key) key = sidecar.key.toLowerCase();
+    if (bpm === undefined && sidecar.bpm !== undefined) bpm = sidecar.bpm;
+  }
+
+  context.progress("decoding");
+  const wav = await loadWav(context, input.absolute);
+  throwIfAborted(context.signal);
+  const mono = monoSignal(wav);
+  if (grid === undefined) {
+    context.progress("tempo");
+    const { envelope, hopSeconds } = onsetEnvelope(mono, wav.sampleRate);
+    const tempo = estimateTempo(envelope, hopSeconds);
+    if (tempo) {
+      bpm = bpm ?? tempo.bpm;
+      bpmConfidence = tempo.confidence;
+      grid = fixedBeatGrid(bpm, probe.durationSeconds, tempo.offsetSeconds);
+    }
+  }
+  throwIfAborted(context.signal);
+  if (key === undefined) {
+    context.progress("key");
+    key =
+      keyFromHistogram(pitchClassHistogram(mono, wav.sampleRate)) ?? undefined;
+  }
+  context.progress("waveform");
+  const peaks = waveformPeaks(wav);
+  const analysis: AudioAnalysis = {
+    file: input.relative,
+    ...probe,
+    ...(bpm !== undefined ? { bpm: Math.round(bpm * 10) / 10 } : {}),
+    ...(bpmConfidence !== undefined
+      ? { bpmConfidence: Math.round(bpmConfidence * 100) / 100 }
+      : {}),
+    ...(key ? { key } : {}),
+    ...(grid ? { grid } : {}),
+    peaks,
+    analyzedAt: (options.now ?? (() => new Date()))().toISOString(),
+  };
+  const out = analysisPath(input.absolute);
+  await writeJsonAtomic(out, analysis);
+  const relativeOut = projectPath(context.projectRoot, out);
+  return {
+    summary: `analyzed ${input.relative}: ${formatSeconds(probe.durationSeconds)}${bpm !== undefined ? ` · ${analysis.bpm} bpm` : ""}${key ? ` · ${key}` : ""}`,
+    outputs: [relativeOut],
+    content: {
+      file: input.relative,
+      analysis: relativeOut,
+      durationSeconds: probe.durationSeconds,
+      sampleRate: probe.sampleRate,
+      channels: probe.channels,
+      codec: probe.codec,
+      ...(analysis.bpm !== undefined ? { bpm: analysis.bpm } : {}),
+      ...(analysis.bpmConfidence !== undefined
+        ? { bpmConfidence: analysis.bpmConfidence }
+        : {}),
+      ...(key ? { key } : {}),
+      ...(grid
+        ? {
+            grid: {
+              detector: grid.detector ?? "unknown",
+              beats: grid.beats.length,
+              firstBeatSeconds: Math.round(grid.beats[0]! * 1000) / 1000,
+              bars: grid.bars.length,
+            },
+          }
+        : {}),
+      peaks: downsample(peaks, 48),
+    },
+  };
+}
+
+/** `<x>.stems/<stem>.wav` → `<x>.wav`, so stems inherit the download's sidecar. */
+function parentDownload(path: string): string {
+  const match = /^(.*)\.stems\/[^/]+\.wav$/.exec(path);
+  return match ? `${match[1]}.wav` : path;
+}
+
+function downsample(values: readonly number[], buckets: number): number[] {
+  if (values.length <= buckets) return [...values];
+  const out: number[] = [];
+  const per = values.length / buckets;
+  for (let bucket = 0; bucket < buckets; bucket += 1) {
+    const start = Math.floor(bucket * per);
+    const end = Math.max(start + 1, Math.floor((bucket + 1) * per));
+    let maximum = 0;
+    for (let index = start; index < end; index += 1)
+      maximum = Math.max(maximum, values[index] ?? 0);
+    out.push(Math.round(maximum * 100) / 100);
+  }
+  return out;
+}
+
+export { analysisPath as analysisPathFor };

package/src/media/backend.ts

@@ -0,0 +1,253 @@
+/**
+ * Which backend a media tool can use. A local StemDeck
+ * (`DAWG_STEMDECK_URL`, default http://127.0.0.1:8000) answers `/api/health`
+ * within a second, or the tools run the binaries directly: `yt-dlp`,
+ * `ffmpeg`, `ffprobe`, `uv` (for `uv tool run demucs` / `basic-pitch`) and
+ * `whisper-cli`. Nothing here installs anything; `doctor()` prints the exact
+ * command for each missing piece instead.
+ */
+import type { CommandRunner } from "../auth/runner.ts";
+import type { MediaHost } from "./types.ts";
+import {
+  finiteNumber,
+  isRecord,
+  normalizeServiceBaseUrl,
+  optionalString,
+} from "./vendor/util.ts";
+
+export const DEFAULT_STEMDECK_URL = "http://127.0.0.1:8000";
+const HEALTH_TIMEOUT_MS = 1_000;
+const UV_LIST_TIMEOUT_MS = 10_000;
+
+export const MEDIA_TOOL_NAMES = [
+  "yt-dlp",
+  "ffmpeg",
+  "ffprobe",
+  "uv",
+  "demucs",
+  "basic-pitch",
+  "whisper-cli",
+] as const;
+export type MediaToolName = (typeof MEDIA_TOOL_NAMES)[number];
+
+export const INSTALL_COMMANDS: Readonly<Record<MediaToolName, string>> = {
+  "yt-dlp": "brew install yt-dlp ffmpeg whisper-cpp",
+  ffmpeg: "brew install yt-dlp ffmpeg whisper-cpp",
+  ffprobe: "brew install yt-dlp ffmpeg whisper-cpp",
+  "whisper-cli": "brew install yt-dlp ffmpeg whisper-cpp",
+  uv: "curl -LsSf https://astral.sh/uv/install.sh | sh",
+  demucs: "uv tool install demucs",
+  "basic-pitch": "uv tool install basic-pitch",
+};
+
+/** What each tool is for, in the doctor report. */
+const PURPOSE: Readonly<Record<MediaToolName, string>> = {
+  "yt-dlp": "download_audio",
+  ffmpeg: "import_sample, decoding for analysis",
+  ffprobe: "analyze_audio",
+  uv: "runs demucs and basic-pitch",
+  demucs: "split_stems (direct backend)",
+  "basic-pitch": "transcribe_notes for pitched stems",
+  "whisper-cli": "transcribe_lyrics",
+};
+
+export type ToolStatus = Readonly<{
+  name: MediaToolName;
+  available: boolean;
+  /** How the tool is started: a PATH binary or `uv tool run <name>`. */
+  command?: readonly string[];
+  install: string;
+  purpose: string;
+  /** First-run model download the tool performs on its own. */
+  firstRun?: string;
+}>;
+
+export type StemDeckHealth = Readonly<{
+  url: string;
+  version?: string;
+  demucsModel?: string;
+  demucsDevice?: string;
+}>;
+
+export type MediaBackend =
+  | Readonly<{ kind: "stemdeck"; stemdeck: StemDeckHealth }>
+  /** `checked` is the StemDeck origin that did not answer (no credentials). */
+  | Readonly<{ kind: "direct"; checked?: string }>;
+
+export const FIRST_RUN_NOTES: Readonly<Partial<Record<MediaToolName, string>>> =
+  {
+    demucs:
+      "demucs downloads the htdemucs_6s model (~80 MB) into ~/.cache/torch on its first run",
+    "basic-pitch":
+      "basic-pitch ships its model in the package; the first run compiles it and is slower",
+    "whisper-cli":
+      "dawg downloads ggml-base.en.bin (~148 MB) into ~/.cache/dawg/whisper before the first transcription",
+  };
+
+/** The StemDeck base URL from the environment, or the default. Never logged with credentials. */
+export function stemdeckUrl(
+  env: Readonly<Record<string, string | undefined>> | undefined,
+): URL {
+  const raw = env?.DAWG_STEMDECK_URL?.trim();
+  return normalizeServiceBaseUrl(raw ? raw : DEFAULT_STEMDECK_URL);
+}
+
+/** `GET /api/health` with a one-second budget; undefined when StemDeck is not there. */
+export async function detectStemDeck(
+  host: Pick<MediaHost, "fetch" | "env"> & { signal?: AbortSignal },
+): Promise<StemDeckHealth | undefined> {
+  let base: URL;
+  try {
+    base = stemdeckUrl(host.env);
+  } catch {
+    return undefined;
+  }
+  const prefix = `${base.origin}${base.pathname.replace(/\/+$/u, "")}`;
+  const fetcher = host.fetch ?? fetch;
+  const timeout = AbortSignal.timeout(HEALTH_TIMEOUT_MS);
+  const signal = host.signal
+    ? AbortSignal.any([host.signal, timeout])
+    : timeout;
+  try {
+    const response = await fetcher(`${prefix}/api/health`, {
+      signal,
+      headers: { accept: "application/json" },
+    });
+    if (!response.ok) return undefined;
+    const text = await response.text();
+    if (text.length > 64 * 1024) return undefined;
+    const body: unknown = JSON.parse(text);
+    if (!isRecord(body) || body.status !== "ok") return undefined;
+    const name = optionalString(body.name, 40);
+    if (name !== undefined && name.toLowerCase() !== "stemdeck")
+      return undefined;
+    const version = optionalString(body.version, 40);
+    const demucsModel = optionalString(body.demucs_model, 40);
+    const demucsDevice = optionalString(body.demucs_device, 40);
+    return {
+      url: prefix,
+      ...(version ? { version } : {}),
+      ...(demucsModel ? { demucsModel } : {}),
+      ...(demucsDevice ? { demucsDevice } : {}),
+    };
+  } catch {
+    return undefined;
+  }
+}
+
+export async function detectBackend(
+  host: Pick<MediaHost, "fetch" | "env"> & { signal?: AbortSignal },
+): Promise<MediaBackend> {
+  const stemdeck = await detectStemDeck(host);
+  if (stemdeck) return { kind: "stemdeck", stemdeck };
+  try {
+    return { kind: "direct", checked: stemdeckUrl(host.env).origin };
+  } catch {
+    return { kind: "direct" };
+  }
+}
+
+/** Names `uv tool list` reports as installed (never installs anything). */
+export async function uvInstalledTools(
+  runner: CommandRunner,
+  signal?: AbortSignal,
+): Promise<ReadonlySet<string>> {
+  if (!runner.which("uv")) return new Set();
+  try {
+    const result = await runner.run("uv", ["tool", "list"], {
+      timeoutMs: UV_LIST_TIMEOUT_MS,
+      maxOutputBytes: 64 * 1024,
+      ...(signal ? { signal } : {}),
+    });
+    if (result.code !== 0) return new Set();
+    const names = new Set<string>();
+    for (const line of result.stdout.split("\n")) {
+      // `demucs v4.0.1` heads each block; indented lines are entry points.
+      const match = /^([A-Za-z0-9_.-]+) v\S+/.exec(line);
+      if (match) names.add(match[1]!.toLowerCase());
+    }
+    return names;
+  } catch {
+    return new Set();
+  }
+}
+
+/** Resolve one tool to the argv prefix that starts it, or undefined. */
+export function toolCommand(
+  name: MediaToolName,
+  runner: CommandRunner,
+  uvTools: ReadonlySet<string>,
+): readonly string[] | undefined {
+  const direct = runner.which(name);
+  if (direct) return [name];
+  if ((name === "demucs" || name === "basic-pitch") && runner.which("uv")) {
+    if (uvTools.has(name)) return ["uv", "tool", "run", name];
+  }
+  return undefined;
+}
+
+export async function probeTools(
+  runner: CommandRunner,
+  signal?: AbortSignal,
+): Promise<readonly ToolStatus[]> {
+  const uvTools = await uvInstalledTools(runner, signal);
+  return MEDIA_TOOL_NAMES.map((name) => {
+    const command = toolCommand(name, runner, uvTools);
+    const firstRun = FIRST_RUN_NOTES[name];
+    return {
+      name,
+      available: command !== undefined,
+      ...(command ? { command } : {}),
+      install: INSTALL_COMMANDS[name],
+      purpose: PURPOSE[name],
+      ...(firstRun ? { firstRun } : {}),
+    };
+  });
+}
+
+export type DoctorReport = Readonly<{
+  backend: MediaBackend;
+  tools: readonly ToolStatus[];
+}>;
+
+export async function doctor(
+  host: Pick<MediaHost, "runner" | "fetch" | "env"> & { signal?: AbortSignal },
+): Promise<DoctorReport> {
+  const [backend, tools] = await Promise.all([
+    detectBackend(host),
+    probeTools(host.runner, host.signal),
+  ]);
+  return { backend, tools };
+}
+
+/** Human lines for `dawg media doctor` and activity cards. */
+export function formatDoctor(report: DoctorReport): readonly string[] {
+  const lines: string[] = [];
+  if (report.backend.kind === "stemdeck") {
+    const s = report.backend.stemdeck;
+    lines.push(
+      `✓ StemDeck at ${s.url}${s.version ? ` v${s.version}` : ""}${s.demucsModel ? ` · ${s.demucsModel}` : ""}${s.demucsDevice ? ` on ${s.demucsDevice}` : ""}`,
+    );
+  } else {
+    lines.push(
+      `· no StemDeck at ${report.backend.checked ?? DEFAULT_STEMDECK_URL} (set DAWG_STEMDECK_URL); using local binaries`,
+    );
+  }
+  for (const tool of report.tools) {
+    lines.push(
+      tool.available
+        ? `✓ ${tool.name} · ${tool.purpose}${tool.command && tool.command.length > 1 ? ` · via ${tool.command.join(" ")}` : ""}`
+        : `✗ ${tool.name} · ${tool.purpose} · install: ${tool.install}`,
+    );
+  }
+  for (const tool of report.tools)
+    if (tool.available && tool.firstRun) lines.push(`  ${tool.firstRun}`);
+  return lines;
+}
+
+/** Error text for a tool a run needs but could not find. */
+export function missingTool(name: MediaToolName): string {
+  return `${name} is not installed; install it with: ${INSTALL_COMMANDS[name]} (dawg never installs tools itself; run \`dawg media doctor\`)`;
+}
+
+export { finiteNumber };