@hraness/dawg 0.0.0-stage → 0.3.0

package/src/media/stems.ts

@@ -0,0 +1,173 @@
+/**
+ * `split_stems(file)`: six stems (vocals, drums, bass, guitar, piano, other)
+ * into `<file>.stems/<stem>.wav` next to the input. A download that came
+ * through StemDeck (sidecar has a job id) or that has a source URL and a
+ * running StemDeck is separated there; otherwise `demucs -n htdemucs_6s`
+ * runs locally (`uv tool run demucs` or a PATH binary).
+ */
+import { readdir, rename, rm } from "node:fs/promises";
+import { join } from "node:path";
+import {
+  detectStemDeck,
+  missingTool,
+  toolCommand,
+  uvInstalledTools,
+  FIRST_RUN_NOTES,
+} from "./backend.ts";
+import { readSidecar, sidecarPath, writeSidecarField } from "./sidecar.ts";
+import { ensureDir, exists, projectPath, resolveInput } from "./paths.ts";
+import {
+  MediaToolError,
+  percentProgress,
+  runHelper,
+  throwIfAborted,
+  withTempDir,
+} from "./process.ts";
+import {
+  STEM_NAMES,
+  createStemDeck,
+  type StemDeckOptions,
+} from "./stemdeck.ts";
+import {
+  MEDIA_LIMITS,
+  type MediaResult,
+  type MediaRunContext,
+} from "./types.ts";
+
+export const DEMUCS_MODEL = "htdemucs_6s";
+
+export type SplitStemsArgs = Readonly<{ file: string }>;
+export type SplitStemsOptions = Readonly<{ stemdeck?: StemDeckOptions }>;
+
+export function stemsDirFor(wavPath: string): string {
+  return wavPath.replace(/\.wav$/i, "") + ".stems";
+}
+
+async function existingStems(
+  dir: string,
+): Promise<readonly string[] | undefined> {
+  if (!(await exists(dir))) return undefined;
+  const present = new Set(await readdir(dir));
+  const all = STEM_NAMES.every((name) => present.has(`${name}.wav`));
+  return all ? STEM_NAMES.map((name) => join(dir, `${name}.wav`)) : undefined;
+}
+
+export async function splitStems(
+  args: SplitStemsArgs,
+  context: MediaRunContext,
+  options: SplitStemsOptions = {},
+): Promise<MediaResult> {
+  const input = await resolveInput(context, args.file);
+  const dir = stemsDirFor(input.absolute);
+  const cached = await existingStems(dir);
+  if (cached)
+    return result(context, input.relative, cached, "already split", "cache");
+  const sidecar = await readSidecar(input.absolute);
+  const health = sidecar?.source ? await detectStemDeck(context) : undefined;
+  if (health && sidecar?.source) {
+    const client = createStemDeck(health, context, options.stemdeck);
+    let jobId = sidecar.stemdeck?.jobId;
+    if (!jobId || sidecar.stemdeck?.url !== health.url) {
+      context.progress("stemdeck submitting");
+      jobId = await client.submit(sidecar.source);
+      await writeSidecarField(input.absolute, "stemdeck", {
+        url: health.url,
+        jobId,
+      });
+    }
+    const job = await client.wait(jobId, MEDIA_LIMITS.stemsTimeoutMs);
+    if (job.stems.length === 0)
+      throw new MediaToolError(
+        "StemDeck finished the job but offered no stems",
+      );
+    await ensureDir(dir);
+    const paths = await client.fetchStems(
+      job,
+      dir,
+      MEDIA_LIMITS.downloadMaxBytes,
+    );
+    return result(context, input.relative, paths, "stemdeck split", "stemdeck");
+  }
+  const demucs = toolCommand(
+    "demucs",
+    context.runner,
+    await uvInstalledTools(context.runner, context.signal),
+  );
+  if (!demucs) throw new MediaToolError(missingTool("demucs"));
+  context.progress(FIRST_RUN_NOTES.demucs ?? "demucs starting");
+  const paths = await withTempDir("stems", async (temp) => {
+    await runHelper(
+      context,
+      [
+        ...demucs,
+        "-n",
+        DEMUCS_MODEL,
+        "-o",
+        temp,
+        "--filename",
+        "{stem}.{ext}",
+        input.absolute,
+      ],
+      {
+        timeoutMs: MEDIA_LIMITS.stemsTimeoutMs,
+        progress: percentProgress("demucs"),
+        env: { PYTHONUNBUFFERED: "1" },
+      },
+    );
+    throwIfAborted(context.signal);
+    const produced = join(temp, DEMUCS_MODEL);
+    const missing = [];
+    for (const name of STEM_NAMES)
+      if (!(await exists(join(produced, `${name}.wav`)))) missing.push(name);
+    if (missing.length > 0)
+      throw new MediaToolError(`demucs did not write ${missing.join(", ")}`);
+    await rm(dir, { recursive: true, force: true });
+    await ensureDir(dir);
+    const moved: string[] = [];
+    for (const name of STEM_NAMES) {
+      const from = join(produced, `${name}.wav`);
+      const to = join(dir, `${name}.wav`);
+      await rename(from, to).catch(async () => {
+        await Bun.write(to, Bun.file(from));
+      });
+      moved.push(to);
+    }
+    return moved;
+  });
+  return result(
+    context,
+    input.relative,
+    paths,
+    `demucs ${DEMUCS_MODEL} split`,
+    "demucs",
+  );
+}
+
+function result(
+  context: MediaRunContext,
+  inputRelative: string,
+  paths: readonly string[],
+  verb: string,
+  backend: string,
+): MediaResult {
+  const outputs = paths.map((path) => projectPath(context.projectRoot, path));
+  const stems = Object.fromEntries(
+    paths.map((path) => [
+      path
+        .split("/")
+        .at(-1)!
+        .replace(/\.wav$/, ""),
+      projectPath(context.projectRoot, path),
+    ]),
+  );
+  return {
+    summary: `${verb}: ${inputRelative} → ${outputs.length} stems`,
+    outputs,
+    content: {
+      file: inputRelative,
+      backend,
+      stems,
+      sidecar: projectPath(context.projectRoot, sidecarPath(inputRelative)),
+    },
+  };
+}

package/src/media/tools.ts

@@ -0,0 +1,292 @@
+/**
+ * The six media tools as `AgentTool`s. `plan()` validates arguments from
+ * `unknown` and returns a `media` plan whose `run()` the agent loop executes
+ * with the host's `MediaHost` (project root, focused slug, runner, fetch),
+ * a cancellation signal and a progress sink. `dawg media <verb>` reuses the
+ * same `run` functions through `src/media/cli.ts`.
+ */
+import type { AgentTool, ToolPlan } from "../agent/tools.ts";
+import { analyzeAudio } from "./analyze.ts";
+import { downloadAudio } from "./download.ts";
+import { importSample } from "./import.ts";
+import { transcribeLyrics } from "./lyrics.ts";
+import { NOTE_KINDS, transcribeNotes, type NoteKind } from "./notes.ts";
+import { MAX_NAME_LENGTH, trackSlug } from "./paths.ts";
+import { splitStems } from "./stems.ts";
+import { validateYoutubeUrl } from "./vendor/util.ts";
+
+export class MediaArgumentError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "MediaArgumentError";
+  }
+}
+
+function text(args: Record<string, unknown>, key: string, max: number): string {
+  const value = args[key];
+  if (
+    typeof value !== "string" ||
+    value.trim().length === 0 ||
+    value.length > max
+  )
+    throw new MediaArgumentError(
+      `${key} must be a non-empty string (≤ ${max} chars)`,
+    );
+  return value.trim();
+}
+
+function optionalText(args: Record<string, unknown>, key: string, max: number) {
+  return args[key] === undefined ? undefined : text(args, key, max);
+}
+
+function optionalFraction(args: Record<string, unknown>, key: string) {
+  const value = args[key];
+  if (value === undefined) return undefined;
+  if (
+    typeof value !== "number" ||
+    !Number.isFinite(value) ||
+    value < 0 ||
+    value > 1
+  )
+    throw new MediaArgumentError(`${key} must be a number in 0..1`);
+  return value;
+}
+
+function optionalSeconds(args: Record<string, unknown>, key: string) {
+  const value = args[key];
+  if (value === undefined) return undefined;
+  if (
+    typeof value !== "number" ||
+    !Number.isFinite(value) ||
+    value < 0 ||
+    value > 7_200
+  )
+    throw new MediaArgumentError(`${key} must be seconds in 0..7200`);
+  return value;
+}
+
+function sampleName(args: Record<string, unknown>): string {
+  const value = text(args, "name", MAX_NAME_LENGTH);
+  if (!/^[a-z][a-z0-9_]{0,31}$/.test(value))
+    throw new MediaArgumentError(
+      "name must be a short identifier: lowercase letter, then letters, digits or _, at most 32 characters",
+    );
+  return value;
+}
+
+const fileSchema = {
+  type: "string",
+  maxLength: 1024,
+  description:
+    "Project-relative path (as listed in the brief's downloads or returned by another media tool).",
+};
+
+type MediaRun = Extract<ToolPlan, { kind: "media" }>["run"];
+
+const media = (summary: string, run: MediaRun): ToolPlan => ({
+  kind: "media",
+  summary,
+  run,
+});
+
+export const MEDIA_TOOLS: readonly AgentTool[] = Object.freeze([
+  {
+    name: "download_audio",
+    description:
+      "Download the audio of a YouTube video as a wav into the focused track's downloads/ folder (with a .json sidecar: title, duration, source, sha256). YouTube URLs only; 500 MiB and 15 minutes of wall time at most. Check the brief's downloads list first so the same video is never fetched twice.",
+    parameters: {
+      type: "object",
+      properties: {
+        url: {
+          type: "string",
+          maxLength: 2048,
+          description: "A youtube.com or youtu.be URL.",
+        },
+        name: {
+          type: "string",
+          maxLength: MAX_NAME_LENGTH,
+          description:
+            "Optional file name (slugified); defaults to the video title.",
+        },
+      },
+      required: ["url"],
+      additionalProperties: false,
+    },
+    plan: (args) => {
+      const url = validateYoutubeUrl(text(args, "url", 2048));
+      const name = optionalText(args, "name", MAX_NAME_LENGTH);
+      return media(
+        `download ${name ? trackSlug(name) : new URL(url).hostname}`,
+        (context) => downloadAudio({ url, ...(name ? { name } : {}) }, context),
+      );
+    },
+  },
+  {
+    name: "split_stems",
+    description:
+      "Separate a downloaded wav into six stems (vocals, drums, bass, guitar, piano, other) in <file>.stems/. Uses a local StemDeck when one is running, otherwise demucs htdemucs_6s (up to 20 minutes; the first run downloads the model).",
+    parameters: {
+      type: "object",
+      properties: { file: fileSchema },
+      required: ["file"],
+      additionalProperties: false,
+    },
+    plan: (args) => {
+      const file = text(args, "file", 1024);
+      return media(`split stems of ${file}`, (context) =>
+        splitStems({ file }, context),
+      );
+    },
+  },
+  {
+    name: "analyze_audio",
+    description:
+      "Measure an audio file: duration, sample rate, tempo (bpm with a confidence), beat grid, key and 240 waveform peaks. Written to <name>.analysis.json; later transcribe_notes calls align to this grid.",
+    parameters: {
+      type: "object",
+      properties: { file: fileSchema },
+      required: ["file"],
+      additionalProperties: false,
+    },
+    plan: (args) => {
+      const file = text(args, "file", 1024);
+      return media(`analyze ${file}`, (context) =>
+        analyzeAudio({ file }, context),
+      );
+    },
+  },
+  {
+    name: "transcribe_notes",
+    description:
+      'Transcribe a stem to notes. kind=drums classifies kick/snare/hat/… hits; other kinds run basic-pitch. Returns up to 2048 timed notes and a quantized snippet (note("A1", startBeat, lengthBeats, velocity) or hit("kick", beat)) aligned to the file\'s beat grid; use from/to (seconds) to transcribe one section.',
+    parameters: {
+      type: "object",
+      properties: {
+        file: fileSchema,
+        kind: {
+          type: "string",
+          enum: [...NOTE_KINDS],
+          description:
+            "Defaults from the file name (drums.wav → drums, bass.wav → bass, else other).",
+        },
+        from: {
+          type: "number",
+          minimum: 0,
+          description: "Start of the window in seconds.",
+        },
+        to: {
+          type: "number",
+          minimum: 0,
+          description: "End of the window in seconds.",
+        },
+      },
+      required: ["file"],
+      additionalProperties: false,
+    },
+    plan: (args) => {
+      const file = text(args, "file", 1024);
+      const kindValue = optionalText(args, "kind", 16);
+      if (
+        kindValue !== undefined &&
+        !(NOTE_KINDS as readonly string[]).includes(kindValue)
+      )
+        throw new MediaArgumentError(
+          `kind must be one of ${NOTE_KINDS.join(", ")}`,
+        );
+      const kind = kindValue as NoteKind | undefined;
+      const from = optionalSeconds(args, "from");
+      const to = optionalSeconds(args, "to");
+      if (from !== undefined && to !== undefined && to <= from)
+        throw new MediaArgumentError("to must be greater than from");
+      return media(`transcribe ${kind ?? "notes"} from ${file}`, (context) =>
+        transcribeNotes(
+          {
+            file,
+            ...(kind ? { kind } : {}),
+            ...(from !== undefined ? { from } : {}),
+            ...(to !== undefined ? { to } : {}),
+          },
+          context,
+        ),
+      );
+    },
+  },
+  {
+    name: "import_sample",
+    description:
+      "Copy an audio file into the focused track's samples/ as 48 kHz wav and return the sampler({...}) snippet for track.ts. begin/end are fractions 0..1 of the file; root is the note the sample is pitched at (default C4).",
+    parameters: {
+      type: "object",
+      properties: {
+        file: fileSchema,
+        name: {
+          type: "string",
+          maxLength: MAX_NAME_LENGTH,
+          description: "Voice name: lowercase identifier such as kick or vox.",
+        },
+        begin: { type: "number", minimum: 0, maximum: 1 },
+        end: { type: "number", minimum: 0, maximum: 1 },
+        root: {
+          type: "string",
+          maxLength: 4,
+          description: "Note name like C4 or A#2.",
+        },
+      },
+      required: ["file", "name"],
+      additionalProperties: false,
+    },
+    plan: (args) => {
+      const file = text(args, "file", 1024);
+      const name = sampleName(args);
+      const begin = optionalFraction(args, "begin");
+      const end = optionalFraction(args, "end");
+      if (begin !== undefined && end !== undefined && end <= begin)
+        throw new MediaArgumentError("end must be greater than begin");
+      const root = optionalText(args, "root", 4);
+      if (root !== undefined && !/^[A-Ga-g][#b]?-?\d$/.test(root))
+        throw new MediaArgumentError("root must be a note name like C4");
+      return media(`import ${file} as sample ${name}`, (context) =>
+        importSample(
+          {
+            file,
+            name,
+            ...(begin !== undefined ? { begin } : {}),
+            ...(end !== undefined ? { end } : {}),
+            ...(root ? { root } : {}),
+          },
+          context,
+        ),
+      );
+    },
+  },
+  {
+    name: "transcribe_lyrics",
+    description:
+      "Transcribe sung or spoken words with whisper.cpp. Writes <name>.lyrics.json (timed segments) and .lyrics.txt; the first run downloads the ~148 MB ggml-base model into ~/.cache/dawg/whisper.",
+    parameters: {
+      type: "object",
+      properties: {
+        file: fileSchema,
+        lang: {
+          type: "string",
+          maxLength: 4,
+          description: "Language code (en default, or auto).",
+        },
+      },
+      required: ["file"],
+      additionalProperties: false,
+    },
+    plan: (args) => {
+      const file = text(args, "file", 1024);
+      const lang = optionalText(args, "lang", 4)?.toLowerCase();
+      return media(`transcribe lyrics of ${file}`, (context) =>
+        transcribeLyrics({ file, ...(lang ? { lang } : {}) }, context),
+      );
+    },
+  },
+]);
+
+/** Agent tool names that touch `downloads/`; the host refreshes its listing after them. */
+export const MEDIA_TOOL_NAME_SET: ReadonlySet<string> = new Set(
+  MEDIA_TOOLS.map((tool) => tool.name),
+);

package/src/media/types.ts

@@ -0,0 +1,92 @@
+/**
+ * Shared types for the local media tools (`download_audio`, `split_stems`,
+ * `analyze_audio`, `transcribe_notes`, `import_sample`, `transcribe_lyrics`).
+ *
+ * Nothing here imports the agent, so `src/agent/tools.ts` can reference these
+ * types without a cycle. Every tool runs through a `MediaRunContext` that the
+ * host supplies (project root, focused track slug, subprocess runner, fetch),
+ * which is what lets tests script yt-dlp, ffprobe, demucs and StemDeck.
+ */
+import type { CommandRunner } from "../auth/runner.ts";
+
+/** One transcribed note or drum hit, in source seconds. */
+export type TimedNote = Readonly<{
+  pitch: number;
+  startSeconds: number;
+  endSeconds: number;
+  /** 0..1, the scale `note()` and `hit()` use. */
+  velocity: number;
+  /** Model confidence 0..1 when the transcriber reports one. */
+  confidence?: number;
+  /** Heuristic ranking score 0..1 for classifier output. */
+  heuristicScore?: number;
+}>;
+
+/** A beat grid in source seconds; `bars` mark downbeats by beat index. */
+export type BeatGrid = Readonly<{
+  beats: readonly number[];
+  bars: readonly Readonly<{ beat: number; beatsPerBar: number }>[];
+  durationSeconds: number;
+  bpm?: number;
+  detector?: string;
+  confidence?: number;
+  intervalCv?: number;
+}>;
+
+/**
+ * Services the agent host lends the media tools. The project root and track
+ * slug come from the workspace tools (`AgentHost.workspace` and the focused
+ * track), so media outputs share their write scope.
+ */
+export type MediaServices = Readonly<{
+  runner: CommandRunner;
+  /** Injectable for tests; defaults to global fetch. */
+  fetch?: typeof fetch;
+  env?: Readonly<Record<string, string | undefined>>;
+  /** Overrides `$HOME` for model caches (`~/.cache/dawg`). */
+  homeDir?: string;
+}>;
+
+/** What every media tool runs with. Paths are absolute. */
+export type MediaHost = MediaServices &
+  Readonly<{
+    /** Workspace root; media outputs live under `tracks/<slug>/downloads/`. */
+    projectRoot: string;
+    /** `trackSlug()` of the focused track's name, as the workspace tools use. */
+    trackSlug: string;
+  }>;
+
+export type MediaRunContext = MediaHost &
+  Readonly<{
+    signal: AbortSignal;
+    /** Short progress line for the activity feed, e.g. `demucs 42%`. */
+    progress: (line: string) => void;
+  }>;
+
+/** What a finished media tool reports; `content` goes back to the model. */
+export type MediaResult = Readonly<{
+  /** One line for the activity card. */
+  summary: string;
+  /** JSON-serialisable result, bounded by the tool. */
+  content: Record<string, unknown>;
+  /** Project-relative output paths, so the model can chain tools. */
+  outputs: readonly string[];
+}>;
+
+export const MEDIA_LIMITS = Object.freeze({
+  downloadTimeoutMs: 15 * 60_000,
+  downloadMaxBytes: 500 * 1024 * 1024,
+  stemsTimeoutMs: 20 * 60_000,
+  analyzeTimeoutMs: 5 * 60_000,
+  notesTimeoutMs: 10 * 60_000,
+  sampleTimeoutMs: 2 * 60_000,
+  lyricsTimeoutMs: 10 * 60_000,
+  /** Largest file any tool reads whole into memory. */
+  maxWavBytes: 256 * 1024 * 1024,
+  maxNotes: 2048,
+  waveformBuckets: 240,
+  /** Bound on captured stdout/stderr of every helper process. */
+  maxToolOutputBytes: 1024 * 1024,
+  /** Model files announced before download (whisper). */
+  whisperModelBytes: 148_000_000,
+});