@hraness/dawg 0.3.0 → 0.4.0

package/src/audio/samples.ts

@@ -7,9 +7,10 @@
  *   int, `sowt`, `fl32`) decode natively in TypeScript.
  * - Anything else (mp3, flac, ogg, m4a, …) decodes through `ffmpeg` when it is
  *   on PATH; dawg never installs it.
- * - Decoded PCM is cached content-addressed at `.dawg/assets/<sha256>.pcm`
- *   (and in memory), least recently used first out, bounded by
- *   `SCORE_LIMITS.maxSampleCacheBytes`.
+ * - Decoded PCM is cached content-addressed at `.dawg/assets/<sha256>.pcm`,
+ *   least recently used first out, bounded by `DAWG_ASSETS_CACHE_MAX`
+ *   (default 1 GiB, see `cache.ts`); files the loaded score uses are never
+ *   evicted. In memory, decoded voices share a 512 MiB LRU.
  *
  * Loading never throws for project problems: a voice that cannot load is
  * reported as a `<what> · <why> · <next step>` problem and renders silent.
@@ -17,20 +18,32 @@
 import { createHash } from "node:crypto";
 import {
   mkdir,
-  readdir,
   readFile,
   rename,
   stat,
-  unlink,
   utimes,
   writeFile,
 } from "node:fs/promises";
 import { join } from "node:path";
 import {
+  PACK_PREFIX,
   SCORE_LIMITS,
   isSamplerInstrument,
+  isLocalTableSrc,
+  isWavetableInstrument,
+  type SampleRef,
   type TrackScore,
 } from "../../core/score.ts";
+import { PackError, PackStore } from "./packs.ts";
+import {
+  DEFAULT_MEMORY_CACHE_BYTES,
+  assetsCacheMax,
+  cacheUsage,
+  pruneLru,
+  type CacheUsage,
+  type PruneResult,
+} from "./cache.ts";
+import { Wavetable, wavetableFromWav } from "./wavetable.ts";
 import { trackDirectories } from "../../core/sdk/print.ts";
 import { WavFormatError, parseWav } from "../media/vendor/wav.ts";
 import { WorkspaceError, resolveReadPath } from "../agent/workspace.ts";
@@ -58,6 +71,10 @@ export type SampleProblem = Readonly<{
 export type SampleBank = Readonly<{
   voices: ReadonlyMap<string, DecodedSample>;
   problems: readonly SampleProblem[];
+  /** Pack wavetables keyed by track id (built-in tables need no loading). */
+  wavetables?: ReadonlyMap<string, Wavetable>;
+  /** Decoded `reverb.ir` samples keyed by track id (built-ins need none). */
+  irs?: ReadonlyMap<string, DecodedSample>;
 }>;
 
 export const EMPTY_SAMPLE_BANK: SampleBank = Object.freeze({
@@ -65,16 +82,51 @@ export const EMPTY_SAMPLE_BANK: SampleBank = Object.freeze({
   problems: Object.freeze([]),
 });
 
-/** True when any track of the score is a sampler with voices. */
+/**
+ * True when the score needs the sample library: a sampler with voices, or a
+ * wavetable track playing a pack table.
+ */
 export function hasSamplerTracks(score: TrackScore): boolean {
   return score.tracks.some(
     (track) =>
-      isSamplerInstrument(track.instrument) &&
-      track.sampler !== undefined &&
-      Object.keys(track.sampler.voices).length > 0,
+      (isSamplerInstrument(track.instrument) &&
+        track.sampler !== undefined &&
+        Object.keys(track.sampler.voices).length > 0) ||
+      packWavetable(track) !== undefined ||
+      localWavetable(track) !== undefined ||
+      reverbIrSample(track) !== undefined,
   );
 }
 
+/** The project WAV a wavetable track plays, if any. */
+export function localWavetable(
+  track: TrackScore["tracks"][number],
+): SampleRef | undefined {
+  if (!isWavetableInstrument(track.instrument)) return undefined;
+  const table = track.wavetable?.table;
+  return table && isLocalTableSrc(table.src) ? table : undefined;
+}
+
+/** The pack sound or project file a track's reverb convolves with, if any. */
+export function reverbIrSample(
+  track: TrackScore["tracks"][number],
+): SampleRef | undefined {
+  const ir = track.reverb?.ir;
+  return ir && !ir.src.startsWith("builtin:") ? ir : undefined;
+}
+
+/** The pinned pack table a wavetable track plays, if any. */
+export function packWavetable(
+  track: TrackScore["tracks"][number],
+): SampleRef | undefined {
+  if (!isWavetableInstrument(track.instrument)) return undefined;
+  const table = track.wavetable?.table;
+  return table?.src.startsWith(PACK_PREFIX) ? table : undefined;
+}
+
+/** Parsed pack wavetables kept in memory (they are small). */
+const MAX_MEMORY_WAVETABLES = 32;
+
 export function sampleKey(trackId: string, voice: string): string {
   return `${trackId}\u0000${voice}`;
 }
@@ -409,37 +461,25 @@ export function decodeCachedPcm(bytes: Uint8Array): PcmData | undefined {
   return { sampleRate, channels, frames: data.length / channels, data };
 }
 
-/** Delete least recently used `.pcm` files until the directory fits `maxBytes`. */
+const ASSET_FILE = /^[0-9a-f]{64}\.pcm$/;
+
+/**
+ * Delete least recently used `.pcm` files until the directory fits
+ * `maxBytes`, never the ones `keep` names (a sha256 or a set of them).
+ * Returns how many files went.
+ */
 export async function pruneAssetCache(
   dir: string,
   maxBytes: number,
-  keep?: string,
+  keep?: string | ReadonlySet<string>,
 ): Promise<number> {
-  let names: string[];
-  try {
-    names = await readdir(dir);
-  } catch {
-    return 0;
-  }
-  const entries: { path: string; bytes: number; used: number }[] = [];
-  for (const name of names) {
-    if (!/^[0-9a-f]{64}\.pcm$/.test(name)) continue;
-    const path = join(dir, name);
-    const info = await stat(path).catch(() => undefined);
-    if (info?.isFile())
-      entries.push({ path, bytes: info.size, used: info.mtimeMs });
-  }
-  let total = entries.reduce((sum, entry) => sum + entry.bytes, 0);
-  entries.sort((a, b) => a.used - b.used || a.path.localeCompare(b.path));
-  let removed = 0;
-  for (const entry of entries) {
-    if (total <= maxBytes) break;
-    if (keep && entry.path.endsWith(`${keep}.pcm`)) continue;
-    await unlink(entry.path).catch(() => undefined);
-    total -= entry.bytes;
-    removed += 1;
-  }
-  return removed;
+  const kept =
+    keep === undefined
+      ? () => false
+      : typeof keep === "string"
+        ? (name: string) => name === `${keep}.pcm`
+        : (name: string) => keep.has(name.slice(0, 64));
+  return (await pruneLru(dir, ASSET_FILE, maxBytes, kept)).removed;
 }
 
 // ---------------------------------------------------------------------------
@@ -450,11 +490,16 @@ export type SampleLibraryOptions = Readonly<{
   projectRoot: string;
   /** Default `<projectRoot>/.dawg/assets`. */
   cacheDir?: string;
-  /** Disk and memory cache budget; default `SCORE_LIMITS.maxSampleCacheBytes`. */
+  /**
+   * Disk cache budget; default `assetsCacheMax()` (`DAWG_ASSETS_CACHE_MAX`,
+   * 1 GiB). Memory uses the smaller of this and 512 MiB.
+   */
   maxCacheBytes?: number;
   /** ffmpeg binary, `null` for none; default `Bun.which("ffmpeg")`. */
   ffmpeg?: string | null;
   runFfmpeg?: FfmpegRunner;
+  /** Resolves `pack:` voices; default a `PackStore` on the user cache. */
+  packs?: PackStore;
 }>;
 
 type MemoryEntry = { sample: DecodedSample; bytes: number };
@@ -466,28 +511,41 @@ type MemoryEntry = { sample: DecodedSample; bytes: number };
 export class SampleLibrary implements SampleSource {
   public readonly projectRoot: string;
   public readonly cacheDir: string;
-  private readonly maxCacheBytes: number;
+  public readonly maxCacheBytes: number;
+  private readonly maxMemoryBytes: number;
+  /** Content hashes the last loaded score uses; pruning skips them. */
+  private inUse = new Set<string>();
   private readonly ffmpeg: string | null;
   private readonly runFfmpeg: FfmpegRunner | undefined;
   private readonly memory = new Map<string, MemoryEntry>();
   private memoryBytes = 0;
   /** `real:size:mtime` → sha256, so unchanged files are not re-hashed. */
   private readonly hashes = new Map<string, string>();
+  private readonly tables = new Map<string, Wavetable>();
   private stats = { decodes: 0, diskHits: 0, memoryHits: 0 };
 
   public constructor(options: SampleLibraryOptions) {
     this.projectRoot = options.projectRoot;
     this.cacheDir =
       options.cacheDir ?? join(options.projectRoot, ".dawg", "assets");
-    this.maxCacheBytes = Math.max(
-      0,
-      options.maxCacheBytes ?? SCORE_LIMITS.maxSampleCacheBytes,
+    this.maxCacheBytes = Math.max(0, options.maxCacheBytes ?? assetsCacheMax());
+    this.maxMemoryBytes = Math.min(
+      this.maxCacheBytes,
+      DEFAULT_MEMORY_CACHE_BYTES,
     );
     this.ffmpeg =
       options.ffmpeg === undefined
         ? (Bun.which("ffmpeg") ?? null)
         : options.ffmpeg;
     this.runFfmpeg = options.runFfmpeg;
+    this.packStore = options.packs;
+  }
+
+  private packStore: PackStore | undefined;
+
+  private get packs(): PackStore {
+    this.packStore ??= new PackStore();
+    return this.packStore;
   }
 
   /** Decode/cache counters, for tests and diagnostics. */
@@ -499,16 +557,137 @@ export class SampleLibrary implements SampleSource {
     return { ...this.stats };
   }
 
+  /** Decoded asset files on disk and the cap. */
+  public async cacheStatus(): Promise<CacheUsage & { max: number }> {
+    return {
+      ...(await cacheUsage(this.cacheDir, ASSET_FILE)),
+      max: this.maxCacheBytes,
+    };
+  }
+
+  /**
+   * Prunes decoded assets down to `maxBytes` (default the cap), keeping the
+   * ones the last loaded score uses.
+   */
+  public pruneCache(maxBytes = this.maxCacheBytes): Promise<PruneResult> {
+    const inUse = this.inUse;
+    return pruneLru(this.cacheDir, ASSET_FILE, maxBytes, (name) =>
+      inUse.has(name.slice(0, 64)),
+    );
+  }
+
   public async load(score: TrackScore): Promise<SampleBank> {
     const voices = new Map<string, DecodedSample>();
     const problems: SampleProblem[] = [];
     let dirs: ReadonlyMap<string, string> | undefined;
+    // The score's pins are this project's working set: neither cache evicts
+    // them while it is open.
+    this.inUse = new Set();
+    const urls: string[] = [];
+    for (const track of score.tracks) {
+      if (!isSamplerInstrument(track.instrument) || !track.sampler) continue;
+      for (const ref of Object.values(track.sampler.voices)) {
+        if (ref.sha256) this.inUse.add(ref.sha256);
+        if (ref.url) urls.push(ref.url);
+      }
+    }
+    for (const track of score.tracks) {
+      const ir = reverbIrSample(track);
+      if (ir?.sha256) this.inUse.add(ir.sha256);
+      if (ir?.url) urls.push(ir.url);
+    }
+    for (const track of score.tracks) {
+      const table = packWavetable(track);
+      if (table?.sha256) this.inUse.add(table.sha256);
+      if (table?.url) urls.push(table.url);
+    }
+    if (urls.length) this.packs.protect(urls);
+    const wavetables = new Map<string, Wavetable>();
+    for (const track of score.tracks) {
+      const ref = packWavetable(track);
+      if (!ref) continue;
+      try {
+        const loaded = await this.loadWavetable(ref);
+        wavetables.set(track.id, loaded.table);
+        if (loaded.warning)
+          problems.push({
+            trackId: track.id,
+            voice: "wavetable",
+            src: ref.src,
+            level: "warning",
+            message: `wavetable · ${ref.src} · ${loaded.warning}`,
+          });
+      } catch (error) {
+        problems.push({
+          trackId: track.id,
+          voice: "wavetable",
+          src: ref.src,
+          level: "error",
+          message: `wavetable · ${ref.src} · ${error instanceof Error ? error.message : String(error)} · /wt pick another table`,
+        });
+      }
+    }
+    for (const track of score.tracks) {
+      const ref = localWavetable(track);
+      if (!ref) continue;
+      try {
+        const loaded = await this.loadLocalWavetable(ref.src);
+        wavetables.set(track.id, loaded);
+        if (ref.sha256 && ref.sha256 !== loaded.id)
+          problems.push({
+            trackId: track.id,
+            voice: "wavetable",
+            src: ref.src,
+            level: "warning",
+            message: `wavetable · ${ref.src} changed since it was picked (sha256 ${loaded.id.slice(0, 12)}… ≠ ${ref.sha256.slice(0, 12)}…) · playing the file on disk; re-pick it with /wt to pin it`,
+          });
+      } catch (error) {
+        problems.push({
+          trackId: track.id,
+          voice: "wavetable",
+          src: ref.src,
+          level: "error",
+          message:
+            error instanceof WorkspaceError &&
+            /no such file/.test(error.message)
+              ? `wavetable · ${ref.src} · file is missing · remake it with make_wavetable or /wt another table`
+              : `wavetable · ${ref.src} · ${error instanceof Error ? error.message : String(error)} · /wt pick another table`,
+        });
+      }
+    }
     for (const track of score.tracks) {
       if (!isSamplerInstrument(track.instrument) || !track.sampler) continue;
       const names = Object.keys(track.sampler.voices).sort();
       for (const voice of names.slice(0, SCORE_LIMITS.maxSamplerVoices)) {
         const ref = track.sampler.voices[voice]!;
         let src = ref.src;
+        if (src.startsWith(PACK_PREFIX)) {
+          try {
+            const loaded = await this.loadPack(ref);
+            this.inUse.add(loaded.sample.sha256);
+            voices.set(sampleKey(track.id, voice), loaded.sample);
+            if (loaded.warning)
+              problems.push({
+                trackId: track.id,
+                voice,
+                src,
+                level: "warning",
+                message: `${voice} · ${src} · ${loaded.warning}`,
+              });
+          } catch (error) {
+            problems.push({
+              trackId: track.id,
+              voice,
+              src,
+              level: "error",
+              message:
+                error instanceof PackError
+                  ? `${voice} · ${src} · ${error.message}`
+                  : describeFailure(voice, src, error, this.ffmpeg),
+            });
+          }
+          continue;
+        }
         if (!src.startsWith("tracks/")) {
           dirs ??= trackDirectories(score);
           src = `tracks/${dirs.get(track.id) ?? track.id}/${src.replace(/^\.\//, "")}`;
@@ -523,6 +702,7 @@ export class SampleLibrary implements SampleSource {
           });
         try {
           const loaded = await this.loadFile(src);
+          this.inUse.add(loaded.sha256);
           if (ref.sha256 && ref.sha256 !== loaded.sha256)
             report(
               "warning",
@@ -534,7 +714,168 @@ export class SampleLibrary implements SampleSource {
         }
       }
     }
-    return Object.freeze({ voices, problems: Object.freeze(problems) });
+    const irs = new Map<string, DecodedSample>();
+    for (const track of score.tracks) {
+      const ref = reverbIrSample(track);
+      if (!ref) continue;
+      const report = (level: SampleProblem["level"], message: string) =>
+        problems.push({
+          trackId: track.id,
+          voice: "ir",
+          src: ref.src,
+          level,
+          message,
+        });
+      try {
+        if (ref.src.startsWith(PACK_PREFIX)) {
+          const loaded = await this.loadPack(ref);
+          this.inUse.add(loaded.sample.sha256);
+          irs.set(track.id, loaded.sample);
+          if (loaded.warning)
+            report("warning", `ir · ${ref.src} · ${loaded.warning}`);
+        } else {
+          dirs ??= trackDirectories(score);
+          const src = ref.src.startsWith("tracks/")
+            ? ref.src
+            : `tracks/${dirs.get(track.id) ?? track.id}/${ref.src.replace(/^\.\//, "")}`;
+          const loaded = await this.loadFile(src);
+          this.inUse.add(loaded.sha256);
+          irs.set(track.id, loaded);
+        }
+      } catch (error) {
+        report(
+          "error",
+          error instanceof PackError
+            ? `ir · ${ref.src} · ${error.message} · fx reverb ir hall uses a built-in`
+            : describeFailure("ir", ref.src, error, this.ffmpeg),
+        );
+      }
+    }
+    return Object.freeze({
+      voices,
+      problems: Object.freeze(problems),
+      ...(wavetables.size ? { wavetables } : {}),
+      ...(irs.size ? { irs } : {}),
+    });
+  }
+
+  /**
+   * A pack wavetable: the raw WAV from the pack file cache (or the network,
+   * checked against the pin), sliced into frames. Tables stay WAV on disk
+   * because the frame layout lives in the file's `clm ` chunk.
+   */
+  private async loadWavetable(
+    ref: SampleRef,
+  ): Promise<{ table: Wavetable; warning?: string }> {
+    const memo = ref.sha256 ? this.tables.get(ref.sha256) : undefined;
+    if (memo) return { table: memo };
+    let url = ref.url;
+    let warning: string | undefined;
+    if (!url) {
+      url = (await this.packs.resolve(ref.src)).url;
+      if (!ref.sha256)
+        warning =
+          "not pinned · re-pick it with /wt so renders stay reproducible";
+    }
+    const file = await this.packs.fetchFile(url, ref.sha256);
+    let table = this.tables.get(file.sha256);
+    if (!table) {
+      table = wavetableFromWav(ref.src, file.sha256, file.bytes);
+      this.tables.set(file.sha256, table);
+      if (this.tables.size > MAX_MEMORY_WAVETABLES)
+        this.tables.delete(this.tables.keys().next().value!);
+    }
+    return { table, ...(warning ? { warning } : {}) };
+  }
+
+  /**
+   * A project wavetable WAV, read within the project root (symlinks that
+   * leave it are refused) and memoized by content hash.
+   */
+  private async loadLocalWavetable(src: string): Promise<Wavetable> {
+    const resolved = await resolveReadPath(
+      { root: this.projectRoot, trackSlug: "" },
+      src,
+      "wavetable",
+    );
+    const info = await stat(resolved.real);
+    if (!info.isFile()) throw new SampleDecodeError("not a regular file");
+    if (info.size > SCORE_LIMITS.maxSampleFileBytes)
+      throw new SampleLimitError(
+        `${formatMiB(info.size)} is over the ${formatMiB(SCORE_LIMITS.maxSampleFileBytes)} file limit`,
+      );
+    const bytes = new Uint8Array(await readFile(resolved.real));
+    const sha256 = createHash("sha256").update(bytes).digest("hex");
+    let table = this.tables.get(sha256);
+    if (!table) {
+      table = wavetableFromWav(src, sha256, bytes);
+      this.tables.set(sha256, table);
+      if (this.tables.size > MAX_MEMORY_WAVETABLES)
+        this.tables.delete(this.tables.keys().next().value!);
+    }
+    return table;
+  }
+
+  /**
+   * A `pack:` voice. With a pinned sha256 the decoded asset cache answers
+   * without the network; otherwise the file comes from the pack store
+   * (its file cache, then a download) and is checked against the pin.
+   */
+  private async loadPack(
+    ref: SampleRef,
+  ): Promise<{ sample: DecodedSample; warning?: string }> {
+    if (ref.sha256) {
+      const hit = this.fromMemory(ref.sha256);
+      if (hit) return { sample: hit };
+      const pcm = await this.fromDisk(ref.sha256);
+      if (pcm) {
+        this.stats.diskHits += 1;
+        return { sample: this.admit(ref.sha256, pcm) };
+      }
+    }
+    let url = ref.url;
+    let warning: string | undefined;
+    if (!url) {
+      url = (await this.packs.resolve(ref.src)).url;
+      if (!ref.sha256)
+        warning =
+          "not pinned · re-pick it with /kit, /pack or use_sound so renders stay reproducible";
+    }
+    const file = await this.packs.fetchFile(url, ref.sha256);
+    if (file.bytes.byteLength > SCORE_LIMITS.maxSampleFileBytes)
+      throw new SampleLimitError(
+        `${formatMiB(file.bytes.byteLength)} is over the ${formatMiB(SCORE_LIMITS.maxSampleFileBytes)} sample file limit`,
+      );
+    const hit = this.fromMemory(file.sha256);
+    if (hit) return { sample: hit, ...(warning ? { warning } : {}) };
+    let pcm = await this.fromDisk(file.sha256);
+    if (pcm) this.stats.diskHits += 1;
+    else {
+      pcm = await this.decode(file.bytes, file.path);
+      this.stats.decodes += 1;
+      await this.toDisk(file.sha256, pcm);
+    }
+    return {
+      sample: this.admit(file.sha256, pcm),
+      ...(warning ? { warning } : {}),
+    };
+  }
+
+  private admit(sha256: string, pcm: PcmData): DecodedSample {
+    if (pcm.frames / pcm.sampleRate > SCORE_LIMITS.maxSampleSeconds)
+      throw new SampleLimitError(
+        `${Math.round(pcm.frames / pcm.sampleRate)} s is over the ${SCORE_LIMITS.maxSampleSeconds / 60} minute sample limit`,
+      );
+    if (pcm.frames === 0) throw new SampleDecodeError("it contains no audio");
+    const sample: DecodedSample = Object.freeze({
+      sha256,
+      sampleRate: pcm.sampleRate,
+      channels: pcm.channels,
+      frames: pcm.frames,
+      mono: monoMix(pcm),
+    });
+    this.remember(sample);
+    return sample;
   }
 
   private async loadFile(src: string): Promise<DecodedSample> {
@@ -606,7 +947,7 @@ export class SampleLibrary implements SampleSource {
     this.memory.set(sample.sha256, { sample, bytes });
     this.memoryBytes += bytes;
     for (const [sha, entry] of this.memory) {
-      if (this.memoryBytes <= this.maxCacheBytes) break;
+      if (this.memoryBytes <= this.maxMemoryBytes) break;
       if (sha === sample.sha256) continue;
       this.memory.delete(sha);
       this.memoryBytes -= entry.bytes;
@@ -635,7 +976,8 @@ export class SampleLibrary implements SampleSource {
       const temporary = `${path}.${process.pid}.tmp`;
       await writeFile(temporary, encodeCachedPcm(pcm));
       await rename(temporary, path);
-      await pruneAssetCache(this.cacheDir, this.maxCacheBytes, sha256);
+      const inUse = new Set(this.inUse).add(sha256);
+      await pruneAssetCache(this.cacheDir, this.maxCacheBytes, inUse);
     } catch {
       /* the cache is an optimisation; a read-only project still renders */
     }