mlx-bun 0.4.0 → 1.0.0

package/src/preflight.ts

@@ -1,158 +0,0 @@
-// Machine-state preflight for fair benchmarks (Phase 15 method rules).
-// Every failure mode here was MEASURED on this machine (PLAN findings):
-//   - 6.4 GB swap residue: 26B decode 32 tok/s → 0.02 tok/s
-//   - session drift: identical bf16 runs 15.5 vs 14.3 tok/s an hour apart
-//   - cold page cache: prefill timer dominated by SSD page-in
-// The harness refuses to record headline numbers when checks fail;
-// the state snapshot is stored in every eval-DB row either way.
-//
-// LIMITS of this instrument (2026-07-05 finding): the snapshot is taken
-// BETWEEN legs, when the bench process has exited and memory is quiet —
-// free_pct reads a near-constant ~92% on an otherwise-idle box and load1m
-// sat under every threshold while a 40-minute mid-pass slow window cut
-// decode rates ~40% across every stack. Preflight ok ≠ clean measurements
-// and preflight-failed ≠ dirty ones (the 07-04 pass was ‡-tagged yet
-// stable). The DIRECT quality signal is per-cell run spread — see the
-// run-stability policy in scripts/bench-h2h.ts (SPREAD_TOL retries +
-// `unstable`/`stabilized` tags). Preflight remains the coarse pre-gate.
-
-export interface MachineState {
-  swapUsedMB: number;
-  freePercent: number;
-  /** pmset -g therm CPU_Speed_Limit (100 = no thermal throttle; -1 if
-   *  unavailable on this platform). */
-  cpuSpeedLimit: number;
-  /** iogpu.wired_limit_mb sysctl (0 = macOS default ≈75% of unified
-   *  memory). A raised ceiling helps every stack equally but MUST be
-   *  recorded — results depend on it. Resets on reboot. */
-  gpuWiredLimitMB: number;
-  /** 1-minute load average — post-boot indexing (Spotlight etc.) shows
-   *  up here; let it settle before benchmarking. */
-  loadAvg1m: number;
-  /** Foreign processes holding > bigRssMB resident (excludes kernel,
-   *  WindowServer, and our own bun/python benchmark processes). */
-  bigProcesses: { rssMB: number; command: string }[];
-  /** Explicitly permitted CPU-only background commands; RSS limits still apply. */
-  allowedCpuProcesses?: string[];
-  backgroundCpuProcesses?: { rssMB: number; command: string }[];
-  ok: boolean;
-  problems: string[];
-  at: string;
-}
-
-export interface PreflightLimits {
-  maxSwapUsedMB?: number;   // default 512
-  minFreePercent?: number;  // default 35
-  bigRssMB?: number;        // default 2048
-  allowCpuProcesses?: string[];
-}
-
-export function foreignProcessFindings(output: string, limits: PreflightLimits = {}) {
-  const bigProcesses: { rssMB: number; command: string }[] = [];
-  const backgroundCpuProcesses: { rssMB: number; command: string }[] = [];
-  for (const line of output.split("\n")) {
-    const m = line.match(/^\s*([\d.]+)\s+(\d+)\s+(.*)$/);
-    if (!m) continue;
-    const cpuPct = Number(m[1]);
-    const rssMB = Number(m[2]) / 1024;
-    const command = m[3]!;
-    if (rssMB < (limits.bigRssMB ?? 2048) && cpuPct < 50) continue;
-    if (/(bun|\.venv\/bin\/python|mlx_lm|optiq)/.test(command) ||
-        /(kernel_task|WindowServer|launchd|mds|spotlight)/i.test(command)) continue;
-    const finding = { rssMB: Math.round(rssMB),
-      command: command.slice(0, 80) + (cpuPct >= 50 ? ` (${cpuPct.toFixed(0)}% cpu)` : "") };
-    // Exact command match; permission to tolerate CPU activity never permits
-    // that process to consume the benchmark's memory headroom.
-    const allowed = rssMB < (limits.bigRssMB ?? 2048) && limits.allowCpuProcesses?.includes(command);
-    (allowed ? backgroundCpuProcesses : bigProcesses).push(finding);
-  }
-  return { bigProcesses, backgroundCpuProcesses };
-}
-
-function sh(cmd: string[]): string {
-  const r = Bun.spawnSync(cmd, { stdout: "pipe", stderr: "pipe" });
-  return r.exitCode === 0 ? r.stdout.toString() : "";
-}
-
-export function checkMachine(limits: PreflightLimits = {}): MachineState {
-  const maxSwap = limits.maxSwapUsedMB ?? 512;
-  const minFree = limits.minFreePercent ?? 35;
-  const problems: string[] = [];
-
-  // swap: "vm.swapusage: total = 7168.00M  used = 6413.06M  free = ..."
-  const swapOut = sh(["sysctl", "vm.swapusage"]);
-  const swapUsedMB = Number(swapOut.match(/used = ([\d.]+)M/)?.[1] ?? -1);
-  if (swapUsedMB < 0) problems.push("could not read vm.swapusage");
-  else if (swapUsedMB > maxSwap)
-    problems.push(`swap in use: ${swapUsedMB.toFixed(0)} MB (> ${maxSwap} MB) — reboot before benchmarking`);
-
-  // free memory: "System-wide memory free percentage: 34%"
-  const memOut = sh(["memory_pressure", "-Q"]);
-  const freePercent = Number(memOut.match(/free percentage: (\d+)/)?.[1] ?? -1);
-  if (freePercent < 0) problems.push("could not read memory_pressure");
-  else if (freePercent < minFree)
-    problems.push(`free memory ${freePercent}% (< ${minFree}%) — close apps / reboot`);
-
-  // thermal: "CPU_Speed_Limit     = 100"
-  const thermOut = sh(["pmset", "-g", "therm"]);
-  const speedMatch = thermOut.match(/CPU_Speed_Limit\s*=\s*(\d+)/);
-  const cpuSpeedLimit = speedMatch ? Number(speedMatch[1]) : -1;
-  if (cpuSpeedLimit !== -1 && cpuSpeedLimit < 100)
-    problems.push(`thermal throttle active: CPU speed limit ${cpuSpeedLimit}% — let the machine cool`);
-
-  // GPU wired ceiling: 0 = stock (~75% of unified). Recorded, not judged
-  // — raising it (sudo sysctl iogpu.wired_limit_mb=N) helps all stacks
-  // equally, but rows measured under different ceilings don't compare.
-  const wiredOut = sh(["sysctl", "-n", "iogpu.wired_limit_mb"]);
-  const gpuWiredLimitMB = Number(wiredOut.trim() || 0) || 0;
-
-  // load average: "{ 4.96 3.21 1.90 }" — post-boot indexing settles fast
-  const loadOut = sh(["sysctl", "-n", "vm.loadavg"]);
-  const loadAvg1m = Number(loadOut.match(/([\d.]+)/)?.[1] ?? -1);
-  if (loadAvg1m > 8)
-    problems.push(`1-min load ${loadAvg1m.toFixed(1)} — post-boot indexing still running; wait a minute`);
-
-  // big foreign processes: high RSS *or* high CPU (pcpu + rss in KB from
-  // ps). The CPU arm is the 2026-07-05 lesson: Apple's
-  // knowledgeconstructiond (Apple Intelligence indexing) burned 87% CPU
-  // with a tiny RSS during the pass's 40-min slow window — a background
-  // ML workload the RSS-only check could never flag. CPU is bursty, so
-  // one clean sample doesn't prove absence (the run-spread stability
-  // policy in bench-h2h.ts is the direct signal); a hot sample here is
-  // still a cheap early warning.
-  const { bigProcesses, backgroundCpuProcesses } = foreignProcessFindings(
-    sh(["ps", "axo", "pcpu=,rss=,command="]), limits);
-  for (const p of bigProcesses)
-    problems.push(`big process: ${p.rssMB} MB — ${p.command}`);
-
-  return {
-    swapUsedMB, freePercent, cpuSpeedLimit, gpuWiredLimitMB, loadAvg1m,
-    bigProcesses,
-    allowedCpuProcesses: limits.allowCpuProcesses ?? [], backgroundCpuProcesses,
-    ok: problems.length === 0,
-    problems,
-    at: new Date().toISOString(),
-  };
-}
-
-/** Compact JSON for the eval-DB machine_state column. Includes machine
- *  identity — rows from different Macs must never be confused (the
- *  eval DB is per-machine, but rendered tables travel in git). */
-export function machineStateJson(s: MachineState): string {
-  const host = (sh(["hostname", "-s"]).trim() || "unknown");
-  const chip = (sh(["sysctl", "-n", "machdep.cpu.brand_string"]).trim() || "unknown");
-  const ramGB = Math.round(Number(sh(["sysctl", "-n", "hw.memsize"]).trim() || 0) / 2 ** 30);
-  return JSON.stringify({
-    host, chip, ram_gb: ramGB,
-    swap_mb: Math.round(s.swapUsedMB),
-    free_pct: s.freePercent,
-    cpu_limit: s.cpuSpeedLimit,
-    gpu_wired_mb: s.gpuWiredLimitMB,
-    load1m: s.loadAvg1m,
-    big_procs: s.bigProcesses.length,
-    allowed_cpu_processes: s.allowedCpuProcesses ?? [],
-    background_cpu_processes: s.backgroundCpuProcesses ?? [],
-    ok: s.ok,
-    at: s.at,
-  });
-}

package/src/prompt-cache.ts

@@ -1,460 +0,0 @@
-import { type CheckpointAttachment, materializeCheckpoint, attachmentBytes, cloneAttachments, disposeAttachments } from "./backends/mlx/checkpoint-state";
-import { leaseCacheState, minimumReusableOffset } from "./backends/mlx/state-views";
-import { withResource, cleanupFailure, disposeResources, ownResource } from "./engine/resources";
-// Byte-capped LRU prompt cache — the RAM tier of the Layer-0 KV store.
-//
-// The mlx-lm lesson (PLAN.md): a count-capped cache of multi-GB KV
-// entries is an OOM footgun. Ours accounts bytes (sum of KV array bytes
-// per entry) and evicts least-recently-used until under the cap.
-//
-// Usage pattern (single generation queue, so take/put is race-free):
-//   const hit = cache.take(promptIds);   // longest-prefix match
-//   generate(model, promptIds, { cache: hit?.caches ?? fresh })
-//   cache.put([...promptIds, ...generated], caches, ns, hit?.retain);
-// take() lends zero-copy views and leaves every RAM donor in place, including
-// recurrent and method state. SSD persistence runs independently of RAM reuse.
-// put() supersedes same-ns prefix-ancestors when the new entry is trimmable.
-//
-// TIERING (Layer 0, unified-engine plan): when a ColdTier is attached,
-// take() itself runs the two-tier dance — RAM peek vs cold find, restore
-// (zero-copy mmap) + trim when the cold tier holds a strictly longer
-// prefix — so EVERY consumer (the serial lane, the batch scheduler, future
-// prefix sharing) gets SSD restores through the same take()/put() it
-// already calls. Eviction spills to the tier (the #spill hook); idle
-// entries DEMOTE to it (demoteIdle — free the GPU memory, keep the prefix
-// reachable); onPut lets the server schedule its debounced write-behind
-// snapshot for both lanes.
-
-import type { Cache } from "./model/gemma4";
-import type { PrefixCache, PrefixCacheHit } from "./contracts/prefix-cache";
-import { cloneKvCaches } from "./kv-store";
-
-/** Reference-counted release: wraps an entry's `retain` (e.g. an mmap
- *  unmap) so DONOR + CLONES can each hold a share — the underlying hook
- *  runs once, after the LAST holder releases. Prefix sharing hands out
- *  zero-copy VIEWS of a donor's buffers; if the donor's unmap ran while a
- *  clone still referenced the mapped pages, the clone would read freed
- *  memory. */
-function makeSharedRetain(retain: (() => void) | undefined): { acquire(): () => void } {
-  let count = 0;
-  let released = false;
-  return {
-    acquire() {
-      count++;
-      let mine = false;
-      return () => {
-        if (mine) return; // idempotent per share
-        mine = true;
-        count--;
-        if (count === 0 && !released) {
-          released = true;
-          retain?.();
-        }
-      };
-    },
-  };
-}
-
-/** The cold (SSD) tier the RAM cache tiers over. Structural — this module
- *  never imports ssd-cache (which imports us); the server binds
- *  SsdCacheStore + the model into this shape (src/server.ts). Lookup and
- *  restore failures return null; store returns false on failure. Operations
- *  that require persistence retain their donor if storing fails. */
-export interface ColdTier {
-  /** Longest stored usable prefix for prompt/ns — index-only, no I/O.
-   *  `handle` is the tier's opaque entry token, passed back to restore. */
-  find(prompt: number[], ns: string): { prefixLen: number; handle: unknown } | null;
-  /** Materialize a found entry as GPU-visible caches (zero-copy COW mmap;
-   *  pages fault in lazily). `retain` must run after the caches are
-   *  disposed (it unmaps the backing file). Null on any failure. */
-  restore(handle: unknown): { tokens: number[]; caches: Cache[]; attachments?: CheckpointAttachment[]; retain: () => void } | null;
-  /** Borrow live caches and finish persistence before returning. A false
-   *  result prevents a required demotion. */
-  store(tokens: number[], caches: Cache[], ns: string, attachments?: CheckpointAttachment[]): boolean | void;
-}
-
-/** Spill sink for eviction/demotion (write-behind, 2026-07-06).
- *
- *  `spillOwned` (preferred, NON-BLOCKING): the cache hands the sink an
- *  entry it OWNS — zero-copy CLONES of the caches (made via the same
- *  injectable cloner take() uses, BEFORE the donor is disposed; entries
- *  are immutable so the clones stay consistent forever) plus a copy of
- *  the tokens. The sink must dispose entry.caches when done, on BOTH
- *  fulfill and reject paths (the server chains storeAsync -> dispose on
- *  ssdWriteChain). A sink that throws SYNCHRONOUSLY has not taken
- *  ownership — the cache disposes the clones and the spill degrades to
- *  a no-op; eviction/demotion never unwinds.
- *
- *  `spillSync` (fallback): called with the LIVE entry about to be
- *  disposed; must finish reading before returning. This is the old
- *  contract — a full synchronous serialize+write under the generation
- *  lock — kept for tests and simple embedders. */
-export interface SpillSink {
-  spillOwned?: (entry: PromptCacheEntry) => void;
-  spillSync?: (entry: PromptCacheEntry) => void;
-}
-
-export interface PromptCacheEntry extends PrefixCacheHit<Cache[], CheckpointAttachment[]> {
-  /** Namespace key — adapter spec for LoRA requests ("" = base model).
-   *  KV computed under one adapter must never seed another's prefill. */
-  ns: string;
-  /** Release hook for entries restored from the SSD cold tier: unmaps the
-   *  backing COW file mapping. MUST run only after `caches` are disposed
-   *  (they may alias file pages); every dispose path here honors that, and
-   *  callers who take() an entry carry the thunk with it. */
-  retain?: () => void;
-}
-
-export function cacheBytes(caches: Cache[]): number {
-  let total = 0;
-  for (const c of caches) {
-    total += withResource(leaseCacheState(c), (state) => state.reduce((bytes, a) => bytes + a.nbytes, 0));
-  }
-  return total;
-}
-
-/** Length of the longest common prefix of two token sequences. */
-export function commonPrefixLength(a: number[], b: number[]): number {
-  const n = Math.min(a.length, b.length);
-  let i = 0;
-  while (i < n && a[i] === b[i]) i++;
-  return i;
-}
-
-interface EntryRecord {
-  entry: PromptCacheEntry;
-  bytes: number;
-  lastUsed: number;
-  lastUsedMs: number;
-  /** Ref-counted retain: the donor holds one share (entry.retain); every
-   *  clone handed out by take() holds another. */
-  share: { acquire(): () => void };
-}
-
-export class PromptCache implements PrefixCache<Cache[], CheckpointAttachment[]> {
-  readonly maxBytes: number;
-  #entries: EntryRecord[] = [];
-  #clock = 0;
-  hits = 0;
-  misses = 0;
-  demotions = 0;
-  /** Cold-tier spill sinks (see SpillSink): spillOwned gets clones and
-   *  runs the write off-lock; spillSync gets the live entry inline.
-   *  Failures are the sink's problem; eviction proceeds regardless. */
-  readonly #spillOwned: ((entry: PromptCacheEntry) => void) | null;
-  readonly #spillSync: ((entry: PromptCacheEntry) => void) | null;
-  /** Optional cold tier: take() tiers over it, demoteIdle() spills into it. */
-  readonly #cold: ColdTier | null;
-  /** Fired after every successful put() — the server hangs its debounced
-   *  write-behind SSD snapshot here so BOTH lanes' entries persist. */
-  onPut: ((tokens: number[], ns: string) => void) | null = null;
-
-  /** Zero-copy view cloner — injectable so model-free tests can stub it. */
-  readonly #clone: (caches: Cache[]) => Cache[];
-
-  constructor(
-    maxBytes: number,
-    // Bare function = legacy sync spill (tests, simple embedders).
-    spill: SpillSink | ((entry: PromptCacheEntry) => void) | null = null,
-    cold: ColdTier | null = null,
-    clone: (caches: Cache[]) => Cache[] = cloneKvCaches,
-    private readonly materialize: (caches: Cache[], attachments?: CheckpointAttachment[]) => void = materializeCheckpoint,
-  ) {
-    this.maxBytes = maxBytes;
-    this.#spillOwned = typeof spill === "function" ? null : spill?.spillOwned ?? null;
-    this.#spillSync = typeof spill === "function" ? spill : spill?.spillSync ?? null;
-    this.#cold = cold;
-    this.#clone = clone;
-  }
-
-  #disposeEntry(entry: PromptCacheEntry, spillFirst: boolean): void {
-    if (spillFirst) {
-      if (this.#spillOwned) {
-        // NON-BLOCKING spill: clone BEFORE disposing (clones are zero-copy
-        // views of the live arrays) and hand ownership to the sink, which
-        // flushes off the generation lock. The underlying GPU buffers stay
-        // alive until the sink disposes the clones — for demoteIdle that
-        // means the memory is freed one bounded write-chain hop later, not
-        // instantly; acceptable because the chain is serial and each write
-        // disposes its clones on settle (fulfill AND reject).
-        // Retain contract: entry.retain still runs below, possibly while
-        // the clones are in flight — safe because cold-restored entries
-        // OWN their bytes (streamed copy-restore, 2026-07-07): there is
-        // no backing mmap for retain to unmap, so nothing the clones
-        // could alias goes away.
-        let clones: Cache[] | null = null;
-        let attachments: CheckpointAttachment[] | undefined;
-        try {
-          clones = this.#clone(entry.caches);
-          attachments = cloneAttachments(entry.attachments);
-          this.#spillOwned({ tokens: [...entry.tokens], caches: clones, ns: entry.ns, attachments });
-        } catch {
-          // Failure degrades to no-spill; eviction/demotion never unwinds.
-          if (clones) for (const c of clones) try { c.dispose(); } catch {}
-          try { disposeAttachments(attachments); } catch {}
-        }
-      } else {
-        const spill =
-          this.#spillSync ??
-          (this.#cold
-            ? (e: PromptCacheEntry) => this.#cold!.store(e.tokens, e.caches, e.ns, e.attachments)
-            : null);
-        if (spill) try { spill(entry); } catch {}
-      }
-    }
-    disposeResources([...entry.caches, { dispose: () => disposeAttachments(entry.attachments) }]);
-    entry.retain?.(); // backing release requires successful cache disposal
-  }
-
-  get totalBytes(): number {
-    return this.#entries.reduce((a, e) => a + e.bytes, 0);
-  }
-
-  get size(): number {
-    return this.#entries.length;
-  }
-
-  /** Release LRU entries until the caller's live-memory check is satisfied.
-   * Unlike background eviction, pressure relief must finish the SSD write
-   * and release the buffers now, not enqueue clones that keep them alive.
-   * Call under the engine lock, after taking the request's reusable prefix. */
-  relievePressure(overBudget: () => boolean): number {
-    let count = 0;
-    while (this.#entries.length && overBudget()) {
-      let oldest = 0;
-      for (let i = 1; i < this.#entries.length; i++)
-        if (this.#entries[i]!.lastUsed < this.#entries[oldest]!.lastUsed) oldest = i;
-      const { entry } = this.#entries[oldest]!;
-      // Keep the entry resident if persistence throws. A failed SSD write
-      // must not be presented as a successful demotion.
-      if (this.#cold) {
-        if (this.#cold.store(entry.tokens, entry.caches, entry.ns, entry.attachments) === false) break;
-      }
-      else this.#spillSync?.(entry);
-      this.#entries.splice(oldest, 1);
-      this.#disposeEntry(entry, false);
-      this.demotions++;
-      count++;
-    }
-    return count;
-  }
-
-  /** Find the entry with the longest usable common prefix of `prompt`
-   *  across BOTH tiers. Every RAM entry lends zero-copy views and retains its
-   *  donor, including exact-boundary recurrent state and method attachments.
-   *  The caller owns the returned views —
-   *  dispose them or put() an extended entry, honoring `retain` either way.
-   *
-   *  Usable prefix = common prefix capped at prompt.length - 1 (at least
-   *  one token must be forwarded to produce logits). Entries longer than
-   *  the prefix need cache.trim(); ring caches lose trimability once
-   *  wrapped — those entries only match in full.
-   *
-   *  Tier order: the cold tier wins only with a STRICTLY longer usable
-   *  prefix (its restore costs a mmap + lazy fault-in; RAM is free). A
-   *  cold entry with an untrimmable divergent tail (ring post-wrap, SSM)
-   *  is dropped and the RAM candidate (or a fresh prefill) serves.
-   *  hits/misses count RAM candidacy only (cold restores are counted by
-   *  the tier itself), preserving the /stats meaning. */
-  take(prompt: number[], ns = ""): PromptCacheEntry | null {
-    let bestIdx = -1;
-    let bestLen = 0;
-    for (let i = 0; i < this.#entries.length; i++) {
-      const e = this.#entries[i]!.entry;
-      if (e.ns !== ns) continue;
-      const p = Math.min(commonPrefixLength(e.tokens, prompt), prompt.length - 1);
-      if (p <= bestLen || p < minimumReusableOffset(e.caches)) continue;
-      const trimNeeded = e.tokens.length - p;
-      if (trimNeeded > 0 && (!!e.attachments?.length || !e.caches.every((c) => c.isTrimmable()))) continue;
-      bestLen = p;
-      bestIdx = i;
-    }
-    if (bestIdx === -1) this.misses++;
-    else this.hits++;
-
-    // Cold tier: strictly longer usable prefix ⇒ restore + trim; the RAM
-    // candidate stays put UNTOUCHED (the old serial-lane flow trimmed it
-    // before comparing — a needless entry degradation, gone now).
-    if (this.#cold) {
-      const hit = this.#cold.find(prompt, ns);
-      if (hit && hit.prefixLen > bestLen) {
-        const loaded = this.#cold.restore(hit.handle);
-        if (loaded) {
-          const owner = ownResource(loaded, (entry) =>
-            this.#disposeEntry({ ...entry, ns }, false));
-          const result = withResource(owner, (entry) => {
-            const trimNeeded = entry.tokens.length - hit.prefixLen;
-            if (hit.prefixLen < minimumReusableOffset(entry.caches)) return null;
-            if (trimNeeded > 0 && (!!entry.attachments?.length || !entry.caches.every((cache) => cache.isTrimmable()))) {
-              console.warn(
-                `[prompt-cache] cold entry unusable: ${entry.tokens.length} tokens, ` +
-                `needs trim ${trimNeeded} but untrimmable (wrapped ring/SSM) — re-prefilling`,
-              );
-              return null;
-            }
-            if (trimNeeded > 0) for (const cache of entry.caches) cache.trim(trimNeeded);
-            const tokens = entry.tokens.slice(0, hit.prefixLen);
-            return { ...owner.transfer(), tokens, ns };
-          });
-          if (result) return result;
-        }
-      }
-    }
-
-    if (bestIdx === -1) return null;
-    // PREFIX SHARING (non-consuming serve): the caller gets zero-copy
-    // CLONES of the donor's caches, trimmed to the matched prefix; the
-    // donor stays in the cache untouched, ready to serve the next agent /
-    // session with the same prefix. Safe because mlx cache updates are
-    // FUNCTIONAL (updateAndFetch reassigns fresh arrays; buffer donation
-    // only fires at refcount 1, and the donor always holds a ref), so a
-    // clone extending itself can never mutate the donor's bytes. The old
-    // consume-and-trim semantics CANNIBALIZED donors: agent B borrowing
-    // agent A's 2k system prompt destroyed A's 10k entry to do it.
-    const rec = this.#entries[bestIdx]!;
-    rec.lastUsed = ++this.#clock;
-    rec.lastUsedMs = Date.now();
-    // Lend immutable views for every backend state. RAM retention and SSD
-    // durability remain cache responsibilities while the caller extends its
-    // own views. This keeps synchronous persistence out of a RAM hit.
-    const clones = this.#clone(rec.entry.caches);
-    let attachments: CheckpointAttachment[] | undefined;
-    const trimNeeded = rec.entry.tokens.length - bestLen;
-    try {
-      attachments = cloneAttachments(rec.entry.attachments);
-      if (trimNeeded > 0) for (const c of clones) c.trim(trimNeeded);
-    } catch (error) {
-      cleanupFailure(error, () => disposeResources([...clones, { dispose: () => disposeAttachments(attachments) }]));
-    }
-    return {
-      tokens: rec.entry.tokens.slice(0, bestLen),
-      caches: clones,
-      attachments,
-      ns,
-      retain: rec.share.acquire(),
-    };
-  }
-
-  /** Longest usable prefix length available for `prompt` WITHOUT taking
-   *  (the SSD tier compares this against its own best before restoring). */
-  peekPrefixLen(prompt: number[], ns = ""): number {
-    let best = 0;
-    for (const { entry: e } of this.#entries) {
-      if (e.ns !== ns) continue;
-      const p = Math.min(commonPrefixLength(e.tokens, prompt), prompt.length - 1);
-      if (p <= best || p < minimumReusableOffset(e.caches)) continue;
-      const trimNeeded = e.tokens.length - p; // same usability rule as take()
-      if (trimNeeded > 0 && (!!e.attachments?.length || !e.caches.every((c) => c.isTrimmable()))) continue;
-      best = p;
-    }
-    return best;
-  }
-
-  /** Read-only exact-token lookup (the write-behind snapshot path: the
-   *  entry stays owned by the cache; the caller only reads array state,
-   *  under the gateway lock so no generation is mutating it). */
-  findExact(tokens: number[], ns = ""): PromptCacheEntry | null {
-    for (const { entry: e } of this.#entries) {
-      if (e.ns !== ns || e.tokens.length !== tokens.length) continue;
-      if (commonPrefixLength(e.tokens, tokens) === tokens.length) return e;
-    }
-    return null;
-  }
-
-  /** Insert (or reinsert) an entry; evicts LRU entries over the byte cap
-   *  (spilling each to the cold tier first, when one is attached). If the
-   *  entry itself exceeds the cap it is spilled + disposed, not stored. */
-  put(tokens: number[], caches: Cache[], ns = "", retain?: () => void, attachments?: CheckpointAttachment[]): void {
-    // All cache-provided validation runs before ownership is adopted.
-    const bytes = cacheBytes(caches) + attachmentBytes(attachments);
-    const trimmable = !attachments?.length && caches.every((cache) => cache.isTrimmable());
-    // Storage takes resolved snapshots, independently of their producer.
-    this.materialize(caches, attachments);
-    const discard = (entry: PromptCacheEntry, spill: boolean) => {
-      try { this.#disposeEntry(entry, spill); }
-      catch (error) {
-        // The new entry is already owned here. Throwing back to its previous
-        // owner would invite disposal of state still held by this store.
-        console.warn(`[prompt-cache] entry cleanup failed: ${error}`);
-      }
-    };
-    const entry: PromptCacheEntry = { tokens, caches, ns, retain, attachments };
-    if (bytes > this.maxBytes) {
-      discard(entry, true);
-      return;
-    }
-    // The donor's retain becomes ref-counted so take()'s clones can hold
-    // shares; the entry's own retain is the donor's share.
-    const share = makeSharedRetain(retain);
-    entry.retain = share.acquire();
-    const rec: EntryRecord = { entry, bytes, lastUsed: ++this.#clock, lastUsedMs: Date.now(), share };
-    this.#entries.push(rec);
-    // Exact duplicates (equal tokens) are redundant REGARDLESS of
-    // trimmability — the new entry serves exactly the matches the old one
-    // did (mlx-lm's trie replaces in place). Without this, every gemma ctx
-    // repeat left another full-size wrapped-ring entry behind (measured
-    // ~0.4 GB retained per repeat on the 12B bench).
-    for (const old of [...this.#entries]) {
-      if (old === rec || old.entry.ns !== ns) continue;
-      if (old.entry.tokens.length !== tokens.length) continue;
-      if (commonPrefixLength(old.entry.tokens, tokens) !== tokens.length) continue;
-      this.#entries = this.#entries.filter((r) => r !== old);
-      discard(old.entry, false);
-    }
-    // SUPERSEDE strict prefix-ancestors (the SSD store's rule, brought to
-    // RAM — prefix sharing would otherwise grow an entry per turn): same-ns
-    // entries whose tokens are a prefix of the new entry are redundant
-    // and disposed WITHOUT spill (the new entry serves their prefixes and
-    // its own write-behind persists them) — but ONLY when the new entry's
-    // caches are all trimmable. An untrimmable new entry (wrapped ring)
-    // can only serve exact-length matches, so shorter ancestors — e.g.
-    // the prompt-boundary snapshot that makes drifted multi-turn agents
-    // hit — must survive.
-    if (trimmable) {
-      for (const old of [...this.#entries]) {
-        if (old === rec || old.entry.ns !== ns) continue;
-        if (old.entry.tokens.length >= tokens.length || old.entry.tokens.length < minimumReusableOffset(caches)) continue;
-        if (commonPrefixLength(old.entry.tokens, tokens) !== old.entry.tokens.length) continue;
-        this.#entries = this.#entries.filter((r) => r !== old);
-        discard(old.entry, false);
-      }
-    }
-    while (this.totalBytes > this.maxBytes && this.#entries.length > 1) {
-      let lruIdx = 0;
-      for (let i = 1; i < this.#entries.length; i++)
-        if (this.#entries[i]!.lastUsed < this.#entries[lruIdx]!.lastUsed) lruIdx = i;
-      const [evicted] = this.#entries.splice(lruIdx, 1);
-      discard(evicted!.entry, true);
-    }
-    try { this.onPut?.(tokens, ns); } catch {}
-  }
-
-  /** Layer-0 idle demotion: spill every entry unused for `idleMs` to the
-   *  cold tier and FREE its GPU memory. The prefix stays reachable — the
-   *  next take() restores it via zero-copy mmap (~0.25 s for a 13.7k-token
-   *  entry, vs a 12 s re-prefill) — but between bursts the RAM tier drains
-   *  toward empty, returning unified memory to the system. Caller must
-   *  hold the generation lock (entries' arrays are disposed here) — with
-   *  a spillOwned sink the lock covers only the zero-copy clone; the
-   *  write runs off-lock and the buffers free when it settles. No-op
-   *  without a cold tier — demotion without a place to demote TO would
-   *  just be data loss. Returns entries demoted. */
-  demoteIdle(idleMs: number, now = Date.now()): number {
-    if (!this.#cold) return 0;
-    let n = 0;
-    for (const rec of [...this.#entries]) {
-      if (now - rec.lastUsedMs < idleMs) continue;
-      this.#entries = this.#entries.filter((r) => r !== rec);
-      this.#disposeEntry(rec.entry, true); // spill-first, then dispose
-      this.demotions++;
-      n++;
-    }
-    return n;
-  }
-
-  clear(): void {
-    const entries = this.#entries;
-    this.#entries = [];
-    disposeResources(entries.map(({ entry }) => ({ dispose: () => this.#disposeEntry(entry, false) })));
-  }
-}