@elizaos/app-core 2.0.0-alpha.85 → 2.0.0-beta.2

package/services/local-inference/dflash-server.js

@@ -0,0 +1,1076 @@
+/**
+ * Out-of-process llama-server backend for DFlash speculative decoding.
+ *
+ * DFlash needs llama-server flags (`-md`, `--spec-type dflash`) that the
+ * in-process node-llama-cpp API does not expose. This backend is deliberately
+ * small: spawn a compatible llama-server, wait for health, and use the
+ * OpenAI-compatible chat endpoint so llama-server applies the model chat
+ * template and reasoning controls consistently with LlamaChatSession.
+ */
+import { spawn, spawnSync } from "node:child_process";
+import fs from "node:fs";
+import net from "node:net";
+import os from "node:os";
+import path from "node:path";
+import { buildModelHash, DEFAULT_CACHE_TTLS, deriveSlotId, evictExpired, readCacheStats, slotSavePath, } from "./cache-bridge";
+import { findCatalogModel } from "./catalog";
+import { diffSnapshots, fetchMetricsSnapshot, } from "./llama-server-metrics";
+import { localInferenceRoot } from "./paths";
+const DEFAULT_HOST = "127.0.0.1";
+const DEFAULT_START_TIMEOUT_MS = 120_000;
+const METAL_UNSUPPORTED_CACHE_TYPES = new Set([
+    "turbo2",
+    "turbo3",
+    "turbo4",
+    "turbo2_0",
+    "turbo3_0",
+    "turbo4_0",
+    "turbo2_tcq",
+    "turbo3_tcq",
+]);
+const DFLASH_METRIC_ALIASES = {
+    decoded: ["llamacpp:n_decode_total", "llamacpp:n_decode"],
+    drafted: ["llamacpp:n_drafted_total", "llamacpp:n_drafted"],
+    accepted: [
+        "llamacpp:n_drafted_accepted_total",
+        "llamacpp:n_drafted_accepted",
+        "llamacpp:n_accepted_total",
+        "llamacpp:n_accepted",
+    ],
+};
+export function parseDflashMetrics(body) {
+    const samples = new Map();
+    for (const rawLine of body.split(/\r?\n/)) {
+        const line = rawLine.trim();
+        if (!line || line.startsWith("#"))
+            continue;
+        const match = line.match(/^([a-zA-Z_:][\w:]*)(?:\{[^}]*\})?\s+([+-]?\d+(?:\.\d+)?(?:e[+-]?\d+)?)/);
+        if (!match)
+            continue;
+        const name = match[1];
+        const value = Number(match[2]);
+        if (!name || !Number.isFinite(value))
+            continue;
+        samples.set(name, (samples.get(name) ?? 0) + value);
+    }
+    const readFirst = (aliases) => {
+        for (const alias of aliases) {
+            const value = samples.get(alias);
+            if (value !== undefined)
+                return value;
+        }
+        return null;
+    };
+    const drafted = readFirst(DFLASH_METRIC_ALIASES.drafted);
+    const accepted = readFirst(DFLASH_METRIC_ALIASES.accepted);
+    if (drafted === null || accepted === null)
+        return null;
+    const decoded = readFirst(DFLASH_METRIC_ALIASES.decoded) ?? 0;
+    return {
+        decoded,
+        drafted,
+        accepted,
+        acceptanceRate: drafted > 0 ? accepted / drafted : Number.NaN,
+    };
+}
+function readBool(name) {
+    const raw = process.env[name]?.trim().toLowerCase();
+    return raw === "1" || raw === "true" || raw === "yes";
+}
+function managedDflashBinaryPath() {
+    return path.join(localInferenceRoot(), "bin", "dflash", platformKey(), "llama-server");
+}
+function managedDflashCapabilitiesPath() {
+    return path.join(localInferenceRoot(), "bin", "dflash", platformKey(), "CAPABILITIES.json");
+}
+let capabilitiesCache = null;
+/**
+ * Read CAPABILITIES.json for the currently-installed binary, if present.
+ * Returns null when the file is missing or unreadable — older binaries
+ * built before the capabilities probe landed simply don't have it.
+ *
+ * Cached by path+mtime so repeated probes are cheap.
+ */
+export function readDflashBinaryCapabilities() {
+    const capsPath = managedDflashCapabilitiesPath();
+    let stat;
+    try {
+        stat = fs.statSync(capsPath);
+    }
+    catch {
+        return null;
+    }
+    if (capabilitiesCache &&
+        capabilitiesCache.path === capsPath &&
+        capabilitiesCache.mtimeMs === stat.mtimeMs) {
+        return capabilitiesCache.caps;
+    }
+    try {
+        const parsed = JSON.parse(fs.readFileSync(capsPath, "utf8"));
+        if (typeof parsed === "object" &&
+            parsed !== null &&
+            typeof parsed.kernels === "object" &&
+            parsed.kernels !== null) {
+            capabilitiesCache = {
+                path: capsPath,
+                mtimeMs: stat.mtimeMs,
+                caps: parsed,
+            };
+            return parsed;
+        }
+    }
+    catch {
+        // fall through
+    }
+    return null;
+}
+function isMetalDflashRuntime() {
+    return platformKey().endsWith("-metal");
+}
+function dflashMetalAutoEnabled() {
+    return (readBool("ELIZA_DFLASH_METAL_AUTO") ||
+        readBool("ELIZA_DFLASH_METAL_ENABLED"));
+}
+function assertCacheTypeSupportedOnBackend(name, value) {
+    if (isMetalDflashRuntime() &&
+        METAL_UNSUPPORTED_CACHE_TYPES.has(value.toLowerCase())) {
+        // Wave-3 hardware-verified all 5 Metal kernels 8/8 PASS on Apple M4 Max
+        // via the JIT harness, but the patch coverage audit (2026-05-10) found
+        // the standalones do NOT actually ship in the v0.4.0-milady binary —
+        // build-llama-cpp-dflash.mjs's patchMetal* hooks are decorative-only.
+        // Until the shader-shipping work lands AND the shipped binary itself
+        // is re-verified, the runtime correctly refuses these cache types on
+        // Metal. Once capabilities.kernels.turbo3 etc. flip to true via a real
+        // build, gate this on the capability instead of the static set.
+        throw new Error(`${name}=${value} is not yet shipped in the Metal binary. Wave-3 verified the kernels in a JIT harness but the build pipeline doesn't compile them into the shipped artifact. Use f16 KV on Metal until the kernel-shipping work lands.`);
+    }
+}
+export function dflashEnabled() {
+    if (readBool("ELIZA_DFLASH_DISABLED"))
+        return false;
+    if (readBool("ELIZA_DFLASH_ENABLED"))
+        return true;
+    if (!fs.existsSync(managedDflashBinaryPath()))
+        return false;
+    if (isMetalDflashRuntime())
+        return dflashMetalAutoEnabled();
+    return true;
+}
+export function dflashRequired() {
+    return readBool("ELIZA_DFLASH_REQUIRED");
+}
+function candidateBinaryPaths() {
+    const explicit = process.env.ELIZA_DFLASH_LLAMA_SERVER?.trim();
+    const out = explicit ? [explicit] : [];
+    out.push(managedDflashBinaryPath());
+    if (readBool("ELIZA_DFLASH_ENABLED"))
+        out.push("llama-server");
+    return out;
+}
+function platformKey() {
+    const forced = process.env.ELIZA_DFLASH_BACKEND?.trim().toLowerCase();
+    if (forced)
+        return `${process.platform}-${process.arch}-${forced}`;
+    const backend = process.platform === "darwin"
+        ? "metal"
+        : process.env.HIP_VISIBLE_DEVICES || process.env.ROCR_VISIBLE_DEVICES
+            ? "rocm"
+            : process.env.CUDA_VISIBLE_DEVICES &&
+                process.env.CUDA_VISIBLE_DEVICES !== "-1"
+                ? "cuda"
+                : "cpu";
+    return `${process.platform}-${process.arch}-${backend}`;
+}
+export function resolveDflashBinary() {
+    for (const candidate of candidateBinaryPaths()) {
+        if (candidate.includes(path.sep)) {
+            if (fs.existsSync(candidate))
+                return candidate;
+            continue;
+        }
+        const pathEnv = process.env.PATH ?? "";
+        for (const dir of pathEnv.split(path.delimiter)) {
+            const resolved = path.join(dir, candidate);
+            if (fs.existsSync(resolved))
+                return resolved;
+        }
+    }
+    return null;
+}
+export function getDflashRuntimeStatus() {
+    const binary = resolveDflashBinary();
+    const capabilities = readDflashBinaryCapabilities();
+    if (!dflashEnabled()) {
+        const managedBinaryExists = fs.existsSync(managedDflashBinaryPath());
+        const reason = managedBinaryExists && isMetalDflashRuntime()
+            ? "DFlash Metal binary found but auto-disabled because the current Eliza-1 Metal path is faster target-only; set ELIZA_DFLASH_ENABLED=1 or ELIZA_DFLASH_METAL_AUTO=1 to force it."
+            : "DFlash auto-enables when the managed llama-server binary is installed; set ELIZA_DFLASH_ENABLED=1 to force a PATH/explicit binary, or run packages/app-core/scripts/build-llama-cpp-dflash.mjs.";
+        return {
+            enabled: false,
+            required: dflashRequired(),
+            binaryPath: binary,
+            reason,
+            capabilities,
+        };
+    }
+    if (!binary) {
+        return {
+            enabled: false,
+            required: dflashRequired(),
+            binaryPath: null,
+            reason: "No compatible llama-server found. Set ELIZA_DFLASH_LLAMA_SERVER or run packages/app-core/scripts/build-llama-cpp-dflash.mjs.",
+            capabilities,
+        };
+    }
+    return {
+        enabled: true,
+        required: dflashRequired(),
+        binaryPath: binary,
+        reason: "DFlash llama-server binary found.",
+        capabilities,
+    };
+}
+function normalizeGpuLayers(value) {
+    return value === "auto" ? "99" : String(value);
+}
+function findPython() {
+    for (const candidate of ["python3", "python"]) {
+        const result = spawnSync(candidate, ["--version"], {
+            stdio: "ignore",
+            env: process.env,
+        });
+        if (result.status === 0)
+            return candidate;
+    }
+    return null;
+}
+function maybeRepairDflashDrafter(binaryPath, targetModelPath, drafterModelPath) {
+    if (readBool("ELIZA_DFLASH_REPAIR_DISABLED"))
+        return drafterModelPath;
+    if (!fs.existsSync(targetModelPath) || !fs.existsSync(drafterModelPath)) {
+        return drafterModelPath;
+    }
+    const repairedPath = drafterModelPath.replace(/\.gguf$/i, ".repaired.gguf");
+    if (repairedPath === drafterModelPath)
+        return drafterModelPath;
+    if (fs.existsSync(repairedPath))
+        return repairedPath;
+    const python = findPython();
+    if (!python)
+        return drafterModelPath;
+    const bundledGgufPy = path.join(path.dirname(binaryPath), "gguf-py");
+    const pythonPath = [
+        fs.existsSync(bundledGgufPy) ? bundledGgufPy : null,
+        process.env.PYTHONPATH,
+    ]
+        .filter((value) => Boolean(value))
+        .join(path.delimiter);
+    const repairCode = `
+import sys
+from pathlib import Path
+
+if len(sys.argv) != 4:
+    raise SystemExit("usage: repair_dflash.py TARGET DRAFTER OUT")
+
+target = Path(sys.argv[1])
+drafter = Path(sys.argv[2])
+out = Path(sys.argv[3])
+
+import gguf
+from gguf.scripts.gguf_new_metadata import MetadataDetails, copy_with_new_metadata, get_field_data
+
+target_reader = gguf.GGUFReader(target, "r")
+draft_reader = gguf.GGUFReader(drafter, "r")
+
+if get_field_data(draft_reader, gguf.Keys.Tokenizer.MERGES):
+    print(drafter)
+    raise SystemExit(0)
+
+merges = get_field_data(target_reader, gguf.Keys.Tokenizer.MERGES)
+if not merges:
+    raise SystemExit("target GGUF has no tokenizer.ggml.merges metadata")
+
+arch = get_field_data(draft_reader, gguf.Keys.General.ARCHITECTURE)
+writer = gguf.GGUFWriter(out, arch=arch, endianess=draft_reader.endianess)
+alignment = get_field_data(draft_reader, gguf.Keys.General.ALIGNMENT)
+if alignment is not None:
+    writer.data_alignment = alignment
+copy_with_new_metadata(
+    draft_reader,
+    writer,
+    {gguf.Keys.Tokenizer.MERGES: MetadataDetails(gguf.GGUFValueType.ARRAY, merges, sub_type=gguf.GGUFValueType.STRING)},
+    [],
+)
+print(out)
+`;
+    const result = spawnSync(python, ["-c", repairCode, targetModelPath, drafterModelPath, repairedPath], {
+        env: {
+            ...process.env,
+            ...(pythonPath ? { PYTHONPATH: pythonPath } : {}),
+        },
+        encoding: "utf8",
+        maxBuffer: 16 * 1024 * 1024,
+    });
+    if (result.status !== 0) {
+        console.warn("[local-inference] DFlash drafter tokenizer repair failed; trying original drafter:", result.stderr || result.stdout);
+        return drafterModelPath;
+    }
+    const outputPath = result.stdout.trim().split(/\r?\n/).at(-1)?.trim();
+    return outputPath && fs.existsSync(outputPath)
+        ? outputPath
+        : drafterModelPath;
+}
+function resolvePort() {
+    const explicit = Number.parseInt(process.env.ELIZA_DFLASH_PORT ?? "", 10);
+    if (Number.isFinite(explicit) && explicit > 0) {
+        return Promise.resolve(explicit);
+    }
+    return new Promise((resolve, reject) => {
+        const server = net.createServer();
+        server.unref();
+        server.on("error", reject);
+        server.listen(0, DEFAULT_HOST, () => {
+            const address = server.address();
+            server.close(() => {
+                if (address && typeof address === "object") {
+                    resolve(address.port);
+                }
+                else {
+                    reject(new Error("Could not allocate a loopback port for llama-server"));
+                }
+            });
+        });
+    });
+}
+async function sleep(ms) {
+    await new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function fetchJson(url, init, timeoutMs = 60_000) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+        const res = await fetch(url, { ...init, signal: controller.signal });
+        if (!res.ok) {
+            const body = await res.text().catch(() => "");
+            throw new Error(`HTTP ${res.status} from ${url}${body ? `: ${body}` : ""}`);
+        }
+        return await res.json();
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+/**
+ * Default `--parallel` when caching is enabled. Higher values give more
+ * distinct cache slots so concurrent prefixes don't evict each other,
+ * at the cost of KV memory. 4 is a balance that works for a single-user
+ * desktop while still saturating a single GPU under load.
+ */
+const DEFAULT_CACHE_PARALLEL = 4;
+/**
+ * Resolve `--parallel`. Order: ELIZA_LOCAL_PARALLEL (generalised) →
+ * ELIZA_DFLASH_PARALLEL (legacy) → catalog `optimizations.parallel` →
+ * DEFAULT_CACHE_PARALLEL. The generalised env wins because it's the
+ * operator's explicit override; the legacy DFlash-specific env stays
+ * for back-compat.
+ */
+function resolveParallel(catalogParallel) {
+    for (const raw of [
+        process.env.ELIZA_LOCAL_PARALLEL,
+        process.env.ELIZA_DFLASH_PARALLEL,
+    ]) {
+        const trimmed = raw?.trim();
+        if (!trimmed)
+            continue;
+        const parsed = Number.parseInt(trimmed, 10);
+        if (Number.isFinite(parsed) && parsed >= 1)
+            return parsed;
+    }
+    if (typeof catalogParallel === "number" &&
+        Number.isFinite(catalogParallel) &&
+        catalogParallel >= 1) {
+        return catalogParallel;
+    }
+    return DEFAULT_CACHE_PARALLEL;
+}
+/**
+ * Append optimization flags driven by env overrides + catalog metadata to a
+ * llama-server arg list. Env wins over the catalog when both supply the
+ * same knob — the operator's escape hatch.
+ *
+ * Env mapping (per AGENTS.md / task brief):
+ *
+ *   ELIZA_LOCAL_LOOKAHEAD=N        → --lookahead N
+ *   ELIZA_LOCAL_NGRAM=on           → enable n-gram drafter (uses
+ *                                    optimizations.ngramDraft when set,
+ *                                    else conservative defaults)
+ *   ELIZA_LOCAL_PARALLEL=N         → --parallel N (handled by resolveParallel
+ *                                    at the call site, not here)
+ *   ELIZA_LOCAL_MOE_OFFLOAD=cpu    → -ot ".*=CPU"
+ *   ELIZA_LOCAL_MLOCK=1            → --mlock
+ *   ELIZA_LOCAL_NO_MMAP=1          → --no-mmap
+ *   ELIZA_LOCAL_FLASH_ATTENTION=on → -fa on (DFlash already implies it via
+ *                                    spec config; this is for non-DFlash
+ *                                    llama-server use cases)
+ */
+function readBoolFlag(name) {
+    const raw = process.env[name]?.trim().toLowerCase();
+    if (raw === undefined)
+        return undefined;
+    if (raw === "1" || raw === "true" || raw === "yes" || raw === "on") {
+        return true;
+    }
+    if (raw === "0" || raw === "false" || raw === "no" || raw === "off") {
+        return false;
+    }
+    return undefined;
+}
+export function appendOptimizationFlags(args, optimizations) {
+    // --lookahead N
+    const lookaheadEnv = process.env.ELIZA_LOCAL_LOOKAHEAD?.trim();
+    const lookaheadValue = lookaheadEnv
+        ? Number.parseInt(lookaheadEnv, 10)
+        : optimizations?.lookahead;
+    if (typeof lookaheadValue === "number" &&
+        Number.isFinite(lookaheadValue) &&
+        lookaheadValue > 0) {
+        args.push("--lookahead", String(lookaheadValue));
+    }
+    // N-gram drafter — only meaningful when DFlash is NOT in use (mutually
+    // exclusive). Caller is responsible for not setting ngramDraft on a
+    // DFlash-configured catalog entry.
+    const ngramEnvOn = readBoolFlag("ELIZA_LOCAL_NGRAM");
+    const ngramConfig = optimizations?.ngramDraft;
+    const ngramEffective = ngramEnvOn === false
+        ? null
+        : (ngramConfig ?? (ngramEnvOn ? { min: 4, max: 8, minProb: 0.5 } : null));
+    if (ngramEffective) {
+        args.push("--draft-min", String(ngramEffective.min));
+        args.push("--draft-max", String(ngramEffective.max));
+        args.push("--draft-min-prob", String(ngramEffective.minProb));
+    }
+    // -ot ".*=CPU" — MoE expert offload to CPU.
+    const moeEnv = process.env.ELIZA_LOCAL_MOE_OFFLOAD?.trim().toLowerCase();
+    const moeMode = moeEnv ?? optimizations?.moeOffload;
+    if (moeMode === "cpu") {
+        args.push("-ot", ".*=CPU");
+    }
+    // --mlock
+    const mlockEnv = readBoolFlag("ELIZA_LOCAL_MLOCK");
+    const mlock = mlockEnv ?? optimizations?.mlock;
+    if (mlock === true)
+        args.push("--mlock");
+    // --no-mmap
+    const noMmapEnv = readBoolFlag("ELIZA_LOCAL_NO_MMAP");
+    const noMmap = noMmapEnv ?? optimizations?.noMmap;
+    if (noMmap === true)
+        args.push("--no-mmap");
+    // --mmproj <path>
+    const mmprojEnv = process.env.ELIZA_LOCAL_MMPROJ?.trim();
+    const mmproj = mmprojEnv || optimizations?.mmproj;
+    if (mmproj)
+        args.push("--mmproj", mmproj);
+    // --alias <name>
+    const aliasEnv = process.env.ELIZA_LOCAL_ALIAS?.trim();
+    const alias = aliasEnv || optimizations?.alias;
+    if (alias)
+        args.push("--alias", alias);
+    // -fa on / -fa off (catalog default off so existing DFlash behaviour
+    // — which compiles flash attention into the spec config — is unchanged
+    // unless the operator opts in).
+    const faEnv = readBoolFlag("ELIZA_LOCAL_FLASH_ATTENTION");
+    const fa = faEnv ?? optimizations?.flashAttention;
+    if (fa === true)
+        args.push("-fa", "on");
+    return args;
+}
+/**
+ * Default eviction sweep interval. Set to 5 minutes to match the short
+ * TTL — a slot file at most one short-TTL window stale before it's
+ * deleted. Override via `ELIZA_LOCAL_EVICTION_INTERVAL_MS`.
+ */
+const DEFAULT_EVICTION_INTERVAL_MS = 5 * 60 * 1000;
+function resolveEvictionIntervalMs() {
+    const raw = process.env.ELIZA_LOCAL_EVICTION_INTERVAL_MS?.trim();
+    if (!raw)
+        return DEFAULT_EVICTION_INTERVAL_MS;
+    const parsed = Number.parseInt(raw, 10);
+    if (!Number.isFinite(parsed) || parsed < 60_000) {
+        // Anything under a minute is almost certainly a typo; clamp to
+        // protect the disk from sweep storms.
+        return DEFAULT_EVICTION_INTERVAL_MS;
+    }
+    return parsed;
+}
+export class DflashLlamaServer {
+    id = "llama-server";
+    child = null;
+    baseUrl = null;
+    stderrTail = [];
+    loadedPlan = null;
+    /**
+     * Cache state captured at `start()`. The model hash + parallel count
+     * stay constant for the lifetime of the spawned process so we record
+     * them once and reuse them on every `generate()` call.
+     */
+    cacheModelHash = null;
+    cacheParallel = DEFAULT_CACHE_PARALLEL;
+    cacheSlotDir = null;
+    evictionTimer = null;
+    /**
+     * Per-conversation slot files persisted on shutdown for cross-restart
+     * KV reuse. Distinct from the per-slot save dir: this directory keys
+     * by conversation id, so a conversation that comes back with a different
+     * slot id (e.g. after a --parallel resize) can still find its KV.
+     */
+    conversationKvDir = null;
+    /**
+     * Track which conversation ids have written KV state so far this
+     * process lifetime. Used for diagnostics and the "did the last save
+     * actually happen" assertion in tests.
+     */
+    persistedConversations = new Set();
+    hasLoadedModel() {
+        return this.child !== null && this.loadedPlan !== null;
+    }
+    currentModelPath() {
+        return this.loadedPlan?.targetModelPath ?? null;
+    }
+    /**
+     * Scrape the running llama-server's `/metrics` endpoint and return a
+     * `DflashMetricsSnapshot` with `drafted` / `accepted` / `decoded` counts.
+     * Returns `null` when no server is loaded, the endpoint isn't reachable,
+     * or the response doesn't contain the expected speculative counters.
+     * Mirrors the UI twin's API so `dflash-doctor` works against either copy.
+     */
+    async getMetrics() {
+        if (!this.baseUrl)
+            return null;
+        try {
+            const res = await fetch(`${this.baseUrl.replace(/\/$/, "")}/metrics`, {
+                method: "GET",
+            });
+            if (!res.ok)
+                return null;
+            return parseDflashMetrics(await res.text());
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Parallel slot count negotiated at `start()`. Used by the engine's
+     * conversation handle API to size new conversations into available
+     * slots. Returns the static default when no server is running.
+     */
+    parallelSlots() {
+        return this.cacheParallel;
+    }
+    /** Soft probe — does the binary resolve and is DFlash enabled. */
+    async available() {
+        return getDflashRuntimeStatus().enabled;
+    }
+    /**
+     * Unified backend contract entry point. Resolves the catalog entry from
+     * the plan and delegates to `start()` if a DFlash plan is configured.
+     * For non-DFlash llama-server use (e.g. `requiresKernel` for turbo3
+     * without spec decoding), the catalog can declare an `optimizations`
+     * block without `dflash` and we still launch the server here.
+     */
+    async load(plan) {
+        const catalog = plan.catalog ??
+            (plan.modelId ? findCatalogModel(plan.modelId) : undefined);
+        const dflash = catalog?.runtime?.dflash;
+        const optimizations = catalog?.runtime?.optimizations ?? null;
+        if (!dflash) {
+            throw new Error(`[dflash] llama-server backend currently requires a catalog 'runtime.dflash' block. Model '${plan.modelId ?? plan.modelPath}' has none — declare DFlash or route this model through node-llama-cpp.`);
+        }
+        // The drafter is resolved from the registry by the engine before this
+        // dispatcher call, but the engine no longer pre-builds the dflash plan,
+        // so we resolve it here. Inline import avoids the engine ↔ dflash-server
+        // import cycle.
+        const { listInstalledModels } = await import("./registry");
+        const installed = await listInstalledModels();
+        const target = installed.find((m) => m.path === plan.modelPath) ??
+            installed.find((m) => m.id === plan.modelId);
+        if (!target) {
+            throw new Error(`[dflash] No installed model matched plan path/id (${plan.modelPath}; ${plan.modelId ?? "no id"}).`);
+        }
+        const drafter = installed.find((m) => m.id === dflash.drafterModelId);
+        if (!drafter) {
+            throw new Error(`[dflash] ${target.displayName} requires companion drafter ${dflash.drafterModelId}; install it first.`);
+        }
+        // Per-load overrides win over catalog defaults. The active-model
+        // coordinator merges these in before the dispatcher is called; this
+        // keeps the same precedence on the llama-server path so a benchmark
+        // run that asks for `contextSize: 131072` actually starts the server
+        // with `--ctx-size 131072` instead of the smaller catalog default.
+        const overrides = plan.overrides;
+        const contextSize = typeof overrides?.contextSize === "number"
+            ? overrides.contextSize
+            : dflash.contextSize;
+        const gpuLayers = typeof overrides?.gpuLayers === "number"
+            ? overrides.gpuLayers
+            : dflash.gpuLayers;
+        await this.start({
+            targetModelPath: target.path,
+            drafterModelPath: drafter.path,
+            contextSize,
+            draftContextSize: dflash.draftContextSize,
+            draftMin: dflash.draftMin,
+            draftMax: dflash.draftMax,
+            gpuLayers,
+            draftGpuLayers: dflash.draftGpuLayers,
+            disableThinking: dflash.disableThinking,
+        }, optimizations);
+    }
+    /** Backend interface alias for stop(). */
+    async unload() {
+        await this.stop();
+    }
+    async start(plan, optimizations) {
+        if (this.child &&
+            this.loadedPlan?.targetModelPath === plan.targetModelPath &&
+            this.loadedPlan.drafterModelPath === plan.drafterModelPath) {
+            return;
+        }
+        await this.stop();
+        const status = getDflashRuntimeStatus();
+        if (!status.enabled || !status.binaryPath) {
+            throw new Error(`[dflash] ${status.reason}`);
+        }
+        const drafterModelPath = maybeRepairDflashDrafter(status.binaryPath, plan.targetModelPath, plan.drafterModelPath);
+        const port = await resolvePort();
+        const host = process.env.ELIZA_DFLASH_HOST?.trim() || DEFAULT_HOST;
+        const cacheTypeK = process.env.ELIZA_DFLASH_CACHE_TYPE_K?.trim();
+        const cacheTypeV = process.env.ELIZA_DFLASH_CACHE_TYPE_V?.trim();
+        const parallel = resolveParallel(optimizations?.parallel);
+        const modelHash = buildModelHash({
+            targetModelPath: plan.targetModelPath,
+            drafterModelPath,
+            cacheTypeK: cacheTypeK ?? null,
+            cacheTypeV: cacheTypeV ?? null,
+            extra: `ctx=${plan.contextSize};parallel=${parallel}`,
+        });
+        const slotDir = slotSavePath(modelHash);
+        // llama-server's slot API treats `filename` as a basename relative to
+        // --slot-save-path. Keep per-conversation KV files in that same root so
+        // save and restore agree on the exact path.
+        const conversationKvDir = slotDir;
+        fs.mkdirSync(slotDir, { recursive: true });
+        // Fire-and-forget eviction: stale slot files on disk shouldn't block
+        // server startup, but we don't want them to grow without bound.
+        void evictExpired(slotDir, DEFAULT_CACHE_TTLS).catch(() => {
+            // Best effort; an EACCES or similar should not prevent server start.
+        });
+        void evictExpired(conversationKvDir, DEFAULT_CACHE_TTLS).catch(() => { });
+        this.cacheModelHash = modelHash;
+        this.cacheParallel = parallel;
+        this.cacheSlotDir = slotDir;
+        this.conversationKvDir = conversationKvDir;
+        const args = [
+            "--model",
+            plan.targetModelPath,
+            "-md",
+            drafterModelPath,
+            "--spec-type",
+            "dflash",
+            "--host",
+            host,
+            "--port",
+            String(port),
+            "--n-gpu-layers",
+            normalizeGpuLayers(plan.gpuLayers),
+            "--n-gpu-layers-draft",
+            normalizeGpuLayers(plan.draftGpuLayers),
+            "--ctx-size",
+            String(plan.contextSize),
+            "--ctx-size-draft",
+            String(plan.draftContextSize),
+            "--draft-min",
+            String(plan.draftMin),
+            "--draft-max",
+            String(plan.draftMax),
+            "--parallel",
+            String(parallel),
+            // Persist per-slot KV state to disk so prefix reuse survives the
+            // process lifetime. llama-server keys files by slot id, not by
+            // prompt hash, but combined with deterministic slot_id derivation
+            // this gives effective prefix caching across restarts.
+            "--slot-save-path",
+            slotDir,
+            // Allow the server to fall back to a similar slot (>= 0.7
+            // similarity) when an exact match isn't loaded — useful when
+            // distinct keys land on the same slot due to hash collision.
+            "--slot-prompt-similarity",
+            "0.7",
+            "--metrics",
+            "--jinja",
+        ];
+        if (plan.disableThinking) {
+            args.push("--reasoning", "off");
+            args.push("--chat-template-kwargs", '{"enable_thinking":false}');
+        }
+        if (cacheTypeK) {
+            assertCacheTypeSupportedOnBackend("ELIZA_DFLASH_CACHE_TYPE_K", cacheTypeK);
+            args.push("--cache-type-k", cacheTypeK);
+        }
+        if (cacheTypeV) {
+            assertCacheTypeSupportedOnBackend("ELIZA_DFLASH_CACHE_TYPE_V", cacheTypeV);
+            args.push("--cache-type-v", cacheTypeV);
+        }
+        appendOptimizationFlags(args, optimizations ?? null);
+        const extra = process.env.ELIZA_DFLASH_LLAMA_ARGS?.trim();
+        if (extra && isMetalDflashRuntime()) {
+            for (const cacheType of METAL_UNSUPPORTED_CACHE_TYPES) {
+                if (extra.toLowerCase().split(/\s+/).includes(cacheType)) {
+                    throw new Error(`ELIZA_DFLASH_LLAMA_ARGS includes ${cacheType}, but that KV cache type is not production-safe on Metal in the DFlash fork. Use f16 KV on Metal or run this variant on CUDA/ROCm.`);
+                }
+            }
+        }
+        if (extra)
+            args.push(...extra.split(/\s+/).filter(Boolean));
+        fs.mkdirSync(path.join(localInferenceRoot(), "logs"), { recursive: true });
+        this.stderrTail = [];
+        const child = spawn(status.binaryPath, args, {
+            env: { ...process.env },
+            stdio: ["ignore", "pipe", "pipe"],
+        });
+        this.child = child;
+        this.baseUrl = `http://${host}:${port}`;
+        this.loadedPlan = plan;
+        child.stdout?.on("data", (chunk) => this.captureLog(chunk));
+        child.stderr?.on("data", (chunk) => this.captureLog(chunk));
+        child.on("exit", (code, signal) => {
+            if (this.child && (code !== null || signal !== null)) {
+                this.child = null;
+                this.baseUrl = null;
+                this.loadedPlan = null;
+            }
+        });
+        await this.waitUntilReady(DEFAULT_START_TIMEOUT_MS);
+        // Periodic eviction sweep — short TTL or shorter, capped to one
+        // sweep per minute. Without this, stale slot files accumulate to
+        // gigabytes over a long-running session because eviction was only
+        // ever fired at startup.
+        this.startEvictionTimer();
+    }
+    /**
+     * Set up the periodic eviction sweep that keeps stale slot files from
+     * accumulating. Runs `evictExpired` against both the per-slot save
+     * directory (used by llama-server's own KV save) and the per-conversation
+     * directory (used by our `persistConversationKv` saves) so neither can
+     * grow without bound.
+     */
+    startEvictionTimer() {
+        if (this.evictionTimer)
+            return;
+        const intervalMs = resolveEvictionIntervalMs();
+        const timer = setInterval(() => {
+            const slotDir = this.cacheSlotDir;
+            const conversationDir = this.conversationKvDir;
+            if (slotDir) {
+                void evictExpired(slotDir, DEFAULT_CACHE_TTLS).catch(() => {
+                    // Don't crash the timer on a single failed sweep — we'll try again
+                    // on the next tick.
+                });
+            }
+            if (conversationDir) {
+                void evictExpired(conversationDir, DEFAULT_CACHE_TTLS).catch(() => { });
+            }
+        }, intervalMs);
+        timer.unref();
+        this.evictionTimer = timer;
+    }
+    async stop() {
+        if (this.evictionTimer) {
+            clearInterval(this.evictionTimer);
+            this.evictionTimer = null;
+        }
+        const child = this.child;
+        const baseUrl = this.baseUrl;
+        const conversationDir = this.conversationKvDir;
+        this.child = null;
+        this.baseUrl = null;
+        this.loadedPlan = null;
+        this.cacheModelHash = null;
+        this.cacheSlotDir = null;
+        this.conversationKvDir = null;
+        this.cacheParallel = DEFAULT_CACHE_PARALLEL;
+        if (!child)
+            return;
+        // Best-effort: tell llama-server to flush per-conversation KV state
+        // to disk before we kill it. If the dispatcher restarts the server
+        // (or the whole process restarts), conversations re-opening with the
+        // same id will lazy-restore from these files on first generate.
+        if (baseUrl && conversationDir) {
+            await this.persistAllConversationsBeforeStop(baseUrl, conversationDir);
+        }
+        child.kill("SIGTERM");
+        await Promise.race([
+            new Promise((resolve) => child.once("exit", () => resolve())),
+            sleep(5_000).then(() => {
+                if (!child.killed)
+                    child.kill("SIGKILL");
+            }),
+        ]);
+    }
+    /**
+     * Issue `POST /slots/<slot_id>?action=save` for every conversation slot
+     * the registry knows about. Best-effort — a single slot's save failing
+     * must not block the rest of the shutdown sequence.
+     *
+     * The slot file path is `<conversationDir>/<conversationId>.bin`, which
+     * is what `restoreConversationKv` reads on the next `generate` for the
+     * same conversation id.
+     */
+    async persistAllConversationsBeforeStop(baseUrl, conversationDir) {
+        // Inline import to avoid a hard cycle with the engine, which imports
+        // this file at module load time.
+        const { conversationRegistry } = await import("./conversation-registry");
+        const handles = conversationRegistry.snapshot();
+        if (handles.length === 0)
+            return;
+        const tasks = handles.map(async (handle) => {
+            const targetPath = path.join(conversationDir, `${handle.conversationId}.bin`);
+            try {
+                await this.requestSlotSave(baseUrl, handle.slotId, targetPath);
+                this.persistedConversations.add(handle.conversationId);
+            }
+            catch {
+                // A single failed slot save must not block the stop path —
+                // the worst case is that conversation cold-prefills on next use.
+            }
+        });
+        await Promise.all(tasks);
+    }
+    /**
+     * Issue a single slot save. Splits out so `persistConversationKv` can
+     * call it from any path (graceful save, periodic checkpoint), not just
+     * the shutdown path.
+     *
+     * The fork's REST API: `POST /slots/<id>?action=save` with a JSON
+     * `{ filename: "<absolute-path>" }` body. llama-server writes the slot
+     * KV to that absolute path. Filename MUST be a basename within the
+     * directory llama-server was started with `--slot-save-path`; we
+     * generate filenames inside that root so the constraint is satisfied
+     * naturally.
+     */
+    async requestSlotSave(baseUrl, slotId, targetPath) {
+        if (slotId < 0)
+            return;
+        // llama-server expects `filename` as a basename inside the
+        // --slot-save-path root, not a full path. Strip the path prefix
+        // before sending; we reconstruct it from `cacheSlotDir` below.
+        const baseName = path.basename(targetPath);
+        await fetchJson(`${baseUrl}/slots/${slotId}?action=save`, {
+            method: "POST",
+            headers: { "content-type": "application/json" },
+            body: JSON.stringify({ filename: baseName }),
+        }, 10_000);
+    }
+    /**
+     * Issue a single slot restore. Mirror of `requestSlotSave`. Skips when
+     * the file doesn't exist — a fresh conversation has no KV to restore.
+     */
+    async requestSlotRestore(baseUrl, slotId, sourcePath) {
+        if (slotId < 0)
+            return false;
+        if (!fs.existsSync(sourcePath))
+            return false;
+        const baseName = path.basename(sourcePath);
+        await fetchJson(`${baseUrl}/slots/${slotId}?action=restore`, {
+            method: "POST",
+            headers: { "content-type": "application/json" },
+            body: JSON.stringify({ filename: baseName }),
+        }, 10_000);
+        return true;
+    }
+    /**
+     * Persist this conversation's KV state to a stable, cross-restart
+     * filename. Callers should fire this on a long-cache TTL boundary, on
+     * `closeConversation`, and at process shutdown.
+     *
+     * Returns true when a save was issued, false when there was no slot to
+     * save (e.g. slot pinning disabled, server not running).
+     */
+    async persistConversationKv(conversationId, slotId) {
+        const baseUrl = this.baseUrl;
+        const conversationDir = this.conversationKvDir;
+        if (!baseUrl || !conversationDir || slotId < 0)
+            return false;
+        const targetPath = path.join(conversationDir, `${conversationId}.bin`);
+        try {
+            await this.requestSlotSave(baseUrl, slotId, targetPath);
+            this.persistedConversations.add(conversationId);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Lazy-restore a previously-persisted conversation's KV state into the
+     * given slot. Called from the generate path on the first request for a
+     * conversation id whose registry handle was just opened.
+     */
+    async restoreConversationKv(conversationId, slotId) {
+        const baseUrl = this.baseUrl;
+        const conversationDir = this.conversationKvDir;
+        if (!baseUrl || !conversationDir || slotId < 0)
+            return false;
+        const sourcePath = path.join(conversationDir, `${conversationId}.bin`);
+        try {
+            const restored = await this.requestSlotRestore(baseUrl, slotId, sourcePath);
+            return restored;
+        }
+        catch {
+            return false;
+        }
+    }
+    /** Diagnostic snapshot of the on-disk slot save directory for this server. */
+    async describeCache() {
+        if (!this.cacheSlotDir) {
+            return {
+                modelHash: this.cacheModelHash,
+                slotDir: null,
+                parallel: this.cacheParallel,
+                files: [],
+            };
+        }
+        return {
+            modelHash: this.cacheModelHash,
+            slotDir: this.cacheSlotDir,
+            parallel: this.cacheParallel,
+            files: await readCacheStats(this.cacheSlotDir),
+        };
+    }
+    async generate(args) {
+        const result = await this.generateWithUsage(args);
+        return result.text;
+    }
+    /**
+     * Run one generation and return both the text AND the Anthropic-shape
+     * usage block. The usage block is built by differencing two `/metrics`
+     * snapshots taken before/after the request, plus the per-call
+     * `usage` body the chat-completion response itself returns.
+     *
+     * Falls back to zero counters when the metrics scrape fails — the
+     * response text is still surfaced. Callers that don't need usage can
+     * keep using `generate()`.
+     */
+    async generateWithUsage(args) {
+        const baseUrl = this.baseUrl;
+        if (!baseUrl) {
+            throw new Error("[dflash] llama-server is not running");
+        }
+        const dflashArgs = args;
+        const slotId = typeof dflashArgs.slotId === "number" && dflashArgs.slotId >= -1
+            ? dflashArgs.slotId
+            : deriveSlotId(args.cacheKey ?? "", this.cacheParallel);
+        const payload = {
+            model: "local-dflash",
+            messages: [{ role: "user", content: args.prompt }],
+            max_tokens: args.maxTokens ?? 2048,
+            temperature: args.temperature ?? 0.7,
+            top_p: args.topP ?? 0.9,
+            stop: args.stopSequences,
+            stream: false,
+            // `cache_prompt: true` is always safe — the worst case is the
+            // server matches no prefix tokens and the request behaves like a
+            // cold call. Pinning by `slot_id` only happens when the runtime
+            // gave us a stable cache key.
+            cache_prompt: true,
+            slot_id: slotId,
+        };
+        const before = await fetchMetricsSnapshot(baseUrl);
+        const json = (await fetchJson(`${baseUrl}/v1/chat/completions`, {
+            method: "POST",
+            headers: { "content-type": "application/json" },
+            body: JSON.stringify(payload),
+        }));
+        const after = await fetchMetricsSnapshot(baseUrl);
+        const text = extractCompletionText(json);
+        const responseUsage = extractResponseUsage(json);
+        const usage = diffSnapshots(before, after, responseUsage);
+        return { text, usage, slotId };
+    }
+    captureLog(chunk) {
+        const text = chunk.toString();
+        for (const line of text.split(/\r?\n/)) {
+            const trimmed = line.trim();
+            if (!trimmed)
+                continue;
+            this.stderrTail.push(trimmed);
+            while (this.stderrTail.length > 30)
+                this.stderrTail.shift();
+        }
+    }
+    async waitUntilReady(timeoutMs) {
+        if (!this.baseUrl)
+            throw new Error("[dflash] llama-server did not start");
+        const started = Date.now();
+        while (Date.now() - started < timeoutMs) {
+            if (!this.child) {
+                throw new Error(`[dflash] llama-server exited during startup: ${this.stderrTail.join(os.EOL)}`);
+            }
+            try {
+                await fetchJson(`${this.baseUrl}/health`, { method: "GET" }, 2_000);
+                return;
+            }
+            catch {
+                try {
+                    await fetchJson(`${this.baseUrl}/v1/models`, { method: "GET" }, 2_000);
+                    return;
+                }
+                catch {
+                    await sleep(500);
+                }
+            }
+        }
+        const detail = this.stderrTail.length
+            ? ` Last logs: ${this.stderrTail.join(os.EOL)}`
+            : "";
+        throw new Error(`[dflash] llama-server did not become ready within ${timeoutMs}ms.${detail}`);
+    }
+}
+export const dflashLlamaServer = new DflashLlamaServer();
+/**
+ * Extract the assistant text from a llama-server `/v1/chat/completions`
+ * response. Throws when no recognised text shape is present — silently
+ * returning empty would mask a real protocol mismatch.
+ */
+function extractCompletionText(json) {
+    const choice = Array.isArray(json.choices) ? json.choices[0] : null;
+    const message = choice && typeof choice === "object"
+        ? choice.message
+        : null;
+    const content = message && typeof message === "object"
+        ? message.content
+        : null;
+    if (typeof content === "string")
+        return content;
+    const text = json.text;
+    if (typeof text === "string")
+        return text;
+    throw new Error(`[dflash] Unexpected llama-server response: ${JSON.stringify(json)}`);
+}
+/**
+ * Extract the per-call usage block from a llama-server response. Returns
+ * undefined when the response did not include one — `diffSnapshots` then
+ * falls back to the metric-delta input/output counts.
+ */
+function extractResponseUsage(json) {
+    const usage = json.usage;
+    if (!usage || typeof usage !== "object")
+        return undefined;
+    const u = usage;
+    const out = {};
+    if (typeof u.prompt_tokens === "number") {
+        out.prompt_tokens = u.prompt_tokens;
+    }
+    if (typeof u.completion_tokens === "number") {
+        out.completion_tokens = u.completion_tokens;
+    }
+    return Object.keys(out).length > 0 ? out : undefined;
+}