@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/services/kv-spill.d.ts

@@ -0,0 +1,189 @@
+/**
+ * CPU-offloaded KV-cache spill policy.
+ *
+ * packages/inference/AGENTS.md §3 item 7 mandates that for context > 64k on a
+ * device whose RAM cannot hold the full KV cache, the runtime MUST implement
+ * *spill* — keep the hot KV pages resident, page the cold ones out to CPU RAM
+ * (or, when even that is insufficient, to disk) — rather than refusing the
+ * request. AGENTS.md §3 "Failure handling" is equally explicit that the spill
+ * is gated by a real latency budget: a device where paging the cold KV back in
+ * would miss the voice first-audio-latency target must HARD-FAIL with a
+ * structured error, not silently serve a slow session.
+ *
+ * This module is the policy core. It is pure arithmetic — no llama-server
+ * process management, no native binding. `ffi-streaming-backend.ts` consults
+ * `planKvSpill()` at activation time:
+ *   - `mode: "resident"`  → no spill needed; load normally.
+ *   - `mode: "spill"`     → pass the resulting `residentPages` /
+ *                           `spillBytes` / tier ("cpu" | "disk") down to the
+ *                           backend as a `--kv-spill` hint.
+ *   - `mode: "unsupported"` → throw `KvSpillUnsupportedError` so the engine
+ *                           surfaces a structured 4xx to the UI.
+ *
+ * Model parameters (page size, per-page bandwidth, voice latency budget) are
+ * documented constants below — the only "measured" inputs are the device's
+ * memory bandwidth class and the KV geometry of the loaded bundle. We do not
+ * pretend to micro-benchmark the disk here; the bandwidth tiers are coarse
+ * and conservative, and the gate fails *closed*.
+ */
+import type { RamBudget } from "./types";
+/** Context length below which spill never applies (AGENTS.md §3 item 7). */
+export declare const KV_SPILL_MIN_CONTEXT = 65536;
+/**
+ * KV-cache page granularity, in tokens. The runtime evicts/restores KV in
+ * page units, not per-token, so spill accounting is page-aligned. 256 tokens
+ * is the buun-llama-cpp fork's default `--kv-page-size` for the spillable
+ * cache; keep this in sync if that default changes.
+ */
+export declare const KV_PAGE_TOKENS = 256;
+/**
+ * First-audio-latency budget for voice mode, in milliseconds. The streaming
+ * contract (AGENTS.md §4) wants the phrase chunker handing the first chunk to
+ * TTS inside a scheduler tick; a cold KV restore at decode time eats directly
+ * into this budget. If the worst-case restore for the spilled pages exceeds
+ * this, spill is not viable for a voice-enabled bundle and we hard-fail.
+ *
+ * Text-only bundles get the looser `KV_SPILL_TEXT_LATENCY_BUDGET_MS`.
+ */
+export declare const KV_SPILL_VOICE_LATENCY_BUDGET_MS = 200;
+export declare const KV_SPILL_TEXT_LATENCY_BUDGET_MS = 1500;
+/**
+ * Effective KV transfer bandwidth back into the attention kernel, by storage
+ * tier and host class, in bytes per millisecond (≈ GB/s). Conservative — the
+ * gate fails closed, so under-estimating bandwidth only makes us refuse more
+ * aggressively, never serve something too slow.
+ *
+ *   - `cpu`/`apple` : Apple Silicon shared memory — "spilling to CPU" is
+ *                     mostly an accounting move (same physical RAM, different
+ *                     residency bookkeeping); effective restore bandwidth is
+ *                     high.
+ *   - `cpu`/`pcie`  : discrete-GPU x86 — cold KV pages live in host RAM and
+ *                     ride the PCIe bus back to VRAM. PCIe 4.0 x16 ≈ 25 GB/s
+ *                     after framing; we budget 12.
+ *   - `disk`/`nvme` : NVMe SSD — sequential read ≈ 3 GB/s; we budget 1.5.
+ *   - `disk`/`sata` : SATA SSD / spinning rust fallback — ≈ 0.4 GB/s; we
+ *                     budget 0.25. (Mostly here so the math is defined; in
+ *                     practice this tier fails the gate immediately.)
+ */
+declare const KV_RESTORE_BANDWIDTH_BYTES_PER_MS: {
+    readonly "cpu-apple": 40000000;
+    readonly "cpu-pcie": 12000000;
+    readonly "disk-nvme": 1500000;
+    readonly "disk-sata": 250000;
+};
+export type KvRestoreClass = keyof typeof KV_RESTORE_BANDWIDTH_BYTES_PER_MS;
+/**
+ * Per-token KV-cache footprint of a loaded bundle, summed across all
+ * full-attention layers, for the *quantized* cache it actually ships with
+ * (QJL K + PolarQuant/TurboQuant V — see packages/training/AGENTS.md §3).
+ * Callers derive this from the bundle's manifest / catalog runtime block;
+ * `estimateQuantizedKvBytesPerToken()` is the fallback when only the param
+ * count is known.
+ */
+export interface KvGeometry {
+    /** Bytes of compressed KV the cache grows by, per generated token. */
+    bytesPerToken: number;
+    /** True when the loaded bundle has voice enabled (tighter latency gate). */
+    voiceEnabled: boolean;
+}
+export declare function estimateQuantizedKvBytesPerToken(params: string): number;
+/**
+ * Where the spilled pages land. `"cpu"` = host RAM (still RAM, just not
+ * counted against the resident budget); `"disk"` = the local-inference cache
+ * directory on persistent storage.
+ */
+export type KvSpillTier = "cpu" | "disk";
+export interface KvSpillPlanResident {
+    mode: "resident";
+    /** The whole KV cache fits in the resident budget; nothing spills. */
+    totalKvBytes: number;
+    residentBytes: number;
+}
+export interface KvSpillPlanSpill {
+    mode: "spill";
+    tier: KvSpillTier;
+    /** Pages kept resident (the hot tail of the context). */
+    residentPages: number;
+    /** Pages paged out to `tier`. */
+    spillPages: number;
+    /** Bytes of KV held resident. */
+    residentBytes: number;
+    /** Bytes of KV spilled to `tier`. */
+    spillBytes: number;
+    /** Total compressed KV footprint at full context. */
+    totalKvBytes: number;
+    /** Worst-case latency to restore one cold page, in ms. */
+    worstCaseRestoreMs: number;
+    /** The latency budget this plan was checked against, in ms. */
+    latencyBudgetMs: number;
+}
+export type KvSpillPlan = KvSpillPlanResident | KvSpillPlanSpill;
+/**
+ * Structured error thrown when spill cannot meet the latency budget. The
+ * engine catches this and surfaces it to the UI as a 4xx with `code` and
+ * `details` intact — there is NO silent-slow fallback (AGENTS.md §3).
+ */
+export declare class KvSpillUnsupportedError extends Error {
+    readonly code = "kv-spill-unsupported";
+    readonly details: {
+        requestedContext: number;
+        totalKvBytes: number;
+        residentBytes: number;
+        spillBytes: number;
+        worstCaseRestoreMs: number;
+        latencyBudgetMs: number;
+        restoreClass: KvRestoreClass;
+        voiceEnabled: boolean;
+    };
+    constructor(details: KvSpillUnsupportedError["details"]);
+}
+/**
+ * Inputs to `planKvSpill`. `residentKvBudgetBytes` is the slice of the RAM
+ * budget the runtime is willing to hand to the *resident* KV cache after
+ * weights + activations + the TTS/ASR working sets are accounted for; callers
+ * derive it from `RamBudget` via `residentKvBudgetFromRamBudget()`.
+ */
+export interface KvSpillInput {
+    requestedContext: number;
+    geometry: KvGeometry;
+    residentKvBudgetBytes: number;
+    restoreClass: KvRestoreClass;
+    /**
+     * True when the host can spill to CPU RAM (host RAM available beyond the
+     * resident budget). When false the spill tier degrades to `"disk"`.
+     */
+    cpuSpillAvailable: boolean;
+}
+/**
+ * Slice the resident-KV budget out of a model's `RamBudget`. The recommended
+ * budget covers weights + activations + voice working sets + KV; we reserve a
+ * fixed fraction for KV. This mirrors what `recommendation.ts` already assumes
+ * implicitly when it sizes tiers — kept as one constant so the spill policy
+ * and the recommender agree.
+ */
+export declare const RESIDENT_KV_BUDGET_FRACTION = 0.25;
+export declare function residentKvBudgetFromRamBudget(budget: RamBudget): number;
+/**
+ * Decide the KV-cache placement for a requested context.
+ *
+ * Returns `{ mode: "resident" }` when the whole compressed KV fits the
+ * resident budget; `{ mode: "spill", ... }` when it fits with paging and the
+ * cold-page restore stays inside the latency budget; throws
+ * `KvSpillUnsupportedError` when spill would miss the budget.
+ *
+ * Below `KV_SPILL_MIN_CONTEXT` this is always `{ mode: "resident" }` — there
+ * is no spill at short context, by contract.
+ */
+export declare function planKvSpill(input: KvSpillInput): KvSpillPlan;
+/**
+ * Map a `HardwareProbe`-shaped descriptor to the KV restore bandwidth class.
+ * Apple Silicon → unified-memory class; discrete-GPU x86 → PCIe class;
+ * CPU-only → NVMe class (no GPU to page back to, so "restore" is a host-RAM
+ * memcpy bounded by the same order as a fast SSD on the conservative side).
+ */
+export declare function restoreClassForHardware(input: {
+    appleSilicon: boolean;
+    hasDiscreteGpu: boolean;
+}): KvRestoreClass;
+export {};
+//# sourceMappingURL=kv-spill.d.ts.map

package/src/services/kv-spill.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"kv-spill.d.ts","sourceRoot":"","sources":["kv-spill.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAEzC,4EAA4E;AAC5E,eAAO,MAAM,oBAAoB,QAAQ,CAAC;AAE1C;;;;;GAKG;AACH,eAAO,MAAM,cAAc,MAAM,CAAC;AAElC;;;;;;;;GAQG;AACH,eAAO,MAAM,gCAAgC,MAAM,CAAC;AACpD,eAAO,MAAM,+BAA+B,OAAO,CAAC;AAEpD;;;;;;;;;;;;;;;;;GAiBG;AACH,QAAA,MAAM,iCAAiC;;;;;CAK7B,CAAC;AAEX,MAAM,MAAM,cAAc,GAAG,MAAM,OAAO,iCAAiC,CAAC;AAE5E;;;;;;;GAOG;AACH,MAAM,WAAW,UAAU;IAC1B,sEAAsE;IACtE,aAAa,EAAE,MAAM,CAAC;IACtB,4EAA4E;IAC5E,YAAY,EAAE,OAAO,CAAC;CACtB;AAmBD,wBAAgB,gCAAgC,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAOvE;AAED;;;;GAIG;AACH,MAAM,MAAM,WAAW,GAAG,KAAK,GAAG,MAAM,CAAC;AAEzC,MAAM,WAAW,mBAAmB;IACnC,IAAI,EAAE,UAAU,CAAC;IACjB,sEAAsE;IACtE,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAChC,IAAI,EAAE,OAAO,CAAC;IACd,IAAI,EAAE,WAAW,CAAC;IAClB,yDAAyD;IACzD,aAAa,EAAE,MAAM,CAAC;IACtB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,iCAAiC;IACjC,aAAa,EAAE,MAAM,CAAC;IACtB,qCAAqC;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,qDAAqD;IACrD,YAAY,EAAE,MAAM,CAAC;IACrB,0DAA0D;IAC1D,kBAAkB,EAAE,MAAM,CAAC;IAC3B,+DAA+D;IAC/D,eAAe,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,WAAW,GAAG,mBAAmB,GAAG,gBAAgB,CAAC;AAEjE;;;;GAIG;AACH,qBAAa,uBAAwB,SAAQ,KAAK;IACjD,QAAQ,CAAC,IAAI,0BAA0B;IACvC,QAAQ,CAAC,OAAO,EAAE;QACjB,gBAAgB,EAAE,MAAM,CAAC;QACzB,YAAY,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,CAAC;QACtB,UAAU,EAAE,MAAM,CAAC;QACnB,kBAAkB,EAAE,MAAM,CAAC;QAC3B,eAAe,EAAE,MAAM,CAAC;QACxB,YAAY,EAAE,cAAc,CAAC;QAC7B,YAAY,EAAE,OAAO,CAAC;KACtB,CAAC;gBAEU,OAAO,EAAE,uBAAuB,CAAC,SAAS,CAAC;CAgBvD;AAED;;;;;GAKG;AACH,MAAM,WAAW,YAAY;IAC5B,gBAAgB,EAAE,MAAM,CAAC;IACzB,QAAQ,EAAE,UAAU,CAAC;IACrB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,YAAY,EAAE,cAAc,CAAC;IAC7B;;;OAGG;IACH,iBAAiB,EAAE,OAAO,CAAC;CAC3B;AAED;;;;;;GAMG;AACH,eAAO,MAAM,2BAA2B,OAAO,CAAC;AAEhD,wBAAgB,6BAA6B,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,CAIvE;AAMD;;;;;;;;;;GAUG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,YAAY,GAAG,WAAW,CAmG5D;AAED;;;;;GAKG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE;IAC9C,YAAY,EAAE,OAAO,CAAC;IACtB,cAAc,EAAE,OAAO,CAAC;CACxB,GAAG,cAAc,CAIjB"}

package/src/services/kv-spill.test.ts

@@ -0,0 +1,222 @@
+import { describe, expect, it } from "vitest";
+import {
+	estimateQuantizedKvBytesPerToken,
+	KV_PAGE_TOKENS,
+	KV_SPILL_MIN_CONTEXT,
+	KV_SPILL_VOICE_LATENCY_BUDGET_MS,
+	type KvGeometry,
+	KvSpillUnsupportedError,
+	planKvSpill,
+	residentKvBudgetFromRamBudget,
+	restoreClassForHardware,
+} from "./kv-spill";
+import type { RamBudget } from "./ram-budget";
+
+const MIB = 1024 * 1024;
+
+function geometry(overrides: Partial<KvGeometry> = {}): KvGeometry {
+	return { bytesPerToken: 2_000, voiceEnabled: false, ...overrides };
+}
+
+describe("planKvSpill", () => {
+	it("returns resident when the whole compressed KV fits the budget", () => {
+		// 32k tokens * 2000 B/token = 64 MiB; budget 128 MiB.
+		const plan = planKvSpill({
+			requestedContext: 32_768,
+			geometry: geometry(),
+			residentKvBudgetBytes: 128 * MIB,
+			restoreClass: "cpu-pcie",
+			cpuSpillAvailable: true,
+		});
+		expect(plan.mode).toBe("resident");
+		if (plan.mode !== "resident") throw new Error("unreachable");
+		expect(plan.residentBytes).toBe(plan.totalKvBytes);
+	});
+
+	it("spills the cold pages on a long context that overruns the resident budget", () => {
+		// 256k tokens * 2000 B/token = 512 MiB; budget 64 MiB → most pages spill.
+		const plan = planKvSpill({
+			requestedContext: 262_144,
+			geometry: geometry(),
+			residentKvBudgetBytes: 64 * MIB,
+			restoreClass: "cpu-pcie",
+			cpuSpillAvailable: true,
+		});
+		expect(plan.mode).toBe("spill");
+		if (plan.mode !== "spill") throw new Error("unreachable");
+		expect(plan.tier).toBe("cpu");
+		expect(plan.spillPages).toBeGreaterThan(0);
+		expect(plan.residentPages).toBeGreaterThanOrEqual(1);
+		expect(plan.residentBytes + plan.spillBytes).toBe(plan.totalKvBytes);
+		// 256-token page * 2000 B = 512_000 B; PCIe budget 12_000_000 B/ms →
+		// ~0.043 ms per page restore, comfortably under any budget.
+		expect(plan.worstCaseRestoreMs).toBeLessThan(plan.latencyBudgetMs);
+		expect(plan.latencyBudgetMs).toBe(1_500);
+	});
+
+	it("uses the tighter voice budget when the bundle has voice enabled", () => {
+		const plan = planKvSpill({
+			requestedContext: 262_144,
+			geometry: geometry({ voiceEnabled: true }),
+			residentKvBudgetBytes: 64 * MIB,
+			restoreClass: "cpu-pcie",
+			cpuSpillAvailable: true,
+		});
+		expect(plan.mode).toBe("spill");
+		if (plan.mode !== "spill") throw new Error("unreachable");
+		expect(plan.latencyBudgetMs).toBe(KV_SPILL_VOICE_LATENCY_BUDGET_MS);
+	});
+
+	it("hard-fails with a structured error when cold-page restore misses the voice budget", () => {
+		// Big per-token KV + slow SATA disk → a single page restore blows the
+		// 200ms voice budget. Page = 256 * 200_000 B = 51.2 MB; SATA budget
+		// 250_000 B/ms → ~205 ms > 200 ms.
+		let thrown: unknown;
+		try {
+			planKvSpill({
+				requestedContext: 262_144,
+				geometry: geometry({ bytesPerToken: 200_000, voiceEnabled: true }),
+				residentKvBudgetBytes: 64 * MIB,
+				restoreClass: "disk-sata",
+				cpuSpillAvailable: false,
+			});
+		} catch (err) {
+			thrown = err;
+		}
+		expect(thrown).toBeInstanceOf(KvSpillUnsupportedError);
+		const e = thrown as KvSpillUnsupportedError;
+		expect(e.code).toBe("kv-spill-unsupported");
+		expect(e.details.voiceEnabled).toBe(true);
+		expect(e.details.restoreClass).toBe("disk-sata");
+		expect(e.details.worstCaseRestoreMs).toBeGreaterThan(
+			e.details.latencyBudgetMs,
+		);
+		expect(e.message).toContain("voice latency budget");
+	});
+
+	it("degrades the spill tier to disk and downgrades a cpu restore class when CPU spill is unavailable", () => {
+		const plan = planKvSpill({
+			requestedContext: 262_144,
+			geometry: geometry(),
+			residentKvBudgetBytes: 64 * MIB,
+			restoreClass: "cpu-pcie",
+			cpuSpillAvailable: false,
+		});
+		expect(plan.mode).toBe("spill");
+		if (plan.mode !== "spill") throw new Error("unreachable");
+		expect(plan.tier).toBe("disk");
+		// worstCaseRestoreMs reflects the NVMe class, not the cpu-pcie one.
+		expect(plan.worstCaseRestoreMs).toBeGreaterThan(
+			// page (512_000 B) / cpu-pcie (12e6) ≈ 0.043 ms
+			512_000 / 12_000_000,
+		);
+	});
+
+	it("page-aligns the resident/spill split", () => {
+		const plan = planKvSpill({
+			requestedContext: 200_000,
+			geometry: geometry(),
+			residentKvBudgetBytes: 30 * MIB,
+			restoreClass: "cpu-apple",
+			cpuSpillAvailable: true,
+		});
+		expect(plan.mode).toBe("spill");
+		if (plan.mode !== "spill") throw new Error("unreachable");
+		const pageBytes = 2_000 * KV_PAGE_TOKENS;
+		expect(plan.residentBytes % pageBytes).toBe(0);
+		expect(plan.spillBytes % pageBytes).toBe(0);
+		const totalPages = Math.ceil(200_000 / KV_PAGE_TOKENS);
+		expect(plan.residentPages + plan.spillPages).toBe(totalPages);
+	});
+
+	it("treats a short context that overruns the resident budget as unsupported (wrong tier for device)", () => {
+		// 64k - 1 tokens, but a per-token KV so large the cache won't fit and
+		// we're below KV_SPILL_MIN_CONTEXT → structured error, no half-load.
+		let thrown: unknown;
+		try {
+			planKvSpill({
+				requestedContext: KV_SPILL_MIN_CONTEXT - 1,
+				geometry: geometry({ bytesPerToken: 50_000 }),
+				residentKvBudgetBytes: 16 * MIB,
+				restoreClass: "cpu-apple",
+				cpuSpillAvailable: true,
+			});
+		} catch (err) {
+			thrown = err;
+		}
+		expect(thrown).toBeInstanceOf(KvSpillUnsupportedError);
+		expect((thrown as KvSpillUnsupportedError).details.requestedContext).toBe(
+			KV_SPILL_MIN_CONTEXT - 1,
+		);
+	});
+
+	it("rejects degenerate inputs loudly", () => {
+		expect(() =>
+			planKvSpill({
+				requestedContext: 0,
+				geometry: geometry(),
+				residentKvBudgetBytes: MIB,
+				restoreClass: "cpu-apple",
+				cpuSpillAvailable: true,
+			}),
+		).toThrow(/positive context/);
+		expect(() =>
+			planKvSpill({
+				requestedContext: 100_000,
+				geometry: geometry(),
+				residentKvBudgetBytes: 0,
+				restoreClass: "cpu-apple",
+				cpuSpillAvailable: true,
+			}),
+		).toThrow(/residentKvBudgetBytes must be positive/);
+	});
+});
+
+describe("residentKvBudgetFromRamBudget", () => {
+	it("reserves the documented KV fraction of the recommended budget", () => {
+		const budget: RamBudget = {
+			minMb: 7000,
+			recommendedMb: 9600,
+			source: "manifest",
+		};
+		// 9600 MiB * 0.25 = 2400 MiB
+		expect(residentKvBudgetFromRamBudget(budget)).toBe(2400 * MIB);
+	});
+});
+
+describe("estimateQuantizedKvBytesPerToken", () => {
+	it("returns the per-tier figure for known param strings", () => {
+		expect(estimateQuantizedKvBytesPerToken("0.8B")).toBeLessThan(
+			estimateQuantizedKvBytesPerToken("9B"),
+		);
+		expect(estimateQuantizedKvBytesPerToken("2B")).toBeGreaterThan(
+			estimateQuantizedKvBytesPerToken("0.8B"),
+		);
+		expect(estimateQuantizedKvBytesPerToken("4B")).toBeGreaterThan(
+			estimateQuantizedKvBytesPerToken("2B"),
+		);
+		expect(estimateQuantizedKvBytesPerToken("27B")).toBeGreaterThan(
+			estimateQuantizedKvBytesPerToken("9B"),
+		);
+	});
+
+	it("fails closed (largest tier) for unknown param strings", () => {
+		expect(estimateQuantizedKvBytesPerToken("999B")).toBe(
+			estimateQuantizedKvBytesPerToken("27B"),
+		);
+	});
+});
+
+describe("restoreClassForHardware", () => {
+	it("maps Apple Silicon, discrete GPU, and CPU-only to the right classes", () => {
+		expect(
+			restoreClassForHardware({ appleSilicon: true, hasDiscreteGpu: false }),
+		).toBe("cpu-apple");
+		expect(
+			restoreClassForHardware({ appleSilicon: false, hasDiscreteGpu: true }),
+		).toBe("cpu-pcie");
+		expect(
+			restoreClassForHardware({ appleSilicon: false, hasDiscreteGpu: false }),
+		).toBe("disk-nvme");
+	});
+});