@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/dist/services/inference-capabilities.d.ts

@@ -0,0 +1,132 @@
+/**
+ * Inference capability detection.
+ *
+ * Centralises "what does this device's local-inference stack expose"
+ * into one struct the runtime can read at startup.  The shape mirrors
+ * the per-platform binding probes (Android + iOS + desktop FFI) so the
+ * runtime doesn't have to import each platform's adapter just to
+ * surface the bits.
+ *
+ * Consumed by:
+ *   - the AOSP local-inference bootstrap, to choose the in-process FFI
+ *     streaming path,
+ *   - the desktop voice lifecycle service, to decide whether to wire the
+ *     FFI streaming runner factory,
+ *   - UI surfaces (model picker, voice toggle) that hide options the
+ *     loaded build cannot honour.
+ *
+ * Naming:
+ *   - `streamingLlm` — `eliza_inference_llm_stream_*` symbols are
+ *     resolved and the build reports `_supported() === 1`.
+ *   - `mtpSupported` — native MTP speculative decoding can actually run.
+ *   - `omnivoiceStreaming` — `eliza_inference_tts_synthesize_stream` is
+ *     present and supported.
+ *   - `mmprojSupported` — the build carries the multi-modal projector
+ *     and the device has the headroom to keep it resident.
+ *   - `thermalState` — best-effort current thermal snapshot from the
+ *     platform (`ProcessInfo.thermalState` on iOS,
+ *     `PowerManager.getCurrentThermalStatus` on Android).
+ *
+ * All fields are read-only snapshots; the runtime re-probes on resume.
+ */
+export type ThermalState = "nominal" | "fair" | "serious" | "critical";
+export interface InferenceCapabilities {
+    streamingLlm: boolean;
+    mtpSupported: boolean;
+    omnivoiceStreaming: boolean;
+    mmprojSupported: boolean;
+    thermalState: ThermalState;
+    /** Platform tag for diagnostics + routing. */
+    platform: "android" | "ios" | "desktop" | "unknown";
+}
+/** Minimal probe surface — what the caller hands in. */
+export interface CapabilityProbes {
+    /** True only when `eliza_inference_llm_stream_supported()` returns 1. */
+    llmStreamSupported(): boolean;
+    /** True only when `eliza_inference_tts_stream_supported()` returns 1. */
+    ttsStreamSupported(): boolean;
+    /** True only when the native MTP path is available for the loaded model. */
+    mtpResident(): boolean;
+    /** True only when the mmproj weights are present in the bundle. */
+    mmprojResident(): boolean;
+    /** Current thermal snapshot.  May return `nominal` on platforms without a thermal API. */
+    thermalState(): ThermalState;
+    /** Platform tag. */
+    platform(): "android" | "ios" | "desktop" | "unknown";
+}
+/**
+ * Build a capability struct from a set of probes.
+ *
+ * Policy decisions encoded here:
+ *   - Native MTP only fires when `llmStreamSupported` and the thermal state is
+ *     at most `fair`.
+ *   - mmproj is gated entirely on the bundle carrying it.  Devices
+ *     short on RAM can still load the chat model — they just lose the
+ *     vision path; the picker UI uses this bit to grey out vision
+ *     uploads.
+ *   - omnivoice streaming is gated entirely on the FFI build: the JS
+ *     side has no fallback path for streaming TTS, only for batch.
+ */
+export declare function probeCapabilities(probes: CapabilityProbes): InferenceCapabilities;
+/**
+ * Defaults probe: every flag off, platform `unknown`, thermal `nominal`.
+ * Used by the runtime when no FFI binding could be loaded (cloud-only
+ * fallback path).  Surfaces as a single struct the UI can render
+ * without branching on "no probe registered".
+ */
+export declare function defaultsForNoBinding(): InferenceCapabilities;
+/**
+ * A live, *sampled* device-resource snapshot from the native probe
+ * (iOS `getResourceSnapshot`, Android `ResourceProbe.getResourceSnapshot`) — as
+ * opposed to the one-shot `InferenceCapabilities` probe. The Mobile Resource
+ * Workbench (issue #8800) samples these on an interval across a sustained
+ * workload to build a thermal/RSS/battery timeline. Every numeric field is
+ * `null` when the platform could not measure it — never a fabricated zero.
+ */
+export interface ResourceSnapshot {
+    /** Current thermal state; `"unknown"` on platforms without a thermal API. */
+    thermalState: ThermalState | "unknown";
+    /** Whether the OS low-power / battery-saver mode is engaged, or null. */
+    lowPowerMode: boolean | null;
+    /** Process resident set size in MB, or null. */
+    residentMemoryMb: number | null;
+    /** Device-wide available RAM in MB, or null. */
+    availableRamMb: number | null;
+    /** Cumulative process CPU time in ms, or null. */
+    cpuTimeMs: number | null;
+    /** Battery level 0..100, or null. */
+    batteryLevelPct: number | null;
+    /** Sample timestamp in ms (epoch). */
+    capturedAtMs: number;
+}
+export interface ThermalThrottleDecision {
+    /**
+     * Whether speculative decoding (MTP) should be disabled for the next step.
+     * MTP burns extra compute for a latency win; under heat that trade flips.
+     */
+    throttleSpeculativeDecode: boolean;
+    /**
+     * Whether to proactively shed load (shrink batch / context, pause warmups)
+     * because the device is at the top of the thermal range.
+     */
+    reduceLoad: boolean;
+    reason: string;
+}
+/**
+ * Decide whether to throttle on-device inference for the current thermal /
+ * power state. Pure and synchronous so the streaming path can call it per token
+ * (the `ProcessInfo.thermalState` throttle hook the iOS streaming bridge still
+ * lists as a TODO) and the workbench can assert the policy without a device.
+ *
+ *   - `serious` / `critical` thermal → stop speculative decoding (matches the
+ *     existing one-shot MTP gate in `probeCapabilities`).
+ *   - `critical` thermal → additionally shed load.
+ *   - low-power mode → stop speculative decoding (honour the user's power intent).
+ *   - `unknown` thermal with no low-power signal → do not throttle (don't
+ *     penalise a device that simply lacks a thermal API).
+ */
+export declare function thermalThrottleDecision(input: {
+    thermalState: ThermalState | "unknown";
+    lowPowerMode?: boolean | null;
+}): ThermalThrottleDecision;
+//# sourceMappingURL=inference-capabilities.d.ts.map

package/dist/services/inference-capabilities.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"inference-capabilities.d.ts","sourceRoot":"","sources":["../../src/services/inference-capabilities.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,GAAG,UAAU,CAAC;AAEvE,MAAM,WAAW,qBAAqB;IACrC,YAAY,EAAE,OAAO,CAAC;IACtB,YAAY,EAAE,OAAO,CAAC;IACtB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,eAAe,EAAE,OAAO,CAAC;IACzB,YAAY,EAAE,YAAY,CAAC;IAC3B,8CAA8C;IAC9C,QAAQ,EAAE,SAAS,GAAG,KAAK,GAAG,SAAS,GAAG,SAAS,CAAC;CACpD;AAED,wDAAwD;AACxD,MAAM,WAAW,gBAAgB;IAChC,yEAAyE;IACzE,kBAAkB,IAAI,OAAO,CAAC;IAC9B,yEAAyE;IACzE,kBAAkB,IAAI,OAAO,CAAC;IAC9B,4EAA4E;IAC5E,WAAW,IAAI,OAAO,CAAC;IACvB,mEAAmE;IACnE,cAAc,IAAI,OAAO,CAAC;IAC1B,0FAA0F;IAC1F,YAAY,IAAI,YAAY,CAAC;IAC7B,oBAAoB;IACpB,QAAQ,IAAI,SAAS,GAAG,KAAK,GAAG,SAAS,GAAG,SAAS,CAAC;CACtD;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,iBAAiB,CAChC,MAAM,EAAE,gBAAgB,GACtB,qBAAqB,CAqBvB;AAED;;;;;GAKG;AACH,wBAAgB,oBAAoB,IAAI,qBAAqB,CAS5D;AAMD;;;;;;;GAOG;AACH,MAAM,WAAW,gBAAgB;IAChC,6EAA6E;IAC7E,YAAY,EAAE,YAAY,GAAG,SAAS,CAAC;IACvC,yEAAyE;IACzE,YAAY,EAAE,OAAO,GAAG,IAAI,CAAC;IAC7B,gDAAgD;IAChD,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,gDAAgD;IAChD,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,kDAAkD;IAClD,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,qCAAqC;IACrC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,sCAAsC;IACtC,YAAY,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,uBAAuB;IACvC;;;OAGG;IACH,yBAAyB,EAAE,OAAO,CAAC;IACnC;;;OAGG;IACH,UAAU,EAAE,OAAO,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;CACf;AASD;;;;;;;;;;;;GAYG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE;IAC9C,YAAY,EAAE,YAAY,GAAG,SAAS,CAAC;IACvC,YAAY,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;CAC9B,GAAG,uBAAuB,CAsB1B"}

package/dist/services/inference-telemetry.d.ts

@@ -0,0 +1,59 @@
+/**
+ * Lightweight structured-metrics sink for local-inference telemetry.
+ *
+ * This module is intentionally small: it is a logger-backed histogram with a
+ * stable, call-site-friendly `record(name, value, tags?)` API so individual
+ * backends (FFI runtime, voice scheduler, …) can emit point-in-time
+ * observations without importing a heavy metrics framework.
+ *
+ * Design constraints
+ * ------------------
+ * - No external dependencies beyond `@elizaos/core` logger (already a dep).
+ * - `record()` is synchronous and must never throw into the call site.
+ * - Each named metric keeps a bounded ring of recent samples (default 256) for
+ *   in-process p50/p95 queries via `summary(name)`. This mirrors the
+ *   `BoundedHistogram` pattern already used by `latency-trace.ts`.
+ * - The module exports a process-wide singleton (`inferenceTelemetry`) that
+ *   all backends share, plus the class for test injection.
+ *
+ * Metric names used today
+ * -----------------------
+ *   inference.ttfa_ms        — time from fetch() to first HTTP chunk (L5)
+ *   inference.first_token_ms — time from fetch() to first decoded token (L5)
+ *   tts.chunk_size_ms        — duration of one PCM chunk in ms (T2)
+ *   tts.chunk_size_bytes     — byte size of one PCM chunk (T2)
+ */
+interface HistogramSummary {
+    count: number;
+    p50: number | null;
+    p95: number | null;
+    mean: number | null;
+}
+export type TelemetryTags = Record<string, string | number | boolean>;
+export declare class InferenceTelemetry {
+    private readonly rings;
+    private readonly capacity;
+    constructor(capacity?: number);
+    /**
+     * Record a scalar observation. Never throws — telemetry must be
+     * instrumentation, never a fault path.
+     *
+     * @param name  Dot-separated metric name, e.g. `"inference.ttfa_ms"`.
+     * @param value Numeric value (non-finite values are silently dropped).
+     * @param tags  Optional key/value labels emitted in the log line.
+     */
+    record(name: string, value: number, tags?: TelemetryTags): void;
+    /**
+     * Summary statistics for a named metric over the retained ring of samples.
+     * Returns `{ count: 0, ... }` when the metric has never been recorded.
+     */
+    summary(name: string): HistogramSummary;
+    /** Names of all metrics that have received at least one sample. */
+    metricNames(): string[];
+    /** Reset all retained samples. Useful in tests. */
+    reset(): void;
+}
+/** Process-wide singleton used by FFI runtime and voice scheduler. */
+export declare const inferenceTelemetry: InferenceTelemetry;
+export {};
+//# sourceMappingURL=inference-telemetry.d.ts.map

package/dist/services/inference-telemetry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"inference-telemetry.d.ts","sourceRoot":"","sources":["../../src/services/inference-telemetry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAQH,UAAU,gBAAgB;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CACpB;AAwCD,MAAM,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,CAAC;AAEtE,qBAAa,kBAAkB;IAC9B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAkC;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;gBAEtB,QAAQ,SAAM;IAI1B;;;;;;;OAOG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,IAAI;IAoB/D;;;OAGG;IACH,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,gBAAgB;IAWvC,mEAAmE;IACnE,WAAW,IAAI,MAAM,EAAE;IAIvB,mDAAmD;IACnD,KAAK,IAAI,IAAI;CAGb;AAED,sEAAsE;AACtE,eAAO,MAAM,kBAAkB,oBAA2B,CAAC"}

package/dist/services/ios-llama-streaming.d.ts

@@ -0,0 +1,119 @@
+/**
+ * iOS streaming-LLM adapter.
+ *
+ * Mirror of the AOSP plugin's `aosp-llama-streaming.ts` for the iOS
+ * platform.  Bridges the JS-side streaming surface (the same shape the
+ * `FfiStreamingRunner` expects) over the `LlamaCpp.xcframework` Swift
+ * bridge that ships with the iOS shell.
+ *
+ * Status (2026-05-12): the XCFramework currently ships without streaming-LLM
+ * support. The Swift implementation against
+ * `libelizainference.dylib` (built by `build-llama-cpp-mtp.mjs` with
+ * the `darwin-arm64-metal-fused` target) is the gating item; until then
+ * `loadIosStreamingLlmBinding` returns `null` and the runtime falls
+ * back to the cloud route.
+ *
+ * Why XCFramework + Swift bridge rather than `bun:ffi` directly:
+ *   - bun:ffi runs only inside the Bun runtime.  On iOS, Eliza ships the
+ *     ElizaBunEngine.xcframework which embeds a Bun runtime — but the
+ *     llama symbols cannot be resolved at `bun:ffi.dlopen` time on a
+ *     codesigned device build (the dynamic linker forbids loading
+ *     arbitrary `.dylib` from the app bundle on a real device).
+ *   - The Swift bridge wraps the symbols at compile time inside the
+ *     XCFramework so the codesignature covers them.  Capacitor's
+ *     `llama-cpp-capacitor` plugin already uses this pattern for the
+ *     non-streaming surface.
+ *
+ * The Swift glue gating this binding needs to:
+ *   1. Re-export the streaming-LLM symbols from `ffi-streaming-llm.h`
+ *      under the Swift module name `LlamaCpp.Streaming.*`.
+ *   2. Expose an Objective-C-bridgeable wrapper class
+ *      (`LlamaStreamingSession`) so Capacitor's plugin can register
+ *      methods on it.
+ *   3. Wire `ProcessInfo.thermalState` into a thermal-throttle hook so
+ *      iOS can bail out of speculative decoding under sustained heat.
+ *
+ * Until that lands, this file:
+ *   - declares the JS contract,
+ *   - probes for `(window as any).LlamaStreaming` (the Capacitor plugin
+ *     entry point Swift will expose),
+ *   - logs + returns null when the bridge isn't there.
+ */
+export type IosLlmStreamHandle = bigint;
+export type IosInferenceContextHandle = bigint;
+export interface IosLlmStreamConfig {
+    maxTokens: number;
+    temperature: number;
+    topP: number;
+    topK: number;
+    repeatPenalty: number;
+    slotId: number;
+    promptCacheKey: string | null;
+    draftMin: number;
+    draftMax: number;
+    mtpDrafterPath: string | null;
+    disableThinking: boolean;
+}
+export interface IosLlmStreamStep {
+    tokens: number[];
+    text: string;
+    done: boolean;
+    drafterDrafted: number;
+    drafterAccepted: number;
+}
+export interface IosStreamingLlmBinding {
+    llmStreamSupported(): boolean;
+    llmStreamOpen(args: {
+        ctx: IosInferenceContextHandle;
+        config: IosLlmStreamConfig;
+    }): IosLlmStreamHandle;
+    llmStreamPrefill(args: {
+        stream: IosLlmStreamHandle;
+        tokens: Int32Array;
+    }): void;
+    llmStreamNext(args: {
+        stream: IosLlmStreamHandle;
+        maxTokensPerStep?: number;
+        maxTextBytes?: number;
+    }): IosLlmStreamStep;
+    llmStreamCancel(stream: IosLlmStreamHandle): void;
+    llmStreamSaveSlot(args: {
+        stream: IosLlmStreamHandle;
+        filename: string;
+    }): void;
+    llmStreamRestoreSlot(args: {
+        stream: IosLlmStreamHandle;
+        filename: string;
+    }): void;
+    llmStreamClose(stream: IosLlmStreamHandle): void;
+}
+/**
+ * Try to load the iOS streaming-LLM binding.  Returns null when:
+ *   - we are not on iOS,
+ *   - the Capacitor `LlamaStreaming` plugin isn't registered,
+ *   - the bridge reports `isAvailable() === false` (e.g. the XCFramework
+ *     was built without streaming-LLM support).
+ *
+ * Returning null is NOT a failure — the runtime then falls back to the
+ * cloud route.  Throws only when the bridge is present but
+ * mis-configured (probe-and-catch is reserved for the boot path).
+ */
+export declare function loadIosStreamingLlmBinding(): Promise<IosStreamingLlmBinding | null>;
+export interface IosInferenceCapabilities {
+    /** True only when the iOS Swift bridge is present AND reports streaming-LLM. */
+    streamingLlm: boolean;
+    /** Always false on iOS until the drafter weights ship in the bundle. */
+    mtpSupported: boolean;
+    /** Whether the XCFramework reports omnivoice streaming. */
+    omnivoiceStreaming: boolean;
+    /** Phone-tier iOS devices rarely have headroom for mmproj. */
+    mmprojSupported: boolean;
+    /**
+     * `ProcessInfo.thermalState` snapshot at probe time.  Surfaced so the
+     * runtime can refuse to start speculative decoding when the device is
+     * already in `serious` / `critical`.  Always `nominal` on the
+     * web/sim fallback path.
+     */
+    thermalState: "nominal" | "fair" | "serious" | "critical";
+}
+//# sourceMappingURL=ios-llama-streaming.d.ts.map

package/dist/services/ios-llama-streaming.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ios-llama-streaming.d.ts","sourceRoot":"","sources":["../../src/services/ios-llama-streaming.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AASH,MAAM,MAAM,kBAAkB,GAAG,MAAM,CAAC;AACxC,MAAM,MAAM,yBAAyB,GAAG,MAAM,CAAC;AAE/C,MAAM,WAAW,kBAAkB;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,eAAe,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,WAAW,gBAAgB;IAChC,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,OAAO,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,sBAAsB;IACtC,kBAAkB,IAAI,OAAO,CAAC;IAC9B,aAAa,CAAC,IAAI,EAAE;QACnB,GAAG,EAAE,yBAAyB,CAAC;QAC/B,MAAM,EAAE,kBAAkB,CAAC;KAC3B,GAAG,kBAAkB,CAAC;IACvB,gBAAgB,CAAC,IAAI,EAAE;QACtB,MAAM,EAAE,kBAAkB,CAAC;QAC3B,MAAM,EAAE,UAAU,CAAC;KACnB,GAAG,IAAI,CAAC;IACT,aAAa,CAAC,IAAI,EAAE;QACnB,MAAM,EAAE,kBAAkB,CAAC;QAC3B,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;KACtB,GAAG,gBAAgB,CAAC;IACrB,eAAe,CAAC,MAAM,EAAE,kBAAkB,GAAG,IAAI,CAAC;IAClD,iBAAiB,CAAC,IAAI,EAAE;QACvB,MAAM,EAAE,kBAAkB,CAAC;QAC3B,QAAQ,EAAE,MAAM,CAAC;KACjB,GAAG,IAAI,CAAC;IACT,oBAAoB,CAAC,IAAI,EAAE;QAC1B,MAAM,EAAE,kBAAkB,CAAC;QAC3B,QAAQ,EAAE,MAAM,CAAC;KACjB,GAAG,IAAI,CAAC;IACT,cAAc,CAAC,MAAM,EAAE,kBAAkB,GAAG,IAAI,CAAC;CACjD;AA8DD;;;;;;;;;;GAUG;AACH,wBAAsB,0BAA0B,IAAI,OAAO,CAAC,sBAAsB,GAAG,IAAI,CAAC,CAuBzF;AAoCD,MAAM,WAAW,wBAAwB;IACxC,gFAAgF;IAChF,YAAY,EAAE,OAAO,CAAC;IACtB,wEAAwE;IACxE,YAAY,EAAE,OAAO,CAAC;IACtB,2DAA2D;IAC3D,kBAAkB,EAAE,OAAO,CAAC;IAC5B,8DAA8D;IAC9D,eAAe,EAAE,OAAO,CAAC;IACzB;;;;;OAKG;IACH,YAAY,EAAE,SAAS,GAAG,MAAM,GAAG,SAAS,GAAG,UAAU,CAAC;CAC1D"}

package/dist/services/kv-spill.d.ts

@@ -0,0 +1,189 @@
+/**
+ * CPU-offloaded KV-cache spill policy.
+ *
+ * packages/inference/AGENTS.md §3 item 7 mandates that for context > 64k on a
+ * device whose RAM cannot hold the full KV cache, the runtime MUST implement
+ * *spill* — keep the hot KV pages resident, page the cold ones out to CPU RAM
+ * (or, when even that is insufficient, to disk) — rather than refusing the
+ * request. AGENTS.md §3 "Failure handling" is equally explicit that the spill
+ * is gated by a real latency budget: a device where paging the cold KV back in
+ * would miss the voice first-audio-latency target must HARD-FAIL with a
+ * structured error, not silently serve a slow session.
+ *
+ * This module is the policy core. It is pure arithmetic — no llama-server
+ * process management, no native binding. `ffi-streaming-backend.ts` consults
+ * `planKvSpill()` at activation time:
+ *   - `mode: "resident"`  → no spill needed; load normally.
+ *   - `mode: "spill"`     → pass the resulting `residentPages` /
+ *                           `spillBytes` / tier ("cpu" | "disk") down to the
+ *                           backend as a `--kv-spill` hint.
+ *   - `mode: "unsupported"` → throw `KvSpillUnsupportedError` so the engine
+ *                           surfaces a structured 4xx to the UI.
+ *
+ * Model parameters (page size, per-page bandwidth, voice latency budget) are
+ * documented constants below — the only "measured" inputs are the device's
+ * memory bandwidth class and the KV geometry of the loaded bundle. We do not
+ * pretend to micro-benchmark the disk here; the bandwidth tiers are coarse
+ * and conservative, and the gate fails *closed*.
+ */
+import type { RamBudget } from "./types";
+/** Context length below which spill never applies (AGENTS.md §3 item 7). */
+export declare const KV_SPILL_MIN_CONTEXT = 65536;
+/**
+ * KV-cache page granularity, in tokens. The runtime evicts/restores KV in
+ * page units, not per-token, so spill accounting is page-aligned. 256 tokens
+ * is the buun-llama-cpp fork's default `--kv-page-size` for the spillable
+ * cache; keep this in sync if that default changes.
+ */
+export declare const KV_PAGE_TOKENS = 256;
+/**
+ * First-audio-latency budget for voice mode, in milliseconds. The streaming
+ * contract (AGENTS.md §4) wants the phrase chunker handing the first chunk to
+ * TTS inside a scheduler tick; a cold KV restore at decode time eats directly
+ * into this budget. If the worst-case restore for the spilled pages exceeds
+ * this, spill is not viable for a voice-enabled bundle and we hard-fail.
+ *
+ * Text-only bundles get the looser `KV_SPILL_TEXT_LATENCY_BUDGET_MS`.
+ */
+export declare const KV_SPILL_VOICE_LATENCY_BUDGET_MS = 200;
+export declare const KV_SPILL_TEXT_LATENCY_BUDGET_MS = 1500;
+/**
+ * Effective KV transfer bandwidth back into the attention kernel, by storage
+ * tier and host class, in bytes per millisecond (≈ GB/s). Conservative — the
+ * gate fails closed, so under-estimating bandwidth only makes us refuse more
+ * aggressively, never serve something too slow.
+ *
+ *   - `cpu`/`apple` : Apple Silicon shared memory — "spilling to CPU" is
+ *                     mostly an accounting move (same physical RAM, different
+ *                     residency bookkeeping); effective restore bandwidth is
+ *                     high.
+ *   - `cpu`/`pcie`  : discrete-GPU x86 — cold KV pages live in host RAM and
+ *                     ride the PCIe bus back to VRAM. PCIe 4.0 x16 ≈ 25 GB/s
+ *                     after framing; we budget 12.
+ *   - `disk`/`nvme` : NVMe SSD — sequential read ≈ 3 GB/s; we budget 1.5.
+ *   - `disk`/`sata` : SATA SSD / spinning rust fallback — ≈ 0.4 GB/s; we
+ *                     budget 0.25. (Mostly here so the math is defined; in
+ *                     practice this tier fails the gate immediately.)
+ */
+declare const KV_RESTORE_BANDWIDTH_BYTES_PER_MS: {
+    readonly "cpu-apple": 40000000;
+    readonly "cpu-pcie": 12000000;
+    readonly "disk-nvme": 1500000;
+    readonly "disk-sata": 250000;
+};
+export type KvRestoreClass = keyof typeof KV_RESTORE_BANDWIDTH_BYTES_PER_MS;
+/**
+ * Per-token KV-cache footprint of a loaded bundle, summed across all
+ * full-attention layers, for the *quantized* cache it actually ships with
+ * (QJL K + PolarQuant/TurboQuant V — see packages/training/AGENTS.md §3).
+ * Callers derive this from the bundle's manifest / catalog runtime block;
+ * `estimateQuantizedKvBytesPerToken()` is the fallback when only the param
+ * count is known.
+ */
+export interface KvGeometry {
+    /** Bytes of compressed KV the cache grows by, per generated token. */
+    bytesPerToken: number;
+    /** True when the loaded bundle has voice enabled (tighter latency gate). */
+    voiceEnabled: boolean;
+}
+export declare function estimateQuantizedKvBytesPerToken(params: string): number;
+/**
+ * Where the spilled pages land. `"cpu"` = host RAM (still RAM, just not
+ * counted against the resident budget); `"disk"` = the local-inference cache
+ * directory on persistent storage.
+ */
+export type KvSpillTier = "cpu" | "disk";
+export interface KvSpillPlanResident {
+    mode: "resident";
+    /** The whole KV cache fits in the resident budget; nothing spills. */
+    totalKvBytes: number;
+    residentBytes: number;
+}
+export interface KvSpillPlanSpill {
+    mode: "spill";
+    tier: KvSpillTier;
+    /** Pages kept resident (the hot tail of the context). */
+    residentPages: number;
+    /** Pages paged out to `tier`. */
+    spillPages: number;
+    /** Bytes of KV held resident. */
+    residentBytes: number;
+    /** Bytes of KV spilled to `tier`. */
+    spillBytes: number;
+    /** Total compressed KV footprint at full context. */
+    totalKvBytes: number;
+    /** Worst-case latency to restore one cold page, in ms. */
+    worstCaseRestoreMs: number;
+    /** The latency budget this plan was checked against, in ms. */
+    latencyBudgetMs: number;
+}
+export type KvSpillPlan = KvSpillPlanResident | KvSpillPlanSpill;
+/**
+ * Structured error thrown when spill cannot meet the latency budget. The
+ * engine catches this and surfaces it to the UI as a 4xx with `code` and
+ * `details` intact — there is NO silent-slow fallback (AGENTS.md §3).
+ */
+export declare class KvSpillUnsupportedError extends Error {
+    readonly code = "kv-spill-unsupported";
+    readonly details: {
+        requestedContext: number;
+        totalKvBytes: number;
+        residentBytes: number;
+        spillBytes: number;
+        worstCaseRestoreMs: number;
+        latencyBudgetMs: number;
+        restoreClass: KvRestoreClass;
+        voiceEnabled: boolean;
+    };
+    constructor(details: KvSpillUnsupportedError["details"]);
+}
+/**
+ * Inputs to `planKvSpill`. `residentKvBudgetBytes` is the slice of the RAM
+ * budget the runtime is willing to hand to the *resident* KV cache after
+ * weights + activations + the TTS/ASR working sets are accounted for; callers
+ * derive it from `RamBudget` via `residentKvBudgetFromRamBudget()`.
+ */
+export interface KvSpillInput {
+    requestedContext: number;
+    geometry: KvGeometry;
+    residentKvBudgetBytes: number;
+    restoreClass: KvRestoreClass;
+    /**
+     * True when the host can spill to CPU RAM (host RAM available beyond the
+     * resident budget). When false the spill tier degrades to `"disk"`.
+     */
+    cpuSpillAvailable: boolean;
+}
+/**
+ * Slice the resident-KV budget out of a model's `RamBudget`. The recommended
+ * budget covers weights + activations + voice working sets + KV; we reserve a
+ * fixed fraction for KV. This mirrors what `recommendation.ts` already assumes
+ * implicitly when it sizes tiers — kept as one constant so the spill policy
+ * and the recommender agree.
+ */
+export declare const RESIDENT_KV_BUDGET_FRACTION = 0.25;
+export declare function residentKvBudgetFromRamBudget(budget: RamBudget): number;
+/**
+ * Decide the KV-cache placement for a requested context.
+ *
+ * Returns `{ mode: "resident" }` when the whole compressed KV fits the
+ * resident budget; `{ mode: "spill", ... }` when it fits with paging and the
+ * cold-page restore stays inside the latency budget; throws
+ * `KvSpillUnsupportedError` when spill would miss the budget.
+ *
+ * Below `KV_SPILL_MIN_CONTEXT` this is always `{ mode: "resident" }` — there
+ * is no spill at short context, by contract.
+ */
+export declare function planKvSpill(input: KvSpillInput): KvSpillPlan;
+/**
+ * Map a `HardwareProbe`-shaped descriptor to the KV restore bandwidth class.
+ * Apple Silicon → unified-memory class; discrete-GPU x86 → PCIe class;
+ * CPU-only → NVMe class (no GPU to page back to, so "restore" is a host-RAM
+ * memcpy bounded by the same order as a fast SSD on the conservative side).
+ */
+export declare function restoreClassForHardware(input: {
+    appleSilicon: boolean;
+    hasDiscreteGpu: boolean;
+}): KvRestoreClass;
+export {};
+//# sourceMappingURL=kv-spill.d.ts.map

package/dist/services/kv-spill.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"kv-spill.d.ts","sourceRoot":"","sources":["../../src/services/kv-spill.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAEzC,4EAA4E;AAC5E,eAAO,MAAM,oBAAoB,QAAQ,CAAC;AAE1C;;;;;GAKG;AACH,eAAO,MAAM,cAAc,MAAM,CAAC;AAElC;;;;;;;;GAQG;AACH,eAAO,MAAM,gCAAgC,MAAM,CAAC;AACpD,eAAO,MAAM,+BAA+B,OAAO,CAAC;AAEpD;;;;;;;;;;;;;;;;;GAiBG;AACH,QAAA,MAAM,iCAAiC;;;;;CAK7B,CAAC;AAEX,MAAM,MAAM,cAAc,GAAG,MAAM,OAAO,iCAAiC,CAAC;AAE5E;;;;;;;GAOG;AACH,MAAM,WAAW,UAAU;IAC1B,sEAAsE;IACtE,aAAa,EAAE,MAAM,CAAC;IACtB,4EAA4E;IAC5E,YAAY,EAAE,OAAO,CAAC;CACtB;AAoBD,wBAAgB,gCAAgC,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAOvE;AAED;;;;GAIG;AACH,MAAM,MAAM,WAAW,GAAG,KAAK,GAAG,MAAM,CAAC;AAEzC,MAAM,WAAW,mBAAmB;IACnC,IAAI,EAAE,UAAU,CAAC;IACjB,sEAAsE;IACtE,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAChC,IAAI,EAAE,OAAO,CAAC;IACd,IAAI,EAAE,WAAW,CAAC;IAClB,yDAAyD;IACzD,aAAa,EAAE,MAAM,CAAC;IACtB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,iCAAiC;IACjC,aAAa,EAAE,MAAM,CAAC;IACtB,qCAAqC;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,qDAAqD;IACrD,YAAY,EAAE,MAAM,CAAC;IACrB,0DAA0D;IAC1D,kBAAkB,EAAE,MAAM,CAAC;IAC3B,+DAA+D;IAC/D,eAAe,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,WAAW,GAAG,mBAAmB,GAAG,gBAAgB,CAAC;AAEjE;;;;GAIG;AACH,qBAAa,uBAAwB,SAAQ,KAAK;IACjD,QAAQ,CAAC,IAAI,0BAA0B;IACvC,QAAQ,CAAC,OAAO,EAAE;QACjB,gBAAgB,EAAE,MAAM,CAAC;QACzB,YAAY,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,CAAC;QACtB,UAAU,EAAE,MAAM,CAAC;QACnB,kBAAkB,EAAE,MAAM,CAAC;QAC3B,eAAe,EAAE,MAAM,CAAC;QACxB,YAAY,EAAE,cAAc,CAAC;QAC7B,YAAY,EAAE,OAAO,CAAC;KACtB,CAAC;gBAEU,OAAO,EAAE,uBAAuB,CAAC,SAAS,CAAC;CAgBvD;AAED;;;;;GAKG;AACH,MAAM,WAAW,YAAY;IAC5B,gBAAgB,EAAE,MAAM,CAAC;IACzB,QAAQ,EAAE,UAAU,CAAC;IACrB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,YAAY,EAAE,cAAc,CAAC;IAC7B;;;OAGG;IACH,iBAAiB,EAAE,OAAO,CAAC;CAC3B;AAED;;;;;;GAMG;AACH,eAAO,MAAM,2BAA2B,OAAO,CAAC;AAEhD,wBAAgB,6BAA6B,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,CAIvE;AAMD;;;;;;;;;;GAUG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,YAAY,GAAG,WAAW,CAmG5D;AAED;;;;;GAKG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE;IAC9C,YAAY,EAAE,OAAO,CAAC;IACtB,cAAc,EAAE,OAAO,CAAC;CACxB,GAAG,cAAc,CAIjB"}