@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/ios-llama-streaming.ts

@@ -0,0 +1,248 @@
+/**
+ * iOS streaming-LLM adapter.
+ *
+ * Mirror of the AOSP plugin's `aosp-llama-streaming.ts` for the iOS
+ * platform.  Bridges the JS-side streaming surface (the same shape the
+ * `FfiStreamingRunner` expects) over the `LlamaCpp.xcframework` Swift
+ * bridge that ships with the iOS shell.
+ *
+ * Status (2026-05-12): the XCFramework currently ships without streaming-LLM
+ * support. The Swift implementation against
+ * `libelizainference.dylib` (built by `build-llama-cpp-mtp.mjs` with
+ * the `darwin-arm64-metal-fused` target) is the gating item; until then
+ * `loadIosStreamingLlmBinding` returns `null` and the runtime falls
+ * back to the cloud route.
+ *
+ * Why XCFramework + Swift bridge rather than `bun:ffi` directly:
+ *   - bun:ffi runs only inside the Bun runtime.  On iOS, Eliza ships the
+ *     ElizaBunEngine.xcframework which embeds a Bun runtime — but the
+ *     llama symbols cannot be resolved at `bun:ffi.dlopen` time on a
+ *     codesigned device build (the dynamic linker forbids loading
+ *     arbitrary `.dylib` from the app bundle on a real device).
+ *   - The Swift bridge wraps the symbols at compile time inside the
+ *     XCFramework so the codesignature covers them.  Capacitor's
+ *     `llama-cpp-capacitor` plugin already uses this pattern for the
+ *     non-streaming surface.
+ *
+ * The Swift glue gating this binding needs to:
+ *   1. Re-export the streaming-LLM symbols from `ffi-streaming-llm.h`
+ *      under the Swift module name `LlamaCpp.Streaming.*`.
+ *   2. Expose an Objective-C-bridgeable wrapper class
+ *      (`LlamaStreamingSession`) so Capacitor's plugin can register
+ *      methods on it.
+ *   3. Wire `ProcessInfo.thermalState` into a thermal-throttle hook so
+ *      iOS can bail out of speculative decoding under sustained heat.
+ *
+ * Until that lands, this file:
+ *   - declares the JS contract,
+ *   - probes for `(window as any).LlamaStreaming` (the Capacitor plugin
+ *     entry point Swift will expose),
+ *   - logs + returns null when the bridge isn't there.
+ */
+
+import { logger } from "@elizaos/core";
+
+/* -------------------------------------------------------------------- */
+/* Public types — identical shape to the AOSP binding so the runner     */
+/* doesn't know or care which platform it's on.                         */
+/* -------------------------------------------------------------------- */
+
+export type IosLlmStreamHandle = bigint;
+export type IosInferenceContextHandle = bigint;
+
+export interface IosLlmStreamConfig {
+	maxTokens: number;
+	temperature: number;
+	topP: number;
+	topK: number;
+	repeatPenalty: number;
+	slotId: number;
+	promptCacheKey: string | null;
+	draftMin: number;
+	draftMax: number;
+	mtpDrafterPath: string | null;
+	disableThinking: boolean;
+}
+
+export interface IosLlmStreamStep {
+	tokens: number[];
+	text: string;
+	done: boolean;
+	drafterDrafted: number;
+	drafterAccepted: number;
+}
+
+export interface IosStreamingLlmBinding {
+	llmStreamSupported(): boolean;
+	llmStreamOpen(args: {
+		ctx: IosInferenceContextHandle;
+		config: IosLlmStreamConfig;
+	}): IosLlmStreamHandle;
+	llmStreamPrefill(args: {
+		stream: IosLlmStreamHandle;
+		tokens: Int32Array;
+	}): void;
+	llmStreamNext(args: {
+		stream: IosLlmStreamHandle;
+		maxTokensPerStep?: number;
+		maxTextBytes?: number;
+	}): IosLlmStreamStep;
+	llmStreamCancel(stream: IosLlmStreamHandle): void;
+	llmStreamSaveSlot(args: {
+		stream: IosLlmStreamHandle;
+		filename: string;
+	}): void;
+	llmStreamRestoreSlot(args: {
+		stream: IosLlmStreamHandle;
+		filename: string;
+	}): void;
+	llmStreamClose(stream: IosLlmStreamHandle): void;
+}
+
+/* -------------------------------------------------------------------- */
+/* Capacitor bridge contract                                            */
+/*                                                                      */
+/* Shape the Swift side must expose.  Capacitor exposes plugin methods  */
+/* via `window.Capacitor.Plugins.LlamaStreaming` once the bridge is     */
+/* registered.  Each method is a Promise on the JS side (Capacitor      */
+/* serialises across the JS↔native bridge), so the synchronous          */
+/* `IosStreamingLlmBinding` shape is built by `loadIosStreamingLlmBinding`*/
+/* over a small adapter — but the underlying bridge is async.           */
+/* -------------------------------------------------------------------- */
+
+interface CapacitorLlamaStreamingPlugin {
+	isAvailable(): Promise<{ available: boolean }>;
+	open(args: {
+		ctxHandle: string; // bigint serialised as string over the bridge
+		config: IosLlmStreamConfig;
+	}): Promise<{ streamHandle: string }>;
+	prefill(args: { streamHandle: string; tokens: number[] }): Promise<void>;
+	/**
+	 * Iteration is event-based on the native side — Capacitor plugin
+	 * `addListener("llmStreamStep", cb)` fires for each step.  The JS
+	 * binding shape we want is synchronous-ish for symmetry with bun:ffi;
+	 * the adapter in `buildIosBinding`
+	 * will turn the listener stream into a `next()`-like queue.
+	 */
+	cancel(args: { streamHandle: string }): Promise<void>;
+	saveSlot(args: { streamHandle: string; filename: string }): Promise<void>;
+	restoreSlot(args: { streamHandle: string; filename: string }): Promise<void>;
+	close(args: { streamHandle: string }): Promise<void>;
+}
+
+interface CapacitorWindow {
+	Capacitor?: {
+		Plugins?: {
+			LlamaStreaming?: CapacitorLlamaStreamingPlugin;
+		};
+		isNativePlatform?(): boolean;
+		getPlatform?(): string;
+	};
+}
+
+function tryGetCapacitorPlugin(): CapacitorLlamaStreamingPlugin | null {
+	const w = globalThis as CapacitorWindow;
+	const plugin = w.Capacitor?.Plugins?.LlamaStreaming;
+	if (!plugin) return null;
+	if (
+		typeof plugin.open !== "function" ||
+		typeof plugin.prefill !== "function" ||
+		typeof plugin.cancel !== "function" ||
+		typeof plugin.close !== "function"
+	) {
+		return null;
+	}
+	return plugin;
+}
+
+/* -------------------------------------------------------------------- */
+/* Loader                                                               */
+/* -------------------------------------------------------------------- */
+
+/**
+ * Try to load the iOS streaming-LLM binding.  Returns null when:
+ *   - we are not on iOS,
+ *   - the Capacitor `LlamaStreaming` plugin isn't registered,
+ *   - the bridge reports `isAvailable() === false` (e.g. the XCFramework
+ *     was built without streaming-LLM support).
+ *
+ * Returning null is NOT a failure — the runtime then falls back to the
+ * cloud route.  Throws only when the bridge is present but
+ * mis-configured (probe-and-catch is reserved for the boot path).
+ */
+export async function loadIosStreamingLlmBinding(): Promise<IosStreamingLlmBinding | null> {
+	const plugin = tryGetCapacitorPlugin();
+	if (!plugin) {
+		logger.info(
+			"[ios-llama-streaming] No Capacitor LlamaStreaming plugin registered. " +
+				"iOS streaming-LLM unavailable.",
+		);
+		return null;
+	}
+	const probe = await plugin.isAvailable().catch((err: unknown) => {
+		logger.warn(
+			`[ios-llama-streaming] isAvailable probe threw: ${formatError(err)}`,
+		);
+		return { available: false };
+	});
+	if (!probe.available) {
+		logger.info(
+			"[ios-llama-streaming] LlamaCpp.xcframework reports streaming-LLM " +
+				"unsupported (shim build).  See docs/eliza-1-ios-streaming-status.md.",
+		);
+		return null;
+	}
+	return buildIosBinding(plugin);
+}
+
+/**
+ * Wrap the async Capacitor plugin in the synchronous-ish JS contract
+ * `FfiStreamingRunner` expects.  Internally this means buffering
+ * `llmStreamStep` events from the Capacitor listener bus into a JS
+ * queue keyed by stream handle, then having `llmStreamNext` block on a
+ * `Promise<step>`.
+ *
+ * Unavailable adapter body. Until the Swift bridge reports streaming support,
+ * this code path is unreachable because the loader returns null first. The
+ * shape is here so app-core wiring compiles against the platform contract.
+ */
+function buildIosBinding(
+	_plugin: CapacitorLlamaStreamingPlugin,
+): IosStreamingLlmBinding {
+	throw new Error(
+		"[ios-llama-streaming] buildIosBinding is unavailable. " +
+			"The Swift bridge (LlamaStreaming Capacitor plugin) is not wired. " +
+			"See docs/eliza-1-ios-streaming-status.md for the rollout plan.",
+	);
+}
+
+function formatError(err: unknown): string {
+	if (err instanceof Error) return err.message;
+	try {
+		return JSON.stringify(err);
+	} catch {
+		return String(err);
+	}
+}
+
+/* -------------------------------------------------------------------- */
+/* Capability probe                                                     */
+/* -------------------------------------------------------------------- */
+
+export interface IosInferenceCapabilities {
+	/** True only when the iOS Swift bridge is present AND reports streaming-LLM. */
+	streamingLlm: boolean;
+	/** Always false on iOS until the drafter weights ship in the bundle. */
+	mtpSupported: boolean;
+	/** Whether the XCFramework reports omnivoice streaming. */
+	omnivoiceStreaming: boolean;
+	/** Phone-tier iOS devices rarely have headroom for mmproj. */
+	mmprojSupported: boolean;
+	/**
+	 * `ProcessInfo.thermalState` snapshot at probe time.  Surfaced so the
+	 * runtime can refuse to start speculative decoding when the device is
+	 * already in `serious` / `critical`.  Always `nominal` on the
+	 * web/sim fallback path.
+	 */
+	thermalState: "nominal" | "fair" | "serious" | "critical";
+}

package/src/services/kv-spill.test.ts

@@ -0,0 +1,222 @@
+import { describe, expect, it } from "vitest";
+import {
+	estimateQuantizedKvBytesPerToken,
+	KV_PAGE_TOKENS,
+	KV_SPILL_MIN_CONTEXT,
+	KV_SPILL_VOICE_LATENCY_BUDGET_MS,
+	type KvGeometry,
+	KvSpillUnsupportedError,
+	planKvSpill,
+	residentKvBudgetFromRamBudget,
+	restoreClassForHardware,
+} from "./kv-spill";
+import type { RamBudget } from "./ram-budget";
+
+const MIB = 1024 * 1024;
+
+function geometry(overrides: Partial<KvGeometry> = {}): KvGeometry {
+	return { bytesPerToken: 2_000, voiceEnabled: false, ...overrides };
+}
+
+describe("planKvSpill", () => {
+	it("returns resident when the whole compressed KV fits the budget", () => {
+		// 32k tokens * 2000 B/token = 64 MiB; budget 128 MiB.
+		const plan = planKvSpill({
+			requestedContext: 32_768,
+			geometry: geometry(),
+			residentKvBudgetBytes: 128 * MIB,
+			restoreClass: "cpu-pcie",
+			cpuSpillAvailable: true,
+		});
+		expect(plan.mode).toBe("resident");
+		if (plan.mode !== "resident") throw new Error("unreachable");
+		expect(plan.residentBytes).toBe(plan.totalKvBytes);
+	});
+
+	it("spills the cold pages on a long context that overruns the resident budget", () => {
+		// 256k tokens * 2000 B/token = 512 MiB; budget 64 MiB → most pages spill.
+		const plan = planKvSpill({
+			requestedContext: 262_144,
+			geometry: geometry(),
+			residentKvBudgetBytes: 64 * MIB,
+			restoreClass: "cpu-pcie",
+			cpuSpillAvailable: true,
+		});
+		expect(plan.mode).toBe("spill");
+		if (plan.mode !== "spill") throw new Error("unreachable");
+		expect(plan.tier).toBe("cpu");
+		expect(plan.spillPages).toBeGreaterThan(0);
+		expect(plan.residentPages).toBeGreaterThanOrEqual(1);
+		expect(plan.residentBytes + plan.spillBytes).toBe(plan.totalKvBytes);
+		// 256-token page * 2000 B = 512_000 B; PCIe budget 12_000_000 B/ms →
+		// ~0.043 ms per page restore, comfortably under any budget.
+		expect(plan.worstCaseRestoreMs).toBeLessThan(plan.latencyBudgetMs);
+		expect(plan.latencyBudgetMs).toBe(1_500);
+	});
+
+	it("uses the tighter voice budget when the bundle has voice enabled", () => {
+		const plan = planKvSpill({
+			requestedContext: 262_144,
+			geometry: geometry({ voiceEnabled: true }),
+			residentKvBudgetBytes: 64 * MIB,
+			restoreClass: "cpu-pcie",
+			cpuSpillAvailable: true,
+		});
+		expect(plan.mode).toBe("spill");
+		if (plan.mode !== "spill") throw new Error("unreachable");
+		expect(plan.latencyBudgetMs).toBe(KV_SPILL_VOICE_LATENCY_BUDGET_MS);
+	});
+
+	it("hard-fails with a structured error when cold-page restore misses the voice budget", () => {
+		// Big per-token KV + slow SATA disk → a single page restore blows the
+		// 200ms voice budget. Page = 256 * 200_000 B = 51.2 MB; SATA budget
+		// 250_000 B/ms → ~205 ms > 200 ms.
+		let thrown: unknown;
+		try {
+			planKvSpill({
+				requestedContext: 262_144,
+				geometry: geometry({ bytesPerToken: 200_000, voiceEnabled: true }),
+				residentKvBudgetBytes: 64 * MIB,
+				restoreClass: "disk-sata",
+				cpuSpillAvailable: false,
+			});
+		} catch (err) {
+			thrown = err;
+		}
+		expect(thrown).toBeInstanceOf(KvSpillUnsupportedError);
+		const e = thrown as KvSpillUnsupportedError;
+		expect(e.code).toBe("kv-spill-unsupported");
+		expect(e.details.voiceEnabled).toBe(true);
+		expect(e.details.restoreClass).toBe("disk-sata");
+		expect(e.details.worstCaseRestoreMs).toBeGreaterThan(
+			e.details.latencyBudgetMs,
+		);
+		expect(e.message).toContain("voice latency budget");
+	});
+
+	it("degrades the spill tier to disk and downgrades a cpu restore class when CPU spill is unavailable", () => {
+		const plan = planKvSpill({
+			requestedContext: 262_144,
+			geometry: geometry(),
+			residentKvBudgetBytes: 64 * MIB,
+			restoreClass: "cpu-pcie",
+			cpuSpillAvailable: false,
+		});
+		expect(plan.mode).toBe("spill");
+		if (plan.mode !== "spill") throw new Error("unreachable");
+		expect(plan.tier).toBe("disk");
+		// worstCaseRestoreMs reflects the NVMe class, not the cpu-pcie one.
+		expect(plan.worstCaseRestoreMs).toBeGreaterThan(
+			// page (512_000 B) / cpu-pcie (12e6) ≈ 0.043 ms
+			512_000 / 12_000_000,
+		);
+	});
+
+	it("page-aligns the resident/spill split", () => {
+		const plan = planKvSpill({
+			requestedContext: 200_000,
+			geometry: geometry(),
+			residentKvBudgetBytes: 30 * MIB,
+			restoreClass: "cpu-apple",
+			cpuSpillAvailable: true,
+		});
+		expect(plan.mode).toBe("spill");
+		if (plan.mode !== "spill") throw new Error("unreachable");
+		const pageBytes = 2_000 * KV_PAGE_TOKENS;
+		expect(plan.residentBytes % pageBytes).toBe(0);
+		expect(plan.spillBytes % pageBytes).toBe(0);
+		const totalPages = Math.ceil(200_000 / KV_PAGE_TOKENS);
+		expect(plan.residentPages + plan.spillPages).toBe(totalPages);
+	});
+
+	it("treats a short context that overruns the resident budget as unsupported (wrong tier for device)", () => {
+		// 64k - 1 tokens, but a per-token KV so large the cache won't fit and
+		// we're below KV_SPILL_MIN_CONTEXT → structured error, no half-load.
+		let thrown: unknown;
+		try {
+			planKvSpill({
+				requestedContext: KV_SPILL_MIN_CONTEXT - 1,
+				geometry: geometry({ bytesPerToken: 50_000 }),
+				residentKvBudgetBytes: 16 * MIB,
+				restoreClass: "cpu-apple",
+				cpuSpillAvailable: true,
+			});
+		} catch (err) {
+			thrown = err;
+		}
+		expect(thrown).toBeInstanceOf(KvSpillUnsupportedError);
+		expect((thrown as KvSpillUnsupportedError).details.requestedContext).toBe(
+			KV_SPILL_MIN_CONTEXT - 1,
+		);
+	});
+
+	it("rejects degenerate inputs loudly", () => {
+		expect(() =>
+			planKvSpill({
+				requestedContext: 0,
+				geometry: geometry(),
+				residentKvBudgetBytes: MIB,
+				restoreClass: "cpu-apple",
+				cpuSpillAvailable: true,
+			}),
+		).toThrow(/positive context/);
+		expect(() =>
+			planKvSpill({
+				requestedContext: 100_000,
+				geometry: geometry(),
+				residentKvBudgetBytes: 0,
+				restoreClass: "cpu-apple",
+				cpuSpillAvailable: true,
+			}),
+		).toThrow(/residentKvBudgetBytes must be positive/);
+	});
+});
+
+describe("residentKvBudgetFromRamBudget", () => {
+	it("reserves the documented KV fraction of the recommended budget", () => {
+		const budget: RamBudget = {
+			minMb: 7000,
+			recommendedMb: 9600,
+			source: "manifest",
+		};
+		// 9600 MiB * 0.25 = 2400 MiB
+		expect(residentKvBudgetFromRamBudget(budget)).toBe(2400 * MIB);
+	});
+});
+
+describe("estimateQuantizedKvBytesPerToken", () => {
+	it("returns the per-tier figure for known param strings", () => {
+		expect(estimateQuantizedKvBytesPerToken("0.8B")).toBeLessThan(
+			estimateQuantizedKvBytesPerToken("9B"),
+		);
+		expect(estimateQuantizedKvBytesPerToken("2B")).toBeGreaterThan(
+			estimateQuantizedKvBytesPerToken("0.8B"),
+		);
+		expect(estimateQuantizedKvBytesPerToken("4B")).toBeGreaterThan(
+			estimateQuantizedKvBytesPerToken("2B"),
+		);
+		expect(estimateQuantizedKvBytesPerToken("27B")).toBeGreaterThan(
+			estimateQuantizedKvBytesPerToken("9B"),
+		);
+	});
+
+	it("fails closed (largest tier) for unknown param strings", () => {
+		expect(estimateQuantizedKvBytesPerToken("999B")).toBe(
+			estimateQuantizedKvBytesPerToken("27B"),
+		);
+	});
+});
+
+describe("restoreClassForHardware", () => {
+	it("maps Apple Silicon, discrete GPU, and CPU-only to the right classes", () => {
+		expect(
+			restoreClassForHardware({ appleSilicon: true, hasDiscreteGpu: false }),
+		).toBe("cpu-apple");
+		expect(
+			restoreClassForHardware({ appleSilicon: false, hasDiscreteGpu: true }),
+		).toBe("cpu-pcie");
+		expect(
+			restoreClassForHardware({ appleSilicon: false, hasDiscreteGpu: false }),
+		).toBe("disk-nvme");
+	});
+});