@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/conversation-registry.ts

@@ -0,0 +1,264 @@
+/**
+ * Conversation registry for the local-inference path.
+ *
+ * Today's slot allocation is purely a hash function: `deriveSlotId` maps a
+ * `promptCacheKey` (or any stable string) to `slot_id` in `[0, parallel)`.
+ * That works for one-shot calls but breaks for long agentic loops:
+ *
+ *   - Two distinct conversations whose cache keys hash to the same slot
+ *     evict each other's KV every turn (slot thrashing).
+ *   - The current high-water mark of concurrent conversations is invisible,
+ *     so `--parallel N` cannot be tuned to fit.
+ *   - There is no notion of an explicit "I am still using this slot" lease,
+ *     so eviction is purely best-effort.
+ *
+ * This registry keeps a per-conversation reservation. `openConversation`
+ * picks the lowest-loaded slot and pins the conversation to it; subsequent
+ * `generateInConversation` calls always land on the same slot. When the
+ * pool is full, slot reuse falls back to the same-as-before hash policy
+ * (two leases on the same slot still serialise correctly via the dispatcher's
+ * generation queue).
+ *
+ * The registry tracks the high-water mark of concurrently-open conversations
+ * so the engine can warn, or later restart llama-server with a higher
+ * --parallel, when the load outgrows the configured slot count.
+ */
+
+import { createHash } from "node:crypto";
+
+/**
+ * Opaque handle returned by `openConversation`. Callers MUST treat this as
+ * opaque — the registry owns the slot id and lifetime.
+ */
+export interface ConversationHandle {
+	readonly conversationId: string;
+	readonly modelId: string;
+	/**
+	 * Pinned slot id in `[0, parallel)`, or `-1` when slot pinning is disabled
+	 * (parallel <= 0). Used by both backends as the cache key:
+	 *   - llama-server: forwarded as `slot_id` in the request payload.
+	 *   - node-llama-cpp: combined with the conversation id to derive the
+	 *     session-pool key so identical conversations share a session.
+	 */
+	readonly slotId: number;
+	/** Wall-clock ms when the handle was opened. */
+	readonly openedAtMs: number;
+	/** Wall-clock ms when the handle was last touched (open or generate). */
+	lastUsedMs: number;
+	/** TTL after which the registry MAY auto-close on the next sweep. */
+	readonly ttlMs: number;
+	/** True when `closeConversation` has been called; further use is rejected. */
+	closed: boolean;
+}
+
+export interface OpenConversationArgs {
+	conversationId: string;
+	modelId: string;
+	/** Slot count from the running server (`--parallel N`). Defaults to 1. */
+	parallel?: number;
+	/**
+	 * TTL after which the handle is considered idle and may be auto-closed
+	 * by `evictIdle`. Defaults to 60 minutes — long enough for an LLM call
+	 * to finish even on a slow drafter, short enough to recover from forgotten
+	 * close calls within the long-cache window.
+	 */
+	ttlMs?: number;
+}
+
+const DEFAULT_HANDLE_TTL_MS = 60 * 60 * 1000;
+
+/**
+ * In-memory registry of open conversation handles. A single instance is
+ * shared by the engine; each backend reads from it on every generate to
+ * decide which slot to pin to.
+ */
+export class ConversationRegistry {
+	private readonly handles = new Map<string, ConversationHandle>();
+	/** Per-slot reference count; lowest-loaded slot wins on next open. */
+	private readonly slotLoad = new Map<number, number>();
+	/** Largest concurrent open count seen; the engine reads this for parallel auto-tune. */
+	private highWaterMark = 0;
+
+	/**
+	 * Lookup / open a conversation handle. Idempotent for the same
+	 * conversation id + model id; callers can call this on every turn
+	 * without leaking handles. When the call is reusing an existing handle,
+	 * `lastUsedMs` is bumped for LRU-style eviction tracking.
+	 */
+	open(args: OpenConversationArgs): ConversationHandle {
+		if (!args.conversationId) {
+			throw new Error("[conversation-registry] conversationId is required");
+		}
+		if (!args.modelId) {
+			throw new Error("[conversation-registry] modelId is required");
+		}
+		const compositeKey = this.compositeKey(args.conversationId, args.modelId);
+		const existing = this.handles.get(compositeKey);
+		if (existing && !existing.closed) {
+			existing.lastUsedMs = Date.now();
+			return existing;
+		}
+
+		const parallel =
+			typeof args.parallel === "number" && args.parallel > 0
+				? Math.floor(args.parallel)
+				: 1;
+		const slotId = this.pickLowestLoadedSlot(parallel, args.conversationId);
+		const now = Date.now();
+		const handle: ConversationHandle = {
+			conversationId: args.conversationId,
+			modelId: args.modelId,
+			slotId,
+			openedAtMs: now,
+			lastUsedMs: now,
+			ttlMs: args.ttlMs ?? DEFAULT_HANDLE_TTL_MS,
+			closed: false,
+		};
+		this.handles.set(compositeKey, handle);
+		this.slotLoad.set(slotId, (this.slotLoad.get(slotId) ?? 0) + 1);
+		if (this.handles.size > this.highWaterMark) {
+			this.highWaterMark = this.handles.size;
+		}
+		return handle;
+	}
+
+	/**
+	 * Lookup an open handle by conversation+model. Returns null when the
+	 * conversation has not been opened or has already been closed. Bumps
+	 * `lastUsedMs` so an LRU sweep treats reads as activity.
+	 */
+	get(conversationId: string, modelId: string): ConversationHandle | null {
+		const handle = this.handles.get(this.compositeKey(conversationId, modelId));
+		if (!handle || handle.closed) return null;
+		handle.lastUsedMs = Date.now();
+		return handle;
+	}
+
+	/**
+	 * Close + drop a handle. Idempotent — closing an unknown / already-closed
+	 * handle has no additional effect, so callers can call this from cleanup paths
+	 * unconditionally.
+	 */
+	close(conversationId: string, modelId: string): void {
+		const compositeKey = this.compositeKey(conversationId, modelId);
+		const handle = this.handles.get(compositeKey);
+		if (!handle) return;
+		handle.closed = true;
+		this.handles.delete(compositeKey);
+		const remaining = (this.slotLoad.get(handle.slotId) ?? 0) - 1;
+		if (remaining <= 0) {
+			this.slotLoad.delete(handle.slotId);
+		} else {
+			this.slotLoad.set(handle.slotId, remaining);
+		}
+	}
+
+	/**
+	 * Sweep handles whose `lastUsedMs` is older than their TTL. Returns the
+	 * conversation ids dropped so callers can persist final KV state to
+	 * disk, etc. Safe to call on a timer.
+	 */
+	evictIdle(now: number = Date.now()): string[] {
+		const dropped: string[] = [];
+		for (const [compositeKey, handle] of this.handles) {
+			if (now - handle.lastUsedMs > handle.ttlMs) {
+				handle.closed = true;
+				this.handles.delete(compositeKey);
+				const remaining = (this.slotLoad.get(handle.slotId) ?? 0) - 1;
+				if (remaining <= 0) {
+					this.slotLoad.delete(handle.slotId);
+				} else {
+					this.slotLoad.set(handle.slotId, remaining);
+				}
+				dropped.push(handle.conversationId);
+			}
+		}
+		return dropped;
+	}
+
+	/**
+	 * Snapshot every currently-open handle. Used by the shutdown path to
+	 * emit a save-state request per slot.
+	 */
+	snapshot(): readonly ConversationHandle[] {
+		return [...this.handles.values()];
+	}
+
+	/** Largest concurrent open count seen since the registry was created. */
+	highWater(): number {
+		return this.highWaterMark;
+	}
+
+	/** Number of currently-open handles. */
+	size(): number {
+		return this.handles.size;
+	}
+
+	/**
+	 * Recommended `--parallel` slot count given the observed high-water mark
+	 * of concurrently-open conversations plus a small headroom (max(2, 25%)).
+	 * The engine's auto-tune (J4) compares this against the running server's
+	 * slot count: when this is larger AND there's RAM headroom, it restarts
+	 * llama-server with the higher value so new conversations get their own
+	 * KV slots instead of thrashing.
+	 *
+	 * `running` is the currently-configured slot count; when the high-water
+	 * mark hasn't outgrown it, this returns `running` (no resize needed) so
+	 * callers can compare against equality without a second branch.
+	 */
+	recommendedParallel(running: number): number {
+		const headroom = Math.max(2, Math.ceil(this.highWaterMark * 0.25));
+		const desired = Math.max(1, this.highWaterMark + headroom);
+		return Math.max(running, desired);
+	}
+
+	/**
+	 * Drop every handle and reset the high-water mark + slot-load bookkeeping.
+	 * Test-only — the module singleton leaks state across files when the suite
+	 * runs together; call this in `beforeEach` to isolate. Not part of the
+	 * runtime contract.
+	 */
+	__resetForTests(): void {
+		for (const handle of this.handles.values()) handle.closed = true;
+		this.handles.clear();
+		this.slotLoad.clear();
+		this.highWaterMark = 0;
+	}
+
+	/**
+	 * Pick the slot with the fewest in-flight handles. Ties are broken by a
+	 * deterministic hash of the conversation id, which avoids consistently
+	 * loading slot 0 when N concurrent opens race.
+	 */
+	private pickLowestLoadedSlot(
+		parallel: number,
+		conversationId: string,
+	): number {
+		if (parallel <= 1) return 0;
+		let bestSlot = 0;
+		let bestLoad = Number.POSITIVE_INFINITY;
+		for (let slot = 0; slot < parallel; slot += 1) {
+			const load = this.slotLoad.get(slot) ?? 0;
+			if (load < bestLoad) {
+				bestLoad = load;
+				bestSlot = slot;
+			}
+		}
+		if (bestLoad === 0) return bestSlot;
+		// All slots are loaded equally — use the conversation hash for a
+		// deterministic tie-break. Same conversation, same slot when reopened.
+		const digest = createHash("sha256").update(conversationId).digest();
+		return digest.readUInt32BE(0) % parallel;
+	}
+
+	private compositeKey(conversationId: string, modelId: string): string {
+		return `${modelId}::${conversationId}`;
+	}
+}
+
+/**
+ * Module-singleton registry. The engine reads this on every generate; the
+ * conversation lifecycle API (`openConversation`, `closeConversation`)
+ * mutates it.
+ */
+export const conversationRegistry = new ConversationRegistry();

package/src/services/desktop-fused-ffi-backend-runtime.ts

@@ -0,0 +1,431 @@
+/**
+ * Desktop production `FfiBackendRuntime` over the FUSED `libelizainference` —
+ * the SOLE desktop text runtime now that libllama has been retired.
+ *
+ * Desktop text generation runs through the fused library: the same
+ * `eliza_inference_llm_stream_*` ABI (v9) the voice subsystem already loads,
+ * so text + voice share one native lib, one GGML pin, and one resident text
+ * model.
+ *
+ *   - The fused lib's `eliza_inference_llm_stream_open` loads the bundle's text
+ *     GGUF (`<bundleRoot>/text/*.gguf`) and applies MTP speculative
+ *     decoding + KV-cache quant + per-load GPU layers natively (ABI v9). The
+ *     path is gated on the capability probes
+ *     (`llmStreamSupported && llmMtpSupported && llmKvQuantSupported`).
+ *   - A fused lib that lacks MTP / KV-quant / native tokenize is REFUSED by
+ *     `supported()` → the engine raises LocalInferenceUnavailable. There is no
+ *     libllama fallback and never an unoptimized fused loop.
+ *
+ * Tokenization runs over the fused handle's resident text vocab via ABI-v9
+ * `eliza_inference_tokenize`: the fused `create()` + first `llmStreamOpen`
+ * already made the text vocab resident, so no second model is loaded.
+ * `tokenizeSupported()` gates this; a pre-v9 lib without the symbol is refused.
+ *
+ * Lifecycle: one fused context per loaded model; `acquire()` builds it,
+ * `release()` tears it down. A throwing native free poisons the runtime so no
+ * new allocation happens over leaked resources.
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+
+import { resolveStateDir } from "@elizaos/core";
+
+import type { BackendPlan } from "./backend";
+import type {
+	FfiBackendRuntime,
+	FfiBackendSession,
+} from "./ffi-streaming-backend";
+import { FfiStreamingRunner } from "./ffi-streaming-runner";
+import { wrapElizaInferenceFfi } from "./llm-streaming-binding";
+import type { ElizaInferenceContextHandle } from "./voice/ffi-bindings";
+import {
+	type ElizaInferenceFfi,
+	loadElizaInferenceFfi,
+} from "./voice/ffi-bindings";
+
+function throwIfAborted(signal: AbortSignal | undefined): void {
+	if (!signal?.aborted) return;
+	throw signal.reason instanceof Error
+		? signal.reason
+		: new DOMException("Aborted", "AbortError");
+}
+
+/**
+ * Candidate filenames for the fused library, per platform. Mirrors
+ * `samantha-preset-regenerator.ts::libraryFilenames` so the runtime and the
+ * voice regenerator resolve the same artifact.
+ */
+function fusedLibraryFilenames(): string[] {
+	if (process.platform === "darwin") return ["libelizainference.dylib"];
+	if (process.platform === "win32") {
+		return ["elizainference.dll", "libelizainference.dll"];
+	}
+	return ["libelizainference.so"];
+}
+
+/**
+ * Resolve the on-disk path to the fused `libelizainference`. Precedence:
+ *   1. `ELIZA_INFERENCE_LIBRARY` — an explicit absolute path.
+ *   2. `<bundleRoot>/lib/<name>` — the bundle-local lib.
+ *   3. `ELIZA_INFERENCE_LIB_DIR/<name>` — an explicit lib directory.
+ *   4. `<stateDir>/local-inference/lib/<name>` — the default staging dir written
+ *      by `scripts/stage-desktop-fused-lib.mjs`, so a staged desktop build is
+ *      found with no env wiring.
+ * Returns null when none of the candidates exist on disk — `supported()` then
+ * reports unavailable and the engine raises LocalInferenceUnavailable.
+ */
+export function resolveFusedLibraryPath(
+	bundleRoot: string | null,
+	env: NodeJS.ProcessEnv = process.env,
+): string | null {
+	const exact = env.ELIZA_INFERENCE_LIBRARY?.trim();
+	if (exact && fs.existsSync(exact)) return exact;
+	const dirs = [
+		bundleRoot ? path.join(bundleRoot, "lib") : null,
+		exact ? path.dirname(exact) : null,
+		env.ELIZA_INFERENCE_LIB_DIR?.trim() || null,
+		path.join(resolveStateDir(env), "local-inference", "lib"),
+	].filter((dir): dir is string => Boolean(dir));
+	for (const dir of dirs) {
+		for (const name of fusedLibraryFilenames()) {
+			const candidate = path.join(dir, name);
+			if (fs.existsSync(candidate)) return candidate;
+		}
+	}
+	return null;
+}
+
+/**
+ * Derive the bundle root (the dir the fused `create()` anchors at) from a
+ * BackendPlan. Eliza-1 bundles set `overrides.bundleRoot` explicitly; otherwise
+ * the GGUF lives at `<bundleRoot>/text/<file>.gguf`, so the bundle root is
+ * `dirname(dirname(modelPath))`.
+ */
+function bundleRootForPlan(plan: BackendPlan): string {
+	if (plan.overrides?.bundleRoot) return plan.overrides.bundleRoot;
+	return path.dirname(path.dirname(plan.modelPath));
+}
+
+interface ActiveFusedSession {
+	ffi: ElizaInferenceFfi;
+	ctx: ElizaInferenceContextHandle;
+	session: FfiBackendSession;
+}
+
+export class DesktopFusedFfiBackendRuntime implements FfiBackendRuntime {
+	private active: ActiveFusedSession | null = null;
+	private poisonedError: Error | null = null;
+	/** Cached `supported()` result so the engine gate and the dispatcher agree. */
+	private supportedCache: boolean | null = null;
+
+	/**
+	 * Viable only when:
+	 *   - bun:ffi resolves on the current runtime,
+	 *   - the fused dylib is present AND reports ABI-v9 capability: the
+	 *     streaming-LLM surface, MTP, KV-cache quant, AND native
+	 *     tokenization (`eliza_inference_tokenize`).
+	 * A pre-v9 fused lib reports the probes as unsupported → refused, and the
+	 * engine raises LocalInferenceUnavailable. libllama has been retired; there
+	 * is no fallback runtime and no tokenizer sidecar.
+	 */
+	supported(): boolean {
+		if (this.supportedCache !== null) return this.supportedCache;
+		this.supportedCache = this.computeSupported();
+		return this.supportedCache;
+	}
+
+	/** Clear the cached `supported()` result (tests / lib swaps). */
+	resetSupportedCache(): void {
+		this.supportedCache = null;
+	}
+
+	private computeSupported(): boolean {
+		try {
+			require.resolve("bun:ffi");
+		} catch {
+			return false;
+		}
+		const libPath = resolveFusedLibraryPath(null);
+		if (!libPath) return false;
+		// Load the lib and probe the v8 LLM capabilities. This dlopen is cheap (no
+		// model load); we close it immediately after probing.
+		let ffi: ElizaInferenceFfi | null = null;
+		try {
+			ffi = loadElizaInferenceFfi(libPath);
+			const llmOk =
+				typeof ffi.llmStreamSupported === "function" &&
+				ffi.llmStreamSupported() === true &&
+				typeof ffi.llmMtpSupported === "function" &&
+				ffi.llmMtpSupported() === true &&
+				typeof ffi.llmKvQuantSupported === "function" &&
+				ffi.llmKvQuantSupported() === true;
+			if (!llmOk) return false;
+			// Native tokenization over the fused handle's resident text vocab
+			// (ABI v9) is required: libllama has been retired, so there is no
+			// tokenizer sidecar. A pre-v9 fused lib without `eliza_inference_tokenize`
+			// is refused → the engine raises LocalInferenceUnavailable.
+			const fusedTokenize =
+				typeof ffi.tokenizeSupported === "function" &&
+				ffi.tokenizeSupported() === true;
+			if (!fusedTokenize) return false;
+			return true;
+		} catch {
+			// dlopen / ABI-mismatch / non-Bun runtime → not viable.
+			return false;
+		} finally {
+			ffi?.close();
+		}
+	}
+
+	async acquire(plan: BackendPlan): Promise<FfiBackendSession> {
+		if (this.poisonedError) {
+			throw new Error(
+				`[desktop-fused-ffi-runtime] native cleanup previously failed; restart required before acquiring a new session: ${this.poisonedError.message}`,
+			);
+		}
+		if (this.active) {
+			throw new Error(
+				"[desktop-fused-ffi-runtime] acquire() called with a live session; release() first",
+			);
+		}
+		const bundleRoot = bundleRootForPlan(plan);
+		const libPath = resolveFusedLibraryPath(bundleRoot);
+		if (!libPath) {
+			throw new Error(
+				`[desktop-fused-ffi-runtime] fused libelizainference not found for bundle ${bundleRoot}. ` +
+					"Dispatcher should not have routed here; check supported().",
+			);
+		}
+
+		// 1. Fused lib + bundle context for the generation path. `create()`
+		//    anchors at the bundle root; the first `llmStreamOpen` loads
+		//    `<bundleRoot>/text/*.gguf` and applies gpuLayers + KV-cache quant
+		//    from the session config (threaded via loadConfig below).
+		const ffi = loadElizaInferenceFfi(libPath);
+		let ctx: ElizaInferenceContextHandle;
+		try {
+			ctx = ffi.create(bundleRoot);
+		} catch (err) {
+			ffi.close();
+			throw err;
+		}
+
+		// 2. Tokenization over the fused handle's resident text vocab via ABI-v9
+		//    `eliza_inference_tokenize` — no second model load. `supported()`
+		//    already refused a pre-v9 lib, so the symbol is present here; this
+		//    guard turns any surprise absence into a loud failure (the session is
+		//    torn down) rather than a silent tokenizer gap. libllama is retired.
+		const fusedTokenizeFn = ffi.tokenize;
+		if (
+			typeof ffi.tokenizeSupported !== "function" ||
+			ffi.tokenizeSupported() !== true ||
+			typeof fusedTokenizeFn !== "function"
+		) {
+			ffi.destroy(ctx);
+			ffi.close();
+			throw new Error(
+				"[desktop-fused-ffi-runtime] fused lib lacks eliza_inference_tokenize (pre-v9). " +
+					"libllama has been retired; rebuild the fused lib with the v9 tokenizer ABI.",
+			);
+		}
+		const tokenizeFn = (prompt: string): Int32Array =>
+			fusedTokenizeFn({ ctx, text: prompt });
+
+		const binding = wrapElizaInferenceFfi(ffi);
+		const runner = new FfiStreamingRunner(binding, ctx);
+		const overrides = plan.overrides;
+		const session: FfiBackendSession = {
+			binding,
+			ctx,
+			runner,
+			tokenize: (prompt) => tokenizeFn(prompt),
+			mtp: plan.catalog?.runtime?.mtp ?? null,
+			draftModelPath: overrides?.draftModelPath ?? null,
+			mmprojPath: overrides?.mmprojPath ?? null,
+			// The fused path applies these at its first `llmStreamOpen`:
+			// context size, gpuLayers, and KV-cache quant types from the
+			// session config.
+			loadConfig: {
+				contextSize:
+					typeof overrides?.contextSize === "number"
+						? overrides.contextSize
+						: undefined,
+				gpuLayers:
+					typeof overrides?.gpuLayers === "number"
+						? overrides.gpuLayers
+						: undefined,
+				cacheTypeK: overrides?.cacheTypeK ?? null,
+				cacheTypeV: overrides?.cacheTypeV ?? null,
+			},
+		};
+		this.active = { ffi, ctx, session };
+		return session;
+	}
+
+	parallelSlots(): number {
+		// The fused runtime holds one resident text context per loaded model;
+		// multi-slot parallelism is not exposed by the fused ABI.
+		return 1;
+	}
+
+	/**
+	 * Whether the LIVE session can describe images through the fused
+	 * `eliza_inference_describe_image`. Mirrors the FfiStreamingBackend gate:
+	 * true only when a session is bound and the fused lib exposes vision.
+	 */
+	visionSupported(): boolean {
+		if (!this.active) return false;
+		return (
+			typeof this.active.ffi.visionSupported === "function" &&
+			this.active.ffi.visionSupported() === true &&
+			typeof this.active.ffi.describeImage === "function"
+		);
+	}
+
+	/**
+	 * Whether the LIVE session can STREAM a vision describe token-by-token
+	 * through `eliza_inference_describe_image_stream_open` + the existing
+	 * `llmStreamNext` loop (ABI v13). A <=v12 lib reports false and the handler
+	 * uses the buffered one-shot `describeImage` path.
+	 */
+	visionStreamSupported(): boolean {
+		if (!this.active) return false;
+		const { ffi } = this.active;
+		return (
+			typeof ffi.visionStreamSupported === "function" &&
+			ffi.visionStreamSupported() === true &&
+			typeof ffi.describeImageStreamOpen === "function" &&
+			typeof ffi.llmStreamNext === "function" &&
+			typeof ffi.llmStreamClose === "function"
+		);
+	}
+
+	/**
+	 * Vision describe through the fused mmproj path. Reuses the mtmd machinery
+	 * linked for ASR over the bundle's text model + the passed mmproj projector.
+	 * The `FfiStreamingBackend` forwards `describeImage`/`visionSupported` to this
+	 * runtime by duck-typing.
+	 *
+	 * When `onTextChunk` is supplied AND the fused lib exposes ABI-v13 streaming
+	 * vision, the description is decoded token-by-token: `describeImageStreamOpen`
+	 * primes a stream with the image+prompt KV and the EXISTING `llmStreamNext`
+	 * loop pulls tokens — the same machinery that streams chat text, so vision
+	 * flows into the dashboard through one pipe. Otherwise it falls back to the
+	 * buffered one-shot `eliza_inference_describe_image`.
+	 */
+	async describeImage(args: {
+		imageBytes: Uint8Array;
+		mmprojPath: string;
+		prompt?: string;
+		maxTokens?: number;
+		temperature?: number;
+		signal?: AbortSignal;
+		onTextChunk?: (chunk: string) => void | Promise<void>;
+		maxTokensPerStep?: number;
+	}): Promise<{ text: string; projectorMs?: number; decodeMs?: number }> {
+		if (!this.active) {
+			throw new Error(
+				"[desktop-fused-ffi-runtime] describeImage before acquire — no session",
+			);
+		}
+		const { ffi, ctx } = this.active;
+		if (
+			typeof ffi.visionSupported !== "function" ||
+			ffi.visionSupported() !== true ||
+			typeof ffi.describeImage !== "function"
+		) {
+			throw new Error(
+				"[desktop-fused-ffi-runtime] describeImage: fused lib was built without " +
+					"vision (eliza_inference_vision_supported() == 0). Rebuild the fused " +
+					"lib with -DELIZA_ENABLE_VISION=ON (verify-fused-symbols requires it).",
+			);
+		}
+
+		// Token-by-token streaming path (ABI v13): open a vision stream and drive
+		// the shared `llmStreamNext` loop, surfacing each decoded piece through
+		// `onTextChunk` so the description renders as it generates.
+		if (
+			typeof args.onTextChunk === "function" &&
+			this.visionStreamSupported() &&
+			typeof ffi.describeImageStreamOpen === "function" &&
+			typeof ffi.llmStreamNext === "function" &&
+			typeof ffi.llmStreamClose === "function"
+		) {
+			throwIfAborted(args.signal);
+			const startedAt = Date.now();
+			const stream = ffi.describeImageStreamOpen({
+				ctx,
+				imageBytes: args.imageBytes,
+				mmprojPath: args.mmprojPath,
+				prompt: args.prompt,
+			});
+			let full = "";
+			let generated = 0;
+			// JS-side token budget: the native ELIZA_VISION_MAX_TOKENS env does not
+			// reliably reach the loaded DLL's getenv across runtimes, so cap here.
+			const tokenBudget =
+				typeof args.maxTokens === "number" && args.maxTokens > 0
+					? args.maxTokens
+					: 256;
+			try {
+				for (;;) {
+					if (args.signal?.aborted) {
+						ffi.llmStreamCancel?.(stream);
+						throwIfAborted(args.signal);
+					}
+					const step = ffi.llmStreamNext({
+						stream,
+						// Fine-grained by default so the description renders token-by-token
+						// in the dashboard rather than in coarse ~32-token jumps (matches
+						// the tuned chat default). Callers may override per request.
+						maxTokensPerStep: args.maxTokensPerStep ?? 8,
+					});
+					if (step.text.length > 0) {
+						full += step.text;
+						await args.onTextChunk(step.text);
+					}
+					generated += step.tokens.length;
+					if (step.done || generated >= tokenBudget) break;
+				}
+			} finally {
+				ffi.llmStreamClose(stream);
+			}
+			return { text: full, decodeMs: Date.now() - startedAt };
+		}
+
+		const startedAt = Date.now();
+		const text = ffi.describeImage({
+			ctx,
+			imageBytes: args.imageBytes,
+			mmprojPath: args.mmprojPath,
+			prompt: args.prompt,
+		});
+		return { text, decodeMs: Date.now() - startedAt };
+	}
+
+	async release(): Promise<void> {
+		if (!this.active) return;
+		const { ffi, ctx } = this.active;
+		// Free the native handles. A throwing free poisons the runtime so a new
+		// model cannot be allocated over leaked resources. Clear `active` in the
+		// finally so a throwing free can't wedge the live-session guard.
+		try {
+			ffi.destroy(ctx);
+			ffi.close();
+		} catch (err) {
+			this.poisonedError = err instanceof Error ? err : new Error(String(err));
+			throw err;
+		} finally {
+			this.active = null;
+		}
+	}
+}
+
+/**
+ * Process singleton — the engine wires this as the sole `FfiBackendRuntime` for
+ * the dispatcher's `"llama-cpp"` slot. The ABI-v9 capability probes in
+ * `supported()` gate whether the fused lib serves text at all.
+ */
+export const desktopFusedFfiBackendRuntime =
+	new DesktopFusedFfiBackendRuntime();