@elizaos/plugin-local-inference 2.0.3-beta.2 → 2.0.3-beta.4

package/src/runtime/ensure-local-inference-handler.test.ts

@@ -2,11 +2,21 @@ import { type AgentRuntime, ModelType } from "@elizaos/core";
 import { beforeEach, describe, expect, it, vi } from "vitest";
 
 const modeState = vi.hoisted(() => ({ mode: "local" }));
+const assignmentsState = vi.hoisted(() => ({
+	assignments: {} as Record<string, string>,
+}));
+const registryState = vi.hoisted(() => ({
+	installed: [] as Array<{ id: string; path: string }>,
+}));
+const hardwareState = vi.hoisted(() => ({
+	probe: { memory: { totalGb: 8 } },
+}));
 const engineState = vi.hoisted(() => ({
 	activeBackendId: vi.fn(() => "llama-server"),
 	available: vi.fn(async () => true),
 	conversation: vi.fn(() => null),
 	currentModelPath: vi.fn(() => null),
+	ensureActiveBundleAsrReady: vi.fn(async () => undefined),
 	ensureActiveBundleVoiceReady: vi.fn(async () => undefined),
 	generate: vi.fn(async () => "ok"),
 	generateInConversation: vi.fn(async () => ({
@@ -42,7 +52,7 @@ vi.mock("../services/active-model", () => ({
 
 vi.mock("../services/assignments", () => ({
 	autoAssignAtBoot: vi.fn(async () => null),
-	readEffectiveAssignments: vi.fn(async () => ({})),
+	readEffectiveAssignments: vi.fn(async () => assignmentsState.assignments),
 }));
 
 vi.mock("../services/cache-bridge", () => ({
@@ -71,12 +81,16 @@ vi.mock("../services/handler-registry", () => ({
 	},
 }));
 
+vi.mock("../services/hardware", () => ({
+	probeHardware: vi.fn(async () => hardwareState.probe),
+}));
+
 vi.mock("../services/memory-arbiter", () => ({
 	tryGetMemoryArbiter: vi.fn(() => arbiterState),
 }));
 
 vi.mock("../services/registry", () => ({
-	listInstalledModels: vi.fn(async () => []),
+	listInstalledModels: vi.fn(async () => registryState.installed),
 }));
 
 vi.mock("../services/router-handler", () => ({
@@ -90,6 +104,8 @@ vi.mock("../services/voice", () => ({
 	})),
 }));
 
+import { resolveLocalInferenceLoadArgs } from "../services/active-model";
+import { probeHardware } from "../services/hardware";
 import { installRouterHandler } from "../services/router-handler";
 import { VoiceStartupError } from "../services/voice/errors";
 import { ensureLocalInferenceHandler } from "./ensure-local-inference-handler";
@@ -151,6 +167,9 @@ function findRegisteredHandler(
 beforeEach(() => {
 	vi.clearAllMocks();
 	modeState.mode = "local";
+	assignmentsState.assignments = {};
+	registryState.installed = [];
+	hardwareState.probe = { memory: { totalGb: 8 } };
 	delete process.env.ELIZA_LOCAL_LLAMA;
 	delete process.env.ELIZA_DEVICE_BRIDGE_ENABLED;
 	delete process.env.ELIZA_DISABLE_LOCAL_EMBEDDINGS;
@@ -164,6 +183,9 @@ beforeEach(() => {
 		title: "A small image",
 		description: "A tiny synthetic image.",
 	});
+	vi.mocked(resolveLocalInferenceLoadArgs).mockImplementation(
+		async (target) => target,
+	);
 });
 
 describe("ensureLocalInferenceHandler", () => {
@@ -297,6 +319,88 @@ describe("ensureLocalInferenceHandler", () => {
 		);
 	});
 
+	it("uses a fine-grained maxTokensPerStep for user-visible streaming, coarse for internal calls", async () => {
+		const prior = process.env.ELIZA_LOCAL_STREAM_TOKENS_PER_STEP;
+		delete process.env.ELIZA_LOCAL_STREAM_TOKENS_PER_STEP;
+		try {
+			const { registrations, runtime } = makeRuntime();
+			engineState.hasLoadedModel.mockReturnValue(true);
+
+			await ensureLocalInferenceHandler(runtime);
+			const handler = findRegisteredHandler(
+				registrations,
+				ModelType.TEXT_LARGE,
+			);
+
+			// Streaming reply (onStreamChunk wired) → tuned fine-grained step (8).
+			await handler(runtime, {
+				prompt: "hi",
+				stream: true,
+				onStreamChunk: () => {},
+			});
+			expect(engineState.generate).toHaveBeenLastCalledWith(
+				expect.objectContaining({ maxTokensPerStep: 8 }),
+			);
+
+			// Internal / non-streamed call → no override (runner keeps coarse 32).
+			await handler(runtime, { prompt: "hi" });
+			expect(engineState.generate).toHaveBeenLastCalledWith(
+				expect.objectContaining({ maxTokensPerStep: undefined }),
+			);
+
+			// The shared env knob overrides the tuned streaming default.
+			process.env.ELIZA_LOCAL_STREAM_TOKENS_PER_STEP = "4";
+			await handler(runtime, {
+				prompt: "hi",
+				stream: true,
+				onStreamChunk: () => {},
+			});
+			expect(engineState.generate).toHaveBeenLastCalledWith(
+				expect.objectContaining({ maxTokensPerStep: 4 }),
+			);
+		} finally {
+			if (prior === undefined) {
+				delete process.env.ELIZA_LOCAL_STREAM_TOKENS_PER_STEP;
+			} else {
+				process.env.ELIZA_LOCAL_STREAM_TOKENS_PER_STEP = prior;
+			}
+		}
+	});
+
+	it("passes hardware-aware load args through desktop lazy assignment loads", async () => {
+		const installed = {
+			id: "eliza-1-2b",
+			path: "/models/eliza-1-2b.gguf",
+		};
+		const resolved = {
+			...installed,
+			modelPath: installed.path,
+			contextSize: 32_768,
+		};
+		assignmentsState.assignments = { TEXT_SMALL: installed.id };
+		registryState.installed = [installed];
+		engineState.hasLoadedModel.mockReturnValue(true);
+		vi.mocked(resolveLocalInferenceLoadArgs).mockResolvedValueOnce(
+			resolved as never,
+		);
+		const { registrations, runtime } = makeRuntime();
+
+		await ensureLocalInferenceHandler(runtime);
+		const handler = findRegisteredHandler(registrations, ModelType.TEXT_SMALL);
+
+		await handler(runtime, {
+			messages: [{ role: "user", content: "hello" }],
+		});
+
+		expect(probeHardware).toHaveBeenCalledTimes(1);
+		expect(resolveLocalInferenceLoadArgs).toHaveBeenCalledWith(
+			installed,
+			undefined,
+			{ hardware: hardwareState.probe },
+		);
+		expect(engineState.load).toHaveBeenCalledWith(installed.path, resolved);
+	});
+
 	it.each([
 		[ModelType.TEXT_SMALL, "TEXT_SMALL"],
 		[ModelType.TEXT_LARGE, "TEXT_LARGE"],
@@ -348,6 +452,8 @@ describe("ensureLocalInferenceHandler", () => {
 
 	it("routes image description through the Eliza-1 vision arbiter", async () => {
 		const { registrations, runtime } = makeRuntime();
+		const signal = new AbortController().signal;
+		const onStreamChunk = vi.fn();
 
 		await ensureLocalInferenceHandler(runtime);
 		const registration = registrations.find(
@@ -365,24 +471,65 @@ describe("ensureLocalInferenceHandler", () => {
 			handler?.(runtime, {
 				imageUrl: "data:image/png;base64,AAAA",
 				prompt: "describe this",
+				stream: true,
+				signal,
+				onStreamChunk,
 			}),
 		).resolves.toEqual({
 			title: "A small image",
 			description: "A tiny synthetic image.",
 		});
 		expect(arbiterState.requestVisionDescribe).toHaveBeenCalledWith({
-			modelKey: "qwen3-vl",
+			modelKey: "gemma-vl",
 			payload: {
 				image: { kind: "dataUrl", dataUrl: "data:image/png;base64,AAAA" },
 				prompt: "describe this",
+				signal,
+				onTextChunk: expect.any(Function),
 			},
 		});
+		const payload = arbiterState.requestVisionDescribe.mock.calls[0]?.[0]
+			?.payload as { onTextChunk?: (chunk: string) => void | Promise<void> };
+		await payload.onTextChunk?.("token");
+		expect(onStreamChunk).toHaveBeenCalledWith("token");
 		expect(runtime.setSetting).toHaveBeenCalledWith(
 			"ELIZA1_VISION_HANDLER_PRESENT",
 			"1",
 		);
 	});
 
+	it("keeps image description buffered unless stream is explicitly true", async () => {
+		const { registrations, runtime } = makeRuntime();
+		const onStreamChunk = vi.fn();
+
+		await ensureLocalInferenceHandler(runtime);
+		const registration = registrations.find(
+			(entry) => entry.modelType === ModelType.IMAGE_DESCRIPTION,
+		);
+		const handler = registration?.handler as
+			| ((
+					runtime: AgentRuntime,
+					params: Record<string, unknown>,
+			  ) => Promise<{ title: string; description: string }>)
+			| undefined;
+		expect(handler).toBeDefined();
+
+		await handler?.(runtime, {
+			imageUrl: "https://example.test/image.png",
+			prompt: "describe this",
+			onStreamChunk,
+		});
+
+		expect(arbiterState.requestVisionDescribe).toHaveBeenCalledWith({
+			modelKey: "gemma-vl",
+			payload: {
+				image: { kind: "url", url: "https://example.test/image.png" },
+				prompt: "describe this",
+			},
+		});
+		expect(onStreamChunk).not.toHaveBeenCalled();
+	});
+
 	it("arms the active voice bundle before TRANSCRIPTION", async () => {
 		const { registrations, runtime } = makeRuntime();
 
@@ -402,10 +549,12 @@ describe("ensureLocalInferenceHandler", () => {
 			handler?.(runtime, { audio: new Uint8Array([82, 73, 70, 70]) }),
 		).resolves.toBe("transcribed");
 
-		expect(engineState.ensureActiveBundleVoiceReady).toHaveBeenCalledTimes(1);
+		expect(engineState.ensureActiveBundleAsrReady).toHaveBeenCalledTimes(1);
+		expect(engineState.ensureActiveBundleVoiceReady).not.toHaveBeenCalled();
 		expect(engineState.transcribePcm).toHaveBeenCalledWith(
 			{ pcm: new Float32Array([0]), sampleRate: 16_000 },
 			undefined,
+			undefined,
 		);
 	});
 
@@ -413,7 +562,7 @@ describe("ensureLocalInferenceHandler", () => {
 		// The fused libelizainference ASR runtime is the sole on-device
 		// transcriber. A startup failure must propagate (AGENTS.md §3) — there is
 		// no whisper.cpp second attempt and no silent empty transcript.
-		engineState.ensureActiveBundleVoiceReady.mockRejectedValueOnce(
+		engineState.ensureActiveBundleAsrReady.mockRejectedValueOnce(
 			new VoiceStartupError("missing-bundle-root", "no bundle"),
 		);
 		const { registrations, runtime } = makeRuntime();
@@ -434,6 +583,7 @@ describe("ensureLocalInferenceHandler", () => {
 			handler?.(runtime, { audio: new Uint8Array([82, 73, 70, 70]) }),
 		).rejects.toThrow(VoiceStartupError);
 
+		expect(engineState.ensureActiveBundleAsrReady).toHaveBeenCalledTimes(1);
 		expect(engineState.transcribePcm).not.toHaveBeenCalled();
 	});
 
@@ -504,6 +654,54 @@ describe("ensureLocalInferenceHandler", () => {
 		expect(received.length).toBeGreaterThan(1);
 	});
 
+	it("wires onTextChunk for a plain (non-structured) stream request", async () => {
+		// The chat path can ask for token streaming via `stream: true` without a
+		// response skeleton. The handler must still bridge onStreamChunk →
+		// onTextChunk so cloud-parity token streaming works for the local model.
+		const { registrations, runtime } = makeRuntime();
+		engineState.hasLoadedModel.mockReturnValue(true);
+
+		await ensureLocalInferenceHandler(runtime);
+		const handler = findRegisteredHandler(
+			registrations,
+			ModelType.RESPONSE_HANDLER,
+		);
+
+		await handler(runtime, {
+			messages: [{ role: "user", content: "hello" }],
+			stream: true,
+			onStreamChunk: vi.fn(),
+		});
+
+		expect(engineState.generate).toHaveBeenCalledWith(
+			expect.objectContaining({ onTextChunk: expect.any(Function) }),
+		);
+	});
+
+	it("does not wire onTextChunk for a non-streaming request", async () => {
+		// Non-streaming callers must not pay the per-chunk callback overhead:
+		// engineGenerateArgsFromParams only bridges the callback when the caller
+		// asked for streaming (`stream` or `streamStructured`).
+		const { registrations, runtime } = makeRuntime();
+		engineState.hasLoadedModel.mockReturnValue(true);
+
+		await ensureLocalInferenceHandler(runtime);
+		const handler = findRegisteredHandler(
+			registrations,
+			ModelType.RESPONSE_HANDLER,
+		);
+
+		await handler(runtime, {
+			messages: [{ role: "user", content: "hello" }],
+			onStreamChunk: vi.fn(),
+		});
+
+		const args = engineState.generate.mock.calls.at(-1)?.[0] as {
+			onTextChunk?: unknown;
+		};
+		expect(args.onTextChunk).toBeUndefined();
+	});
+
 	it("threads eliza thinking provider options into local engine args", async () => {
 		const { registrations, runtime } = makeRuntime();
 		engineState.hasLoadedModel.mockReturnValue(true);

package/src/runtime/ensure-local-inference-handler.ts

@@ -57,6 +57,7 @@ import {
 import { deviceBridge } from "../services/device-bridge";
 import { localInferenceEngine } from "../services/engine";
 import { handlerRegistry } from "../services/handler-registry";
+import { probeHardware } from "../services/hardware";
 import { tryGetMemoryArbiter } from "../services/memory-arbiter";
 import { listInstalledModels } from "../services/registry";
 import { installRouterHandler } from "../services/router-handler";
@@ -270,10 +271,18 @@ async function ensureAssignedModelLoaded(
 	}
 
 	if (loader) {
+		const hardware = await probeHardware();
+		const resolved = await resolveLocalInferenceLoadArgs(target, undefined, {
+			hardware,
+		});
 		await loader.unloadModel();
-		await loader.loadModel(await resolveLocalInferenceLoadArgs(target));
+		await loader.loadModel(resolved);
 	} else {
-		await localInferenceEngine.load(target.path);
+		const hardware = await probeHardware();
+		const resolved = await resolveLocalInferenceLoadArgs(target, undefined, {
+			hardware,
+		});
+		await localInferenceEngine.load(target.path, resolved);
 	}
 }
 
@@ -337,6 +346,26 @@ function extractThinkingControl(
  * honours them (the forced-span / prefill / grammar / prefill-plan path is
  * local-model-only).
  */
+/**
+ * Per-step token cap for USER-VISIBLE local streaming (chat replies).
+ *
+ * Benchmarked on the fused eliza-1 model (#9174): the per-`llmStreamNext` step
+ * carries a large fixed FFI overhead, so the throughput↔smoothness curve has a
+ * knee around 8 — `8` yields ~10 UI updates per 80 tokens (clearly streaming)
+ * at a modest decode-throughput cost, whereas 1–4 fall off a throughput cliff
+ * and 16–32 look jumpy. Internal / planner / voice calls do NOT set this and
+ * keep the coarse, throughput-tuned runner default (32). Overridable via the
+ * shared `ELIZA_LOCAL_STREAM_TOKENS_PER_STEP` env knob; the runner clamps it.
+ */
+const DEFAULT_CHAT_STREAM_TOKENS_PER_STEP = 8;
+function resolveChatStreamTokensPerStep(): number {
+	const raw = process.env.ELIZA_LOCAL_STREAM_TOKENS_PER_STEP?.trim();
+	const parsed = raw ? Number.parseInt(raw, 10) : Number.NaN;
+	return Number.isFinite(parsed) && parsed > 0
+		? parsed
+		: DEFAULT_CHAT_STREAM_TOKENS_PER_STEP;
+}
+
 function engineGenerateArgsFromParams(
 	params: GenerateTextParams,
 	cacheKey: string | undefined,
@@ -356,6 +385,7 @@ function engineGenerateArgsFromParams(
 	spanSamplerPlan?: GenerateTextParams["spanSamplerPlan"];
 	thinking?: "auto" | "on" | "off";
 	onTextChunk?: (chunk: string) => void | Promise<void>;
+	maxTokensPerStep?: number;
 	voiceOutput?: "user-visible" | "internal";
 } {
 	const renderContent = (content: unknown): string => {
@@ -419,6 +449,13 @@ function engineGenerateArgsFromParams(
 		spanSamplerPlan: params.spanSamplerPlan,
 		thinking: extractThinkingControl(params.providerOptions),
 		onTextChunk,
+		// Stream user-visible replies in fine-grained steps so the dashboard
+		// renders token-by-token instead of in ~32-token jumps. Only when
+		// streaming (onTextChunk set) — internal/planner calls keep the coarse,
+		// throughput-tuned default. See resolveChatStreamTokensPerStep (#9174).
+		maxTokensPerStep: onTextChunk
+			? resolveChatStreamTokensPerStep()
+			: undefined,
 		voiceOutput:
 			params.voiceOutput ??
 			(typeof params.onStreamChunk === "function" ? "user-visible" : undefined),
@@ -847,9 +884,32 @@ function makeTranscriptionHandler(): TranscriptionHandler {
 		// The fused libelizainference ASR runtime is the sole on-device
 		// transcriber. A startup/availability failure propagates (AGENTS.md §3) —
 		// there is no whisper.cpp second attempt and no silent empty transcript.
-		await localInferenceEngine.ensureActiveBundleVoiceReady();
+		await localInferenceEngine.ensureActiveBundleAsrReady();
 		throwIfAborted(signal);
-		const transcript = await localInferenceEngine.transcribePcm(audio, signal);
+		// Stream partial transcripts through the same pipe as chat text when the
+		// runtime wired a chunk sink (useModel injects onStreamChunk into local
+		// model params inside a streaming reply turn). The fused streaming-ASR
+		// session surfaces each running partial; we forward the deltas. Read the
+		// sink structurally so this stays robust to the core param type surface.
+		const streamSink =
+			params && typeof params === "object"
+				? (
+						params as {
+							onStreamChunk?: (chunk: string) => void | Promise<void>;
+						}
+					).onStreamChunk
+				: undefined;
+		const onPartial =
+			typeof streamSink === "function"
+				? (delta: string) => {
+						void streamSink(delta);
+					}
+				: undefined;
+		const transcript = await localInferenceEngine.transcribePcm(
+			audio,
+			signal,
+			onPartial,
+		);
 		throwIfAborted(signal);
 		return transcript;
 	};
@@ -858,6 +918,8 @@ function makeTranscriptionHandler(): TranscriptionHandler {
 function paramsToVisionRequest(params: ImageDescriptionParams | string): {
 	image: { kind: "dataUrl"; dataUrl: string } | { kind: "url"; url: string };
 	prompt?: string;
+	signal?: AbortSignal;
+	onTextChunk?: (chunk: string) => void | Promise<void>;
 } {
 	const url = typeof params === "string" ? params : params.imageUrl;
 	if (typeof url !== "string" || url.length === 0) {
@@ -866,10 +928,36 @@ function paramsToVisionRequest(params: ImageDescriptionParams | string): {
 		);
 	}
 	const prompt = typeof params === "object" ? params.prompt : undefined;
+	const signal =
+		typeof params === "object"
+			? (params as { signal?: AbortSignal }).signal
+			: undefined;
+	const wantsStream =
+		typeof params === "object" &&
+		(params as { stream?: boolean }).stream === true;
+	const streamSink =
+		wantsStream && typeof params === "object"
+			? (params as { onStreamChunk?: (chunk: string) => void | Promise<void> })
+					.onStreamChunk
+			: undefined;
+	const onTextChunk =
+		typeof streamSink === "function"
+			? (chunk: string) => streamSink(chunk)
+			: undefined;
 	if (url.startsWith("data:")) {
-		return { image: { kind: "dataUrl", dataUrl: url }, prompt };
+		return {
+			image: { kind: "dataUrl", dataUrl: url },
+			prompt,
+			...(signal ? { signal } : {}),
+			...(onTextChunk ? { onTextChunk } : {}),
+		};
 	}
-	return { image: { kind: "url", url }, prompt };
+	return {
+		image: { kind: "url", url },
+		prompt,
+		...(signal ? { signal } : {}),
+		...(onTextChunk ? { onTextChunk } : {}),
+	};
 }
 
 function normalizeImageDescription(
@@ -948,7 +1036,7 @@ function makeImageDescriptionHandler(): ImageDescriptionHandler {
 		const modelKey =
 			typeof modelKeyCandidate === "string" && modelKeyCandidate
 				? modelKeyCandidate
-				: "qwen3-vl";
+				: "gemma-vl";
 		const request = paramsToVisionRequest(params);
 		const result = await arbiter.requestVisionDescribe<
 			typeof request,
@@ -958,6 +1046,102 @@ function makeImageDescriptionHandler(): ImageDescriptionHandler {
 	};
 }
 
+// ── Bionic-host TRANSCRIPTION / IMAGE_DESCRIPTION (Android GPU delegation) ──
+//
+// On the bionic-delegated path the musl agent can't load the fused
+// libelizainference, so the engine-driven transcriber / memory-arbiter vision
+// paths above can't run here. Instead the audio / image bytes are forwarded to
+// the in-process bionic host over the UDS (op="asr" / op="image"), which runs
+// the fused Gemma ASR + mmproj vision path on the Mali GPU and returns text.
+// This is the same delegation `BionicHostLoader` already does for text
+// generation.
+
+/** The bionic-host loader when registered (exposes transcribe + describeImage). */
+function getBionicHostLoader(runtime: IAgentRuntime): BionicHostLoader | null {
+	const svc = (
+		runtime as { getService?: (name: string) => unknown }
+	).getService?.("localInferenceLoader");
+	if (
+		svc &&
+		typeof (svc as BionicHostLoader).transcribe === "function" &&
+		typeof (svc as BionicHostLoader).describeImage === "function"
+	) {
+		return svc as BionicHostLoader;
+	}
+	return null;
+}
+
+/** Pack a mono fp32 PCM buffer little-endian and base64-encode it for the UDS frame. */
+export function float32ToBase64LE(pcm: Float32Array): string {
+	const buf = Buffer.allocUnsafe(pcm.length * 4);
+	for (let i = 0; i < pcm.length; i++) {
+		buf.writeFloatLE(pcm[i] ?? 0, i * 4);
+	}
+	return buf.toString("base64");
+}
+
+/** Resolve a vision request to base64 image bytes for the bionic host. */
+export async function imageRequestToBase64(image: {
+	kind: "dataUrl" | "url";
+	dataUrl?: string;
+	url?: string;
+}): Promise<string> {
+	if (image.kind === "dataUrl" && image.dataUrl) {
+		const comma = image.dataUrl.indexOf(",");
+		return comma >= 0 ? image.dataUrl.slice(comma + 1) : image.dataUrl;
+	}
+	if (image.kind === "url" && image.url) {
+		const resp = await fetch(image.url);
+		if (!resp.ok) {
+			throw new Error(
+				`[local-inference] IMAGE_DESCRIPTION failed to fetch ${image.url}: ${resp.status}`,
+			);
+		}
+		return Buffer.from(await resp.arrayBuffer()).toString("base64");
+	}
+	throw new Error(
+		"[local-inference] IMAGE_DESCRIPTION could not resolve image bytes",
+	);
+}
+
+function makeBionicTranscriptionHandler(): TranscriptionHandler {
+	return async (runtime, params) => {
+		const signal = extractTranscriptionSignal(params);
+		throwIfAborted(signal);
+		const loader = getBionicHostLoader(runtime);
+		if (!loader) {
+			throw new Error(
+				"[local-inference] bionic-host TRANSCRIPTION requires the bionic-host loader (localInferenceLoader service)",
+			);
+		}
+		const audio = extractTranscriptionAudio(params);
+		throwIfAborted(signal);
+		const transcript = await loader.transcribe({
+			pcmBase64: float32ToBase64LE(audio.pcm),
+			sampleRate: audio.sampleRate,
+		});
+		throwIfAborted(signal);
+		return transcript;
+	};
+}
+
+function makeBionicImageDescriptionHandler(): ImageDescriptionHandler {
+	return async (runtime, params) => {
+		const loader = getBionicHostLoader(runtime);
+		if (!loader) {
+			throw new Error(
+				"[local-inference] bionic-host IMAGE_DESCRIPTION requires the bionic-host loader (localInferenceLoader service)",
+			);
+		}
+		const request = paramsToVisionRequest(params);
+		const description = await loader.describeImage({
+			imageBase64: await imageRequestToBase64(request.image),
+			prompt: request.prompt,
+		});
+		return normalizeImageDescription(description);
+	};
+}
+
 /**
  * Register the device-bridge loader on the runtime. Accepts load/generate
  * calls whether or not a mobile device is currently connected — parked
@@ -1395,25 +1579,33 @@ export async function ensureLocalInferenceHandler(
 		// priority (0). It is the last-resort handler: any cloud / other-plugin
 		// TRANSCRIPTION handler registers above 0 and wins. When the handler
 		// does run, it drives the fused libelizainference ASR runtime — the sole
-		// on-device transcriber (Qwen3-ASR streaming → fused batch interim →
+		// on-device transcriber (Gemma ASR streaming → fused batch interim →
 		// AsrUnavailableError) via the engine's armed voice bridge — see
 		// makeTranscriptionHandler / EngineVoiceBridge.createStreamingTranscriber.
 		// (The old ELIZA_LOCAL_TRANSCRIPTION env gate is removed — voice is a
 		// first-class Eliza-1 surface, not opt-in.)
+		// On the bionic-delegated path the fused lib lives in the app process, not
+		// this musl agent — so transcription + vision must forward audio/image
+		// bytes to the bionic host (op="asr" / op="image") rather than the
+		// in-process engine / memory-arbiter, which can't load the lib here.
 		runtimeWithRegistration.registerModel(
 			ModelType.TRANSCRIPTION,
-			makeTranscriptionHandler(),
+			bionicHostRegistered
+				? makeBionicTranscriptionHandler()
+				: makeTranscriptionHandler(),
 			provider,
 			LOCAL_INFERENCE_PRIORITY,
 		);
 		runtimeWithRegistration.registerModel(
 			ModelType.IMAGE_DESCRIPTION,
-			makeImageDescriptionHandler(),
+			bionicHostRegistered
+				? makeBionicImageDescriptionHandler()
+				: makeImageDescriptionHandler(),
 			provider,
 			LOCAL_INFERENCE_PRIORITY,
 		);
 		logger.info(
-			`[local-inference] Registered ${provider} voice and vision handlers for TEXT_TO_SPEECH / TRANSCRIPTION / IMAGE_DESCRIPTION at priority ${LOCAL_INFERENCE_PRIORITY}`,
+			`[local-inference] Registered ${provider} voice and vision handlers for TEXT_TO_SPEECH / TRANSCRIPTION / IMAGE_DESCRIPTION at priority ${LOCAL_INFERENCE_PRIORITY}${bionicHostRegistered ? " (bionic-host delegated)" : ""}`,
 		);
 	} catch (err) {
 		logger.warn(

package/src/runtime/index.ts

@@ -23,7 +23,10 @@ export {
 export { detectEmbeddingPreset } from "./embedding-presets.js";
 export { shouldWarmupLocalEmbeddingModel } from "./embedding-warmup-policy.js";
 export { ensureLocalInferenceHandler } from "./ensure-local-inference-handler.js";
-export { shouldEnableMobileLocalInference } from "./mobile-local-inference-gate.js";
+export {
+	shouldEnableMobileLocalInference,
+	warnIfMobileGateActiveWithoutPlatform,
+} from "./mobile-local-inference-gate.js";
 export {
 	type EmitVoiceTurnObservedArgs,
 	emitVoiceTurnObserved,