@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/bionic-host-loader.test.ts

@@ -0,0 +1,226 @@
+import fs from "node:fs";
+import net from "node:net";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { BionicHostLoader } from "./bionic-host-loader";
+
+/**
+ * Real-IPC test: stand up an actual abstract-namespace AF_UNIX server (the same
+ * transport ElizaBionicInferenceServer.java binds on the device) and drive the
+ * loader against it. No mocks — this exercises the real node:net framing.
+ */
+
+function frame(json: string): Buffer {
+	const payload = Buffer.from(json, "utf8");
+	const out = Buffer.allocUnsafe(4 + payload.length);
+	out.writeUInt32BE(payload.length, 0);
+	payload.copy(out, 4);
+	return out;
+}
+
+/** A test host that decodes one request frame and replies with `respond(req)`. */
+function startHost(
+	name: string,
+	respond: (req: Record<string, unknown>) => string,
+): net.Server {
+	const server = net.createServer((sock) => {
+		let buf = Buffer.alloc(0);
+		let expected = -1;
+		sock.on("data", (d) => {
+			buf = Buffer.concat([buf, d]);
+			if (expected < 0 && buf.length >= 4) expected = buf.readUInt32BE(0);
+			if (expected >= 0 && buf.length >= 4 + expected) {
+				const req = JSON.parse(buf.subarray(4, 4 + expected).toString("utf8"));
+				sock.write(frame(respond(req)));
+			}
+		});
+	});
+	server.listen({ path: `\0${name}` });
+	return server;
+}
+
+let host: net.Server | null = null;
+const tempDirs: string[] = [];
+afterEach(() => {
+	host?.close();
+	host = null;
+	for (const dir of tempDirs.splice(0)) {
+		fs.rmSync(dir, { recursive: true, force: true });
+	}
+});
+
+const SOCK = `eliza-bionic-test-${process.pid}`;
+const describeLinuxOnly =
+	process.platform === "linux" ? describe : describe.skip;
+
+function makeBundleModelPath(manifest: unknown = {}): string {
+	const bundleRoot = fs.mkdtempSync(
+		path.join(os.tmpdir(), "eliza-bionic-bundle-"),
+	);
+	tempDirs.push(bundleRoot);
+	fs.mkdirSync(path.join(bundleRoot, "text"), { recursive: true });
+	fs.mkdirSync(path.join(bundleRoot, "asr"), { recursive: true });
+	fs.writeFileSync(path.join(bundleRoot, "asr", "gemma-asr.gguf"), "asr");
+	fs.writeFileSync(
+		path.join(bundleRoot, "eliza-1.manifest.json"),
+		JSON.stringify(manifest),
+	);
+	return path.join(bundleRoot, "text", "model.gguf");
+}
+
+describeLinuxOnly("BionicHostLoader (real abstract-UDS)", () => {
+	it("round-trips a buffered generate and returns the host completion", async () => {
+		let seen: Record<string, unknown> | null = null;
+		host = startHost(SOCK, (req) => {
+			seen = req;
+			return JSON.stringify({
+				ok: true,
+				text: "Two plus two equals four.",
+				tokens: 7,
+				ms: 500,
+				tokS: 14,
+			});
+		});
+		const loader = new BionicHostLoader(SOCK);
+		await loader.loadModel({
+			modelPath: "/data/x/eliza-1/bundle/text/model.gguf",
+		});
+		expect(loader.currentModelPath()).toBe(
+			"/data/x/eliza-1/bundle/text/model.gguf",
+		);
+		const out = await loader.generate({
+			prompt: "what is 2+2?",
+			maxTokens: 32,
+		});
+		expect(out).toBe("Two plus two equals four.");
+		// bundleDir derived from the .../text/<model>.gguf layout.
+		expect(seen).toMatchObject({
+			op: "generate",
+			prompt: "what is 2+2?",
+			maxTokens: 32,
+			bundleDir: "/data/x/eliza-1/bundle",
+		});
+	});
+
+	it("forwards an empty bundleDir when the model is not in a text/ bundle", async () => {
+		let seen: Record<string, unknown> | null = null;
+		host = startHost(SOCK, (req) => {
+			seen = req;
+			return JSON.stringify({ ok: true, text: "hi" });
+		});
+		const loader = new BionicHostLoader(SOCK);
+		await loader.loadModel({ modelPath: "/models/flat-model.gguf" });
+		await loader.generate({ prompt: "hi" });
+		expect((seen as { bundleDir?: string } | null)?.bundleDir).toBe("");
+	});
+
+	it("throws when the host returns ok:false", async () => {
+		host = startHost(SOCK, () =>
+			JSON.stringify({ ok: false, error: "no vulkan device" }),
+		);
+		const loader = new BionicHostLoader(SOCK);
+		await loader.loadModel({ modelPath: "/m/text/x.gguf" });
+		await expect(loader.generate({ prompt: "x" })).rejects.toThrow(
+			/no vulkan device/,
+		);
+	});
+
+	it("survives a response split across multiple data chunks (multibyte safe)", async () => {
+		const text = `héllo 🌊 ünïcode ${"x".repeat(5000)}`;
+		host = net.createServer((sock) => {
+			let buf = Buffer.alloc(0);
+			let expected = -1;
+			sock.on("data", (d) => {
+				buf = Buffer.concat([buf, d]);
+				if (expected < 0 && buf.length >= 4) expected = buf.readUInt32BE(0);
+				if (expected >= 0 && buf.length >= 4 + expected) {
+					const full = frame(JSON.stringify({ ok: true, text }));
+					// Write in two pieces, splitting mid-buffer to exercise reassembly.
+					sock.write(full.subarray(0, 10));
+					setTimeout(() => sock.write(full.subarray(10)), 5);
+				}
+			});
+		});
+		host.listen({ path: `\0${SOCK}` });
+		const loader = new BionicHostLoader(SOCK);
+		await loader.loadModel({ modelPath: "/m/text/x.gguf" });
+		const out = await loader.generate({ prompt: "x" });
+		expect(out).toBe(text);
+	});
+
+	it("rejects when the host is unreachable", async () => {
+		const loader = new BionicHostLoader(`eliza-bionic-absent-${process.pid}`);
+		await loader.loadModel({ modelPath: "/m/text/x.gguf" });
+		await expect(loader.generate({ prompt: "x" })).rejects.toThrow();
+	});
+
+	it("transcribe forwards op=asr with pcm + sampleRate and returns the transcript", async () => {
+		let seen: Record<string, unknown> | null = null;
+		host = startHost(SOCK, (req) => {
+			seen = req;
+			return JSON.stringify({ ok: true, text: "the quick brown fox" });
+		});
+		const loader = new BionicHostLoader(SOCK);
+		const modelPath = makeBundleModelPath();
+		await loader.loadModel({ modelPath });
+		const out = await loader.transcribe({
+			pcmBase64: "AAAA",
+			sampleRate: 16000,
+		});
+		expect(out).toBe("the quick brown fox");
+		expect(seen).toMatchObject({
+			op: "asr",
+			pcmBase64: "AAAA",
+			sampleRate: 16000,
+			bundleDir: path.dirname(path.dirname(modelPath)),
+		});
+	});
+
+	it("transcribe refuses Qwen ASR provenance before contacting the host", async () => {
+		const loader = new BionicHostLoader(SOCK);
+		await loader.loadModel({
+			modelPath: makeBundleModelPath({
+				lineage: { asr: { base: "Qwen3-ASR" } },
+			}),
+		});
+		await expect(
+			loader.transcribe({ pcmBase64: "AAAA", sampleRate: 16000 }),
+		).rejects.toThrow(/Qwen ASR provenance/);
+	});
+
+	it("describeImage forwards op=image with bytes + prompt and returns the description", async () => {
+		let seen: Record<string, unknown> | null = null;
+		host = startHost(SOCK, (req) => {
+			seen = req;
+			return JSON.stringify({ ok: true, text: "a cat on a desk" });
+		});
+		const loader = new BionicHostLoader(SOCK);
+		await loader.loadModel({
+			modelPath: "/data/x/eliza-1/bundle/text/model.gguf",
+		});
+		const out = await loader.describeImage({
+			imageBase64: "iVBORw0K",
+			prompt: "what is this?",
+		});
+		expect(out).toBe("a cat on a desk");
+		expect(seen).toMatchObject({
+			op: "image",
+			imageBase64: "iVBORw0K",
+			prompt: "what is this?",
+			mmprojPath: "",
+			bundleDir: "/data/x/eliza-1/bundle",
+		});
+	});
+
+	it("transcribe throws on host ok:false", async () => {
+		host = startHost(SOCK, () =>
+			JSON.stringify({ ok: false, error: "no asr weights staged" }),
+		);
+		const loader = new BionicHostLoader(SOCK);
+		await loader.loadModel({ modelPath: makeBundleModelPath() });
+		await expect(
+			loader.transcribe({ pcmBase64: "AAAA", sampleRate: 16000 }),
+		).rejects.toThrow(/no asr weights staged/);
+	});
+});

package/src/services/bionic-host-loader.ts

@@ -0,0 +1,252 @@
+/**
+ * BionicHostLoader — the agent-side half of the on-device GPU delegation path.
+ *
+ * On Android the elizaOS agent runs as embedded bun under the musl loader, whose
+ * restricted linker namespace cannot load the bionic Android Vulkan driver (its
+ * HIDL/HAL closure) — so the musl agent can only run inference on the CPU. The
+ * GPU is reachable only from the normal bionic `ai.elizaos.app` process, where
+ * `ElizaBionicInferenceServer` (Java) has loaded `libelizainference.so` +
+ * `libggml-vulkan.so` and offloads the model to the Mali GPU.
+ *
+ * This loader implements the standard {@link LocalInferenceLoader} contract, so
+ * the TEXT_SMALL / TEXT_LARGE handlers in `ensure-local-inference-handler.ts`
+ * route through it transparently. `generate()` sends the prompt to the bionic
+ * host over an abstract-namespace `AF_UNIX` socket and gets the GPU completion
+ * back — the whole decode loop runs server-side, so there is no per-token
+ * two-process round trip.
+ *
+ * This is the buffered first slice (one GENERATE request → one full completion).
+ * Server-push per-step streaming, embed, and cancel are layered on later via the
+ * shared `LlmStreamingBinding`; the wire framing already carries an `op`
+ * discriminator for that.
+ */
+
+import net from "node:net";
+import path from "node:path";
+import { logger } from "@elizaos/core";
+import type {
+	LocalInferenceLoadArgs,
+	LocalInferenceLoader,
+} from "./active-model";
+import {
+	bundleHasAsrModelFiles,
+	readBundleAsrProvenanceBlockers,
+} from "./asr-provenance";
+
+/** Connect + full round-trip budget. A cold GPU decode of a long reply fits. */
+const REQUEST_TIMEOUT_MS = 120_000;
+/** Defensive ceiling on a single response frame (a full completion). */
+const MAX_FRAME_BYTES = 64 * 1024 * 1024;
+
+interface BionicGenerateResponse {
+	ok: boolean;
+	text?: string;
+	error?: string;
+	tokens?: number;
+	ms?: number;
+	tokS?: number;
+}
+
+/** {ok, text} response for the asr / image ops (transcript / description). */
+interface BionicTextResponse {
+	ok: boolean;
+	text?: string;
+	error?: string;
+}
+
+/**
+ * Derive the fused-bundle root from a model GGUF path. The host's
+ * `eliza_inference_create(bundleDir)` expects the directory that contains
+ * `text/<model>.gguf`; when the installed model is laid out that way we forward
+ * it, otherwise we send empty and let the host fall back to its default bundle.
+ */
+function deriveBundleDir(modelPath: string): string {
+	if (!modelPath) return "";
+	const dir = path.dirname(modelPath);
+	if (path.basename(dir) === "text") return path.dirname(dir);
+	return "";
+}
+
+export class BionicHostLoader implements LocalInferenceLoader {
+	private modelPath: string | null = null;
+	private bundleDir = "";
+
+	/** @param socketName abstract-namespace socket name (no leading NUL). */
+	constructor(private readonly socketName: string) {}
+
+	async loadModel(args: LocalInferenceLoadArgs): Promise<void> {
+		this.modelPath = args.modelPath;
+		this.bundleDir = deriveBundleDir(args.modelPath);
+		logger.info(
+			`[BionicHostLoader] active model ${args.modelPath} (bundle ${this.bundleDir || "<host-default>"})`,
+		);
+	}
+
+	async unloadModel(): Promise<void> {
+		this.modelPath = null;
+	}
+
+	currentModelPath(): string | null {
+		return this.modelPath;
+	}
+
+	async generate(args: {
+		prompt: string;
+		stopSequences?: string[];
+		maxTokens?: number;
+		temperature?: number;
+		cacheKey?: string;
+	}): Promise<string> {
+		const res = await this.roundTrip<BionicGenerateResponse>({
+			op: "generate",
+			bundleDir: this.bundleDir,
+			prompt: args.prompt,
+			maxTokens: args.maxTokens ?? 256,
+			temperature: args.temperature ?? 0,
+		});
+		if (!res.ok) {
+			throw new Error(
+				`[BionicHostLoader] host generate failed: ${res.error ?? "unknown error"}`,
+			);
+		}
+		if (typeof res.tokS === "number") {
+			logger.debug(
+				`[BionicHostLoader] generated ${res.tokens ?? "?"} tok @ ${res.tokS.toFixed(1)} tok/s on the bionic GPU host`,
+			);
+		}
+		return res.text ?? "";
+	}
+
+	/**
+	 * On-device STT: transcribe mono fp32 PCM via the bionic host's fused
+	 * Gemma ASR path (op="asr"). The musl agent can't load the fused lib, so
+	 * the TRANSCRIPTION delegate routes the audio here over the UDS and gets
+	 * the transcript back. `pcm` is little-endian fp32 already base64-encoded.
+	 */
+	async transcribe(args: {
+		pcmBase64: string;
+		sampleRate: number;
+	}): Promise<string> {
+		if (!this.bundleDir || !bundleHasAsrModelFiles(this.bundleDir)) {
+			throw new Error(
+				"[BionicHostLoader] host asr requires an active Gemma ASR-capable bundle; refusing to use the bionic host default bundle",
+			);
+		}
+		const blockers = readBundleAsrProvenanceBlockers(this.bundleDir);
+		if (blockers.length > 0) {
+			throw new Error(
+				`[BionicHostLoader] host asr refused non-Gemma ASR provenance: ${blockers.join("; ")}`,
+			);
+		}
+		const res = await this.roundTrip<BionicTextResponse>({
+			op: "asr",
+			bundleDir: this.bundleDir,
+			pcmBase64: args.pcmBase64,
+			sampleRate: args.sampleRate,
+		});
+		if (!res.ok) {
+			throw new Error(
+				`[BionicHostLoader] host asr failed: ${res.error ?? "unknown error"}`,
+			);
+		}
+		return res.text ?? "";
+	}
+
+	/**
+	 * On-device vision / screen-recognition: describe a raw image (PNG/JPEG/WebP
+	 * bytes, base64) via the bionic host's mmproj describe-image (op="image").
+	 * `mmprojPath` may be empty — the host resolves the projector from the
+	 * bundle's `vision/` dir.
+	 */
+	async describeImage(args: {
+		imageBase64: string;
+		mmprojPath?: string;
+		prompt?: string;
+	}): Promise<string> {
+		const res = await this.roundTrip<BionicTextResponse>({
+			op: "image",
+			bundleDir: this.bundleDir,
+			imageBase64: args.imageBase64,
+			mmprojPath: args.mmprojPath ?? "",
+			prompt: args.prompt ?? "",
+		});
+		if (!res.ok) {
+			throw new Error(
+				`[BionicHostLoader] host image describe failed: ${res.error ?? "unknown error"}`,
+			);
+		}
+		return res.text ?? "";
+	}
+
+	/**
+	 * One request → one response over a fresh connection. Length-prefixed frames:
+	 * `[int32 BE byte length][UTF-8 JSON]` in each direction.
+	 */
+	private roundTrip<T>(request: Record<string, unknown>): Promise<T> {
+		const payload = Buffer.from(JSON.stringify(request), "utf8");
+		const frame = Buffer.allocUnsafe(4 + payload.length);
+		frame.writeUInt32BE(payload.length, 0);
+		payload.copy(frame, 4);
+
+		return new Promise<T>((resolve, reject) => {
+			// Abstract-namespace socket: a leading NUL byte in the path.
+			const sock = net.connect({ path: `\0${this.socketName}` });
+			let settled = false;
+			let chunks: Buffer = Buffer.alloc(0);
+			let expected = -1;
+
+			const finish = (err: Error | null, value?: T) => {
+				if (settled) return;
+				settled = true;
+				clearTimeout(timer);
+				sock.destroy();
+				if (err) reject(err);
+				else resolve(value as T);
+			};
+
+			const timer = setTimeout(
+				() => finish(new Error("[BionicHostLoader] request timed out")),
+				REQUEST_TIMEOUT_MS,
+			);
+
+			sock.on("connect", () => sock.write(frame));
+			sock.on("data", (d: Buffer) => {
+				chunks = Buffer.concat([chunks, d]);
+				if (expected < 0 && chunks.length >= 4) {
+					expected = chunks.readUInt32BE(0);
+					if (expected < 0 || expected > MAX_FRAME_BYTES) {
+						finish(
+							new Error(
+								`[BionicHostLoader] bad response frame length ${expected}`,
+							),
+						);
+						return;
+					}
+				}
+				if (expected >= 0 && chunks.length >= 4 + expected) {
+					const json = chunks.subarray(4, 4 + expected).toString("utf8");
+					try {
+						finish(null, JSON.parse(json) as T);
+					} catch (e) {
+						finish(
+							new Error(
+								`[BionicHostLoader] malformed response: ${e instanceof Error ? e.message : String(e)}`,
+							),
+						);
+					}
+				}
+			});
+			sock.on("error", (e: Error) =>
+				finish(new Error(`[BionicHostLoader] socket error: ${e.message}`)),
+			);
+			sock.on("close", () => {
+				if (!settled)
+					finish(
+						new Error(
+							"[BionicHostLoader] host closed the connection before responding",
+						),
+					);
+			});
+		});
+	}
+}

package/src/services/bundled-models.ts

@@ -0,0 +1,129 @@
+/**
+ * Bundled-models bootstrap for AOSP / on-device installs.
+ *
+ * The AOSP build pipeline stages Eliza-1 models into the APK at
+ * `assets/agent/models/{file}.gguf` plus a
+ * `manifest.json` describing each one (id, role, sha256, sizeBytes).
+ * `ElizaAgentService.extractAssetsIfNeeded()` copies those files into
+ * `$ELIZA_STATE_DIR/local-inference/models/` on first launch.
+ *
+ * This module reads the manifest at runtime startup and registers each
+ * file as a eliza-owned model in the local-inference registry, so the
+ * auto-assign pass picks them up for TEXT_LARGE / TEXT_SMALL /
+ * TEXT_EMBEDDING slots without needing the user to download anything.
+ *
+ * Idempotent: re-running with the registry already populated is a
+ * no-op for unchanged entries (`upsertElizaModel` overwrites entries
+ * with the same id, so updated sha256s on a later re-bundle replace
+ * the old metadata cleanly).
+ *
+ * Source classification: the runtime treats bundled models as
+ * `source: "eliza-download"` because Eliza ships the file and Eliza
+ * owns it on disk — same lifecycle as a user-initiated download
+ * (uninstall removes the file, the registry tracks the install). The
+ * only difference is the file arrived via APK extraction rather than
+ * an HTTP transfer.
+ */
+
+import fs from "node:fs/promises";
+import path from "node:path";
+import {
+	ensureDefaultAssignment,
+	readAssignments,
+	writeAssignments,
+} from "./assignments";
+import { elizaModelsDir } from "./paths";
+import { upsertElizaModel } from "./registry";
+import type { InstalledModel } from "./types";
+
+interface BundledModelEntry {
+	id: string;
+	displayName: string;
+	hfRepo: string;
+	ggufFile: string;
+	role: "chat" | "embedding";
+	sizeBytes: number;
+	sha256: string | null;
+}
+
+interface BundledModelManifest {
+	version: 1;
+	models: BundledModelEntry[];
+}
+
+function manifestPath(): string {
+	return path.join(elizaModelsDir(), "manifest.json");
+}
+
+async function readManifest(): Promise<BundledModelManifest | null> {
+	try {
+		const raw = await fs.readFile(manifestPath(), "utf8");
+		const parsed = JSON.parse(raw) as BundledModelManifest;
+		if (parsed.version !== 1 || !Array.isArray(parsed.models)) {
+			return null;
+		}
+		return parsed;
+	} catch {
+		return null;
+	}
+}
+
+async function ensureBundledAssignment(
+	modelId: string,
+	role: BundledModelEntry["role"],
+): Promise<void> {
+	if (role !== "embedding") {
+		await ensureDefaultAssignment(modelId);
+		return;
+	}
+
+	const current = await readAssignments();
+	if (current.TEXT_EMBEDDING) return;
+	await writeAssignments({ ...current, TEXT_EMBEDDING: modelId });
+}
+
+/**
+ * Walk the manifest and register every bundled GGUF file in the
+ * local-inference registry. Returns the number of entries successfully
+ * registered. A missing manifest is normal on Capacitor / desktop /
+ * non-AOSP installs and returns 0 silently.
+ */
+export async function registerBundledModels(): Promise<number> {
+	const manifest = await readManifest();
+	if (!manifest) return 0;
+	const dir = elizaModelsDir();
+	let registered = 0;
+	for (const entry of manifest.models) {
+		const filePath = path.join(dir, entry.ggufFile);
+		let sizeBytes = entry.sizeBytes;
+		try {
+			const stat = await fs.stat(filePath);
+			sizeBytes = stat.size;
+		} catch {
+			// File didn't extract — manifest references something the APK
+			// didn't ship. Skip this entry rather than registering a broken
+			// path. AOSP build's stage-default-models.mjs is the source of
+			// truth; if a file is missing the build is broken upstream.
+			continue;
+		}
+		const installed: InstalledModel = {
+			id: entry.id,
+			displayName: entry.displayName,
+			path: filePath,
+			sizeBytes,
+			hfRepo: entry.hfRepo,
+			installedAt: new Date().toISOString(),
+			lastUsedAt: null,
+			source: "eliza-download",
+			sha256: entry.sha256 ?? undefined,
+		};
+		await upsertElizaModel(installed);
+		// Auto-assign each bundled model to its manifest role if the user
+		// hasn't already assigned that slot. This keeps Eliza-1 chat models
+		// eligible for TEXT_EMBEDDING when the bundle explicitly marks them
+		// as the local embedding bootstrap model.
+		await ensureBundledAssignment(entry.id, entry.role);
+		registered += 1;
+	}
+	return registered;
+}