@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/system-memory.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, it } from "vitest";
+import { readSystemMemory } from "./system-memory.js";
+
+const SAMPLE_MEMINFO = `MemTotal:       16277856 kB
+MemFree:          812044 kB
+MemAvailable:   10342988 kB
+Buffers:          204512 kB
+Cached:          7651200 kB
+SwapTotal:             0 kB
+`;
+
+describe("readSystemMemory", () => {
+	it("prefers MemAvailable + MemTotal from /proc/meminfo", () => {
+		const mem = readSystemMemory(() => SAMPLE_MEMINFO);
+		expect(mem.totalBytes).toBe(16_277_856 * 1024);
+		// MemAvailable (10.3 GB), NOT MemFree (0.8 GB) — the whole point.
+		expect(mem.freeBytes).toBe(10_342_988 * 1024);
+	});
+
+	it("does not regress to MemFree when reclaimable cache is large", () => {
+		const mem = readSystemMemory(() => SAMPLE_MEMINFO);
+		const memFreeBytes = 812_044 * 1024;
+		expect(mem.freeBytes).toBeGreaterThan(memFreeBytes * 10);
+	});
+
+	it("falls back to os when the reader returns null (non-Linux)", () => {
+		const mem = readSystemMemory(() => null);
+		expect(mem.totalBytes).toBeGreaterThan(0);
+		expect(mem.freeBytes).toBeGreaterThan(0);
+		expect(mem.freeBytes).toBeLessThanOrEqual(mem.totalBytes);
+	});
+
+	it("falls back to os when MemAvailable is absent (pre-3.14 kernel)", () => {
+		const noAvail = "MemTotal:  16277856 kB\nMemFree:  812044 kB\n";
+		const mem = readSystemMemory(() => noAvail);
+		// No MemAvailable → os fallback, so freeBytes is the live os.freemem(),
+		// not the parsed MemFree.
+		expect(mem.totalBytes).toBeGreaterThan(0);
+		expect(mem.freeBytes).toBeGreaterThan(0);
+	});
+
+	it("falls back to os on malformed meminfo", () => {
+		const mem = readSystemMemory(() => "garbage\nnot meminfo\n");
+		expect(mem.totalBytes).toBeGreaterThan(0);
+		expect(mem.freeBytes).toBeGreaterThan(0);
+	});
+});

package/src/services/system-memory.ts

@@ -0,0 +1,67 @@
+/**
+ * System memory reader — the single source of "how much RAM can we actually
+ * allocate right now" for the local-inference memory arbiter and pressure
+ * sources.
+ *
+ * Node's `os.freemem()` returns the kernel's `MemFree` on Linux, which counts
+ * only never-touched pages and EXCLUDES reclaimable page cache + slab. On
+ * Android — where the app process runs under a large page cache — `MemFree`
+ * undercounts allocatable memory by gigabytes, so a `MemFree`-driven arbiter
+ * evicts models it didn't need to and refuses loads that would have fit.
+ *
+ * `/proc/meminfo`'s `MemAvailable` is the kernel's own estimate of how much
+ * memory is available for starting new applications without swapping (free +
+ * reclaimable cache/slab, minus the low watermark). That is exactly the number
+ * the arbiter wants. Read it on Linux/Android; fall back to `os.freemem()` /
+ * `os.totalmem()` everywhere else (macOS, Windows) or if `/proc/meminfo` is
+ * unreadable or pre-3.14 (no `MemAvailable`).
+ */
+
+import { readFileSync } from "node:fs";
+import os from "node:os";
+
+export interface SystemMemory {
+	freeBytes: number;
+	totalBytes: number;
+}
+
+/** Injectable for tests: returns the raw `/proc/meminfo` text, or null. */
+export type MeminfoReader = () => string | null;
+
+const defaultMeminfoReader: MeminfoReader = () => {
+	if (os.platform() !== "linux") return null;
+	try {
+		return readFileSync("/proc/meminfo", "utf8");
+	} catch {
+		return null;
+	}
+};
+
+function parseMeminfoKb(text: string, key: string): number | null {
+	// Lines look like: "MemAvailable:   12345678 kB"
+	const match = new RegExp(`^${key}:\\s+(\\d+)\\s*kB`, "m").exec(text);
+	if (!match) return null;
+	const kb = Number.parseInt(match[1], 10);
+	return Number.isFinite(kb) ? kb : null;
+}
+
+/**
+ * Read available + total system memory in bytes. Prefers `/proc/meminfo`
+ * `MemAvailable`/`MemTotal` on Linux; falls back to `os.freemem()/totalmem()`.
+ *
+ * @param read injectable meminfo reader (tests). Defaults to reading
+ *   `/proc/meminfo` on Linux and returning null elsewhere.
+ */
+export function readSystemMemory(
+	read: MeminfoReader = defaultMeminfoReader,
+): SystemMemory {
+	const text = read();
+	if (text) {
+		const availKb = parseMeminfoKb(text, "MemAvailable");
+		const totalKb = parseMeminfoKb(text, "MemTotal");
+		if (availKb !== null && totalKb !== null && totalKb > 0) {
+			return { freeBytes: availKb * 1024, totalBytes: totalKb * 1024 };
+		}
+	}
+	return { freeBytes: os.freemem(), totalBytes: os.totalmem() };
+}

package/src/services/transcription-priority.test.ts

@@ -0,0 +1,211 @@
+/**
+ * Tests for the `TRANSCRIPTION` model handler priority + provider hint
+ * semantics on `AgentRuntime`. These exercise the same `registerModel`
+ * machinery used by plugin-elizacloud (remote OpenAI Whisper API) and the
+ * local fused Gemma ASR handler so callers know exactly:
+ *
+ *   - which handler wins when priorities tie (first-registered),
+ *   - how an explicit `priority` overrides that,
+ *   - that a `provider` hint to `useModel` overrides priority entirely,
+ *   - that the runtime does NOT auto-fall-through on handler errors,
+ *   - and how a caller can implement an explicit try/catch fallback.
+ *
+ * The test is intentionally provider-agnostic — it registers fake handlers
+ * named "local" and "openai" so the contract is observable without booting
+ * any real ASR backend.
+ */
+
+import { describe, expect, it } from "vitest";
+
+import { InMemoryDatabaseAdapter } from "../../../../packages/core/src/database/inMemoryAdapter";
+import { AgentRuntime } from "../../../../packages/core/src/runtime";
+import { ModelType } from "../../../../packages/core/src/types";
+
+interface TranscriptionParams {
+	audio?: Float32Array | Uint8Array | Buffer | string;
+}
+
+interface TestRuntimeCtx {
+	runtime: AgentRuntime;
+	calls: string[];
+}
+
+function makeRuntime(): TestRuntimeCtx {
+	const runtime = new AgentRuntime({
+		character: {
+			name: "TranscriptionPriorityTest",
+			bio: "asr-priority test",
+			settings: {},
+		} as never,
+		adapter: new InMemoryDatabaseAdapter(),
+		logLevel: "fatal",
+	});
+	const calls: string[] = [];
+	return { runtime, calls };
+}
+
+function makeHandler(
+	calls: string[],
+	label: string,
+	result: string | (() => string),
+	options: { throws?: boolean } = {},
+) {
+	return async (_runtime: unknown, _params: unknown): Promise<string> => {
+		calls.push(label);
+		if (options.throws) {
+			throw new Error(`${label} handler failed`);
+		}
+		return typeof result === "function" ? result() : result;
+	};
+}
+
+describe("TRANSCRIPTION handler priority on AgentRuntime", () => {
+	it("local Gemma ASR wins over remote OpenAI Whisper by registration order when priorities tie", async () => {
+		const { runtime, calls } = makeRuntime();
+		runtime.registerModel(
+			ModelType.TRANSCRIPTION,
+			makeHandler(calls, "local", "local transcript"),
+			"eliza-local-inference",
+			0,
+		);
+		runtime.registerModel(
+			ModelType.TRANSCRIPTION,
+			makeHandler(calls, "openai", "openai transcript"),
+			"openai",
+			0,
+		);
+
+		const text = await runtime.useModel(ModelType.TRANSCRIPTION, {
+			audio: new Float32Array(160),
+		} as TranscriptionParams as never);
+
+		expect(text).toBe("local transcript");
+		expect(calls).toEqual(["local"]);
+	});
+
+	it("explicit higher priority always wins regardless of registration order", async () => {
+		const { runtime, calls } = makeRuntime();
+		runtime.registerModel(
+			ModelType.TRANSCRIPTION,
+			makeHandler(calls, "remote", "remote transcript"),
+			"openai",
+			100,
+		);
+		runtime.registerModel(
+			ModelType.TRANSCRIPTION,
+			makeHandler(calls, "local", "local transcript"),
+			"eliza-local-inference",
+			0,
+		);
+
+		// Remote wins because priority 100 > 0.
+		let text = await runtime.useModel(ModelType.TRANSCRIPTION, {
+			audio: new Float32Array(160),
+		} as TranscriptionParams as never);
+		expect(text).toBe("remote transcript");
+		expect(calls).toEqual(["remote"]);
+
+		// Re-register local with a higher priority — now local wins.
+		runtime.registerModel(
+			ModelType.TRANSCRIPTION,
+			makeHandler(calls, "local-priority", "local-priority transcript"),
+			"eliza-local-inference",
+			200,
+		);
+
+		text = await runtime.useModel(ModelType.TRANSCRIPTION, {
+			audio: new Float32Array(160),
+		} as TranscriptionParams as never);
+		expect(text).toBe("local-priority transcript");
+		expect(calls).toEqual(["remote", "local-priority"]);
+	});
+
+	it("provider hint to useModel overrides priority", async () => {
+		const { runtime, calls } = makeRuntime();
+		runtime.registerModel(
+			ModelType.TRANSCRIPTION,
+			makeHandler(calls, "local", "local transcript"),
+			"eliza-local-inference",
+			200,
+		);
+		runtime.registerModel(
+			ModelType.TRANSCRIPTION,
+			makeHandler(calls, "openai", "openai transcript"),
+			"openai",
+			0,
+		);
+
+		// Default useModel: local wins (priority 200 > 0).
+		let text = await runtime.useModel(ModelType.TRANSCRIPTION, {
+			audio: new Float32Array(160),
+		} as TranscriptionParams as never);
+		expect(text).toBe("local transcript");
+
+		// Explicit provider hint flips it to openai despite the priority gap.
+		text = await runtime.useModel(
+			ModelType.TRANSCRIPTION,
+			{ audio: new Float32Array(160) } as TranscriptionParams as never,
+			"openai",
+		);
+		expect(text).toBe("openai transcript");
+		expect(calls).toEqual(["local", "openai"]);
+	});
+
+	it("runtime does NOT auto-fall-through on handler errors", async () => {
+		const { runtime, calls } = makeRuntime();
+		runtime.registerModel(
+			ModelType.TRANSCRIPTION,
+			makeHandler(calls, "local", "should never reach this", { throws: true }),
+			"eliza-local-inference",
+			200,
+		);
+		runtime.registerModel(
+			ModelType.TRANSCRIPTION,
+			makeHandler(calls, "openai", "openai transcript"),
+			"openai",
+			0,
+		);
+
+		await expect(
+			runtime.useModel(ModelType.TRANSCRIPTION, {
+				audio: new Float32Array(160),
+			} as TranscriptionParams as never),
+		).rejects.toThrow(/local handler failed/);
+		// The remote handler should NOT have been invoked — there is no
+		// implicit fallback chain on the runtime.
+		expect(calls).toEqual(["local"]);
+	});
+
+	it("caller can implement fallback via try/catch + provider hint", async () => {
+		const { runtime, calls } = makeRuntime();
+		runtime.registerModel(
+			ModelType.TRANSCRIPTION,
+			makeHandler(calls, "local", "boom", { throws: true }),
+			"eliza-local-inference",
+			200,
+		);
+		runtime.registerModel(
+			ModelType.TRANSCRIPTION,
+			makeHandler(calls, "openai", "openai transcript"),
+			"openai",
+			0,
+		);
+
+		let result: string;
+		try {
+			result = (await runtime.useModel(ModelType.TRANSCRIPTION, {
+				audio: new Float32Array(160),
+			} as TranscriptionParams as never)) as string;
+		} catch {
+			// Caller-controlled fallback: explicit retry with provider hint.
+			result = (await runtime.useModel(
+				ModelType.TRANSCRIPTION,
+				{ audio: new Float32Array(160) } as TranscriptionParams as never,
+				"openai",
+			)) as string;
+		}
+
+		expect(result).toBe("openai transcript");
+		expect(calls).toEqual(["local", "openai"]);
+	});
+});

package/src/services/types.ts

@@ -0,0 +1,59 @@
+/**
+ * Local inference type re-exports.
+ *
+ * The canonical definitions live in `@elizaos/shared/local-inference`.
+ * This shim preserves the historical import path
+ * `../services/local-inference/types` for server-side code.
+ */
+
+export {
+	type ActiveModelState,
+	AGENT_MODEL_SLOTS,
+	type AgentModelSlot,
+	type CatalogModel,
+	type CatalogQuantizationId,
+	type CatalogQuantizationMatrix,
+	type CatalogQuantizationVariant,
+	type CpuFeatureProbe,
+	classifyCatalogModelRuntimeClass,
+	classifyInstalledModelRuntimeClass,
+	type DownloadEvent,
+	type DownloadJob,
+	type DownloadState,
+	type GpuProfile,
+	type GpuProfileId,
+	type HardwareFitLevel,
+	type HardwareProbe,
+	type InstalledModel,
+	type KvCacheType,
+	type LocalInferenceDownloadStatus,
+	type LocalInferenceReadiness,
+	type LocalInferenceSlotReadiness,
+	type LocalRuntimeAcceleration,
+	type LocalRuntimeBackend,
+	type LocalRuntimeKernel,
+	type LocalRuntimeOptimizations,
+	type MobileHardwareProbe,
+	type ModelAssignments,
+	type ModelBucket,
+	type ModelCategory,
+	type ModelHubSnapshot,
+	type OpenVinoDeviceKind,
+	type OpenVinoHardwareProbe,
+	type RuntimeClass,
+	TEXT_GENERATION_SLOTS,
+	type TextGenerationSlot,
+	type TokenizerFamily,
+	withRuntimeClass,
+} from "@elizaos/shared";
+
+/** RAM requirements for a model bundle. */
+export interface RamBudget {
+	/** Minimum RAM the bundle will boot under, in megabytes. */
+	minMb: number;
+	/** RAM the bundle expects for nominal workloads, in megabytes. */
+	recommendedMb: number;
+	/** Where the numbers came from. `manifest` only when both came from
+	 *  a validated `eliza-1.manifest.json` next to the installed bundle. */
+	source: "manifest" | "catalog";
+}

package/src/services/verify-on-device.test.ts

@@ -0,0 +1,87 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { createVerifyBundleOnDevice } from "./verify-on-device";
+
+const engineMock = {
+	load: vi.fn(async () => {}),
+	generate: vi.fn(async () => "ok"),
+	ensureActiveBundleVoiceReady: vi.fn(async () => ({})),
+	startVoice: vi.fn(() => {}),
+	armVoice: vi.fn(async () => {}),
+	synthesizeSpeech: vi.fn(async () => new Uint8Array([1, 2, 3, 4])),
+	triggerBargeIn: vi.fn(() => {}),
+	stopVoice: vi.fn(async () => {}),
+	unload: vi.fn(async () => {}),
+};
+
+const manifestState: { voiceFiles: number } = { voiceFiles: 0 };
+
+function verifier() {
+	const deps = {
+		engine: engineMock,
+		readFile: vi.fn(async () => "{}"),
+		parseManifest: () => ({
+			files: {
+				voice: Array.from({ length: manifestState.voiceFiles }, (_, i) => ({
+					path: `tts/v${i}.gguf`,
+					sha256: "x",
+				})),
+			},
+		}),
+	} as unknown as Parameters<typeof createVerifyBundleOnDevice>[0];
+	return createVerifyBundleOnDevice(deps);
+}
+
+const ARGS = {
+	modelId: "eliza-1-2b",
+	bundleRoot: "/tmp/bundle",
+	manifestPath: "/tmp/bundle/eliza-1.manifest.json",
+	textGgufPath: "/tmp/bundle/text/eliza-1-2b.gguf",
+};
+
+afterEach(() => {
+	vi.clearAllMocks();
+	manifestState.voiceFiles = 0;
+});
+
+describe("verifyBundleOnDevice", () => {
+	it("loads, runs a 1-token text gen, and unloads for a text-only bundle", async () => {
+		manifestState.voiceFiles = 0;
+		await verifier()(ARGS);
+		expect(engineMock.load).toHaveBeenCalledWith(ARGS.textGgufPath, {
+			modelPath: ARGS.textGgufPath,
+			modelId: ARGS.modelId,
+		});
+		expect(engineMock.generate).toHaveBeenCalledWith(
+			expect.objectContaining({ maxTokens: 1 }),
+		);
+		expect(engineMock.ensureActiveBundleVoiceReady).not.toHaveBeenCalled();
+		expect(engineMock.startVoice).not.toHaveBeenCalled();
+		expect(engineMock.unload).toHaveBeenCalled();
+	});
+
+	it("also runs a 1-phrase voice gen + barge-in cancel when the bundle ships voice", async () => {
+		manifestState.voiceFiles = 1;
+		await verifier()(ARGS);
+		expect(engineMock.ensureActiveBundleVoiceReady).toHaveBeenCalled();
+		expect(engineMock.startVoice).not.toHaveBeenCalled();
+		expect(engineMock.armVoice).not.toHaveBeenCalled();
+		expect(engineMock.synthesizeSpeech).toHaveBeenCalled();
+		expect(engineMock.triggerBargeIn).toHaveBeenCalled();
+		expect(engineMock.stopVoice).toHaveBeenCalled();
+		expect(engineMock.unload).toHaveBeenCalled();
+	});
+
+	it("rethrows verify failures and still unloads", async () => {
+		manifestState.voiceFiles = 0;
+		engineMock.generate.mockRejectedValueOnce(new Error("kernel missing"));
+		await expect(verifier()(ARGS)).rejects.toThrow("kernel missing");
+		expect(engineMock.unload).toHaveBeenCalled();
+	});
+
+	it("fails verify when voice synthesis yields no PCM", async () => {
+		manifestState.voiceFiles = 1;
+		engineMock.synthesizeSpeech.mockResolvedValueOnce(new Uint8Array(0));
+		await expect(verifier()(ARGS)).rejects.toThrow(/no PCM bytes/);
+		expect(engineMock.stopVoice).toHaveBeenCalled();
+	});
+});

package/src/services/verify-on-device.ts

@@ -0,0 +1,127 @@
+/**
+ * Engine-backed `verifyOnDevice` hook for the Eliza-1 downloader
+ * (`packages/inference/AGENTS.md` §7): after a bundle is materialized and
+ * every file's sha256 checks out, run one cold smoke pass —
+ *
+ *   load → 1-token text generation → (if the bundle ships voice assets)
+ *   1-phrase voice generation → barge-in cancel → unload
+ *
+ * — before the bundle is allowed to auto-fill an empty default slot. The
+ * downloader stays decoupled from the engine: it only knows the
+ * {@link VerifyBundleOnDevice} shape; this module is the concrete
+ * implementation the service layer injects.
+ *
+ * Failure semantics: any error throws. The downloader catches it and leaves
+ * `bundleVerifiedAt` unset, so an unverified bundle is registered but does
+ * not become the recommended default. There is no "verified anyway" path —
+ * voice is mandatory for Eliza-1 voice tiers (AGENTS.md §3), so a bundle
+ * whose fused voice ABI is not loadable on this device legitimately fails
+ * verify until the fused build is present.
+ */
+
+import fs from "node:fs/promises";
+import type { VerifyBundleOnDevice } from "./downloader";
+import { localInferenceEngine } from "./engine";
+import { parseManifestOrThrow } from "./manifest";
+
+/** A short, deterministic prompt — we only care that one token comes back. */
+const VERIFY_PROMPT = "Reply with one word.";
+/** A short phrase to drive a single TTS dispatch through the voice scheduler. */
+const VERIFY_PHRASE = "Ready.";
+
+type VerifyEngine = Pick<
+	typeof localInferenceEngine,
+	| "load"
+	| "generate"
+	| "ensureActiveBundleVoiceReady"
+	| "startVoice"
+	| "armVoice"
+	| "synthesizeSpeech"
+	| "triggerBargeIn"
+	| "stopVoice"
+	| "unload"
+>;
+
+interface VerifyBundleOnDeviceDeps {
+	readonly engine: VerifyEngine;
+	readonly readFile: typeof fs.readFile;
+	readonly parseManifest: typeof parseManifestOrThrow;
+}
+
+async function manifestDeclaresVoice(
+	manifestPath: string,
+	deps: Pick<VerifyBundleOnDeviceDeps, "readFile" | "parseManifest">,
+): Promise<boolean> {
+	const raw = await deps.readFile(manifestPath, "utf8");
+	const manifest = deps.parseManifest(JSON.parse(String(raw)));
+	// Voice tiers ship a TTS GGUF under `files.voice`; the ASR/VAD files are
+	// gated on top of that. If there is no voice file, this is a text-only
+	// bundle and the voice leg of the smoke is skipped.
+	return manifest.files.voice.length > 0;
+}
+
+async function verifyText(
+	engine: VerifyEngine,
+	modelId: string,
+	textGgufPath: string,
+): Promise<void> {
+	await engine.load(textGgufPath, { modelPath: textGgufPath, modelId });
+	const out = await engine.generate({
+		prompt: VERIFY_PROMPT,
+		maxTokens: 1,
+		temperature: 0,
+	});
+	if (typeof out !== "string") {
+		throw new Error(
+			`[verify-on-device] text generation returned ${typeof out}, expected string`,
+		);
+	}
+}
+
+async function verifyVoice(
+	engine: VerifyEngine,
+	_bundleRoot: string,
+): Promise<void> {
+	await engine.ensureActiveBundleVoiceReady();
+	try {
+		// One real synthesis through the voice bridge.
+		const pcm = await engine.synthesizeSpeech(VERIFY_PHRASE);
+		if (!(pcm instanceof Uint8Array) || pcm.byteLength === 0) {
+			throw new Error(
+				"[verify-on-device] voice synthesis produced no PCM bytes",
+			);
+		}
+		// Barge-in cancel must be accepted without throwing — exercises the
+		// hard-stop path the voice loop uses to abort speculative TTS.
+		engine.triggerBargeIn();
+	} finally {
+		await engine.stopVoice();
+	}
+}
+
+export function createVerifyBundleOnDevice(
+	deps: Partial<VerifyBundleOnDeviceDeps> = {},
+): VerifyBundleOnDevice {
+	const engine = deps.engine ?? localInferenceEngine;
+	const manifestDeps = {
+		readFile: deps.readFile ?? fs.readFile,
+		parseManifest: deps.parseManifest ?? parseManifestOrThrow,
+	};
+
+	return async ({ modelId, bundleRoot, manifestPath, textGgufPath }) => {
+		try {
+			await verifyText(engine, modelId, textGgufPath);
+			if (await manifestDeclaresVoice(manifestPath, manifestDeps)) {
+				await verifyVoice(engine, bundleRoot);
+			}
+		} finally {
+			// Always release the model the verify pass loaded — the bundle is not
+			// "active" yet, and the active-model coordinator owns load/unload from
+			// here on.
+			await engine.unload().catch(() => {});
+		}
+	};
+}
+
+export const verifyBundleOnDevice: VerifyBundleOnDevice =
+	createVerifyBundleOnDevice();

package/src/services/verify.ts

@@ -0,0 +1,13 @@
+/**
+ * Re-export of the shared model-file verification module. The canonical
+ * implementation lives in `@elizaos/shared/local-inference` because both
+ * the server (`@elizaos/app-core`) and the UI client (`@elizaos/ui`)
+ * compute the same SHA256 / GGUF-magic checks against on-disk models.
+ */
+export {
+	__registryPathForTests,
+	hashFile,
+	type VerifyResult,
+	type VerifyState,
+	verifyInstalledModel,
+} from "@elizaos/shared/local-inference/verify";