@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/gpu-detect.ts

@@ -0,0 +1,139 @@
+/**
+ * NVIDIA GPU detection + profile mapping.
+ *
+ * Wraps `nvidia-smi` to identify the host GPU and maps it to a
+ * `GpuProfileId` from `@elizaos/shared/local-inference/gpu-profiles`.
+ *
+ * Single-GPU only: when multiple GPUs are present, we use the first one
+ * `nvidia-smi` reports (canonical CUDA device 0). We do NOT try to split
+ * the model across multiple cards — that is an explicit non-goal of the
+ * single-GPU profile system.
+ *
+ * Detection is cached after first call. Pass `force: true` to bypass the
+ * cache when a fresh probe is needed (e.g. after a GPU hot-swap on a
+ * laptop dock — unusual but possible).
+ */
+
+import { type SpawnSyncReturns, spawnSync } from "node:child_process";
+import {
+	GPU_PROFILES,
+	type GpuProfile,
+	type GpuProfileId,
+	matchGpuProfile,
+} from "@elizaos/shared";
+
+export interface DetectedGpu {
+	/** Raw GPU name from `nvidia-smi --query-gpu=name`. */
+	name: string;
+	/** Total VRAM in MiB as reported by `nvidia-smi`. */
+	totalMemoryMiB: number;
+	/** Matched profile id, or `null` when the card is not in the supported set. */
+	profileId: GpuProfileId | null;
+}
+
+export interface GpuDetectionResult {
+	/** `true` when `nvidia-smi` ran successfully (even if no GPU matched a profile). */
+	nvidiaPresent: boolean;
+	/** First GPU reported by `nvidia-smi`; `null` when no NVIDIA GPU is present. */
+	gpu: DetectedGpu | null;
+	/** Resolved profile, or `null` for unsupported / non-NVIDIA hosts. */
+	profile: GpuProfile | null;
+}
+
+const EMPTY_RESULT: GpuDetectionResult = {
+	nvidiaPresent: false,
+	gpu: null,
+	profile: null,
+};
+
+let cached: GpuDetectionResult | null = null;
+let spawnSyncForTests:
+	| ((
+			command: string,
+			args: string[],
+			options: Parameters<typeof spawnSync>[2],
+	  ) => SpawnSyncReturns<string>)
+	| null = null;
+
+/**
+ * Detect the primary NVIDIA GPU and resolve it to a profile. Returns
+ * `{ nvidiaPresent: false }` on hosts without `nvidia-smi` on PATH or
+ * without an NVIDIA GPU.
+ *
+ * The probe runs `nvidia-smi --query-gpu=name,memory.total --format=csv,noheader,nounits`
+ * with a 3-second timeout so a misbehaving driver cannot stall boot.
+ */
+export function detectGpu(opts: { force?: boolean } = {}): GpuDetectionResult {
+	if (cached && !opts.force) return cached;
+	cached = probe();
+	return cached;
+}
+
+/** Clear the cached detection result. Used by tests. */
+export function __resetGpuDetectionCacheForTests(): void {
+	cached = null;
+	spawnSyncForTests = null;
+}
+
+/** Override the nvidia-smi runner. Used by tests without mutating ESM exports. */
+export function __setGpuDetectionSpawnSyncForTests(
+	runner:
+		| ((
+				command: string,
+				args: string[],
+				options: Parameters<typeof spawnSync>[2],
+		  ) => SpawnSyncReturns<string>)
+		| null,
+): void {
+	spawnSyncForTests = runner;
+}
+
+function probe(): GpuDetectionResult {
+	const run = spawnSyncForTests ?? spawnSync;
+	const result = run(
+		"nvidia-smi",
+		["--query-gpu=name,memory.total", "--format=csv,noheader,nounits"],
+		{
+			encoding: "utf8",
+			timeout: 3000,
+			stdio: ["ignore", "pipe", "pipe"],
+		},
+	);
+	if (result.error || result.status !== 0) {
+		return EMPTY_RESULT;
+	}
+	const stdout =
+		typeof result.stdout === "string"
+			? result.stdout
+			: String(result.stdout ?? "");
+	const firstLine = stdout
+		.split(/\r?\n/)
+		.find((line: string) => line.trim() !== "");
+	if (!firstLine) return EMPTY_RESULT;
+
+	// Format: "NVIDIA H200, 141248"
+	const parts = firstLine.split(",").map((part: string) => part.trim());
+	if (parts.length < 2) return EMPTY_RESULT;
+	const name = parts[0] ?? "";
+	const memMiBRaw = Number.parseInt(parts[1] ?? "", 10);
+	const totalMemoryMiB = Number.isFinite(memMiBRaw) ? memMiBRaw : 0;
+
+	const profileId = matchGpuProfile(name);
+	const profile = profileId ? GPU_PROFILES[profileId] : null;
+
+	return {
+		nvidiaPresent: true,
+		gpu: { name, totalMemoryMiB, profileId },
+		profile,
+	};
+}
+
+/**
+ * Recommend a `GpuProfileId` for a synthetic GPU descriptor — used by the
+ * recommender service when it already has a `HardwareProbe` and does not
+ * want to re-shell out to `nvidia-smi`. Returns `null` when nothing
+ * matches.
+ */
+export function recommendProfileFromName(name: string): GpuProfileId | null {
+	return matchGpuProfile(name);
+}

package/src/services/handler-registry.ts

@@ -0,0 +1,240 @@
+/**
+ * Side-registry of model handlers registered on an AgentRuntime.
+ *
+ * The elizaOS core exposes `runtime.registerModel(type, handler, provider,
+ * priority)` but no way to list who registered what. This module intercepts
+ * `registerModel` at runtime to record every registration in a Map keyed by
+ * model type, plus fires status listeners so the UI can render a live
+ * [ModelType × Provider] routing table.
+ *
+ * Because we monkey-patch `registerModel` we also keep the original
+ * handler reference — the router-handler (see `router-handler.ts`) uses
+ * this to dispatch inference calls by policy without going through
+ * `runtime.useModel` (which would loop back to us and recurse).
+ */
+
+import { AgentRuntime, type IAgentRuntime } from "@elizaos/core";
+
+export interface HandlerRegistration {
+	modelType: string;
+	provider: string;
+	priority: number;
+	registeredAt: string;
+	/**
+	 * The original handler function. Captured so the router-handler can
+	 * dispatch to it directly, bypassing `runtime.useModel` which would
+	 * re-enter the router itself.
+	 */
+	handler: (
+		runtime: IAgentRuntime,
+		params: Record<string, unknown>,
+	) => Promise<unknown>;
+}
+
+type Listener = (registrations: HandlerRegistration[]) => void;
+
+class HandlerRegistry {
+	private readonly registrations = new Map<string, HandlerRegistration[]>();
+	private readonly listeners = new Set<Listener>();
+	private installedOn: WeakSet<object> = new WeakSet();
+
+	/**
+	 * Snapshot of all registrations grouped by model type, sorted by
+	 * priority descending inside each group (matches core's selection
+	 * order). Callers must not mutate the returned array.
+	 */
+	getAll(): HandlerRegistration[] {
+		const out: HandlerRegistration[] = [];
+		for (const list of this.registrations.values()) {
+			out.push(...list);
+		}
+		return out;
+	}
+
+	/** All registrations for a given model type, sorted by priority desc. */
+	getForType(modelType: string): HandlerRegistration[] {
+		const list = this.registrations.get(modelType);
+		return list ? [...list] : [];
+	}
+
+	/**
+	 * Registrations excluding a specific provider. Used by the router-handler
+	 * to find "all providers except me" when dispatching.
+	 */
+	getForTypeExcluding(
+		modelType: string,
+		excludeProvider: string,
+	): HandlerRegistration[] {
+		return this.getForType(modelType).filter(
+			(r) => r.provider !== excludeProvider,
+		);
+	}
+
+	subscribe(listener: Listener): () => void {
+		this.listeners.add(listener);
+		return () => {
+			this.listeners.delete(listener);
+		};
+	}
+
+	private emit(): void {
+		const snapshot = this.getAll();
+		for (const listener of this.listeners) {
+			try {
+				listener(snapshot);
+			} catch {
+				this.listeners.delete(listener);
+			}
+		}
+	}
+
+	private record(reg: HandlerRegistration): void {
+		const existing = this.registrations.get(reg.modelType) ?? [];
+		// Replace any prior registration from the same provider for this
+		// model type. Core allows multiple providers per type but only one
+		// registration per (type, provider) pair — last write wins.
+		const filtered = existing.filter((r) => r.provider !== reg.provider);
+		filtered.push(reg);
+		filtered.sort((a, b) => b.priority - a.priority);
+		this.registrations.set(reg.modelType, filtered);
+		this.emit();
+	}
+
+	/**
+	 * Install the interception on a runtime. Idempotent per runtime instance.
+	 * For most boot paths the prototype-level patch below already covers the
+	 * runtime before any plugin registers; this method is the belt-and-braces
+	 * fallback for runtimes constructed before the patch ran.
+	 */
+	installOn(runtime: AgentRuntime): void {
+		installPrototypePatch();
+		const rt = runtime as AgentRuntime & {
+			registerModel?: unknown;
+		};
+		if (typeof rt.registerModel !== "function") return;
+		if (this.installedOn.has(rt)) return;
+		this.installedOn.add(rt);
+
+		// If the runtime already inherited the patched prototype method the
+		// prototype handles every call. Nothing to do per-instance.
+		const protoMethod = Object.getPrototypeOf(rt)?.registerModel as
+			| PatchMarkedRegisterModel
+			| undefined;
+		if (protoMethod?.[PATCH_MARK]) {
+			return;
+		}
+
+		// Per-instance wrap only for legacy runtimes whose prototype pre-dates
+		// our prototype patch (shouldn't happen in practice).
+		const original = rt.registerModel.bind(runtime) as (
+			modelType: string,
+			handler: HandlerRegistration["handler"],
+			provider: string,
+			priority?: number,
+		) => void;
+		rt.registerModel = ((
+			modelType: string,
+			handler: HandlerRegistration["handler"],
+			provider: string,
+			priority?: number,
+		) => {
+			this.record({
+				modelType: String(modelType),
+				provider: String(provider),
+				priority: typeof priority === "number" ? priority : 0,
+				registeredAt: new Date().toISOString(),
+				handler,
+			});
+			return original(modelType, handler, provider, priority);
+		}) as typeof rt.registerModel;
+	}
+
+	/** Exposed so the prototype patch can record through the singleton. */
+	recordFromPrototype(reg: HandlerRegistration): void {
+		this.record(reg);
+	}
+}
+
+const PATCH_MARK = Symbol.for("eliza.local-inference.registerModel.patched");
+let prototypePatched = false;
+
+type RegisterModelMethod = (
+	this: AgentRuntime,
+	modelType: string,
+	handler: HandlerRegistration["handler"],
+	provider: string,
+	priority?: number,
+) => void;
+
+type PatchMarkedRegisterModel = RegisterModelMethod & {
+	[PATCH_MARK]?: true;
+};
+
+/**
+ * One-shot patch of `AgentRuntime.prototype.registerModel` so every runtime
+ * instance — including ones constructed later in boot — records through
+ * the singleton handler registry. Idempotent.
+ */
+function installPrototypePatch(): void {
+	if (prototypePatched) return;
+	const proto = AgentRuntime.prototype as AgentRuntime & {
+		registerModel: RegisterModelMethod;
+	};
+	const original = proto.registerModel;
+	if (typeof original !== "function") return;
+	if ((original as PatchMarkedRegisterModel)[PATCH_MARK]) {
+		prototypePatched = true;
+		return;
+	}
+	const patched = function patchedRegisterModel(
+		this: AgentRuntime,
+		modelType: string,
+		handler: HandlerRegistration["handler"],
+		provider: string,
+		priority?: number,
+	): void {
+		try {
+			handlerRegistry.recordFromPrototype({
+				modelType: String(modelType),
+				provider: String(provider),
+				priority: typeof priority === "number" ? priority : 0,
+				registeredAt: new Date().toISOString(),
+				handler,
+			});
+		} catch {
+			// Never let registry bookkeeping break the registration path.
+		}
+		original.call(this, modelType, handler, provider, priority);
+	} as typeof original & { [PATCH_MARK]?: true };
+	patched[PATCH_MARK] = true;
+	proto.registerModel = patched;
+	prototypePatched = true;
+}
+
+// Install at module-import time. This is a process-wide side effect but a
+// benign one: the patch is idempotent and forwards to the original.
+installPrototypePatch();
+
+export const handlerRegistry = new HandlerRegistry();
+
+/**
+ * Public type used by the API/UI — omits the handler function for
+ * serialisation and to prevent UI code from accidentally calling it.
+ */
+export interface PublicRegistration {
+	modelType: string;
+	provider: string;
+	priority: number;
+	registeredAt: string;
+}
+
+export function toPublicRegistration(
+	reg: HandlerRegistration,
+): PublicRegistration {
+	return {
+		modelType: reg.modelType,
+		provider: reg.provider,
+		priority: reg.priority,
+		registeredAt: reg.registeredAt,
+	};
+}

package/src/services/hardware.test.ts

@@ -0,0 +1,236 @@
+import { afterEach, describe, expect, it } from "vitest";
+import { classifyDeviceTier } from "./device-tier";
+import {
+	__resetGpuDetectionCacheForTests,
+	__setGpuDetectionSpawnSyncForTests,
+} from "./gpu-detect";
+import {
+	detectCpuFeatures,
+	detectOpenVinoDevices,
+	deviceCapsFromProbe,
+	probeHardware,
+} from "./hardware";
+
+function detector(files: Record<string, string[] | true>) {
+	const existsSync = (path: string): boolean => path in files;
+	const readdirSync = (path: string): string[] => {
+		const value = files[path];
+		return Array.isArray(value) ? value : [];
+	};
+	return { existsSync, readdirSync };
+}
+
+describe("detectOpenVinoDevices", () => {
+	it("prefers NPU for ASR when OpenVINO runtime and accel node are present", () => {
+		const probe = detectOpenVinoDevices({
+			platform: "linux",
+			env: { OpenVINO_DIR: "/opt/intel/openvino_2026/runtime/cmake" },
+			...detector({
+				"/dev/accel": ["accel0"],
+			}),
+		});
+
+		expect(probe.runtimeAvailable).toBe(true);
+		expect(probe.devices).toEqual(["CPU", "NPU"]);
+		expect(probe.npu.accelNodes).toEqual(["/dev/accel/accel0"]);
+		expect(probe.recommendedAsrDevice).toBe("NPU");
+	});
+
+	it("warns when Intel render nodes exist without the Compute Runtime stack", () => {
+		const probe = detectOpenVinoDevices({
+			platform: "linux",
+			env: { OpenVINO_DIR: "/opt/intel/openvino_2026/runtime/cmake" },
+			...detector({
+				"/dev/dri": ["card0", "renderD128"],
+			}),
+		});
+
+		expect(probe.devices).toEqual(["CPU"]);
+		expect(probe.gpu.renderNodes).toEqual(["/dev/dri/renderD128"]);
+		expect(probe.gpu.computeRuntimeReady).toBe(false);
+		expect(probe.gpu.missingLinuxPackages).toEqual([
+			"intel-opencl-icd",
+			"libigc2",
+			"libigdfcl2",
+		]);
+		expect(probe.warnings[0]).toContain("Intel Compute Runtime packages");
+	});
+
+	it("does not report OpenVINO devices when the runtime is not installed", () => {
+		const probe = detectOpenVinoDevices({
+			platform: "linux",
+			env: {},
+			...detector({
+				"/dev/accel": ["accel0"],
+				"/dev/dri": ["renderD128"],
+				"/usr/lib/x86_64-linux-gnu/intel-opencl/libigdrcl.so": true,
+			}),
+		});
+
+		expect(probe.runtimeAvailable).toBe(false);
+		expect(probe.devices).toEqual([]);
+		expect(probe.recommendedAsrDevice).toBeNull();
+		expect(probe.warnings).toContain(
+			"Intel accelerator nodes are present, but OpenVINO Runtime was not found; source setupvars.sh or set OpenVINO_DIR.",
+		);
+	});
+
+	it("treats sandbox-denied optional system paths as unavailable", () => {
+		const existsSync = (path: string): boolean => {
+			throw Object.assign(
+				new Error(`mobile-fs-shim: path escapes workspace root: ${path}`),
+				{ code: "EACCES" },
+			);
+		};
+		const readdirSync = (path: string): string[] => {
+			throw new Error(`unexpected readdir for ${path}`);
+		};
+
+		const probe = detectOpenVinoDevices({
+			platform: "linux",
+			env: {},
+			existsSync,
+			readdirSync,
+		});
+
+		expect(probe.runtimeAvailable).toBe(false);
+		expect(probe.devices).toEqual([]);
+		expect(probe.gpu.renderNodes).toEqual([]);
+		expect(probe.npu.accelNodes).toEqual([]);
+		expect(probe.recommendedAsrDevice).toBeNull();
+	});
+});
+
+describe("detectCpuFeatures", () => {
+	it("maps Linux ARM cpuinfo feature tokens to the shared probe shape", () => {
+		const cpuinfo = [
+			"processor\t: 0",
+			"Features\t: fp asimd evtstrm aes asimddp i8mm sve sve2",
+			"",
+		].join("\n");
+
+		expect(
+			detectCpuFeatures({
+				platform: "linux",
+				arch: "arm64",
+				readFileSync: () => cpuinfo,
+			}),
+		).toEqual({
+			neon: true,
+			dotprod: true,
+			i8mm: true,
+			sve: true,
+			sve2: true,
+		});
+	});
+
+	it("maps Android ARM cpuinfo aliases without assuming missing features", () => {
+		const cpuinfo = "Features\t: fp neon dotprod\n";
+
+		expect(
+			detectCpuFeatures({
+				platform: "android",
+				arch: "arm64",
+				readFileSync: () => cpuinfo,
+			}),
+		).toEqual({
+			neon: true,
+			dotprod: true,
+			i8mm: false,
+			sve: false,
+			sve2: false,
+		});
+	});
+
+	it("uses Darwin sysctl keys where available and treats missing keys as false", () => {
+		const values: Record<string, string> = {
+			"hw.optional.arm.FEAT_DotProd": "1\n",
+			"hw.optional.arm.FEAT_I8MM": "0\n",
+		};
+
+		expect(
+			detectCpuFeatures({
+				platform: "darwin",
+				arch: "arm64",
+				execFileSync: (_file, args) => {
+					const key = args[1];
+					if (key in values) return values[key];
+					throw new Error(`unknown sysctl ${key}`);
+				},
+			}),
+		).toEqual({
+			neon: true,
+			dotprod: true,
+			i8mm: false,
+			sve: false,
+			sve2: false,
+		});
+	});
+
+	it("does not claim CPU backend support for ARM when NEON evidence is absent", () => {
+		expect(
+			deviceCapsFromProbe({
+				totalRamGb: 16,
+				freeRamGb: 12,
+				gpu: null,
+				cpuCores: 8,
+				platform: "linux",
+				arch: "arm64",
+				appleSilicon: false,
+				recommendedBucket: "mid",
+				source: "os-fallback",
+			}),
+		).toEqual({
+			availableBackends: [],
+			ramMb: 16 * 1024,
+			cpuFeatures: undefined,
+		});
+	});
+});
+
+describe("probeHardware GPU detection", () => {
+	afterEach(() => {
+		__setGpuDetectionSpawnSyncForTests(null);
+		__resetGpuDetectionCacheForTests();
+	});
+
+	const fakeSpawn =
+		(stdout: string, status: number) =>
+		(): { stdout: string; stderr: string; status: number; signal: null } => ({
+			stdout,
+			stderr: "",
+			status,
+			signal: null,
+		});
+
+	it("populates a CUDA GPU + VRAM from nvidia-smi and lifts the tier off CPU", async () => {
+		// reset clears BOTH the cache and any prior override, so reset first.
+		__resetGpuDetectionCacheForTests();
+		__setGpuDetectionSpawnSyncForTests(
+			fakeSpawn("NVIDIA GeForce RTX 4090, 24564\n", 0) as never,
+		);
+		const probe = await probeHardware();
+		expect(probe.gpu).not.toBeNull();
+		expect(probe.gpu?.backend).toBe("cuda");
+		expect(probe.gpu?.totalVramGb).toBeGreaterThanOrEqual(23);
+		// A 24 GB discrete GPU must not be mis-tiered as a CPU box (POOR). MAX/GOOD
+		// additionally require >=16/>=8 GB *free* RAM at session start, which this test
+		// only mocks the GPU for; under parallel-suite memory pressure real free RAM
+		// can dip below those gates, legitimately tiering a 24 GB-GPU box to OKAY
+		// (still GPU-backed, not a CPU box). Assert "off CPU" (not POOR) -- the stated
+		// intent -- which is deterministic regardless of host free RAM.
+		expect(["MAX", "GOOD", "OKAY"]).toContain(classifyDeviceTier(probe).tier);
+	});
+
+	it("reports gpu:null when nvidia-smi is absent on a non-Apple host", async () => {
+		__resetGpuDetectionCacheForTests();
+		__setGpuDetectionSpawnSyncForTests(fakeSpawn("", 1) as never);
+		const probe = await probeHardware();
+		if (probe.appleSilicon) {
+			// Unified memory: the Apple GPU is always present.
+			expect(probe.gpu?.backend).toBe("metal");
+		} else {
+			expect(probe.gpu).toBeNull();
+		}
+	});
+});