@elizaos/plugin-local-inference 2.0.3-beta.2 → 2.0.3-beta.3

package/src/services/recommendation.test.ts

@@ -0,0 +1,216 @@
+import type { CatalogModel, HardwareProbe } from "@elizaos/shared";
+import { describe, expect, it } from "vitest";
+import { MODEL_CATALOG } from "./catalog.js";
+import {
+	assessCatalogModelFit,
+	canBundleBeDefaultOnDevice,
+	catalogDownloadSizeBytes,
+	catalogDownloadSizeGb,
+	chooseSmallerFallbackModel,
+	classifyRecommendationPlatform,
+	deviceCapsFromProbe,
+	selectBestQuantizationVariant,
+	selectRecommendedModelForSlot,
+} from "./recommendation.js";
+
+/**
+ * Pure recommendation helpers that decide which model the device can run.
+ * Platform/backend classification drives the slot ladders and the on-device
+ * default gate (#8848), so the branch table is pinned here.
+ */
+
+const probe = (o: Partial<HardwareProbe> = {}): HardwareProbe =>
+	({
+		totalRamGb: 16,
+		freeRamGb: 8,
+		gpu: null,
+		cpuCores: 8,
+		platform: "linux",
+		arch: "x64",
+		appleSilicon: false,
+		recommendedBucket: "medium",
+		source: "os-fallback",
+		...o,
+	}) as HardwareProbe;
+
+const metalGpu = { backend: "metal" as const, totalVramGb: 24, freeVramGb: 20 };
+const cudaGpu = { backend: "cuda" as const, totalVramGb: 24, freeVramGb: 20 };
+
+describe("classifyRecommendationPlatform", () => {
+	it("maps each hardware shape to its platform class", () => {
+		expect(
+			classifyRecommendationPlatform(
+				probe({ mobile: { platform: "android" } }),
+			),
+		).toBe("mobile");
+		expect(
+			classifyRecommendationPlatform(
+				probe({ platform: "darwin", arch: "arm64", appleSilicon: true }),
+			),
+		).toBe("apple-silicon");
+		expect(
+			classifyRecommendationPlatform(
+				probe({ platform: "linux", gpu: cudaGpu }),
+			),
+		).toBe("linux-gpu");
+		expect(classifyRecommendationPlatform(probe({ platform: "linux" }))).toBe(
+			"linux-cpu",
+		);
+		expect(
+			classifyRecommendationPlatform(
+				probe({ platform: "win32", gpu: cudaGpu }),
+			),
+		).toBe("desktop-gpu");
+		expect(classifyRecommendationPlatform(probe({ platform: "win32" }))).toBe(
+			"desktop-cpu",
+		);
+	});
+
+	it("treats mobile and apple-silicon as higher precedence than gpu/platform", () => {
+		// mobile wins even with a GPU present.
+		expect(
+			classifyRecommendationPlatform(
+				probe({ gpu: cudaGpu, mobile: { platform: "ios" } }),
+			),
+		).toBe("mobile");
+	});
+});
+
+describe("deviceCapsFromProbe", () => {
+	it("includes cpu for x86 and adds the probed GPU backend, RAM in MB", () => {
+		expect(
+			deviceCapsFromProbe(probe({ totalRamGb: 8, gpu: metalGpu })),
+		).toEqual({
+			availableBackends: ["cpu", "metal"],
+			ramMb: 8192,
+			cpuFeatures: undefined,
+		});
+	});
+
+	it("requires NEON for an arm CPU backend to count", () => {
+		expect(
+			deviceCapsFromProbe(probe({ arch: "arm64", cpuFeatures: { neon: true } }))
+				.availableBackends,
+		).toEqual(["cpu"]);
+		expect(
+			deviceCapsFromProbe(probe({ arch: "arm64" })).availableBackends,
+		).toEqual([]);
+		// arm without neon but with a GPU still exposes the GPU backend only.
+		expect(
+			deviceCapsFromProbe(probe({ arch: "arm64", gpu: metalGpu }))
+				.availableBackends,
+		).toEqual(["metal"]);
+	});
+});
+
+describe("catalog download size", () => {
+	it("reads sizeGb and converts to bytes", () => {
+		const model = { sizeGb: 2 } as CatalogModel;
+		expect(catalogDownloadSizeGb(model)).toBe(2);
+		expect(catalogDownloadSizeBytes(model)).toBe(2 * 1024 ** 3);
+	});
+});
+
+describe("selectBestQuantizationVariant", () => {
+	const variant = (id: string, status: "published" | "planned") => ({
+		id,
+		label: "4-bit",
+		ggufFile: `${id}.gguf`,
+		sizeGb: 1,
+		minRamGb: 2,
+		status,
+	});
+
+	it("prefers the default variant, then published, then first, else null", () => {
+		expect(
+			selectBestQuantizationVariant({
+				quantization: {
+					defaultVariantId: "q4",
+					variants: [variant("q4", "published"), variant("q8", "published")],
+				},
+			} as unknown as CatalogModel)?.id,
+		).toBe("q4");
+
+		expect(
+			selectBestQuantizationVariant({
+				quantization: {
+					defaultVariantId: "missing",
+					variants: [variant("q5", "planned"), variant("q6", "published")],
+				},
+			} as unknown as CatalogModel)?.id,
+		).toBe("q6");
+
+		expect(
+			selectBestQuantizationVariant({
+				quantization: {
+					defaultVariantId: "missing",
+					variants: [variant("q3", "planned")],
+				},
+			} as unknown as CatalogModel)?.id,
+		).toBe("q3");
+
+		expect(selectBestQuantizationVariant({} as CatalogModel)).toBeNull();
+	});
+});
+
+describe("canBundleBeDefaultOnDevice gate (#8848)", () => {
+	// Types derived from the function signature so the test needs no extra
+	// type imports; the manifest-loader injection keeps it model-independent.
+	type Installed = Parameters<typeof canBundleBeDefaultOnDevice>[0];
+	type Loader = NonNullable<
+		Parameters<typeof canBundleBeDefaultOnDevice>[2]
+	>["manifestLoader"];
+
+	it("refuses a bundle with no validated manifest", () => {
+		const result = canBundleBeDefaultOnDevice(
+			{ id: "test-bundle" } as unknown as Installed,
+			probe(),
+			{ manifestLoader: (() => null) as Loader },
+		);
+		expect(result.canBeDefault).toBe(false);
+		expect(result.reason).toBe("no-manifest");
+	});
+
+	it("refuses a bundle whose on-device verify pass has not run", () => {
+		// Manifest present but `bundleVerifiedAt` unset → the verify gate fires
+		// before any RAM/kernel check, so the manifest content is irrelevant.
+		const result = canBundleBeDefaultOnDevice(
+			{ id: "test-bundle" } as unknown as Installed,
+			probe(),
+			{ manifestLoader: (() => ({})) as Loader },
+		);
+		expect(result.canBeDefault).toBe(false);
+		expect(result.reason).toBe("not-verified-on-device");
+	});
+});
+
+describe("model-fit + smaller-fallback ladder (#8848)", () => {
+	const bySize = [...MODEL_CATALOG].sort(
+		(a, b) => catalogDownloadSizeGb(a) - catalogDownloadSizeGb(b),
+	);
+	const smallest = bySize[0];
+	const largest = bySize[bySize.length - 1];
+	const tiny = probe({ totalRamGb: 1, freeRamGb: 0.5 });
+	const huge = probe({ totalRamGb: 128, freeRamGb: 100 });
+
+	it("won't fit even the smallest catalog model on a 1 GB device, but fits on a big one", () => {
+		expect(assessCatalogModelFit(tiny, smallest)).toBe("wontfit");
+		expect(assessCatalogModelFit(huge, smallest)).toBe("fits");
+	});
+
+	it("has no smaller fallback on a 1 GB device, and a strictly-smaller one on a big device", () => {
+		expect(chooseSmallerFallbackModel(largest.id, tiny)).toBeNull();
+		const fallback = chooseSmallerFallbackModel(largest.id, huge);
+		expect(fallback).not.toBeNull();
+		expect(fallback?.id).not.toBe(largest.id);
+		expect(catalogDownloadSizeGb(fallback as CatalogModel)).toBeLessThan(
+			catalogDownloadSizeGb(largest),
+		);
+	});
+
+	it("selectRecommendedModelForSlot picks a real, fitting catalog model on a capable host", () => {
+		const sel = selectRecommendedModelForSlot("TEXT_LARGE", huge);
+		expect(sel.model).not.toBeNull();
+		expect(MODEL_CATALOG.some((m) => m.id === sel.model?.id)).toBe(true);
+	});
+});

package/src/services/registry.ts

@@ -1,22 +1,18 @@
 /**
  * On-disk registry of installed models.
  *
- * Two sources feed the registry:
- *   1. Eliza-owned downloads (source: "eliza-download") — written on
- *      successful completion by the downloader.
- *   2. External scans (source: "external-scan") — merged in at read time
- *      from `scanExternalModels()`. These are never persisted to the
- *      registry file; a rescan runs whenever we read.
- *
- * The JSON file only holds Eliza-owned entries. That way, if a user
- * cleans up LM Studio models we don't show stale ghosts.
+ * The default registry contains only Eliza-owned downloads
+ * (source: "eliza-download") written on successful completion by the
+ * curated bundle downloader. External scans are developer-only diagnostics
+ * behind `ELIZA_LOCAL_INFERENCE_ENABLE_EXTERNAL_SCAN=1`; they never enter
+ * first-run, setup, or normal Settings surfaces.
  */
 
 import fs from "node:fs/promises";
 import path from "node:path";
 import { scanExternalModels } from "./external-scanner";
 import { isWithinElizaRoot, localInferenceRoot, registryPath } from "./paths";
-import type { InstalledModel } from "./types";
+import { type InstalledModel, withRuntimeClass } from "./types";
 
 interface RegistryFile {
 	version: 1;
@@ -51,26 +47,36 @@ async function writeElizaOwned(models: InstalledModel[]): Promise<void> {
 	await fs.rename(tmp, registryPath());
 }
 
+function externalScanEnabled(): boolean {
+	const value =
+		process.env.ELIZA_LOCAL_INFERENCE_ENABLE_EXTERNAL_SCAN?.trim().toLowerCase();
+	return value === "1" || value === "true" || value === "yes";
+}
+
 /**
- * Return all models currently usable: persisted Eliza downloads plus a
- * fresh external-tool scan. External duplicates of Eliza-owned files are
- * filtered out by path.
+ * Return models currently usable by the curated local-inference path.
+ *
+ * Normal product behavior is Eliza-1 only. The external scan remains available
+ * only to developers who explicitly opt into the old arbitrary-GGUF diagnostic
+ * path with `ELIZA_LOCAL_INFERENCE_ENABLE_EXTERNAL_SCAN=1`.
  */
 export async function listInstalledModels(): Promise<InstalledModel[]> {
-	const [ownedRaw, external] = await Promise.all([
-		readElizaOwned(),
-		scanExternalModels(),
-	]);
-	const owned = ownedRaw;
+	const owned = await readElizaOwned();
+	if (!externalScanEnabled()) return owned;
 
 	// Filter out Eliza-owned files that also survived a reboot of the local
 	// file and got re-detected by the scanner.
+	const external = await scanExternalModels();
 	const ownedPaths = new Set(owned.map((m) => path.resolve(m.path)));
 	const dedupedExternal = external.filter(
 		(m) => !ownedPaths.has(path.resolve(m.path)),
 	);
 
-	return [...owned, ...dedupedExternal];
+	// Backfill `runtimeClass` once, at the canonical read boundary: legacy
+	// registry rows and freshly scanned external models predate the field.
+	// Downstream (dispatcher, load-arg resolver, UI) reads the field rather
+	// than re-deriving the class from the id.
+	return [...owned, ...dedupedExternal].map(withRuntimeClass);
 }
 
 /** Add or update a Eliza-owned entry. External entries are rejected. */

package/src/services/required-kernels-gate.test.ts

@@ -0,0 +1,64 @@
+import { describe, expect, it } from "vitest";
+import {
+	assertRequiredKernelsPresent,
+	MissingRequiredKernelsError,
+} from "./active-model";
+import type { Eliza1Manifest } from "./manifest";
+import { REQUIRED_KERNELS_BY_TIER } from "./manifest";
+import type { ManifestLoader } from "./ram-budget";
+import type { InstalledModel } from "./types";
+
+function installed2b(): InstalledModel {
+	return {
+		id: "eliza-1-2b",
+		displayName: "Eliza-1 2B",
+		path: "/tmp/eliza-1-2b/text/model.gguf",
+		sizeBytes: 1024,
+		installedAt: "2026-05-15T00:00:00.000Z",
+		lastUsedAt: null,
+		source: "eliza-download",
+	};
+}
+
+/**
+ * `assertRequiredKernelsPresent` reads only `manifest.tier` and
+ * `manifest.kernels.required`, so a minimal manifest is sufficient.
+ */
+function manifestWithKernels(required: readonly string[]): Eliza1Manifest {
+	return {
+		tier: "2b",
+		kernels: { required },
+	} as unknown as Eliza1Manifest;
+}
+
+describe("assertRequiredKernelsPresent (native/CLAUDE.md §3#5)", () => {
+	it("is a no-op when no manifest is present (bare-GGUF/dev path)", () => {
+		const noManifest: ManifestLoader = () => null;
+		expect(() =>
+			assertRequiredKernelsPresent(installed2b(), noManifest),
+		).not.toThrow();
+	});
+
+	it("is a no-op when the manifest declares every required kernel", () => {
+		const ok: ManifestLoader = () =>
+			manifestWithKernels(REQUIRED_KERNELS_BY_TIER["2b"]);
+		expect(() => assertRequiredKernelsPresent(installed2b(), ok)).not.toThrow();
+	});
+
+	it("throws MissingRequiredKernelsError when a required kernel is absent", () => {
+		// The 2B tier requires `turboquant_q4`; a manifest that only declares an
+		// optional KV kernel is missing it.
+		const broken: ManifestLoader = () => manifestWithKernels(["qjl"]);
+		let thrown: unknown;
+		try {
+			assertRequiredKernelsPresent(installed2b(), broken);
+		} catch (err) {
+			thrown = err;
+		}
+		expect(thrown).toBeInstanceOf(MissingRequiredKernelsError);
+		const err = thrown as MissingRequiredKernelsError;
+		expect(err.tier).toBe("2b");
+		expect(err.missing).toContain("turboquant_q4");
+		expect(err.modelId).toBe("eliza-1-2b");
+	});
+});

package/src/services/router-handler.ts

@@ -54,8 +54,8 @@
  *   6. **Edge-TTS (`edge-tts`)** — free Microsoft Edge endpoint, no key.
  *
  * Same precedence applies to `TRANSCRIPTION`, with the local side using
- * Qwen3-ASR via the fused libelizainference (no whisper.cpp; see
- * `plugin-local-inference/native/AGENTS.md` §1).
+ * the fused Gemma ASR path when the active bundle stages eligible ASR
+ * artifacts (no whisper.cpp fallback).
  *
  * Users can override this per slot via the routing-preferences settings
  * panel (`prefer-local` ↔ `manual` + explicit `preferredProvider`).
@@ -73,9 +73,11 @@ import { localInferenceEngine } from "./engine";
 import type { HandlerRegistration } from "./handler-registry";
 import { handlerRegistry } from "./handler-registry";
 import { probeHardware } from "./hardware";
+import { type LiveDeviceSignals, readLiveDeviceSignals } from "./live-signals";
 import { policyEngine } from "./routing-policy";
 import {
 	DEFAULT_ROUTING_POLICY,
+	type RoutingPolicy,
 	readRoutingPreferences,
 } from "./routing-preferences";
 import { AGENT_MODEL_SLOTS, type AgentModelSlot } from "./types";
@@ -92,7 +94,9 @@ const ROUTER_PRIORITY = Number.MAX_SAFE_INTEGER;
  * The device-tier assessment drives the `auto` policy (and softly hints
  * `prefer-local`). Probing hardware is cheap but not free, so cache the
  * assessment for a short window — long enough to avoid re-probing on every
- * model call, short enough that free-RAM demotion stays roughly current.
+ * model call, short enough that the live free-RAM demotion stays roughly
+ * current. The live thermal / throughput signals are read fresh per call (they
+ * change fast and are cheap to read), not cached.
  */
 const DEVICE_TIER_TTL_MS = 30_000;
 let cachedDeviceTier: { at: number; assessment: DeviceTierAssessment } | null =
@@ -154,6 +158,10 @@ function shouldForceLocalInference(
 	policy: string,
 	preferredProvider: string | null,
 ): boolean {
+	// Keep the local-inference candidate even when no text model is assigned/loaded
+	// when the policy guarantees on-device routing: an explicit manual pin, or the
+	// `local-only` policy (which must never fall back to cloud).
+	if (policy === "local-only") return true;
 	return policy === "manual" && preferredProvider === "eliza-local-inference";
 }
 
@@ -210,14 +218,20 @@ export async function filterUnavailableLocalInference(
 	preferredProvider: string | null,
 	candidates: HandlerRegistration[],
 ): Promise<HandlerRegistration[]> {
-	// Voice slots (TTS / STT) are self-sufficient — their handlers call
-	// ensureActiveBundleVoiceReady() internally which loads the voice model
-	// on-demand. Don't gate them behind text model availability; the text
-	// model assignment check only makes sense for text generation slots.
-	if (slot === "TEXT_TO_SPEECH" || slot === "TRANSCRIPTION") {
+	// TTS is self-sufficient: its handler calls ensureActiveBundleVoiceReady()
+	// internally and can use the Kokoro-only bridge on demand.
+	if (slot === "TEXT_TO_SPEECH") {
 		return candidates;
 	}
 
+	if (slot === "TRANSCRIPTION") {
+		return filterUnavailableLocalInferenceCandidates(
+			candidates,
+			await localInferenceEngine.canTranscribeLocally(),
+			shouldForceLocalInference(policy, preferredProvider),
+		);
+	}
+
 	const hasLocalInference = candidates.some(
 		(candidate) => candidate.provider === "eliza-local-inference",
 	);
@@ -240,18 +254,16 @@ function makeRouterHandler(slot: AgentModelSlot): AnyHandler {
 			throw new Error(`[router] Unknown agent slot: ${slot}`);
 		}
 
-		// Global local-only override: ELIZA_LOCAL_ONLY=1 pins every slot to
-		// manual + eliza-local-inference, disabling cloud fallback entirely.
-		const globalLocalOnly = readBooleanEnv("ELIZA_LOCAL_ONLY");
-
-		// Read the user's policy for this slot. Absent = local-first fallback.
+		// Read the user's policy for this slot. The per-slot policy is canonical;
+		// when absent it falls back to the local-first default. ELIZA_LOCAL_ONLY
+		// is retained for back-compat only: it sets the *global default* to
+		// `local-only`, but an explicit per-slot policy always wins.
 		const prefs = await readRoutingPreferences();
-		const policy = globalLocalOnly
-			? "manual"
-			: (prefs.policy[slot] ?? DEFAULT_ROUTING_POLICY);
-		const preferred = globalLocalOnly
-			? "eliza-local-inference"
-			: (prefs.preferredProvider[slot] ?? null);
+		const globalDefault: RoutingPolicy = readBooleanEnv("ELIZA_LOCAL_ONLY")
+			? "local-only"
+			: DEFAULT_ROUTING_POLICY;
+		const policy: RoutingPolicy = prefs.policy[slot] ?? globalDefault;
+		const preferred = prefs.preferredProvider[slot] ?? null;
 
 		// Ask the policy engine which handler to dispatch to. For automatic
 		// policies, honor the documented fallback behaviour: if the selected
@@ -270,11 +282,16 @@ function makeRouterHandler(slot: AgentModelSlot): AnyHandler {
 				: getRuntimeModelCandidates(runtime, modelType),
 		);
 
-		// Only the capability-aware policies need a hardware assessment.
-		const deviceTier =
-			policy === "auto" || policy === "prefer-local"
-				? await resolveDeviceTier()
-				: null;
+		// Only the capability-aware policies need the hardware assessment + live
+		// signals. The tier is cached; the live signals are read fresh.
+		let deviceTier: DeviceTierAssessment | null = null;
+		let liveSignals: LiveDeviceSignals | null = null;
+		if (policy === "auto" || policy === "prefer-local") {
+			deviceTier = await resolveDeviceTier();
+		}
+		if (policy === "auto") {
+			liveSignals = readLiveDeviceSignals();
+		}
 
 		const failedProviders = new Set<string>();
 		let lastError: unknown = null;
@@ -289,7 +306,9 @@ function makeRouterHandler(slot: AgentModelSlot): AnyHandler {
 				preferredProvider: preferred,
 				candidates: remaining,
 				selfProvider: ROUTER_PROVIDER,
+				slot,
 				deviceTier,
+				liveSignals,
 			});
 
 			if (!pick) {