@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/services/assignments.d.ts

@@ -0,0 +1,71 @@
+/**
+ * Per-ModelType model assignment store.
+ *
+ * Separate from the "active loaded model" concept in `ActiveModelCoordinator`.
+ * Assignments are a *policy* — the user's declared intent that
+ * `ModelType.TEXT_SMALL` should be served by model X and `TEXT_LARGE` by
+ * model Y. The runtime's model handlers lazy-load whichever assignment
+ * fires; the coordinator handles the actual swap in and out of memory.
+ *
+ * Stored in `$ELIZA_STATE_DIR/local-inference/assignments.json`. Cheap
+ * enough to rewrite on every change — we never mutate in place.
+ */
+import type { AgentModelSlot, InstalledModel, ModelAssignments } from "./types";
+export declare function readAssignments(): Promise<ModelAssignments>;
+/**
+ * Build slot recommendations from currently-installed models.
+ *
+ * Only default-eligible Eliza-1 downloads are auto-recommended.
+ * External-scan blobs and ad-hoc Hugging Face downloads are surfaced only
+ * through explicit search/manual selection; they are NOT auto-assigned to
+ * any agent slot.
+ *
+ * Why: external blobs may use newer architectures or quant formats outside
+ * the bundled `capacitor-llama` binding's supported set. Auto-loading
+ * an external blob the user never selected silently breaks PROACTIVE_AGENT
+ * and other background tasks at boot. The user opted into the external
+ * tool, not into Eliza loading those weights through llama.cpp.
+ */
+export declare function buildRecommendedAssignments(installed: InstalledModel[]): ModelAssignments;
+export declare function readEffectiveAssignments(): Promise<ModelAssignments>;
+export declare function writeAssignments(assignments: ModelAssignments): Promise<void>;
+export declare function setAssignment(slot: AgentModelSlot, modelId: string | null): Promise<ModelAssignments>;
+/**
+ * Decide which slots a freshly-installed model is a sensible default for.
+ *
+ * Today the curated catalog tags models with `category` ∈
+ * `chat | code | tools | tiny | reasoning` and `bucket` ∈
+ * `small | mid | large | xl` — no explicit "embedding" tag, because the
+ * default catalog ships only generative models. The defensive check below
+ * still recognizes an "embedding" category/bucket for catalog additions and
+ * for external-scan models whose ids contain a recognizable
+ * embedding-family marker (`nomic-embed`, `bge`, `all-minilm`, `gte`,
+ * `e5-`). External GGUFs without a catalog entry default to generative.
+ */
+export declare function isEmbeddingModelId(modelId: string): boolean;
+/**
+ * Fill empty assignment slots with `modelId`. Idempotent: never overwrites
+ * an existing slot. Embedding models only fill `TEXT_EMBEDDING`; generative
+ * models only fill `TEXT_SMALL` and `TEXT_LARGE`. Returns the resulting
+ * assignment map (read state is `readAssignments()`, not effective +
+ * recommended).
+ *
+ * Wired from the downloader's success path and the runtime boot's
+ * "exactly one model installed, no assignments" branch so first-light
+ * users land in chat without a Settings detour. The hard error in
+ * `ensure-local-inference-handler.ts` only fires when the operator has
+ * actively cleared the assignment.
+ */
+export declare function ensureDefaultAssignment(modelId: string): Promise<ModelAssignments>;
+/**
+ * Boot-time helper. If exactly one default-eligible Eliza-1 model is
+ * installed and no assignment file exists yet, auto-fill its slots so
+ * the first session works without the user opening Settings. No-op when
+ * assignments are already present or when more than one default-eligible
+ * model is installed (we cannot guess intent).
+ *
+ * External-scan blobs and custom Hugging Face downloads are intentionally
+ * excluded - see `buildRecommendedAssignments` for the rationale.
+ */
+export declare function autoAssignAtBoot(installed: InstalledModel[]): Promise<ModelAssignments | null>;
+//# sourceMappingURL=assignments.d.ts.map

package/src/services/assignments.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"assignments.d.ts","sourceRoot":"","sources":["assignments.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAOH,OAAO,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAiBhF,wBAAsB,eAAe,IAAI,OAAO,CAAC,gBAAgB,CAAC,CASjE;AAYD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,2BAA2B,CAC1C,SAAS,EAAE,cAAc,EAAE,GACzB,gBAAgB,CAmBlB;AAED,wBAAsB,wBAAwB,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAS1E;AAED,wBAAsB,gBAAgB,CACrC,WAAW,EAAE,gBAAgB,GAC3B,OAAO,CAAC,IAAI,CAAC,CAMf;AAED,wBAAsB,aAAa,CAClC,IAAI,EAAE,cAAc,EACpB,OAAO,EAAE,MAAM,GAAG,IAAI,GACpB,OAAO,CAAC,gBAAgB,CAAC,CAU3B;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAe3D;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,uBAAuB,CAC5C,OAAO,EAAE,MAAM,GACb,OAAO,CAAC,gBAAgB,CAAC,CA4B3B;AAED;;;;;;;;;GASG;AACH,wBAAsB,gBAAgB,CACrC,SAAS,EAAE,cAAc,EAAE,GACzB,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAclC"}

package/src/services/assignments.test.ts

@@ -0,0 +1,80 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import {
+	buildRecommendedAssignments,
+	ensureDefaultAssignment,
+	readAssignments,
+} from "./assignments";
+import type { InstalledModel } from "./types";
+
+const originalEnv = { ...process.env };
+
+afterEach(() => {
+	process.env = { ...originalEnv };
+});
+
+function installed(
+	id: string,
+	sizeBytes: number,
+	overrides: Partial<InstalledModel> = {},
+): InstalledModel {
+	return {
+		id,
+		displayName: id,
+		path: `/tmp/${id}.gguf`,
+		sizeBytes,
+		installedAt: "2026-05-11T00:00:00.000Z",
+		lastUsedAt: null,
+		source: "eliza-download",
+		...overrides,
+	};
+}
+
+describe("local inference assignments", () => {
+	it("does not auto-recommend ad-hoc Hugging Face downloads", () => {
+		const assignments = buildRecommendedAssignments([
+			installed("hf:some-org/some-model::model.Q4_K_M.gguf", 10_000),
+			installed("eliza-1-2b", 1_000, {
+				bundleVerifiedAt: "2026-05-11T01:00:00.000Z",
+			}),
+		]);
+
+		expect(assignments).toEqual({
+			TEXT_SMALL: "eliza-1-2b",
+			TEXT_LARGE: "eliza-1-2b",
+			TEXT_TO_SPEECH: "eliza-1-2b",
+			TRANSCRIPTION: "eliza-1-2b",
+		});
+	});
+
+	it("keeps custom-only installs out of recommended assignments", () => {
+		const assignments = buildRecommendedAssignments([
+			installed("llama-3.2-3b-instruct", 3_000, {
+				source: "external-scan",
+			}),
+			installed("hf:some-org/custom-model::model.Q4_K_M.gguf", 2_000),
+		]);
+
+		expect(assignments).toEqual({});
+	});
+
+	it("does not auto-recommend unverified Eliza-1 downloads", () => {
+		const assignments = buildRecommendedAssignments([
+			installed("eliza-1-2b", 1_000),
+		]);
+
+		expect(assignments).toEqual({});
+	});
+
+	it("does not fill defaults for custom Hugging Face model ids", async () => {
+		process.env.ELIZA_STATE_DIR = fs.mkdtempSync(
+			path.join(os.tmpdir(), "eliza-assignments-test-"),
+		);
+
+		await ensureDefaultAssignment("hf:some-org/some-model::model.Q4_K_M.gguf");
+
+		expect(await readAssignments()).toEqual({});
+	});
+});

package/src/services/assignments.ts

@@ -0,0 +1,230 @@
+/**
+ * Per-ModelType model assignment store.
+ *
+ * Separate from the "active loaded model" concept in `ActiveModelCoordinator`.
+ * Assignments are a *policy* — the user's declared intent that
+ * `ModelType.TEXT_SMALL` should be served by model X and `TEXT_LARGE` by
+ * model Y. The runtime's model handlers lazy-load whichever assignment
+ * fires; the coordinator handles the actual swap in and out of memory.
+ *
+ * Stored in `$ELIZA_STATE_DIR/local-inference/assignments.json`. Cheap
+ * enough to rewrite on every change — we never mutate in place.
+ */
+
+import fs from "node:fs/promises";
+import path from "node:path";
+import { findCatalogModel, isDefaultEligibleId } from "./catalog";
+import { localInferenceRoot } from "./paths";
+import { listInstalledModels } from "./registry";
+import type { AgentModelSlot, InstalledModel, ModelAssignments } from "./types";
+
+const ASSIGNMENTS_FILENAME = "assignments.json";
+
+interface AssignmentsFile {
+	version: 1;
+	assignments: ModelAssignments;
+}
+
+function assignmentsPath(): string {
+	return path.join(localInferenceRoot(), ASSIGNMENTS_FILENAME);
+}
+
+async function ensureRoot(): Promise<void> {
+	await fs.mkdir(localInferenceRoot(), { recursive: true });
+}
+
+export async function readAssignments(): Promise<ModelAssignments> {
+	try {
+		const raw = await fs.readFile(assignmentsPath(), "utf8");
+		const parsed = JSON.parse(raw) as AssignmentsFile;
+		if (parsed?.version !== 1 || !parsed.assignments) return {};
+		return parsed.assignments;
+	} catch {
+		return {};
+	}
+}
+
+function pickLargestInstalledModel(
+	installed: InstalledModel[],
+): InstalledModel | null {
+	return (
+		installed
+			.filter((model) => typeof model.id === "string" && model.id.length > 0)
+			.sort((left, right) => right.sizeBytes - left.sizeBytes)[0] ?? null
+	);
+}
+
+/**
+ * Build slot recommendations from currently-installed models.
+ *
+ * Only default-eligible Eliza-1 downloads are auto-recommended.
+ * External-scan blobs and ad-hoc Hugging Face downloads are surfaced only
+ * through explicit search/manual selection; they are NOT auto-assigned to
+ * any agent slot.
+ *
+ * Why: external blobs may use newer architectures or quant formats outside
+ * the bundled `capacitor-llama` binding's supported set. Auto-loading
+ * an external blob the user never selected silently breaks PROACTIVE_AGENT
+ * and other background tasks at boot. The user opted into the external
+ * tool, not into Eliza loading those weights through llama.cpp.
+ */
+export function buildRecommendedAssignments(
+	installed: InstalledModel[],
+): ModelAssignments {
+	const ownDownloads = installed.filter(
+		(model) =>
+			model.source === "eliza-download" &&
+			isDefaultEligibleId(model.id) &&
+			typeof model.bundleVerifiedAt === "string" &&
+			model.bundleVerifiedAt.length > 0,
+	);
+	const best = pickLargestInstalledModel(ownDownloads);
+	if (best) {
+		return {
+			TEXT_SMALL: best.id,
+			TEXT_LARGE: best.id,
+			TEXT_TO_SPEECH: best.id,
+			TRANSCRIPTION: best.id,
+		};
+	}
+
+	return {};
+}
+
+export async function readEffectiveAssignments(): Promise<ModelAssignments> {
+	const [saved, installed] = await Promise.all([
+		readAssignments(),
+		listInstalledModels(),
+	]);
+	return {
+		...buildRecommendedAssignments(installed),
+		...saved,
+	};
+}
+
+export async function writeAssignments(
+	assignments: ModelAssignments,
+): Promise<void> {
+	await ensureRoot();
+	const payload: AssignmentsFile = { version: 1, assignments };
+	const tmp = `${assignmentsPath()}.tmp`;
+	await fs.writeFile(tmp, JSON.stringify(payload, null, 2), "utf8");
+	await fs.rename(tmp, assignmentsPath());
+}
+
+export async function setAssignment(
+	slot: AgentModelSlot,
+	modelId: string | null,
+): Promise<ModelAssignments> {
+	const current = await readAssignments();
+	const next: ModelAssignments = { ...current };
+	if (modelId) {
+		next[slot] = modelId;
+	} else {
+		delete next[slot];
+	}
+	await writeAssignments(next);
+	return next;
+}
+
+/**
+ * Decide which slots a freshly-installed model is a sensible default for.
+ *
+ * Today the curated catalog tags models with `category` ∈
+ * `chat | code | tools | tiny | reasoning` and `bucket` ∈
+ * `small | mid | large | xl` — no explicit "embedding" tag, because the
+ * default catalog ships only generative models. The defensive check below
+ * still recognizes an "embedding" category/bucket for catalog additions and
+ * for external-scan models whose ids contain a recognizable
+ * embedding-family marker (`nomic-embed`, `bge`, `all-minilm`, `gte`,
+ * `e5-`). External GGUFs without a catalog entry default to generative.
+ */
+export function isEmbeddingModelId(modelId: string): boolean {
+	const catalog = findCatalogModel(modelId);
+	if (catalog) {
+		if ((catalog.category as string) === "embedding") return true;
+		if ((catalog.bucket as string) === "embedding") return true;
+		return false;
+	}
+	const lowered = modelId.toLowerCase();
+	return (
+		lowered.includes("nomic-embed") ||
+		lowered.includes("bge-") ||
+		lowered.includes("all-minilm") ||
+		lowered.includes("gte-") ||
+		lowered.includes("e5-")
+	);
+}
+
+/**
+ * Fill empty assignment slots with `modelId`. Idempotent: never overwrites
+ * an existing slot. Embedding models only fill `TEXT_EMBEDDING`; generative
+ * models only fill `TEXT_SMALL` and `TEXT_LARGE`. Returns the resulting
+ * assignment map (read state is `readAssignments()`, not effective +
+ * recommended).
+ *
+ * Wired from the downloader's success path and the runtime boot's
+ * "exactly one model installed, no assignments" branch so first-light
+ * users land in chat without a Settings detour. The hard error in
+ * `ensure-local-inference-handler.ts` only fires when the operator has
+ * actively cleared the assignment.
+ */
+export async function ensureDefaultAssignment(
+	modelId: string,
+): Promise<ModelAssignments> {
+	const current = await readAssignments();
+	if (!isDefaultEligibleId(modelId)) return current;
+
+	const next: ModelAssignments = { ...current };
+
+	if (isEmbeddingModelId(modelId)) {
+		if (!next.TEXT_EMBEDDING) next.TEXT_EMBEDDING = modelId;
+	} else {
+		if (!next.TEXT_SMALL) next.TEXT_SMALL = modelId;
+		if (!next.TEXT_LARGE) next.TEXT_LARGE = modelId;
+		if (!next.TEXT_TO_SPEECH) next.TEXT_TO_SPEECH = modelId;
+		if (!next.TRANSCRIPTION) next.TRANSCRIPTION = modelId;
+	}
+
+	// Cheap shortcut: skip the rewrite when nothing changed.
+	if (
+		next.TEXT_SMALL === current.TEXT_SMALL &&
+		next.TEXT_LARGE === current.TEXT_LARGE &&
+		next.TEXT_EMBEDDING === current.TEXT_EMBEDDING &&
+		next.TEXT_TO_SPEECH === current.TEXT_TO_SPEECH &&
+		next.TRANSCRIPTION === current.TRANSCRIPTION
+	) {
+		return current;
+	}
+
+	await writeAssignments(next);
+	return next;
+}
+
+/**
+ * Boot-time helper. If exactly one default-eligible Eliza-1 model is
+ * installed and no assignment file exists yet, auto-fill its slots so
+ * the first session works without the user opening Settings. No-op when
+ * assignments are already present or when more than one default-eligible
+ * model is installed (we cannot guess intent).
+ *
+ * External-scan blobs and custom Hugging Face downloads are intentionally
+ * excluded - see `buildRecommendedAssignments` for the rationale.
+ */
+export async function autoAssignAtBoot(
+	installed: InstalledModel[],
+): Promise<ModelAssignments | null> {
+	const ownDownloads = installed.filter(
+		(model) =>
+			model.source === "eliza-download" &&
+			isDefaultEligibleId(model.id) &&
+			typeof model.bundleVerifiedAt === "string" &&
+			model.bundleVerifiedAt.length > 0,
+	);
+	if (ownDownloads.length !== 1) return null;
+	const current = await readAssignments();
+	if (Object.keys(current).length > 0) return null;
+	const onlyInstalled = ownDownloads[0];
+	if (!onlyInstalled || typeof onlyInstalled.id !== "string") return null;
+	return ensureDefaultAssignment(onlyInstalled.id);
+}

package/src/services/backend-selector.ts

@@ -0,0 +1,95 @@
+/**
+ * Local-inference backend selector.
+ *
+ * One choice per host process: route generation through the in-process
+ * FFI streaming runner (`ffi-streaming-runner.ts`). The selection is
+ * deterministic so callers can mock it in tests and so no platform falls
+ * back to a server sidecar by accident.
+ *
+ * Rules:
+ *   - Mobile (Android / iOS) ALWAYS uses `ffi-streaming`. The
+ *     `llama-server` child-process path cannot ship on mobile (sandbox
+ *     restrictions, App Store review, ~10–30 ms HTTP round-trip per
+ *     token, no slot persistence on Android's APK private dir). When
+ *     the FFI symbols are absent on a mobile build we throw — there is
+ *     no second backend to fall back to.
+ *   - Desktop also uses `ffi-streaming`. Missing streaming-LLM symbols are
+ *     a bad build, not a reason to start a server backend.
+ *   - `ELIZA_INFERENCE_BACKEND=ffi` forces the streaming runner and
+ *     throws if the symbols are absent. `=auto` (or unset) follows the
+ *     rules above.
+ *
+ * The selector intentionally does NOT inspect the live process to detect
+ * mobile — callers pass that information in. The mobile bootstrap
+ * (`aosp-mtp-adapter.ts` / the iOS bridge) knows what it is; tests
+ * pass synthetic values. Keeping detection out of the selector matches
+ * AGENTS.md §7 (single source of truth for inputs) and lets the
+ * decision be replayed offline.
+ */
+
+// MLX (Apple Silicon) is not a separate provider/plugin. If we ever need it
+// for better Apple compatibility than llama.cpp's Metal path, it belongs here
+// as an additional backend compiled into the fused libelizainference target —
+// never an external mlx_lm.server HTTP sidecar.
+export type LocalInferenceBackend = "ffi-streaming";
+
+export type LocalInferencePlatform = "desktop" | "mobile";
+
+export interface BackendSelectInput {
+	/** Where the host is running. */
+	platform: LocalInferencePlatform;
+	/**
+	 * `llmStreamSupported()` from the loaded FFI binding. All builds MUST
+	 * have this true; false means the runtime was built without the unified
+	 * llama.cpp FFI path.
+	 */
+	ffiSupported: boolean;
+	/**
+	 * Optional env override (`ELIZA_INFERENCE_BACKEND`). `"ffi"` forces
+	 * the FFI path, while `"auto"` or unset follows the default rule.
+	 */
+	envOverride?: string | null;
+}
+
+/** Read the `ELIZA_INFERENCE_BACKEND` env var into a normalised value. */
+export function readBackendEnvOverride(
+	env: NodeJS.ProcessEnv = process.env,
+): "ffi" | "auto" | null {
+	const raw = env.ELIZA_INFERENCE_BACKEND?.trim().toLowerCase();
+	if (!raw || raw === "auto") return raw === "auto" ? "auto" : null;
+	if (raw === "ffi" || raw === "ffi-streaming") return "ffi";
+	return null;
+}
+
+/**
+ * Decide which local-inference backend should service text generation.
+ * See file header for the full rule set. Throws when the chosen
+ * combination is incoherent (no FFI support, explicit FFI with a bad
+ * runtime build, …).
+ */
+export function selectBackend(
+	input: BackendSelectInput,
+): LocalInferenceBackend {
+	const { platform, ffiSupported, envOverride } = input;
+	const override = (envOverride ?? "").toLowerCase();
+
+	if (override === "ffi") {
+		if (!ffiSupported) {
+			throw new Error(
+				"[backend-selector] ELIZA_INFERENCE_BACKEND=ffi but the loaded " +
+					"libelizainference does not export the streaming-LLM symbols. " +
+					"Rebuild the omnivoice fuse against the current ffi-streaming-llm.h.",
+			);
+		}
+		return "ffi-streaming";
+	}
+
+	if (!ffiSupported) {
+		const target = platform === "mobile" ? "Mobile build" : "Desktop build";
+		throw new Error(
+			`[backend-selector] ${target} missing streaming-LLM FFI symbols. ` +
+				"Rebuild libelizainference against the current ffi-streaming-llm.h.",
+		);
+	}
+	return "ffi-streaming";
+}