@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.2

package/src/routes/local-inference-asr-transcribe.ts

@@ -0,0 +1,97 @@
+import { type AgentRuntime, ModelType } from "@elizaos/core";
+import { localInferenceEngine } from "../services/engine";
+import { type AsrWordTiming, decodeMonoPcm16Wav } from "../services/voice";
+
+/** A transcript plus optional per-word timings (ms from audio start). */
+export interface LocalAsrTranscript {
+	text: string;
+	words: AsrWordTiming[];
+}
+
+/**
+ * Providers the multi-platform `useModel` fallback tries in order. Exactly one
+ * is registered per platform; the loop skips "no handler" providers and stops
+ * at the first that answers. This path carries no word timings — the locked
+ * `ModelType.TRANSCRIPTION ⇒ string` contract returns text only.
+ */
+const LOCAL_TRANSCRIPTION_PROVIDER_IDS = [
+	"eliza-local-inference",
+	"capacitor-llama",
+	"eliza-device-bridge",
+	"eliza-aosp-llama",
+] as const;
+
+function isMissingTranscriptionProviderError(error: unknown): boolean {
+	return (
+		error instanceof Error &&
+		/No handler found for delegate type: TRANSCRIPTION/.test(error.message)
+	);
+}
+
+function normalizeTranscriptText(value: unknown): string {
+	if (typeof value === "string") return value.trim();
+	if (value && typeof value === "object") {
+		const text = (value as { text?: unknown }).text;
+		if (typeof text === "string") return text.trim();
+	}
+	throw new Error("TRANSCRIPTION returned an invalid transcript");
+}
+
+/**
+ * Transcribe WAV audio to text + per-word timings.
+ *
+ * The primary path is the single fused libelizainference FFI pipe
+ * (`transcribePcmTimed`, ASR ABI v12) — the cross-platform on-device runtime
+ * that emits per-word `[startMs,endMs)` timings the transcript player
+ * highlights against. When the in-process engine is not the active local
+ * backend (a device-bridge / capacitor / AOSP topology owns transcription),
+ * fall back to the `useModel` provider chain for the transcript text; word
+ * timings are unavailable on that path (`words: []`).
+ */
+export async function transcribeWavWithWords(
+	runtime: AgentRuntime,
+	audioWav: Uint8Array,
+	signal?: AbortSignal,
+): Promise<LocalAsrTranscript> {
+	if (await localInferenceEngine.available()) {
+		const audio = decodeMonoPcm16Wav(audioWav);
+		await localInferenceEngine.ensureActiveBundleVoiceReady();
+		const { text, words } = await localInferenceEngine.transcribePcmTimed(
+			audio,
+			signal,
+		);
+		return { text: text.trim(), words: [...words] };
+	}
+	return {
+		text: await transcribeViaModelChain(runtime, audioWav, signal),
+		words: [],
+	};
+}
+
+async function transcribeViaModelChain(
+	runtime: AgentRuntime,
+	audio: Uint8Array,
+	signal?: AbortSignal,
+): Promise<string> {
+	let lastError: unknown;
+	for (const provider of LOCAL_TRANSCRIPTION_PROVIDER_IDS) {
+		try {
+			const transcript = normalizeTranscriptText(
+				await runtime.useModel(
+					ModelType.TRANSCRIPTION,
+					{ audio, ...(signal ? { signal } : {}) } as never,
+					provider,
+				),
+			);
+			if (!transcript) {
+				throw new Error("TRANSCRIPTION returned an empty transcript");
+			}
+			return transcript;
+		} catch (err) {
+			lastError = err;
+			if (!isMissingTranscriptionProviderError(err)) throw err;
+		}
+	}
+	if (lastError instanceof Error) throw lastError;
+	throw new Error("No local-inference TRANSCRIPTION provider is registered");
+}

package/src/routes/local-inference-compat-routes.d.ts

@@ -0,0 +1,16 @@
+/**
+ * HTTP routes for the local-inference / model management feature.
+ *
+ * Route shape and auth follow the established `*-compat-routes.ts` pattern:
+ *   - `handleLocalInferenceCompatRoutes` returns `true` when it handles a
+ *     request and `false` to pass through to the next handler.
+ *   - Regular reads use `ensureCompatApiAuthorized`.
+ *   - Mutating routes (download start/cancel, active switch, uninstall)
+ *     use `ensureCompatSensitiveRouteAuthorized`.
+ *   - SSE allows `?token=...` as an alternative to the auth header, via
+ *     `isStreamAuthorized`.
+ */
+import type http from "node:http";
+import { type CompatRuntimeState } from "./compat-helpers";
+export declare function handleLocalInferenceCompatRoutes(req: http.IncomingMessage, res: http.ServerResponse, state: CompatRuntimeState): Promise<boolean>;
+//# sourceMappingURL=local-inference-compat-routes.d.ts.map

package/src/routes/local-inference-compat-routes.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"local-inference-compat-routes.d.ts","sourceRoot":"","sources":["local-inference-compat-routes.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AA0BlC,OAAO,EACN,KAAK,kBAAkB,EASvB,MAAM,kBAAkB,CAAC;AAwL1B,wBAAsB,gCAAgC,CACrD,GAAG,EAAE,IAAI,CAAC,eAAe,EACzB,GAAG,EAAE,IAAI,CAAC,cAAc,EACxB,KAAK,EAAE,kBAAkB,GACvB,OAAO,CAAC,OAAO,CAAC,CA0jBlB"}

package/src/routes/local-inference-compat-routes.test.ts

@@ -0,0 +1,485 @@
+import * as http from "node:http";
+import { Socket } from "node:net";
+import { afterEach, beforeAll, describe, expect, it, vi } from "vitest";
+import type { HardwareProbe } from "../services/types";
+import type { CompatRuntimeState } from "./compat-route-shared";
+
+// ── mocks ──────────────────────────────────────────────────────────────
+
+const setActiveMock = vi.fn();
+
+vi.mock("@elizaos/core", async (importOriginal) => {
+	const actual = await importOriginal<typeof import("@elizaos/core")>();
+	return {
+		...actual,
+		logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn(), error: vi.fn() },
+		ModelType: {
+			TEXT_LARGE: "TEXT_LARGE",
+			TEXT_SMALL: "TEXT_SMALL",
+			TEXT_EMBEDDING: "TEXT_EMBEDDING",
+			TEXT_TO_SPEECH: "TEXT_TO_SPEECH",
+			TRANSCRIPTION: "TRANSCRIPTION",
+		},
+		ResponseSkeletonStreamExtractor: class {
+			done = false;
+			push(chunk: string) {
+				return chunk;
+			}
+			flush() {
+				this.done = true;
+				return "";
+			}
+			reset() {
+				this.done = false;
+			}
+		},
+		stringToUuid: (value: string) => value,
+	};
+});
+
+vi.mock("@elizaos/agent", () => ({
+	loadElizaConfig: () => ({ meta: {}, agents: {} }),
+}));
+
+vi.mock("./auth", async (importOriginal) => {
+	const actual = await importOriginal<typeof import("./auth")>();
+	return {
+		...actual,
+		ensureRouteAuthorized: vi.fn(async () => true),
+		ensureCompatSensitiveRouteAuthorized: () => true,
+		getCompatApiToken: () => null,
+		getProvidedApiToken: () => null,
+		tokenMatches: () => true,
+	};
+});
+
+vi.mock("./auth/sessions", async (importOriginal) => {
+	const actual = await importOriginal<typeof import("./auth/sessions")>();
+	return {
+		...actual,
+		findActiveSession: vi.fn(async () => null),
+		parseSessionCookie: vi.fn(() => null),
+	};
+});
+
+vi.mock("./server-first-run-helpers", () => ({
+	isCloudProvisioned: () => false,
+}));
+
+vi.mock("../services/service", () => ({
+	localInferenceService: {
+		setActive: setActiveMock,
+		getActive: () => ({ modelId: null, loadedAt: null, status: "idle" }),
+		clearActive: vi.fn(async () => ({
+			modelId: null,
+			loadedAt: null,
+			status: "idle",
+		})),
+		getCatalog: () => [],
+		snapshot: vi.fn(),
+		getInstalled: vi.fn(),
+		getHardware: vi.fn(),
+		getDownloads: () => [],
+		getAssignments: vi.fn(),
+		getTextReadiness: vi.fn(),
+		setSlotAssignment: vi.fn(),
+		startDownload: vi.fn(),
+		cancelDownload: vi.fn(),
+		subscribeDownloads: vi.fn(),
+		subscribeActive: vi.fn(),
+		searchHuggingFace: vi.fn(),
+		verifyModel: vi.fn(),
+		uninstall: vi.fn(),
+		getRecommendedModel: vi.fn(),
+		getRecommendedModels: vi.fn(),
+		startSmallerFallbackDownload: vi.fn(),
+		getLocalCacheStats: vi.fn(),
+	},
+}));
+
+vi.mock("../services/device-bridge", () => ({
+	deviceBridge: { status: () => ({ connected: false, devices: [] }) },
+}));
+
+vi.mock("../services/handler-registry", () => ({
+	handlerRegistry: { getAll: () => [] },
+	toPublicRegistration: (r: unknown) => r,
+}));
+
+vi.mock("../services/providers", () => ({
+	snapshotProviders: vi.fn(async () => []),
+}));
+
+vi.mock("../services/routing-preferences", () => ({
+	readRoutingPreferences: vi.fn(async () => ({})),
+	setPolicy: vi.fn(),
+	setPreferredProvider: vi.fn(),
+}));
+
+const STATE: CompatRuntimeState = {
+	current: null,
+	pendingAgentName: null,
+	pendingRestartReasons: [],
+};
+
+// ── test helpers ───────────────────────────────────────────────────────
+
+let handleLocalInferenceCompatRoutes: typeof import("./local-inference-compat-routes").handleLocalInferenceCompatRoutes;
+
+interface FakeRes {
+	res: http.ServerResponse;
+	body(): unknown;
+	status(): number;
+}
+
+function fakeRes(): FakeRes {
+	let bodyText = "";
+	const req = new http.IncomingMessage(new Socket());
+	const res = new http.ServerResponse(req);
+	res.statusCode = 200;
+	res.setHeader = () => res;
+	res.end = ((chunk?: string | Buffer) => {
+		if (typeof chunk === "string") bodyText += chunk;
+		else if (chunk) bodyText += chunk.toString("utf8");
+		return res;
+	}) as typeof res.end;
+	return {
+		res,
+		body() {
+			return bodyText.length > 0 ? JSON.parse(bodyText) : null;
+		},
+		status() {
+			return res.statusCode;
+		},
+	};
+}
+
+function fakeReq(opts: {
+	method: string;
+	pathname: string;
+	body?: unknown;
+}): http.IncomingMessage {
+	const req = new http.IncomingMessage(new Socket());
+	req.method = opts.method;
+	req.url = opts.pathname;
+	req.headers = { host: "localhost:2138" };
+	Object.defineProperty(req.socket, "remoteAddress", {
+		value: "127.0.0.1",
+		configurable: true,
+	});
+	if (opts.body !== undefined) {
+		(req as { body?: unknown }).body = opts.body;
+	}
+	return req;
+}
+
+// ── tests ──────────────────────────────────────────────────────────────
+
+describe("POST /api/local-inference/active", () => {
+	beforeAll(async () => {
+		handleLocalInferenceCompatRoutes = (
+			await import("./local-inference-compat-routes")
+		).handleLocalInferenceCompatRoutes;
+	}, 120_000);
+
+	afterEach(() => {
+		setActiveMock.mockReset();
+	});
+
+	it("accepts the legacy { modelId } body shape (no overrides)", async () => {
+		setActiveMock.mockResolvedValue({
+			modelId: "eliza-1-2b",
+			loadedAt: "2026-05-09T00:00:00.000Z",
+			status: "ready",
+		});
+
+		const res = fakeRes();
+		const handled = await handleLocalInferenceCompatRoutes(
+			fakeReq({
+				method: "POST",
+				pathname: "/api/local-inference/active",
+				body: { modelId: "eliza-1-2b" },
+			}),
+			res.res,
+			STATE,
+		);
+
+		expect(handled).toBe(true);
+		expect(res.status()).toBe(200);
+		expect(setActiveMock).toHaveBeenCalledWith(null, "eliza-1-2b", undefined);
+	});
+
+	it("forwards a parsed overrides block to setActive", async () => {
+		setActiveMock.mockResolvedValue({
+			modelId: "eliza-1-2b",
+			loadedAt: "2026-05-09T00:00:00.000Z",
+			status: "ready",
+		});
+
+		const res = fakeRes();
+		await handleLocalInferenceCompatRoutes(
+			fakeReq({
+				method: "POST",
+				pathname: "/api/local-inference/active",
+				body: {
+					modelId: "eliza-1-2b",
+					overrides: {
+						contextSize: 131072,
+						cacheTypeK: "f16",
+						cacheTypeV: "q8_0",
+						gpuLayers: 32,
+						flashAttention: true,
+					},
+				},
+			}),
+			res.res,
+			STATE,
+		);
+
+		expect(res.status()).toBe(200);
+		expect(setActiveMock).toHaveBeenCalledWith(null, "eliza-1-2b", {
+			contextSize: 131072,
+			cacheTypeK: "f16",
+			cacheTypeV: "q8_0",
+			gpuLayers: 32,
+			flashAttention: true,
+		});
+	});
+
+	it("forwards fork-only KV cache types to the optimized backend gate", async () => {
+		setActiveMock.mockResolvedValue({
+			modelId: "eliza-1-2b",
+			loadedAt: "2026-05-09T00:00:00.000Z",
+			status: "ready",
+		});
+
+		const res = fakeRes();
+		await handleLocalInferenceCompatRoutes(
+			fakeReq({
+				method: "POST",
+				pathname: "/api/local-inference/active",
+				body: {
+					modelId: "eliza-1-2b",
+					overrides: { cacheTypeK: "tbq4_0", cacheTypeV: "tbq3_0" },
+				},
+			}),
+			res.res,
+			STATE,
+		);
+
+		expect(res.status()).toBe(200);
+		expect(setActiveMock).toHaveBeenCalledWith(null, "eliza-1-2b", {
+			cacheTypeK: "tbq4_0",
+			cacheTypeV: "tbq3_0",
+		});
+	});
+
+	it("rejects illegal contextSize", async () => {
+		const res = fakeRes();
+		await handleLocalInferenceCompatRoutes(
+			fakeReq({
+				method: "POST",
+				pathname: "/api/local-inference/active",
+				body: {
+					modelId: "eliza-1-2b",
+					overrides: { contextSize: 100 },
+				},
+			}),
+			res.res,
+			STATE,
+		);
+
+		expect(res.status()).toBe(400);
+		expect((res.body() as { error: string }).error).toMatch(/contextSize/);
+		expect(setActiveMock).not.toHaveBeenCalled();
+	});
+
+	it("rejects illegal kvOffload values", async () => {
+		const res = fakeRes();
+		await handleLocalInferenceCompatRoutes(
+			fakeReq({
+				method: "POST",
+				pathname: "/api/local-inference/active",
+				body: {
+					modelId: "eliza-1-2b",
+					overrides: { kvOffload: "magic" },
+				},
+			}),
+			res.res,
+			STATE,
+		);
+
+		expect(res.status()).toBe(400);
+		expect((res.body() as { error: string }).error).toMatch(/kvOffload/);
+	});
+
+	it("accepts kvOffload object form { gpuLayers: N }", async () => {
+		setActiveMock.mockResolvedValue({
+			modelId: "eliza-1-2b",
+			loadedAt: "2026-05-09T00:00:00.000Z",
+			status: "ready",
+		});
+
+		const res = fakeRes();
+		await handleLocalInferenceCompatRoutes(
+			fakeReq({
+				method: "POST",
+				pathname: "/api/local-inference/active",
+				body: {
+					modelId: "eliza-1-2b",
+					overrides: { kvOffload: { gpuLayers: 16 } },
+				},
+			}),
+			res.res,
+			STATE,
+		);
+
+		expect(res.status()).toBe(200);
+		expect(setActiveMock).toHaveBeenCalledWith(null, "eliza-1-2b", {
+			kvOffload: { gpuLayers: 16 },
+		});
+	});
+
+	it("rejects an overrides field that isn't an object", async () => {
+		const res = fakeRes();
+		await handleLocalInferenceCompatRoutes(
+			fakeReq({
+				method: "POST",
+				pathname: "/api/local-inference/active",
+				body: { modelId: "eliza-1-2b", overrides: "nope" },
+			}),
+			res.res,
+			STATE,
+		);
+
+		expect(res.status()).toBe(400);
+		expect((res.body() as { error: string }).error).toMatch(
+			/overrides must be an object/,
+		);
+	});
+
+	// #7679: refuse to activate a candidate-only / weights-staged bundle
+	// whose own manifest reports `evals.textEval.passed=false`.
+	it("returns 422 with manifestVersion + failedEvals when the bundle is candidate-only", async () => {
+		const { CandidateModelActivationError } = await import(
+			"../services/active-model"
+		);
+		setActiveMock.mockRejectedValue(
+			new CandidateModelActivationError({
+				modelId: "eliza-1-0_8b",
+				manifestVersion: "1.0.0-candidate.1",
+				failedEvals: ["textEval", "voiceRtf", "asrWer", "expressive"],
+			}),
+		);
+
+		const res = fakeRes();
+		await handleLocalInferenceCompatRoutes(
+			fakeReq({
+				method: "POST",
+				pathname: "/api/local-inference/active",
+				body: { modelId: "eliza-1-0_8b" },
+			}),
+			res.res,
+			STATE,
+		);
+
+		expect(res.status()).toBe(422);
+		const body = res.body() as {
+			error: string;
+			modelId: string;
+			manifestVersion: string;
+			failedEvals: string[];
+		};
+		expect(body.modelId).toBe("eliza-1-0_8b");
+		expect(body.manifestVersion).toBe("1.0.0-candidate.1");
+		expect(body.failedEvals).toContain("textEval");
+		expect(body.failedEvals).toContain("voiceRtf");
+		expect(body.error).toMatch(/candidate-only/);
+		expect(body.error).toMatch(/textEval/);
+	});
+
+	it("returns 200 (delegated to setActive) when the bundle is a strict release", async () => {
+		setActiveMock.mockResolvedValue({
+			modelId: "eliza-1-2b",
+			loadedAt: "2026-05-14T00:00:00.000Z",
+			status: "ready",
+		});
+
+		const res = fakeRes();
+		await handleLocalInferenceCompatRoutes(
+			fakeReq({
+				method: "POST",
+				pathname: "/api/local-inference/active",
+				body: { modelId: "eliza-1-2b" },
+			}),
+			res.res,
+			STATE,
+		);
+
+		expect(res.status()).toBe(200);
+		const body = res.body() as { modelId: string; status: string };
+		expect(body.modelId).toBe("eliza-1-2b");
+		expect(body.status).toBe("ready");
+	});
+});
+
+describe("GET /api/local-inference/device-tier", () => {
+	beforeAll(async () => {
+		handleLocalInferenceCompatRoutes = (
+			await import("./local-inference-compat-routes")
+		).handleLocalInferenceCompatRoutes;
+	}, 120_000);
+
+	const probe: HardwareProbe = {
+		totalRamGb: 32,
+		freeRamGb: 16,
+		gpu: null,
+		cpuCores: 16,
+		platform: "linux",
+		arch: "x64",
+		appleSilicon: false,
+		recommendedBucket: "mid",
+		source: "capacitor-llama",
+	};
+
+	it("returns the device tier + live memory budget", async () => {
+		const { localInferenceService } = await import("../services/service");
+		vi.mocked(localInferenceService.getHardware).mockResolvedValue(probe);
+		const res = fakeRes();
+		const handled = await handleLocalInferenceCompatRoutes(
+			fakeReq({ method: "GET", pathname: "/api/local-inference/device-tier" }),
+			res.res,
+			STATE,
+		);
+		expect(handled).toBe(true);
+		expect(res.status()).toBe(200);
+		const body = res.body() as {
+			tier: { tier: string };
+			memory: { availableBytes: number; totalBytes: number };
+			resident: unknown;
+		};
+		expect(["MAX", "GOOD", "OKAY", "POOR"]).toContain(body.tier.tier);
+		// Memory comes from the live system reader, not the probe.
+		expect(body.memory.totalBytes).toBeGreaterThan(0);
+		expect(body.memory.availableBytes).toBeGreaterThan(0);
+		expect(body.memory.availableBytes).toBeLessThanOrEqual(
+			body.memory.totalBytes,
+		);
+		// No arbiter configured in this unit context → resident is null, not a throw.
+		expect(body.resident).toBeNull();
+	});
+
+	it("surfaces a 500 when the hardware probe fails", async () => {
+		const { localInferenceService } = await import("../services/service");
+		vi.mocked(localInferenceService.getHardware).mockRejectedValue(
+			new Error("probe boom"),
+		);
+		const res = fakeRes();
+		await handleLocalInferenceCompatRoutes(
+			fakeReq({ method: "GET", pathname: "/api/local-inference/device-tier" }),
+			res.res,
+			STATE,
+		);
+		expect(res.status()).toBe(500);
+	});
+});