@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/routes/local-inference-compat-routes.test.ts

@@ -0,0 +1,485 @@
+import * as http from "node:http";
+import { Socket } from "node:net";
+import { afterEach, beforeAll, describe, expect, it, vi } from "vitest";
+import type { HardwareProbe } from "../services/types";
+import type { CompatRuntimeState } from "./compat-route-shared";
+
+// ── mocks ──────────────────────────────────────────────────────────────
+
+const setActiveMock = vi.fn();
+
+vi.mock("@elizaos/core", async (importOriginal) => {
+	const actual = await importOriginal<typeof import("@elizaos/core")>();
+	return {
+		...actual,
+		logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn(), error: vi.fn() },
+		ModelType: {
+			TEXT_LARGE: "TEXT_LARGE",
+			TEXT_SMALL: "TEXT_SMALL",
+			TEXT_EMBEDDING: "TEXT_EMBEDDING",
+			TEXT_TO_SPEECH: "TEXT_TO_SPEECH",
+			TRANSCRIPTION: "TRANSCRIPTION",
+		},
+		ResponseSkeletonStreamExtractor: class {
+			done = false;
+			push(chunk: string) {
+				return chunk;
+			}
+			flush() {
+				this.done = true;
+				return "";
+			}
+			reset() {
+				this.done = false;
+			}
+		},
+		stringToUuid: (value: string) => value,
+	};
+});
+
+vi.mock("@elizaos/agent", () => ({
+	loadElizaConfig: () => ({ meta: {}, agents: {} }),
+}));
+
+vi.mock("./auth", async (importOriginal) => {
+	const actual = await importOriginal<typeof import("./auth")>();
+	return {
+		...actual,
+		ensureRouteAuthorized: vi.fn(async () => true),
+		ensureCompatSensitiveRouteAuthorized: () => true,
+		getCompatApiToken: () => null,
+		getProvidedApiToken: () => null,
+		tokenMatches: () => true,
+	};
+});
+
+vi.mock("./auth/sessions", async (importOriginal) => {
+	const actual = await importOriginal<typeof import("./auth/sessions")>();
+	return {
+		...actual,
+		findActiveSession: vi.fn(async () => null),
+		parseSessionCookie: vi.fn(() => null),
+	};
+});
+
+vi.mock("./server-first-run-helpers", () => ({
+	isCloudProvisioned: () => false,
+}));
+
+vi.mock("../services/service", () => ({
+	localInferenceService: {
+		setActive: setActiveMock,
+		getActive: () => ({ modelId: null, loadedAt: null, status: "idle" }),
+		clearActive: vi.fn(async () => ({
+			modelId: null,
+			loadedAt: null,
+			status: "idle",
+		})),
+		getCatalog: () => [],
+		snapshot: vi.fn(),
+		getInstalled: vi.fn(),
+		getHardware: vi.fn(),
+		getDownloads: () => [],
+		getAssignments: vi.fn(),
+		getTextReadiness: vi.fn(),
+		setSlotAssignment: vi.fn(),
+		startDownload: vi.fn(),
+		cancelDownload: vi.fn(),
+		subscribeDownloads: vi.fn(),
+		subscribeActive: vi.fn(),
+		searchHuggingFace: vi.fn(),
+		verifyModel: vi.fn(),
+		uninstall: vi.fn(),
+		getRecommendedModel: vi.fn(),
+		getRecommendedModels: vi.fn(),
+		startSmallerFallbackDownload: vi.fn(),
+		getLocalCacheStats: vi.fn(),
+	},
+}));
+
+vi.mock("../services/device-bridge", () => ({
+	deviceBridge: { status: () => ({ connected: false, devices: [] }) },
+}));
+
+vi.mock("../services/handler-registry", () => ({
+	handlerRegistry: { getAll: () => [] },
+	toPublicRegistration: (r: unknown) => r,
+}));
+
+vi.mock("../services/providers", () => ({
+	snapshotProviders: vi.fn(async () => []),
+}));
+
+vi.mock("../services/routing-preferences", () => ({
+	readRoutingPreferences: vi.fn(async () => ({})),
+	setPolicy: vi.fn(),
+	setPreferredProvider: vi.fn(),
+}));
+
+const STATE: CompatRuntimeState = {
+	current: null,
+	pendingAgentName: null,
+	pendingRestartReasons: [],
+};
+
+// ── test helpers ───────────────────────────────────────────────────────
+
+let handleLocalInferenceCompatRoutes: typeof import("./local-inference-compat-routes").handleLocalInferenceCompatRoutes;
+
+interface FakeRes {
+	res: http.ServerResponse;
+	body(): unknown;
+	status(): number;
+}
+
+function fakeRes(): FakeRes {
+	let bodyText = "";
+	const req = new http.IncomingMessage(new Socket());
+	const res = new http.ServerResponse(req);
+	res.statusCode = 200;
+	res.setHeader = () => res;
+	res.end = ((chunk?: string | Buffer) => {
+		if (typeof chunk === "string") bodyText += chunk;
+		else if (chunk) bodyText += chunk.toString("utf8");
+		return res;
+	}) as typeof res.end;
+	return {
+		res,
+		body() {
+			return bodyText.length > 0 ? JSON.parse(bodyText) : null;
+		},
+		status() {
+			return res.statusCode;
+		},
+	};
+}
+
+function fakeReq(opts: {
+	method: string;
+	pathname: string;
+	body?: unknown;
+}): http.IncomingMessage {
+	const req = new http.IncomingMessage(new Socket());
+	req.method = opts.method;
+	req.url = opts.pathname;
+	req.headers = { host: "localhost:2138" };
+	Object.defineProperty(req.socket, "remoteAddress", {
+		value: "127.0.0.1",
+		configurable: true,
+	});
+	if (opts.body !== undefined) {
+		(req as { body?: unknown }).body = opts.body;
+	}
+	return req;
+}
+
+// ── tests ──────────────────────────────────────────────────────────────
+
+describe("POST /api/local-inference/active", () => {
+	beforeAll(async () => {
+		handleLocalInferenceCompatRoutes = (
+			await import("./local-inference-compat-routes")
+		).handleLocalInferenceCompatRoutes;
+	}, 120_000);
+
+	afterEach(() => {
+		setActiveMock.mockReset();
+	});
+
+	it("accepts the legacy { modelId } body shape (no overrides)", async () => {
+		setActiveMock.mockResolvedValue({
+			modelId: "eliza-1-2b",
+			loadedAt: "2026-05-09T00:00:00.000Z",
+			status: "ready",
+		});
+
+		const res = fakeRes();
+		const handled = await handleLocalInferenceCompatRoutes(
+			fakeReq({
+				method: "POST",
+				pathname: "/api/local-inference/active",
+				body: { modelId: "eliza-1-2b" },
+			}),
+			res.res,
+			STATE,
+		);
+
+		expect(handled).toBe(true);
+		expect(res.status()).toBe(200);
+		expect(setActiveMock).toHaveBeenCalledWith(null, "eliza-1-2b", undefined);
+	});
+
+	it("forwards a parsed overrides block to setActive", async () => {
+		setActiveMock.mockResolvedValue({
+			modelId: "eliza-1-2b",
+			loadedAt: "2026-05-09T00:00:00.000Z",
+			status: "ready",
+		});
+
+		const res = fakeRes();
+		await handleLocalInferenceCompatRoutes(
+			fakeReq({
+				method: "POST",
+				pathname: "/api/local-inference/active",
+				body: {
+					modelId: "eliza-1-2b",
+					overrides: {
+						contextSize: 131072,
+						cacheTypeK: "f16",
+						cacheTypeV: "q8_0",
+						gpuLayers: 32,
+						flashAttention: true,
+					},
+				},
+			}),
+			res.res,
+			STATE,
+		);
+
+		expect(res.status()).toBe(200);
+		expect(setActiveMock).toHaveBeenCalledWith(null, "eliza-1-2b", {
+			contextSize: 131072,
+			cacheTypeK: "f16",
+			cacheTypeV: "q8_0",
+			gpuLayers: 32,
+			flashAttention: true,
+		});
+	});
+
+	it("forwards fork-only KV cache types to the optimized backend gate", async () => {
+		setActiveMock.mockResolvedValue({
+			modelId: "eliza-1-2b",
+			loadedAt: "2026-05-09T00:00:00.000Z",
+			status: "ready",
+		});
+
+		const res = fakeRes();
+		await handleLocalInferenceCompatRoutes(
+			fakeReq({
+				method: "POST",
+				pathname: "/api/local-inference/active",
+				body: {
+					modelId: "eliza-1-2b",
+					overrides: { cacheTypeK: "tbq4_0", cacheTypeV: "tbq3_0" },
+				},
+			}),
+			res.res,
+			STATE,
+		);
+
+		expect(res.status()).toBe(200);
+		expect(setActiveMock).toHaveBeenCalledWith(null, "eliza-1-2b", {
+			cacheTypeK: "tbq4_0",
+			cacheTypeV: "tbq3_0",
+		});
+	});
+
+	it("rejects illegal contextSize", async () => {
+		const res = fakeRes();
+		await handleLocalInferenceCompatRoutes(
+			fakeReq({
+				method: "POST",
+				pathname: "/api/local-inference/active",
+				body: {
+					modelId: "eliza-1-2b",
+					overrides: { contextSize: 100 },
+				},
+			}),
+			res.res,
+			STATE,
+		);
+
+		expect(res.status()).toBe(400);
+		expect((res.body() as { error: string }).error).toMatch(/contextSize/);
+		expect(setActiveMock).not.toHaveBeenCalled();
+	});
+
+	it("rejects illegal kvOffload values", async () => {
+		const res = fakeRes();
+		await handleLocalInferenceCompatRoutes(
+			fakeReq({
+				method: "POST",
+				pathname: "/api/local-inference/active",
+				body: {
+					modelId: "eliza-1-2b",
+					overrides: { kvOffload: "magic" },
+				},
+			}),
+			res.res,
+			STATE,
+		);
+
+		expect(res.status()).toBe(400);
+		expect((res.body() as { error: string }).error).toMatch(/kvOffload/);
+	});
+
+	it("accepts kvOffload object form { gpuLayers: N }", async () => {
+		setActiveMock.mockResolvedValue({
+			modelId: "eliza-1-2b",
+			loadedAt: "2026-05-09T00:00:00.000Z",
+			status: "ready",
+		});
+
+		const res = fakeRes();
+		await handleLocalInferenceCompatRoutes(
+			fakeReq({
+				method: "POST",
+				pathname: "/api/local-inference/active",
+				body: {
+					modelId: "eliza-1-2b",
+					overrides: { kvOffload: { gpuLayers: 16 } },
+				},
+			}),
+			res.res,
+			STATE,
+		);
+
+		expect(res.status()).toBe(200);
+		expect(setActiveMock).toHaveBeenCalledWith(null, "eliza-1-2b", {
+			kvOffload: { gpuLayers: 16 },
+		});
+	});
+
+	it("rejects an overrides field that isn't an object", async () => {
+		const res = fakeRes();
+		await handleLocalInferenceCompatRoutes(
+			fakeReq({
+				method: "POST",
+				pathname: "/api/local-inference/active",
+				body: { modelId: "eliza-1-2b", overrides: "nope" },
+			}),
+			res.res,
+			STATE,
+		);
+
+		expect(res.status()).toBe(400);
+		expect((res.body() as { error: string }).error).toMatch(
+			/overrides must be an object/,
+		);
+	});
+
+	// #7679: refuse to activate a candidate-only / weights-staged bundle
+	// whose own manifest reports `evals.textEval.passed=false`.
+	it("returns 422 with manifestVersion + failedEvals when the bundle is candidate-only", async () => {
+		const { CandidateModelActivationError } = await import(
+			"../services/active-model"
+		);
+		setActiveMock.mockRejectedValue(
+			new CandidateModelActivationError({
+				modelId: "eliza-1-2b",
+				manifestVersion: "1.0.0-candidate.1",
+				failedEvals: ["textEval", "voiceRtf", "asrWer", "expressive"],
+			}),
+		);
+
+		const res = fakeRes();
+		await handleLocalInferenceCompatRoutes(
+			fakeReq({
+				method: "POST",
+				pathname: "/api/local-inference/active",
+				body: { modelId: "eliza-1-2b" },
+			}),
+			res.res,
+			STATE,
+		);
+
+		expect(res.status()).toBe(422);
+		const body = res.body() as {
+			error: string;
+			modelId: string;
+			manifestVersion: string;
+			failedEvals: string[];
+		};
+		expect(body.modelId).toBe("eliza-1-2b");
+		expect(body.manifestVersion).toBe("1.0.0-candidate.1");
+		expect(body.failedEvals).toContain("textEval");
+		expect(body.failedEvals).toContain("voiceRtf");
+		expect(body.error).toMatch(/candidate-only/);
+		expect(body.error).toMatch(/textEval/);
+	});
+
+	it("returns 200 (delegated to setActive) when the bundle is a strict release", async () => {
+		setActiveMock.mockResolvedValue({
+			modelId: "eliza-1-2b",
+			loadedAt: "2026-05-14T00:00:00.000Z",
+			status: "ready",
+		});
+
+		const res = fakeRes();
+		await handleLocalInferenceCompatRoutes(
+			fakeReq({
+				method: "POST",
+				pathname: "/api/local-inference/active",
+				body: { modelId: "eliza-1-2b" },
+			}),
+			res.res,
+			STATE,
+		);
+
+		expect(res.status()).toBe(200);
+		const body = res.body() as { modelId: string; status: string };
+		expect(body.modelId).toBe("eliza-1-2b");
+		expect(body.status).toBe("ready");
+	});
+});
+
+describe("GET /api/local-inference/device-tier", () => {
+	beforeAll(async () => {
+		handleLocalInferenceCompatRoutes = (
+			await import("./local-inference-compat-routes")
+		).handleLocalInferenceCompatRoutes;
+	}, 120_000);
+
+	const probe: HardwareProbe = {
+		totalRamGb: 32,
+		freeRamGb: 16,
+		gpu: null,
+		cpuCores: 16,
+		platform: "linux",
+		arch: "x64",
+		appleSilicon: false,
+		recommendedBucket: "mid",
+		source: "capacitor-llama",
+	};
+
+	it("returns the device tier + live memory budget", async () => {
+		const { localInferenceService } = await import("../services/service");
+		vi.mocked(localInferenceService.getHardware).mockResolvedValue(probe);
+		const res = fakeRes();
+		const handled = await handleLocalInferenceCompatRoutes(
+			fakeReq({ method: "GET", pathname: "/api/local-inference/device-tier" }),
+			res.res,
+			STATE,
+		);
+		expect(handled).toBe(true);
+		expect(res.status()).toBe(200);
+		const body = res.body() as {
+			tier: { tier: string };
+			memory: { availableBytes: number; totalBytes: number };
+			resident: unknown;
+		};
+		expect(["MAX", "GOOD", "OKAY", "POOR"]).toContain(body.tier.tier);
+		// Memory comes from the live system reader, not the probe.
+		expect(body.memory.totalBytes).toBeGreaterThan(0);
+		expect(body.memory.availableBytes).toBeGreaterThan(0);
+		expect(body.memory.availableBytes).toBeLessThanOrEqual(
+			body.memory.totalBytes,
+		);
+		// No arbiter configured in this unit context → resident is null, not a throw.
+		expect(body.resident).toBeNull();
+	});
+
+	it("surfaces a 500 when the hardware probe fails", async () => {
+		const { localInferenceService } = await import("../services/service");
+		vi.mocked(localInferenceService.getHardware).mockRejectedValue(
+			new Error("probe boom"),
+		);
+		const res = fakeRes();
+		await handleLocalInferenceCompatRoutes(
+			fakeReq({ method: "GET", pathname: "/api/local-inference/device-tier" }),
+			res.res,
+			STATE,
+		);
+		expect(res.status()).toBe(500);
+	});
+});