@elizaos/plugin-local-inference 2.0.3-beta.2 → 2.0.3-beta.4

package/src/routes/local-inference-compat-routes.ts

@@ -18,6 +18,7 @@ import {
 	type LocalInferenceLoadOverrides,
 	validateLocalInferenceLoadArgs,
 } from "../services/active-model";
+import { AssignmentRejectedError } from "../services/assignments";
 import { deviceBridge } from "../services/device-bridge";
 import { classifyDeviceTier } from "../services/device-tier";
 import {
@@ -35,7 +36,7 @@ import {
 } from "../services/routing-preferences";
 import { localInferenceService } from "../services/service";
 import { readSystemMemory } from "../services/system-memory";
-import type { AgentModelSlot, CatalogModel } from "../services/types";
+import type { AgentModelSlot } from "../services/types";
 import { AGENT_MODEL_SLOTS } from "../services/types";
 import {
 	type CompatRuntimeState,
@@ -49,26 +50,6 @@ import {
 	tokenMatches,
 } from "./compat-helpers";
 
-function isCatalogModel(value: unknown): value is CatalogModel {
-	if (!value || typeof value !== "object" || Array.isArray(value)) {
-		return false;
-	}
-	const record = value as Partial<CatalogModel>;
-	return (
-		typeof record.id === "string" &&
-		typeof record.displayName === "string" &&
-		typeof record.hfRepo === "string" &&
-		typeof record.ggufFile === "string" &&
-		typeof record.params === "string" &&
-		typeof record.quant === "string" &&
-		typeof record.sizeGb === "number" &&
-		typeof record.minRamGb === "number" &&
-		typeof record.category === "string" &&
-		typeof record.bucket === "string" &&
-		typeof record.blurb === "string"
-	);
-}
-
 function isStreamAuthorized(
 	req: http.IncomingMessage,
 	res: http.ServerResponse,
@@ -391,8 +372,7 @@ export async function handleLocalInferenceCompatRoutes(
 	}
 
 	// ── POST: start download ────────────────────────────────────────────
-	// Body: either `{ modelId }` for a curated entry, or
-	// `{ spec: CatalogModel }` for a HuggingFace-search result.
+	// Body: `{ modelId }` for a curated Eliza-1 entry.
 	if (method === "POST" && pathname === "/api/local-inference/downloads") {
 		if (!ensureCompatSensitiveRouteAuthorized(req, res)) return true;
 		const body = await readCompatJsonBody(req, res);
@@ -402,15 +382,16 @@ export async function handleLocalInferenceCompatRoutes(
 		try {
 			let job: Awaited<ReturnType<typeof localInferenceService.startDownload>>;
 			if (rawSpec) {
-				if (!isCatalogModel(rawSpec)) {
-					sendJsonErrorResponse(res, 400, "Invalid model spec");
-					return true;
-				}
-				job = await localInferenceService.startDownload(rawSpec);
+				sendJsonErrorResponse(
+					res,
+					400,
+					"Custom model downloads are disabled; choose an Eliza-1 tier.",
+				);
+				return true;
 			} else if (modelId) {
 				job = await localInferenceService.startDownload(modelId);
 			} else {
-				sendJsonErrorResponse(res, 400, "modelId or spec is required");
+				sendJsonErrorResponse(res, 400, "modelId is required");
 				return true;
 			}
 			sendJsonResponse(res, 202, { job });
@@ -583,6 +564,13 @@ export async function handleLocalInferenceCompatRoutes(
 			);
 			sendJsonResponse(res, 200, { assignments });
 		} catch (err) {
+			if (err instanceof AssignmentRejectedError) {
+				// Eliza-1-only stack: the only assignable models are the curated
+				// tiers. A rejected pick is a typed, user-visible 422 rather than a
+				// silent deferred load failure.
+				sendJsonErrorResponse(res, 422, err.message, { code: err.code });
+				return true;
+			}
 			sendJsonErrorResponse(
 				res,
 				500,
@@ -629,35 +617,14 @@ export async function handleLocalInferenceCompatRoutes(
 		return true;
 	}
 
-	// ── GET: explicit custom model search (Hugging Face / ModelScope) ───
+	// ── GET: legacy custom model search (disabled; curated Eliza-1 only) ─
 	if (method === "GET" && pathname === "/api/local-inference/hf-search") {
 		if (!(await ensureRouteAuthorized(req, res, state))) return true;
-		const q = url.searchParams.get("q")?.trim() ?? "";
-		if (q.length === 0) {
-			sendJsonResponse(res, 200, { models: [] });
-			return true;
-		}
-		const limitRaw = url.searchParams.get("limit");
-		const limit = limitRaw
-			? Math.max(1, Math.min(50, Number.parseInt(limitRaw, 10) || 12))
-			: 12;
-		const hub =
-			url.searchParams.get("hub")?.trim().toLowerCase() === "modelscope"
-				? "modelscope"
-				: "huggingface";
-		try {
-			const models =
-				typeof localInferenceService.searchModelHub === "function"
-					? await localInferenceService.searchModelHub(q, hub, limit)
-					: await localInferenceService.searchHuggingFace(q, limit);
-			sendJsonResponse(res, 200, { models });
-		} catch (err) {
-			sendJsonErrorResponse(
-				res,
-				502,
-				err instanceof Error ? err.message : "Model search failed",
-			);
-		}
+		sendJsonResponse(res, 200, {
+			models: [],
+			disabled: true,
+			reason: "custom-model-search-disabled",
+		});
 		return true;
 	}
 

package/src/routes/native-pcm-turn-route.test.ts

@@ -0,0 +1,136 @@
+import type http from "node:http";
+import { Readable } from "node:stream";
+import { logger } from "@elizaos/core";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { handleNativePcmTurnRoute } from "./native-pcm-turn-route";
+
+const engineMock = vi.hoisted(() => ({
+	ensureActiveBundleAsrReady: vi.fn(async () => undefined),
+	runVoiceTurn: vi.fn(
+		async (): Promise<"done" | "token-cap" | "cancelled"> => "done",
+	),
+}));
+
+vi.mock("../services/engine", () => ({
+	localInferenceEngine: engineMock,
+}));
+
+class FakeRes {
+	statusCode = 200;
+	headersSent = false;
+	private readonly headers = new Map<string, string>();
+	body = "";
+	setHeader(name: string, value: string): void {
+		this.headers.set(name.toLowerCase(), value);
+	}
+	end(chunk?: string): void {
+		if (chunk) this.body += chunk;
+		this.headersSent = true;
+	}
+	json(): unknown {
+		return JSON.parse(this.body);
+	}
+}
+
+function makeReq(opts: {
+	method?: string;
+	url?: string;
+	body?: unknown;
+}): http.IncomingMessage {
+	const payload = opts.body !== undefined ? JSON.stringify(opts.body) : "";
+	const stream = Readable.from(payload ? [Buffer.from(payload)] : []);
+	const req = stream as unknown as http.IncomingMessage & {
+		method: string;
+		url: string;
+		headers: Record<string, string>;
+	};
+	req.method = opts.method ?? "POST";
+	req.url = opts.url ?? "/api/voice/native-pcm-turn";
+	req.headers = {
+		host: "127.0.0.1:31337",
+		"content-type": "application/json",
+	};
+	return req;
+}
+
+function float32Base64(samples: ReadonlyArray<number>): string {
+	const bytes = Buffer.alloc(samples.length * 4);
+	samples.forEach((sample, index) => {
+		bytes.writeFloatLE(sample, index * 4);
+	});
+	return bytes.toString("base64");
+}
+
+beforeEach(() => {
+	vi.clearAllMocks();
+	engineMock.ensureActiveBundleAsrReady.mockResolvedValue(undefined);
+	engineMock.runVoiceTurn.mockResolvedValue("done");
+});
+
+describe("handleNativePcmTurnRoute", () => {
+	it("runs completed native PCM turns through the local voice engine", async () => {
+		const infoSpy = vi
+			.spyOn(logger, "info")
+			.mockImplementation(() => undefined);
+		const res = new FakeRes();
+
+		const handled = await handleNativePcmTurnRoute(
+			makeReq({
+				body: {
+					turnId: "native-turn-1",
+					pcm: float32Base64([0, 0.5, -0.25]),
+					sampleRate: 24_000,
+					signal: "end-of-speech",
+				},
+			}),
+			res as unknown as http.ServerResponse,
+		);
+
+		expect(handled).toBe(true);
+		expect(engineMock.ensureActiveBundleAsrReady).toHaveBeenCalledTimes(1);
+		expect(engineMock.runVoiceTurn).toHaveBeenCalledTimes(1);
+		const [audio, opts] = engineMock.runVoiceTurn.mock.calls[0] as [
+			{ pcm: Float32Array; sampleRate: number },
+			{ events?: { onComplete?: (reason: "done") => void } },
+		];
+		expect(audio.sampleRate).toBe(24_000);
+		expect(Array.from(audio.pcm)).toEqual([0, 0.5, -0.25]);
+		opts.events?.onComplete?.("done");
+		expect(infoSpy).toHaveBeenCalledWith(
+			expect.objectContaining({
+				turnId: "native-turn-1",
+				exitReason: "done",
+				nativeSignal: "end-of-speech",
+			}),
+			expect.stringContaining("completed native PCM voice turn"),
+		);
+		expect(res.statusCode).toBe(200);
+		expect(res.json()).toEqual({
+			ok: true,
+			turnId: "native-turn-1",
+			exitReason: "done",
+		});
+	});
+
+	it("rejects malformed Float32 PCM before touching the engine", async () => {
+		const res = new FakeRes();
+
+		const handled = await handleNativePcmTurnRoute(
+			makeReq({
+				body: {
+					pcm: Buffer.from([0, 1]).toString("base64"),
+					sampleRate: 16_000,
+				},
+			}),
+			res as unknown as http.ServerResponse,
+		);
+
+		expect(handled).toBe(true);
+		expect(res.statusCode).toBe(400);
+		expect(res.json()).toEqual({
+			error: "pcm byte length must be a multiple of 4 for Float32 PCM",
+		});
+		expect(engineMock.ensureActiveBundleAsrReady).not.toHaveBeenCalled();
+		expect(engineMock.runVoiceTurn).not.toHaveBeenCalled();
+	});
+});

package/src/routes/native-pcm-turn-route.ts

@@ -0,0 +1,121 @@
+import type http from "node:http";
+import { logger, readJsonBody, sendJson, sendJsonError } from "@elizaos/core";
+import { localInferenceEngine } from "../services/engine";
+import type { VoicePipelineEvents } from "../services/voice/pipeline";
+
+const ROUTE_PATH = "/api/voice/native-pcm-turn";
+const MAX_PCM_BYTES = 20 * 1024 * 1024;
+const DEFAULT_SAMPLE_RATE = 16_000;
+
+type NativePcmTurnBody = {
+	turnId?: unknown;
+	pcm?: unknown;
+	sampleRate?: unknown;
+	signal?: unknown;
+};
+
+function decodeFloat32Base64(value: unknown): Float32Array | string {
+	if (typeof value !== "string" || value.length === 0) {
+		return "pcm must be a non-empty base64 string";
+	}
+	const bytes = Buffer.from(value, "base64");
+	if (bytes.byteLength === 0 || bytes.byteLength > MAX_PCM_BYTES) {
+		return `pcm byte length must be between 1 and ${MAX_PCM_BYTES}`;
+	}
+	if (bytes.byteLength % 4 !== 0) {
+		return "pcm byte length must be a multiple of 4 for Float32 PCM";
+	}
+	const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+	const pcm = new Float32Array(bytes.byteLength / 4);
+	for (let i = 0; i < pcm.length; i += 1) {
+		pcm[i] = view.getFloat32(i * 4, true);
+	}
+	return pcm;
+}
+
+function normalizeSampleRate(value: unknown): number | string {
+	if (value === undefined) return DEFAULT_SAMPLE_RATE;
+	if (
+		typeof value !== "number" ||
+		!Number.isFinite(value) ||
+		!Number.isInteger(value) ||
+		value <= 0 ||
+		value > 192_000
+	) {
+		return "sampleRate must be an integer between 1 and 192000";
+	}
+	return value;
+}
+
+function normalizeTurnId(value: unknown): string | undefined {
+	if (typeof value !== "string") return undefined;
+	const trimmed = value.trim();
+	return trimmed.length > 0 ? trimmed : undefined;
+}
+
+function voiceEventsForTurn(args: {
+	turnId?: string;
+	nativeSignal?: unknown;
+}): VoicePipelineEvents {
+	return {
+		onComplete(exitReason) {
+			logger.info(
+				{
+					turnId: args.turnId,
+					exitReason,
+					nativeSignal: args.nativeSignal,
+				},
+				"[native-pcm-turn] completed native PCM voice turn",
+			);
+		},
+	};
+}
+
+export async function handleNativePcmTurnRoute(
+	req: http.IncomingMessage,
+	res: http.ServerResponse,
+): Promise<boolean> {
+	const method = (req.method ?? "GET").toUpperCase();
+	const pathname = new URL(req.url ?? "/", "http://localhost").pathname;
+	if (method !== "POST" || pathname !== ROUTE_PATH) return false;
+
+	const body = await readJsonBody<NativePcmTurnBody>(req, res, {
+		maxBytes: MAX_PCM_BYTES + 4096,
+	});
+	if (!body) return true;
+
+	const pcm = decodeFloat32Base64(body.pcm);
+	if (typeof pcm === "string") {
+		sendJsonError(res, pcm, 400);
+		return true;
+	}
+	const sampleRate = normalizeSampleRate(body.sampleRate);
+	if (typeof sampleRate === "string") {
+		sendJsonError(res, sampleRate, 400);
+		return true;
+	}
+	const turnId = normalizeTurnId(body.turnId);
+
+	try {
+		await localInferenceEngine.ensureActiveBundleAsrReady();
+		const exitReason = await localInferenceEngine.runVoiceTurn(
+			{ pcm, sampleRate },
+			{ events: voiceEventsForTurn({ turnId, nativeSignal: body.signal }) },
+		);
+		sendJson(res, { ok: true, turnId: turnId ?? null, exitReason });
+	} catch (error) {
+		logger.warn(
+			{
+				turnId,
+				error: error instanceof Error ? error.message : String(error),
+			},
+			"[native-pcm-turn] failed to run native PCM voice turn",
+		);
+		sendJsonError(
+			res,
+			error instanceof Error ? error.message : "native PCM voice turn failed",
+			503,
+		);
+	}
+	return true;
+}

package/src/routes/transcripts-routes.test.ts

@@ -40,6 +40,12 @@ function fakeRuntime(): { rows: Map<string, Memory>; runtime: unknown } {
 		},
 		getMemories: async () => [...rows.values()],
 		getMemoryById: async (id: UUID) => rows.get(id) ?? null,
+		updateMemory: async (m: Partial<Memory> & { id: UUID }) => {
+			const existing = rows.get(m.id);
+			if (!existing) return false;
+			rows.set(m.id, { ...existing, ...m });
+			return true;
+		},
 		deleteMemory: async (id: UUID) => {
 			rows.delete(id);
 		},
@@ -141,4 +147,49 @@ describe("transcripts routes", () => {
 		)(ctx({ runtime: runtime as never, body: { segments: [] } }));
 		expect(res.status).toBe(400);
 	});
+
+	it("PUT edits an existing transcript and 404s on a missing one", async () => {
+		const { runtime } = fakeRuntime();
+		const created = await handlerFor(
+			"POST",
+			"/api/transcripts",
+		)(ctx({ runtime: runtime as never, body: { title: "Raw", segments } }));
+		const id = (created.body as { transcript: { id: string } }).transcript.id;
+
+		const put = await handlerFor(
+			"PUT",
+			"/api/transcripts/:id",
+		)(
+			ctx({
+				runtime: runtime as never,
+				params: { id },
+				body: { title: "Fixed title" },
+			}),
+		);
+		expect(put.status).toBe(200);
+		expect(
+			(put.body as { transcript: { title: string } }).transcript.title,
+		).toBe("Fixed title");
+
+		const missing = await handlerFor(
+			"PUT",
+			"/api/transcripts/:id",
+		)(
+			ctx({
+				runtime: runtime as never,
+				params: { id: "00000000-0000-0000-0000-0000000000ff" },
+				body: { title: "x" },
+			}),
+		);
+		expect(missing.status).toBe(404);
+	});
+
+	it("PUT rejects a body with neither title nor segments", async () => {
+		const { runtime } = fakeRuntime();
+		const res = await handlerFor(
+			"PUT",
+			"/api/transcripts/:id",
+		)(ctx({ runtime: runtime as never, params: { id: "x" }, body: {} }));
+		expect(res.status).toBe(400);
+	});
 });

package/src/routes/transcripts-routes.ts

@@ -29,9 +29,7 @@ import {
 import { persistTranscriptAudioWav } from "./transcript-audio-store.js";
 
 function service(ctx: RouteHandlerContext): TranscriptService {
-	return new TranscriptService(
-		ctx.runtime as unknown as TranscriptServiceRuntime,
-	);
+	return new TranscriptService(ctx.runtime as TranscriptServiceRuntime);
 }
 
 /** The body a recording session POSTs to create a transcript record. */
@@ -116,6 +114,39 @@ const deleteRoute: Route = {
 	},
 };
 
+/** The body a transcript editor PUTs to persist a user edit. */
+export interface UpdateTranscriptRequest {
+	worldId?: UUID;
+	roomId?: UUID;
+	entityId?: UUID;
+	title?: string;
+	segments?: TranscriptSegment[];
+}
+
+const updateRoute: Route = {
+	type: "PUT",
+	path: "/api/transcripts/:id",
+	rawPath: true,
+	routeHandler: async (ctx): Promise<RouteHandlerResult> => {
+		const body = (ctx.body ?? {}) as UpdateTranscriptRequest;
+		if (body.title === undefined && body.segments === undefined) {
+			return { status: 400, body: { error: "title or segments is required" } };
+		}
+		if (body.segments !== undefined && !Array.isArray(body.segments)) {
+			return { status: 400, body: { error: "segments must be an array" } };
+		}
+		const agentId = ctx.runtime.agentId as UUID;
+		const updated = await service(ctx).update(ctx.params.id as UUID, {
+			worldId: (body.worldId ?? agentId) as UUID,
+			roomId: (body.roomId ?? agentId) as UUID,
+			entityId: (body.entityId ?? agentId) as UUID,
+			patch: { title: body.title, segments: body.segments },
+		});
+		if (!updated) return { status: 404, body: { error: "not found" } };
+		return { status: 200, body: { transcript: updated } };
+	},
+};
+
 const createRoute: Route = {
 	type: "POST",
 	path: "/api/transcripts",
@@ -155,5 +186,6 @@ export const transcriptsRoutes: Route[] = [
 	listRoute,
 	createRoute,
 	getRoute,
+	updateRoute,
 	deleteRoute,
 ];

package/src/runtime/bionic-wire-encoding.test.ts

@@ -0,0 +1,147 @@
+/**
+ * Bionic-host UDS wire encoding (#8848).
+ *
+ * On the bionic-delegated path the musl agent forwards audio + image bytes to
+ * the in-process bionic host over a Unix socket. Two pure helpers do the
+ * framing: `float32ToBase64LE` packs mono fp32 PCM little-endian, and
+ * `imageRequestToBase64` resolves a vision request to base64 image bytes. A
+ * byte-order or prefix-stripping bug here corrupts every ASR/vision frame
+ * silently (the host just gets garbage), so this pins the exact wire format.
+ *
+ * `ensure-local-inference-handler` has heavy import side-effects (the service
+ * layer), so — like its sibling `ensure-local-inference-handler.test.ts` — the
+ * service modules are stubbed before the import so the two encoders can be
+ * exercised in isolation.
+ */
+
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+vi.mock("../services/active-model", () => ({
+	resolveLocalInferenceLoadArgs: vi.fn(async (t) => t),
+}));
+vi.mock("../services/assignments", () => ({
+	autoAssignAtBoot: vi.fn(async () => null),
+	readEffectiveAssignments: vi.fn(async () => ({})),
+}));
+vi.mock("../services/cache-bridge", () => ({
+	extractConversationId: vi.fn(() => null),
+	extractPromptCacheKey: vi.fn(() => null),
+	resolveLocalCacheKey: vi.fn(() => null),
+}));
+vi.mock("../services/device-bridge", () => ({
+	deviceBridge: {
+		currentModelPath: vi.fn(() => null),
+		embed: vi.fn(),
+		generate: vi.fn(),
+		loadModel: vi.fn(),
+		unloadModel: vi.fn(),
+	},
+}));
+vi.mock("../services/engine", () => ({ localInferenceEngine: {} }));
+vi.mock("../services/handler-registry", () => ({
+	handlerRegistry: { installOn: vi.fn() },
+}));
+vi.mock("../services/hardware", () => ({
+	probeHardware: vi.fn(async () => ({ memory: { totalGb: 8 } })),
+}));
+vi.mock("../services/memory-arbiter", () => ({
+	tryGetMemoryArbiter: vi.fn(() => null),
+}));
+vi.mock("../services/registry", () => ({
+	listInstalledModels: vi.fn(async () => []),
+}));
+vi.mock("../services/router-handler", () => ({
+	installRouterHandler: vi.fn(),
+}));
+vi.mock("../services/voice", () => ({
+	decodeMonoPcm16Wav: vi.fn(() => ({
+		pcm: new Float32Array([0]),
+		sampleRate: 16_000,
+	})),
+}));
+
+import {
+	float32ToBase64LE,
+	imageRequestToBase64,
+} from "./ensure-local-inference-handler";
+
+const originalFetch = globalThis.fetch;
+afterEach(() => {
+	globalThis.fetch = originalFetch;
+	vi.restoreAllMocks();
+});
+
+describe("float32ToBase64LE", () => {
+	it("round-trips exactly-representable fp32 samples through base64 → readFloatLE", () => {
+		// Every value is exactly representable in IEEE-754 binary32, so the decoded
+		// values must equal the originals bit-for-bit (no float32 rounding drift).
+		const pcm = new Float32Array([
+			0, 1, -1, 0.5, -0.5, 0.25, -0.75, 3.5, -128.5, 32767.5,
+		]);
+		const buf = Buffer.from(float32ToBase64LE(pcm), "base64");
+		expect(buf.length).toBe(pcm.length * 4);
+
+		const decoded = new Float32Array(pcm.length);
+		for (let i = 0; i < pcm.length; i += 1) {
+			decoded[i] = buf.readFloatLE(i * 4);
+		}
+		expect(Array.from(decoded)).toEqual(Array.from(pcm));
+	});
+
+	it("encodes little-endian (1.0 → 00 00 80 3f)", () => {
+		const buf = Buffer.from(float32ToBase64LE(new Float32Array([1])), "base64");
+		expect([...buf]).toEqual([0x00, 0x00, 0x80, 0x3f]);
+	});
+
+	it("encodes an empty buffer as the empty string", () => {
+		expect(float32ToBase64LE(new Float32Array(0))).toBe("");
+	});
+});
+
+describe("imageRequestToBase64", () => {
+	it("strips the data-URL header and returns the base64 payload", async () => {
+		const payload = Buffer.from("hello bionic").toString("base64");
+		await expect(
+			imageRequestToBase64({
+				kind: "dataUrl",
+				dataUrl: `data:image/png;base64,${payload}`,
+			}),
+		).resolves.toBe(payload);
+	});
+
+	it("returns a comma-less dataUrl verbatim", async () => {
+		await expect(
+			imageRequestToBase64({ kind: "dataUrl", dataUrl: "rawbase64nocomma" }),
+		).resolves.toBe("rawbase64nocomma");
+	});
+
+	it("fetches a url and base64-encodes the response bytes", async () => {
+		const bytes = new Uint8Array([1, 2, 3, 4]);
+		globalThis.fetch = vi.fn(
+			async () => new Response(bytes, { status: 200 }),
+		) as unknown as typeof fetch;
+
+		await expect(
+			imageRequestToBase64({ kind: "url", url: "https://example.test/i.png" }),
+		).resolves.toBe(Buffer.from(bytes).toString("base64"));
+	});
+
+	it("throws when the url fetch is not ok", async () => {
+		globalThis.fetch = vi.fn(
+			async () => new Response("nope", { status: 404 }),
+		) as unknown as typeof fetch;
+
+		await expect(
+			imageRequestToBase64({
+				kind: "url",
+				url: "https://example.test/missing",
+			}),
+		).rejects.toThrow(/404/);
+	});
+
+	it("throws when neither a dataUrl nor a url is resolvable", async () => {
+		await expect(imageRequestToBase64({ kind: "dataUrl" })).rejects.toThrow(
+			/could not resolve image bytes/,
+		);
+	});
+});