@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/routes/local-inference-asr-route.test.ts

@@ -0,0 +1,246 @@
+import * as http from "node:http";
+import { Socket } from "node:net";
+import { ModelType } from "@elizaos/core";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import type { CompatRuntimeState } from "./compat-helpers";
+import { handleLocalInferenceAsrRoute } from "./local-inference-asr-route";
+import { transcribeWavWithWords } from "./local-inference-asr-transcribe";
+
+const engineMock = vi.hoisted(() => ({
+	canTranscribeLocally: vi.fn(async () => true),
+}));
+
+vi.mock("../services/engine", () => ({
+	localInferenceEngine: engineMock,
+}));
+
+vi.mock("./local-inference-asr-transcribe", () => ({
+	transcribeWavWithWords: vi.fn(),
+}));
+
+const transcribeWavWithWordsMock = vi.mocked(transcribeWavWithWords);
+
+beforeEach(() => {
+	vi.clearAllMocks();
+	engineMock.canTranscribeLocally.mockResolvedValue(true);
+});
+
+function wavBytes(): Uint8Array {
+	const pcm = new Int16Array([0, 900, -900, 0]);
+	const buffer = new ArrayBuffer(44 + pcm.length * 2);
+	const view = new DataView(buffer);
+	const writeAscii = (offset: number, value: string) => {
+		for (let index = 0; index < value.length; index += 1) {
+			view.setUint8(offset + index, value.charCodeAt(index));
+		}
+	};
+	writeAscii(0, "RIFF");
+	view.setUint32(4, 36 + pcm.length * 2, true);
+	writeAscii(8, "WAVE");
+	writeAscii(12, "fmt ");
+	view.setUint32(16, 16, true);
+	view.setUint16(20, 1, true);
+	view.setUint16(22, 1, true);
+	view.setUint32(24, 16_000, true);
+	view.setUint32(28, 16_000 * 2, true);
+	view.setUint16(32, 2, true);
+	view.setUint16(34, 16, true);
+	writeAscii(36, "data");
+	view.setUint32(40, pcm.length * 2, true);
+	for (let index = 0; index < pcm.length; index += 1) {
+		view.setInt16(44 + index * 2, pcm[index] ?? 0, true);
+	}
+	return new Uint8Array(buffer);
+}
+
+function fakeReq(
+	body?: unknown,
+	opts?: { method?: string; url?: string },
+): http.IncomingMessage {
+	const req = new http.IncomingMessage(new Socket());
+	req.method = opts?.method ?? "POST";
+	req.url = opts?.url ?? "/api/asr/local-inference";
+	req.headers = {
+		host: "localhost:2138",
+		"content-type": "audio/wav",
+	};
+	Object.defineProperty(req.socket, "remoteAddress", {
+		value: "127.0.0.1",
+		configurable: true,
+	});
+	if (body !== undefined) {
+		(req as { body?: unknown }).body = body;
+	}
+	return req;
+}
+
+function fakeRes(): {
+	res: http.ServerResponse;
+	bodyJson: () => Record<string, unknown>;
+	status: () => number;
+} {
+	const req = new http.IncomingMessage(new Socket());
+	const res = new http.ServerResponse(req);
+	let body = Buffer.alloc(0);
+	let status = 200;
+	res.setHeader = (() => res) as typeof res.setHeader;
+	res.writeHead = ((code: number) => {
+		status = code;
+		res.statusCode = code;
+		return res;
+	}) as typeof res.writeHead;
+	res.end = ((chunk?: string | Uint8Array | Buffer) => {
+		if (typeof chunk === "string") {
+			body = Buffer.concat([body, Buffer.from(chunk)]);
+		} else if (chunk) {
+			body = Buffer.concat([body, Buffer.from(chunk)]);
+		}
+		return res;
+	}) as typeof res.end;
+	return {
+		res,
+		bodyJson: () => JSON.parse(body.toString("utf8")),
+		status: () => status,
+	};
+}
+
+describe("local inference ASR route", () => {
+	it("reports readiness from the registered handler and eligible ASR bundle", async () => {
+		engineMock.canTranscribeLocally.mockResolvedValue(true);
+		const getModel = vi.fn(() => () => "transcript");
+		const useModel = vi.fn();
+		const state: CompatRuntimeState = {
+			current: {
+				getModel,
+				useModel,
+			} as unknown as CompatRuntimeState["current"],
+		};
+		const out = fakeRes();
+
+		const handled = await handleLocalInferenceAsrRoute(
+			fakeReq(undefined, {
+				method: "GET",
+				url: "/api/asr/local-inference/status",
+			}),
+			out.res,
+			state,
+		);
+
+		expect(handled).toBe(true);
+		expect(out.status()).toBe(200);
+		expect(out.bodyJson()).toEqual({
+			ready: true,
+			provider: "local-inference",
+		});
+		expect(getModel).toHaveBeenCalledWith(ModelType.TRANSCRIPTION);
+		expect(engineMock.canTranscribeLocally).toHaveBeenCalledTimes(1);
+		expect(useModel).not.toHaveBeenCalled();
+	});
+
+	it("reports not-ready when the handler is registered but no ASR bundle is eligible", async () => {
+		engineMock.canTranscribeLocally.mockResolvedValue(false);
+		const getModel = vi.fn(() => () => "transcript");
+		const state: CompatRuntimeState = {
+			current: {
+				getModel,
+			} as unknown as CompatRuntimeState["current"],
+		};
+		const out = fakeRes();
+
+		const handled = await handleLocalInferenceAsrRoute(
+			fakeReq(undefined, {
+				method: "GET",
+				url: "/api/asr/local-inference/status",
+			}),
+			out.res,
+			state,
+		);
+
+		expect(handled).toBe(true);
+		expect(out.status()).toBe(200);
+		expect(out.bodyJson()).toEqual({ ready: false, provider: null });
+		expect(engineMock.canTranscribeLocally).toHaveBeenCalledTimes(1);
+	});
+
+	it("reports not-ready when no TRANSCRIPTION handler is registered", async () => {
+		const getModel = vi.fn(() => undefined);
+		const state: CompatRuntimeState = {
+			current: {
+				getModel,
+			} as unknown as CompatRuntimeState["current"],
+		};
+		const out = fakeRes();
+
+		const handled = await handleLocalInferenceAsrRoute(
+			fakeReq(undefined, {
+				method: "GET",
+				url: "/api/asr/local-inference/status",
+			}),
+			out.res,
+			state,
+		);
+
+		expect(handled).toBe(true);
+		expect(out.status()).toBe(200);
+		expect(out.bodyJson()).toEqual({ ready: false, provider: null });
+		expect(engineMock.canTranscribeLocally).not.toHaveBeenCalled();
+	});
+
+	it("transcribes raw WAV audio and returns text + per-word timings", async () => {
+		transcribeWavWithWordsMock.mockResolvedValue({
+			text: "hello local voice",
+			words: [
+				{ text: "hello", startMs: 0, endMs: 400 },
+				{ text: "local", startMs: 400, endMs: 700 },
+				{ text: "voice", startMs: 700, endMs: 1000 },
+			],
+		});
+		const state: CompatRuntimeState = {
+			current: {} as unknown as CompatRuntimeState["current"],
+		};
+		const out = fakeRes();
+
+		const handled = await handleLocalInferenceAsrRoute(
+			fakeReq(wavBytes()),
+			out.res,
+			state,
+		);
+
+		expect(handled).toBe(true);
+		// The raw WAV bytes are forwarded to the single FFI-pipe transcriber.
+		expect(
+			Array.from(transcribeWavWithWordsMock.mock.calls[0]?.[1] as Uint8Array),
+		).toEqual(Array.from(wavBytes()));
+		expect(out.status()).toBe(200);
+		expect(out.bodyJson()).toEqual({
+			text: "hello local voice",
+			words: [
+				{ text: "hello", startMs: 0, endMs: 400 },
+				{ text: "local", startMs: 400, endMs: 700 },
+				{ text: "voice", startMs: 700, endMs: 1000 },
+			],
+		});
+	});
+
+	it("accepts JSON base64 audio for route clients that cannot send raw WAV", async () => {
+		transcribeWavWithWordsMock.mockResolvedValue({
+			text: "hello from json",
+			words: [],
+		});
+		const state: CompatRuntimeState = {
+			current: {} as unknown as CompatRuntimeState["current"],
+		};
+		const req = fakeReq({
+			audioBase64: Buffer.from(wavBytes()).toString("base64"),
+		});
+		req.headers["content-type"] = "application/json";
+		const out = fakeRes();
+
+		await handleLocalInferenceAsrRoute(req, out.res, state);
+
+		expect(
+			Array.from(transcribeWavWithWordsMock.mock.calls[0]?.[1] as Uint8Array),
+		).toEqual(Array.from(wavBytes()));
+		expect(out.bodyJson()).toEqual({ text: "hello from json", words: [] });
+	});
+});

package/src/routes/local-inference-asr-route.ts

@@ -0,0 +1,166 @@
+import type http from "node:http";
+import { ModelType } from "@elizaos/core";
+import { localInferenceEngine } from "../services/engine";
+import {
+	type CompatRuntimeState,
+	ensureRouteAuthorized,
+	readCompatJsonBody,
+	sendJson,
+} from "./compat-helpers";
+import { transcribeWavWithWords } from "./local-inference-asr-transcribe";
+
+const MAX_LOCAL_ASR_AUDIO_BYTES = 16 * 1024 * 1024;
+
+function toUint8Array(value: Uint8Array | ArrayBuffer): Uint8Array {
+	return value instanceof Uint8Array ? value : new Uint8Array(value);
+}
+
+function coercePreParsedAudio(value: unknown): Uint8Array | null {
+	if (value instanceof Uint8Array) return toUint8Array(value);
+	if (value instanceof ArrayBuffer) return toUint8Array(value);
+	if (ArrayBuffer.isView(value)) {
+		return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
+	}
+	return null;
+}
+
+async function readRawAudioBody(
+	req: http.IncomingMessage,
+	res: http.ServerResponse,
+): Promise<Uint8Array | null> {
+	const preParsed = coercePreParsedAudio((req as { body?: unknown }).body);
+	if (preParsed) return preParsed;
+
+	const chunks: Buffer[] = [];
+	let totalBytes = 0;
+	try {
+		for await (const chunk of req) {
+			const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+			totalBytes += buf.byteLength;
+			if (totalBytes > MAX_LOCAL_ASR_AUDIO_BYTES) {
+				req.destroy();
+				sendJson(res, 413, { error: "Audio body too large" });
+				return null;
+			}
+			chunks.push(buf);
+		}
+	} catch {
+		sendJson(res, 400, { error: "Invalid audio body" });
+		return null;
+	}
+
+	return new Uint8Array(Buffer.concat(chunks));
+}
+
+function firstHeaderValue(value: string | string[] | undefined): string {
+	return Array.isArray(value) ? (value[0] ?? "") : (value ?? "");
+}
+
+async function readLocalInferenceAsrAudio(
+	req: http.IncomingMessage,
+	res: http.ServerResponse,
+): Promise<Uint8Array | null> {
+	const contentType = firstHeaderValue(req.headers["content-type"])
+		.toLowerCase()
+		.split(";", 1)[0]
+		.trim();
+
+	if (contentType === "application/json") {
+		const body = await readCompatJsonBody(req, res);
+		if (!body) return null;
+		if (typeof body.audioBase64 === "string") {
+			return new Uint8Array(Buffer.from(body.audioBase64, "base64"));
+		}
+		sendJson(res, 400, { error: "Missing audioBase64" });
+		return null;
+	}
+
+	return readRawAudioBody(req, res);
+}
+
+function isClosed(res: http.ServerResponse): boolean {
+	return res.destroyed || res.writableEnded;
+}
+
+export async function handleLocalInferenceAsrRoute(
+	req: http.IncomingMessage,
+	res: http.ServerResponse,
+	state: CompatRuntimeState,
+): Promise<boolean> {
+	const method = req.method?.toUpperCase() ?? "GET";
+	const url = new URL(req.url ?? "/", "http://localhost");
+	if (method === "GET" && url.pathname === "/api/asr/local-inference/status") {
+		if (!(await ensureRouteAuthorized(req, res, state))) return true;
+		// Transcription runs through the registered TRANSCRIPTION model handler,
+		// backed by the fused Gemma ASR runtime. A handler alone is not enough:
+		// the active or assigned Eliza-1 bundle must stage an eligible ASR model.
+		const getModel = state.current?.getModel;
+		const runtimeAsr =
+			typeof getModel === "function" &&
+			Boolean(getModel.call(state.current, ModelType.TRANSCRIPTION));
+		const ready =
+			runtimeAsr && (await localInferenceEngine.canTranscribeLocally());
+		sendJson(res, 200, {
+			ready,
+			provider: ready ? "local-inference" : null,
+		});
+		return true;
+	}
+
+	if (method !== "POST" || url.pathname !== "/api/asr/local-inference") {
+		return false;
+	}
+
+	if (!(await ensureRouteAuthorized(req, res, state))) return true;
+
+	const audio = await readLocalInferenceAsrAudio(req, res);
+	if (!audio) return true;
+	if (audio.byteLength === 0) {
+		sendJson(res, 400, { error: "Missing audio" });
+		return true;
+	}
+
+	const abortController = new AbortController();
+	let completed = false;
+	let clientClosed = false;
+	const abortOnClose = () => {
+		clientClosed = true;
+		if (!completed && !abortController.signal.aborted) {
+			abortController.abort();
+		}
+	};
+	req.on("close", abortOnClose);
+	res.on("close", abortOnClose);
+
+	try {
+		const runtime = state.current;
+		if (!runtime) {
+			completed = true;
+			sendJson(res, 503, {
+				error: "Local inference TRANSCRIPTION is not available",
+			});
+			return true;
+		}
+		const { text, words } = await transcribeWavWithWords(
+			runtime,
+			audio,
+			abortController.signal,
+		);
+		completed = true;
+		sendJson(res, 200, { text, words });
+	} catch (err) {
+		if (!clientClosed && !abortController.signal.aborted && !isClosed(res)) {
+			sendJson(res, 502, {
+				error: `Local inference ASR error: ${
+					err instanceof Error ? err.message : String(err)
+				}`,
+			});
+		}
+	} finally {
+		completed = true;
+		req.off("close", abortOnClose);
+		res.off("close", abortOnClose);
+	}
+
+	return true;
+}

package/src/routes/local-inference-asr-transcribe.test.ts

@@ -0,0 +1,118 @@
+import { type AgentRuntime, ModelType } from "@elizaos/core";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { localInferenceEngine } from "../services/engine";
+import { transcribeWavWithWords } from "./local-inference-asr-transcribe";
+
+vi.mock("../services/engine", () => ({
+	localInferenceEngine: {
+		available: vi.fn(),
+		ensureActiveBundleAsrReady: vi.fn(),
+		transcribePcmTimed: vi.fn(),
+	},
+}));
+
+const engine = vi.mocked(localInferenceEngine, true) as unknown as {
+	available: ReturnType<typeof vi.fn>;
+	ensureActiveBundleAsrReady: ReturnType<typeof vi.fn>;
+	transcribePcmTimed: ReturnType<typeof vi.fn>;
+};
+
+/** Minimal mono PCM16 16 kHz WAV with four samples. */
+function wavBytes(): Uint8Array {
+	const pcm = new Int16Array([0, 900, -900, 0]);
+	const buffer = new ArrayBuffer(44 + pcm.length * 2);
+	const view = new DataView(buffer);
+	const writeAscii = (offset: number, value: string) => {
+		for (let i = 0; i < value.length; i += 1) {
+			view.setUint8(offset + i, value.charCodeAt(i));
+		}
+	};
+	writeAscii(0, "RIFF");
+	view.setUint32(4, 36 + pcm.length * 2, true);
+	writeAscii(8, "WAVE");
+	writeAscii(12, "fmt ");
+	view.setUint32(16, 16, true);
+	view.setUint16(20, 1, true);
+	view.setUint16(22, 1, true);
+	view.setUint32(24, 16_000, true);
+	view.setUint32(28, 16_000 * 2, true);
+	view.setUint16(32, 2, true);
+	view.setUint16(34, 16, true);
+	writeAscii(36, "data");
+	view.setUint32(40, pcm.length * 2, true);
+	for (let i = 0; i < pcm.length; i += 1) {
+		view.setInt16(44 + i * 2, pcm[i] ?? 0, true);
+	}
+	return new Uint8Array(buffer);
+}
+
+describe("transcribeWavWithWords", () => {
+	afterEach(() => {
+		vi.clearAllMocks();
+	});
+
+	it("runs the fused FFI timed pipe when the in-process engine is active", async () => {
+		engine.available.mockResolvedValue(true);
+		engine.ensureActiveBundleAsrReady.mockResolvedValue(undefined);
+		engine.transcribePcmTimed.mockResolvedValue({
+			text: "  hello world  ",
+			words: [
+				{ text: "hello", startMs: 0, endMs: 500 },
+				{ text: "world", startMs: 500, endMs: 1000 },
+			],
+		});
+		const runtime = {
+			useModel: vi.fn(),
+		} as unknown as AgentRuntime;
+
+		const result = await transcribeWavWithWords(runtime, wavBytes());
+
+		// useModel is never touched — the single FFI pipe owns this path.
+		expect(runtime.useModel).not.toHaveBeenCalled();
+		expect(engine.ensureActiveBundleAsrReady).toHaveBeenCalledTimes(1);
+		// The WAV decoded to a Float32Array @ 16 kHz before the timed call.
+		const [audioArg] = engine.transcribePcmTimed.mock.calls[0] ?? [];
+		expect((audioArg as { pcm: Float32Array }).pcm).toBeInstanceOf(
+			Float32Array,
+		);
+		expect((audioArg as { sampleRate: number }).sampleRate).toBe(16_000);
+		expect(result).toEqual({
+			text: "hello world",
+			words: [
+				{ text: "hello", startMs: 0, endMs: 500 },
+				{ text: "world", startMs: 500, endMs: 1000 },
+			],
+		});
+	});
+
+	it("falls back to the useModel provider chain when the engine is inactive", async () => {
+		engine.available.mockResolvedValue(false);
+		const useModel = vi
+			.fn()
+			.mockRejectedValueOnce(
+				new Error("No handler found for delegate type: TRANSCRIPTION"),
+			)
+			.mockResolvedValueOnce({ text: "hello local voice" });
+		const runtime = { useModel } as unknown as AgentRuntime;
+
+		const result = await transcribeWavWithWords(runtime, wavBytes());
+
+		expect(useModel).toHaveBeenCalledTimes(2);
+		expect(useModel.mock.calls[0]?.[0]).toBe(ModelType.TRANSCRIPTION);
+		expect(useModel.mock.calls[0]?.[2]).toBe("eliza-local-inference");
+		expect(useModel.mock.calls[1]?.[2]).toBe("capacitor-llama");
+		// Fallback path carries no word timings (TRANSCRIPTION ⇒ string).
+		expect(result).toEqual({ text: "hello local voice", words: [] });
+		expect(engine.transcribePcmTimed).not.toHaveBeenCalled();
+	});
+
+	it("rejects an invalid transcript shape from the provider chain", async () => {
+		engine.available.mockResolvedValue(false);
+		const useModel = vi.fn().mockResolvedValue(42);
+		const runtime = { useModel } as unknown as AgentRuntime;
+
+		await expect(transcribeWavWithWords(runtime, wavBytes())).rejects.toThrow(
+			/invalid transcript/,
+		);
+	});
+});

package/src/routes/local-inference-asr-transcribe.ts

@@ -0,0 +1,97 @@
+import { type AgentRuntime, ModelType } from "@elizaos/core";
+import { localInferenceEngine } from "../services/engine";
+import { type AsrWordTiming, decodeMonoPcm16Wav } from "../services/voice";
+
+/** A transcript plus optional per-word timings (ms from audio start). */
+export interface LocalAsrTranscript {
+	text: string;
+	words: AsrWordTiming[];
+}
+
+/**
+ * Providers the multi-platform `useModel` fallback tries in order. Exactly one
+ * is registered per platform; the loop skips "no handler" providers and stops
+ * at the first that answers. This path carries no word timings — the locked
+ * `ModelType.TRANSCRIPTION ⇒ string` contract returns text only.
+ */
+const LOCAL_TRANSCRIPTION_PROVIDER_IDS = [
+	"eliza-local-inference",
+	"capacitor-llama",
+	"eliza-device-bridge",
+	"eliza-aosp-llama",
+] as const;
+
+function isMissingTranscriptionProviderError(error: unknown): boolean {
+	return (
+		error instanceof Error &&
+		/No handler found for delegate type: TRANSCRIPTION/.test(error.message)
+	);
+}
+
+function normalizeTranscriptText(value: unknown): string {
+	if (typeof value === "string") return value.trim();
+	if (value && typeof value === "object") {
+		const text = (value as { text?: unknown }).text;
+		if (typeof text === "string") return text.trim();
+	}
+	throw new Error("TRANSCRIPTION returned an invalid transcript");
+}
+
+/**
+ * Transcribe WAV audio to text + per-word timings.
+ *
+ * The primary path is the single fused libelizainference FFI pipe
+ * (`transcribePcmTimed`, ASR ABI v12) — the cross-platform on-device runtime
+ * that emits per-word `[startMs,endMs)` timings the transcript player
+ * highlights against. When the in-process engine is not the active local
+ * backend (a device-bridge / capacitor / AOSP topology owns transcription),
+ * fall back to the `useModel` provider chain for the transcript text; word
+ * timings are unavailable on that path (`words: []`).
+ */
+export async function transcribeWavWithWords(
+	runtime: AgentRuntime,
+	audioWav: Uint8Array,
+	signal?: AbortSignal,
+): Promise<LocalAsrTranscript> {
+	if (await localInferenceEngine.available()) {
+		const audio = decodeMonoPcm16Wav(audioWav);
+		await localInferenceEngine.ensureActiveBundleAsrReady();
+		const { text, words } = await localInferenceEngine.transcribePcmTimed(
+			audio,
+			signal,
+		);
+		return { text: text.trim(), words: [...words] };
+	}
+	return {
+		text: await transcribeViaModelChain(runtime, audioWav, signal),
+		words: [],
+	};
+}
+
+async function transcribeViaModelChain(
+	runtime: AgentRuntime,
+	audio: Uint8Array,
+	signal?: AbortSignal,
+): Promise<string> {
+	let lastError: unknown;
+	for (const provider of LOCAL_TRANSCRIPTION_PROVIDER_IDS) {
+		try {
+			const transcript = normalizeTranscriptText(
+				await runtime.useModel(
+					ModelType.TRANSCRIPTION,
+					{ audio, ...(signal ? { signal } : {}) } as never,
+					provider,
+				),
+			);
+			if (!transcript) {
+				throw new Error("TRANSCRIPTION returned an empty transcript");
+			}
+			return transcript;
+		} catch (err) {
+			lastError = err;
+			if (!isMissingTranscriptionProviderError(err)) throw err;
+		}
+	}
+	if (lastError instanceof Error) throw lastError;
+	throw new Error("No local-inference TRANSCRIPTION provider is registered");
+}