@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/routes/live-diarization-route.test.ts

@@ -0,0 +1,347 @@
+/**
+ * live-diarization-route unit tests (HTTP-level, no GGUF models).
+ *
+ * Exercises the WebView → agent transport route's request/response contract:
+ *   - loopback (trusted-local) auth pass-through;
+ *   - non-loopback rejection (401);
+ *   - frame-shape validation (rejects malformed frames);
+ *   - the status route surfaces the model/lib resolution (and, on a host with
+ *     no on-device GGUFs, the precise "voice GGUFs missing" blocker — the same
+ *     readiness payload the device read returns).
+ *
+ * The model-heavy path (real ggml VAD/encoder/diarizer) is covered by the
+ * host smoke harness (`packages/app-core/scripts/voice-attribution-smoke.ts`),
+ * which exercises the same AudioFrameConsumer with real models.
+ */
+
+import type http from "node:http";
+import { Readable } from "node:stream";
+import { afterEach, describe, expect, it } from "vitest";
+import { buildLiveDiarizationConsumerDeps } from "../services/voice/live-diarization-session.js";
+import type { CompatRuntimeState } from "./compat-helpers.js";
+import {
+	handleLiveDiarizationRoute,
+	resetLiveDiarizationSession,
+} from "./live-diarization-route.js";
+
+class FakeRes {
+	statusCode = 200;
+	headersSent = false;
+	private readonly headers = new Map<string, string>();
+	body = "";
+	ended = false;
+	setHeader(name: string, value: string): void {
+		this.headers.set(name.toLowerCase(), value);
+	}
+	end(chunk?: string): void {
+		if (chunk) this.body += chunk;
+		this.ended = true;
+		this.headersSent = true;
+	}
+	json(): unknown {
+		return JSON.parse(this.body);
+	}
+}
+
+function makeReq(opts: {
+	method: string;
+	url: string;
+	body?: unknown;
+	remoteAddress?: string;
+	host?: string;
+}): http.IncomingMessage {
+	const payload = opts.body !== undefined ? JSON.stringify(opts.body) : "";
+	const stream = Readable.from(payload ? [Buffer.from(payload)] : []);
+	const req = stream as unknown as http.IncomingMessage & {
+		method: string;
+		url: string;
+		headers: Record<string, string>;
+		socket: { remoteAddress: string };
+	};
+	req.method = opts.method;
+	req.url = opts.url;
+	req.headers = { host: opts.host ?? "127.0.0.1:31337" };
+	req.socket = { remoteAddress: opts.remoteAddress ?? "127.0.0.1" } as never;
+	return req;
+}
+
+const runtimeState = (): CompatRuntimeState => ({
+	current: {
+		emitEvent: async () => {},
+	} as never,
+});
+
+/** A well-formed AudioFrameEvent (20 ms silence frame). */
+function silentFrame(frameIndex: number) {
+	const samples = 320;
+	const pcm16 = Buffer.alloc(samples * 2).toString("base64");
+	return {
+		pcm16,
+		sampleRate: 16_000,
+		channels: 1,
+		samples,
+		rms: 0,
+		timestamp: frameIndex * 20,
+		frameIndex,
+	};
+}
+
+afterEach(async () => {
+	await resetLiveDiarizationSession();
+});
+
+describe("handleLiveDiarizationRoute", () => {
+	it("builds AudioFrameConsumer deps with an optional echoReference provider", () => {
+		const echoReference = () => new Float32Array(320);
+		const deps = buildLiveDiarizationConsumerDeps({
+			vad: {
+				inSpeech: false,
+				onVadEvent: () => () => {},
+				pushFrame: async () => {},
+				flush: async () => {},
+				reset: () => {},
+			},
+			pipeline: {
+				attribute: async () => {
+					throw new Error("not used");
+				},
+			},
+			runtime: { emitEvent: async () => {} },
+			echoReference,
+		});
+		expect(deps.echoReference).toBe(echoReference);
+		expect(
+			buildLiveDiarizationConsumerDeps({
+				vad: deps.vad,
+				pipeline: deps.pipeline,
+				runtime: deps.runtime,
+			}),
+		).not.toHaveProperty("echoReference");
+	});
+
+	it("returns false for an unrelated path (passes through)", async () => {
+		const res = new FakeRes();
+		const handled = await handleLiveDiarizationRoute(
+			makeReq({ method: "GET", url: "/api/unrelated" }),
+			res as unknown as http.ServerResponse,
+			runtimeState(),
+		);
+		expect(handled).toBe(false);
+		expect(res.ended).toBe(false);
+	});
+
+	it("rejects a non-loopback caller with 401", async () => {
+		const res = new FakeRes();
+		const handled = await handleLiveDiarizationRoute(
+			makeReq({
+				method: "GET",
+				url: "/api/voice/audio-frames/status",
+				remoteAddress: "10.0.0.5",
+				host: "example.com",
+			}),
+			res as unknown as http.ServerResponse,
+			runtimeState(),
+		);
+		expect(handled).toBe(true);
+		expect(res.statusCode).toBe(401);
+	});
+
+	it("status route surfaces model/lib resolution (blocker on a host without device GGUFs)", async () => {
+		const res = new FakeRes();
+		const handled = await handleLiveDiarizationRoute(
+			makeReq({ method: "GET", url: "/api/voice/audio-frames/status" }),
+			res as unknown as http.ServerResponse,
+			runtimeState(),
+		);
+		expect(handled).toBe(true);
+		expect(res.statusCode).toBe(200);
+		const status = res.json() as {
+			ready: boolean;
+			libs: { fusedInference: string | null };
+			models: { dir: string };
+			framesReceived: number;
+			turnsObserved: number;
+			aec: { echoReferenceWired: boolean };
+			error?: string;
+		};
+		// On CI/host there is no fused libelizainference, so readiness fails with
+		// a precise blocker rather than a silent default — the device-evidence
+		// read. The session now runs the whole stack (VAD/encoder/diarizer)
+		// through the one fused FFI handle, not separate bun:ffi-musl libs.
+		expect(typeof status.models.dir).toBe("string");
+		expect("fusedInference" in status.libs).toBe(true);
+		expect(status.framesReceived).toBe(0);
+		expect(status.turnsObserved).toBe(0);
+		expect(status.aec.echoReferenceWired).toBe(false);
+		if (!status.ready) {
+			expect(status.error).toMatch(
+				/fused libelizainference|ABI|FFI|libelizainference/i,
+			);
+		}
+	});
+
+	it("threads a runtime echoReference provider into live AEC status", async () => {
+		const res = new FakeRes();
+		const echoReference = () => new Float32Array(320);
+		const handled = await handleLiveDiarizationRoute(
+			makeReq({ method: "GET", url: "/api/voice/audio-frames/status" }),
+			res as unknown as http.ServerResponse,
+			{
+				current: {
+					emitEvent: async () => {},
+					voiceEchoReferenceProvider: echoReference,
+				} as never,
+			},
+		);
+		expect(handled).toBe(true);
+		expect(res.statusCode).toBe(200);
+		const status = res.json() as {
+			aec: { echoReferenceWired: boolean };
+		};
+		expect(status.aec.echoReferenceWired).toBe(true);
+	});
+
+	it("rejects a malformed frame batch with 400", async () => {
+		const res = new FakeRes();
+		const handled = await handleLiveDiarizationRoute(
+			makeReq({
+				method: "POST",
+				url: "/api/voice/audio-frames",
+				body: { frames: [{ pcm16: "AA==" /* missing fields */ }] },
+			}),
+			res as unknown as http.ServerResponse,
+			runtimeState(),
+		);
+		expect(handled).toBe(true);
+		expect(res.statusCode).toBe(400);
+		expect((res.json() as { error: string }).error).toMatch(/Malformed/);
+	});
+
+	it("rejects a non-array frames field with 400", async () => {
+		const res = new FakeRes();
+		const handled = await handleLiveDiarizationRoute(
+			makeReq({
+				method: "POST",
+				url: "/api/voice/audio-frames",
+				body: { frames: "nope" },
+			}),
+			res as unknown as http.ServerResponse,
+			runtimeState(),
+		);
+		expect(handled).toBe(true);
+		expect(res.statusCode).toBe(400);
+	});
+
+	it("accepts a well-formed batch shape (validation passes before model build)", async () => {
+		// Well-formed frames clear the route's shape gate. Without on-device GGUFs
+		// the session build then throws; the route surfaces that as a 500-class
+		// failure, NOT a 400 — proving the wire contract is satisfied.
+		const res = new FakeRes();
+		const handled = await handleLiveDiarizationRoute(
+			makeReq({
+				method: "POST",
+				url: "/api/voice/audio-frames",
+				body: { frames: [silentFrame(0), silentFrame(1)] },
+			}),
+			res as unknown as http.ServerResponse,
+			runtimeState(),
+		).catch(() => "threw");
+		// Either it threw inside ingest (no models) or returned true; either way
+		// the request was NOT rejected as malformed (no 400).
+		expect(res.statusCode).not.toBe(400);
+		expect(handled === true || handled === "threw").toBe(true);
+	});
+
+	it("returns 503 when the runtime is not ready", async () => {
+		const res = new FakeRes();
+		const handled = await handleLiveDiarizationRoute(
+			makeReq({ method: "GET", url: "/api/voice/audio-frames/status" }),
+			res as unknown as http.ServerResponse,
+			{ current: null },
+		);
+		expect(handled).toBe(true);
+		expect(res.statusCode).toBe(503);
+	});
+
+	it("accepts agent-playback (far-end) frames for echo cancellation", async () => {
+		// Unlike the mic path, pushing playback needs no model build — it only
+		// decodes + appends to the alignment buffer — so this exercises the full
+		// route end-to-end on a host with no on-device GGUFs.
+		const res = new FakeRes();
+		const handled = await handleLiveDiarizationRoute(
+			makeReq({
+				method: "POST",
+				url: "/api/voice/playback-frames",
+				body: { frames: [silentFrame(0), silentFrame(1)] },
+			}),
+			res as unknown as http.ServerResponse,
+			runtimeState(),
+		);
+		expect(handled).toBe(true);
+		expect(res.statusCode).toBe(200);
+		expect(res.json()).toMatchObject({ ok: true, framesPushed: 2 });
+	});
+
+	it("accepts a reset-only playback request (barge-in / playback stop)", async () => {
+		const res = new FakeRes();
+		const handled = await handleLiveDiarizationRoute(
+			makeReq({
+				method: "POST",
+				url: "/api/voice/playback-frames",
+				body: { reset: true },
+			}),
+			res as unknown as http.ServerResponse,
+			runtimeState(),
+		);
+		expect(handled).toBe(true);
+		expect(res.statusCode).toBe(200);
+		expect(res.json()).toMatchObject({ ok: true, framesPushed: 0 });
+	});
+
+	it("resets stale playback before accepting a fresh playback batch", async () => {
+		const res = new FakeRes();
+		const handled = await handleLiveDiarizationRoute(
+			makeReq({
+				method: "POST",
+				url: "/api/voice/playback-frames",
+				body: { reset: true, frames: [silentFrame(0)] },
+			}),
+			res as unknown as http.ServerResponse,
+			runtimeState(),
+		);
+		expect(handled).toBe(true);
+		expect(res.statusCode).toBe(200);
+		expect(res.json()).toMatchObject({ ok: true, framesPushed: 1 });
+	});
+
+	it("rejects malformed playback frames with 400", async () => {
+		const res = new FakeRes();
+		const handled = await handleLiveDiarizationRoute(
+			makeReq({
+				method: "POST",
+				url: "/api/voice/playback-frames",
+				body: { frames: [{ pcm16: "AA==" /* missing fields */ }] },
+			}),
+			res as unknown as http.ServerResponse,
+			runtimeState(),
+		);
+		expect(handled).toBe(true);
+		expect(res.statusCode).toBe(400);
+		expect((res.json() as { error: string }).error).toMatch(/Malformed/);
+	});
+
+	it("rejects a non-array playback frames field with 400", async () => {
+		const res = new FakeRes();
+		const handled = await handleLiveDiarizationRoute(
+			makeReq({
+				method: "POST",
+				url: "/api/voice/playback-frames",
+				body: { frames: "nope" },
+			}),
+			res as unknown as http.ServerResponse,
+			runtimeState(),
+		);
+		expect(handled).toBe(true);
+		expect(res.statusCode).toBe(400);
+	});
+});

package/src/routes/live-diarization-route.ts

@@ -0,0 +1,198 @@
+/**
+ * `/api/voice/audio-frames` — WebView → agent transport for live on-device
+ * speaker diarization.
+ *
+ * The Android `audioFrame` PCM stream is captured in the Capacitor WebView but
+ * the bun:ffi voice libs run in the agent process. The WebView batches frames
+ * (~49 fps) and POSTs them here; this route feeds them to the single
+ * {@link LiveDiarizationSession}, which runs the real ggml VAD / encoder /
+ * diarizer / attribution pipeline and emits VOICE_TURN_OBSERVED.
+ *
+ * Routes:
+ *   POST /api/voice/audio-frames        body: { frames: AudioFrameEvent[],
+ *                                               flush?: boolean }
+ *                                       → { ok, framesReceived, turnsObserved }
+ *   POST /api/voice/playback-frames     body: { frames?: AudioFrameEvent[],
+ *                                               reset?: boolean }
+ *                                       → { ok, framesPushed }
+ *     The agent's own TTS playback (far-end) in the same base64 LE-s16 16 kHz
+ *     mono wire format, streamed in real time so the session's NLMS canceller
+ *     removes the agent's echo before VAD/attribution (#9455/#9583). Send
+ *     `reset: true` when playback stops / on barge-in.
+ *   GET  /api/voice/audio-frames/status → LiveDiarizationStatus (device evidence)
+ *
+ * Auth follows the compat pattern: trusted-loopback OR the compat API token.
+ * The WebView reaches this over 127.0.0.1 (trusted local), matching the rest of
+ * the on-device agent surface.
+ */
+
+import type http from "node:http";
+import type {
+	AudioFrameEvent,
+	EchoReferenceProvider,
+} from "../services/voice/audio-frame-consumer.js";
+import {
+	LiveDiarizationSession,
+	type RuntimeEventSink,
+} from "../services/voice/live-diarization-session.js";
+import {
+	type CompatRuntimeState,
+	ensureCompatApiAuthorized,
+	readCompatJsonBody,
+	sendJson,
+	sendJsonError,
+} from "./compat-helpers.js";
+
+let session: LiveDiarizationSession | null = null;
+
+type RuntimeEchoReferenceSource = RuntimeEventSink & {
+	/**
+	 * Optional live far-end playback provider. Hosts that can tap agent TTS PCM
+	 * expose this so the diarization consumer can run AEC before VAD.
+	 */
+	voiceEchoReferenceProvider?: EchoReferenceProvider | null;
+	getVoiceEchoReferenceProvider?: () =>
+		| EchoReferenceProvider
+		| null
+		| undefined;
+};
+
+function resolveRuntimeEchoReference(
+	runtime: RuntimeEventSink,
+): EchoReferenceProvider | null {
+	const source = runtime as RuntimeEchoReferenceSource;
+	if (typeof source.getVoiceEchoReferenceProvider === "function") {
+		return source.getVoiceEchoReferenceProvider() ?? null;
+	}
+	return source.voiceEchoReferenceProvider ?? null;
+}
+
+/** Lazily own one session per agent process, bound to the live runtime. */
+function getSession(state: CompatRuntimeState): LiveDiarizationSession | null {
+	const runtime = state.current as RuntimeEventSink | null;
+	if (!runtime || typeof runtime.emitEvent !== "function") return null;
+	if (!session) {
+		// Thread a host-supplied echo reference (if any) into the live AEC path
+		// so the NLMS canceller has a far-end signal without the playback-frames
+		// ingest route (#9583).
+		const echoReference = resolveRuntimeEchoReference(runtime);
+		session = new LiveDiarizationSession(
+			runtime,
+			echoReference ? { echoReference } : {},
+		);
+	}
+	return session;
+}
+
+/** Reset the module-level session (tests + capture teardown). */
+export async function resetLiveDiarizationSession(): Promise<void> {
+	const current = session;
+	session = null;
+	if (current) await current.close();
+}
+
+function isAudioFrameEvent(value: unknown): value is AudioFrameEvent {
+	if (!value || typeof value !== "object") return false;
+	const f = value as Partial<AudioFrameEvent>;
+	return (
+		typeof f.pcm16 === "string" &&
+		typeof f.sampleRate === "number" &&
+		typeof f.channels === "number" &&
+		typeof f.samples === "number" &&
+		typeof f.rms === "number" &&
+		typeof f.timestamp === "number" &&
+		typeof f.frameIndex === "number"
+	);
+}
+
+export async function handleLiveDiarizationRoute(
+	req: http.IncomingMessage,
+	res: http.ServerResponse,
+	state: CompatRuntimeState,
+): Promise<boolean> {
+	const url = new URL(req.url ?? "/", "http://localhost");
+	const method = req.method ?? "GET";
+
+	if (url.pathname === "/api/voice/audio-frames/status" && method === "GET") {
+		if (!ensureCompatApiAuthorized(req, res)) return true;
+		const current = getSession(state);
+		if (!current) {
+			sendJsonError(res, 503, "Runtime not ready");
+			return true;
+		}
+		sendJson(res, 200, await current.status());
+		return true;
+	}
+
+	if (url.pathname === "/api/voice/audio-frames" && method === "POST") {
+		if (!ensureCompatApiAuthorized(req, res)) return true;
+		const current = getSession(state);
+		if (!current) {
+			sendJsonError(res, 503, "Runtime not ready");
+			return true;
+		}
+		const body = await readCompatJsonBody(req, res);
+		if (!body) return true;
+		const rawFrames = body.frames;
+		if (!Array.isArray(rawFrames)) {
+			sendJsonError(res, 400, "Expected { frames: AudioFrameEvent[] }");
+			return true;
+		}
+		const frames = rawFrames.filter(isAudioFrameEvent);
+		if (frames.length !== rawFrames.length) {
+			sendJsonError(
+				res,
+				400,
+				`Malformed frame(s): ${rawFrames.length - frames.length} of ${rawFrames.length} did not match AudioFrameEvent`,
+			);
+			return true;
+		}
+		await current.ingest(frames);
+		if (body.flush === true) await current.flush();
+		const status = await current.status();
+		sendJson(res, 200, {
+			ok: true,
+			framesReceived: status.framesReceived,
+			framesDropped: status.framesDropped,
+			turnsObserved: status.turnsObserved,
+		});
+		return true;
+	}
+
+	if (url.pathname === "/api/voice/playback-frames" && method === "POST") {
+		if (!ensureCompatApiAuthorized(req, res)) return true;
+		const current = getSession(state);
+		if (!current) {
+			sendJsonError(res, 503, "Runtime not ready");
+			return true;
+		}
+		const body = await readCompatJsonBody(req, res);
+		if (!body) return true;
+		if (body.reset === true) current.resetPlayback();
+		const rawFrames = body.frames;
+		if (rawFrames !== undefined && !Array.isArray(rawFrames)) {
+			sendJsonError(
+				res,
+				400,
+				"Expected { frames?: AudioFrameEvent[], reset?: boolean }",
+			);
+			return true;
+		}
+		const frames = Array.isArray(rawFrames)
+			? rawFrames.filter(isAudioFrameEvent)
+			: [];
+		if (Array.isArray(rawFrames) && frames.length !== rawFrames.length) {
+			sendJsonError(
+				res,
+				400,
+				`Malformed playback frame(s): ${rawFrames.length - frames.length} of ${rawFrames.length} did not match AudioFrameEvent`,
+			);
+			return true;
+		}
+		current.pushPlayback(frames);
+		sendJson(res, 200, { ok: true, framesPushed: frames.length });
+		return true;
+	}
+
+	return false;
+}