@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.2

package/src/routes/live-diarization-route.ts

@@ -0,0 +1,122 @@
+/**
+ * `/api/voice/audio-frames` — WebView → agent transport for live on-device
+ * speaker diarization.
+ *
+ * The Android `audioFrame` PCM stream is captured in the Capacitor WebView but
+ * the bun:ffi voice libs run in the agent process. The WebView batches frames
+ * (~49 fps) and POSTs them here; this route feeds them to the single
+ * {@link LiveDiarizationSession}, which runs the real ggml VAD / encoder /
+ * diarizer / attribution pipeline and emits VOICE_TURN_OBSERVED.
+ *
+ * Routes:
+ *   POST /api/voice/audio-frames        body: { frames: AudioFrameEvent[],
+ *                                               flush?: boolean }
+ *                                       → { ok, framesReceived, turnsObserved }
+ *   GET  /api/voice/audio-frames/status → LiveDiarizationStatus (device evidence)
+ *
+ * Auth follows the compat pattern: trusted-loopback OR the compat API token.
+ * The WebView reaches this over 127.0.0.1 (trusted local), matching the rest of
+ * the on-device agent surface.
+ */
+
+import type http from "node:http";
+import type { AudioFrameEvent } from "../services/voice/audio-frame-consumer.js";
+import {
+	LiveDiarizationSession,
+	type RuntimeEventSink,
+} from "../services/voice/live-diarization-session.js";
+import {
+	type CompatRuntimeState,
+	ensureCompatApiAuthorized,
+	readCompatJsonBody,
+	sendJson,
+	sendJsonError,
+} from "./compat-helpers.js";
+
+let session: LiveDiarizationSession | null = null;
+
+/** Lazily own one session per agent process, bound to the live runtime. */
+function getSession(state: CompatRuntimeState): LiveDiarizationSession | null {
+	const runtime = state.current as unknown as RuntimeEventSink | null;
+	if (!runtime || typeof runtime.emitEvent !== "function") return null;
+	if (!session) session = new LiveDiarizationSession(runtime);
+	return session;
+}
+
+/** Reset the module-level session (tests + capture teardown). */
+export async function resetLiveDiarizationSession(): Promise<void> {
+	const current = session;
+	session = null;
+	if (current) await current.close();
+}
+
+function isAudioFrameEvent(value: unknown): value is AudioFrameEvent {
+	if (!value || typeof value !== "object") return false;
+	const f = value as Partial<AudioFrameEvent>;
+	return (
+		typeof f.pcm16 === "string" &&
+		typeof f.sampleRate === "number" &&
+		typeof f.channels === "number" &&
+		typeof f.samples === "number" &&
+		typeof f.rms === "number" &&
+		typeof f.timestamp === "number" &&
+		typeof f.frameIndex === "number"
+	);
+}
+
+export async function handleLiveDiarizationRoute(
+	req: http.IncomingMessage,
+	res: http.ServerResponse,
+	state: CompatRuntimeState,
+): Promise<boolean> {
+	const url = new URL(req.url ?? "/", "http://localhost");
+	const method = req.method ?? "GET";
+
+	if (url.pathname === "/api/voice/audio-frames/status" && method === "GET") {
+		if (!ensureCompatApiAuthorized(req, res)) return true;
+		const current = getSession(state);
+		if (!current) {
+			sendJsonError(res, 503, "Runtime not ready");
+			return true;
+		}
+		sendJson(res, 200, await current.status());
+		return true;
+	}
+
+	if (url.pathname === "/api/voice/audio-frames" && method === "POST") {
+		if (!ensureCompatApiAuthorized(req, res)) return true;
+		const current = getSession(state);
+		if (!current) {
+			sendJsonError(res, 503, "Runtime not ready");
+			return true;
+		}
+		const body = await readCompatJsonBody(req, res);
+		if (!body) return true;
+		const rawFrames = body.frames;
+		if (!Array.isArray(rawFrames)) {
+			sendJsonError(res, 400, "Expected { frames: AudioFrameEvent[] }");
+			return true;
+		}
+		const frames = rawFrames.filter(isAudioFrameEvent);
+		if (frames.length !== rawFrames.length) {
+			sendJsonError(
+				res,
+				400,
+				`Malformed frame(s): ${rawFrames.length - frames.length} of ${rawFrames.length} did not match AudioFrameEvent`,
+			);
+			return true;
+		}
+		await current.ingest(frames);
+		if (body.flush === true) await current.flush();
+		const status = await current.status();
+		sendJson(res, 200, {
+			ok: true,
+			framesReceived: status.framesReceived,
+			framesDropped: status.framesDropped,
+			turnsObserved: status.turnsObserved,
+		});
+		return true;
+	}
+
+	return false;
+}

package/src/routes/local-inference-asr-route.d.ts

@@ -0,0 +1,4 @@
+import type http from "node:http";
+import { type CompatRuntimeState } from "./compat-helpers";
+export declare function handleLocalInferenceAsrRoute(req: http.IncomingMessage, res: http.ServerResponse, state: CompatRuntimeState): Promise<boolean>;
+//# sourceMappingURL=local-inference-asr-route.d.ts.map

package/src/routes/local-inference-asr-route.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"local-inference-asr-route.d.ts","sourceRoot":"","sources":["local-inference-asr-route.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EACN,KAAK,kBAAkB,EAIvB,MAAM,kBAAkB,CAAC;AA4E1B,wBAAsB,4BAA4B,CACjD,GAAG,EAAE,IAAI,CAAC,eAAe,EACzB,GAAG,EAAE,IAAI,CAAC,cAAc,EACxB,KAAK,EAAE,kBAAkB,GACvB,OAAO,CAAC,OAAO,CAAC,CA2ElB"}

package/src/routes/local-inference-asr-route.test.ts

@@ -0,0 +1,205 @@
+import * as http from "node:http";
+import { Socket } from "node:net";
+import { ModelType } from "@elizaos/core";
+import { describe, expect, it, vi } from "vitest";
+import type { CompatRuntimeState } from "./compat-helpers";
+import { handleLocalInferenceAsrRoute } from "./local-inference-asr-route";
+import { transcribeWavWithWords } from "./local-inference-asr-transcribe";
+
+vi.mock("./local-inference-asr-transcribe", () => ({
+	transcribeWavWithWords: vi.fn(),
+}));
+
+const transcribeWavWithWordsMock = vi.mocked(transcribeWavWithWords);
+
+function wavBytes(): Uint8Array {
+	const pcm = new Int16Array([0, 900, -900, 0]);
+	const buffer = new ArrayBuffer(44 + pcm.length * 2);
+	const view = new DataView(buffer);
+	const writeAscii = (offset: number, value: string) => {
+		for (let index = 0; index < value.length; index += 1) {
+			view.setUint8(offset + index, value.charCodeAt(index));
+		}
+	};
+	writeAscii(0, "RIFF");
+	view.setUint32(4, 36 + pcm.length * 2, true);
+	writeAscii(8, "WAVE");
+	writeAscii(12, "fmt ");
+	view.setUint32(16, 16, true);
+	view.setUint16(20, 1, true);
+	view.setUint16(22, 1, true);
+	view.setUint32(24, 16_000, true);
+	view.setUint32(28, 16_000 * 2, true);
+	view.setUint16(32, 2, true);
+	view.setUint16(34, 16, true);
+	writeAscii(36, "data");
+	view.setUint32(40, pcm.length * 2, true);
+	for (let index = 0; index < pcm.length; index += 1) {
+		view.setInt16(44 + index * 2, pcm[index] ?? 0, true);
+	}
+	return new Uint8Array(buffer);
+}
+
+function fakeReq(
+	body?: unknown,
+	opts?: { method?: string; url?: string },
+): http.IncomingMessage {
+	const req = new http.IncomingMessage(new Socket());
+	req.method = opts?.method ?? "POST";
+	req.url = opts?.url ?? "/api/asr/local-inference";
+	req.headers = {
+		host: "localhost:2138",
+		"content-type": "audio/wav",
+	};
+	Object.defineProperty(req.socket, "remoteAddress", {
+		value: "127.0.0.1",
+		configurable: true,
+	});
+	if (body !== undefined) {
+		(req as { body?: unknown }).body = body;
+	}
+	return req;
+}
+
+function fakeRes(): {
+	res: http.ServerResponse;
+	bodyJson: () => Record<string, unknown>;
+	status: () => number;
+} {
+	const req = new http.IncomingMessage(new Socket());
+	const res = new http.ServerResponse(req);
+	let body = Buffer.alloc(0);
+	let status = 200;
+	res.setHeader = (() => res) as typeof res.setHeader;
+	res.writeHead = ((code: number) => {
+		status = code;
+		res.statusCode = code;
+		return res;
+	}) as typeof res.writeHead;
+	res.end = ((chunk?: string | Uint8Array | Buffer) => {
+		if (typeof chunk === "string") {
+			body = Buffer.concat([body, Buffer.from(chunk)]);
+		} else if (chunk) {
+			body = Buffer.concat([body, Buffer.from(chunk)]);
+		}
+		return res;
+	}) as typeof res.end;
+	return {
+		res,
+		bodyJson: () => JSON.parse(body.toString("utf8")),
+		status: () => status,
+	};
+}
+
+describe("local inference ASR route", () => {
+	it("reports readiness off the registered fused TRANSCRIPTION handler", async () => {
+		const getModel = vi.fn(() => () => "transcript");
+		const useModel = vi.fn();
+		const state: CompatRuntimeState = {
+			current: {
+				getModel,
+				useModel,
+			} as unknown as CompatRuntimeState["current"],
+		};
+		const out = fakeRes();
+
+		const handled = await handleLocalInferenceAsrRoute(
+			fakeReq(undefined, {
+				method: "GET",
+				url: "/api/asr/local-inference/status",
+			}),
+			out.res,
+			state,
+		);
+
+		expect(handled).toBe(true);
+		expect(out.status()).toBe(200);
+		expect(out.bodyJson()).toEqual({
+			ready: true,
+			provider: "local-inference",
+		});
+		expect(getModel).toHaveBeenCalledWith(ModelType.TRANSCRIPTION);
+		expect(useModel).not.toHaveBeenCalled();
+	});
+
+	it("reports not-ready when no TRANSCRIPTION handler is registered", async () => {
+		const getModel = vi.fn(() => undefined);
+		const state: CompatRuntimeState = {
+			current: {
+				getModel,
+			} as unknown as CompatRuntimeState["current"],
+		};
+		const out = fakeRes();
+
+		const handled = await handleLocalInferenceAsrRoute(
+			fakeReq(undefined, {
+				method: "GET",
+				url: "/api/asr/local-inference/status",
+			}),
+			out.res,
+			state,
+		);
+
+		expect(handled).toBe(true);
+		expect(out.status()).toBe(200);
+		expect(out.bodyJson()).toEqual({ ready: false, provider: null });
+	});
+
+	it("transcribes raw WAV audio and returns text + per-word timings", async () => {
+		transcribeWavWithWordsMock.mockResolvedValue({
+			text: "hello local voice",
+			words: [
+				{ text: "hello", startMs: 0, endMs: 400 },
+				{ text: "local", startMs: 400, endMs: 700 },
+				{ text: "voice", startMs: 700, endMs: 1000 },
+			],
+		});
+		const state: CompatRuntimeState = {
+			current: {} as unknown as CompatRuntimeState["current"],
+		};
+		const out = fakeRes();
+
+		const handled = await handleLocalInferenceAsrRoute(
+			fakeReq(wavBytes()),
+			out.res,
+			state,
+		);
+
+		expect(handled).toBe(true);
+		// The raw WAV bytes are forwarded to the single FFI-pipe transcriber.
+		expect(
+			Array.from(transcribeWavWithWordsMock.mock.calls[0]?.[1] as Uint8Array),
+		).toEqual(Array.from(wavBytes()));
+		expect(out.status()).toBe(200);
+		expect(out.bodyJson()).toEqual({
+			text: "hello local voice",
+			words: [
+				{ text: "hello", startMs: 0, endMs: 400 },
+				{ text: "local", startMs: 400, endMs: 700 },
+				{ text: "voice", startMs: 700, endMs: 1000 },
+			],
+		});
+	});
+
+	it("accepts JSON base64 audio for route clients that cannot send raw WAV", async () => {
+		transcribeWavWithWordsMock.mockResolvedValue({
+			text: "hello from json",
+			words: [],
+		});
+		const state: CompatRuntimeState = {
+			current: {} as unknown as CompatRuntimeState["current"],
+		};
+		const req = fakeReq({
+			audioBase64: Buffer.from(wavBytes()).toString("base64"),
+		});
+		req.headers["content-type"] = "application/json";
+		const out = fakeRes();
+
+		await handleLocalInferenceAsrRoute(req, out.res, state);
+
+		expect(
+			Array.from(transcribeWavWithWordsMock.mock.calls[0]?.[1] as Uint8Array),
+		).toEqual(Array.from(wavBytes()));
+		expect(out.bodyJson()).toEqual({ text: "hello from json", words: [] });
+	});
+});

package/src/routes/local-inference-asr-route.ts

@@ -0,0 +1,163 @@
+import type http from "node:http";
+import { ModelType } from "@elizaos/core";
+import {
+	type CompatRuntimeState,
+	ensureRouteAuthorized,
+	readCompatJsonBody,
+	sendJson,
+} from "./compat-helpers";
+import { transcribeWavWithWords } from "./local-inference-asr-transcribe";
+
+const MAX_LOCAL_ASR_AUDIO_BYTES = 16 * 1024 * 1024;
+
+function toUint8Array(value: Uint8Array | ArrayBuffer): Uint8Array {
+	return value instanceof Uint8Array ? value : new Uint8Array(value);
+}
+
+function coercePreParsedAudio(value: unknown): Uint8Array | null {
+	if (value instanceof Uint8Array) return toUint8Array(value);
+	if (value instanceof ArrayBuffer) return toUint8Array(value);
+	if (ArrayBuffer.isView(value)) {
+		return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
+	}
+	return null;
+}
+
+async function readRawAudioBody(
+	req: http.IncomingMessage,
+	res: http.ServerResponse,
+): Promise<Uint8Array | null> {
+	const preParsed = coercePreParsedAudio((req as { body?: unknown }).body);
+	if (preParsed) return preParsed;
+
+	const chunks: Buffer[] = [];
+	let totalBytes = 0;
+	try {
+		for await (const chunk of req) {
+			const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+			totalBytes += buf.byteLength;
+			if (totalBytes > MAX_LOCAL_ASR_AUDIO_BYTES) {
+				req.destroy();
+				sendJson(res, 413, { error: "Audio body too large" });
+				return null;
+			}
+			chunks.push(buf);
+		}
+	} catch {
+		sendJson(res, 400, { error: "Invalid audio body" });
+		return null;
+	}
+
+	return new Uint8Array(Buffer.concat(chunks));
+}
+
+function firstHeaderValue(value: string | string[] | undefined): string {
+	return Array.isArray(value) ? (value[0] ?? "") : (value ?? "");
+}
+
+async function readLocalInferenceAsrAudio(
+	req: http.IncomingMessage,
+	res: http.ServerResponse,
+): Promise<Uint8Array | null> {
+	const contentType = firstHeaderValue(req.headers["content-type"])
+		.toLowerCase()
+		.split(";", 1)[0]
+		.trim();
+
+	if (contentType === "application/json") {
+		const body = await readCompatJsonBody(req, res);
+		if (!body) return null;
+		if (typeof body.audioBase64 === "string") {
+			return new Uint8Array(Buffer.from(body.audioBase64, "base64"));
+		}
+		sendJson(res, 400, { error: "Missing audioBase64" });
+		return null;
+	}
+
+	return readRawAudioBody(req, res);
+}
+
+function isClosed(res: http.ServerResponse): boolean {
+	return res.destroyed || res.writableEnded;
+}
+
+export async function handleLocalInferenceAsrRoute(
+	req: http.IncomingMessage,
+	res: http.ServerResponse,
+	state: CompatRuntimeState,
+): Promise<boolean> {
+	const method = req.method?.toUpperCase() ?? "GET";
+	const url = new URL(req.url ?? "/", "http://localhost");
+	if (method === "GET" && url.pathname === "/api/asr/local-inference/status") {
+		if (!(await ensureRouteAuthorized(req, res, state))) return true;
+		// Transcription runs through the registered TRANSCRIPTION model handler,
+		// which is backed by the fused libelizainference ASR runtime. Report ready
+		// when a TRANSCRIPTION handler is registered.
+		const getModel = state.current?.getModel;
+		const runtimeAsr =
+			typeof getModel === "function" &&
+			Boolean(getModel.call(state.current, ModelType.TRANSCRIPTION));
+		sendJson(res, 200, {
+			ready: runtimeAsr,
+			provider: runtimeAsr ? "local-inference" : null,
+		});
+		return true;
+	}
+
+	if (method !== "POST" || url.pathname !== "/api/asr/local-inference") {
+		return false;
+	}
+
+	if (!(await ensureRouteAuthorized(req, res, state))) return true;
+
+	const audio = await readLocalInferenceAsrAudio(req, res);
+	if (!audio) return true;
+	if (audio.byteLength === 0) {
+		sendJson(res, 400, { error: "Missing audio" });
+		return true;
+	}
+
+	const abortController = new AbortController();
+	let completed = false;
+	let clientClosed = false;
+	const abortOnClose = () => {
+		clientClosed = true;
+		if (!completed && !abortController.signal.aborted) {
+			abortController.abort();
+		}
+	};
+	req.on("close", abortOnClose);
+	res.on("close", abortOnClose);
+
+	try {
+		const runtime = state.current;
+		if (!runtime) {
+			completed = true;
+			sendJson(res, 503, {
+				error: "Local inference TRANSCRIPTION is not available",
+			});
+			return true;
+		}
+		const { text, words } = await transcribeWavWithWords(
+			runtime,
+			audio,
+			abortController.signal,
+		);
+		completed = true;
+		sendJson(res, 200, { text, words });
+	} catch (err) {
+		if (!clientClosed && !abortController.signal.aborted && !isClosed(res)) {
+			sendJson(res, 502, {
+				error: `Local inference ASR error: ${
+					err instanceof Error ? err.message : String(err)
+				}`,
+			});
+		}
+	} finally {
+		completed = true;
+		req.off("close", abortOnClose);
+		res.off("close", abortOnClose);
+	}
+
+	return true;
+}

package/src/routes/local-inference-asr-transcribe.d.ts

@@ -0,0 +1,20 @@
+import { type AgentRuntime } from "@elizaos/core";
+import { type AsrWordTiming } from "../services/voice";
+/** A transcript plus optional per-word timings (ms from audio start). */
+export interface LocalAsrTranscript {
+    text: string;
+    words: AsrWordTiming[];
+}
+/**
+ * Transcribe WAV audio to text + per-word timings.
+ *
+ * The primary path is the single fused libelizainference FFI pipe
+ * (`transcribePcmTimed`, ASR ABI v12) — the cross-platform on-device runtime
+ * that emits per-word `[startMs,endMs)` timings the transcript player
+ * highlights against. When the in-process engine is not the active local
+ * backend (a device-bridge / capacitor / AOSP topology owns transcription),
+ * fall back to the `useModel` provider chain for the transcript text; word
+ * timings are unavailable on that path (`words: []`).
+ */
+export declare function transcribeWavWithWords(runtime: AgentRuntime, audioWav: Uint8Array, signal?: AbortSignal): Promise<LocalAsrTranscript>;
+//# sourceMappingURL=local-inference-asr-transcribe.d.ts.map

package/src/routes/local-inference-asr-transcribe.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"local-inference-asr-transcribe.d.ts","sourceRoot":"","sources":["local-inference-asr-transcribe.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,YAAY,EAAa,MAAM,eAAe,CAAC;AAE7D,OAAO,EAAE,KAAK,aAAa,EAAsB,MAAM,mBAAmB,CAAC;AAE3E,yEAAyE;AACzE,MAAM,WAAW,kBAAkB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,aAAa,EAAE,CAAC;CACvB;AA+BD;;;;;;;;;;GAUG;AACH,wBAAsB,sBAAsB,CAC3C,OAAO,EAAE,YAAY,EACrB,QAAQ,EAAE,UAAU,EACpB,MAAM,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,kBAAkB,CAAC,CAc7B"}

package/src/routes/local-inference-asr-transcribe.test.ts

@@ -0,0 +1,118 @@
+import { type AgentRuntime, ModelType } from "@elizaos/core";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { localInferenceEngine } from "../services/engine";
+import { transcribeWavWithWords } from "./local-inference-asr-transcribe";
+
+vi.mock("../services/engine", () => ({
+	localInferenceEngine: {
+		available: vi.fn(),
+		ensureActiveBundleVoiceReady: vi.fn(),
+		transcribePcmTimed: vi.fn(),
+	},
+}));
+
+const engine = vi.mocked(localInferenceEngine, true) as unknown as {
+	available: ReturnType<typeof vi.fn>;
+	ensureActiveBundleVoiceReady: ReturnType<typeof vi.fn>;
+	transcribePcmTimed: ReturnType<typeof vi.fn>;
+};
+
+/** Minimal mono PCM16 16 kHz WAV with four samples. */
+function wavBytes(): Uint8Array {
+	const pcm = new Int16Array([0, 900, -900, 0]);
+	const buffer = new ArrayBuffer(44 + pcm.length * 2);
+	const view = new DataView(buffer);
+	const writeAscii = (offset: number, value: string) => {
+		for (let i = 0; i < value.length; i += 1) {
+			view.setUint8(offset + i, value.charCodeAt(i));
+		}
+	};
+	writeAscii(0, "RIFF");
+	view.setUint32(4, 36 + pcm.length * 2, true);
+	writeAscii(8, "WAVE");
+	writeAscii(12, "fmt ");
+	view.setUint32(16, 16, true);
+	view.setUint16(20, 1, true);
+	view.setUint16(22, 1, true);
+	view.setUint32(24, 16_000, true);
+	view.setUint32(28, 16_000 * 2, true);
+	view.setUint16(32, 2, true);
+	view.setUint16(34, 16, true);
+	writeAscii(36, "data");
+	view.setUint32(40, pcm.length * 2, true);
+	for (let i = 0; i < pcm.length; i += 1) {
+		view.setInt16(44 + i * 2, pcm[i] ?? 0, true);
+	}
+	return new Uint8Array(buffer);
+}
+
+describe("transcribeWavWithWords", () => {
+	afterEach(() => {
+		vi.clearAllMocks();
+	});
+
+	it("runs the fused FFI timed pipe when the in-process engine is active", async () => {
+		engine.available.mockResolvedValue(true);
+		engine.ensureActiveBundleVoiceReady.mockResolvedValue(undefined);
+		engine.transcribePcmTimed.mockResolvedValue({
+			text: "  hello world  ",
+			words: [
+				{ text: "hello", startMs: 0, endMs: 500 },
+				{ text: "world", startMs: 500, endMs: 1000 },
+			],
+		});
+		const runtime = {
+			useModel: vi.fn(),
+		} as unknown as AgentRuntime;
+
+		const result = await transcribeWavWithWords(runtime, wavBytes());
+
+		// useModel is never touched — the single FFI pipe owns this path.
+		expect(runtime.useModel).not.toHaveBeenCalled();
+		expect(engine.ensureActiveBundleVoiceReady).toHaveBeenCalledTimes(1);
+		// The WAV decoded to a Float32Array @ 16 kHz before the timed call.
+		const [audioArg] = engine.transcribePcmTimed.mock.calls[0] ?? [];
+		expect((audioArg as { pcm: Float32Array }).pcm).toBeInstanceOf(
+			Float32Array,
+		);
+		expect((audioArg as { sampleRate: number }).sampleRate).toBe(16_000);
+		expect(result).toEqual({
+			text: "hello world",
+			words: [
+				{ text: "hello", startMs: 0, endMs: 500 },
+				{ text: "world", startMs: 500, endMs: 1000 },
+			],
+		});
+	});
+
+	it("falls back to the useModel provider chain when the engine is inactive", async () => {
+		engine.available.mockResolvedValue(false);
+		const useModel = vi
+			.fn()
+			.mockRejectedValueOnce(
+				new Error("No handler found for delegate type: TRANSCRIPTION"),
+			)
+			.mockResolvedValueOnce({ text: "hello local voice" });
+		const runtime = { useModel } as unknown as AgentRuntime;
+
+		const result = await transcribeWavWithWords(runtime, wavBytes());
+
+		expect(useModel).toHaveBeenCalledTimes(2);
+		expect(useModel.mock.calls[0]?.[0]).toBe(ModelType.TRANSCRIPTION);
+		expect(useModel.mock.calls[0]?.[2]).toBe("eliza-local-inference");
+		expect(useModel.mock.calls[1]?.[2]).toBe("capacitor-llama");
+		// Fallback path carries no word timings (TRANSCRIPTION ⇒ string).
+		expect(result).toEqual({ text: "hello local voice", words: [] });
+		expect(engine.transcribePcmTimed).not.toHaveBeenCalled();
+	});
+
+	it("rejects an invalid transcript shape from the provider chain", async () => {
+		engine.available.mockResolvedValue(false);
+		const useModel = vi.fn().mockResolvedValue(42);
+		const runtime = { useModel } as unknown as AgentRuntime;
+
+		await expect(transcribeWavWithWords(runtime, wavBytes())).rejects.toThrow(
+			/invalid transcript/,
+		);
+	});
+});