@elizaos/plugin-local-inference 2.0.3-beta.2 → 2.0.3-beta.3

package/src/provider.ts

@@ -42,6 +42,8 @@ export const LOCAL_INFERENCE_MODEL_TYPES = [
 	ModelType.TRANSCRIPTION,
 ] as const;
 
+const OMIT_MAX_TOKENS_LOCAL_BUDGET = 64_000;
+
 export type LocalInferenceUnavailableReason =
 	| "backend_unavailable"
 	| "capability_unavailable"
@@ -290,7 +292,9 @@ function textGenerationArgsFromParams(
 	return {
 		prompt: promptFromParams(params),
 		stopSequences: params.stopSequences,
-		maxTokens: params.maxTokens,
+		maxTokens: params.omitMaxTokens
+			? (params.maxTokens ?? OMIT_MAX_TOKENS_LOCAL_BUDGET)
+			: params.maxTokens,
 		temperature: params.temperature,
 		topP: params.topP,
 		signal: params.signal,
@@ -723,6 +727,8 @@ function tryGetImageGenArbiter(
 function paramsToVisionRequest(params: ImageDescriptionParams | string): {
 	image: { kind: "dataUrl"; dataUrl: string } | { kind: "url"; url: string };
 	prompt?: string;
+	signal?: AbortSignal;
+	onTextChunk?: (chunk: string) => void | Promise<void>;
 } {
 	const url = typeof params === "string" ? params : params.imageUrl;
 	if (typeof url !== "string" || !url) {
@@ -733,15 +739,38 @@ function paramsToVisionRequest(params: ImageDescriptionParams | string): {
 		);
 	}
 	const prompt = typeof params === "object" ? params.prompt : undefined;
+	const signal =
+		typeof params === "object"
+			? (params as { signal?: AbortSignal }).signal
+			: undefined;
+	// Token-by-token streaming is intentionally explicit for vision. Hidden image
+	// preprocessing can happen inside a streaming chat turn; only forward the
+	// runtime callback when the call itself asks for `stream: true`.
+	const wantsStream =
+		typeof params === "object" &&
+		(params as { stream?: boolean }).stream === true;
+	const streamSink =
+		wantsStream && typeof params === "object"
+			? (params as { onStreamChunk?: (chunk: string) => void | Promise<void> })
+					.onStreamChunk
+			: undefined;
+	const onTextChunk =
+		typeof streamSink === "function"
+			? (chunk: string) => streamSink(chunk)
+			: undefined;
 	if (url.startsWith("data:")) {
 		return {
 			image: { kind: "dataUrl", dataUrl: url },
 			prompt,
+			...(signal ? { signal } : {}),
+			...(onTextChunk ? { onTextChunk } : {}),
 		};
 	}
 	return {
 		image: { kind: "url", url },
 		prompt,
+		...(signal ? { signal } : {}),
+		...(onTextChunk ? { onTextChunk } : {}),
 	};
 }
 
@@ -785,12 +814,12 @@ function createImageDescriptionHandler() {
 			markEliza1VisionHandlerPresent(runtime);
 			const modelKeyCandidate =
 				typeof params === "object"
-					? (params as unknown as { modelKey?: unknown }).modelKey
+					? (params as { modelKey?: unknown }).modelKey
 					: undefined;
 			const modelKey =
 				typeof modelKeyCandidate === "string" && modelKeyCandidate
 					? modelKeyCandidate
-					: "qwen3-vl";
+					: "gemma-vl";
 			const request = paramsToVisionRequest(params);
 			const result = await arbiter.requestVisionDescribe<
 				typeof request,

package/src/routes/compat-helpers.ts

@@ -186,8 +186,9 @@ export function sendJsonError(
 	res: http.ServerResponse,
 	status: number,
 	message: string,
+	extra?: Record<string, unknown>,
 ): void {
-	sendJson(res, status, { error: message });
+	sendJson(res, status, { error: message, ...extra });
 }
 
 export function ensureCompatApiAuthorized(

package/src/routes/index.ts

@@ -29,6 +29,7 @@ export {
 export * from "./local-inference-asr-route.js";
 export * from "./local-inference-compat-routes.js";
 export * from "./local-inference-tts-route.js";
+export { handleNativePcmTurnRoute } from "./native-pcm-turn-route.js";
 export {
 	__resetVoiceFirstRunSessions,
 	type EncoderFactory as VoiceFirstRunEncoderFactory,

package/src/routes/live-diarization-route.test.ts

@@ -17,6 +17,7 @@
 import type http from "node:http";
 import { Readable } from "node:stream";
 import { afterEach, describe, expect, it } from "vitest";
+import { buildLiveDiarizationConsumerDeps } from "../services/voice/live-diarization-session.js";
 import type { CompatRuntimeState } from "./compat-helpers.js";
 import {
 	handleLiveDiarizationRoute,
@@ -90,6 +91,34 @@ afterEach(async () => {
 });
 
 describe("handleLiveDiarizationRoute", () => {
+	it("builds AudioFrameConsumer deps with an optional echoReference provider", () => {
+		const echoReference = () => new Float32Array(320);
+		const deps = buildLiveDiarizationConsumerDeps({
+			vad: {
+				inSpeech: false,
+				onVadEvent: () => () => {},
+				pushFrame: async () => {},
+				flush: async () => {},
+				reset: () => {},
+			},
+			pipeline: {
+				attribute: async () => {
+					throw new Error("not used");
+				},
+			},
+			runtime: { emitEvent: async () => {} },
+			echoReference,
+		});
+		expect(deps.echoReference).toBe(echoReference);
+		expect(
+			buildLiveDiarizationConsumerDeps({
+				vad: deps.vad,
+				pipeline: deps.pipeline,
+				runtime: deps.runtime,
+			}),
+		).not.toHaveProperty("echoReference");
+	});
+
 	it("returns false for an unrelated path (passes through)", async () => {
 		const res = new FakeRes();
 		const handled = await handleLiveDiarizationRoute(
@@ -132,6 +161,7 @@ describe("handleLiveDiarizationRoute", () => {
 			models: { dir: string };
 			framesReceived: number;
 			turnsObserved: number;
+			aec: { echoReferenceWired: boolean };
 			error?: string;
 		};
 		// On CI/host there is no fused libelizainference, so readiness fails with
@@ -142,6 +172,7 @@ describe("handleLiveDiarizationRoute", () => {
 		expect("fusedInference" in status.libs).toBe(true);
 		expect(status.framesReceived).toBe(0);
 		expect(status.turnsObserved).toBe(0);
+		expect(status.aec.echoReferenceWired).toBe(false);
 		if (!status.ready) {
 			expect(status.error).toMatch(
 				/fused libelizainference|ABI|FFI|libelizainference/i,
@@ -149,6 +180,27 @@ describe("handleLiveDiarizationRoute", () => {
 		}
 	});
 
+	it("threads a runtime echoReference provider into live AEC status", async () => {
+		const res = new FakeRes();
+		const echoReference = () => new Float32Array(320);
+		const handled = await handleLiveDiarizationRoute(
+			makeReq({ method: "GET", url: "/api/voice/audio-frames/status" }),
+			res as unknown as http.ServerResponse,
+			{
+				current: {
+					emitEvent: async () => {},
+					voiceEchoReferenceProvider: echoReference,
+				} as never,
+			},
+		);
+		expect(handled).toBe(true);
+		expect(res.statusCode).toBe(200);
+		const status = res.json() as {
+			aec: { echoReferenceWired: boolean };
+		};
+		expect(status.aec.echoReferenceWired).toBe(true);
+	});
+
 	it("rejects a malformed frame batch with 400", async () => {
 		const res = new FakeRes();
 		const handled = await handleLiveDiarizationRoute(
@@ -210,4 +262,86 @@ describe("handleLiveDiarizationRoute", () => {
 		expect(handled).toBe(true);
 		expect(res.statusCode).toBe(503);
 	});
+
+	it("accepts agent-playback (far-end) frames for echo cancellation", async () => {
+		// Unlike the mic path, pushing playback needs no model build — it only
+		// decodes + appends to the alignment buffer — so this exercises the full
+		// route end-to-end on a host with no on-device GGUFs.
+		const res = new FakeRes();
+		const handled = await handleLiveDiarizationRoute(
+			makeReq({
+				method: "POST",
+				url: "/api/voice/playback-frames",
+				body: { frames: [silentFrame(0), silentFrame(1)] },
+			}),
+			res as unknown as http.ServerResponse,
+			runtimeState(),
+		);
+		expect(handled).toBe(true);
+		expect(res.statusCode).toBe(200);
+		expect(res.json()).toMatchObject({ ok: true, framesPushed: 2 });
+	});
+
+	it("accepts a reset-only playback request (barge-in / playback stop)", async () => {
+		const res = new FakeRes();
+		const handled = await handleLiveDiarizationRoute(
+			makeReq({
+				method: "POST",
+				url: "/api/voice/playback-frames",
+				body: { reset: true },
+			}),
+			res as unknown as http.ServerResponse,
+			runtimeState(),
+		);
+		expect(handled).toBe(true);
+		expect(res.statusCode).toBe(200);
+		expect(res.json()).toMatchObject({ ok: true, framesPushed: 0 });
+	});
+
+	it("resets stale playback before accepting a fresh playback batch", async () => {
+		const res = new FakeRes();
+		const handled = await handleLiveDiarizationRoute(
+			makeReq({
+				method: "POST",
+				url: "/api/voice/playback-frames",
+				body: { reset: true, frames: [silentFrame(0)] },
+			}),
+			res as unknown as http.ServerResponse,
+			runtimeState(),
+		);
+		expect(handled).toBe(true);
+		expect(res.statusCode).toBe(200);
+		expect(res.json()).toMatchObject({ ok: true, framesPushed: 1 });
+	});
+
+	it("rejects malformed playback frames with 400", async () => {
+		const res = new FakeRes();
+		const handled = await handleLiveDiarizationRoute(
+			makeReq({
+				method: "POST",
+				url: "/api/voice/playback-frames",
+				body: { frames: [{ pcm16: "AA==" /* missing fields */ }] },
+			}),
+			res as unknown as http.ServerResponse,
+			runtimeState(),
+		);
+		expect(handled).toBe(true);
+		expect(res.statusCode).toBe(400);
+		expect((res.json() as { error: string }).error).toMatch(/Malformed/);
+	});
+
+	it("rejects a non-array playback frames field with 400", async () => {
+		const res = new FakeRes();
+		const handled = await handleLiveDiarizationRoute(
+			makeReq({
+				method: "POST",
+				url: "/api/voice/playback-frames",
+				body: { frames: "nope" },
+			}),
+			res as unknown as http.ServerResponse,
+			runtimeState(),
+		);
+		expect(handled).toBe(true);
+		expect(res.statusCode).toBe(400);
+	});
 });

package/src/routes/live-diarization-route.ts

@@ -12,6 +12,13 @@
  *   POST /api/voice/audio-frames        body: { frames: AudioFrameEvent[],
  *                                               flush?: boolean }
  *                                       → { ok, framesReceived, turnsObserved }
+ *   POST /api/voice/playback-frames     body: { frames?: AudioFrameEvent[],
+ *                                               reset?: boolean }
+ *                                       → { ok, framesPushed }
+ *     The agent's own TTS playback (far-end) in the same base64 LE-s16 16 kHz
+ *     mono wire format, streamed in real time so the session's NLMS canceller
+ *     removes the agent's echo before VAD/attribution (#9455/#9583). Send
+ *     `reset: true` when playback stops / on barge-in.
  *   GET  /api/voice/audio-frames/status → LiveDiarizationStatus (device evidence)
  *
  * Auth follows the compat pattern: trusted-loopback OR the compat API token.
@@ -20,7 +27,10 @@
  */
 
 import type http from "node:http";
-import type { AudioFrameEvent } from "../services/voice/audio-frame-consumer.js";
+import type {
+	AudioFrameEvent,
+	EchoReferenceProvider,
+} from "../services/voice/audio-frame-consumer.js";
 import {
 	LiveDiarizationSession,
 	type RuntimeEventSink,
@@ -35,11 +45,42 @@ import {
 
 let session: LiveDiarizationSession | null = null;
 
+type RuntimeEchoReferenceSource = RuntimeEventSink & {
+	/**
+	 * Optional live far-end playback provider. Hosts that can tap agent TTS PCM
+	 * expose this so the diarization consumer can run AEC before VAD.
+	 */
+	voiceEchoReferenceProvider?: EchoReferenceProvider | null;
+	getVoiceEchoReferenceProvider?: () =>
+		| EchoReferenceProvider
+		| null
+		| undefined;
+};
+
+function resolveRuntimeEchoReference(
+	runtime: RuntimeEventSink,
+): EchoReferenceProvider | null {
+	const source = runtime as RuntimeEchoReferenceSource;
+	if (typeof source.getVoiceEchoReferenceProvider === "function") {
+		return source.getVoiceEchoReferenceProvider() ?? null;
+	}
+	return source.voiceEchoReferenceProvider ?? null;
+}
+
 /** Lazily own one session per agent process, bound to the live runtime. */
 function getSession(state: CompatRuntimeState): LiveDiarizationSession | null {
-	const runtime = state.current as unknown as RuntimeEventSink | null;
+	const runtime = state.current as RuntimeEventSink | null;
 	if (!runtime || typeof runtime.emitEvent !== "function") return null;
-	if (!session) session = new LiveDiarizationSession(runtime);
+	if (!session) {
+		// Thread a host-supplied echo reference (if any) into the live AEC path
+		// so the NLMS canceller has a far-end signal without the playback-frames
+		// ingest route (#9583).
+		const echoReference = resolveRuntimeEchoReference(runtime);
+		session = new LiveDiarizationSession(
+			runtime,
+			echoReference ? { echoReference } : {},
+		);
+	}
 	return session;
 }
 
@@ -118,5 +159,40 @@ export async function handleLiveDiarizationRoute(
 		return true;
 	}
 
+	if (url.pathname === "/api/voice/playback-frames" && method === "POST") {
+		if (!ensureCompatApiAuthorized(req, res)) return true;
+		const current = getSession(state);
+		if (!current) {
+			sendJsonError(res, 503, "Runtime not ready");
+			return true;
+		}
+		const body = await readCompatJsonBody(req, res);
+		if (!body) return true;
+		if (body.reset === true) current.resetPlayback();
+		const rawFrames = body.frames;
+		if (rawFrames !== undefined && !Array.isArray(rawFrames)) {
+			sendJsonError(
+				res,
+				400,
+				"Expected { frames?: AudioFrameEvent[], reset?: boolean }",
+			);
+			return true;
+		}
+		const frames = Array.isArray(rawFrames)
+			? rawFrames.filter(isAudioFrameEvent)
+			: [];
+		if (Array.isArray(rawFrames) && frames.length !== rawFrames.length) {
+			sendJsonError(
+				res,
+				400,
+				`Malformed playback frame(s): ${rawFrames.length - frames.length} of ${rawFrames.length} did not match AudioFrameEvent`,
+			);
+			return true;
+		}
+		current.pushPlayback(frames);
+		sendJson(res, 200, { ok: true, framesPushed: frames.length });
+		return true;
+	}
+
 	return false;
 }

package/src/routes/local-inference-asr-route.test.ts

@@ -1,17 +1,30 @@
 import * as http from "node:http";
 import { Socket } from "node:net";
 import { ModelType } from "@elizaos/core";
-import { describe, expect, it, vi } from "vitest";
+import { beforeEach, describe, expect, it, vi } from "vitest";
 import type { CompatRuntimeState } from "./compat-helpers";
 import { handleLocalInferenceAsrRoute } from "./local-inference-asr-route";
 import { transcribeWavWithWords } from "./local-inference-asr-transcribe";
 
+const engineMock = vi.hoisted(() => ({
+	canTranscribeLocally: vi.fn(async () => true),
+}));
+
+vi.mock("../services/engine", () => ({
+	localInferenceEngine: engineMock,
+}));
+
 vi.mock("./local-inference-asr-transcribe", () => ({
 	transcribeWavWithWords: vi.fn(),
 }));
 
 const transcribeWavWithWordsMock = vi.mocked(transcribeWavWithWords);
 
+beforeEach(() => {
+	vi.clearAllMocks();
+	engineMock.canTranscribeLocally.mockResolvedValue(true);
+});
+
 function wavBytes(): Uint8Array {
 	const pcm = new Int16Array([0, 900, -900, 0]);
 	const buffer = new ArrayBuffer(44 + pcm.length * 2);
@@ -92,7 +105,8 @@ function fakeRes(): {
 }
 
 describe("local inference ASR route", () => {
-	it("reports readiness off the registered fused TRANSCRIPTION handler", async () => {
+	it("reports readiness from the registered handler and eligible ASR bundle", async () => {
+		engineMock.canTranscribeLocally.mockResolvedValue(true);
 		const getModel = vi.fn(() => () => "transcript");
 		const useModel = vi.fn();
 		const state: CompatRuntimeState = {
@@ -119,9 +133,35 @@ describe("local inference ASR route", () => {
 			provider: "local-inference",
 		});
 		expect(getModel).toHaveBeenCalledWith(ModelType.TRANSCRIPTION);
+		expect(engineMock.canTranscribeLocally).toHaveBeenCalledTimes(1);
 		expect(useModel).not.toHaveBeenCalled();
 	});
 
+	it("reports not-ready when the handler is registered but no ASR bundle is eligible", async () => {
+		engineMock.canTranscribeLocally.mockResolvedValue(false);
+		const getModel = vi.fn(() => () => "transcript");
+		const state: CompatRuntimeState = {
+			current: {
+				getModel,
+			} as unknown as CompatRuntimeState["current"],
+		};
+		const out = fakeRes();
+
+		const handled = await handleLocalInferenceAsrRoute(
+			fakeReq(undefined, {
+				method: "GET",
+				url: "/api/asr/local-inference/status",
+			}),
+			out.res,
+			state,
+		);
+
+		expect(handled).toBe(true);
+		expect(out.status()).toBe(200);
+		expect(out.bodyJson()).toEqual({ ready: false, provider: null });
+		expect(engineMock.canTranscribeLocally).toHaveBeenCalledTimes(1);
+	});
+
 	it("reports not-ready when no TRANSCRIPTION handler is registered", async () => {
 		const getModel = vi.fn(() => undefined);
 		const state: CompatRuntimeState = {
@@ -143,6 +183,7 @@ describe("local inference ASR route", () => {
 		expect(handled).toBe(true);
 		expect(out.status()).toBe(200);
 		expect(out.bodyJson()).toEqual({ ready: false, provider: null });
+		expect(engineMock.canTranscribeLocally).not.toHaveBeenCalled();
 	});
 
 	it("transcribes raw WAV audio and returns text + per-word timings", async () => {

package/src/routes/local-inference-asr-route.ts

@@ -1,5 +1,6 @@
 import type http from "node:http";
 import { ModelType } from "@elizaos/core";
+import { localInferenceEngine } from "../services/engine";
 import {
 	type CompatRuntimeState,
 	ensureRouteAuthorized,
@@ -91,15 +92,17 @@ export async function handleLocalInferenceAsrRoute(
 	if (method === "GET" && url.pathname === "/api/asr/local-inference/status") {
 		if (!(await ensureRouteAuthorized(req, res, state))) return true;
 		// Transcription runs through the registered TRANSCRIPTION model handler,
-		// which is backed by the fused libelizainference ASR runtime. Report ready
-		// when a TRANSCRIPTION handler is registered.
+		// backed by the fused Gemma ASR runtime. A handler alone is not enough:
+		// the active or assigned Eliza-1 bundle must stage an eligible ASR model.
 		const getModel = state.current?.getModel;
 		const runtimeAsr =
 			typeof getModel === "function" &&
 			Boolean(getModel.call(state.current, ModelType.TRANSCRIPTION));
+		const ready =
+			runtimeAsr && (await localInferenceEngine.canTranscribeLocally());
 		sendJson(res, 200, {
-			ready: runtimeAsr,
-			provider: runtimeAsr ? "local-inference" : null,
+			ready,
+			provider: ready ? "local-inference" : null,
 		});
 		return true;
 	}

package/src/routes/local-inference-asr-transcribe.test.ts

@@ -6,14 +6,14 @@ import { transcribeWavWithWords } from "./local-inference-asr-transcribe";
 vi.mock("../services/engine", () => ({
 	localInferenceEngine: {
 		available: vi.fn(),
-		ensureActiveBundleVoiceReady: vi.fn(),
+		ensureActiveBundleAsrReady: vi.fn(),
 		transcribePcmTimed: vi.fn(),
 	},
 }));
 
 const engine = vi.mocked(localInferenceEngine, true) as unknown as {
 	available: ReturnType<typeof vi.fn>;
-	ensureActiveBundleVoiceReady: ReturnType<typeof vi.fn>;
+	ensureActiveBundleAsrReady: ReturnType<typeof vi.fn>;
 	transcribePcmTimed: ReturnType<typeof vi.fn>;
 };
 
@@ -53,7 +53,7 @@ describe("transcribeWavWithWords", () => {
 
 	it("runs the fused FFI timed pipe when the in-process engine is active", async () => {
 		engine.available.mockResolvedValue(true);
-		engine.ensureActiveBundleVoiceReady.mockResolvedValue(undefined);
+		engine.ensureActiveBundleAsrReady.mockResolvedValue(undefined);
 		engine.transcribePcmTimed.mockResolvedValue({
 			text: "  hello world  ",
 			words: [
@@ -69,7 +69,7 @@ describe("transcribeWavWithWords", () => {
 
 		// useModel is never touched — the single FFI pipe owns this path.
 		expect(runtime.useModel).not.toHaveBeenCalled();
-		expect(engine.ensureActiveBundleVoiceReady).toHaveBeenCalledTimes(1);
+		expect(engine.ensureActiveBundleAsrReady).toHaveBeenCalledTimes(1);
 		// The WAV decoded to a Float32Array @ 16 kHz before the timed call.
 		const [audioArg] = engine.transcribePcmTimed.mock.calls[0] ?? [];
 		expect((audioArg as { pcm: Float32Array }).pcm).toBeInstanceOf(

package/src/routes/local-inference-asr-transcribe.ts

@@ -55,7 +55,7 @@ export async function transcribeWavWithWords(
 ): Promise<LocalAsrTranscript> {
 	if (await localInferenceEngine.available()) {
 		const audio = decodeMonoPcm16Wav(audioWav);
-		await localInferenceEngine.ensureActiveBundleVoiceReady();
+		await localInferenceEngine.ensureActiveBundleAsrReady();
 		const { text, words } = await localInferenceEngine.transcribePcmTimed(
 			audio,
 			signal,

package/src/routes/local-inference-compat-routes.test.ts

@@ -366,7 +366,7 @@ describe("POST /api/local-inference/active", () => {
 		);
 		setActiveMock.mockRejectedValue(
 			new CandidateModelActivationError({
-				modelId: "eliza-1-0_8b",
+				modelId: "eliza-1-2b",
 				manifestVersion: "1.0.0-candidate.1",
 				failedEvals: ["textEval", "voiceRtf", "asrWer", "expressive"],
 			}),
@@ -377,7 +377,7 @@ describe("POST /api/local-inference/active", () => {
 			fakeReq({
 				method: "POST",
 				pathname: "/api/local-inference/active",
-				body: { modelId: "eliza-1-0_8b" },
+				body: { modelId: "eliza-1-2b" },
 			}),
 			res.res,
 			STATE,
@@ -390,7 +390,7 @@ describe("POST /api/local-inference/active", () => {
 			manifestVersion: string;
 			failedEvals: string[];
 		};
-		expect(body.modelId).toBe("eliza-1-0_8b");
+		expect(body.modelId).toBe("eliza-1-2b");
 		expect(body.manifestVersion).toBe("1.0.0-candidate.1");
 		expect(body.failedEvals).toContain("textEval");
 		expect(body.failedEvals).toContain("voiceRtf");