@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/routes/local-inference-tts-route.test.ts

@@ -0,0 +1,179 @@
+import * as http from "node:http";
+import { Socket } from "node:net";
+import { describe, expect, it, vi } from "vitest";
+import type { CompatRuntimeState } from "./compat-helpers";
+import {
+	handleLocalInferenceTtsRoute,
+	sanitizeLocalInferenceSpeechText,
+} from "./local-inference-tts-route";
+
+function wavBytes(): Uint8Array {
+	return new Uint8Array([
+		0x52, 0x49, 0x46, 0x46, 0x24, 0x00, 0x00, 0x00, 0x57, 0x41, 0x56, 0x45,
+		0x66, 0x6d, 0x74, 0x20,
+	]);
+}
+
+function fakeReq(body?: unknown): http.IncomingMessage {
+	const req = new http.IncomingMessage(new Socket());
+	req.method = "POST";
+	req.url = "/api/tts/local-inference";
+	req.headers = { host: "localhost:2138" };
+	Object.defineProperty(req.socket, "remoteAddress", {
+		value: "127.0.0.1",
+		configurable: true,
+	});
+	if (body !== undefined) {
+		(req as { body?: unknown }).body = body;
+	}
+	return req;
+}
+
+function fakeRes(): {
+	res: http.ServerResponse;
+	bodyBuffer: () => Buffer;
+	status: () => number;
+	header: (name: string) => string | undefined;
+} {
+	const req = new http.IncomingMessage(new Socket());
+	const res = new http.ServerResponse(req);
+	let body = Buffer.alloc(0);
+	let status = 200;
+	const headers = new Map<string, string>();
+	res.setHeader = ((
+		name: string,
+		value: number | string | readonly string[],
+	) => {
+		headers.set(String(name).toLowerCase(), String(value));
+		return res;
+	}) as typeof res.setHeader;
+	res.writeHead = ((code: number, values?: http.OutgoingHttpHeaders) => {
+		status = code;
+		res.statusCode = code;
+		if (values) {
+			for (const [key, value] of Object.entries(values)) {
+				headers.set(key.toLowerCase(), String(value));
+			}
+		}
+		return res;
+	}) as typeof res.writeHead;
+	res.end = ((chunk?: string | Uint8Array | Buffer) => {
+		if (typeof chunk === "string") {
+			body = Buffer.concat([body, Buffer.from(chunk)]);
+		} else if (chunk) {
+			body = Buffer.concat([body, Buffer.from(chunk)]);
+		}
+		return res;
+	}) as typeof res.end;
+	return {
+		res,
+		bodyBuffer: () => body,
+		status: () => status,
+		header: (name) => headers.get(name.toLowerCase()),
+	};
+}
+
+describe("local inference TTS route", () => {
+	it("sanitizes assistant markup before synthesis", () => {
+		expect(
+			sanitizeLocalInferenceSpeechText(
+				"<think>hidden</think>Hello `there` [friend](https://example.com) https://example.com",
+			),
+		).toBe("Hello there friend");
+	});
+
+	it("falls through missing providers and returns WAV bytes", async () => {
+		const useModel = vi
+			.fn()
+			.mockRejectedValueOnce(
+				new Error("No handler found for delegate type: TEXT_TO_SPEECH"),
+			)
+			.mockResolvedValueOnce(wavBytes());
+		const state: CompatRuntimeState = {
+			current: { useModel } as unknown as CompatRuntimeState["current"],
+		};
+		const out = fakeRes();
+
+		const handled = await handleLocalInferenceTtsRoute(
+			fakeReq({ text: "Hello" }),
+			out.res,
+			state,
+		);
+
+		const secondParams = useModel.mock.calls[1]?.[1] as
+			| { text?: string; signal?: AbortSignal; voice?: string }
+			| undefined;
+		expect(handled).toBe(true);
+		expect(useModel).toHaveBeenCalledTimes(2);
+		expect(useModel.mock.calls[1]?.[2]).toBe("capacitor-llama");
+		expect(secondParams).toMatchObject({ text: "Hello" });
+		expect(secondParams?.signal).toBeInstanceOf(AbortSignal);
+		expect(out.status()).toBe(200);
+		expect(out.header("content-type")).toBe("audio/wav");
+		expect(out.bodyBuffer()).toEqual(Buffer.from(wavBytes()));
+	});
+
+	it("forwards voice, model, speed, sample rate, and format hints", async () => {
+		const useModel = vi.fn().mockResolvedValue(wavBytes());
+		const state: CompatRuntimeState = {
+			current: { useModel } as unknown as CompatRuntimeState["current"],
+		};
+		const out = fakeRes();
+
+		await handleLocalInferenceTtsRoute(
+			fakeReq({
+				text: "Hello",
+				voiceId: "narrator",
+				modelId: "omnivoice-q4",
+				speed: 1.15,
+				sampleRate: 24_000,
+				format: "wav",
+			}),
+			out.res,
+			state,
+		);
+
+		expect(useModel.mock.calls[0]?.[1]).toMatchObject({
+			text: "Hello",
+			voice: "narrator",
+			modelId: "omnivoice-q4",
+			speed: 1.15,
+			sampleRate: 24_000,
+			format: "wav",
+		});
+	});
+
+	it("aborts TTS on client close without writing a synthetic 502", async () => {
+		let aborted = false;
+		let signalReady!: (signal: AbortSignal | undefined) => void;
+		const signalReceived = new Promise<AbortSignal | undefined>((resolve) => {
+			signalReady = resolve;
+		});
+		const useModel = vi.fn((_type, params: { signal?: AbortSignal }) => {
+			signalReady(params.signal);
+			return new Promise<Uint8Array>((_resolve, reject) => {
+				params.signal?.addEventListener(
+					"abort",
+					() => {
+						aborted = true;
+						reject(new Error("aborted"));
+					},
+					{ once: true },
+				);
+			});
+		});
+		const state: CompatRuntimeState = {
+			current: { useModel } as unknown as CompatRuntimeState["current"],
+		};
+		const req = fakeReq({ text: "Hello" });
+		const out = fakeRes();
+
+		const handledPromise = handleLocalInferenceTtsRoute(req, out.res, state);
+		await expect(signalReceived).resolves.toBeInstanceOf(AbortSignal);
+		req.emit("close");
+		await expect(handledPromise).resolves.toBe(true);
+
+		expect(aborted).toBe(true);
+		expect(out.bodyBuffer().length).toBe(0);
+	});
+});

package/src/routes/local-inference-tts-route.ts

@@ -0,0 +1,230 @@
+import type http from "node:http";
+import { type AgentRuntime, ModelType } from "@elizaos/core";
+import {
+	type CompatRuntimeState,
+	ensureRouteAuthorized,
+	readCompatJsonBody,
+	sendJson,
+} from "./compat-helpers";
+
+const LOCAL_TTS_PROVIDER_IDS = [
+	"eliza-local-inference",
+	"capacitor-llama",
+	"eliza-device-bridge",
+	"eliza-aosp-llama",
+] as const;
+
+export function sanitizeLocalInferenceSpeechText(input: string): string {
+	let text = input.normalize("NFKC");
+	text = text.replace(/<think\b[^>]*>[\s\S]*?(?:<\/think>|$)/gi, " ");
+	text = text.replace(
+		/<(analysis|reasoning|tool_calls?|tools?)\b[^>]*>[\s\S]*?(?:<\/\1>|$)/gi,
+		" ",
+	);
+	text = text.replace(/```[\s\S]*?```/g, " ");
+	text = text.replace(/`([^`]+)`/g, "$1");
+	text = text.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1");
+	text = text.replace(/<[^>\n]+>/g, " ");
+	text = text.replace(/\bhttps?:\/\/\S+/gi, " ");
+	return text.replace(/\s+/g, " ").trim();
+}
+
+export function normalizeAudioBytes(value: unknown): Uint8Array {
+	if (value instanceof Uint8Array) {
+		return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
+	}
+	if (value instanceof ArrayBuffer) {
+		return new Uint8Array(value);
+	}
+	if (ArrayBuffer.isView(value)) {
+		return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
+	}
+	throw new Error("TEXT_TO_SPEECH returned a non-binary payload");
+}
+
+export function sniffAudioContentType(bytes: Uint8Array): string {
+	if (
+		bytes.length >= 12 &&
+		bytes[0] === 0x52 &&
+		bytes[1] === 0x49 &&
+		bytes[2] === 0x46 &&
+		bytes[3] === 0x46 &&
+		bytes[8] === 0x57 &&
+		bytes[9] === 0x41 &&
+		bytes[10] === 0x56 &&
+		bytes[11] === 0x45
+	) {
+		return "audio/wav";
+	}
+	if (
+		bytes.length >= 3 &&
+		bytes[0] === 0x49 &&
+		bytes[1] === 0x44 &&
+		bytes[2] === 0x33
+	) {
+		return "audio/mpeg";
+	}
+	if (bytes.length >= 2 && bytes[0] === 0xff && (bytes[1] & 0xe0) === 0xe0) {
+		return "audio/mpeg";
+	}
+	return "application/octet-stream";
+}
+
+function isMissingTtsProviderError(error: unknown): boolean {
+	return (
+		error instanceof Error &&
+		/No handler found for delegate type: TEXT_TO_SPEECH/.test(error.message)
+	);
+}
+
+interface LocalInferenceTtsRequest {
+	text: string;
+	voice?: string;
+	model?: string;
+	modelId?: string;
+	speed?: number;
+	sampleRate?: number;
+	format?: string;
+	signal?: AbortSignal;
+}
+
+function optionalString(value: unknown): string | undefined {
+	return typeof value === "string" && value.trim() ? value.trim() : undefined;
+}
+
+function optionalPositiveNumber(value: unknown): number | undefined {
+	return typeof value === "number" && Number.isFinite(value) && value > 0
+		? value
+		: undefined;
+}
+
+async function useLocalInferenceTts(
+	runtime: AgentRuntime,
+	request: LocalInferenceTtsRequest,
+	signal?: AbortSignal,
+): Promise<Uint8Array> {
+	let lastError: unknown;
+	for (const provider of LOCAL_TTS_PROVIDER_IDS) {
+		try {
+			return normalizeAudioBytes(
+				await runtime.useModel(
+					ModelType.TEXT_TO_SPEECH,
+					{ ...request, ...(signal ? { signal } : {}) },
+					provider,
+				),
+			);
+		} catch (err) {
+			lastError = err;
+			if (!isMissingTtsProviderError(err)) throw err;
+		}
+	}
+	if (lastError instanceof Error) throw lastError;
+	throw new Error("No local-inference TEXT_TO_SPEECH provider is registered");
+}
+
+function isClosed(res: http.ServerResponse): boolean {
+	return res.destroyed || res.writableEnded;
+}
+
+export async function handleLocalInferenceTtsRoute(
+	req: http.IncomingMessage,
+	res: http.ServerResponse,
+	state: CompatRuntimeState,
+): Promise<boolean> {
+	const method = req.method?.toUpperCase() ?? "GET";
+	const url = new URL(req.url ?? "/", "http://localhost");
+	if (method !== "POST" || url.pathname !== "/api/tts/local-inference") {
+		return false;
+	}
+
+	if (!(await ensureRouteAuthorized(req, res, state))) return true;
+
+	const body = await readCompatJsonBody(req, res);
+	if (!body || typeof body !== "object") return true;
+
+	const rawText = body.text;
+	const text =
+		typeof rawText === "string"
+			? sanitizeLocalInferenceSpeechText(rawText)
+			: "";
+	if (!text) {
+		sendJson(res, 400, { error: "Missing text" });
+		return true;
+	}
+
+	const ttsRequest: LocalInferenceTtsRequest = {
+		text,
+		...(optionalString(body.voice)
+			? { voice: optionalString(body.voice) }
+			: {}),
+		...(optionalString(body.voiceId)
+			? { voice: optionalString(body.voiceId) }
+			: {}),
+		...(optionalString(body.model)
+			? { model: optionalString(body.model) }
+			: {}),
+		...(optionalString(body.modelId)
+			? { modelId: optionalString(body.modelId) }
+			: {}),
+		...(optionalPositiveNumber(body.speed)
+			? { speed: optionalPositiveNumber(body.speed) }
+			: {}),
+		...(optionalPositiveNumber(body.sampleRate)
+			? { sampleRate: optionalPositiveNumber(body.sampleRate) }
+			: {}),
+		...(optionalString(body.format)
+			? { format: optionalString(body.format) }
+			: {}),
+	};
+
+	const runtime = state.current;
+	if (!runtime) {
+		sendJson(res, 503, {
+			error: "Local inference TEXT_TO_SPEECH is not available",
+		});
+		return true;
+	}
+
+	const abortController = new AbortController();
+	let completed = false;
+	let clientClosed = false;
+	const abortOnClose = () => {
+		clientClosed = true;
+		if (!completed && !abortController.signal.aborted) {
+			abortController.abort();
+		}
+	};
+	req.on("close", abortOnClose);
+	res.on("close", abortOnClose);
+	try {
+		const bytes = await useLocalInferenceTts(
+			runtime,
+			ttsRequest,
+			abortController.signal,
+		);
+		if (bytes.length === 0) {
+			sendJson(res, 502, {
+				error: "Local inference TEXT_TO_SPEECH returned empty audio",
+			});
+			return true;
+		}
+		completed = true;
+		res.writeHead(200, {
+			"Content-Type": sniffAudioContentType(bytes),
+			"Cache-Control": "no-store",
+			"Content-Length": String(bytes.byteLength),
+		});
+		res.end(Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength));
+	} catch (err) {
+		if (!clientClosed && !abortController.signal.aborted && !isClosed(res)) {
+			sendJson(res, 502, {
+				error: `Local inference TTS error: ${err instanceof Error ? err.message : String(err)}`,
+			});
+		}
+	} finally {
+		completed = true;
+		req.off("close", abortOnClose);
+		res.off("close", abortOnClose);
+	}
+	return true;
+}

package/src/routes/native-pcm-turn-route.test.ts

@@ -0,0 +1,136 @@
+import type http from "node:http";
+import { Readable } from "node:stream";
+import { logger } from "@elizaos/core";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { handleNativePcmTurnRoute } from "./native-pcm-turn-route";
+
+const engineMock = vi.hoisted(() => ({
+	ensureActiveBundleAsrReady: vi.fn(async () => undefined),
+	runVoiceTurn: vi.fn(
+		async (): Promise<"done" | "token-cap" | "cancelled"> => "done",
+	),
+}));
+
+vi.mock("../services/engine", () => ({
+	localInferenceEngine: engineMock,
+}));
+
+class FakeRes {
+	statusCode = 200;
+	headersSent = false;
+	private readonly headers = new Map<string, string>();
+	body = "";
+	setHeader(name: string, value: string): void {
+		this.headers.set(name.toLowerCase(), value);
+	}
+	end(chunk?: string): void {
+		if (chunk) this.body += chunk;
+		this.headersSent = true;
+	}
+	json(): unknown {
+		return JSON.parse(this.body);
+	}
+}
+
+function makeReq(opts: {
+	method?: string;
+	url?: string;
+	body?: unknown;
+}): http.IncomingMessage {
+	const payload = opts.body !== undefined ? JSON.stringify(opts.body) : "";
+	const stream = Readable.from(payload ? [Buffer.from(payload)] : []);
+	const req = stream as unknown as http.IncomingMessage & {
+		method: string;
+		url: string;
+		headers: Record<string, string>;
+	};
+	req.method = opts.method ?? "POST";
+	req.url = opts.url ?? "/api/voice/native-pcm-turn";
+	req.headers = {
+		host: "127.0.0.1:31337",
+		"content-type": "application/json",
+	};
+	return req;
+}
+
+function float32Base64(samples: ReadonlyArray<number>): string {
+	const bytes = Buffer.alloc(samples.length * 4);
+	samples.forEach((sample, index) => {
+		bytes.writeFloatLE(sample, index * 4);
+	});
+	return bytes.toString("base64");
+}
+
+beforeEach(() => {
+	vi.clearAllMocks();
+	engineMock.ensureActiveBundleAsrReady.mockResolvedValue(undefined);
+	engineMock.runVoiceTurn.mockResolvedValue("done");
+});
+
+describe("handleNativePcmTurnRoute", () => {
+	it("runs completed native PCM turns through the local voice engine", async () => {
+		const infoSpy = vi
+			.spyOn(logger, "info")
+			.mockImplementation(() => undefined);
+		const res = new FakeRes();
+
+		const handled = await handleNativePcmTurnRoute(
+			makeReq({
+				body: {
+					turnId: "native-turn-1",
+					pcm: float32Base64([0, 0.5, -0.25]),
+					sampleRate: 24_000,
+					signal: "end-of-speech",
+				},
+			}),
+			res as unknown as http.ServerResponse,
+		);
+
+		expect(handled).toBe(true);
+		expect(engineMock.ensureActiveBundleAsrReady).toHaveBeenCalledTimes(1);
+		expect(engineMock.runVoiceTurn).toHaveBeenCalledTimes(1);
+		const [audio, opts] = engineMock.runVoiceTurn.mock.calls[0] as [
+			{ pcm: Float32Array; sampleRate: number },
+			{ events?: { onComplete?: (reason: "done") => void } },
+		];
+		expect(audio.sampleRate).toBe(24_000);
+		expect(Array.from(audio.pcm)).toEqual([0, 0.5, -0.25]);
+		opts.events?.onComplete?.("done");
+		expect(infoSpy).toHaveBeenCalledWith(
+			expect.objectContaining({
+				turnId: "native-turn-1",
+				exitReason: "done",
+				nativeSignal: "end-of-speech",
+			}),
+			expect.stringContaining("completed native PCM voice turn"),
+		);
+		expect(res.statusCode).toBe(200);
+		expect(res.json()).toEqual({
+			ok: true,
+			turnId: "native-turn-1",
+			exitReason: "done",
+		});
+	});
+
+	it("rejects malformed Float32 PCM before touching the engine", async () => {
+		const res = new FakeRes();
+
+		const handled = await handleNativePcmTurnRoute(
+			makeReq({
+				body: {
+					pcm: Buffer.from([0, 1]).toString("base64"),
+					sampleRate: 16_000,
+				},
+			}),
+			res as unknown as http.ServerResponse,
+		);
+
+		expect(handled).toBe(true);
+		expect(res.statusCode).toBe(400);
+		expect(res.json()).toEqual({
+			error: "pcm byte length must be a multiple of 4 for Float32 PCM",
+		});
+		expect(engineMock.ensureActiveBundleAsrReady).not.toHaveBeenCalled();
+		expect(engineMock.runVoiceTurn).not.toHaveBeenCalled();
+	});
+});

package/src/routes/native-pcm-turn-route.ts

@@ -0,0 +1,121 @@
+import type http from "node:http";
+import { logger, readJsonBody, sendJson, sendJsonError } from "@elizaos/core";
+import { localInferenceEngine } from "../services/engine";
+import type { VoicePipelineEvents } from "../services/voice/pipeline";
+
+const ROUTE_PATH = "/api/voice/native-pcm-turn";
+const MAX_PCM_BYTES = 20 * 1024 * 1024;
+const DEFAULT_SAMPLE_RATE = 16_000;
+
+type NativePcmTurnBody = {
+	turnId?: unknown;
+	pcm?: unknown;
+	sampleRate?: unknown;
+	signal?: unknown;
+};
+
+function decodeFloat32Base64(value: unknown): Float32Array | string {
+	if (typeof value !== "string" || value.length === 0) {
+		return "pcm must be a non-empty base64 string";
+	}
+	const bytes = Buffer.from(value, "base64");
+	if (bytes.byteLength === 0 || bytes.byteLength > MAX_PCM_BYTES) {
+		return `pcm byte length must be between 1 and ${MAX_PCM_BYTES}`;
+	}
+	if (bytes.byteLength % 4 !== 0) {
+		return "pcm byte length must be a multiple of 4 for Float32 PCM";
+	}
+	const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+	const pcm = new Float32Array(bytes.byteLength / 4);
+	for (let i = 0; i < pcm.length; i += 1) {
+		pcm[i] = view.getFloat32(i * 4, true);
+	}
+	return pcm;
+}
+
+function normalizeSampleRate(value: unknown): number | string {
+	if (value === undefined) return DEFAULT_SAMPLE_RATE;
+	if (
+		typeof value !== "number" ||
+		!Number.isFinite(value) ||
+		!Number.isInteger(value) ||
+		value <= 0 ||
+		value > 192_000
+	) {
+		return "sampleRate must be an integer between 1 and 192000";
+	}
+	return value;
+}
+
+function normalizeTurnId(value: unknown): string | undefined {
+	if (typeof value !== "string") return undefined;
+	const trimmed = value.trim();
+	return trimmed.length > 0 ? trimmed : undefined;
+}
+
+function voiceEventsForTurn(args: {
+	turnId?: string;
+	nativeSignal?: unknown;
+}): VoicePipelineEvents {
+	return {
+		onComplete(exitReason) {
+			logger.info(
+				{
+					turnId: args.turnId,
+					exitReason,
+					nativeSignal: args.nativeSignal,
+				},
+				"[native-pcm-turn] completed native PCM voice turn",
+			);
+		},
+	};
+}
+
+export async function handleNativePcmTurnRoute(
+	req: http.IncomingMessage,
+	res: http.ServerResponse,
+): Promise<boolean> {
+	const method = (req.method ?? "GET").toUpperCase();
+	const pathname = new URL(req.url ?? "/", "http://localhost").pathname;
+	if (method !== "POST" || pathname !== ROUTE_PATH) return false;
+
+	const body = await readJsonBody<NativePcmTurnBody>(req, res, {
+		maxBytes: MAX_PCM_BYTES + 4096,
+	});
+	if (!body) return true;
+
+	const pcm = decodeFloat32Base64(body.pcm);
+	if (typeof pcm === "string") {
+		sendJsonError(res, pcm, 400);
+		return true;
+	}
+	const sampleRate = normalizeSampleRate(body.sampleRate);
+	if (typeof sampleRate === "string") {
+		sendJsonError(res, sampleRate, 400);
+		return true;
+	}
+	const turnId = normalizeTurnId(body.turnId);
+
+	try {
+		await localInferenceEngine.ensureActiveBundleAsrReady();
+		const exitReason = await localInferenceEngine.runVoiceTurn(
+			{ pcm, sampleRate },
+			{ events: voiceEventsForTurn({ turnId, nativeSignal: body.signal }) },
+		);
+		sendJson(res, { ok: true, turnId: turnId ?? null, exitReason });
+	} catch (error) {
+		logger.warn(
+			{
+				turnId,
+				error: error instanceof Error ? error.message : String(error),
+			},
+			"[native-pcm-turn] failed to run native PCM voice turn",
+		);
+		sendJsonError(
+			res,
+			error instanceof Error ? error.message : "native PCM voice turn failed",
+			503,
+		);
+	}
+	return true;
+}

package/src/routes/transcript-audio-store.ts

@@ -0,0 +1,27 @@
+/**
+ * Transcript audio persistence (#8789).
+ *
+ * The transcript create route runs in plugin-local-inference, which cannot
+ * import the agent-internal media store (`packages/agent/src/api/media-store.ts`
+ * is not exported). But that store is just a content-addressed write under
+ * `<stateDir>/media/<sha256>.<ext>`, served by the agent's `serveMediaFile` at
+ * `/api/media/<sha256>.<ext>` (with HTTP Range, so `<audio>` scrubbing works).
+ * Writing the session WAV the same way — via core's `resolveStateDir` — lands it
+ * in the exact dir the agent already serves, so transcript audio plays back
+ * with zero extra wiring. Idempotent (sha256 filename).
+ */
+
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { resolveStateDir } from "@elizaos/core";
+
+/** Persist mono PCM16 WAV bytes; returns the served `/api/media/<hash>.wav` URL. */
+export function persistTranscriptAudioWav(wav: Buffer): string {
+	const hash = createHash("sha256").update(wav).digest("hex");
+	const dir = join(resolveStateDir(), "media");
+	mkdirSync(dir, { recursive: true });
+	const file = join(dir, `${hash}.wav`);
+	if (!existsSync(file)) writeFileSync(file, wav);
+	return `/api/media/${hash}.wav`;
+}