@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/routes/local-inference-tts-route.d.ts

@@ -0,0 +1,7 @@
+import type http from "node:http";
+import { type CompatRuntimeState } from "./compat-helpers";
+export declare function sanitizeLocalInferenceSpeechText(input: string): string;
+export declare function normalizeAudioBytes(value: unknown): Uint8Array;
+export declare function sniffAudioContentType(bytes: Uint8Array): string;
+export declare function handleLocalInferenceTtsRoute(req: http.IncomingMessage, res: http.ServerResponse, state: CompatRuntimeState): Promise<boolean>;
+//# sourceMappingURL=local-inference-tts-route.d.ts.map

package/src/routes/local-inference-tts-route.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"local-inference-tts-route.d.ts","sourceRoot":"","sources":["local-inference-tts-route.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EACN,KAAK,kBAAkB,EAIvB,MAAM,kBAAkB,CAAC;AAS1B,wBAAgB,gCAAgC,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAatE;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,GAAG,UAAU,CAW9D;AAED,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CA0B/D;AA0DD,wBAAsB,4BAA4B,CACjD,GAAG,EAAE,IAAI,CAAC,eAAe,EACzB,GAAG,EAAE,IAAI,CAAC,cAAc,EACxB,KAAK,EAAE,kBAAkB,GACvB,OAAO,CAAC,OAAO,CAAC,CAiGlB"}

package/src/routes/local-inference-tts-route.test.ts

@@ -0,0 +1,179 @@
+import * as http from "node:http";
+import { Socket } from "node:net";
+import { describe, expect, it, vi } from "vitest";
+import type { CompatRuntimeState } from "./compat-helpers";
+import {
+	handleLocalInferenceTtsRoute,
+	sanitizeLocalInferenceSpeechText,
+} from "./local-inference-tts-route";
+
+function wavBytes(): Uint8Array {
+	return new Uint8Array([
+		0x52, 0x49, 0x46, 0x46, 0x24, 0x00, 0x00, 0x00, 0x57, 0x41, 0x56, 0x45,
+		0x66, 0x6d, 0x74, 0x20,
+	]);
+}
+
+function fakeReq(body?: unknown): http.IncomingMessage {
+	const req = new http.IncomingMessage(new Socket());
+	req.method = "POST";
+	req.url = "/api/tts/local-inference";
+	req.headers = { host: "localhost:2138" };
+	Object.defineProperty(req.socket, "remoteAddress", {
+		value: "127.0.0.1",
+		configurable: true,
+	});
+	if (body !== undefined) {
+		(req as { body?: unknown }).body = body;
+	}
+	return req;
+}
+
+function fakeRes(): {
+	res: http.ServerResponse;
+	bodyBuffer: () => Buffer;
+	status: () => number;
+	header: (name: string) => string | undefined;
+} {
+	const req = new http.IncomingMessage(new Socket());
+	const res = new http.ServerResponse(req);
+	let body = Buffer.alloc(0);
+	let status = 200;
+	const headers = new Map<string, string>();
+	res.setHeader = ((
+		name: string,
+		value: number | string | readonly string[],
+	) => {
+		headers.set(String(name).toLowerCase(), String(value));
+		return res;
+	}) as typeof res.setHeader;
+	res.writeHead = ((code: number, values?: http.OutgoingHttpHeaders) => {
+		status = code;
+		res.statusCode = code;
+		if (values) {
+			for (const [key, value] of Object.entries(values)) {
+				headers.set(key.toLowerCase(), String(value));
+			}
+		}
+		return res;
+	}) as typeof res.writeHead;
+	res.end = ((chunk?: string | Uint8Array | Buffer) => {
+		if (typeof chunk === "string") {
+			body = Buffer.concat([body, Buffer.from(chunk)]);
+		} else if (chunk) {
+			body = Buffer.concat([body, Buffer.from(chunk)]);
+		}
+		return res;
+	}) as typeof res.end;
+	return {
+		res,
+		bodyBuffer: () => body,
+		status: () => status,
+		header: (name) => headers.get(name.toLowerCase()),
+	};
+}
+
+describe("local inference TTS route", () => {
+	it("sanitizes assistant markup before synthesis", () => {
+		expect(
+			sanitizeLocalInferenceSpeechText(
+				"<think>hidden</think>Hello `there` [friend](https://example.com) https://example.com",
+			),
+		).toBe("Hello there friend");
+	});
+
+	it("falls through missing providers and returns WAV bytes", async () => {
+		const useModel = vi
+			.fn()
+			.mockRejectedValueOnce(
+				new Error("No handler found for delegate type: TEXT_TO_SPEECH"),
+			)
+			.mockResolvedValueOnce(wavBytes());
+		const state: CompatRuntimeState = {
+			current: { useModel } as unknown as CompatRuntimeState["current"],
+		};
+		const out = fakeRes();
+
+		const handled = await handleLocalInferenceTtsRoute(
+			fakeReq({ text: "Hello" }),
+			out.res,
+			state,
+		);
+
+		const secondParams = useModel.mock.calls[1]?.[1] as
+			| { text?: string; signal?: AbortSignal; voice?: string }
+			| undefined;
+		expect(handled).toBe(true);
+		expect(useModel).toHaveBeenCalledTimes(2);
+		expect(useModel.mock.calls[1]?.[2]).toBe("capacitor-llama");
+		expect(secondParams).toMatchObject({ text: "Hello" });
+		expect(secondParams?.signal).toBeInstanceOf(AbortSignal);
+		expect(out.status()).toBe(200);
+		expect(out.header("content-type")).toBe("audio/wav");
+		expect(out.bodyBuffer()).toEqual(Buffer.from(wavBytes()));
+	});
+
+	it("forwards voice, model, speed, sample rate, and format hints", async () => {
+		const useModel = vi.fn().mockResolvedValue(wavBytes());
+		const state: CompatRuntimeState = {
+			current: { useModel } as unknown as CompatRuntimeState["current"],
+		};
+		const out = fakeRes();
+
+		await handleLocalInferenceTtsRoute(
+			fakeReq({
+				text: "Hello",
+				voiceId: "narrator",
+				modelId: "omnivoice-q4",
+				speed: 1.15,
+				sampleRate: 24_000,
+				format: "wav",
+			}),
+			out.res,
+			state,
+		);
+
+		expect(useModel.mock.calls[0]?.[1]).toMatchObject({
+			text: "Hello",
+			voice: "narrator",
+			modelId: "omnivoice-q4",
+			speed: 1.15,
+			sampleRate: 24_000,
+			format: "wav",
+		});
+	});
+
+	it("aborts TTS on client close without writing a synthetic 502", async () => {
+		let aborted = false;
+		let signalReady!: (signal: AbortSignal | undefined) => void;
+		const signalReceived = new Promise<AbortSignal | undefined>((resolve) => {
+			signalReady = resolve;
+		});
+		const useModel = vi.fn((_type, params: { signal?: AbortSignal }) => {
+			signalReady(params.signal);
+			return new Promise<Uint8Array>((_resolve, reject) => {
+				params.signal?.addEventListener(
+					"abort",
+					() => {
+						aborted = true;
+						reject(new Error("aborted"));
+					},
+					{ once: true },
+				);
+			});
+		});
+		const state: CompatRuntimeState = {
+			current: { useModel } as unknown as CompatRuntimeState["current"],
+		};
+		const req = fakeReq({ text: "Hello" });
+		const out = fakeRes();
+
+		const handledPromise = handleLocalInferenceTtsRoute(req, out.res, state);
+		await expect(signalReceived).resolves.toBeInstanceOf(AbortSignal);
+		req.emit("close");
+		await expect(handledPromise).resolves.toBe(true);
+
+		expect(aborted).toBe(true);
+		expect(out.bodyBuffer().length).toBe(0);
+	});
+});

package/src/routes/local-inference-tts-route.ts

@@ -0,0 +1,230 @@
+import type http from "node:http";
+import { type AgentRuntime, ModelType } from "@elizaos/core";
+import {
+	type CompatRuntimeState,
+	ensureRouteAuthorized,
+	readCompatJsonBody,
+	sendJson,
+} from "./compat-helpers";
+
+const LOCAL_TTS_PROVIDER_IDS = [
+	"eliza-local-inference",
+	"capacitor-llama",
+	"eliza-device-bridge",
+	"eliza-aosp-llama",
+] as const;
+
+export function sanitizeLocalInferenceSpeechText(input: string): string {
+	let text = input.normalize("NFKC");
+	text = text.replace(/<think\b[^>]*>[\s\S]*?(?:<\/think>|$)/gi, " ");
+	text = text.replace(
+		/<(analysis|reasoning|tool_calls?|tools?)\b[^>]*>[\s\S]*?(?:<\/\1>|$)/gi,
+		" ",
+	);
+	text = text.replace(/```[\s\S]*?```/g, " ");
+	text = text.replace(/`([^`]+)`/g, "$1");
+	text = text.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1");
+	text = text.replace(/<[^>\n]+>/g, " ");
+	text = text.replace(/\bhttps?:\/\/\S+/gi, " ");
+	return text.replace(/\s+/g, " ").trim();
+}
+
+export function normalizeAudioBytes(value: unknown): Uint8Array {
+	if (value instanceof Uint8Array) {
+		return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
+	}
+	if (value instanceof ArrayBuffer) {
+		return new Uint8Array(value);
+	}
+	if (ArrayBuffer.isView(value)) {
+		return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
+	}
+	throw new Error("TEXT_TO_SPEECH returned a non-binary payload");
+}
+
+export function sniffAudioContentType(bytes: Uint8Array): string {
+	if (
+		bytes.length >= 12 &&
+		bytes[0] === 0x52 &&
+		bytes[1] === 0x49 &&
+		bytes[2] === 0x46 &&
+		bytes[3] === 0x46 &&
+		bytes[8] === 0x57 &&
+		bytes[9] === 0x41 &&
+		bytes[10] === 0x56 &&
+		bytes[11] === 0x45
+	) {
+		return "audio/wav";
+	}
+	if (
+		bytes.length >= 3 &&
+		bytes[0] === 0x49 &&
+		bytes[1] === 0x44 &&
+		bytes[2] === 0x33
+	) {
+		return "audio/mpeg";
+	}
+	if (bytes.length >= 2 && bytes[0] === 0xff && (bytes[1] & 0xe0) === 0xe0) {
+		return "audio/mpeg";
+	}
+	return "application/octet-stream";
+}
+
+function isMissingTtsProviderError(error: unknown): boolean {
+	return (
+		error instanceof Error &&
+		/No handler found for delegate type: TEXT_TO_SPEECH/.test(error.message)
+	);
+}
+
+interface LocalInferenceTtsRequest {
+	text: string;
+	voice?: string;
+	model?: string;
+	modelId?: string;
+	speed?: number;
+	sampleRate?: number;
+	format?: string;
+	signal?: AbortSignal;
+}
+
+function optionalString(value: unknown): string | undefined {
+	return typeof value === "string" && value.trim() ? value.trim() : undefined;
+}
+
+function optionalPositiveNumber(value: unknown): number | undefined {
+	return typeof value === "number" && Number.isFinite(value) && value > 0
+		? value
+		: undefined;
+}
+
+async function useLocalInferenceTts(
+	runtime: AgentRuntime,
+	request: LocalInferenceTtsRequest,
+	signal?: AbortSignal,
+): Promise<Uint8Array> {
+	let lastError: unknown;
+	for (const provider of LOCAL_TTS_PROVIDER_IDS) {
+		try {
+			return normalizeAudioBytes(
+				await runtime.useModel(
+					ModelType.TEXT_TO_SPEECH,
+					{ ...request, ...(signal ? { signal } : {}) },
+					provider,
+				),
+			);
+		} catch (err) {
+			lastError = err;
+			if (!isMissingTtsProviderError(err)) throw err;
+		}
+	}
+	if (lastError instanceof Error) throw lastError;
+	throw new Error("No local-inference TEXT_TO_SPEECH provider is registered");
+}
+
+function isClosed(res: http.ServerResponse): boolean {
+	return res.destroyed || res.writableEnded;
+}
+
+export async function handleLocalInferenceTtsRoute(
+	req: http.IncomingMessage,
+	res: http.ServerResponse,
+	state: CompatRuntimeState,
+): Promise<boolean> {
+	const method = req.method?.toUpperCase() ?? "GET";
+	const url = new URL(req.url ?? "/", "http://localhost");
+	if (method !== "POST" || url.pathname !== "/api/tts/local-inference") {
+		return false;
+	}
+
+	if (!(await ensureRouteAuthorized(req, res, state))) return true;
+
+	const body = await readCompatJsonBody(req, res);
+	if (!body || typeof body !== "object") return true;
+
+	const rawText = body.text;
+	const text =
+		typeof rawText === "string"
+			? sanitizeLocalInferenceSpeechText(rawText)
+			: "";
+	if (!text) {
+		sendJson(res, 400, { error: "Missing text" });
+		return true;
+	}
+
+	const ttsRequest: LocalInferenceTtsRequest = {
+		text,
+		...(optionalString(body.voice)
+			? { voice: optionalString(body.voice) }
+			: {}),
+		...(optionalString(body.voiceId)
+			? { voice: optionalString(body.voiceId) }
+			: {}),
+		...(optionalString(body.model)
+			? { model: optionalString(body.model) }
+			: {}),
+		...(optionalString(body.modelId)
+			? { modelId: optionalString(body.modelId) }
+			: {}),
+		...(optionalPositiveNumber(body.speed)
+			? { speed: optionalPositiveNumber(body.speed) }
+			: {}),
+		...(optionalPositiveNumber(body.sampleRate)
+			? { sampleRate: optionalPositiveNumber(body.sampleRate) }
+			: {}),
+		...(optionalString(body.format)
+			? { format: optionalString(body.format) }
+			: {}),
+	};
+
+	const runtime = state.current;
+	if (!runtime) {
+		sendJson(res, 503, {
+			error: "Local inference TEXT_TO_SPEECH is not available",
+		});
+		return true;
+	}
+
+	const abortController = new AbortController();
+	let completed = false;
+	let clientClosed = false;
+	const abortOnClose = () => {
+		clientClosed = true;
+		if (!completed && !abortController.signal.aborted) {
+			abortController.abort();
+		}
+	};
+	req.on("close", abortOnClose);
+	res.on("close", abortOnClose);
+	try {
+		const bytes = await useLocalInferenceTts(
+			runtime,
+			ttsRequest,
+			abortController.signal,
+		);
+		if (bytes.length === 0) {
+			sendJson(res, 502, {
+				error: "Local inference TEXT_TO_SPEECH returned empty audio",
+			});
+			return true;
+		}
+		completed = true;
+		res.writeHead(200, {
+			"Content-Type": sniffAudioContentType(bytes),
+			"Cache-Control": "no-store",
+			"Content-Length": String(bytes.byteLength),
+		});
+		res.end(Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength));
+	} catch (err) {
+		if (!clientClosed && !abortController.signal.aborted && !isClosed(res)) {
+			sendJson(res, 502, {
+				error: `Local inference TTS error: ${err instanceof Error ? err.message : String(err)}`,
+			});
+		}
+	} finally {
+		completed = true;
+		req.off("close", abortOnClose);
+		res.off("close", abortOnClose);
+	}
+	return true;
+}

package/src/routes/voice-first-run-routes.d.ts

@@ -0,0 +1,62 @@
+/**
+ * Voice first-run HTTP routes.
+ *
+ * The flow (R2-speaker.md §6):
+ *
+ *   POST /api/voice/first-run/profile/start
+ *     → { sessionId, script: ScriptStep[], embeddingModel }
+ *
+ *   POST /api/voice/first-run/profile/append?id=<sessionId>
+ *     Content-Type: application/octet-stream
+ *     Body: PCM (Float32, 16 kHz, mono) for one capture window
+ *     → { sessionId, samplesReceived, totalSamples, durationMs }
+ *
+ *   POST /api/voice/first-run/profile/finalize?id=<sessionId>&entityId=<id>
+ *     → { profileId, entityId, samples, durationMs }
+ *
+ *   POST /api/voice/first-run/complete
+ *     Body: { entityId: string }
+ *     → { ownerEntityId, settingsWritten: ["ELIZA_ADMIN_ENTITY_ID"] }
+ *
+ * Sessions are in-memory. They expire after 30 minutes of inactivity.
+ * The encoder is loaded lazily; the route handlers return a structured
+ * 503 if the native GGML library is missing.
+ *
+ * Audio storage: when the user grants `audioRefs` consent, sample WAVs
+ * land under `$ELIZA_STATE_DIR/voice-profiles/audio/<profileId>/...`.
+ * Otherwise nothing is written to disk except the centroid + variance.
+ */
+import type * as http from "node:http";
+import { VoiceProfileStore } from "../services/voice/profile-store";
+import { type SpeakerEncoder } from "../services/voice/speaker/encoder";
+/** Verbatim first-run script (R2-speaker.md §6.2). */
+export interface FirstRunScriptStep {
+    id: string;
+    role: "consent" | "calibration" | "phonetic" | "prosody" | "quiet" | "open";
+    prompt: string;
+    expectedDurationMs: number;
+    requiresUserSpeech: boolean;
+}
+export declare const FIRST_RUN_SCRIPT: ReadonlyArray<FirstRunScriptStep>;
+/**
+ * Encoder factory. By default the route handlers load the WeSpeaker
+ * ResNet34-LM speaker encoder through the fused `libelizainference`
+ * `eliza_inference_speaker_*` ABI — the sole on-device speaker runtime.
+ * Tests inject a fake encoder via `setVoiceFirstRunEncoderFactory()`.
+ */
+export type EncoderFactory = () => Promise<SpeakerEncoder>;
+export declare function setVoiceFirstRunEncoderFactory(factory: EncoderFactory | null): void;
+export declare function setVoiceFirstRunProfileStore(store: VoiceProfileStore | null): void;
+/** Settings-write hook. The runtime overrides this with `runtime.setSetting`. */
+type SettingsWriter = (key: string, value: string) => void | Promise<void>;
+export declare function setVoiceFirstRunSettingsWriter(writer: SettingsWriter | null): void;
+/**
+ * Mount-point: returns `true` if the request was handled, `false` if
+ * the path is not one of the voice-first-run routes (so the caller
+ * can fall through to the next handler).
+ */
+export declare function handleVoiceFirstRunRoutes(req: http.IncomingMessage, res: http.ServerResponse): Promise<boolean>;
+/** Test helper: clear in-memory sessions so a test starts clean. */
+export declare function __resetVoiceFirstRunSessions(): void;
+export {};
+//# sourceMappingURL=voice-first-run-routes.d.ts.map

package/src/routes/voice-first-run-routes.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"voice-first-run-routes.d.ts","sourceRoot":"","sources":["voice-first-run-routes.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAGH,OAAO,KAAK,KAAK,IAAI,MAAM,WAAW,CAAC;AAWvC,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AACpE,OAAO,EAEN,KAAK,cAAc,EAMnB,MAAM,mCAAmC,CAAC;AAG3C,sDAAsD;AACtD,MAAM,WAAW,kBAAkB;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,SAAS,GAAG,aAAa,GAAG,UAAU,GAAG,SAAS,GAAG,OAAO,GAAG,MAAM,CAAC;IAC5E,MAAM,EAAE,MAAM,CAAC;IACf,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,OAAO,CAAC;CAC5B;AAED,eAAO,MAAM,gBAAgB,EAAE,aAAa,CAAC,kBAAkB,CA4E9D,CAAC;AA2BF;;;;;GAKG;AACH,MAAM,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,cAAc,CAAC,CAAC;AAK3D,wBAAgB,8BAA8B,CAC7C,OAAO,EAAE,cAAc,GAAG,IAAI,GAC5B,IAAI,CAGN;AAyCD,wBAAgB,4BAA4B,CAC3C,KAAK,EAAE,iBAAiB,GAAG,IAAI,GAC7B,IAAI,CAEN;AAWD,iFAAiF;AACjF,KAAK,cAAc,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAG3E,wBAAgB,8BAA8B,CAC7C,MAAM,EAAE,cAAc,GAAG,IAAI,GAC3B,IAAI,CAEN;AAyCD;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC9C,GAAG,EAAE,IAAI,CAAC,eAAe,EACzB,GAAG,EAAE,IAAI,CAAC,cAAc,GACtB,OAAO,CAAC,OAAO,CAAC,CAgOlB;AAED,oEAAoE;AACpE,wBAAgB,4BAA4B,IAAI,IAAI,CAEnD"}