@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/routes/voice-first-run-routes.ts

@@ -0,0 +1,524 @@
+/**
+ * Voice first-run HTTP routes.
+ *
+ * The flow (R2-speaker.md §6):
+ *
+ *   POST /api/voice/first-run/profile/start
+ *     → { sessionId, script: ScriptStep[], embeddingModel }
+ *
+ *   POST /api/voice/first-run/profile/append?id=<sessionId>
+ *     Content-Type: application/octet-stream
+ *     Body: PCM (Float32, 16 kHz, mono) for one capture window
+ *     → { sessionId, samplesReceived, totalSamples, durationMs }
+ *
+ *   POST /api/voice/first-run/profile/finalize?id=<sessionId>&entityId=<id>
+ *     → { profileId, entityId, samples, durationMs }
+ *
+ *   POST /api/voice/first-run/complete
+ *     Body: { entityId: string }
+ *     → { ownerEntityId, settingsWritten: ["ELIZA_ADMIN_ENTITY_ID"] }
+ *
+ * Sessions are in-memory. They expire after 30 minutes of inactivity.
+ * The encoder is loaded lazily; the route handlers return a structured
+ * 503 if the native GGML library is missing.
+ *
+ * Audio storage: when the user grants `audioRefs` consent, sample WAVs
+ * land under `$ELIZA_STATE_DIR/voice-profiles/audio/<profileId>/...`.
+ * Otherwise nothing is written to disk except the centroid + variance.
+ */
+
+import crypto from "node:crypto";
+import type * as http from "node:http";
+import path from "node:path";
+import {
+	logger,
+	readJsonBody,
+	resolveStateDir,
+	sendJson,
+	sendJsonError,
+} from "@elizaos/core";
+import { resolveFusedLibraryPath } from "../services/desktop-fused-ffi-backend-runtime";
+import { loadElizaInferenceFfi } from "../services/voice/ffi-bindings";
+import { VoiceProfileStore } from "../services/voice/profile-store";
+import {
+	averageEmbeddings,
+	type SpeakerEncoder,
+	SpeakerEncoderUnavailableError,
+	WESPEAKER_EMBEDDING_DIM,
+	WESPEAKER_MIN_SAMPLES,
+	WESPEAKER_RESNET34_LM_INT8_MODEL_ID,
+	WESPEAKER_SAMPLE_RATE,
+} from "../services/voice/speaker/encoder";
+import { FusedSpeakerEncoder } from "../services/voice/speaker/encoder-fused";
+
+/** Verbatim first-run script (R2-speaker.md §6.2). */
+export interface FirstRunScriptStep {
+	id: string;
+	role: "consent" | "calibration" | "phonetic" | "prosody" | "quiet" | "open";
+	prompt: string;
+	expectedDurationMs: number;
+	requiresUserSpeech: boolean;
+}
+
+export const FIRST_RUN_SCRIPT: ReadonlyArray<FirstRunScriptStep> = [
+	{
+		id: "consent-1",
+		role: "consent",
+		prompt:
+			"Before we start, I'd like to record a short voice sample so I can recognize you when you talk to me. The recording stays on this device. Is that okay?",
+		expectedDurationMs: 4_000,
+		requiresUserSpeech: true,
+	},
+	{
+		id: "consent-2",
+		role: "consent",
+		prompt:
+			"One more — do you want me to also be able to imitate your voice for outgoing messages? You can change this any time.",
+		expectedDurationMs: 4_000,
+		requiresUserSpeech: true,
+	},
+	{
+		id: "calibration",
+		role: "calibration",
+		prompt: 'Please say "Hello, my name is" and then your full name.',
+		expectedDurationMs: 5_000,
+		requiresUserSpeech: true,
+	},
+	{
+		id: "phonetic-1",
+		role: "phonetic",
+		prompt: '"The quick brown fox jumps over the lazy dog."',
+		expectedDurationMs: 10_000,
+		requiresUserSpeech: true,
+	},
+	{
+		id: "phonetic-2",
+		role: "phonetic",
+		prompt: '"Pack my box with five dozen liquor jugs."',
+		expectedDurationMs: 10_000,
+		requiresUserSpeech: true,
+	},
+	{
+		id: "phonetic-3",
+		role: "phonetic",
+		prompt: '"How razorback-jumping frogs can level six piqued gymnasts."',
+		expectedDurationMs: 10_000,
+		requiresUserSpeech: true,
+	},
+	{
+		id: "prosody-1",
+		role: "prosody",
+		prompt: '"Did you remember to lock the back door?"',
+		expectedDurationMs: 7_500,
+		requiresUserSpeech: true,
+	},
+	{
+		id: "prosody-2",
+		role: "prosody",
+		prompt:
+			'"I left the keys on the kitchen counter, near the coffee machine."',
+		expectedDurationMs: 7_500,
+		requiresUserSpeech: true,
+	},
+	{
+		id: "quiet",
+		role: "quiet",
+		prompt:
+			'Now read this one as if someone next to you is sleeping: "Just checking in quickly — everything\'s fine, talk to you tomorrow."',
+		expectedDurationMs: 10_000,
+		requiresUserSpeech: true,
+	},
+	{
+		id: "open",
+		role: "open",
+		prompt:
+			"Last one — tell me, in your own words, what you'd like me to help with most in the next few weeks.",
+		expectedDurationMs: 15_000,
+		requiresUserSpeech: true,
+	},
+];
+
+const SESSION_TIMEOUT_MS = 30 * 60 * 1000;
+
+interface FirstRunSession {
+	id: string;
+	createdAt: number;
+	lastAccessedAt: number;
+	embeddings: Float32Array[];
+	totalSamples: number;
+	totalDurationMs: number;
+	consent: {
+		attributionAuthorized: boolean;
+		synthesisAuthorized: boolean;
+	};
+}
+
+const sessions = new Map<string, FirstRunSession>();
+
+function pruneExpiredSessions(now: number): void {
+	for (const [id, session] of sessions.entries()) {
+		if (now - session.lastAccessedAt > SESSION_TIMEOUT_MS) {
+			sessions.delete(id);
+		}
+	}
+}
+
+/**
+ * Encoder factory. By default the route handlers load the WeSpeaker
+ * ResNet34-LM speaker encoder through the fused `libelizainference`
+ * `eliza_inference_speaker_*` ABI — the sole on-device speaker runtime.
+ * Tests inject a fake encoder via `setVoiceFirstRunEncoderFactory()`.
+ */
+export type EncoderFactory = () => Promise<SpeakerEncoder>;
+
+let encoderFactoryOverride: EncoderFactory | null = null;
+let cachedEncoder: SpeakerEncoder | null = null;
+
+export function setVoiceFirstRunEncoderFactory(
+	factory: EncoderFactory | null,
+): void {
+	encoderFactoryOverride = factory;
+	cachedEncoder = null;
+}
+
+async function loadEncoder(): Promise<SpeakerEncoder> {
+	if (cachedEncoder) return cachedEncoder;
+	if (encoderFactoryOverride) {
+		cachedEncoder = await encoderFactoryOverride();
+		return cachedEncoder;
+	}
+	cachedEncoder = await loadFusedSpeakerEncoder();
+	return cachedEncoder;
+}
+
+/**
+ * Load the fused speaker encoder. Resolves the fused `libelizainference`,
+ * creates a context anchored at the voice-profiles dir, and probes the speaker
+ * ABI up front: a build that lacks `eliza_inference_speaker_*` raises a
+ * structured `SpeakerEncoderUnavailableError` (the route surfaces it as a 503)
+ * instead of silently degrading. There is no standalone-lib fallback.
+ */
+async function loadFusedSpeakerEncoder(): Promise<SpeakerEncoder> {
+	const bundleRoot = path.join(resolveStateDir(), "voice-profiles");
+	const libPath = resolveFusedLibraryPath(bundleRoot);
+	if (!libPath) {
+		throw new SpeakerEncoderUnavailableError(
+			"library-missing",
+			"[voice-first-run] fused libelizainference not found. Set $ELIZA_INFERENCE_LIBRARY (exact path) or $ELIZA_INFERENCE_LIB_DIR, or build it via packages/app-core/scripts/build-llama-cpp-mtp.mjs.",
+		);
+	}
+	const ffi = loadElizaInferenceFfi(libPath);
+	if (!FusedSpeakerEncoder.isSupported(ffi)) {
+		throw new SpeakerEncoderUnavailableError(
+			"native-missing",
+			`[voice-first-run] the fused libelizainference at ${libPath} (ABI v${ffi.libraryAbiVersion}) lacks the speaker ABI (eliza_inference_speaker_supported() == 0). Rebuild with the WeSpeaker forward graph linked in.`,
+		);
+	}
+	const ctx = ffi.create(bundleRoot);
+	return FusedSpeakerEncoder.load({ ffi, ctx });
+}
+
+let profileStoreOverride: VoiceProfileStore | null = null;
+
+export function setVoiceFirstRunProfileStore(
+	store: VoiceProfileStore | null,
+): void {
+	profileStoreOverride = store;
+}
+
+async function getProfileStore(): Promise<VoiceProfileStore> {
+	if (profileStoreOverride) return profileStoreOverride;
+	const store = new VoiceProfileStore({
+		rootDir: path.join(resolveStateDir(), "voice-profiles"),
+	});
+	await store.init();
+	return store;
+}
+
+/** Settings-write hook. The runtime overrides this with `runtime.setSetting`. */
+type SettingsWriter = (key: string, value: string) => void | Promise<void>;
+let settingsWriter: SettingsWriter | null = null;
+
+export function setVoiceFirstRunSettingsWriter(
+	writer: SettingsWriter | null,
+): void {
+	settingsWriter = writer;
+}
+
+function startSession(consent: FirstRunSession["consent"]): FirstRunSession {
+	const id = `obs_${crypto.randomUUID()}`;
+	const now = Date.now();
+	const session: FirstRunSession = {
+		id,
+		createdAt: now,
+		lastAccessedAt: now,
+		embeddings: [],
+		totalSamples: 0,
+		totalDurationMs: 0,
+		consent,
+	};
+	sessions.set(id, session);
+	return session;
+}
+
+function decodeFloat32(buf: Buffer): Float32Array {
+	if (buf.byteLength % 4 !== 0) {
+		throw new Error(
+			`[voice-first-run] PCM buffer length ${buf.byteLength} is not a multiple of 4`,
+		);
+	}
+	// Copy into an owned Float32Array (Buffer.buffer may have an offset).
+	const out = new Float32Array(buf.byteLength / 4);
+	const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
+	for (let i = 0; i < out.length; i += 1) {
+		out[i] = view.getFloat32(i * 4, true);
+	}
+	return out;
+}
+
+async function readBinaryBody(req: http.IncomingMessage): Promise<Buffer> {
+	const chunks: Buffer[] = [];
+	for await (const chunk of req) {
+		chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+	}
+	return Buffer.concat(chunks);
+}
+
+/**
+ * Mount-point: returns `true` if the request was handled, `false` if
+ * the path is not one of the voice-first-run routes (so the caller
+ * can fall through to the next handler).
+ */
+export async function handleVoiceFirstRunRoutes(
+	req: http.IncomingMessage,
+	res: http.ServerResponse,
+): Promise<boolean> {
+	const method = (req.method ?? "GET").toUpperCase();
+	const url = new URL(req.url ?? "/", "http://localhost");
+	const pathname = url.pathname;
+	if (!pathname.startsWith("/api/voice/first-run/")) return false;
+
+	pruneExpiredSessions(Date.now());
+
+	if (method === "POST" && pathname === "/api/voice/first-run/profile/start") {
+		// Empty body is valid here (the consent flags default to a safe
+		// "attribution-yes / synthesis-no" pair). We read the body only when
+		// the caller has actually sent one — otherwise `readJsonBody` would
+		// emit a 400 for the empty string and then we'd double-write below.
+		const hasJsonBody =
+			(req.headers["content-type"] ?? "").includes("application/json") &&
+			req.headers["content-length"] !== "0";
+		const body = hasJsonBody
+			? await readJsonBody<Record<string, unknown>>(req, res, {
+					requireObject: false,
+				})
+			: null;
+		// If the body read failed, `readJsonBody` already sent a 4xx.
+		if (hasJsonBody && body === null) return true;
+		const consent = {
+			attributionAuthorized:
+				typeof body?.attributionAuthorized === "boolean"
+					? body.attributionAuthorized
+					: true,
+			synthesisAuthorized:
+				typeof body?.synthesisAuthorized === "boolean"
+					? body.synthesisAuthorized
+					: false,
+		};
+		const session = startSession(consent);
+		sendJson(res, {
+			sessionId: session.id,
+			script: FIRST_RUN_SCRIPT,
+			embeddingModel: WESPEAKER_RESNET34_LM_INT8_MODEL_ID,
+			expectedSampleRate: WESPEAKER_SAMPLE_RATE,
+			minSamplesPerCapture: WESPEAKER_MIN_SAMPLES,
+		});
+		return true;
+	}
+
+	if (method === "POST" && pathname === "/api/voice/first-run/profile/append") {
+		const sessionId = url.searchParams.get("id");
+		if (!sessionId) {
+			sendJsonError(res, "id query parameter is required");
+			return true;
+		}
+		const session = sessions.get(sessionId);
+		if (!session) {
+			sendJsonError(res, "session not found", 404);
+			return true;
+		}
+		let buffer: Buffer;
+		try {
+			buffer = await readBinaryBody(req);
+		} catch (err) {
+			sendJsonError(
+				res,
+				err instanceof Error ? err.message : "failed to read body",
+				400,
+			);
+			return true;
+		}
+		let pcm: Float32Array;
+		try {
+			pcm = decodeFloat32(buffer);
+		} catch (err) {
+			sendJsonError(
+				res,
+				err instanceof Error ? err.message : "invalid PCM body",
+				400,
+			);
+			return true;
+		}
+		if (pcm.length < WESPEAKER_MIN_SAMPLES) {
+			sendJsonError(
+				res,
+				`capture too short: ${pcm.length} samples (< ${WESPEAKER_MIN_SAMPLES})`,
+				400,
+			);
+			return true;
+		}
+		let encoder: SpeakerEncoder;
+		try {
+			encoder = await loadEncoder();
+		} catch (err) {
+			if (err instanceof SpeakerEncoderUnavailableError) {
+				sendJsonError(res, err.message, 503);
+				return true;
+			}
+			throw err;
+		}
+		try {
+			const embedding = await encoder.encode(pcm);
+			session.embeddings.push(embedding);
+			session.totalSamples += pcm.length;
+			session.totalDurationMs += Math.round(
+				(pcm.length / WESPEAKER_SAMPLE_RATE) * 1000,
+			);
+			session.lastAccessedAt = Date.now();
+			sendJson(res, {
+				sessionId: session.id,
+				samplesReceived: session.embeddings.length,
+				totalSamples: session.totalSamples,
+				durationMs: session.totalDurationMs,
+			});
+		} catch (err) {
+			if (err instanceof SpeakerEncoderUnavailableError) {
+				sendJsonError(
+					res,
+					err.message,
+					err.code === "invalid-input" ? 400 : 503,
+				);
+				return true;
+			}
+			throw err;
+		}
+		return true;
+	}
+
+	if (
+		method === "POST" &&
+		pathname === "/api/voice/first-run/profile/finalize"
+	) {
+		const sessionId = url.searchParams.get("id");
+		const entityId = url.searchParams.get("entityId");
+		if (!sessionId) {
+			sendJsonError(res, "id query parameter is required");
+			return true;
+		}
+		const session = sessions.get(sessionId);
+		if (!session) {
+			sendJsonError(res, "session not found", 404);
+			return true;
+		}
+		if (session.embeddings.length === 0) {
+			sendJsonError(res, "no embeddings captured yet", 400);
+			return true;
+		}
+		let centroid: Float32Array;
+		try {
+			centroid = averageEmbeddings(session.embeddings);
+		} catch (err) {
+			sendJsonError(
+				res,
+				err instanceof Error ? err.message : "failed to compute centroid",
+				400,
+			);
+			return true;
+		}
+		if (centroid.length !== WESPEAKER_EMBEDDING_DIM) {
+			sendJsonError(
+				res,
+				`centroid dim mismatch: ${centroid.length} != ${WESPEAKER_EMBEDDING_DIM}`,
+				500,
+			);
+			return true;
+		}
+		const store = await getProfileStore();
+		const profile = await store.createProfile({
+			centroid,
+			embeddingModel: WESPEAKER_RESNET34_LM_INT8_MODEL_ID,
+			entityId: entityId ?? null,
+			confidence: 0.95,
+			durationMs: session.totalDurationMs,
+			consent: {
+				attributionAuthorized: session.consent.attributionAuthorized,
+				synthesisAuthorized: session.consent.synthesisAuthorized,
+				grantedAt: new Date().toISOString(),
+				grantedBy: entityId ?? undefined,
+			},
+		});
+		sessions.delete(sessionId);
+		sendJson(res, {
+			profileId: profile.profileId,
+			entityId: profile.entityId,
+			samples: profile.sampleCount,
+			durationMs: profile.totalDurationMs,
+		});
+		return true;
+	}
+
+	if (method === "POST" && pathname === "/api/voice/first-run/complete") {
+		const body = await readJsonBody<Record<string, unknown>>(req, res);
+		if (!body) return true;
+		const entityId =
+			typeof body.entityId === "string" ? body.entityId.trim() : "";
+		if (!entityId) {
+			sendJsonError(res, "entityId is required", 400);
+			return true;
+		}
+		if (!settingsWriter) {
+			sendJsonError(
+				res,
+				"voice first-run settings writer not configured; runtime must call setVoiceFirstRunSettingsWriter()",
+				503,
+			);
+			return true;
+		}
+		try {
+			await settingsWriter("ELIZA_ADMIN_ENTITY_ID", entityId);
+		} catch (err) {
+			logger.error(
+				{ err },
+				"[voice-first-run] failed to write ELIZA_ADMIN_ENTITY_ID",
+			);
+			sendJsonError(
+				res,
+				err instanceof Error ? err.message : "failed to write owner entity id",
+				500,
+			);
+			return true;
+		}
+		sendJson(res, {
+			ownerEntityId: entityId,
+			settingsWritten: ["ELIZA_ADMIN_ENTITY_ID"],
+		});
+		return true;
+	}
+
+	return false;
+}
+
+/** Test helper: clear in-memory sessions so a test starts clean. */
+export function __resetVoiceFirstRunSessions(): void {
+	sessions.clear();
+}