@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/routes/compat-helpers.ts

@@ -0,0 +1,275 @@
+import crypto from "node:crypto";
+import type http from "node:http";
+import { isIP } from "node:net";
+import type { AgentRuntime } from "@elizaos/core";
+import { isLoopbackBindHost, resolveApiToken } from "@elizaos/shared";
+
+const MAX_BODY_BYTES = 1_048_576;
+
+export interface CompatRuntimeState {
+	current: AgentRuntime | null;
+	pendingAgentName?: string | null;
+	pendingRestartReasons?: string[];
+}
+
+function firstHeaderValue(value: string | string[] | undefined): string | null {
+	if (typeof value === "string") return value;
+	if (Array.isArray(value) && typeof value[0] === "string") return value[0];
+	return null;
+}
+
+export function getCompatApiToken(): string | null {
+	return resolveApiToken(process.env);
+}
+
+export function getProvidedApiToken(
+	req: Pick<http.IncomingMessage, "headers">,
+): string | null {
+	const authHeader = firstHeaderValue(req.headers.authorization)
+		?.slice(0, 1024)
+		.trim();
+	if (authHeader) {
+		const match = /^Bearer\s{1,8}(.+)$/i.exec(authHeader);
+		if (match?.[1]) return match[1].trim();
+	}
+	return (
+		(
+			firstHeaderValue(req.headers["x-eliza-token"]) ??
+			firstHeaderValue(req.headers["x-elizaos-token"]) ??
+			firstHeaderValue(req.headers["x-api-key"]) ??
+			firstHeaderValue(req.headers["x-api-token"])
+		)?.trim() || null
+	);
+}
+
+export function tokenMatches(expected: string, provided: string): boolean {
+	const expectedBytes = Buffer.from(expected);
+	const providedBytes = Buffer.from(provided);
+	return (
+		expectedBytes.length === providedBytes.length &&
+		crypto.timingSafeEqual(expectedBytes, providedBytes)
+	);
+}
+
+function isLoopbackRemoteAddress(
+	remoteAddress: string | null | undefined,
+): boolean {
+	if (!remoteAddress) return false;
+	const normalized = remoteAddress.trim().toLowerCase();
+	return (
+		normalized === "127.0.0.1" ||
+		normalized === "::1" ||
+		normalized === "0:0:0:0:0:0:0:1" ||
+		normalized === "::ffff:127.0.0.1" ||
+		normalized === "::ffff:0:127.0.0.1"
+	);
+}
+
+function headerList(value: string | string[] | undefined): string[] {
+	if (!value) return [];
+	return (Array.isArray(value) ? value : [value])
+		.flatMap((entry) => entry.split(","))
+		.map((entry) => entry.trim())
+		.filter(Boolean);
+}
+
+function proxyClientHeaderBlocksLocalTrust(
+	headers: http.IncomingHttpHeaders,
+): boolean {
+	for (const name of [
+		"forwarded",
+		"forwarded-for",
+		"x-forwarded",
+		"x-forwarded-for",
+		"x-original-forwarded-for",
+		"x-real-ip",
+		"x-client-ip",
+		"x-forwarded-client-ip",
+		"x-cluster-client-ip",
+		"cf-connecting-ip",
+		"true-client-ip",
+		"fastly-client-ip",
+		"x-appengine-user-ip",
+		"x-azure-clientip",
+	]) {
+		const values = headerList(headers[name]);
+		for (const value of values) {
+			const host = value.replace(/^\[|\]$/g, "").split(":")[0];
+			if (host && isIP(host) && !isLoopbackRemoteAddress(host)) return true;
+		}
+	}
+	return false;
+}
+
+function isCloudProvisionedByEnv(): boolean {
+	return process.env.ELIZA_CLOUD_PROVISIONED === "1";
+}
+
+function isLocalAuthRequiredByEnv(): boolean {
+	return process.env.ELIZA_REQUIRE_LOCAL_AUTH === "1";
+}
+
+function isTrustedLocalOrigin(raw: string): boolean {
+	const trimmed = raw.trim();
+	if (!trimmed || trimmed === "null") return true;
+	try {
+		const parsed = new URL(trimmed);
+		if (
+			parsed.protocol === "file:" ||
+			parsed.protocol === "app:" ||
+			parsed.protocol === "tauri:" ||
+			parsed.protocol === "capacitor:" ||
+			parsed.protocol === "capacitor-electron:" ||
+			parsed.protocol === "electrobun:" ||
+			parsed.protocol === "views:"
+		) {
+			return true;
+		}
+		return isLoopbackBindHost(parsed.hostname);
+	} catch {
+		return false;
+	}
+}
+
+export function isTrustedLocalRequest(
+	req: Pick<http.IncomingMessage, "headers" | "socket">,
+): boolean {
+	if (isLocalAuthRequiredByEnv()) return false;
+	if (isCloudProvisionedByEnv()) return false;
+	if (!isLoopbackRemoteAddress(req.socket.remoteAddress)) return false;
+	if (proxyClientHeaderBlocksLocalTrust(req.headers)) return false;
+
+	const host = firstHeaderValue(req.headers.host);
+	if (host && !isLoopbackBindHost(host)) return false;
+
+	const secFetchSite = firstHeaderValue(
+		req.headers["sec-fetch-site"],
+	)?.toLowerCase();
+	if (secFetchSite === "cross-site") return false;
+
+	const origin = firstHeaderValue(req.headers.origin);
+	if (origin && !isTrustedLocalOrigin(origin)) return false;
+
+	const referer = firstHeaderValue(req.headers.referer);
+	if (!origin && referer && !isTrustedLocalOrigin(referer)) return false;
+
+	return true;
+}
+
+function scrubStackFields(value: unknown): unknown {
+	if (value instanceof Error)
+		return { error: value.message || "Internal error" };
+	if (Array.isArray(value)) return value.map(scrubStackFields);
+	if (value && typeof value === "object") {
+		const out: Record<string, unknown> = {};
+		for (const [key, nested] of Object.entries(value)) {
+			if (key === "stack" || key === "stackTrace") continue;
+			out[key] = scrubStackFields(nested);
+		}
+		return out;
+	}
+	return value;
+}
+
+export function sendJson(
+	res: http.ServerResponse,
+	status: number,
+	body: unknown,
+): void {
+	if (res.headersSent) return;
+	res.statusCode = status;
+	res.setHeader("content-type", "application/json; charset=utf-8");
+	res.end(JSON.stringify(scrubStackFields(body)));
+}
+
+export function sendJsonError(
+	res: http.ServerResponse,
+	status: number,
+	message: string,
+	extra?: Record<string, unknown>,
+): void {
+	sendJson(res, status, { error: message, ...extra });
+}
+
+export function ensureCompatApiAuthorized(
+	req: Pick<http.IncomingMessage, "headers" | "socket">,
+	res: http.ServerResponse,
+): boolean {
+	if (isTrustedLocalRequest(req)) return true;
+	const expectedToken = getCompatApiToken();
+	if (!expectedToken) {
+		sendJsonError(res, 401, "Unauthorized");
+		return false;
+	}
+	const providedToken = getProvidedApiToken(req);
+	if (providedToken && tokenMatches(expectedToken, providedToken)) return true;
+	sendJsonError(res, 401, "Unauthorized");
+	return false;
+}
+
+export function ensureCompatSensitiveRouteAuthorized(
+	req: Pick<http.IncomingMessage, "headers" | "socket">,
+	res: http.ServerResponse,
+): boolean {
+	if (!getCompatApiToken()) {
+		if (isTrustedLocalRequest(req)) return true;
+		sendJsonError(
+			res,
+			403,
+			"Sensitive endpoint requires API token authentication",
+		);
+		return false;
+	}
+	return ensureCompatApiAuthorized(req, res);
+}
+
+export async function ensureRouteAuthorized(
+	req: Pick<http.IncomingMessage, "headers" | "socket" | "method">,
+	res: http.ServerResponse,
+	_state: CompatRuntimeState,
+): Promise<boolean> {
+	return ensureCompatApiAuthorized(req, res);
+}
+
+export async function readCompatJsonBody(
+	req: http.IncomingMessage,
+	res: http.ServerResponse,
+): Promise<Record<string, unknown> | null> {
+	const preParsed = (req as { body?: unknown }).body;
+	if (preParsed && typeof preParsed === "object" && !Array.isArray(preParsed)) {
+		return preParsed as Record<string, unknown>;
+	}
+
+	const chunks: Buffer[] = [];
+	let totalBytes = 0;
+	try {
+		for await (const chunk of req) {
+			const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+			totalBytes += buf.length;
+			if (totalBytes > MAX_BODY_BYTES) {
+				req.destroy();
+				sendJsonError(res, 413, "Request body too large");
+				return null;
+			}
+			chunks.push(buf);
+		}
+	} catch {
+		sendJsonError(res, 400, "Invalid request body");
+		return null;
+	}
+
+	if (chunks.length === 0) return {};
+	try {
+		const parsed = JSON.parse(
+			Buffer.concat(chunks).toString("utf8"),
+		) as unknown;
+		if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+			sendJsonError(res, 400, "Invalid JSON body");
+			return null;
+		}
+		return parsed as Record<string, unknown>;
+	} catch {
+		sendJsonError(res, 400, "Invalid JSON body");
+		return null;
+	}
+}

package/src/routes/family-member-route.ts

@@ -0,0 +1,353 @@
+/**
+ * Voice first-run route: create a family-member voice profile.
+ *
+ * Route:
+ *   POST /v1/voice/first-run/family-member
+ *     Body (JSON):
+ *       {
+ *         audioBase64: string,   // raw base64-encoded audio (webm/wav/ogg)
+ *         durationMs: number,    // client-measured capture length
+ *         displayName: string,   // e.g. "Alex"
+ *         relationship: string,  // free-form label, e.g. "spouse", "colleague"
+ *         ownerEntityId?: string // optional — stored on the profile for edge creation
+ *       }
+ *     Response:
+ *       {
+ *         profileId: string,    // content-addressed vp_<sha> id
+ *         entityId: string,     // freshly minted UUID for the family-member entity
+ *         displayName: string,
+ *         relationship: string,
+ *         relationshipTag: "family_of"
+ *       }
+ *
+ * Audio pipeline (mirrors voice-first-run-routes.ts):
+ *   1. Decode base64 → Buffer → Float32 PCM (assumes 16 kHz, mono).
+ *   2. Encode via the fused `FusedSpeakerEncoder` → 256-dim centroid.
+ *   3. Store in VoiceProfileStore (non-OWNER entity binding).
+ *
+ * Graceful degradation: when the fused libelizainference (or its speaker ABI)
+ * is unavailable the route returns a 503. The client-side adapter
+ * (`VoiceProfilesClient`) falls back to a client-side pending profile so the UI
+ * flow is not blocked.
+ *
+ * The `family_of` relationship edge is recorded in profile metadata so a
+ * runtime-level consumer (e.g. voice-first-run-complete handler) can create
+ * the real Entity + Relationship rows after the runtime is up. The route
+ * itself does not call IAgentRuntime because HTTP route handlers in this
+ * plugin do not hold a runtime reference.
+ */
+
+import crypto from "node:crypto";
+import type * as http from "node:http";
+import path from "node:path";
+import {
+	logger,
+	readJsonBody,
+	resolveStateDir,
+	sendJson,
+	sendJsonError,
+} from "@elizaos/core";
+import { resolveFusedLibraryPath } from "../services/desktop-fused-ffi-backend-runtime.js";
+import { loadElizaInferenceFfi } from "../services/voice/ffi-bindings.js";
+import { VoiceProfileStore } from "../services/voice/profile-store.js";
+import {
+	type SpeakerEncoder,
+	SpeakerEncoderUnavailableError,
+	WESPEAKER_EMBEDDING_DIM,
+	WESPEAKER_MIN_SAMPLES,
+	WESPEAKER_RESNET34_LM_INT8_MODEL_ID,
+	WESPEAKER_SAMPLE_RATE,
+} from "../services/voice/speaker/encoder.js";
+import { FusedSpeakerEncoder } from "../services/voice/speaker/encoder-fused.js";
+
+// ---------------------------------------------------------------------------
+// Injectable test hooks (mirrors voice-first-run-routes.ts)
+// ---------------------------------------------------------------------------
+
+export type FamilyMemberEncoderFactory = () => Promise<SpeakerEncoder>;
+
+let encoderFactoryOverride: FamilyMemberEncoderFactory | null = null;
+let cachedEncoder: SpeakerEncoder | null = null;
+
+export function setFamilyMemberEncoderFactory(
+	factory: FamilyMemberEncoderFactory | null,
+): void {
+	encoderFactoryOverride = factory;
+	cachedEncoder = null;
+}
+
+let profileStoreOverride: VoiceProfileStore | null = null;
+
+export function setFamilyMemberProfileStore(
+	store: VoiceProfileStore | null,
+): void {
+	profileStoreOverride = store;
+}
+
+// ---------------------------------------------------------------------------
+// Loader helpers
+// ---------------------------------------------------------------------------
+
+async function loadEncoder(): Promise<SpeakerEncoder> {
+	if (cachedEncoder) return cachedEncoder;
+	if (encoderFactoryOverride) {
+		cachedEncoder = await encoderFactoryOverride();
+		return cachedEncoder;
+	}
+	cachedEncoder = await loadFusedSpeakerEncoder();
+	return cachedEncoder;
+}
+
+/**
+ * Load the fused speaker encoder through the `eliza_inference_speaker_*` ABI —
+ * the sole on-device speaker runtime. Probes the speaker ABI up front: a build
+ * that lacks it raises a structured `SpeakerEncoderUnavailableError` (the route
+ * surfaces a 503) rather than degrading silently. No standalone-lib fallback.
+ */
+async function loadFusedSpeakerEncoder(): Promise<SpeakerEncoder> {
+	const bundleRoot = path.join(resolveStateDir(), "voice-profiles");
+	const libPath = resolveFusedLibraryPath(bundleRoot);
+	if (!libPath) {
+		throw new SpeakerEncoderUnavailableError(
+			"library-missing",
+			"[family-member-route] fused libelizainference not found. Set $ELIZA_INFERENCE_LIBRARY (exact path) or $ELIZA_INFERENCE_LIB_DIR, or build it via packages/app-core/scripts/build-llama-cpp-mtp.mjs.",
+		);
+	}
+	const ffi = loadElizaInferenceFfi(libPath);
+	if (!FusedSpeakerEncoder.isSupported(ffi)) {
+		throw new SpeakerEncoderUnavailableError(
+			"native-missing",
+			`[family-member-route] the fused libelizainference at ${libPath} (ABI v${ffi.libraryAbiVersion}) lacks the speaker ABI (eliza_inference_speaker_supported() == 0). Rebuild with the WeSpeaker forward graph linked in.`,
+		);
+	}
+	const ctx = ffi.create(bundleRoot);
+	return FusedSpeakerEncoder.load({ ffi, ctx });
+}
+
+async function getProfileStore(): Promise<VoiceProfileStore> {
+	if (profileStoreOverride) return profileStoreOverride;
+	const store = new VoiceProfileStore({
+		rootDir: path.join(resolveStateDir(), "voice-profiles"),
+	});
+	await store.init();
+	return store;
+}
+
+// ---------------------------------------------------------------------------
+// Request body validation
+// ---------------------------------------------------------------------------
+
+interface FamilyMemberBody {
+	audioBase64: string;
+	durationMs: number;
+	displayName: string;
+	relationship: string;
+	ownerEntityId?: string | null;
+}
+
+function parseFamilyMemberBody(
+	raw: Record<string, unknown>,
+): FamilyMemberBody | string {
+	const audioBase64 =
+		typeof raw.audioBase64 === "string" ? raw.audioBase64.trim() : null;
+	if (!audioBase64) return "audioBase64 is required";
+
+	const durationMs =
+		typeof raw.durationMs === "number" && raw.durationMs > 0
+			? raw.durationMs
+			: null;
+	if (durationMs === null) return "durationMs must be a positive number";
+
+	const displayName =
+		typeof raw.displayName === "string" && raw.displayName.trim().length > 0
+			? raw.displayName.trim()
+			: null;
+	if (!displayName) return "displayName is required";
+
+	const relationship =
+		typeof raw.relationship === "string" && raw.relationship.trim().length > 0
+			? raw.relationship.trim()
+			: "family";
+
+	const ownerEntityId =
+		typeof raw.ownerEntityId === "string" && raw.ownerEntityId.trim()
+			? raw.ownerEntityId.trim()
+			: null;
+
+	return { audioBase64, durationMs, displayName, relationship, ownerEntityId };
+}
+
+// ---------------------------------------------------------------------------
+// PCM decode
+// ---------------------------------------------------------------------------
+
+function decodeBase64ToPcm(audioBase64: string): Float32Array | string {
+	let rawBuf: Buffer;
+	try {
+		rawBuf = Buffer.from(audioBase64, "base64");
+	} catch {
+		return "failed to decode audioBase64";
+	}
+	if (rawBuf.byteLength % 4 !== 0) {
+		return `PCM buffer length ${rawBuf.byteLength} is not a multiple of 4 (expected raw Float32 PCM)`;
+	}
+	const out = new Float32Array(rawBuf.byteLength / 4);
+	const view = new DataView(
+		rawBuf.buffer,
+		rawBuf.byteOffset,
+		rawBuf.byteLength,
+	);
+	for (let i = 0; i < out.length; i += 1) {
+		out[i] = view.getFloat32(i * 4, true);
+	}
+	return out;
+}
+
+// ---------------------------------------------------------------------------
+// Route handler
+// ---------------------------------------------------------------------------
+
+/** Relationship tag written to profile metadata and echoed in response. */
+export const FAMILY_OF_TAG = "family_of" as const;
+
+export interface FamilyMemberResult {
+	profileId: string;
+	entityId: string;
+	displayName: string;
+	relationship: string;
+	relationshipTag: typeof FAMILY_OF_TAG;
+	ownerEntityId: string | null;
+}
+
+/**
+ * Handle `POST /v1/voice/first-run/family-member`.
+ *
+ * Returns `true` when the request was handled (success or error response
+ * written), `false` when the path does not match this handler.
+ */
+export async function handleFamilyMemberRoute(
+	req: http.IncomingMessage,
+	res: http.ServerResponse,
+): Promise<boolean> {
+	const method = (req.method ?? "GET").toUpperCase();
+	const url = new URL(req.url ?? "/", "http://localhost");
+	const pathname = url.pathname;
+
+	if (method !== "POST" || pathname !== "/v1/voice/first-run/family-member") {
+		return false;
+	}
+
+	const raw = await readJsonBody<Record<string, unknown>>(req, res);
+	if (!raw) return true; // readJsonBody already sent a 4xx
+
+	const parsed = parseFamilyMemberBody(raw);
+	if (typeof parsed === "string") {
+		sendJsonError(res, parsed, 400);
+		return true;
+	}
+
+	const { audioBase64, durationMs, displayName, relationship, ownerEntityId } =
+		parsed;
+
+	// Decode audio.
+	const pcmOrError = decodeBase64ToPcm(audioBase64);
+	if (typeof pcmOrError === "string") {
+		sendJsonError(res, pcmOrError, 400);
+		return true;
+	}
+	const pcm = pcmOrError;
+
+	if (pcm.length < WESPEAKER_MIN_SAMPLES) {
+		sendJsonError(
+			res,
+			`capture too short: ${pcm.length} PCM samples (< ${WESPEAKER_MIN_SAMPLES} required for ${WESPEAKER_SAMPLE_RATE} Hz encoder). Minimum ~${Math.ceil((WESPEAKER_MIN_SAMPLES / WESPEAKER_SAMPLE_RATE) * 1000)}ms.`,
+			400,
+		);
+		return true;
+	}
+
+	// Encode via WeSpeaker.
+	let encoder: SpeakerEncoder;
+	try {
+		encoder = await loadEncoder();
+	} catch (err) {
+		if (err instanceof SpeakerEncoderUnavailableError) {
+			sendJsonError(res, err.message, 503);
+			return true;
+		}
+		throw err;
+	}
+
+	let centroid: Float32Array;
+	try {
+		centroid = await encoder.encode(pcm);
+	} catch (err) {
+		if (err instanceof SpeakerEncoderUnavailableError) {
+			sendJsonError(res, err.message, err.code === "invalid-input" ? 400 : 503);
+			return true;
+		}
+		throw err;
+	}
+
+	if (centroid.length !== WESPEAKER_EMBEDDING_DIM) {
+		sendJsonError(
+			res,
+			`centroid dim mismatch: ${centroid.length} != ${WESPEAKER_EMBEDDING_DIM}`,
+			500,
+		);
+		return true;
+	}
+
+	// Generate a stable entity ID for the family member.
+	const entityId = crypto.randomUUID();
+
+	// Store the profile in VoiceProfileStore.
+	const store = await getProfileStore();
+	let profile: Awaited<ReturnType<VoiceProfileStore["createProfile"]>>;
+	try {
+		profile = await store.createProfile({
+			centroid,
+			embeddingModel: WESPEAKER_RESNET34_LM_INT8_MODEL_ID,
+			entityId,
+			confidence: 0.85,
+			durationMs,
+			consent: {
+				attributionAuthorized: true,
+				synthesisAuthorized: false,
+				grantedAt: new Date().toISOString(),
+				grantedBy: ownerEntityId ?? undefined,
+			},
+			metadata: {
+				displayName,
+				relationship,
+				cohort: "family",
+				source: "first-run",
+				relationshipTag: FAMILY_OF_TAG,
+				ownerEntityId: ownerEntityId ?? null,
+			},
+		});
+	} catch (err) {
+		logger.error(
+			{ err, displayName, relationship },
+			"[family-member-route] failed to create voice profile",
+		);
+		sendJsonError(
+			res,
+			err instanceof Error ? err.message : "failed to create voice profile",
+			500,
+		);
+		return true;
+	}
+
+	const result: FamilyMemberResult = {
+		profileId: profile.profileId,
+		entityId,
+		displayName,
+		relationship,
+		relationshipTag: FAMILY_OF_TAG,
+		ownerEntityId: ownerEntityId ?? null,
+	};
+
+	sendJson(res, result);
+	return true;
+}

package/src/routes/index.ts

@@ -0,0 +1,61 @@
+/**
+ * Route-side exports for plugin-local-inference.
+ *
+ * Consumers (app-core/api/server.ts) import from
+ * `@elizaos/plugin-local-inference/routes` to mount the HTTP compat routes
+ * for model catalog, downloads, status, and chat commands.
+ */
+
+export {
+	handleVoiceProfileRoutes,
+	registerProfileInCatalog,
+	resolveDefaultProfileId,
+	type VoiceProfileCatalog,
+	type VoiceProfileCatalogEntry,
+	type VoiceProfileRouteOptions,
+} from "../services/voice/voice-profile-routes.js";
+export {
+	FAMILY_OF_TAG,
+	type FamilyMemberEncoderFactory,
+	type FamilyMemberResult,
+	handleFamilyMemberRoute,
+	setFamilyMemberEncoderFactory,
+	setFamilyMemberProfileStore,
+} from "./family-member-route.js";
+export {
+	handleLiveDiarizationRoute,
+	resetLiveDiarizationSession,
+} from "./live-diarization-route.js";
+export * from "./local-inference-asr-route.js";
+export * from "./local-inference-compat-routes.js";
+export * from "./local-inference-tts-route.js";
+export { handleNativePcmTurnRoute } from "./native-pcm-turn-route.js";
+export {
+	__resetVoiceFirstRunSessions,
+	type EncoderFactory as VoiceFirstRunEncoderFactory,
+	FIRST_RUN_SCRIPT,
+	type FirstRunScriptStep,
+	handleVoiceFirstRunRoutes,
+	setVoiceFirstRunEncoderFactory,
+	setVoiceFirstRunProfileStore,
+	setVoiceFirstRunSettingsWriter,
+} from "./voice-first-run-routes.js";
+export {
+	handleVoiceModelsRoutes,
+	resolveInstalledVersions as resolveInstalledVoiceModelVersions,
+	setVoiceModelDownloader,
+	setVoiceModelsBundleVersionForTest,
+	setVoiceModelsUpdater,
+	type VoiceModelInstallationView,
+} from "./voice-models-routes.js";
+export { voiceProfilePluginRoutes } from "./voice-profile-plugin-routes.js";
+export {
+	handleVoiceProfilesManagementRoutes,
+	setVoiceProfilesManagementStore,
+	type VoiceProfileDto,
+} from "./voice-profiles-management-routes.js";
+export {
+	handleVoiceSpeakerProfileRoutes,
+	type SpeakerProfileSummary,
+	setVoiceSpeakerProfileStore,
+} from "./voice-speaker-profile-routes.js";