@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.2

package/src/runtime/voice-entity-binding.ts

@@ -0,0 +1,328 @@
+/**
+ * Voice ⇄ entity binding seam (producer + round-trip consumer).
+ *
+ * Producer (`emitVoiceTurnObserved`): emit `VOICE_TURN_OBSERVED` so a
+ * merge-engine owner (plugin-lifeops) can fold the recognized voice turn
+ * into the entity/relationship graph. The voice-profile store is owned
+ * here; the entity graph is owned there; the only shared surface is the
+ * core event seam — neither plugin imports the other.
+ *
+ * Consumer (`handleVoiceEntityBound`): when the merge engine reports a
+ * binding via `VOICE_ENTITY_BOUND`, persist the resulting `entityId` onto
+ * every profile in that imprint cluster (`VoiceProfileStore.bindEntity`).
+ * This is the runtime path that was missing in issue #8234 — without it a
+ * profile's `entityId` stayed `null` and recognized speakers never reached
+ * the relationship graph.
+ */
+
+import crypto from "node:crypto";
+import path from "node:path";
+import {
+	EventType,
+	type IAgentRuntime,
+	logger,
+	resolveStateDir,
+	type VoiceEntityBoundPayload,
+} from "@elizaos/core";
+import type {
+	VoiceNextSpeaker,
+	VoiceTurnSignal,
+} from "../services/voice/eot-classifier.js";
+import { VoiceProfileStore } from "../services/voice/profile-store.js";
+import type { VoiceAttributionOutput } from "../services/voice/speaker/attribution-pipeline.js";
+
+// ---------------------------------------------------------------------------
+// Store wiring (injectable for tests, mirrors the route handlers)
+// ---------------------------------------------------------------------------
+
+let storeOverride: VoiceProfileStore | null = null;
+
+export function setVoiceEntityBindingStore(
+	store: VoiceProfileStore | null,
+): void {
+	storeOverride = store;
+}
+
+export async function getVoiceProfileStore(): Promise<VoiceProfileStore> {
+	if (storeOverride) return storeOverride;
+	const store = new VoiceProfileStore({
+		rootDir: path.join(resolveStateDir(), "voice-profiles"),
+	});
+	await store.init();
+	return store;
+}
+
+// ---------------------------------------------------------------------------
+// Producer
+// ---------------------------------------------------------------------------
+
+export interface EmitVoiceTurnObservedArgs {
+	/** Stable utterance id; a random one is minted when omitted. */
+	turnId?: string;
+	/** Recognized text (drives name/partner-claim extraction downstream). */
+	text: string;
+	/** Imprint cluster id from the voice-profile store. */
+	imprintClusterId: string;
+	/** Confidence of the imprint match (0..1). */
+	matchConfidence: number;
+	/** Entity the imprint already resolved to, or `null`/omitted when unbound. */
+	matchedEntityId?: string | null;
+	/** True when the OWNER spoke this turn. */
+	isOwner?: boolean;
+	/** ISO timestamp; defaults to now. */
+	observedAt?: string;
+}
+
+/**
+ * Emit `VOICE_TURN_OBSERVED`. No-op in effect when no merge-engine plugin
+ * is loaded (the event simply has no handler). `emitEvent` awaits every
+ * handler, so by the time this resolves the binding round-trip (including
+ * `VOICE_ENTITY_BOUND` → profile persist) has completed.
+ */
+export async function emitVoiceTurnObserved(
+	runtime: IAgentRuntime,
+	args: EmitVoiceTurnObservedArgs,
+): Promise<void> {
+	await runtime.emitEvent(EventType.VOICE_TURN_OBSERVED, {
+		runtime,
+		turnId: args.turnId ?? `vturn_${crypto.randomUUID()}`,
+		text: args.text,
+		imprintClusterId: args.imprintClusterId,
+		matchConfidence: args.matchConfidence,
+		matchedEntityId: args.matchedEntityId ?? null,
+		observedAt: args.observedAt ?? new Date().toISOString(),
+		...(args.isOwner !== undefined ? { isOwner: args.isOwner } : {}),
+	});
+}
+
+// ---------------------------------------------------------------------------
+// Live-turn attribution → VOICE_TURN_OBSERVED + voiceTurnSignal (gating)
+// ---------------------------------------------------------------------------
+
+/** Server SUPPRESS threshold for EOT — below this reads as "user still talking". */
+const SERVER_EOT_SUPPRESS_THRESHOLD = 0.4;
+/** Only a CONFIDENT bystander attribution is allowed to silence a turn. */
+const BYSTANDER_SUPPRESS_CONFIDENCE = 0.7;
+
+export interface HandleLiveVoiceAttributionOptions {
+	/**
+	 * Entity id the agent treats as the device owner / primary enrolled
+	 * speaker. A turn attributed to this entity is always allowed to speak.
+	 */
+	ownerEntityId?: string | null;
+	/**
+	 * Entity ids the agent answers to without a wake word (owner + enrolled
+	 * household members). A confident bystander is anyone attributed to an
+	 * entity NOT in this set.
+	 */
+	knownSpeakerEntityIds?: readonly string[];
+	/**
+	 * The EOT-based turn signal the turn-controller already computed for this
+	 * turn (from `eot-classifier` / `turn-controller`). The speaker decision is
+	 * folded into it. When omitted, a neutral base is synthesized from
+	 * `endOfTurnProbability` (default 0.5 — "unknown", fail open).
+	 */
+	baseSignal?: VoiceTurnSignal;
+	/** P(turn complete) when no `baseSignal` is supplied (default 0.5). */
+	endOfTurnProbability?: number;
+	/** True when a wake word fired within the recent listen window. */
+	wakeWordActive?: boolean;
+	/**
+	 * The ASR transcript for this turn, joined from the streaming-ASR path. When
+	 * provided it rides on `VOICE_TURN_OBSERVED` (and the turn signal) so the
+	 * merge engine's name/partner extraction (`VoiceObserver.ingestTurn`) runs
+	 * from LIVE audio — previously this was hardcoded `""`, so live recognition
+	 * could identify *who* spoke but never *what* they said (#8786). Diarization-
+	 * only callers (audio-frame path) leave it unset; the in-process voice engine
+	 * (which has both ASR + diarization) passes the real transcript.
+	 */
+	transcript?: string;
+}
+
+/**
+ * Resolve owner / enrolled state for the attributed primary speaker.
+ *
+ * `isOwner` is `entityId === ownerEntityId`; "enrolled" is owner OR an entity
+ * id present in `knownSpeakerEntityIds`. An unbound speaker (`entityId == null`)
+ * is neither — it can never be a "confident bystander" (fail open).
+ */
+function resolveSpeakerStanding(
+	output: VoiceAttributionOutput,
+	opts: HandleLiveVoiceAttributionOptions,
+): {
+	entityId: string | null;
+	confidence: number;
+	isOwner: boolean;
+	enrolled: boolean;
+} {
+	const speaker = output.primarySpeaker;
+	const entityId = speaker?.entityId ?? output.observation?.entityId ?? null;
+	const confidence = speaker?.confidence ?? output.observation?.confidence ?? 0;
+	const ownerEntityId = opts.ownerEntityId ?? null;
+	const isOwner = entityId !== null && entityId === ownerEntityId;
+	const known = new Set<string>(opts.knownSpeakerEntityIds ?? []);
+	const enrolled = isOwner || (entityId !== null && known.has(entityId));
+	return { entityId, confidence, isOwner, enrolled };
+}
+
+/**
+ * Compose the EOT base signal with the live speaker decision.
+ *
+ * Mirrors `packages/ui/src/voice/voice-turn-signal.ts buildVoiceTurnSignal`
+ * (the transcript-only producer) on the audio-frame side: a CONFIDENT bystander
+ * who did NOT say the wake word is cross-talk → suppress. A wake word is an
+ * explicit address → always speak. Uncertain attribution never silences a real
+ * turn. The server gate `core.voice_turn_signal` reads the returned object.
+ */
+function foldSpeakerIntoSignal(
+	base: VoiceTurnSignal,
+	standing: {
+		entityId: string | null;
+		confidence: number;
+		isOwner: boolean;
+		enrolled: boolean;
+	},
+	opts: HandleLiveVoiceAttributionOptions,
+): VoiceTurnSignal {
+	let agentShouldSpeak = base.agentShouldSpeak !== false;
+
+	const confidentBystander =
+		!standing.enrolled &&
+		standing.entityId !== null &&
+		standing.confidence >= BYSTANDER_SUPPRESS_CONFIDENCE;
+	if (agentShouldSpeak && opts.wakeWordActive !== true && confidentBystander) {
+		agentShouldSpeak = false;
+	}
+
+	// Wake word overrides bystander doubt — the user deliberately summoned us.
+	if (opts.wakeWordActive === true) agentShouldSpeak = true;
+
+	const eot = base.endOfTurnProbability;
+	const nextSpeaker: VoiceNextSpeaker = !agentShouldSpeak
+		? "user"
+		: eot < SERVER_EOT_SUPPRESS_THRESHOLD
+			? "user"
+			: "agent";
+
+	const source = opts.wakeWordActive
+		? "voice-bridge+wakeword"
+		: "voice-bridge+diarization";
+
+	return {
+		endOfTurnProbability: eot,
+		nextSpeaker,
+		agentShouldSpeak,
+		source: "custom",
+		transcript: base.transcript,
+		...(base.model ? { model: base.model } : {}),
+		...(base.latencyMs !== undefined ? { latencyMs: base.latencyMs } : {}),
+		// Stash the human-readable provenance so traces show the fold source even
+		// though the typed `source` enum stays "custom".
+		metadata: { provenance: source },
+	} as VoiceTurnSignal & { metadata: { provenance: string } };
+}
+
+/**
+ * Handle a live per-turn attribution result. This is the single automatic seam
+ * the engine bridge calls from its `onAttribution` path: any caller that wires a
+ * `profileStore` gets diarization-driven gating for free.
+ *
+ * 1. Emits `VOICE_TURN_OBSERVED` when the turn produced a profile observation
+ *    (so the merge engine can fold the recognized speaker into the entity
+ *    graph and round-trip the binding back onto the profile).
+ * 2. Composes the EOT-based turn signal with the speaker decision and stamps it
+ *    onto `output.turn.metadata.voiceTurnSignal`, which the chat-view producer
+ *    forwards to the server gate verbatim.
+ *
+ * Returns the composed signal (also written onto the turn metadata in place).
+ * Never throws on the emit path — observation emission is best-effort and is
+ * logged, never propagated, so an attribution turn never crashes a voice turn.
+ */
+export async function handleLiveVoiceAttribution(
+	runtime: IAgentRuntime,
+	output: VoiceAttributionOutput,
+	opts: HandleLiveVoiceAttributionOptions = {},
+): Promise<VoiceTurnSignal> {
+	const standing = resolveSpeakerStanding(output, opts);
+	// Carry the real ASR transcript when the caller joined it (in-process engine);
+	// fall back to a base-signal transcript, else "" for diarization-only callers.
+	const transcript = opts.transcript ?? opts.baseSignal?.transcript ?? "";
+
+	if (output.observation) {
+		const obs = output.observation;
+		try {
+			await emitVoiceTurnObserved(runtime, {
+				turnId: output.turnId,
+				text: transcript,
+				imprintClusterId: obs.imprintClusterId,
+				matchConfidence: obs.confidence,
+				matchedEntityId: obs.entityId,
+				isOwner: standing.isOwner,
+			});
+		} catch (err) {
+			logger.warn(
+				{
+					turnId: output.turnId,
+					imprintClusterId: obs.imprintClusterId,
+					error: err instanceof Error ? err.message : String(err),
+				},
+				"[local-inference] VOICE_TURN_OBSERVED emit failed during live attribution",
+			);
+		}
+	}
+
+	const base: VoiceTurnSignal = opts.baseSignal ?? {
+		endOfTurnProbability: opts.endOfTurnProbability ?? 0.5,
+		nextSpeaker: "unknown",
+		agentShouldSpeak: null,
+		source: "custom",
+		transcript,
+	};
+
+	const signal = foldSpeakerIntoSignal(base, standing, opts);
+
+	const turn = output.turn;
+	turn.metadata = { ...(turn.metadata ?? {}), voiceTurnSignal: signal };
+
+	return signal;
+}
+
+// ---------------------------------------------------------------------------
+// Consumer
+// ---------------------------------------------------------------------------
+
+/**
+ * Handler for `VOICE_ENTITY_BOUND`. Persists `entityId` onto every profile
+ * in the cluster that is not already bound to it. Returns nothing (the
+ * `EventHandler` contract); the bound count is logged.
+ */
+export async function handleVoiceEntityBound(
+	payload: VoiceEntityBoundPayload,
+): Promise<void> {
+	const store = await getVoiceProfileStore();
+	const records = await store.list();
+	const targets = records.filter(
+		(r) =>
+			r.imprintClusterId === payload.imprintClusterId &&
+			r.entityId !== payload.entityId,
+	);
+	let bound = 0;
+	for (const record of targets) {
+		const updated = await store.bindEntity({
+			profileId: record.profileId,
+			entityId: payload.entityId,
+			...(payload.displayName ? { label: payload.displayName } : {}),
+		});
+		if (updated) bound += 1;
+	}
+	if (bound > 0) {
+		logger.info(
+			{
+				imprintClusterId: payload.imprintClusterId,
+				entityId: payload.entityId,
+				bound,
+			},
+			"[local-inference] persisted voice→entity binding onto profile(s)",
+		);
+	}
+}

package/src/services/README.md

@@ -0,0 +1,71 @@
+# `services/local-inference/` (app-core, server-side)
+
+This is the **server-side** local-inference service used by the agent
+runtime (`@elizaos/app-core`). It owns:
+
+- KV-cache slot management (`cache-bridge.ts`, `mtp-cache-flow.test.ts`).
+- llama-server lifecycle (`ffi-streaming-backend.ts`, `mtp-doctor.ts`,
+  `llama-server-metrics.ts`, `session-pool.ts`).
+- Conversation registry (`conversation-registry.ts`).
+- Backend dispatch (`backend.ts`, `engine.ts`, `handler-registry.ts`,
+  `router-handler.ts`, `service.ts`).
+- Provider snapshot, hardware probe, model catalog, recommendation, and
+  download orchestration (`providers.ts`, `hardware.ts`, `catalog.ts`,
+  `recommendation.ts`, `downloader.ts`, `assignments.ts`,
+  `bundled-models.ts`, `external-scanner.ts`, `hf-search.ts`,
+  `registry.ts`, `paths.ts`, `routing-policy.ts`).
+
+## Server / client split
+
+The UI client mirror lives in
+[`packages/ui/src/services/local-inference/`](../../../../ui/src/services/local-inference/README.md).
+That mirror exists because UI code (panels, hooks, the iOS/Android local
+agent kernel) needs access to the **catalog**, **recommendation**, and
+the **type contract** for status payloads, but must not pull in the
+server runtime (KV cache, llama-server lifecycle, conversation
+registry).
+
+Some files are byte-identical between the two trees and have identical
+semantics. Those have been extracted to `@elizaos/shared/local-inference`
+and the local files in this directory are thin re-exports:
+
+- `paths.ts` — re-exports `localInferenceRoot` etc. from
+  `@elizaos/shared`.
+- `routing-preferences.ts` — re-exports `readRoutingPreferences` etc.
+  from `@elizaos/shared`.
+- `verify.ts` — re-exports `verifyInstalledModel` etc. from
+  `@elizaos/shared`.
+- `types.ts` — re-exports `AgentModelSlot`, `InstalledModel`,
+  `ModelAssignments`, `TextGenerationSlot`, and `AGENT_MODEL_SLOTS` from
+  `@elizaos/shared`. Server-only types (MTP kernel metadata,
+  `LocalRuntimeOptimizations`, `loadedCacheTypeK`/`...V`/`GpuLayers`,
+  etc.) remain declared in this file because the UI public client has
+  no consumer for them.
+
+## What stays a twin (and why)
+
+These files exist in both `packages/app-core` and `packages/ui` and are
+intentionally **not bundled:
+
+- `catalog.ts` — server adds `contextLength`, `optimizations.requiresKernel`,
+  and MTP drafter variants that the UI public catalog does not surface.
+- `recommendation.ts` — server has kernel-availability filtering
+  (`recommendation.test.ts` covers MTP gating) that depends on
+  server-only `LocalRuntimeOptimizations.requiresKernel`.
+- `active-model.ts` — server resolves load args against the loader's
+  KV-cache type / GPU-layer overrides (server-only types).
+- `device-bridge.ts` — server forwards `promptCacheKey` from the runtime
+  cache plan.
+- `ffi-streaming-backend.ts` — server owns the full llama-server lifecycle, the
+  in-process binding fallback, and metrics scraping.
+- `mtp-doctor.ts` — uses tokenizer parity catalog metadata that only
+  the server catalog declares.
+- `engine.ts`, `handler-registry.ts`, `hardware.ts`, `index.ts`,
+  `providers.ts`, `router-handler.ts`, `service.ts` — server-side
+  superset; the UI mirror keeps a slim subset of the same surface for
+  type-only / catalog-only consumers.
+
+If a twin pair becomes byte-identical with identical semantics, extract
+it to `packages/shared/src/local-inference/` and replace both copies
+with a thin re-export, the same way `paths.ts` / `verify.ts` /
+`routing-preferences.ts` were handled.

package/src/services/__tests__/backend-selector.test.ts

@@ -0,0 +1,101 @@
+import { describe, expect, it } from "vitest";
+
+import { readBackendEnvOverride, selectBackend } from "../backend-selector";
+
+describe("selectBackend", () => {
+	it("forces ffi-streaming on mobile when FFI is supported", () => {
+		expect(
+			selectBackend({
+				platform: "mobile",
+				ffiSupported: true,
+			}),
+		).toBe("ffi-streaming");
+	});
+
+	it("throws on mobile when FFI is missing", () => {
+		expect(() =>
+			selectBackend({
+				platform: "mobile",
+				ffiSupported: false,
+			}),
+		).toThrow(/streaming-LLM FFI symbols/);
+	});
+
+	it("desktop defaults to ffi-streaming when FFI is supported", () => {
+		expect(
+			selectBackend({
+				platform: "desktop",
+				ffiSupported: true,
+			}),
+		).toBe("ffi-streaming");
+	});
+
+	it("throws on desktop when FFI symbols are absent", () => {
+		expect(() =>
+			selectBackend({
+				platform: "desktop",
+				ffiSupported: false,
+			}),
+		).toThrow(/streaming-LLM FFI symbols/);
+	});
+
+	it("envOverride=ffi wins on desktop when supported", () => {
+		expect(
+			selectBackend({
+				platform: "desktop",
+				ffiSupported: true,
+				envOverride: "ffi",
+			}),
+		).toBe("ffi-streaming");
+	});
+
+	it("envOverride=ffi without ffiSupported throws", () => {
+		expect(() =>
+			selectBackend({
+				platform: "desktop",
+				ffiSupported: false,
+				envOverride: "ffi",
+			}),
+		).toThrow(/does not export the streaming-LLM symbols/);
+	});
+
+	it("envOverride=auto on desktop with FFI keeps the new default (ffi-streaming)", () => {
+		expect(
+			selectBackend({
+				platform: "desktop",
+				ffiSupported: true,
+				envOverride: "auto",
+			}),
+		).toBe("ffi-streaming");
+	});
+});
+
+describe("readBackendEnvOverride", () => {
+	it("returns auto for explicit auto", () => {
+		expect(readBackendEnvOverride({ ELIZA_INFERENCE_BACKEND: "auto" })).toBe(
+			"auto",
+		);
+	});
+
+	it("normalizes ffi aliases and ignores server aliases", () => {
+		expect(readBackendEnvOverride({ ELIZA_INFERENCE_BACKEND: "FFI" })).toBe(
+			"ffi",
+		);
+		expect(
+			readBackendEnvOverride({ ELIZA_INFERENCE_BACKEND: "ffi-streaming" }),
+		).toBe("ffi");
+		expect(
+			readBackendEnvOverride({ ELIZA_INFERENCE_BACKEND: "http-server" }),
+		).toBeNull();
+		expect(readBackendEnvOverride({ ELIZA_INFERENCE_BACKEND: "server" })).toBe(
+			null,
+		);
+	});
+
+	it("returns null for unset / unknown", () => {
+		expect(readBackendEnvOverride({})).toBeNull();
+		expect(
+			readBackendEnvOverride({ ELIZA_INFERENCE_BACKEND: "foo" }),
+		).toBeNull();
+	});
+});