@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/services/voice/lifecycle.ts

@@ -0,0 +1,301 @@
+/**
+ * Voice on/off state machine.
+ *
+ * Per `packages/inference/AGENTS.md` §4 + this scope's design goals,
+ * voice is OFF by default. Text + native MTP are hot; TTS, ASR, the
+ * speaker preset cache and phrase cache, the chunker, the rollback
+ * queue, the barge-in controller, and the ring buffer are NOT in RAM.
+ *
+ * Transitions are explicit. Illegal transitions throw — no
+ * "log-and-continue" (AGENTS.md §9). The transition to `voice-off`
+ * MUST issue a real page-eviction call on the TTS/ASR mmap regions
+ * (see `MmapRegionHandle.evictPages()` in `shared-resources.ts`) so
+ * the OS can reclaim those pages.
+ *
+ *   ┌──────────┐ start()  ┌──────────────┐ armed   ┌──────────┐
+ *   │ voice-off│─────────▶│ voice-arming │────────▶│ voice-on │
+ *   └──────────┘          └──────────────┘         └──────────┘
+ *        ▲                       │ start fails           │ stop()
+ *        │ disarmed              ▼                       ▼
+ *  ┌──────────────────┐    ┌──────────────┐  ┌────────────────────┐
+ *  │ voice-disarming  │◀───│ voice-error  │  │  voice-disarming   │
+ *  └──────────────────┘    └──────────────┘  └────────────────────┘
+ *        │                                            │
+ *        └────────────────── disarmed ◀───────────────┘
+ *
+ * `voice-error` is terminal until `reset()` is called. There is no
+ * automatic retry: a missing kernel, mmap fail, or RAM-pressure
+ * refusal MUST surface to the caller.
+ */
+
+import type {
+	KernelSet,
+	MmapRegionHandle,
+	MtpDraftHandle,
+	RefCountedResource,
+	SchedulerSlot,
+	SharedResourceRegistry,
+	SharedTokenizer,
+} from "./shared-resources";
+
+/**
+ * Discriminated union — never widened to `string`. Each state may carry
+ * payload (the `armed` payload includes the loaded mmap regions so the
+ * disarm path can call `evictPages()` on them).
+ */
+export type VoiceLifecycleState =
+	| { readonly kind: "voice-off" }
+	| { readonly kind: "voice-arming" }
+	| { readonly kind: "voice-on"; readonly resources: ArmedResources }
+	| { readonly kind: "voice-disarming"; readonly resources: ArmedResources }
+	| { readonly kind: "voice-error"; readonly error: VoiceLifecycleError };
+
+/**
+ * Resources held while voice is armed. Released in reverse order on
+ * disarm; the mmap regions get an explicit `evictPages()` call before
+ * `release()` so the OS reclaims pages even if the FFI keeps the file
+ * descriptor open for the next re-arm.
+ */
+export interface ArmedResources {
+	readonly tts: MmapRegionHandle;
+	readonly asr: MmapRegionHandle;
+	/** Speaker preset + phrase cache — kept in a small LRU after disarm. */
+	readonly voiceCaches: RefCountedResource;
+	/** Voice-specific scheduler nodes (chunker, rollback, ring buffer, barge-in). */
+	readonly voiceSchedulerNodes: RefCountedResource;
+}
+
+/**
+ * Resources held while text is up. Acquired by the engine when the
+ * bundle is activated; voice piggy-backs on these without re-loading.
+ */
+export interface TextResources {
+	readonly tokenizer: SharedTokenizer;
+	readonly textWeights: MmapRegionHandle;
+	readonly kernels: KernelSet;
+	readonly scheduler: SchedulerSlot;
+	readonly mtp: MtpDraftHandle;
+}
+
+/**
+ * Structured failure surfaced to the caller. Never a generic `Error` —
+ * the caller (engine + UI) needs to distinguish RAM pressure from a
+ * missing kernel from a manifest mismatch (AGENTS.md §3).
+ */
+export class VoiceLifecycleError extends Error {
+	readonly code:
+		| "ram-pressure"
+		| "mmap-fail"
+		| "kernel-missing"
+		| "illegal-transition"
+		| "arm-failed"
+		| "disarm-failed";
+
+	constructor(code: VoiceLifecycleError["code"], message: string) {
+		super(message);
+		this.name = "VoiceLifecycleError";
+		this.code = code;
+	}
+}
+
+/**
+ * Loader functions injected at construction. Splitting these out keeps
+ * `VoiceLifecycle` independent of FFI specifics — the engine wires real
+ * loaders at runtime; tests inject mocks. Each loader MUST throw on
+ * failure (AGENTS.md §3 — no silent fallback).
+ */
+export interface VoiceLifecycleLoaders {
+	loadTtsRegion(): Promise<MmapRegionHandle>;
+	loadAsrRegion(): Promise<MmapRegionHandle>;
+	loadVoiceCaches(): Promise<RefCountedResource>;
+	loadVoiceSchedulerNodes(): Promise<RefCountedResource>;
+}
+
+export interface VoiceLifecycleEvents {
+	onTransition?(prev: VoiceLifecycleState, next: VoiceLifecycleState): void;
+}
+
+export class VoiceLifecycle {
+	private state: VoiceLifecycleState = { kind: "voice-off" };
+	private readonly registry: SharedResourceRegistry;
+	private readonly loaders: VoiceLifecycleLoaders;
+	private readonly events: VoiceLifecycleEvents;
+
+	constructor(args: {
+		registry: SharedResourceRegistry;
+		loaders: VoiceLifecycleLoaders;
+		events?: VoiceLifecycleEvents;
+	}) {
+		this.registry = args.registry;
+		this.loaders = args.loaders;
+		this.events = args.events ?? {};
+	}
+
+	current(): VoiceLifecycleState {
+		return this.state;
+	}
+
+	/**
+	 * Transition `voice-off → voice-arming → voice-on`. Loads TTS + ASR
+	 * mmap regions, voice caches, voice scheduler nodes. Each load throws
+	 * on failure; a thrown loader transitions the state to `voice-error`
+	 * and re-throws so the caller sees the structured cause. No partial
+	 * arm: either all four resources are held or none are.
+	 */
+	async arm(): Promise<ArmedResources> {
+		if (this.state.kind !== "voice-off") {
+			throw new VoiceLifecycleError(
+				"illegal-transition",
+				`[voice-lifecycle] arm() called in state ${this.state.kind} — must be voice-off`,
+			);
+		}
+		this.transition({ kind: "voice-arming" });
+
+		let tts: MmapRegionHandle | null = null;
+		let asr: MmapRegionHandle | null = null;
+		let voiceCaches: RefCountedResource | null = null;
+		let voiceSchedulerNodes: RefCountedResource | null = null;
+		try {
+			tts = this.registry.acquire(await this.loaders.loadTtsRegion());
+			asr = this.registry.acquire(await this.loaders.loadAsrRegion());
+			voiceCaches = this.registry.acquire(await this.loaders.loadVoiceCaches());
+			voiceSchedulerNodes = this.registry.acquire(
+				await this.loaders.loadVoiceSchedulerNodes(),
+			);
+		} catch (err) {
+			// Roll back partial acquisitions before surfacing the error so the
+			// registry doesn't leak refs on a failed arm. Evict heavy mmap
+			// regions before release; release() only drops the refcount and may
+			// intentionally keep file descriptors alive for the next re-page.
+			await Promise.allSettled([
+				tts?.evictPages() ?? Promise.resolve(),
+				asr?.evictPages() ?? Promise.resolve(),
+			]);
+			const rollback: Array<RefCountedResource | null> = [
+				voiceSchedulerNodes,
+				voiceCaches,
+				asr,
+				tts,
+			];
+			for (const res of rollback) {
+				if (res) await this.registry.release(res.id);
+			}
+			const lifecycleErr = toLifecycleError("arm-failed", err);
+			this.transition({ kind: "voice-error", error: lifecycleErr });
+			throw lifecycleErr;
+		}
+
+		const resources: ArmedResources = {
+			tts,
+			asr,
+			voiceCaches,
+			voiceSchedulerNodes,
+		};
+		this.transition({ kind: "voice-on", resources });
+		return resources;
+	}
+
+	/**
+	 * Transition `voice-on → voice-disarming → voice-off`. Calls
+	 * `evictPages()` on the TTS + ASR mmap regions before releasing them
+	 * so the OS reclaims the pages even if another consumer keeps the
+	 * file descriptor open. The voice caches stay in the registry as
+	 * tiny (KB-scale) entries — only the heavy mmap regions get evicted.
+	 */
+	async disarm(): Promise<void> {
+		if (this.state.kind !== "voice-on") {
+			throw new VoiceLifecycleError(
+				"illegal-transition",
+				`[voice-lifecycle] disarm() called in state ${this.state.kind} — must be voice-on`,
+			);
+		}
+		const resources = this.state.resources;
+		this.transition({ kind: "voice-disarming", resources });
+
+		let evictionFailure: unknown = null;
+		// Eviction first — the mmap region is still mapped, the kernel can
+		// still drop the pages. If eviction fails we still proceed to
+		// release; the failure is captured and re-thrown after release so
+		// the registry stays consistent.
+		//
+		// `evictPages()` on production handles wires through to the
+		// `libelizainference` FFI (`ffi.mmapEvict(ctx, "tts" | "asr")`,
+		// declared in `tools/omnivoice/include/eliza-inference-ffi.h`).
+		// The fused build implements it by tearing down the OmniVoice /
+		// ASR model context (`ov_free` + `eliza_free_asr`), which lets
+		// the llama.cpp / OmniVoice destructors run their own
+		// platform-appropriate unmap (`munmap` on POSIX, `UnmapViewOfFile`
+		// on Windows). The TS layer is platform-agnostic — all
+		// platform-specific eviction lives in the C ABI. The stub library
+		// returns ELIZA_ERR_NOT_IMPLEMENTED, which the binding raises as
+		// `VoiceLifecycleError({code:"kernel-missing"})` — this method
+		// captures it and re-classifies as `disarm-failed` after release
+		// runs (so registry refs don't leak on a bad eviction).
+		const evictResults = await Promise.allSettled([
+			resources.tts.evictPages(),
+			resources.asr.evictPages(),
+		]);
+		for (const r of evictResults) {
+			if (r.status === "rejected" && evictionFailure === null) {
+				evictionFailure = r.reason;
+			}
+		}
+
+		// Release in reverse acquisition order.
+		await this.registry.release(resources.voiceSchedulerNodes.id);
+		await this.registry.release(resources.voiceCaches.id);
+		await this.registry.release(resources.asr.id);
+		await this.registry.release(resources.tts.id);
+
+		if (evictionFailure !== null) {
+			const err = toLifecycleError("disarm-failed", evictionFailure);
+			this.transition({ kind: "voice-error", error: err });
+			throw err;
+		}
+		this.transition({ kind: "voice-off" });
+	}
+
+	/**
+	 * Reset from `voice-error` back to `voice-off`. Required because
+	 * `voice-error` is terminal — the engine must explicitly acknowledge
+	 * the failure before the user can re-attempt voice. There is no
+	 * automatic retry path.
+	 */
+	reset(): void {
+		if (this.state.kind !== "voice-error") {
+			throw new VoiceLifecycleError(
+				"illegal-transition",
+				`[voice-lifecycle] reset() called in state ${this.state.kind} — must be voice-error`,
+			);
+		}
+		this.transition({ kind: "voice-off" });
+	}
+
+	private transition(next: VoiceLifecycleState): void {
+		const prev = this.state;
+		this.state = next;
+		this.events.onTransition?.(prev, next);
+	}
+}
+
+function toLifecycleError(
+	fallbackCode: VoiceLifecycleError["code"],
+	err: unknown,
+): VoiceLifecycleError {
+	if (err instanceof VoiceLifecycleError) return err;
+	const message = err instanceof Error ? err.message : String(err);
+	// Heuristic mapping of common platform-level signals into the
+	// structured codes documented above. The lifecycle never fabricates
+	// a code it didn't receive evidence for — anything that doesn't match
+	// one of these falls back to the caller-provided code.
+	if (/ENOMEM|out of memory|RAM/i.test(message)) {
+		return new VoiceLifecycleError("ram-pressure", message);
+	}
+	if (/mmap|MAP_FAILED/i.test(message)) {
+		return new VoiceLifecycleError("mmap-fail", message);
+	}
+	if (/kernel|missing kernel/i.test(message)) {
+		return new VoiceLifecycleError("kernel-missing", message);
+	}
+	return new VoiceLifecycleError(fallbackCode, message);
+}

package/src/services/voice/live-diarization-session.d.ts

@@ -0,0 +1,96 @@
+/**
+ * Live on-device diarization session — the agent-process owner of an
+ * {@link AudioFrameConsumer} wired to the REAL fused VAD / encoder / diarizer /
+ * attribution stack.
+ *
+ * The Android `audioFrame` PCM stream is produced in the Capacitor WebView
+ * (JS renderer) but the voice FFI runs in the embedded bun agent process. The
+ * agent's `/api/voice/audio-frames` route pumps batched frames into the single
+ * session this module owns, where the consumer segments turns, runs
+ * diarization + speaker attribution, and emits VOICE_TURN_OBSERVED.
+ *
+ * This module is the agent-side mirror of the host smoke harness
+ * (`packages/app-core/scripts/voice-attribution-smoke.ts`): same real models,
+ * same consumer, fed live frames over HTTP instead of a WAV.
+ *
+ * Single fused engine: VAD, the WeSpeaker speaker encoder, and the pyannote
+ * diarizer all run through the ONE fused `libelizainference` handle via its
+ * `eliza_inference_vad_*` / `_speaker_*` / `_diariz_*` ABI (the user directive:
+ * no separate bun:ffi-musl libs). Resolution:
+ *   - fused lib: `$ELIZA_INFERENCE_LIBRARY` (exact) or `$ELIZA_INFERENCE_LIB_DIR`
+ *     (dir) — both exported by ElizaAgentService on Android to the app
+ *     nativeLibraryDir.
+ *   - context bundle root: `$ELIZA_VOICE_MODEL_DIR` (the same dir the GGUFs
+ *     live under); the fused runtime resolves the per-model GGUFs from there.
+ */
+import { type AudioFrameEvent, type RuntimeEventSink } from "./audio-frame-consumer.js";
+export type { RuntimeEventSink } from "./audio-frame-consumer.js";
+export interface LiveDiarizationStatus {
+    /** True once the consumer + real fused deps are loaded and accepting frames. */
+    ready: boolean;
+    /** Resolved fused-library path (null when it could not be resolved). */
+    libs: {
+        fusedInference: string | null;
+    };
+    /** Resolved context-bundle dir for the fused runtime. */
+    models: {
+        dir: string;
+    };
+    /** Frames received from the WebView across this session. */
+    framesReceived: number;
+    /** Frames dropped at the decode boundary. */
+    framesDropped: number;
+    /** Turns segmented + attributed so far. */
+    turnsObserved: number;
+    /** The most recent attributed turns (capped), for device-evidence reads. */
+    recentTurns: LiveDiarizationTurnSummary[];
+    /** Populated only when readiness failed — the precise blocker. */
+    error?: string;
+}
+/** A compact, JSON-safe summary of one attributed turn (no PCM/embeddings). */
+export interface LiveDiarizationTurnSummary {
+    turnId: string;
+    startedAtMs: number;
+    endedAtMs: number;
+    samples: number;
+    durationMs: number;
+    hasSpeaker: boolean;
+    speakerEntityId: string | null;
+    speakerConfidence: number | null;
+    segments: number;
+    agentShouldSpeak: boolean | null;
+    nextSpeaker: string | null;
+}
+/**
+ * Owns the single live diarization consumer for the agent process. Built
+ * lazily on first frame batch so it does not load voice models at boot.
+ */
+export declare class LiveDiarizationSession {
+    private readonly runtime;
+    private consumer;
+    private ffi;
+    private ctx;
+    private encoder;
+    private diarizer;
+    private vad;
+    private building;
+    private framesReceived;
+    private turnsObserved;
+    private readonly recentTurns;
+    private resolvedLibPath;
+    private buildError;
+    constructor(runtime: RuntimeEventSink);
+    /** Ensure the real-deps consumer exists; idempotent + concurrency-safe. */
+    private ensureBuilt;
+    private build;
+    private recordTurn;
+    /** Feed a batch of WebView-captured frames; resolves once VAD has processed them. */
+    ingest(frames: AudioFrameEvent[]): Promise<void>;
+    /** Flush any open segment (call on stopAudioFrames) and await attribution. */
+    flush(): Promise<void>;
+    /** Build (if needed) and report status — the device-evidence read. */
+    status(): Promise<LiveDiarizationStatus>;
+    /** Release native handles + listeners. */
+    close(): Promise<void>;
+}
+//# sourceMappingURL=live-diarization-session.d.ts.map

package/src/services/voice/live-diarization-session.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"live-diarization-session.d.ts","sourceRoot":"","sources":["live-diarization-session.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAKH,OAAO,EAIN,KAAK,eAAe,EACpB,KAAK,gBAAgB,EACrB,MAAM,2BAA2B,CAAC;AAYnC,YAAY,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAqClE,MAAM,WAAW,qBAAqB;IACrC,gFAAgF;IAChF,KAAK,EAAE,OAAO,CAAC;IACf,wEAAwE;IACxE,IAAI,EAAE;QACL,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;KAC9B,CAAC;IACF,yDAAyD;IACzD,MAAM,EAAE;QACP,GAAG,EAAE,MAAM,CAAC;KACZ,CAAC;IACF,4DAA4D;IAC5D,cAAc,EAAE,MAAM,CAAC;IACvB,6CAA6C;IAC7C,aAAa,EAAE,MAAM,CAAC;IACtB,2CAA2C;IAC3C,aAAa,EAAE,MAAM,CAAC;IACtB,4EAA4E;IAC5E,WAAW,EAAE,0BAA0B,EAAE,CAAC;IAC1C,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;CACf;AAED,+EAA+E;AAC/E,MAAM,WAAW,0BAA0B;IAC1C,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,OAAO,CAAC;IACpB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,OAAO,GAAG,IAAI,CAAC;IACjC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AAID;;;GAGG;AACH,qBAAa,sBAAsB;IActB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAbpC,OAAO,CAAC,QAAQ,CAAmC;IACnD,OAAO,CAAC,GAAG,CAAkC;IAC7C,OAAO,CAAC,GAAG,CAA4C;IACvD,OAAO,CAAC,OAAO,CAAoC;IACnD,OAAO,CAAC,QAAQ,CAA8B;IAC9C,OAAO,CAAC,GAAG,CAA8B;IACzC,OAAO,CAAC,QAAQ,CAA8B;IAC9C,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoC;IAChE,OAAO,CAAC,eAAe,CAAuB;IAC9C,OAAO,CAAC,UAAU,CAAuB;gBAEZ,OAAO,EAAE,gBAAgB;IAEtD,2EAA2E;IAC3E,OAAO,CAAC,WAAW;YAUL,KAAK;IAmEnB,OAAO,CAAC,UAAU;IAoBlB,qFAAqF;IAC/E,MAAM,CAAC,MAAM,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAStD,8EAA8E;IACxE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,sEAAsE;IAChE,MAAM,IAAI,OAAO,CAAC,qBAAqB,CAAC;IAkB9C,0CAA0C;IACpC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAW5B"}

package/src/services/voice/live-diarization-session.ts

@@ -0,0 +1,289 @@
+/**
+ * Live on-device diarization session — the agent-process owner of an
+ * {@link AudioFrameConsumer} wired to the REAL fused VAD / encoder / diarizer /
+ * attribution stack.
+ *
+ * The Android `audioFrame` PCM stream is produced in the Capacitor WebView
+ * (JS renderer) but the voice FFI runs in the embedded bun agent process. The
+ * agent's `/api/voice/audio-frames` route pumps batched frames into the single
+ * session this module owns, where the consumer segments turns, runs
+ * diarization + speaker attribution, and emits VOICE_TURN_OBSERVED.
+ *
+ * This module is the agent-side mirror of the host smoke harness
+ * (`packages/app-core/scripts/voice-attribution-smoke.ts`): same real models,
+ * same consumer, fed live frames over HTTP instead of a WAV.
+ *
+ * Single fused engine: VAD, the WeSpeaker speaker encoder, and the pyannote
+ * diarizer all run through the ONE fused `libelizainference` handle via its
+ * `eliza_inference_vad_*` / `_speaker_*` / `_diariz_*` ABI (the user directive:
+ * no separate bun:ffi-musl libs). Resolution:
+ *   - fused lib: `$ELIZA_INFERENCE_LIBRARY` (exact) or `$ELIZA_INFERENCE_LIB_DIR`
+ *     (dir) — both exported by ElizaAgentService on Android to the app
+ *     nativeLibraryDir.
+ *   - context bundle root: `$ELIZA_VOICE_MODEL_DIR` (the same dir the GGUFs
+ *     live under); the fused runtime resolves the per-model GGUFs from there.
+ */
+
+import { existsSync } from "node:fs";
+import path from "node:path";
+import { resolveStateDir } from "@elizaos/core";
+import {
+	type AttributedTurn,
+	AudioFrameConsumer,
+	type AudioFrameConsumerConfig,
+	type AudioFrameEvent,
+	type RuntimeEventSink,
+} from "./audio-frame-consumer.js";
+import type {
+	ElizaInferenceContextHandle,
+	ElizaInferenceFfi,
+} from "./ffi-bindings.js";
+import { loadElizaInferenceFfi } from "./ffi-bindings.js";
+import { VoiceProfileStore } from "./profile-store.js";
+import { VoiceAttributionPipeline } from "./speaker/attribution-pipeline.js";
+import { FusedDiarizer } from "./speaker/diarizer-fused.js";
+import { FusedSpeakerEncoder } from "./speaker/encoder-fused.js";
+import { GgmlSileroVad, VadDetector } from "./vad.js";
+
+export type { RuntimeEventSink } from "./audio-frame-consumer.js";
+
+/** Resolve the on-device voice-model directory (env override wins). Doubles as
+ *  the fused context bundle root — the runtime resolves per-model GGUFs from it. */
+function voiceModelDir(): string {
+	const override = process.env.ELIZA_VOICE_MODEL_DIR?.trim();
+	if (override) return override;
+	return path.join(resolveStateDir(process.env), "models", "voice");
+}
+
+/** Candidate filenames for the fused library on this platform. */
+function fusedLibraryFilenames(): string[] {
+	if (process.platform === "darwin") return ["libelizainference.dylib"];
+	if (process.platform === "win32") {
+		return ["elizainference.dll", "libelizainference.dll"];
+	}
+	return ["libelizainference.so"];
+}
+
+/**
+ * Resolve the fused `libelizainference` path from the environment. Returns
+ * `null` when neither an exact path nor a containing dir yields a file —
+ * the session then surfaces that as a structured build error.
+ */
+function resolveFusedLibrary(): string | null {
+	const exact = process.env.ELIZA_INFERENCE_LIBRARY?.trim();
+	if (exact && existsSync(exact)) return exact;
+	const dir = process.env.ELIZA_INFERENCE_LIB_DIR?.trim();
+	if (dir) {
+		for (const name of fusedLibraryFilenames()) {
+			const candidate = path.join(dir, name);
+			if (existsSync(candidate)) return candidate;
+		}
+	}
+	return null;
+}
+
+export interface LiveDiarizationStatus {
+	/** True once the consumer + real fused deps are loaded and accepting frames. */
+	ready: boolean;
+	/** Resolved fused-library path (null when it could not be resolved). */
+	libs: {
+		fusedInference: string | null;
+	};
+	/** Resolved context-bundle dir for the fused runtime. */
+	models: {
+		dir: string;
+	};
+	/** Frames received from the WebView across this session. */
+	framesReceived: number;
+	/** Frames dropped at the decode boundary. */
+	framesDropped: number;
+	/** Turns segmented + attributed so far. */
+	turnsObserved: number;
+	/** The most recent attributed turns (capped), for device-evidence reads. */
+	recentTurns: LiveDiarizationTurnSummary[];
+	/** Populated only when readiness failed — the precise blocker. */
+	error?: string;
+}
+
+/** A compact, JSON-safe summary of one attributed turn (no PCM/embeddings). */
+export interface LiveDiarizationTurnSummary {
+	turnId: string;
+	startedAtMs: number;
+	endedAtMs: number;
+	samples: number;
+	durationMs: number;
+	hasSpeaker: boolean;
+	speakerEntityId: string | null;
+	speakerConfidence: number | null;
+	segments: number;
+	agentShouldSpeak: boolean | null;
+	nextSpeaker: string | null;
+}
+
+const MAX_RECENT_TURNS = 20;
+
+/**
+ * Owns the single live diarization consumer for the agent process. Built
+ * lazily on first frame batch so it does not load voice models at boot.
+ */
+export class LiveDiarizationSession {
+	private consumer: AudioFrameConsumer | null = null;
+	private ffi: ElizaInferenceFfi | null = null;
+	private ctx: ElizaInferenceContextHandle | null = null;
+	private encoder: FusedSpeakerEncoder | null = null;
+	private diarizer: FusedDiarizer | null = null;
+	private vad: GgmlSileroVad | null = null;
+	private building: Promise<void> | null = null;
+	private framesReceived = 0;
+	private turnsObserved = 0;
+	private readonly recentTurns: LiveDiarizationTurnSummary[] = [];
+	private resolvedLibPath: string | null = null;
+	private buildError: string | null = null;
+
+	constructor(private readonly runtime: RuntimeEventSink) {}
+
+	/** Ensure the real-deps consumer exists; idempotent + concurrency-safe. */
+	private ensureBuilt(): Promise<void> {
+		if (this.consumer) return Promise.resolve();
+		if (this.building) return this.building;
+		this.building = this.build().catch((err) => {
+			this.buildError = err instanceof Error ? err.message : String(err);
+			throw err;
+		});
+		return this.building;
+	}
+
+	private async build(): Promise<void> {
+		const dir = voiceModelDir();
+		const libPath = resolveFusedLibrary();
+		if (!libPath) {
+			throw new Error(
+				`fused libelizainference not found on device. Set $ELIZA_INFERENCE_LIBRARY (exact path) or $ELIZA_INFERENCE_LIB_DIR (containing one of ${fusedLibraryFilenames().join(", ")}).`,
+			);
+		}
+		this.resolvedLibPath = libPath;
+		const ffi = loadElizaInferenceFfi(libPath);
+		this.ffi = ffi;
+		// One context anchored at the voice-model dir; the fused runtime resolves
+		// the VAD / speaker / diarizer GGUFs from it.
+		const ctx = ffi.create(dir);
+		this.ctx = ctx;
+
+		if (!GgmlSileroVad.isSupported(ffi)) {
+			throw new Error(
+				"fused libelizainference does not export the VAD ABI (eliza_inference_vad_supported() == 0). Rebuild with the fused voice runtime linked in.",
+			);
+		}
+		if (!FusedSpeakerEncoder.isSupported(ffi)) {
+			throw new Error(
+				"fused libelizainference does not export the speaker ABI (eliza_inference_speaker_supported() == 0).",
+			);
+		}
+		if (!FusedDiarizer.isSupported(ffi)) {
+			throw new Error(
+				"fused libelizainference does not export the diarizer ABI (eliza_inference_diariz_supported() == 0).",
+			);
+		}
+
+		const vad = await GgmlSileroVad.load({ ffi, ctx });
+		this.vad = vad;
+		const detector = new VadDetector(vad, {
+			onsetThreshold: 0.5,
+			pauseHangoverMs: 120,
+			endHangoverMs: 500,
+			minSpeechMs: 250,
+		});
+		const encoder = await FusedSpeakerEncoder.load({ ffi, ctx });
+		this.encoder = encoder;
+		const diarizer = await FusedDiarizer.load({ ffi, ctx });
+		this.diarizer = diarizer;
+		const store = new VoiceProfileStore({
+			rootDir: path.join(resolveStateDir(process.env), "voice-profiles"),
+		});
+		await store.init();
+
+		const pipeline = new VoiceAttributionPipeline({
+			encoder,
+			diarizer,
+			profileStore: store,
+		});
+		const config: AudioFrameConsumerConfig = {
+			source: { kind: "local_mic", deviceId: "android-audioframe" },
+			preRollSeconds: 0.3,
+			maxTurnSeconds: 30,
+		};
+		const consumer = new AudioFrameConsumer(
+			{ vad: detector, pipeline, runtime: this.runtime },
+			config,
+		);
+		consumer.onTurn((turn) => this.recordTurn(turn));
+		this.consumer = consumer;
+	}
+
+	private recordTurn(turn: AttributedTurn): void {
+		this.turnsObserved += 1;
+		const speaker = turn.output.primarySpeaker;
+		const summary: LiveDiarizationTurnSummary = {
+			turnId: turn.turnId,
+			startedAtMs: turn.startedAtMs,
+			endedAtMs: turn.endedAtMs,
+			samples: turn.samples,
+			durationMs: Math.round((turn.samples / 16_000) * 1000),
+			hasSpeaker: speaker != null,
+			speakerEntityId: speaker?.entityId ?? null,
+			speakerConfidence: speaker?.confidence ?? null,
+			segments: turn.output.segments.length,
+			agentShouldSpeak: turn.signal.agentShouldSpeak,
+			nextSpeaker: turn.signal.nextSpeaker ?? null,
+		};
+		this.recentTurns.push(summary);
+		if (this.recentTurns.length > MAX_RECENT_TURNS) this.recentTurns.shift();
+	}
+
+	/** Feed a batch of WebView-captured frames; resolves once VAD has processed them. */
+	async ingest(frames: AudioFrameEvent[]): Promise<void> {
+		await this.ensureBuilt();
+		if (!this.consumer) return;
+		for (const frame of frames) {
+			this.framesReceived += 1;
+			await this.consumer.onAudioFrame(frame);
+		}
+	}
+
+	/** Flush any open segment (call on stopAudioFrames) and await attribution. */
+	async flush(): Promise<void> {
+		if (this.consumer) await this.consumer.flush();
+	}
+
+	/** Build (if needed) and report status — the device-evidence read. */
+	async status(): Promise<LiveDiarizationStatus> {
+		try {
+			await this.ensureBuilt();
+		} catch {
+			// Surface the blocker in the status payload rather than throwing.
+		}
+		return {
+			ready: this.consumer != null,
+			libs: { fusedInference: this.resolvedLibPath },
+			models: { dir: voiceModelDir() },
+			framesReceived: this.framesReceived,
+			framesDropped: this.consumer?.droppedFrames ?? 0,
+			turnsObserved: this.turnsObserved,
+			recentTurns: [...this.recentTurns],
+			...(this.buildError ? { error: this.buildError } : {}),
+		};
+	}
+
+	/** Release native handles + listeners. */
+	async close(): Promise<void> {
+		await this.consumer?.close();
+		await this.encoder?.dispose();
+		await this.diarizer?.dispose();
+		this.vad?.close();
+		if (this.ffi && this.ctx !== null) this.ffi.destroy(this.ctx);
+		this.ffi?.close();
+		this.consumer = null;
+		this.ffi = null;
+		this.ctx = null;
+	}
+}