@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/voice/lifecycle.ts

@@ -0,0 +1,301 @@
+/**
+ * Voice on/off state machine.
+ *
+ * Per `packages/inference/AGENTS.md` §4 + this scope's design goals,
+ * voice is OFF by default. Text + native MTP are hot; TTS, ASR, the
+ * speaker preset cache and phrase cache, the chunker, the rollback
+ * queue, the barge-in controller, and the ring buffer are NOT in RAM.
+ *
+ * Transitions are explicit. Illegal transitions throw — no
+ * "log-and-continue" (AGENTS.md §9). The transition to `voice-off`
+ * MUST issue a real page-eviction call on the TTS/ASR mmap regions
+ * (see `MmapRegionHandle.evictPages()` in `shared-resources.ts`) so
+ * the OS can reclaim those pages.
+ *
+ *   ┌──────────┐ start()  ┌──────────────┐ armed   ┌──────────┐
+ *   │ voice-off│─────────▶│ voice-arming │────────▶│ voice-on │
+ *   └──────────┘          └──────────────┘         └──────────┘
+ *        ▲                       │ start fails           │ stop()
+ *        │ disarmed              ▼                       ▼
+ *  ┌──────────────────┐    ┌──────────────┐  ┌────────────────────┐
+ *  │ voice-disarming  │◀───│ voice-error  │  │  voice-disarming   │
+ *  └──────────────────┘    └──────────────┘  └────────────────────┘
+ *        │                                            │
+ *        └────────────────── disarmed ◀───────────────┘
+ *
+ * `voice-error` is terminal until `reset()` is called. There is no
+ * automatic retry: a missing kernel, mmap fail, or RAM-pressure
+ * refusal MUST surface to the caller.
+ */
+
+import type {
+	KernelSet,
+	MmapRegionHandle,
+	MtpDraftHandle,
+	RefCountedResource,
+	SchedulerSlot,
+	SharedResourceRegistry,
+	SharedTokenizer,
+} from "./shared-resources";
+
+/**
+ * Discriminated union — never widened to `string`. Each state may carry
+ * payload (the `armed` payload includes the loaded mmap regions so the
+ * disarm path can call `evictPages()` on them).
+ */
+export type VoiceLifecycleState =
+	| { readonly kind: "voice-off" }
+	| { readonly kind: "voice-arming" }
+	| { readonly kind: "voice-on"; readonly resources: ArmedResources }
+	| { readonly kind: "voice-disarming"; readonly resources: ArmedResources }
+	| { readonly kind: "voice-error"; readonly error: VoiceLifecycleError };
+
+/**
+ * Resources held while voice is armed. Released in reverse order on
+ * disarm; the mmap regions get an explicit `evictPages()` call before
+ * `release()` so the OS reclaims pages even if the FFI keeps the file
+ * descriptor open for the next re-arm.
+ */
+export interface ArmedResources {
+	readonly tts: MmapRegionHandle;
+	readonly asr: MmapRegionHandle;
+	/** Speaker preset + phrase cache — kept in a small LRU after disarm. */
+	readonly voiceCaches: RefCountedResource;
+	/** Voice-specific scheduler nodes (chunker, rollback, ring buffer, barge-in). */
+	readonly voiceSchedulerNodes: RefCountedResource;
+}
+
+/**
+ * Resources held while text is up. Acquired by the engine when the
+ * bundle is activated; voice piggy-backs on these without re-loading.
+ */
+export interface TextResources {
+	readonly tokenizer: SharedTokenizer;
+	readonly textWeights: MmapRegionHandle;
+	readonly kernels: KernelSet;
+	readonly scheduler: SchedulerSlot;
+	readonly mtp: MtpDraftHandle;
+}
+
+/**
+ * Structured failure surfaced to the caller. Never a generic `Error` —
+ * the caller (engine + UI) needs to distinguish RAM pressure from a
+ * missing kernel from a manifest mismatch (AGENTS.md §3).
+ */
+export class VoiceLifecycleError extends Error {
+	readonly code:
+		| "ram-pressure"
+		| "mmap-fail"
+		| "kernel-missing"
+		| "illegal-transition"
+		| "arm-failed"
+		| "disarm-failed";
+
+	constructor(code: VoiceLifecycleError["code"], message: string) {
+		super(message);
+		this.name = "VoiceLifecycleError";
+		this.code = code;
+	}
+}
+
+/**
+ * Loader functions injected at construction. Splitting these out keeps
+ * `VoiceLifecycle` independent of FFI specifics — the engine wires real
+ * loaders at runtime; tests inject mocks. Each loader MUST throw on
+ * failure (AGENTS.md §3 — no silent fallback).
+ */
+export interface VoiceLifecycleLoaders {
+	loadTtsRegion(): Promise<MmapRegionHandle>;
+	loadAsrRegion(): Promise<MmapRegionHandle>;
+	loadVoiceCaches(): Promise<RefCountedResource>;
+	loadVoiceSchedulerNodes(): Promise<RefCountedResource>;
+}
+
+export interface VoiceLifecycleEvents {
+	onTransition?(prev: VoiceLifecycleState, next: VoiceLifecycleState): void;
+}
+
+export class VoiceLifecycle {
+	private state: VoiceLifecycleState = { kind: "voice-off" };
+	private readonly registry: SharedResourceRegistry;
+	private readonly loaders: VoiceLifecycleLoaders;
+	private readonly events: VoiceLifecycleEvents;
+
+	constructor(args: {
+		registry: SharedResourceRegistry;
+		loaders: VoiceLifecycleLoaders;
+		events?: VoiceLifecycleEvents;
+	}) {
+		this.registry = args.registry;
+		this.loaders = args.loaders;
+		this.events = args.events ?? {};
+	}
+
+	current(): VoiceLifecycleState {
+		return this.state;
+	}
+
+	/**
+	 * Transition `voice-off → voice-arming → voice-on`. Loads TTS + ASR
+	 * mmap regions, voice caches, voice scheduler nodes. Each load throws
+	 * on failure; a thrown loader transitions the state to `voice-error`
+	 * and re-throws so the caller sees the structured cause. No partial
+	 * arm: either all four resources are held or none are.
+	 */
+	async arm(): Promise<ArmedResources> {
+		if (this.state.kind !== "voice-off") {
+			throw new VoiceLifecycleError(
+				"illegal-transition",
+				`[voice-lifecycle] arm() called in state ${this.state.kind} — must be voice-off`,
+			);
+		}
+		this.transition({ kind: "voice-arming" });
+
+		let tts: MmapRegionHandle | null = null;
+		let asr: MmapRegionHandle | null = null;
+		let voiceCaches: RefCountedResource | null = null;
+		let voiceSchedulerNodes: RefCountedResource | null = null;
+		try {
+			tts = this.registry.acquire(await this.loaders.loadTtsRegion());
+			asr = this.registry.acquire(await this.loaders.loadAsrRegion());
+			voiceCaches = this.registry.acquire(await this.loaders.loadVoiceCaches());
+			voiceSchedulerNodes = this.registry.acquire(
+				await this.loaders.loadVoiceSchedulerNodes(),
+			);
+		} catch (err) {
+			// Roll back partial acquisitions before surfacing the error so the
+			// registry doesn't leak refs on a failed arm. Evict heavy mmap
+			// regions before release; release() only drops the refcount and may
+			// intentionally keep file descriptors alive for the next re-page.
+			await Promise.allSettled([
+				tts?.evictPages() ?? Promise.resolve(),
+				asr?.evictPages() ?? Promise.resolve(),
+			]);
+			const rollback: Array<RefCountedResource | null> = [
+				voiceSchedulerNodes,
+				voiceCaches,
+				asr,
+				tts,
+			];
+			for (const res of rollback) {
+				if (res) await this.registry.release(res.id);
+			}
+			const lifecycleErr = toLifecycleError("arm-failed", err);
+			this.transition({ kind: "voice-error", error: lifecycleErr });
+			throw lifecycleErr;
+		}
+
+		const resources: ArmedResources = {
+			tts,
+			asr,
+			voiceCaches,
+			voiceSchedulerNodes,
+		};
+		this.transition({ kind: "voice-on", resources });
+		return resources;
+	}
+
+	/**
+	 * Transition `voice-on → voice-disarming → voice-off`. Calls
+	 * `evictPages()` on the TTS + ASR mmap regions before releasing them
+	 * so the OS reclaims the pages even if another consumer keeps the
+	 * file descriptor open. The voice caches stay in the registry as
+	 * tiny (KB-scale) entries — only the heavy mmap regions get evicted.
+	 */
+	async disarm(): Promise<void> {
+		if (this.state.kind !== "voice-on") {
+			throw new VoiceLifecycleError(
+				"illegal-transition",
+				`[voice-lifecycle] disarm() called in state ${this.state.kind} — must be voice-on`,
+			);
+		}
+		const resources = this.state.resources;
+		this.transition({ kind: "voice-disarming", resources });
+
+		let evictionFailure: unknown = null;
+		// Eviction first — the mmap region is still mapped, the kernel can
+		// still drop the pages. If eviction fails we still proceed to
+		// release; the failure is captured and re-thrown after release so
+		// the registry stays consistent.
+		//
+		// `evictPages()` on production handles wires through to the
+		// `libelizainference` FFI (`ffi.mmapEvict(ctx, "tts" | "asr")`,
+		// declared in `tools/omnivoice/include/eliza-inference-ffi.h`).
+		// The fused build implements it by tearing down the OmniVoice /
+		// ASR model context (`ov_free` + `eliza_free_asr`), which lets
+		// the llama.cpp / OmniVoice destructors run their own
+		// platform-appropriate unmap (`munmap` on POSIX, `UnmapViewOfFile`
+		// on Windows). The TS layer is platform-agnostic — all
+		// platform-specific eviction lives in the C ABI. The stub library
+		// returns ELIZA_ERR_NOT_IMPLEMENTED, which the binding raises as
+		// `VoiceLifecycleError({code:"kernel-missing"})` — this method
+		// captures it and re-classifies as `disarm-failed` after release
+		// runs (so registry refs don't leak on a bad eviction).
+		const evictResults = await Promise.allSettled([
+			resources.tts.evictPages(),
+			resources.asr.evictPages(),
+		]);
+		for (const r of evictResults) {
+			if (r.status === "rejected" && evictionFailure === null) {
+				evictionFailure = r.reason;
+			}
+		}
+
+		// Release in reverse acquisition order.
+		await this.registry.release(resources.voiceSchedulerNodes.id);
+		await this.registry.release(resources.voiceCaches.id);
+		await this.registry.release(resources.asr.id);
+		await this.registry.release(resources.tts.id);
+
+		if (evictionFailure !== null) {
+			const err = toLifecycleError("disarm-failed", evictionFailure);
+			this.transition({ kind: "voice-error", error: err });
+			throw err;
+		}
+		this.transition({ kind: "voice-off" });
+	}
+
+	/**
+	 * Reset from `voice-error` back to `voice-off`. Required because
+	 * `voice-error` is terminal — the engine must explicitly acknowledge
+	 * the failure before the user can re-attempt voice. There is no
+	 * automatic retry path.
+	 */
+	reset(): void {
+		if (this.state.kind !== "voice-error") {
+			throw new VoiceLifecycleError(
+				"illegal-transition",
+				`[voice-lifecycle] reset() called in state ${this.state.kind} — must be voice-error`,
+			);
+		}
+		this.transition({ kind: "voice-off" });
+	}
+
+	private transition(next: VoiceLifecycleState): void {
+		const prev = this.state;
+		this.state = next;
+		this.events.onTransition?.(prev, next);
+	}
+}
+
+function toLifecycleError(
+	fallbackCode: VoiceLifecycleError["code"],
+	err: unknown,
+): VoiceLifecycleError {
+	if (err instanceof VoiceLifecycleError) return err;
+	const message = err instanceof Error ? err.message : String(err);
+	// Heuristic mapping of common platform-level signals into the
+	// structured codes documented above. The lifecycle never fabricates
+	// a code it didn't receive evidence for — anything that doesn't match
+	// one of these falls back to the caller-provided code.
+	if (/ENOMEM|out of memory|RAM/i.test(message)) {
+		return new VoiceLifecycleError("ram-pressure", message);
+	}
+	if (/mmap|MAP_FAILED/i.test(message)) {
+		return new VoiceLifecycleError("mmap-fail", message);
+	}
+	if (/kernel|missing kernel/i.test(message)) {
+		return new VoiceLifecycleError("kernel-missing", message);
+	}
+	return new VoiceLifecycleError(fallbackCode, message);
+}

package/src/services/voice/live-diarization-session.echo.test.ts

@@ -0,0 +1,232 @@
+/**
+ * LiveDiarizationSession echo-reference wiring (#9455/#9583).
+ *
+ * Proves the agent-side AEC seam without the fused FFI: the session decodes
+ * agent-playback (far-end) frames into its alignment buffer, and the
+ * `echoReference` read seam (the closure handed to AudioFrameConsumer) returns
+ * the time-aligned far-end slice — zero-filled until playback is pushed, and
+ * reset on barge-in. The model-heavy path (real NLMS cancellation over the
+ * fused VAD/encoder/diarizer) is covered by the host smoke harness.
+ */
+
+import { describe, expect, it } from "vitest";
+import type { AudioFrameEvent } from "./audio-frame-consumer.js";
+import { platformPlaybackDelaySamples } from "./echo-delay.js";
+import {
+	LiveDiarizationSession,
+	type RuntimeEventSink,
+} from "./live-diarization-session.js";
+
+const SAMPLE_RATE = 16_000;
+
+function fakeRuntime(): RuntimeEventSink {
+	return { emitEvent: async () => {} } as unknown as RuntimeEventSink;
+}
+
+/** Build a well-formed playback frame from Float32 [-1,1] samples. */
+function playbackFrame(
+	samples: Float32Array,
+	frameIndex: number,
+): AudioFrameEvent {
+	const buf = Buffer.alloc(samples.length * 2);
+	for (let i = 0; i < samples.length; i += 1) {
+		const clamped = Math.max(-1, Math.min(1, samples[i] ?? 0));
+		buf.writeInt16LE(Math.round(clamped * 32_768) | 0, i * 2);
+	}
+	return {
+		pcm16: buf.toString("base64"),
+		sampleRate: SAMPLE_RATE,
+		channels: 1,
+		samples: samples.length,
+		rms: 0,
+		timestamp: frameIndex * 20,
+		frameIndex,
+	};
+}
+
+/** A deterministic ramp in [-0.5, 0.5]. */
+function ramp(n: number): Float32Array {
+	const out = new Float32Array(n);
+	for (let i = 0; i < n; i += 1) out[i] = i / n - 0.5;
+	return out;
+}
+
+function noise(n: number): Float32Array {
+	const out = new Float32Array(n);
+	let seed = 0x12345678;
+	for (let i = 0; i < n; i += 1) {
+		seed = (seed * 1_664_525 + 1_013_904_223) >>> 0;
+		out[i] = ((seed / 0xffffffff) * 2 - 1) * 0.6;
+	}
+	return out;
+}
+
+describe("LiveDiarizationSession echo reference", () => {
+	it("returns a zero far-end reference before any playback is pushed", () => {
+		const session = new LiveDiarizationSession(fakeRuntime());
+		const ref = session.echoReferenceFrame(0, 320);
+		expect(ref).toHaveLength(320);
+		expect(ref.every((v) => v === 0)).toBe(true);
+	});
+
+	it("aligns playback by frame timestamp as the far-end (delay 0)", () => {
+		const session = new LiveDiarizationSession(fakeRuntime());
+		const playback = ramp(320);
+		session.pushPlayback([playbackFrame(playback, 0)]);
+
+		const ref = session.echoReferenceFrame(0, 320);
+		expect(ref).toHaveLength(320);
+		// Not zero-filled — the canceller now has a real far-end to cancel.
+		expect(ref.some((v) => v !== 0)).toBe(true);
+		// s16 round-trip is exact to ~1/32768; assert close alignment to the
+		// pushed ramp, not silence.
+		for (let i = 0; i < 320; i += 1) {
+			expect(Math.abs((ref[i] ?? 0) - (playback[i] ?? 0))).toBeLessThan(1e-3);
+		}
+	});
+
+	it("returns the trailing window when asked for fewer samples than pushed", () => {
+		const session = new LiveDiarizationSession(fakeRuntime());
+		const playback = ramp(640);
+		session.pushPlayback([playbackFrame(playback, 0)]);
+
+		const ref = session.echoReferenceFrame(20, 320);
+		expect(ref).toHaveLength(320);
+		// The aligned window is the LAST 320 of the 640 pushed (delay 0).
+		for (let i = 0; i < 320; i += 1) {
+			expect(Math.abs((ref[i] ?? 0) - (playback[320 + i] ?? 0))).toBeLessThan(
+				1e-3,
+			);
+		}
+	});
+
+	it("resetPlayback drops buffered far-end (barge-in / playback stop)", () => {
+		const session = new LiveDiarizationSession(fakeRuntime());
+		session.pushPlayback([playbackFrame(ramp(320), 0)]);
+		expect(session.echoReferenceFrame(0, 320).some((v) => v !== 0)).toBe(true);
+
+		session.resetPlayback();
+		expect(session.echoReferenceFrame(0, 320).every((v) => v === 0)).toBe(true);
+	});
+
+	it("zero-fills natural gaps between playback frames", () => {
+		const session = new LiveDiarizationSession(fakeRuntime());
+		const first = ramp(320);
+		const later = ramp(320);
+		session.pushPlayback([playbackFrame(first, 0), playbackFrame(later, 5)]);
+
+		expect(session.echoReferenceFrame(0, 320).some((v) => v !== 0)).toBe(true);
+		expect(session.echoReferenceFrame(40, 320).every((v) => v === 0)).toBe(
+			true,
+		);
+		expect(session.echoReferenceFrame(100, 320).some((v) => v !== 0)).toBe(
+			true,
+		);
+	});
+
+	it("self-calibrates playback-to-mic delay from correlated echo", () => {
+		const session = new LiveDiarizationSession(fakeRuntime());
+		const frameSamples = 320;
+		const totalSamples = 16_000;
+		const delaySamples = 240;
+		const playback = noise(totalSamples);
+
+		for (let offset = 0; offset < totalSamples; offset += frameSamples) {
+			session.pushPlayback([
+				playbackFrame(
+					playback.slice(offset, offset + frameSamples),
+					offset / frameSamples,
+				),
+			]);
+		}
+
+		for (let offset = 0; offset < totalSamples; offset += frameSamples) {
+			const near = new Float32Array(frameSamples);
+			for (let i = 0; i < frameSamples; i += 1) {
+				near[i] = playback[offset + i - delaySamples] ?? 0;
+			}
+			session.observeForDelayCalibration(near, (offset / SAMPLE_RATE) * 1000);
+			if (session.aecDelayState().calibrated) break;
+		}
+
+		const state = session.aecDelayState();
+		expect(state.calibrated).toBe(true);
+		expect(Math.abs(state.delaySamples - delaySamples)).toBeLessThanOrEqual(1);
+		expect(state.confidence).toBeGreaterThan(0.95);
+	});
+
+	it("seeds the echo delay from the platform default when ELIZA_VOICE_ECHO_DELAY_MS=auto", () => {
+		const prev = process.env.ELIZA_VOICE_ECHO_DELAY_MS;
+		process.env.ELIZA_VOICE_ECHO_DELAY_MS = "auto";
+		try {
+			const session = new LiveDiarizationSession(fakeRuntime());
+			// Seed comes straight from the per-platform table (#9583); runtime
+			// calibration would refine it later, but at construction it equals the
+			// platform default for the host the test runs on.
+			expect(session.aecDelayState().delaySamples).toBe(
+				platformPlaybackDelaySamples(process.platform, SAMPLE_RATE),
+			);
+		} finally {
+			if (prev === undefined) delete process.env.ELIZA_VOICE_ECHO_DELAY_MS;
+			else process.env.ELIZA_VOICE_ECHO_DELAY_MS = prev;
+		}
+	});
+
+	it("resolves the ELIZA_PLATFORM id (ios) for the auto seed, not the host's darwin seed (#9583)", () => {
+		const prevDelay = process.env.ELIZA_VOICE_ECHO_DELAY_MS;
+		const prevPlatform = process.env.ELIZA_PLATFORM;
+		process.env.ELIZA_VOICE_ECHO_DELAY_MS = "auto";
+		process.env.ELIZA_PLATFORM = "ios";
+		try {
+			const session = new LiveDiarizationSession(fakeRuntime());
+			// The mobile shell reports ELIZA_PLATFORM=ios even though the host's
+			// process.platform is darwin. The auto seed must follow the device id
+			// (#9653 ios table = 400 samples @16kHz), NOT the darwin host seed
+			// (320 samples) — otherwise the deliberate per-platform seeds are
+			// unreachable on device.
+			expect(session.aecDelayState().delaySamples).toBe(
+				platformPlaybackDelaySamples("ios", SAMPLE_RATE),
+			);
+			expect(session.aecDelayState().delaySamples).toBe(400);
+			expect(session.aecDelayState().delaySamples).not.toBe(
+				platformPlaybackDelaySamples("darwin", SAMPLE_RATE),
+			);
+		} finally {
+			if (prevDelay === undefined) delete process.env.ELIZA_VOICE_ECHO_DELAY_MS;
+			else process.env.ELIZA_VOICE_ECHO_DELAY_MS = prevDelay;
+			if (prevPlatform === undefined) delete process.env.ELIZA_PLATFORM;
+			else process.env.ELIZA_PLATFORM = prevPlatform;
+		}
+	});
+
+	it("resolves the ELIZA_PLATFORM id (android) for the auto seed (#9583)", () => {
+		const prevDelay = process.env.ELIZA_VOICE_ECHO_DELAY_MS;
+		const prevPlatform = process.env.ELIZA_PLATFORM;
+		process.env.ELIZA_VOICE_ECHO_DELAY_MS = "auto";
+		process.env.ELIZA_PLATFORM = "android";
+		try {
+			const session = new LiveDiarizationSession(fakeRuntime());
+			expect(session.aecDelayState().delaySamples).toBe(
+				platformPlaybackDelaySamples("android", SAMPLE_RATE),
+			);
+			expect(session.aecDelayState().delaySamples).toBe(720);
+		} finally {
+			if (prevDelay === undefined) delete process.env.ELIZA_VOICE_ECHO_DELAY_MS;
+			else process.env.ELIZA_VOICE_ECHO_DELAY_MS = prevDelay;
+			if (prevPlatform === undefined) delete process.env.ELIZA_PLATFORM;
+			else process.env.ELIZA_PLATFORM = prevPlatform;
+		}
+	});
+
+	it("defaults the echo delay seed to 0 when no override is set", () => {
+		const prev = process.env.ELIZA_VOICE_ECHO_DELAY_MS;
+		delete process.env.ELIZA_VOICE_ECHO_DELAY_MS;
+		try {
+			const session = new LiveDiarizationSession(fakeRuntime());
+			expect(session.aecDelayState().delaySamples).toBe(0);
+		} finally {
+			if (prev === undefined) delete process.env.ELIZA_VOICE_ECHO_DELAY_MS;
+			else process.env.ELIZA_VOICE_ECHO_DELAY_MS = prev;
+		}
+	});
+});