@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/voice/kokoro/types.ts

@@ -0,0 +1,95 @@
+/**
+ * Public types for the Kokoro-82M TTS backend (Apache-2.0, hexgrad/Kokoro-82M
+ * upstream).
+ *
+ * Kokoro is a small (~82M-param) StyleTTS-2 derivative that ships with a set
+ * of "voice packs" — pre-baked 256-dim style vectors (one .bin per voice).
+ * Adding voices is a cheap (~512KB) extra download.
+ */
+
+/** Canonical voice-pack id. Convention: `<lang>_<name>` (af_bella, am_michael). */
+export type KokoroVoiceId = string;
+
+/** One bundled voice pack — small fp32 style tensor on disk. */
+export interface KokoroVoicePack {
+	/** `af_bella`, `af_sarah`, `am_michael`, ... */
+	id: KokoroVoiceId;
+	/** Human-readable name shown in UI. */
+	displayName: string;
+	/** Two-letter language tag (`a` = American English, `b` = British English, etc. per Kokoro convention). */
+	lang: string;
+	/** Filename inside the voices/ directory, relative to the Kokoro model root. */
+	file: string;
+	/** Style-vector dim (256 for v1.0). */
+	dim: number;
+	/** Genre/voice tags for picker UIs. */
+	tags?: ReadonlyArray<string>;
+}
+
+/** Where the runtime expects to find Kokoro on disk. */
+export interface KokoroModelLayout {
+	/** Directory under `<stateDir>/local-inference/models/kokoro/`. */
+	root: string;
+	/** Model file — the Kokoro GGUF carried by our llama.cpp fork. */
+	modelFile: string;
+	/** Directory containing the per-voice style tensors. */
+	voicesDir: string;
+	/** Model output sample rate (Kokoro v1.0 = 24000). */
+	sampleRate: number;
+}
+
+/** Construction-time configuration for `KokoroTtsBackend`. */
+export interface KokoroBackendOptions {
+	/** Resolved on-disk layout. Required — the backend never guesses paths. */
+	layout: KokoroModelLayout;
+	/**
+	 * Voice id to use when the caller's `SpeakerPreset.voiceId` is not in the
+	 * voice-pack registry. The named voice MUST be present in `layout.voicesDir`.
+	 */
+	defaultVoiceId: KokoroVoiceId;
+	/**
+	 * Optional phonemizer override. Defaults to the bundled lazy phonemizer,
+	 * which uses `phonemize` if installed and falls back to a deterministic
+	 * grapheme-to-phoneme adapter otherwise (documented tradeoff in README).
+	 */
+	phonemizer?: KokoroPhonemizer;
+	/**
+	 * Max samples emitted in a single streaming chunk. Defaults to a quarter-
+	 * second at 24kHz (6000) so the scheduler ring buffer sees a continuous
+	 * trickle and TTFB stays close to the first inference completion.
+	 */
+	streamingChunkSamples?: number;
+}
+
+/** A pure (or async-pure) text → phoneme-id sequence converter. */
+export interface KokoroPhonemizer {
+	/** Phonemize a single utterance into a sequence of integer ids. */
+	phonemize(text: string, lang: string): Promise<KokoroPhonemeSequence>;
+	/** Human-facing id (`"phonemize"`, `"espeak-ng"`, `"fallback-g2p"`, ...). */
+	readonly id: string;
+}
+
+export interface KokoroPhonemeSequence {
+	/** Token ids for the model's `input_ids` tensor. */
+	ids: Int32Array;
+	/** Original phoneme string, for debugging and tests. */
+	phonemes: string;
+}
+
+/** Raised when the on-disk model layout is missing or malformed. */
+export class KokoroModelMissingError extends Error {
+	readonly code = "kokoro-model-missing" as const;
+	constructor(message: string) {
+		super(message);
+		this.name = "KokoroModelMissingError";
+	}
+}
+
+/** Raised when phonemization cannot proceed (no phonemizer + non-ASCII text). */
+export class KokoroPhonemizerError extends Error {
+	readonly code = "kokoro-phonemizer-error" as const;
+	constructor(message: string) {
+		super(message);
+		this.name = "KokoroPhonemizerError";
+	}
+}

package/src/services/voice/kokoro/voice-presets.ts

@@ -0,0 +1,129 @@
+/**
+ * Registry of bundled Kokoro voice packs (upstream: hexgrad/Kokoro-82M).
+ *
+ * Each entry maps a stable `KokoroVoiceId` (the `voices/<id>.bin` filename
+ * Kokoro ships) onto display metadata. The runtime resolves a caller's
+ * `SpeakerPreset.voiceId` against this table; an unknown id falls through to
+ * the backend's `defaultVoiceId`.
+ *
+ * The actual style tensor (256 fp32 values) lives at
+ * `<modelRoot>/voices/<file>` and is loaded lazily on first use.
+ *
+ * Reference: https://huggingface.co/hexgrad/Kokoro-82M
+ */
+
+import type { KokoroVoicePack } from "./types";
+
+export const KOKORO_VOICE_PACKS: ReadonlyArray<KokoroVoicePack> = [
+	// American English — female
+	{
+		id: "af_bella",
+		displayName: "Bella (US English)",
+		lang: "a",
+		file: "af_bella.bin",
+		dim: 256,
+		tags: ["female", "warm", "default"],
+	},
+	{
+		id: "af_sarah",
+		displayName: "Sarah (US English)",
+		lang: "a",
+		file: "af_sarah.bin",
+		dim: 256,
+		tags: ["female", "professional"],
+	},
+	{
+		id: "af_nicole",
+		displayName: "Nicole (US English, breathy)",
+		lang: "a",
+		file: "af_nicole.bin",
+		dim: 256,
+		tags: ["female", "breathy"],
+	},
+	{
+		id: "af_sky",
+		displayName: "Sky (US English)",
+		lang: "a",
+		file: "af_sky.bin",
+		dim: 256,
+		tags: ["female", "young"],
+	},
+	// American English — male
+	{
+		id: "am_michael",
+		displayName: "Michael (US English)",
+		lang: "a",
+		file: "am_michael.bin",
+		dim: 256,
+		tags: ["male", "warm"],
+	},
+	{
+		id: "am_adam",
+		displayName: "Adam (US English)",
+		lang: "a",
+		file: "am_adam.bin",
+		dim: 256,
+		tags: ["male", "neutral"],
+	},
+	// British English
+	{
+		id: "bf_emma",
+		displayName: "Emma (British English)",
+		lang: "b",
+		file: "bf_emma.bin",
+		dim: 256,
+		tags: ["female", "british"],
+	},
+	{
+		id: "bf_isabella",
+		displayName: "Isabella (British English)",
+		lang: "b",
+		file: "bf_isabella.bin",
+		dim: 256,
+		tags: ["female", "british"],
+	},
+	{
+		id: "bm_george",
+		displayName: "George (British English)",
+		lang: "b",
+		file: "bm_george.bin",
+		dim: 256,
+		tags: ["male", "british"],
+	},
+	{
+		id: "bm_lewis",
+		displayName: "Lewis (British English)",
+		lang: "b",
+		file: "bm_lewis.bin",
+		dim: 256,
+		tags: ["male", "british"],
+	},
+	// Eliza-1 fine-tuned voice — same (research-only, derivative of *Her* 2013).
+	// Voice pack lives at `elizaos/eliza-1` under `voice/kokoro/voices/af_same.bin`
+	// (first push is private; do not promote to default without a public-release sign-off).
+	// Source corpus: `lalalune/ai_voices/sam` upstream subset, landed locally as
+	// `same` (58 clips, 3.51 min, research-only).
+	// Voice id obeys the Kokoro `<lang><sex>_<name>` convention (US English, female).
+	{
+		id: "af_same",
+		displayName: "Same (Eliza-1, US English)",
+		lang: "a",
+		file: "af_same.bin",
+		dim: 256,
+		tags: ["female", "same", "eliza-1-voice", "research-only"],
+	},
+];
+
+const VOICE_BY_ID = new Map(KOKORO_VOICE_PACKS.map((v) => [v.id, v] as const));
+
+/** Look up a voice pack by id. Returns `undefined` for unknown ids — the
+ *  backend chooses how to fall back (typically `defaultVoiceId`). */
+export function findKokoroVoice(id: string): KokoroVoicePack | undefined {
+	return VOICE_BY_ID.get(id);
+}
+
+/** The voice the runtime selects when nothing is configured. */
+export const KOKORO_DEFAULT_VOICE_ID = "af_same";
+
+/** Conservative fallback voice when a configured/default preset is not staged. */
+export const KOKORO_FALLBACK_VOICE_ID = "af_bella";

package/src/services/voice/kokoro/voices.ts

@@ -0,0 +1,64 @@
+/**
+ * Kokoro-82M voice catalog.
+ *
+ * Re-exports the bundled `KOKORO_VOICE_PACKS` table from `voice-presets.ts`
+ * under the canonical filename the integration spec calls out, and adds a few
+ * search helpers used by the runtime-selection layer and the picker UI.
+ *
+ * Each voice pack ships as a small (~256 fp32 values × 4 bytes ≈ 1 KB) style
+ * tensor under `<modelRoot>/voices/<file>`. Adding voices is cheap (download
+ * a single .bin), and a hot ONNX session can swap voices per-utterance with
+ * no re-init cost — the model is the same.
+ *
+ * Upstream registry: https://huggingface.co/hexgrad/Kokoro-82M/tree/main/voices
+ */
+
+import type { KokoroVoiceId, KokoroVoicePack } from "./types";
+import {
+	findKokoroVoice,
+	KOKORO_DEFAULT_VOICE_ID,
+	KOKORO_VOICE_PACKS,
+} from "./voice-presets";
+
+export {
+	findKokoroVoice,
+	KOKORO_DEFAULT_VOICE_ID,
+	KOKORO_VOICE_PACKS,
+} from "./voice-presets";
+
+/** All voice ids that ship with the upstream Kokoro v1.0 release. */
+export function listKokoroVoiceIds(): ReadonlyArray<KokoroVoiceId> {
+	return KOKORO_VOICE_PACKS.map((v) => v.id);
+}
+
+/** Filter voice packs by language tag (`a` US, `b` UK, etc.). */
+export function listKokoroVoicesByLang(
+	lang: string,
+): ReadonlyArray<KokoroVoicePack> {
+	return KOKORO_VOICE_PACKS.filter((v) => v.lang === lang);
+}
+
+/** Filter by an exact tag (`female`, `male`, `british`, `breathy`, ...). */
+export function listKokoroVoicesByTag(
+	tag: string,
+): ReadonlyArray<KokoroVoicePack> {
+	return KOKORO_VOICE_PACKS.filter((v) => v.tags?.includes(tag) === true);
+}
+
+/**
+ * Resolve a caller-supplied voice id against the registry. Returns the
+ * matching pack, or the default pack when the id is not bundled. Never
+ * throws — runtime selection treats this as a graceful fallback because the
+ * caller's `SpeakerPreset.voiceId` may have been authored for OmniVoice.
+ */
+export function resolveKokoroVoiceOrDefault(id: string): KokoroVoicePack {
+	const match = findKokoroVoice(id);
+	if (match) return match;
+	const fallback = findKokoroVoice(KOKORO_DEFAULT_VOICE_ID);
+	if (!fallback) {
+		throw new Error(
+			`[kokoro] default voice id ${KOKORO_DEFAULT_VOICE_ID} is missing from KOKORO_VOICE_PACKS — registry is corrupted`,
+		);
+	}
+	return fallback;
+}

package/src/services/voice/lifecycle.test.ts

@@ -0,0 +1,315 @@
+/**
+ * Voice lifecycle state-machine tests.
+ *
+ * Exercises every documented transition in `lifecycle.ts`:
+ *   - default state is `voice-off`
+ *   - `arm()` transitions through `voice-arming` → `voice-on`
+ *   - `disarm()` transitions through `voice-disarming` → `voice-off`
+ *     and invokes `evictPages()` on the TTS + ASR mmap regions
+ *   - illegal transitions throw `VoiceLifecycleError`
+ *   - 100x arm/disarm cycles do not leak refs in the registry
+ *   - mmap failure surfaces as `voice-error` with the right code,
+ *     not a silent fallback (AGENTS.md §3 + §9)
+ *   - `reset()` is the only way out of `voice-error`
+ */
+
+import { describe, expect, it, vi } from "vitest";
+import {
+	VoiceLifecycle,
+	VoiceLifecycleError,
+	type VoiceLifecycleLoaders,
+	type VoiceLifecycleState,
+} from "./lifecycle";
+import {
+	type MmapRegionHandle,
+	type RefCountedResource,
+	SharedResourceRegistry,
+} from "./shared-resources";
+
+interface FakeMmap extends MmapRegionHandle {
+	evictCalls: number;
+	releaseCalls: number;
+}
+
+function fakeMmap(id: string, opts: { evictThrows?: Error } = {}): FakeMmap {
+	const region: FakeMmap = {
+		id,
+		path: `/tmp/${id}`,
+		sizeBytes: 1024,
+		evictCalls: 0,
+		releaseCalls: 0,
+		async evictPages() {
+			region.evictCalls++;
+			if (opts.evictThrows) throw opts.evictThrows;
+		},
+		async release() {
+			region.releaseCalls++;
+		},
+	};
+	return region;
+}
+
+function fakeResource(
+	id: string,
+): RefCountedResource & { releaseCalls: number } {
+	const r = {
+		id,
+		releaseCalls: 0,
+		async release() {
+			r.releaseCalls++;
+		},
+	};
+	return r;
+}
+
+function loadersOk(): VoiceLifecycleLoaders & {
+	tts: FakeMmap;
+	asr: FakeMmap;
+	caches: ReturnType<typeof fakeResource>;
+	nodes: ReturnType<typeof fakeResource>;
+} {
+	const tts = fakeMmap("tts:omnivoice-default");
+	const asr = fakeMmap("asr:default");
+	const caches = fakeResource("voice-caches");
+	const nodes = fakeResource("voice-scheduler-nodes");
+	return {
+		tts,
+		asr,
+		caches,
+		nodes,
+		loadTtsRegion: async () => tts,
+		loadAsrRegion: async () => asr,
+		loadVoiceCaches: async () => caches,
+		loadVoiceSchedulerNodes: async () => nodes,
+	};
+}
+
+describe("VoiceLifecycle", () => {
+	it("defaults to voice-off", () => {
+		const reg = new SharedResourceRegistry();
+		const lc = new VoiceLifecycle({ registry: reg, loaders: loadersOk() });
+		expect(lc.current().kind).toBe("voice-off");
+		expect(reg.size()).toBe(0);
+	});
+
+	it("arm() transitions voice-off → voice-arming → voice-on with all resources held", async () => {
+		const reg = new SharedResourceRegistry();
+		const loaders = loadersOk();
+		const transitions: VoiceLifecycleState["kind"][] = [];
+		const lc = new VoiceLifecycle({
+			registry: reg,
+			loaders,
+			events: {
+				onTransition: (_prev, next) => transitions.push(next.kind),
+			},
+		});
+		const armed = await lc.arm();
+		expect(transitions).toEqual(["voice-arming", "voice-on"]);
+		expect(lc.current().kind).toBe("voice-on");
+		expect(armed.tts.id).toBe(loaders.tts.id);
+		// All four resources tracked at refcount 1.
+		expect(reg.refCount(loaders.tts.id)).toBe(1);
+		expect(reg.refCount(loaders.asr.id)).toBe(1);
+		expect(reg.refCount(loaders.caches.id)).toBe(1);
+		expect(reg.refCount(loaders.nodes.id)).toBe(1);
+	});
+
+	it("disarm() transitions voice-on → voice-disarming → voice-off and invokes evictPages on tts+asr", async () => {
+		const reg = new SharedResourceRegistry();
+		const loaders = loadersOk();
+		const transitions: VoiceLifecycleState["kind"][] = [];
+		const lc = new VoiceLifecycle({
+			registry: reg,
+			loaders,
+			events: {
+				onTransition: (_prev, next) => transitions.push(next.kind),
+			},
+		});
+		await lc.arm();
+		transitions.length = 0;
+		await lc.disarm();
+		expect(transitions).toEqual(["voice-disarming", "voice-off"]);
+		expect(lc.current().kind).toBe("voice-off");
+		// madvise was called on both heavy regions.
+		expect(loaders.tts.evictCalls).toBe(1);
+		expect(loaders.asr.evictCalls).toBe(1);
+		// Underlying release ran for every resource.
+		expect(loaders.tts.releaseCalls).toBe(1);
+		expect(loaders.asr.releaseCalls).toBe(1);
+		expect(loaders.caches.releaseCalls).toBe(1);
+		expect(loaders.nodes.releaseCalls).toBe(1);
+		// Registry empty post-disarm.
+		expect(reg.size()).toBe(0);
+	});
+
+	it("arm() in voice-on throws illegal-transition", async () => {
+		const reg = new SharedResourceRegistry();
+		const lc = new VoiceLifecycle({ registry: reg, loaders: loadersOk() });
+		await lc.arm();
+		let thrown: unknown;
+		try {
+			await lc.arm();
+		} catch (e) {
+			thrown = e;
+		}
+		expect(thrown).toBeInstanceOf(VoiceLifecycleError);
+		if (thrown instanceof VoiceLifecycleError) {
+			expect(thrown.code).toBe("illegal-transition");
+		}
+	});
+
+	it("disarm() in voice-off throws illegal-transition", async () => {
+		const reg = new SharedResourceRegistry();
+		const lc = new VoiceLifecycle({ registry: reg, loaders: loadersOk() });
+		let thrown: unknown;
+		try {
+			await lc.disarm();
+		} catch (e) {
+			thrown = e;
+		}
+		expect(thrown).toBeInstanceOf(VoiceLifecycleError);
+		if (thrown instanceof VoiceLifecycleError) {
+			expect(thrown.code).toBe("illegal-transition");
+		}
+	});
+
+	it("100x arm/disarm cycles do not leak resources in the registry", async () => {
+		const reg = new SharedResourceRegistry();
+		// Build loaders that emit fresh handles on every call so refcount
+		// accounting actually exercises acquire/release rather than alias
+		// dedup.
+		let counter = 0;
+		const loaders: VoiceLifecycleLoaders = {
+			loadTtsRegion: async () => fakeMmap(`tts-${counter++}`),
+			loadAsrRegion: async () => fakeMmap(`asr-${counter++}`),
+			loadVoiceCaches: async () => fakeResource(`caches-${counter++}`),
+			loadVoiceSchedulerNodes: async () => fakeResource(`nodes-${counter++}`),
+		};
+		const lc = new VoiceLifecycle({ registry: reg, loaders });
+		for (let i = 0; i < 100; i++) {
+			await lc.arm();
+			await lc.disarm();
+		}
+		expect(lc.current().kind).toBe("voice-off");
+		expect(reg.size()).toBe(0);
+	});
+
+	it("simulated mmap failure surfaces as voice-error (no silent fallback)", async () => {
+		const reg = new SharedResourceRegistry();
+		const tts = fakeMmap("tts");
+		const ttsLoad = vi.fn(async () => tts);
+		const asrLoad = vi.fn(async () => {
+			throw new Error("mmap MAP_FAILED: cannot allocate memory");
+		});
+		const lc = new VoiceLifecycle({
+			registry: reg,
+			loaders: {
+				loadTtsRegion: ttsLoad,
+				loadAsrRegion: asrLoad,
+				loadVoiceCaches: async () => fakeResource("caches"),
+				loadVoiceSchedulerNodes: async () => fakeResource("nodes"),
+			},
+		});
+		let thrown: unknown;
+		try {
+			await lc.arm();
+		} catch (e) {
+			thrown = e;
+		}
+		expect(thrown).toBeInstanceOf(VoiceLifecycleError);
+		if (thrown instanceof VoiceLifecycleError) {
+			// The error message matched both /mmap|MAP_FAILED/ and
+			// /ENOMEM|out of memory|RAM/. The mmap branch comes second in the
+			// mapper — but the RAM regex matches "memory" first. Either is a
+			// structured code; the contract is "not silent fallback".
+			expect(["ram-pressure", "mmap-fail"]).toContain(thrown.code);
+		}
+		expect(lc.current().kind).toBe("voice-error");
+		// Partial acquisition rolled back — mapped pages are evicted first
+		// and registry refs are released.
+		expect(tts.evictCalls).toBe(1);
+		expect(reg.size()).toBe(0);
+	});
+
+	it("RAM-pressure error code is preferred when message says ENOMEM", async () => {
+		const reg = new SharedResourceRegistry();
+		const lc = new VoiceLifecycle({
+			registry: reg,
+			loaders: {
+				loadTtsRegion: async () => {
+					throw new Error("ENOMEM: out of memory while mapping TTS weights");
+				},
+				loadAsrRegion: async () => fakeMmap("asr"),
+				loadVoiceCaches: async () => fakeResource("caches"),
+				loadVoiceSchedulerNodes: async () => fakeResource("nodes"),
+			},
+		});
+		let thrown: unknown;
+		try {
+			await lc.arm();
+		} catch (e) {
+			thrown = e;
+		}
+		expect(thrown).toBeInstanceOf(VoiceLifecycleError);
+		if (thrown instanceof VoiceLifecycleError) {
+			expect(thrown.code).toBe("ram-pressure");
+		}
+		expect(lc.current().kind).toBe("voice-error");
+	});
+
+	it("voice-error is terminal until reset()", async () => {
+		const reg = new SharedResourceRegistry();
+		const lc = new VoiceLifecycle({
+			registry: reg,
+			loaders: {
+				loadTtsRegion: async () => {
+					throw new Error("kernel missing: turboquant_q4 unavailable");
+				},
+				loadAsrRegion: async () => fakeMmap("asr"),
+				loadVoiceCaches: async () => fakeResource("caches"),
+				loadVoiceSchedulerNodes: async () => fakeResource("nodes"),
+			},
+		});
+		await expect(lc.arm()).rejects.toBeInstanceOf(VoiceLifecycleError);
+		expect(lc.current().kind).toBe("voice-error");
+		// Cannot arm again from voice-error.
+		await expect(lc.arm()).rejects.toMatchObject({
+			code: "illegal-transition",
+		});
+		// Cannot disarm from voice-error.
+		await expect(lc.disarm()).rejects.toMatchObject({
+			code: "illegal-transition",
+		});
+		lc.reset();
+		expect(lc.current().kind).toBe("voice-off");
+	});
+
+	it("reset() in voice-off throws illegal-transition", () => {
+		const reg = new SharedResourceRegistry();
+		const lc = new VoiceLifecycle({ registry: reg, loaders: loadersOk() });
+		expect(() => lc.reset()).toThrow(VoiceLifecycleError);
+	});
+});
+
+describe("SharedResourceRegistry", () => {
+	it("dedupes by id and refcounts acquire/release", async () => {
+		const reg = new SharedResourceRegistry();
+		const a = fakeResource("foo");
+		const aliased = fakeResource("foo");
+		const first = reg.acquire(a);
+		const second = reg.acquire(aliased);
+		expect(second).toBe(first);
+		expect(reg.refCount("foo")).toBe(2);
+		await reg.release("foo");
+		expect(reg.refCount("foo")).toBe(1);
+		expect(a.releaseCalls).toBe(0);
+		await reg.release("foo");
+		expect(a.releaseCalls).toBe(1);
+		expect(reg.size()).toBe(0);
+	});
+
+	it("release of unknown id throws — no silent leak", async () => {
+		const reg = new SharedResourceRegistry();
+		await expect(reg.release("ghost")).rejects.toThrow(/unknown resource/);
+	});
+});