@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.2

package/src/services/voice/kokoro/__tests__/kokoro-ffi-runtime.test.ts

@@ -0,0 +1,235 @@
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+import type {
+	ElizaInferenceContextHandle,
+	ElizaInferenceFfi,
+} from "../ffi-bindings";
+import { KokoroFfiRuntime } from "../kokoro-ffi-runtime";
+import type { KokoroRuntimeInputs } from "../kokoro-runtime";
+import type { KokoroModelLayout, KokoroVoicePack } from "../types";
+
+const MODEL_FILE = "kokoro-82m-v1_0.gguf";
+
+interface FakeFfiCalls {
+	loads: Array<{ ggufPath: string; voiceBinPath: string; styleDim?: number }>;
+	synths: Array<{ text: string; maxSamples: number; speed?: number }>;
+	destroyed: number;
+	closed: number;
+}
+
+function fakeKokoroFfi(opts: {
+	supported?: boolean;
+	pcm?: Float32Array;
+	calls: FakeFfiCalls;
+}): ElizaInferenceFfi {
+	const pcm = opts.pcm ?? Float32Array.from([0.1, 0.2, 0.3, 0.4]);
+	return {
+		libraryPath: "/fake/libelizainference.so",
+		libraryAbiVersion: "10",
+		create: () => 7n as ElizaInferenceContextHandle,
+		destroy: () => {
+			opts.calls.destroyed++;
+		},
+		kokoroSupported: () => opts.supported ?? true,
+		kokoroLoad: (a) => {
+			opts.calls.loads.push(a);
+		},
+		kokoroSynthesize: (a) => {
+			opts.calls.synths.push({
+				text: a.text,
+				maxSamples: a.maxSamples,
+				speed: a.speed,
+			});
+			return pcm.slice();
+		},
+		kokoroSampleRate: () => 24_000,
+		close: () => {
+			opts.calls.closed++;
+		},
+	} as unknown as ElizaInferenceFfi;
+}
+
+function makeLayout(root: string): KokoroModelLayout {
+	return {
+		root,
+		modelFile: MODEL_FILE,
+		voicesDir: path.join(root, "voices"),
+		sampleRate: 24_000,
+	};
+}
+
+function voice(id: string, file: string): KokoroVoicePack {
+	return { id, displayName: id, lang: "a", file, dim: 256 };
+}
+
+function makeInputs(
+	v: KokoroVoicePack,
+	onChunk: KokoroRuntimeInputs["onChunk"],
+	cancelSignal = { cancelled: false },
+): KokoroRuntimeInputs {
+	return {
+		phonemes: { ids: Int32Array.from([1, 2, 3]), phonemes: "hɛˈloʊ" },
+		voice: v,
+		cancelSignal,
+		onChunk,
+	};
+}
+
+describe("KokoroFfiRuntime", () => {
+	let root: string;
+	let calls: FakeFfiCalls;
+
+	beforeEach(() => {
+		root = mkdtempSync(path.join(os.tmpdir(), "kokoro-ffi-test-"));
+		mkdirSync(path.join(root, "voices"), { recursive: true });
+		writeFileSync(path.join(root, MODEL_FILE), Buffer.alloc(8));
+		writeFileSync(path.join(root, "voices", "af_same.bin"), Buffer.alloc(1024));
+		writeFileSync(
+			path.join(root, "voices", "af_bella.bin"),
+			Buffer.alloc(1024),
+		);
+		calls = { loads: [], synths: [], destroyed: 0, closed: 0 };
+	});
+
+	afterEach(() => {
+		rmSync(root, { recursive: true, force: true });
+		vi.restoreAllMocks();
+	});
+
+	it("throws when the loaded build does not link the Kokoro engine (no TCP fallback)", () => {
+		const ffi = fakeKokoroFfi({ supported: false, calls });
+		expect(
+			() =>
+				new KokoroFfiRuntime({
+					layout: makeLayout(root),
+					ffi,
+					ctx: 7n as ElizaInferenceContextHandle,
+				}),
+		).toThrow(/does not link the in-process Eliza-1 Kokoro engine/);
+	});
+
+	it("loads the GGUF + voice once and emits one body chunk + final tail", async () => {
+		const ffi = fakeKokoroFfi({ calls });
+		const rt = new KokoroFfiRuntime({
+			layout: makeLayout(root),
+			ffi,
+			ctx: 7n as ElizaInferenceContextHandle,
+		});
+
+		const chunks: Array<{ isFinal: boolean; len: number }> = [];
+		const result = await rt.synthesize(
+			makeInputs(voice("af_same", "af_same.bin"), (c) => {
+				chunks.push({ isFinal: c.isFinal, len: c.pcm.length });
+				return undefined;
+			}),
+		);
+
+		expect(result.cancelled).toBe(false);
+		expect(calls.loads).toHaveLength(1);
+		expect(calls.loads[0]?.ggufPath).toBe(path.join(root, MODEL_FILE));
+		expect(calls.loads[0]?.voiceBinPath).toBe(
+			path.join(root, "voices", "af_same.bin"),
+		);
+		// The fork phonemizes internally — we hand it the phoneme string, the
+		// same `input` the llama-server /v1/audio/speech path sends.
+		expect(calls.synths[0]?.text).toBe("hɛˈloʊ");
+		expect(chunks.filter((c) => !c.isFinal)).toHaveLength(1);
+		expect(chunks.at(-1)?.isFinal).toBe(true);
+		expect(chunks[0]?.len).toBe(4);
+	});
+
+	it("does not reload the voice across synthesize calls for the same voice", async () => {
+		const ffi = fakeKokoroFfi({ calls });
+		const rt = new KokoroFfiRuntime({
+			layout: makeLayout(root),
+			ffi,
+			ctx: 7n as ElizaInferenceContextHandle,
+		});
+
+		const v = voice("af_same", "af_same.bin");
+		await rt.synthesize(makeInputs(v, () => undefined));
+		await rt.synthesize(makeInputs(v, () => undefined));
+
+		expect(calls.loads).toHaveLength(1);
+		expect(calls.synths).toHaveLength(2);
+	});
+
+	it("reloads when the requested voice changes", async () => {
+		const ffi = fakeKokoroFfi({ calls });
+		const rt = new KokoroFfiRuntime({
+			layout: makeLayout(root),
+			ffi,
+			ctx: 7n as ElizaInferenceContextHandle,
+		});
+
+		await rt.synthesize(
+			makeInputs(voice("af_same", "af_same.bin"), () => undefined),
+		);
+		await rt.synthesize(
+			makeInputs(voice("af_bella", "af_bella.bin"), () => undefined),
+		);
+
+		expect(calls.loads).toHaveLength(2);
+		expect(calls.loads[1]?.voiceBinPath).toBe(
+			path.join(root, "voices", "af_bella.bin"),
+		);
+	});
+
+	it("honours a pre-set cancel signal without synthesizing", async () => {
+		const ffi = fakeKokoroFfi({ calls });
+		const rt = new KokoroFfiRuntime({
+			layout: makeLayout(root),
+			ffi,
+			ctx: 7n as ElizaInferenceContextHandle,
+		});
+
+		const chunks: boolean[] = [];
+		const result = await rt.synthesize(
+			makeInputs(
+				voice("af_same", "af_same.bin"),
+				(c) => {
+					chunks.push(c.isFinal);
+					return undefined;
+				},
+				{ cancelled: true },
+			),
+		);
+
+		expect(result.cancelled).toBe(true);
+		// Only the final tail is emitted; no synthesis happened.
+		expect(calls.synths).toHaveLength(0);
+		expect(chunks).toEqual([true]);
+	});
+
+	it("throws when the model file is missing on disk", async () => {
+		rmSync(path.join(root, MODEL_FILE), { force: true });
+		const ffi = fakeKokoroFfi({ calls });
+		const rt = new KokoroFfiRuntime({
+			layout: makeLayout(root),
+			ffi,
+			ctx: 7n as ElizaInferenceContextHandle,
+		});
+
+		await expect(
+			rt.synthesize(
+				makeInputs(voice("af_same", "af_same.bin"), () => undefined),
+			),
+		).rejects.toThrow(/Kokoro model file not found/);
+	});
+
+	it("destroys the ctx and closes the lib only when it owns them", () => {
+		const ffi = fakeKokoroFfi({ calls });
+		const rt = new KokoroFfiRuntime({
+			layout: makeLayout(root),
+			ffi,
+			ctx: 7n as ElizaInferenceContextHandle,
+		});
+		rt.dispose();
+		// ctx + ffi were injected → not owned → not torn down.
+		expect(calls.destroyed).toBe(0);
+		expect(calls.closed).toBe(0);
+	});
+});

package/src/services/voice/kokoro/__tests__/kokoro-runtime.test.ts

@@ -0,0 +1,95 @@
+import { afterEach, describe, expect, it } from "vitest";
+
+import { KokoroMockRuntime } from "../kokoro-runtime";
+import type { KokoroVoicePack } from "../types";
+
+const OLD_ENV = { ...process.env };
+
+afterEach(() => {
+	process.env = { ...OLD_ENV };
+});
+
+function makeVoice(): KokoroVoicePack {
+	return {
+		id: "af_test",
+		displayName: "Test",
+		lang: "a",
+		file: "af_test.bin",
+		dim: 256,
+		tags: ["test"],
+	};
+}
+
+describe("KokoroMockRuntime", () => {
+	it("emits chunks and a final marker", async () => {
+		const runtime = new KokoroMockRuntime({
+			sampleRate: 24_000,
+			totalSamples: 100,
+			chunkCount: 4,
+		});
+		const chunks: Array<{ isFinal: boolean; len: number }> = [];
+		await runtime.synthesize({
+			phonemes: { ids: Int32Array.from([1, 2, 3]), phonemes: "abc" },
+			voice: makeVoice(),
+			cancelSignal: { cancelled: false },
+			onChunk: (c) => {
+				chunks.push({ isFinal: c.isFinal, len: c.pcm.length });
+				return undefined;
+			},
+		});
+		expect(chunks.at(-1)?.isFinal).toBe(true);
+		const bodyChunks = chunks.filter((c) => !c.isFinal);
+		expect(bodyChunks.length).toBeGreaterThan(0);
+		const total = bodyChunks.reduce((s, c) => s + c.len, 0);
+		expect(total).toBe(100);
+	});
+
+	it("increments calls counter", async () => {
+		const runtime = new KokoroMockRuntime({ sampleRate: 24_000 });
+		expect(runtime.calls).toBe(0);
+		await runtime.synthesize({
+			phonemes: { ids: Int32Array.from([1]), phonemes: "a" },
+			voice: makeVoice(),
+			cancelSignal: { cancelled: false },
+			onChunk: () => undefined,
+		});
+		expect(runtime.calls).toBe(1);
+	});
+
+	it("honours cancel signal", async () => {
+		const runtime = new KokoroMockRuntime({
+			sampleRate: 24_000,
+			totalSamples: 1000,
+			chunkCount: 10,
+		});
+		const signal = { cancelled: false };
+		let bodyCount = 0;
+		const result = await runtime.synthesize({
+			phonemes: { ids: Int32Array.from([1, 2]), phonemes: "ab" },
+			voice: makeVoice(),
+			cancelSignal: signal,
+			onChunk: (c) => {
+				if (!c.isFinal) {
+					bodyCount++;
+					if (bodyCount >= 2) {
+						signal.cancelled = true;
+					}
+				}
+				return undefined;
+			},
+		});
+		expect(result.cancelled).toBe(true);
+		expect(bodyCount).toBeLessThan(10);
+	});
+
+	it("returns cancelled=true when onChunk returns true", async () => {
+		const runtime = new KokoroMockRuntime({ sampleRate: 24_000 });
+		const result = await runtime.synthesize({
+			phonemes: { ids: Int32Array.from([1]), phonemes: "a" },
+			voice: makeVoice(),
+			cancelSignal: { cancelled: false },
+			onChunk: () => true,
+		});
+		expect(result.cancelled).toBe(true);
+	});
+});

package/src/services/voice/kokoro/__tests__/phonemizer.test.ts

@@ -0,0 +1,53 @@
+import { describe, expect, it } from "vitest";
+
+import {
+	FallbackG2PPhonemizer,
+	KOKORO_PAD_ID,
+	kokoroLangToPhonemizerLanguage,
+	NpmPhonemizePhonemizer,
+} from "../phonemizer";
+
+describe("FallbackG2PPhonemizer", () => {
+	it("uses Kokoro tokenizer boundary ids and IPA for common smoke phrases", async () => {
+		const seq = await new FallbackG2PPhonemizer().phonemize(
+			"Hello there.",
+			"a",
+		);
+
+		expect(seq.phonemes).toBe("hɛloʊ ðɛɹ.");
+		expect(Array.from(seq.ids)).toEqual([
+			KOKORO_PAD_ID,
+			50,
+			86,
+			54,
+			57,
+			135,
+			16,
+			81,
+			86,
+			123,
+			4,
+			KOKORO_PAD_ID,
+		]);
+	});
+
+	it("maps Kokoro voice language ids to phonemizer locales", () => {
+		expect(kokoroLangToPhonemizerLanguage("a")).toBe("en-us");
+		expect(kokoroLangToPhonemizerLanguage("b")).toBe("en-gb");
+		expect(kokoroLangToPhonemizerLanguage("en-us")).toBe("en-us");
+	});
+
+	it("loads the bundled phonemizer package before falling back to pseudo phonemes", async () => {
+		const phonemizer = await NpmPhonemizePhonemizer.tryLoad();
+		expect(phonemizer?.id).toBe("phonemizer");
+		if (!phonemizer) {
+			throw new Error("phonemizer package did not load");
+		}
+
+		const seq = await phonemizer.phonemize("Hello there.", "a");
+
+		expect(seq.phonemes).not.toBe("hɛloʊ ðɛɹ.");
+		expect(seq.phonemes).toContain("h");
+		expect(Array.from(seq.ids)).toContain(156);
+	});
+});

package/src/services/voice/kokoro/__tests__/runtime-selection.test.ts

@@ -0,0 +1,231 @@
+import { describe, expect, it } from "vitest";
+import {
+	readVoiceBackendModeFromEnv,
+	selectVoiceBackend,
+} from "../runtime-selection";
+
+describe("selectVoiceBackend", () => {
+	it("mobile: always picks Kokoro, ignoring mode and tier policy", () => {
+		const d = selectVoiceBackend({
+			mobile: true,
+			mode: "omnivoice",
+			kokoroAvailable: true,
+			omnivoiceAvailable: true,
+			tierVoiceBackends: ["omnivoice"],
+		});
+		expect(d.backend).toBe("kokoro");
+		expect(d.reason).toMatch(/mobile/);
+	});
+
+	it("mobile: throws when Kokoro artifacts are missing (no OmniVoice fallback)", () => {
+		expect(() =>
+			selectVoiceBackend({
+				mobile: true,
+				kokoroAvailable: false,
+				omnivoiceAvailable: true,
+			}),
+		).toThrow(/mobile/);
+	});
+
+	it("forces Kokoro when mode=kokoro and artifacts are present", () => {
+		const d = selectVoiceBackend({
+			mode: "kokoro",
+			kokoroAvailable: true,
+			omnivoiceAvailable: true,
+		});
+		expect(d.backend).toBe("kokoro");
+		expect(d.reason).toMatch(/forced/);
+	});
+
+	it("throws when mode=kokoro but artifacts are missing", () => {
+		expect(() =>
+			selectVoiceBackend({
+				mode: "kokoro",
+				kokoroAvailable: false,
+				omnivoiceAvailable: true,
+			}),
+		).toThrow(/Kokoro/);
+	});
+
+	it("forces OmniVoice when mode=omnivoice", () => {
+		const d = selectVoiceBackend({
+			mode: "omnivoice",
+			kokoroAvailable: true,
+			omnivoiceAvailable: true,
+		});
+		expect(d.backend).toBe("omnivoice");
+	});
+
+	it("auto: requireVoiceCloning routes to OmniVoice", () => {
+		const d = selectVoiceBackend({
+			mode: "auto",
+			requireVoiceCloning: true,
+			kokoroAvailable: true,
+			omnivoiceAvailable: true,
+		});
+		expect(d.backend).toBe("omnivoice");
+		expect(d.reason).toMatch(/cloning/);
+	});
+
+	it("auto: requireVoiceCloning with no OmniVoice throws", () => {
+		expect(() =>
+			selectVoiceBackend({
+				mode: "auto",
+				requireVoiceCloning: true,
+				kokoroAvailable: true,
+				omnivoiceAvailable: false,
+			}),
+		).toThrow(/cloning/);
+	});
+
+	it("auto: low targetTtfaMs (<200) prefers Kokoro", () => {
+		const d = selectVoiceBackend({
+			mode: "auto",
+			targetTtfaMs: 120,
+			kokoroAvailable: true,
+			omnivoiceAvailable: true,
+		});
+		expect(d.backend).toBe("kokoro");
+		expect(d.reason).toMatch(/97ms/);
+	});
+
+	it("auto: low targetTtfaMs with no Kokoro falls back to OmniVoice", () => {
+		const d = selectVoiceBackend({
+			mode: "auto",
+			targetTtfaMs: 120,
+			kokoroAvailable: false,
+			omnivoiceAvailable: true,
+		});
+		expect(d.backend).toBe("omnivoice");
+	});
+
+	it("auto: Kokoro RTF beats OmniVoice by ≥10% → Kokoro", () => {
+		const d = selectVoiceBackend({
+			mode: "auto",
+			kokoroRtf: 2.5,
+			omnivoiceRtf: 2.0,
+			kokoroAvailable: true,
+			omnivoiceAvailable: true,
+		});
+		expect(d.backend).toBe("kokoro");
+	});
+
+	it("auto: Kokoro RTF only marginally better → OmniVoice default", () => {
+		const d = selectVoiceBackend({
+			mode: "auto",
+			kokoroRtf: 2.05,
+			omnivoiceRtf: 2.0,
+			kokoroAvailable: true,
+			omnivoiceAvailable: true,
+		});
+		expect(d.backend).toBe("omnivoice");
+	});
+
+	it("auto: Kokoro RTF measured but OmniVoice unmeasured → Kokoro", () => {
+		const d = selectVoiceBackend({
+			mode: "auto",
+			kokoroRtf: 1.5,
+			omnivoiceRtf: null,
+			kokoroAvailable: true,
+			omnivoiceAvailable: true,
+		});
+		expect(d.backend).toBe("kokoro");
+	});
+
+	it("auto: OmniVoice missing but Kokoro present → Kokoro", () => {
+		const d = selectVoiceBackend({
+			mode: "auto",
+			kokoroAvailable: true,
+			omnivoiceAvailable: false,
+		});
+		expect(d.backend).toBe("kokoro");
+	});
+
+	it("auto: neither backend available throws", () => {
+		expect(() =>
+			selectVoiceBackend({
+				mode: "auto",
+				kokoroAvailable: false,
+				omnivoiceAvailable: false,
+			}),
+		).toThrow(/no TTS backend/);
+	});
+
+	it("auto: default (no overrides, no tier policy) picks OmniVoice", () => {
+		const d = selectVoiceBackend({
+			mode: "auto",
+			kokoroAvailable: true,
+			omnivoiceAvailable: true,
+		});
+		expect(d.backend).toBe("omnivoice");
+		expect(d.reason).toMatch(/default/);
+	});
+
+	it("auto: tier policy [kokoro] picks Kokoro", () => {
+		const d = selectVoiceBackend({
+			mode: "auto",
+			kokoroAvailable: true,
+			omnivoiceAvailable: true,
+			tierVoiceBackends: ["kokoro"],
+		});
+		expect(d.backend).toBe("kokoro");
+		expect(d.reason).toMatch(/tier default/);
+	});
+
+	it("auto: tier policy [kokoro, omnivoice] picks Kokoro (first wins)", () => {
+		const d = selectVoiceBackend({
+			mode: "auto",
+			kokoroAvailable: true,
+			omnivoiceAvailable: true,
+			tierVoiceBackends: ["kokoro", "omnivoice"],
+		});
+		expect(d.backend).toBe("kokoro");
+	});
+
+	it("auto: tier policy [omnivoice] picks OmniVoice", () => {
+		const d = selectVoiceBackend({
+			mode: "auto",
+			kokoroAvailable: true,
+			omnivoiceAvailable: true,
+			tierVoiceBackends: ["omnivoice"],
+		});
+		expect(d.backend).toBe("omnivoice");
+		expect(d.reason).toMatch(/tier default/);
+	});
+
+	it("auto: tier policy is overridden by requireVoiceCloning", () => {
+		const d = selectVoiceBackend({
+			mode: "auto",
+			kokoroAvailable: true,
+			omnivoiceAvailable: true,
+			tierVoiceBackends: ["kokoro"],
+			requireVoiceCloning: true,
+		});
+		expect(d.backend).toBe("omnivoice");
+		expect(d.reason).toMatch(/cloning/);
+	});
+});
+
+describe("readVoiceBackendModeFromEnv", () => {
+	it("returns undefined when env var is unset", () => {
+		expect(readVoiceBackendModeFromEnv({})).toBeUndefined();
+	});
+
+	it("parses each valid value", () => {
+		expect(readVoiceBackendModeFromEnv({ ELIZA_TTS_BACKEND: "kokoro" })).toBe(
+			"kokoro",
+		);
+		expect(
+			readVoiceBackendModeFromEnv({ ELIZA_TTS_BACKEND: "OMNIVOICE" }),
+		).toBe("omnivoice");
+		expect(readVoiceBackendModeFromEnv({ ELIZA_TTS_BACKEND: "auto" })).toBe(
+			"auto",
+		);
+	});
+
+	it("throws on an invalid value", () => {
+		expect(() =>
+			readVoiceBackendModeFromEnv({ ELIZA_TTS_BACKEND: "garbage" }),
+		).toThrow();
+	});
+});

package/src/services/voice/kokoro/__tests__/voices.test.ts

@@ -0,0 +1,57 @@
+import { describe, expect, it } from "vitest";
+import {
+	findKokoroVoice,
+	KOKORO_DEFAULT_VOICE_ID,
+	KOKORO_VOICE_PACKS,
+	listKokoroVoiceIds,
+	listKokoroVoicesByLang,
+	listKokoroVoicesByTag,
+	resolveKokoroVoiceOrDefault,
+} from "../voices";
+
+describe("KOKORO_VOICE_PACKS", () => {
+	it("is non-empty and every entry has consistent metadata", () => {
+		expect(KOKORO_VOICE_PACKS.length).toBeGreaterThanOrEqual(8);
+		for (const v of KOKORO_VOICE_PACKS) {
+			expect(v.id).toMatch(/^[a-z]{2}_[a-z]+$/);
+			expect(v.file).toBe(`${v.id}.bin`);
+			expect(v.dim).toBe(256);
+			expect(["a", "b"]).toContain(v.lang);
+			expect(v.displayName.length).toBeGreaterThan(0);
+		}
+	});
+
+	it("default voice id is registered", () => {
+		expect(findKokoroVoice(KOKORO_DEFAULT_VOICE_ID)).toBeDefined();
+	});
+
+	it("ids are unique", () => {
+		const ids = listKokoroVoiceIds();
+		expect(new Set(ids).size).toBe(ids.length);
+	});
+
+	it("listKokoroVoicesByLang filters correctly", () => {
+		const us = listKokoroVoicesByLang("a");
+		const uk = listKokoroVoicesByLang("b");
+		expect(us.length).toBeGreaterThan(0);
+		expect(uk.length).toBeGreaterThan(0);
+		expect(us.every((v) => v.lang === "a")).toBe(true);
+		expect(uk.every((v) => v.lang === "b")).toBe(true);
+	});
+
+	it("listKokoroVoicesByTag filters by tag membership", () => {
+		const female = listKokoroVoicesByTag("female");
+		expect(female.length).toBeGreaterThan(0);
+		expect(female.every((v) => v.tags?.includes("female"))).toBe(true);
+	});
+
+	it("resolveKokoroVoiceOrDefault returns the requested voice when present", () => {
+		const v = resolveKokoroVoiceOrDefault("af_sarah");
+		expect(v.id).toBe("af_sarah");
+	});
+
+	it("resolveKokoroVoiceOrDefault falls back to the default for unknown ids", () => {
+		const v = resolveKokoroVoiceOrDefault("not_a_real_voice_id");
+		expect(v.id).toBe(KOKORO_DEFAULT_VOICE_ID);
+	});
+});