@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/voice/kokoro/phonemizer.ts

@@ -0,0 +1,344 @@
+/**
+ * Text → phoneme-id adapter for Kokoro-82M.
+ *
+ * Kokoro is trained against espeak-ng IPA tokens with a small fixed vocab
+ * (~178 entries: IPA symbols + stress/punct markers + special <s>/<pad>).
+ * Production deployments should bring real espeak-ng phonemization
+ * (`phonemizer` is the pure-JS eSpeak NG package); the bundled fallback here is a
+ * deterministic letter-to-pseudo-phoneme adapter that produces audible
+ * speech for ASCII English text but loses prosodic accuracy.
+ *
+ * Resolution order:
+ *   1. Caller-provided `KokoroPhonemizer` (preferred — bring your own).
+ *   2. Dynamically-imported `phonemizer`/`phonemize` npm package, if installed.
+ *   3. Bundled `FallbackG2PPhonemizer` (degrades gracefully, never throws on
+ *      ASCII input).
+ *
+ * Non-ASCII text with no real phonemizer raises `KokoroPhonemizerError` —
+ * silent garbage out is worse than a surfaced error (AGENTS.md §3).
+ */
+
+import {
+	type KokoroPhonemeSequence,
+	type KokoroPhonemizer,
+	KokoroPhonemizerError,
+} from "./types";
+
+/**
+ * Kokoro v1.0 phoneme vocabulary. These ids must match the bundled
+ * `tts/kokoro/tokenizer.json` asset. The boundary token is `$` (id 0);
+ * feeding invented `<s>` / `</s>` ids shifts the whole utterance and produces
+ * plausible-sounding but lexically wrong audio.
+ */
+const VOCAB: Readonly<Record<string, number>> = {
+	$: 0,
+	";": 1,
+	":": 2,
+	",": 3,
+	".": 4,
+	"!": 5,
+	"?": 6,
+	"—": 9,
+	"…": 10,
+	'"': 11,
+	"(": 12,
+	")": 13,
+	"“": 14,
+	"”": 15,
+	" ": 16,
+	"̃": 17,
+	ʣ: 18,
+	ʥ: 19,
+	ʦ: 20,
+	ʨ: 21,
+	ᵝ: 22,
+	ꭧ: 23,
+	A: 24,
+	I: 25,
+	O: 31,
+	Q: 33,
+	S: 35,
+	T: 36,
+	W: 39,
+	Y: 41,
+	ᵊ: 42,
+	a: 43,
+	b: 44,
+	c: 45,
+	d: 46,
+	e: 47,
+	f: 48,
+	h: 50,
+	i: 51,
+	j: 52,
+	k: 53,
+	l: 54,
+	m: 55,
+	n: 56,
+	o: 57,
+	p: 58,
+	q: 59,
+	r: 60,
+	s: 61,
+	t: 62,
+	u: 63,
+	v: 64,
+	w: 65,
+	x: 66,
+	y: 67,
+	z: 68,
+	ɑ: 69,
+	ɐ: 70,
+	ɒ: 71,
+	æ: 72,
+	β: 75,
+	ɔ: 76,
+	ɕ: 77,
+	ç: 78,
+	ɖ: 80,
+	ð: 81,
+	ʤ: 82,
+	ə: 83,
+	ɚ: 85,
+	ɛ: 86,
+	ɜ: 87,
+	ɟ: 90,
+	ɡ: 92,
+	ɥ: 99,
+	ɨ: 101,
+	ɪ: 102,
+	ʝ: 103,
+	ɯ: 110,
+	ɰ: 111,
+	ŋ: 112,
+	ɳ: 113,
+	ɲ: 114,
+	ɴ: 115,
+	ø: 116,
+	ɸ: 118,
+	θ: 119,
+	œ: 120,
+	ɹ: 123,
+	ɾ: 125,
+	ɻ: 126,
+	ʁ: 128,
+	ɽ: 129,
+	ʂ: 130,
+	ʃ: 131,
+	ʈ: 132,
+	ʧ: 133,
+	ʊ: 135,
+	ʋ: 136,
+	ʌ: 138,
+	ɣ: 139,
+	ɤ: 140,
+	χ: 142,
+	ʎ: 143,
+	ʒ: 147,
+	ʔ: 148,
+	ˈ: 156,
+	ˌ: 157,
+	ː: 158,
+	ʰ: 162,
+	ʲ: 164,
+	"↓": 169,
+	"→": 171,
+	"↗": 172,
+	"↘": 173,
+	ᵻ: 177,
+};
+
+const PAD = VOCAB.$;
+const BOS = VOCAB.$;
+const EOS = VOCAB.$;
+
+const FALLBACK_WORD_IPA: Readonly<Record<string, string>> = {
+	a: "ə",
+	am: "æm",
+	and: "ænd",
+	are: "ɑɹ",
+	cal: "kæl",
+	capital: "kæpɪtəl",
+	can: "kæn",
+	france: "fɹæns",
+	hello: "hɛloʊ",
+	hear: "hiɹ",
+	is: "ɪz",
+	me: "mi",
+	meeting: "mitɪŋ",
+	of: "ʌv",
+	the: "ðə",
+	there: "ðɛɹ",
+	to: "tu",
+	you: "ju",
+};
+
+const FALLBACK_DIGRAPH_IPA: Readonly<Record<string, string>> = {
+	ch: "ʧ",
+	ng: "ŋ",
+	sh: "ʃ",
+	th: "θ",
+	wh: "w",
+	zh: "ʒ",
+};
+
+function fallbackWordToIpa(word: string): string {
+	const known = FALLBACK_WORD_IPA[word];
+	if (known) return known;
+	let out = "";
+	for (let i = 0; i < word.length; i += 1) {
+		const pair = word.slice(i, i + 2);
+		const digraph = FALLBACK_DIGRAPH_IPA[pair];
+		if (digraph) {
+			out += digraph;
+			i += 1;
+			continue;
+		}
+		out += word[i];
+	}
+	return out;
+}
+
+function fallbackTextToIpa(cleaned: string): string {
+	return cleaned.replace(/[a-z]+|[^a-z]+/g, (part) =>
+		/^[a-z]+$/.test(part) ? fallbackWordToIpa(part) : part,
+	);
+}
+
+/**
+ * Deterministic ASCII-only G2P used when no real phonemizer is installed.
+ * Lossy by design — this exists so dev environments without espeak-ng still
+ * produce lexically useful smoke output for common English phrases, not to
+ * replace a production Misaki/espeak phonemizer.
+ */
+export class FallbackG2PPhonemizer implements KokoroPhonemizer {
+	readonly id = "fallback-g2p";
+
+	async phonemize(text: string, _lang: string): Promise<KokoroPhonemeSequence> {
+		const cleaned = text.normalize("NFKD").toLowerCase();
+		for (const ch of cleaned) {
+			const cp = ch.codePointAt(0);
+			if (cp === undefined) continue;
+			// Allow ASCII printable + whitespace; refuse anything else so we
+			// surface non-English text rather than emit silence.
+			if (cp > 127) {
+				throw new KokoroPhonemizerError(
+					`[kokoro] fallback phonemizer cannot handle non-ASCII character '${ch}' (U+${cp.toString(16).padStart(4, "0")}). Install the 'phonemizer' npm package or pass a custom KokoroPhonemizer for full Unicode coverage.`,
+				);
+			}
+		}
+		const phonemes = fallbackTextToIpa(cleaned);
+		const ids: number[] = [BOS];
+		for (const ch of phonemes) {
+			const id = VOCAB[ch];
+			if (id !== undefined) ids.push(id);
+			// Unknown char: skip (acts as a pad). The model's training data did
+			// not contain raw graphemes anyway — best effort.
+		}
+		ids.push(EOS);
+		return {
+			ids: Int32Array.from(ids),
+			phonemes,
+		};
+	}
+}
+
+interface PhonemizeMod {
+	// The `phonemizer` / legacy `phonemize` npm package typing varies between
+	// packages and versions; treat it structurally so minor updates do not break
+	// mobile TTS.
+	phonemize?: (
+		text: string,
+		langOrOpts?: unknown,
+	) => string | string[] | Promise<string | string[]>;
+	default?: { phonemize?: PhonemizeMod["phonemize"] };
+}
+
+/**
+ * Wraps the npm `phonemizer` package when present. It returns an IPA string
+ * which we tokenise with the same VOCAB above. Real Kokoro inference should
+ * use a proper espeak tokenizer — production deployments bring their own;
+ * this is the "install npm and it works" middle ground.
+ */
+export class NpmPhonemizePhonemizer implements KokoroPhonemizer {
+	readonly id: string;
+	private constructor(
+		private readonly mod: PhonemizeMod,
+		id = "phonemizer",
+		private readonly callStyle: "language" | "options" = "language",
+	) {
+		this.id = id;
+	}
+
+	static async tryLoad(): Promise<NpmPhonemizePhonemizer | null> {
+		try {
+			const mod = (await import("phonemizer")) as PhonemizeMod;
+			const phon = mod.phonemize ?? mod.default?.phonemize;
+			if (typeof phon !== "function") return null;
+			return new NpmPhonemizePhonemizer(mod);
+		} catch {
+			// Older local installs used a package named `phonemize`. Keep it as a
+			// secondary, deliberately non-bundled fallback for developer machines.
+		}
+		try {
+			const spec = "phonemize";
+			const mod = (await import(/* @vite-ignore */ spec)) as PhonemizeMod;
+			const phon = mod.phonemize ?? mod.default?.phonemize;
+			if (typeof phon !== "function") return null;
+			return new NpmPhonemizePhonemizer(mod, "phonemize", "options");
+		} catch {
+			return null;
+		}
+	}
+
+	async phonemize(text: string, lang: string): Promise<KokoroPhonemeSequence> {
+		const phon = this.mod.phonemize ?? this.mod.default?.phonemize;
+		if (!phon) {
+			throw new KokoroPhonemizerError(
+				"[kokoro] 'phonemize' module loaded but does not export a phonemize() function",
+			);
+		}
+		const out = await phon(
+			text,
+			this.callStyle === "language"
+				? kokoroLangToPhonemizerLanguage(lang)
+				: { lang },
+		);
+		const phonemes = Array.isArray(out)
+			? out.join(" ")
+			: typeof out === "string"
+				? out
+				: String(out);
+		const ids: number[] = [BOS];
+		for (const ch of phonemes.toLowerCase()) {
+			const id = VOCAB[ch];
+			if (id !== undefined) ids.push(id);
+		}
+		ids.push(EOS);
+		return { ids: Int32Array.from(ids), phonemes };
+	}
+}
+
+export function kokoroLangToPhonemizerLanguage(lang: string): string {
+	switch (lang.trim().toLowerCase()) {
+		case "a":
+			return "en-us";
+		case "b":
+			return "en-gb";
+		default:
+			return lang || "en-us";
+	}
+}
+
+/** Lazy resolver: caller override → npm `phonemizer` → bundled fallback. */
+export async function resolvePhonemizer(
+	override?: KokoroPhonemizer,
+): Promise<KokoroPhonemizer> {
+	if (override) return override;
+	const npm = await NpmPhonemizePhonemizer.tryLoad();
+	if (npm) return npm;
+	return new FallbackG2PPhonemizer();
+}
+
+/** Exported for tests and bench-time diagnostics. */
+export const KOKORO_PAD_ID = PAD;

package/src/services/voice/kokoro/pick-runtime.test.ts

@@ -0,0 +1,91 @@
+import { describe, expect, it } from "vitest";
+
+import type { ElizaInferenceFfi } from "../ffi-bindings";
+import { pickKokoroRuntimeBackend } from "./pick-runtime";
+import type { KokoroModelLayout } from "./types";
+
+const LAYOUT: KokoroModelLayout = {
+	root: "/tmp/kokoro",
+	modelFile: "kokoro-82m-v1_0.gguf",
+	voicesDir: "/tmp/kokoro/voices",
+	sampleRate: 24_000,
+};
+
+/**
+ * Minimal FFI stand-in for the in-process Kokoro path: only the symbols
+ * `KokoroFfiRuntime`'s constructor touches (`kokoroSupported`, `create`) are
+ * non-trivial. Synthesis itself is exercised in the runtime test; here we only
+ * need a handle that constructs.
+ */
+function fakeKokoroFfi(supported = true): ElizaInferenceFfi {
+	return {
+		libraryPath: "/fake/libelizainference.so",
+		libraryAbiVersion: "10",
+		create: () => 1n,
+		destroy: () => {},
+		kokoroSupported: () => supported,
+		kokoroLoad: () => {},
+		kokoroSynthesize: () => new Float32Array(0),
+		kokoroSampleRate: () => 24_000,
+	} as unknown as ElizaInferenceFfi;
+}
+
+describe("pickKokoroRuntimeBackend", () => {
+	it("defaults to the in-process ffi runtime when no backend or env override is set", () => {
+		const decision = pickKokoroRuntimeBackend({
+			env: {},
+			ffi: { layout: LAYOUT, ffi: fakeKokoroFfi(), ctx: 1n },
+		});
+
+		expect(decision.backend).toBe("ffi");
+		expect(decision.runtime.id).toBe("gguf");
+		expect(decision.reason).toMatch(/default/);
+		expect(decision.reason).toMatch(/in-process/);
+	});
+
+	it("uses ffi when KOKORO_BACKEND=ffi is set via env", () => {
+		const decision = pickKokoroRuntimeBackend({
+			env: { KOKORO_BACKEND: "ffi" },
+			ffi: { layout: LAYOUT, ffi: fakeKokoroFfi(), ctx: 1n },
+		});
+
+		expect(decision.backend).toBe("ffi");
+		expect(decision.reason).toMatch(/KOKORO_BACKEND=ffi/);
+	});
+
+	it("rejects the removed HTTP fork/server backend with an actionable error", () => {
+		for (const value of ["fork", "server"]) {
+			expect(() =>
+				pickKokoroRuntimeBackend({
+					env: { KOKORO_BACKEND: value },
+					ffi: { layout: LAYOUT, ffi: fakeKokoroFfi(), ctx: 1n },
+				}),
+			).toThrow(/llama-server HTTP\) was removed/);
+		}
+	});
+
+	it("uses mock when KOKORO_BACKEND=mock is set via env", () => {
+		const decision = pickKokoroRuntimeBackend({
+			env: { KOKORO_BACKEND: "mock" },
+			mock: { sampleRate: 24_000 },
+		});
+
+		expect(decision.backend).toBe("mock");
+		expect(decision.runtime.id).toBe("mock");
+	});
+
+	it("throws on unrecognized KOKORO_BACKEND value", () => {
+		expect(() =>
+			pickKokoroRuntimeBackend({
+				env: { KOKORO_BACKEND: "bogus" },
+				ffi: { layout: LAYOUT, ffi: fakeKokoroFfi(), ctx: 1n },
+			}),
+		).toThrow(/KOKORO_BACKEND must be one of/);
+	});
+
+	it("throws when the ffi backend is selected without its options block", () => {
+		expect(() =>
+			pickKokoroRuntimeBackend({ env: { KOKORO_BACKEND: "ffi" } }),
+		).toThrow(/requires `inputs.ffi`/);
+	});
+});

package/src/services/voice/kokoro/pick-runtime.ts

@@ -0,0 +1,130 @@
+/**
+ * Kokoro runtime selector — picks the resolved-runtime path.
+ *
+ * The env knob is `KOKORO_BACKEND`:
+ *
+ *   ffi   (default)  → KokoroFfiRuntime → in-process synthesis through the
+ *                       fused `libelizainference` handle (the
+ *                       `eliza_inference_kokoro_*` exports, introduced at ABI
+ *                       v10; the fused library is currently ABI v12). This is
+ *                       the SOLE synthesis path on every platform.
+ *   mock             → KokoroMockRuntime. Tests only.
+ *
+ * The legacy `fork` / `server` value (POST `/v1/audio/speech` on a running
+ * llama-server) was removed — Kokoro is folded into the fused lib, so there is
+ * one runtime, not a duplicate HTTP transport. The "onnx" value was removed
+ * earlier with `onnxruntime-node`.
+ */
+
+import {
+	KokoroFfiRuntime,
+	type KokoroFfiRuntimeOptions,
+} from "./kokoro-ffi-runtime";
+import {
+	KokoroMockRuntime,
+	type KokoroMockRuntimeOptions,
+	type KokoroRuntime,
+} from "./kokoro-runtime";
+
+export type KokoroBackendId = "ffi" | "mock";
+
+export interface KokoroBackendInputs {
+	/** Override the env-resolved backend (tests / programmatic selection). */
+	backend?: KokoroBackendId;
+	/** Default backend derived from the discovered model layout. Used when no
+	 *  explicit backend and no `KOKORO_BACKEND` env override are set. When
+	 *  omitted the selector defaults to the in-process `ffi` path. */
+	defaultBackend?: KokoroBackendId;
+	/** Construction options for the in-process FFI path. Used iff backend === "ffi". */
+	ffi?: KokoroFfiRuntimeOptions;
+	/** Construction options for the mock path. */
+	mock?: KokoroMockRuntimeOptions;
+	/** Override the process.env source. */
+	env?: NodeJS.ProcessEnv;
+}
+
+export interface KokoroBackendDecision {
+	backend: KokoroBackendId;
+	/** One-line reason — surfaced to telemetry. */
+	reason: string;
+	runtime: KokoroRuntime;
+}
+
+/**
+ * Resolve the `KOKORO_BACKEND` env variable. Throws on an unrecognized value
+ * — silent fallback would hide a misconfiguration (AGENTS.md §3 "no silent
+ * fallback"). The legacy `fork` / `server` (HTTP) value is rejected with a
+ * pointer to the in-process path.
+ */
+export function readKokoroBackendFromEnv(
+	env: NodeJS.ProcessEnv = process.env,
+): KokoroBackendId | undefined {
+	const raw = env.KOKORO_BACKEND?.trim().toLowerCase();
+	if (!raw) return undefined;
+	if (raw === "ffi" || raw === "mock") return raw;
+	if (raw === "fork" || raw === "server") {
+		throw new Error(
+			"[voice/kokoro] KOKORO_BACKEND='fork'/'server' (llama-server HTTP) was " +
+				"removed — Kokoro runs in-process through the fused libelizainference. " +
+				"Use 'ffi' (default).",
+		);
+	}
+	throw new Error(
+		`[voice/kokoro] KOKORO_BACKEND must be one of 'ffi', 'mock' (got '${raw}')`,
+	);
+}
+
+/**
+ * Pick the Kokoro runtime backend.
+ *
+ *   1. Explicit `inputs.backend` wins.
+ *   2. Else env (`KOKORO_BACKEND`).
+ *   3. Else `inputs.defaultBackend`.
+ *   4. Else default → `ffi` (in-process fused handle, the only mobile-safe path).
+ *
+ * If the chosen backend's options block is missing the call throws a
+ * structured error (no silent downgrade). Callers must wire the options
+ * for the backends they enable.
+ */
+export function pickKokoroRuntimeBackend(
+	inputs: KokoroBackendInputs,
+): KokoroBackendDecision {
+	const fromEnv = readKokoroBackendFromEnv(inputs.env);
+	const fromDefault = inputs.backend === undefined && fromEnv === undefined;
+	const backend: KokoroBackendId =
+		inputs.backend ?? fromEnv ?? inputs.defaultBackend ?? "ffi";
+
+	if (backend === "ffi") {
+		if (!inputs.ffi) {
+			throw new Error(
+				"[voice/kokoro] KOKORO_BACKEND=ffi requires `inputs.ffi` " +
+					"(layout). Pass the resolved Kokoro layout so the in-process " +
+					"fused engine can load the GGUF + voice .bin.",
+			);
+		}
+		return {
+			backend,
+			reason: inputs.backend
+				? "explicit backend=ffi (in-process fused libelizainference)"
+				: fromEnv
+					? `KOKORO_BACKEND=${fromEnv} → ffi (in-process fused libelizainference)`
+					: fromDefault && inputs.defaultBackend === "ffi"
+						? "model layout default → ffi (in-process fused libelizainference)"
+						: "default → ffi (in-process fused libelizainference)",
+			runtime: new KokoroFfiRuntime(inputs.ffi),
+		};
+	}
+
+	// backend === "mock"
+	if (!inputs.mock) {
+		throw new Error(
+			"[voice/kokoro] KOKORO_BACKEND=mock requires `inputs.mock` " +
+				"(sampleRate). Construct the runtime with explicit test options.",
+		);
+	}
+	return {
+		backend,
+		reason: "explicit backend=mock (test fixture)",
+		runtime: new KokoroMockRuntime(inputs.mock),
+	};
+}

package/src/services/voice/kokoro/runtime-selection.ts

@@ -0,0 +1,64 @@
+/**
+ * Voice backend selection — Kokoro is the only on-device TTS backend.
+ *
+ * OmniVoice TTS was retired (it was an autoregressive LM-based synth: heavier,
+ * slower TTFB, and only it could voice-clone). Kokoro (StyleTTS2 / iSTFTNet,
+ * non-autoregressive) is faster, smaller, and ships in every bundle — desktop
+ * and mobile alike — so the selector collapses to a single auditable answer.
+ * The function + env reader are kept (rather than inlined) so the engine layer
+ * and tests retain one seam to ask "is a TTS backend available?".
+ */
+
+export type VoiceBackendChoice = "kokoro";
+
+/** Retained for the env override; `auto` and `kokoro` both resolve to Kokoro. */
+export type VoiceBackendMode = "kokoro" | "auto";
+
+export interface VoiceBackendInputs {
+	/** Caller-set mode. Defaults to `auto`; both modes resolve to Kokoro. */
+	mode?: VoiceBackendMode;
+	/** Whether Kokoro model artifacts are present on disk. The selector throws
+	 *  rather than returning a backend when this is `false` — no silent downgrade. */
+	kokoroAvailable: boolean;
+	/** True on mobile (iOS / Android) builds — informational only now. */
+	mobile?: boolean;
+}
+
+export interface VoiceBackendDecision {
+	backend: VoiceBackendChoice;
+	/** One-line reason — surfaced to telemetry. */
+	reason: string;
+}
+
+/** Resolve the env override (`ELIZA_TTS_BACKEND=kokoro|auto`). */
+export function readVoiceBackendModeFromEnv(
+	env: NodeJS.ProcessEnv = process.env,
+): VoiceBackendMode | undefined {
+	const raw = env.ELIZA_TTS_BACKEND?.trim().toLowerCase();
+	if (!raw) return undefined;
+	if (raw === "kokoro" || raw === "auto") return raw;
+	if (raw === "omnivoice") {
+		throw new Error(
+			"[voice] ELIZA_TTS_BACKEND=omnivoice is retired — OmniVoice TTS was removed; Kokoro is the only on-device TTS backend.",
+		);
+	}
+	throw new Error(
+		`[voice] ELIZA_TTS_BACKEND must be 'kokoro' or 'auto' (got '${raw}')`,
+	);
+}
+
+export function selectVoiceBackend(
+	inputs: VoiceBackendInputs,
+): VoiceBackendDecision {
+	if (!inputs.kokoroAvailable) {
+		throw new Error(
+			"[voice] Kokoro model artifacts are not present on disk; Kokoro is the only on-device TTS backend (install an Eliza-1 bundle or stage the Kokoro GGUF + at least one voice .bin).",
+		);
+	}
+	return {
+		backend: "kokoro",
+		reason: inputs.mobile
+			? "mobile platform — Kokoro (the only on-device TTS backend)"
+			: "Kokoro (the only on-device TTS backend)",
+	};
+}