@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/voice/kokoro/kokoro-ffi-runtime.ts

@@ -0,0 +1,233 @@
+/**
+ * In-process Kokoro-82M runtime over the fused `libelizainference` FFI
+ * (the `eliza_inference_kokoro_*` exports — introduced at ABI v10; the fused
+ * library is currently ABI v12, which adds EOT (v11) and ASR word timestamps
+ * (v12) on top, so these symbols are present in every current build — see
+ * `ELIZA_INFERENCE_ABI_VERSION` in ffi-bindings.ts).
+ *
+ * This is the canonical Kokoro execution path on every platform. It replaces
+ * the local-TCP `KokoroGgufRuntime` (POST `/v1/audio/speech` on a running
+ * llama-server) for the mobile case — iOS and Google Play forbid the app
+ * opening a local TCP socket, so the HTTP→llama-server route cannot ship there.
+ * Kokoro synthesizes through the same dlopen()-ed handle as OmniVoice: the
+ * fused build links Eliza-1's Kokoro engine (its own GGUF reader + iSTFT
+ * decoder) behind `eliza_inference_kokoro_supported/load/synthesize/sample_rate`.
+ *
+ * Ownership: this runtime owns its own FFI handle + context. The context is
+ * created with `create(bundleRoot)` anchored at the bundle root (or the Kokoro
+ * model root when there is no Eliza-1 bundle), mirroring how the desktop fused
+ * text runtime obtains its ctx. The GGUF + the active voice `.bin` are loaded
+ * once via `kokoroLoad` and reloaded only when the requested voice changes.
+ *
+ * No silent fallback (AGENTS.md §3): when the loaded library does not export
+ * the Kokoro symbols (`kokoroSupported() === false`) or the model/voice files
+ * are missing, construction / first synthesis throws a structured
+ * `VoiceLifecycleError` rather than dropping back to the TCP route.
+ */
+
+import { existsSync } from "node:fs";
+import path from "node:path";
+import { logger } from "@elizaos/core";
+import { resolveFusedLibraryPath } from "../../desktop-fused-ffi-backend-runtime";
+import {
+	type ElizaInferenceContextHandle,
+	type ElizaInferenceFfi,
+	loadElizaInferenceFfi,
+} from "../ffi-bindings";
+import { VoiceLifecycleError } from "../lifecycle";
+import type { KokoroRuntime, KokoroRuntimeInputs } from "./kokoro-runtime";
+import type { KokoroModelLayout } from "./types";
+import { resolveKokoroVoiceOrDefault } from "./voices";
+
+/** Kokoro v1.0 style-vector inner dimension. */
+const KOKORO_STYLE_DIM = 256;
+
+/**
+ * Per-synthesis output ceiling. Kokoro v1.0 emits 24 kHz fp32 PCM; 30 s of
+ * headroom (720 000 samples) bounds a single phrase synthesis well past the
+ * longest chunk the phrase chunker will hand us. The library returns the real
+ * sample count, which we slice to — this is only the allocation cap.
+ */
+const MAX_OUTPUT_SAMPLES = 30 * 24_000;
+
+export interface KokoroFfiRuntimeOptions {
+	/** Resolved on-disk Kokoro layout (GGUF filename + voices dir + root). */
+	layout: KokoroModelLayout;
+	/**
+	 * Directory the FFI context anchors at (`create(bundleRoot)`). Defaults to
+	 * the Kokoro model root, which is sufficient for the standalone Kokoro
+	 * engine — it loads the GGUF + voice `.bin` by explicit absolute path, not
+	 * by bundle convention.
+	 */
+	bundleRoot?: string;
+	/**
+	 * Inject a pre-loaded FFI handle (the desktop fused engine already owns one).
+	 * When omitted the runtime loads its own via `resolveFusedLibraryPath`.
+	 */
+	ffi?: ElizaInferenceFfi;
+	/**
+	 * Inject a context to reuse. When omitted the runtime creates its own with
+	 * `ffi.create(bundleRoot)` and destroys it on `dispose`.
+	 */
+	ctx?: ElizaInferenceContextHandle;
+}
+
+export class KokoroFfiRuntime implements KokoroRuntime {
+	readonly id = "gguf" as const;
+	readonly sampleRate: number;
+
+	private readonly layout: KokoroModelLayout;
+	private readonly ffi: ElizaInferenceFfi;
+	private readonly ownsFfi: boolean;
+	private readonly ctx: ElizaInferenceContextHandle;
+	private readonly ownsCtx: boolean;
+	/** Voice id currently resident on the ctx (null until first load). */
+	private loadedVoiceId: string | null = null;
+	private disposed = false;
+
+	constructor(opts: KokoroFfiRuntimeOptions) {
+		this.layout = opts.layout;
+		const bundleRoot = opts.bundleRoot ?? opts.layout.root;
+
+		const provided = opts.ffi;
+		if (provided) {
+			this.ffi = provided;
+			this.ownsFfi = false;
+		} else {
+			const libPath = resolveFusedLibraryPath(bundleRoot);
+			if (!libPath) {
+				throw new VoiceLifecycleError(
+					"kernel-missing",
+					`[KokoroFfiRuntime] fused libelizainference not found for the in-process Eliza-1 Kokoro engine (anchored at ${bundleRoot}). ` +
+						"Set ELIZA_INFERENCE_LIBRARY or build via packages/app-core/scripts/build-llama-cpp-mtp.mjs.",
+				);
+			}
+			this.ffi = loadElizaInferenceFfi(libPath);
+			this.ownsFfi = true;
+		}
+
+		if (
+			typeof this.ffi.kokoroSupported !== "function" ||
+			!this.ffi.kokoroSupported()
+		) {
+			if (this.ownsFfi) this.ffi.close();
+			throw new VoiceLifecycleError(
+				"kernel-missing",
+				`[KokoroFfiRuntime] the loaded libelizainference (ABI v${this.ffi.libraryAbiVersion}) does not link the in-process Eliza-1 Kokoro engine. ` +
+					"Rebuild with the Kokoro engine enabled — the mobile path must not fall back to the local-TCP /v1/audio/speech route.",
+			);
+		}
+
+		if (opts.ctx !== undefined) {
+			this.ctx = opts.ctx;
+			this.ownsCtx = false;
+		} else {
+			this.ctx = this.ffi.create(bundleRoot);
+			this.ownsCtx = true;
+		}
+
+		this.sampleRate = this.layout.sampleRate;
+	}
+
+	async synthesize(args: KokoroRuntimeInputs): Promise<{ cancelled: boolean }> {
+		if (this.disposed) {
+			throw new VoiceLifecycleError(
+				"kernel-missing",
+				"[KokoroFfiRuntime] synthesize called after dispose",
+			);
+		}
+		this.ensureVoiceLoaded(args.voice.id);
+
+		if (args.cancelSignal.cancelled) {
+			args.onChunk({
+				pcm: new Float32Array(0),
+				sampleRate: this.sampleRate,
+				isFinal: true,
+			});
+			return { cancelled: true };
+		}
+
+		const maxSamples = args.maxSamples ?? MAX_OUTPUT_SAMPLES;
+		// The Kokoro engine produces the full waveform in one synchronous
+		// forward. The text it phonemizes internally is the same phoneme string
+		// the llama-server `/v1/audio/speech` path sends as `input`.
+		const pcm = this.kokoroSynthesize(args.phonemes.phonemes, maxSamples);
+
+		let cancelled = false;
+		if (args.cancelSignal.cancelled) {
+			cancelled = true;
+		} else if (pcm.length > 0) {
+			const want = args.onChunk({
+				pcm,
+				sampleRate: this.sampleRate,
+				isFinal: false,
+			});
+			if (want === true || args.cancelSignal.cancelled) cancelled = true;
+		}
+
+		args.onChunk({
+			pcm: new Float32Array(0),
+			sampleRate: this.sampleRate,
+			isFinal: true,
+		});
+		return { cancelled };
+	}
+
+	dispose(): void {
+		if (this.disposed) return;
+		this.disposed = true;
+		if (this.ownsCtx) this.ffi.destroy(this.ctx);
+		if (this.ownsFfi) this.ffi.close();
+	}
+
+	/**
+	 * Load the GGUF + the requested voice `.bin` into the ctx, reloading only
+	 * when the voice changes (Kokoro keeps the model resident; swapping voices
+	 * is a cheap re-load of the 256-float style tensor).
+	 */
+	private ensureVoiceLoaded(requestedVoiceId: string): void {
+		const voice = resolveKokoroVoiceOrDefault(requestedVoiceId);
+		if (this.loadedVoiceId === voice.id) return;
+
+		const ggufPath = path.join(this.layout.root, this.layout.modelFile);
+		const voiceBinPath = path.join(this.layout.voicesDir, voice.file);
+		if (!existsSync(ggufPath)) {
+			throw new VoiceLifecycleError(
+				"kernel-missing",
+				`[KokoroFfiRuntime] Eliza-1 Kokoro model file not found at ${ggufPath}`,
+			);
+		}
+		if (!existsSync(voiceBinPath)) {
+			throw new VoiceLifecycleError(
+				"kernel-missing",
+				`[KokoroFfiRuntime] Eliza-1 voice preset not found at ${voiceBinPath} for voice ${voice.id}`,
+			);
+		}
+		if (typeof this.ffi.kokoroLoad !== "function") {
+			throw new VoiceLifecycleError(
+				"kernel-missing",
+				"[KokoroFfiRuntime] eliza_inference_kokoro_load is not exported by the loaded build",
+			);
+		}
+		this.ffi.kokoroLoad({
+			ctx: this.ctx,
+			ggufPath,
+			voiceBinPath,
+			styleDim: voice.dim ?? KOKORO_STYLE_DIM,
+		});
+		this.loadedVoiceId = voice.id;
+		logger.info(
+			`[KokoroFfiRuntime] loaded Eliza-1 voice ${voice.id} from ${voiceBinPath}`,
+		);
+	}
+
+	private kokoroSynthesize(text: string, maxSamples: number): Float32Array {
+		if (typeof this.ffi.kokoroSynthesize !== "function") {
+			throw new VoiceLifecycleError(
+				"kernel-missing",
+				"[KokoroFfiRuntime] eliza_inference_kokoro_synthesize is not exported by the loaded build",
+			);
+		}
+		return this.ffi.kokoroSynthesize({ ctx: this.ctx, text, maxSamples });
+	}
+}

package/src/services/voice/kokoro/kokoro-runtime.ts

@@ -0,0 +1,170 @@
+/**
+ * Kokoro-82M model runner.
+ *
+ * Execution paths:
+ *
+ *   1. GGUF via llama-server (default). When the host llama-server advertises
+ *      a Kokoro-capable build and exposes `/v1/audio/speech`, we POST text in
+ *      and stream PCM out.
+ *
+ *   2. Python subprocess — eval-loop only. Spawns `python -m kokoro_tts`.
+ *      Never the default in production.
+ */
+
+import type { KokoroPhonemeSequence, KokoroVoicePack } from "./types";
+
+/** Pinned GGUF candidate location (carried by our llama.cpp fork). The
+ *  runtime references this only for diagnostics; the fork-side builder
+ *  produces the file at this path. */
+export const KOKORO_GGUF_REL_PATH = "voice/kokoro-82m-v1_0.gguf";
+
+/** One synthesized PCM segment delivered to the streaming backend. */
+export interface KokoroRuntimeChunk {
+	pcm: Float32Array;
+	sampleRate: number;
+	isFinal: boolean;
+}
+
+/**
+ * Construction-time inputs for a runtime instance. The voice pack contains
+ * the style tensor reference; the runtime is responsible for resolving the
+ * bytes off `layout.voicesDir/<file>`.
+ */
+export interface KokoroRuntimeInputs {
+	phonemes: KokoroPhonemeSequence;
+	voice: KokoroVoicePack;
+	/**
+	 * Output sample budget. The runtime always honours the model's native
+	 * sample rate (`layout.sampleRate`, usually 24 kHz) — this caps the
+	 * total samples to prevent runaway generation. Defaults to 16 seconds
+	 * at the layout sample rate (matches the longest phrase the chunker
+	 * will emit + headroom).
+	 */
+	maxSamples?: number;
+	/** Cancellation signal — polled at chunk boundaries. */
+	cancelSignal: { cancelled: boolean };
+	/** Per-chunk callback; returning `true` cancels the rest of the run. */
+	onChunk: (chunk: KokoroRuntimeChunk) => boolean | undefined;
+}
+
+/** Shared runtime contract — `KokoroTtsBackend` depends on this, not the
+ *  concrete classes. Tests inject a mock. */
+export interface KokoroRuntime {
+	readonly id: "gguf" | "python" | "mock";
+	readonly sampleRate: number;
+	synthesize(args: KokoroRuntimeInputs): Promise<{ cancelled: boolean }>;
+	dispose(): void;
+}
+
+// ---------------------------------------------------------------------------
+// Python subprocess path — eval-loop only.
+// ---------------------------------------------------------------------------
+
+export interface KokoroPythonRuntimeOptions {
+	pythonBinary: string;
+	/** Resolved layout — the subprocess discovers the model under here. */
+	layout: { root: string; sampleRate: number };
+	/** Optional env passed through to the subprocess. */
+	env?: NodeJS.ProcessEnv;
+}
+
+/**
+ * Subprocess-backed runtime. Spawns `python -m kokoro_tts ...` per
+ * synthesis call (no warm-pool — the Python path is the *eval* path, not
+ * the realtime path). Production code paths never select this; the
+ * fine-tune evaluator wires it explicitly.
+ */
+export class KokoroPythonRuntime implements KokoroRuntime {
+	readonly id = "python" as const;
+	readonly sampleRate: number;
+
+	constructor(opts: KokoroPythonRuntimeOptions) {
+		this.sampleRate = opts.layout.sampleRate;
+	}
+
+	async synthesize(
+		_args: KokoroRuntimeInputs,
+	): Promise<{ cancelled: boolean }> {
+		// The eval driver in `packages/training` is the canonical caller and
+		// already wires `child_process.spawn`. Surfacing a clear error here
+		// keeps the production runtime from accidentally enabling this path.
+		throw new Error(
+			"[kokoro] KokoroPythonRuntime is eval-only — use it from the fine-tune driver, not the runtime scheduler",
+		);
+	}
+
+	dispose(): void {
+		// No long-lived state.
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Mock runtime — synthesizes a sine sweep keyed to phoneme count so tests
+// can observe deterministic PCM without loading a model.
+// ---------------------------------------------------------------------------
+
+export interface KokoroMockRuntimeOptions {
+	sampleRate: number;
+	/** Total samples emitted per synthesis call. */
+	totalSamples?: number;
+	/** Number of body chunks to split the output across. */
+	chunkCount?: number;
+}
+
+export class KokoroMockRuntime implements KokoroRuntime {
+	readonly id = "mock" as const;
+	readonly sampleRate: number;
+	private readonly opts: Required<KokoroMockRuntimeOptions>;
+	calls = 0;
+
+	constructor(opts: KokoroMockRuntimeOptions) {
+		this.sampleRate = opts.sampleRate;
+		this.opts = {
+			sampleRate: opts.sampleRate,
+			totalSamples: opts.totalSamples ?? Math.floor(opts.sampleRate * 0.2),
+			chunkCount: opts.chunkCount ?? 4,
+		};
+	}
+
+	async synthesize(args: KokoroRuntimeInputs): Promise<{ cancelled: boolean }> {
+		this.calls++;
+		const { totalSamples, chunkCount } = this.opts;
+		const perChunk = Math.max(1, Math.ceil(totalSamples / chunkCount));
+		const freqHz = 100 + (args.phonemes.ids.length % 200);
+		let written = 0;
+		let cancelled = false;
+		for (let off = 0; off < totalSamples; off += perChunk) {
+			if (args.cancelSignal.cancelled) {
+				cancelled = true;
+				break;
+			}
+			const n = Math.min(perChunk, totalSamples - off);
+			const pcm = new Float32Array(n);
+			for (let i = 0; i < n; i++) {
+				const t = (off + i) / this.sampleRate;
+				pcm[i] = Math.sin(2 * Math.PI * freqHz * t) * 0.1;
+			}
+			written += n;
+			const want = args.onChunk({
+				pcm,
+				sampleRate: this.sampleRate,
+				isFinal: false,
+			});
+			if (want === true || args.cancelSignal.cancelled) {
+				cancelled = true;
+				break;
+			}
+		}
+		args.onChunk({
+			pcm: new Float32Array(0),
+			sampleRate: this.sampleRate,
+			isFinal: true,
+		});
+		void written;
+		return { cancelled };
+	}
+
+	dispose(): void {
+		/* nothing */
+	}
+}

package/src/services/voice/kokoro/phoneme-stream.ts

@@ -0,0 +1,123 @@
+/**
+ * Phoneme streaming for Kokoro-82M.
+ *
+ * Kokoro consumes a sequence of phoneme ids (espeak-ng IPA tokenised against a
+ * small fixed vocab). The scheduler emits phrases at punctuation or
+ * `phoneme-stream` boundaries (see `voice/phrase-chunker.ts` `chunkOn`
+ * option). This module is the seam between those phrase boundaries and the
+ * model's input tensor:
+ *
+ *   text → phonemizer.phonemize() → KokoroPhonemeSequence (ids) → runtime
+ *
+ * For maximum responsiveness the runtime can call `streamPhonemes()` against
+ * an async text iterator (chunked draft tokens) and forward each window of
+ * accumulated ids as soon as a phoneme boundary fires. The default `flushAt`
+ * is one phoneme — i.e. emit progress per id — but production deployments
+ * lift this to ~8 phonemes to amortise the ONNX forward pass on small
+ * windows. This file intentionally has no dependency on the rest of the
+ * voice scaffold so it can be reused by the fine-tune evaluator script.
+ */
+
+import type { KokoroPhonemeSequence, KokoroPhonemizer } from "./types";
+
+export interface PhonemeStreamWindow {
+	/** Cumulative ids since stream start. The runtime can re-tokenise or
+	 *  carry state by id; the simplest implementation forwards the full
+	 *  window each call. */
+	ids: Int32Array;
+	/** Cumulative phoneme string for debugging / display. */
+	phonemes: string;
+	/** True for the final window in the stream. */
+	isFinal: boolean;
+}
+
+export interface StreamPhonemesOptions {
+	phonemizer: KokoroPhonemizer;
+	lang: string;
+	/** Emit a window every N new phoneme ids. Default 8 (≈ first audio after a
+	 *  short syllable cluster — matches the phrase chunker's default cap). */
+	flushAt?: number;
+}
+
+/**
+ * Phonemize an async text source and emit cumulative windows. The caller
+ * consumes the iterator with `for await (const window of streamPhonemes(…))`.
+ * A pull-style API keeps this independent of the scheduler's event loop —
+ * the bench harness and the eval loop both reuse it without taking on a
+ * scheduler dependency.
+ */
+export async function* streamPhonemes(
+	textChunks: AsyncIterable<string>,
+	opts: StreamPhonemesOptions,
+): AsyncIterable<PhonemeStreamWindow> {
+	const flushAt = Math.max(1, opts.flushAt ?? 8);
+	const idsAcc: number[] = [];
+	let phonemesAcc = "";
+	let lastFlushAt = 0;
+	let leftover = "";
+
+	for await (const chunk of textChunks) {
+		if (!chunk) continue;
+		leftover += chunk;
+		// Only phonemize when we have at least a whole word to feed to the
+		// phonemizer — espeak-ng is significantly more accurate when fed
+		// word-aligned input. Look back to the last whitespace as the split.
+		const split = leftover.lastIndexOf(" ");
+		if (split === -1) continue;
+		const head = leftover.slice(0, split);
+		leftover = leftover.slice(split + 1);
+		const seq = await opts.phonemizer.phonemize(head, opts.lang);
+		appendSeq(seq, idsAcc);
+		phonemesAcc += seq.phonemes;
+		if (idsAcc.length - lastFlushAt >= flushAt) {
+			lastFlushAt = idsAcc.length;
+			yield {
+				ids: Int32Array.from(idsAcc),
+				phonemes: phonemesAcc,
+				isFinal: false,
+			};
+		}
+	}
+
+	if (leftover.length > 0) {
+		const seq = await opts.phonemizer.phonemize(leftover, opts.lang);
+		appendSeq(seq, idsAcc);
+		phonemesAcc += seq.phonemes;
+	}
+	yield {
+		ids: Int32Array.from(idsAcc),
+		phonemes: phonemesAcc,
+		isFinal: true,
+	};
+}
+
+function appendSeq(seq: KokoroPhonemeSequence, target: number[]): void {
+	// The phonemizer emits a sequence framed with BOS/EOS — strip both when
+	// accumulating windows so the model sees one BOS at the head and one EOS
+	// at the tail. Defensive against phonemizers that omit framing (the
+	// accumulator simply appends raw ids in that case).
+	const ids = seq.ids;
+	let start = 0;
+	let end = ids.length;
+	if (ids.length >= 2) {
+		// Heuristic: ids ≤ 2 are <pad>/<s>/</s> in the bundled vocab.
+		if (ids[0] !== undefined && ids[0] <= 2) start = 1;
+		if (ids[end - 1] !== undefined && (ids[end - 1] as number) <= 2) end -= 1;
+	}
+	for (let i = start; i < end; i++) {
+		const id = ids[i];
+		if (id !== undefined) target.push(id);
+	}
+}
+
+/** Synchronous variant for whole-phrase callers (the scheduler dispatches
+ *  one phrase at a time in the default `punctuation` mode). Returns the
+ *  full id array — equivalent to draining `streamPhonemes` on a single-item
+ *  iterator and taking the last window. */
+export async function phonemizePhrase(
+	text: string,
+	opts: StreamPhonemesOptions,
+): Promise<PhonemeStreamWindow> {
+	const seq = await opts.phonemizer.phonemize(text, opts.lang);
+	return { ids: seq.ids, phonemes: seq.phonemes, isFinal: true };
+}