@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/services/voice/kokoro/index.ts

@@ -0,0 +1,79 @@
+/**
+ * Public barrel for the Kokoro-82M TTS adapter.
+ *
+ * External callers (the engine layer, the bench harness, tests) should
+ * import from `./kokoro` rather than reaching into individual files. The
+ * internal layout may change; this surface is stable.
+ */
+
+export type { KokoroTtsBackendDeps } from "./kokoro-backend";
+export { KokoroTtsBackend } from "./kokoro-backend";
+export type { KokoroFfiRuntimeOptions } from "./kokoro-ffi-runtime";
+export { KokoroFfiRuntime } from "./kokoro-ffi-runtime";
+export type {
+	KokoroMockRuntimeOptions,
+	KokoroPythonRuntimeOptions,
+	KokoroRuntime,
+	KokoroRuntimeChunk,
+	KokoroRuntimeInputs,
+} from "./kokoro-runtime";
+export {
+	KOKORO_GGUF_REL_PATH,
+	KokoroMockRuntime,
+	KokoroPythonRuntime,
+} from "./kokoro-runtime";
+export type {
+	PhonemeStreamWindow,
+	StreamPhonemesOptions,
+} from "./phoneme-stream";
+
+export {
+	phonemizePhrase,
+	streamPhonemes,
+} from "./phoneme-stream";
+export {
+	FallbackG2PPhonemizer,
+	KOKORO_PAD_ID,
+	NpmPhonemizePhonemizer,
+	resolvePhonemizer,
+} from "./phonemizer";
+export type {
+	KokoroBackendDecision,
+	KokoroBackendId,
+	KokoroBackendInputs,
+} from "./pick-runtime";
+export {
+	pickKokoroRuntimeBackend,
+	readKokoroBackendFromEnv,
+} from "./pick-runtime";
+export type {
+	VoiceBackendChoice,
+	VoiceBackendDecision,
+	VoiceBackendInputs,
+	VoiceBackendMode,
+} from "./runtime-selection";
+export {
+	readVoiceBackendModeFromEnv,
+	selectVoiceBackend,
+} from "./runtime-selection";
+export type {
+	KokoroBackendOptions,
+	KokoroModelLayout,
+	KokoroPhonemeSequence,
+	KokoroPhonemizer,
+	KokoroVoiceId,
+	KokoroVoicePack,
+} from "./types";
+export {
+	KokoroModelMissingError,
+	KokoroPhonemizerError,
+} from "./types";
+export {
+	findKokoroVoice,
+	KOKORO_DEFAULT_VOICE_ID,
+	KOKORO_VOICE_PACKS,
+	listKokoroVoiceIds,
+	listKokoroVoicesByLang,
+	listKokoroVoicesByTag,
+	resolveKokoroVoiceOrDefault,
+} from "./voices";

package/src/services/voice/kokoro/kokoro-backend.d.ts

@@ -0,0 +1,72 @@
+/**
+ * Kokoro-82M TTS backend.
+ *
+ * Implements the same `OmniVoiceBackend + StreamingTtsBackend` seam that
+ * `FfiOmniVoiceBackend` (the OmniVoice path) satisfies, so a
+ * `VoiceScheduler` instance does not need to know which TTS family it is
+ * driving. The runtime selection layer (`runtime-selection.ts`) picks
+ * between this and `FfiOmniVoiceBackend` based on hardware tier and the
+ * caller's first-audio-latency target.
+ *
+ * The actual model inference is delegated to a `KokoroRuntime` instance
+ * (GGUF / mock) — this class owns:
+ *   - phonemizer resolution + per-phrase phonemize call,
+ *   - voice-pack resolution against `SpeakerPreset.voiceId`,
+ *   - streaming-protocol bookkeeping (cancel signal polling, final tail).
+ *
+ * No fallback sludge: if the runtime is unavailable, the backend throws
+ * on first synthesis rather than emitting silent zero PCM (AGENTS.md §3).
+ */
+import type { AudioChunk, OmniVoiceBackend, Phrase, SpeakerPreset, StreamingTtsBackend, TtsPcmChunk } from "../types";
+import type { KokoroRuntime } from "./kokoro-runtime";
+import type { KokoroBackendOptions } from "./types";
+export interface KokoroTtsBackendDeps extends KokoroBackendOptions {
+    /** The concrete model runner. Wire `KokoroFfiRuntime` (in-process fused
+     *  libelizainference) in production and `KokoroMockRuntime` in tests. */
+    runtime: KokoroRuntime;
+}
+/**
+ * `KokoroTtsBackend` is a streaming-only TTS backend. The model produces
+ * the full waveform in one forward, but we surface it as one body chunk +
+ * tail so the scheduler protocol is identical for both backends.
+ */
+export declare class KokoroTtsBackend implements OmniVoiceBackend, StreamingTtsBackend {
+    readonly id: "kokoro";
+    private readonly runtime;
+    private readonly defaultVoiceId;
+    private readonly streamingChunkSamples;
+    private phonemizer;
+    private readonly phonemizerOverride?;
+    constructor(deps: KokoroTtsBackendDeps);
+    /** Native sample rate of the model output (24 kHz for Kokoro v1.0). */
+    get sampleRate(): number;
+    /** Always true — `KokoroTtsBackend` satisfies `StreamingTtsBackend`. */
+    supportsStreamingTts(): boolean;
+    /**
+     * One-shot synthesis. Drives the streaming path internally and
+     * concatenates chunks. Cancellation observed at chunk boundaries.
+     */
+    synthesize(args: {
+        phrase: Phrase;
+        preset: SpeakerPreset;
+        cancelSignal: {
+            cancelled: boolean;
+        };
+        onKernelTick?: () => void;
+    }): Promise<AudioChunk>;
+    synthesizeStream(args: {
+        phrase: Phrase;
+        preset: SpeakerPreset;
+        cancelSignal: {
+            cancelled: boolean;
+        };
+        onChunk: (chunk: TtsPcmChunk) => boolean | undefined;
+        onKernelTick?: () => void;
+    }): Promise<{
+        cancelled: boolean;
+    }>;
+    dispose(): void;
+    private resolveVoice;
+    private ensurePhonemizer;
+}
+//# sourceMappingURL=kokoro-backend.d.ts.map

package/src/services/voice/kokoro/kokoro-backend.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"kokoro-backend.d.ts","sourceRoot":"","sources":["kokoro-backend.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,KAAK,EACX,UAAU,EACV,gBAAgB,EAChB,MAAM,EACN,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,MAAM,UAAU,CAAC;AAClB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEtD,OAAO,KAAK,EACX,oBAAoB,EAGpB,MAAM,SAAS,CAAC;AAGjB,MAAM,WAAW,oBAAqB,SAAQ,oBAAoB;IACjE;6EACyE;IACzE,OAAO,EAAE,aAAa,CAAC;CACvB;AAED;;;;GAIG;AACH,qBAAa,gBAAiB,YAAW,gBAAgB,EAAE,mBAAmB;IAC7E,QAAQ,CAAC,EAAE,EAAG,QAAQ,CAAU;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAgB;IACxC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAS;IAC/C,OAAO,CAAC,UAAU,CAAiC;IACnD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAmB;gBAE3C,IAAI,EAAE,oBAAoB;IAQtC,uEAAuE;IACvE,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED,wEAAwE;IACxE,oBAAoB,IAAI,OAAO;IAI/B;;;OAGG;IACG,UAAU,CAAC,IAAI,EAAE;QACtB,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,aAAa,CAAC;QACtB,YAAY,EAAE;YAAE,SAAS,EAAE,OAAO,CAAA;SAAE,CAAC;QACrC,YAAY,CAAC,EAAE,MAAM,IAAI,CAAC;KAC1B,GAAG,OAAO,CAAC,UAAU,CAAC;IAgCjB,gBAAgB,CAAC,IAAI,EAAE;QAC5B,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,aAAa,CAAC;QACtB,YAAY,EAAE;YAAE,SAAS,EAAE,OAAO,CAAA;SAAE,CAAC;QACrC,OAAO,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,OAAO,GAAG,SAAS,CAAC;QACrD,YAAY,CAAC,EAAE,MAAM,IAAI,CAAC;KAC1B,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,OAAO,CAAA;KAAE,CAAC;IAkEnC,OAAO,IAAI,IAAI;IAIf,OAAO,CAAC,YAAY;YASN,gBAAgB;CAM9B"}

package/src/services/voice/kokoro/kokoro-backend.ts

@@ -0,0 +1,207 @@
+/**
+ * Kokoro-82M TTS backend.
+ *
+ * Implements the same `OmniVoiceBackend + StreamingTtsBackend` seam that
+ * `FfiOmniVoiceBackend` (the OmniVoice path) satisfies, so a
+ * `VoiceScheduler` instance does not need to know which TTS family it is
+ * driving. The runtime selection layer (`runtime-selection.ts`) picks
+ * between this and `FfiOmniVoiceBackend` based on hardware tier and the
+ * caller's first-audio-latency target.
+ *
+ * The actual model inference is delegated to a `KokoroRuntime` instance
+ * (GGUF / mock) — this class owns:
+ *   - phonemizer resolution + per-phrase phonemize call,
+ *   - voice-pack resolution against `SpeakerPreset.voiceId`,
+ *   - streaming-protocol bookkeeping (cancel signal polling, final tail).
+ *
+ * No fallback sludge: if the runtime is unavailable, the backend throws
+ * on first synthesis rather than emitting silent zero PCM (AGENTS.md §3).
+ */
+
+import type {
+	AudioChunk,
+	OmniVoiceBackend,
+	Phrase,
+	SpeakerPreset,
+	StreamingTtsBackend,
+	TtsPcmChunk,
+} from "../types";
+import type { KokoroRuntime } from "./kokoro-runtime";
+import { resolvePhonemizer } from "./phonemizer";
+import type {
+	KokoroBackendOptions,
+	KokoroPhonemizer,
+	KokoroVoicePack,
+} from "./types";
+import { resolveKokoroVoiceOrDefault } from "./voices";
+
+export interface KokoroTtsBackendDeps extends KokoroBackendOptions {
+	/** The concrete model runner. Wire `KokoroFfiRuntime` (in-process fused
+	 *  libelizainference) in production and `KokoroMockRuntime` in tests. */
+	runtime: KokoroRuntime;
+}
+
+/**
+ * `KokoroTtsBackend` is a streaming-only TTS backend. The model produces
+ * the full waveform in one forward, but we surface it as one body chunk +
+ * tail so the scheduler protocol is identical for both backends.
+ */
+export class KokoroTtsBackend implements OmniVoiceBackend, StreamingTtsBackend {
+	readonly id = "kokoro" as const;
+	private readonly runtime: KokoroRuntime;
+	private readonly defaultVoiceId: string;
+	private readonly streamingChunkSamples: number;
+	private phonemizer: KokoroPhonemizer | null = null;
+	private readonly phonemizerOverride?: KokoroPhonemizer;
+
+	constructor(deps: KokoroTtsBackendDeps) {
+		this.runtime = deps.runtime;
+		this.defaultVoiceId = deps.defaultVoiceId;
+		this.streamingChunkSamples =
+			deps.streamingChunkSamples ?? Math.floor(deps.layout.sampleRate / 4);
+		this.phonemizerOverride = deps.phonemizer;
+	}
+
+	/** Native sample rate of the model output (24 kHz for Kokoro v1.0). */
+	get sampleRate(): number {
+		return this.runtime.sampleRate;
+	}
+
+	/** Always true — `KokoroTtsBackend` satisfies `StreamingTtsBackend`. */
+	supportsStreamingTts(): boolean {
+		return true;
+	}
+
+	/**
+	 * One-shot synthesis. Drives the streaming path internally and
+	 * concatenates chunks. Cancellation observed at chunk boundaries.
+	 */
+	async synthesize(args: {
+		phrase: Phrase;
+		preset: SpeakerPreset;
+		cancelSignal: { cancelled: boolean };
+		onKernelTick?: () => void;
+	}): Promise<AudioChunk> {
+		const collected: Float32Array[] = [];
+		let total = 0;
+		await this.synthesizeStream({
+			phrase: args.phrase,
+			preset: args.preset,
+			cancelSignal: args.cancelSignal,
+			onKernelTick: args.onKernelTick,
+			onChunk: ({ pcm, isFinal }) => {
+				args.onKernelTick?.();
+				if (!isFinal && pcm.length > 0) {
+					collected.push(pcm);
+					total += pcm.length;
+				}
+				return args.cancelSignal.cancelled;
+			},
+		});
+		const merged = new Float32Array(total);
+		let off = 0;
+		for (const part of collected) {
+			merged.set(part, off);
+			off += part.length;
+		}
+		return {
+			phraseId: args.phrase.id,
+			fromIndex: args.phrase.fromIndex,
+			toIndex: args.phrase.toIndex,
+			pcm: merged,
+			sampleRate: this.runtime.sampleRate,
+		};
+	}
+
+	async synthesizeStream(args: {
+		phrase: Phrase;
+		preset: SpeakerPreset;
+		cancelSignal: { cancelled: boolean };
+		onChunk: (chunk: TtsPcmChunk) => boolean | undefined;
+		onKernelTick?: () => void;
+	}): Promise<{ cancelled: boolean }> {
+		const voice = this.resolveVoice(args.preset);
+		const phonemizer = await this.ensurePhonemizer();
+		args.onKernelTick?.();
+		const phonemes = await phonemizer.phonemize(args.phrase.text, voice.lang);
+		if (args.cancelSignal.cancelled) {
+			args.onChunk({
+				pcm: new Float32Array(0),
+				sampleRate: this.runtime.sampleRate,
+				isFinal: true,
+			});
+			return { cancelled: true };
+		}
+
+		// The runtime emits one (or a few) body chunks. We re-chunk to
+		// `streamingChunkSamples` so the scheduler's ring buffer sees a
+		// continuous trickle even when ONNX returns the whole waveform at
+		// once — this is how Kokoro's ~97ms TTFB becomes audible to the
+		// listener before the full phrase finishes decoding.
+		const limit = this.streamingChunkSamples;
+		let cancelled = false;
+		const result = await this.runtime.synthesize({
+			phonemes,
+			voice,
+			cancelSignal: args.cancelSignal,
+			onChunk: ({ pcm, isFinal }) => {
+				args.onKernelTick?.();
+				if (cancelled || args.cancelSignal.cancelled) {
+					cancelled = true;
+					return true;
+				}
+				if (pcm.length === 0) {
+					// Pass through tail markers from the runtime — the final tail is
+					// emitted by us below, so swallow runtime-side finals to avoid
+					// double-tails.
+					if (!isFinal) return false;
+					return false;
+				}
+				for (let off = 0; off < pcm.length; off += limit) {
+					if (args.cancelSignal.cancelled) {
+						cancelled = true;
+						return true;
+					}
+					const end = Math.min(pcm.length, off + limit);
+					const slice = pcm.subarray(off, end);
+					const want = args.onChunk({
+						pcm: slice,
+						sampleRate: this.runtime.sampleRate,
+						isFinal: false,
+					});
+					if (want === true || args.cancelSignal.cancelled) {
+						cancelled = true;
+						return true;
+					}
+				}
+				return false;
+			},
+		});
+		args.onChunk({
+			pcm: new Float32Array(0),
+			sampleRate: this.runtime.sampleRate,
+			isFinal: true,
+		});
+		return { cancelled: cancelled || result.cancelled };
+	}
+
+	dispose(): void {
+		this.runtime.dispose();
+	}
+
+	private resolveVoice(preset: SpeakerPreset): KokoroVoicePack {
+		// The scheduler's `SpeakerPreset.voiceId` is the canonical caller
+		// hook for picking a Kokoro voice; an unknown id falls back to the
+		// configured default rather than throwing (so OmniVoice-authored
+		// presets still produce audio when routed through Kokoro).
+		const id = preset.voiceId || this.defaultVoiceId;
+		return resolveKokoroVoiceOrDefault(id);
+	}
+
+	private async ensurePhonemizer(): Promise<KokoroPhonemizer> {
+		if (this.phonemizer) return this.phonemizer;
+		this.phonemizer = await resolvePhonemizer(this.phonemizerOverride);
+		console.info(`[kokoro] using phonemizer=${this.phonemizer.id}`);
+		return this.phonemizer;
+	}
+}

package/src/services/voice/kokoro/kokoro-engine-discovery.d.ts

@@ -0,0 +1,58 @@
+/**
+ * On-disk discovery for the Kokoro-only voice mode. Probes
+ * `<stateDir>/local-inference/models/kokoro/` (or `$ELIZA_KOKORO_MODEL_DIR`)
+ * for a Kokoro GGUF model file plus at least one voice `.bin` under
+ * `voices/`. Callers can pass an explicit model root to probe bundle-local
+ * Kokoro artifacts first. Returns null when anything is missing — no
+ * auto-download (AGENTS.md §3). GGUF-only: the ONNX path has been retired
+ * (see `runtimeKind` below).
+ *
+ * TRANSPORT NOTE — Kokoro synthesizes in-process through the fused
+ * `libelizainference` handle (ABI v10 `eliza_inference_kokoro_*`), the same
+ * dlopen()-ed lib as OmniVoice. The fork links Kokoro's native engine
+ * (`tools/kokoro/kokoro_lib`, its own GGUF reader + iSTFT decoder) into the
+ * fused build, and `KokoroFfiRuntime` drives it via `kokoroLoad` /
+ * `kokoroSynthesize`. This is the canonical path on every platform and the
+ * only one that ships on iOS / Google Play (those forbid the app opening a
+ * local TCP socket). The legacy `KokoroGgufRuntime` — POST `/v1/audio/speech`
+ * on a Kokoro-capable llama-server (the MTP gateway launched with
+ * `--kokoro-model`) — stays as an explicit dev/desktop opt-in
+ * (`KOKORO_BACKEND=fork`) and is never resolved on the mobile path. The GGUF
+ * is produced by the fork's `tools/kokoro/convert_kokoro_pth_to_gguf.py`.
+ *
+ * Env overrides:
+ *   ELIZA_KOKORO_MODEL_DIR        — directory root
+ *   ELIZA_KOKORO_MODEL_FILE       — exact filename inside the root
+ *                                   (ONNX or GGUF; the loader auto-detects)
+ *   ELIZA_KOKORO_DEFAULT_VOICE_ID — default voice id (e.g. `af_same`, `af_bella`)
+ */
+import type { KokoroModelLayout } from "./types";
+/** Canonical Kokoro v1.0 output sample rate. */
+export declare const KOKORO_DEFAULT_SAMPLE_RATE = 24000;
+/** True iff the candidate filename routes to the fused GGUF path. */
+export declare function isKokoroGgufFile(filename: string): boolean;
+export interface KokoroEngineDiscoveryResult {
+    layout: KokoroModelLayout;
+    /**
+     * Resolved default voice id. Prefers the catalog default
+     * (`KOKORO_DEFAULT_VOICE_ID` = `af_same`, Samantha) when its preset is on
+     * disk; falls back loudly to `KOKORO_FALLBACK_VOICE_ID` (`af_bella`) when
+     * Samantha's preset has not been produced yet; otherwise picks the first
+     * voice pack whose `.bin` is actually staged.
+     */
+    defaultVoiceId: string;
+    /**
+     * Resolved runtime kind. Always `"gguf"` — only GGUF model files are
+     * accepted by the discovery (ONNX paths have been retired).
+     */
+    runtimeKind: "gguf";
+}
+/** Returns the on-disk directory the discovery probes. */
+export declare function kokoroEngineModelDir(rootOverride?: string): string;
+/**
+ * Probe disk for a usable Kokoro layout. Returns null when any required
+ * piece is missing — the engine then falls back to its existing behaviour
+ * (fused omnivoice or `StubOmniVoiceBackend`).
+ */
+export declare function resolveKokoroEngineConfig(rootOverride?: string): KokoroEngineDiscoveryResult | null;
+//# sourceMappingURL=kokoro-engine-discovery.d.ts.map

package/src/services/voice/kokoro/kokoro-engine-discovery.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"kokoro-engine-discovery.d.ts","sourceRoot":"","sources":["kokoro-engine-discovery.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAKH,OAAO,KAAK,EAAE,iBAAiB,EAAmB,MAAM,SAAS,CAAC;AAOlE,gDAAgD;AAChD,eAAO,MAAM,0BAA0B,QAAS,CAAC;AAiBjD,qEAAqE;AACrE,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAE1D;AAED,MAAM,WAAW,2BAA2B;IAC3C,MAAM,EAAE,iBAAiB,CAAC;IAC1B;;;;;;OAMG;IACH,cAAc,EAAE,MAAM,CAAC;IACvB;;;OAGG;IACH,WAAW,EAAE,MAAM,CAAC;CACpB;AAED,0DAA0D;AAC1D,wBAAgB,oBAAoB,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,MAAM,CAMlE;AAED;;;;GAIG;AACH,wBAAgB,yBAAyB,CACxC,YAAY,CAAC,EAAE,MAAM,GACnB,2BAA2B,GAAG,IAAI,CAuBpC"}

package/src/services/voice/kokoro/kokoro-engine-discovery.ts

@@ -0,0 +1,177 @@
+/**
+ * On-disk discovery for the Kokoro-only voice mode. Probes
+ * `<stateDir>/local-inference/models/kokoro/` (or `$ELIZA_KOKORO_MODEL_DIR`)
+ * for a Kokoro GGUF model file plus at least one voice `.bin` under
+ * `voices/`. Callers can pass an explicit model root to probe bundle-local
+ * Kokoro artifacts first. Returns null when anything is missing — no
+ * auto-download (AGENTS.md §3). GGUF-only: the ONNX path has been retired
+ * (see `runtimeKind` below).
+ *
+ * TRANSPORT NOTE — Kokoro synthesizes in-process through the fused
+ * `libelizainference` handle (ABI v10 `eliza_inference_kokoro_*`), the same
+ * dlopen()-ed lib as OmniVoice. The fork links Kokoro's native engine
+ * (`tools/kokoro/kokoro_lib`, its own GGUF reader + iSTFT decoder) into the
+ * fused build, and `KokoroFfiRuntime` drives it via `kokoroLoad` /
+ * `kokoroSynthesize`. This is the canonical path on every platform and the
+ * only one that ships on iOS / Google Play (those forbid the app opening a
+ * local TCP socket). The legacy `KokoroGgufRuntime` — POST `/v1/audio/speech`
+ * on a Kokoro-capable llama-server (the MTP gateway launched with
+ * `--kokoro-model`) — stays as an explicit dev/desktop opt-in
+ * (`KOKORO_BACKEND=fork`) and is never resolved on the mobile path. The GGUF
+ * is produced by the fork's `tools/kokoro/convert_kokoro_pth_to_gguf.py`.
+ *
+ * Env overrides:
+ *   ELIZA_KOKORO_MODEL_DIR        — directory root
+ *   ELIZA_KOKORO_MODEL_FILE       — exact filename inside the root
+ *                                   (ONNX or GGUF; the loader auto-detects)
+ *   ELIZA_KOKORO_DEFAULT_VOICE_ID — default voice id (e.g. `af_same`, `af_bella`)
+ */
+
+import { existsSync, readdirSync } from "node:fs";
+import path from "node:path";
+import { elizaModelsDir } from "../../paths";
+import type { KokoroModelLayout, KokoroVoicePack } from "./types";
+import {
+	KOKORO_DEFAULT_VOICE_ID,
+	KOKORO_FALLBACK_VOICE_ID,
+	KOKORO_VOICE_PACKS,
+} from "./voice-presets";
+
+/** Canonical Kokoro v1.0 output sample rate. */
+export const KOKORO_DEFAULT_SAMPLE_RATE = 24_000;
+
+/**
+ * Filenames the loader will accept if `ELIZA_KOKORO_MODEL_FILE` is unset.
+ * Order is preference-first: a fused-GGUF beats an ONNX of the same
+ * quantization tier, and within ONNX the int8 export beats fp32.
+ *
+ * The Q4_K_M GGUF is what the elizaOS/llama.cpp fork's
+ * `tools/kokoro/convert_kokoro_pth_to_gguf.py` produces for shipping
+ * tiers; `kokoro-82m-v1_0.gguf` is the unquantized canonical filename
+ * the runtime documents at `kokoro-runtime.ts:KOKORO_GGUF_REL_PATH`.
+ */
+const CANDIDATE_MODEL_FILES: ReadonlyArray<string> = [
+	"kokoro-82m-v1_0-Q4_K_M.gguf",
+	"kokoro-82m-v1_0.gguf",
+];
+
+/** True iff the candidate filename routes to the fused GGUF path. */
+export function isKokoroGgufFile(filename: string): boolean {
+	return /\.gguf$/i.test(filename);
+}
+
+export interface KokoroEngineDiscoveryResult {
+	layout: KokoroModelLayout;
+	/**
+	 * Resolved default voice id. Prefers the catalog default
+	 * (`KOKORO_DEFAULT_VOICE_ID` = `af_same`, Samantha) when its preset is on
+	 * disk; falls back loudly to `KOKORO_FALLBACK_VOICE_ID` (`af_bella`) when
+	 * Samantha's preset has not been produced yet; otherwise picks the first
+	 * voice pack whose `.bin` is actually staged.
+	 */
+	defaultVoiceId: string;
+	/**
+	 * Resolved runtime kind. Always `"gguf"` — only GGUF model files are
+	 * accepted by the discovery (ONNX paths have been retired).
+	 */
+	runtimeKind: "gguf";
+}
+
+/** Returns the on-disk directory the discovery probes. */
+export function kokoroEngineModelDir(rootOverride?: string): string {
+	const explicit = rootOverride?.trim();
+	if (explicit) return explicit;
+	const env = process.env.ELIZA_KOKORO_MODEL_DIR?.trim();
+	if (env) return env;
+	return path.join(elizaModelsDir(), "kokoro");
+}
+
+/**
+ * Probe disk for a usable Kokoro layout. Returns null when any required
+ * piece is missing — the engine then falls back to its existing behaviour
+ * (fused omnivoice or `StubOmniVoiceBackend`).
+ */
+export function resolveKokoroEngineConfig(
+	rootOverride?: string,
+): KokoroEngineDiscoveryResult | null {
+	const root = kokoroEngineModelDir(rootOverride);
+	if (!existsSync(root)) return null;
+
+	const modelFile = resolveModelFile(root);
+	if (!modelFile) return null;
+
+	const voicesDir = path.join(root, "voices");
+	if (!existsSync(voicesDir)) return null;
+
+	const defaultVoiceId = resolveDefaultVoiceId(voicesDir);
+	if (!defaultVoiceId) return null;
+
+	return {
+		layout: {
+			root,
+			modelFile,
+			voicesDir,
+			sampleRate: KOKORO_DEFAULT_SAMPLE_RATE,
+		},
+		defaultVoiceId,
+		runtimeKind: "gguf",
+	};
+}
+
+function resolveModelFile(root: string): string | null {
+	const env = process.env.ELIZA_KOKORO_MODEL_FILE?.trim();
+	if (env) {
+		return existsSync(path.join(root, env)) ? env : null;
+	}
+	for (const candidate of CANDIDATE_MODEL_FILES) {
+		if (existsSync(path.join(root, candidate))) return candidate;
+	}
+	return null;
+}
+
+function resolveDefaultVoiceId(voicesDir: string): string | null {
+	const env = process.env.ELIZA_KOKORO_DEFAULT_VOICE_ID?.trim();
+	if (env) {
+		const pack = findVoicePack(env);
+		if (pack && existsSync(path.join(voicesDir, pack.file))) return pack.id;
+		return null;
+	}
+	// Prefer the catalog default (Samantha) when its file is staged.
+	const defaultPack = findVoicePack(KOKORO_DEFAULT_VOICE_ID);
+	if (defaultPack && existsSync(path.join(voicesDir, defaultPack.file))) {
+		return defaultPack.id;
+	}
+	// Samantha preset bytes not staged — fall back to the bundled fallback
+	// voice (af_bella). This MUST be loud so operators see the degradation:
+	// the canonical default is Samantha and we only land here when the LoRA
+	// pipeline has not produced a real `af_same.bin` yet.
+	const fallbackPack = findVoicePack(KOKORO_FALLBACK_VOICE_ID);
+	if (fallbackPack && existsSync(path.join(voicesDir, fallbackPack.file))) {
+		// eslint-disable-next-line no-console -- this is the one place where
+		// the runtime must surface the fallback to the operator console; the
+		// structured logger is unavailable at discovery time.
+		console.warn(
+			`[kokoro] default voice ${KOKORO_DEFAULT_VOICE_ID} preset not staged at ${path.join(voicesDir, defaultPack?.file ?? `${KOKORO_DEFAULT_VOICE_ID}.bin`)} — falling back to ${KOKORO_FALLBACK_VOICE_ID}. Run packages/training/scripts/voice/samantha_lora/RUNBOOK.md to produce a real Samantha preset, or regenerate via plugins/plugin-local-inference/scripts/regenerate-samantha-preset.mjs.`,
+		);
+		return fallbackPack.id;
+	}
+	// Otherwise pick the first catalog voice whose file is on disk. This
+	// lets operators stage a single voice (any voice) and have it just work.
+	const staged = listStagedVoiceIds(voicesDir);
+	return staged[0] ?? null;
+}
+
+function findVoicePack(id: string): KokoroVoicePack | null {
+	return KOKORO_VOICE_PACKS.find((v) => v.id === id) ?? null;
+}
+
+function listStagedVoiceIds(voicesDir: string): string[] {
+	try {
+		const present = new Set(readdirSync(voicesDir));
+		return KOKORO_VOICE_PACKS.filter((v) => present.has(v.file)).map(
+			(v) => v.id,
+		);
+	} catch {
+		return [];
+	}
+}

package/src/services/voice/kokoro/kokoro-ffi-runtime.d.ts

@@ -0,0 +1,75 @@
+/**
+ * In-process Kokoro-82M runtime over the fused `libelizainference` FFI
+ * (the `eliza_inference_kokoro_*` exports — introduced at ABI v10; the fused
+ * library is currently ABI v11, which adds EOT on top, so these symbols are
+ * present in every current build — see `ELIZA_INFERENCE_ABI_VERSION` in
+ * ffi-bindings.ts).
+ *
+ * This is the canonical Kokoro execution path on every platform. It replaces
+ * the local-TCP `KokoroGgufRuntime` (POST `/v1/audio/speech` on a running
+ * llama-server) for the mobile case — iOS and Google Play forbid the app
+ * opening a local TCP socket, so the HTTP→llama-server route cannot ship there.
+ * Kokoro synthesizes through the same dlopen()-ed handle as OmniVoice: the
+ * fused build links Eliza-1's Kokoro engine (its own GGUF reader + iSTFT
+ * decoder) behind `eliza_inference_kokoro_supported/load/synthesize/sample_rate`.
+ *
+ * Ownership: this runtime owns its own FFI handle + context. The context is
+ * created with `create(bundleRoot)` anchored at the bundle root (or the Kokoro
+ * model root when there is no Eliza-1 bundle), mirroring how the desktop fused
+ * text runtime obtains its ctx. The GGUF + the active voice `.bin` are loaded
+ * once via `kokoroLoad` and reloaded only when the requested voice changes.
+ *
+ * No silent fallback (AGENTS.md §3): when the loaded library does not export
+ * the Kokoro symbols (`kokoroSupported() === false`) or the model/voice files
+ * are missing, construction / first synthesis throws a structured
+ * `VoiceLifecycleError` rather than dropping back to the TCP route.
+ */
+import { type ElizaInferenceContextHandle, type ElizaInferenceFfi } from "../ffi-bindings";
+import type { KokoroRuntime, KokoroRuntimeInputs } from "./kokoro-runtime";
+import type { KokoroModelLayout } from "./types";
+export interface KokoroFfiRuntimeOptions {
+    /** Resolved on-disk Kokoro layout (GGUF filename + voices dir + root). */
+    layout: KokoroModelLayout;
+    /**
+     * Directory the FFI context anchors at (`create(bundleRoot)`). Defaults to
+     * the Kokoro model root, which is sufficient for the standalone Kokoro
+     * engine — it loads the GGUF + voice `.bin` by explicit absolute path, not
+     * by bundle convention.
+     */
+    bundleRoot?: string;
+    /**
+     * Inject a pre-loaded FFI handle (the desktop fused engine already owns one).
+     * When omitted the runtime loads its own via `resolveFusedLibraryPath`.
+     */
+    ffi?: ElizaInferenceFfi;
+    /**
+     * Inject a context to reuse. When omitted the runtime creates its own with
+     * `ffi.create(bundleRoot)` and destroys it on `dispose`.
+     */
+    ctx?: ElizaInferenceContextHandle;
+}
+export declare class KokoroFfiRuntime implements KokoroRuntime {
+    readonly id: "gguf";
+    readonly sampleRate: number;
+    private readonly layout;
+    private readonly ffi;
+    private readonly ownsFfi;
+    private readonly ctx;
+    private readonly ownsCtx;
+    /** Voice id currently resident on the ctx (null until first load). */
+    private loadedVoiceId;
+    private disposed;
+    constructor(opts: KokoroFfiRuntimeOptions);
+    synthesize(args: KokoroRuntimeInputs): Promise<{
+        cancelled: boolean;
+    }>;
+    dispose(): void;
+    /**
+     * Load the GGUF + the requested voice `.bin` into the ctx, reloading only
+     * when the voice changes (Kokoro keeps the model resident; swapping voices
+     * is a cheap re-load of the 256-float style tensor).
+     */
+    private ensureVoiceLoaded;
+    private kokoroSynthesize;
+}
+//# sourceMappingURL=kokoro-ffi-runtime.d.ts.map

package/src/services/voice/kokoro/kokoro-ffi-runtime.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"kokoro-ffi-runtime.d.ts","sourceRoot":"","sources":["kokoro-ffi-runtime.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAMH,OAAO,EACN,KAAK,2BAA2B,EAChC,KAAK,iBAAiB,EAEtB,MAAM,iBAAiB,CAAC;AAEzB,OAAO,KAAK,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAC3E,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAcjD,MAAM,WAAW,uBAAuB;IACvC,0EAA0E;IAC1E,MAAM,EAAE,iBAAiB,CAAC;IAC1B;;;;;OAKG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,GAAG,CAAC,EAAE,iBAAiB,CAAC;IACxB;;;OAGG;IACH,GAAG,CAAC,EAAE,2BAA2B,CAAC;CAClC;AAED,qBAAa,gBAAiB,YAAW,aAAa;IACrD,QAAQ,CAAC,EAAE,EAAG,MAAM,CAAU;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAE5B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoB;IAC3C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAoB;IACxC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAA8B;IAClD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAClC,sEAAsE;IACtE,OAAO,CAAC,aAAa,CAAuB;IAC5C,OAAO,CAAC,QAAQ,CAAS;gBAEb,IAAI,EAAE,uBAAuB;IA4CnC,UAAU,CAAC,IAAI,EAAE,mBAAmB,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,OAAO,CAAA;KAAE,CAAC;IA4C5E,OAAO,IAAI,IAAI;IAOf;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IAoCzB,OAAO,CAAC,gBAAgB;CASxB"}