@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.2

package/src/services/voice/kokoro/kokoro-ffi-runtime.ts

@@ -0,0 +1,233 @@
+/**
+ * In-process Kokoro-82M runtime over the fused `libelizainference` FFI
+ * (the `eliza_inference_kokoro_*` exports — introduced at ABI v10; the fused
+ * library is currently ABI v11, which adds EOT on top, so these symbols are
+ * present in every current build — see `ELIZA_INFERENCE_ABI_VERSION` in
+ * ffi-bindings.ts).
+ *
+ * This is the canonical Kokoro execution path on every platform. It replaces
+ * the local-TCP `KokoroGgufRuntime` (POST `/v1/audio/speech` on a running
+ * llama-server) for the mobile case — iOS and Google Play forbid the app
+ * opening a local TCP socket, so the HTTP→llama-server route cannot ship there.
+ * Kokoro synthesizes through the same dlopen()-ed handle as OmniVoice: the
+ * fused build links Eliza-1's Kokoro engine (its own GGUF reader + iSTFT
+ * decoder) behind `eliza_inference_kokoro_supported/load/synthesize/sample_rate`.
+ *
+ * Ownership: this runtime owns its own FFI handle + context. The context is
+ * created with `create(bundleRoot)` anchored at the bundle root (or the Kokoro
+ * model root when there is no Eliza-1 bundle), mirroring how the desktop fused
+ * text runtime obtains its ctx. The GGUF + the active voice `.bin` are loaded
+ * once via `kokoroLoad` and reloaded only when the requested voice changes.
+ *
+ * No silent fallback (AGENTS.md §3): when the loaded library does not export
+ * the Kokoro symbols (`kokoroSupported() === false`) or the model/voice files
+ * are missing, construction / first synthesis throws a structured
+ * `VoiceLifecycleError` rather than dropping back to the TCP route.
+ */
+
+import { existsSync } from "node:fs";
+import path from "node:path";
+import { logger } from "@elizaos/core";
+import { resolveFusedLibraryPath } from "../../desktop-fused-ffi-backend-runtime";
+import {
+	type ElizaInferenceContextHandle,
+	type ElizaInferenceFfi,
+	loadElizaInferenceFfi,
+} from "../ffi-bindings";
+import { VoiceLifecycleError } from "../lifecycle";
+import type { KokoroRuntime, KokoroRuntimeInputs } from "./kokoro-runtime";
+import type { KokoroModelLayout } from "./types";
+import { resolveKokoroVoiceOrDefault } from "./voices";
+
+/** Kokoro v1.0 style-vector inner dimension. */
+const KOKORO_STYLE_DIM = 256;
+
+/**
+ * Per-synthesis output ceiling. Kokoro v1.0 emits 24 kHz fp32 PCM; 30 s of
+ * headroom (720 000 samples) bounds a single phrase synthesis well past the
+ * longest chunk the phrase chunker will hand us. The library returns the real
+ * sample count, which we slice to — this is only the allocation cap.
+ */
+const MAX_OUTPUT_SAMPLES = 30 * 24_000;
+
+export interface KokoroFfiRuntimeOptions {
+	/** Resolved on-disk Kokoro layout (GGUF filename + voices dir + root). */
+	layout: KokoroModelLayout;
+	/**
+	 * Directory the FFI context anchors at (`create(bundleRoot)`). Defaults to
+	 * the Kokoro model root, which is sufficient for the standalone Kokoro
+	 * engine — it loads the GGUF + voice `.bin` by explicit absolute path, not
+	 * by bundle convention.
+	 */
+	bundleRoot?: string;
+	/**
+	 * Inject a pre-loaded FFI handle (the desktop fused engine already owns one).
+	 * When omitted the runtime loads its own via `resolveFusedLibraryPath`.
+	 */
+	ffi?: ElizaInferenceFfi;
+	/**
+	 * Inject a context to reuse. When omitted the runtime creates its own with
+	 * `ffi.create(bundleRoot)` and destroys it on `dispose`.
+	 */
+	ctx?: ElizaInferenceContextHandle;
+}
+
+export class KokoroFfiRuntime implements KokoroRuntime {
+	readonly id = "gguf" as const;
+	readonly sampleRate: number;
+
+	private readonly layout: KokoroModelLayout;
+	private readonly ffi: ElizaInferenceFfi;
+	private readonly ownsFfi: boolean;
+	private readonly ctx: ElizaInferenceContextHandle;
+	private readonly ownsCtx: boolean;
+	/** Voice id currently resident on the ctx (null until first load). */
+	private loadedVoiceId: string | null = null;
+	private disposed = false;
+
+	constructor(opts: KokoroFfiRuntimeOptions) {
+		this.layout = opts.layout;
+		const bundleRoot = opts.bundleRoot ?? opts.layout.root;
+
+		const provided = opts.ffi;
+		if (provided) {
+			this.ffi = provided;
+			this.ownsFfi = false;
+		} else {
+			const libPath = resolveFusedLibraryPath(bundleRoot);
+			if (!libPath) {
+				throw new VoiceLifecycleError(
+					"kernel-missing",
+					`[KokoroFfiRuntime] fused libelizainference not found for the in-process Eliza-1 Kokoro engine (anchored at ${bundleRoot}). ` +
+						"Set ELIZA_INFERENCE_LIBRARY or build via packages/app-core/scripts/build-llama-cpp-mtp.mjs.",
+				);
+			}
+			this.ffi = loadElizaInferenceFfi(libPath);
+			this.ownsFfi = true;
+		}
+
+		if (
+			typeof this.ffi.kokoroSupported !== "function" ||
+			!this.ffi.kokoroSupported()
+		) {
+			if (this.ownsFfi) this.ffi.close();
+			throw new VoiceLifecycleError(
+				"kernel-missing",
+				`[KokoroFfiRuntime] the loaded libelizainference (ABI v${this.ffi.libraryAbiVersion}) does not link the in-process Eliza-1 Kokoro engine. ` +
+					"Rebuild with the Kokoro engine enabled — the mobile path must not fall back to the local-TCP /v1/audio/speech route.",
+			);
+		}
+
+		if (opts.ctx !== undefined) {
+			this.ctx = opts.ctx;
+			this.ownsCtx = false;
+		} else {
+			this.ctx = this.ffi.create(bundleRoot);
+			this.ownsCtx = true;
+		}
+
+		this.sampleRate = this.layout.sampleRate;
+	}
+
+	async synthesize(args: KokoroRuntimeInputs): Promise<{ cancelled: boolean }> {
+		if (this.disposed) {
+			throw new VoiceLifecycleError(
+				"kernel-missing",
+				"[KokoroFfiRuntime] synthesize called after dispose",
+			);
+		}
+		this.ensureVoiceLoaded(args.voice.id);
+
+		if (args.cancelSignal.cancelled) {
+			args.onChunk({
+				pcm: new Float32Array(0),
+				sampleRate: this.sampleRate,
+				isFinal: true,
+			});
+			return { cancelled: true };
+		}
+
+		const maxSamples = args.maxSamples ?? MAX_OUTPUT_SAMPLES;
+		// The Kokoro engine produces the full waveform in one synchronous
+		// forward. The text it phonemizes internally is the same phoneme string
+		// the llama-server `/v1/audio/speech` path sends as `input`.
+		const pcm = this.kokoroSynthesize(args.phonemes.phonemes, maxSamples);
+
+		let cancelled = false;
+		if (args.cancelSignal.cancelled) {
+			cancelled = true;
+		} else if (pcm.length > 0) {
+			const want = args.onChunk({
+				pcm,
+				sampleRate: this.sampleRate,
+				isFinal: false,
+			});
+			if (want === true || args.cancelSignal.cancelled) cancelled = true;
+		}
+
+		args.onChunk({
+			pcm: new Float32Array(0),
+			sampleRate: this.sampleRate,
+			isFinal: true,
+		});
+		return { cancelled };
+	}
+
+	dispose(): void {
+		if (this.disposed) return;
+		this.disposed = true;
+		if (this.ownsCtx) this.ffi.destroy(this.ctx);
+		if (this.ownsFfi) this.ffi.close();
+	}
+
+	/**
+	 * Load the GGUF + the requested voice `.bin` into the ctx, reloading only
+	 * when the voice changes (Kokoro keeps the model resident; swapping voices
+	 * is a cheap re-load of the 256-float style tensor).
+	 */
+	private ensureVoiceLoaded(requestedVoiceId: string): void {
+		const voice = resolveKokoroVoiceOrDefault(requestedVoiceId);
+		if (this.loadedVoiceId === voice.id) return;
+
+		const ggufPath = path.join(this.layout.root, this.layout.modelFile);
+		const voiceBinPath = path.join(this.layout.voicesDir, voice.file);
+		if (!existsSync(ggufPath)) {
+			throw new VoiceLifecycleError(
+				"kernel-missing",
+				`[KokoroFfiRuntime] Eliza-1 Kokoro model file not found at ${ggufPath}`,
+			);
+		}
+		if (!existsSync(voiceBinPath)) {
+			throw new VoiceLifecycleError(
+				"kernel-missing",
+				`[KokoroFfiRuntime] Eliza-1 voice preset not found at ${voiceBinPath} for voice ${voice.id}`,
+			);
+		}
+		if (typeof this.ffi.kokoroLoad !== "function") {
+			throw new VoiceLifecycleError(
+				"kernel-missing",
+				"[KokoroFfiRuntime] eliza_inference_kokoro_load is not exported by the loaded build",
+			);
+		}
+		this.ffi.kokoroLoad({
+			ctx: this.ctx,
+			ggufPath,
+			voiceBinPath,
+			styleDim: voice.dim ?? KOKORO_STYLE_DIM,
+		});
+		this.loadedVoiceId = voice.id;
+		logger.info(
+			`[KokoroFfiRuntime] loaded Eliza-1 voice ${voice.id} from ${voiceBinPath}`,
+		);
+	}
+
+	private kokoroSynthesize(text: string, maxSamples: number): Float32Array {
+		if (typeof this.ffi.kokoroSynthesize !== "function") {
+			throw new VoiceLifecycleError(
+				"kernel-missing",
+				"[KokoroFfiRuntime] eliza_inference_kokoro_synthesize is not exported by the loaded build",
+			);
+		}
+		return this.ffi.kokoroSynthesize({ ctx: this.ctx, text, maxSamples });
+	}
+}

package/src/services/voice/kokoro/kokoro-runtime.d.ts

@@ -0,0 +1,100 @@
+/**
+ * Kokoro-82M model runner.
+ *
+ * Execution paths:
+ *
+ *   1. GGUF via llama-server (default). When the host llama-server advertises
+ *      a Kokoro-capable build and exposes `/v1/audio/speech`, we POST text in
+ *      and stream PCM out.
+ *
+ *   2. Python subprocess — eval-loop only. Spawns `python -m kokoro_tts`.
+ *      Never the default in production.
+ */
+import type { KokoroPhonemeSequence, KokoroVoicePack } from "./types";
+/** Pinned GGUF candidate location (carried by our llama.cpp fork). The
+ *  runtime references this only for diagnostics; the fork-side builder
+ *  produces the file at this path. */
+export declare const KOKORO_GGUF_REL_PATH = "voice/kokoro-82m-v1_0.gguf";
+/** One synthesized PCM segment delivered to the streaming backend. */
+export interface KokoroRuntimeChunk {
+    pcm: Float32Array;
+    sampleRate: number;
+    isFinal: boolean;
+}
+/**
+ * Construction-time inputs for a runtime instance. The voice pack contains
+ * the style tensor reference; the runtime is responsible for resolving the
+ * bytes off `layout.voicesDir/<file>`.
+ */
+export interface KokoroRuntimeInputs {
+    phonemes: KokoroPhonemeSequence;
+    voice: KokoroVoicePack;
+    /**
+     * Output sample budget. The runtime always honours the model's native
+     * sample rate (`layout.sampleRate`, usually 24 kHz) — this caps the
+     * total samples to prevent runaway generation. Defaults to 16 seconds
+     * at the layout sample rate (matches the longest phrase the chunker
+     * will emit + headroom).
+     */
+    maxSamples?: number;
+    /** Cancellation signal — polled at chunk boundaries. */
+    cancelSignal: {
+        cancelled: boolean;
+    };
+    /** Per-chunk callback; returning `true` cancels the rest of the run. */
+    onChunk: (chunk: KokoroRuntimeChunk) => boolean | undefined;
+}
+/** Shared runtime contract — `KokoroTtsBackend` depends on this, not the
+ *  concrete classes. Tests inject a mock. */
+export interface KokoroRuntime {
+    readonly id: "gguf" | "python" | "mock";
+    readonly sampleRate: number;
+    synthesize(args: KokoroRuntimeInputs): Promise<{
+        cancelled: boolean;
+    }>;
+    dispose(): void;
+}
+export interface KokoroPythonRuntimeOptions {
+    pythonBinary: string;
+    /** Resolved layout — the subprocess discovers the model under here. */
+    layout: {
+        root: string;
+        sampleRate: number;
+    };
+    /** Optional env passed through to the subprocess. */
+    env?: NodeJS.ProcessEnv;
+}
+/**
+ * Subprocess-backed runtime. Spawns `python -m kokoro_tts ...` per
+ * synthesis call (no warm-pool — the Python path is the *eval* path, not
+ * the realtime path). Production code paths never select this; the
+ * fine-tune evaluator wires it explicitly.
+ */
+export declare class KokoroPythonRuntime implements KokoroRuntime {
+    readonly id: "python";
+    readonly sampleRate: number;
+    constructor(opts: KokoroPythonRuntimeOptions);
+    synthesize(_args: KokoroRuntimeInputs): Promise<{
+        cancelled: boolean;
+    }>;
+    dispose(): void;
+}
+export interface KokoroMockRuntimeOptions {
+    sampleRate: number;
+    /** Total samples emitted per synthesis call. */
+    totalSamples?: number;
+    /** Number of body chunks to split the output across. */
+    chunkCount?: number;
+}
+export declare class KokoroMockRuntime implements KokoroRuntime {
+    readonly id: "mock";
+    readonly sampleRate: number;
+    private readonly opts;
+    calls: number;
+    constructor(opts: KokoroMockRuntimeOptions);
+    synthesize(args: KokoroRuntimeInputs): Promise<{
+        cancelled: boolean;
+    }>;
+    dispose(): void;
+}
+//# sourceMappingURL=kokoro-runtime.d.ts.map

package/src/services/voice/kokoro/kokoro-runtime.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"kokoro-runtime.d.ts","sourceRoot":"","sources":["kokoro-runtime.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,qBAAqB,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAEtE;;sCAEsC;AACtC,eAAO,MAAM,oBAAoB,+BAA+B,CAAC;AAEjE,sEAAsE;AACtE,MAAM,WAAW,kBAAkB;IAClC,GAAG,EAAE,YAAY,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;CACjB;AAED;;;;GAIG;AACH,MAAM,WAAW,mBAAmB;IACnC,QAAQ,EAAE,qBAAqB,CAAC;IAChC,KAAK,EAAE,eAAe,CAAC;IACvB;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,wDAAwD;IACxD,YAAY,EAAE;QAAE,SAAS,EAAE,OAAO,CAAA;KAAE,CAAC;IACrC,wEAAwE;IACxE,OAAO,EAAE,CAAC,KAAK,EAAE,kBAAkB,KAAK,OAAO,GAAG,SAAS,CAAC;CAC5D;AAED;6CAC6C;AAC7C,MAAM,WAAW,aAAa;IAC7B,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;IACxC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,UAAU,CAAC,IAAI,EAAE,mBAAmB,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACvE,OAAO,IAAI,IAAI,CAAC;CAChB;AAMD,MAAM,WAAW,0BAA0B;IAC1C,YAAY,EAAE,MAAM,CAAC;IACrB,uEAAuE;IACvE,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7C,qDAAqD;IACrD,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;CACxB;AAED;;;;;GAKG;AACH,qBAAa,mBAAoB,YAAW,aAAa;IACxD,QAAQ,CAAC,EAAE,EAAG,QAAQ,CAAU;IAChC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;gBAEhB,IAAI,EAAE,0BAA0B;IAItC,UAAU,CACf,KAAK,EAAE,mBAAmB,GACxB,OAAO,CAAC;QAAE,SAAS,EAAE,OAAO,CAAA;KAAE,CAAC;IASlC,OAAO,IAAI,IAAI;CAGf;AAOD,MAAM,WAAW,wBAAwB;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,gDAAgD;IAChD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,wDAAwD;IACxD,UAAU,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,iBAAkB,YAAW,aAAa;IACtD,QAAQ,CAAC,EAAE,EAAG,MAAM,CAAU;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAqC;IAC1D,KAAK,SAAK;gBAEE,IAAI,EAAE,wBAAwB;IASpC,UAAU,CAAC,IAAI,EAAE,mBAAmB,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,OAAO,CAAA;KAAE,CAAC;IAsC5E,OAAO,IAAI,IAAI;CAGf"}

package/src/services/voice/kokoro/kokoro-runtime.ts

@@ -0,0 +1,170 @@
+/**
+ * Kokoro-82M model runner.
+ *
+ * Execution paths:
+ *
+ *   1. GGUF via llama-server (default). When the host llama-server advertises
+ *      a Kokoro-capable build and exposes `/v1/audio/speech`, we POST text in
+ *      and stream PCM out.
+ *
+ *   2. Python subprocess — eval-loop only. Spawns `python -m kokoro_tts`.
+ *      Never the default in production.
+ */
+
+import type { KokoroPhonemeSequence, KokoroVoicePack } from "./types";
+
+/** Pinned GGUF candidate location (carried by our llama.cpp fork). The
+ *  runtime references this only for diagnostics; the fork-side builder
+ *  produces the file at this path. */
+export const KOKORO_GGUF_REL_PATH = "voice/kokoro-82m-v1_0.gguf";
+
+/** One synthesized PCM segment delivered to the streaming backend. */
+export interface KokoroRuntimeChunk {
+	pcm: Float32Array;
+	sampleRate: number;
+	isFinal: boolean;
+}
+
+/**
+ * Construction-time inputs for a runtime instance. The voice pack contains
+ * the style tensor reference; the runtime is responsible for resolving the
+ * bytes off `layout.voicesDir/<file>`.
+ */
+export interface KokoroRuntimeInputs {
+	phonemes: KokoroPhonemeSequence;
+	voice: KokoroVoicePack;
+	/**
+	 * Output sample budget. The runtime always honours the model's native
+	 * sample rate (`layout.sampleRate`, usually 24 kHz) — this caps the
+	 * total samples to prevent runaway generation. Defaults to 16 seconds
+	 * at the layout sample rate (matches the longest phrase the chunker
+	 * will emit + headroom).
+	 */
+	maxSamples?: number;
+	/** Cancellation signal — polled at chunk boundaries. */
+	cancelSignal: { cancelled: boolean };
+	/** Per-chunk callback; returning `true` cancels the rest of the run. */
+	onChunk: (chunk: KokoroRuntimeChunk) => boolean | undefined;
+}
+
+/** Shared runtime contract — `KokoroTtsBackend` depends on this, not the
+ *  concrete classes. Tests inject a mock. */
+export interface KokoroRuntime {
+	readonly id: "gguf" | "python" | "mock";
+	readonly sampleRate: number;
+	synthesize(args: KokoroRuntimeInputs): Promise<{ cancelled: boolean }>;
+	dispose(): void;
+}
+
+// ---------------------------------------------------------------------------
+// Python subprocess path — eval-loop only.
+// ---------------------------------------------------------------------------
+
+export interface KokoroPythonRuntimeOptions {
+	pythonBinary: string;
+	/** Resolved layout — the subprocess discovers the model under here. */
+	layout: { root: string; sampleRate: number };
+	/** Optional env passed through to the subprocess. */
+	env?: NodeJS.ProcessEnv;
+}
+
+/**
+ * Subprocess-backed runtime. Spawns `python -m kokoro_tts ...` per
+ * synthesis call (no warm-pool — the Python path is the *eval* path, not
+ * the realtime path). Production code paths never select this; the
+ * fine-tune evaluator wires it explicitly.
+ */
+export class KokoroPythonRuntime implements KokoroRuntime {
+	readonly id = "python" as const;
+	readonly sampleRate: number;
+
+	constructor(opts: KokoroPythonRuntimeOptions) {
+		this.sampleRate = opts.layout.sampleRate;
+	}
+
+	async synthesize(
+		_args: KokoroRuntimeInputs,
+	): Promise<{ cancelled: boolean }> {
+		// The eval driver in `packages/training` is the canonical caller and
+		// already wires `child_process.spawn`. Surfacing a clear error here
+		// keeps the production runtime from accidentally enabling this path.
+		throw new Error(
+			"[kokoro] KokoroPythonRuntime is eval-only — use it from the fine-tune driver, not the runtime scheduler",
+		);
+	}
+
+	dispose(): void {
+		// No long-lived state.
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Mock runtime — synthesizes a sine sweep keyed to phoneme count so tests
+// can observe deterministic PCM without loading a model.
+// ---------------------------------------------------------------------------
+
+export interface KokoroMockRuntimeOptions {
+	sampleRate: number;
+	/** Total samples emitted per synthesis call. */
+	totalSamples?: number;
+	/** Number of body chunks to split the output across. */
+	chunkCount?: number;
+}
+
+export class KokoroMockRuntime implements KokoroRuntime {
+	readonly id = "mock" as const;
+	readonly sampleRate: number;
+	private readonly opts: Required<KokoroMockRuntimeOptions>;
+	calls = 0;
+
+	constructor(opts: KokoroMockRuntimeOptions) {
+		this.sampleRate = opts.sampleRate;
+		this.opts = {
+			sampleRate: opts.sampleRate,
+			totalSamples: opts.totalSamples ?? Math.floor(opts.sampleRate * 0.2),
+			chunkCount: opts.chunkCount ?? 4,
+		};
+	}
+
+	async synthesize(args: KokoroRuntimeInputs): Promise<{ cancelled: boolean }> {
+		this.calls++;
+		const { totalSamples, chunkCount } = this.opts;
+		const perChunk = Math.max(1, Math.ceil(totalSamples / chunkCount));
+		const freqHz = 100 + (args.phonemes.ids.length % 200);
+		let written = 0;
+		let cancelled = false;
+		for (let off = 0; off < totalSamples; off += perChunk) {
+			if (args.cancelSignal.cancelled) {
+				cancelled = true;
+				break;
+			}
+			const n = Math.min(perChunk, totalSamples - off);
+			const pcm = new Float32Array(n);
+			for (let i = 0; i < n; i++) {
+				const t = (off + i) / this.sampleRate;
+				pcm[i] = Math.sin(2 * Math.PI * freqHz * t) * 0.1;
+			}
+			written += n;
+			const want = args.onChunk({
+				pcm,
+				sampleRate: this.sampleRate,
+				isFinal: false,
+			});
+			if (want === true || args.cancelSignal.cancelled) {
+				cancelled = true;
+				break;
+			}
+		}
+		args.onChunk({
+			pcm: new Float32Array(0),
+			sampleRate: this.sampleRate,
+			isFinal: true,
+		});
+		void written;
+		return { cancelled };
+	}
+
+	dispose(): void {
+		/* nothing */
+	}
+}

package/src/services/voice/kokoro/phoneme-stream.ts

@@ -0,0 +1,123 @@
+/**
+ * Phoneme streaming for Kokoro-82M.
+ *
+ * Kokoro consumes a sequence of phoneme ids (espeak-ng IPA tokenised against a
+ * small fixed vocab). The scheduler emits phrases at punctuation or
+ * `phoneme-stream` boundaries (see `voice/phrase-chunker.ts` `chunkOn`
+ * option). This module is the seam between those phrase boundaries and the
+ * model's input tensor:
+ *
+ *   text → phonemizer.phonemize() → KokoroPhonemeSequence (ids) → runtime
+ *
+ * For maximum responsiveness the runtime can call `streamPhonemes()` against
+ * an async text iterator (chunked draft tokens) and forward each window of
+ * accumulated ids as soon as a phoneme boundary fires. The default `flushAt`
+ * is one phoneme — i.e. emit progress per id — but production deployments
+ * lift this to ~8 phonemes to amortise the ONNX forward pass on small
+ * windows. This file intentionally has no dependency on the rest of the
+ * voice scaffold so it can be reused by the fine-tune evaluator script.
+ */
+
+import type { KokoroPhonemeSequence, KokoroPhonemizer } from "./types";
+
+export interface PhonemeStreamWindow {
+	/** Cumulative ids since stream start. The runtime can re-tokenise or
+	 *  carry state by id; the simplest implementation forwards the full
+	 *  window each call. */
+	ids: Int32Array;
+	/** Cumulative phoneme string for debugging / display. */
+	phonemes: string;
+	/** True for the final window in the stream. */
+	isFinal: boolean;
+}
+
+export interface StreamPhonemesOptions {
+	phonemizer: KokoroPhonemizer;
+	lang: string;
+	/** Emit a window every N new phoneme ids. Default 8 (≈ first audio after a
+	 *  short syllable cluster — matches the phrase chunker's default cap). */
+	flushAt?: number;
+}
+
+/**
+ * Phonemize an async text source and emit cumulative windows. The caller
+ * consumes the iterator with `for await (const window of streamPhonemes(…))`.
+ * A pull-style API keeps this independent of the scheduler's event loop —
+ * the bench harness and the eval loop both reuse it without taking on a
+ * scheduler dependency.
+ */
+export async function* streamPhonemes(
+	textChunks: AsyncIterable<string>,
+	opts: StreamPhonemesOptions,
+): AsyncIterable<PhonemeStreamWindow> {
+	const flushAt = Math.max(1, opts.flushAt ?? 8);
+	const idsAcc: number[] = [];
+	let phonemesAcc = "";
+	let lastFlushAt = 0;
+	let leftover = "";
+
+	for await (const chunk of textChunks) {
+		if (!chunk) continue;
+		leftover += chunk;
+		// Only phonemize when we have at least a whole word to feed to the
+		// phonemizer — espeak-ng is significantly more accurate when fed
+		// word-aligned input. Look back to the last whitespace as the split.
+		const split = leftover.lastIndexOf(" ");
+		if (split === -1) continue;
+		const head = leftover.slice(0, split);
+		leftover = leftover.slice(split + 1);
+		const seq = await opts.phonemizer.phonemize(head, opts.lang);
+		appendSeq(seq, idsAcc);
+		phonemesAcc += seq.phonemes;
+		if (idsAcc.length - lastFlushAt >= flushAt) {
+			lastFlushAt = idsAcc.length;
+			yield {
+				ids: Int32Array.from(idsAcc),
+				phonemes: phonemesAcc,
+				isFinal: false,
+			};
+		}
+	}
+
+	if (leftover.length > 0) {
+		const seq = await opts.phonemizer.phonemize(leftover, opts.lang);
+		appendSeq(seq, idsAcc);
+		phonemesAcc += seq.phonemes;
+	}
+	yield {
+		ids: Int32Array.from(idsAcc),
+		phonemes: phonemesAcc,
+		isFinal: true,
+	};
+}
+
+function appendSeq(seq: KokoroPhonemeSequence, target: number[]): void {
+	// The phonemizer emits a sequence framed with BOS/EOS — strip both when
+	// accumulating windows so the model sees one BOS at the head and one EOS
+	// at the tail. Defensive against phonemizers that omit framing (the
+	// accumulator simply appends raw ids in that case).
+	const ids = seq.ids;
+	let start = 0;
+	let end = ids.length;
+	if (ids.length >= 2) {
+		// Heuristic: ids ≤ 2 are <pad>/<s>/</s> in the bundled vocab.
+		if (ids[0] !== undefined && ids[0] <= 2) start = 1;
+		if (ids[end - 1] !== undefined && (ids[end - 1] as number) <= 2) end -= 1;
+	}
+	for (let i = start; i < end; i++) {
+		const id = ids[i];
+		if (id !== undefined) target.push(id);
+	}
+}
+
+/** Synchronous variant for whole-phrase callers (the scheduler dispatches
+ *  one phrase at a time in the default `punctuation` mode). Returns the
+ *  full id array — equivalent to draining `streamPhonemes` on a single-item
+ *  iterator and taking the last window. */
+export async function phonemizePhrase(
+	text: string,
+	opts: StreamPhonemesOptions,
+): Promise<PhonemeStreamWindow> {
+	const seq = await opts.phonemizer.phonemize(text, opts.lang);
+	return { ids: seq.ids, phonemes: seq.phonemes, isFinal: true };
+}

package/src/services/voice/kokoro/phonemizer.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Text → phoneme-id adapter for Kokoro-82M.
+ *
+ * Kokoro is trained against espeak-ng IPA tokens with a small fixed vocab
+ * (~178 entries: IPA symbols + stress/punct markers + special <s>/<pad>).
+ * Production deployments should bring real espeak-ng phonemization
+ * (`phonemizer` is the pure-JS eSpeak NG package); the bundled fallback here is a
+ * deterministic letter-to-pseudo-phoneme adapter that produces audible
+ * speech for ASCII English text but loses prosodic accuracy.
+ *
+ * Resolution order:
+ *   1. Caller-provided `KokoroPhonemizer` (preferred — bring your own).
+ *   2. Dynamically-imported `phonemizer`/`phonemize` npm package, if installed.
+ *   3. Bundled `FallbackG2PPhonemizer` (degrades gracefully, never throws on
+ *      ASCII input).
+ *
+ * Non-ASCII text with no real phonemizer raises `KokoroPhonemizerError` —
+ * silent garbage out is worse than a surfaced error (AGENTS.md §3).
+ */
+import { type KokoroPhonemeSequence, type KokoroPhonemizer } from "./types";
+/**
+ * Deterministic ASCII-only G2P used when no real phonemizer is installed.
+ * Lossy by design — this exists so dev environments without espeak-ng still
+ * produce lexically useful smoke output for common English phrases, not to
+ * replace a production Misaki/espeak phonemizer.
+ */
+export declare class FallbackG2PPhonemizer implements KokoroPhonemizer {
+    readonly id = "fallback-g2p";
+    phonemize(text: string, _lang: string): Promise<KokoroPhonemeSequence>;
+}
+/**
+ * Wraps the npm `phonemizer` package when present. It returns an IPA string
+ * which we tokenise with the same VOCAB above. Real Kokoro inference should
+ * use a proper espeak tokenizer — production deployments bring their own;
+ * this is the "install npm and it works" middle ground.
+ */
+export declare class NpmPhonemizePhonemizer implements KokoroPhonemizer {
+    private readonly mod;
+    private readonly callStyle;
+    readonly id: string;
+    private constructor();
+    static tryLoad(): Promise<NpmPhonemizePhonemizer | null>;
+    phonemize(text: string, lang: string): Promise<KokoroPhonemeSequence>;
+}
+export declare function kokoroLangToPhonemizerLanguage(lang: string): string;
+/** Lazy resolver: caller override → npm `phonemizer` → bundled fallback. */
+export declare function resolvePhonemizer(override?: KokoroPhonemizer): Promise<KokoroPhonemizer>;
+/** Exported for tests and bench-time diagnostics. */
+export declare const KOKORO_PAD_ID: number;
+//# sourceMappingURL=phonemizer.d.ts.map

package/src/services/voice/kokoro/phonemizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"phonemizer.d.ts","sourceRoot":"","sources":["phonemizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EACN,KAAK,qBAAqB,EAC1B,KAAK,gBAAgB,EAErB,MAAM,SAAS,CAAC;AAuLjB;;;;;GAKG;AACH,qBAAa,qBAAsB,YAAW,gBAAgB;IAC7D,QAAQ,CAAC,EAAE,kBAAkB;IAEvB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,CAAC;CA2B5E;AAaD;;;;;GAKG;AACH,qBAAa,sBAAuB,YAAW,gBAAgB;IAG7D,OAAO,CAAC,QAAQ,CAAC,GAAG;IAEpB,OAAO,CAAC,QAAQ,CAAC,SAAS;IAJ3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,OAAO;WAQM,OAAO,IAAI,OAAO,CAAC,sBAAsB,GAAG,IAAI,CAAC;IAqBxD,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,CAAC;CA0B3E;AAED,wBAAgB,8BAA8B,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CASnE;AAED,4EAA4E;AAC5E,wBAAsB,iBAAiB,CACtC,QAAQ,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,gBAAgB,CAAC,CAK3B;AAED,qDAAqD;AACrD,eAAO,MAAM,aAAa,QAAM,CAAC"}