@elizaos/plugin-local-inference 2.0.3-beta.2 → 2.0.3-beta.4

package/src/services/voice/kokoro/__tests__/kokoro-engine-bridge.real.test.ts

@@ -23,7 +23,11 @@ import {
 } from "vitest";
 
 import { resolveFusedLibraryPath } from "../../../desktop-fused-ffi-backend-runtime";
-import { EngineVoiceBridge } from "../../engine-bridge";
+import {
+	createKokoroSpeakerPreset,
+	createKokoroTtsBackend,
+	EngineVoiceBridge,
+} from "../../engine-bridge";
 import {
 	type ElizaInferenceFfi,
 	loadElizaInferenceFfi,
@@ -33,8 +37,15 @@ import type {
 	MmapRegionHandle,
 	RefCountedResource,
 } from "../../shared-resources";
-import type { KokoroTtsBackend } from "../kokoro-backend";
-import type { KokoroEngineDiscoveryResult } from "../kokoro-engine-discovery";
+import type { Phrase } from "../../types";
+import {
+	KOKORO_MOBILE_TTFA_BUDGET_MS,
+	type KokoroTtsBackend,
+} from "../kokoro-backend";
+import {
+	type KokoroEngineDiscoveryResult,
+	resolveKokoroEngineConfig,
+} from "../kokoro-engine-discovery";
 
 const isBun = typeof (globalThis as { Bun?: unknown }).Bun !== "undefined";
 const LIB_PATH = resolveFusedLibraryPath(null, process.env);
@@ -149,3 +160,77 @@ describe.skipIf(!isBun || !LIB_PATH)(
 		});
 	},
 );
+
+// ── Gated real TTFA synth (issue #8787 acceptance criteria 4 + 7) ──
+//
+// Drives the REAL fused Kokoro engine over a REAL discovered Kokoro GGUF +
+// voice .bin, synthesizes a phrase, and asserts (a) non-empty 24 kHz PCM is
+// produced and (b) the first AUDIBLE chunk arrives within the mobile-class
+// TTFA budget. Skipped (never faked) when the fused lib is unresolved, the
+// build does not link Kokoro, or no Kokoro model is staged on disk
+// (`resolveKokoroEngineConfig() === null`). Stage a model under
+// `$ELIZA_KOKORO_MODEL_DIR` (or `~/.eliza/local-inference/models/kokoro/`).
+const KOKORO_MODEL = isBun && LIB_PATH ? resolveKokoroEngineConfig() : null;
+
+describe.skipIf(!isBun || !LIB_PATH || !KOKORO_MODEL)(
+	"Kokoro real synth — TTFA on the in-process fused engine",
+	() => {
+		let ffi: ElizaInferenceFfi;
+
+		beforeAll(() => {
+			ffi = loadElizaInferenceFfi(LIB_PATH as string);
+			if (typeof ffi.kokoroSupported !== "function" || !ffi.kokoroSupported()) {
+				throw new Error(
+					`[test] the fused lib at ${LIB_PATH} does not link the in-process Kokoro engine.`,
+				);
+			}
+		});
+		afterAll(() => {
+			(ffi as unknown as { close?: () => void }).close?.();
+		});
+
+		it("synthesizes real PCM and reaches first audio within the mobile TTFA budget", async () => {
+			// Non-null inside the skipIf guard.
+			const kokoro = KOKORO_MODEL as KokoroEngineDiscoveryResult;
+			const backend = createKokoroTtsBackend(kokoro, { ffi });
+			const preset = createKokoroSpeakerPreset(kokoro);
+			const phrase: Phrase = {
+				id: 1,
+				text: "Hello, this is a time to first audio probe.",
+				fromIndex: 0,
+				toIndex: 42,
+				terminator: "punctuation",
+			};
+
+			const startedAt = performance.now();
+			let firstAudibleAtMs: number | null = null;
+			let totalSamples = 0;
+			let sampleRate = 0;
+			await backend.synthesizeStream({
+				phrase,
+				preset,
+				cancelSignal: { cancelled: false },
+				onChunk: (c) => {
+					if (!c.isFinal && c.pcm.length > 0) {
+						if (firstAudibleAtMs === null) {
+							firstAudibleAtMs = performance.now() - startedAt;
+						}
+						totalSamples += c.pcm.length;
+						sampleRate = c.sampleRate;
+					}
+					return undefined;
+				},
+			});
+			backend.dispose();
+
+			// Real, non-empty 24 kHz PCM — never a silent stub (acceptance criterion 4).
+			expect(totalSamples).toBeGreaterThan(0);
+			expect(sampleRate).toBe(24_000);
+			// First audible chunk within the mobile-class TTFA budget (criterion 7).
+			expect(firstAudibleAtMs).not.toBeNull();
+			expect(firstAudibleAtMs as unknown as number).toBeLessThanOrEqual(
+				KOKORO_MOBILE_TTFA_BUDGET_MS,
+			);
+		});
+	},
+);

package/src/services/voice/kokoro/__tests__/runtime-selection.test.ts

@@ -4,228 +4,64 @@ import {
 	selectVoiceBackend,
 } from "../runtime-selection";
 
+// OmniVoice TTS was retired — Kokoro is the only on-device TTS backend, so the
+// selector collapses to a single answer: Kokoro when its artifacts are present,
+// else a hard error (never a silent downgrade).
 describe("selectVoiceBackend", () => {
-	it("mobile: always picks Kokoro, ignoring mode and tier policy", () => {
-		const d = selectVoiceBackend({
-			mobile: true,
-			mode: "omnivoice",
-			kokoroAvailable: true,
-			omnivoiceAvailable: true,
-			tierVoiceBackends: ["omnivoice"],
-		});
+	it("returns Kokoro when artifacts are present", () => {
+		const d = selectVoiceBackend({ kokoroAvailable: true });
 		expect(d.backend).toBe("kokoro");
-		expect(d.reason).toMatch(/mobile/);
-	});
-
-	it("mobile: throws when Kokoro artifacts are missing (no OmniVoice fallback)", () => {
-		expect(() =>
-			selectVoiceBackend({
-				mobile: true,
-				kokoroAvailable: false,
-				omnivoiceAvailable: true,
-			}),
-		).toThrow(/mobile/);
-	});
-
-	it("forces Kokoro when mode=kokoro and artifacts are present", () => {
-		const d = selectVoiceBackend({
-			mode: "kokoro",
-			kokoroAvailable: true,
-			omnivoiceAvailable: true,
-		});
-		expect(d.backend).toBe("kokoro");
-		expect(d.reason).toMatch(/forced/);
-	});
-
-	it("throws when mode=kokoro but artifacts are missing", () => {
-		expect(() =>
-			selectVoiceBackend({
-				mode: "kokoro",
-				kokoroAvailable: false,
-				omnivoiceAvailable: true,
-			}),
-		).toThrow(/Kokoro/);
-	});
-
-	it("forces OmniVoice when mode=omnivoice", () => {
-		const d = selectVoiceBackend({
-			mode: "omnivoice",
-			kokoroAvailable: true,
-			omnivoiceAvailable: true,
-		});
-		expect(d.backend).toBe("omnivoice");
-	});
-
-	it("auto: requireVoiceCloning routes to OmniVoice", () => {
-		const d = selectVoiceBackend({
-			mode: "auto",
-			requireVoiceCloning: true,
-			kokoroAvailable: true,
-			omnivoiceAvailable: true,
-		});
-		expect(d.backend).toBe("omnivoice");
-		expect(d.reason).toMatch(/cloning/);
-	});
-
-	it("auto: requireVoiceCloning with no OmniVoice throws", () => {
-		expect(() =>
-			selectVoiceBackend({
-				mode: "auto",
-				requireVoiceCloning: true,
-				kokoroAvailable: true,
-				omnivoiceAvailable: false,
-			}),
-		).toThrow(/cloning/);
-	});
-
-	it("auto: low targetTtfaMs (<200) prefers Kokoro", () => {
-		const d = selectVoiceBackend({
-			mode: "auto",
-			targetTtfaMs: 120,
-			kokoroAvailable: true,
-			omnivoiceAvailable: true,
-		});
-		expect(d.backend).toBe("kokoro");
-		expect(d.reason).toMatch(/97ms/);
-	});
-
-	it("auto: low targetTtfaMs with no Kokoro falls back to OmniVoice", () => {
-		const d = selectVoiceBackend({
-			mode: "auto",
-			targetTtfaMs: 120,
-			kokoroAvailable: false,
-			omnivoiceAvailable: true,
-		});
-		expect(d.backend).toBe("omnivoice");
-	});
-
-	it("auto: Kokoro RTF beats OmniVoice by ≥10% → Kokoro", () => {
-		const d = selectVoiceBackend({
-			mode: "auto",
-			kokoroRtf: 2.5,
-			omnivoiceRtf: 2.0,
-			kokoroAvailable: true,
-			omnivoiceAvailable: true,
-		});
-		expect(d.backend).toBe("kokoro");
-	});
-
-	it("auto: Kokoro RTF only marginally better → OmniVoice default", () => {
-		const d = selectVoiceBackend({
-			mode: "auto",
-			kokoroRtf: 2.05,
-			omnivoiceRtf: 2.0,
-			kokoroAvailable: true,
-			omnivoiceAvailable: true,
-		});
-		expect(d.backend).toBe("omnivoice");
+		expect(d.reason).toMatch(/Kokoro/);
 	});
 
-	it("auto: Kokoro RTF measured but OmniVoice unmeasured → Kokoro", () => {
-		const d = selectVoiceBackend({
-			mode: "auto",
-			kokoroRtf: 1.5,
-			omnivoiceRtf: null,
-			kokoroAvailable: true,
-			omnivoiceAvailable: true,
-		});
+	it("returns Kokoro on mobile", () => {
+		const d = selectVoiceBackend({ mobile: true, kokoroAvailable: true });
 		expect(d.backend).toBe("kokoro");
+		expect(d.reason).toMatch(/mobile/);
 	});
 
-	it("auto: OmniVoice missing but Kokoro present → Kokoro", () => {
-		const d = selectVoiceBackend({
-			mode: "auto",
-			kokoroAvailable: true,
-			omnivoiceAvailable: false,
-		});
-		expect(d.backend).toBe("kokoro");
+	it("returns Kokoro regardless of mode (kokoro and auto both resolve to Kokoro)", () => {
+		expect(
+			selectVoiceBackend({ mode: "kokoro", kokoroAvailable: true }).backend,
+		).toBe("kokoro");
+		expect(
+			selectVoiceBackend({ mode: "auto", kokoroAvailable: true }).backend,
+		).toBe("kokoro");
 	});
 
-	it("auto: neither backend available throws", () => {
+	it("throws when Kokoro artifacts are missing — no fallback", () => {
+		expect(() => selectVoiceBackend({ kokoroAvailable: false })).toThrow(
+			/Kokoro model artifacts are not present/,
+		);
 		expect(() =>
-			selectVoiceBackend({
-				mode: "auto",
-				kokoroAvailable: false,
-				omnivoiceAvailable: false,
-			}),
-		).toThrow(/no TTS backend/);
-	});
-
-	it("auto: default (no overrides, no tier policy) picks OmniVoice", () => {
-		const d = selectVoiceBackend({
-			mode: "auto",
-			kokoroAvailable: true,
-			omnivoiceAvailable: true,
-		});
-		expect(d.backend).toBe("omnivoice");
-		expect(d.reason).toMatch(/default/);
-	});
-
-	it("auto: tier policy [kokoro] picks Kokoro", () => {
-		const d = selectVoiceBackend({
-			mode: "auto",
-			kokoroAvailable: true,
-			omnivoiceAvailable: true,
-			tierVoiceBackends: ["kokoro"],
-		});
-		expect(d.backend).toBe("kokoro");
-		expect(d.reason).toMatch(/tier default/);
-	});
-
-	it("auto: tier policy [kokoro, omnivoice] picks Kokoro (first wins)", () => {
-		const d = selectVoiceBackend({
-			mode: "auto",
-			kokoroAvailable: true,
-			omnivoiceAvailable: true,
-			tierVoiceBackends: ["kokoro", "omnivoice"],
-		});
-		expect(d.backend).toBe("kokoro");
-	});
-
-	it("auto: tier policy [omnivoice] picks OmniVoice", () => {
-		const d = selectVoiceBackend({
-			mode: "auto",
-			kokoroAvailable: true,
-			omnivoiceAvailable: true,
-			tierVoiceBackends: ["omnivoice"],
-		});
-		expect(d.backend).toBe("omnivoice");
-		expect(d.reason).toMatch(/tier default/);
-	});
-
-	it("auto: tier policy is overridden by requireVoiceCloning", () => {
-		const d = selectVoiceBackend({
-			mode: "auto",
-			kokoroAvailable: true,
-			omnivoiceAvailable: true,
-			tierVoiceBackends: ["kokoro"],
-			requireVoiceCloning: true,
-		});
-		expect(d.backend).toBe("omnivoice");
-		expect(d.reason).toMatch(/cloning/);
+			selectVoiceBackend({ mobile: true, kokoroAvailable: false }),
+		).toThrow(/Kokoro model artifacts are not present/);
 	});
 });
 
 describe("readVoiceBackendModeFromEnv", () => {
-	it("returns undefined when env var is unset", () => {
-		expect(readVoiceBackendModeFromEnv({})).toBeUndefined();
-	});
-
-	it("parses each valid value", () => {
+	it("parses 'kokoro' and 'auto'", () => {
 		expect(readVoiceBackendModeFromEnv({ ELIZA_TTS_BACKEND: "kokoro" })).toBe(
 			"kokoro",
 		);
-		expect(
-			readVoiceBackendModeFromEnv({ ELIZA_TTS_BACKEND: "OMNIVOICE" }),
-		).toBe("omnivoice");
-		expect(readVoiceBackendModeFromEnv({ ELIZA_TTS_BACKEND: "auto" })).toBe(
+		expect(readVoiceBackendModeFromEnv({ ELIZA_TTS_BACKEND: "AUTO" })).toBe(
 			"auto",
 		);
 	});
 
-	it("throws on an invalid value", () => {
+	it("returns undefined when unset", () => {
+		expect(readVoiceBackendModeFromEnv({})).toBeUndefined();
+	});
+
+	it("rejects the retired 'omnivoice' value with a migration message", () => {
+		expect(() =>
+			readVoiceBackendModeFromEnv({ ELIZA_TTS_BACKEND: "omnivoice" }),
+		).toThrow(/retired/);
+	});
+
+	it("rejects any other value", () => {
 		expect(() =>
-			readVoiceBackendModeFromEnv({ ELIZA_TTS_BACKEND: "garbage" }),
-		).toThrow();
+			readVoiceBackendModeFromEnv({ ELIZA_TTS_BACKEND: "bogus" }),
+		).toThrow(/must be 'kokoro' or 'auto'/);
 	});
 });

package/src/services/voice/kokoro/kokoro-backend.ts

@@ -41,6 +41,22 @@ export interface KokoroTtsBackendDeps extends KokoroBackendOptions {
 	runtime: KokoroRuntime;
 }
 
+/**
+ * Wall-clock budget for time-to-first-audio (TTFA) on the Kokoro path for a
+ * mobile-class tier (0.8b/2b/4b — where Kokoro is the *exclusive* backend per
+ * `ELIZA_1_VOICE_BACKENDS`). TTFA is measured to the first **audible** chunk:
+ * the first non-empty sub-phrase PCM slice the scheduler can begin playing,
+ * NOT the whole-phrase synthesis latency. The re-chunking in
+ * `synthesizeStream` (one runtime forward → `streamingChunkSamples` slices) is
+ * exactly what makes those two numbers differ — the listener hears audio after
+ * the first slice, well before the phrase finishes decoding.
+ *
+ * Conservative gate (catches gross regressions without CI flakiness): real
+ * Kokoro-82M first-phrase TTFB is ~97 ms on a warm desktop handle; mobile CPU
+ * plus phonemize headroom lands comfortably under this ceiling.
+ */
+export const KOKORO_MOBILE_TTFA_BUDGET_MS = 700;
+
 /**
  * `KokoroTtsBackend` is a streaming-only TTS backend. The model produces
  * the full waveform in one forward, but we surface it as one body chunk +

package/src/services/voice/kokoro/kokoro-engine-discovery.ts

@@ -8,7 +8,7 @@
  * (see `runtimeKind` below).
  *
  * TRANSPORT NOTE — Kokoro synthesizes in-process through the fused
- * `libelizainference` handle (ABI v10 `eliza_inference_kokoro_*`), the same
+ * `libelizainference` handle (`eliza_inference_kokoro_*`, added at ABI v10), the same
  * dlopen()-ed lib as OmniVoice. The fork links Kokoro's native engine
  * (`tools/kokoro/kokoro_lib`, its own GGUF reader + iSTFT decoder) into the
  * fused build, and `KokoroFfiRuntime` drives it via `kokoroLoad` /

package/src/services/voice/kokoro/kokoro-ffi-runtime.ts

@@ -1,9 +1,9 @@
 /**
  * In-process Kokoro-82M runtime over the fused `libelizainference` FFI
  * (the `eliza_inference_kokoro_*` exports — introduced at ABI v10; the fused
- * library is currently ABI v11, which adds EOT on top, so these symbols are
- * present in every current build — see `ELIZA_INFERENCE_ABI_VERSION` in
- * ffi-bindings.ts).
+ * library is currently ABI v12, which adds EOT (v11) and ASR word timestamps
+ * (v12) on top, so these symbols are present in every current build — see
+ * `ELIZA_INFERENCE_ABI_VERSION` in ffi-bindings.ts).
  *
  * This is the canonical Kokoro execution path on every platform. It replaces
  * the local-TCP `KokoroGgufRuntime` (POST `/v1/audio/speech` on a running

package/src/services/voice/kokoro/pick-runtime.ts

@@ -6,7 +6,7 @@
  *   ffi   (default)  → KokoroFfiRuntime → in-process synthesis through the
  *                       fused `libelizainference` handle (the
  *                       `eliza_inference_kokoro_*` exports, introduced at ABI
- *                       v10; the fused library is currently ABI v11). This is
+ *                       v10; the fused library is currently ABI v12). This is
  *                       the SOLE synthesis path on every platform.
  *   mock             → KokoroMockRuntime. Tests only.
  *