@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/voice/kokoro/__tests__/kokoro-backend.test.ts

@@ -0,0 +1,262 @@
+import { describe, expect, it } from "vitest";
+import { scoreFirstResponseLatency } from "../../e2e-harness";
+import type {
+	AudioSink,
+	Phrase,
+	SpeakerPreset,
+	TtsPcmChunk,
+} from "../../types";
+import {
+	KOKORO_MOBILE_TTFA_BUDGET_MS,
+	KokoroTtsBackend,
+} from "../kokoro-backend";
+import type { KokoroRuntime, KokoroRuntimeInputs } from "../kokoro-runtime";
+import { KokoroMockRuntime } from "../kokoro-runtime";
+import type { KokoroPhonemizer } from "../types";
+import { KOKORO_DEFAULT_VOICE_ID } from "../voices";
+
+function fixedPhonemizer(): KokoroPhonemizer {
+	return {
+		id: "fixed",
+		async phonemize() {
+			return { ids: Int32Array.from([1, 43, 60, 2]), phonemes: "ab" };
+		},
+	};
+}
+
+function makePreset(voiceId: string): SpeakerPreset {
+	return {
+		voiceId,
+		embedding: new Float32Array(8),
+		bytes: new Uint8Array(8),
+	};
+}
+
+function makePhrase(text: string): Phrase {
+	return {
+		id: 1,
+		text,
+		fromIndex: 0,
+		toIndex: text.length - 1,
+		terminator: "punctuation",
+	};
+}
+
+function makeBackend(opts?: { totalSamples?: number; chunkCount?: number }): {
+	backend: KokoroTtsBackend;
+	runtime: KokoroMockRuntime;
+} {
+	const runtime = new KokoroMockRuntime({
+		sampleRate: 24000,
+		totalSamples: opts?.totalSamples ?? 9600, // 0.4s
+		chunkCount: opts?.chunkCount ?? 4,
+	});
+	const backend = new KokoroTtsBackend({
+		runtime,
+		layout: {
+			root: "/tmp/kokoro",
+			modelFile: "kokoro-82m-v1_0.gguf",
+			voicesDir: "/tmp/kokoro/voices",
+			sampleRate: 24000,
+		},
+		defaultVoiceId: KOKORO_DEFAULT_VOICE_ID,
+		phonemizer: fixedPhonemizer(),
+		streamingChunkSamples: 1200, // 50ms — re-chunks the mock output
+	});
+	return { backend, runtime };
+}
+
+describe("KokoroTtsBackend", () => {
+	it("streams PCM chunks and emits a zero-length final tail", async () => {
+		const { backend, runtime } = makeBackend({
+			totalSamples: 9600,
+			chunkCount: 4,
+		});
+		const chunks: TtsPcmChunk[] = [];
+		const result = await backend.synthesizeStream({
+			phrase: makePhrase("hello"),
+			preset: makePreset(KOKORO_DEFAULT_VOICE_ID),
+			cancelSignal: { cancelled: false },
+			onChunk: (c) => {
+				chunks.push({
+					pcm: new Float32Array(c.pcm),
+					sampleRate: c.sampleRate,
+					isFinal: c.isFinal,
+				});
+				return undefined;
+			},
+		});
+		expect(result.cancelled).toBe(false);
+		expect(runtime.calls).toBe(1);
+		expect(chunks.length).toBeGreaterThan(1);
+		const last = chunks[chunks.length - 1];
+		expect(last).toBeDefined();
+		expect(last?.isFinal).toBe(true);
+		expect(last?.pcm.length).toBe(0);
+		const bodyChunks = chunks.slice(0, -1);
+		expect(bodyChunks.every((c) => !c.isFinal)).toBe(true);
+		expect(bodyChunks.every((c) => c.sampleRate === 24000)).toBe(true);
+		// Re-chunking honoured: every body chunk respects the 1200-sample cap.
+		expect(bodyChunks.every((c) => c.pcm.length <= 1200)).toBe(true);
+		const totalBody = bodyChunks.reduce((n, c) => n + c.pcm.length, 0);
+		expect(totalBody).toBe(9600);
+	});
+
+	it("propagates cancelSignal at chunk boundaries", async () => {
+		const { backend } = makeBackend({ totalSamples: 24000, chunkCount: 8 });
+		const cancelSignal = { cancelled: false };
+		let received = 0;
+		const result = await backend.synthesizeStream({
+			phrase: makePhrase("a longer line"),
+			preset: makePreset(KOKORO_DEFAULT_VOICE_ID),
+			cancelSignal,
+			onChunk: (c) => {
+				if (!c.isFinal) {
+					received++;
+					if (received === 2) cancelSignal.cancelled = true;
+				}
+				return undefined;
+			},
+		});
+		expect(result.cancelled).toBe(true);
+		// Final tail is always emitted, even on cancel.
+		// received counts body chunks only; the runtime stops once cancelled.
+		expect(received).toBeLessThanOrEqual(3);
+	});
+
+	it("synthesize() concatenates streamed chunks into one AudioChunk", async () => {
+		const { backend } = makeBackend({ totalSamples: 9600, chunkCount: 4 });
+		const chunk = await backend.synthesize({
+			phrase: makePhrase("hi"),
+			preset: makePreset(KOKORO_DEFAULT_VOICE_ID),
+			cancelSignal: { cancelled: false },
+		});
+		expect(chunk.sampleRate).toBe(24000);
+		expect(chunk.pcm.length).toBe(9600);
+		expect(chunk.phraseId).toBe(1);
+	});
+
+	it("falls back to the default voice when preset.voiceId is unknown", async () => {
+		const { backend } = makeBackend();
+		const chunk = await backend.synthesize({
+			phrase: makePhrase("hi"),
+			preset: makePreset("does_not_exist"),
+			cancelSignal: { cancelled: false },
+		});
+		expect(chunk.pcm.length).toBeGreaterThan(0);
+	});
+
+	it("supportsStreamingTts() returns true (satisfies the streaming seam)", () => {
+		const { backend } = makeBackend();
+		expect(backend.supportsStreamingTts()).toBe(true);
+	});
+});
+
+// ── TTFA: the first AUDIBLE chunk streams out before the phrase finishes ──
+//
+// These lock the streaming-TTFA contract (issue #8787 acceptance criterion 7)
+// deterministically — no native model, no wall-clock flakiness. The gated
+// real-FFI synth that measures TTFA against a true Kokoro forward lives in
+// `kokoro-engine-bridge.real.test.ts` and skips when artifacts are absent.
+
+/**
+ * A runtime that emits `chunkCount` body chunks and records, for every body
+ * chunk, how many runtime chunks had been produced when the backend re-emitted
+ * its first audible slice. Proves TTFA is bounded by ONE runtime forward
+ * boundary, not the whole phrase.
+ */
+class CountingKokoroRuntime implements KokoroRuntime {
+	readonly id = "mock" as const;
+	readonly sampleRate = 24000;
+	emitted = 0;
+	constructor(
+		private readonly totalSamples: number,
+		private readonly chunkCount: number,
+	) {}
+	async synthesize(args: KokoroRuntimeInputs): Promise<{ cancelled: boolean }> {
+		const perChunk = Math.max(
+			1,
+			Math.ceil(this.totalSamples / this.chunkCount),
+		);
+		for (let off = 0; off < this.totalSamples; off += perChunk) {
+			if (args.cancelSignal.cancelled) return { cancelled: true };
+			const n = Math.min(perChunk, this.totalSamples - off);
+			this.emitted++;
+			const want = args.onChunk({
+				pcm: new Float32Array(n).fill(0.05),
+				sampleRate: this.sampleRate,
+				isFinal: false,
+			});
+			if (want === true) return { cancelled: true };
+		}
+		args.onChunk({
+			pcm: new Float32Array(0),
+			sampleRate: this.sampleRate,
+			isFinal: true,
+		});
+		return { cancelled: false };
+	}
+	dispose(): void {}
+}
+
+describe("KokoroTtsBackend — streaming TTFA", () => {
+	it("emits the first audible chunk from the first runtime forward (sub-phrase TTFA)", async () => {
+		const runtime = new CountingKokoroRuntime(48000, 4); // 2s across 4 forwards
+		const backend = new KokoroTtsBackend({
+			runtime,
+			layout: {
+				root: "/tmp/kokoro",
+				modelFile: "kokoro-82m-v1_0.gguf",
+				voicesDir: "/tmp/kokoro/voices",
+				sampleRate: 24000,
+			},
+			defaultVoiceId: KOKORO_DEFAULT_VOICE_ID,
+			phonemizer: fixedPhonemizer(),
+			streamingChunkSamples: 1200, // 50ms slices
+		});
+
+		let firstAudibleAtRuntimeEmits = -1;
+		let firstAudibleSamples = -1;
+		await backend.synthesizeStream({
+			phrase: makePhrase("a full sentence that decodes in one forward"),
+			preset: makePreset(KOKORO_DEFAULT_VOICE_ID),
+			cancelSignal: { cancelled: false },
+			onChunk: (c) => {
+				if (!c.isFinal && c.pcm.length > 0 && firstAudibleAtRuntimeEmits < 0) {
+					firstAudibleAtRuntimeEmits = runtime.emitted;
+					firstAudibleSamples = c.pcm.length;
+				}
+				return undefined;
+			},
+		});
+
+		// The listener hears audio after the FIRST runtime forward, not all 4.
+		expect(firstAudibleAtRuntimeEmits).toBe(1);
+		// The first audible chunk is a bounded sub-phrase slice, not the phrase.
+		expect(firstAudibleSamples).toBeGreaterThan(0);
+		expect(firstAudibleSamples).toBeLessThanOrEqual(1200);
+	});
+
+	it("a mobile-class TTFA gate passes within budget and fails past it", () => {
+		// Representative warm-handle Kokoro first-phrase TTFB (~97ms TTFB +
+		// phonemize). Well within the mobile budget.
+		const within = scoreFirstResponseLatency({
+			turnStartedAtMs: 1_000,
+			ttsFirstAudioAtMs: 1_000 + 180,
+			maxFirstAudioMs: KOKORO_MOBILE_TTFA_BUDGET_MS,
+		});
+		expect(within.firstAudioMs).toBe(180);
+		expect(within.passed).toBe(true);
+
+		// A regression that blows the budget must fail the gate, never silently pass.
+		const blown = scoreFirstResponseLatency({
+			turnStartedAtMs: 1_000,
+			ttsFirstAudioAtMs: 1_000 + KOKORO_MOBILE_TTFA_BUDGET_MS + 50,
+			maxFirstAudioMs: KOKORO_MOBILE_TTFA_BUDGET_MS,
+		});
+		expect(blown.passed).toBe(false);
+	});
+});
+
+// Local declaration so the test file does not import the audio sink (unused).
+void (null as unknown as AudioSink);

package/src/services/voice/kokoro/__tests__/kokoro-engine-bridge.real.test.ts

@@ -0,0 +1,236 @@
+/**
+ * Real-FFI tests for `EngineVoiceBridge.startKokoroOnly`: construct the
+ * Kokoro-backed bridge against the ACTUAL fused `libelizainference` — loaded,
+ * `create`d, and probed for `kokoroSupported()` — never a stub.
+ *
+ * Skipped (not faked) when the fused lib is not resolvable. To run them, point
+ * `ELIZA_INFERENCE_LIBRARY` (or `ELIZA_INFERENCE_LIB_DIR`) at a built
+ * `libelizainference` or build one via
+ * `packages/app-core/scripts/build-llama-cpp-mtp.mjs`.
+ */
+
+import { mkdtempSync, rmSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import {
+	afterAll,
+	afterEach,
+	beforeAll,
+	beforeEach,
+	describe,
+	expect,
+	it,
+} from "vitest";
+
+import { resolveFusedLibraryPath } from "../../../desktop-fused-ffi-backend-runtime";
+import {
+	createKokoroSpeakerPreset,
+	createKokoroTtsBackend,
+	EngineVoiceBridge,
+} from "../../engine-bridge";
+import {
+	type ElizaInferenceFfi,
+	loadElizaInferenceFfi,
+} from "../../ffi-bindings";
+import type { VoiceLifecycleLoaders } from "../../lifecycle";
+import type {
+	MmapRegionHandle,
+	RefCountedResource,
+} from "../../shared-resources";
+import type { Phrase } from "../../types";
+import {
+	KOKORO_MOBILE_TTFA_BUDGET_MS,
+	type KokoroTtsBackend,
+} from "../kokoro-backend";
+import {
+	type KokoroEngineDiscoveryResult,
+	resolveKokoroEngineConfig,
+} from "../kokoro-engine-discovery";
+
+const isBun = typeof (globalThis as { Bun?: unknown }).Bun !== "undefined";
+const LIB_PATH = resolveFusedLibraryPath(null, process.env);
+
+// The real fused `create()` validates that the anchor dir exists, so the
+// layout root points at a live temp dir (`tmp`, per-test) rather than a
+// never-created path.
+function makeKokoroConfig(root: string): KokoroEngineDiscoveryResult {
+	return {
+		layout: {
+			root,
+			modelFile: "kokoro-82m-v1_0-Q4_K_M.gguf",
+			voicesDir: path.join(root, "voices"),
+			sampleRate: 24_000,
+		},
+		defaultVoiceId: "af_bella",
+	};
+}
+
+function lifecycleLoadersOk(): VoiceLifecycleLoaders {
+	const region: MmapRegionHandle = {
+		id: "test-region",
+		path: "/tmp/test",
+		sizeBytes: 0,
+		async evictPages() {},
+		async release() {},
+	};
+	const refc: RefCountedResource = { id: "refc", async release() {} };
+	return {
+		loadTtsRegion: async () => region,
+		loadAsrRegion: async () => region,
+		loadVoiceCaches: async () => refc,
+		loadVoiceSchedulerNodes: async () => refc,
+	};
+}
+
+describe.skipIf(!isBun || !LIB_PATH)(
+	"EngineVoiceBridge — kokoroOnly real FFI construction",
+	() => {
+		let ffi: ElizaInferenceFfi;
+		let tmp: string;
+
+		beforeAll(() => {
+			// LIB_PATH is non-null inside the skipIf-guarded block.
+			ffi = loadElizaInferenceFfi(LIB_PATH as string);
+			if (typeof ffi.kokoroSupported !== "function" || !ffi.kokoroSupported()) {
+				throw new Error(
+					`[test] the fused lib at ${LIB_PATH} (ABI v${ffi.libraryAbiVersion}) does not link the in-process Kokoro engine — rebuild with the Kokoro engine enabled.`,
+				);
+			}
+		});
+		afterAll(() => {
+			(ffi as unknown as { close?: () => void }).close?.();
+		});
+		beforeEach(() => {
+			tmp = mkdtempSync(path.join(os.tmpdir(), "kokoro-bridge-real-"));
+		});
+		afterEach(() => {
+			rmSync(tmp, { recursive: true, force: true });
+		});
+
+		it("constructs a KokoroTtsBackend against the real fused lib", () => {
+			const bridge = EngineVoiceBridge.start({
+				bundleRoot: "", // kokoroOnly skips the existsSync check
+				useFfiBackend: false,
+				kokoroOnly: makeKokoroConfig(tmp),
+				kokoroFfi: ffi,
+				lifecycleLoaders: lifecycleLoadersOk(),
+			});
+			expect(bridge.backend?.id).toBe("kokoro");
+			expect(bridge.asrAvailable).toBe(false); // ASR is not served from this path
+			expect(bridge.ffi).toBeNull();
+			bridge.dispose();
+		});
+
+		it("uses the provided bundleRoot as working dir when it exists", () => {
+			const bridge = EngineVoiceBridge.start({
+				bundleRoot: tmp,
+				useFfiBackend: false,
+				kokoroOnly: makeKokoroConfig(tmp),
+				kokoroFfi: ffi,
+				lifecycleLoaders: lifecycleLoadersOk(),
+			});
+			expect(bridge.backend?.id).toBe("kokoro");
+			bridge.dispose();
+		});
+
+		it("arms with no-op lifecycle loaders by default (no real mmap regions)", async () => {
+			const bridge = EngineVoiceBridge.start({
+				bundleRoot: "",
+				useFfiBackend: false,
+				kokoroOnly: makeKokoroConfig(tmp),
+				kokoroFfi: ffi,
+			});
+			await expect(bridge.arm()).resolves.toBeUndefined();
+			bridge.dispose();
+		});
+
+		it("preserves the requested sample rate from the kokoroOnly layout", () => {
+			const bridge = EngineVoiceBridge.start({
+				bundleRoot: "",
+				useFfiBackend: false,
+				kokoroOnly: {
+					...makeKokoroConfig(tmp),
+					layout: { ...makeKokoroConfig(tmp).layout, sampleRate: 16_000 },
+				},
+				kokoroFfi: ffi,
+				lifecycleLoaders: lifecycleLoadersOk(),
+			});
+			expect((bridge.backend as KokoroTtsBackend).sampleRate).toBe(16_000);
+			bridge.dispose();
+		});
+	},
+);
+
+// ── Gated real TTFA synth (issue #8787 acceptance criteria 4 + 7) ──
+//
+// Drives the REAL fused Kokoro engine over a REAL discovered Kokoro GGUF +
+// voice .bin, synthesizes a phrase, and asserts (a) non-empty 24 kHz PCM is
+// produced and (b) the first AUDIBLE chunk arrives within the mobile-class
+// TTFA budget. Skipped (never faked) when the fused lib is unresolved, the
+// build does not link Kokoro, or no Kokoro model is staged on disk
+// (`resolveKokoroEngineConfig() === null`). Stage a model under
+// `$ELIZA_KOKORO_MODEL_DIR` (or `~/.eliza/local-inference/models/kokoro/`).
+const KOKORO_MODEL = isBun && LIB_PATH ? resolveKokoroEngineConfig() : null;
+
+describe.skipIf(!isBun || !LIB_PATH || !KOKORO_MODEL)(
+	"Kokoro real synth — TTFA on the in-process fused engine",
+	() => {
+		let ffi: ElizaInferenceFfi;
+
+		beforeAll(() => {
+			ffi = loadElizaInferenceFfi(LIB_PATH as string);
+			if (typeof ffi.kokoroSupported !== "function" || !ffi.kokoroSupported()) {
+				throw new Error(
+					`[test] the fused lib at ${LIB_PATH} does not link the in-process Kokoro engine.`,
+				);
+			}
+		});
+		afterAll(() => {
+			(ffi as unknown as { close?: () => void }).close?.();
+		});
+
+		it("synthesizes real PCM and reaches first audio within the mobile TTFA budget", async () => {
+			// Non-null inside the skipIf guard.
+			const kokoro = KOKORO_MODEL as KokoroEngineDiscoveryResult;
+			const backend = createKokoroTtsBackend(kokoro, { ffi });
+			const preset = createKokoroSpeakerPreset(kokoro);
+			const phrase: Phrase = {
+				id: 1,
+				text: "Hello, this is a time to first audio probe.",
+				fromIndex: 0,
+				toIndex: 42,
+				terminator: "punctuation",
+			};
+
+			const startedAt = performance.now();
+			let firstAudibleAtMs: number | null = null;
+			let totalSamples = 0;
+			let sampleRate = 0;
+			await backend.synthesizeStream({
+				phrase,
+				preset,
+				cancelSignal: { cancelled: false },
+				onChunk: (c) => {
+					if (!c.isFinal && c.pcm.length > 0) {
+						if (firstAudibleAtMs === null) {
+							firstAudibleAtMs = performance.now() - startedAt;
+						}
+						totalSamples += c.pcm.length;
+						sampleRate = c.sampleRate;
+					}
+					return undefined;
+				},
+			});
+			backend.dispose();
+
+			// Real, non-empty 24 kHz PCM — never a silent stub (acceptance criterion 4).
+			expect(totalSamples).toBeGreaterThan(0);
+			expect(sampleRate).toBe(24_000);
+			// First audible chunk within the mobile-class TTFA budget (criterion 7).
+			expect(firstAudibleAtMs).not.toBeNull();
+			expect(firstAudibleAtMs as unknown as number).toBeLessThanOrEqual(
+				KOKORO_MOBILE_TTFA_BUDGET_MS,
+			);
+		});
+	},
+);

package/src/services/voice/kokoro/__tests__/kokoro-engine-bridge.test.ts

@@ -0,0 +1,60 @@
+/**
+ * Unit tests for `EngineVoiceBridge.startKokoroOnly` option validation — the
+ * mutually-exclusive backend-path checks that throw before any FFI work, so
+ * they need no library.
+ *
+ * Real FFI construction (loading `libelizainference` + `create` + Kokoro
+ * support) is exercised against the ACTUAL library — never a stub — in
+ * `kokoro-engine-bridge.real.test.ts`, gated on the fused lib being built.
+ */
+
+import path from "node:path";
+import { describe, expect, it } from "vitest";
+
+import { EngineVoiceBridge, VoiceStartupError } from "../../engine-bridge";
+import type { KokoroEngineDiscoveryResult } from "../kokoro-engine-discovery";
+
+function makeKokoroConfig(): KokoroEngineDiscoveryResult {
+	return {
+		layout: {
+			root: "/tmp/fake-kokoro",
+			modelFile: "kokoro-82m-v1_0-Q4_K_M.gguf",
+			voicesDir: path.join("/tmp/fake-kokoro", "voices"),
+			sampleRate: 24_000,
+		},
+		defaultVoiceId: "af_bella",
+	};
+}
+
+describe("EngineVoiceBridge — kokoroOnly option validation", () => {
+	it("throws when kokoroOnly is combined with useFfiBackend:true", () => {
+		expect(() =>
+			EngineVoiceBridge.start({
+				bundleRoot: "",
+				useFfiBackend: true,
+				kokoroOnly: makeKokoroConfig(),
+			}),
+		).toThrow(VoiceStartupError);
+	});
+
+	it("throws when kokoroOnly is combined with backendOverride", () => {
+		expect(() =>
+			EngineVoiceBridge.start({
+				bundleRoot: "",
+				useFfiBackend: false,
+				kokoroOnly: makeKokoroConfig(),
+				backendOverride: {
+					async synthesize() {
+						return {
+							phraseId: 0,
+							fromIndex: 0,
+							toIndex: 0,
+							pcm: new Float32Array(0),
+							sampleRate: 24000,
+						};
+					},
+				} as never,
+			}),
+		).toThrow(VoiceStartupError);
+	});
+});