@elizaos/plugin-local-inference 2.0.3-beta.2 → 2.0.3-beta.3

package/src/services/voice/kokoro/runtime-selection.ts

@@ -1,88 +1,27 @@
 /**
- * Voice backend selection — picks between OmniVoice and Kokoro at engine
- * arm time.
+ * Voice backend selection — Kokoro is the only on-device TTS backend.
  *
- * The scheduler is backend-agnostic — both implementations satisfy
- * `OmniVoiceBackend + StreamingTtsBackend`. This module isolates the
- * decision logic so the engine layer (or a test) can ask "which backend
- * should I instantiate?" and get a single, auditable answer.
- *
- * Decision modes:
- *
- *   - `omnivoice` — force OmniVoice. Used when the caller needs voice
- *      cloning (Kokoro v1.0 has fixed voice packs, no per-user cloning).
- *   - `kokoro`    — force Kokoro. Used when the caller cares about
- *      first-audio latency over voice fidelity (Kokoro ≈ 97ms CPU TTFB,
- *      OmniVoice ≈ 200ms on the fused build).
- *   - `auto`      — apply the documented heuristic below.
- *
- * Mobile precedence: when `mobile === true` the selector returns Kokoro
- * unconditionally (it is smaller + faster and the only backend shipped in
- * mobile-class bundles). This short-circuits every mode and heuristic below.
- *
- * `auto` heuristic (deterministic, no model probes — those go through the
- * autotune layer in `voice/scheduler.ts`):
- *
- *   1. If `requireVoiceCloning === true` → OmniVoice.
- *   2. If `targetTtfaMs` is set and < 200 → Kokoro.
- *   3. If a Kokoro RTF measurement is available and OmniVoice RTF is not,
- *      or Kokoro RTF beats OmniVoice by ≥ 10% → Kokoro.
- *   4. Else → first entry of `tierVoiceBackends` if provided (the
- *      catalog's declared per-tier default). Falls back to OmniVoice
- *      only when no tier policy is supplied.
- *
- * Tier policy comes from `ELIZA_1_VOICE_BACKENDS` in
- * `packages/shared/src/local-inference/catalog.ts`. Callers should pass
- * the active bundle's `voiceBackends` array so the selection is
- * data-driven (small tiers → Kokoro default; large tiers → OmniVoice).
- *
- * The decision returns a tagged discriminated union, not a backend
- * instance, so the engine layer can instantiate the chosen backend with
- * its own dependencies (FFI handle / Kokoro runtime / etc.). This keeps
- * the selection logic unit-testable without dragging the ORT or FFI
- * surfaces into the test graph.
+ * OmniVoice TTS was retired (it was an autoregressive LM-based synth: heavier,
+ * slower TTFB, and only it could voice-clone). Kokoro (StyleTTS2 / iSTFTNet,
+ * non-autoregressive) is faster, smaller, and ships in every bundle — desktop
+ * and mobile alike — so the selector collapses to a single auditable answer.
+ * The function + env reader are kept (rather than inlined) so the engine layer
+ * and tests retain one seam to ask "is a TTS backend available?".
  */
 
-export type VoiceBackendChoice = "omnivoice" | "kokoro";
+export type VoiceBackendChoice = "kokoro";
 
-export type VoiceBackendMode = VoiceBackendChoice | "auto";
+/** Retained for the env override; `auto` and `kokoro` both resolve to Kokoro. */
+export type VoiceBackendMode = "kokoro" | "auto";
 
 export interface VoiceBackendInputs {
-	/** Caller-set mode. Defaults to `auto`. */
+	/** Caller-set mode. Defaults to `auto`; both modes resolve to Kokoro. */
 	mode?: VoiceBackendMode;
-	/** Time-to-first-audio target (ms). Lower → prefer Kokoro. */
-	targetTtfaMs?: number;
-	/** Whether the caller needs per-user voice cloning. */
-	requireVoiceCloning?: boolean;
-	/** Latest measured RTF for Kokoro on this device (audio_seconds / wall_seconds). */
-	kokoroRtf?: number | null;
-	/** Latest measured RTF for OmniVoice on this device. */
-	omnivoiceRtf?: number | null;
-	/** Whether Kokoro model artifacts are present on disk. The selector
-	 *  never returns Kokoro when this is `false` — no silent downgrade. */
+	/** Whether Kokoro model artifacts are present on disk. The selector throws
+	 *  rather than returning a backend when this is `false` — no silent downgrade. */
 	kokoroAvailable: boolean;
-	/** Whether the OmniVoice FFI library is present on disk. */
-	omnivoiceAvailable: boolean;
-	/**
-	 * True on mobile (iOS / Android) builds. Mobile uses Kokoro exclusively —
-	 * it is smaller and faster than OmniVoice and is the only TTS backend
-	 * shipped in mobile-class bundles. When set, the selector returns Kokoro
-	 * unconditionally (ignoring `mode`, RTF, and TTFA heuristics) and throws
-	 * if Kokoro artifacts are missing rather than falling back to OmniVoice.
-	 */
+	/** True on mobile (iOS / Android) builds — informational only now. */
 	mobile?: boolean;
-	/**
-	 * The active bundle's per-tier voice backend policy, as declared in
-	 * `ELIZA_1_VOICE_BACKENDS`. First entry is the catalog default for
-	 * the tier; later entries are also bundled. The selector reads this
-	 * to make the `auto` default tier-aware rather than hard-coding a
-	 * single backend.
-	 *
-	 * Omit when called outside the Eliza-1 catalog context (e.g. ad-hoc
-	 * smoke benches) — the selector falls back to OmniVoice as the
-	 * historical default in that case.
-	 */
-	tierVoiceBackends?: ReadonlyArray<VoiceBackendChoice>;
 }
 
 export interface VoiceBackendDecision {
@@ -91,147 +30,35 @@ export interface VoiceBackendDecision {
 	reason: string;
 }
 
-const TTFA_CUTOFF_MS = 200;
-const RTF_MARGIN = 1.1; // Kokoro must beat OmniVoice by 10% to win on RTF.
-
-/** Resolve the env override (`ELIZA_TTS_BACKEND=kokoro|omnivoice|auto`). */
+/** Resolve the env override (`ELIZA_TTS_BACKEND=kokoro|auto`). */
 export function readVoiceBackendModeFromEnv(
 	env: NodeJS.ProcessEnv = process.env,
 ): VoiceBackendMode | undefined {
 	const raw = env.ELIZA_TTS_BACKEND?.trim().toLowerCase();
 	if (!raw) return undefined;
-	if (raw === "kokoro" || raw === "omnivoice" || raw === "auto") return raw;
+	if (raw === "kokoro" || raw === "auto") return raw;
+	if (raw === "omnivoice") {
+		throw new Error(
+			"[voice] ELIZA_TTS_BACKEND=omnivoice is retired — OmniVoice TTS was removed; Kokoro is the only on-device TTS backend.",
+		);
+	}
 	throw new Error(
-		`[voice] ELIZA_TTS_BACKEND must be one of 'kokoro', 'omnivoice', 'auto' (got '${raw}')`,
+		`[voice] ELIZA_TTS_BACKEND must be 'kokoro' or 'auto' (got '${raw}')`,
 	);
 }
 
 export function selectVoiceBackend(
 	inputs: VoiceBackendInputs,
 ): VoiceBackendDecision {
-	// Mobile is Kokoro-exclusive: it is smaller + faster and is the only TTS
-	// backend shipped in mobile-class bundles. This wins over every mode and
-	// heuristic below — no OmniVoice fallback on phones.
-	if (inputs.mobile) {
-		if (!inputs.kokoroAvailable) {
-			throw new Error(
-				"[voice] mobile builds use Kokoro exclusively but its model artifacts are not present on disk",
-			);
-		}
-		return {
-			backend: "kokoro",
-			reason:
-				"mobile platform — Kokoro exclusively (OmniVoice not shipped on mobile)",
-		};
-	}
-
-	const mode = inputs.mode ?? "auto";
-
-	if (mode === "kokoro") {
-		if (!inputs.kokoroAvailable) {
-			throw new Error(
-				"[voice] ELIZA_TTS_BACKEND=kokoro but Kokoro model artifacts are not present on disk",
-			);
-		}
-		return { backend: "kokoro", reason: "forced via mode=kokoro" };
-	}
-
-	if (mode === "omnivoice") {
-		if (!inputs.omnivoiceAvailable) {
-			throw new Error(
-				"[voice] ELIZA_TTS_BACKEND=omnivoice but the OmniVoice FFI library is not present",
-			);
-		}
-		return { backend: "omnivoice", reason: "forced via mode=omnivoice" };
-	}
-
-	// `auto` — apply heuristics.
-	if (inputs.requireVoiceCloning) {
-		if (!inputs.omnivoiceAvailable) {
-			throw new Error(
-				"[voice] voice cloning required but OmniVoice FFI library is not available; Kokoro v1.0 has no per-user cloning",
-			);
-		}
-		return {
-			backend: "omnivoice",
-			reason: "voice cloning required (Kokoro v1.0 cannot clone)",
-		};
-	}
-
-	if (
-		inputs.targetTtfaMs !== undefined &&
-		inputs.targetTtfaMs < TTFA_CUTOFF_MS
-	) {
-		if (inputs.kokoroAvailable) {
-			return {
-				backend: "kokoro",
-				reason: `targetTtfaMs=${inputs.targetTtfaMs} < ${TTFA_CUTOFF_MS} → Kokoro (~97ms CPU TTFB)`,
-			};
-		}
-		if (!inputs.omnivoiceAvailable) {
-			throw new Error(
-				"[voice] no TTS backend available (neither Kokoro model nor OmniVoice FFI library on disk)",
-			);
-		}
-		return {
-			backend: "omnivoice",
-			reason: "targetTtfaMs requested but Kokoro artifacts missing",
-		};
-	}
-
-	if (
-		inputs.kokoroAvailable &&
-		inputs.kokoroRtf !== null &&
-		inputs.kokoroRtf !== undefined &&
-		inputs.kokoroRtf > 0
-	) {
-		if (
-			inputs.omnivoiceRtf === null ||
-			inputs.omnivoiceRtf === undefined ||
-			inputs.omnivoiceRtf <= 0
-		) {
-			return {
-				backend: "kokoro",
-				reason: `Kokoro RTF=${inputs.kokoroRtf.toFixed(2)} measured; OmniVoice RTF unknown`,
-			};
-		}
-		if (inputs.kokoroRtf >= inputs.omnivoiceRtf * RTF_MARGIN) {
-			return {
-				backend: "kokoro",
-				reason: `Kokoro RTF=${inputs.kokoroRtf.toFixed(2)} beats OmniVoice RTF=${inputs.omnivoiceRtf.toFixed(2)} by ≥10%`,
-			};
-		}
-	}
-
-	if (!inputs.omnivoiceAvailable && inputs.kokoroAvailable) {
-		return {
-			backend: "kokoro",
-			reason: "OmniVoice FFI library not available; Kokoro is the only option",
-		};
-	}
-	if (!inputs.omnivoiceAvailable && !inputs.kokoroAvailable) {
+	if (!inputs.kokoroAvailable) {
 		throw new Error(
-			"[voice] no TTS backend available (neither Kokoro model nor OmniVoice FFI library on disk)",
+			"[voice] Kokoro model artifacts are not present on disk; Kokoro is the only on-device TTS backend (install an Eliza-1 bundle or stage the Kokoro GGUF + at least one voice .bin).",
 		);
 	}
-	// Both backends available, no override. Honor the tier's declared
-	// default if the caller supplied one (catalog-driven), else fall back
-	// to OmniVoice to preserve historical behavior for non-Eliza-1 contexts.
-	const tierDefault = inputs.tierVoiceBackends?.[0];
-	if (tierDefault === "kokoro") {
-		return {
-			backend: "kokoro",
-			reason: "tier default — kokoro per ELIZA_1_VOICE_BACKENDS",
-		};
-	}
-	if (tierDefault === "omnivoice") {
-		return {
-			backend: "omnivoice",
-			reason: "tier default — omnivoice per ELIZA_1_VOICE_BACKENDS",
-		};
-	}
 	return {
-		backend: "omnivoice",
-		reason: "default — OmniVoice on the fused build (no tier policy supplied)",
+		backend: "kokoro",
+		reason: inputs.mobile
+			? "mobile platform — Kokoro (the only on-device TTS backend)"
+			: "Kokoro (the only on-device TTS backend)",
 	};
 }

package/src/services/voice/live-diarization-session.echo.test.ts

@@ -0,0 +1,232 @@
+/**
+ * LiveDiarizationSession echo-reference wiring (#9455/#9583).
+ *
+ * Proves the agent-side AEC seam without the fused FFI: the session decodes
+ * agent-playback (far-end) frames into its alignment buffer, and the
+ * `echoReference` read seam (the closure handed to AudioFrameConsumer) returns
+ * the time-aligned far-end slice — zero-filled until playback is pushed, and
+ * reset on barge-in. The model-heavy path (real NLMS cancellation over the
+ * fused VAD/encoder/diarizer) is covered by the host smoke harness.
+ */
+
+import { describe, expect, it } from "vitest";
+import type { AudioFrameEvent } from "./audio-frame-consumer.js";
+import { platformPlaybackDelaySamples } from "./echo-delay.js";
+import {
+	LiveDiarizationSession,
+	type RuntimeEventSink,
+} from "./live-diarization-session.js";
+
+const SAMPLE_RATE = 16_000;
+
+function fakeRuntime(): RuntimeEventSink {
+	return { emitEvent: async () => {} } as unknown as RuntimeEventSink;
+}
+
+/** Build a well-formed playback frame from Float32 [-1,1] samples. */
+function playbackFrame(
+	samples: Float32Array,
+	frameIndex: number,
+): AudioFrameEvent {
+	const buf = Buffer.alloc(samples.length * 2);
+	for (let i = 0; i < samples.length; i += 1) {
+		const clamped = Math.max(-1, Math.min(1, samples[i] ?? 0));
+		buf.writeInt16LE(Math.round(clamped * 32_768) | 0, i * 2);
+	}
+	return {
+		pcm16: buf.toString("base64"),
+		sampleRate: SAMPLE_RATE,
+		channels: 1,
+		samples: samples.length,
+		rms: 0,
+		timestamp: frameIndex * 20,
+		frameIndex,
+	};
+}
+
+/** A deterministic ramp in [-0.5, 0.5]. */
+function ramp(n: number): Float32Array {
+	const out = new Float32Array(n);
+	for (let i = 0; i < n; i += 1) out[i] = i / n - 0.5;
+	return out;
+}
+
+function noise(n: number): Float32Array {
+	const out = new Float32Array(n);
+	let seed = 0x12345678;
+	for (let i = 0; i < n; i += 1) {
+		seed = (seed * 1_664_525 + 1_013_904_223) >>> 0;
+		out[i] = ((seed / 0xffffffff) * 2 - 1) * 0.6;
+	}
+	return out;
+}
+
+describe("LiveDiarizationSession echo reference", () => {
+	it("returns a zero far-end reference before any playback is pushed", () => {
+		const session = new LiveDiarizationSession(fakeRuntime());
+		const ref = session.echoReferenceFrame(0, 320);
+		expect(ref).toHaveLength(320);
+		expect(ref.every((v) => v === 0)).toBe(true);
+	});
+
+	it("aligns playback by frame timestamp as the far-end (delay 0)", () => {
+		const session = new LiveDiarizationSession(fakeRuntime());
+		const playback = ramp(320);
+		session.pushPlayback([playbackFrame(playback, 0)]);
+
+		const ref = session.echoReferenceFrame(0, 320);
+		expect(ref).toHaveLength(320);
+		// Not zero-filled — the canceller now has a real far-end to cancel.
+		expect(ref.some((v) => v !== 0)).toBe(true);
+		// s16 round-trip is exact to ~1/32768; assert close alignment to the
+		// pushed ramp, not silence.
+		for (let i = 0; i < 320; i += 1) {
+			expect(Math.abs((ref[i] ?? 0) - (playback[i] ?? 0))).toBeLessThan(1e-3);
+		}
+	});
+
+	it("returns the trailing window when asked for fewer samples than pushed", () => {
+		const session = new LiveDiarizationSession(fakeRuntime());
+		const playback = ramp(640);
+		session.pushPlayback([playbackFrame(playback, 0)]);
+
+		const ref = session.echoReferenceFrame(20, 320);
+		expect(ref).toHaveLength(320);
+		// The aligned window is the LAST 320 of the 640 pushed (delay 0).
+		for (let i = 0; i < 320; i += 1) {
+			expect(Math.abs((ref[i] ?? 0) - (playback[320 + i] ?? 0))).toBeLessThan(
+				1e-3,
+			);
+		}
+	});
+
+	it("resetPlayback drops buffered far-end (barge-in / playback stop)", () => {
+		const session = new LiveDiarizationSession(fakeRuntime());
+		session.pushPlayback([playbackFrame(ramp(320), 0)]);
+		expect(session.echoReferenceFrame(0, 320).some((v) => v !== 0)).toBe(true);
+
+		session.resetPlayback();
+		expect(session.echoReferenceFrame(0, 320).every((v) => v === 0)).toBe(true);
+	});
+
+	it("zero-fills natural gaps between playback frames", () => {
+		const session = new LiveDiarizationSession(fakeRuntime());
+		const first = ramp(320);
+		const later = ramp(320);
+		session.pushPlayback([playbackFrame(first, 0), playbackFrame(later, 5)]);
+
+		expect(session.echoReferenceFrame(0, 320).some((v) => v !== 0)).toBe(true);
+		expect(session.echoReferenceFrame(40, 320).every((v) => v === 0)).toBe(
+			true,
+		);
+		expect(session.echoReferenceFrame(100, 320).some((v) => v !== 0)).toBe(
+			true,
+		);
+	});
+
+	it("self-calibrates playback-to-mic delay from correlated echo", () => {
+		const session = new LiveDiarizationSession(fakeRuntime());
+		const frameSamples = 320;
+		const totalSamples = 16_000;
+		const delaySamples = 240;
+		const playback = noise(totalSamples);
+
+		for (let offset = 0; offset < totalSamples; offset += frameSamples) {
+			session.pushPlayback([
+				playbackFrame(
+					playback.slice(offset, offset + frameSamples),
+					offset / frameSamples,
+				),
+			]);
+		}
+
+		for (let offset = 0; offset < totalSamples; offset += frameSamples) {
+			const near = new Float32Array(frameSamples);
+			for (let i = 0; i < frameSamples; i += 1) {
+				near[i] = playback[offset + i - delaySamples] ?? 0;
+			}
+			session.observeForDelayCalibration(near, (offset / SAMPLE_RATE) * 1000);
+			if (session.aecDelayState().calibrated) break;
+		}
+
+		const state = session.aecDelayState();
+		expect(state.calibrated).toBe(true);
+		expect(Math.abs(state.delaySamples - delaySamples)).toBeLessThanOrEqual(1);
+		expect(state.confidence).toBeGreaterThan(0.95);
+	});
+
+	it("seeds the echo delay from the platform default when ELIZA_VOICE_ECHO_DELAY_MS=auto", () => {
+		const prev = process.env.ELIZA_VOICE_ECHO_DELAY_MS;
+		process.env.ELIZA_VOICE_ECHO_DELAY_MS = "auto";
+		try {
+			const session = new LiveDiarizationSession(fakeRuntime());
+			// Seed comes straight from the per-platform table (#9583); runtime
+			// calibration would refine it later, but at construction it equals the
+			// platform default for the host the test runs on.
+			expect(session.aecDelayState().delaySamples).toBe(
+				platformPlaybackDelaySamples(process.platform, SAMPLE_RATE),
+			);
+		} finally {
+			if (prev === undefined) delete process.env.ELIZA_VOICE_ECHO_DELAY_MS;
+			else process.env.ELIZA_VOICE_ECHO_DELAY_MS = prev;
+		}
+	});
+
+	it("resolves the ELIZA_PLATFORM id (ios) for the auto seed, not the host's darwin seed (#9583)", () => {
+		const prevDelay = process.env.ELIZA_VOICE_ECHO_DELAY_MS;
+		const prevPlatform = process.env.ELIZA_PLATFORM;
+		process.env.ELIZA_VOICE_ECHO_DELAY_MS = "auto";
+		process.env.ELIZA_PLATFORM = "ios";
+		try {
+			const session = new LiveDiarizationSession(fakeRuntime());
+			// The mobile shell reports ELIZA_PLATFORM=ios even though the host's
+			// process.platform is darwin. The auto seed must follow the device id
+			// (#9653 ios table = 400 samples @16kHz), NOT the darwin host seed
+			// (320 samples) — otherwise the deliberate per-platform seeds are
+			// unreachable on device.
+			expect(session.aecDelayState().delaySamples).toBe(
+				platformPlaybackDelaySamples("ios", SAMPLE_RATE),
+			);
+			expect(session.aecDelayState().delaySamples).toBe(400);
+			expect(session.aecDelayState().delaySamples).not.toBe(
+				platformPlaybackDelaySamples("darwin", SAMPLE_RATE),
+			);
+		} finally {
+			if (prevDelay === undefined) delete process.env.ELIZA_VOICE_ECHO_DELAY_MS;
+			else process.env.ELIZA_VOICE_ECHO_DELAY_MS = prevDelay;
+			if (prevPlatform === undefined) delete process.env.ELIZA_PLATFORM;
+			else process.env.ELIZA_PLATFORM = prevPlatform;
+		}
+	});
+
+	it("resolves the ELIZA_PLATFORM id (android) for the auto seed (#9583)", () => {
+		const prevDelay = process.env.ELIZA_VOICE_ECHO_DELAY_MS;
+		const prevPlatform = process.env.ELIZA_PLATFORM;
+		process.env.ELIZA_VOICE_ECHO_DELAY_MS = "auto";
+		process.env.ELIZA_PLATFORM = "android";
+		try {
+			const session = new LiveDiarizationSession(fakeRuntime());
+			expect(session.aecDelayState().delaySamples).toBe(
+				platformPlaybackDelaySamples("android", SAMPLE_RATE),
+			);
+			expect(session.aecDelayState().delaySamples).toBe(720);
+		} finally {
+			if (prevDelay === undefined) delete process.env.ELIZA_VOICE_ECHO_DELAY_MS;
+			else process.env.ELIZA_VOICE_ECHO_DELAY_MS = prevDelay;
+			if (prevPlatform === undefined) delete process.env.ELIZA_PLATFORM;
+			else process.env.ELIZA_PLATFORM = prevPlatform;
+		}
+	});
+
+	it("defaults the echo delay seed to 0 when no override is set", () => {
+		const prev = process.env.ELIZA_VOICE_ECHO_DELAY_MS;
+		delete process.env.ELIZA_VOICE_ECHO_DELAY_MS;
+		try {
+			const session = new LiveDiarizationSession(fakeRuntime());
+			expect(session.aecDelayState().delaySamples).toBe(0);
+		} finally {
+			if (prev === undefined) delete process.env.ELIZA_VOICE_ECHO_DELAY_MS;
+			else process.env.ELIZA_VOICE_ECHO_DELAY_MS = prev;
+		}
+	});
+});