@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/services/transcription-priority.test.ts

@@ -0,0 +1,211 @@
+/**
+ * Tests for the `TRANSCRIPTION` model handler priority + provider hint
+ * semantics on `AgentRuntime`. These exercise the same `registerModel`
+ * machinery used by plugin-elizacloud (remote OpenAI Whisper API) and the
+ * local fused Qwen3-ASR handler so callers know exactly:
+ *
+ *   - which handler wins when priorities tie (first-registered),
+ *   - how an explicit `priority` overrides that,
+ *   - that a `provider` hint to `useModel` overrides priority entirely,
+ *   - that the runtime does NOT auto-fall-through on handler errors,
+ *   - and how a caller can implement an explicit try/catch fallback.
+ *
+ * The test is intentionally provider-agnostic — it registers fake handlers
+ * named "local" and "openai" so the contract is observable without booting
+ * any real ASR backend.
+ */
+
+import { describe, expect, it } from "vitest";
+
+import { InMemoryDatabaseAdapter } from "../../../../packages/core/src/database/inMemoryAdapter";
+import { AgentRuntime } from "../../../../packages/core/src/runtime";
+import { ModelType } from "../../../../packages/core/src/types";
+
+interface TranscriptionParams {
+	audio?: Float32Array | Uint8Array | Buffer | string;
+}
+
+interface TestRuntimeCtx {
+	runtime: AgentRuntime;
+	calls: string[];
+}
+
+function makeRuntime(): TestRuntimeCtx {
+	const runtime = new AgentRuntime({
+		character: {
+			name: "TranscriptionPriorityTest",
+			bio: "asr-priority test",
+			settings: {},
+		} as never,
+		adapter: new InMemoryDatabaseAdapter(),
+		logLevel: "fatal",
+	});
+	const calls: string[] = [];
+	return { runtime, calls };
+}
+
+function makeHandler(
+	calls: string[],
+	label: string,
+	result: string | (() => string),
+	options: { throws?: boolean } = {},
+) {
+	return async (_runtime: unknown, _params: unknown): Promise<string> => {
+		calls.push(label);
+		if (options.throws) {
+			throw new Error(`${label} handler failed`);
+		}
+		return typeof result === "function" ? result() : result;
+	};
+}
+
+describe("TRANSCRIPTION handler priority on AgentRuntime", () => {
+	it("local Qwen3-ASR wins over remote OpenAI Whisper by registration order when priorities tie", async () => {
+		const { runtime, calls } = makeRuntime();
+		runtime.registerModel(
+			ModelType.TRANSCRIPTION,
+			makeHandler(calls, "local", "local transcript"),
+			"eliza-local-inference",
+			0,
+		);
+		runtime.registerModel(
+			ModelType.TRANSCRIPTION,
+			makeHandler(calls, "openai", "openai transcript"),
+			"openai",
+			0,
+		);
+
+		const text = await runtime.useModel(ModelType.TRANSCRIPTION, {
+			audio: new Float32Array(160),
+		} as TranscriptionParams as never);
+
+		expect(text).toBe("local transcript");
+		expect(calls).toEqual(["local"]);
+	});
+
+	it("explicit higher priority always wins regardless of registration order", async () => {
+		const { runtime, calls } = makeRuntime();
+		runtime.registerModel(
+			ModelType.TRANSCRIPTION,
+			makeHandler(calls, "remote", "remote transcript"),
+			"openai",
+			100,
+		);
+		runtime.registerModel(
+			ModelType.TRANSCRIPTION,
+			makeHandler(calls, "local", "local transcript"),
+			"eliza-local-inference",
+			0,
+		);
+
+		// Remote wins because priority 100 > 0.
+		let text = await runtime.useModel(ModelType.TRANSCRIPTION, {
+			audio: new Float32Array(160),
+		} as TranscriptionParams as never);
+		expect(text).toBe("remote transcript");
+		expect(calls).toEqual(["remote"]);
+
+		// Re-register local with a higher priority — now local wins.
+		runtime.registerModel(
+			ModelType.TRANSCRIPTION,
+			makeHandler(calls, "local-priority", "local-priority transcript"),
+			"eliza-local-inference",
+			200,
+		);
+
+		text = await runtime.useModel(ModelType.TRANSCRIPTION, {
+			audio: new Float32Array(160),
+		} as TranscriptionParams as never);
+		expect(text).toBe("local-priority transcript");
+		expect(calls).toEqual(["remote", "local-priority"]);
+	});
+
+	it("provider hint to useModel overrides priority", async () => {
+		const { runtime, calls } = makeRuntime();
+		runtime.registerModel(
+			ModelType.TRANSCRIPTION,
+			makeHandler(calls, "local", "local transcript"),
+			"eliza-local-inference",
+			200,
+		);
+		runtime.registerModel(
+			ModelType.TRANSCRIPTION,
+			makeHandler(calls, "openai", "openai transcript"),
+			"openai",
+			0,
+		);
+
+		// Default useModel: local wins (priority 200 > 0).
+		let text = await runtime.useModel(ModelType.TRANSCRIPTION, {
+			audio: new Float32Array(160),
+		} as TranscriptionParams as never);
+		expect(text).toBe("local transcript");
+
+		// Explicit provider hint flips it to openai despite the priority gap.
+		text = await runtime.useModel(
+			ModelType.TRANSCRIPTION,
+			{ audio: new Float32Array(160) } as TranscriptionParams as never,
+			"openai",
+		);
+		expect(text).toBe("openai transcript");
+		expect(calls).toEqual(["local", "openai"]);
+	});
+
+	it("runtime does NOT auto-fall-through on handler errors", async () => {
+		const { runtime, calls } = makeRuntime();
+		runtime.registerModel(
+			ModelType.TRANSCRIPTION,
+			makeHandler(calls, "local", "should never reach this", { throws: true }),
+			"eliza-local-inference",
+			200,
+		);
+		runtime.registerModel(
+			ModelType.TRANSCRIPTION,
+			makeHandler(calls, "openai", "openai transcript"),
+			"openai",
+			0,
+		);
+
+		await expect(
+			runtime.useModel(ModelType.TRANSCRIPTION, {
+				audio: new Float32Array(160),
+			} as TranscriptionParams as never),
+		).rejects.toThrow(/local handler failed/);
+		// The remote handler should NOT have been invoked — there is no
+		// implicit fallback chain on the runtime.
+		expect(calls).toEqual(["local"]);
+	});
+
+	it("caller can implement fallback via try/catch + provider hint", async () => {
+		const { runtime, calls } = makeRuntime();
+		runtime.registerModel(
+			ModelType.TRANSCRIPTION,
+			makeHandler(calls, "local", "boom", { throws: true }),
+			"eliza-local-inference",
+			200,
+		);
+		runtime.registerModel(
+			ModelType.TRANSCRIPTION,
+			makeHandler(calls, "openai", "openai transcript"),
+			"openai",
+			0,
+		);
+
+		let result: string;
+		try {
+			result = (await runtime.useModel(ModelType.TRANSCRIPTION, {
+				audio: new Float32Array(160),
+			} as TranscriptionParams as never)) as string;
+		} catch {
+			// Caller-controlled fallback: explicit retry with provider hint.
+			result = (await runtime.useModel(
+				ModelType.TRANSCRIPTION,
+				{ audio: new Float32Array(160) } as TranscriptionParams as never,
+				"openai",
+			)) as string;
+		}
+
+		expect(result).toBe("openai transcript");
+		expect(calls).toEqual(["local", "openai"]);
+	});
+});

package/src/services/tts/errors.ts

@@ -0,0 +1,46 @@
+/**
+ * Structured error every TTS backend throws when it can't serve a
+ * request. Mirrors `ImageGenBackendUnavailableError` so the arbiter / WS5
+ * provider handler can surface a single typed failure mode upward.
+ *
+ * `unsupported_request` is the only reason callers retry against a
+ * different backend (e.g. unknown voice id on Kokoro → fall through to
+ * Edge TTS via the runtime model registry priority list). All other
+ * reasons indicate a missing local install and the caller should surface
+ * the actionable message verbatim — no silent fallback to a cloud
+ * provider for installer / weights issues (AGENTS.md §3).
+ */
+export class TtsBackendUnavailableError extends Error {
+	readonly code = "TTS_BACKEND_UNAVAILABLE";
+	constructor(
+		readonly backendId: string,
+		readonly reason:
+			| "binary_missing"
+			| "binary_version_mismatch"
+			| "model_missing"
+			| "tokenizer_missing"
+			| "voice_preset_missing"
+			| "voice_unknown"
+			| "binding_unavailable"
+			| "unsupported_runtime"
+			| "unsupported_request"
+			| "subprocess_failed",
+		message: string,
+		options?: { cause?: unknown },
+	) {
+		super(message, options);
+		this.name = "TtsBackendUnavailableError";
+	}
+}
+
+/** Tells callers whether a thrown error came from a backend availability check. */
+export function isTtsUnavailable(
+	err: unknown,
+): err is TtsBackendUnavailableError {
+	return (
+		err instanceof TtsBackendUnavailableError ||
+		(typeof err === "object" &&
+			err !== null &&
+			(err as { code?: unknown }).code === "TTS_BACKEND_UNAVAILABLE")
+	);
+}

package/src/services/tts/index.ts

@@ -0,0 +1,214 @@
+/**
+ * Local text-to-speech capability (WS5) — public entry point.
+ *
+ * This module is what `provider.ts` (`createTextToSpeechHandler`), the
+ * voice lifecycle service, and TTS skill imports use to register
+ * the capability with the WS1 `MemoryArbiter`.
+ *
+ * Wiring:
+ *
+ *   const arbiter = service.getMemoryArbiter();
+ *   const registration = createTtsCapabilityRegistration({
+ *     loader: createDefaultTtsLoader({ ... }),
+ *     cache: arbiterTtsCache,
+ *   });
+ *   arbiter.registerCapability(registration);
+ *
+ * `createTtsCapabilityRegistration` wraps the underlying backend so the
+ * arbiter's `run(request)` path:
+ *
+ *   1. Hashes the request (`hashTtsRequest`) against the loaded model id.
+ *   2. Checks the capability-level audio cache (`tts-audio` namespace).
+ *      On hit it short-circuits the backend and returns the cached bytes
+ *      with `metadata.cacheHit = true`.
+ *   3. On miss: calls `backend.supports(request)`. If false the arbiter
+ *      throws `TtsBackendUnavailableError` with `unsupported_request` so
+ *      the runtime model registry can fall through to the next provider
+ *      in priority order (typically `plugin-edge-tts` as the cloud
+ *      fallback). No silent fallback inside the capability — the runtime
+ *      owns provider-cascade routing, AGENTS.md §3.
+ *   4. Calls `backend.synthesize(request)`, populates the cache, and
+ *      returns the result.
+ *
+ * The capability registers with the default `"tts"` resident role
+ * (`memory-arbiter.ts#CAPABILITY_ROLE.speak === "tts"`). Co-evicts with
+ * NOTHING in particular — TTS has its own role in the `RESIDENT_ROLE_PRIORITY`
+ * table and is the first non-text bucket the arbiter drops under critical
+ * pressure. That ordering is deliberate: voice output is the cheapest
+ * audible feature to defer; vision/image-gen weights take longer to
+ * re-load.
+ */
+
+export { isTtsUnavailable, TtsBackendUnavailableError } from "./errors";
+export {
+	hashTtsRequest,
+	TtsAudioCache,
+	type TtsAudioCacheConfig,
+	type TtsAudioEntry,
+} from "./tts-audio-cache";
+export type {
+	TtsBackend,
+	TtsBackendLoader,
+	TtsLoadArgs,
+	TtsMimeType,
+	TtsRequest,
+	TtsResult,
+} from "./types";
+
+import type {
+	ArbiterCapability,
+	CapabilityRegistration,
+} from "../memory-arbiter";
+import { TtsBackendUnavailableError } from "./errors";
+import { hashTtsRequest, type TtsAudioEntry } from "./tts-audio-cache";
+import type {
+	TtsBackend,
+	TtsBackendLoader,
+	TtsMimeType,
+	TtsRequest,
+	TtsResult,
+} from "./types";
+
+/**
+ * Minimal cache shape the registration needs. Lets tests inject a fake
+ * cache without instantiating the full `TtsAudioCache`.
+ */
+export interface TtsAudioCacheLike {
+	get(hash: string): TtsAudioEntry | null;
+	set(
+		hash: string,
+		entry: {
+			audio: Uint8Array;
+			mime: TtsMimeType;
+			sampleRate: number;
+			voice: string;
+			model: string;
+		},
+		ttlMs?: number,
+	): void;
+}
+
+export interface CreateTtsCapabilityRegistrationOptions {
+	loader: TtsBackendLoader;
+	/**
+	 * Optional audio cache. When provided the wrapper performs hash →
+	 * cache lookup before calling the backend's `synthesize`. The cache
+	 * is keyed on `(provider, model, voice, text, speed, sampleRate)` via
+	 * {@link hashTtsRequest}; the provider name is fixed to
+	 * `"eliza-local-inference"` so cache entries don't collide with a
+	 * different runtime's TTS cache living in the same process.
+	 */
+	cache?: TtsAudioCacheLike;
+	/**
+	 * Default provider name used in the cache key. Defaults to
+	 * `"eliza-local-inference"`. Tests may override to assert keying.
+	 */
+	provider?: string;
+	/**
+	 * Best-effort RAM footprint estimate for the loaded weights. The
+	 * arbiter only uses this for telemetry; eviction is by priority.
+	 * Defaults: Kokoro ≈ 350 MB int8 / 410 MB fp32; OmniVoice ranges
+	 * 800–1700 MB across the Q3..Q8 ladder. The right call site is
+	 * `loader.estimatedMb` once it's resolved against the active tier.
+	 */
+	estimatedMb?: number;
+	/**
+	 * Optional override for the cache TTL on insert. Defaults to the
+	 * `TtsAudioCache` default (10 min).
+	 */
+	cacheTtlMs?: number;
+}
+
+/**
+ * Build a `CapabilityRegistration` ready to feed to
+ * `arbiter.registerCapability()`. Mirrors
+ * `createVisionCapabilityRegistration` from WS2 and
+ * `createImageGenCapabilityRegistration` from WS3.
+ */
+export function createTtsCapabilityRegistration(
+	opts: CreateTtsCapabilityRegistrationOptions,
+): CapabilityRegistration<TtsBackend, TtsRequest, TtsResult> {
+	const capability: ArbiterCapability = "speak";
+	const loader = opts.loader;
+	const cache = opts.cache;
+	const provider = opts.provider ?? "eliza-local-inference";
+	const cacheTtlMs = opts.cacheTtlMs;
+	return {
+		capability,
+		estimatedMb: opts.estimatedMb ?? 600,
+		async load(modelKey: string): Promise<TtsBackend> {
+			return await loader(modelKey);
+		},
+		async unload(backend: TtsBackend): Promise<void> {
+			await backend.dispose();
+		},
+		async run(backend: TtsBackend, request: TtsRequest): Promise<TtsResult> {
+			// 1. Reject unsupported requests cleanly — no silent fallback.
+			if (!backend.supports(request)) {
+				throw new TtsBackendUnavailableError(
+					backend.id,
+					"unsupported_request",
+					`[tts] backend "${backend.id}" does not support this request (voice=${request.voice ?? "default"} sampleRate=${request.sampleRate ?? "native"})`,
+				);
+			}
+
+			// 2. Cache lookup (if a cache is wired).
+			let cacheKey: string | null = null;
+			if (cache) {
+				cacheKey = hashTtsRequest({
+					provider,
+					model: backend.id,
+					voice: request.voice ?? "default",
+					text: request.text,
+					...(typeof request.speed === "number"
+						? { speed: request.speed }
+						: {}),
+					...(typeof request.sampleRate === "number"
+						? { sampleRate: request.sampleRate }
+						: {}),
+				});
+				const hit = cache.get(cacheKey);
+				if (hit && hit.live !== false) {
+					return {
+						audio: hit.audio,
+						mime: hit.mime,
+						sampleRate: hit.sampleRate,
+						metadata: {
+							model: hit.model,
+							voice: hit.voice,
+							text: request.text,
+							inferenceTimeMs: 0,
+							cacheHit: true,
+						},
+					};
+				}
+			}
+
+			// 3. Miss — synthesize through the backend.
+			const result = await backend.synthesize(request);
+
+			// 4. Populate cache (best-effort).
+			if (cache && cacheKey) {
+				cache.set(
+					cacheKey,
+					{
+						audio: result.audio,
+						mime: result.mime,
+						sampleRate: result.sampleRate,
+						voice: result.metadata.voice,
+						model: result.metadata.model,
+					},
+					cacheTtlMs,
+				);
+			}
+
+			return {
+				...result,
+				metadata: {
+					...result.metadata,
+					cacheHit: false,
+				},
+			};
+		},
+	};
+}

package/src/services/tts/tts-audio-cache.ts

@@ -0,0 +1,235 @@
+/**
+ * Content-hashed cache for synthesized TTS audio. WS5 capability-level
+ * cache that sits between the WS1 `MemoryArbiter` and the concrete TTS
+ * backend (OmniVoice / Kokoro / fake).
+ *
+ * Cache namespace: `tts-audio`.
+ *
+ * Why a capability-level cache (in addition to the existing
+ * `wrap-with-first-line-cache.ts`):
+ *
+ *   - `wrap-with-first-line-cache.ts` is a *first-sentence* cache. It
+ *     accelerates the audible TTFA on the planner's typical "short
+ *     acknowledgement" leading sentence. It's keyed on `(provider,
+ *     voiceId, voiceRevision, codec, voiceSettingsFingerprint,
+ *     normalizedFirstSentence)`.
+ *
+ *   - This cache is a *full-utterance* cache. It accelerates repeated
+ *     synthesis of identical inputs across turns (system prompts, idle
+ *     fillers, the user replaying a previously-spoken sentence). It's
+ *     keyed on `sha256(provider || model || voice || text)`. Hits short-
+ *     circuit `backend.synthesize` entirely; the arbiter returns the
+ *     cached bytes and records `metadata.cacheHit = true`.
+ *
+ * The two layers are independent. A request can hit the first-line cache,
+ * a partially-matching prefix can hit this cache, both can miss — and the
+ * keying spaces don't collide because the first-line cache keys on the
+ * *first sentence's* normalized text, while this one keys on the full
+ * text. Wire both at boot for maximum hit rate.
+ *
+ * Contract:
+ *   - Caller computes a stable hash from a deterministic key (the request
+ *     plus the active model). The hash is the cache key.
+ *   - Caller pairs the hash with the synthesized bytes AND the
+ *     `(mime, sampleRate, voice, model)` quadruple so a reader can
+ *     reproduce a full `TtsResult` on hit.
+ *   - `get(hash)` returns `null` on miss or expiry, the entry on hit. A
+ *     hit also "touches" the entry to keep it warm under LRU.
+ *   - `set(hash, entry, ttlMs?)` inserts with a TTL (default 10 min);
+ *     if the LRU is full, the coldest entry is evicted.
+ */
+
+import { createHash } from "node:crypto";
+import type { TtsMimeType } from "./types";
+
+interface CacheEntry {
+	audio: Uint8Array;
+	mime: TtsMimeType;
+	sampleRate: number;
+	voice: string;
+	model: string;
+	expiresAtMs: number;
+}
+
+export interface TtsAudioEntry {
+	audio: Uint8Array;
+	mime: TtsMimeType;
+	sampleRate: number;
+	voice: string;
+	model: string;
+	/** True when this entry is still within its TTL. */
+	live: boolean;
+}
+
+export interface TtsAudioCacheConfig {
+	/** Max entries retained. LRU evicts beyond this. Default 64. */
+	maxEntries: number;
+	/** Default TTL when `set()` is called without one. Default 10 min. */
+	defaultTtlMs: number;
+	/** Optional max total bytes; an LRU sweep enforces this on insert. Default 64 MB. */
+	maxBytes: number;
+}
+
+const DEFAULTS: TtsAudioCacheConfig = {
+	maxEntries: 64,
+	defaultTtlMs: 10 * 60_000,
+	maxBytes: 64 * 1024 * 1024,
+};
+
+/**
+ * Build the canonical cache key for a TTS request + active model. The
+ * fields are joined with a separator that can't appear in any of them
+ * (a NUL byte) and SHA-256'd. The model id participates so a tier swap
+ * doesn't yield phantom hits from a different voice family.
+ */
+export function hashTtsRequest(args: {
+	provider: string;
+	model: string;
+	voice: string;
+	text: string;
+	speed?: number;
+	sampleRate?: number;
+}): string {
+	const h = createHash("sha256");
+	h.update(args.provider);
+	h.update("\0");
+	h.update(args.model);
+	h.update("\0");
+	h.update(args.voice);
+	h.update("\0");
+	h.update(args.text);
+	h.update("\0");
+	h.update(typeof args.speed === "number" ? String(args.speed) : "");
+	h.update("\0");
+	h.update(typeof args.sampleRate === "number" ? String(args.sampleRate) : "");
+	return h.digest("hex");
+}
+
+export class TtsAudioCache {
+	private readonly config: TtsAudioCacheConfig;
+	/**
+	 * `Map` preserves insertion order; we re-insert on hit to bubble
+	 * entries to the back, so the first key in iteration order is the
+	 * LRU candidate.
+	 */
+	private readonly entries = new Map<string, CacheEntry>();
+	private readonly now: () => number;
+	private totalBytes = 0;
+
+	constructor(
+		opts: {
+			config?: Partial<TtsAudioCacheConfig>;
+			now?: () => number;
+		} = {},
+	) {
+		this.config = {
+			maxEntries: Math.max(1, opts.config?.maxEntries ?? DEFAULTS.maxEntries),
+			defaultTtlMs: Math.max(
+				0,
+				opts.config?.defaultTtlMs ?? DEFAULTS.defaultTtlMs,
+			),
+			maxBytes: Math.max(1024, opts.config?.maxBytes ?? DEFAULTS.maxBytes),
+		};
+		this.now = opts.now ?? (() => Date.now());
+	}
+
+	get(hash: string): TtsAudioEntry | null {
+		const found = this.entries.get(hash);
+		if (!found) return null;
+		const live = found.expiresAtMs > this.now();
+		if (!live) {
+			this.entries.delete(hash);
+			this.totalBytes -= found.audio.byteLength;
+			return null;
+		}
+		// Bubble to MRU position.
+		this.entries.delete(hash);
+		this.entries.set(hash, found);
+		return {
+			audio: found.audio,
+			mime: found.mime,
+			sampleRate: found.sampleRate,
+			voice: found.voice,
+			model: found.model,
+			live: true,
+		};
+	}
+
+	set(
+		hash: string,
+		entry: {
+			audio: Uint8Array;
+			mime: TtsMimeType;
+			sampleRate: number;
+			voice: string;
+			model: string;
+		},
+		ttlMs?: number,
+	): void {
+		const ttl = typeof ttlMs === "number" ? ttlMs : this.config.defaultTtlMs;
+		if (ttl <= 0) return;
+		const existing = this.entries.get(hash);
+		if (existing) this.totalBytes -= existing.audio.byteLength;
+		const record: CacheEntry = {
+			audio: entry.audio,
+			mime: entry.mime,
+			sampleRate: entry.sampleRate,
+			voice: entry.voice,
+			model: entry.model,
+			expiresAtMs: this.now() + ttl,
+		};
+		this.entries.set(hash, record);
+		this.totalBytes += entry.audio.byteLength;
+		this.enforceBudget();
+	}
+
+	has(hash: string): boolean {
+		const found = this.entries.get(hash);
+		if (!found) return false;
+		if (found.expiresAtMs <= this.now()) {
+			this.entries.delete(hash);
+			this.totalBytes -= found.audio.byteLength;
+			return false;
+		}
+		return true;
+	}
+
+	purgeExpired(nowMs: number = this.now()): number {
+		let removed = 0;
+		for (const [key, value] of this.entries) {
+			if (value.expiresAtMs <= nowMs) {
+				this.entries.delete(key);
+				this.totalBytes -= value.audio.byteLength;
+				removed += 1;
+			}
+		}
+		return removed;
+	}
+
+	clear(): void {
+		this.entries.clear();
+		this.totalBytes = 0;
+	}
+
+	size(): number {
+		return this.entries.size;
+	}
+
+	bytes(): number {
+		return this.totalBytes;
+	}
+
+	private enforceBudget(): void {
+		while (
+			this.entries.size > this.config.maxEntries ||
+			this.totalBytes > this.config.maxBytes
+		) {
+			const oldest = this.entries.keys().next();
+			if (oldest.done) break;
+			const key = oldest.value;
+			const removed = this.entries.get(key);
+			this.entries.delete(key);
+			if (removed) this.totalBytes -= removed.audio.byteLength;
+		}
+	}
+}