@elizaos/plugin-local-inference 2.0.3-beta.2 → 2.0.3-beta.3

package/src/services/voice/voice-hardening.fuzz.test.ts

@@ -0,0 +1,116 @@
+// Fuzz / hardening pass for the pure voice decision + validation helpers.
+// validateVoiceScenario gates an arbitrary (possibly malformed) scenario object
+// before the corpus build, and turnSignalFromProbability maps an arbitrary
+// model probability (including NaN / +-Infinity) onto the speak/stay decision.
+// Invariants under any input: never throw, and always return a value honoring
+// the declared contract. A seeded LCG makes failures reproducible.
+
+import { describe, expect, it } from "vitest";
+import { clampProbability, turnSignalFromProbability } from "./eot-classifier";
+import { type VoiceScenario, validateVoiceScenario } from "./voice-scenario";
+
+function makeRng(seed: number): () => number {
+	let s = seed >>> 0;
+	return () => {
+		s = (Math.imul(s, 1664525) + 1013904223) >>> 0;
+		return s / 0x100000000;
+	};
+}
+
+const KEYS = [
+	"id",
+	"classes",
+	"participants",
+	"turns",
+	"agents",
+	"environment",
+	"label",
+	"speaker",
+	"text",
+	"audioRef",
+	"expectRespond",
+	"expectEndOfTurn",
+	"reverb",
+	"farFieldDb",
+];
+const PRIMS = ["1", '"s"', "true", "false", "null", "-3.5", '""', '"alice"'];
+
+function randomJson(rng: () => number, depth: number): string {
+	if (depth <= 0 || rng() < 0.4) return PRIMS[Math.floor(rng() * PRIMS.length)];
+	const k = 1 + Math.floor(rng() * 4);
+	if (rng() < 0.5) {
+		const items: string[] = [];
+		for (let i = 0; i < k; i++) items.push(randomJson(rng, depth - 1));
+		return `[${items.join(",")}]`;
+	}
+	const entries: string[] = [];
+	for (let i = 0; i < k; i++) {
+		const key = KEYS[Math.floor(rng() * KEYS.length)];
+		entries.push(`${JSON.stringify(key)}:${randomJson(rng, depth - 1)}`);
+	}
+	return `{${entries.join(",")}}`;
+}
+
+describe("validateVoiceScenario - fuzz", () => {
+	it("never throws and always returns a consistent {valid, errors} report", () => {
+		const rng = makeRng(0x5ce4a);
+		for (let i = 0; i < 3000; i++) {
+			const value = JSON.parse(randomJson(rng, 4)) as unknown;
+			const result = validateVoiceScenario(value as VoiceScenario);
+			expect(typeof result.valid).toBe("boolean");
+			expect(Array.isArray(result.errors)).toBe(true);
+			for (const e of result.errors) expect(typeof e).toBe("string");
+			// valid is exactly the no-errors case.
+			expect(result.valid).toBe(result.errors.length === 0);
+		}
+	});
+});
+
+describe("turnSignalFromProbability / clampProbability - fuzz", () => {
+	const EXTREMES = [
+		Number.NaN,
+		Number.POSITIVE_INFINITY,
+		Number.NEGATIVE_INFINITY,
+		Number.MAX_VALUE,
+		-Number.MAX_VALUE,
+		0,
+		1,
+		-1,
+		2,
+		1e-12,
+	];
+
+	it("clampProbability always yields a finite value in [0,1]", () => {
+		const rng = makeRng(0x110b);
+		for (let i = 0; i < 3000; i++) {
+			const p =
+				rng() < 0.3
+					? EXTREMES[Math.floor(rng() * EXTREMES.length)]
+					: (rng() - 0.5) * 1e6;
+			const c = clampProbability(p);
+			expect(Number.isFinite(c)).toBe(true);
+			expect(c).toBeGreaterThanOrEqual(0);
+			expect(c).toBeLessThanOrEqual(1);
+		}
+	});
+
+	it("turnSignalFromProbability always returns a well-formed signal", () => {
+		const rng = makeRng(0xd00d);
+		const speakers = new Set(["agent", "user", "unknown"]);
+		for (let i = 0; i < 3000; i++) {
+			const probability =
+				rng() < 0.3
+					? EXTREMES[Math.floor(rng() * EXTREMES.length)]
+					: (rng() - 0.5) * 1e6;
+			const sig = turnSignalFromProbability({
+				probability,
+				transcript: "x",
+				source: "heuristic",
+			});
+			expect(speakers.has(sig.nextSpeaker)).toBe(true);
+			expect(sig.endOfTurnProbability).toBeGreaterThanOrEqual(0);
+			expect(sig.endOfTurnProbability).toBeLessThanOrEqual(1);
+			expect([true, false, null]).toContain(sig.agentShouldSpeak);
+		}
+	});
+});

package/src/services/voice/voice-preload-predictor.test.ts

@@ -0,0 +1,130 @@
+/**
+ * Voice next-stage preload predictor — real-MemoryArbiter test (#8809 C5).
+ *
+ * Mirrors `packages/benchmarks/memperf/co-residency.test.ts`: drives the REAL
+ * `MemoryArbiter` (no models, no FFI) with a synthetic, SIZED "text" capability
+ * and a real `capacitorPressureSource`, then asserts the predictor's
+ * `onAsrStageComplete()` warms the next-stage text model under nominal pressure
+ * (resident before its first request) and is REFUSED under critical pressure
+ * (not resident). This exercises the genuine `preload` guard — not a stub.
+ */
+
+import { describe, expect, it } from "vitest";
+
+import { MemoryArbiter } from "../memory-arbiter";
+import { capacitorPressureSource } from "../memory-pressure";
+import { SharedResourceRegistry } from "./shared-resources";
+import { VoicePreloadPredictor } from "./voice-preload-predictor";
+
+const TEXT_MODEL_KEY = "eliza-1-4b";
+const TEXT_MODEL_MB = 1200;
+
+function makeHarness(budgetMb: number) {
+	const pressure = capacitorPressureSource();
+	const arbiter = new MemoryArbiter({
+		registry: new SharedResourceRegistry(),
+		pressureSource: pressure,
+		budgetMb: () => budgetMb,
+	});
+	// Synthetic SIZED text loader — the next-stage model the arbiter owns on the
+	// mobile path. No FFI; load() just yields a marker object.
+	arbiter.registerCapability({
+		capability: "text",
+		estimatedMb: TEXT_MODEL_MB,
+		load: async () => ({ cap: "text" as const }),
+		unload: async () => {},
+		run: async () => ({}),
+	});
+	arbiter.start();
+	const predictor = new VoicePreloadPredictor({
+		arbiter,
+		resolveTextModelKey: () => TEXT_MODEL_KEY,
+	});
+	return { arbiter, pressure, predictor };
+}
+
+function textResident(arbiter: MemoryArbiter): boolean {
+	return arbiter
+		.residentSnapshot()
+		.some((e) => e.capability === "text" && e.modelKey === TEXT_MODEL_KEY);
+}
+
+describe("VoicePreloadPredictor — real arbiter next-stage preload", () => {
+	it("warms the next-stage text model under nominal pressure + sufficient budget", async () => {
+		const { arbiter, predictor } = makeHarness(8000);
+		try {
+			// Before the ASR stage finishes the text model is not resident.
+			expect(textResident(arbiter)).toBe(false);
+
+			const warmed = await predictor.onAsrStageComplete();
+
+			// The predictor warmed the text model: it is RESIDENT before its first
+			// request, so the post-ASR verifier pays no cold load.
+			expect(warmed).toBe(TEXT_MODEL_KEY);
+			expect(textResident(arbiter)).toBe(true);
+		} finally {
+			await arbiter.shutdown();
+		}
+	});
+
+	it("refuses the preload under critical pressure (next-stage model not resident)", async () => {
+		const { arbiter, pressure, predictor } = makeHarness(8000);
+		try {
+			// The OS flags critical memory pressure while ASR is running.
+			pressure.dispatch("critical", 32);
+			await new Promise((r) => setTimeout(r, 10));
+			expect(arbiter.currentPressureLevel()).toBe("critical");
+
+			const warmed = await predictor.onAsrStageComplete();
+
+			// preload is REFUSED: returns null (not the key), and the text model is
+			// NOT resident — the predictor never forces a load onto a system the OS
+			// has already flagged as in trouble.
+			expect(warmed).toBeNull();
+			expect(textResident(arbiter)).toBe(false);
+		} finally {
+			await arbiter.shutdown();
+		}
+	});
+
+	it("refuses the preload when the next-stage footprint exceeds the budget", async () => {
+		// Budget below the text model's footprint → preload declines (no eviction
+		// theatre, no forced load) even under nominal pressure.
+		const { arbiter, predictor } = makeHarness(TEXT_MODEL_MB - 1);
+		try {
+			const warmed = await predictor.onAsrStageComplete();
+			expect(warmed).toBeNull();
+			expect(textResident(arbiter)).toBe(false);
+		} finally {
+			await arbiter.shutdown();
+		}
+	});
+
+	it("declines to predict when no text model is assigned", async () => {
+		const pressure = capacitorPressureSource();
+		const noModelArbiter = new MemoryArbiter({
+			registry: new SharedResourceRegistry(),
+			pressureSource: pressure,
+			budgetMb: () => 8000,
+		});
+		noModelArbiter.registerCapability({
+			capability: "text",
+			estimatedMb: TEXT_MODEL_MB,
+			load: async () => ({ cap: "text" as const }),
+			unload: async () => {},
+			run: async () => ({}),
+		});
+		noModelArbiter.start();
+		const predictor = new VoicePreloadPredictor({
+			arbiter: noModelArbiter,
+			resolveTextModelKey: () => null,
+		});
+		try {
+			const warmed = await predictor.onAsrStageComplete();
+			expect(warmed).toBeNull();
+			expect(textResident(noModelArbiter)).toBe(false);
+		} finally {
+			await noModelArbiter.shutdown();
+		}
+	});
+});

package/src/services/voice/voice-preload-predictor.ts

@@ -0,0 +1,113 @@
+/**
+ * Voice next-stage preload predictor (#8809 C5).
+ *
+ * A voice turn runs its stages sequentially on a memory-constrained device:
+ * ASR → text-response → TTS (AGENTS.md §4). The text-response model is the one
+ * the {@link MemoryArbiter} owns on the mobile / Capacitor path (where text
+ * generation routes through `arbiter` rather than the desktop direct-engine
+ * dispatcher). That model is NOT resident while ASR is running — so the first
+ * text request of the turn pays the full cold load on the critical path,
+ * stalling time-to-first-token right when the user has stopped speaking.
+ *
+ * This predictor closes that gap with a single, deterministic prediction: the
+ * instant ASR finishes, the next arbiter-managed stage is the `"text"` model,
+ * so warm it now. It calls {@link ArbiterPreloader.preload} with the resolved
+ * text model key. `preload` is intentionally conservative — it only loads under
+ * nominal pressure when the configured budget proves the footprint fits, and
+ * returns `false` otherwise (low / critical pressure, or no headroom). The
+ * predictor never forces a load; it never touches the engine directly; it owns
+ * no model handle. It is the prediction, nothing more.
+ *
+ * Injectable by construction: it depends only on the `preload` surface of the
+ * arbiter and the resolved text model key, so it is unit-testable against the
+ * real arbiter with a synthetic capability and carries no transitive coupling
+ * to the FFI engine, the voice pipeline, or the device bridge.
+ */
+
+import type { ArbiterCapability } from "../memory-arbiter";
+
+/**
+ * The slice of {@link MemoryArbiter} the predictor needs. Narrowed to `preload`
+ * so the predictor can be injected with the real arbiter (which satisfies this
+ * structurally) or a test double, with no other surface area in scope.
+ */
+export interface ArbiterPreloader {
+	preload(capability: ArbiterCapability, modelKey: string): Promise<boolean>;
+}
+
+export interface VoicePreloadPredictorOptions {
+	/** The arbiter (or anything exposing its `preload`) to warm the model on. */
+	arbiter: ArbiterPreloader;
+	/**
+	 * Resolves the arbiter-managed text-response model key for the current turn.
+	 * A function (not a bare string) so the predictor reflects a model swap
+	 * between turns without being rebuilt. Returning `null` means "no text model
+	 * is assigned yet" — the predictor then declines to predict rather than
+	 * guessing a key.
+	 */
+	resolveTextModelKey: () => string | null;
+	/** Optional structured logger. Messages are prefixed `[voice-preload-predictor]`. */
+	logger?: {
+		debug?: (message: string) => void;
+		warn?: (message: string) => void;
+	};
+}
+
+/**
+ * The capability the next voice stage uses. On the arbiter-routed mobile path
+ * the response after ASR is plain text generation.
+ */
+const NEXT_STAGE_CAPABILITY: ArbiterCapability = "text";
+
+/**
+ * Predicts the next arbiter-managed model for a voice turn and warms it.
+ *
+ * Wire {@link VoicePreloadPredictor.onAsrStageComplete} to the voice pipeline's
+ * `onAsrComplete` event (the instant ASR emits its final token, which is also
+ * the drafter/verifier kick-off point): that is the genuine seam where the next
+ * stage is known and the arbiter owns its model.
+ */
+export class VoicePreloadPredictor {
+	private readonly arbiter: ArbiterPreloader;
+	private readonly resolveTextModelKey: () => string | null;
+	private readonly logger: VoicePreloadPredictorOptions["logger"];
+
+	constructor(options: VoicePreloadPredictorOptions) {
+		this.arbiter = options.arbiter;
+		this.resolveTextModelKey = options.resolveTextModelKey;
+		this.logger = options.logger;
+	}
+
+	/**
+	 * Called when the ASR stage of the current turn completes. Predicts the
+	 * next-stage text model and asks the arbiter to warm it.
+	 *
+	 * Resolves to the model key that was warmed (resident or freshly loaded), or
+	 * `null` when no prediction was made or the arbiter declined the preload
+	 * (pressure / no budget headroom). Never throws on a declined preload — a
+	 * predictor that breaks the voice loop is worse than one that does nothing.
+	 */
+	async onAsrStageComplete(): Promise<string | null> {
+		const textModelKey = this.resolveTextModelKey();
+		if (!textModelKey) {
+			this.logger?.debug?.(
+				"[voice-preload-predictor] no text model assigned; skipping next-stage preload",
+			);
+			return null;
+		}
+		const warmed = await this.arbiter.preload(
+			NEXT_STAGE_CAPABILITY,
+			textModelKey,
+		);
+		if (!warmed) {
+			this.logger?.debug?.(
+				`[voice-preload-predictor] arbiter declined preload of text/${textModelKey} (pressure or no budget headroom)`,
+			);
+			return null;
+		}
+		this.logger?.debug?.(
+			`[voice-preload-predictor] warmed next-stage text/${textModelKey} during ASR stage`,
+		);
+		return textModelKey;
+	}
+}

package/src/services/voice/voice-preset-format.fuzz.test.ts

@@ -0,0 +1,89 @@
+// Mutation + random-byte fuzz for the voice-preset binary parser.
+// readVoicePresetFile is documented as "the single defensive boundary for the
+// format", so the hardening invariant is: on ANY bytes it either returns a
+// well-formed VoicePresetFile or throws a VoicePresetFormatError -- never an
+// unexpected RangeError/TypeError from an unchecked DataView read. A seeded LCG
+// makes failures reproducible.
+
+import { describe, expect, it } from "vitest";
+import {
+	readVoicePresetFile,
+	VoicePresetFormatError,
+	writeVoicePresetFileV2,
+} from "./voice-preset-format";
+
+function makeRng(seed: number): () => number {
+	let s = seed >>> 0;
+	return () => {
+		s = (Math.imul(s, 1664525) + 1013904223) >>> 0;
+		return s / 0x100000000;
+	};
+}
+
+function randomFloat32(rng: () => number, n: number): Float32Array {
+	const out = new Float32Array(n);
+	for (let i = 0; i < n; i++) out[i] = (rng() - 0.5) * 4;
+	return out;
+}
+
+function validBlob(rng: () => number): Uint8Array {
+	const phrases = Array.from({ length: Math.floor(rng() * 3) }, (_, i) => ({
+		text: `phrase ${i}`,
+		sampleRate: 24000,
+		pcm: randomFloat32(rng, Math.floor(rng() * 8)),
+	}));
+	return writeVoicePresetFileV2({
+		embedding: randomFloat32(rng, Math.floor(rng() * 10)),
+		phrases,
+		refText: rng() < 0.5 ? "reference" : "",
+		instruct: rng() < 0.5 ? "speak" : "",
+		metadata: rng() < 0.5 ? { voice: "af" } : {},
+	});
+}
+
+/** Either a valid parse or the format's own error -- never anything else. */
+function assertSafeParse(bytes: Uint8Array): void {
+	let result: ReturnType<typeof readVoicePresetFile> | undefined;
+	try {
+		result = readVoicePresetFile(bytes);
+	} catch (err) {
+		expect(
+			err instanceof VoicePresetFormatError,
+			`expected VoicePresetFormatError, got ${(err as Error)?.name}: ${(err as Error)?.message}`,
+		).toBe(true);
+		return;
+	}
+	// A non-throwing parse must still be a well-formed record.
+	expect(typeof result.version).toBe("number");
+	expect(result.embedding).toBeInstanceOf(Float32Array);
+	expect(Array.isArray(result.phrases)).toBe(true);
+}
+
+describe("readVoicePresetFile - fuzz", () => {
+	it("never throws anything but VoicePresetFormatError on random bytes", () => {
+		const rng = makeRng(0xb10b);
+		for (let i = 0; i < 1500; i++) {
+			const len = Math.floor(rng() * 96);
+			const bytes = new Uint8Array(len);
+			for (let j = 0; j < len; j++) bytes[j] = Math.floor(rng() * 256);
+			assertSafeParse(bytes);
+		}
+	});
+
+	it("never throws anything but VoicePresetFormatError on mutated valid blobs", () => {
+		const rng = makeRng(0x7a11);
+		for (let i = 0; i < 1500; i++) {
+			const blob = validBlob(rng);
+			// Round-trip sanity: an unmutated blob parses cleanly.
+			expect(() => readVoicePresetFile(blob)).not.toThrow();
+			// Now flip a handful of random bytes and re-parse.
+			const mutated = blob.slice();
+			const flips = 1 + Math.floor(rng() * 6);
+			for (let f = 0; f < flips && mutated.length > 0; f++) {
+				const idx = Math.floor(rng() * mutated.length);
+				mutated[idx] = Math.floor(rng() * 256);
+			}
+			assertSafeParse(mutated);
+		}
+	});
+});

package/src/services/voice/voice-preset-format.test.ts

@@ -0,0 +1,75 @@
+// Round-trip coverage for the voice-preset binary (de)serializer. The format is
+// the on-disk shape for enrolled TTS voice presets, and readVoicePresetFile is
+// "the single defensive boundary for the format" — so we pin that a written
+// preset reads back byte-for-byte (v1 + v2) and that malformed input throws.
+
+import { describe, expect, it } from "vitest";
+import {
+	readVoicePresetFile,
+	VOICE_PRESET_VERSION_V1,
+	VOICE_PRESET_VERSION_V2,
+	VoicePresetFormatError,
+	writeVoicePresetFile,
+	writeVoicePresetFileV2,
+} from "./voice-preset-format";
+
+// Values chosen to be exactly representable in float32 so the round-trip is
+// bit-exact (no rounding to reason about).
+const embedding = new Float32Array([0.5, -0.25, 0.75, -1]);
+const phrases = [
+	{
+		text: "hello there",
+		sampleRate: 24000,
+		pcm: new Float32Array([0.5, -0.5]),
+	},
+	{ text: "", sampleRate: 16000, pcm: new Float32Array([0.25]) },
+];
+
+describe("voice-preset-format round-trip", () => {
+	it("writes and reads back a v1 preset (embedding + phrases)", () => {
+		const parsed = readVoicePresetFile(
+			writeVoicePresetFile({ embedding, phrases }),
+		);
+		expect(parsed.version).toBe(VOICE_PRESET_VERSION_V1);
+		expect(Array.from(parsed.embedding)).toEqual(Array.from(embedding));
+		expect(parsed.phrases).toHaveLength(2);
+		expect(parsed.phrases[0].text).toBe("hello there");
+		expect(parsed.phrases[0].sampleRate).toBe(24000);
+		expect(Array.from(parsed.phrases[0].pcm)).toEqual([0.5, -0.5]);
+		expect(parsed.phrases[1].text).toBe("");
+	});
+
+	it("writes and reads back a v2 preset (refText/instruct/metadata/tokens)", () => {
+		const parsed = readVoicePresetFile(
+			writeVoicePresetFileV2({
+				embedding,
+				phrases,
+				refText: "reference line",
+				instruct: "speak warmly",
+				metadata: { voiceId: "af_bella", lang: "en" },
+				refAudioTokens: { K: 2, refT: 2, tokens: new Int32Array([1, 2, 3, 4]) },
+			}),
+		);
+		expect(parsed.version).toBe(VOICE_PRESET_VERSION_V2);
+		expect(Array.from(parsed.embedding)).toEqual(Array.from(embedding));
+		expect(parsed.refText).toBe("reference line");
+		expect(parsed.instruct).toBe("speak warmly");
+		expect(parsed.metadata).toEqual({ voiceId: "af_bella", lang: "en" });
+		expect(parsed.refAudioTokens.K).toBe(2);
+		expect(Array.from(parsed.refAudioTokens.tokens)).toEqual([1, 2, 3, 4]);
+	});
+
+	it("rejects a ref_audio_tokens shape mismatch", () => {
+		expect(() =>
+			writeVoicePresetFileV2({
+				refAudioTokens: { K: 2, refT: 2, tokens: new Int32Array([1, 2, 3]) },
+			}),
+		).toThrow(VoicePresetFormatError);
+	});
+
+	it("throws on a malformed blob (the single defensive boundary)", () => {
+		expect(() => readVoicePresetFile(new Uint8Array([1, 2, 3, 4]))).toThrow(
+			VoicePresetFormatError,
+		);
+	});
+});

package/src/services/voice/voice-preset-format.ts

@@ -133,7 +133,8 @@ export class VoicePresetFormatError extends Error {
 			| "bad-phrase-record"
 			| "bad-embedding-length"
 			| "bad-ref-tokens"
-			| "bad-metadata",
+			| "bad-metadata"
+			| "bad-utf8",
 	) {
 		super(message);
 		this.name = "VoicePresetFormatError";
@@ -349,10 +350,23 @@ function readRefAudioTokens(
 	return { K, refT, tokens };
 }
 
+/** Strict UTF-8 decode that surfaces invalid bytes as the format's own error
+ * rather than leaking a raw TextDecoder TypeError past the defensive boundary. */
+function decodeUtf8Strict(bytes: Uint8Array): string {
+	try {
+		return new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+	} catch {
+		throw new VoicePresetFormatError(
+			"voice preset contains invalid UTF-8 text",
+			"bad-utf8",
+		);
+	}
+}
+
 function readUtf8(bytes: Uint8Array, sec: SectionView): string {
 	if (sec.length === 0) return "";
 	const slice = bytes.subarray(sec.offset, sec.offset + sec.length);
-	return new TextDecoder("utf-8", { fatal: true }).decode(slice);
+	return decodeUtf8Strict(slice);
 }
 
 function readMetadata(
@@ -389,7 +403,6 @@ function readPhrases(
 		bytes.byteOffset + sec.offset,
 		sec.length,
 	);
-	const decoder = new TextDecoder("utf-8", { fatal: true });
 	let pos = 0;
 	if (sec.length < 4) {
 		throw new VoicePresetFormatError(
@@ -420,7 +433,7 @@ function readPhrases(
 			bytes.byteOffset + sec.offset + pos,
 			textLen,
 		);
-		const text = decoder.decode(textBytes);
+		const text = decodeUtf8Strict(textBytes);
 		pos += textLen;
 		if (pos + 8 > sec.length) {
 			throw new VoicePresetFormatError(