@elizaos/plugin-local-inference 2.0.3-beta.2 → 2.0.3-beta.4

package/src/services/voice/wav-codec.fuzz.test.ts

@@ -0,0 +1,59 @@
+// Fuzz / hardening regression guard for the WAV decoder. decodeMonoPcm16Wav
+// parses untrusted audio bytes (uploads / mic captures), so the invariant under
+// random + mutated input is: it either returns a well-formed {pcm, sampleRate}
+// or throws a plain Error -- never an unexpected throw, and never hangs. A
+// seeded LCG makes failures reproducible.
+
+import { describe, expect, it } from "vitest";
+import { decodeMonoPcm16Wav, encodeMonoPcm16Wav } from "./wav-codec";
+
+function makeRng(seed: number): () => number {
+	let s = seed >>> 0;
+	return () => {
+		s = (Math.imul(s, 1664525) + 1013904223) >>> 0;
+		return s / 0x100000000;
+	};
+}
+
+function assertSafeDecode(bytes: Uint8Array): void {
+	try {
+		const out = decodeMonoPcm16Wav(bytes);
+		expect(out.pcm).toBeInstanceOf(Float32Array);
+		expect(typeof out.sampleRate).toBe("number");
+		expect(Number.isFinite(out.sampleRate)).toBe(true);
+	} catch (err) {
+		expect(err instanceof Error).toBe(true);
+	}
+}
+
+describe("decodeMonoPcm16Wav - fuzz", () => {
+	it("handles random bytes without an unexpected throw", () => {
+		const rng = makeRng(0x7a7);
+		for (let i = 0; i < 2000; i++) {
+			const len = Math.floor(rng() * 80);
+			const bytes = new Uint8Array(len);
+			for (let j = 0; j < len; j++) bytes[j] = Math.floor(rng() * 256);
+			assertSafeDecode(bytes);
+		}
+	});
+
+	it("round-trips a valid blob and survives mutation of it", () => {
+		const rng = makeRng(0x317);
+		for (let i = 0; i < 2000; i++) {
+			const n = Math.floor(rng() * 16);
+			const pcm = new Float32Array(n);
+			for (let k = 0; k < n; k++) pcm[k] = (rng() - 0.5) * 1.8;
+			const blob = encodeMonoPcm16Wav(pcm, 16000);
+
+			// Unmutated decodes to the same length.
+			expect(decodeMonoPcm16Wav(blob).pcm.length).toBe(n);
+
+			const mutated = blob.slice();
+			const flips = 1 + Math.floor(rng() * 6);
+			for (let f = 0; f < flips && mutated.length > 0; f++) {
+				mutated[Math.floor(rng() * mutated.length)] = Math.floor(rng() * 256);
+			}
+			assertSafeDecode(mutated);
+		}
+	});
+});

package/src/services/voice/wav-codec.test.ts

@@ -0,0 +1,32 @@
+import { describe, expect, it } from "vitest";
+import { decodeMonoPcm16Wav, encodeMonoPcm16Wav } from "./wav-codec";
+
+describe("wav-codec", () => {
+	it("round-trips mono PCM16 within quantization error", () => {
+		const pcm = new Float32Array(256);
+		for (let i = 0; i < pcm.length; i++) {
+			pcm[i] = Math.sin((2 * Math.PI * 220 * i) / 16_000) * 0.5;
+		}
+		const wav = encodeMonoPcm16Wav(pcm, 16_000);
+		const decoded = decodeMonoPcm16Wav(wav);
+		expect(decoded.sampleRate).toBe(16_000);
+		expect(decoded.pcm.length).toBe(pcm.length);
+		for (let i = 0; i < pcm.length; i++) {
+			expect(Math.abs(decoded.pcm[i] - pcm[i])).toBeLessThan(1 / 0x7fff + 1e-6);
+		}
+	});
+
+	it("clamps out-of-range samples and emits a 44-byte header", () => {
+		const wav = encodeMonoPcm16Wav(Float32Array.from([2, -2, 0]), 24_000);
+		expect(wav.length).toBe(44 + 3 * 2);
+		const decoded = decodeMonoPcm16Wav(wav);
+		expect(decoded.sampleRate).toBe(24_000);
+		expect(decoded.pcm[0]).toBeCloseTo(1, 3);
+		expect(decoded.pcm[1]).toBeCloseTo(-1, 3);
+	});
+
+	it("rejects non-WAV / truncated input (fail loud)", () => {
+		expect(() => decodeMonoPcm16Wav(new Uint8Array(10))).toThrow(/too short/);
+		expect(() => decodeMonoPcm16Wav(new Uint8Array(64))).toThrow(/WAV/);
+	});
+});

package/src/services/voice/wav-codec.ts

@@ -0,0 +1,101 @@
+/**
+ * Mono PCM16 WAV codec — dependency-light so corpus / fixture / test code can
+ * encode + decode WAV without dragging in the heavy `engine-bridge` module
+ * graph. `engine-bridge` re-exports these for its existing callers.
+ */
+
+import type { TranscriptionAudio } from "./types";
+
+function writeAscii(out: Uint8Array, offset: number, text: string): void {
+	for (let i = 0; i < text.length; i++) {
+		out[offset + i] = text.charCodeAt(i);
+	}
+}
+
+function readAscii(bytes: Uint8Array, offset: number, length: number): string {
+	let out = "";
+	for (let i = 0; i < length; i++) {
+		out += String.fromCharCode(bytes[offset + i]);
+	}
+	return out;
+}
+
+/** Encode mono float PCM as a 16-bit little-endian WAV byte stream. */
+export function encodeMonoPcm16Wav(
+	pcm: Float32Array,
+	sampleRate: number,
+): Uint8Array {
+	const channels = 1;
+	const bytesPerSample = 2;
+	const dataBytes = pcm.length * bytesPerSample;
+	const out = new Uint8Array(44 + dataBytes);
+	const view = new DataView(out.buffer);
+	writeAscii(out, 0, "RIFF");
+	view.setUint32(4, 36 + dataBytes, true);
+	writeAscii(out, 8, "WAVE");
+	writeAscii(out, 12, "fmt ");
+	view.setUint32(16, 16, true);
+	view.setUint16(20, 1, true);
+	view.setUint16(22, channels, true);
+	view.setUint32(24, sampleRate, true);
+	view.setUint32(28, sampleRate * channels * bytesPerSample, true);
+	view.setUint16(32, channels * bytesPerSample, true);
+	view.setUint16(34, bytesPerSample * 8, true);
+	writeAscii(out, 36, "data");
+	view.setUint32(40, dataBytes, true);
+
+	let offset = 44;
+	for (const sample of pcm) {
+		const clamped = Math.max(-1, Math.min(1, sample));
+		const value = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff;
+		view.setInt16(offset, Math.round(value), true);
+		offset += bytesPerSample;
+	}
+	return out;
+}
+
+/** Decode a mono PCM16 WAV byte stream into float PCM + sample rate. */
+export function decodeMonoPcm16Wav(bytes: Uint8Array): TranscriptionAudio {
+	if (bytes.byteLength < 44) {
+		throw new Error("[voice] WAV input is too short to contain a header");
+	}
+	const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+	if (
+		readAscii(bytes, 0, 4) !== "RIFF" ||
+		readAscii(bytes, 8, 4) !== "WAVE" ||
+		readAscii(bytes, 12, 4) !== "fmt "
+	) {
+		throw new Error("[voice] Local transcription expects mono PCM16 WAV bytes");
+	}
+	const audioFormat = view.getUint16(20, true);
+	const channels = view.getUint16(22, true);
+	const sampleRate = view.getUint32(24, true);
+	const bitsPerSample = view.getUint16(34, true);
+	if (audioFormat !== 1 || channels !== 1 || bitsPerSample !== 16) {
+		throw new Error(
+			`[voice] Local transcription expects mono PCM16 WAV (format=1 channels=1 bits=16); got format=${audioFormat} channels=${channels} bits=${bitsPerSample}`,
+		);
+	}
+
+	let pos = 36;
+	while (pos + 8 <= bytes.byteLength) {
+		const chunkId = readAscii(bytes, pos, 4);
+		const chunkBytes = view.getUint32(pos + 4, true);
+		const dataStart = pos + 8;
+		if (chunkId === "data") {
+			if (dataStart + chunkBytes > bytes.byteLength) {
+				throw new Error("[voice] WAV data chunk exceeds input length");
+			}
+			if (chunkBytes % 2 !== 0) {
+				throw new Error("[voice] WAV PCM16 data chunk has odd byte length");
+			}
+			const pcm = new Float32Array(chunkBytes / 2);
+			for (let i = 0; i < pcm.length; i++) {
+				pcm[i] = view.getInt16(dataStart + i * 2, true) / 0x8000;
+			}
+			return { pcm, sampleRate };
+		}
+		pos = dataStart + chunkBytes + (chunkBytes % 2);
+	}
+	throw new Error("[voice] WAV input is missing a data chunk");
+}

package/src/services/voice/workbench-entrypoint.test.ts

@@ -0,0 +1,55 @@
+import { mkdtempSync, readFileSync, rmSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import {
+	buildAndRunVoiceWorkbench,
+	writeVoiceWorkbenchResult,
+} from "./workbench-entrypoint";
+import { groundTruthMockServices } from "./workbench-scenarios";
+
+describe("buildAndRunVoiceWorkbench", () => {
+	it("the mocked lane runs the whole matrix to an overall PASS + Markdown", async () => {
+		const { report, markdown } = await buildAndRunVoiceWorkbench({
+			services: groundTruthMockServices(),
+		});
+		expect(report.overall).toBe("pass");
+		expect(report.scenariosSkipped).toBe(0);
+		expect(report.scenariosRan).toBeGreaterThan(0);
+		expect(markdown).toContain("Voice Workbench");
+		expect(markdown).toContain("WER");
+	});
+
+	it("the real lane with no backend skips every scenario (never pass)", async () => {
+		const { report } = await buildAndRunVoiceWorkbench({ services: null });
+		expect(report.overall).toBe("skipped");
+		expect(report.scenariosRan).toBe(0);
+	});
+});
+
+describe("writeVoiceWorkbenchResult", () => {
+	let dir: string;
+	beforeEach(() => {
+		dir = mkdtempSync(path.join(os.tmpdir(), "voice-workbench-"));
+	});
+	afterEach(() => {
+		rmSync(dir, { recursive: true, force: true });
+	});
+
+	it("writes report.json + report.md", async () => {
+		const result = await buildAndRunVoiceWorkbench({
+			services: groundTruthMockServices(),
+		});
+		const { reportJsonPath, reportMarkdownPath } = writeVoiceWorkbenchResult(
+			result,
+			dir,
+		);
+		const json = JSON.parse(readFileSync(reportJsonPath, "utf8")) as {
+			overall: string;
+		};
+		expect(json.overall).toBe("pass");
+		expect(readFileSync(reportMarkdownPath, "utf8")).toContain(
+			"Voice Workbench",
+		);
+	});
+});

package/src/services/voice/workbench-entrypoint.ts

@@ -0,0 +1,88 @@
+/**
+ * Voice Workbench entrypoint (#8785).
+ *
+ * One call that generates the corpus for a scenario matrix, runs them headless
+ * through the (injected) voice services, and rolls the result into the single
+ * JSON + Markdown benchmark report. The CLI (`voice:workbench`) and the CI lane
+ * are thin shells over this:
+ *   - mocked lane (always): pass `groundTruthMockServices()` → runs + passes,
+ *     exercising corpus → runner → scorers → report end-to-end with no model;
+ *   - real lane (where provisioned): pass a real services adapter; absent
+ *     backend → `services: null` → every scenario `skipped` (never `pass`).
+ *
+ * Pure orchestration over the already-tested pieces, so it is unit-testable
+ * without a model or a browser.
+ */
+
+import { mkdirSync, writeFileSync } from "node:fs";
+import path from "node:path";
+import {
+	type CorpusTtsSynthesizer,
+	generateVoiceCorpus,
+} from "./corpus-generator";
+import type { VoiceScenario } from "./voice-scenario";
+import {
+	buildVoiceWorkbenchReport,
+	formatVoiceWorkbenchMarkdown,
+	type VoiceWorkbenchReport,
+} from "./voice-workbench-report";
+import {
+	runVoiceWorkbenchHeadless,
+	type VoiceWorkbenchServices,
+} from "./workbench-headless-runner";
+import { VOICE_WORKBENCH_SCENARIOS } from "./workbench-scenarios";
+
+export interface BuildAndRunVoiceWorkbenchArgs {
+	/** Scenario matrix to run (defaults to the built-in set). */
+	scenarios?: ReadonlyArray<VoiceScenario>;
+	/** Voice services; null → every scenario is skipped (no backend). */
+	services: VoiceWorkbenchServices | null;
+	/** Corpus sample rate (default 16 kHz). */
+	sampleRate?: number;
+	/** Real TTS for the corpus; omitted → deterministic synthetic speech. */
+	synthesizer?: CorpusTtsSynthesizer;
+}
+
+export interface VoiceWorkbenchResult {
+	report: VoiceWorkbenchReport;
+	markdown: string;
+}
+
+/** Generate corpus → run headless → build the report + Markdown. */
+export async function buildAndRunVoiceWorkbench(
+	args: BuildAndRunVoiceWorkbenchArgs,
+): Promise<VoiceWorkbenchResult> {
+	const scenarios = args.scenarios ?? VOICE_WORKBENCH_SCENARIOS;
+	const entries = [];
+	for (const scenario of scenarios) {
+		const corpus = await generateVoiceCorpus(scenario, {
+			...(args.sampleRate !== undefined ? { sampleRate: args.sampleRate } : {}),
+			...(args.synthesizer ? { synthesizer: args.synthesizer } : {}),
+		});
+		entries.push({ scenario, corpus });
+	}
+	const runs = await runVoiceWorkbenchHeadless({
+		scenarios: entries,
+		services: args.services,
+	});
+	const report = buildVoiceWorkbenchReport(runs);
+	return { report, markdown: formatVoiceWorkbenchMarkdown(report) };
+}
+
+export interface VoiceWorkbenchArtifacts {
+	reportJsonPath: string;
+	reportMarkdownPath: string;
+}
+
+/** Persist the workbench result as `report.json` + `report.md` under `outDir`. */
+export function writeVoiceWorkbenchResult(
+	result: VoiceWorkbenchResult,
+	outDir: string,
+): VoiceWorkbenchArtifacts {
+	mkdirSync(outDir, { recursive: true });
+	const reportJsonPath = path.join(outDir, "report.json");
+	const reportMarkdownPath = path.join(outDir, "report.md");
+	writeFileSync(reportJsonPath, `${JSON.stringify(result.report, null, 2)}\n`);
+	writeFileSync(reportMarkdownPath, result.markdown);
+	return { reportJsonPath, reportMarkdownPath };
+}

package/src/services/voice/workbench-headless-runner.test.ts

@@ -0,0 +1,162 @@
+import { describe, expect, it } from "vitest";
+import { generateVoiceCorpus } from "./corpus-generator";
+import type { VoiceScenario } from "./voice-scenario";
+import { buildVoiceWorkbenchReport } from "./voice-workbench-report";
+import {
+	runVoiceScenarioHeadless,
+	runVoiceWorkbenchHeadless,
+	type VoiceWorkbenchServices,
+} from "./workbench-headless-runner";
+import {
+	groundTruthMockServices,
+	VOICE_WORKBENCH_SCENARIOS,
+} from "./workbench-scenarios";
+
+const SCENARIO: VoiceScenario = {
+	id: "runner-demo",
+	classes: ["multi-speaker", "respond-no-respond", "voice-recognition"],
+	participants: [
+		{ label: "alice", entityId: "entity-alice" },
+		{ label: "bob", entityId: "entity-bob" },
+	],
+	turns: [
+		{ speaker: "alice", text: "Eliza what time is it", expectRespond: true },
+		{ speaker: "bob", text: "hey alice not you", expectRespond: false },
+	],
+	assertions: { maxWer: 0.2, maxDer: 0.2, minRespondAccuracy: 0.9 },
+};
+
+describe("runVoiceScenarioHeadless — honesty contract", () => {
+	it("skips (never passes) when the backend is absent", async () => {
+		const corpus = await generateVoiceCorpus(SCENARIO);
+		const run = await runVoiceScenarioHeadless({
+			scenario: SCENARIO,
+			corpus,
+			services: null,
+		});
+		expect(run.status).toBe("skipped");
+		expect(run.cases).toHaveLength(0);
+		expect(run.skipReason).toMatch(/no voice backend/);
+	});
+
+	it("skips when the corpus is absent", async () => {
+		const run = await runVoiceScenarioHeadless({
+			scenario: SCENARIO,
+			corpus: null,
+			services: groundTruthMockServices(),
+		});
+		expect(run.status).toBe("skipped");
+		expect(run.skipReason).toMatch(/corpus/);
+	});
+});
+
+describe("runVoiceScenarioHeadless — scoring", () => {
+	it("a ground-truth-perfect backend passes every scorer", async () => {
+		const corpus = await generateVoiceCorpus(SCENARIO);
+		const run = await runVoiceScenarioHeadless({
+			scenario: SCENARIO,
+			corpus,
+			services: groundTruthMockServices(),
+		});
+		expect(run.status).toBe("ran");
+		expect(run.cases.every((c) => c.passed)).toBe(true);
+		const kinds = new Set(run.cases.map((c) => c.kind));
+		expect(kinds.has("tts-asr-roundtrip")).toBe(true);
+		expect(kinds.has("eot-decision")).toBe(true);
+		expect(kinds.has("diarization")).toBe(true);
+		expect(kinds.has("respond-decision")).toBe(true);
+		expect(kinds.has("voice-entity-match")).toBe(true);
+	});
+
+	it("a faulty backend fails the scorers it regressed", async () => {
+		const corpus = await generateVoiceCorpus(SCENARIO);
+		// Diarization always says "alice" + the agent always responds.
+		const faulty: VoiceWorkbenchServices = {
+			async observeTurn({ label }) {
+				return {
+					hypothesisTranscript: label.referenceTranscript,
+					predictedSpeakerLabel: "alice", // wrong for bob's turn
+					eotDecided: true,
+					responded: true, // wrong for the bystander turn
+					inferredEntities: [],
+					matchedEntityId: label.entityId ?? null,
+				};
+			},
+		};
+		const run = await runVoiceScenarioHeadless({
+			scenario: SCENARIO,
+			corpus,
+			services: faulty,
+		});
+		expect(run.status).toBe("ran");
+		const failed = new Set(
+			run.cases.filter((c) => !c.passed).map((c) => c.kind),
+		);
+		expect(failed.has("diarization")).toBe(true);
+		expect(failed.has("respond-decision")).toBe(true);
+	});
+
+	it("fails EOT when a mid-utterance pause is treated as a boundary", async () => {
+		const scenario = VOICE_WORKBENCH_SCENARIOS.find(
+			(candidate) => candidate.id === "pauses-midutterance",
+		);
+		if (!scenario) throw new Error("missing pauses-midutterance scenario");
+		const corpus = await generateVoiceCorpus(scenario);
+		const eagerEot: VoiceWorkbenchServices = {
+			async observeTurn({ label }) {
+				return {
+					hypothesisTranscript: label.referenceTranscript,
+					predictedSpeakerLabel: label.speaker,
+					eotDecided: true,
+					responded: label.expectRespond,
+					inferredEntities: [],
+					matchedEntityId: label.entityId ?? null,
+				};
+			},
+		};
+
+		const run = await runVoiceScenarioHeadless({
+			scenario,
+			corpus,
+			services: eagerEot,
+		});
+		const eot = run.cases.find((c) => c.kind === "eot-decision");
+		expect(eot?.passed).toBe(false);
+		expect(eot).toMatchObject({ falseTriggerRate: 0.5 });
+	});
+});
+
+describe("runVoiceWorkbenchHeadless over the built-in scenario matrix", () => {
+	it("the ground-truth mock lane produces an overall PASS report", async () => {
+		const entries = await Promise.all(
+			VOICE_WORKBENCH_SCENARIOS.map(async (scenario) => ({
+				scenario,
+				corpus: await generateVoiceCorpus(scenario),
+			})),
+		);
+		const runs = await runVoiceWorkbenchHeadless({
+			scenarios: entries,
+			services: groundTruthMockServices(),
+		});
+		const report = buildVoiceWorkbenchReport(runs);
+		expect(report.overall).toBe("pass");
+		expect(report.scenariosRan).toBe(VOICE_WORKBENCH_SCENARIOS.length);
+		expect(report.scenariosSkipped).toBe(0);
+	});
+
+	it("an absent backend skips the whole matrix (overall skipped, never pass)", async () => {
+		const entries = await Promise.all(
+			VOICE_WORKBENCH_SCENARIOS.map(async (scenario) => ({
+				scenario,
+				corpus: await generateVoiceCorpus(scenario),
+			})),
+		);
+		const runs = await runVoiceWorkbenchHeadless({
+			scenarios: entries,
+			services: null,
+		});
+		const report = buildVoiceWorkbenchReport(runs);
+		expect(report.overall).toBe("skipped");
+		expect(report.scenariosRan).toBe(0);
+	});
+});