@elizaos/plugin-local-inference 2.0.3-beta.2 → 2.0.3-beta.3

package/src/services/voice/workbench-headless-runner.ts

@@ -0,0 +1,396 @@
+/**
+ * Voice Workbench headless runner (#8785).
+ *
+ * Drives a {@link VoiceScenario} + its generated corpus through the real voice
+ * services WITHOUT a browser, scores every turn with the shared scorers
+ * (`e2e-harness.ts`), and emits one {@link VoiceWorkbenchScenarioRun} the report
+ * layer (`voice-workbench-report.ts`) aggregates. The services are injected
+ * through {@link VoiceWorkbenchServices} so:
+ *   - a provisioned local backend wires the real ASR / diarization / EOT /
+ *     respond / entity / TTS path,
+ *   - a mock returns ground-truth-derived observations for the CI plumbing lane,
+ *   - and an ABSENT backend (`services === null`) or absent corpus
+ *     (`corpus === null`) yields a `skipped` run — never a `pass` (honesty
+ *     contract).
+ *
+ * Pure orchestration: it slices the corpus per turn, asks the services to
+ * observe the turn, and maps observations onto scorer inputs. No model loading
+ * here, so it is unit-testable with a fake services adapter.
+ */
+
+import { mkdirSync, writeFileSync } from "node:fs";
+import path from "node:path";
+import type {
+	CorpusGroundTruth,
+	CorpusTurnLabel,
+	GeneratedVoiceCorpus,
+} from "./corpus-generator";
+import {
+	type DiarizationTurnSample,
+	scoreDiarizationTimeline,
+	scoreEchoRejection,
+	scoreEntityExtraction,
+	scoreEotDecision,
+	scoreFirstResponseLatency,
+	scoreOwnerSecurity,
+	scoreRespondDecision,
+	scoreTtsAsrRoundTrip,
+	scoreVoiceEntityMatch,
+	type VoiceE2eCaseResult,
+} from "./e2e-harness";
+import type { VoiceScenario } from "./voice-scenario";
+import type {
+	VoiceAudioArtifact,
+	VoiceWorkbenchScenarioRun,
+} from "./voice-workbench-report";
+import { encodeMonoPcm16Wav } from "./wav-codec";
+
+/** What the real (or mock) services observed for one turn of audio. */
+export interface VoiceTurnObservation {
+	/** ASR hypothesis transcript for the turn's audio. */
+	hypothesisTranscript: string;
+	/** Diarized speaker label, or null when the diarizer missed the turn. */
+	predictedSpeakerLabel: string | null;
+	/** The EOT classifier decided end-of-turn at this turn's boundary. */
+	eotDecided: boolean;
+	/** EOT decision latency (ms) from the true boundary, when measured. */
+	eotLatencyMs?: number;
+	/** The agent decided to respond to this turn. */
+	responded: boolean;
+	/** Entities inferred from this turn's transcript (name/partner extraction). */
+	inferredEntities: string[];
+	/** Voice→entity match: the entity the recognized voice resolved to (or null). */
+	matchedEntityId: string | null;
+	/** First-audio latency (ms) of the agent's spoken reply, when it replied. */
+	firstAudioMs?: number;
+	/** The system judged this turn to be the device owner (owner-security). */
+	predictedOwner?: boolean;
+}
+
+export interface VoiceWorkbenchServices {
+	/**
+	 * Optional one-shot hook before a scenario's turns are scored. Real services
+	 * use this to enroll speaker centroids from the generated corpus; mock
+	 * services can omit it.
+	 */
+	prepareScenario?(args: {
+		scenario: VoiceScenario;
+		corpus: GeneratedVoiceCorpus;
+	}): Promise<void> | void;
+	/**
+	 * Feed one turn's audio slice through the real services and report what was
+	 * observed. The `label` carries the turn's ground truth (so a mock can echo
+	 * it); the real adapter ignores it and measures.
+	 */
+	observeTurn(args: {
+		turnIndex: number;
+		audio: Float32Array;
+		sampleRate: number;
+		label: CorpusTurnLabel;
+		/** Scenario-level ground truth (participants, owner set, agents). */
+		groundTruth: CorpusGroundTruth;
+	}): Promise<VoiceTurnObservation>;
+}
+
+/**
+ * Where to write per-run `.wav` artifacts. When present, the runner encodes the
+ * full corpus + each consumed per-turn slice to disk under `dir` and records the
+ * paths (relative to `relativeTo`) on the run's `audioArtifacts`. Absent ⇒ no IO.
+ */
+export interface VoiceAudioCaptureSink {
+	/** Directory the `.wav` files are written into (created recursively). */
+	dir: string;
+	/** Artifact paths are recorded relative to this dir (the scenario run dir). */
+	relativeTo: string;
+}
+
+export interface RunVoiceScenarioHeadlessArgs {
+	scenario: VoiceScenario;
+	/** The generated/loaded corpus; null when its artifacts are absent. */
+	corpus: GeneratedVoiceCorpus | null;
+	/** The voice services; null when no backend is provisioned. */
+	services: VoiceWorkbenchServices | null;
+	/** When set, write the run's audio as `.wav` artifacts and record their paths. */
+	captureAudio?: VoiceAudioCaptureSink;
+}
+
+/** Encode `pcm` to `<dir>/<fileName>` and return a relative-path artifact. */
+function writeAudioArtifact(args: {
+	sink: VoiceAudioCaptureSink;
+	fileName: string;
+	pcm: Float32Array;
+	sampleRate: number;
+	turnIndex: number;
+	kind: VoiceAudioArtifact["kind"];
+	speakerLabel?: string;
+}): VoiceAudioArtifact {
+	const absolutePath = path.join(args.sink.dir, args.fileName);
+	writeFileSync(absolutePath, encodeMonoPcm16Wav(args.pcm, args.sampleRate));
+	const relativePath = path
+		.relative(args.sink.relativeTo, absolutePath)
+		.split(path.sep)
+		.join("/");
+	return {
+		turnIndex: args.turnIndex,
+		kind: args.kind,
+		path: relativePath,
+		sampleRate: args.sampleRate,
+		durationMs: Math.round((args.pcm.length / args.sampleRate) * 1000),
+		...(args.speakerLabel !== undefined
+			? { speakerLabel: args.speakerLabel }
+			: {}),
+	};
+}
+
+function skipped(
+	scenario: VoiceScenario,
+	skipReason: string,
+): VoiceWorkbenchScenarioRun {
+	return {
+		scenarioId: scenario.id,
+		classes: scenario.classes,
+		status: "skipped",
+		cases: [],
+		skipReason,
+	};
+}
+
+/**
+ * Run one scenario headless and score it. Returns a `skipped` run (never a
+ * pass) when the corpus or the backend is absent.
+ */
+export async function runVoiceScenarioHeadless(
+	args: RunVoiceScenarioHeadlessArgs,
+): Promise<VoiceWorkbenchScenarioRun> {
+	const { scenario, corpus, services, captureAudio } = args;
+	if (!corpus) return skipped(scenario, "corpus artifacts absent");
+	if (!services) return skipped(scenario, "no voice backend provisioned");
+
+	const assertions = scenario.assertions ?? {};
+	const cases: VoiceE2eCaseResult[] = [];
+	await services.prepareScenario?.({ scenario, corpus });
+
+	// When a capture sink is active, write the full corpus once + each per-turn
+	// slice as `.wav` artifacts and record their (run-dir-relative) paths.
+	const audioArtifacts: VoiceAudioArtifact[] = [];
+	if (captureAudio) {
+		mkdirSync(captureAudio.dir, { recursive: true });
+		audioArtifacts.push(
+			writeAudioArtifact({
+				sink: captureAudio,
+				fileName: "corpus.wav",
+				pcm: corpus.pcm,
+				sampleRate: corpus.sampleRate,
+				turnIndex: 0,
+				kind: "generated",
+			}),
+		);
+	}
+
+	const eotSamples: Array<{
+		decided: boolean;
+		expected: boolean;
+		latencyMs?: number;
+	}> = [];
+	const diarTurns: DiarizationTurnSample[] = [];
+	const respondSamples: Array<{ responded: boolean; expectRespond: boolean }> =
+		[];
+	const voiceEntitySamples: Array<{
+		matchedEntityId: string | null;
+		expectedEntityId: string;
+	}> = [];
+	const inferredEntities: string[] = [];
+	const expectedEntities: string[] = [];
+	const echoSamples: Array<{ isAgentEcho: boolean; responded: boolean }> = [];
+	const ownerSamples: Array<{
+		predictedOwner: boolean;
+		expectedOwner: boolean;
+	}> = [];
+	const wantsOwnerScoring = scenario.classes.includes("owner-security");
+
+	for (const label of corpus.groundTruth.turns) {
+		const audio = corpus.pcm.subarray(
+			label.segmentStartSample,
+			label.segmentEndSample,
+		);
+		if (captureAudio) {
+			audioArtifacts.push(
+				writeAudioArtifact({
+					sink: captureAudio,
+					fileName: `turn-${label.index}.wav`,
+					pcm: audio,
+					sampleRate: corpus.sampleRate,
+					turnIndex: label.index,
+					kind: "consumed",
+					speakerLabel: label.speaker,
+				}),
+			);
+		}
+		const obs = await services.observeTurn({
+			turnIndex: label.index,
+			audio,
+			sampleRate: corpus.sampleRate,
+			label,
+			groundTruth: corpus.groundTruth,
+		});
+
+		// WER — one round-trip case per turn (referenceTranscript vs ASR hypothesis).
+		cases.push(
+			scoreTtsAsrRoundTrip({
+				referenceText: label.referenceTranscript,
+				hypothesisText: obs.hypothesisTranscript,
+				...(assertions.maxWer !== undefined
+					? { maxWer: assertions.maxWer }
+					: {}),
+			}),
+		);
+
+		// EOT — some corpus segments deliberately stop mid-utterance so the
+		// classifier can be scored for false triggers.
+		eotSamples.push({
+			decided: obs.eotDecided,
+			expected: label.expectEndOfTurn ?? true,
+			...(obs.eotLatencyMs !== undefined
+				? { latencyMs: obs.eotLatencyMs }
+				: {}),
+		});
+		// Diarization scores REAL speaker turns only — an agent-echo turn is the
+		// agent's own voice bleeding back, not a speaker to attribute, so it is
+		// handled by the echo-rejection scorer instead.
+		if (!label.isAgentEcho) {
+			diarTurns.push({
+				predictedLabel: obs.predictedSpeakerLabel,
+				expectedLabel: label.speaker,
+				startMs: (label.speechStartSample / corpus.sampleRate) * 1000,
+				endMs: (label.speechEndSample / corpus.sampleRate) * 1000,
+			});
+		}
+		respondSamples.push({
+			responded: obs.responded,
+			expectRespond: label.expectRespond,
+		});
+		if (label.entityId) {
+			voiceEntitySamples.push({
+				matchedEntityId: obs.matchedEntityId,
+				expectedEntityId: label.entityId,
+			});
+		}
+		if (label.expectedEntity) expectedEntities.push(label.expectedEntity);
+		inferredEntities.push(...obs.inferredEntities);
+
+		if (label.isAgentEcho) {
+			echoSamples.push({ isAgentEcho: true, responded: obs.responded });
+		}
+		if (wantsOwnerScoring && typeof obs.predictedOwner === "boolean") {
+			ownerSamples.push({
+				predictedOwner: obs.predictedOwner,
+				expectedOwner: label.isOwner === true,
+			});
+		}
+
+		if (obs.responded && typeof obs.firstAudioMs === "number") {
+			cases.push(
+				scoreFirstResponseLatency({
+					turnStartedAtMs: 0,
+					ttsFirstAudioAtMs: obs.firstAudioMs,
+					...(assertions.maxFirstAudioMs !== undefined
+						? { maxFirstAudioMs: assertions.maxFirstAudioMs }
+						: {}),
+				}),
+			);
+		}
+	}
+
+	cases.push(
+		scoreEotDecision(eotSamples, {
+			...(assertions.maxEotFalseTriggerRate !== undefined
+				? { maxFalseTriggerRate: assertions.maxEotFalseTriggerRate }
+				: {}),
+		}),
+	);
+	if (diarTurns.length > 0) {
+		cases.push(
+			scoreDiarizationTimeline(diarTurns, {
+				...(assertions.maxDer !== undefined
+					? { maxDer: assertions.maxDer }
+					: {}),
+			}),
+		);
+	}
+	cases.push(
+		scoreRespondDecision(respondSamples, {
+			...(assertions.minRespondAccuracy !== undefined
+				? { minAccuracy: assertions.minRespondAccuracy }
+				: {}),
+		}),
+	);
+	// Entity extraction + voice→entity match only when the scenario asserts them.
+	if (expectedEntities.length > 0 || inferredEntities.length > 0) {
+		cases.push(
+			scoreEntityExtraction({
+				expected: expectedEntities,
+				inferred: inferredEntities,
+			}),
+		);
+	}
+	if (voiceEntitySamples.length > 0) {
+		cases.push(
+			scoreVoiceEntityMatch(voiceEntitySamples, {
+				...(assertions.minVoiceEntityMatchRate !== undefined
+					? { minMatchRate: assertions.minVoiceEntityMatchRate }
+					: {}),
+			}),
+		);
+	}
+	if (echoSamples.length > 0) {
+		cases.push(
+			scoreEchoRejection(echoSamples, {
+				...(assertions.minEchoRejectionRate !== undefined
+					? { minRejectionRate: assertions.minEchoRejectionRate }
+					: {}),
+			}),
+		);
+	}
+	if (ownerSamples.length > 0) {
+		cases.push(
+			scoreOwnerSecurity(ownerSamples, {
+				...(assertions.minOwnerAccuracy !== undefined
+					? { minAccuracy: assertions.minOwnerAccuracy }
+					: {}),
+			}),
+		);
+	}
+
+	return {
+		scenarioId: scenario.id,
+		classes: scenario.classes,
+		status: "ran",
+		cases,
+		...(audioArtifacts.length > 0 ? { audioArtifacts } : {}),
+	};
+}
+
+export interface RunVoiceWorkbenchArgs {
+	scenarios: ReadonlyArray<{
+		scenario: VoiceScenario;
+		corpus: GeneratedVoiceCorpus | null;
+	}>;
+	services: VoiceWorkbenchServices | null;
+}
+
+/** Run a matrix of scenarios headless, returning one run per scenario. */
+export async function runVoiceWorkbenchHeadless(
+	args: RunVoiceWorkbenchArgs,
+): Promise<VoiceWorkbenchScenarioRun[]> {
+	const runs: VoiceWorkbenchScenarioRun[] = [];
+	for (const entry of args.scenarios) {
+		runs.push(
+			await runVoiceScenarioHeadless({
+				scenario: entry.scenario,
+				corpus: entry.corpus,
+				services: args.services,
+			}),
+		);
+	}
+	return runs;
+}

package/src/services/voice/workbench-logic-services.test.ts

@@ -0,0 +1,225 @@
+import { describe, expect, it } from "vitest";
+import { generateVoiceCorpus } from "./corpus-generator";
+import type { VoiceScenario } from "./voice-scenario";
+import { buildVoiceWorkbenchReport } from "./voice-workbench-report";
+import { runVoiceScenarioHeadless } from "./workbench-headless-runner";
+import { realDecisionLogicServices } from "./workbench-logic-services";
+import { VOICE_WORKBENCH_SCENARIOS } from "./workbench-scenarios";
+
+describe("realDecisionLogicServices over the built-in scenario matrix", () => {
+	it("every built-in scenario PASSES against the real decision logic", async () => {
+		const services = realDecisionLogicServices();
+		const runs = [];
+		for (const scenario of VOICE_WORKBENCH_SCENARIOS) {
+			const corpus = await generateVoiceCorpus(scenario);
+			runs.push(await runVoiceScenarioHeadless({ scenario, corpus, services }));
+		}
+		const report = buildVoiceWorkbenchReport(runs);
+		const failures = report.scenarios.filter((s) => s.verdict === "fail");
+		expect(
+			failures,
+			`failing scenarios: ${JSON.stringify(failures, null, 2)}`,
+		).toEqual([]);
+		expect(report.overall).toBe("pass");
+		expect(report.scenariosRan).toBe(VOICE_WORKBENCH_SCENARIOS.length);
+	});
+
+	it("attributes the multi-speaker scenarios from AUDIO, scoring DER 0", async () => {
+		// The real diarization gate: blind acoustic clustering must partition the
+		// distinct voices correctly — proving `predictedSpeakerLabel` is derived,
+		// not copied from the ground-truth label (#9427).
+		const services = realDecisionLogicServices();
+		for (const id of ["multi-voice-greeting", "multi-speaker-name-capture"]) {
+			const scenario = VOICE_WORKBENCH_SCENARIOS.find((s) => s.id === id);
+			if (!scenario) throw new Error(`scenario ${id} missing`);
+			const corpus = await generateVoiceCorpus(scenario);
+			const run = await runVoiceScenarioHeadless({
+				scenario,
+				corpus,
+				services,
+			});
+			const diar = run.cases.find((c) => c.kind === "diarization");
+			expect(diar, `${id} has a diarization case`).toBeDefined();
+			expect(diar?.kind === "diarization" && diar.der).toBe(0);
+			expect(diar?.passed).toBe(true);
+		}
+	});
+
+	it("the DER gate FAILS on a real misattribution — not a tautology (#9427)", async () => {
+		// Two participants, SAME words so their speech regions are the same length.
+		const scenario: VoiceScenario = {
+			id: "diar-divergence-probe",
+			classes: ["diarization", "multi-speaker"],
+			participants: [
+				{ label: "alice", entityId: "entity-alice" },
+				{ label: "bob", entityId: "entity-bob" },
+			],
+			turns: [
+				{
+					speaker: "alice",
+					text: "eliza what time is it now",
+					expectRespond: true,
+				},
+				{
+					speaker: "bob",
+					text: "eliza what time is it now",
+					expectRespond: true,
+				},
+			],
+			assertions: { maxDer: 0.2 },
+		};
+		const corpus = await generateVoiceCorpus(scenario);
+
+		// Honest: the two distinct voices cluster apart → DER 0, gate passes.
+		const honest = await runVoiceScenarioHeadless({
+			scenario,
+			corpus,
+			services: realDecisionLogicServices(),
+		});
+		const honestDer = honest.cases.find((c) => c.kind === "diarization");
+		expect(honestDer?.der).toBe(0);
+		expect(honestDer?.passed).toBe(true);
+
+		// Tamper ONLY the audio: overwrite bob's speech with alice's voice. Ground
+		// truth still labels the turns alice/bob, so a tautological gate would keep
+		// passing — but the real acoustic clusterer hears one voice, merges the
+		// turns, and the DER gate trips.
+		const [aliceTurn, bobTurn] = corpus.groundTruth.turns;
+		const tamperedPcm = corpus.pcm.slice();
+		tamperedPcm.set(
+			corpus.pcm.subarray(
+				aliceTurn.speechStartSample,
+				aliceTurn.speechEndSample,
+			),
+			bobTurn.speechStartSample,
+		);
+		const tampered = await runVoiceScenarioHeadless({
+			scenario,
+			corpus: { ...corpus, pcm: tamperedPcm },
+			services: realDecisionLogicServices(),
+		});
+		const tamperedDer = tampered.cases.find((c) => c.kind === "diarization");
+		expect(tamperedDer?.der).toBeGreaterThan(0.2);
+		expect(tamperedDer?.passed).toBe(false);
+	});
+
+	it("genuinely SUPPRESSES a confident bystander (not just echoing ground truth)", async () => {
+		const scenario = VOICE_WORKBENCH_SCENARIOS.find(
+			(s) => s.id === "respond-vs-bystander",
+		);
+		if (!scenario) throw new Error("scenario missing");
+		const corpus = await generateVoiceCorpus(scenario);
+		const services = realDecisionLogicServices();
+		const responded: boolean[] = [];
+		for (const label of corpus.groundTruth.turns) {
+			const obs = await services.observeTurn({
+				turnIndex: label.index,
+				audio: corpus.pcm.subarray(
+					label.segmentStartSample,
+					label.segmentEndSample,
+				),
+				sampleRate: corpus.sampleRate,
+				label,
+				groundTruth: corpus.groundTruth,
+			});
+			responded.push(obs.responded);
+		}
+		// alice (owner) → respond, bob (bystander) → silent, alice → respond.
+		expect(responded).toEqual([true, false, true]);
+	});
+
+	it("genuinely REJECTS the agent's own echoed reply via word-overlap", async () => {
+		const scenario = VOICE_WORKBENCH_SCENARIOS.find(
+			(s) => s.id === "echo-self-trigger",
+		);
+		if (!scenario) throw new Error("scenario missing");
+		const corpus = await generateVoiceCorpus(scenario);
+		const services = realDecisionLogicServices();
+		const responded: boolean[] = [];
+		for (const label of corpus.groundTruth.turns) {
+			const obs = await services.observeTurn({
+				turnIndex: label.index,
+				audio: corpus.pcm.subarray(
+					label.segmentStartSample,
+					label.segmentEndSample,
+				),
+				sampleRate: corpus.sampleRate,
+				label,
+				groundTruth: corpus.groundTruth,
+			});
+			responded.push(obs.responded);
+		}
+		// real reply (respond) → echoed reply (suppressed) → thanks (respond).
+		expect(responded).toEqual([true, false, true]);
+	});
+
+	it("genuinely HOLDS on a mid-utterance pause (EOT gate), then commits", async () => {
+		const scenario = VOICE_WORKBENCH_SCENARIOS.find(
+			(s) => s.id === "pauses-midutterance",
+		);
+		if (!scenario) throw new Error("scenario missing");
+		const corpus = await generateVoiceCorpus(scenario);
+		const services = realDecisionLogicServices();
+		const decided: boolean[] = [];
+		const responded: boolean[] = [];
+		for (const label of corpus.groundTruth.turns) {
+			const obs = await services.observeTurn({
+				turnIndex: label.index,
+				audio: corpus.pcm.subarray(
+					label.segmentStartSample,
+					label.segmentEndSample,
+				),
+				sampleRate: corpus.sampleRate,
+				label,
+				groundTruth: corpus.groundTruth,
+			});
+			decided.push(obs.eotDecided);
+			responded.push(obs.responded);
+		}
+		// "...schedule a meeting with" trails off → not end-of-turn, no response;
+		// "Bob tomorrow at noon" completes → end-of-turn, respond.
+		expect(decided).toEqual([false, true]);
+		expect(responded).toEqual([false, true]);
+	});
+
+	it("resets reply state between scenarios (no cross-scenario echo leak)", async () => {
+		const services = realDecisionLogicServices();
+		const echo = VOICE_WORKBENCH_SCENARIOS.find(
+			(s) => s.id === "echo-self-trigger",
+		);
+		const greeting = VOICE_WORKBENCH_SCENARIOS.find(
+			(s) => s.id === "multi-voice-greeting",
+		);
+		if (!echo || !greeting) throw new Error("scenarios missing");
+		// Run echo scenario first (populates lastAgentReply), then greeting.
+		for (const scenario of [echo, greeting]) {
+			const corpus = await generateVoiceCorpus(scenario);
+			for (const label of corpus.groundTruth.turns) {
+				await services.observeTurn({
+					turnIndex: label.index,
+					audio: corpus.pcm.subarray(
+						label.segmentStartSample,
+						label.segmentEndSample,
+					),
+					sampleRate: corpus.sampleRate,
+					label,
+					groundTruth: corpus.groundTruth,
+				});
+			}
+		}
+		// Greeting's first turn must still be answered (no stale reply suppressing it).
+		const corpus = await generateVoiceCorpus(greeting);
+		const first = corpus.groundTruth.turns[0];
+		const obs = await services.observeTurn({
+			turnIndex: 0,
+			audio: corpus.pcm.subarray(
+				first.segmentStartSample,
+				first.segmentEndSample,
+			),
+			sampleRate: corpus.sampleRate,
+			label: first,
+			groundTruth: corpus.groundTruth,
+		});
+		expect(obs.responded).toBe(true);
+	});
+});