@elizaos/plugin-local-inference 2.0.3-beta.2 → 2.0.3-beta.4

package/src/services/voice/diarization-error-rate.ts

@@ -0,0 +1,249 @@
+// Diarization Error Rate (DER) scorer for the voice test matrix (issue #9147).
+//
+// The voice-scenario schema already carries a `maxDer` threshold and an
+// `expectedSpeakerLabel` per turn, but nothing computed DER — diarization was
+// only ever scored on the respond-decision, so a wrong speaker attribution or a
+// missed overlapping talker passed silently. This is that missing scorer: a
+// pure, frame-based DER (the standard NIST md-eval decomposition) so the
+// diarization / multi-speaker / overlapping-speech scenario classes can assert
+// per-turn labels AND enforce `maxDer`.
+//
+// Frame-based (default 10ms) so OVERLAPPING speech is handled correctly: each
+// frame compares the SET of reference speakers against the SET of hypothesis
+// speakers, after an optimal one-to-one speaker mapping that maximizes matched
+// time. Everything here is deterministic and dependency-free.
+
+export interface DiarizationSegment {
+	/** Speaker label (any stable string — ground-truth and hypothesis label
+	 * spaces are mapped against each other, so the raw strings need not match). */
+	speaker: string;
+	/** Segment start, milliseconds. */
+	startMs: number;
+	/** Segment end, milliseconds (must be ≥ startMs). */
+	endMs: number;
+}
+
+export interface DerResult {
+	/** Diarization Error Rate: (missed + falseAlarm + confusion) / referenceMs.
+	 * 0 = perfect; can exceed 1 when false alarms dominate. */
+	der: number;
+	/** Reference speech the system failed to attribute to anyone (ms). */
+	missedMs: number;
+	/** System speech with no reference speaker present (ms). */
+	falseAlarmMs: number;
+	/** Reference speech attributed to the wrong (mapped) speaker (ms). */
+	confusionMs: number;
+	/** Total reference speaker-time (Σ |ref speakers in frame| · frame), the DER denominator. */
+	totalReferenceMs: number;
+	/** The chosen hypothesis→reference speaker mapping (optimal for matched time). */
+	mapping: Record<string, string>;
+}
+
+export interface DerOptions {
+	/** Frame size in ms (default 10). Smaller = more precise, more work. */
+	frameMs?: number;
+	/** Above this combined speaker count, fall back to a greedy mapping instead
+	 * of the exact permutation search (keeps it O(n) not O(n!)). Default 7. */
+	maxExactSpeakers?: number;
+}
+
+function totalDurationMs(segments: readonly DiarizationSegment[]): number {
+	return segments.reduce((max, s) => Math.max(max, s.endMs), 0);
+}
+
+/** Per-frame active-speaker sets. `frames[f]` is the set of speakers whose
+ * segment covers the start of frame `f`. */
+function frameize(
+	segments: readonly DiarizationSegment[],
+	frameMs: number,
+	numFrames: number,
+): Array<Set<string>> {
+	const frames: Array<Set<string>> = Array.from(
+		{ length: numFrames },
+		() => new Set<string>(),
+	);
+	for (const seg of segments) {
+		if (seg.endMs <= seg.startMs) continue;
+		const first = Math.max(0, Math.floor(seg.startMs / frameMs));
+		// A frame f (covering [f·frameMs, (f+1)·frameMs)) is active if its start
+		// time falls within [startMs, endMs).
+		const last = Math.min(numFrames - 1, Math.ceil(seg.endMs / frameMs) - 1);
+		for (let f = first; f <= last; f++) {
+			if (f * frameMs >= seg.startMs && f * frameMs < seg.endMs) {
+				frames[f].add(seg.speaker);
+			}
+		}
+	}
+	return frames;
+}
+
+function uniqueSpeakers(segments: readonly DiarizationSegment[]): string[] {
+	return [...new Set(segments.map((s) => s.speaker))];
+}
+
+/** Frames where reference speaker `r` and hypothesis speaker `h` are both active. */
+function coOccurrence(
+	refFrames: Array<Set<string>>,
+	hypFrames: Array<Set<string>>,
+	refSpeakers: string[],
+	hypSpeakers: string[],
+): Map<string, Map<string, number>> {
+	const co = new Map<string, Map<string, number>>();
+	for (const r of refSpeakers)
+		co.set(r, new Map(hypSpeakers.map((h) => [h, 0])));
+	for (let f = 0; f < refFrames.length; f++) {
+		const rs = refFrames[f];
+		const hs = hypFrames[f];
+		if (rs.size === 0 || hs.size === 0) continue;
+		for (const r of rs) {
+			const row = co.get(r);
+			if (!row) continue;
+			for (const h of hs) row.set(h, (row.get(h) ?? 0) + 1);
+		}
+	}
+	return co;
+}
+
+/** Optimal injective hyp→ref mapping maximizing total co-occurrence (exact for
+ * small speaker counts, greedy beyond `maxExactSpeakers`). */
+function bestMapping(
+	co: Map<string, Map<string, number>>,
+	refSpeakers: string[],
+	hypSpeakers: string[],
+	maxExact: number,
+): Record<string, string> {
+	const score = (h: string, r: string) => co.get(r)?.get(h) ?? 0;
+
+	if (refSpeakers.length + hypSpeakers.length <= maxExact) {
+		// Exact: try every injective assignment of hyp speakers onto ref speakers.
+		let best: Record<string, string> = {};
+		let bestScore = -1;
+		const assign = (
+			i: number,
+			usedRefs: Set<string>,
+			current: Record<string, string>,
+			running: number,
+		) => {
+			if (i === hypSpeakers.length) {
+				if (running > bestScore) {
+					bestScore = running;
+					best = { ...current };
+				}
+				return;
+			}
+			const h = hypSpeakers[i];
+			// Option: leave this hypothesis speaker unmapped (pure false alarm).
+			assign(i + 1, usedRefs, current, running);
+			for (const r of refSpeakers) {
+				if (usedRefs.has(r)) continue;
+				usedRefs.add(r);
+				current[h] = r;
+				assign(i + 1, usedRefs, current, running + score(h, r));
+				delete current[h];
+				usedRefs.delete(r);
+			}
+		};
+		assign(0, new Set(), {}, 0);
+		return best;
+	}
+
+	// Greedy: repeatedly take the highest-scoring (h, r) pair not yet used.
+	const pairs: Array<{ h: string; r: string; s: number }> = [];
+	for (const h of hypSpeakers) {
+		for (const r of refSpeakers) pairs.push({ h, r, s: score(h, r) });
+	}
+	pairs.sort((a, b) => b.s - a.s);
+	const usedHyp = new Set<string>();
+	const usedRef = new Set<string>();
+	const mapping: Record<string, string> = {};
+	for (const { h, r, s } of pairs) {
+		if (s <= 0 || usedHyp.has(h) || usedRef.has(r)) continue;
+		mapping[h] = r;
+		usedHyp.add(h);
+		usedRef.add(r);
+	}
+	return mapping;
+}
+
+/**
+ * Frame-based Diarization Error Rate between a reference (ground-truth) speaker
+ * timeline and a hypothesis (diarizer output) timeline. Returns DER plus its
+ * missed / false-alarm / confusion decomposition and the speaker mapping used.
+ */
+export function computeDiarizationErrorRate(
+	reference: readonly DiarizationSegment[],
+	hypothesis: readonly DiarizationSegment[],
+	options: DerOptions = {},
+): DerResult {
+	const frameMs = options.frameMs && options.frameMs > 0 ? options.frameMs : 10;
+	const maxExact = options.maxExactSpeakers ?? 7;
+	const durationMs = Math.max(
+		totalDurationMs(reference),
+		totalDurationMs(hypothesis),
+	);
+	const numFrames = Math.ceil(durationMs / frameMs);
+
+	const refSpeakers = uniqueSpeakers(reference);
+	const hypSpeakers = uniqueSpeakers(hypothesis);
+	const refFrames = frameize(reference, frameMs, numFrames);
+	const hypFrames = frameize(hypothesis, frameMs, numFrames);
+
+	const co = coOccurrence(refFrames, hypFrames, refSpeakers, hypSpeakers);
+	const mapping = bestMapping(co, refSpeakers, hypSpeakers, maxExact);
+	// inverse: ref speaker -> the hyp speaker mapped onto it.
+	const inverse = new Map<string, string>();
+	for (const [h, r] of Object.entries(mapping)) inverse.set(r, h);
+
+	let missedFrames = 0;
+	let falseAlarmFrames = 0;
+	let confusionFrames = 0;
+	let referenceSpeakerFrames = 0;
+
+	for (let f = 0; f < numFrames; f++) {
+		const R = refFrames[f];
+		const H = hypFrames[f];
+		const nRef = R.size;
+		const nSys = H.size;
+		referenceSpeakerFrames += nRef;
+		if (nRef === 0 && nSys === 0) continue;
+
+		// Correctly attributed: a ref speaker whose mapped hyp speaker is active.
+		let correct = 0;
+		for (const r of R) {
+			const h = inverse.get(r);
+			if (h !== undefined && H.has(h)) correct += 1;
+		}
+
+		missedFrames += Math.max(0, nRef - nSys);
+		falseAlarmFrames += Math.max(0, nSys - nRef);
+		confusionFrames += Math.min(nRef, nSys) - correct;
+	}
+
+	const missedMs = missedFrames * frameMs;
+	const falseAlarmMs = falseAlarmFrames * frameMs;
+	const confusionMs = confusionFrames * frameMs;
+	const totalReferenceMs = referenceSpeakerFrames * frameMs;
+	const der =
+		totalReferenceMs > 0
+			? (missedMs + falseAlarmMs + confusionMs) / totalReferenceMs
+			: falseAlarmMs > 0
+				? Number.POSITIVE_INFINITY
+				: 0;
+
+	return {
+		der,
+		missedMs,
+		falseAlarmMs,
+		confusionMs,
+		totalReferenceMs,
+		mapping,
+	};
+}
+
+/** Whether a diarization hypothesis is within a scenario's `maxDer` budget. */
+export function diarizationWithinBudget(
+	result: Pick<DerResult, "der">,
+	maxDer: number,
+): boolean {
+	return result.der <= maxDer;
+}

package/src/services/voice/e2e-harness.der.test.ts

@@ -0,0 +1,94 @@
+import { describe, expect, it } from "vitest";
+import { type DiarizationSample, scoreDiarization } from "./e2e-harness";
+
+// #9147 — diarization error rate (DER) is the third heavy voice case the issue
+// flags as having a scorer that "does not run anywhere that gates merges". Like
+// scoreEchoRejection / scoreOwnerSecurity, scoreDiarization is pure and
+// GGUF-independent, so pin its decision math here to give merges an offline gate.
+//
+// DER here is the per-segment label-agreement form used by the harness:
+//   der = (speakerConfusion + missed) / totalReference
+// where a null predicted label is a miss and a non-null mismatch is a confusion.
+
+const seg = (
+	predictedLabel: string | null,
+	expectedLabel: string,
+): DiarizationSample => ({ predictedLabel, expectedLabel });
+
+describe("scoreDiarization (#9147)", () => {
+	it("perfect alignment (reference == hypothesis) → DER 0, passes", () => {
+		const r = scoreDiarization([
+			seg("alice", "alice"),
+			seg("bob", "bob"),
+			seg("alice", "alice"),
+			seg("bob", "bob"),
+		]);
+		expect(r.total).toBe(4);
+		expect(r.confusions).toBe(0);
+		expect(r.misses).toBe(0);
+		expect(r.der).toBe(0);
+		expect(r.passed).toBe(true);
+	});
+
+	it("one swapped speaker over four turns → DER 0.25 (1 confusion / 4)", () => {
+		const r = scoreDiarization([
+			seg("alice", "alice"),
+			seg("alice", "bob"), // swapped: predicted alice, truth bob → confusion
+			seg("alice", "alice"),
+			seg("bob", "bob"),
+		]);
+		expect(r.total).toBe(4);
+		expect(r.confusions).toBe(1);
+		expect(r.misses).toBe(0);
+		expect(r.der).toBe(0.25); // (1 + 0) / 4
+		expect(r.passed).toBe(false); // 0.25 > default maxDer 0.2
+	});
+
+	it("one missed segment over four turns → DER 0.25 (1 miss / 4)", () => {
+		const r = scoreDiarization([
+			seg("alice", "alice"),
+			seg(null, "bob"), // missed: no predicted label → miss
+			seg("alice", "alice"),
+			seg("bob", "bob"),
+		]);
+		expect(r.total).toBe(4);
+		expect(r.confusions).toBe(0);
+		expect(r.misses).toBe(1);
+		expect(r.der).toBe(0.25); // (0 + 1) / 4
+		expect(r.passed).toBe(false); // 0.25 > default maxDer 0.2
+	});
+
+	it("empty hypothesis vs non-empty reference → all-missed, DER 1, fails closed", () => {
+		const r = scoreDiarization([
+			seg(null, "alice"),
+			seg(null, "bob"),
+			seg(null, "alice"),
+		]);
+		expect(r.total).toBe(3);
+		expect(r.confusions).toBe(0);
+		expect(r.misses).toBe(3);
+		expect(r.der).toBe(1); // 3 missed / 3 reference
+		expect(r.passed).toBe(false);
+	});
+
+	it("fails closed on an empty sample set (nothing proven)", () => {
+		const r = scoreDiarization([]);
+		expect(r.total).toBe(0);
+		expect(r.der).toBe(0);
+		expect(r.passed).toBe(false);
+	});
+
+	it("honors a custom maxDer floor", () => {
+		// 1 confusion over 5 → DER 0.2, exactly the default ceiling.
+		const samples = [
+			seg("alice", "alice"),
+			seg("bob", "alice"), // confusion
+			seg("alice", "alice"),
+			seg("bob", "bob"),
+			seg("alice", "alice"),
+		];
+		expect(scoreDiarization(samples).der).toBe(0.2);
+		expect(scoreDiarization(samples).passed).toBe(true); // 0.2 <= 0.2 default
+		expect(scoreDiarization(samples, { maxDer: 0.1 }).passed).toBe(false); // 0.2 > 0.1
+	});
+});

package/src/services/voice/e2e-harness.respond-eot-entity.test.ts

@@ -0,0 +1,277 @@
+import { describe, expect, it } from "vitest";
+import {
+	type EotDecisionSample,
+	type RespondDecisionSample,
+	scoreEntityExtraction,
+	scoreEotDecision,
+	scoreRespondDecision,
+	scoreVoiceEntityMatch,
+	summarizeVoiceE2e,
+	type VoiceEntityMatchSample,
+} from "./e2e-harness";
+
+// #9147 — four more pure, GGUF-independent voice e2e scorers that the issue's
+// test matrix relies on but which had no offline coverage (the matrix "scorers
+// exist but don't gate merges" gap). Companions to the already-pinned
+// scoreDiarization (#9222), scoreEchoRejection, and scoreOwnerSecurity:
+//   - scoreRespondDecision  → respond-when-should vs answer-a-bystander (FP/FN)
+//   - scoreEotDecision      → VAD/EOT don't false-trigger (noise/music rows)
+//   - scoreEntityExtraction → owner-inference entity precision/recall/f1
+//   - scoreVoiceEntityMatch → recognized voice resolves to the right entity
+// All are pure decision math, so they give merges an offline gate today; the
+// real-audio lanes stay blocked on the pending speaker-encoder/diarizer GGUFs.
+
+const eot = (
+	decided: boolean,
+	expected: boolean,
+	latencyMs?: number,
+): EotDecisionSample => ({ decided, expected, latencyMs });
+
+const respond = (
+	responded: boolean,
+	expectRespond: boolean,
+): RespondDecisionSample => ({ responded, expectRespond });
+
+const match = (
+	matchedEntityId: string | null,
+	expectedEntityId: string,
+): VoiceEntityMatchSample => ({ matchedEntityId, expectedEntityId });
+
+describe("scoreEotDecision (#9147)", () => {
+	it("perfect EOT decisions → 0 false-trigger / 0 false-suppress, passes", () => {
+		const r = scoreEotDecision([
+			eot(true, true, 120),
+			eot(false, false),
+			eot(true, true, 80),
+			eot(false, false),
+		]);
+		expect(r.total).toBe(4);
+		expect(r.falseTriggerRate).toBe(0);
+		expect(r.falseSuppressionRate).toBe(0);
+		expect(r.accuracy).toBe(1);
+		expect(r.latencyP50Ms).not.toBeNull();
+		expect(r.passed).toBe(true);
+	});
+
+	it("a single false trigger over 4 turns exceeds the default 0.1 ceiling → fails", () => {
+		// decided where there was NO real boundary = jumped in over a noise/music
+		// false-trigger; 1/4 = 0.25 > 0.1.
+		const r = scoreEotDecision([
+			eot(true, false, 50), // false trigger
+			eot(false, false),
+			eot(true, true, 90),
+			eot(false, false),
+		]);
+		expect(r.falseTriggerRate).toBe(0.25);
+		expect(r.passed).toBe(false);
+	});
+
+	it("false suppression alone does NOT fail the gate (only false-trigger gates)", () => {
+		// held when it should have ended the turn; falseTriggerRate stays 0.
+		const r = scoreEotDecision([
+			eot(false, true), // missed boundary
+			eot(false, false),
+			eot(true, true, 60),
+			eot(false, false),
+		]);
+		expect(r.falseSuppressionRate).toBe(0.25);
+		expect(r.falseTriggerRate).toBe(0);
+		expect(r.passed).toBe(true);
+	});
+
+	it("a relaxed maxFalseTriggerRate admits the borderline case", () => {
+		const r = scoreEotDecision(
+			[
+				eot(true, false, 40),
+				eot(false, false),
+				eot(true, true, 70),
+				eot(false, false),
+			],
+			{ maxFalseTriggerRate: 0.25 },
+		);
+		expect(r.falseTriggerRate).toBe(0.25);
+		expect(r.passed).toBe(true);
+	});
+
+	it("latency percentiles are null when no decided turn carries a latency", () => {
+		const r = scoreEotDecision([eot(true, true), eot(false, false)]);
+		expect(r.latencyP50Ms).toBeNull();
+		expect(r.latencyP95Ms).toBeNull();
+	});
+
+	it("empty sample set fails closed (no evidence the gate works)", () => {
+		const r = scoreEotDecision([]);
+		expect(r.total).toBe(0);
+		expect(r.passed).toBe(false);
+	});
+});
+
+describe("scoreRespondDecision (#9147)", () => {
+	it("all decisions correct → accuracy 1, no FP/FN, passes", () => {
+		const r = scoreRespondDecision([
+			respond(true, true),
+			respond(false, false),
+			respond(true, true),
+			respond(false, false),
+		]);
+		expect(r.total).toBe(4);
+		expect(r.accuracy).toBe(1);
+		expect(r.falsePositiveRate).toBe(0);
+		expect(r.falseNegativeRate).toBe(0);
+		expect(r.passed).toBe(true);
+	});
+
+	it("answering a bystander is a false positive scored over the should-NOT turns", () => {
+		// 2 should-not turns, 1 answered → FPR 0.5; accuracy 3/4 = 0.75 < 0.9.
+		const r = scoreRespondDecision([
+			respond(true, true),
+			respond(true, false), // answered a bystander
+			respond(false, false),
+			respond(true, true),
+		]);
+		expect(r.falsePositiveRate).toBe(0.5);
+		expect(r.falseNegativeRate).toBe(0);
+		expect(r.accuracy).toBe(0.75);
+		expect(r.passed).toBe(false);
+	});
+
+	it("staying silent on a real turn is a false negative scored over the should turns", () => {
+		// 3 should turns, 1 missed → FNR 1/3 ≈ 0.3333.
+		const r = scoreRespondDecision([
+			respond(true, true),
+			respond(false, true), // missed a real turn
+			respond(true, true),
+			respond(false, false),
+		]);
+		expect(r.falseNegativeRate).toBe(0.3333);
+		expect(r.falsePositiveRate).toBe(0);
+		expect(r.passed).toBe(false);
+	});
+
+	it("a high enough accuracy passes the default 0.9 gate", () => {
+		const samples = [
+			...Array.from({ length: 9 }, () => respond(true, true)),
+			respond(true, false), // one bystander answered
+		];
+		const r = scoreRespondDecision(samples);
+		expect(r.accuracy).toBe(0.9);
+		expect(r.passed).toBe(true);
+	});
+
+	it("empty sample set fails closed", () => {
+		const r = scoreRespondDecision([]);
+		expect(r.total).toBe(0);
+		expect(r.passed).toBe(false);
+	});
+});
+
+describe("scoreEntityExtraction (#9147)", () => {
+	it("exact match (case/whitespace-insensitive) → precision/recall/f1 = 1", () => {
+		const r = scoreEntityExtraction({
+			expected: ["Alice", "Bob"],
+			inferred: ["alice ", " BOB"],
+		});
+		expect(r.precision).toBe(1);
+		expect(r.recall).toBe(1);
+		expect(r.f1).toBe(1);
+		expect(r.passed).toBe(true);
+	});
+
+	it("an extra inferred entity lowers precision; a missed one lowers recall", () => {
+		const r = scoreEntityExtraction({
+			expected: ["alice", "bob"],
+			inferred: ["alice", "carol"],
+		});
+		// tp=1, precision 1/2=0.5, recall 1/2=0.5, f1=0.5 < 0.8.
+		expect(r.precision).toBe(0.5);
+		expect(r.recall).toBe(0.5);
+		expect(r.f1).toBe(0.5);
+		expect(r.passed).toBe(false);
+	});
+
+	it("both sides empty → vacuously perfect (precision/recall/f1 = 1, passes)", () => {
+		const r = scoreEntityExtraction({ expected: [], inferred: [] });
+		expect(r.precision).toBe(1);
+		expect(r.recall).toBe(1);
+		expect(r.f1).toBe(1);
+		expect(r.passed).toBe(true);
+	});
+
+	it("inferring entities when none were expected → precision 0, f1 0, fails", () => {
+		const r = scoreEntityExtraction({ expected: [], inferred: ["alice"] });
+		expect(r.precision).toBe(0);
+		expect(r.recall).toBe(1);
+		expect(r.f1).toBe(0);
+		expect(r.passed).toBe(false);
+	});
+
+	it("a custom minF1 admits a partial match", () => {
+		const r = scoreEntityExtraction(
+			{ expected: ["alice", "bob"], inferred: ["alice", "carol"] },
+			{ minF1: 0.5 },
+		);
+		expect(r.f1).toBe(0.5);
+		expect(r.passed).toBe(true);
+	});
+});
+
+describe("scoreVoiceEntityMatch (#9147)", () => {
+	it("every recognized voice resolves to the right entity → matchRate 1, passes", () => {
+		const r = scoreVoiceEntityMatch([
+			match("owner-1", "owner-1"),
+			match("guest-2", "guest-2"),
+		]);
+		expect(r.total).toBe(2);
+		expect(r.correct).toBe(2);
+		expect(r.matchRate).toBe(1);
+		expect(r.passed).toBe(true);
+	});
+
+	it("a misattribution and a null (unresolved) both count as misses", () => {
+		const r = scoreVoiceEntityMatch([
+			match("owner-1", "owner-1"),
+			match("guest-2", "owner-1"), // wrong entity
+			match(null, "owner-1"), // unresolved
+			match("owner-1", "owner-1"),
+		]);
+		expect(r.correct).toBe(2);
+		expect(r.matchRate).toBe(0.5);
+		expect(r.passed).toBe(false);
+	});
+
+	it("meets the default 0.9 gate at 9/10 correct", () => {
+		const samples = [
+			...Array.from({ length: 9 }, () => match("owner-1", "owner-1")),
+			match("guest-2", "owner-1"),
+		];
+		const r = scoreVoiceEntityMatch(samples);
+		expect(r.matchRate).toBe(0.9);
+		expect(r.passed).toBe(true);
+	});
+
+	it("empty sample set fails closed", () => {
+		const r = scoreVoiceEntityMatch([]);
+		expect(r.total).toBe(0);
+		expect(r.passed).toBe(false);
+	});
+});
+
+describe("summarizeVoiceE2e over the respond/EOT/entity scorers (#9147)", () => {
+	it("is passed only when every case passes", () => {
+		const ok = summarizeVoiceE2e([
+			scoreRespondDecision([respond(true, true), respond(false, false)]),
+			scoreEotDecision([eot(true, true, 10), eot(false, false)]),
+			scoreVoiceEntityMatch([match("a", "a")]),
+		]);
+		expect(ok.passed).toBe(true);
+		expect(ok.cases).toHaveLength(3);
+	});
+
+	it("one failing case fails the whole summary", () => {
+		const bad = summarizeVoiceE2e([
+			scoreRespondDecision([respond(true, true), respond(false, false)]),
+			scoreVoiceEntityMatch([match("a", "b")]), // miss → fails
+		]);
+		expect(bad.passed).toBe(false);
+	});
+});