@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/voice/e2e-harness.security-echo.test.ts

@@ -0,0 +1,103 @@
+import { describe, expect, it } from "vitest";
+import {
+	type EchoRejectionSample,
+	type OwnerSecuritySample,
+	scoreEchoRejection,
+	scoreOwnerSecurity,
+} from "./e2e-harness";
+
+// #9147 — self-echo rejection ("did the agent talk to itself?") and owner-vs-
+// impostor gating are two of the three heavy voice cases the issue calls out as
+// having scorers that "do not run anywhere that gates merges". These scorers are
+// pure + GGUF-independent, so pin their decision math here so the gate runs.
+
+const echo = (
+	isAgentEcho: boolean,
+	responded: boolean,
+): EchoRejectionSample => ({
+	isAgentEcho,
+	responded,
+});
+
+describe("scoreEchoRejection (#9147)", () => {
+	it("passes when every agent-echo turn is suppressed", () => {
+		const r = scoreEchoRejection([echo(true, false), echo(true, false)]);
+		expect(r.total).toBe(2);
+		expect(r.rejected).toBe(2);
+		expect(r.rejectionRate).toBe(1);
+		expect(r.passed).toBe(true);
+	});
+
+	it("scores ONLY agent-echo turns (real turns are ignored here)", () => {
+		// 2 echo (one leaked) + 2 real turns the agent answered (must not count).
+		const r = scoreEchoRejection([
+			echo(true, false),
+			echo(true, true),
+			echo(false, true),
+			echo(false, true),
+		]);
+		expect(r.total).toBe(2);
+		expect(r.rejected).toBe(1);
+		expect(r.rejectionRate).toBe(0.5);
+		expect(r.passed).toBe(false); // 0.5 < default 0.9 floor
+	});
+
+	it("fails closed when there are no echo samples to prove rejection", () => {
+		const r = scoreEchoRejection([echo(false, true)]);
+		expect(r.total).toBe(0);
+		expect(r.rejectionRate).toBe(0);
+		expect(r.passed).toBe(false);
+	});
+
+	it("honors a custom minRejectionRate floor", () => {
+		const samples = [echo(true, false), echo(true, false), echo(true, true)];
+		expect(scoreEchoRejection(samples).passed).toBe(false); // 0.6667 < 0.9
+		expect(scoreEchoRejection(samples, { minRejectionRate: 0.6 }).passed).toBe(
+			true,
+		); // 0.6667 >= 0.6
+	});
+});
+
+const sec = (
+	predictedOwner: boolean,
+	expectedOwner: boolean,
+): OwnerSecuritySample => ({
+	predictedOwner,
+	expectedOwner,
+});
+
+describe("scoreOwnerSecurity (#9147)", () => {
+	it("passes only when accuracy is high AND no impostor is accepted", () => {
+		const r = scoreOwnerSecurity([sec(true, true), sec(false, false)]);
+		expect(r.accuracy).toBe(1);
+		expect(r.impostorAcceptRate).toBe(0);
+		expect(r.passed).toBe(true);
+	});
+
+	it("FAILS on a single impostor-accept even at high overall accuracy", () => {
+		// 9 correct + 1 stranger let in as owner → accuracy 0.9 but a real breach.
+		const samples = [
+			...Array.from({ length: 9 }, () => sec(true, true)),
+			sec(true, false),
+		];
+		const r = scoreOwnerSecurity(samples);
+		expect(r.accuracy).toBe(0.9);
+		expect(r.impostorAcceptRate).toBeGreaterThan(0);
+		expect(r.passed).toBe(false); // default maxImpostorAcceptRate is 0
+	});
+
+	it("treats rejecting the real owner as friction, not a security failure", () => {
+		// owner rejected once (friction) but no impostor accepted.
+		const r = scoreOwnerSecurity(
+			[sec(false, true), sec(false, false), sec(false, false)],
+			{ minAccuracy: 0.6 },
+		);
+		expect(r.ownerRejectRate).toBe(1);
+		expect(r.impostorAcceptRate).toBe(0);
+		expect(r.passed).toBe(true); // 2/3 accuracy >= 0.6 floor, no impostor in
+	});
+
+	it("fails closed on an empty sample set", () => {
+		expect(scoreOwnerSecurity([]).passed).toBe(false);
+	});
+});

package/src/services/voice/e2e-harness.test.ts

@@ -0,0 +1,182 @@
+import { describe, expect, it } from "vitest";
+import {
+	assertRequiredVoiceArtifacts,
+	scoreBargeInInterruption,
+	scoreFirstResponseLatency,
+	scoreOptimisticRollbackRestart,
+	scorePauseContinuation,
+	scoreTtsAsrRoundTrip,
+	summarizeVoiceE2e,
+	VoiceE2eHarnessError,
+	wordErrorRate,
+} from "./e2e-harness";
+
+describe("voice E2E harness WER scoring", () => {
+	it("normalizes punctuation and computes word error rate", () => {
+		expect(wordErrorRate("Hello, local voice!", "hello local voice")).toBe(0);
+		expect(wordErrorRate("alpha beta gamma", "alpha gamma")).toBeCloseTo(
+			1 / 3,
+			4,
+		);
+	});
+
+	it("scores TTS -> ASR roundtrip against a WER threshold", () => {
+		const pass = scoreTtsAsrRoundTrip({
+			referenceText: "Eliza local voice smoke.",
+			hypothesisText: "eliza local voice smoke",
+			maxWer: 0,
+		});
+		expect(pass.passed).toBe(true);
+		expect(pass.wer).toBe(0);
+
+		const fail = scoreTtsAsrRoundTrip({
+			referenceText: "one two three four",
+			hypothesisText: "one four",
+			maxWer: 0.25,
+		});
+		expect(fail.passed).toBe(false);
+		expect(fail.wer).toBe(0.5);
+	});
+});
+
+describe("voice E2E harness artifact validation", () => {
+	it("fails clearly when a required model artifact is missing", () => {
+		expect(() =>
+			assertRequiredVoiceArtifacts(
+				[
+					{ kind: "bundle-root", path: "/models/eliza-1-2b.bundle" },
+					{
+						kind: "asr-model",
+						path: "/models/eliza-1-2b.bundle/asr/eliza-1-asr.gguf",
+						magic: "GGUF",
+					},
+				],
+				{
+					exists: (p) => p.endsWith(".bundle"),
+					size: () => null,
+					readMagic: () => null,
+				},
+			),
+		).toThrow(/asr-model.*not found/);
+	});
+
+	it("rejects a tiny or non-GGUF model instead of accepting a placeholder", () => {
+		try {
+			assertRequiredVoiceArtifacts(
+				[
+					{
+						kind: "tts-model",
+						path: "/tmp/placeholder.gguf",
+						minBytes: 1024,
+						magic: "GGUF",
+					},
+				],
+				{
+					exists: () => true,
+					size: () => 12,
+					readMagic: () => "NOPE",
+				},
+			);
+			throw new Error("expected artifact validation to fail");
+		} catch (err) {
+			expect(err).toBeInstanceOf(VoiceE2eHarnessError);
+			expect((err as VoiceE2eHarnessError).code).toBe("missing-artifact");
+			expect(String((err as Error).message)).toContain("too small");
+		}
+	});
+});
+
+describe("voice E2E harness barge-in scoring", () => {
+	it("passes when TTS, LLM, and audio drain cancel inside the budget", () => {
+		const result = scoreBargeInInterruption({
+			voiceDetectedAtMs: 1000,
+			ttsCancelledAtMs: 1060,
+			llmCancelledAtMs: 1100,
+			audioDrainedAtMs: 1030,
+			maxCancelMs: 250,
+		});
+		expect(result.passed).toBe(true);
+		expect(result.bargeInCancelMs).toBe(100);
+	});
+
+	it("does not pass a missing LLM cancel measurement by default", () => {
+		expect(() =>
+			scoreBargeInInterruption({
+				voiceDetectedAtMs: 1000,
+				ttsCancelledAtMs: 1030,
+			}),
+		).toThrow(/llmCancelledAtMs/);
+	});
+});
+
+describe("voice E2E harness pause and rollback scoring", () => {
+	it("scores user continuation within the 4s pause window", () => {
+		const result = scorePauseContinuation({
+			speechPauseAtMs: 1000,
+			speculativeStartedAtMs: 1200,
+			continuationAtMs: 4700,
+			speculativeAbortedAtMs: 4740,
+			finalRestartedAtMs: 4900,
+		});
+		expect(result.passed).toBe(true);
+		expect(result.continuationGapMs).toBe(3700);
+	});
+
+	it("fails when the partial response committed before the user continued", () => {
+		const result = scorePauseContinuation({
+			speechPauseAtMs: 1000,
+			continuationAtMs: 3000,
+			speculativeStartedAtMs: 1100,
+			committedBeforeContinuationAtMs: 2500,
+			speculativeAbortedAtMs: 3020,
+			finalRestartedAtMs: 3100,
+		});
+		expect(result.passed).toBe(false);
+	});
+
+	it("scores optimistic rollback restore and restart timing", () => {
+		const result = scoreOptimisticRollbackRestart({
+			speechPauseAtMs: 1000,
+			checkpointSavedAtMs: 1025,
+			speculativeStartedAtMs: 1030,
+			continuationAtMs: 1300,
+			speculativeAbortedAtMs: 1315,
+			checkpointRestoredAtMs: 1370,
+			restartedAtMs: 1460,
+		});
+		expect(result.passed).toBe(true);
+		expect(result.restoreAfterContinuationMs).toBe(70);
+		expect(result.restartAfterRestoreMs).toBe(90);
+	});
+});
+
+describe("voice E2E harness latency summary", () => {
+	it("scores first response latency from a real timestamp set", () => {
+		const result = scoreFirstResponseLatency({
+			turnStartedAtMs: 100,
+			asrFinalAtMs: 420,
+			llmFirstTokenAtMs: 700,
+			ttsFirstAudioAtMs: 980,
+			audioFirstPlayedAtMs: 1005,
+			maxFirstAudioMs: 1000,
+		});
+		expect(result.passed).toBe(true);
+		expect(result.firstAudioMs).toBe(880);
+		expect(result.firstPlayedMs).toBe(905);
+	});
+
+	it("summarizes all case pass/fail flags", () => {
+		const summary = summarizeVoiceE2e([
+			scoreTtsAsrRoundTrip({
+				referenceText: "hello",
+				hypothesisText: "hello",
+			}),
+			scoreFirstResponseLatency({
+				turnStartedAtMs: 0,
+				ttsFirstAudioAtMs: 400,
+			}),
+		]);
+		expect(summary.passed).toBe(true);
+		expect(summary.cases).toHaveLength(2);
+	});
+});