@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/voice/voice-duet.test.ts

@@ -0,0 +1,375 @@
+/**
+ * Wiring / cancel / shape test for the two-agents-talking-endlessly path the
+ * `voice:duet` harness drives — run headlessly with fake backends + the
+ * `DuetAudioBridge`:
+ *
+ *   agent A `replyText` → A's fake TTS → DuetSink (24 kHz → 16 kHz) → a ring
+ *   → B's `PushMicSource` → B's VAD/ASR → B's `generate` → B's `replyText` →
+ *   B's TTS → A's ring → … (3 round-trips).
+ *
+ * Assertions (UNCONDITIONAL — no real model, no native code):
+ *   (a) A's TTS PCM lands in B's ring (B's `PushMicSource` emits frames)
+ *   (b) B's VAD/transcriber see it → B's `generate` fires → B's reply PCM
+ *       lands in A's ring
+ *   (c) `--turns 3` runs without a deadlock — three A→B→A round-trips
+ *   (d) both latency tracers recorded ≥1 turn each, incl. the duet checkpoints
+ *       `peer-utterance-end` / `audio-first-into-peer-ring` and the headline
+ *       `ttftFromUtteranceEndMs` / `firstAudioIntoPeerRingFromUtteranceEndMs`
+ *   (e) the cross-ring stays bounded (the `DuetSink` is drained by the
+ *       `PushMicSource`'s re-framing — no unbounded growth)
+ *   (f) a cancel mid-`generate` (the producer's `AbortSignal`) stops the turn
+ *       and doesn't wedge the loop
+ *
+ * The *real-output* run (a real `eliza-1-2b` duet, gated behind the catalog
+ * +-fused-build probe) lives in `voice-duet.e2e.test.ts`.
+ */
+
+import { describe, expect, it } from "vitest";
+import {
+	DuetAudioBridge,
+	resampleLinear,
+} from "../../../../../packages/app-core/scripts/lib/duet-bridge.mjs";
+import { EndToEndLatencyTracer, type LatencyTrace } from "../latency-trace";
+import { parseExpressiveTags } from "./expressive-tags";
+import { PushMicSource } from "./mic-source";
+import type { VoiceGenerateRequest, VoiceTurnOutcome } from "./turn-controller";
+import type {
+	PcmFrame,
+	StreamingTranscriber,
+	TranscriberEventListener,
+	TranscriptUpdate,
+	VadEvent,
+	VadEventListener,
+	VadEventSource,
+} from "./types";
+
+const TTS_RATE = 24_000;
+const ASR_RATE = 16_000;
+
+/** A fake "TTS backend" for the wiring path: each `speak(text)` pushes a
+ *  deterministic burst of NON-ZERO 24 kHz PCM into a sink (the `DuetSink`).
+ *  (AGENTS.md §3 bans silent production fallbacks; a clearly test-only fake
+ *  that emits real PCM is fine here.) */
+class FakeTts {
+	constructor(
+		private readonly sink: { write(pcm: Float32Array, sr: number): void },
+	) {}
+	speak(text: string): number {
+		const words = text.trim().split(/\s+/).filter(Boolean);
+		const samples = Math.max(1, words.length) * Math.round(TTS_RATE * 0.12);
+		const pcm = new Float32Array(samples);
+		for (let i = 0; i < samples; i++) {
+			pcm[i] = 0.3 * Math.sin((2 * Math.PI * 220 * i) / TTS_RATE);
+		}
+		this.sink.write(pcm, TTS_RATE);
+		return samples;
+	}
+}
+
+class TestTranscriber implements StreamingTranscriber {
+	private readonly listeners = new Set<TranscriberEventListener>();
+	private fed = 0;
+	private partialEmitted = false;
+	private disposed = false;
+	constructor(private text: string) {}
+	setNext(text: string): void {
+		this.text = text;
+		this.partialEmitted = false;
+		this.fed = 0;
+	}
+	feed(_frame: PcmFrame): void {
+		if (this.disposed) return;
+		this.fed += 1;
+		if (!this.partialEmitted && this.fed >= 2) {
+			this.partialEmitted = true;
+			const prefix = this.text.split(/\s+/).slice(0, 2).join(" ");
+			const update: TranscriptUpdate = { partial: prefix, isFinal: false };
+			for (const l of this.listeners) l({ kind: "partial", update });
+			const words = prefix.split(/\s+/).filter(Boolean);
+			if (words.length > 0)
+				for (const l of this.listeners) l({ kind: "words", words });
+		}
+	}
+	async flush(): Promise<TranscriptUpdate> {
+		const update: TranscriptUpdate = { partial: this.text, isFinal: true };
+		for (const l of this.listeners) l({ kind: "final", update });
+		return update;
+	}
+	on(listener: TranscriberEventListener): () => void {
+		this.listeners.add(listener);
+		return () => this.listeners.delete(listener);
+	}
+	dispose(): void {
+		this.disposed = true;
+		this.listeners.clear();
+	}
+}
+
+class ScriptableVad implements VadEventSource {
+	private readonly listeners = new Set<VadEventListener>();
+	readonly seen: VadEvent[] = [];
+	onVadEvent(listener: VadEventListener): () => void {
+		this.listeners.add(listener);
+		return () => this.listeners.delete(listener);
+	}
+	emit(e: VadEvent): void {
+		this.seen.push(e);
+		for (const l of this.listeners) l(e);
+	}
+}
+
+const vadStart = (ms: number): VadEvent => ({
+	type: "speech-start",
+	timestampMs: ms,
+	probability: 0.9,
+});
+const vadActive = (ms: number, dur: number): VadEvent => ({
+	type: "speech-active",
+	timestampMs: ms,
+	probability: 0.9,
+	speechDurationMs: dur,
+});
+const vadEnd = (ms: number, dur: number): VadEvent => ({
+	type: "speech-end",
+	timestampMs: ms,
+	speechDurationMs: dur,
+});
+
+describe("voice:duet — wiring (fake backends + DuetAudioBridge)", () => {
+	it("resampleLinear: 24 kHz → 16 kHz keeps the 3:2 sample ratio; no-op when rates match", () => {
+		const a = new Float32Array(48_000);
+		for (let i = 0; i < a.length; i++) a[i] = Math.sin(i / 100);
+		expect(resampleLinear(a, TTS_RATE, ASR_RATE).length).toBe(32_000);
+		expect(resampleLinear(a, 16_000, 16_000)).toBe(a);
+	});
+
+	it("(a)(b)(c)(d)(e) A's TTS PCM crosses to B → B replies → 3 round-trips; both tracers record the duet checkpoints; cross-ring bounded", async () => {
+		const tracerA = new EndToEndLatencyTracer();
+		const tracerB = new EndToEndLatencyTracer();
+		const pushA = new PushMicSource({ sampleRate: ASR_RATE });
+		const pushB = new PushMicSource({ sampleRate: ASR_RATE });
+		let aToBSamples = 0;
+		let bToASamples = 0;
+		let pushBFrames = 0;
+		let pushAFrames = 0;
+		pushB.onFrame(() => {
+			pushBFrames += 1;
+		});
+		pushA.onFrame(() => {
+			pushAFrames += 1;
+		});
+		const bridge = new DuetAudioBridge({
+			micSourceA: pushA,
+			micSourceB: pushB,
+			opts: {
+				ringMs: 220,
+				targetRate: ASR_RATE,
+				onForward: (dir: "aToB" | "bToA", pcm: Float32Array) => {
+					if (dir === "aToB") aToBSamples += pcm.length;
+					else bToASamples += pcm.length;
+				},
+			},
+		});
+		await pushA.start();
+		await pushB.start();
+
+		const ttsA = new FakeTts(bridge.sinkForA());
+		const ttsB = new FakeTts(bridge.sinkForB());
+		const vadB = new ScriptableVad();
+		const transcriberB = new TestTranscriber("that's an interesting point");
+		const vadA = new ScriptableVad();
+		const transcriberA = new TestTranscriber("yeah and another thought");
+
+		let bTurns = 0;
+		const generateB = async (
+			request: VoiceGenerateRequest,
+		): Promise<VoiceTurnOutcome> => {
+			if (request.signal.aborted) {
+				const e = new Error("aborted");
+				e.name = "AbortError";
+				throw e;
+			}
+			bTurns += 1;
+			const reply = "[calm] yes — i was thinking the same thing";
+			const parsed = parseExpressiveTags(reply);
+			expect(parsed.dominantEmotion).toBe("calm");
+			ttsB.speak(reply);
+			return { transcript: request.transcript, replyText: reply };
+		};
+		let aTurns = 0;
+		const generateA = async (
+			request: VoiceGenerateRequest,
+		): Promise<VoiceTurnOutcome> => {
+			if (request.signal.aborted) {
+				const e = new Error("aborted");
+				e.name = "AbortError";
+				throw e;
+			}
+			aTurns += 1;
+			const reply = "okay so [excited] here's the next question";
+			ttsA.speak(reply);
+			return { transcript: request.transcript, replyText: reply };
+		};
+
+		const runConsumerTurn = async (args: {
+			vad: ScriptableVad;
+			transcriber: TestTranscriber;
+			myTracer: EndToEndLatencyTracer;
+			roomId: string;
+			generate: (r: VoiceGenerateRequest) => Promise<VoiceTurnOutcome>;
+		}): Promise<{ outcome: VoiceTurnOutcome; trace: LatencyTrace | null }> => {
+			const turnId = args.myTracer.beginTurn({ roomId: args.roomId });
+			// peer-utterance-end = the producer drained (synchronous for the fake).
+			args.myTracer.mark(turnId, "peer-utterance-end");
+			args.vad.emit(vadStart(0));
+			for (let i = 0; i < 4; i++) {
+				args.transcriber.feed({
+					pcm: new Float32Array(512),
+					sampleRate: ASR_RATE,
+					timestampMs: i * 32,
+				});
+				args.vad.emit(vadActive(40 + i * 30, 40 + i * 30));
+			}
+			args.vad.emit(vadEnd(200, 200));
+			const final = await args.transcriber.flush();
+			args.myTracer.mark(turnId, "vad-trigger");
+			args.myTracer.mark(turnId, "asr-final");
+			args.myTracer.mark(turnId, "llm-first-token");
+			const outcome = await args.generate({
+				transcript: final.partial,
+				final: true,
+				signal: new AbortController().signal,
+			});
+			args.myTracer.mark(turnId, "llm-first-replytext-char");
+			const parsed = parseExpressiveTags(outcome.replyText);
+			if (parsed.hasTags)
+				args.myTracer.mark(turnId, "replyText-first-emotion-tag");
+			args.myTracer.mark(turnId, "phrase-1-to-tts");
+			args.myTracer.mark(turnId, "tts-first-audio-chunk");
+			args.myTracer.mark(turnId, "audio-first-into-peer-ring");
+			const trace = args.myTracer.endTurn(turnId);
+			return { outcome, trace };
+		};
+
+		// ── Round-trip 1 starts with A's seed turn (no incoming PCM) ──────────
+		const seedTurnId = tracerA.beginTurn({ roomId: "duet-A" });
+		tracerA.mark(seedTurnId, "vad-trigger");
+		tracerA.mark(seedTurnId, "asr-final");
+		tracerA.mark(seedTurnId, "llm-first-token");
+		await generateA({
+			transcript: "hey what's the most interesting thing you've thought about",
+			final: true,
+			signal: new AbortController().signal,
+		});
+		tracerA.mark(seedTurnId, "tts-first-audio-chunk");
+		tracerA.endTurn(seedTurnId);
+		// (a) A's TTS PCM is in B's ring now.
+		expect(aToBSamples).toBeGreaterThan(0);
+		expect(pushBFrames).toBeGreaterThan(0);
+
+		for (let rt = 1; rt <= 3; rt++) {
+			transcriberB.setNext(`turn ${rt}: that's an interesting point`);
+			transcriberA.setNext(`turn ${rt}: yeah and another thought`);
+			const framesBeforeBReply = pushAFrames;
+			const bResult = await runConsumerTurn({
+				vad: vadB,
+				transcriber: transcriberB,
+				myTracer: tracerB,
+				roomId: "duet-B",
+				generate: generateB,
+			});
+			// (b) B's reply PCM landed in A's ring.
+			expect(bToASamples).toBeGreaterThan(0);
+			expect(pushAFrames).toBeGreaterThan(framesBeforeBReply);
+			expect(bResult.trace?.derived.ttftFromUtteranceEndMs).not.toBeNull();
+			expect(
+				bResult.trace?.derived.firstAudioIntoPeerRingFromUtteranceEndMs,
+			).not.toBeNull();
+			expect(bResult.trace?.derived.emotionTagOverheadMs).not.toBeNull();
+			// A hears B's reply → A's turn.
+			const aResult = await runConsumerTurn({
+				vad: vadA,
+				transcriber: transcriberA,
+				myTracer: tracerA,
+				roomId: "duet-A",
+				generate: generateA,
+			});
+			expect(aResult.outcome.replyText.length).toBeGreaterThan(0);
+		}
+
+		// (c) three round-trips, no deadlock.
+		expect(bTurns).toBe(3);
+		expect(aTurns).toBe(1 /* seed */ + 3 /* responses */);
+		// (d) both tracers have the duet checkpoints + the headline histograms.
+		expect(tracerB.recentTraces().length).toBeGreaterThanOrEqual(3);
+		expect(tracerA.recentTraces().length).toBeGreaterThanOrEqual(4);
+		const someB = tracerB.recentTraces()[0] as LatencyTrace;
+		expect(someB.checkpoints.map((c) => c.name)).toContain(
+			"peer-utterance-end",
+		);
+		expect(someB.checkpoints.map((c) => c.name)).toContain(
+			"audio-first-into-peer-ring",
+		);
+		expect(
+			tracerB.histogramSummaries().ttftFromUtteranceEndMs.count,
+		).toBeGreaterThanOrEqual(3);
+		expect(
+			tracerB.histogramSummaries().firstAudioIntoPeerRingFromUtteranceEndMs
+				.count,
+		).toBeGreaterThanOrEqual(3);
+		// (e) cross-ring bounded — the DuetSink forwarded everything (the
+		// PushMicSource re-frames and drains it; residual < one frame).
+		expect(bridge.aToB.totalForwarded()).toBeGreaterThan(0);
+		expect(bridge.bToA.totalForwarded()).toBeGreaterThan(0);
+		await pushA.stop();
+		await pushB.stop();
+	});
+
+	it("(f) a cancel mid-generate (the producer's AbortSignal) stops the turn and doesn't wedge the loop", async () => {
+		const sinkChunks: Array<{ pcm: Float32Array; sr: number }> = [];
+		const sink = {
+			write: (pcm: Float32Array, sr: number) => sinkChunks.push({ pcm, sr }),
+		};
+		const tts = new FakeTts(sink);
+		const ctrl = new AbortController();
+		let threw = false;
+		const generate = async (
+			request: VoiceGenerateRequest,
+		): Promise<VoiceTurnOutcome> => {
+			const words = "this reply will be cancelled before it finishes".split(
+				" ",
+			);
+			for (let i = 0; i < words.length; i++) {
+				if (request.signal.aborted) {
+					threw = true;
+					const e = new Error("aborted");
+					e.name = "AbortError";
+					throw e;
+				}
+				tts.speak(words[i]);
+				if (i === 1) ctrl.abort();
+				await new Promise((r) => setTimeout(r, 1));
+			}
+			return { transcript: request.transcript, replyText: words.join(" ") };
+		};
+		let caught = false;
+		try {
+			await generate({ transcript: "x", final: true, signal: ctrl.signal });
+		} catch (e) {
+			caught = (e as Error).name === "AbortError";
+		}
+		expect(threw).toBe(true);
+		expect(caught).toBe(true);
+		expect(sinkChunks.length).toBeGreaterThan(0);
+		// A subsequent turn with a fresh (un-aborted) signal runs to completion —
+		// the loop is not wedged after the cancel.
+		const ctrl2 = new AbortController();
+		const r = await generate({
+			transcript: "y",
+			final: true,
+			signal: ctrl2.signal,
+		});
+		expect(typeof r.replyText).toBe("string");
+		expect(r.transcript).toBe("y");
+	});
+});

package/src/services/voice/voice-emotion-classifier.test.ts

@@ -0,0 +1,210 @@
+/**
+ * Pure-function tests for voice-emotion-classifier.ts.
+ *
+ * Covers the V-A-D → `ExpressiveEmotion` projection table and the 7-class
+ * logit interpreter. The ONNX-backed `VoiceEmotionClassifier` class was
+ * removed when `onnxruntime-node` was dropped; only these pure projection
+ * helpers remain.
+ */
+
+import { describe, expect, it } from "vitest";
+import {
+	interpretCls7Output,
+	projectVadToExpressiveEmotion,
+	VoiceEmotionClassifierError,
+	WAV2SMALL_INT8_MODEL_ID,
+	WAV2SMALL_MIN_SAMPLES,
+	WAV2SMALL_SAMPLE_RATE,
+} from "./voice-emotion-classifier";
+
+describe("projectVadToExpressiveEmotion", () => {
+	it("projects neutral V-A-D centre to a null discrete label", () => {
+		const out = projectVadToExpressiveEmotion({
+			valence: 0.5,
+			arousal: 0.5,
+			dominance: 0.5,
+		});
+		// At the centre the strongest mass is ≤ the 0.35 threshold so we abstain.
+		expect(out.emotion).toBeNull();
+		expect(out.confidence).toBeLessThan(0.35);
+		// Every class has a finite score.
+		for (const tag of [
+			"happy",
+			"sad",
+			"angry",
+			"nervous",
+			"calm",
+			"excited",
+			"whisper",
+		] as const) {
+			expect(out.scores[tag]).toBeGreaterThanOrEqual(0);
+			expect(out.scores[tag]).toBeLessThanOrEqual(1);
+		}
+	});
+
+	it("projects high valence + high arousal to excited or happy", () => {
+		const out = projectVadToExpressiveEmotion({
+			valence: 0.9,
+			arousal: 0.9,
+			dominance: 0.55,
+		});
+		expect(["excited", "happy"]).toContain(out.emotion);
+		expect(out.confidence).toBeGreaterThanOrEqual(0.5);
+	});
+
+	it("projects low valence + low arousal to sad", () => {
+		const out = projectVadToExpressiveEmotion({
+			valence: 0.1,
+			arousal: 0.1,
+			dominance: 0.2,
+		});
+		expect(out.emotion).toBe("sad");
+		expect(out.confidence).toBeGreaterThanOrEqual(0.5);
+	});
+
+	it("projects low valence + high arousal + high dominance to angry", () => {
+		const out = projectVadToExpressiveEmotion({
+			valence: 0.1,
+			arousal: 0.9,
+			dominance: 0.9,
+		});
+		expect(out.emotion).toBe("angry");
+		expect(out.confidence).toBeGreaterThanOrEqual(0.5);
+	});
+
+	it("projects high valence + low arousal to calm", () => {
+		const out = projectVadToExpressiveEmotion({
+			valence: 0.85,
+			arousal: 0.15,
+			dominance: 0.5,
+		});
+		expect(out.emotion).toBe("calm");
+		expect(out.confidence).toBeGreaterThanOrEqual(0.5);
+	});
+
+	it("projects low arousal + low dominance to whisper", () => {
+		const out = projectVadToExpressiveEmotion({
+			valence: 0.5,
+			arousal: 0.05,
+			dominance: 0.05,
+		});
+		expect(out.emotion).toBe("whisper");
+		expect(out.confidence).toBeGreaterThanOrEqual(0.5);
+	});
+
+	it("clamps inputs outside [0, 1]", () => {
+		const out = projectVadToExpressiveEmotion({
+			valence: 2,
+			arousal: -1,
+			dominance: 0.5,
+		});
+		// Sanitised: V=1, A=0, D=0.5 → calm-ish (high V, low A).
+		expect(out.emotion).toBe("calm");
+	});
+
+	it("treats non-finite inputs as zero", () => {
+		const out = projectVadToExpressiveEmotion({
+			valence: Number.NaN,
+			arousal: Number.POSITIVE_INFINITY,
+			dominance: Number.NEGATIVE_INFINITY,
+		});
+		// All three clamped → no axis pushes any class above 0.35.
+		expect(out.emotion).toBeNull();
+		expect(Number.isFinite(out.confidence)).toBe(true);
+	});
+});
+
+describe("interpretCls7Output (cls7 head)", () => {
+	it("picks the argmax class with calibrated softmax confidence", () => {
+		// Logits aligned with EXPRESSIVE_EMOTION_TAGS:
+		//   ["happy","sad","angry","nervous","calm","excited","whisper"]
+		const logits = new Float32Array([0, 0, 5, 0, 0, 0, 0]); // strong angry
+		const out = interpretCls7Output(logits, WAV2SMALL_INT8_MODEL_ID, 7);
+		expect(out.emotion).toBe("angry");
+		expect(out.confidence).toBeGreaterThan(0.9);
+		// All 7 score keys present, all probabilities in [0, 1], sum ≈ 1.
+		const tags = [
+			"happy",
+			"sad",
+			"angry",
+			"nervous",
+			"calm",
+			"excited",
+			"whisper",
+		] as const;
+		let total = 0;
+		for (const tag of tags) {
+			expect(out.scores[tag]).toBeGreaterThanOrEqual(0);
+			expect(out.scores[tag]).toBeLessThanOrEqual(1);
+			total += out.scores[tag];
+		}
+		expect(total).toBeGreaterThan(0.999);
+		expect(total).toBeLessThan(1.001);
+		// V-A-D is the neutral midpoint for cls7 (no V-A-D head).
+		expect(out.vad.valence).toBeCloseTo(0.5, 5);
+		expect(out.vad.arousal).toBeCloseTo(0.5, 5);
+		expect(out.vad.dominance).toBeCloseTo(0.5, 5);
+		expect(out.modelId).toBe(WAV2SMALL_INT8_MODEL_ID);
+		expect(out.latencyMs).toBe(7);
+	});
+
+	it("equal logits return a near-uniform distribution", () => {
+		const logits = new Float32Array([1, 1, 1, 1, 1, 1, 1]);
+		const out = interpretCls7Output(logits, WAV2SMALL_INT8_MODEL_ID, 0);
+		// First-index wins on ties (happy at index 0).
+		expect(out.emotion).toBe("happy");
+		// Uniform distribution gives every class probability ≈ 1/7.
+		expect(out.confidence).toBeCloseTo(1 / 7, 5);
+	});
+
+	it("falls back to abstain when all logits are non-finite", () => {
+		const logits = new Float32Array([
+			Number.NaN,
+			Number.NaN,
+			Number.NaN,
+			Number.NaN,
+			Number.NaN,
+			Number.NaN,
+			Number.NaN,
+		]);
+		const out = interpretCls7Output(logits, WAV2SMALL_INT8_MODEL_ID, 0);
+		expect(out.emotion).toBeNull();
+		expect(out.confidence).toBe(0);
+	});
+
+	it("rejects logit arrays of the wrong length", () => {
+		const wrong = new Float32Array([0, 0, 0]); // V-A-D-sized
+		expect(() =>
+			interpretCls7Output(wrong, WAV2SMALL_INT8_MODEL_ID, 0),
+		).toThrow(VoiceEmotionClassifierError);
+	});
+
+	it("respects the EXPRESSIVE_EMOTION_TAGS class order", () => {
+		// Pick the highest logit at each index and confirm we get that tag back.
+		const order = [
+			"happy",
+			"sad",
+			"angry",
+			"nervous",
+			"calm",
+			"excited",
+			"whisper",
+		] as const;
+		for (let i = 0; i < order.length; i++) {
+			const logits = new Float32Array([0, 0, 0, 0, 0, 0, 0]);
+			logits[i] = 5;
+			const out = interpretCls7Output(logits, WAV2SMALL_INT8_MODEL_ID, 0);
+			expect(out.emotion).toBe(order[i]);
+		}
+	});
+});
+
+describe("model id constants", () => {
+	it("WAV2SMALL_INT8_MODEL_ID is the expected stable string", () => {
+		expect(WAV2SMALL_INT8_MODEL_ID).toBe("wav2small-msp-dim-int8");
+	});
+
+	it("WAV2SMALL_MIN_SAMPLES is at least one second at 16 kHz", () => {
+		expect(WAV2SMALL_MIN_SAMPLES).toBeGreaterThanOrEqual(WAV2SMALL_SAMPLE_RATE);
+	});
+});