@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.2

package/src/services/voice/turn-controller.test.ts

@@ -0,0 +1,575 @@
+/**
+ * `VoiceTurnController` tests — turn-taking above the scheduler.
+ *
+ * Drives the controller with a hand-cranked fake `VadEvent` stream + a fake
+ * `StreamingTranscriber` and asserts the brief's A4/A5 contract:
+ *   - speech-start → `prewarm()` fires
+ *   - speech-pause(ms ≥ threshold) → a SPECULATIVE generate kicks off the
+ *     current partial transcript with an `AbortSignal`
+ *   - speech-active again → the speculative generate is ABORTED
+ *   - speech-end + matching final transcript → the speculative is PROMOTED
+ *   - speech-end + diverged final transcript → speculative discarded, a
+ *     fresh FINAL generate runs
+ *   - the transcriber's `words` event → `bargeIn.onWordsDetected`
+ *     (provisional `pause-tts` → `hard-stop`)
+ */
+
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { StubOmniVoiceBackend } from "./engine-bridge";
+import type { EotClassifier } from "./eot-classifier";
+import { InMemoryAudioSink } from "./ring-buffer";
+import { VoiceScheduler } from "./scheduler";
+import {
+	type VoiceGenerateRequest,
+	VoiceTurnController,
+	type VoiceTurnOutcome,
+} from "./turn-controller";
+import type {
+	SpeakerPreset,
+	StreamingTranscriber,
+	TranscriberEvent,
+	TranscriberEventListener,
+	TranscriptUpdate,
+	VadEvent,
+	VadEventListener,
+	VoiceInputSource,
+	VoiceSegment,
+	VoiceSpeaker,
+	VoiceTurnMetadata,
+} from "./types";
+
+function makePreset(): SpeakerPreset {
+	const embedding = new Float32Array([0.1, 0.2, 0.3]);
+	return {
+		voiceId: "default",
+		embedding,
+		bytes: new Uint8Array(embedding.buffer.slice(0)),
+	};
+}
+
+function makeScheduler(): VoiceScheduler {
+	return new VoiceScheduler(
+		{
+			chunkerConfig: { maxTokensPerPhrase: 30 },
+			preset: makePreset(),
+			ringBufferCapacity: 4096,
+			sampleRate: 24000,
+		},
+		{ backend: new StubOmniVoiceBackend(24000), sink: new InMemoryAudioSink() },
+	);
+}
+
+class FakeVad {
+	private readonly listeners = new Set<VadEventListener>();
+	onVadEvent(listener: VadEventListener): () => void {
+		this.listeners.add(listener);
+		return () => this.listeners.delete(listener);
+	}
+	emit(event: VadEvent): void {
+		for (const l of this.listeners) l(event);
+	}
+}
+
+class FakeTranscriber implements StreamingTranscriber {
+	private readonly listeners = new Set<TranscriberEventListener>();
+	partial = "";
+	finalText = "";
+	finalSource: VoiceInputSource | undefined;
+	finalSpeaker: VoiceSpeaker | undefined;
+	finalSegments: VoiceSegment[] | undefined;
+	finalTurn: VoiceTurnMetadata | undefined;
+	flushCalls = 0;
+	feed(): void {}
+	async flush(): Promise<TranscriptUpdate> {
+		this.flushCalls++;
+		return {
+			partial: this.finalText,
+			isFinal: true,
+			...(this.finalSource ? { source: this.finalSource } : {}),
+			...(this.finalSpeaker ? { speaker: this.finalSpeaker } : {}),
+			...(this.finalSegments ? { segments: this.finalSegments } : {}),
+			...(this.finalTurn ? { turn: this.finalTurn } : {}),
+		};
+	}
+	on(listener: TranscriberEventListener): () => void {
+		this.listeners.add(listener);
+		return () => this.listeners.delete(listener);
+	}
+	dispose(): void {}
+	emit(event: TranscriberEvent): void {
+		for (const l of this.listeners) l(event);
+	}
+	setPartial(text: string, source?: VoiceInputSource): void {
+		this.partial = text;
+		this.emit({
+			kind: "partial",
+			update: {
+				partial: text,
+				isFinal: false,
+				...(source ? { source } : {}),
+			},
+		});
+	}
+}
+
+let ts = 0;
+function vadEvent(
+	event: Partial<VadEvent> & { type: VadEvent["type"] },
+): VadEvent {
+	ts += 100;
+	switch (event.type) {
+		case "speech-start":
+			return { type: "speech-start", timestampMs: ts, probability: 0.9 };
+		case "speech-active":
+			return {
+				type: "speech-active",
+				timestampMs: ts,
+				probability: 0.9,
+				speechDurationMs: 500,
+			};
+		case "speech-pause":
+			return {
+				type: "speech-pause",
+				timestampMs: ts,
+				pauseDurationMs:
+					(event as { pauseDurationMs?: number }).pauseDurationMs ?? 400,
+			};
+		case "speech-end":
+			return { type: "speech-end", timestampMs: ts, speechDurationMs: 1000 };
+		case "blip":
+			return { type: "blip", timestampMs: ts, durationMs: 30, peakRms: 0.2 };
+	}
+}
+
+interface Harness {
+	vad: FakeVad;
+	transcriber: FakeTranscriber;
+	scheduler: VoiceScheduler;
+	controller: VoiceTurnController;
+	prewarm: ReturnType<typeof vi.fn>;
+	generateCalls: VoiceGenerateRequest[];
+	/** Resolve the n-th pending generate (0-based) with the given reply. */
+	resolveGenerate(index: number, replyText: string): void;
+	events: {
+		speculativeStart: string[];
+		speculativeAbort: number;
+		speculativePromoted: VoiceTurnOutcome[];
+		turnComplete: VoiceTurnOutcome[];
+		turnSuppressed: Array<{
+			transcript: string;
+			probability: number;
+		}>;
+		errors: Error[];
+	};
+}
+
+function makeHarness(
+	opts: { speculatePauseMs?: number; turnDetector?: EotClassifier } = {},
+): Harness {
+	const vad = new FakeVad();
+	const transcriber = new FakeTranscriber();
+	const scheduler = makeScheduler();
+	const prewarm = vi.fn(async () => {});
+	const generateCalls: VoiceGenerateRequest[] = [];
+	const pending: Array<(o: VoiceTurnOutcome) => void> = [];
+	const events: Harness["events"] = {
+		speculativeStart: [],
+		speculativeAbort: 0,
+		speculativePromoted: [],
+		turnComplete: [],
+		turnSuppressed: [],
+		errors: [],
+	};
+	const controller = new VoiceTurnController(
+		{
+			vad,
+			transcriber,
+			scheduler,
+			prewarm,
+			...(opts.turnDetector ? { turnDetector: opts.turnDetector } : {}),
+			generate: (request) => {
+				generateCalls.push(request);
+				return new Promise<VoiceTurnOutcome>((resolve, reject) => {
+					// Reject if the request is aborted before it's resolved.
+					request.signal.addEventListener(
+						"abort",
+						() =>
+							reject(
+								Object.assign(new Error("aborted"), { name: "AbortError" }),
+							),
+						{ once: true },
+					);
+					pending.push((o) => resolve(o));
+				});
+			},
+		},
+		{
+			roomId: "room-1",
+			...(opts.speculatePauseMs !== undefined
+				? { speculatePauseMs: opts.speculatePauseMs }
+				: {}),
+		},
+		{
+			onSpeculativeStart: (t) => events.speculativeStart.push(t),
+			onSpeculativeAbort: () => {
+				events.speculativeAbort++;
+			},
+			onSpeculativePromoted: (o) => events.speculativePromoted.push(o),
+			onTurnComplete: (o) => events.turnComplete.push(o),
+			onTurnSuppressed: (transcript, signal) =>
+				events.turnSuppressed.push({
+					transcript,
+					probability: signal.endOfTurnProbability,
+				}),
+			onError: (e) => events.errors.push(e),
+		},
+	);
+	return {
+		vad,
+		transcriber,
+		scheduler,
+		controller,
+		prewarm,
+		generateCalls,
+		resolveGenerate(index, replyText) {
+			const transcript = generateCalls[index]?.transcript ?? "";
+			pending[index]?.({ transcript, replyText });
+		},
+		events,
+	};
+}
+
+describe("VoiceTurnController", () => {
+	beforeEach(() => {
+		ts = 0;
+	});
+	afterEach(() => {
+		vi.restoreAllMocks();
+	});
+
+	it("fires prewarm on speech-start, before STT finishes", () => {
+		const h = makeHarness();
+		h.controller.start();
+		h.vad.emit(vadEvent({ type: "speech-start" }));
+		expect(h.prewarm).toHaveBeenCalledWith("room-1");
+	});
+
+	it("kicks a speculative generate on a long-enough speech-pause", () => {
+		const h = makeHarness({ speculatePauseMs: 300 });
+		h.controller.start();
+		h.vad.emit(vadEvent({ type: "speech-start" }));
+		h.transcriber.setPartial("turn on the lights");
+		// Too-short pause — no speculation.
+		h.vad.emit(vadEvent({ type: "speech-pause", pauseDurationMs: 200 }));
+		expect(h.generateCalls).toHaveLength(0);
+		// Long-enough pause — speculation kicks off the current partial.
+		h.vad.emit(vadEvent({ type: "speech-pause", pauseDurationMs: 400 }));
+		expect(h.generateCalls).toHaveLength(1);
+		expect(h.generateCalls[0]).toMatchObject({
+			transcript: "turn on the lights",
+			final: false,
+		});
+		expect(h.events.speculativeStart).toEqual(["turn on the lights"]);
+	});
+
+	it("attaches semantic turn signal to speculative generate requests", async () => {
+		const h = makeHarness({
+			speculatePauseMs: 300,
+			turnDetector: { score: async () => 0.95 },
+		});
+		h.controller.start();
+		h.vad.emit(vadEvent({ type: "speech-start" }));
+		h.transcriber.setPartial("turn on the lights");
+		h.vad.emit(vadEvent({ type: "speech-pause", pauseDurationMs: 400 }));
+		await new Promise((r) => setTimeout(r, 0));
+		expect(h.generateCalls).toHaveLength(1);
+		expect(h.generateCalls[0].turnSignal).toMatchObject({
+			endOfTurnProbability: 0.95,
+			nextSpeaker: "agent",
+			agentShouldSpeak: true,
+		});
+	});
+
+	it("suppresses speculative generation when turn detector says user continues", async () => {
+		const h = makeHarness({
+			speculatePauseMs: 300,
+			turnDetector: { score: async () => 0.1 },
+		});
+		h.controller.start();
+		h.vad.emit(vadEvent({ type: "speech-start" }));
+		h.transcriber.setPartial("can you check the");
+		h.vad.emit(vadEvent({ type: "speech-pause", pauseDurationMs: 400 }));
+		await new Promise((r) => setTimeout(r, 0));
+		expect(h.generateCalls).toHaveLength(0);
+		expect(h.events.turnSuppressed).toEqual([
+			{ transcript: "can you check the", probability: 0.1 },
+		]);
+	});
+
+	it("aborts the speculative generate when speech resumes (speech-active)", () => {
+		const h = makeHarness({ speculatePauseMs: 300 });
+		h.controller.start();
+		h.vad.emit(vadEvent({ type: "speech-start" }));
+		h.transcriber.setPartial("what is");
+		h.vad.emit(vadEvent({ type: "speech-pause", pauseDurationMs: 400 }));
+		expect(h.generateCalls).toHaveLength(1);
+		expect(h.generateCalls[0].signal.aborted).toBe(false);
+		h.vad.emit(vadEvent({ type: "speech-active" }));
+		expect(h.generateCalls[0].signal.aborted).toBe(true);
+		expect(h.events.speculativeAbort).toBe(1);
+	});
+
+	it("promotes the speculative result on speech-end when it matches the final transcript", async () => {
+		const h = makeHarness({ speculatePauseMs: 300 });
+		h.controller.start();
+		h.vad.emit(vadEvent({ type: "speech-start" }));
+		h.transcriber.setPartial("hello there");
+		h.vad.emit(vadEvent({ type: "speech-pause", pauseDurationMs: 400 }));
+		expect(h.generateCalls).toHaveLength(1);
+		// The speculative produces a reply...
+		h.resolveGenerate(0, "Hi! How can I help?");
+		// ...and the segment ends with the SAME transcript the speculation used.
+		h.transcriber.finalText = "hello there";
+		h.vad.emit(vadEvent({ type: "speech-end" }));
+		// Let the finalize promise (flush + await speculative) settle.
+		await new Promise((r) => setTimeout(r, 0));
+		expect(h.transcriber.flushCalls).toBe(1);
+		// No second generate — the speculative was promoted.
+		expect(h.generateCalls).toHaveLength(1);
+		expect(h.events.speculativePromoted).toHaveLength(1);
+		expect(h.events.speculativePromoted[0].replyText).toBe(
+			"Hi! How can I help?",
+		);
+		expect(h.events.turnComplete).toHaveLength(1);
+	});
+
+	it("discards a stale speculative and runs a fresh final turn when the transcript diverged", async () => {
+		const h = makeHarness({ speculatePauseMs: 300 });
+		h.controller.start();
+		h.vad.emit(vadEvent({ type: "speech-start" }));
+		h.transcriber.setPartial("turn on");
+		h.vad.emit(vadEvent({ type: "speech-pause", pauseDurationMs: 400 }));
+		expect(h.generateCalls).toHaveLength(1);
+		h.resolveGenerate(0, "(speculative reply)");
+		// The full utterance was actually longer.
+		h.transcriber.finalText = "turn on the kitchen lights";
+		h.vad.emit(vadEvent({ type: "speech-end" }));
+		await new Promise((r) => setTimeout(r, 0));
+		// The speculative was aborted/discarded; a fresh FINAL generate ran on
+		// the finalized transcript.
+		expect(h.generateCalls).toHaveLength(2);
+		expect(h.generateCalls[1]).toMatchObject({
+			transcript: "turn on the kitchen lights",
+			final: true,
+		});
+		h.resolveGenerate(1, "Turning on the kitchen lights.");
+		await new Promise((r) => setTimeout(r, 0));
+		expect(h.events.speculativeAbort).toBe(1);
+		expect(h.events.speculativePromoted).toHaveLength(0);
+		expect(h.events.turnComplete).toHaveLength(1);
+		expect(h.events.turnComplete[0].replyText).toBe(
+			"Turning on the kitchen lights.",
+		);
+	});
+
+	it("runs a final turn directly when no speculation happened", async () => {
+		const h = makeHarness();
+		h.controller.start();
+		h.vad.emit(vadEvent({ type: "speech-start" }));
+		h.transcriber.setPartial("");
+		h.transcriber.finalText = "good morning";
+		h.vad.emit(vadEvent({ type: "speech-end" }));
+		await new Promise((r) => setTimeout(r, 0));
+		expect(h.generateCalls).toHaveLength(1);
+		expect(h.generateCalls[0]).toMatchObject({
+			transcript: "good morning",
+			final: true,
+		});
+	});
+
+	it("suppresses final generation when semantic turn detector says the next speaker is user", async () => {
+		const h = makeHarness({
+			turnDetector: { score: async () => 0.15 },
+		});
+		h.controller.start();
+		h.vad.emit(vadEvent({ type: "speech-start" }));
+		h.transcriber.finalText = "can you look at the";
+		h.vad.emit(vadEvent({ type: "speech-end" }));
+		await new Promise((r) => setTimeout(r, 0));
+		expect(h.generateCalls).toHaveLength(0);
+		expect(h.events.turnSuppressed).toEqual([
+			{ transcript: "can you look at the", probability: 0.15 },
+		]);
+	});
+
+	it("passes transcript source metadata into speculative and final generate requests", async () => {
+		const h = makeHarness({ speculatePauseMs: 300 });
+		const source: VoiceInputSource = {
+			kind: "local_mic",
+			deviceId: "default-input",
+			roomId: "room-1",
+		};
+		h.controller.start();
+		h.vad.emit(vadEvent({ type: "speech-start" }));
+		h.transcriber.setPartial("who is speaking", source);
+		h.vad.emit(vadEvent({ type: "speech-pause", pauseDurationMs: 400 }));
+		expect(h.generateCalls[0]).toMatchObject({
+			transcript: "who is speaking",
+			final: false,
+			source,
+		});
+
+		h.resolveGenerate(0, "(speculative reply)");
+		h.transcriber.finalText = "who is speaking now";
+		h.transcriber.finalSource = source;
+		h.vad.emit(vadEvent({ type: "speech-end" }));
+		await new Promise((r) => setTimeout(r, 0));
+		expect(h.generateCalls[1]).toMatchObject({
+			transcript: "who is speaking now",
+			final: true,
+			source,
+		});
+	});
+
+	it("passes speaker attribution and diarized segments into final generate requests", async () => {
+		const h = makeHarness();
+		const source: VoiceInputSource = {
+			kind: "local_mic",
+			deviceId: "default-input",
+			roomId: "room-1",
+		};
+		const speaker: VoiceSpeaker = {
+			id: "entity-owner",
+			label: "Owner",
+			displayName: "Owner",
+			source,
+			imprintClusterId: "cluster-owner",
+			imprintObservationId: "observation-owner-1",
+			entityId: "entity-owner",
+			confidence: 0.91,
+			isLocalUser: true,
+		};
+		const segments: VoiceSegment[] = [
+			{
+				id: "seg-1",
+				text: "this is the owner",
+				startMs: 120,
+				endMs: 1480,
+				speaker,
+				source,
+				confidence: 0.89,
+			},
+		];
+		const turn: VoiceTurnMetadata = {
+			turnId: "turn-1",
+			source,
+			primarySpeaker: speaker,
+			segments,
+			startedAtMs: 120,
+			endedAtMs: 1480,
+			diarization: {
+				provider: "local",
+				model: "eliza-voice-imprint-v1",
+				confidence: 0.89,
+			},
+		};
+
+		h.controller.start();
+		h.vad.emit(vadEvent({ type: "speech-start" }));
+		h.transcriber.finalText = "this is the owner";
+		h.transcriber.finalSource = source;
+		h.transcriber.finalSpeaker = speaker;
+		h.transcriber.finalSegments = segments;
+		h.transcriber.finalTurn = turn;
+		h.vad.emit(vadEvent({ type: "speech-end" }));
+		await new Promise((r) => setTimeout(r, 0));
+
+		expect(h.generateCalls).toHaveLength(1);
+		expect(h.generateCalls[0]).toMatchObject({
+			transcript: "this is the owner",
+			final: true,
+			source,
+			speaker,
+			segments,
+			turn,
+		});
+	});
+
+	it("a new speech-start aborts an in-flight speculative (VAD re-trigger)", () => {
+		const h = makeHarness({ speculatePauseMs: 300 });
+		h.controller.start();
+		h.vad.emit(vadEvent({ type: "speech-start" }));
+		h.transcriber.setPartial("hey");
+		h.vad.emit(vadEvent({ type: "speech-pause", pauseDurationMs: 400 }));
+		expect(h.generateCalls).toHaveLength(1);
+		// A brand-new utterance onset before the segment ended.
+		h.vad.emit(vadEvent({ type: "speech-start" }));
+		expect(h.generateCalls[0].signal.aborted).toBe(true);
+		expect(h.prewarm).toHaveBeenCalledTimes(2);
+	});
+
+	it("routes the transcriber 'words' event into the barge-in word-confirm gate", () => {
+		const h = makeHarness();
+		h.controller.start();
+		// Simulate the agent speaking + a provisional barge-in.
+		h.scheduler.bargeIn.setAgentSpeaking(true);
+		let hardStopped = false;
+		h.scheduler.bargeIn.onSignal((s) => {
+			if (s.type === "hard-stop") hardStopped = true;
+		});
+		// A blip alone (no words) only pauses — emit a pause-style provisional.
+		h.scheduler.bargeIn.onSignal(() => {});
+		h.vad.emit(vadEvent({ type: "speech-active" })); // → pause-tts
+		expect(hardStopped).toBe(false);
+		// ASR confirms real words → hard-stop.
+		h.transcriber.emit({ kind: "words", words: ["wait", "stop"] });
+		expect(hardStopped).toBe(true);
+	});
+
+	it("surfaces a prewarm rejection via onError without killing the turn", async () => {
+		const h = makeHarness();
+		h.prewarm.mockRejectedValueOnce(new Error("kv prefill failed"));
+		h.controller.start();
+		h.vad.emit(vadEvent({ type: "speech-start" }));
+		await new Promise((r) => setTimeout(r, 0));
+		expect(h.events.errors.map((e) => e.message)).toContain(
+			"kv prefill failed",
+		);
+		// Turn-taking still works.
+		h.transcriber.finalText = "ok";
+		h.vad.emit(vadEvent({ type: "speech-end" }));
+		await new Promise((r) => setTimeout(r, 0));
+		expect(h.generateCalls).toHaveLength(1);
+	});
+
+	it("C2: prewarmOnIdle fires the prewarm without blocking the caller", async () => {
+		// Hold the prewarm open so we can confirm prewarmOnIdle returned
+		// synchronously (i.e. did not await).
+		let releasePrewarm: () => void = () => {};
+		const pending = new Promise<void>((resolve) => {
+			releasePrewarm = resolve;
+		});
+		const h = makeHarness();
+		h.prewarm.mockImplementationOnce(async () => {
+			await pending;
+		});
+		const ret = h.controller.prewarmOnIdle();
+		expect(ret).toBeUndefined();
+		expect(h.prewarm).toHaveBeenCalledTimes(1);
+		expect(h.prewarm).toHaveBeenCalledWith("room-1");
+		// The prewarm is still in flight — the caller is not blocked on it.
+		releasePrewarm();
+		await new Promise((r) => setTimeout(r, 0));
+		expect(h.events.errors).toEqual([]);
+	});
+
+	it("C2: prewarmOnIdle errors surface via onError, do not throw to the caller", async () => {
+		const h = makeHarness();
+		h.prewarm.mockRejectedValueOnce(new Error("idle prewarm failed"));
+		expect(() => h.controller.prewarmOnIdle()).not.toThrow();
+		await new Promise((r) => setTimeout(r, 0));
+		expect(h.events.errors.map((e) => e.message)).toContain(
+			"idle prewarm failed",
+		);
+	});
+});