@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/voice/barge-in.test.ts

@@ -0,0 +1,244 @@
+/**
+ * BargeInController tests — the blip-vs-words distinction.
+ *
+ *   - legacy `attach`/`onMicActive`/`cancelSignal`/`reset` still work
+ *     (`VoiceScheduler` depends on them).
+ *   - VAD-driven flow while the agent is speaking:
+ *       speech-active → pause-tts
+ *       blip          → resume-tts
+ *       ASR word      → hard-stop (cancel token tripped, AbortSignal aborted)
+ *       no ASR word   → resume-tts after the grace window
+ *   - barge-in is inert while the agent is NOT speaking.
+ */
+
+import { describe, expect, it, vi } from "vitest";
+import { BargeInController } from "./barge-in";
+import type { BargeInSignal, VadEvent, VadEventListener } from "./types";
+
+/** A standalone fake VAD event source the controller can bind to. */
+class FakeVad {
+	private readonly listeners = new Set<VadEventListener>();
+	onVadEvent(l: VadEventListener): () => void {
+		this.listeners.add(l);
+		return () => this.listeners.delete(l);
+	}
+	emit(e: VadEvent): void {
+		for (const l of this.listeners) l(e);
+	}
+}
+
+function speechActive(ts: number): VadEvent {
+	return {
+		type: "speech-active",
+		timestampMs: ts,
+		probability: 0.9,
+		speechDurationMs: 200,
+	};
+}
+function blip(ts: number): VadEvent {
+	return { type: "blip", timestampMs: ts, durationMs: 80, peakRms: 0.2 };
+}
+
+describe("BargeInController — legacy API", () => {
+	it("flips the cancel signal and notifies listeners on onMicActive", () => {
+		const c = new BargeInController();
+		let n = 0;
+		c.attach({ onCancel: () => n++ });
+		expect(c.cancelSignal().cancelled).toBe(false);
+		c.onMicActive();
+		expect(c.cancelSignal().cancelled).toBe(true);
+		expect(n).toBe(1);
+	});
+
+	it("reset issues a fresh cancel signal", () => {
+		const c = new BargeInController();
+		c.onMicActive();
+		expect(c.cancelSignal().cancelled).toBe(true);
+		c.reset();
+		expect(c.cancelSignal().cancelled).toBe(false);
+		expect(c.currentCancelToken()).toBeNull();
+	});
+});
+
+describe("BargeInController — VAD-driven barge-in", () => {
+	it("does nothing while the agent is not speaking", () => {
+		const c = new BargeInController();
+		const vad = new FakeVad();
+		c.bindVad(vad);
+		const signals: BargeInSignal[] = [];
+		c.onSignal((s) => signals.push(s));
+		vad.emit(speechActive(100));
+		vad.emit(blip(200));
+		expect(signals).toHaveLength(0);
+	});
+
+	it("pauses TTS on speech-active and resumes on a blip", () => {
+		const c = new BargeInController();
+		const vad = new FakeVad();
+		c.bindVad(vad);
+		c.setAgentSpeaking(true);
+		const signals: BargeInSignal[] = [];
+		c.onSignal((s) => signals.push(s));
+
+		vad.emit(speechActive(100));
+		expect(signals.map((s) => s.type)).toEqual(["pause-tts"]);
+		// A second speech-active while already paused — no duplicate pause.
+		vad.emit(speechActive(132));
+		expect(signals.map((s) => s.type)).toEqual(["pause-tts"]);
+		// Blip → resume.
+		vad.emit(blip(300));
+		expect(signals.map((s) => s.type)).toEqual(["pause-tts", "resume-tts"]);
+	});
+
+	it("hard-stops (cancel token + AbortSignal) when ASR confirms a word", () => {
+		const c = new BargeInController();
+		const vad = new FakeVad();
+		c.bindVad(vad);
+		c.setAgentSpeaking(true);
+		const signals: BargeInSignal[] = [];
+		c.onSignal((s) => signals.push(s));
+		let onCancelCalls = 0;
+		c.attach({ onCancel: () => onCancelCalls++ });
+
+		vad.emit(speechActive(100)); // pause-tts
+		expect(signals.map((s) => s.type)).toEqual(["pause-tts"]);
+
+		c.onWordsDetected({
+			wordCount: 1,
+			partialText: "hey wait",
+			timestampMs: 250,
+		});
+
+		expect(signals.map((s) => s.type)).toEqual(["pause-tts", "hard-stop"]);
+		const hard = signals.find((s) => s.type === "hard-stop");
+		expect(hard && hard.type === "hard-stop").toBe(true);
+		if (hard && hard.type === "hard-stop") {
+			expect(hard.token.cancelled).toBe(true);
+			expect(hard.token.reason).toBe("barge-in-words");
+			expect(hard.token.signal.aborted).toBe(true);
+		}
+		expect(c.currentCancelToken()?.cancelled).toBe(true);
+		expect(c.cancelSignal().cancelled).toBe(true);
+		expect(onCancelCalls).toBe(1);
+	});
+
+	it("ignores onWordsDetected with zero words", () => {
+		const c = new BargeInController();
+		const vad = new FakeVad();
+		c.bindVad(vad);
+		c.setAgentSpeaking(true);
+		const signals: BargeInSignal[] = [];
+		c.onSignal((s) => signals.push(s));
+		vad.emit(speechActive(100));
+		c.onWordsDetected({ wordCount: 0, partialText: "", timestampMs: 200 });
+		expect(signals.map((s) => s.type)).toEqual(["pause-tts"]);
+	});
+
+	it("resumes TTS after the grace window when ASR never confirms a word", () => {
+		vi.useFakeTimers();
+		try {
+			const c = new BargeInController({ wordsGraceMs: 500 });
+			const vad = new FakeVad();
+			c.bindVad(vad);
+			c.setAgentSpeaking(true);
+			const signals: BargeInSignal[] = [];
+			c.onSignal((s) => signals.push(s));
+
+			vad.emit(speechActive(100)); // pause-tts, arms 500ms deadline
+			expect(signals.map((s) => s.type)).toEqual(["pause-tts"]);
+			vad.emit({
+				type: "speech-pause",
+				timestampMs: 300,
+				pauseDurationMs: 200,
+			});
+			vad.emit({ type: "speech-end", timestampMs: 500, speechDurationMs: 350 });
+			// Still inside the grace window.
+			vi.advanceTimersByTime(400);
+			expect(signals.map((s) => s.type)).toEqual(["pause-tts"]);
+			// Past it → resume.
+			vi.advanceTimersByTime(200);
+			expect(signals.map((s) => s.type)).toEqual(["pause-tts", "resume-tts"]);
+		} finally {
+			vi.useRealTimers();
+		}
+	});
+
+	it("a word arriving before the grace deadline cancels the resume", () => {
+		vi.useFakeTimers();
+		try {
+			const c = new BargeInController({ wordsGraceMs: 500 });
+			const vad = new FakeVad();
+			c.bindVad(vad);
+			c.setAgentSpeaking(true);
+			const signals: BargeInSignal[] = [];
+			c.onSignal((s) => signals.push(s));
+			vad.emit(speechActive(100));
+			vi.advanceTimersByTime(300);
+			c.onWordsDetected({
+				wordCount: 2,
+				partialText: "stop please",
+				timestampMs: 400,
+			});
+			vi.advanceTimersByTime(500);
+			expect(signals.map((s) => s.type)).toEqual(["pause-tts", "hard-stop"]);
+		} finally {
+			vi.useRealTimers();
+		}
+	});
+
+	it("ignores stale ASR words after the interruption window resumes TTS", () => {
+		vi.useFakeTimers();
+		try {
+			const c = new BargeInController({ wordsGraceMs: 500 });
+			const vad = new FakeVad();
+			c.bindVad(vad);
+			c.setAgentSpeaking(true);
+			const signals: BargeInSignal[] = [];
+			c.onSignal((s) => signals.push(s));
+
+			vad.emit(speechActive(100));
+			vi.advanceTimersByTime(600);
+			expect(signals.map((s) => s.type)).toEqual(["pause-tts", "resume-tts"]);
+
+			c.onWordsDetected({
+				wordCount: 2,
+				partialText: "late stale",
+				timestampMs: 800,
+			});
+			expect(signals.map((s) => s.type)).toEqual(["pause-tts", "resume-tts"]);
+			expect(c.currentCancelToken()).toBeNull();
+		} finally {
+			vi.useRealTimers();
+		}
+	});
+
+	it("setAgentSpeaking(false) clears a pending word-confirm without resuming", () => {
+		vi.useFakeTimers();
+		try {
+			const c = new BargeInController({ wordsGraceMs: 500 });
+			const vad = new FakeVad();
+			c.bindVad(vad);
+			c.setAgentSpeaking(true);
+			const signals: BargeInSignal[] = [];
+			c.onSignal((s) => signals.push(s));
+			vad.emit(speechActive(100)); // pause-tts
+			c.setAgentSpeaking(false); // agent finished its turn
+			vi.advanceTimersByTime(1000);
+			expect(signals.map((s) => s.type)).toEqual(["pause-tts"]);
+		} finally {
+			vi.useRealTimers();
+		}
+	});
+
+	it("unbindVad detaches the controller from VAD events", () => {
+		const c = new BargeInController();
+		const vad = new FakeVad();
+		const unbind = c.bindVad(vad);
+		c.setAgentSpeaking(true);
+		const signals: BargeInSignal[] = [];
+		c.onSignal((s) => signals.push(s));
+		unbind();
+		vad.emit(speechActive(100));
+		expect(signals).toHaveLength(0);
+	});
+});

package/src/services/voice/barge-in.ts

@@ -0,0 +1,335 @@
+/**
+ * Barge-in controller — distinguishes a blip from real speech while the
+ * agent is talking, and turns that into TTS pause/resume/hard-stop plus an
+ * LLM-generation abort.
+ *
+ * Inputs:
+ *   - the `VadEvent` stream from `VadDetector` (subscribe via `bindVad()`),
+ *   - W2's ASR word-confirm callback (`onWordsDetected()` — the
+ *     `WordsDetectedSink` contract).
+ *
+ * Behaviour while the agent is speaking (`agentSpeaking === true`):
+ *   - `speech-active`  → emit `pause-tts`. (Provisional — could still be a
+ *                        blip; the energy-duration heuristic guesses, ASR
+ *                        confirms.)
+ *   - `blip` (or a short `speech-end` before any words)
+ *                      → emit `resume-tts`. The agent keeps talking.
+ *   - `onWordsDetected({wordCount ≥ 1})` → emit `hard-stop` with a fresh
+ *                        `BargeInCancelToken`. Hard-stop means: cancel TTS
+ *                        *and* abort the in-flight LLM / MTP drafter
+ *                        generation. The engine layer (W9) threads
+ *                        `token.signal` into `dispatcher.generate` and polls
+ *                        `token.cancelled` at kernel boundaries.
+ *   - `speech-end` with a long-enough segment but no ASR words yet →
+ *                        treated as words-pending: emit `hard-stop` only
+ *                        once ASR confirms; if ASR never confirms within
+ *                        `wordsGraceMs`, resume TTS (it was non-speech the
+ *                        Silero VAD let through).
+ *
+ * Legacy API (still used by `VoiceScheduler` and `EngineVoiceBridge`):
+ *   `attach({onCancel})`, `onMicActive()`, `cancelSignal()`, `reset()` — a
+ *   thin "everything cancelled" path. `onMicActive()` is now equivalent to
+ *   `hardStop("manual")`.
+ *
+ * No fallback sludge: a `hard-stop` always carries a real `AbortSignal`; the
+ * controller never swallows a VAD event.
+ */
+
+import type {
+	BargeInCancelToken,
+	BargeInSignal,
+	BargeInSignalListener,
+	VadEvent,
+	VadEventListener,
+	WordsDetectedSink,
+} from "./types";
+
+/** Minimal structural view of `VadDetector` — avoids a module dependency on
+ *  `vad.ts` (which pulls in the fused `libelizainference` VAD FFI surface). */
+interface VadEventSource {
+	onVadEvent(listener: VadEventListener): () => void;
+}
+
+// --- Legacy interfaces (kept; `VoiceScheduler` depends on them) ------------
+
+export interface BargeInListener {
+	onCancel(): void;
+}
+
+export interface CancelSignal {
+	cancelled: boolean;
+}
+
+// --- New: cancel token --------------------------------------------------------
+
+function makeCancelToken(
+	reason: BargeInCancelToken["reason"],
+): BargeInCancelToken {
+	const controller = new AbortController();
+	const token: BargeInCancelToken = {
+		cancelled: false,
+		reason: null,
+		signal: controller.signal,
+	};
+	const trip = (r: BargeInCancelToken["reason"]) => {
+		if (token.cancelled) return;
+		token.cancelled = true;
+		token.reason = r;
+		controller.abort();
+	};
+	if (reason) trip(reason);
+	// Expose the tripper on a non-enumerable slot for the controller to use.
+	Object.defineProperty(token, "__trip", { value: trip, enumerable: false });
+	return token;
+}
+
+function tripToken(
+	token: BargeInCancelToken,
+	reason: BargeInCancelToken["reason"],
+): void {
+	const trip = (token as { __trip?: (r: BargeInCancelToken["reason"]) => void })
+		.__trip;
+	if (trip) trip(reason);
+}
+
+export interface BargeInControllerConfig {
+	/**
+	 * After a `speech-active` (TTS paused) with no ASR word confirmation,
+	 * resume TTS if ASR has not reported ≥1 word within this window. Default
+	 * 600 ms. Long enough for a streaming ASR partial; short enough that a
+	 * cough doesn't keep the agent muted.
+	 */
+	wordsGraceMs?: number;
+}
+
+export class BargeInController implements WordsDetectedSink {
+	private readonly listeners = new Set<BargeInListener>();
+	private readonly signalListeners = new Set<BargeInSignalListener>();
+	private readonly wordsGraceMs: number;
+
+	/** Legacy single-shot cancel flag, reset by `reset()`. */
+	private signal: CancelSignal = { cancelled: false };
+
+	/** True while the agent's TTS is playing. The turn controller / scheduler
+	 *  flips this via `setAgentSpeaking()`. Barge-in logic only acts while
+	 *  this is true. */
+	private agentSpeaking = false;
+	/** True while we have emitted `pause-tts` and are waiting on the
+	 *  blip-vs-words decision. */
+	private awaitingWordConfirm = false;
+	private wordConfirmDeadlineTimer: ReturnType<typeof setTimeout> | null = null;
+	private wordConfirmExpiresAtMs: number | null = null;
+	private lastEventTimestampMs = 0;
+	private vadUnsub: (() => void) | null = null;
+
+	constructor(config: BargeInControllerConfig = {}) {
+		this.wordsGraceMs = config.wordsGraceMs ?? 600;
+	}
+
+	// --- New subscription API ---------------------------------------------------
+
+	/** Subscribe to `pause-tts` / `resume-tts` / `hard-stop`. */
+	onSignal(listener: BargeInSignalListener): () => void {
+		this.signalListeners.add(listener);
+		return () => this.signalListeners.delete(listener);
+	}
+
+	/** Wire this controller to a `VadDetector`. Returns an unsubscribe fn. */
+	bindVad(detector: VadEventSource): () => void {
+		this.unbindVad();
+		this.vadUnsub = detector.onVadEvent((e: VadEvent) => this.onVadEvent(e));
+		return () => this.unbindVad();
+	}
+
+	unbindVad(): void {
+		if (this.vadUnsub) {
+			this.vadUnsub();
+			this.vadUnsub = null;
+		}
+	}
+
+	/** The turn controller flips this when TTS starts/stops playing. */
+	setAgentSpeaking(speaking: boolean): void {
+		if (this.agentSpeaking === speaking) return;
+		this.agentSpeaking = speaking;
+		if (!speaking) {
+			// Agent stopped talking on its own — drop any pending word-confirm.
+			this.clearWordConfirm();
+			this.awaitingWordConfirm = false;
+		}
+	}
+
+	get isAgentSpeaking(): boolean {
+		return this.agentSpeaking;
+	}
+
+	// --- VAD event handling -----------------------------------------------------
+
+	private onVadEvent(event: VadEvent): void {
+		this.lastEventTimestampMs = event.timestampMs;
+		if (!this.agentSpeaking) return;
+		switch (event.type) {
+			case "speech-start":
+			case "speech-active": {
+				if (!this.awaitingWordConfirm) {
+					this.awaitingWordConfirm = true;
+					this.emitSignal({
+						type: "pause-tts",
+						timestampMs: event.timestampMs,
+					});
+					this.armWordConfirmDeadline(event.timestampMs);
+				}
+				break;
+			}
+			case "blip": {
+				// Definitely not speech — resume immediately.
+				if (this.awaitingWordConfirm) {
+					this.awaitingWordConfirm = false;
+					// Stop the pending auto-resume timer, but keep the ASR grace
+					// window alive. A VAD blip decision can arrive before the ASR
+					// partial for the same audio; if words land inside the original
+					// window, they are authoritative and should still hard-stop.
+					this.clearWordConfirm({ keepWindow: true });
+					this.emitSignal({
+						type: "resume-tts",
+						timestampMs: event.timestampMs,
+					});
+				}
+				break;
+			}
+			case "speech-pause":
+				// Still ambiguous; keep TTS paused, wait on ASR / the deadline.
+				break;
+			case "speech-end": {
+				// The Silero VAD considers this a finished segment. If ASR hasn't
+				// confirmed words by now, the grace deadline will resume TTS; if it
+				// has, `onWordsDetected` already hard-stopped. Nothing extra here.
+				break;
+			}
+		}
+	}
+
+	// --- ASR word-confirm sink (WordsDetectedSink) ------------------------------
+
+	onWordsDetected(args: {
+		wordCount: number;
+		partialText: string;
+		timestampMs: number;
+	}): void {
+		if (args.wordCount < 1) return;
+		const withinConfirmWindow =
+			this.wordConfirmExpiresAtMs != null &&
+			args.timestampMs <= this.wordConfirmExpiresAtMs;
+		if (
+			!this.agentSpeaking ||
+			(!this.awaitingWordConfirm && !withinConfirmWindow)
+		) {
+			return;
+		}
+		// Authoritative: real user speech. Hard-stop.
+		this.hardStop("barge-in-words", args.timestampMs);
+	}
+
+	// --- Hard stop --------------------------------------------------------------
+
+	/**
+	 * Cancel TTS + abort the in-flight LLM / drafter generation. Returns the
+	 * `BargeInCancelToken` whose `signal` the engine layer aborts on. Idempotent
+	 * within a single barge-in episode — calling it again returns the same
+	 * token until `reset()`.
+	 */
+	hardStop(
+		reason: NonNullable<BargeInCancelToken["reason"]> = "manual",
+		timestampMs: number = this.lastEventTimestampMs || Date.now(),
+	): BargeInCancelToken {
+		this.clearWordConfirm();
+		this.awaitingWordConfirm = false;
+		if (!this.activeToken) {
+			this.activeToken = makeCancelToken(null);
+		}
+		tripToken(this.activeToken, reason);
+		// Legacy cancel flag + listeners.
+		this.signal.cancelled = true;
+		for (const l of this.listeners) l.onCancel();
+		this.emitSignal({
+			type: "hard-stop",
+			timestampMs,
+			token: this.activeToken,
+		});
+		return this.activeToken;
+	}
+
+	private activeToken: BargeInCancelToken | null = null;
+
+	/** The cancel token for the current barge-in episode (null until a
+	 *  `hard-stop`). The engine threads `.signal` into generation. */
+	currentCancelToken(): BargeInCancelToken | null {
+		return this.activeToken;
+	}
+
+	// --- Legacy API (VoiceScheduler / EngineVoiceBridge) ------------------------
+
+	/** @deprecated Use `currentCancelToken()`; kept for `VoiceScheduler`. */
+	cancelSignal(): CancelSignal {
+		return this.signal;
+	}
+
+	attach(listener: BargeInListener): () => void {
+		this.listeners.add(listener);
+		return () => this.listeners.delete(listener);
+	}
+
+	/** @deprecated Equivalent to `hardStop("manual")`; kept for the bridge. */
+	onMicActive(): void {
+		this.hardStop("manual");
+	}
+
+	reset(): void {
+		this.clearWordConfirm();
+		this.awaitingWordConfirm = false;
+		this.activeToken = null;
+		this.signal = { cancelled: false };
+	}
+
+	// --- internals --------------------------------------------------------------
+
+	private emitSignal(signal: BargeInSignal): void {
+		for (const l of this.signalListeners) l(signal);
+	}
+
+	private armWordConfirmDeadline(timestampMs: number): void {
+		this.clearWordConfirm();
+		this.wordConfirmExpiresAtMs = timestampMs + this.wordsGraceMs;
+		this.wordConfirmDeadlineTimer = setTimeout(() => {
+			this.wordConfirmDeadlineTimer = null;
+			if (this.awaitingWordConfirm && this.agentSpeaking) {
+				// ASR never confirmed a word — the Silero VAD let through
+				// non-speech. Resume TTS.
+				this.awaitingWordConfirm = false;
+				this.emitSignal({
+					type: "resume-tts",
+					timestampMs: timestampMs + this.wordsGraceMs,
+				});
+			}
+			this.wordConfirmExpiresAtMs = null;
+		}, this.wordsGraceMs);
+		// Don't keep the event loop alive on this timer.
+		if (
+			this.wordConfirmDeadlineTimer &&
+			typeof (this.wordConfirmDeadlineTimer as { unref?: () => void }).unref ===
+				"function"
+		) {
+			(this.wordConfirmDeadlineTimer as { unref: () => void }).unref();
+		}
+	}
+
+	private clearWordConfirm(options: { keepWindow?: boolean } = {}): void {
+		if (this.wordConfirmDeadlineTimer) {
+			clearTimeout(this.wordConfirmDeadlineTimer);
+			this.wordConfirmDeadlineTimer = null;
+		}
+		if (!options.keepWindow) {
+			this.wordConfirmExpiresAtMs = null;
+		}
+	}
+}