@elizaos/plugin-local-inference 2.0.3-beta.2 → 2.0.3-beta.4

package/src/services/voice/eot-classifier.test.ts

@@ -0,0 +1,98 @@
+// Coverage for the pure end-of-turn decision helpers (#9147, "eot" matrix
+// class). turnSignalFromProbability maps an EOT probability onto the
+// agent/user/unknown next-speaker decision via the commit/tentative thresholds;
+// these boundaries gate whether the agent speaks, so they are pinned here.
+
+import { describe, expect, it } from "vitest";
+import {
+	clampProbability,
+	EOT_MID_CLAUSE_THRESHOLD,
+	EOT_TENTATIVE_THRESHOLD,
+	LIVEKIT_TURN_DETECTOR_EN_REVISION,
+	LIVEKIT_TURN_DETECTOR_INTL_REVISION,
+	turnDetectorRevisionForTier,
+	turnSignalFromProbability,
+} from "./eot-classifier";
+
+describe("clampProbability", () => {
+	it("clamps to [0,1] and defaults non-finite input to 0.5", () => {
+		expect(clampProbability(-1)).toBe(0);
+		expect(clampProbability(2)).toBe(1);
+		expect(clampProbability(0.7)).toBe(0.7);
+		expect(clampProbability(Number.NaN)).toBe(0.5);
+		expect(clampProbability(Number.POSITIVE_INFINITY)).toBe(0.5);
+	});
+});
+
+describe("turnSignalFromProbability", () => {
+	const base = { transcript: "are we done", source: "heuristic" as const };
+
+	it("yields agent (speak) at/above the tentative threshold", () => {
+		const sig = turnSignalFromProbability({
+			...base,
+			probability: EOT_TENTATIVE_THRESHOLD,
+		});
+		expect(sig.nextSpeaker).toBe("agent");
+		expect(sig.agentShouldSpeak).toBe(true);
+	});
+
+	it("yields user (stay silent) below the mid-clause threshold", () => {
+		const sig = turnSignalFromProbability({
+			...base,
+			probability: EOT_MID_CLAUSE_THRESHOLD - 0.01,
+		});
+		expect(sig.nextSpeaker).toBe("user");
+		expect(sig.agentShouldSpeak).toBe(false);
+	});
+
+	it("yields unknown (null) in the mid-clause band", () => {
+		const sig = turnSignalFromProbability({
+			...base,
+			probability: EOT_MID_CLAUSE_THRESHOLD,
+		});
+		expect(sig.nextSpeaker).toBe("unknown");
+		expect(sig.agentShouldSpeak).toBeNull();
+	});
+
+	it("clamps a non-finite probability into the unknown band", () => {
+		const sig = turnSignalFromProbability({
+			...base,
+			probability: Number.NaN,
+		});
+		expect(sig.endOfTurnProbability).toBe(0.5);
+		expect(sig.nextSpeaker).toBe("unknown");
+	});
+
+	it("passes transcript through and includes model/latency only when supplied", () => {
+		const withOpts = turnSignalFromProbability({
+			...base,
+			probability: 0.9,
+			model: "eot-v2",
+			latencyMs: 12,
+		});
+		expect(withOpts.transcript).toBe("are we done");
+		expect(withOpts.model).toBe("eot-v2");
+		expect(withOpts.latencyMs).toBe(12);
+
+		const without = turnSignalFromProbability({ ...base, probability: 0.9 });
+		expect("model" in without).toBe(false);
+		expect("latencyMs" in without).toBe(false);
+	});
+});
+
+describe("turnDetectorRevisionForTier", () => {
+	it("maps the 2b entry tier to the English variant, others to multilingual", () => {
+		expect(turnDetectorRevisionForTier("2b")).toBe(
+			LIVEKIT_TURN_DETECTOR_EN_REVISION,
+		);
+		expect(turnDetectorRevisionForTier("eliza-1-2b")).toBe(
+			LIVEKIT_TURN_DETECTOR_EN_REVISION,
+		);
+		expect(turnDetectorRevisionForTier("4b")).toBe(
+			LIVEKIT_TURN_DETECTOR_INTL_REVISION,
+		);
+		expect(turnDetectorRevisionForTier("eliza-1-8b")).toBe(
+			LIVEKIT_TURN_DETECTOR_INTL_REVISION,
+		);
+	});
+});

package/src/services/voice/eot-classifier.ts

@@ -34,6 +34,7 @@
  * suppress (via `BargeInCancelToken.signal` with reason `"turn-suppressed"`).
  */
 
+import { scoreEndOfTurnHeuristic } from "@elizaos/shared/voice-eot";
 import type {
 	Eliza1EotScoreResult,
 	Eliza1EotScorerOptions,
@@ -120,136 +121,24 @@ export function turnSignalFromProbability(args: {
 // ---------------------------------------------------------------------------
 
 /**
- * Rules-of-thumb EOT classifier. The rules fire in priority order; the first
- * match wins.
+ * Rules-of-thumb EOT classifier. Delegates to the single canonical heuristic in
+ * `@elizaos/shared/voice-eot` — the SAME scorer the UI shell capture path
+ * (`packages/ui/src/voice/end-of-turn.ts`) uses, so the two surfaces can never
+ * drift. The rules fire in priority order; the first match wins:
  *
  * Priority  Signal                                       P(done)
  * --------  -------------------------------------------  -------
- *   1       Sentence-final punctuation (. ! ?)            0.95
- *   2       Question-tag words ("right?", "yeah?", "ok?") 0.85
- *   3       Short utterance (< 3 words)                   0.70
+ *   1       Trailing ellipsis ("…" / "..")               0.20
+ *   2       Sentence-final punctuation (. ! ?)            0.95
+ *   3       Question-tag words ("right?", "yeah", …)      0.85
  *   4       Trailing conjunction (and/but/or/because/…)   0.15
  *   5       Last word is a preposition or article         0.20
- *   6       No signal                                     0.50
+ *   6       Short utterance (< 3 words, no trail-off)     0.70
+ *   7       No signal                                     0.50
  */
 export class HeuristicEotClassifier implements EotClassifier {
-	/** Conjunctions that strongly suggest the user is mid-clause. */
-	private static readonly TRAILING_CONJUNCTIONS = new Set([
-		"and",
-		"but",
-		"or",
-		"nor",
-		"yet",
-		"so",
-		"because",
-		"although",
-		"though",
-		"while",
-		"whereas",
-		"if",
-		"unless",
-		"until",
-		"since",
-		"when",
-		"where",
-		"which",
-		"that",
-		"who",
-		"whom",
-		"whose",
-	]);
-
-	/** Prepositions and articles that suggest an incomplete NP follows. */
-	private static readonly TRAILING_INCOMPLETE = new Set([
-		"a",
-		"an",
-		"the",
-		"to",
-		"of",
-		"in",
-		"on",
-		"at",
-		"by",
-		"for",
-		"with",
-		"from",
-		"into",
-		"about",
-		"through",
-		"between",
-		"against",
-		"during",
-		"before",
-		"after",
-		"without",
-		"under",
-		"over",
-		"above",
-		"below",
-		"around",
-		"beside",
-		"beyond",
-		"like",
-		"near",
-		"past",
-		"via",
-	]);
-
-	/** Question-tag suffixes that end an utterance (case-insensitive). */
-	private static readonly QUESTION_TAGS = [
-		"right?",
-		"yeah?",
-		"ok?",
-		"okay?",
-		"right",
-		"yeah",
-		"correct?",
-		"correct",
-		"hm?",
-		"huh?",
-		"eh?",
-	];
-
 	score(partialTranscript: string): Promise<number> {
-		const text = partialTranscript.trim();
-		if (text.length === 0) return Promise.resolve(0.5);
-
-		// Rule 1 — sentence-final punctuation.
-		if (/[.!?]$/.test(text)) {
-			return Promise.resolve(0.95);
-		}
-
-		// Rule 2 — question-tag words at the end.
-		const lower = text.toLowerCase();
-		for (const tag of HeuristicEotClassifier.QUESTION_TAGS) {
-			if (lower.endsWith(tag)) return Promise.resolve(0.85);
-		}
-
-		// Split into words for word-level checks.
-		const words = text
-			.toLowerCase()
-			.replace(/[^a-z0-9'\s-]/gi, "")
-			.split(/\s+/)
-			.filter(Boolean);
-		if (words.length === 0) return Promise.resolve(0.5);
-
-		const lastWord = words[words.length - 1].replace(/[',;:-]+$/, "");
-
-		// Rule 3 — short utterance (< 3 words) → likely complete.
-		if (words.length < 3) return Promise.resolve(0.7);
-
-		// Rule 4 — trailing conjunction → mid-clause.
-		if (HeuristicEotClassifier.TRAILING_CONJUNCTIONS.has(lastWord)) {
-			return Promise.resolve(0.15);
-		}
-
-		// Rule 5 — trailing preposition or article → incomplete NP.
-		if (HeuristicEotClassifier.TRAILING_INCOMPLETE.has(lastWord)) {
-			return Promise.resolve(0.2);
-		}
-
-		// Rule 6 — no signal.
-		return Promise.resolve(0.5);
+		return Promise.resolve(scoreEndOfTurnHeuristic(partialTranscript));
 	}
 
 	async signal(partialTranscript: string): Promise<VoiceTurnSignal> {

package/src/services/voice/errors.ts

@@ -18,7 +18,9 @@ export class VoiceStartupError extends Error {
 		| "missing-ffi"
 		| "missing-speaker-preset"
 		| "missing-bundle-root"
+		| "missing-asr-model"
 		| "missing-fused-build"
+		| "blocked-asr-provenance"
 		| "missing-turn-detector"
 		| "already-started"
 		| "not-started"

package/src/services/voice/expressive-tags.asr.test.ts

@@ -0,0 +1,77 @@
+/**
+ * Unit coverage for the ASR-emotion → expressive-tag helpers (#9147 voice).
+ *
+ * normalizeAsrEmotionLabel / asrEmotionToTag / expressiveTagPromptClause are
+ * pure mappers between local ASR emotion labels and the TTS expressive-tag
+ * vocabulary; they were untested (the parse/strip helpers in the same file are
+ * covered separately). No GGUF / audio.
+ */
+
+import { describe, expect, it } from "vitest";
+import {
+	asrEmotionToTag,
+	EXPRESSIVE_EMOTION_TAGS,
+	EXPRESSIVE_NONVERBAL_TAGS,
+	EXPRESSIVE_SINGING_TAG,
+	expressiveTagPromptClause,
+	normalizeAsrEmotionLabel,
+} from "./expressive-tags";
+
+describe("normalizeAsrEmotionLabel", () => {
+	it("returns null for empty/missing input", () => {
+		expect(normalizeAsrEmotionLabel(null)).toBeNull();
+		expect(normalizeAsrEmotionLabel(undefined)).toBeNull();
+		expect(normalizeAsrEmotionLabel("")).toBeNull();
+	});
+
+	it("passes through a canonical noun label (case/space-insensitive)", () => {
+		expect(normalizeAsrEmotionLabel("happiness")).toBe("happiness");
+		expect(normalizeAsrEmotionLabel("  HAPPINESS ")).toBe("happiness");
+	});
+
+	it("maps common adjective forms to noun labels", () => {
+		expect(normalizeAsrEmotionLabel("happy")).toBe("happiness");
+		expect(normalizeAsrEmotionLabel("sad")).toBe("sadness");
+		expect(normalizeAsrEmotionLabel("angry")).toBe("anger");
+		expect(normalizeAsrEmotionLabel("afraid")).toBe("fear");
+		expect(normalizeAsrEmotionLabel("scared")).toBe("fear");
+		expect(normalizeAsrEmotionLabel("surprised")).toBe("surprise");
+		expect(normalizeAsrEmotionLabel("disgusted")).toBe("disgust");
+	});
+
+	it("returns null for an unknown label", () => {
+		expect(normalizeAsrEmotionLabel("gibberish-xyz")).toBeNull();
+	});
+});
+
+describe("asrEmotionToTag", () => {
+	it("maps a recognized emotion to an expressive emotion tag", () => {
+		const tag = asrEmotionToTag("happy");
+		expect(tag).not.toBeNull();
+		expect(EXPRESSIVE_EMOTION_TAGS).toContain(tag);
+	});
+
+	it("returns null for null / unknown input", () => {
+		expect(asrEmotionToTag(null)).toBeNull();
+		expect(asrEmotionToTag("gibberish-xyz")).toBeNull();
+	});
+});
+
+describe("expressiveTagPromptClause", () => {
+	it("lists the emotion + nonverbal tags and excludes singing by default", () => {
+		const clause = expressiveTagPromptClause();
+		for (const tag of EXPRESSIVE_EMOTION_TAGS) {
+			expect(clause).toContain(`[${tag}]`);
+		}
+		for (const tag of EXPRESSIVE_NONVERBAL_TAGS) {
+			expect(clause).toContain(`[${tag}]`);
+		}
+		expect(clause).not.toContain(`[${EXPRESSIVE_SINGING_TAG}]`);
+	});
+
+	it("includes the singing tag only when singingAllowed", () => {
+		expect(expressiveTagPromptClause({ singingAllowed: true })).toContain(
+			`[${EXPRESSIVE_SINGING_TAG}]`,
+		);
+	});
+});

package/src/services/voice/expressive-tags.test.ts

@@ -0,0 +1,102 @@
+import { describe, expect, it } from "vitest";
+import {
+	emotionToEnum,
+	enumToEmotion,
+	isExpressiveEmotionEnum,
+	isExpressiveTag,
+	parseExpressiveTags,
+	stripExpressiveTags,
+} from "./expressive-tags.js";
+
+/**
+ * Inline expressive-tag parsing for the voice path (#9147). Tags are scoped:
+ * an emotion/`[singing]` tag starts a new segment; `[laughter]`/`[sigh]` are
+ * recorded non-verbals; unrecognized `[tokens]` are preserved as model text
+ * (and flagged) rather than silently consumed.
+ */
+
+describe("isExpressiveTag / isExpressiveEmotionEnum", () => {
+	it("recognizes vocabulary tags, case/space-insensitively", () => {
+		expect(isExpressiveTag("happy")).toBe(true);
+		expect(isExpressiveTag("  SINGING ")).toBe(true);
+		expect(isExpressiveTag("laughter")).toBe(true);
+		expect(isExpressiveTag("grumpy")).toBe(false);
+	});
+
+	it("treats none + emotions (but not singing) as enum values", () => {
+		expect(isExpressiveEmotionEnum("none")).toBe(true);
+		expect(isExpressiveEmotionEnum("excited")).toBe(true);
+		expect(isExpressiveEmotionEnum("singing")).toBe(false);
+		expect(isExpressiveEmotionEnum("grumpy")).toBe(false);
+	});
+});
+
+describe("parseExpressiveTags", () => {
+	it("returns one neutral segment for untagged text", () => {
+		const parsed = parseExpressiveTags("just a normal reply");
+		expect(parsed.hasTags).toBe(false);
+		expect(parsed.dominantEmotion).toBeNull();
+		expect(parsed.segments).toHaveLength(1);
+		expect(parsed.segments[0]).toMatchObject({
+			emotion: null,
+			singing: false,
+			cleanText: "just a normal reply",
+		});
+	});
+
+	it("segments on each scope-setting emotion tag", () => {
+		const parsed = parseExpressiveTags("[happy] hello [sad] world");
+		expect(parsed.segments.map((s) => [s.emotion, s.cleanText])).toEqual([
+			["happy", "hello"],
+			["sad", "world"],
+		]);
+		expect(parsed.dominantEmotion).toBe("happy");
+		expect(parsed.cleanText).toBe("hello world");
+		expect(parsed.hasTags).toBe(true);
+	});
+
+	it("allows a mid-sentence shift with a leading neutral segment", () => {
+		const parsed = parseExpressiveTags("that's [excited] amazing");
+		expect(parsed.segments.map((s) => [s.emotion, s.cleanText])).toEqual([
+			[null, "that's"],
+			["excited", "amazing"],
+		]);
+		expect(parsed.dominantEmotion).toBe("excited");
+	});
+
+	it("tracks singing scope and preserves non-verbals without segmenting", () => {
+		const singing = parseExpressiveTags("[singing] la la la");
+		expect(singing.anySinging).toBe(true);
+		expect(singing.segments[0]).toMatchObject({ singing: true, emotion: null });
+
+		const nonverbal = parseExpressiveTags("hello [laughter] there");
+		expect(nonverbal.segments).toHaveLength(1);
+		expect(nonverbal.segments[0].nonverbals).toEqual(["laughter"]);
+		expect(nonverbal.cleanText).toBe("hello there");
+	});
+
+	it("records unknown bracket tokens instead of treating them as tags", () => {
+		const parsed = parseExpressiveTags("this is [grumpy] text");
+		expect(parsed.unknownTags).toEqual(["[grumpy]"]);
+		expect(parsed.hasTags).toBe(false);
+		expect(parsed.cleanText).toBe("this is text");
+	});
+});
+
+describe("stripExpressiveTags", () => {
+	it("removes recognized tags but leaves unknown bracket tokens", () => {
+		expect(stripExpressiveTags("[happy] hi [sigh] there")).toBe("hi there");
+		expect(stripExpressiveTags("[grumpy] hi")).toBe("[grumpy] hi");
+	});
+});
+
+describe("emotionToEnum / enumToEmotion", () => {
+	it("round-trips an emotion and maps null ⇄ none", () => {
+		expect(emotionToEnum("happy")).toBe("happy");
+		expect(emotionToEnum(null)).toBe("none");
+		expect(enumToEmotion("none")).toBeNull();
+		expect(enumToEmotion("happy")).toBe("happy");
+		expect(enumToEmotion("singing")).toBeNull();
+		expect(enumToEmotion(undefined)).toBeNull();
+	});
+});

package/src/services/voice/expressive-tags.ts

@@ -308,7 +308,7 @@ export function enumToEmotion(
  * as model-native emotion recognition unless that backend explicitly advertises
  * such a field; callers should record heuristic attribution separately.
  */
-export const QWEN3_ASR_EMOTION_LABELS = [
+export const ASR_EMOTION_LABELS = [
 	"surprise",
 	"calm",
 	"happiness",
@@ -318,7 +318,7 @@ export const QWEN3_ASR_EMOTION_LABELS = [
 	"fear",
 ] as const;
 
-export type Qwen3AsrEmotionLabel = (typeof QWEN3_ASR_EMOTION_LABELS)[number];
+export type AsrEmotionLabel = (typeof ASR_EMOTION_LABELS)[number];
 
 /**
  * Explicit mapping from an ASR-perceived emotion label to the tag vocab the
@@ -328,7 +328,7 @@ export type Qwen3AsrEmotionLabel = (typeof QWEN3_ASR_EMOTION_LABELS)[number];
  * `null` (counted as "no agreement" rather than forced).
  */
 export const ASR_LABEL_TO_EMOTION_TAG: Readonly<
-	Record<Qwen3AsrEmotionLabel, ExpressiveEmotion | null>
+	Record<AsrEmotionLabel, ExpressiveEmotion | null>
 > = {
 	happiness: "happy",
 	sadness: "sad",
@@ -340,18 +340,18 @@ export const ASR_LABEL_TO_EMOTION_TAG: Readonly<
 };
 
 /** Normalize an arbitrary ASR-emitted emotion string (any casing, possibly an
- *  adjective form) to a `Qwen3AsrEmotionLabel` if it matches, else `null`. */
+ *  adjective form) to an `AsrEmotionLabel` if it matches, else `null`. */
 export function normalizeAsrEmotionLabel(
 	raw: string | null | undefined,
-): Qwen3AsrEmotionLabel | null {
+): AsrEmotionLabel | null {
 	if (!raw) return null;
 	const v = raw.trim().toLowerCase();
 	// Direct hit.
-	if ((QWEN3_ASR_EMOTION_LABELS as readonly string[]).includes(v)) {
-		return v as Qwen3AsrEmotionLabel;
+	if ((ASR_EMOTION_LABELS as readonly string[]).includes(v)) {
+		return v as AsrEmotionLabel;
 	}
 	// Common adjective forms → noun labels.
-	const ADJ: Record<string, Qwen3AsrEmotionLabel> = {
+	const ADJ: Record<string, AsrEmotionLabel> = {
 		happy: "happiness",
 		sad: "sadness",
 		angry: "anger",

package/src/services/voice/ffi-bindings.test.ts

@@ -187,9 +187,16 @@ const FFI_STUB_DIR = path.resolve(
 );
 const STUB_DYLIB = path.join(
 	FFI_STUB_DIR,
+	// Per-OS shared-library naming. Windows must load a PE `.dll` — attempting to
+	// `dlopen` the committed Linux `.so` fails with error 193 (ERROR_BAD_EXE_FORMAT)
+	// rather than skipping. No Windows stub is built/committed, so the `existsSync`
+	// guards below skip these integration tests on Windows (same as Linux before
+	// `make -C scripts/ffi-stub`), instead of hard-failing on a format mismatch.
 	process.platform === "darwin"
 		? "libelizainference_stub.dylib"
-		: "libelizainference_stub.so",
+		: process.platform === "win32"
+			? "libelizainference_stub.dll"
+			: "libelizainference_stub.so",
 );
 
 function bunOnPath(): string | null {
@@ -215,8 +222,8 @@ function bunOnPath(): string | null {
 }
 
 describe("ffi-bindings — pure unit (no Bun, no dylib)", () => {
-	it("ELIZA_INFERENCE_ABI_VERSION is 12 (per-word ASR timestamps)", () => {
-		expect(ELIZA_INFERENCE_ABI_VERSION).toBe(12);
+	it("ELIZA_INFERENCE_ABI_VERSION is 13 (streaming vision describe)", () => {
+		expect(ELIZA_INFERENCE_ABI_VERSION).toBe(13);
 	});
 
 	it("loadElizaInferenceFfi throws VoiceLifecycleError when FFI is unavailable", () => {