@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/voice/partial-stabilizer.ts

@@ -0,0 +1,140 @@
+/**
+ * A2 — LocalAgreement-n streaming-ASR partial stabilizer.
+ *
+ * Streaming ASR (the fused Gemma ASR build running on partial windows)
+ * emits a fresh partial transcript on every
+ * audio frame. Each partial can revise tokens the previous partial
+ * already showed — "the cat sa" → "the cat sat" → "the cat sat on" is
+ * fine, but "the cat sa" → "the cap sat" rewrites earlier text. Handing
+ * a TTS chunker every revision causes audible stutter when the agent's
+ * drafter starts speaking text the verifier later rejects.
+ *
+ * The LocalAgreement-n trick: only commit a prefix to downstream once it
+ * has appeared identically in `n` consecutive partials. Below that
+ * threshold the text is "pending" — visible to UI for confirmation
+ * latency, but never sent to the drafter / phrase chunker. n=2 is the
+ * sweet spot for voice — large enough to suppress single-frame ASR
+ * jitter, small enough that the stable prefix tracks the speaker
+ * within ~one extra frame.
+ *
+ * Wiring: the streaming-ASR adapter calls `feed(partial)` per frame; the
+ * `stable` portion is what flows into `splitTranscriptToTokens` and the
+ * drafter. `pending` is suffix-only, fed to UI for visual feedback. The
+ * stabilizer is feature-flagged off until the streaming-ASR backend is
+ * wired (see `voice/pipeline.ts`'s `usePartialStabilizer`).
+ *
+ * No `any`, no fallbacks: a malformed partial (e.g. an empty string)
+ * collapses the stable prefix to whatever the agreement window still
+ * supports — this is correctness, not a swallow.
+ */
+
+export interface PartialStabilizerOptions {
+	/**
+	 * Number of consecutive identical partials a token has to appear in
+	 * before it migrates from `pending` → `stable`. Default 2 (the
+	 * LocalAgreement-2 setting that the streaming-ASR literature finds
+	 * close to optimal for English voice input).
+	 */
+	agreementCount?: number;
+}
+
+export interface StabilizerOutput {
+	/** The agreed-on prefix — safe to hand the drafter / phrase chunker. */
+	stable: string;
+	/**
+	 * The suffix still awaiting `n` matching partials — surface in UI but
+	 * do not commit. Concatenating `stable + pending` reconstructs the
+	 * latest raw partial.
+	 */
+	pending: string;
+}
+
+const DEFAULT_AGREEMENT_COUNT = 2;
+
+/**
+ * Find the longest character prefix shared by both strings. Character-level
+ * (not word-level) so a partial that finished a word ("sa" → "sat") still
+ * shows agreement on the shared prefix "sa" and only "t" stays pending.
+ */
+function commonPrefixLength(a: string, b: string): number {
+	const n = Math.min(a.length, b.length);
+	let i = 0;
+	while (i < n && a.charCodeAt(i) === b.charCodeAt(i)) i++;
+	return i;
+}
+
+export class PartialStabilizer {
+	private readonly agreementCount: number;
+	/**
+	 * The most recent partials, oldest first. We only need the last
+	 * `agreementCount` entries — the agreed prefix is the intersection of
+	 * all of them. Length 0 before any feed.
+	 */
+	private history: string[] = [];
+	/** The longest committed stable prefix so far. Monotonically grows. */
+	private committed = "";
+
+	constructor(options: PartialStabilizerOptions = {}) {
+		const requested = options.agreementCount ?? DEFAULT_AGREEMENT_COUNT;
+		if (!Number.isFinite(requested) || requested < 1) {
+			throw new Error(
+				`[partial-stabilizer] agreementCount must be a finite integer >= 1; got ${String(requested)}`,
+			);
+		}
+		this.agreementCount = Math.floor(requested);
+	}
+
+	/**
+	 * Feed the latest streaming-ASR partial. Returns the stable / pending
+	 * split. The stable prefix is monotonically non-decreasing across calls
+	 * — once a span has been agreed `n` times it stays committed even if a
+	 * later partial briefly disagrees (the ASR will catch up; rolling back
+	 * would cause downstream stutter).
+	 */
+	feed(partial: string): StabilizerOutput {
+		this.history.push(partial);
+		if (this.history.length > this.agreementCount) {
+			this.history.shift();
+		}
+		if (this.history.length < this.agreementCount) {
+			// Not enough partials yet to confirm anything new — only the
+			// already-committed prefix is stable.
+			return {
+				stable: this.committed,
+				pending: partial.startsWith(this.committed)
+					? partial.slice(this.committed.length)
+					: partial,
+			};
+		}
+		// Intersect: agreed prefix = common prefix across the whole agreement
+		// window.
+		let agreed = this.history[0];
+		for (let i = 1; i < this.history.length; i++) {
+			const sharedLen = commonPrefixLength(agreed, this.history[i]);
+			if (sharedLen < agreed.length) {
+				agreed = agreed.slice(0, sharedLen);
+			}
+			if (agreed.length === 0) break;
+		}
+		if (agreed.length > this.committed.length) {
+			this.committed = agreed;
+		}
+		return {
+			stable: this.committed,
+			pending: partial.startsWith(this.committed)
+				? partial.slice(this.committed.length)
+				: partial,
+		};
+	}
+
+	/** The current committed stable prefix (read-only view). */
+	stable(): string {
+		return this.committed;
+	}
+
+	/** Clear all history. Call at utterance boundaries. */
+	reset(): void {
+		this.history = [];
+		this.committed = "";
+	}
+}

package/src/services/voice/phoneme-tokenizer.ts

@@ -0,0 +1,158 @@
+/**
+ * Phoneme tokenizer interface used by the IPA-mode phrase chunker.
+ *
+ * The chunker consumes a stream of accepted text tokens and re-emits them
+ * as sub-phrase chunks at phoneme boundaries. This lets TTS start
+ * synthesizing partial phrases earlier than the punctuation-only mode, at
+ * the cost of slightly less prosody coherence per chunk.
+ *
+ * The default tokenizer is synchronous because the chunker runs in the
+ * accepted-token hot path. Full espeak-ng / phonemizer integrations can
+ * implement this interface by resolving their native or package dependency
+ * before constructing the scheduler.
+ */
+
+export interface Phoneme {
+	/** IPA symbol(s) for this phoneme. */
+	ipa: string;
+	/** Index of the source `TextToken` this phoneme came from. Used by the
+	 *  chunker to map sub-phrases back to token-index ranges so that the
+	 *  rollback queue can still drop the right audio on a verifier reject. */
+	sourceTokenIndex: number;
+}
+
+export interface PhonemeTokenizer {
+	/** Stable tokenizer name, used for logging and cache keys. */
+	readonly name: string;
+	/** Relative quality signal for telemetry and debugging. */
+	readonly quality: "ipa" | "approximate";
+	/**
+	 * Tokenize a single text token's text into phonemes. The chunker calls
+	 * this once per accepted token; the tokenizer returns the phonemes for
+	 * that token only. Returning an empty array is legal (e.g. whitespace
+	 * tokens) and is treated as "no phoneme boundary added by this token".
+	 */
+	tokenize(text: string, sourceTokenIndex: number): readonly Phoneme[];
+}
+
+const WORD_IPA: Readonly<Record<string, readonly string[]>> = {
+	a: ["ə"],
+	an: ["æ", "n"],
+	and: ["æ", "n", "d"],
+	are: ["ɑː", "r"],
+	be: ["b", "iː"],
+	eliza: ["ə", "l", "iː", "z", "ə"],
+	hello: ["h", "ə", "l", "oʊ"],
+	is: ["ɪ", "z"],
+	of: ["ʌ", "v"],
+	the: ["ð", "ə"],
+	to: ["t", "uː"],
+	world: ["w", "ɜː", "r", "l", "d"],
+};
+
+const DIGRAPH_IPA: Readonly<Record<string, string>> = {
+	ch: "tʃ",
+	ck: "k",
+	ng: "ŋ",
+	ph: "f",
+	qu: "kʷ",
+	sh: "ʃ",
+	th: "θ",
+	wh: "w",
+};
+
+const LETTER_IPA: Readonly<Record<string, string>> = {
+	a: "æ",
+	b: "b",
+	c: "k",
+	d: "d",
+	e: "ɛ",
+	f: "f",
+	g: "ɡ",
+	h: "h",
+	i: "ɪ",
+	j: "dʒ",
+	k: "k",
+	l: "l",
+	m: "m",
+	n: "n",
+	o: "ɑ",
+	p: "p",
+	q: "k",
+	r: "r",
+	s: "s",
+	t: "t",
+	u: "ʌ",
+	v: "v",
+	w: "w",
+	x: "k",
+	y: "j",
+	z: "z",
+};
+
+const DIGIT_IPA: Readonly<Record<string, readonly string[]>> = {
+	"0": ["z", "iː", "r", "oʊ"],
+	"1": ["w", "ʌ", "n"],
+	"2": ["t", "uː"],
+	"3": ["θ", "r", "iː"],
+	"4": ["f", "ɔː", "r"],
+	"5": ["f", "aɪ", "v"],
+	"6": ["s", "ɪ", "k", "s"],
+	"7": ["s", "ɛ", "v", "ə", "n"],
+	"8": ["eɪ", "t"],
+	"9": ["n", "aɪ", "n"],
+};
+
+function ipaForWord(word: string): readonly string[] {
+	const known = WORD_IPA[word];
+	if (known) return known;
+
+	const out: string[] = [];
+	for (let i = 0; i < word.length; ) {
+		const pair = word.slice(i, i + 2);
+		const digraph = DIGRAPH_IPA[pair];
+		if (digraph) {
+			out.push(digraph);
+			i += 2;
+			continue;
+		}
+
+		const mapped = LETTER_IPA[word[i]];
+		if (mapped) out.push(mapped);
+		i += 1;
+	}
+	return out;
+}
+
+/**
+ * Synchronous English IPA approximation for phrase chunking.
+ *
+ * This is not a pronunciation dictionary; it is a deterministic tokenizer
+ * whose output is close enough for boundary counting and rollback range
+ * mapping. Deployments that need accent-accurate phonemization can inject a
+ * higher-quality `PhonemeTokenizer` built around espeak-ng or phonemizer.
+ */
+export class RuleBasedEnglishPhonemeTokenizer implements PhonemeTokenizer {
+	readonly name = "RuleBasedEnglishPhonemeTokenizer";
+	readonly quality = "approximate" as const;
+
+	tokenize(text: string, sourceTokenIndex: number): readonly Phoneme[] {
+		const out: Phoneme[] = [];
+		const pieces = text.match(/[A-Za-z]+|\d/g) ?? [];
+
+		for (const piece of pieces) {
+			const phonemes = /^\d$/.test(piece)
+				? (DIGIT_IPA[piece] ?? [])
+				: ipaForWord(piece.toLowerCase());
+			for (const ipa of phonemes) {
+				out.push({ ipa, sourceTokenIndex });
+			}
+		}
+
+		return out;
+	}
+}
+
+export function createDefaultPhonemeTokenizer(): PhonemeTokenizer {
+	return new RuleBasedEnglishPhonemeTokenizer();
+}

package/src/services/voice/phrase-cache.test.ts

@@ -0,0 +1,242 @@
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { EngineVoiceBridge, StubOmniVoiceBackend } from "./engine-bridge";
+import type { VoiceLifecycleLoaders } from "./lifecycle";
+import {
+	canonicalizePhraseText,
+	DEFAULT_PHRASE_CACHE_SEED,
+	estimatePhraseTokenCount,
+	FIRST_AUDIO_FILLERS,
+	PhraseCache,
+} from "./phrase-cache";
+import type { MmapRegionHandle, RefCountedResource } from "./shared-resources";
+import type {
+	AudioChunk,
+	OmniVoiceBackend,
+	Phrase,
+	SpeakerPreset,
+} from "./types";
+import { writeVoicePresetFile } from "./voice-preset-format";
+
+// --- the canonical seed/filler constants -----------------------------------
+
+describe("DEFAULT_PHRASE_CACHE_SEED", () => {
+	it("is non-empty and stored in canonical form (lowercase, single-spaced, trimmed)", () => {
+		expect(DEFAULT_PHRASE_CACHE_SEED.length).toBeGreaterThan(5);
+		for (const text of DEFAULT_PHRASE_CACHE_SEED) {
+			expect(text).toBe(canonicalizePhraseText(text));
+		}
+	});
+
+	it("has no duplicates", () => {
+		expect(new Set(DEFAULT_PHRASE_CACHE_SEED).size).toBe(
+			DEFAULT_PHRASE_CACHE_SEED.length,
+		);
+	});
+
+	it("includes the documented common openers/acks", () => {
+		for (const phrase of ["okay", "got it", "one sec", "hmm", "okay so"]) {
+			expect(DEFAULT_PHRASE_CACHE_SEED).toContain(phrase);
+		}
+	});
+});
+
+describe("FIRST_AUDIO_FILLERS", () => {
+	it("is a non-empty subset of DEFAULT_PHRASE_CACHE_SEED", () => {
+		expect(FIRST_AUDIO_FILLERS.length).toBeGreaterThan(0);
+		const seed = new Set(DEFAULT_PHRASE_CACHE_SEED);
+		for (const filler of FIRST_AUDIO_FILLERS) {
+			expect(seed.has(filler)).toBe(true);
+		}
+	});
+});
+
+// --- PhraseCache LRU semantics (new symbols) -------------------------------
+
+describe("PhraseCache LRU", () => {
+	it("estimates short acknowledgement token counts", () => {
+		expect(estimatePhraseTokenCount("  Got it!  ")).toBe(2);
+		expect(estimatePhraseTokenCount("")).toBe(0);
+	});
+
+	it("get() promotes an entry to most-recently-used", () => {
+		const c = new PhraseCache({ maxEntries: 2 });
+		c.put({ text: "a", pcm: new Float32Array([1]), sampleRate: 24000 });
+		c.put({ text: "b", pcm: new Float32Array([2]), sampleRate: 24000 });
+		// Touch "a" — now "b" is the LRU.
+		expect(c.get("a")).toBeDefined();
+		c.put({ text: "c", pcm: new Float32Array([3]), sampleRate: 24000 });
+		expect(c.has("a")).toBe(true);
+		expect(c.has("b")).toBe(false);
+		expect(c.has("c")).toBe(true);
+	});
+
+	it("keeps live phrase caching focused on sub-10-token first sentences", () => {
+		const c = new PhraseCache();
+		expect(
+			c.put({
+				text: "one two three four five six seven eight nine",
+				pcm: new Float32Array([1]),
+				sampleRate: 24000,
+			}),
+		).toBe(true);
+		expect(
+			c.put({
+				text: "one two three four five six seven eight nine ten",
+				pcm: new Float32Array([2]),
+				sampleRate: 24000,
+			}),
+		).toBe(false);
+		expect(c.has("one two three four five six seven eight nine")).toBe(true);
+		expect(c.has("one two three four five six seven eight nine ten")).toBe(
+			false,
+		);
+	});
+});
+
+// --- engine-bridge prewarm / filler gating ---------------------------------
+
+class RecordingBackend implements OmniVoiceBackend {
+	readonly synthesized: string[] = [];
+	async synthesize(args: {
+		phrase: Phrase;
+		preset: SpeakerPreset;
+		cancelSignal: { cancelled: boolean };
+		onKernelTick?: () => void;
+	}): Promise<AudioChunk> {
+		this.synthesized.push(args.phrase.text);
+		args.onKernelTick?.();
+		// Distinct non-zero PCM so the phrase cache holds real audio.
+		return {
+			phraseId: args.phrase.id,
+			fromIndex: args.phrase.fromIndex,
+			toIndex: args.phrase.toIndex,
+			pcm: new Float32Array([0.3, -0.3, 0.2]),
+			sampleRate: 24000,
+		};
+	}
+}
+
+function lifecycleLoadersOk(): VoiceLifecycleLoaders {
+	const region: MmapRegionHandle = {
+		id: "region-ok",
+		path: "/tmp/region-ok",
+		sizeBytes: 1024,
+		async evictPages() {},
+		async release() {},
+	};
+	const refc: RefCountedResource = { id: "refc-ok", async release() {} };
+	return {
+		loadTtsRegion: async () => region,
+		loadAsrRegion: async () => region,
+		loadVoiceCaches: async () => refc,
+		loadVoiceSchedulerNodes: async () => refc,
+	};
+}
+
+function writePresetBundle(root: string): void {
+	mkdirSync(path.join(root, "cache"), { recursive: true });
+	const embedding = new Float32Array(8);
+	for (let i = 0; i < embedding.length; i++) embedding[i] = (i + 1) / 100;
+	writeFileSync(
+		path.join(root, "cache", "voice-preset-default.bin"),
+		Buffer.from(writeVoicePresetFile({ embedding, phrases: [] })),
+	);
+}
+
+describe("EngineVoiceBridge phrase prewarm + first-audio filler", () => {
+	let bundleRoot: string;
+
+	beforeEach(() => {
+		bundleRoot = mkdtempSync(path.join(tmpdir(), "eliza-phrase-cache-"));
+		writePresetBundle(bundleRoot);
+	});
+
+	afterEach(() => {
+		rmSync(bundleRoot, { recursive: true, force: true });
+	});
+
+	it("hasRealTtsBackend() is false for the silent backend", () => {
+		const bridge = EngineVoiceBridge.start({
+			bundleRoot,
+			useFfiBackend: false,
+		});
+		expect(bridge.backend).toBeInstanceOf(StubOmniVoiceBackend);
+		expect(bridge.hasRealTtsBackend()).toBe(false);
+	});
+
+	it("prewarmIdlePhrases() is a no-op without a real backend (never caches zeros)", async () => {
+		const bridge = EngineVoiceBridge.start({
+			bundleRoot,
+			useFfiBackend: false,
+			lifecycleLoaders: lifecycleLoadersOk(),
+		});
+		await bridge.arm();
+		const result = await bridge.prewarmIdlePhrases();
+		expect(result).toEqual({ warmed: 0, cached: 0 });
+	});
+
+	it("playFirstAudioFiller() is a no-op without a real backend", async () => {
+		const bridge = EngineVoiceBridge.start({
+			bundleRoot,
+			useFfiBackend: false,
+			lifecycleLoaders: lifecycleLoadersOk(),
+		});
+		await bridge.arm();
+		expect(bridge.playFirstAudioFiller()).toBeNull();
+	});
+
+	it("prewarmIdlePhrases() is a no-op when voice is not armed", async () => {
+		const backend = new RecordingBackend();
+		const bridge = EngineVoiceBridge.start({
+			bundleRoot,
+			useFfiBackend: false,
+			backendOverride: backend,
+		});
+		// Not armed.
+		const result = await bridge.prewarmIdlePhrases();
+		expect(result).toEqual({ warmed: 0, cached: 0 });
+		expect(backend.synthesized).toHaveLength(0);
+	});
+
+	it("prewarmIdlePhrases() synthesizes the seed list with a real backend", async () => {
+		const backend = new RecordingBackend();
+		const bridge = EngineVoiceBridge.start({
+			bundleRoot,
+			useFfiBackend: false,
+			backendOverride: backend,
+			lifecycleLoaders: lifecycleLoadersOk(),
+		});
+		expect(bridge.hasRealTtsBackend()).toBe(true);
+		await bridge.arm();
+		const result = await bridge.prewarmIdlePhrases();
+		expect(result.warmed).toBe(DEFAULT_PHRASE_CACHE_SEED.length);
+		expect(result.cached).toBe(0);
+		expect(new Set(backend.synthesized)).toEqual(
+			new Set(DEFAULT_PHRASE_CACHE_SEED),
+		);
+		// A second pass is fully cached.
+		const second = await bridge.prewarmIdlePhrases();
+		expect(second.cached).toBe(DEFAULT_PHRASE_CACHE_SEED.length);
+		expect(second.warmed).toBe(0);
+	});
+
+	it("playFirstAudioFiller() plays a cached filler once a real backend prewarmed it", async () => {
+		const backend = new RecordingBackend();
+		const bridge = EngineVoiceBridge.start({
+			bundleRoot,
+			useFfiBackend: false,
+			backendOverride: backend,
+			lifecycleLoaders: lifecycleLoadersOk(),
+		});
+		await bridge.arm();
+		// No filler cached yet.
+		expect(bridge.playFirstAudioFiller()).toBeNull();
+		await bridge.prewarmIdlePhrases();
+		const played = bridge.playFirstAudioFiller();
+		expect(played).not.toBeNull();
+		expect(FIRST_AUDIO_FILLERS).toContain(played);
+	});
+});