@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/services/voice/phrase-cache.test.ts

@@ -0,0 +1,242 @@
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { EngineVoiceBridge, StubOmniVoiceBackend } from "./engine-bridge";
+import type { VoiceLifecycleLoaders } from "./lifecycle";
+import {
+	canonicalizePhraseText,
+	DEFAULT_PHRASE_CACHE_SEED,
+	estimatePhraseTokenCount,
+	FIRST_AUDIO_FILLERS,
+	PhraseCache,
+} from "./phrase-cache";
+import type { MmapRegionHandle, RefCountedResource } from "./shared-resources";
+import type {
+	AudioChunk,
+	OmniVoiceBackend,
+	Phrase,
+	SpeakerPreset,
+} from "./types";
+import { writeVoicePresetFile } from "./voice-preset-format";
+
+// --- the canonical seed/filler constants -----------------------------------
+
+describe("DEFAULT_PHRASE_CACHE_SEED", () => {
+	it("is non-empty and stored in canonical form (lowercase, single-spaced, trimmed)", () => {
+		expect(DEFAULT_PHRASE_CACHE_SEED.length).toBeGreaterThan(5);
+		for (const text of DEFAULT_PHRASE_CACHE_SEED) {
+			expect(text).toBe(canonicalizePhraseText(text));
+		}
+	});
+
+	it("has no duplicates", () => {
+		expect(new Set(DEFAULT_PHRASE_CACHE_SEED).size).toBe(
+			DEFAULT_PHRASE_CACHE_SEED.length,
+		);
+	});
+
+	it("includes the documented common openers/acks", () => {
+		for (const phrase of ["okay", "got it", "one sec", "hmm", "okay so"]) {
+			expect(DEFAULT_PHRASE_CACHE_SEED).toContain(phrase);
+		}
+	});
+});
+
+describe("FIRST_AUDIO_FILLERS", () => {
+	it("is a non-empty subset of DEFAULT_PHRASE_CACHE_SEED", () => {
+		expect(FIRST_AUDIO_FILLERS.length).toBeGreaterThan(0);
+		const seed = new Set(DEFAULT_PHRASE_CACHE_SEED);
+		for (const filler of FIRST_AUDIO_FILLERS) {
+			expect(seed.has(filler)).toBe(true);
+		}
+	});
+});
+
+// --- PhraseCache LRU semantics (new symbols) -------------------------------
+
+describe("PhraseCache LRU", () => {
+	it("estimates short acknowledgement token counts", () => {
+		expect(estimatePhraseTokenCount("  Got it!  ")).toBe(2);
+		expect(estimatePhraseTokenCount("")).toBe(0);
+	});
+
+	it("get() promotes an entry to most-recently-used", () => {
+		const c = new PhraseCache({ maxEntries: 2 });
+		c.put({ text: "a", pcm: new Float32Array([1]), sampleRate: 24000 });
+		c.put({ text: "b", pcm: new Float32Array([2]), sampleRate: 24000 });
+		// Touch "a" — now "b" is the LRU.
+		expect(c.get("a")).toBeDefined();
+		c.put({ text: "c", pcm: new Float32Array([3]), sampleRate: 24000 });
+		expect(c.has("a")).toBe(true);
+		expect(c.has("b")).toBe(false);
+		expect(c.has("c")).toBe(true);
+	});
+
+	it("keeps live phrase caching focused on sub-10-token first sentences", () => {
+		const c = new PhraseCache();
+		expect(
+			c.put({
+				text: "one two three four five six seven eight nine",
+				pcm: new Float32Array([1]),
+				sampleRate: 24000,
+			}),
+		).toBe(true);
+		expect(
+			c.put({
+				text: "one two three four five six seven eight nine ten",
+				pcm: new Float32Array([2]),
+				sampleRate: 24000,
+			}),
+		).toBe(false);
+		expect(c.has("one two three four five six seven eight nine")).toBe(true);
+		expect(c.has("one two three four five six seven eight nine ten")).toBe(
+			false,
+		);
+	});
+});
+
+// --- engine-bridge prewarm / filler gating ---------------------------------
+
+class RecordingBackend implements OmniVoiceBackend {
+	readonly synthesized: string[] = [];
+	async synthesize(args: {
+		phrase: Phrase;
+		preset: SpeakerPreset;
+		cancelSignal: { cancelled: boolean };
+		onKernelTick?: () => void;
+	}): Promise<AudioChunk> {
+		this.synthesized.push(args.phrase.text);
+		args.onKernelTick?.();
+		// Distinct non-zero PCM so the phrase cache holds real audio.
+		return {
+			phraseId: args.phrase.id,
+			fromIndex: args.phrase.fromIndex,
+			toIndex: args.phrase.toIndex,
+			pcm: new Float32Array([0.3, -0.3, 0.2]),
+			sampleRate: 24000,
+		};
+	}
+}
+
+function lifecycleLoadersOk(): VoiceLifecycleLoaders {
+	const region: MmapRegionHandle = {
+		id: "region-ok",
+		path: "/tmp/region-ok",
+		sizeBytes: 1024,
+		async evictPages() {},
+		async release() {},
+	};
+	const refc: RefCountedResource = { id: "refc-ok", async release() {} };
+	return {
+		loadTtsRegion: async () => region,
+		loadAsrRegion: async () => region,
+		loadVoiceCaches: async () => refc,
+		loadVoiceSchedulerNodes: async () => refc,
+	};
+}
+
+function writePresetBundle(root: string): void {
+	mkdirSync(path.join(root, "cache"), { recursive: true });
+	const embedding = new Float32Array(8);
+	for (let i = 0; i < embedding.length; i++) embedding[i] = (i + 1) / 100;
+	writeFileSync(
+		path.join(root, "cache", "voice-preset-default.bin"),
+		Buffer.from(writeVoicePresetFile({ embedding, phrases: [] })),
+	);
+}
+
+describe("EngineVoiceBridge phrase prewarm + first-audio filler", () => {
+	let bundleRoot: string;
+
+	beforeEach(() => {
+		bundleRoot = mkdtempSync(path.join(tmpdir(), "eliza-phrase-cache-"));
+		writePresetBundle(bundleRoot);
+	});
+
+	afterEach(() => {
+		rmSync(bundleRoot, { recursive: true, force: true });
+	});
+
+	it("hasRealTtsBackend() is false for the silent backend", () => {
+		const bridge = EngineVoiceBridge.start({
+			bundleRoot,
+			useFfiBackend: false,
+		});
+		expect(bridge.backend).toBeInstanceOf(StubOmniVoiceBackend);
+		expect(bridge.hasRealTtsBackend()).toBe(false);
+	});
+
+	it("prewarmIdlePhrases() is a no-op without a real backend (never caches zeros)", async () => {
+		const bridge = EngineVoiceBridge.start({
+			bundleRoot,
+			useFfiBackend: false,
+			lifecycleLoaders: lifecycleLoadersOk(),
+		});
+		await bridge.arm();
+		const result = await bridge.prewarmIdlePhrases();
+		expect(result).toEqual({ warmed: 0, cached: 0 });
+	});
+
+	it("playFirstAudioFiller() is a no-op without a real backend", async () => {
+		const bridge = EngineVoiceBridge.start({
+			bundleRoot,
+			useFfiBackend: false,
+			lifecycleLoaders: lifecycleLoadersOk(),
+		});
+		await bridge.arm();
+		expect(bridge.playFirstAudioFiller()).toBeNull();
+	});
+
+	it("prewarmIdlePhrases() is a no-op when voice is not armed", async () => {
+		const backend = new RecordingBackend();
+		const bridge = EngineVoiceBridge.start({
+			bundleRoot,
+			useFfiBackend: false,
+			backendOverride: backend,
+		});
+		// Not armed.
+		const result = await bridge.prewarmIdlePhrases();
+		expect(result).toEqual({ warmed: 0, cached: 0 });
+		expect(backend.synthesized).toHaveLength(0);
+	});
+
+	it("prewarmIdlePhrases() synthesizes the seed list with a real backend", async () => {
+		const backend = new RecordingBackend();
+		const bridge = EngineVoiceBridge.start({
+			bundleRoot,
+			useFfiBackend: false,
+			backendOverride: backend,
+			lifecycleLoaders: lifecycleLoadersOk(),
+		});
+		expect(bridge.hasRealTtsBackend()).toBe(true);
+		await bridge.arm();
+		const result = await bridge.prewarmIdlePhrases();
+		expect(result.warmed).toBe(DEFAULT_PHRASE_CACHE_SEED.length);
+		expect(result.cached).toBe(0);
+		expect(new Set(backend.synthesized)).toEqual(
+			new Set(DEFAULT_PHRASE_CACHE_SEED),
+		);
+		// A second pass is fully cached.
+		const second = await bridge.prewarmIdlePhrases();
+		expect(second.cached).toBe(DEFAULT_PHRASE_CACHE_SEED.length);
+		expect(second.warmed).toBe(0);
+	});
+
+	it("playFirstAudioFiller() plays a cached filler once a real backend prewarmed it", async () => {
+		const backend = new RecordingBackend();
+		const bridge = EngineVoiceBridge.start({
+			bundleRoot,
+			useFfiBackend: false,
+			backendOverride: backend,
+			lifecycleLoaders: lifecycleLoadersOk(),
+		});
+		await bridge.arm();
+		// No filler cached yet.
+		expect(bridge.playFirstAudioFiller()).toBeNull();
+		await bridge.prewarmIdlePhrases();
+		const played = bridge.playFirstAudioFiller();
+		expect(played).not.toBeNull();
+		expect(FIRST_AUDIO_FILLERS).toContain(played);
+	});
+});

package/src/services/voice/phrase-cache.ts

@@ -0,0 +1,186 @@
+export interface CachedPhraseAudio {
+	text: string;
+	pcm: Float32Array;
+	sampleRate: number;
+}
+
+/**
+ * Canonical seed list for the voice phrase cache: short openers, fillers, and
+ * acknowledgements the assistant emits constantly. Pre-synthesizing these and
+ * holding their PCM in `PhraseCache` removes the TTS forward pass from the
+ * critical path for the most common first utterances — `dispatchPhrase` hits
+ * the cache and writes audio to the ring buffer on the same tick.
+ *
+ * Used by:
+ *   - the preset generator (`scripts/voice-preset/build-default-voice-preset.mjs`),
+ *     which synthesizes these against a real OmniVoice TTS backend and writes
+ *     the PCM into `cache/voice-preset-default.bin` — the seeded source of truth.
+ *   - `EngineVoiceBridge.start()` indirectly: the bundle's preset ships these
+ *     phrases with their PCM, which `PhraseCache.seed(...)` loads at startup.
+ *   - the idle-time auto-prewarm hook (`EngineVoiceBridge.prewarmIdlePhrases`),
+ *     which only runs when a real TTS backend is present — never against the
+ *     silent backend (caching zeros is not a phrase cache).
+ *   - the first-audio filler (`FIRST_AUDIO_FILLERS` is a subset).
+ *
+ * Entries are kept here in canonical form (lowercase, single-spaced, trimmed)
+ * so the preset generator and the runtime agree byte-for-byte on the keys.
+ */
+export const DEFAULT_PHRASE_CACHE_SEED: ReadonlyArray<string> = [
+	// Immediate acknowledgements — "I heard you, working on it".
+	"okay",
+	"got it",
+	"sure",
+	"right",
+	"on it",
+	"one sec",
+	"one second",
+	"let me check",
+	"let me see",
+	"give me a moment",
+	// Conversational openers / fillers — natural sentence starters the planner
+	// emits before the substantive answer streams in.
+	"okay so",
+	"so",
+	"hmm",
+	"well",
+	"alright",
+	"sure thing",
+	"of course",
+	"no problem",
+	"good question",
+	"let me think",
+];
+
+/**
+ * The subset of `DEFAULT_PHRASE_CACHE_SEED` suitable to play the instant VAD
+ * fires `speech-start`, masking first-token latency (AGENTS.md §4 / H4). Kept
+ * short and uncommitted — anything that takes a stance ("of course") is
+ * excluded so the filler never contradicts the eventual reply. The first
+ * entry found in the phrase cache wins.
+ */
+export const FIRST_AUDIO_FILLERS: ReadonlyArray<string> = [
+	"one sec",
+	"okay",
+	"let me check",
+	"hmm",
+	"got it",
+];
+
+export interface PhraseCacheOptions {
+	/** Maximum distinct phrase texts retained. Older non-accessed entries
+	 * are evicted first. */
+	maxEntries?: number;
+	/**
+	 * Opportunistic live-cache guardrail. Voice mode primarily benefits from
+	 * cached acknowledgements and first sentence fragments; longer text is less
+	 * likely to repeat and can evict useful hot phrases.
+	 */
+	maxEstimatedTokensPerEntry?: number;
+	/**
+	 * Guardrail for live opportunistic caching. Long-form direct TTS can be
+	 * megabytes of PCM and is not a good phrase-cache resident.
+	 */
+	maxPcmSamplesPerEntry?: number;
+}
+
+export function canonicalizePhraseText(text: string): string {
+	return text.trim().toLowerCase().replace(/\s+/g, " ");
+}
+
+export function estimatePhraseTokenCount(text: string): number {
+	const normalized = canonicalizePhraseText(text);
+	if (!normalized) return 0;
+	return normalized.split(/\s+/).length;
+}
+
+const DEFAULT_MAX_ENTRIES = 128;
+const DEFAULT_MAX_ESTIMATED_TOKENS_PER_ENTRY = 9;
+const DEFAULT_MAX_PCM_SAMPLES_PER_ENTRY = 24000 * 8;
+
+export class PhraseCache {
+	private readonly entries = new Map<string, CachedPhraseAudio>();
+	private readonly maxEntries: number;
+	private readonly maxEstimatedTokensPerEntry: number;
+	private readonly maxPcmSamplesPerEntry: number;
+
+	constructor(opts: PhraseCacheOptions = {}) {
+		this.maxEntries = Math.max(
+			1,
+			Math.floor(opts.maxEntries ?? DEFAULT_MAX_ENTRIES),
+		);
+		this.maxEstimatedTokensPerEntry = Math.max(
+			1,
+			Math.floor(
+				opts.maxEstimatedTokensPerEntry ??
+					DEFAULT_MAX_ESTIMATED_TOKENS_PER_ENTRY,
+			),
+		);
+		this.maxPcmSamplesPerEntry = Math.max(
+			1,
+			Math.floor(
+				opts.maxPcmSamplesPerEntry ?? DEFAULT_MAX_PCM_SAMPLES_PER_ENTRY,
+			),
+		);
+	}
+
+	put(entry: CachedPhraseAudio): boolean {
+		const key = canonicalizePhraseText(entry.text);
+		if (!key) return false;
+		if (entry.pcm.length > this.maxPcmSamplesPerEntry) return false;
+		if (
+			estimatePhraseTokenCount(entry.text) > this.maxEstimatedTokensPerEntry
+		) {
+			return false;
+		}
+		this.entries.delete(key);
+		this.entries.set(key, entry);
+		this.evictOverflow();
+		return true;
+	}
+
+	/**
+	 * Pre-populate the cache from a voice-preset seed list. Texts are stored
+	 * verbatim — callers (the format reader) are responsible for canonicalizing
+	 * before serialization, but we re-canonicalize on insert to be safe.
+	 */
+	seed(
+		entries: ReadonlyArray<{
+			text: string;
+			pcm: Float32Array;
+			sampleRate: number;
+		}>,
+	): void {
+		for (const e of entries) {
+			this.put({
+				text: e.text,
+				pcm: e.pcm,
+				sampleRate: e.sampleRate,
+			});
+		}
+	}
+
+	get(text: string): CachedPhraseAudio | undefined {
+		const key = canonicalizePhraseText(text);
+		const entry = this.entries.get(key);
+		if (!entry) return undefined;
+		this.entries.delete(key);
+		this.entries.set(key, entry);
+		return entry;
+	}
+
+	has(text: string): boolean {
+		return this.entries.has(canonicalizePhraseText(text));
+	}
+
+	size(): number {
+		return this.entries.size;
+	}
+
+	private evictOverflow(): void {
+		while (this.entries.size > this.maxEntries) {
+			const oldest = this.entries.keys().next().value;
+			if (oldest === undefined) return;
+			this.entries.delete(oldest);
+		}
+	}
+}

package/src/services/voice/phrase-chunker.d.ts

@@ -0,0 +1,62 @@
+import type { PhonemeTokenizer } from "./phoneme-tokenizer";
+import type { AcceptedToken, Phrase, PhraseChunkerConfig, TextToken } from "./types";
+/** Wall-clock source the chunker uses. Tests inject a deterministic clock. */
+export type ClockMs = () => number;
+export declare class PhraseChunker {
+    private buffer;
+    private nextPhraseId;
+    private readonly terminators;
+    private readonly chunkOn;
+    private readonly phonemesPerChunk;
+    private readonly maxTokensPerPhrase;
+    private readonly tokenizer;
+    private phonemeCount;
+    /**
+     * T3 — time-budget flush. `firstTokenAtMs` is captured on the first
+     * `push()` after an empty buffer; once `clock() - firstTokenAtMs >=
+     * maxAccumulationMs` the chunker force-flushes even without a
+     * punctuation / phoneme / cap boundary. `maxAccumulationMs <= 0`
+     * disables the time budget.
+     */
+    private readonly maxAccumulationMs;
+    /** Shorter budget applied only while no phrase has flushed yet this reply. */
+    private readonly firstPhraseMaxAccumulationMs;
+    private readonly clock;
+    private firstTokenAtMs;
+    /** Phrases emitted since the last {@link reset}; gates the first-phrase budget. */
+    private phrasesEmitted;
+    constructor(config: PhraseChunkerConfig, tokenizer?: PhonemeTokenizer | null, clock?: ClockMs);
+    push(token: AcceptedToken): Phrase | null;
+    /** Active time budget: the shorter first-phrase budget until the reply's
+     * first phrase has flushed, then the full budget. */
+    private currentBudgetMs;
+    /**
+     * T3 — caller-driven check. Returns a phrase when the time budget has
+     * elapsed for the current buffer, otherwise null. The scheduler polls
+     * this from a `setTimeout` so even a producer that goes silent before
+     * pushing the next token still gets its in-flight phrase flushed.
+     */
+    flushIfTimeBudgetExceeded(): Phrase | null;
+    /**
+     * T3 — milliseconds remaining until the time budget elapses for the
+     * current buffer. Negative when the budget has already been exceeded;
+     * `Number.POSITIVE_INFINITY` when the buffer is empty or the budget is
+     * disabled. Callers compute their flush timer off this.
+     */
+    msUntilTimeBudget(): number;
+    flushPending(): Phrase | null;
+    /**
+     * Drop buffered tokens that have not flushed whose token index is ≥
+     * `fromIndex`. Used by the pipeline's rollback path: when the target
+     * verifier rejects a draft tail, any draft tokens still sitting in the
+     * chunker's buffer before phrase packing MUST be discarded so
+     * the verifier's correction does not get glued onto stale text.
+     * Phonemes are recounted from scratch over what remains.
+     */
+    dropPendingFrom(fromIndex: number): void;
+    reset(): void;
+    private endsWithTerminator;
+    private flushAs;
+}
+export declare function chunkTokens(tokens: TextToken[], config: PhraseChunkerConfig, acceptedAt?: number, tokenizer?: PhonemeTokenizer | null): Phrase[];
+//# sourceMappingURL=phrase-chunker.d.ts.map

package/src/services/voice/phrase-chunker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"phrase-chunker.d.ts","sourceRoot":"","sources":["phrase-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,KAAK,EACX,aAAa,EACb,MAAM,EACN,mBAAmB,EACnB,SAAS,EACT,MAAM,SAAS,CAAC;AA8DjB,8EAA8E;AAC9E,MAAM,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC;AAInC,qBAAa,aAAa;IACzB,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAsB;IAClD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAC3D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA0B;IACpD,OAAO,CAAC,YAAY,CAAK;IACzB;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAC3C,8EAA8E;IAC9E,OAAO,CAAC,QAAQ,CAAC,4BAA4B,CAAS;IACtD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAU;IAChC,OAAO,CAAC,cAAc,CAAK;IAC3B,mFAAmF;IACnF,OAAO,CAAC,cAAc,CAAK;gBAG1B,MAAM,EAAE,mBAAmB,EAC3B,SAAS,GAAE,gBAAgB,GAAG,IAAW,EACzC,KAAK,GAAE,OAAuB;IAgC/B,IAAI,CAAC,KAAK,EAAE,aAAa,GAAG,MAAM,GAAG,IAAI;IAmCzC;yDACqD;IACrD,OAAO,CAAC,eAAe;IAMvB;;;;;OAKG;IACH,yBAAyB,IAAI,MAAM,GAAG,IAAI;IAU1C;;;;;OAKG;IACH,iBAAiB,IAAI,MAAM;IAO3B,YAAY,IAAI,MAAM,GAAG,IAAI;IAK7B;;;;;;;OAOG;IACH,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAexC,KAAK,IAAI,IAAI;IAOb,OAAO,CAAC,kBAAkB;IAM1B,OAAO,CAAC,OAAO;CAkBf;AAED,wBAAgB,WAAW,CAC1B,MAAM,EAAE,SAAS,EAAE,EACnB,MAAM,EAAE,mBAAmB,EAC3B,UAAU,SAAI,EACd,SAAS,GAAE,gBAAgB,GAAG,IAAW,GACvC,MAAM,EAAE,CAUV"}