@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/services/vision-embedding-cache.d.ts

@@ -0,0 +1,98 @@
+/**
+ * Content-hashed cache for projected vision-language tokens (WS1 deliverable).
+ *
+ * Vision models in the Eliza-1 stack (Qwen3-VL, Florence-2, Apothic-VL) all
+ * go through the same expensive projector step: raw pixel
+ * bytes → patch embeddings → projector → tokens that the text decoder
+ * actually consumes. When the user pastes the same screenshot three times
+ * in a row, or when computer-use takes near-duplicate frames of an idle
+ * screen, we want to skip the projector entirely and reuse the cached
+ * tokens.
+ *
+ * Contract:
+ *   - Caller computes a stable hash of the *normalized* input bytes
+ *     (downscaled to the model's input resolution, then SHA-256 of the
+ *     packed pixels). The hash is the cache key.
+ *   - Caller pairs the hash with the projected token tensor (a flat
+ *     `Float32Array` of length `tokens * hiddenSize`) AND the geometry
+ *     `{ tokens, hiddenSize }` so a reader can reshape on the way out.
+ *   - `get(hash)` returns `null` on miss or expiry, the entry on hit.
+ *     A hit also "touches" the entry to keep it warm under LRU.
+ *   - `set(hash, entry, ttlMs?)` inserts with a TTL (default 5 min); if
+ *     the LRU is full, the coldest entry is evicted.
+ *
+ * Why a separate module:
+ *   - The arbiter owns the *model handle*; the cache holds *per-input
+ *     projected weights* that survive across model loads/unloads of the
+ *     same family. Keeping the cache in a sibling module lets the vision
+ *     plugin reuse it even when the arbiter swapped the underlying model
+ *     for memory pressure (the projector tokens are still valid as long
+ *     as the model family + hash match — we encode the family in the key
+ *     to be safe).
+ *
+ * What this is NOT:
+ *   - A blob cache for the encoder *weights*. Those live in mmap regions
+ *     owned by the arbiter / SharedResourceRegistry and are evicted via
+ *     `MmapRegionHandle.evictPages()`.
+ *   - A cache for downstream LLM generations. Prefix-cache for text is
+ *     handled by `cache-bridge.ts` and the backend session pool.
+ */
+export interface VisionEmbeddingEntry {
+    /** Flat row-major buffer: `tokenCount * hiddenSize` floats. */
+    tokens: Float32Array;
+    tokenCount: number;
+    hiddenSize: number;
+    /** True when this entry is still within its TTL. */
+    live: boolean;
+}
+export interface VisionEmbeddingCacheConfig {
+    /** Max entries retained. LRU evicts beyond this. Default 32. */
+    maxEntries: number;
+    /** Default TTL when `set()` is called without one. Default 5 min. */
+    defaultTtlMs: number;
+}
+export declare class VisionEmbeddingCache {
+    private readonly config;
+    /**
+     * `Map` preserves insertion order; we re-insert on hit to bubble entries
+     * to the back, so the first key in iteration order is the LRU candidate.
+     */
+    private readonly entries;
+    private readonly now;
+    constructor(opts?: {
+        config?: Partial<VisionEmbeddingCacheConfig>;
+        now?: () => number;
+    });
+    /**
+     * Lookup. Returns the entry on hit (and refreshes LRU position), or null
+     * on miss / expiry. Expired entries are deleted on read so they don't
+     * silently consume the LRU budget.
+     */
+    get(hash: string): VisionEmbeddingEntry | null;
+    /**
+     * Insert. Replaces any existing entry under the same hash. Evicts the
+     * coldest entry if we're at capacity. `ttlMs` overrides the configured
+     * default; pass 0 to use the default.
+     */
+    set(hash: string, entry: {
+        tokens: Float32Array;
+        tokenCount: number;
+        hiddenSize: number;
+    }, ttlMs?: number): void;
+    /** Diagnostic: current entry count. */
+    size(): number;
+    /** Diagnostic: snapshot of (hash, byteSize, expiresAtMs) for each entry. */
+    snapshot(): ReadonlyArray<{
+        hash: string;
+        bytes: number;
+        expiresAtMs: number;
+    }>;
+    /** Drop everything. Cheap; only releases JS-side refs to the Float32Arrays. */
+    clear(): void;
+    /**
+     * Drop entries whose TTL has expired. Returns the number removed. Cheap
+     * to call from the arbiter's pressure tick.
+     */
+    purgeExpired(nowMs?: number): number;
+}
+//# sourceMappingURL=vision-embedding-cache.d.ts.map

package/src/services/vision-embedding-cache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vision-embedding-cache.d.ts","sourceRoot":"","sources":["vision-embedding-cache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AASH,MAAM,WAAW,oBAAoB;IACpC,+DAA+D;IAC/D,MAAM,EAAE,YAAY,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,oDAAoD;IACpD,IAAI,EAAE,OAAO,CAAC;CACd;AAED,MAAM,WAAW,0BAA0B;IAC1C,gEAAgE;IAChE,UAAU,EAAE,MAAM,CAAC;IACnB,qEAAqE;IACrE,YAAY,EAAE,MAAM,CAAC;CACrB;AAOD,qBAAa,oBAAoB;IAChC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA6B;IACpD;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiC;IACzD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;gBAGlC,IAAI,GAAE;QACL,MAAM,CAAC,EAAE,OAAO,CAAC,0BAA0B,CAAC,CAAC;QAC7C,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;KACd;IAYP;;;;OAIG;IACH,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,oBAAoB,GAAG,IAAI;IAkB9C;;;;OAIG;IACH,GAAG,CACF,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE;QAAE,MAAM,EAAE,YAAY,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,EACvE,KAAK,CAAC,EAAE,MAAM,GACZ,IAAI;IAsBP,uCAAuC;IACvC,IAAI,IAAI,MAAM;IAId,4EAA4E;IAC5E,QAAQ,IAAI,aAAa,CAAC;QACzB,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;KACpB,CAAC;IAYF,+EAA+E;IAC/E,KAAK,IAAI,IAAI;IAIb;;;OAGG;IACH,YAAY,CAAC,KAAK,GAAE,MAAmB,GAAG,MAAM;CAUhD"}

package/src/services/vision-embedding-cache.ts

@@ -0,0 +1,189 @@
+/**
+ * Content-hashed cache for projected vision-language tokens (WS1 deliverable).
+ *
+ * Vision models in the Eliza-1 stack (Qwen3-VL, Florence-2, Apothic-VL) all
+ * go through the same expensive projector step: raw pixel
+ * bytes → patch embeddings → projector → tokens that the text decoder
+ * actually consumes. When the user pastes the same screenshot three times
+ * in a row, or when computer-use takes near-duplicate frames of an idle
+ * screen, we want to skip the projector entirely and reuse the cached
+ * tokens.
+ *
+ * Contract:
+ *   - Caller computes a stable hash of the *normalized* input bytes
+ *     (downscaled to the model's input resolution, then SHA-256 of the
+ *     packed pixels). The hash is the cache key.
+ *   - Caller pairs the hash with the projected token tensor (a flat
+ *     `Float32Array` of length `tokens * hiddenSize`) AND the geometry
+ *     `{ tokens, hiddenSize }` so a reader can reshape on the way out.
+ *   - `get(hash)` returns `null` on miss or expiry, the entry on hit.
+ *     A hit also "touches" the entry to keep it warm under LRU.
+ *   - `set(hash, entry, ttlMs?)` inserts with a TTL (default 5 min); if
+ *     the LRU is full, the coldest entry is evicted.
+ *
+ * Why a separate module:
+ *   - The arbiter owns the *model handle*; the cache holds *per-input
+ *     projected weights* that survive across model loads/unloads of the
+ *     same family. Keeping the cache in a sibling module lets the vision
+ *     plugin reuse it even when the arbiter swapped the underlying model
+ *     for memory pressure (the projector tokens are still valid as long
+ *     as the model family + hash match — we encode the family in the key
+ *     to be safe).
+ *
+ * What this is NOT:
+ *   - A blob cache for the encoder *weights*. Those live in mmap regions
+ *     owned by the arbiter / SharedResourceRegistry and are evicted via
+ *     `MmapRegionHandle.evictPages()`.
+ *   - A cache for downstream LLM generations. Prefix-cache for text is
+ *     handled by `cache-bridge.ts` and the backend session pool.
+ */
+
+interface CacheEntry {
+	tokens: Float32Array;
+	tokenCount: number;
+	hiddenSize: number;
+	expiresAtMs: number;
+}
+
+export interface VisionEmbeddingEntry {
+	/** Flat row-major buffer: `tokenCount * hiddenSize` floats. */
+	tokens: Float32Array;
+	tokenCount: number;
+	hiddenSize: number;
+	/** True when this entry is still within its TTL. */
+	live: boolean;
+}
+
+export interface VisionEmbeddingCacheConfig {
+	/** Max entries retained. LRU evicts beyond this. Default 32. */
+	maxEntries: number;
+	/** Default TTL when `set()` is called without one. Default 5 min. */
+	defaultTtlMs: number;
+}
+
+const DEFAULTS: VisionEmbeddingCacheConfig = {
+	maxEntries: 32,
+	defaultTtlMs: 5 * 60_000,
+};
+
+export class VisionEmbeddingCache {
+	private readonly config: VisionEmbeddingCacheConfig;
+	/**
+	 * `Map` preserves insertion order; we re-insert on hit to bubble entries
+	 * to the back, so the first key in iteration order is the LRU candidate.
+	 */
+	private readonly entries = new Map<string, CacheEntry>();
+	private readonly now: () => number;
+
+	constructor(
+		opts: {
+			config?: Partial<VisionEmbeddingCacheConfig>;
+			now?: () => number;
+		} = {},
+	) {
+		this.config = {
+			maxEntries: Math.max(1, opts.config?.maxEntries ?? DEFAULTS.maxEntries),
+			defaultTtlMs: Math.max(
+				0,
+				opts.config?.defaultTtlMs ?? DEFAULTS.defaultTtlMs,
+			),
+		};
+		this.now = opts.now ?? (() => Date.now());
+	}
+
+	/**
+	 * Lookup. Returns the entry on hit (and refreshes LRU position), or null
+	 * on miss / expiry. Expired entries are deleted on read so they don't
+	 * silently consume the LRU budget.
+	 */
+	get(hash: string): VisionEmbeddingEntry | null {
+		const found = this.entries.get(hash);
+		if (!found) return null;
+		if (found.expiresAtMs <= this.now()) {
+			this.entries.delete(hash);
+			return null;
+		}
+		// Touch — re-insert so it moves to the back of the iteration order.
+		this.entries.delete(hash);
+		this.entries.set(hash, found);
+		return {
+			tokens: found.tokens,
+			tokenCount: found.tokenCount,
+			hiddenSize: found.hiddenSize,
+			live: true,
+		};
+	}
+
+	/**
+	 * Insert. Replaces any existing entry under the same hash. Evicts the
+	 * coldest entry if we're at capacity. `ttlMs` overrides the configured
+	 * default; pass 0 to use the default.
+	 */
+	set(
+		hash: string,
+		entry: { tokens: Float32Array; tokenCount: number; hiddenSize: number },
+		ttlMs?: number,
+	): void {
+		if (entry.tokens.length !== entry.tokenCount * entry.hiddenSize) {
+			throw new Error(
+				`[vision-embedding-cache] token buffer length ${entry.tokens.length} does not match tokenCount*hiddenSize (${entry.tokenCount}*${entry.hiddenSize})`,
+			);
+		}
+		const ttl = ttlMs && ttlMs > 0 ? ttlMs : this.config.defaultTtlMs;
+		const expiresAtMs = this.now() + ttl;
+		this.entries.delete(hash);
+		this.entries.set(hash, {
+			tokens: entry.tokens,
+			tokenCount: entry.tokenCount,
+			hiddenSize: entry.hiddenSize,
+			expiresAtMs,
+		});
+		while (this.entries.size > this.config.maxEntries) {
+			const firstKey = this.entries.keys().next().value;
+			if (firstKey === undefined) break;
+			this.entries.delete(firstKey);
+		}
+	}
+
+	/** Diagnostic: current entry count. */
+	size(): number {
+		return this.entries.size;
+	}
+
+	/** Diagnostic: snapshot of (hash, byteSize, expiresAtMs) for each entry. */
+	snapshot(): ReadonlyArray<{
+		hash: string;
+		bytes: number;
+		expiresAtMs: number;
+	}> {
+		const out: { hash: string; bytes: number; expiresAtMs: number }[] = [];
+		for (const [hash, entry] of this.entries) {
+			out.push({
+				hash,
+				bytes: entry.tokens.byteLength,
+				expiresAtMs: entry.expiresAtMs,
+			});
+		}
+		return out;
+	}
+
+	/** Drop everything. Cheap; only releases JS-side refs to the Float32Arrays. */
+	clear(): void {
+		this.entries.clear();
+	}
+
+	/**
+	 * Drop entries whose TTL has expired. Returns the number removed. Cheap
+	 * to call from the arbiter's pressure tick.
+	 */
+	purgeExpired(nowMs: number = this.now()): number {
+		let removed = 0;
+		for (const [hash, entry] of this.entries) {
+			if (entry.expiresAtMs <= nowMs) {
+				this.entries.delete(hash);
+				removed++;
+			}
+		}
+		return removed;
+	}
+}

package/src/services/voice/VOICE_WORKBENCH.md

@@ -0,0 +1,88 @@
+# Voice Workbench
+
+Tracking issue: [elizaOS/eliza#8785](https://github.com/elizaOS/eliza/issues/8785).
+
+elizaOS ships a mature voice pipeline (VAD, streaming ASR, EOT classifier,
+barge-in, diarization, speaker imprint/profiles, Kokoro/OmniVoice TTS) but its
+test harnesses were **fragmented** across five families with no shared scenario
+format, no shared corpus, divergent metric definitions, and a headful surface
+that only covered a single-speaker, single-turn round-trip. The Voice Workbench
+unifies them onto **one scenario format, one metric module, and one report**.
+
+## Status
+
+This directory holds the **pure, framework-level foundation** — the parts that
+can be implemented, tested, and shipped without an audio corpus, native models,
+or a browser. The execution runners that actually drive real services/audio are
+intentionally **gated** (they need a provisioned Eliza-1 local backend + a
+synthesized corpus) and are listed under *Remaining* below.
+
+### Implemented (this directory, unit-tested, no native artifacts)
+
+| Piece | File | What it is |
+| --- | --- | --- |
+| **Scenario schema** | `voice-scenario.ts` | The declarative `VoiceScenario` format: named `participants` (voice→entity), ordered `turns` (`expectRespond`, `expectedTranscript`, `expectedSpeakerLabel`, `expectedEntity`, `pausesMs`), scenario `assertions` (WER/DER/EOT/latency ceilings), and `classes`. Pure `validateVoiceScenario` reports every consistency error at once. |
+| **Metric module (single source of truth)** | `e2e-harness.ts` | All voice scoring lives here. WER is delegated to `@elizaos/shared/voice-wer` (one definition for headless + headful). Added scorers: `scoreEotDecision` (latency p50/p95 + false-trigger/false-suppression rate), `scoreRespondDecision` (FP/FN split), `scoreDiarization` (DER + confusions/misses), `scoreEntityExtraction` (precision/recall/F1), `scoreVoiceEntityMatch` (recognized-voice→entity accuracy). |
+| **Benchmark report** | `voice-workbench-report.ts` | `buildVoiceWorkbenchReport` rolls a matrix of per-scenario scorer results into one gating report (per-metric mean/worst + percentiles, per-scenario verdict). `formatVoiceWorkbenchMarkdown` renders it; `regressionsAgainstBaseline` flags metrics that worsened past a tolerance. |
+| **WER consolidation** | `@elizaos/shared/voice-wer` | The previously-duplicated `wordErrorRate` (`e2e-harness.ts` **and** `voice-selftest-harness.ts`, with subtly different normalization) is now defined once — Unicode-aware, contraction-preserving — and imported by both. |
+
+Tests: `voice-workbench.test.ts`, `voice-workbench-report.test.ts`,
+`e2e-harness.test.ts`.
+
+### Honesty contract
+
+A scenario whose corpus/backend artifacts are absent is reported `skipped`,
+**never `pass`** — matching the existing self-test contract. A workbench report
+is `skipped` overall only when *every* scenario was skipped; one ran-and-failed
+scenario makes the whole report `fail`.
+
+## Execution modes (the three the schema feeds)
+
+1. **Headless** — feed corpus audio through the real services without a browser:
+   `/api/asr/local-inference`, `LiveDiarizationSession` / `/api/voice/audio-frames`,
+   the `ELIZA_VOICE_EOT_BACKEND` classifier, respond/room decisions over a real
+   `AgentRuntime` (scenario-runner PGLite boot), `VOICE_TURN_OBSERVED` /
+   `VOICE_ENTITY_BOUND` / `IDENTIFY_SPEAKER`, and `/api/tts/local-inference`.
+2. **Headful** — extend `VoiceSelfTestShell` (`packages/ui/src/voice/voice-selftest/`)
+   from a single-turn self-test into a scenario player that drives the real
+   client pipeline (capture → ASR → SSE → TTS → playback) turn-by-turn, with
+   per-turn machine-readable + DOM-mirrored verdicts.
+3. **Benchmark/report** — a single `voice:workbench` entrypoint that runs the
+   matrix in both modes and rolls up via `voice-workbench-report.ts` into one
+   JSON + Markdown report with regression baselines.
+
+All three consume the **same** `VoiceScenario` and the **same** scorers, so a
+metric is defined exactly once regardless of where the audio is driven.
+
+## Consolidation map (what converges here)
+
+The workbench is the convergence point for these previously-disjoint harnesses:
+
+| Legacy harness | Convergence |
+| --- | --- |
+| `e2e-harness.ts:wordErrorRate` + `voice-selftest-harness.ts:wordErrorRate` | **Done** — one `@elizaos/shared/voice-wer`. |
+| Pure scoring lib (`e2e-harness.ts`) | **Promoted** to the single metric module (EOT/diarization/respond/entity scorers added). |
+| `packages/app-core/scripts/voice-duet.mjs` (`voice:duet`), `voice-e2e-hardware.ts`, `voice-vad-smoke.ts`, `voice-attribution-smoke.ts`, `lib/duet-bridge.mjs` | Feed measurements into the shared scorers + report (planned absorb). |
+| `packages/benchmarks/voice/three-voice-scenario.mjs`, `three-voice-e2e-real.mjs` | Corpus-generation precedent the `VoiceScenario` corpus generator extends (planned). |
+| `packages/benchmarks/voicebench/` (TS latency p95/p99) | The report layer mirrors its p95/p99 shape; remains a research bench linked from the workbench. |
+| Per-spec inline `tinyWav()` fixtures (`packages/app/test/ui-smoke/voice-*.spec.ts`) | Replaced by the versioned corpus (planned). |
+
+## Remaining (gated — needs corpus + real backend)
+
+These are tracked on #8785 and are **not** stubbed here (no LARP):
+
+- **Corpus generator + versioned labeled corpus** — TTS-synthesize each turn,
+  splice pauses, mix multi-speaker streams; persist labeled WAV + ground-truth
+  JSON. Needs the real TTS routes / Kokoro voices. (`__test-helpers__/synthetic-speech.ts`
+  is the synthesis seed.)
+- **Headless runner** — wire the scenario through the real ASR/diarization/EOT/
+  respond/entity/TTS services + `AgentRuntime`.
+- **scenario-runner audio turn kind** — add an `audio`/`voice` `ScenarioTurnExecution`
+  so voice scenarios become first-class `.scenario.ts` files.
+- **Headful scenario player** — `VoiceSelfTestShell` → multi-turn player +
+  `packages/app/test/ui-smoke/voice-workbench-*.spec.ts` per scenario class.
+- **`voice:workbench` entrypoint + CI lane** — run the matrix, emit the report
+  (`buildVoiceWorkbenchReport`), `skipped` (never `pass`) when artifacts absent.
+- **Multi-agent room semantics** — the canonical ≥3-participant "who responds"
+  contract (an open question on the issue) must be settled before the workbench
+  can assert against it rather than inventing a rule.

package/src/services/voice/__test-helpers__/fake-ffi.ts

@@ -0,0 +1,92 @@
+/**
+ * Test-only `ElizaInferenceFfi` stand-in. Only the methods the voice
+ * pipeline exercises are non-trivial: `asrTranscribe` returns the supplied
+ * fixed transcript; `ttsSynthesize` writes a constant number of samples;
+ * `ttsSynthesizeStream` emits the same PCM as two chunks (one body + one
+ * `isFinal` tail) and honours `onChunk` returning `true` as a cancel.
+ * The ABI-v2 streaming-ASR symbols report "no working decoder" by
+ * default (the same as the C unsupported-build path) so the pipeline routes through the v1
+ * batch path unless a test opts into `asrStreamSupported`. Everything
+ * else is a no-op / identity so a test can wire a "fused" FFI without a
+ * real `.dylib`.
+ */
+import type { ElizaInferenceFfi, TtsStreamChunk } from "../ffi-bindings";
+
+export function fakeFfi(
+	transcript: string,
+	opts: {
+		ttsSamples?: number;
+		ttsStreamSupported?: boolean;
+		asrStreamSupported?: boolean;
+		vadSupported?: boolean;
+		vadProbs?: readonly number[];
+		speakerSupported?: boolean;
+		/** The 256-float embedding `speakerEmbed` returns (defaults to zeros). */
+		speakerEmbedding?: Float32Array;
+		diarizSupported?: boolean;
+		/** The per-frame label sequence `diarizSegment` returns. */
+		diarizLabels?: Int8Array;
+	} = {},
+): ElizaInferenceFfi {
+	const ttsSamples = opts.ttsSamples ?? 8;
+	const ttsStreamSupported = opts.ttsStreamSupported ?? true;
+	const asrStreamSupported = opts.asrStreamSupported ?? false;
+	const vadSupported = opts.vadSupported ?? false;
+	const vadProbs = opts.vadProbs ?? [0];
+	const speakerSupported = opts.speakerSupported ?? false;
+	const diarizSupported = opts.diarizSupported ?? false;
+	let vadIdx = 0;
+	return {
+		libraryPath: "/fake/libelizainference.so",
+		libraryAbiVersion: "3",
+		create: () => 1n,
+		destroy: () => {},
+		mmapAcquire: () => {},
+		mmapEvict: () => {},
+		ttsSynthesize: ({ out }) => {
+			const n = Math.min(ttsSamples, out.length);
+			out.fill(0.1, 0, n);
+			return n;
+		},
+		asrTranscribe: () => transcript,
+		ttsStreamSupported: () => ttsStreamSupported,
+		ttsSynthesizeStream: ({ onChunk }) => {
+			const body = new Float32Array(ttsSamples).fill(0.1);
+			const wantCancel = onChunk({
+				pcm: body,
+				isFinal: false,
+			} as TtsStreamChunk);
+			onChunk({ pcm: new Float32Array(0), isFinal: true });
+			return { cancelled: wantCancel === true };
+		},
+		cancelTts: () => {},
+		setVerifierCallback: () => ({ close: () => {} }),
+		encodeReferenceSupported: () => false,
+		vadSupported: () => vadSupported,
+		vadOpen: () => 2n,
+		vadProcess: ({ pcm }) => {
+			if (pcm.length !== 512) throw new Error("fake VAD expected 512 samples");
+			const p = vadProbs[vadIdx] ?? vadProbs[vadProbs.length - 1] ?? 0;
+			vadIdx++;
+			return p;
+		},
+		vadReset: () => {},
+		vadClose: () => {},
+		speakerSupported: () => speakerSupported,
+		speakerOpen: () => 3n,
+		speakerEmbed: () =>
+			(opts.speakerEmbedding ?? new Float32Array(256)).slice(),
+		speakerClose: () => {},
+		diarizSupported: () => diarizSupported,
+		diarizOpen: () => 4n,
+		diarizSegment: () => (opts.diarizLabels ?? new Int8Array(293)).slice(),
+		diarizClose: () => {},
+		asrStreamSupported: () => asrStreamSupported,
+		asrStreamOpen: () => 1n,
+		asrStreamFeed: () => {},
+		asrStreamPartial: () => ({ partial: transcript }),
+		asrStreamFinish: () => ({ partial: transcript }),
+		asrStreamClose: () => {},
+		close: () => {},
+	};
+}

package/src/services/voice/__test-helpers__/synthetic-speech.ts

@@ -0,0 +1,124 @@
+/**
+ * Deterministic-ish speech-like audio generator for VAD/wake-word smoke
+ * tests. Pure synthesis (glottal pulse train through a three-formant
+ * resonator bank with a syllable-rate amplitude envelope and mild f0
+ * jitter) — close enough to real speech in the time/frequency domain that
+ * the Silero VAD reads it as speech, without shipping a recorded WAV.
+ *
+ * `silence + speech + silence` is the canonical smoke fixture: the VAD
+ * should detect exactly one speech segment whose boundaries land inside
+ * the voiced region, and `VadDetector` should drop the leading/trailing
+ * silence windows from its speech-state timeline.
+ */
+
+export interface SpeechFixtureOptions {
+	sampleRate?: number;
+	/** Seconds of leading silence. */
+	leadSilenceSec?: number;
+	/** Seconds of synthesized speech. */
+	speechSec?: number;
+	/** Seconds of trailing silence. */
+	tailSilenceSec?: number;
+	/** Deterministic seed for the f0 jitter. */
+	seed?: number;
+}
+
+export interface SpeechFixture {
+	pcm: Float32Array;
+	sampleRate: number;
+	speechStartSample: number;
+	speechEndSample: number;
+}
+
+function mulberry32(seed: number): () => number {
+	let a = seed >>> 0;
+	return () => {
+		a |= 0;
+		a = (a + 0x6d2b79f5) | 0;
+		let t = Math.imul(a ^ (a >>> 15), 1 | a);
+		t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+		return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+	};
+}
+
+/** A three-formant resonator bank state. */
+class FormantBank {
+	private readonly r: number[];
+	private readonly a1: number[];
+	private readonly a2: number[];
+	private readonly z1: number[];
+	private readonly z2: number[];
+	constructor(
+		sampleRate: number,
+		formants: ReadonlyArray<readonly [number, number]>,
+	) {
+		this.r = [];
+		this.a1 = [];
+		this.a2 = [];
+		this.z1 = [];
+		this.z2 = [];
+		for (const [fc, bw] of formants) {
+			const r = Math.exp((-Math.PI * bw) / sampleRate);
+			const theta = (2 * Math.PI * fc) / sampleRate;
+			this.r.push(r);
+			this.a1.push(-2 * r * Math.cos(theta));
+			this.a2.push(r * r);
+			this.z1.push(0);
+			this.z2.push(0);
+		}
+	}
+	step(excitation: number): number {
+		let v = 0;
+		for (let k = 0; k < this.r.length; k++) {
+			const y = excitation - this.a1[k] * this.z1[k] - this.a2[k] * this.z2[k];
+			this.z2[k] = this.z1[k];
+			this.z1[k] = y;
+			v += y * (1 - k * 0.25);
+		}
+		return v;
+	}
+}
+
+const DEFAULT_FORMANTS: ReadonlyArray<readonly [number, number]> = [
+	[700, 80],
+	[1220, 90],
+	[2600, 120],
+];
+
+/** Build a `silence + synthesized speech + silence` PCM buffer. */
+export function makeSpeechWithSilenceFixture(
+	opts: SpeechFixtureOptions = {},
+): SpeechFixture {
+	const sampleRate = opts.sampleRate ?? 16_000;
+	const leadSec = opts.leadSilenceSec ?? 0.5;
+	const speechSec = opts.speechSec ?? 1.2;
+	const tailSec = opts.tailSilenceSec ?? 0.5;
+	const totalSec = leadSec + speechSec + tailSec;
+	const n = Math.floor(totalSec * sampleRate);
+	const pcm = new Float32Array(n);
+	const speechStartSample = Math.floor(leadSec * sampleRate);
+	const speechEndSample = Math.floor((leadSec + speechSec) * sampleRate);
+
+	const rng = mulberry32(opts.seed ?? 0xe11a);
+	const bank = new FormantBank(sampleRate, DEFAULT_FORMANTS);
+	let phase = 0;
+	for (let i = speechStartSample; i < speechEndSample; i++) {
+		const tInSpeech = (i - speechStartSample) / sampleRate;
+		const f0 =
+			110 + 30 * Math.sin(2 * Math.PI * 5 * tInSpeech) + (rng() - 0.5) * 4;
+		phase += f0 / sampleRate;
+		let excitation = 0;
+		if (phase >= 1) {
+			phase -= 1;
+			excitation = 1;
+		}
+		// Syllable-rate amplitude envelope (~4 Hz).
+		const amp = Math.max(
+			0,
+			0.6 * (1 + Math.sin(2 * Math.PI * 4 * tInSpeech - Math.PI / 2)),
+		);
+		excitation *= amp;
+		pcm[i] = bank.step(excitation) * 0.15;
+	}
+	return { pcm, sampleRate, speechStartSample, speechEndSample };
+}