@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/vision/vast-fallback.ts

@@ -0,0 +1,127 @@
+/**
+ * Final optional IMAGE_DESCRIPTION fallback layer.
+ *
+ * This mirrors the cloud wrapper shape but only runs when the previous
+ * handler explicitly returned a typed fallback outcome.
+ */
+
+import type {
+	ImageDescriptionParams,
+	ImageDescriptionResult,
+} from "@elizaos/core";
+import {
+	isVisionFallbackOutcome,
+	type LocalVisionOutcome,
+	type LocalVisionResult,
+	normalizeVisionDescription,
+	type VisionFallbackReason,
+	type WrappedImageDescriptionHandler,
+} from "./cloud-fallback";
+
+export interface VisionVastFallbackOptions {
+	enabled?: boolean;
+	apiKey?: string;
+	baseUrl?: string;
+	fetch?: typeof fetch;
+	handler?: (
+		params: ImageDescriptionParams | string,
+		reason: VisionFallbackReason,
+	) => Promise<LocalVisionOutcome>;
+	log?: (message: string, detail?: Record<string, unknown>) => void;
+}
+
+function resolveVastApiKey(options: VisionVastFallbackOptions): string | null {
+	return (
+		options.apiKey?.trim() || process.env.ELIZA_VAST_API_KEY?.trim() || null
+	);
+}
+
+function resolveVastBaseUrl(options: VisionVastFallbackOptions): string {
+	return (
+		options.baseUrl?.trim() ||
+		process.env.ELIZA_VAST_BASE_URL?.trim() ||
+		"https://api.vast.ai"
+	).replace(/\/+$/, "");
+}
+
+function imageRequestBody(params: ImageDescriptionParams | string): {
+	image: { kind: "url"; url: string } | { kind: "data"; data: string };
+	prompt?: string;
+} {
+	if (typeof params === "string") {
+		return params.startsWith("data:")
+			? { image: { kind: "data", data: params } }
+			: { image: { kind: "url", url: params } };
+	}
+	const imageUrl = (params as { imageUrl?: string }).imageUrl;
+	const image = (params as { image?: string }).image;
+	const source = imageUrl ?? image;
+	const body = source?.startsWith("data:")
+		? { image: { kind: "data" as const, data: source } }
+		: { image: { kind: "url" as const, url: source ?? "" } };
+	if (params.prompt) return { ...body, prompt: params.prompt };
+	return body;
+}
+
+async function callVastVision(
+	params: ImageDescriptionParams | string,
+	options: VisionVastFallbackOptions,
+): Promise<ImageDescriptionResult> {
+	const apiKey = resolveVastApiKey(options);
+	if (!apiKey) {
+		throw new Error("VAST image fallback is not configured");
+	}
+	const fetchImpl = options.fetch ?? fetch;
+	const response = await fetchImpl(
+		`${resolveVastBaseUrl(options)}/v1/vision/describe`,
+		{
+			method: "POST",
+			headers: {
+				"content-type": "application/json",
+				authorization: `Bearer ${apiKey}`,
+			},
+			body: JSON.stringify(imageRequestBody(params)),
+		},
+	);
+	if (!response.ok) {
+		throw new Error(`VAST image fallback failed with ${response.status}`);
+	}
+	return normalizeVisionDescription(
+		(await response.json()) as LocalVisionResult,
+	);
+}
+
+export function wrapImageDescriptionHandlerWithVastFallback(
+	previous: WrappedImageDescriptionHandler,
+	options: VisionVastFallbackOptions = {},
+): WrappedImageDescriptionHandler {
+	const enabled = options.enabled ?? true;
+	const log = options.log ?? (() => undefined);
+	return async (params): Promise<LocalVisionOutcome> => {
+		const outcome = await previous(params);
+		if (!isVisionFallbackOutcome(outcome)) {
+			return normalizeVisionDescription(outcome);
+		}
+		if (!enabled) return outcome;
+
+		const apiKey = resolveVastApiKey(options);
+		if (!options.handler && !apiKey) return outcome;
+
+		log("[vision/vast-fallback] upstream IMAGE_DESCRIPTION fallback", {
+			reason: outcome.reason,
+		});
+		try {
+			const vastOutcome = options.handler
+				? await options.handler(params, outcome.reason)
+				: await callVastVision(params, options);
+			if (isVisionFallbackOutcome(vastOutcome)) return vastOutcome;
+			return normalizeVisionDescription(vastOutcome);
+		} catch (error) {
+			return {
+				kind: "fallback",
+				reason: "vast-error",
+				cause: error instanceof Error ? error : new Error(String(error)),
+			};
+		}
+	};
+}

package/src/services/vision-embedding-cache.ts

@@ -0,0 +1,189 @@
+/**
+ * Content-hashed cache for projected vision-language tokens (WS1 deliverable).
+ *
+ * Vision models in the Eliza-1 stack (Gemma-VL, Florence-2, Apothic-VL) all
+ * go through the same expensive projector step: raw pixel
+ * bytes → patch embeddings → projector → tokens that the text decoder
+ * actually consumes. When the user pastes the same screenshot three times
+ * in a row, or when computer-use takes near-duplicate frames of an idle
+ * screen, we want to skip the projector entirely and reuse the cached
+ * tokens.
+ *
+ * Contract:
+ *   - Caller computes a stable hash of the *normalized* input bytes
+ *     (downscaled to the model's input resolution, then SHA-256 of the
+ *     packed pixels). The hash is the cache key.
+ *   - Caller pairs the hash with the projected token tensor (a flat
+ *     `Float32Array` of length `tokens * hiddenSize`) AND the geometry
+ *     `{ tokens, hiddenSize }` so a reader can reshape on the way out.
+ *   - `get(hash)` returns `null` on miss or expiry, the entry on hit.
+ *     A hit also "touches" the entry to keep it warm under LRU.
+ *   - `set(hash, entry, ttlMs?)` inserts with a TTL (default 5 min); if
+ *     the LRU is full, the coldest entry is evicted.
+ *
+ * Why a separate module:
+ *   - The arbiter owns the *model handle*; the cache holds *per-input
+ *     projected weights* that survive across model loads/unloads of the
+ *     same family. Keeping the cache in a sibling module lets the vision
+ *     plugin reuse it even when the arbiter swapped the underlying model
+ *     for memory pressure (the projector tokens are still valid as long
+ *     as the model family + hash match — we encode the family in the key
+ *     to be safe).
+ *
+ * What this is NOT:
+ *   - A blob cache for the encoder *weights*. Those live in mmap regions
+ *     owned by the arbiter / SharedResourceRegistry and are evicted via
+ *     `MmapRegionHandle.evictPages()`.
+ *   - A cache for downstream LLM generations. Prefix-cache for text is
+ *     handled by `cache-bridge.ts` and the backend session pool.
+ */
+
+interface CacheEntry {
+	tokens: Float32Array;
+	tokenCount: number;
+	hiddenSize: number;
+	expiresAtMs: number;
+}
+
+export interface VisionEmbeddingEntry {
+	/** Flat row-major buffer: `tokenCount * hiddenSize` floats. */
+	tokens: Float32Array;
+	tokenCount: number;
+	hiddenSize: number;
+	/** True when this entry is still within its TTL. */
+	live: boolean;
+}
+
+export interface VisionEmbeddingCacheConfig {
+	/** Max entries retained. LRU evicts beyond this. Default 32. */
+	maxEntries: number;
+	/** Default TTL when `set()` is called without one. Default 5 min. */
+	defaultTtlMs: number;
+}
+
+const DEFAULTS: VisionEmbeddingCacheConfig = {
+	maxEntries: 32,
+	defaultTtlMs: 5 * 60_000,
+};
+
+export class VisionEmbeddingCache {
+	private readonly config: VisionEmbeddingCacheConfig;
+	/**
+	 * `Map` preserves insertion order; we re-insert on hit to bubble entries
+	 * to the back, so the first key in iteration order is the LRU candidate.
+	 */
+	private readonly entries = new Map<string, CacheEntry>();
+	private readonly now: () => number;
+
+	constructor(
+		opts: {
+			config?: Partial<VisionEmbeddingCacheConfig>;
+			now?: () => number;
+		} = {},
+	) {
+		this.config = {
+			maxEntries: Math.max(1, opts.config?.maxEntries ?? DEFAULTS.maxEntries),
+			defaultTtlMs: Math.max(
+				0,
+				opts.config?.defaultTtlMs ?? DEFAULTS.defaultTtlMs,
+			),
+		};
+		this.now = opts.now ?? (() => Date.now());
+	}
+
+	/**
+	 * Lookup. Returns the entry on hit (and refreshes LRU position), or null
+	 * on miss / expiry. Expired entries are deleted on read so they don't
+	 * silently consume the LRU budget.
+	 */
+	get(hash: string): VisionEmbeddingEntry | null {
+		const found = this.entries.get(hash);
+		if (!found) return null;
+		if (found.expiresAtMs <= this.now()) {
+			this.entries.delete(hash);
+			return null;
+		}
+		// Touch — re-insert so it moves to the back of the iteration order.
+		this.entries.delete(hash);
+		this.entries.set(hash, found);
+		return {
+			tokens: found.tokens,
+			tokenCount: found.tokenCount,
+			hiddenSize: found.hiddenSize,
+			live: true,
+		};
+	}
+
+	/**
+	 * Insert. Replaces any existing entry under the same hash. Evicts the
+	 * coldest entry if we're at capacity. `ttlMs` overrides the configured
+	 * default; pass 0 to use the default.
+	 */
+	set(
+		hash: string,
+		entry: { tokens: Float32Array; tokenCount: number; hiddenSize: number },
+		ttlMs?: number,
+	): void {
+		if (entry.tokens.length !== entry.tokenCount * entry.hiddenSize) {
+			throw new Error(
+				`[vision-embedding-cache] token buffer length ${entry.tokens.length} does not match tokenCount*hiddenSize (${entry.tokenCount}*${entry.hiddenSize})`,
+			);
+		}
+		const ttl = ttlMs && ttlMs > 0 ? ttlMs : this.config.defaultTtlMs;
+		const expiresAtMs = this.now() + ttl;
+		this.entries.delete(hash);
+		this.entries.set(hash, {
+			tokens: entry.tokens,
+			tokenCount: entry.tokenCount,
+			hiddenSize: entry.hiddenSize,
+			expiresAtMs,
+		});
+		while (this.entries.size > this.config.maxEntries) {
+			const firstKey = this.entries.keys().next().value;
+			if (firstKey === undefined) break;
+			this.entries.delete(firstKey);
+		}
+	}
+
+	/** Diagnostic: current entry count. */
+	size(): number {
+		return this.entries.size;
+	}
+
+	/** Diagnostic: snapshot of (hash, byteSize, expiresAtMs) for each entry. */
+	snapshot(): ReadonlyArray<{
+		hash: string;
+		bytes: number;
+		expiresAtMs: number;
+	}> {
+		const out: { hash: string; bytes: number; expiresAtMs: number }[] = [];
+		for (const [hash, entry] of this.entries) {
+			out.push({
+				hash,
+				bytes: entry.tokens.byteLength,
+				expiresAtMs: entry.expiresAtMs,
+			});
+		}
+		return out;
+	}
+
+	/** Drop everything. Cheap; only releases JS-side refs to the Float32Arrays. */
+	clear(): void {
+		this.entries.clear();
+	}
+
+	/**
+	 * Drop entries whose TTL has expired. Returns the number removed. Cheap
+	 * to call from the arbiter's pressure tick.
+	 */
+	purgeExpired(nowMs: number = this.now()): number {
+		let removed = 0;
+		for (const [hash, entry] of this.entries) {
+			if (entry.expiresAtMs <= nowMs) {
+				this.entries.delete(hash);
+				removed++;
+			}
+		}
+		return removed;
+	}
+}

package/src/services/voice/VOICE_WORKBENCH.md

@@ -0,0 +1,133 @@
+# Voice Workbench
+
+Tracking issue: [elizaOS/eliza#8785](https://github.com/elizaOS/eliza/issues/8785).
+
+elizaOS ships a mature voice pipeline (VAD, streaming ASR, EOT classifier,
+barge-in, diarization, speaker imprint/profiles, Kokoro/OmniVoice TTS) but its
+test harnesses were **fragmented** across five families with no shared scenario
+format, no shared corpus, divergent metric definitions, and a headful surface
+that only covered a single-speaker, single-turn round-trip. The Voice Workbench
+unifies them onto **one scenario format, one metric module, and one report**.
+
+> **Capability assessment + evidence map** (what is CI-proven vs hardware/
+> credential-gated, mapped to every #8785 AC and the product-owner questions):
+> [research/VOICE_8785_ASSESSMENT.md](./research/VOICE_8785_ASSESSMENT.md).
+> Research evidence base (pause lengths, VAD, AEC, diarization, owner verification,
+> model landscape, latency math): [research/VOICE_PIPELINE_RESEARCH_2026.md](./research/VOICE_PIPELINE_RESEARCH_2026.md).
+
+## Status
+
+The schema, corpus generator (incl. acoustic degradation), metric module,
+headless runner, report, scenario-runner `voice` turn kind, headful scenario
+player, and the `voice:workbench` CLI are all **implemented and unit-tested**.
+The real **acoustic-model** lane (`--real`) is implemented as a provisioned
+hardware lane: it generates distinct human voices with ElevenLabs, synthesizes
+agent echoes with the fused local TTS, and scores the corpus through fused ASR,
+WeSpeaker, pyannote, and the shipped respond/self-voice gate. Missing real
+artifacts are a hard failure in `--real`, not all-skipped evidence.
+
+### Execution lanes (`voice:workbench`)
+
+| Lane | Services | Proves | CI |
+| --- | --- | --- | --- |
+| `--mock` (default) | `groundTruthMockServices` echoes ground truth | runner → scorers → report wiring | ✅ always |
+| `--logic` | `realDecisionLogicServices` runs the SHIPPED EOT + respond/echo/bystander/wake-word gate + name extraction + owner inference | the **decision logic** (catches a regression the moment it lands) | ✅ always (no models) |
+| `--real` | ElevenLabs human speech + fused local TTS/ASR + WeSpeaker + pyannote | real WER/DER/EOT/respond/self-voice/owner-security measurements | ✅ provisioned nightly/hardware lane |
+
+The `--logic` lane is the key anti-hollow guarantee: it does NOT echo the corpus,
+it runs the same gate the UI client ships (`@elizaos/shared/voice/respond-gate`),
+so the workbench genuinely suppresses a bystander, rejects the agent's echoed
+reply, and holds on a mid-utterance pause — asserted by tests, not assumed.
+
+### Implemented (this directory, unit-tested, no native artifacts)
+
+| Piece | File | What it is |
+| --- | --- | --- |
+| **Scenario schema** | `voice-scenario.ts` | The declarative `VoiceScenario` format: named `participants` (voice→entity), ordered `turns` (`expectRespond`, `expectedTranscript`, `expectedSpeakerLabel`, `expectedEntity`, `pausesMs`), scenario `assertions` (WER/DER/EOT/latency ceilings), and `classes`. Pure `validateVoiceScenario` reports every consistency error at once. |
+| **Metric module (single source of truth)** | `e2e-harness.ts` | All voice scoring lives here. WER is delegated to `@elizaos/shared/voice-wer` (one definition for headless + headful). Added scorers: `scoreEotDecision` (latency p50/p95 + false-trigger/false-suppression rate), `scoreRespondDecision` (FP/FN split), `scoreDiarization` (DER + confusions/misses), `scoreEntityExtraction` (precision/recall/F1), `scoreVoiceEntityMatch` (recognized-voice→entity accuracy). |
+| **Benchmark report** | `voice-workbench-report.ts` | `buildVoiceWorkbenchReport` rolls a matrix of per-scenario scorer results into one gating report (per-metric mean/worst + percentiles, per-scenario verdict). `formatVoiceWorkbenchMarkdown` renders it; `regressionsAgainstBaseline` flags metrics that worsened past a tolerance. |
+| **WER consolidation** | `@elizaos/shared/voice-wer` | The previously-duplicated `wordErrorRate` (`e2e-harness.ts` **and** `voice-selftest-harness.ts`, with subtly different normalization) is now defined once — Unicode-aware, contraction-preserving — and imported by both. |
+| **Acoustic robustness corpus** | `corpus-augment.ts` | Seeded, deterministic degradation DSP: additive room noise (white/pink at a target SNR), Freeverb reverb, far-field attenuation, telephone/low-quality line (band-limit + µ-law), and competing background talkers. Wired into the corpus generator via a per-turn / per-scenario `environment` so a clean scenario and a noisy one share one schema. |
+| **Real-decision-logic adapter** | `workbench-logic-services.ts` | Runs the SHIPPED EOT + respond/echo/bystander/wake-word gate + name extraction over the corpus (no models). The `--logic` lane. |
+| **Real acoustic adapter** | `workbench-real-services.ts` | The `--real` lane: ElevenLabs-generated human speech, fused local agent TTS, fused ASR, WeSpeaker speaker centroids, pyannote speech/overlap labels, live `selfVoiceSimilarity`, owner inference, and the same respond gate. |
+| **Respond/echo gate (single source)** | `@elizaos/shared/voice/respond-gate` | `shouldRespondToVoiceTurn` + `buildVoiceTurnSignal`, promoted out of the UI so the client and the workbench share one definition. The UI re-exports it. |
+| **Owner inference** | `@elizaos/shared/voice/owner-inference` | `resolveOwnerCandidate` — proposes the owner from who speaks most/most-confidently, only when sufficient AND unambiguous, else UNDECIDED. The logic an owner-detection provider/evaluator runs when no owner is enrolled. |
+| **Echo + owner scorers** | `e2e-harness.ts` | `scoreEchoRejection` (agent-echo turns correctly suppressed) and `scoreOwnerSecurity` (owner-vs-intruder accuracy + impostor-accept rate). |
+
+Tests: `voice-workbench.test.ts`, `voice-workbench-report.test.ts`,
+`e2e-harness.test.ts`, `corpus-augment.test.ts`,
+`workbench-logic-services.test.ts`, `corpus-generator.test.ts`, and (in shared)
+`voice/owner-inference.test.ts`.
+
+### Scenario classes
+
+`multi-voice`, `pauses`, `respond-no-respond`, `multi-speaker`, `diarization`,
+`entity-extraction`, `voice-recognition`, `eot`, `transcription-mode`,
+`multi-agent-room`, `long-form-monologue`, **`robustness`** (noise / reverb /
+far-field / low-quality), **`echo-rejection`** (agent self-voice), **`owner-security`**
+(owner vs intruder), **`overlapping-speech`** (interrupting talkers). The 12
+built-in scenarios in `workbench-scenarios.ts` span every class.
+
+### Honesty contract
+
+A scenario whose corpus/backend artifacts are absent is reported `skipped`,
+**never `pass`** — matching the existing self-test contract. A workbench report
+is `skipped` overall only when *every* scenario was skipped; one ran-and-failed
+scenario makes the whole report `fail`.
+
+## Execution modes (the three the schema feeds)
+
+1. **Headless** — feed corpus audio through the real services without a browser:
+   `/api/asr/local-inference`, `LiveDiarizationSession` / `/api/voice/audio-frames`,
+   the `ELIZA_VOICE_EOT_BACKEND` classifier, respond/room decisions over a real
+   `AgentRuntime` (scenario-runner PGLite boot), `VOICE_TURN_OBSERVED` /
+   `VOICE_ENTITY_BOUND` / `IDENTIFY_SPEAKER`, and `/api/tts/local-inference`.
+2. **Headful** — extend `VoiceSelfTestShell` (`packages/ui/src/voice/voice-selftest/`)
+   from a single-turn self-test into a scenario player that drives the real
+   client pipeline (capture → ASR → SSE → TTS → playback) turn-by-turn, with
+   per-turn machine-readable + DOM-mirrored verdicts.
+3. **Benchmark/report** — a single `voice:workbench` entrypoint that runs the
+   matrix in both modes and rolls up via `voice-workbench-report.ts` into one
+   JSON + Markdown report with regression baselines.
+
+All three consume the **same** `VoiceScenario` and the **same** scorers, so a
+metric is defined exactly once regardless of where the audio is driven.
+
+## Consolidation map (what converges here)
+
+The workbench is the convergence point for these previously-disjoint harnesses:
+
+| Legacy harness | Convergence |
+| --- | --- |
+| `e2e-harness.ts:wordErrorRate` + `voice-selftest-harness.ts:wordErrorRate` | **Done** — one `@elizaos/shared/voice-wer`. |
+| Pure scoring lib (`e2e-harness.ts`) | **Promoted** to the single metric module (EOT/diarization/respond/entity scorers added). |
+| `packages/app-core/scripts/voice-duet.mjs` (`voice:duet`), `voice-e2e-hardware.ts`, `voice-vad-smoke.ts`, `voice-attribution-smoke.ts`, `lib/duet-bridge.mjs` | Feed measurements into the shared scorers + report (planned absorb). |
+| `packages/benchmarks/voice/three-voice-scenario.mjs` | Corpus-generation precedent the `VoiceScenario` corpus generator extends (planned). |
+| `packages/benchmarks/voicebench/` (TS latency p95/p99) | The report layer mirrors its p95/p99 shape; remains a research bench linked from the workbench. |
+| Per-spec inline `tinyWav()` fixtures (`packages/app/test/ui-smoke/voice-*.spec.ts`) | Replaced by the versioned corpus (planned). |
+
+## External / Device Follow-Ups
+
+Full detail + why in
+[research/VOICE_8785_ASSESSMENT.md §5](./research/VOICE_8785_ASSESSMENT.md).
+
+- **Live cloud STT/TTS round-trip** — ElevenLabs via `/api/v1/voice/*`; needs an
+  authenticated Cloud session (the test account returns HTTP 402 — a billing
+  state, not a code bug).
+- **PCM-level AEC** — still a product/runtime feature beyond the workbench
+  scorer: it needs a time-aligned playback reference and cancellation path in
+  the live audio transport, then the workbench can score the resulting echo
+  corpus.
+- **Headful real-backend + recorded A/V** — the 10 `voice-workbench-*.spec.ts`
+  run with mocked backends; a real-backend headful lane with audio+video capture
+  needs a provisioned local backend on the CI host.
+- **iOS device** — blocked on Apple ID provisioning; simulator local-inference is
+  Metal-limited.
+
+## Open follow-up: PCM-level acoustic echo cancellation
+
+Self-echo is caught at the transcript level only (word overlap). The recommended
+next step is an `agentSpeaking` flag + ~1.5 s post-TTS cooldown (cheap, robust),
+then WebRTC AEC3 with a time-aligned reference, then speaker-embedding self-voice
+rejection. The `scoreEchoRejection` scorer is ready to gate it. See
+[research/VOICE_8785_ASSESSMENT.md §6](./research/VOICE_8785_ASSESSMENT.md).

package/src/services/voice/__fixtures__/voice-workbench-logic-baseline.json

@@ -0,0 +1,180 @@
+{
+	"schemaVersion": 1,
+	"overall": "pass",
+	"scenariosTotal": 14,
+	"scenariosRan": 14,
+	"scenariosSkipped": 0,
+	"scenarios": [
+		{
+			"scenarioId": "multi-voice-greeting",
+			"classes": ["multi-voice", "diarization"],
+			"status": "ran",
+			"verdict": "pass",
+			"caseCount": 8,
+			"failedCaseKinds": []
+		},
+		{
+			"scenarioId": "respond-vs-bystander",
+			"classes": ["respond-no-respond", "multi-speaker"],
+			"status": "ran",
+			"verdict": "pass",
+			"caseCount": 9,
+			"failedCaseKinds": []
+		},
+		{
+			"scenarioId": "pauses-midutterance",
+			"classes": ["pauses", "eot"],
+			"status": "ran",
+			"verdict": "pass",
+			"caseCount": 7,
+			"failedCaseKinds": []
+		},
+		{
+			"scenarioId": "entity-from-speech",
+			"classes": ["entity-extraction", "voice-recognition"],
+			"status": "ran",
+			"verdict": "pass",
+			"caseCount": 7,
+			"failedCaseKinds": []
+		},
+		{
+			"scenarioId": "transcription-mode-dictation",
+			"classes": ["transcription-mode", "long-form-monologue"],
+			"status": "ran",
+			"verdict": "pass",
+			"caseCount": 5,
+			"failedCaseKinds": []
+		},
+		{
+			"scenarioId": "multi-agent-room-address",
+			"classes": ["multi-agent-room", "respond-no-respond"],
+			"status": "ran",
+			"verdict": "pass",
+			"caseCount": 8,
+			"failedCaseKinds": []
+		},
+		{
+			"scenarioId": "noisy-room-commands",
+			"classes": ["robustness", "respond-no-respond"],
+			"status": "ran",
+			"verdict": "pass",
+			"caseCount": 8,
+			"failedCaseKinds": []
+		},
+		{
+			"scenarioId": "far-field-reverb",
+			"classes": ["robustness", "respond-no-respond"],
+			"status": "ran",
+			"verdict": "pass",
+			"caseCount": 6,
+			"failedCaseKinds": []
+		},
+		{
+			"scenarioId": "background-talkers",
+			"classes": ["robustness", "overlapping-speech", "multi-speaker"],
+			"status": "ran",
+			"verdict": "pass",
+			"caseCount": 6,
+			"failedCaseKinds": []
+		},
+		{
+			"scenarioId": "echo-self-trigger",
+			"classes": ["echo-rejection", "respond-no-respond"],
+			"status": "ran",
+			"verdict": "pass",
+			"caseCount": 10,
+			"failedCaseKinds": []
+		},
+		{
+			"scenarioId": "multi-speaker-name-capture",
+			"classes": [
+				"diarization",
+				"entity-extraction",
+				"multi-speaker",
+				"voice-recognition"
+			],
+			"status": "ran",
+			"verdict": "pass",
+			"caseCount": 13,
+			"failedCaseKinds": []
+		},
+		{
+			"scenarioId": "echo-mistranscribed",
+			"classes": ["echo-rejection"],
+			"status": "ran",
+			"verdict": "pass",
+			"caseCount": 8,
+			"failedCaseKinds": []
+		},
+		{
+			"scenarioId": "owner-enrollment-inference",
+			"classes": ["owner-security", "voice-recognition"],
+			"status": "ran",
+			"verdict": "pass",
+			"caseCount": 15,
+			"failedCaseKinds": []
+		},
+		{
+			"scenarioId": "owner-vs-intruder",
+			"classes": ["owner-security", "respond-no-respond", "multi-speaker"],
+			"status": "ran",
+			"verdict": "pass",
+			"caseCount": 10,
+			"failedCaseKinds": []
+		}
+	],
+	"metrics": {
+		"wer": {
+			"count": 32,
+			"mean": 0,
+			"worst": 0
+		},
+		"eotFalseTriggerRate": {
+			"count": 14,
+			"mean": 0,
+			"worst": 0
+		},
+		"eotLatencyP50Ms": null,
+		"eotLatencyP95Ms": null,
+		"der": {
+			"count": 14,
+			"mean": 0,
+			"worst": 0
+		},
+		"respondAccuracy": {
+			"count": 14,
+			"mean": 1,
+			"worst": 1
+		},
+		"entityF1": {
+			"count": 2,
+			"mean": 1,
+			"worst": 1
+		},
+		"voiceEntityMatchRate": {
+			"count": 14,
+			"mean": 1,
+			"worst": 1
+		},
+		"firstAudioMs": {
+			"count": 26,
+			"mean": 250,
+			"worst": 250
+		},
+		"echoRejectionRate": {
+			"count": 2,
+			"mean": 1,
+			"worst": 1
+		},
+		"ownerAccuracy": {
+			"count": 2,
+			"mean": 1,
+			"worst": 1
+		},
+		"impostorAcceptRate": {
+			"count": 2,
+			"mean": 0,
+			"worst": 0
+		}
+	}
+}