@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.2

package/src/services/vision/types.ts

@@ -0,0 +1,154 @@
+/**
+ * Vision-language describe-image types (WS2).
+ *
+ * Two layers live here:
+ *
+ *   1. The **request/result** contract every WS2 backend implements
+ *      (`VisionDescribeRequest`, `VisionDescribeResult`). Callers pass
+ *      raw image bytes + a prompt; backends return a title+description.
+ *
+ *   2. The **backend** interface (`VisionDescribeBackend`) that the
+ *      `MemoryArbiter` registers as a capability handler. One backend
+ *      per binding family (node-llama-cpp / llama-server / AOSP libllama
+ *      shim). All three implement the same `load → describe → unload`
+ *      shape so the arbiter can swap between them without caring how
+ *      the projector is wired underneath.
+ *
+ * Why a separate file: the arbiter's `CapabilityRegistration<TBackend,
+ * TRequest, TResult>` is generic; pinning concrete shapes here keeps
+ * the registration sites short and removes a dozen casts at the
+ * call-site.
+ */
+
+/**
+ * Channel order for the raw pixel buffer. Most platforms hand us RGBA
+ * (HTMLCanvasElement, Capacitor `Camera`, the desktop `puppeteer-core`
+ * screenshot pipeline). The encoder normalizes internally; this enum
+ * stays so the hash step can pick a stable byte layout that doesn't
+ * depend on the platform-provided buffer order.
+ */
+export type VisionImageChannelOrder = "rgba" | "rgb" | "bgra" | "bgr";
+
+/**
+ * The raw image data the backend will encode. The arbiter does not see
+ * this — it gets handed straight to the backend's `run()`. The reason
+ * we accept multiple wrappers (URL / base64 / bytes) is that the three
+ * upstream entry points (HTTP route, agent runtime model handler,
+ * computer-use frame loop) each prefer a different shape. The backend
+ * resolves to bytes once.
+ */
+export type VisionImageInput =
+	| { kind: "bytes"; bytes: Uint8Array; mimeType?: string }
+	| { kind: "base64"; base64: string; mimeType?: string }
+	| { kind: "dataUrl"; dataUrl: string }
+	| { kind: "url"; url: string; mimeType?: string };
+
+/**
+ * Caller request to `describeImage`. The `modelFamily` distinguishes
+ * projected-token cache entries from different VL families that share
+ * the same hash space — Qwen3-VL tokens are not interchangeable with
+ * Florence-2 tokens. Default is `qwen3-vl` (the WS2 deliverable);
+ * each additional family registers under its own identifier.
+ */
+export interface VisionDescribeRequest {
+	image: VisionImageInput;
+	prompt?: string;
+	/**
+	 * The model family identifier. Used to namespace the projector cache
+	 * so swapping the backend's model family invalidates cached tokens.
+	 * Defaults to `"qwen3-vl"` when omitted.
+	 */
+	modelFamily?: string;
+	/** Max output tokens; defaults to 256 (description-length budget). */
+	maxTokens?: number;
+	/** 0..1, default 0.2 (descriptions should be deterministic-ish). */
+	temperature?: number;
+	signal?: AbortSignal;
+}
+
+/** Backend response — same shape that ImageDescriptionResult expects. */
+export interface VisionDescribeResult {
+	title: string;
+	description: string;
+	/** Best-effort: ms spent in the projector (for arbiter telemetry). */
+	projectorMs?: number;
+	/** Best-effort: ms spent in the decoder. */
+	decodeMs?: number;
+	/** Whether the projected tokens came from the WS1 vision cache. */
+	cacheHit?: boolean;
+}
+
+/**
+ * Per-load arguments for a vision-describe backend. The arbiter's
+ * `load(modelKey)` only carries an opaque key; the binding resolves
+ * that key to real model+mmproj paths through this struct, which
+ * `createVisionCapabilityRegistration` populates from the catalog.
+ */
+export interface VisionDescribeLoadArgs {
+	/** Absolute path to the text decoder GGUF (the "main" model). */
+	modelPath: string;
+	/** Absolute path to the matching mmproj projector GGUF. */
+	mmprojPath: string;
+	/**
+	 * GPU offload preference. The backend translates this to its native
+	 * knob: node-llama-cpp `gpuLayers`, llama-server `--n-gpu-layers`,
+	 * AOSP libllama shim `eliza_llama_model_params_set_n_gpu_layers`.
+	 * `"auto"` lets the binding decide; numeric is honoured verbatim.
+	 */
+	gpuLayers?: number | "auto" | "max";
+	/** Max sampled context window in tokens. Defaults to 4096. */
+	contextSize?: number;
+}
+
+/**
+ * The contract every WS2 backend implements. The shape is intentionally
+ * narrow: the arbiter only ever calls `describe`. `dispose` is wrapped
+ * by the arbiter's `unload` so the backend can free GPU/VRAM and drop
+ * file descriptors on eviction.
+ */
+export interface VisionDescribeBackend {
+	/** Stable identifier — `"capacitor-llama"`, `"llama-server"`, `"aosp"`, or `"fake"` (tests). */
+	readonly id: "capacitor-llama" | "llama-server" | "aosp" | "fake";
+	/**
+	 * Run a describe pass. Backends MAY consult an injected projector cache
+	 * via `args.projectedTokens` (when the caller's hash already produced
+	 * a cache hit) instead of running the projector again; backends that
+	 * don't implement projector-token reuse ignore the field.
+	 */
+	describe(
+		request: VisionDescribeRequest,
+		args?: VisionDescribeBackendOptions,
+	): Promise<VisionDescribeResult>;
+	/** Release the loaded weights. Idempotent. */
+	dispose(): Promise<void>;
+}
+
+/**
+ * Per-call options the arbiter wrapper passes into the backend. Lives
+ * here (rather than on `VisionDescribeRequest`) so the caller-facing
+ * request type stays free of arbiter implementation details.
+ */
+export interface VisionDescribeBackendOptions {
+	/**
+	 * Pre-computed projected tokens from the WS1 vision-embedding cache.
+	 * When present the backend SHOULD skip its own projector step and
+	 * decode against these tokens directly. Backends that can't do this
+	 * still produce a correct result by ignoring the field; the arbiter's
+	 * wrapper will measure `cacheHit: false` in that case.
+	 */
+	projectedTokens?: {
+		tokens: Float32Array;
+		tokenCount: number;
+		hiddenSize: number;
+	};
+}
+
+/**
+ * Capability handler load function. The arbiter calls it with a model
+ * key (e.g. `"qwen3-vl-2b"`); the implementation resolves to a real
+ * `(modelPath, mmprojPath)` pair from the catalog + installed registry
+ * and returns a live backend.
+ */
+export type VisionDescribeBackendLoader = (
+	modelKey: string,
+) => Promise<VisionDescribeBackend>;

package/src/services/vision/vast-fallback.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Final optional IMAGE_DESCRIPTION fallback layer.
+ *
+ * This mirrors the cloud wrapper shape but only runs when the previous
+ * handler explicitly returned a typed fallback outcome.
+ */
+import type { ImageDescriptionParams } from "@elizaos/core";
+import { type LocalVisionOutcome, type VisionFallbackReason, type WrappedImageDescriptionHandler } from "./cloud-fallback";
+export interface VisionVastFallbackOptions {
+    enabled?: boolean;
+    apiKey?: string;
+    baseUrl?: string;
+    fetch?: typeof fetch;
+    handler?: (params: ImageDescriptionParams | string, reason: VisionFallbackReason) => Promise<LocalVisionOutcome>;
+    log?: (message: string, detail?: Record<string, unknown>) => void;
+}
+export declare function wrapImageDescriptionHandlerWithVastFallback(previous: WrappedImageDescriptionHandler, options?: VisionVastFallbackOptions): WrappedImageDescriptionHandler;
+//# sourceMappingURL=vast-fallback.d.ts.map

package/src/services/vision/vast-fallback.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vast-fallback.d.ts","sourceRoot":"","sources":["vast-fallback.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EACX,sBAAsB,EAEtB,MAAM,eAAe,CAAC;AACvB,OAAO,EAEN,KAAK,kBAAkB,EAGvB,KAAK,oBAAoB,EACzB,KAAK,8BAA8B,EACnC,MAAM,kBAAkB,CAAC;AAE1B,MAAM,WAAW,yBAAyB;IACzC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;IACrB,OAAO,CAAC,EAAE,CACT,MAAM,EAAE,sBAAsB,GAAG,MAAM,EACvC,MAAM,EAAE,oBAAoB,KACxB,OAAO,CAAC,kBAAkB,CAAC,CAAC;IACjC,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CAClE;AA+DD,wBAAgB,2CAA2C,CAC1D,QAAQ,EAAE,8BAA8B,EACxC,OAAO,GAAE,yBAA8B,GACrC,8BAA8B,CA8BhC"}

package/src/services/vision/vast-fallback.ts

@@ -0,0 +1,127 @@
+/**
+ * Final optional IMAGE_DESCRIPTION fallback layer.
+ *
+ * This mirrors the cloud wrapper shape but only runs when the previous
+ * handler explicitly returned a typed fallback outcome.
+ */
+
+import type {
+	ImageDescriptionParams,
+	ImageDescriptionResult,
+} from "@elizaos/core";
+import {
+	isVisionFallbackOutcome,
+	type LocalVisionOutcome,
+	type LocalVisionResult,
+	normalizeVisionDescription,
+	type VisionFallbackReason,
+	type WrappedImageDescriptionHandler,
+} from "./cloud-fallback";
+
+export interface VisionVastFallbackOptions {
+	enabled?: boolean;
+	apiKey?: string;
+	baseUrl?: string;
+	fetch?: typeof fetch;
+	handler?: (
+		params: ImageDescriptionParams | string,
+		reason: VisionFallbackReason,
+	) => Promise<LocalVisionOutcome>;
+	log?: (message: string, detail?: Record<string, unknown>) => void;
+}
+
+function resolveVastApiKey(options: VisionVastFallbackOptions): string | null {
+	return (
+		options.apiKey?.trim() || process.env.ELIZA_VAST_API_KEY?.trim() || null
+	);
+}
+
+function resolveVastBaseUrl(options: VisionVastFallbackOptions): string {
+	return (
+		options.baseUrl?.trim() ||
+		process.env.ELIZA_VAST_BASE_URL?.trim() ||
+		"https://api.vast.ai"
+	).replace(/\/+$/, "");
+}
+
+function imageRequestBody(params: ImageDescriptionParams | string): {
+	image: { kind: "url"; url: string } | { kind: "data"; data: string };
+	prompt?: string;
+} {
+	if (typeof params === "string") {
+		return params.startsWith("data:")
+			? { image: { kind: "data", data: params } }
+			: { image: { kind: "url", url: params } };
+	}
+	const imageUrl = (params as { imageUrl?: string }).imageUrl;
+	const image = (params as { image?: string }).image;
+	const source = imageUrl ?? image;
+	const body = source?.startsWith("data:")
+		? { image: { kind: "data" as const, data: source } }
+		: { image: { kind: "url" as const, url: source ?? "" } };
+	if (params.prompt) return { ...body, prompt: params.prompt };
+	return body;
+}
+
+async function callVastVision(
+	params: ImageDescriptionParams | string,
+	options: VisionVastFallbackOptions,
+): Promise<ImageDescriptionResult> {
+	const apiKey = resolveVastApiKey(options);
+	if (!apiKey) {
+		throw new Error("VAST image fallback is not configured");
+	}
+	const fetchImpl = options.fetch ?? fetch;
+	const response = await fetchImpl(
+		`${resolveVastBaseUrl(options)}/v1/vision/describe`,
+		{
+			method: "POST",
+			headers: {
+				"content-type": "application/json",
+				authorization: `Bearer ${apiKey}`,
+			},
+			body: JSON.stringify(imageRequestBody(params)),
+		},
+	);
+	if (!response.ok) {
+		throw new Error(`VAST image fallback failed with ${response.status}`);
+	}
+	return normalizeVisionDescription(
+		(await response.json()) as LocalVisionResult,
+	);
+}
+
+export function wrapImageDescriptionHandlerWithVastFallback(
+	previous: WrappedImageDescriptionHandler,
+	options: VisionVastFallbackOptions = {},
+): WrappedImageDescriptionHandler {
+	const enabled = options.enabled ?? true;
+	const log = options.log ?? (() => undefined);
+	return async (params): Promise<LocalVisionOutcome> => {
+		const outcome = await previous(params);
+		if (!isVisionFallbackOutcome(outcome)) {
+			return normalizeVisionDescription(outcome);
+		}
+		if (!enabled) return outcome;
+
+		const apiKey = resolveVastApiKey(options);
+		if (!options.handler && !apiKey) return outcome;
+
+		log("[vision/vast-fallback] upstream IMAGE_DESCRIPTION fallback", {
+			reason: outcome.reason,
+		});
+		try {
+			const vastOutcome = options.handler
+				? await options.handler(params, outcome.reason)
+				: await callVastVision(params, options);
+			if (isVisionFallbackOutcome(vastOutcome)) return vastOutcome;
+			return normalizeVisionDescription(vastOutcome);
+		} catch (error) {
+			return {
+				kind: "fallback",
+				reason: "vast-error",
+				cause: error instanceof Error ? error : new Error(String(error)),
+			};
+		}
+	};
+}

package/src/services/vision-embedding-cache.d.ts

@@ -0,0 +1,98 @@
+/**
+ * Content-hashed cache for projected vision-language tokens (WS1 deliverable).
+ *
+ * Vision models in the Eliza-1 stack (Qwen3-VL, Florence-2, Apothic-VL) all
+ * go through the same expensive projector step: raw pixel
+ * bytes → patch embeddings → projector → tokens that the text decoder
+ * actually consumes. When the user pastes the same screenshot three times
+ * in a row, or when computer-use takes near-duplicate frames of an idle
+ * screen, we want to skip the projector entirely and reuse the cached
+ * tokens.
+ *
+ * Contract:
+ *   - Caller computes a stable hash of the *normalized* input bytes
+ *     (downscaled to the model's input resolution, then SHA-256 of the
+ *     packed pixels). The hash is the cache key.
+ *   - Caller pairs the hash with the projected token tensor (a flat
+ *     `Float32Array` of length `tokens * hiddenSize`) AND the geometry
+ *     `{ tokens, hiddenSize }` so a reader can reshape on the way out.
+ *   - `get(hash)` returns `null` on miss or expiry, the entry on hit.
+ *     A hit also "touches" the entry to keep it warm under LRU.
+ *   - `set(hash, entry, ttlMs?)` inserts with a TTL (default 5 min); if
+ *     the LRU is full, the coldest entry is evicted.
+ *
+ * Why a separate module:
+ *   - The arbiter owns the *model handle*; the cache holds *per-input
+ *     projected weights* that survive across model loads/unloads of the
+ *     same family. Keeping the cache in a sibling module lets the vision
+ *     plugin reuse it even when the arbiter swapped the underlying model
+ *     for memory pressure (the projector tokens are still valid as long
+ *     as the model family + hash match — we encode the family in the key
+ *     to be safe).
+ *
+ * What this is NOT:
+ *   - A blob cache for the encoder *weights*. Those live in mmap regions
+ *     owned by the arbiter / SharedResourceRegistry and are evicted via
+ *     `MmapRegionHandle.evictPages()`.
+ *   - A cache for downstream LLM generations. Prefix-cache for text is
+ *     handled by `cache-bridge.ts` and the backend session pool.
+ */
+export interface VisionEmbeddingEntry {
+    /** Flat row-major buffer: `tokenCount * hiddenSize` floats. */
+    tokens: Float32Array;
+    tokenCount: number;
+    hiddenSize: number;
+    /** True when this entry is still within its TTL. */
+    live: boolean;
+}
+export interface VisionEmbeddingCacheConfig {
+    /** Max entries retained. LRU evicts beyond this. Default 32. */
+    maxEntries: number;
+    /** Default TTL when `set()` is called without one. Default 5 min. */
+    defaultTtlMs: number;
+}
+export declare class VisionEmbeddingCache {
+    private readonly config;
+    /**
+     * `Map` preserves insertion order; we re-insert on hit to bubble entries
+     * to the back, so the first key in iteration order is the LRU candidate.
+     */
+    private readonly entries;
+    private readonly now;
+    constructor(opts?: {
+        config?: Partial<VisionEmbeddingCacheConfig>;
+        now?: () => number;
+    });
+    /**
+     * Lookup. Returns the entry on hit (and refreshes LRU position), or null
+     * on miss / expiry. Expired entries are deleted on read so they don't
+     * silently consume the LRU budget.
+     */
+    get(hash: string): VisionEmbeddingEntry | null;
+    /**
+     * Insert. Replaces any existing entry under the same hash. Evicts the
+     * coldest entry if we're at capacity. `ttlMs` overrides the configured
+     * default; pass 0 to use the default.
+     */
+    set(hash: string, entry: {
+        tokens: Float32Array;
+        tokenCount: number;
+        hiddenSize: number;
+    }, ttlMs?: number): void;
+    /** Diagnostic: current entry count. */
+    size(): number;
+    /** Diagnostic: snapshot of (hash, byteSize, expiresAtMs) for each entry. */
+    snapshot(): ReadonlyArray<{
+        hash: string;
+        bytes: number;
+        expiresAtMs: number;
+    }>;
+    /** Drop everything. Cheap; only releases JS-side refs to the Float32Arrays. */
+    clear(): void;
+    /**
+     * Drop entries whose TTL has expired. Returns the number removed. Cheap
+     * to call from the arbiter's pressure tick.
+     */
+    purgeExpired(nowMs?: number): number;
+}
+//# sourceMappingURL=vision-embedding-cache.d.ts.map

package/src/services/vision-embedding-cache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vision-embedding-cache.d.ts","sourceRoot":"","sources":["vision-embedding-cache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AASH,MAAM,WAAW,oBAAoB;IACpC,+DAA+D;IAC/D,MAAM,EAAE,YAAY,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,oDAAoD;IACpD,IAAI,EAAE,OAAO,CAAC;CACd;AAED,MAAM,WAAW,0BAA0B;IAC1C,gEAAgE;IAChE,UAAU,EAAE,MAAM,CAAC;IACnB,qEAAqE;IACrE,YAAY,EAAE,MAAM,CAAC;CACrB;AAOD,qBAAa,oBAAoB;IAChC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA6B;IACpD;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiC;IACzD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;gBAGlC,IAAI,GAAE;QACL,MAAM,CAAC,EAAE,OAAO,CAAC,0BAA0B,CAAC,CAAC;QAC7C,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;KACd;IAYP;;;;OAIG;IACH,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,oBAAoB,GAAG,IAAI;IAkB9C;;;;OAIG;IACH,GAAG,CACF,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE;QAAE,MAAM,EAAE,YAAY,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,EACvE,KAAK,CAAC,EAAE,MAAM,GACZ,IAAI;IAsBP,uCAAuC;IACvC,IAAI,IAAI,MAAM;IAId,4EAA4E;IAC5E,QAAQ,IAAI,aAAa,CAAC;QACzB,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;KACpB,CAAC;IAYF,+EAA+E;IAC/E,KAAK,IAAI,IAAI;IAIb;;;OAGG;IACH,YAAY,CAAC,KAAK,GAAE,MAAmB,GAAG,MAAM;CAUhD"}

package/src/services/vision-embedding-cache.ts

@@ -0,0 +1,189 @@
+/**
+ * Content-hashed cache for projected vision-language tokens (WS1 deliverable).
+ *
+ * Vision models in the Eliza-1 stack (Qwen3-VL, Florence-2, Apothic-VL) all
+ * go through the same expensive projector step: raw pixel
+ * bytes → patch embeddings → projector → tokens that the text decoder
+ * actually consumes. When the user pastes the same screenshot three times
+ * in a row, or when computer-use takes near-duplicate frames of an idle
+ * screen, we want to skip the projector entirely and reuse the cached
+ * tokens.
+ *
+ * Contract:
+ *   - Caller computes a stable hash of the *normalized* input bytes
+ *     (downscaled to the model's input resolution, then SHA-256 of the
+ *     packed pixels). The hash is the cache key.
+ *   - Caller pairs the hash with the projected token tensor (a flat
+ *     `Float32Array` of length `tokens * hiddenSize`) AND the geometry
+ *     `{ tokens, hiddenSize }` so a reader can reshape on the way out.
+ *   - `get(hash)` returns `null` on miss or expiry, the entry on hit.
+ *     A hit also "touches" the entry to keep it warm under LRU.
+ *   - `set(hash, entry, ttlMs?)` inserts with a TTL (default 5 min); if
+ *     the LRU is full, the coldest entry is evicted.
+ *
+ * Why a separate module:
+ *   - The arbiter owns the *model handle*; the cache holds *per-input
+ *     projected weights* that survive across model loads/unloads of the
+ *     same family. Keeping the cache in a sibling module lets the vision
+ *     plugin reuse it even when the arbiter swapped the underlying model
+ *     for memory pressure (the projector tokens are still valid as long
+ *     as the model family + hash match — we encode the family in the key
+ *     to be safe).
+ *
+ * What this is NOT:
+ *   - A blob cache for the encoder *weights*. Those live in mmap regions
+ *     owned by the arbiter / SharedResourceRegistry and are evicted via
+ *     `MmapRegionHandle.evictPages()`.
+ *   - A cache for downstream LLM generations. Prefix-cache for text is
+ *     handled by `cache-bridge.ts` and the backend session pool.
+ */
+
+interface CacheEntry {
+	tokens: Float32Array;
+	tokenCount: number;
+	hiddenSize: number;
+	expiresAtMs: number;
+}
+
+export interface VisionEmbeddingEntry {
+	/** Flat row-major buffer: `tokenCount * hiddenSize` floats. */
+	tokens: Float32Array;
+	tokenCount: number;
+	hiddenSize: number;
+	/** True when this entry is still within its TTL. */
+	live: boolean;
+}
+
+export interface VisionEmbeddingCacheConfig {
+	/** Max entries retained. LRU evicts beyond this. Default 32. */
+	maxEntries: number;
+	/** Default TTL when `set()` is called without one. Default 5 min. */
+	defaultTtlMs: number;
+}
+
+const DEFAULTS: VisionEmbeddingCacheConfig = {
+	maxEntries: 32,
+	defaultTtlMs: 5 * 60_000,
+};
+
+export class VisionEmbeddingCache {
+	private readonly config: VisionEmbeddingCacheConfig;
+	/**
+	 * `Map` preserves insertion order; we re-insert on hit to bubble entries
+	 * to the back, so the first key in iteration order is the LRU candidate.
+	 */
+	private readonly entries = new Map<string, CacheEntry>();
+	private readonly now: () => number;
+
+	constructor(
+		opts: {
+			config?: Partial<VisionEmbeddingCacheConfig>;
+			now?: () => number;
+		} = {},
+	) {
+		this.config = {
+			maxEntries: Math.max(1, opts.config?.maxEntries ?? DEFAULTS.maxEntries),
+			defaultTtlMs: Math.max(
+				0,
+				opts.config?.defaultTtlMs ?? DEFAULTS.defaultTtlMs,
+			),
+		};
+		this.now = opts.now ?? (() => Date.now());
+	}
+
+	/**
+	 * Lookup. Returns the entry on hit (and refreshes LRU position), or null
+	 * on miss / expiry. Expired entries are deleted on read so they don't
+	 * silently consume the LRU budget.
+	 */
+	get(hash: string): VisionEmbeddingEntry | null {
+		const found = this.entries.get(hash);
+		if (!found) return null;
+		if (found.expiresAtMs <= this.now()) {
+			this.entries.delete(hash);
+			return null;
+		}
+		// Touch — re-insert so it moves to the back of the iteration order.
+		this.entries.delete(hash);
+		this.entries.set(hash, found);
+		return {
+			tokens: found.tokens,
+			tokenCount: found.tokenCount,
+			hiddenSize: found.hiddenSize,
+			live: true,
+		};
+	}
+
+	/**
+	 * Insert. Replaces any existing entry under the same hash. Evicts the
+	 * coldest entry if we're at capacity. `ttlMs` overrides the configured
+	 * default; pass 0 to use the default.
+	 */
+	set(
+		hash: string,
+		entry: { tokens: Float32Array; tokenCount: number; hiddenSize: number },
+		ttlMs?: number,
+	): void {
+		if (entry.tokens.length !== entry.tokenCount * entry.hiddenSize) {
+			throw new Error(
+				`[vision-embedding-cache] token buffer length ${entry.tokens.length} does not match tokenCount*hiddenSize (${entry.tokenCount}*${entry.hiddenSize})`,
+			);
+		}
+		const ttl = ttlMs && ttlMs > 0 ? ttlMs : this.config.defaultTtlMs;
+		const expiresAtMs = this.now() + ttl;
+		this.entries.delete(hash);
+		this.entries.set(hash, {
+			tokens: entry.tokens,
+			tokenCount: entry.tokenCount,
+			hiddenSize: entry.hiddenSize,
+			expiresAtMs,
+		});
+		while (this.entries.size > this.config.maxEntries) {
+			const firstKey = this.entries.keys().next().value;
+			if (firstKey === undefined) break;
+			this.entries.delete(firstKey);
+		}
+	}
+
+	/** Diagnostic: current entry count. */
+	size(): number {
+		return this.entries.size;
+	}
+
+	/** Diagnostic: snapshot of (hash, byteSize, expiresAtMs) for each entry. */
+	snapshot(): ReadonlyArray<{
+		hash: string;
+		bytes: number;
+		expiresAtMs: number;
+	}> {
+		const out: { hash: string; bytes: number; expiresAtMs: number }[] = [];
+		for (const [hash, entry] of this.entries) {
+			out.push({
+				hash,
+				bytes: entry.tokens.byteLength,
+				expiresAtMs: entry.expiresAtMs,
+			});
+		}
+		return out;
+	}
+
+	/** Drop everything. Cheap; only releases JS-side refs to the Float32Arrays. */
+	clear(): void {
+		this.entries.clear();
+	}
+
+	/**
+	 * Drop entries whose TTL has expired. Returns the number removed. Cheap
+	 * to call from the arbiter's pressure tick.
+	 */
+	purgeExpired(nowMs: number = this.now()): number {
+		let removed = 0;
+		for (const [hash, entry] of this.entries) {
+			if (entry.expiresAtMs <= nowMs) {
+				this.entries.delete(hash);
+				removed++;
+			}
+		}
+		return removed;
+	}
+}

package/src/services/voice/VOICE_WORKBENCH.md

@@ -0,0 +1,88 @@
+# Voice Workbench
+
+Tracking issue: [elizaOS/eliza#8785](https://github.com/elizaOS/eliza/issues/8785).
+
+elizaOS ships a mature voice pipeline (VAD, streaming ASR, EOT classifier,
+barge-in, diarization, speaker imprint/profiles, Kokoro/OmniVoice TTS) but its
+test harnesses were **fragmented** across five families with no shared scenario
+format, no shared corpus, divergent metric definitions, and a headful surface
+that only covered a single-speaker, single-turn round-trip. The Voice Workbench
+unifies them onto **one scenario format, one metric module, and one report**.
+
+## Status
+
+This directory holds the **pure, framework-level foundation** — the parts that
+can be implemented, tested, and shipped without an audio corpus, native models,
+or a browser. The execution runners that actually drive real services/audio are
+intentionally **gated** (they need a provisioned Eliza-1 local backend + a
+synthesized corpus) and are listed under *Remaining* below.
+
+### Implemented (this directory, unit-tested, no native artifacts)
+
+| Piece | File | What it is |
+| --- | --- | --- |
+| **Scenario schema** | `voice-scenario.ts` | The declarative `VoiceScenario` format: named `participants` (voice→entity), ordered `turns` (`expectRespond`, `expectedTranscript`, `expectedSpeakerLabel`, `expectedEntity`, `pausesMs`), scenario `assertions` (WER/DER/EOT/latency ceilings), and `classes`. Pure `validateVoiceScenario` reports every consistency error at once. |
+| **Metric module (single source of truth)** | `e2e-harness.ts` | All voice scoring lives here. WER is delegated to `@elizaos/shared/voice-wer` (one definition for headless + headful). Added scorers: `scoreEotDecision` (latency p50/p95 + false-trigger/false-suppression rate), `scoreRespondDecision` (FP/FN split), `scoreDiarization` (DER + confusions/misses), `scoreEntityExtraction` (precision/recall/F1), `scoreVoiceEntityMatch` (recognized-voice→entity accuracy). |
+| **Benchmark report** | `voice-workbench-report.ts` | `buildVoiceWorkbenchReport` rolls a matrix of per-scenario scorer results into one gating report (per-metric mean/worst + percentiles, per-scenario verdict). `formatVoiceWorkbenchMarkdown` renders it; `regressionsAgainstBaseline` flags metrics that worsened past a tolerance. |
+| **WER consolidation** | `@elizaos/shared/voice-wer` | The previously-duplicated `wordErrorRate` (`e2e-harness.ts` **and** `voice-selftest-harness.ts`, with subtly different normalization) is now defined once — Unicode-aware, contraction-preserving — and imported by both. |
+
+Tests: `voice-workbench.test.ts`, `voice-workbench-report.test.ts`,
+`e2e-harness.test.ts`.
+
+### Honesty contract
+
+A scenario whose corpus/backend artifacts are absent is reported `skipped`,
+**never `pass`** — matching the existing self-test contract. A workbench report
+is `skipped` overall only when *every* scenario was skipped; one ran-and-failed
+scenario makes the whole report `fail`.
+
+## Execution modes (the three the schema feeds)
+
+1. **Headless** — feed corpus audio through the real services without a browser:
+   `/api/asr/local-inference`, `LiveDiarizationSession` / `/api/voice/audio-frames`,
+   the `ELIZA_VOICE_EOT_BACKEND` classifier, respond/room decisions over a real
+   `AgentRuntime` (scenario-runner PGLite boot), `VOICE_TURN_OBSERVED` /
+   `VOICE_ENTITY_BOUND` / `IDENTIFY_SPEAKER`, and `/api/tts/local-inference`.
+2. **Headful** — extend `VoiceSelfTestShell` (`packages/ui/src/voice/voice-selftest/`)
+   from a single-turn self-test into a scenario player that drives the real
+   client pipeline (capture → ASR → SSE → TTS → playback) turn-by-turn, with
+   per-turn machine-readable + DOM-mirrored verdicts.
+3. **Benchmark/report** — a single `voice:workbench` entrypoint that runs the
+   matrix in both modes and rolls up via `voice-workbench-report.ts` into one
+   JSON + Markdown report with regression baselines.
+
+All three consume the **same** `VoiceScenario` and the **same** scorers, so a
+metric is defined exactly once regardless of where the audio is driven.
+
+## Consolidation map (what converges here)
+
+The workbench is the convergence point for these previously-disjoint harnesses:
+
+| Legacy harness | Convergence |
+| --- | --- |
+| `e2e-harness.ts:wordErrorRate` + `voice-selftest-harness.ts:wordErrorRate` | **Done** — one `@elizaos/shared/voice-wer`. |
+| Pure scoring lib (`e2e-harness.ts`) | **Promoted** to the single metric module (EOT/diarization/respond/entity scorers added). |
+| `packages/app-core/scripts/voice-duet.mjs` (`voice:duet`), `voice-e2e-hardware.ts`, `voice-vad-smoke.ts`, `voice-attribution-smoke.ts`, `lib/duet-bridge.mjs` | Feed measurements into the shared scorers + report (planned absorb). |
+| `packages/benchmarks/voice/three-voice-scenario.mjs`, `three-voice-e2e-real.mjs` | Corpus-generation precedent the `VoiceScenario` corpus generator extends (planned). |
+| `packages/benchmarks/voicebench/` (TS latency p95/p99) | The report layer mirrors its p95/p99 shape; remains a research bench linked from the workbench. |
+| Per-spec inline `tinyWav()` fixtures (`packages/app/test/ui-smoke/voice-*.spec.ts`) | Replaced by the versioned corpus (planned). |
+
+## Remaining (gated — needs corpus + real backend)
+
+These are tracked on #8785 and are **not** stubbed here (no LARP):
+
+- **Corpus generator + versioned labeled corpus** — TTS-synthesize each turn,
+  splice pauses, mix multi-speaker streams; persist labeled WAV + ground-truth
+  JSON. Needs the real TTS routes / Kokoro voices. (`__test-helpers__/synthetic-speech.ts`
+  is the synthesis seed.)
+- **Headless runner** — wire the scenario through the real ASR/diarization/EOT/
+  respond/entity/TTS services + `AgentRuntime`.
+- **scenario-runner audio turn kind** — add an `audio`/`voice` `ScenarioTurnExecution`
+  so voice scenarios become first-class `.scenario.ts` files.
+- **Headful scenario player** — `VoiceSelfTestShell` → multi-turn player +
+  `packages/app/test/ui-smoke/voice-workbench-*.spec.ts` per scenario class.
+- **`voice:workbench` entrypoint + CI lane** — run the matrix, emit the report
+  (`buildVoiceWorkbenchReport`), `skipped` (never `pass`) when artifacts absent.
+- **Multi-agent room semantics** — the canonical ≥3-participant "who responds"
+  contract (an open question on the issue) must be settled before the workbench
+  can assert against it rather than inventing a rule.