@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/services/asr/errors.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Typed unavailability error for ASR backends. Mirrors
+ * `ImageGenBackendUnavailableError` from `../imagegen/errors`.
+ *
+ * Backends throw this to signal that a specific request cannot be served
+ * (`unsupported_request`, `model_missing`, `load_failed`, ...) so the
+ * arbiter / caller can surface an actionable message rather than a generic
+ * "transcription failed".
+ */
+export type AsrUnavailableReason = "unsupported_request" | "model_missing" | "load_failed" | "decode_failed" | "aborted";
+export declare class AsrBackendUnavailableError extends Error {
+    readonly backendId: string;
+    readonly reason: AsrUnavailableReason;
+    readonly code = "ASR_BACKEND_UNAVAILABLE";
+    constructor(backendId: string, reason: AsrUnavailableReason, message: string, options?: {
+        cause?: unknown;
+    });
+    toJSON(): Record<string, string>;
+}
+export declare function isAsrBackendUnavailable(err: unknown): err is AsrBackendUnavailableError;
+//# sourceMappingURL=errors.d.ts.map

package/src/services/asr/errors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["errors.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,MAAM,MAAM,oBAAoB,GAC7B,qBAAqB,GACrB,eAAe,GACf,aAAa,GACb,eAAe,GACf,SAAS,CAAC;AAEb,qBAAa,0BAA2B,SAAQ,KAAK;IAInD,QAAQ,CAAC,SAAS,EAAE,MAAM;IAC1B,QAAQ,CAAC,MAAM,EAAE,oBAAoB;IAJtC,QAAQ,CAAC,IAAI,6BAA6B;gBAGhC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,oBAAoB,EACrC,OAAO,EAAE,MAAM,EACf,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE;IAM9B,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;CAQhC;AAED,wBAAgB,uBAAuB,CACtC,GAAG,EAAE,OAAO,GACV,GAAG,IAAI,0BAA0B,CAOnC"}

package/src/services/asr/errors.ts

@@ -0,0 +1,50 @@
+/**
+ * Typed unavailability error for ASR backends. Mirrors
+ * `ImageGenBackendUnavailableError` from `../imagegen/errors`.
+ *
+ * Backends throw this to signal that a specific request cannot be served
+ * (`unsupported_request`, `model_missing`, `load_failed`, ...) so the
+ * arbiter / caller can surface an actionable message rather than a generic
+ * "transcription failed".
+ */
+
+export type AsrUnavailableReason =
+	| "unsupported_request"
+	| "model_missing"
+	| "load_failed"
+	| "decode_failed"
+	| "aborted";
+
+export class AsrBackendUnavailableError extends Error {
+	readonly code = "ASR_BACKEND_UNAVAILABLE";
+
+	constructor(
+		readonly backendId: string,
+		readonly reason: AsrUnavailableReason,
+		message: string,
+		options?: { cause?: unknown },
+	) {
+		super(message, options);
+		this.name = "AsrBackendUnavailableError";
+	}
+
+	toJSON(): Record<string, string> {
+		return {
+			code: this.code,
+			backendId: this.backendId,
+			reason: this.reason,
+			message: this.message,
+		};
+	}
+}
+
+export function isAsrBackendUnavailable(
+	err: unknown,
+): err is AsrBackendUnavailableError {
+	return (
+		err instanceof AsrBackendUnavailableError ||
+		(typeof err === "object" &&
+			err !== null &&
+			(err as { code?: unknown }).code === "ASR_BACKEND_UNAVAILABLE")
+	);
+}

package/src/services/asr/hash.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Content-hash keys for the ASR transcript cache.
+ *
+ * The arbiter caches `AsrResult.text` by the SHA-256 of the
+ * sample-rate-normalized PCM bytes. The hash is fast (one pass over the
+ * fp32 buffer + a 4-byte sample-rate prefix) and stable across runs, so a
+ * caller who re-issues the same WAV gets the cached transcript without
+ * re-running the backend.
+ *
+ * We hash the underlying byte view of the Float32Array, not a re-quantized
+ * representation, so caller-supplied PCM that came from a 16-bit WAV
+ * decode (the common case from the `runtime.useModel(TRANSCRIPTION, ...)`
+ * path) deduplicates correctly with caller-supplied PCM that came from a
+ * fp32 mic capture.
+ */
+import type { AsrRequest } from "./types";
+/**
+ * Hash a PCM buffer + sample rate into a stable cache key. The output is
+ * `<modelFamily>::sha256(<sampleRateLE><pcmBytes>)`.
+ *
+ * - `sampleRateLE`: 4-byte little-endian uint32 prefix so 16 kHz / 24 kHz
+ *   inputs of the same float samples produce different keys.
+ * - `pcmBytes`: the raw fp32 view (`pcm.buffer.slice(pcm.byteOffset, pcm.byteOffset + pcm.byteLength)`).
+ *
+ * Returns the hex digest namespaced by family.
+ */
+export declare function hashAsrInput(req: AsrRequest, family?: string): string;
+//# sourceMappingURL=hash.d.ts.map

package/src/services/asr/hash.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hash.d.ts","sourceRoot":"","sources":["hash.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAIH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAK1C;;;;;;;;;GASG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,UAAU,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAerE"}

package/src/services/asr/hash.ts

@@ -0,0 +1,49 @@
+/**
+ * Content-hash keys for the ASR transcript cache.
+ *
+ * The arbiter caches `AsrResult.text` by the SHA-256 of the
+ * sample-rate-normalized PCM bytes. The hash is fast (one pass over the
+ * fp32 buffer + a 4-byte sample-rate prefix) and stable across runs, so a
+ * caller who re-issues the same WAV gets the cached transcript without
+ * re-running the backend.
+ *
+ * We hash the underlying byte view of the Float32Array, not a re-quantized
+ * representation, so caller-supplied PCM that came from a 16-bit WAV
+ * decode (the common case from the `runtime.useModel(TRANSCRIPTION, ...)`
+ * path) deduplicates correctly with caller-supplied PCM that came from a
+ * fp32 mic capture.
+ */
+
+import { createHash } from "node:crypto";
+
+import type { AsrRequest } from "./types";
+
+/** Family namespace prepended to every hash so caches scoped to different ASR families don't collide. */
+const HASH_FAMILY_DEFAULT = "qwen3-asr";
+
+/**
+ * Hash a PCM buffer + sample rate into a stable cache key. The output is
+ * `<modelFamily>::sha256(<sampleRateLE><pcmBytes>)`.
+ *
+ * - `sampleRateLE`: 4-byte little-endian uint32 prefix so 16 kHz / 24 kHz
+ *   inputs of the same float samples produce different keys.
+ * - `pcmBytes`: the raw fp32 view (`pcm.buffer.slice(pcm.byteOffset, pcm.byteOffset + pcm.byteLength)`).
+ *
+ * Returns the hex digest namespaced by family.
+ */
+export function hashAsrInput(req: AsrRequest, family?: string): string {
+	const hasher = createHash("sha256");
+	const rateBuf = new ArrayBuffer(4);
+	new DataView(rateBuf).setUint32(
+		0,
+		Math.max(0, Math.floor(req.sampleRateHz)),
+		true,
+	);
+	hasher.update(new Uint8Array(rateBuf));
+	const pcm = req.pcm;
+	const bytes = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
+	hasher.update(bytes);
+	if (req.language) hasher.update(`lang:${req.language}`);
+	const fam = req.modelFamily ?? family ?? HASH_FAMILY_DEFAULT;
+	return `${fam}::${hasher.digest("hex")}`;
+}

package/src/services/asr/index.d.ts

@@ -0,0 +1,76 @@
+/**
+ * Local ASR (transcription) capability — public entry point.
+ *
+ * This module is what `provider.ts` (`createTranscriptionHandler`), the
+ * voice pipeline bridge, and additional ASR consumers import to register the
+ * capability with the WS1 MemoryArbiter.
+ *
+ * Wiring:
+ *
+ *   const arbiter = service.getMemoryArbiter();
+ *   const registration = createAsrCapabilityRegistration({
+ *     loader: createDefaultAsrLoader({ ... }),
+ *     transcriptCache: arbiter,
+ *   });
+ *   arbiter.registerCapability(registration);
+ *
+ * `createAsrCapabilityRegistration` wraps the underlying backend so the
+ * arbiter's `run(request)` path:
+ *
+ *   1. Calls `backend.supports(request)`. If false, the arbiter throws
+ *      `AsrBackendUnavailableError` with `unsupported_request`.
+ *   2. Hashes the request's PCM bytes (sample-rate-normalized) and looks
+ *      it up in the optional `transcriptCache`. On a hit, the backend is
+ *      skipped entirely and the cached text is returned.
+ *   3. On a miss, calls `backend.transcribe(request)`, stores the
+ *      transcript under the same hash, and returns the result.
+ *
+ * The capability registers with `residentRole: "asr"` — distinct from the
+ * `vision` slot that `vision-describe` and `image-gen` co-evict in.
+ * Qwen-ASR cohabits comfortably with text on most desktops (≈600 MB for
+ * Q4_K_M Qwen3-ASR-0.6B), but on a 6 GB iPhone / 8 GB low-tier Android
+ * the arbiter's resident-role priority still lets a text-large request
+ * evict the ASR handle when memory is tight. See `memory-arbiter.ts`
+ * `RESIDENT_ROLE_PRIORITY`.
+ */
+export { AsrBackendUnavailableError, type AsrUnavailableReason, isAsrBackendUnavailable, } from "./errors";
+export { hashAsrInput } from "./hash";
+export type { AsrBackend, AsrBackendLoader, AsrLoadArgs, AsrRequest, AsrResult, } from "./types";
+import type { CapabilityRegistration } from "../memory-arbiter";
+import type { AsrBackend, AsrBackendLoader, AsrRequest, AsrResult } from "./types";
+/**
+ * Minimal cache shape the registration accepts. Lets tests inject a fake
+ * cache without pulling in the whole MemoryArbiter. The arbiter
+ * implements this surface (see `memory-arbiter.ts` `getCachedAsrTranscript`
+ * / `setCachedAsrTranscript`).
+ */
+export interface AsrTranscriptCacheLike {
+    getCachedAsrTranscript(hash: string): {
+        text: string;
+        live?: boolean;
+    } | null;
+    setCachedAsrTranscript(hash: string, entry: {
+        text: string;
+    }, ttlMs?: number): void;
+}
+export interface CreateAsrCapabilityRegistrationOptions {
+    loader: AsrBackendLoader;
+    /** Optional content-hash cache. When provided, identical PCM inputs skip the backend. */
+    transcriptCache?: AsrTranscriptCacheLike;
+    /** Default model family for the cache key. Defaults to `qwen3-asr`. */
+    modelFamily?: string;
+    /**
+     * Best-effort RAM footprint estimate for the loaded weights. The
+     * arbiter only uses this for telemetry; eviction is by priority. The
+     * default (600 MB) matches Q4_K_M Qwen3-ASR-0.6B; larger ASR models
+     * SHOULD pass a higher estimate.
+     */
+    estimatedMb?: number;
+}
+/**
+ * Build a `CapabilityRegistration` ready to feed to
+ * `arbiter.registerCapability()`. Mirrors `createVisionCapabilityRegistration`
+ * (WS2) and `createImageGenCapabilityRegistration` (WS3).
+ */
+export declare function createAsrCapabilityRegistration(opts: CreateAsrCapabilityRegistrationOptions): CapabilityRegistration<AsrBackend, AsrRequest, AsrResult>;
+//# sourceMappingURL=index.d.ts.map

package/src/services/asr/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,OAAO,EACN,0BAA0B,EAC1B,KAAK,oBAAoB,EACzB,uBAAuB,GACvB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,YAAY,EACX,UAAU,EACV,gBAAgB,EAChB,WAAW,EACX,UAAU,EACV,SAAS,GACT,MAAM,SAAS,CAAC;AAEjB,OAAO,KAAK,EAEX,sBAAsB,EACtB,MAAM,mBAAmB,CAAC;AAG3B,OAAO,KAAK,EACX,UAAU,EACV,gBAAgB,EAChB,UAAU,EACV,SAAS,EACT,MAAM,SAAS,CAAC;AAEjB;;;;;GAKG;AACH,MAAM,WAAW,sBAAsB;IACtC,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI,CAAC;IAC9E,sBAAsB,CACrB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,EACvB,KAAK,CAAC,EAAE,MAAM,GACZ,IAAI,CAAC;CACR;AAED,MAAM,WAAW,sCAAsC;IACtD,MAAM,EAAE,gBAAgB,CAAC;IACzB,yFAAyF;IACzF,eAAe,CAAC,EAAE,sBAAsB,CAAC;IACzC,uEAAuE;IACvE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;GAIG;AACH,wBAAgB,+BAA+B,CAC9C,IAAI,EAAE,sCAAsC,GAC1C,sBAAsB,CAAC,UAAU,EAAE,UAAU,EAAE,SAAS,CAAC,CA6E3D"}

package/src/services/asr/index.ts

@@ -0,0 +1,178 @@
+/**
+ * Local ASR (transcription) capability — public entry point.
+ *
+ * This module is what `provider.ts` (`createTranscriptionHandler`), the
+ * voice pipeline bridge, and additional ASR consumers import to register the
+ * capability with the WS1 MemoryArbiter.
+ *
+ * Wiring:
+ *
+ *   const arbiter = service.getMemoryArbiter();
+ *   const registration = createAsrCapabilityRegistration({
+ *     loader: createDefaultAsrLoader({ ... }),
+ *     transcriptCache: arbiter,
+ *   });
+ *   arbiter.registerCapability(registration);
+ *
+ * `createAsrCapabilityRegistration` wraps the underlying backend so the
+ * arbiter's `run(request)` path:
+ *
+ *   1. Calls `backend.supports(request)`. If false, the arbiter throws
+ *      `AsrBackendUnavailableError` with `unsupported_request`.
+ *   2. Hashes the request's PCM bytes (sample-rate-normalized) and looks
+ *      it up in the optional `transcriptCache`. On a hit, the backend is
+ *      skipped entirely and the cached text is returned.
+ *   3. On a miss, calls `backend.transcribe(request)`, stores the
+ *      transcript under the same hash, and returns the result.
+ *
+ * The capability registers with `residentRole: "asr"` — distinct from the
+ * `vision` slot that `vision-describe` and `image-gen` co-evict in.
+ * Qwen-ASR cohabits comfortably with text on most desktops (≈600 MB for
+ * Q4_K_M Qwen3-ASR-0.6B), but on a 6 GB iPhone / 8 GB low-tier Android
+ * the arbiter's resident-role priority still lets a text-large request
+ * evict the ASR handle when memory is tight. See `memory-arbiter.ts`
+ * `RESIDENT_ROLE_PRIORITY`.
+ */
+
+export {
+	AsrBackendUnavailableError,
+	type AsrUnavailableReason,
+	isAsrBackendUnavailable,
+} from "./errors";
+export { hashAsrInput } from "./hash";
+export type {
+	AsrBackend,
+	AsrBackendLoader,
+	AsrLoadArgs,
+	AsrRequest,
+	AsrResult,
+} from "./types";
+
+import type {
+	ArbiterCapability,
+	CapabilityRegistration,
+} from "../memory-arbiter";
+import { AsrBackendUnavailableError } from "./errors";
+import { hashAsrInput } from "./hash";
+import type {
+	AsrBackend,
+	AsrBackendLoader,
+	AsrRequest,
+	AsrResult,
+} from "./types";
+
+/**
+ * Minimal cache shape the registration accepts. Lets tests inject a fake
+ * cache without pulling in the whole MemoryArbiter. The arbiter
+ * implements this surface (see `memory-arbiter.ts` `getCachedAsrTranscript`
+ * / `setCachedAsrTranscript`).
+ */
+export interface AsrTranscriptCacheLike {
+	getCachedAsrTranscript(hash: string): { text: string; live?: boolean } | null;
+	setCachedAsrTranscript(
+		hash: string,
+		entry: { text: string },
+		ttlMs?: number,
+	): void;
+}
+
+export interface CreateAsrCapabilityRegistrationOptions {
+	loader: AsrBackendLoader;
+	/** Optional content-hash cache. When provided, identical PCM inputs skip the backend. */
+	transcriptCache?: AsrTranscriptCacheLike;
+	/** Default model family for the cache key. Defaults to `qwen3-asr`. */
+	modelFamily?: string;
+	/**
+	 * Best-effort RAM footprint estimate for the loaded weights. The
+	 * arbiter only uses this for telemetry; eviction is by priority. The
+	 * default (600 MB) matches Q4_K_M Qwen3-ASR-0.6B; larger ASR models
+	 * SHOULD pass a higher estimate.
+	 */
+	estimatedMb?: number;
+}
+
+/**
+ * Build a `CapabilityRegistration` ready to feed to
+ * `arbiter.registerCapability()`. Mirrors `createVisionCapabilityRegistration`
+ * (WS2) and `createImageGenCapabilityRegistration` (WS3).
+ */
+export function createAsrCapabilityRegistration(
+	opts: CreateAsrCapabilityRegistrationOptions,
+): CapabilityRegistration<AsrBackend, AsrRequest, AsrResult> {
+	const capability: ArbiterCapability = "transcribe";
+	const loader = opts.loader;
+	const cache = opts.transcriptCache;
+	const family = opts.modelFamily ?? "qwen3-asr";
+	return {
+		capability,
+		// "asr" is its own resident-role slot in RESIDENT_ROLE_PRIORITY.
+		// Qwen-ASR doesn't coexist in the same VRAM band as vision/image-gen
+		// — the arbiter only evicts ASR when a higher-priority role
+		// (text-target, text-drafter) needs the budget back.
+		residentRole: "asr",
+		estimatedMb: opts.estimatedMb ?? 600,
+		async load(modelKey: string): Promise<AsrBackend> {
+			return await loader(modelKey);
+		},
+		async unload(backend: AsrBackend): Promise<void> {
+			await backend.dispose();
+		},
+		async run(backend: AsrBackend, request: AsrRequest): Promise<AsrResult> {
+			if (!(request.pcm instanceof Float32Array) || request.pcm.length === 0) {
+				throw new AsrBackendUnavailableError(
+					backend.id,
+					"unsupported_request",
+					`[asr] backend "${backend.id}" requires non-empty Float32Array pcm`,
+				);
+			}
+			if (!Number.isFinite(request.sampleRateHz) || request.sampleRateHz <= 0) {
+				throw new AsrBackendUnavailableError(
+					backend.id,
+					"unsupported_request",
+					`[asr] backend "${backend.id}" requires a positive sampleRateHz; got ${request.sampleRateHz}`,
+				);
+			}
+			if (!backend.supports(request)) {
+				throw new AsrBackendUnavailableError(
+					backend.id,
+					"unsupported_request",
+					`[asr] backend "${backend.id}" does not support this request (sampleRateHz=${request.sampleRateHz} pcmSamples=${request.pcm.length})`,
+				);
+			}
+			// Content-hash cache lookup. Honour `signal` between cache + backend.
+			if (request.signal?.aborted) {
+				throw request.signal.reason instanceof Error
+					? request.signal.reason
+					: new DOMException("Aborted", "AbortError");
+			}
+			let hash: string | null = null;
+			if (cache) {
+				try {
+					hash = hashAsrInput(request, family);
+					const hit = cache.getCachedAsrTranscript(hash);
+					if (hit && hit.live !== false) {
+						return { text: hit.text, cacheHit: true };
+					}
+				} catch {
+					// Hashing failed (zero-length pcm guarded above; this catches
+					// hash backend issues). Fall through to the backend.
+					hash = null;
+				}
+			}
+			const result = await backend.transcribe(request);
+			if (request.signal?.aborted) {
+				throw request.signal.reason instanceof Error
+					? request.signal.reason
+					: new DOMException("Aborted", "AbortError");
+			}
+			if (cache && hash) {
+				try {
+					cache.setCachedAsrTranscript(hash, { text: result.text });
+				} catch {
+					// Caching is best-effort; never let a cache write fail a request.
+				}
+			}
+			return { ...result, cacheHit: false };
+		},
+	};
+}

package/src/services/asr/types.d.ts

@@ -0,0 +1,91 @@
+/**
+ * Local ASR (transcription) types — mirrors the WS2 vision-describe and
+ * WS3 image-gen splits.
+ *
+ * Two layers live here:
+ *
+ *   1. The **request/result** contract every backend implements
+ *      (`AsrRequest`, `AsrResult`). Callers pass mono 16 kHz fp32 PCM (or
+ *      a sample rate the backend can resample from), backends return a
+ *      transcript string plus optional token / segment metadata.
+ *
+ *   2. The **backend** interface (`AsrBackend`) the `MemoryArbiter`
+ *      (WS1) registers as a capability handler. One backend per
+ *      per-platform path:
+ *
+ *        - `fused`           — `eliza_inference_asr_*` ABI (Qwen3-ASR
+ *          inside the fused libelizainference). The sole on-device ASR
+ *          runtime: Linux / macOS / Windows desktop, AOSP system app via
+ *          the same fused build.
+ *        - `coreml`          — Capacitor bridge to a Core ML ASR
+ *          model on iOS (unavailable until the bridge ships).
+ *        - `aosp-ffi`        — bun:ffi shim around the AOSP NDK
+ *          Qwen-ASR JNI handle (unavailable until the AOSP fused
+ *          ASR symbols are exported).
+ *        - `fake`            — deterministic in-process backend used by tests.
+ *
+ * Cache contract:
+ *
+ *   The arbiter caches transcript text by **content hash** of the input PCM
+ *   (sample-rate-normalized to 16 kHz mono). The cache namespace is
+ *   `asr-transcripts` — distinct from the WS2 vision-embedding cache
+ *   (`vision-projector-tokens`) and the WS3 image-gen request key space.
+ *   Re-transcribing the same audio is a fast text return; the backend
+ *   never re-runs.
+ */
+/** A request the ASR capability handler accepts. */
+export interface AsrRequest {
+    /**
+     * Mono 32-bit float PCM samples. Range [-1, 1]. The backend will resample
+     * to 16 kHz internally if `sampleRateHz` is not 16000.
+     */
+    pcm: Float32Array;
+    /** Sample rate of `pcm`, in Hz. Common values: 16000, 24000, 48000. */
+    sampleRateHz: number;
+    /** Optional caller-supplied abort signal. Backends MUST honour it. */
+    signal?: AbortSignal;
+    /** Optional model family tag for cache scoping. Defaults to `qwen3-asr`. */
+    modelFamily?: string;
+    /** Optional BCP-47 language hint (e.g. `"en"`, `"zh"`). Backends may ignore. */
+    language?: string;
+}
+/** Backend response. The arbiter normalizes this back to a plain string for the model handler. */
+export interface AsrResult {
+    /** Final transcript. Whitespace-trimmed. Never undefined; empty string is a real "no speech detected" result. */
+    text: string;
+    /** Optional per-segment timings. Backends that support diarization emit one segment per speaker turn. */
+    segments?: ReadonlyArray<{
+        text: string;
+        startMs: number;
+        endMs: number;
+        speaker?: string;
+    }>;
+    /** Optional Qwen2-BPE token ids — the fused build emits these so STT-finish token injection skips re-tokenization. */
+    tokens?: ReadonlyArray<number>;
+    /** Optional inference time in ms (wall clock, not GPU compute). */
+    inferenceTimeMs?: number;
+    /** True when the response came from the arbiter's content-hash cache. */
+    cacheHit?: boolean;
+}
+/** Arguments the arbiter passes to the loader. */
+export interface AsrLoadArgs {
+    modelKey: string;
+}
+/** Backend contract. Every per-platform ASR runtime implements this. */
+export interface AsrBackend {
+    /** Stable identifier for telemetry / errors (`"fused"`, `"coreml"`, `"aosp-ffi"`, ...). */
+    readonly id: string;
+    /**
+     * Whether this backend supports the request as-is. False → the arbiter
+     * throws `AsrBackendUnavailableError` with `unsupported_request`. Most
+     * backends return `true` for any non-empty PCM ≤ a hard length cap.
+     */
+    supports(req: AsrRequest): boolean;
+    /** Run a transcription. The backend MUST honour `req.signal`. */
+    transcribe(req: AsrRequest): Promise<AsrResult>;
+    /** Release native resources held by this backend handle. */
+    dispose(): Promise<void>;
+}
+/** Async loader the registration takes — invoked by the arbiter on first use of a given modelKey. */
+export type AsrBackendLoader = (modelKey: string) => Promise<AsrBackend>;
+//# sourceMappingURL=types.d.ts.map

package/src/services/asr/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,oDAAoD;AACpD,MAAM,WAAW,UAAU;IAC1B;;;OAGG;IACH,GAAG,EAAE,YAAY,CAAC;IAClB,uEAAuE;IACvE,YAAY,EAAE,MAAM,CAAC;IACrB,sEAAsE;IACtE,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,4EAA4E;IAC5E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gFAAgF;IAChF,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,kGAAkG;AAClG,MAAM,WAAW,SAAS;IACzB,iHAAiH;IACjH,IAAI,EAAE,MAAM,CAAC;IACb,yGAAyG;IACzG,QAAQ,CAAC,EAAE,aAAa,CAAC;QACxB,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;QAChB,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;IACH,sHAAsH;IACtH,MAAM,CAAC,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;IAC/B,mEAAmE;IACnE,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,yEAAyE;IACzE,QAAQ,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,kDAAkD;AAClD,MAAM,WAAW,WAAW;IAC3B,QAAQ,EAAE,MAAM,CAAC;CACjB;AAED,wEAAwE;AACxE,MAAM,WAAW,UAAU;IAC1B,2FAA2F;IAC3F,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC;IACnC,iEAAiE;IACjE,UAAU,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IAChD,4DAA4D;IAC5D,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB;AAED,qGAAqG;AACrG,MAAM,MAAM,gBAAgB,GAAG,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,UAAU,CAAC,CAAC"}

package/src/services/asr/types.ts

@@ -0,0 +1,95 @@
+/**
+ * Local ASR (transcription) types — mirrors the WS2 vision-describe and
+ * WS3 image-gen splits.
+ *
+ * Two layers live here:
+ *
+ *   1. The **request/result** contract every backend implements
+ *      (`AsrRequest`, `AsrResult`). Callers pass mono 16 kHz fp32 PCM (or
+ *      a sample rate the backend can resample from), backends return a
+ *      transcript string plus optional token / segment metadata.
+ *
+ *   2. The **backend** interface (`AsrBackend`) the `MemoryArbiter`
+ *      (WS1) registers as a capability handler. One backend per
+ *      per-platform path:
+ *
+ *        - `fused`           — `eliza_inference_asr_*` ABI (Qwen3-ASR
+ *          inside the fused libelizainference). The sole on-device ASR
+ *          runtime: Linux / macOS / Windows desktop, AOSP system app via
+ *          the same fused build.
+ *        - `coreml`          — Capacitor bridge to a Core ML ASR
+ *          model on iOS (unavailable until the bridge ships).
+ *        - `aosp-ffi`        — bun:ffi shim around the AOSP NDK
+ *          Qwen-ASR JNI handle (unavailable until the AOSP fused
+ *          ASR symbols are exported).
+ *        - `fake`            — deterministic in-process backend used by tests.
+ *
+ * Cache contract:
+ *
+ *   The arbiter caches transcript text by **content hash** of the input PCM
+ *   (sample-rate-normalized to 16 kHz mono). The cache namespace is
+ *   `asr-transcripts` — distinct from the WS2 vision-embedding cache
+ *   (`vision-projector-tokens`) and the WS3 image-gen request key space.
+ *   Re-transcribing the same audio is a fast text return; the backend
+ *   never re-runs.
+ */
+
+/** A request the ASR capability handler accepts. */
+export interface AsrRequest {
+	/**
+	 * Mono 32-bit float PCM samples. Range [-1, 1]. The backend will resample
+	 * to 16 kHz internally if `sampleRateHz` is not 16000.
+	 */
+	pcm: Float32Array;
+	/** Sample rate of `pcm`, in Hz. Common values: 16000, 24000, 48000. */
+	sampleRateHz: number;
+	/** Optional caller-supplied abort signal. Backends MUST honour it. */
+	signal?: AbortSignal;
+	/** Optional model family tag for cache scoping. Defaults to `qwen3-asr`. */
+	modelFamily?: string;
+	/** Optional BCP-47 language hint (e.g. `"en"`, `"zh"`). Backends may ignore. */
+	language?: string;
+}
+
+/** Backend response. The arbiter normalizes this back to a plain string for the model handler. */
+export interface AsrResult {
+	/** Final transcript. Whitespace-trimmed. Never undefined; empty string is a real "no speech detected" result. */
+	text: string;
+	/** Optional per-segment timings. Backends that support diarization emit one segment per speaker turn. */
+	segments?: ReadonlyArray<{
+		text: string;
+		startMs: number;
+		endMs: number;
+		speaker?: string;
+	}>;
+	/** Optional Qwen2-BPE token ids — the fused build emits these so STT-finish token injection skips re-tokenization. */
+	tokens?: ReadonlyArray<number>;
+	/** Optional inference time in ms (wall clock, not GPU compute). */
+	inferenceTimeMs?: number;
+	/** True when the response came from the arbiter's content-hash cache. */
+	cacheHit?: boolean;
+}
+
+/** Arguments the arbiter passes to the loader. */
+export interface AsrLoadArgs {
+	modelKey: string;
+}
+
+/** Backend contract. Every per-platform ASR runtime implements this. */
+export interface AsrBackend {
+	/** Stable identifier for telemetry / errors (`"fused"`, `"coreml"`, `"aosp-ffi"`, ...). */
+	readonly id: string;
+	/**
+	 * Whether this backend supports the request as-is. False → the arbiter
+	 * throws `AsrBackendUnavailableError` with `unsupported_request`. Most
+	 * backends return `true` for any non-empty PCM ≤ a hard length cap.
+	 */
+	supports(req: AsrRequest): boolean;
+	/** Run a transcription. The backend MUST honour `req.signal`. */
+	transcribe(req: AsrRequest): Promise<AsrResult>;
+	/** Release native resources held by this backend handle. */
+	dispose(): Promise<void>;
+}
+
+/** Async loader the registration takes — invoked by the arbiter on first use of a given modelKey. */
+export type AsrBackendLoader = (modelKey: string) => Promise<AsrBackend>;