@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/services/tts/types.ts

@@ -0,0 +1,157 @@
+/**
+ * Local text-to-speech capability — public type surface.
+ *
+ * Mirrors the layout of `services/imagegen/types.ts` and `services/vision/types.ts`:
+ *
+ *   - The **request/result** contract every TTS backend implements
+ *     (`TtsRequest`, `TtsResult`). The synth target is bytes (PCM/WAV/MP3).
+ *
+ *   - The **backend** interface (`TtsBackend`) that the WS1 `MemoryArbiter`
+ *     registers as the `"speak"` capability handler. One backend per
+ *     per-platform TTS runtime:
+ *
+ *       - `omnivoice` — fused omnivoice.cpp llama-server (desktop + AOSP)
+ *         and the FFI streaming path (mobile). Default for tiers 9b / 27b
+ *         / 27b-256k and a co-default for 9b.
+ *       - `kokoro`    — Kokoro-82M ONNX/GGUF. Default for tiers 0_8b / 2b
+ *         / 4b (the low-memory mobile floor). Co-default for 9b.
+ *       - `edge-tts`  — Microsoft Edge cloud TTS, wired by
+ *         `@elizaos/plugin-edge-tts`. Fallback when both local backends
+ *         are unavailable (no GPU, missing weights). Routed via priority
+ *         order in the runtime model registry, NOT as a sub-backend of
+ *         this capability — Edge TTS is a different `provider` from the
+ *         runtime's perspective.
+ *       - `fake`      — deterministic in-process backend used by tests.
+ *
+ * Audio format contract:
+ *   The arbiter's `speak` request returns Uint8Array. The bytes carry the
+ *   audio container declared in `TtsResult.mime`:
+ *     - `audio/wav`       — PCM in a RIFF/WAVE container (OmniVoice path
+ *                            produces 24 kHz mono int16 by default).
+ *     - `audio/mpeg`      — MP3 (Edge TTS default).
+ *     - `audio/pcm-f32`   — bare 32-bit float PCM. Used internally between
+ *                            Kokoro's runtime and the streaming sink; the
+ *                            capability layer wraps it as WAV before
+ *                            returning to TEXT_TO_SPEECH callers.
+ *
+ * Cache namespace:
+ *   `tts-audio`. Keyed on `sha256(provider || model || voice || text)`.
+ *   The first-line cache (`services/voice/wrap-with-first-line-cache.ts`)
+ *   uses its own keying space and lives at a different layer; the
+ *   capability-level cache (when wired) sits between the arbiter and the
+ *   backend so it works for both local backends without each having to
+ *   re-implement it.
+ */
+
+/** Audio container the backend returns. */
+export type TtsMimeType =
+	| "audio/wav"
+	| "audio/mpeg"
+	| "audio/pcm-f32"
+	| "audio/opus";
+
+/**
+ * Caller request to `synthesize`. `text` is mandatory; everything else is
+ * optional with backend-specific defaults.
+ *
+ * Knob semantics (consistent across backends):
+ *   - `voice`: voice id. Kokoro uses ids like `"af_bella"`; OmniVoice uses
+ *     a preset path / preset id from the speaker-preset store. Edge TTS
+ *     uses Microsoft voice names (`"en-US-JennyNeural"`).
+ *   - `speed`: speech rate. `1.0` is normal; `0.5` half; `2.0` double. The
+ *     concrete backend maps this to its native knob (Edge TTS percentage
+ *     string; Kokoro phoneme stretch; OmniVoice maskgit-step pacing).
+ *   - `sampleRate`: target output sample rate when the container supports
+ *     it. Backends MAY downsample / upsample to match; omitted = backend
+ *     native (24 kHz for Kokoro and Edge TTS; 22.05 kHz for OmniVoice).
+ *   - `signal`: abort hook. Backends honor this at chunk boundaries.
+ */
+export interface TtsRequest {
+	text: string;
+	voice?: string;
+	speed?: number;
+	sampleRate?: number;
+	/** Hint to the backend that the caller wants streaming chunks. */
+	streaming?: boolean;
+	signal?: AbortSignal;
+}
+
+/**
+ * Backend response. `audio` is the raw bytes in the container declared by
+ * `mime`. `sampleRate` reports the *actual* sample rate the bytes carry
+ * (which may differ from the requested `sampleRate` if the backend chose
+ * its native rate).
+ */
+export interface TtsResult {
+	audio: Uint8Array;
+	mime: TtsMimeType;
+	sampleRate: number;
+	metadata: {
+		/** The model id (catalog key) the run used (`"tts-omnivoice-base-Q8_0"`, `"tts-kokoro"`, etc.). */
+		model: string;
+		/** Resolved voice id (post-default-resolve). */
+		voice: string;
+		/** Echo of the text that was synthesized. */
+		text: string;
+		/** End-to-end wall-clock time inside the backend. */
+		inferenceTimeMs: number;
+		/** Whether the result came from the capability-level cache. */
+		cacheHit?: boolean;
+	};
+}
+
+/**
+ * Per-load arguments for a TTS backend. The arbiter's `load(modelKey)`
+ * only carries an opaque key; the binding resolves it to real
+ * model+tokenizer+voice-preset paths through this struct, which
+ * `createTtsCapabilityRegistration` populates from the catalog +
+ * `ELIZA_1_GGUF_PLATFORM_PLAN.json`.
+ *
+ * The optional `voicePresetPath` exists because the runtime's default
+ * voice ships as a precomputed `cache/voice-preset-default.bin` (ELZ2
+ * v2). Backends that can use a frozen preset short-circuit the encoder
+ * pass; backends that can't ignore the hint.
+ */
+export interface TtsLoadArgs {
+	/** Absolute path to the primary TTS weights (GGUF for OmniVoice, ONNX for Kokoro). */
+	modelPath: string;
+	/** Optional tokenizer path (OmniVoice ships a separate `omnivoice-tokenizer-*.gguf`). */
+	tokenizerPath?: string;
+	/** Optional voice-preset cache path (`cache/voice-preset-default.bin`). */
+	voicePresetPath?: string;
+	/** Optional voice pack directory (Kokoro `voices/af_bella.bin`, etc.). */
+	voicePackDir?: string;
+	/** Cancel a slow load (model file read + weight upload). */
+	signal?: AbortSignal;
+}
+
+/**
+ * The contract every TTS backend implements. The shape is intentionally
+ * narrow: the arbiter only ever calls `synthesize`. `dispose` is wrapped
+ * by the arbiter's `unload` so the backend can free GPU/VRAM and drop
+ * file descriptors / kill subprocesses on eviction.
+ */
+export interface TtsBackend {
+	/** Stable identifier — matches the backend module name. */
+	readonly id: "omnivoice" | "kokoro" | "edge-tts" | "fake";
+	/**
+	 * Best-effort capability check. Backends return `false` for requests
+	 * whose `voice` / `sampleRate` aren't supported, so the arbiter can
+	 * surface a clear error rather than synthesizing garbage. Default
+	 * implementations accept anything reasonable; the gate matters for
+	 * Kokoro (fixed voice pack list) and OmniVoice (cloned-voice presets
+	 * tied to a specific bundle).
+	 */
+	supports(request: TtsRequest): boolean;
+	synthesize(request: TtsRequest): Promise<TtsResult>;
+	/** Release the loaded weights / subprocess. Idempotent. */
+	dispose(): Promise<void>;
+}
+
+/**
+ * Capability handler loader. The arbiter calls it with a model key (e.g.
+ * `"tts-omnivoice-base-Q8_0"`); the implementation resolves to a real
+ * `TtsLoadArgs` from `ELIZA_1_GGUF_PLATFORM_PLAN.json` + the installed
+ * bundle and returns a live backend.
+ */
+export type TtsBackendLoader = (modelKey: string) => Promise<TtsBackend>;

package/src/services/types.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Local inference type re-exports.
+ *
+ * The canonical definitions live in `@elizaos/shared/local-inference`.
+ * This shim preserves the historical import path
+ * `../services/local-inference/types` for server-side code.
+ */
+export { type ActiveModelState, AGENT_MODEL_SLOTS, type AgentModelSlot, type CatalogModel, type CatalogQuantizationId, type CatalogQuantizationMatrix, type CatalogQuantizationVariant, type CpuFeatureProbe, type DownloadEvent, type DownloadJob, type DownloadState, type GpuProfile, type GpuProfileId, type HardwareFitLevel, type HardwareProbe, type InstalledModel, type KvCacheType, type LocalInferenceDownloadStatus, type LocalInferenceReadiness, type LocalInferenceSlotReadiness, type LocalRuntimeAcceleration, type LocalRuntimeBackend, type LocalRuntimeKernel, type LocalRuntimeOptimizations, type MobileHardwareProbe, type ModelAssignments, type ModelBucket, type ModelCategory, type ModelHubSnapshot, type OpenVinoDeviceKind, type OpenVinoHardwareProbe, TEXT_GENERATION_SLOTS, type TextGenerationSlot, type TokenizerFamily, } from "@elizaos/shared";
+/** RAM requirements for a model bundle. */
+export interface RamBudget {
+    /** Minimum RAM the bundle will boot under, in megabytes. */
+    minMb: number;
+    /** RAM the bundle expects for nominal workloads, in megabytes. */
+    recommendedMb: number;
+    /** Where the numbers came from. `manifest` only when both came from
+     *  a validated `eliza-1.manifest.json` next to the installed bundle. */
+    source: "manifest" | "catalog";
+}
+//# sourceMappingURL=types.d.ts.map

package/src/services/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EACN,KAAK,gBAAgB,EACrB,iBAAiB,EACjB,KAAK,cAAc,EACnB,KAAK,YAAY,EACjB,KAAK,qBAAqB,EAC1B,KAAK,yBAAyB,EAC9B,KAAK,0BAA0B,EAC/B,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,WAAW,EAChB,KAAK,aAAa,EAClB,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,gBAAgB,EACrB,KAAK,aAAa,EAClB,KAAK,cAAc,EACnB,KAAK,WAAW,EAChB,KAAK,4BAA4B,EACjC,KAAK,uBAAuB,EAC5B,KAAK,2BAA2B,EAChC,KAAK,wBAAwB,EAC7B,KAAK,mBAAmB,EACxB,KAAK,kBAAkB,EACvB,KAAK,yBAAyB,EAC9B,KAAK,mBAAmB,EACxB,KAAK,gBAAgB,EACrB,KAAK,WAAW,EAChB,KAAK,aAAa,EAClB,KAAK,gBAAgB,EACrB,KAAK,kBAAkB,EACvB,KAAK,qBAAqB,EAC1B,qBAAqB,EACrB,KAAK,kBAAkB,EACvB,KAAK,eAAe,GACpB,MAAM,iBAAiB,CAAC;AAEzB,2CAA2C;AAC3C,MAAM,WAAW,SAAS;IACzB,4DAA4D;IAC5D,KAAK,EAAE,MAAM,CAAC;IACd,kEAAkE;IAClE,aAAa,EAAE,MAAM,CAAC;IACtB;4EACwE;IACxE,MAAM,EAAE,UAAU,GAAG,SAAS,CAAC;CAC/B"}

package/src/services/types.ts

@@ -0,0 +1,55 @@
+/**
+ * Local inference type re-exports.
+ *
+ * The canonical definitions live in `@elizaos/shared/local-inference`.
+ * This shim preserves the historical import path
+ * `../services/local-inference/types` for server-side code.
+ */
+
+export {
+	type ActiveModelState,
+	AGENT_MODEL_SLOTS,
+	type AgentModelSlot,
+	type CatalogModel,
+	type CatalogQuantizationId,
+	type CatalogQuantizationMatrix,
+	type CatalogQuantizationVariant,
+	type CpuFeatureProbe,
+	type DownloadEvent,
+	type DownloadJob,
+	type DownloadState,
+	type GpuProfile,
+	type GpuProfileId,
+	type HardwareFitLevel,
+	type HardwareProbe,
+	type InstalledModel,
+	type KvCacheType,
+	type LocalInferenceDownloadStatus,
+	type LocalInferenceReadiness,
+	type LocalInferenceSlotReadiness,
+	type LocalRuntimeAcceleration,
+	type LocalRuntimeBackend,
+	type LocalRuntimeKernel,
+	type LocalRuntimeOptimizations,
+	type MobileHardwareProbe,
+	type ModelAssignments,
+	type ModelBucket,
+	type ModelCategory,
+	type ModelHubSnapshot,
+	type OpenVinoDeviceKind,
+	type OpenVinoHardwareProbe,
+	TEXT_GENERATION_SLOTS,
+	type TextGenerationSlot,
+	type TokenizerFamily,
+} from "@elizaos/shared";
+
+/** RAM requirements for a model bundle. */
+export interface RamBudget {
+	/** Minimum RAM the bundle will boot under, in megabytes. */
+	minMb: number;
+	/** RAM the bundle expects for nominal workloads, in megabytes. */
+	recommendedMb: number;
+	/** Where the numbers came from. `manifest` only when both came from
+	 *  a validated `eliza-1.manifest.json` next to the installed bundle. */
+	source: "manifest" | "catalog";
+}

package/src/services/verify-on-device.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Engine-backed `verifyOnDevice` hook for the Eliza-1 downloader
+ * (`packages/inference/AGENTS.md` §7): after a bundle is materialized and
+ * every file's sha256 checks out, run one cold smoke pass —
+ *
+ *   load → 1-token text generation → (if the bundle ships voice assets)
+ *   1-phrase voice generation → barge-in cancel → unload
+ *
+ * — before the bundle is allowed to auto-fill an empty default slot. The
+ * downloader stays decoupled from the engine: it only knows the
+ * {@link VerifyBundleOnDevice} shape; this module is the concrete
+ * implementation the service layer injects.
+ *
+ * Failure semantics: any error throws. The downloader catches it and leaves
+ * `bundleVerifiedAt` unset, so an unverified bundle is registered but does
+ * not become the recommended default. There is no "verified anyway" path —
+ * voice is mandatory for Eliza-1 voice tiers (AGENTS.md §3), so a bundle
+ * whose fused voice ABI is not loadable on this device legitimately fails
+ * verify until the fused build is present.
+ */
+import fs from "node:fs/promises";
+import type { VerifyBundleOnDevice } from "./downloader";
+import { localInferenceEngine } from "./engine";
+import { parseManifestOrThrow } from "./manifest";
+type VerifyEngine = Pick<typeof localInferenceEngine, "load" | "generate" | "ensureActiveBundleVoiceReady" | "startVoice" | "armVoice" | "synthesizeSpeech" | "triggerBargeIn" | "stopVoice" | "unload">;
+interface VerifyBundleOnDeviceDeps {
+    readonly engine: VerifyEngine;
+    readonly readFile: typeof fs.readFile;
+    readonly parseManifest: typeof parseManifestOrThrow;
+}
+export declare function createVerifyBundleOnDevice(deps?: Partial<VerifyBundleOnDeviceDeps>): VerifyBundleOnDevice;
+export declare const verifyBundleOnDevice: VerifyBundleOnDevice;
+export {};
+//# sourceMappingURL=verify-on-device.d.ts.map

package/src/services/verify-on-device.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"verify-on-device.d.ts","sourceRoot":"","sources":["verify-on-device.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAOlD,KAAK,YAAY,GAAG,IAAI,CACvB,OAAO,oBAAoB,EACzB,MAAM,GACN,UAAU,GACV,8BAA8B,GAC9B,YAAY,GACZ,UAAU,GACV,kBAAkB,GAClB,gBAAgB,GAChB,WAAW,GACX,QAAQ,CACV,CAAC;AAEF,UAAU,wBAAwB;IACjC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAC,QAAQ,CAAC;IACtC,QAAQ,CAAC,aAAa,EAAE,OAAO,oBAAoB,CAAC;CACpD;AAqDD,wBAAgB,0BAA0B,CACzC,IAAI,GAAE,OAAO,CAAC,wBAAwB,CAAM,GAC1C,oBAAoB,CAoBtB;AAED,eAAO,MAAM,oBAAoB,EAAE,oBACN,CAAC"}

package/src/services/verify-on-device.test.ts

@@ -0,0 +1,87 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { createVerifyBundleOnDevice } from "./verify-on-device";
+
+const engineMock = {
+	load: vi.fn(async () => {}),
+	generate: vi.fn(async () => "ok"),
+	ensureActiveBundleVoiceReady: vi.fn(async () => ({})),
+	startVoice: vi.fn(() => {}),
+	armVoice: vi.fn(async () => {}),
+	synthesizeSpeech: vi.fn(async () => new Uint8Array([1, 2, 3, 4])),
+	triggerBargeIn: vi.fn(() => {}),
+	stopVoice: vi.fn(async () => {}),
+	unload: vi.fn(async () => {}),
+};
+
+const manifestState: { voiceFiles: number } = { voiceFiles: 0 };
+
+function verifier() {
+	const deps = {
+		engine: engineMock,
+		readFile: vi.fn(async () => "{}"),
+		parseManifest: () => ({
+			files: {
+				voice: Array.from({ length: manifestState.voiceFiles }, (_, i) => ({
+					path: `tts/v${i}.gguf`,
+					sha256: "x",
+				})),
+			},
+		}),
+	} as unknown as Parameters<typeof createVerifyBundleOnDevice>[0];
+	return createVerifyBundleOnDevice(deps);
+}
+
+const ARGS = {
+	modelId: "eliza-1-0_8b",
+	bundleRoot: "/tmp/bundle",
+	manifestPath: "/tmp/bundle/eliza-1.manifest.json",
+	textGgufPath: "/tmp/bundle/text/eliza-1-0_8b.gguf",
+};
+
+afterEach(() => {
+	vi.clearAllMocks();
+	manifestState.voiceFiles = 0;
+});
+
+describe("verifyBundleOnDevice", () => {
+	it("loads, runs a 1-token text gen, and unloads for a text-only bundle", async () => {
+		manifestState.voiceFiles = 0;
+		await verifier()(ARGS);
+		expect(engineMock.load).toHaveBeenCalledWith(ARGS.textGgufPath, {
+			modelPath: ARGS.textGgufPath,
+			modelId: ARGS.modelId,
+		});
+		expect(engineMock.generate).toHaveBeenCalledWith(
+			expect.objectContaining({ maxTokens: 1 }),
+		);
+		expect(engineMock.ensureActiveBundleVoiceReady).not.toHaveBeenCalled();
+		expect(engineMock.startVoice).not.toHaveBeenCalled();
+		expect(engineMock.unload).toHaveBeenCalled();
+	});
+
+	it("also runs a 1-phrase voice gen + barge-in cancel when the bundle ships voice", async () => {
+		manifestState.voiceFiles = 1;
+		await verifier()(ARGS);
+		expect(engineMock.ensureActiveBundleVoiceReady).toHaveBeenCalled();
+		expect(engineMock.startVoice).not.toHaveBeenCalled();
+		expect(engineMock.armVoice).not.toHaveBeenCalled();
+		expect(engineMock.synthesizeSpeech).toHaveBeenCalled();
+		expect(engineMock.triggerBargeIn).toHaveBeenCalled();
+		expect(engineMock.stopVoice).toHaveBeenCalled();
+		expect(engineMock.unload).toHaveBeenCalled();
+	});
+
+	it("rethrows verify failures and still unloads", async () => {
+		manifestState.voiceFiles = 0;
+		engineMock.generate.mockRejectedValueOnce(new Error("kernel missing"));
+		await expect(verifier()(ARGS)).rejects.toThrow("kernel missing");
+		expect(engineMock.unload).toHaveBeenCalled();
+	});
+
+	it("fails verify when voice synthesis yields no PCM", async () => {
+		manifestState.voiceFiles = 1;
+		engineMock.synthesizeSpeech.mockResolvedValueOnce(new Uint8Array(0));
+		await expect(verifier()(ARGS)).rejects.toThrow(/no PCM bytes/);
+		expect(engineMock.stopVoice).toHaveBeenCalled();
+	});
+});

package/src/services/verify-on-device.ts

@@ -0,0 +1,127 @@
+/**
+ * Engine-backed `verifyOnDevice` hook for the Eliza-1 downloader
+ * (`packages/inference/AGENTS.md` §7): after a bundle is materialized and
+ * every file's sha256 checks out, run one cold smoke pass —
+ *
+ *   load → 1-token text generation → (if the bundle ships voice assets)
+ *   1-phrase voice generation → barge-in cancel → unload
+ *
+ * — before the bundle is allowed to auto-fill an empty default slot. The
+ * downloader stays decoupled from the engine: it only knows the
+ * {@link VerifyBundleOnDevice} shape; this module is the concrete
+ * implementation the service layer injects.
+ *
+ * Failure semantics: any error throws. The downloader catches it and leaves
+ * `bundleVerifiedAt` unset, so an unverified bundle is registered but does
+ * not become the recommended default. There is no "verified anyway" path —
+ * voice is mandatory for Eliza-1 voice tiers (AGENTS.md §3), so a bundle
+ * whose fused voice ABI is not loadable on this device legitimately fails
+ * verify until the fused build is present.
+ */
+
+import fs from "node:fs/promises";
+import type { VerifyBundleOnDevice } from "./downloader";
+import { localInferenceEngine } from "./engine";
+import { parseManifestOrThrow } from "./manifest";
+
+/** A short, deterministic prompt — we only care that one token comes back. */
+const VERIFY_PROMPT = "Reply with one word.";
+/** A short phrase to drive a single TTS dispatch through the voice scheduler. */
+const VERIFY_PHRASE = "Ready.";
+
+type VerifyEngine = Pick<
+	typeof localInferenceEngine,
+	| "load"
+	| "generate"
+	| "ensureActiveBundleVoiceReady"
+	| "startVoice"
+	| "armVoice"
+	| "synthesizeSpeech"
+	| "triggerBargeIn"
+	| "stopVoice"
+	| "unload"
+>;
+
+interface VerifyBundleOnDeviceDeps {
+	readonly engine: VerifyEngine;
+	readonly readFile: typeof fs.readFile;
+	readonly parseManifest: typeof parseManifestOrThrow;
+}
+
+async function manifestDeclaresVoice(
+	manifestPath: string,
+	deps: Pick<VerifyBundleOnDeviceDeps, "readFile" | "parseManifest">,
+): Promise<boolean> {
+	const raw = await deps.readFile(manifestPath, "utf8");
+	const manifest = deps.parseManifest(JSON.parse(String(raw)));
+	// Voice tiers ship a TTS GGUF under `files.voice`; the ASR/VAD files are
+	// gated on top of that. If there is no voice file, this is a text-only
+	// bundle and the voice leg of the smoke is skipped.
+	return manifest.files.voice.length > 0;
+}
+
+async function verifyText(
+	engine: VerifyEngine,
+	modelId: string,
+	textGgufPath: string,
+): Promise<void> {
+	await engine.load(textGgufPath, { modelPath: textGgufPath, modelId });
+	const out = await engine.generate({
+		prompt: VERIFY_PROMPT,
+		maxTokens: 1,
+		temperature: 0,
+	});
+	if (typeof out !== "string") {
+		throw new Error(
+			`[verify-on-device] text generation returned ${typeof out}, expected string`,
+		);
+	}
+}
+
+async function verifyVoice(
+	engine: VerifyEngine,
+	_bundleRoot: string,
+): Promise<void> {
+	await engine.ensureActiveBundleVoiceReady();
+	try {
+		// One real synthesis through the voice bridge.
+		const pcm = await engine.synthesizeSpeech(VERIFY_PHRASE);
+		if (!(pcm instanceof Uint8Array) || pcm.byteLength === 0) {
+			throw new Error(
+				"[verify-on-device] voice synthesis produced no PCM bytes",
+			);
+		}
+		// Barge-in cancel must be accepted without throwing — exercises the
+		// hard-stop path the voice loop uses to abort speculative TTS.
+		engine.triggerBargeIn();
+	} finally {
+		await engine.stopVoice();
+	}
+}
+
+export function createVerifyBundleOnDevice(
+	deps: Partial<VerifyBundleOnDeviceDeps> = {},
+): VerifyBundleOnDevice {
+	const engine = deps.engine ?? localInferenceEngine;
+	const manifestDeps = {
+		readFile: deps.readFile ?? fs.readFile,
+		parseManifest: deps.parseManifest ?? parseManifestOrThrow,
+	};
+
+	return async ({ modelId, bundleRoot, manifestPath, textGgufPath }) => {
+		try {
+			await verifyText(engine, modelId, textGgufPath);
+			if (await manifestDeclaresVoice(manifestPath, manifestDeps)) {
+				await verifyVoice(engine, bundleRoot);
+			}
+		} finally {
+			// Always release the model the verify pass loaded — the bundle is not
+			// "active" yet, and the active-model coordinator owns load/unload from
+			// here on.
+			await engine.unload().catch(() => {});
+		}
+	};
+}
+
+export const verifyBundleOnDevice: VerifyBundleOnDevice =
+	createVerifyBundleOnDevice();

package/src/services/verify.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Re-export of the shared model-file verification module. The canonical
+ * implementation lives in `@elizaos/shared/local-inference` because both
+ * the server (`@elizaos/app-core`) and the UI client (`@elizaos/ui`)
+ * compute the same SHA256 / GGUF-magic checks against on-disk models.
+ */
+export { __registryPathForTests, hashFile, type VerifyResult, type VerifyState, verifyInstalledModel, } from "@elizaos/shared/local-inference/verify";
+//# sourceMappingURL=verify.d.ts.map

package/src/services/verify.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"verify.d.ts","sourceRoot":"","sources":["verify.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,EACN,sBAAsB,EACtB,QAAQ,EACR,KAAK,YAAY,EACjB,KAAK,WAAW,EAChB,oBAAoB,GACpB,MAAM,wCAAwC,CAAC"}

package/src/services/verify.ts

@@ -0,0 +1,13 @@
+/**
+ * Re-export of the shared model-file verification module. The canonical
+ * implementation lives in `@elizaos/shared/local-inference` because both
+ * the server (`@elizaos/app-core`) and the UI client (`@elizaos/ui`)
+ * compute the same SHA256 / GGUF-magic checks against on-disk models.
+ */
+export {
+	__registryPathForTests,
+	hashFile,
+	type VerifyResult,
+	type VerifyState,
+	verifyInstalledModel,
+} from "@elizaos/shared/local-inference/verify";

package/src/services/vision/aosp-unavailable.d.ts

@@ -0,0 +1,115 @@
+/**
+ * AOSP vision-describe backend contract (WS2).
+ *
+ * The bun:ffi llama.cpp binding in `@elizaos/plugin-aosp-local-inference`
+ * already exposes the embedding helpers (`llama_set_embeddings`,
+ * `llama_get_embeddings_seq`) and the model/context lifecycle. mtmd
+ * (multi-modal definition) is part of upstream llama.cpp from b8198+
+ * but the libeliza-llama-shim that the AOSP binding loads does NOT
+ * yet export the matching `eliza_llama_mtmd_*` symbols.
+ *
+ * This file is the JS contract for the FFI binding the AOSP plugin
+ * will add when the native side ships. It lives in
+ * `plugin-local-inference` (not in `plugin-aosp-local-inference`) so
+ * cross-plugin consumers can compile against the same interface
+ * regardless of platform. When the AOSP shim ships the native
+ * symbols, the implementation in `plugin-aosp-local-inference/src/
+ * aosp-llama-vision.ts` (to be added) will satisfy this contract.
+ *
+ * Required native symbols (libeliza-llama-shim):
+ *
+ *   eliza_llama_mtmd_init_from_file(
+ *     mmproj_path: const char *,
+ *     out_handle: void **
+ *   ) -> int32  // 0 = ok; non-zero = error code
+ *
+ *   eliza_llama_mtmd_free(mtmd_handle: void *) -> void
+ *
+ *   eliza_llama_mtmd_encode(
+ *     mtmd_handle: void *,
+ *     image_bytes: const uint8_t *,
+ *     image_len: size_t,
+ *     out_tokens_ptr: float **,   // owned by mtmd; valid until next encode
+ *     out_token_count: int32 *,
+ *     out_hidden_size: int32 *
+ *   ) -> int32
+ *
+ *   eliza_llama_decode_with_mmproj(
+ *     ctx: void *,                  // existing llama_context
+ *     mtmd_tokens: const float *,
+ *     mtmd_token_count: int32,
+ *     mtmd_hidden_size: int32,
+ *     prompt: const char *,
+ *     prompt_len: size_t,
+ *     max_tokens: int32,
+ *     temperature: float,
+ *     out_text_buf: char *,         // caller-allocated
+ *     out_text_buf_cap: size_t,
+ *     out_text_written: size_t *
+ *   ) -> int32
+ *
+ * Optional fast path: if the shim adds
+ *
+ *   eliza_llama_mtmd_describe(
+ *     model: void *,                // existing llama_model
+ *     ctx: void *,                  // existing llama_context
+ *     mtmd_handle: void *,
+ *     image_bytes: const uint8_t *,
+ *     image_len: size_t,
+ *     prompt: const char *,
+ *     prompt_len: size_t,
+ *     max_tokens: int32,
+ *     temperature: float,
+ *     out_buf: char *,
+ *     out_buf_cap: size_t,
+ *     out_written: size_t *
+ *   ) -> int32
+ *
+ * that single-call wrapper is preferred — it lets the shim fuse the
+ * encode+decode steps internally and avoids the round-trip of token
+ * pointers across the FFI boundary (which bun:ffi handles, but at the
+ * cost of two extra pointer dereferences per frame).
+ *
+ * Until the native side exports those symbols, this module's `loadAospVisionBackend`
+ * throws a structured error that the arbiter / handler can surface as
+ * "vision not available on this platform". It does NOT register
+ * silently — silent unavailability would let the runtime fall back to
+ * the cloud path on a mobile device that explicitly disabled it.
+ */
+import type { VisionDescribeBackend, VisionDescribeLoadArgs } from "./types";
+/**
+ * The AOSP binding's mtmd surface, when present. The AOSP plugin
+ * registers an instance under the runtime service name
+ * `"aosp-llama-mtmd"` once the native shim exports the symbols above.
+ */
+export interface AospLlamaMtmdBinding {
+    /** True when libeliza-llama-shim.so was loaded and exports the mtmd symbols. */
+    hasMtmd(): boolean;
+    /** Initialize an mtmd handle for the given mmproj path. */
+    initMtmd(args: {
+        mmprojPath: string;
+    }): Promise<AospMtmdHandle>;
+}
+export interface AospMtmdHandle {
+    /**
+     * Single-call describe — wraps mtmd_encode + decode_with_chunks. The
+     * AOSP shim's `eliza_llama_mtmd_describe` lands here. Backends that
+     * only have the split encode/decode symbols implement this on top
+     * of two FFI calls; backends with the fused symbol use one.
+     */
+    describe(args: {
+        imageBytes: Uint8Array;
+        prompt: string;
+        maxTokens?: number;
+        temperature?: number;
+        signal?: AbortSignal;
+    }): Promise<string>;
+    /** Free the mtmd handle (and any cached encode buffers). */
+    dispose(): Promise<void>;
+}
+export interface LoadAospVisionBackendOptions {
+    loadArgs: VisionDescribeLoadArgs;
+    mtmdBinding?: AospLlamaMtmdBinding;
+}
+export declare function loadAospVisionBackend(opts: LoadAospVisionBackendOptions): Promise<VisionDescribeBackend>;
+//# sourceMappingURL=aosp-unavailable.d.ts.map

package/src/services/vision/aosp-unavailable.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"aosp-unavailable.d.ts","sourceRoot":"","sources":["aosp-unavailable.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6EG;AAIH,OAAO,KAAK,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,MAAM,SAAS,CAAC;AAE7E;;;;GAIG;AACH,MAAM,WAAW,oBAAoB;IACpC,gFAAgF;IAChF,OAAO,IAAI,OAAO,CAAC;IACnB,2DAA2D;IAC3D,QAAQ,CAAC,IAAI,EAAE;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;CAChE;AAED,MAAM,WAAW,cAAc;IAC9B;;;;;OAKG;IACH,QAAQ,CAAC,IAAI,EAAE;QACd,UAAU,EAAE,UAAU,CAAC;QACvB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,CAAC,EAAE,WAAW,CAAC;KACrB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACpB,4DAA4D;IAC5D,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB;AAED,MAAM,WAAW,4BAA4B;IAC5C,QAAQ,EAAE,sBAAsB,CAAC;IACjC,WAAW,CAAC,EAAE,oBAAoB,CAAC;CACnC;AAED,wBAAsB,qBAAqB,CAC1C,IAAI,EAAE,4BAA4B,GAChC,OAAO,CAAC,qBAAqB,CAAC,CA0ChC"}