@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/services/vision/aosp-unavailable.ts

@@ -0,0 +1,163 @@
+/**
+ * AOSP vision-describe backend contract (WS2).
+ *
+ * The bun:ffi llama.cpp binding in `@elizaos/plugin-aosp-local-inference`
+ * already exposes the embedding helpers (`llama_set_embeddings`,
+ * `llama_get_embeddings_seq`) and the model/context lifecycle. mtmd
+ * (multi-modal definition) is part of upstream llama.cpp from b8198+
+ * but the libeliza-llama-shim that the AOSP binding loads does NOT
+ * yet export the matching `eliza_llama_mtmd_*` symbols.
+ *
+ * This file is the JS contract for the FFI binding the AOSP plugin
+ * will add when the native side ships. It lives in
+ * `plugin-local-inference` (not in `plugin-aosp-local-inference`) so
+ * cross-plugin consumers can compile against the same interface
+ * regardless of platform. When the AOSP shim ships the native
+ * symbols, the implementation in `plugin-aosp-local-inference/src/
+ * aosp-llama-vision.ts` (to be added) will satisfy this contract.
+ *
+ * Required native symbols (libeliza-llama-shim):
+ *
+ *   eliza_llama_mtmd_init_from_file(
+ *     mmproj_path: const char *,
+ *     out_handle: void **
+ *   ) -> int32  // 0 = ok; non-zero = error code
+ *
+ *   eliza_llama_mtmd_free(mtmd_handle: void *) -> void
+ *
+ *   eliza_llama_mtmd_encode(
+ *     mtmd_handle: void *,
+ *     image_bytes: const uint8_t *,
+ *     image_len: size_t,
+ *     out_tokens_ptr: float **,   // owned by mtmd; valid until next encode
+ *     out_token_count: int32 *,
+ *     out_hidden_size: int32 *
+ *   ) -> int32
+ *
+ *   eliza_llama_decode_with_mmproj(
+ *     ctx: void *,                  // existing llama_context
+ *     mtmd_tokens: const float *,
+ *     mtmd_token_count: int32,
+ *     mtmd_hidden_size: int32,
+ *     prompt: const char *,
+ *     prompt_len: size_t,
+ *     max_tokens: int32,
+ *     temperature: float,
+ *     out_text_buf: char *,         // caller-allocated
+ *     out_text_buf_cap: size_t,
+ *     out_text_written: size_t *
+ *   ) -> int32
+ *
+ * Optional fast path: if the shim adds
+ *
+ *   eliza_llama_mtmd_describe(
+ *     model: void *,                // existing llama_model
+ *     ctx: void *,                  // existing llama_context
+ *     mtmd_handle: void *,
+ *     image_bytes: const uint8_t *,
+ *     image_len: size_t,
+ *     prompt: const char *,
+ *     prompt_len: size_t,
+ *     max_tokens: int32,
+ *     temperature: float,
+ *     out_buf: char *,
+ *     out_buf_cap: size_t,
+ *     out_written: size_t *
+ *   ) -> int32
+ *
+ * that single-call wrapper is preferred — it lets the shim fuse the
+ * encode+decode steps internally and avoids the round-trip of token
+ * pointers across the FFI boundary (which bun:ffi handles, but at the
+ * cost of two extra pointer dereferences per frame).
+ *
+ * Until the native side exports those symbols, this module's `loadAospVisionBackend`
+ * throws a structured error that the arbiter / handler can surface as
+ * "vision not available on this platform". It does NOT register
+ * silently — silent unavailability would let the runtime fall back to
+ * the cloud path on a mobile device that explicitly disabled it.
+ */
+
+import { existsSync } from "node:fs";
+import { VisionBackendUnavailableError } from "./capacitor-llama";
+import type { VisionDescribeBackend, VisionDescribeLoadArgs } from "./types";
+
+/**
+ * The AOSP binding's mtmd surface, when present. The AOSP plugin
+ * registers an instance under the runtime service name
+ * `"aosp-llama-mtmd"` once the native shim exports the symbols above.
+ */
+export interface AospLlamaMtmdBinding {
+	/** True when libeliza-llama-shim.so was loaded and exports the mtmd symbols. */
+	hasMtmd(): boolean;
+	/** Initialize an mtmd handle for the given mmproj path. */
+	initMtmd(args: { mmprojPath: string }): Promise<AospMtmdHandle>;
+}
+
+export interface AospMtmdHandle {
+	/**
+	 * Single-call describe — wraps mtmd_encode + decode_with_chunks. The
+	 * AOSP shim's `eliza_llama_mtmd_describe` lands here. Backends that
+	 * only have the split encode/decode symbols implement this on top
+	 * of two FFI calls; backends with the fused symbol use one.
+	 */
+	describe(args: {
+		imageBytes: Uint8Array;
+		prompt: string;
+		maxTokens?: number;
+		temperature?: number;
+		signal?: AbortSignal;
+	}): Promise<string>;
+	/** Free the mtmd handle (and any cached encode buffers). */
+	dispose(): Promise<void>;
+}
+
+export interface LoadAospVisionBackendOptions {
+	loadArgs: VisionDescribeLoadArgs;
+	mtmdBinding?: AospLlamaMtmdBinding;
+}
+
+export async function loadAospVisionBackend(
+	opts: LoadAospVisionBackendOptions,
+): Promise<VisionDescribeBackend> {
+	const { loadArgs, mtmdBinding } = opts;
+	if (!mtmdBinding?.hasMtmd()) {
+		throw new VisionBackendUnavailableError(
+			"aosp",
+			"binding_missing_mtmd",
+			"[vision/aosp] libeliza-llama-shim does not export the mtmd symbols. The AOSP shim needs eliza_llama_mtmd_init_from_file / _encode / _describe / _free. Until the native side exports them, vision-describe on AOSP falls back to the cloud path (or stays disabled when the user opted out of cloud).",
+		);
+	}
+	if (!existsSync(loadArgs.mmprojPath)) {
+		throw new VisionBackendUnavailableError(
+			"aosp",
+			"mmproj_missing",
+			`[vision/aosp] mmproj GGUF not found: ${loadArgs.mmprojPath}`,
+		);
+	}
+	const handle = await mtmdBinding.initMtmd({
+		mmprojPath: loadArgs.mmprojPath,
+	});
+	return {
+		id: "aosp",
+		async describe(request) {
+			const { resolveImageBytes } = await import("./hash");
+			const { bytes } = resolveImageBytes(request.image);
+			const text = await handle.describe({
+				imageBytes: bytes,
+				prompt: request.prompt ?? "Describe what is in this image.",
+				maxTokens: request.maxTokens,
+				temperature: request.temperature,
+				signal: request.signal,
+			});
+			const trimmed = text.trim();
+			if (!trimmed) {
+				throw new Error("[vision/aosp] empty text from mtmd_describe");
+			}
+			const title = trimmed.split(/[.!?]/, 1)[0]?.trim() || "Image";
+			return { title, description: trimmed, cacheHit: false };
+		},
+		async dispose() {
+			await handle.dispose();
+		},
+	};
+}

package/src/services/vision/capacitor-llama.d.ts

@@ -0,0 +1,99 @@
+/**
+ * Capacitor-llama vision-describe backend (WS2).
+ *
+ * Wraps the in-process capacitor-llama binding's multimodal projector
+ * (mtmd) surface and exposes the WS2 `VisionDescribeBackend` contract.
+ *
+ * State of the binding (2026-05-19):
+ *   `llama-cpp-capacitor`'s `LlamaContext` exposes `initMultimodal` and
+ *   `getMultimodalSupport`, which load an mmproj alongside the chat
+ *   target. The desktop bun:ffi shim does not yet bind those symbols —
+ *   the desktop FFI path returns `binding_missing_mtmd` until the shim
+ *   adds `mtmd_init_from_file` + the encode/decode helpers.
+ *
+ * What this module does today:
+ *   - Provides the WS2-shaped backend so plugin-vision / plugin-image-gen /
+ *     computer-use can compile against a stable contract.
+ *   - When the binding exposes the mtmd API, the backend dispatches
+ *     through it.
+ *   - Otherwise, the backend falls back to an injected
+ *     `VisionManagerLike` implementation when one is supplied (kept as a
+ *     pluggable seam for tests and out-of-tree integrations).
+ *   - When neither path is wired, `describe()` throws a structured
+ *     `VisionBackendUnavailableError` the arbiter surfaces upward.
+ *
+ * GPU validation status (this host has neither GPU):
+ *   The mtmd encode path is GPU-accelerated when the underlying llama.cpp
+ *   build dispatches `llama_image_t` through the model's batch path. We
+ *   document the on-device validation that's required for each GPU
+ *   family at the bottom of this file's tests (see
+ *   `__tests__/vision-describe.test.ts`). Until those run on real
+ *   hardware, GPU-backed vision is "implementation present, not
+ *   validated".
+ */
+import type { VisionDescribeBackend, VisionDescribeBackendOptions, VisionDescribeLoadArgs } from "./types";
+export declare class VisionBackendUnavailableError extends Error {
+    readonly backendId: string;
+    readonly reason: "binding_missing_mtmd" | "no_fallback_present" | "mmproj_missing";
+    readonly code = "VISION_BACKEND_UNAVAILABLE";
+    constructor(backendId: string, reason: "binding_missing_mtmd" | "no_fallback_present" | "mmproj_missing", message: string);
+}
+/**
+ * Optional shape the Capacitor-llama binding exposes once the mtmd typed
+ * wrappers land in the shared adapter. The backend only consumes
+ * `describeWithMmproj`, which wraps `LlamaContext.initMultimodal` +
+ * `completion(...)` with `media_paths`. Backends that don't satisfy this
+ * shape are treated as "binding without mtmd support" and the fallback
+ * path is used.
+ */
+export interface CapacitorLlamaMtmdBinding {
+    loadVisionModel(args: {
+        modelPath: string;
+        mmprojPath: string;
+        gpuLayers?: number | "auto" | "max";
+        contextSize?: number;
+    }): Promise<CapacitorLlamaMtmdHandle>;
+}
+export interface CapacitorLlamaMtmdHandle {
+    describeWithMmproj(args: {
+        imageBytes: Uint8Array;
+        mimeType?: string;
+        prompt: string;
+        maxTokens?: number;
+        temperature?: number;
+        signal?: AbortSignal;
+        projectedTokens?: VisionDescribeBackendOptions["projectedTokens"];
+    }): Promise<{
+        text: string;
+        projectorMs?: number;
+        decodeMs?: number;
+    }>;
+    dispose(): Promise<void>;
+}
+/**
+ * Optional VisionManager-shape fallback. Kept available as a pluggable
+ * injection point for tests and out-of-tree integrations that want to
+ * supply their own image captioning implementation.
+ */
+export interface VisionManagerLike {
+    processImage(dataUrl: string): Promise<{
+        title: string;
+        description: string;
+    }>;
+}
+export interface CapacitorLlamaVisionBackendOptions {
+    loadArgs: VisionDescribeLoadArgs;
+    /**
+     * Injected by tests and by the shared mtmd typed wrappers. When
+     * provided the backend uses the mtmd path.
+     */
+    mtmd?: CapacitorLlamaMtmdBinding;
+    /**
+     * Caption-only fallback. Optional — when present the backend uses it
+     * as last resort, after mtmd. Backends that have neither throw a
+     * structured `VisionBackendUnavailableError`.
+     */
+    visionManager?: VisionManagerLike;
+}
+export declare function loadCapacitorLlamaVisionBackend(opts: CapacitorLlamaVisionBackendOptions): Promise<VisionDescribeBackend>;
+//# sourceMappingURL=capacitor-llama.d.ts.map

package/src/services/vision/capacitor-llama.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"capacitor-llama.d.ts","sourceRoot":"","sources":["capacitor-llama.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAIH,OAAO,KAAK,EACX,qBAAqB,EACrB,4BAA4B,EAC5B,sBAAsB,EAGtB,MAAM,SAAS,CAAC;AAEjB,qBAAa,6BAA8B,SAAQ,KAAK;IAGtD,QAAQ,CAAC,SAAS,EAAE,MAAM;IAC1B,QAAQ,CAAC,MAAM,EACZ,sBAAsB,GACtB,qBAAqB,GACrB,gBAAgB;IANpB,QAAQ,CAAC,IAAI,gCAAgC;gBAEnC,SAAS,EAAE,MAAM,EACjB,MAAM,EACZ,sBAAsB,GACtB,qBAAqB,GACrB,gBAAgB,EACnB,OAAO,EAAE,MAAM;CAKhB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,yBAAyB;IACzC,eAAe,CAAC,IAAI,EAAE;QACrB,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC;QACpC,WAAW,CAAC,EAAE,MAAM,CAAC;KACrB,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;CACtC;AAED,MAAM,WAAW,wBAAwB;IACxC,kBAAkB,CAAC,IAAI,EAAE;QACxB,UAAU,EAAE,UAAU,CAAC;QACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,eAAe,CAAC,EAAE,4BAA4B,CAAC,iBAAiB,CAAC,CAAC;KAClE,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACvE,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB;AAED;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IACjC,YAAY,CACX,OAAO,EAAE,MAAM,GACb,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACnD;AAED,MAAM,WAAW,kCAAkC;IAClD,QAAQ,EAAE,sBAAsB,CAAC;IACjC;;;OAGG;IACH,IAAI,CAAC,EAAE,yBAAyB,CAAC;IACjC;;;;OAIG;IACH,aAAa,CAAC,EAAE,iBAAiB,CAAC;CAClC;AAID,wBAAsB,+BAA+B,CACpD,IAAI,EAAE,kCAAkC,GACtC,OAAO,CAAC,qBAAqB,CAAC,CA2EhC"}

package/src/services/vision/capacitor-llama.ts

@@ -0,0 +1,255 @@
+/**
+ * Capacitor-llama vision-describe backend (WS2).
+ *
+ * Wraps the in-process capacitor-llama binding's multimodal projector
+ * (mtmd) surface and exposes the WS2 `VisionDescribeBackend` contract.
+ *
+ * State of the binding (2026-05-19):
+ *   `llama-cpp-capacitor`'s `LlamaContext` exposes `initMultimodal` and
+ *   `getMultimodalSupport`, which load an mmproj alongside the chat
+ *   target. The desktop bun:ffi shim does not yet bind those symbols —
+ *   the desktop FFI path returns `binding_missing_mtmd` until the shim
+ *   adds `mtmd_init_from_file` + the encode/decode helpers.
+ *
+ * What this module does today:
+ *   - Provides the WS2-shaped backend so plugin-vision / plugin-image-gen /
+ *     computer-use can compile against a stable contract.
+ *   - When the binding exposes the mtmd API, the backend dispatches
+ *     through it.
+ *   - Otherwise, the backend falls back to an injected
+ *     `VisionManagerLike` implementation when one is supplied (kept as a
+ *     pluggable seam for tests and out-of-tree integrations).
+ *   - When neither path is wired, `describe()` throws a structured
+ *     `VisionBackendUnavailableError` the arbiter surfaces upward.
+ *
+ * GPU validation status (this host has neither GPU):
+ *   The mtmd encode path is GPU-accelerated when the underlying llama.cpp
+ *   build dispatches `llama_image_t` through the model's batch path. We
+ *   document the on-device validation that's required for each GPU
+ *   family at the bottom of this file's tests (see
+ *   `__tests__/vision-describe.test.ts`). Until those run on real
+ *   hardware, GPU-backed vision is "implementation present, not
+ *   validated".
+ */
+
+import { existsSync, promises as fs } from "node:fs";
+import { resolveImageBytes } from "./hash";
+import type {
+	VisionDescribeBackend,
+	VisionDescribeBackendOptions,
+	VisionDescribeLoadArgs,
+	VisionDescribeRequest,
+	VisionDescribeResult,
+} from "./types";
+
+export class VisionBackendUnavailableError extends Error {
+	readonly code = "VISION_BACKEND_UNAVAILABLE";
+	constructor(
+		readonly backendId: string,
+		readonly reason:
+			| "binding_missing_mtmd"
+			| "no_fallback_present"
+			| "mmproj_missing",
+		message: string,
+	) {
+		super(message);
+		this.name = "VisionBackendUnavailableError";
+	}
+}
+
+/**
+ * Optional shape the Capacitor-llama binding exposes once the mtmd typed
+ * wrappers land in the shared adapter. The backend only consumes
+ * `describeWithMmproj`, which wraps `LlamaContext.initMultimodal` +
+ * `completion(...)` with `media_paths`. Backends that don't satisfy this
+ * shape are treated as "binding without mtmd support" and the fallback
+ * path is used.
+ */
+export interface CapacitorLlamaMtmdBinding {
+	loadVisionModel(args: {
+		modelPath: string;
+		mmprojPath: string;
+		gpuLayers?: number | "auto" | "max";
+		contextSize?: number;
+	}): Promise<CapacitorLlamaMtmdHandle>;
+}
+
+export interface CapacitorLlamaMtmdHandle {
+	describeWithMmproj(args: {
+		imageBytes: Uint8Array;
+		mimeType?: string;
+		prompt: string;
+		maxTokens?: number;
+		temperature?: number;
+		signal?: AbortSignal;
+		projectedTokens?: VisionDescribeBackendOptions["projectedTokens"];
+	}): Promise<{ text: string; projectorMs?: number; decodeMs?: number }>;
+	dispose(): Promise<void>;
+}
+
+/**
+ * Optional VisionManager-shape fallback. Kept available as a pluggable
+ * injection point for tests and out-of-tree integrations that want to
+ * supply their own image captioning implementation.
+ */
+export interface VisionManagerLike {
+	processImage(
+		dataUrl: string,
+	): Promise<{ title: string; description: string }>;
+}
+
+export interface CapacitorLlamaVisionBackendOptions {
+	loadArgs: VisionDescribeLoadArgs;
+	/**
+	 * Injected by tests and by the shared mtmd typed wrappers. When
+	 * provided the backend uses the mtmd path.
+	 */
+	mtmd?: CapacitorLlamaMtmdBinding;
+	/**
+	 * Caption-only fallback. Optional — when present the backend uses it
+	 * as last resort, after mtmd. Backends that have neither throw a
+	 * structured `VisionBackendUnavailableError`.
+	 */
+	visionManager?: VisionManagerLike;
+}
+
+const DEFAULT_PROMPT = "Describe what is in this image.";
+
+export async function loadCapacitorLlamaVisionBackend(
+	opts: CapacitorLlamaVisionBackendOptions,
+): Promise<VisionDescribeBackend> {
+	const { loadArgs, mtmd, visionManager } = opts;
+
+	if (mtmd) {
+		// Validate mmproj presence here so we surface a clean error before
+		// burning a load (the binding's own error would be cryptic).
+		if (!existsSync(loadArgs.mmprojPath)) {
+			throw new VisionBackendUnavailableError(
+				"capacitor-llama",
+				"mmproj_missing",
+				`[vision/capacitor-llama] mmproj GGUF not found: ${loadArgs.mmprojPath}`,
+			);
+		}
+		const handle = await mtmd.loadVisionModel({
+			modelPath: loadArgs.modelPath,
+			mmprojPath: loadArgs.mmprojPath,
+			gpuLayers: loadArgs.gpuLayers,
+			contextSize: loadArgs.contextSize,
+		});
+		return {
+			id: "capacitor-llama",
+			async describe(
+				request: VisionDescribeRequest,
+				args?: VisionDescribeBackendOptions,
+			): Promise<VisionDescribeResult> {
+				const { bytes, mimeType } = resolveImageBytes(request.image);
+				const result = await handle.describeWithMmproj({
+					imageBytes: bytes,
+					mimeType,
+					prompt: request.prompt ?? DEFAULT_PROMPT,
+					maxTokens: request.maxTokens,
+					temperature: request.temperature,
+					signal: request.signal,
+					projectedTokens: args?.projectedTokens,
+				});
+				return shapeResult(result.text, {
+					projectorMs: result.projectorMs,
+					decodeMs: result.decodeMs,
+					cacheHit: Boolean(args?.projectedTokens),
+				});
+			},
+			async dispose() {
+				await handle.dispose();
+			},
+		};
+	}
+
+	if (visionManager) {
+		return {
+			id: "capacitor-llama",
+			async describe(
+				request: VisionDescribeRequest,
+			): Promise<VisionDescribeResult> {
+				const dataUrl = await imageInputToDataUrl(request.image);
+				const result = await visionManager.processImage(dataUrl);
+				return {
+					title: result.title,
+					description: result.description,
+					cacheHit: false,
+				};
+			},
+			async dispose() {
+				// VisionManager is a process-singleton owned by LocalAIManager;
+				// its lifetime is decoupled from the WS2 backend. Disposing the
+				// backend here is a no-op — the manager stays warm for legacy
+				// callers that haven't moved off LocalAIManager.describeImage yet.
+			},
+		};
+	}
+
+	throw new VisionBackendUnavailableError(
+		"capacitor-llama",
+		"binding_missing_mtmd",
+		"[vision/capacitor-llama] no mtmd binding and no VisionManager fallback was provided. Wire up the Capacitor-llama mtmd adapter (initMultimodal + media_paths completion) or pass a VisionManager fallback in options.",
+	);
+}
+
+function shapeResult(
+	text: string,
+	telemetry: { projectorMs?: number; decodeMs?: number; cacheHit?: boolean },
+): VisionDescribeResult {
+	const trimmed = text.trim();
+	if (!trimmed) {
+		throw new Error("[vision/capacitor-llama] backend returned empty text");
+	}
+	const title = trimmed.split(/[.!?]/, 1)[0]?.trim() || "Image";
+	return {
+		title,
+		description: trimmed,
+		...telemetry,
+	};
+}
+
+async function imageInputToDataUrl(
+	input: VisionDescribeRequest["image"],
+): Promise<string> {
+	switch (input.kind) {
+		case "dataUrl":
+			return input.dataUrl;
+		case "base64":
+			return `data:${input.mimeType ?? "image/png"};base64,${input.base64}`;
+		case "bytes": {
+			const mimeType = input.mimeType ?? "image/png";
+			const base64 = Buffer.from(input.bytes).toString("base64");
+			return `data:${mimeType};base64,${base64}`;
+		}
+		case "url": {
+			const url = input.url;
+			if (url.startsWith("data:")) return url;
+			if (url.startsWith("file://") || url.startsWith("/")) {
+				const filePath = url.startsWith("file://") ? url.slice(7) : url;
+				const bytes = await fs.readFile(filePath);
+				const mimeType = input.mimeType ?? guessMimeFromPath(filePath);
+				return `data:${mimeType};base64,${bytes.toString("base64")}`;
+			}
+			const res = await fetch(url);
+			if (!res.ok) {
+				throw new Error(
+					`[vision/capacitor-llama] failed to fetch image: ${res.status} ${res.statusText}`,
+				);
+			}
+			const buf = new Uint8Array(await res.arrayBuffer());
+			const mimeType =
+				input.mimeType ?? res.headers.get("content-type") ?? "image/png";
+			return `data:${mimeType};base64,${Buffer.from(buf).toString("base64")}`;
+		}
+	}
+}
+
+function guessMimeFromPath(p: string): string {
+	const lower = p.toLowerCase();
+	if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) return "image/jpeg";
+	if (lower.endsWith(".webp")) return "image/webp";
+	if (lower.endsWith(".gif")) return "image/gif";
+	return "image/png";
+}

package/src/services/vision/cloud-fallback.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Soft cloud fallback wrapper for local IMAGE_DESCRIPTION handlers.
+ *
+ * The local vision path can report recoverable unavailability without forcing
+ * callers to know which remote provider is paired. The wrapper keeps that
+ * state explicit: handlers return a normal image description or a typed
+ * fallback outcome that the next layer can handle.
+ */
+import type { ImageDescriptionParams, ImageDescriptionResult } from "@elizaos/core";
+export type VisionFallbackReason = "local-unavailable" | "local-overloaded" | "local-error" | "local-aborted-pre-completion" | "local-not-registered" | "cloud-unavailable" | "cloud-error" | "vast-unavailable" | "vast-error";
+export type LocalVisionOutcome = ImageDescriptionResult | string | {
+    kind: "ok";
+    result: ImageDescriptionResult | string;
+} | {
+    kind: "fallback";
+    reason: VisionFallbackReason;
+    cause?: Error;
+};
+export type LocalVisionResult = Exclude<LocalVisionOutcome, {
+    kind: "fallback";
+    reason: VisionFallbackReason;
+    cause?: Error;
+}>;
+export type LocalImageDescriptionHandler = (params: ImageDescriptionParams | string) => Promise<LocalVisionOutcome>;
+export type WrappedImageDescriptionHandler = LocalImageDescriptionHandler;
+export interface VisionCloudFallbackOptions {
+    enabled?: boolean;
+    token?: string;
+    apiKey?: string;
+    baseUrl?: string;
+    fetch?: typeof fetch;
+    handler?: (params: ImageDescriptionParams | string, reason: VisionFallbackReason) => Promise<LocalVisionOutcome>;
+    log?: (message: string, detail?: Record<string, unknown>) => void;
+}
+export declare function classifyLocalVisionError(error: unknown): {
+    fallback: boolean;
+    reason: VisionFallbackReason;
+    cause?: Error;
+};
+export declare function isVisionFallbackOutcome(outcome: LocalVisionOutcome): outcome is {
+    kind: "fallback";
+    reason: VisionFallbackReason;
+    cause?: Error;
+};
+export declare function normalizeVisionDescription(result: LocalVisionResult): ImageDescriptionResult;
+export declare function wrapImageDescriptionHandlerWithCloudFallback(local: LocalImageDescriptionHandler, options?: VisionCloudFallbackOptions): WrappedImageDescriptionHandler;
+//# sourceMappingURL=cloud-fallback.d.ts.map

package/src/services/vision/cloud-fallback.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cloud-fallback.d.ts","sourceRoot":"","sources":["cloud-fallback.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EACX,sBAAsB,EACtB,sBAAsB,EACtB,MAAM,eAAe,CAAC;AAEvB,MAAM,MAAM,oBAAoB,GAC7B,mBAAmB,GACnB,kBAAkB,GAClB,aAAa,GACb,8BAA8B,GAC9B,sBAAsB,GACtB,mBAAmB,GACnB,aAAa,GACb,kBAAkB,GAClB,YAAY,CAAC;AAEhB,MAAM,MAAM,kBAAkB,GAC3B,sBAAsB,GACtB,MAAM,GACN;IAAE,IAAI,EAAE,IAAI,CAAC;IAAC,MAAM,EAAE,sBAAsB,GAAG,MAAM,CAAA;CAAE,GACvD;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,MAAM,EAAE,oBAAoB,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,CAAC;AAErE,MAAM,MAAM,iBAAiB,GAAG,OAAO,CACtC,kBAAkB,EAClB;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,MAAM,EAAE,oBAAoB,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,CACjE,CAAC;AAEF,MAAM,MAAM,4BAA4B,GAAG,CAC1C,MAAM,EAAE,sBAAsB,GAAG,MAAM,KACnC,OAAO,CAAC,kBAAkB,CAAC,CAAC;AAEjC,MAAM,MAAM,8BAA8B,GAAG,4BAA4B,CAAC;AAE1E,MAAM,WAAW,0BAA0B;IAC1C,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;IACrB,OAAO,CAAC,EAAE,CACT,MAAM,EAAE,sBAAsB,GAAG,MAAM,EACvC,MAAM,EAAE,oBAAoB,KACxB,OAAO,CAAC,kBAAkB,CAAC,CAAC;IACjC,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CAClE;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,OAAO,GAAG;IACzD,QAAQ,EAAE,OAAO,CAAC;IAClB,MAAM,EAAE,oBAAoB,CAAC;IAC7B,KAAK,CAAC,EAAE,KAAK,CAAC;CACd,CA2CA;AAED,wBAAgB,uBAAuB,CACtC,OAAO,EAAE,kBAAkB,GACzB,OAAO,IAAI;IACb,IAAI,EAAE,UAAU,CAAC;IACjB,MAAM,EAAE,oBAAoB,CAAC;IAC7B,KAAK,CAAC,EAAE,KAAK,CAAC;CACd,CAOA;AAED,wBAAgB,0BAA0B,CACzC,MAAM,EAAE,iBAAiB,GACvB,sBAAsB,CAiCxB;AAqED,wBAAgB,4CAA4C,CAC3D,KAAK,EAAE,4BAA4B,EACnC,OAAO,GAAE,0BAA+B,GACtC,8BAA8B,CAyChC"}