@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/services/vision/llama-server.ts

@@ -0,0 +1,177 @@
+/**
+ * llama-server vision-describe backend (WS2).
+ *
+ * Wraps the out-of-process llama-server's `/completion` endpoint with
+ * the `image_data` array (base64-encoded payloads) and shapes the
+ * response to the WS2 `VisionDescribeBackend` contract.
+ *
+ * llama-server image-data API recap (verified against llama.cpp commit
+ * b8198+, May 2026):
+ *
+ *   POST /completion
+ *   { "prompt": "<...>USER: [img-12] What's in this image?\nASSISTANT:",
+ *     "image_data": [
+ *       { "data": "<base64 png/jpeg>", "id": 12 }
+ *     ],
+ *     "n_predict": 256,
+ *     "temperature": 0.2,
+ *     "stream": false }
+ *
+ *   Response:
+ *   { "content": "A photo of a cat.", "stop": true,
+ *     "timings": { "prompt_ms": 180.4, "predicted_ms": 423.1 } }
+ *
+ * Server-side mmproj is loaded via the `--mmproj <path>` flag on
+ * llama-server startup. The FFI runtime wrapper passes this flag
+ * already for tiers with vision enabled; this backend assumes the
+ * server has been started with the right mmproj for the active model.
+ *
+ * Backend responsibility:
+ *   - Encode the image as base64 (when not already).
+ *   - Build the prompt with the `[img-N]` placeholder convention.
+ *   - POST to `/completion`, parse the text + timings.
+ *   - Honour AbortSignal by passing it through to the fetch call.
+ *
+ * Backend explicitly does NOT:
+ *   - Start / stop the server. That's the FFI runtime wrapper's job.
+ *   - Resolve the mmproj path — the server already has it. The arbiter's
+ *     `--mmproj` was set when the text model loaded.
+ *   - Implement projector-token reuse. llama-server has no API to
+ *     accept pre-projected tokens; if the WS1 cache hit happens, this
+ *     backend ignores the hint and re-runs the projector. The cache
+ *     is more useful with the in-process node-llama-cpp backend.
+ *
+ * Metal / CUDA validation:
+ *   The llama-server build embeds the same mtmd_encode path the
+ *   in-process binding will eventually expose. On a Metal build the
+ *   image encode dispatches through the Metal compute encoder; on a
+ *   CUDA build through cuBLAS. We have no GPU on this host — see the
+ *   `__tests__/vision-describe.test.ts` notes for the GPU smoke check.
+ */
+
+import { resolveImageBytes } from "./hash";
+import type {
+	VisionDescribeBackend,
+	VisionDescribeRequest,
+	VisionDescribeResult,
+} from "./types";
+
+export interface LlamaServerVisionBackendOptions {
+	/**
+	 * Base URL of the llama-server. The FFI runtime wrapper exposes
+	 * this via `currentBaseUrl()`; pass the resolved URL here at load
+	 * time. The backend keeps it as-is across calls.
+	 */
+	baseUrl: string;
+	/**
+	 * Optional fetch override. Tests inject a fake fetch; production
+	 * uses global fetch. The signature mirrors `fetch` so the test
+	 * surface is the same as the real one.
+	 */
+	fetch?: typeof fetch;
+	/**
+	 * Default `n_predict` budget when the caller doesn't specify
+	 * `maxTokens`. 256 matches the description-length budget the
+	 * Florence-2 / VisionManager path uses today.
+	 */
+	defaultMaxTokens?: number;
+}
+
+export function createLlamaServerVisionBackend(
+	opts: LlamaServerVisionBackendOptions,
+): VisionDescribeBackend {
+	const fetchImpl = opts.fetch ?? globalThis.fetch;
+	const defaultMaxTokens = opts.defaultMaxTokens ?? 256;
+	let baseUrl = opts.baseUrl.replace(/\/$/, "");
+
+	if (!baseUrl) {
+		throw new Error(
+			"[vision/llama-server] baseUrl is required; pass FFI runtime's currentBaseUrl()",
+		);
+	}
+
+	return {
+		id: "llama-server",
+		async describe(
+			request: VisionDescribeRequest,
+		): Promise<VisionDescribeResult> {
+			const { bytes, mimeType } = resolveImageBytes(request.image);
+			const base64 = Buffer.from(bytes).toString("base64");
+			const prompt = buildVisionPrompt(request.prompt);
+			const startMs = Date.now();
+			const body = JSON.stringify({
+				prompt,
+				image_data: [{ data: base64, id: 12 }],
+				n_predict: request.maxTokens ?? defaultMaxTokens,
+				temperature: request.temperature ?? 0.2,
+				stream: false,
+				// `cache_prompt: false` here so each describe call gets a
+				// fresh slot; the WS1 vision-embedding cache handles repeat-
+				// frame reuse on the JS side, and the server-side prompt
+				// cache would only conflict with that (different KV state
+				// for the same projector tokens).
+				cache_prompt: false,
+			});
+			const res = await fetchImpl(`${baseUrl}/completion`, {
+				method: "POST",
+				headers: {
+					"content-type": "application/json",
+					...(mimeType ? { "x-image-mime": mimeType } : {}),
+				},
+				body,
+				signal: request.signal,
+			});
+			if (!res.ok) {
+				const text = await res.text().catch(() => "<unreadable>");
+				throw new Error(
+					`[vision/llama-server] /completion returned ${res.status}: ${text.slice(0, 200)}`,
+				);
+			}
+			const payload = (await res.json()) as {
+				content?: unknown;
+				timings?: { prompt_ms?: number; predicted_ms?: number };
+			};
+			if (typeof payload.content !== "string") {
+				throw new Error(
+					"[vision/llama-server] /completion response missing string `content`",
+				);
+			}
+			const elapsed = Date.now() - startMs;
+			return shape(payload.content, {
+				projectorMs: payload.timings?.prompt_ms,
+				decodeMs: payload.timings?.predicted_ms ?? elapsed,
+				cacheHit: false,
+			});
+		},
+		async dispose() {
+			// llama-server lifetime is owned by the FFI runtime wrapper.
+			// This backend just holds the baseUrl; nothing to free.
+			baseUrl = "";
+		},
+	};
+}
+
+function buildVisionPrompt(userPrompt?: string): string {
+	const ask = userPrompt?.trim() || "Describe what is in this image.";
+	// `[img-N]` is the placeholder llama-server's mtmd path replaces with
+	// the encoded image tokens. The `N` must match the `image_data[*].id`
+	// we send in the body; we use 12 because llama-server's stock
+	// example uses small integer ids — any positive integer works.
+	return `<|im_start|>user\n[img-12]\n${ask}<|im_end|>\n<|im_start|>assistant\n`;
+}
+
+function shape(
+	text: string,
+	telemetry: { projectorMs?: number; decodeMs?: number; cacheHit?: boolean },
+): VisionDescribeResult {
+	const trimmed = text.trim();
+	if (!trimmed) {
+		throw new Error("[vision/llama-server] empty text from /completion");
+	}
+	const title = trimmed.split(/[.!?]/, 1)[0]?.trim() || "Image";
+	return {
+		title,
+		description: trimmed,
+		...telemetry,
+	};
+}

package/src/services/vision/types.d.ts

@@ -0,0 +1,153 @@
+/**
+ * Vision-language describe-image types (WS2).
+ *
+ * Two layers live here:
+ *
+ *   1. The **request/result** contract every WS2 backend implements
+ *      (`VisionDescribeRequest`, `VisionDescribeResult`). Callers pass
+ *      raw image bytes + a prompt; backends return a title+description.
+ *
+ *   2. The **backend** interface (`VisionDescribeBackend`) that the
+ *      `MemoryArbiter` registers as a capability handler. One backend
+ *      per binding family (node-llama-cpp / llama-server / AOSP libllama
+ *      shim). All three implement the same `load → describe → unload`
+ *      shape so the arbiter can swap between them without caring how
+ *      the projector is wired underneath.
+ *
+ * Why a separate file: the arbiter's `CapabilityRegistration<TBackend,
+ * TRequest, TResult>` is generic; pinning concrete shapes here keeps
+ * the registration sites short and removes a dozen casts at the
+ * call-site.
+ */
+/**
+ * Channel order for the raw pixel buffer. Most platforms hand us RGBA
+ * (HTMLCanvasElement, Capacitor `Camera`, the desktop `puppeteer-core`
+ * screenshot pipeline). The encoder normalizes internally; this enum
+ * stays so the hash step can pick a stable byte layout that doesn't
+ * depend on the platform-provided buffer order.
+ */
+export type VisionImageChannelOrder = "rgba" | "rgb" | "bgra" | "bgr";
+/**
+ * The raw image data the backend will encode. The arbiter does not see
+ * this — it gets handed straight to the backend's `run()`. The reason
+ * we accept multiple wrappers (URL / base64 / bytes) is that the three
+ * upstream entry points (HTTP route, agent runtime model handler,
+ * computer-use frame loop) each prefer a different shape. The backend
+ * resolves to bytes once.
+ */
+export type VisionImageInput = {
+    kind: "bytes";
+    bytes: Uint8Array;
+    mimeType?: string;
+} | {
+    kind: "base64";
+    base64: string;
+    mimeType?: string;
+} | {
+    kind: "dataUrl";
+    dataUrl: string;
+} | {
+    kind: "url";
+    url: string;
+    mimeType?: string;
+};
+/**
+ * Caller request to `describeImage`. The `modelFamily` distinguishes
+ * projected-token cache entries from different VL families that share
+ * the same hash space — Qwen3-VL tokens are not interchangeable with
+ * Florence-2 tokens. Default is `qwen3-vl` (the WS2 deliverable);
+ * each additional family registers under its own identifier.
+ */
+export interface VisionDescribeRequest {
+    image: VisionImageInput;
+    prompt?: string;
+    /**
+     * The model family identifier. Used to namespace the projector cache
+     * so swapping the backend's model family invalidates cached tokens.
+     * Defaults to `"qwen3-vl"` when omitted.
+     */
+    modelFamily?: string;
+    /** Max output tokens; defaults to 256 (description-length budget). */
+    maxTokens?: number;
+    /** 0..1, default 0.2 (descriptions should be deterministic-ish). */
+    temperature?: number;
+    signal?: AbortSignal;
+}
+/** Backend response — same shape that ImageDescriptionResult expects. */
+export interface VisionDescribeResult {
+    title: string;
+    description: string;
+    /** Best-effort: ms spent in the projector (for arbiter telemetry). */
+    projectorMs?: number;
+    /** Best-effort: ms spent in the decoder. */
+    decodeMs?: number;
+    /** Whether the projected tokens came from the WS1 vision cache. */
+    cacheHit?: boolean;
+}
+/**
+ * Per-load arguments for a vision-describe backend. The arbiter's
+ * `load(modelKey)` only carries an opaque key; the binding resolves
+ * that key to real model+mmproj paths through this struct, which
+ * `createVisionCapabilityRegistration` populates from the catalog.
+ */
+export interface VisionDescribeLoadArgs {
+    /** Absolute path to the text decoder GGUF (the "main" model). */
+    modelPath: string;
+    /** Absolute path to the matching mmproj projector GGUF. */
+    mmprojPath: string;
+    /**
+     * GPU offload preference. The backend translates this to its native
+     * knob: node-llama-cpp `gpuLayers`, llama-server `--n-gpu-layers`,
+     * AOSP libllama shim `eliza_llama_model_params_set_n_gpu_layers`.
+     * `"auto"` lets the binding decide; numeric is honoured verbatim.
+     */
+    gpuLayers?: number | "auto" | "max";
+    /** Max sampled context window in tokens. Defaults to 4096. */
+    contextSize?: number;
+}
+/**
+ * The contract every WS2 backend implements. The shape is intentionally
+ * narrow: the arbiter only ever calls `describe`. `dispose` is wrapped
+ * by the arbiter's `unload` so the backend can free GPU/VRAM and drop
+ * file descriptors on eviction.
+ */
+export interface VisionDescribeBackend {
+    /** Stable identifier — `"capacitor-llama"`, `"llama-server"`, `"aosp"`, or `"fake"` (tests). */
+    readonly id: "capacitor-llama" | "llama-server" | "aosp" | "fake";
+    /**
+     * Run a describe pass. Backends MAY consult an injected projector cache
+     * via `args.projectedTokens` (when the caller's hash already produced
+     * a cache hit) instead of running the projector again; backends that
+     * don't implement projector-token reuse ignore the field.
+     */
+    describe(request: VisionDescribeRequest, args?: VisionDescribeBackendOptions): Promise<VisionDescribeResult>;
+    /** Release the loaded weights. Idempotent. */
+    dispose(): Promise<void>;
+}
+/**
+ * Per-call options the arbiter wrapper passes into the backend. Lives
+ * here (rather than on `VisionDescribeRequest`) so the caller-facing
+ * request type stays free of arbiter implementation details.
+ */
+export interface VisionDescribeBackendOptions {
+    /**
+     * Pre-computed projected tokens from the WS1 vision-embedding cache.
+     * When present the backend SHOULD skip its own projector step and
+     * decode against these tokens directly. Backends that can't do this
+     * still produce a correct result by ignoring the field; the arbiter's
+     * wrapper will measure `cacheHit: false` in that case.
+     */
+    projectedTokens?: {
+        tokens: Float32Array;
+        tokenCount: number;
+        hiddenSize: number;
+    };
+}
+/**
+ * Capability handler load function. The arbiter calls it with a model
+ * key (e.g. `"qwen3-vl-2b"`); the implementation resolves to a real
+ * `(modelPath, mmprojPath)` pair from the catalog + installed registry
+ * and returns a live backend.
+ */
+export type VisionDescribeBackendLoader = (modelKey: string) => Promise<VisionDescribeBackend>;
+//# sourceMappingURL=types.d.ts.map

package/src/services/vision/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH;;;;;;GAMG;AACH,MAAM,MAAM,uBAAuB,GAAG,MAAM,GAAG,KAAK,GAAG,MAAM,GAAG,KAAK,CAAC;AAEtE;;;;;;;GAOG;AACH,MAAM,MAAM,gBAAgB,GACzB;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,GACvD;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,GACrD;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GACpC;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAEnD;;;;;;GAMG;AACH,MAAM,WAAW,qBAAqB;IACrC,KAAK,EAAE,gBAAgB,CAAC;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,sEAAsE;IACtE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,oEAAoE;IACpE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,WAAW,CAAC;CACrB;AAED,yEAAyE;AACzE,MAAM,WAAW,oBAAoB;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,sEAAsE;IACtE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,4CAA4C;IAC5C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,mEAAmE;IACnE,QAAQ,CAAC,EAAE,OAAO,CAAC;CACnB;AAED;;;;;GAKG;AACH,MAAM,WAAW,sBAAsB;IACtC,iEAAiE;IACjE,SAAS,EAAE,MAAM,CAAC;IAClB,2DAA2D;IAC3D,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC;IACpC,8DAA8D;IAC9D,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;;GAKG;AACH,MAAM,WAAW,qBAAqB;IACrC,gGAAgG;IAChG,QAAQ,CAAC,EAAE,EAAE,iBAAiB,GAAG,cAAc,GAAG,MAAM,GAAG,MAAM,CAAC;IAClE;;;;;OAKG;IACH,QAAQ,CACP,OAAO,EAAE,qBAAqB,EAC9B,IAAI,CAAC,EAAE,4BAA4B,GACjC,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACjC,8CAA8C;IAC9C,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB;AAED;;;;GAIG;AACH,MAAM,WAAW,4BAA4B;IAC5C;;;;;;OAMG;IACH,eAAe,CAAC,EAAE;QACjB,MAAM,EAAE,YAAY,CAAC;QACrB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;KACnB,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,MAAM,2BAA2B,GAAG,CACzC,QAAQ,EAAE,MAAM,KACZ,OAAO,CAAC,qBAAqB,CAAC,CAAC"}

package/src/services/vision/types.ts

@@ -0,0 +1,154 @@
+/**
+ * Vision-language describe-image types (WS2).
+ *
+ * Two layers live here:
+ *
+ *   1. The **request/result** contract every WS2 backend implements
+ *      (`VisionDescribeRequest`, `VisionDescribeResult`). Callers pass
+ *      raw image bytes + a prompt; backends return a title+description.
+ *
+ *   2. The **backend** interface (`VisionDescribeBackend`) that the
+ *      `MemoryArbiter` registers as a capability handler. One backend
+ *      per binding family (node-llama-cpp / llama-server / AOSP libllama
+ *      shim). All three implement the same `load → describe → unload`
+ *      shape so the arbiter can swap between them without caring how
+ *      the projector is wired underneath.
+ *
+ * Why a separate file: the arbiter's `CapabilityRegistration<TBackend,
+ * TRequest, TResult>` is generic; pinning concrete shapes here keeps
+ * the registration sites short and removes a dozen casts at the
+ * call-site.
+ */
+
+/**
+ * Channel order for the raw pixel buffer. Most platforms hand us RGBA
+ * (HTMLCanvasElement, Capacitor `Camera`, the desktop `puppeteer-core`
+ * screenshot pipeline). The encoder normalizes internally; this enum
+ * stays so the hash step can pick a stable byte layout that doesn't
+ * depend on the platform-provided buffer order.
+ */
+export type VisionImageChannelOrder = "rgba" | "rgb" | "bgra" | "bgr";
+
+/**
+ * The raw image data the backend will encode. The arbiter does not see
+ * this — it gets handed straight to the backend's `run()`. The reason
+ * we accept multiple wrappers (URL / base64 / bytes) is that the three
+ * upstream entry points (HTTP route, agent runtime model handler,
+ * computer-use frame loop) each prefer a different shape. The backend
+ * resolves to bytes once.
+ */
+export type VisionImageInput =
+	| { kind: "bytes"; bytes: Uint8Array; mimeType?: string }
+	| { kind: "base64"; base64: string; mimeType?: string }
+	| { kind: "dataUrl"; dataUrl: string }
+	| { kind: "url"; url: string; mimeType?: string };
+
+/**
+ * Caller request to `describeImage`. The `modelFamily` distinguishes
+ * projected-token cache entries from different VL families that share
+ * the same hash space — Qwen3-VL tokens are not interchangeable with
+ * Florence-2 tokens. Default is `qwen3-vl` (the WS2 deliverable);
+ * each additional family registers under its own identifier.
+ */
+export interface VisionDescribeRequest {
+	image: VisionImageInput;
+	prompt?: string;
+	/**
+	 * The model family identifier. Used to namespace the projector cache
+	 * so swapping the backend's model family invalidates cached tokens.
+	 * Defaults to `"qwen3-vl"` when omitted.
+	 */
+	modelFamily?: string;
+	/** Max output tokens; defaults to 256 (description-length budget). */
+	maxTokens?: number;
+	/** 0..1, default 0.2 (descriptions should be deterministic-ish). */
+	temperature?: number;
+	signal?: AbortSignal;
+}
+
+/** Backend response — same shape that ImageDescriptionResult expects. */
+export interface VisionDescribeResult {
+	title: string;
+	description: string;
+	/** Best-effort: ms spent in the projector (for arbiter telemetry). */
+	projectorMs?: number;
+	/** Best-effort: ms spent in the decoder. */
+	decodeMs?: number;
+	/** Whether the projected tokens came from the WS1 vision cache. */
+	cacheHit?: boolean;
+}
+
+/**
+ * Per-load arguments for a vision-describe backend. The arbiter's
+ * `load(modelKey)` only carries an opaque key; the binding resolves
+ * that key to real model+mmproj paths through this struct, which
+ * `createVisionCapabilityRegistration` populates from the catalog.
+ */
+export interface VisionDescribeLoadArgs {
+	/** Absolute path to the text decoder GGUF (the "main" model). */
+	modelPath: string;
+	/** Absolute path to the matching mmproj projector GGUF. */
+	mmprojPath: string;
+	/**
+	 * GPU offload preference. The backend translates this to its native
+	 * knob: node-llama-cpp `gpuLayers`, llama-server `--n-gpu-layers`,
+	 * AOSP libllama shim `eliza_llama_model_params_set_n_gpu_layers`.
+	 * `"auto"` lets the binding decide; numeric is honoured verbatim.
+	 */
+	gpuLayers?: number | "auto" | "max";
+	/** Max sampled context window in tokens. Defaults to 4096. */
+	contextSize?: number;
+}
+
+/**
+ * The contract every WS2 backend implements. The shape is intentionally
+ * narrow: the arbiter only ever calls `describe`. `dispose` is wrapped
+ * by the arbiter's `unload` so the backend can free GPU/VRAM and drop
+ * file descriptors on eviction.
+ */
+export interface VisionDescribeBackend {
+	/** Stable identifier — `"capacitor-llama"`, `"llama-server"`, `"aosp"`, or `"fake"` (tests). */
+	readonly id: "capacitor-llama" | "llama-server" | "aosp" | "fake";
+	/**
+	 * Run a describe pass. Backends MAY consult an injected projector cache
+	 * via `args.projectedTokens` (when the caller's hash already produced
+	 * a cache hit) instead of running the projector again; backends that
+	 * don't implement projector-token reuse ignore the field.
+	 */
+	describe(
+		request: VisionDescribeRequest,
+		args?: VisionDescribeBackendOptions,
+	): Promise<VisionDescribeResult>;
+	/** Release the loaded weights. Idempotent. */
+	dispose(): Promise<void>;
+}
+
+/**
+ * Per-call options the arbiter wrapper passes into the backend. Lives
+ * here (rather than on `VisionDescribeRequest`) so the caller-facing
+ * request type stays free of arbiter implementation details.
+ */
+export interface VisionDescribeBackendOptions {
+	/**
+	 * Pre-computed projected tokens from the WS1 vision-embedding cache.
+	 * When present the backend SHOULD skip its own projector step and
+	 * decode against these tokens directly. Backends that can't do this
+	 * still produce a correct result by ignoring the field; the arbiter's
+	 * wrapper will measure `cacheHit: false` in that case.
+	 */
+	projectedTokens?: {
+		tokens: Float32Array;
+		tokenCount: number;
+		hiddenSize: number;
+	};
+}
+
+/**
+ * Capability handler load function. The arbiter calls it with a model
+ * key (e.g. `"qwen3-vl-2b"`); the implementation resolves to a real
+ * `(modelPath, mmprojPath)` pair from the catalog + installed registry
+ * and returns a live backend.
+ */
+export type VisionDescribeBackendLoader = (
+	modelKey: string,
+) => Promise<VisionDescribeBackend>;

package/src/services/vision/vast-fallback.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Final optional IMAGE_DESCRIPTION fallback layer.
+ *
+ * This mirrors the cloud wrapper shape but only runs when the previous
+ * handler explicitly returned a typed fallback outcome.
+ */
+import type { ImageDescriptionParams } from "@elizaos/core";
+import { type LocalVisionOutcome, type VisionFallbackReason, type WrappedImageDescriptionHandler } from "./cloud-fallback";
+export interface VisionVastFallbackOptions {
+    enabled?: boolean;
+    apiKey?: string;
+    baseUrl?: string;
+    fetch?: typeof fetch;
+    handler?: (params: ImageDescriptionParams | string, reason: VisionFallbackReason) => Promise<LocalVisionOutcome>;
+    log?: (message: string, detail?: Record<string, unknown>) => void;
+}
+export declare function wrapImageDescriptionHandlerWithVastFallback(previous: WrappedImageDescriptionHandler, options?: VisionVastFallbackOptions): WrappedImageDescriptionHandler;
+//# sourceMappingURL=vast-fallback.d.ts.map

package/src/services/vision/vast-fallback.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vast-fallback.d.ts","sourceRoot":"","sources":["vast-fallback.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EACX,sBAAsB,EAEtB,MAAM,eAAe,CAAC;AACvB,OAAO,EAEN,KAAK,kBAAkB,EAGvB,KAAK,oBAAoB,EACzB,KAAK,8BAA8B,EACnC,MAAM,kBAAkB,CAAC;AAE1B,MAAM,WAAW,yBAAyB;IACzC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;IACrB,OAAO,CAAC,EAAE,CACT,MAAM,EAAE,sBAAsB,GAAG,MAAM,EACvC,MAAM,EAAE,oBAAoB,KACxB,OAAO,CAAC,kBAAkB,CAAC,CAAC;IACjC,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CAClE;AA+DD,wBAAgB,2CAA2C,CAC1D,QAAQ,EAAE,8BAA8B,EACxC,OAAO,GAAE,yBAA8B,GACrC,8BAA8B,CA8BhC"}

package/src/services/vision/vast-fallback.ts

@@ -0,0 +1,127 @@
+/**
+ * Final optional IMAGE_DESCRIPTION fallback layer.
+ *
+ * This mirrors the cloud wrapper shape but only runs when the previous
+ * handler explicitly returned a typed fallback outcome.
+ */
+
+import type {
+	ImageDescriptionParams,
+	ImageDescriptionResult,
+} from "@elizaos/core";
+import {
+	isVisionFallbackOutcome,
+	type LocalVisionOutcome,
+	type LocalVisionResult,
+	normalizeVisionDescription,
+	type VisionFallbackReason,
+	type WrappedImageDescriptionHandler,
+} from "./cloud-fallback";
+
+export interface VisionVastFallbackOptions {
+	enabled?: boolean;
+	apiKey?: string;
+	baseUrl?: string;
+	fetch?: typeof fetch;
+	handler?: (
+		params: ImageDescriptionParams | string,
+		reason: VisionFallbackReason,
+	) => Promise<LocalVisionOutcome>;
+	log?: (message: string, detail?: Record<string, unknown>) => void;
+}
+
+function resolveVastApiKey(options: VisionVastFallbackOptions): string | null {
+	return (
+		options.apiKey?.trim() || process.env.ELIZA_VAST_API_KEY?.trim() || null
+	);
+}
+
+function resolveVastBaseUrl(options: VisionVastFallbackOptions): string {
+	return (
+		options.baseUrl?.trim() ||
+		process.env.ELIZA_VAST_BASE_URL?.trim() ||
+		"https://api.vast.ai"
+	).replace(/\/+$/, "");
+}
+
+function imageRequestBody(params: ImageDescriptionParams | string): {
+	image: { kind: "url"; url: string } | { kind: "data"; data: string };
+	prompt?: string;
+} {
+	if (typeof params === "string") {
+		return params.startsWith("data:")
+			? { image: { kind: "data", data: params } }
+			: { image: { kind: "url", url: params } };
+	}
+	const imageUrl = (params as { imageUrl?: string }).imageUrl;
+	const image = (params as { image?: string }).image;
+	const source = imageUrl ?? image;
+	const body = source?.startsWith("data:")
+		? { image: { kind: "data" as const, data: source } }
+		: { image: { kind: "url" as const, url: source ?? "" } };
+	if (params.prompt) return { ...body, prompt: params.prompt };
+	return body;
+}
+
+async function callVastVision(
+	params: ImageDescriptionParams | string,
+	options: VisionVastFallbackOptions,
+): Promise<ImageDescriptionResult> {
+	const apiKey = resolveVastApiKey(options);
+	if (!apiKey) {
+		throw new Error("VAST image fallback is not configured");
+	}
+	const fetchImpl = options.fetch ?? fetch;
+	const response = await fetchImpl(
+		`${resolveVastBaseUrl(options)}/v1/vision/describe`,
+		{
+			method: "POST",
+			headers: {
+				"content-type": "application/json",
+				authorization: `Bearer ${apiKey}`,
+			},
+			body: JSON.stringify(imageRequestBody(params)),
+		},
+	);
+	if (!response.ok) {
+		throw new Error(`VAST image fallback failed with ${response.status}`);
+	}
+	return normalizeVisionDescription(
+		(await response.json()) as LocalVisionResult,
+	);
+}
+
+export function wrapImageDescriptionHandlerWithVastFallback(
+	previous: WrappedImageDescriptionHandler,
+	options: VisionVastFallbackOptions = {},
+): WrappedImageDescriptionHandler {
+	const enabled = options.enabled ?? true;
+	const log = options.log ?? (() => undefined);
+	return async (params): Promise<LocalVisionOutcome> => {
+		const outcome = await previous(params);
+		if (!isVisionFallbackOutcome(outcome)) {
+			return normalizeVisionDescription(outcome);
+		}
+		if (!enabled) return outcome;
+
+		const apiKey = resolveVastApiKey(options);
+		if (!options.handler && !apiKey) return outcome;
+
+		log("[vision/vast-fallback] upstream IMAGE_DESCRIPTION fallback", {
+			reason: outcome.reason,
+		});
+		try {
+			const vastOutcome = options.handler
+				? await options.handler(params, outcome.reason)
+				: await callVastVision(params, options);
+			if (isVisionFallbackOutcome(vastOutcome)) return vastOutcome;
+			return normalizeVisionDescription(vastOutcome);
+		} catch (error) {
+			return {
+				kind: "fallback",
+				reason: "vast-error",
+				cause: error instanceof Error ? error : new Error(String(error)),
+			};
+		}
+	};
+}