@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/services/vision/hash.ts

@@ -0,0 +1,157 @@
+/**
+ * Vision-embedding cache key derivation (WS2).
+ *
+ * The arbiter's vision-embedding cache (WS1, `vision-embedding-cache.ts`)
+ * is keyed by SHA-256 of a *normalized* representation of the input
+ * image. The normalization step is what makes the cache useful across
+ * platforms: two JPEG encodings of the same screenshot, or an RGBA vs
+ * RGB frame captured by different platforms, must hash to the same key
+ * or the cache hit rate collapses.
+ *
+ * Normalization is deliberately minimal:
+ *
+ *   1. Resolve the input to raw bytes (decoding base64/data-url wrappers).
+ *   2. Hash with the model-family prefix so the cache can hold tokens
+ *      for multiple VL families without collision.
+ *
+ * What we DO NOT do here:
+ *
+ *   - Resize the image. The backend's projector enforces its own input
+ *     resolution; the bytes the projector sees are what gets projected.
+ *      Re-encoding here would add work without changing the hit rate
+ *      (the platform-provided buffer is already at the camera's native
+ *      resolution).
+ *   - Strip JPEG/PNG headers. They contribute to the hash; two
+ *      reencodings of the same pixel array land in different cache
+ *      slots intentionally. Reuse only the exact same byte stream.
+ *
+ * If a downstream caller wants finer-grained cache hits (e.g. dedupe
+ * across re-encodings of the same screen frame), it should decode to
+ * RGBA pixels itself and call `hashRawPixels`. The default
+ * `hashVisionInput` path is the conservative, byte-stream-only path.
+ */
+
+import { createHash } from "node:crypto";
+import type { VisionImageInput } from "./types";
+
+const DEFAULT_FAMILY = "qwen3-vl";
+
+/**
+ * Resolve a `VisionImageInput` to its raw bytes. Returns the decoded
+ * payload plus an optional MIME type the caller can forward to the
+ * backend. Throws on `url:` inputs — those must be fetched by the
+ * caller; the hash step does not own HTTP.
+ */
+export function resolveImageBytes(input: VisionImageInput): {
+	bytes: Uint8Array;
+	mimeType?: string;
+} {
+	switch (input.kind) {
+		case "bytes":
+			return { bytes: input.bytes, mimeType: input.mimeType };
+		case "base64": {
+			const bytes = Uint8Array.from(Buffer.from(input.base64, "base64"));
+			return { bytes, mimeType: input.mimeType };
+		}
+		case "dataUrl": {
+			const match = /^data:([^;,]+)(?:;[^,]*)?,(.*)$/s.exec(input.dataUrl);
+			if (!match) {
+				throw new Error(
+					"[vision/hash] malformed data URL — expected data:<mime>;base64,<payload>",
+				);
+			}
+			const mimeType = match[1];
+			const payload = match[2];
+			const isBase64 = /;base64/i.test(input.dataUrl);
+			const bytes = Uint8Array.from(
+				Buffer.from(payload, isBase64 ? "base64" : "utf8"),
+			);
+			return { bytes, mimeType };
+		}
+		case "url":
+			throw new Error(
+				"[vision/hash] url inputs must be fetched by the caller before hashing — the hash step does not own HTTP",
+			);
+	}
+}
+
+/**
+ * Hash an opaque byte stream with the model-family prefix. The result
+ * is stable across processes and platforms (Node, Bun, and the
+ * Capacitor JS bridge all return the same hex string for the same
+ * input).
+ */
+export function hashImageBytes(
+	bytes: Uint8Array,
+	modelFamily: string = DEFAULT_FAMILY,
+): string {
+	const h = createHash("sha256");
+	h.update(modelFamily);
+	// Length prefix prevents a `family || bytes` collision against a
+	// crafted family string that ends with the leading bytes of the
+	// payload. Cheap, defensible.
+	const lenBuf = Buffer.alloc(4);
+	lenBuf.writeUInt32BE(bytes.byteLength, 0);
+	h.update(lenBuf);
+	h.update(bytes);
+	return h.digest("hex");
+}
+
+/**
+ * Hash a raw pixel buffer (RGBA / RGB / BGRA / BGR). The channel order
+ * is folded into the prefix so the same image captured on two different
+ * platforms (Android = RGBA, macOS screenshot = BGRA) produces the same
+ * key when normalized. Width / height are also included so the cache
+ * doesn't conflate two scaled versions of the same source.
+ */
+export function hashRawPixels(args: {
+	bytes: Uint8Array;
+	width: number;
+	height: number;
+	channelOrder: "rgba" | "rgb" | "bgra" | "bgr";
+	modelFamily?: string;
+}): string {
+	const h = createHash("sha256");
+	h.update(args.modelFamily ?? DEFAULT_FAMILY);
+	h.update("|raw|");
+	const prefix = Buffer.alloc(12);
+	prefix.writeUInt32BE(args.width, 0);
+	prefix.writeUInt32BE(args.height, 4);
+	prefix.write(args.channelOrder.padEnd(4, " "), 8, "ascii");
+	h.update(prefix);
+	// Channel-order normalization: rewrite BGRA→RGBA and BGR→RGB in
+	// place into a new buffer so all three platforms land on the same
+	// hash even when the input buffer order differs.
+	const normalized = normalizeChannels(args.bytes, args.channelOrder);
+	h.update(normalized);
+	return h.digest("hex");
+}
+
+function normalizeChannels(
+	bytes: Uint8Array,
+	order: "rgba" | "rgb" | "bgra" | "bgr",
+): Uint8Array {
+	if (order === "rgba" || order === "rgb") return bytes;
+	const stride = order === "bgra" ? 4 : 3;
+	const out = new Uint8Array(bytes.byteLength);
+	for (let i = 0; i + stride <= bytes.byteLength; i += stride) {
+		out[i] = bytes[i + 2];
+		out[i + 1] = bytes[i + 1];
+		out[i + 2] = bytes[i];
+		if (stride === 4) out[i + 3] = bytes[i + 3];
+	}
+	return out;
+}
+
+/**
+ * Convenience wrapper used by the provider: takes a `VisionImageInput`
+ * and a model family, returns the cache key. URL inputs throw —
+ * callers must fetch first.
+ */
+export function hashVisionInput(
+	input: VisionImageInput,
+	modelFamily: string = DEFAULT_FAMILY,
+): string {
+	const { bytes } = resolveImageBytes(input);
+	return hashImageBytes(bytes, modelFamily);
+}

package/src/services/vision/index.d.ts

@@ -0,0 +1,95 @@
+/**
+ * Vision-describe capability (WS2) — public entry point.
+ *
+ * This module is what plugin-vision (WS4), the IMAGE_DESCRIPTION
+ * handler in `provider.ts`, and computer-use (WS9) import to register
+ * vision capability with the WS1 MemoryArbiter.
+ *
+ * Wiring:
+ *
+ *   const arbiter = service.getMemoryArbiter();
+ *   const registration = createVisionCapabilityRegistration({
+ *     loader: createDefaultVisionLoader({ ... }),
+ *     arbiterCache: arbiter,
+ *   });
+ *   arbiter.registerCapability(registration);
+ *
+ * `createVisionCapabilityRegistration` wraps the underlying backend so
+ * the arbiter's `run(request)` path:
+ *
+ *   1. Hashes the request's image bytes (model-family-scoped).
+ *   2. Checks the arbiter's vision-embedding cache.
+ *   3. On miss: calls `backend.describe(request)`, lets the backend
+ *      run its own projector + decoder. Backends that cannot expose projected
+ *      tokens return decoder text only, so the cache stays empty for this hash.
+ *      The decoder text is what the caller wanted anyway.
+ *   4. On hit: calls `backend.describe(request, { projectedTokens })`.
+ *      Backends that support pre-projected token reuse skip the
+ *      projector entirely. Backends that don't ignore the hint; the
+ *      result is still correct but the projector cost is paid again.
+ */
+export { type AospLlamaMtmdBinding, type AospMtmdHandle, type LoadAospVisionBackendOptions, loadAospVisionBackend, } from "./aosp-unavailable";
+export { type CapacitorLlamaMtmdBinding, type CapacitorLlamaMtmdHandle, type CapacitorLlamaVisionBackendOptions, loadCapacitorLlamaVisionBackend, VisionBackendUnavailableError, type VisionManagerLike, } from "./capacitor-llama";
+export { classifyLocalVisionError, type LocalImageDescriptionHandler, type LocalVisionOutcome, type VisionCloudFallbackOptions, type VisionFallbackReason, type WrappedImageDescriptionHandler, wrapImageDescriptionHandlerWithCloudFallback, } from "./cloud-fallback";
+export { hashImageBytes, hashRawPixels, hashVisionInput, resolveImageBytes, } from "./hash";
+export { createLlamaServerVisionBackend, type LlamaServerVisionBackendOptions, } from "./llama-server";
+export type { VisionDescribeBackend, VisionDescribeBackendLoader, VisionDescribeBackendOptions, VisionDescribeLoadArgs, VisionDescribeRequest, VisionDescribeResult, VisionImageChannelOrder, VisionImageInput, } from "./types";
+export { type VisionVastFallbackOptions, wrapImageDescriptionHandlerWithVastFallback, } from "./vast-fallback";
+import type { CapabilityRegistration } from "../memory-arbiter";
+import type { VisionDescribeBackend, VisionDescribeBackendLoader, VisionDescribeRequest, VisionDescribeResult } from "./types";
+/**
+ * Minimal arbiter shape we need from the cache. Lets tests inject a
+ * fake cache without pulling in the whole MemoryArbiter.
+ */
+export interface VisionEmbeddingCacheLike {
+    getCachedVisionEmbedding(hash: string): {
+        tokens: Float32Array;
+        tokenCount: number;
+        hiddenSize: number;
+        live?: boolean;
+    } | null;
+    setCachedVisionEmbedding(hash: string, entry: {
+        tokens: Float32Array;
+        tokenCount: number;
+        hiddenSize: number;
+    }, ttlMs?: number): void;
+}
+export interface CreateVisionCapabilityRegistrationOptions {
+    /**
+     * The arbiter (or any object with the cache passthroughs). When
+     * provided the wrapper performs hash → cache lookup before calling
+     * the backend's `describe`.
+     */
+    arbiterCache?: VisionEmbeddingCacheLike;
+    loader: VisionDescribeBackendLoader;
+    /** Default model family for the cache key. Defaults to `qwen3-vl`. */
+    modelFamily?: string;
+    estimatedMb?: number;
+}
+/**
+ * Build a `CapabilityRegistration` ready to feed to
+ * `arbiter.registerCapability()`. The wrapper plumbs the cache hint
+ * into the backend's describe call so backends that support
+ * pre-projected tokens skip the projector.
+ */
+export declare function createVisionCapabilityRegistration(opts: CreateVisionCapabilityRegistrationOptions): CapabilityRegistration<VisionDescribeBackend, VisionDescribeRequest, VisionDescribeResult>;
+import type { IAgentRuntime, ImageDescriptionParams, ImageDescriptionResult } from "@elizaos/core";
+import { type LocalImageDescriptionHandler, type VisionCloudFallbackOptions } from "./cloud-fallback";
+import { type VisionVastFallbackOptions } from "./vast-fallback";
+/**
+ * Compose the full local → cloud → vast IMAGE_DESCRIPTION chain and
+ * terminate it as a runtime-shaped `ImageDescriptionHandler`. When all
+ * three paths return `{ kind: "fallback" }`, the terminator throws the
+ * underlying cause (or a structured upstream-fail message) so the runtime
+ * surfaces the failure cleanly rather than serving a sentinel result.
+ *
+ * This is the single entry point `ensure-local-inference-handler.ts`
+ * uses at the IMAGE_DESCRIPTION model registration site. Tests
+ * exercise the composition via the individual `wrap*` helpers; this
+ * function is the production wiring.
+ */
+export declare function withVisionFallbackChain(local: LocalImageDescriptionHandler, options?: {
+    cloud?: VisionCloudFallbackOptions;
+    vast?: VisionVastFallbackOptions;
+}): (runtime: IAgentRuntime, params: ImageDescriptionParams | string) => Promise<ImageDescriptionResult>;
+//# sourceMappingURL=index.d.ts.map

package/src/services/vision/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,OAAO,EACN,KAAK,oBAAoB,EACzB,KAAK,cAAc,EACnB,KAAK,4BAA4B,EACjC,qBAAqB,GACrB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACN,KAAK,yBAAyB,EAC9B,KAAK,wBAAwB,EAC7B,KAAK,kCAAkC,EACvC,+BAA+B,EAC/B,6BAA6B,EAC7B,KAAK,iBAAiB,GACtB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACN,wBAAwB,EACxB,KAAK,4BAA4B,EACjC,KAAK,kBAAkB,EACvB,KAAK,0BAA0B,EAC/B,KAAK,oBAAoB,EACzB,KAAK,8BAA8B,EACnC,4CAA4C,GAC5C,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACN,cAAc,EACd,aAAa,EACb,eAAe,EACf,iBAAiB,GACjB,MAAM,QAAQ,CAAC;AAChB,OAAO,EACN,8BAA8B,EAC9B,KAAK,+BAA+B,GACpC,MAAM,gBAAgB,CAAC;AACxB,YAAY,EACX,qBAAqB,EACrB,2BAA2B,EAC3B,4BAA4B,EAC5B,sBAAsB,EACtB,qBAAqB,EACrB,oBAAoB,EACpB,uBAAuB,EACvB,gBAAgB,GAChB,MAAM,SAAS,CAAC;AACjB,OAAO,EACN,KAAK,yBAAyB,EAC9B,2CAA2C,GAC3C,MAAM,iBAAiB,CAAC;AAEzB,OAAO,KAAK,EAEX,sBAAsB,EACtB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,KAAK,EACX,qBAAqB,EACrB,2BAA2B,EAC3B,qBAAqB,EACrB,oBAAoB,EACpB,MAAM,SAAS,CAAC;AAEjB;;;GAGG;AACH,MAAM,WAAW,wBAAwB;IACxC,wBAAwB,CAAC,IAAI,EAAE,MAAM,GAAG;QACvC,MAAM,EAAE,YAAY,CAAC;QACrB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;QACnB,IAAI,CAAC,EAAE,OAAO,CAAC;KACf,GAAG,IAAI,CAAC;IACT,wBAAwB,CACvB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE;QACN,MAAM,EAAE,YAAY,CAAC;QACrB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;KACnB,EACD,KAAK,CAAC,EAAE,MAAM,GACZ,IAAI,CAAC;CACR;AAED,MAAM,WAAW,yCAAyC;IACzD;;;;OAIG;IACH,YAAY,CAAC,EAAE,wBAAwB,CAAC;IACxC,MAAM,EAAE,2BAA2B,CAAC;IACpC,sEAAsE;IACtE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;;GAKG;AACH,wBAAgB,kCAAkC,CACjD,IAAI,EAAE,yCAAyC,GAC7C,sBAAsB,CACxB,qBAAqB,EACrB,qBAAqB,EACrB,oBAAoB,CACpB,CAkDA;AAED,OAAO,KAAK,EACX,aAAa,EACb,sBAAsB,EACtB,sBAAsB,EACtB,MAAM,eAAe,CAAC;AACvB,OAAO,EACN,KAAK,4BAA4B,EACjC,KAAK,0BAA0B,EAE/B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACN,KAAK,yBAAyB,EAE9B,MAAM,iBAAiB,CAAC;AAEzB;;;;;;;;;;;GAWG;AACH,wBAAgB,uBAAuB,CACtC,KAAK,EAAE,4BAA4B,EACnC,OAAO,GAAE;IACR,KAAK,CAAC,EAAE,0BAA0B,CAAC;IACnC,IAAI,CAAC,EAAE,yBAAyB,CAAC;CAC5B,GACJ,CACF,OAAO,EAAE,aAAa,EACtB,MAAM,EAAE,sBAAsB,GAAG,MAAM,KACnC,OAAO,CAAC,sBAAsB,CAAC,CAwBnC"}

package/src/services/vision/index.ts

@@ -0,0 +1,251 @@
+/**
+ * Vision-describe capability (WS2) — public entry point.
+ *
+ * This module is what plugin-vision (WS4), the IMAGE_DESCRIPTION
+ * handler in `provider.ts`, and computer-use (WS9) import to register
+ * vision capability with the WS1 MemoryArbiter.
+ *
+ * Wiring:
+ *
+ *   const arbiter = service.getMemoryArbiter();
+ *   const registration = createVisionCapabilityRegistration({
+ *     loader: createDefaultVisionLoader({ ... }),
+ *     arbiterCache: arbiter,
+ *   });
+ *   arbiter.registerCapability(registration);
+ *
+ * `createVisionCapabilityRegistration` wraps the underlying backend so
+ * the arbiter's `run(request)` path:
+ *
+ *   1. Hashes the request's image bytes (model-family-scoped).
+ *   2. Checks the arbiter's vision-embedding cache.
+ *   3. On miss: calls `backend.describe(request)`, lets the backend
+ *      run its own projector + decoder. Backends that cannot expose projected
+ *      tokens return decoder text only, so the cache stays empty for this hash.
+ *      The decoder text is what the caller wanted anyway.
+ *   4. On hit: calls `backend.describe(request, { projectedTokens })`.
+ *      Backends that support pre-projected token reuse skip the
+ *      projector entirely. Backends that don't ignore the hint; the
+ *      result is still correct but the projector cost is paid again.
+ */
+
+export {
+	type AospLlamaMtmdBinding,
+	type AospMtmdHandle,
+	type LoadAospVisionBackendOptions,
+	loadAospVisionBackend,
+} from "./aosp-unavailable";
+export {
+	type CapacitorLlamaMtmdBinding,
+	type CapacitorLlamaMtmdHandle,
+	type CapacitorLlamaVisionBackendOptions,
+	loadCapacitorLlamaVisionBackend,
+	VisionBackendUnavailableError,
+	type VisionManagerLike,
+} from "./capacitor-llama";
+export {
+	classifyLocalVisionError,
+	type LocalImageDescriptionHandler,
+	type LocalVisionOutcome,
+	type VisionCloudFallbackOptions,
+	type VisionFallbackReason,
+	type WrappedImageDescriptionHandler,
+	wrapImageDescriptionHandlerWithCloudFallback,
+} from "./cloud-fallback";
+export {
+	hashImageBytes,
+	hashRawPixels,
+	hashVisionInput,
+	resolveImageBytes,
+} from "./hash";
+export {
+	createLlamaServerVisionBackend,
+	type LlamaServerVisionBackendOptions,
+} from "./llama-server";
+export type {
+	VisionDescribeBackend,
+	VisionDescribeBackendLoader,
+	VisionDescribeBackendOptions,
+	VisionDescribeLoadArgs,
+	VisionDescribeRequest,
+	VisionDescribeResult,
+	VisionImageChannelOrder,
+	VisionImageInput,
+} from "./types";
+export {
+	type VisionVastFallbackOptions,
+	wrapImageDescriptionHandlerWithVastFallback,
+} from "./vast-fallback";
+
+import type {
+	ArbiterCapability,
+	CapabilityRegistration,
+} from "../memory-arbiter";
+import { hashVisionInput } from "./hash";
+import type {
+	VisionDescribeBackend,
+	VisionDescribeBackendLoader,
+	VisionDescribeRequest,
+	VisionDescribeResult,
+} from "./types";
+
+/**
+ * Minimal arbiter shape we need from the cache. Lets tests inject a
+ * fake cache without pulling in the whole MemoryArbiter.
+ */
+export interface VisionEmbeddingCacheLike {
+	getCachedVisionEmbedding(hash: string): {
+		tokens: Float32Array;
+		tokenCount: number;
+		hiddenSize: number;
+		live?: boolean;
+	} | null;
+	setCachedVisionEmbedding(
+		hash: string,
+		entry: {
+			tokens: Float32Array;
+			tokenCount: number;
+			hiddenSize: number;
+		},
+		ttlMs?: number,
+	): void;
+}
+
+export interface CreateVisionCapabilityRegistrationOptions {
+	/**
+	 * The arbiter (or any object with the cache passthroughs). When
+	 * provided the wrapper performs hash → cache lookup before calling
+	 * the backend's `describe`.
+	 */
+	arbiterCache?: VisionEmbeddingCacheLike;
+	loader: VisionDescribeBackendLoader;
+	/** Default model family for the cache key. Defaults to `qwen3-vl`. */
+	modelFamily?: string;
+	estimatedMb?: number;
+}
+
+/**
+ * Build a `CapabilityRegistration` ready to feed to
+ * `arbiter.registerCapability()`. The wrapper plumbs the cache hint
+ * into the backend's describe call so backends that support
+ * pre-projected tokens skip the projector.
+ */
+export function createVisionCapabilityRegistration(
+	opts: CreateVisionCapabilityRegistrationOptions,
+): CapabilityRegistration<
+	VisionDescribeBackend,
+	VisionDescribeRequest,
+	VisionDescribeResult
+> {
+	const capability: ArbiterCapability = "vision-describe";
+	const family = opts.modelFamily ?? "qwen3-vl";
+	const cache = opts.arbiterCache;
+	const loader = opts.loader;
+	return {
+		capability,
+		residentRole: "vision",
+		estimatedMb: opts.estimatedMb ?? 600,
+		async load(modelKey) {
+			return await loader(modelKey);
+		},
+		async unload(backend) {
+			await backend.dispose();
+		},
+		async run(backend, request) {
+			const effectiveFamily = request.modelFamily ?? family;
+			const cached = (() => {
+				if (!cache) return null;
+				if (request.image.kind === "url") {
+					// URL inputs can't be hashed without first fetching; skip
+					// the cache lookup rather than paying the fetch cost twice.
+					return null;
+				}
+				try {
+					const hash = hashVisionInput(request.image, effectiveFamily);
+					const hit = cache.getCachedVisionEmbedding(hash);
+					if (hit && hit.live !== false) return { hash, hit };
+				} catch {
+					// Hashing failed (malformed data URL etc.); proceed without
+					// cache rather than failing the request.
+				}
+				return null;
+			})();
+			const projected = cached?.hit
+				? {
+						tokens: cached.hit.tokens,
+						tokenCount: cached.hit.tokenCount,
+						hiddenSize: cached.hit.hiddenSize,
+					}
+				: undefined;
+			const result = await backend.describe(request, {
+				projectedTokens: projected,
+			});
+			return {
+				...result,
+				cacheHit: Boolean(projected),
+			};
+		},
+	};
+}
+
+import type {
+	IAgentRuntime,
+	ImageDescriptionParams,
+	ImageDescriptionResult,
+} from "@elizaos/core";
+import {
+	type LocalImageDescriptionHandler,
+	type VisionCloudFallbackOptions,
+	wrapImageDescriptionHandlerWithCloudFallback,
+} from "./cloud-fallback";
+import {
+	type VisionVastFallbackOptions,
+	wrapImageDescriptionHandlerWithVastFallback,
+} from "./vast-fallback";
+
+/**
+ * Compose the full local → cloud → vast IMAGE_DESCRIPTION chain and
+ * terminate it as a runtime-shaped `ImageDescriptionHandler`. When all
+ * three paths return `{ kind: "fallback" }`, the terminator throws the
+ * underlying cause (or a structured upstream-fail message) so the runtime
+ * surfaces the failure cleanly rather than serving a sentinel result.
+ *
+ * This is the single entry point `ensure-local-inference-handler.ts`
+ * uses at the IMAGE_DESCRIPTION model registration site. Tests
+ * exercise the composition via the individual `wrap*` helpers; this
+ * function is the production wiring.
+ */
+export function withVisionFallbackChain(
+	local: LocalImageDescriptionHandler,
+	options: {
+		cloud?: VisionCloudFallbackOptions;
+		vast?: VisionVastFallbackOptions;
+	} = {},
+): (
+	runtime: IAgentRuntime,
+	params: ImageDescriptionParams | string,
+) => Promise<ImageDescriptionResult> {
+	const wrapped = wrapImageDescriptionHandlerWithVastFallback(
+		wrapImageDescriptionHandlerWithCloudFallback(local, options.cloud),
+		options.vast,
+	);
+	return async (_runtime, params) => {
+		const outcome = await wrapped(params);
+		if (
+			outcome &&
+			typeof outcome === "object" &&
+			"kind" in outcome &&
+			outcome.kind === "fallback"
+		) {
+			const causeMsg = outcome.cause?.message ?? outcome.reason;
+			const err = new Error(
+				`[VisionFallback] all IMAGE_DESCRIPTION providers exhausted (reason=${outcome.reason}): ${causeMsg}`,
+			);
+			if (outcome.cause) {
+				(err as Error & { cause?: unknown }).cause = outcome.cause;
+			}
+			throw err;
+		}
+		return outcome as ImageDescriptionResult;
+	};
+}

package/src/services/vision/llama-server.d.ts

@@ -0,0 +1,73 @@
+/**
+ * llama-server vision-describe backend (WS2).
+ *
+ * Wraps the out-of-process llama-server's `/completion` endpoint with
+ * the `image_data` array (base64-encoded payloads) and shapes the
+ * response to the WS2 `VisionDescribeBackend` contract.
+ *
+ * llama-server image-data API recap (verified against llama.cpp commit
+ * b8198+, May 2026):
+ *
+ *   POST /completion
+ *   { "prompt": "<...>USER: [img-12] What's in this image?\nASSISTANT:",
+ *     "image_data": [
+ *       { "data": "<base64 png/jpeg>", "id": 12 }
+ *     ],
+ *     "n_predict": 256,
+ *     "temperature": 0.2,
+ *     "stream": false }
+ *
+ *   Response:
+ *   { "content": "A photo of a cat.", "stop": true,
+ *     "timings": { "prompt_ms": 180.4, "predicted_ms": 423.1 } }
+ *
+ * Server-side mmproj is loaded via the `--mmproj <path>` flag on
+ * llama-server startup. The FFI runtime wrapper passes this flag
+ * already for tiers with vision enabled; this backend assumes the
+ * server has been started with the right mmproj for the active model.
+ *
+ * Backend responsibility:
+ *   - Encode the image as base64 (when not already).
+ *   - Build the prompt with the `[img-N]` placeholder convention.
+ *   - POST to `/completion`, parse the text + timings.
+ *   - Honour AbortSignal by passing it through to the fetch call.
+ *
+ * Backend explicitly does NOT:
+ *   - Start / stop the server. That's the FFI runtime wrapper's job.
+ *   - Resolve the mmproj path — the server already has it. The arbiter's
+ *     `--mmproj` was set when the text model loaded.
+ *   - Implement projector-token reuse. llama-server has no API to
+ *     accept pre-projected tokens; if the WS1 cache hit happens, this
+ *     backend ignores the hint and re-runs the projector. The cache
+ *     is more useful with the in-process node-llama-cpp backend.
+ *
+ * Metal / CUDA validation:
+ *   The llama-server build embeds the same mtmd_encode path the
+ *   in-process binding will eventually expose. On a Metal build the
+ *   image encode dispatches through the Metal compute encoder; on a
+ *   CUDA build through cuBLAS. We have no GPU on this host — see the
+ *   `__tests__/vision-describe.test.ts` notes for the GPU smoke check.
+ */
+import type { VisionDescribeBackend } from "./types";
+export interface LlamaServerVisionBackendOptions {
+    /**
+     * Base URL of the llama-server. The FFI runtime wrapper exposes
+     * this via `currentBaseUrl()`; pass the resolved URL here at load
+     * time. The backend keeps it as-is across calls.
+     */
+    baseUrl: string;
+    /**
+     * Optional fetch override. Tests inject a fake fetch; production
+     * uses global fetch. The signature mirrors `fetch` so the test
+     * surface is the same as the real one.
+     */
+    fetch?: typeof fetch;
+    /**
+     * Default `n_predict` budget when the caller doesn't specify
+     * `maxTokens`. 256 matches the description-length budget the
+     * Florence-2 / VisionManager path uses today.
+     */
+    defaultMaxTokens?: number;
+}
+export declare function createLlamaServerVisionBackend(opts: LlamaServerVisionBackendOptions): VisionDescribeBackend;
+//# sourceMappingURL=llama-server.d.ts.map

package/src/services/vision/llama-server.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llama-server.d.ts","sourceRoot":"","sources":["llama-server.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiDG;AAGH,OAAO,KAAK,EACX,qBAAqB,EAGrB,MAAM,SAAS,CAAC;AAEjB,MAAM,WAAW,+BAA+B;IAC/C;;;;OAIG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;IACrB;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,wBAAgB,8BAA8B,CAC7C,IAAI,EAAE,+BAA+B,GACnC,qBAAqB,CAsEvB"}