@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/vision/cloud-fallback.ts

@@ -0,0 +1,268 @@
+/**
+ * Soft cloud fallback wrapper for local IMAGE_DESCRIPTION handlers.
+ *
+ * The local vision path can report recoverable unavailability without forcing
+ * callers to know which remote provider is paired. The wrapper keeps that
+ * state explicit: handlers return a normal image description or a typed
+ * fallback outcome that the next layer can handle.
+ */
+
+import type {
+	ImageDescriptionParams,
+	ImageDescriptionResult,
+} from "@elizaos/core";
+
+export type VisionFallbackReason =
+	| "local-unavailable"
+	| "local-overloaded"
+	| "local-error"
+	| "local-aborted-pre-completion"
+	| "local-not-registered"
+	| "cloud-unavailable"
+	| "cloud-error"
+	| "vast-unavailable"
+	| "vast-error";
+
+export type LocalVisionOutcome =
+	| ImageDescriptionResult
+	| string
+	| { kind: "ok"; result: ImageDescriptionResult | string }
+	| { kind: "fallback"; reason: VisionFallbackReason; cause?: Error };
+
+export type LocalVisionResult = Exclude<
+	LocalVisionOutcome,
+	{ kind: "fallback"; reason: VisionFallbackReason; cause?: Error }
+>;
+
+export type LocalImageDescriptionHandler = (
+	params: ImageDescriptionParams | string,
+) => Promise<LocalVisionOutcome>;
+
+export type WrappedImageDescriptionHandler = LocalImageDescriptionHandler;
+
+export interface VisionCloudFallbackOptions {
+	enabled?: boolean;
+	token?: string;
+	apiKey?: string;
+	baseUrl?: string;
+	fetch?: typeof fetch;
+	handler?: (
+		params: ImageDescriptionParams | string,
+		reason: VisionFallbackReason,
+	) => Promise<LocalVisionOutcome>;
+	log?: (message: string, detail?: Record<string, unknown>) => void;
+}
+
+export function classifyLocalVisionError(error: unknown): {
+	fallback: boolean;
+	reason: VisionFallbackReason;
+	cause?: Error;
+} {
+	const cause = error instanceof Error ? error : new Error(String(error));
+	const message = cause.message.toLowerCase();
+	if (cause.name === "AbortError") {
+		return {
+			fallback: false,
+			reason: "local-aborted-pre-completion",
+			cause,
+		};
+	}
+	if (
+		message.includes("no local") ||
+		message.includes("not registered") ||
+		message.includes("not installed") ||
+		message.includes("requires an active") ||
+		message.includes("capability_unavailable") ||
+		message.includes("backend_unavailable") ||
+		message.includes("no mtmd binding") ||
+		/not\s+implemented/u.test(message) ||
+		message.includes("not available") ||
+		message.includes("missing") ||
+		message.includes("dlopen")
+	) {
+		return { fallback: true, reason: "local-unavailable", cause };
+	}
+	if (
+		message.includes("busy") ||
+		message.includes("overloaded") ||
+		message.includes("thermal") ||
+		message.includes("low-power") ||
+		message.includes("timeout")
+	) {
+		return { fallback: true, reason: "local-overloaded", cause };
+	}
+	if (
+		message.includes("llama_decode") ||
+		message.includes("mtmd") ||
+		message.includes("projector") ||
+		message.includes("ggml_assert")
+	) {
+		return { fallback: true, reason: "local-error", cause };
+	}
+	return { fallback: true, reason: "local-error", cause };
+}
+
+export function isVisionFallbackOutcome(
+	outcome: LocalVisionOutcome,
+): outcome is {
+	kind: "fallback";
+	reason: VisionFallbackReason;
+	cause?: Error;
+} {
+	return (
+		typeof outcome === "object" &&
+		outcome !== null &&
+		"kind" in outcome &&
+		outcome.kind === "fallback"
+	);
+}
+
+export function normalizeVisionDescription(
+	result: LocalVisionResult,
+): ImageDescriptionResult {
+	if (typeof result === "object" && result !== null && "kind" in result) {
+		return normalizeVisionDescription(result.result);
+	}
+	if (typeof result === "string") {
+		const description = result.trim();
+		if (!description) {
+			throw new Error(
+				"[vision-fallback] IMAGE_DESCRIPTION backend returned an empty description",
+			);
+		}
+		return {
+			title: description.split(/[.!?]/, 1)[0]?.trim() || "Image",
+			description,
+		};
+	}
+	if (
+		result &&
+		typeof result.title === "string" &&
+		typeof result.description === "string" &&
+		result.title.trim() &&
+		result.description.trim()
+	) {
+		const title = result.title.trim();
+		const description = result.description.trim();
+		if (title === result.title && description === result.description) {
+			return result;
+		}
+		return { title, description };
+	}
+	throw new Error(
+		"[vision-fallback] IMAGE_DESCRIPTION backend returned an invalid description",
+	);
+}
+
+function resolveCloudToken(options: VisionCloudFallbackOptions): string | null {
+	return (
+		options.token?.trim() ||
+		options.apiKey?.trim() ||
+		process.env.ELIZA_CLOUD_TOKEN?.trim() ||
+		process.env.ELIZA_CLOUD_API_KEY?.trim() ||
+		null
+	);
+}
+
+function resolveCloudBaseUrl(options: VisionCloudFallbackOptions): string {
+	return (
+		options.baseUrl?.trim() ||
+		process.env.ELIZA_CLOUD_BASE_URL?.trim() ||
+		"https://api.elizacloud.ai"
+	).replace(/\/+$/, "");
+}
+
+function imageRequestBody(params: ImageDescriptionParams | string): {
+	image: { kind: "url"; url: string } | { kind: "data"; data: string };
+	prompt?: string;
+} {
+	if (typeof params === "string") {
+		return params.startsWith("data:")
+			? { image: { kind: "data", data: params } }
+			: { image: { kind: "url", url: params } };
+	}
+	const imageUrl = (params as { imageUrl?: string }).imageUrl;
+	const image = (params as { image?: string }).image;
+	const source = imageUrl ?? image;
+	const body = source?.startsWith("data:")
+		? { image: { kind: "data" as const, data: source } }
+		: { image: { kind: "url" as const, url: source ?? "" } };
+	if (params.prompt) return { ...body, prompt: params.prompt };
+	return body;
+}
+
+async function callCloudVision(
+	params: ImageDescriptionParams | string,
+	options: VisionCloudFallbackOptions,
+): Promise<ImageDescriptionResult> {
+	const token = resolveCloudToken(options);
+	if (!token) {
+		throw new Error("Eliza Cloud image fallback is not configured");
+	}
+	const fetchImpl = options.fetch ?? fetch;
+	const response = await fetchImpl(
+		`${resolveCloudBaseUrl(options)}/v1/vision/describe`,
+		{
+			method: "POST",
+			headers: {
+				"content-type": "application/json",
+				authorization: `Bearer ${token}`,
+			},
+			body: JSON.stringify(imageRequestBody(params)),
+		},
+	);
+	if (!response.ok) {
+		throw new Error(
+			`Eliza Cloud image fallback failed with ${response.status}`,
+		);
+	}
+	return normalizeVisionDescription(
+		(await response.json()) as LocalVisionResult,
+	);
+}
+
+export function wrapImageDescriptionHandlerWithCloudFallback(
+	local: LocalImageDescriptionHandler,
+	options: VisionCloudFallbackOptions = {},
+): WrappedImageDescriptionHandler {
+	const enabled = options.enabled ?? true;
+	const log = options.log ?? (() => undefined);
+	return async (params) => {
+		let localOutcome: LocalVisionOutcome;
+		try {
+			localOutcome = await local(params);
+		} catch (error) {
+			const classified = classifyLocalVisionError(error);
+			if (!classified.fallback) throw error;
+			localOutcome = {
+				kind: "fallback",
+				reason: classified.reason,
+				cause: classified.cause,
+			};
+		}
+
+		if (!isVisionFallbackOutcome(localOutcome)) {
+			return normalizeVisionDescription(localOutcome);
+		}
+		if (!enabled) return localOutcome;
+
+		const token = resolveCloudToken(options);
+		if (!options.handler && !token) return localOutcome;
+
+		log("[vision/cloud-fallback] local IMAGE_DESCRIPTION fallback", {
+			reason: localOutcome.reason,
+		});
+		try {
+			const cloudOutcome = options.handler
+				? await options.handler(params, localOutcome.reason)
+				: await callCloudVision(params, options);
+			if (isVisionFallbackOutcome(cloudOutcome)) return cloudOutcome;
+			return normalizeVisionDescription(cloudOutcome);
+		} catch (error) {
+			return {
+				...localOutcome,
+				cause: error instanceof Error ? error : new Error(String(error)),
+			};
+		}
+	};
+}

package/src/services/vision/fallback-chain.test.ts

@@ -0,0 +1,86 @@
+import type { ImageDescriptionResult } from "@elizaos/core";
+import { describe, expect, it } from "vitest";
+import type { LocalImageDescriptionHandler } from "./cloud-fallback";
+import { withVisionFallbackChain } from "./index";
+
+const localResult: ImageDescriptionResult = {
+	title: "Local",
+	description: "local description",
+};
+
+const cloudResult: ImageDescriptionResult = {
+	title: "Cloud",
+	description: "cloud description",
+};
+
+const vastResult: ImageDescriptionResult = {
+	title: "Vast",
+	description: "vast description",
+};
+
+describe("vision fallback chain", () => {
+	it("returns local IMAGE_DESCRIPTION results without calling fallbacks", async () => {
+		let cloudCalls = 0;
+		const handler = withVisionFallbackChain(async () => localResult, {
+			cloud: {
+				handler: async () => {
+					cloudCalls += 1;
+					return cloudResult;
+				},
+			},
+		});
+
+		await expect(
+			handler({} as never, "data:image/png;base64,a"),
+		).resolves.toEqual(localResult);
+		expect(cloudCalls).toBe(0);
+	});
+
+	it("falls through local to cloud", async () => {
+		const local: LocalImageDescriptionHandler = async () => ({
+			kind: "fallback",
+			reason: "local-unavailable",
+		});
+		const handler = withVisionFallbackChain(local, {
+			cloud: { handler: async () => cloudResult },
+		});
+
+		await expect(
+			handler({} as never, "data:image/png;base64,a"),
+		).resolves.toEqual(cloudResult);
+	});
+
+	it("falls through cloud to vast", async () => {
+		const handler = withVisionFallbackChain(
+			async () => ({ kind: "fallback", reason: "local-error" }),
+			{
+				cloud: {
+					handler: async () => ({ kind: "fallback", reason: "cloud-error" }),
+				},
+				vast: { handler: async () => vastResult },
+			},
+		);
+
+		await expect(
+			handler({} as never, "data:image/png;base64,a"),
+		).resolves.toEqual(vastResult);
+	});
+
+	it("throws a structured error when every provider falls back", async () => {
+		const handler = withVisionFallbackChain(
+			async () => ({ kind: "fallback", reason: "local-error" }),
+			{
+				cloud: {
+					handler: async () => ({ kind: "fallback", reason: "cloud-error" }),
+				},
+				vast: {
+					handler: async () => ({ kind: "fallback", reason: "vast-error" }),
+				},
+			},
+		);
+
+		await expect(
+			handler({} as never, "data:image/png;base64,a"),
+		).rejects.toThrow("all IMAGE_DESCRIPTION providers exhausted");
+	});
+});

package/src/services/vision/hash.ts

@@ -0,0 +1,157 @@
+/**
+ * Vision-embedding cache key derivation (WS2).
+ *
+ * The arbiter's vision-embedding cache (WS1, `vision-embedding-cache.ts`)
+ * is keyed by SHA-256 of a *normalized* representation of the input
+ * image. The normalization step is what makes the cache useful across
+ * platforms: two JPEG encodings of the same screenshot, or an RGBA vs
+ * RGB frame captured by different platforms, must hash to the same key
+ * or the cache hit rate collapses.
+ *
+ * Normalization is deliberately minimal:
+ *
+ *   1. Resolve the input to raw bytes (decoding base64/data-url wrappers).
+ *   2. Hash with the model-family prefix so the cache can hold tokens
+ *      for multiple VL families without collision.
+ *
+ * What we DO NOT do here:
+ *
+ *   - Resize the image. The backend's projector enforces its own input
+ *     resolution; the bytes the projector sees are what gets projected.
+ *      Re-encoding here would add work without changing the hit rate
+ *      (the platform-provided buffer is already at the camera's native
+ *      resolution).
+ *   - Strip JPEG/PNG headers. They contribute to the hash; two
+ *      reencodings of the same pixel array land in different cache
+ *      slots intentionally. Reuse only the exact same byte stream.
+ *
+ * If a downstream caller wants finer-grained cache hits (e.g. dedupe
+ * across re-encodings of the same screen frame), it should decode to
+ * RGBA pixels itself and call `hashRawPixels`. The default
+ * `hashVisionInput` path is the conservative, byte-stream-only path.
+ */
+
+import { createHash } from "node:crypto";
+import type { VisionImageInput } from "./types";
+
+const DEFAULT_FAMILY = "gemma-vl";
+
+/**
+ * Resolve a `VisionImageInput` to its raw bytes. Returns the decoded
+ * payload plus an optional MIME type the caller can forward to the
+ * backend. Throws on `url:` inputs — those must be fetched by the
+ * caller; the hash step does not own HTTP.
+ */
+export function resolveImageBytes(input: VisionImageInput): {
+	bytes: Uint8Array;
+	mimeType?: string;
+} {
+	switch (input.kind) {
+		case "bytes":
+			return { bytes: input.bytes, mimeType: input.mimeType };
+		case "base64": {
+			const bytes = Uint8Array.from(Buffer.from(input.base64, "base64"));
+			return { bytes, mimeType: input.mimeType };
+		}
+		case "dataUrl": {
+			const match = /^data:([^;,]+)(?:;[^,]*)?,(.*)$/s.exec(input.dataUrl);
+			if (!match) {
+				throw new Error(
+					"[vision/hash] malformed data URL — expected data:<mime>;base64,<payload>",
+				);
+			}
+			const mimeType = match[1];
+			const payload = match[2];
+			const isBase64 = /;base64/i.test(input.dataUrl);
+			const bytes = Uint8Array.from(
+				Buffer.from(payload, isBase64 ? "base64" : "utf8"),
+			);
+			return { bytes, mimeType };
+		}
+		case "url":
+			throw new Error(
+				"[vision/hash] url inputs must be fetched by the caller before hashing — the hash step does not own HTTP",
+			);
+	}
+}
+
+/**
+ * Hash an opaque byte stream with the model-family prefix. The result
+ * is stable across processes and platforms (Node, Bun, and the
+ * Capacitor JS bridge all return the same hex string for the same
+ * input).
+ */
+export function hashImageBytes(
+	bytes: Uint8Array,
+	modelFamily: string = DEFAULT_FAMILY,
+): string {
+	const h = createHash("sha256");
+	h.update(modelFamily);
+	// Length prefix prevents a `family || bytes` collision against a
+	// crafted family string that ends with the leading bytes of the
+	// payload. Cheap, defensible.
+	const lenBuf = Buffer.alloc(4);
+	lenBuf.writeUInt32BE(bytes.byteLength, 0);
+	h.update(lenBuf);
+	h.update(bytes);
+	return h.digest("hex");
+}
+
+/**
+ * Hash a raw pixel buffer (RGBA / RGB / BGRA / BGR). The channel order
+ * is folded into the prefix so the same image captured on two different
+ * platforms (Android = RGBA, macOS screenshot = BGRA) produces the same
+ * key when normalized. Width / height are also included so the cache
+ * doesn't conflate two scaled versions of the same source.
+ */
+export function hashRawPixels(args: {
+	bytes: Uint8Array;
+	width: number;
+	height: number;
+	channelOrder: "rgba" | "rgb" | "bgra" | "bgr";
+	modelFamily?: string;
+}): string {
+	const h = createHash("sha256");
+	h.update(args.modelFamily ?? DEFAULT_FAMILY);
+	h.update("|raw|");
+	const prefix = Buffer.alloc(12);
+	prefix.writeUInt32BE(args.width, 0);
+	prefix.writeUInt32BE(args.height, 4);
+	prefix.write(args.channelOrder.padEnd(4, " "), 8, "ascii");
+	h.update(prefix);
+	// Channel-order normalization: rewrite BGRA→RGBA and BGR→RGB in
+	// place into a new buffer so all three platforms land on the same
+	// hash even when the input buffer order differs.
+	const normalized = normalizeChannels(args.bytes, args.channelOrder);
+	h.update(normalized);
+	return h.digest("hex");
+}
+
+function normalizeChannels(
+	bytes: Uint8Array,
+	order: "rgba" | "rgb" | "bgra" | "bgr",
+): Uint8Array {
+	if (order === "rgba" || order === "rgb") return bytes;
+	const stride = order === "bgra" ? 4 : 3;
+	const out = new Uint8Array(bytes.byteLength);
+	for (let i = 0; i + stride <= bytes.byteLength; i += stride) {
+		out[i] = bytes[i + 2];
+		out[i + 1] = bytes[i + 1];
+		out[i + 2] = bytes[i];
+		if (stride === 4) out[i + 3] = bytes[i + 3];
+	}
+	return out;
+}
+
+/**
+ * Convenience wrapper used by the provider: takes a `VisionImageInput`
+ * and a model family, returns the cache key. URL inputs throw —
+ * callers must fetch first.
+ */
+export function hashVisionInput(
+	input: VisionImageInput,
+	modelFamily: string = DEFAULT_FAMILY,
+): string {
+	const { bytes } = resolveImageBytes(input);
+	return hashImageBytes(bytes, modelFamily);
+}