@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/dist/services/verify-on-device.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Engine-backed `verifyOnDevice` hook for the Eliza-1 downloader
+ * (`packages/inference/AGENTS.md` §7): after a bundle is materialized and
+ * every file's sha256 checks out, run one cold smoke pass —
+ *
+ *   load → 1-token text generation → (if the bundle ships voice assets)
+ *   1-phrase voice generation → barge-in cancel → unload
+ *
+ * — before the bundle is allowed to auto-fill an empty default slot. The
+ * downloader stays decoupled from the engine: it only knows the
+ * {@link VerifyBundleOnDevice} shape; this module is the concrete
+ * implementation the service layer injects.
+ *
+ * Failure semantics: any error throws. The downloader catches it and leaves
+ * `bundleVerifiedAt` unset, so an unverified bundle is registered but does
+ * not become the recommended default. There is no "verified anyway" path —
+ * voice is mandatory for Eliza-1 voice tiers (AGENTS.md §3), so a bundle
+ * whose fused voice ABI is not loadable on this device legitimately fails
+ * verify until the fused build is present.
+ */
+import fs from "node:fs/promises";
+import type { VerifyBundleOnDevice } from "./downloader";
+import { localInferenceEngine } from "./engine";
+import { parseManifestOrThrow } from "./manifest";
+type VerifyEngine = Pick<typeof localInferenceEngine, "load" | "generate" | "ensureActiveBundleVoiceReady" | "startVoice" | "armVoice" | "synthesizeSpeech" | "triggerBargeIn" | "stopVoice" | "unload">;
+interface VerifyBundleOnDeviceDeps {
+    readonly engine: VerifyEngine;
+    readonly readFile: typeof fs.readFile;
+    readonly parseManifest: typeof parseManifestOrThrow;
+}
+export declare function createVerifyBundleOnDevice(deps?: Partial<VerifyBundleOnDeviceDeps>): VerifyBundleOnDevice;
+export declare const verifyBundleOnDevice: VerifyBundleOnDevice;
+export {};
+//# sourceMappingURL=verify-on-device.d.ts.map

package/dist/services/verify-on-device.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"verify-on-device.d.ts","sourceRoot":"","sources":["../../src/services/verify-on-device.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAOlD,KAAK,YAAY,GAAG,IAAI,CACvB,OAAO,oBAAoB,EACzB,MAAM,GACN,UAAU,GACV,8BAA8B,GAC9B,YAAY,GACZ,UAAU,GACV,kBAAkB,GAClB,gBAAgB,GAChB,WAAW,GACX,QAAQ,CACV,CAAC;AAEF,UAAU,wBAAwB;IACjC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAC,QAAQ,CAAC;IACtC,QAAQ,CAAC,aAAa,EAAE,OAAO,oBAAoB,CAAC;CACpD;AAqDD,wBAAgB,0BAA0B,CACzC,IAAI,GAAE,OAAO,CAAC,wBAAwB,CAAM,GAC1C,oBAAoB,CAoBtB;AAED,eAAO,MAAM,oBAAoB,EAAE,oBACN,CAAC"}

package/dist/services/verify.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Re-export of the shared model-file verification module. The canonical
+ * implementation lives in `@elizaos/shared/local-inference` because both
+ * the server (`@elizaos/app-core`) and the UI client (`@elizaos/ui`)
+ * compute the same SHA256 / GGUF-magic checks against on-disk models.
+ */
+export { __registryPathForTests, hashFile, type VerifyResult, type VerifyState, verifyInstalledModel, } from "@elizaos/shared/local-inference/verify";
+//# sourceMappingURL=verify.d.ts.map

package/dist/services/verify.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"verify.d.ts","sourceRoot":"","sources":["../../src/services/verify.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,EACN,sBAAsB,EACtB,QAAQ,EACR,KAAK,YAAY,EACjB,KAAK,WAAW,EAChB,oBAAoB,GACpB,MAAM,wCAAwC,CAAC"}

package/dist/services/vision/aosp-unavailable.d.ts

@@ -0,0 +1,115 @@
+/**
+ * AOSP vision-describe backend contract (WS2).
+ *
+ * The bun:ffi llama.cpp binding in `@elizaos/plugin-aosp-local-inference`
+ * already exposes the embedding helpers (`llama_set_embeddings`,
+ * `llama_get_embeddings_seq`) and the model/context lifecycle. mtmd
+ * (multi-modal definition) is part of upstream llama.cpp from b8198+
+ * but the libeliza-llama-shim that the AOSP binding loads does NOT
+ * yet export the matching `eliza_llama_mtmd_*` symbols.
+ *
+ * This file is the JS contract for the FFI binding the AOSP plugin
+ * will add when the native side ships. It lives in
+ * `plugin-local-inference` (not in `plugin-aosp-local-inference`) so
+ * cross-plugin consumers can compile against the same interface
+ * regardless of platform. When the AOSP shim ships the native
+ * symbols, the implementation in `plugin-aosp-local-inference/src/
+ * aosp-llama-vision.ts` (to be added) will satisfy this contract.
+ *
+ * Required native symbols (libeliza-llama-shim):
+ *
+ *   eliza_llama_mtmd_init_from_file(
+ *     mmproj_path: const char *,
+ *     out_handle: void **
+ *   ) -> int32  // 0 = ok; non-zero = error code
+ *
+ *   eliza_llama_mtmd_free(mtmd_handle: void *) -> void
+ *
+ *   eliza_llama_mtmd_encode(
+ *     mtmd_handle: void *,
+ *     image_bytes: const uint8_t *,
+ *     image_len: size_t,
+ *     out_tokens_ptr: float **,   // owned by mtmd; valid until next encode
+ *     out_token_count: int32 *,
+ *     out_hidden_size: int32 *
+ *   ) -> int32
+ *
+ *   eliza_llama_decode_with_mmproj(
+ *     ctx: void *,                  // existing llama_context
+ *     mtmd_tokens: const float *,
+ *     mtmd_token_count: int32,
+ *     mtmd_hidden_size: int32,
+ *     prompt: const char *,
+ *     prompt_len: size_t,
+ *     max_tokens: int32,
+ *     temperature: float,
+ *     out_text_buf: char *,         // caller-allocated
+ *     out_text_buf_cap: size_t,
+ *     out_text_written: size_t *
+ *   ) -> int32
+ *
+ * Optional fast path: if the shim adds
+ *
+ *   eliza_llama_mtmd_describe(
+ *     model: void *,                // existing llama_model
+ *     ctx: void *,                  // existing llama_context
+ *     mtmd_handle: void *,
+ *     image_bytes: const uint8_t *,
+ *     image_len: size_t,
+ *     prompt: const char *,
+ *     prompt_len: size_t,
+ *     max_tokens: int32,
+ *     temperature: float,
+ *     out_buf: char *,
+ *     out_buf_cap: size_t,
+ *     out_written: size_t *
+ *   ) -> int32
+ *
+ * that single-call wrapper is preferred — it lets the shim fuse the
+ * encode+decode steps internally and avoids the round-trip of token
+ * pointers across the FFI boundary (which bun:ffi handles, but at the
+ * cost of two extra pointer dereferences per frame).
+ *
+ * Until the native side exports those symbols, this module's `loadAospVisionBackend`
+ * throws a structured error that the arbiter / handler can surface as
+ * "vision not available on this platform". It does NOT register
+ * silently — silent unavailability would let the runtime fall back to
+ * the cloud path on a mobile device that explicitly disabled it.
+ */
+import type { VisionDescribeBackend, VisionDescribeLoadArgs } from "./types";
+/**
+ * The AOSP binding's mtmd surface, when present. The AOSP plugin
+ * registers an instance under the runtime service name
+ * `"aosp-llama-mtmd"` once the native shim exports the symbols above.
+ */
+export interface AospLlamaMtmdBinding {
+    /** True when libeliza-llama-shim.so was loaded and exports the mtmd symbols. */
+    hasMtmd(): boolean;
+    /** Initialize an mtmd handle for the given mmproj path. */
+    initMtmd(args: {
+        mmprojPath: string;
+    }): Promise<AospMtmdHandle>;
+}
+export interface AospMtmdHandle {
+    /**
+     * Single-call describe — wraps mtmd_encode + decode_with_chunks. The
+     * AOSP shim's `eliza_llama_mtmd_describe` lands here. Backends that
+     * only have the split encode/decode symbols implement this on top
+     * of two FFI calls; backends with the fused symbol use one.
+     */
+    describe(args: {
+        imageBytes: Uint8Array;
+        prompt: string;
+        maxTokens?: number;
+        temperature?: number;
+        signal?: AbortSignal;
+    }): Promise<string>;
+    /** Free the mtmd handle (and any cached encode buffers). */
+    dispose(): Promise<void>;
+}
+export interface LoadAospVisionBackendOptions {
+    loadArgs: VisionDescribeLoadArgs;
+    mtmdBinding?: AospLlamaMtmdBinding;
+}
+export declare function loadAospVisionBackend(opts: LoadAospVisionBackendOptions): Promise<VisionDescribeBackend>;
+//# sourceMappingURL=aosp-unavailable.d.ts.map

package/dist/services/vision/aosp-unavailable.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"aosp-unavailable.d.ts","sourceRoot":"","sources":["../../../src/services/vision/aosp-unavailable.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6EG;AAIH,OAAO,KAAK,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,MAAM,SAAS,CAAC;AAE7E;;;;GAIG;AACH,MAAM,WAAW,oBAAoB;IACpC,gFAAgF;IAChF,OAAO,IAAI,OAAO,CAAC;IACnB,2DAA2D;IAC3D,QAAQ,CAAC,IAAI,EAAE;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;CAChE;AAED,MAAM,WAAW,cAAc;IAC9B;;;;;OAKG;IACH,QAAQ,CAAC,IAAI,EAAE;QACd,UAAU,EAAE,UAAU,CAAC;QACvB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,CAAC,EAAE,WAAW,CAAC;KACrB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACpB,4DAA4D;IAC5D,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB;AAED,MAAM,WAAW,4BAA4B;IAC5C,QAAQ,EAAE,sBAAsB,CAAC;IACjC,WAAW,CAAC,EAAE,oBAAoB,CAAC;CACnC;AAED,wBAAsB,qBAAqB,CAC1C,IAAI,EAAE,4BAA4B,GAChC,OAAO,CAAC,qBAAqB,CAAC,CA0ChC"}

package/dist/services/vision/capacitor-llama.d.ts

@@ -0,0 +1,99 @@
+/**
+ * Capacitor-llama vision-describe backend (WS2).
+ *
+ * Wraps the in-process capacitor-llama binding's multimodal projector
+ * (mtmd) surface and exposes the WS2 `VisionDescribeBackend` contract.
+ *
+ * State of the binding (2026-05-19):
+ *   `llama-cpp-capacitor`'s `LlamaContext` exposes `initMultimodal` and
+ *   `getMultimodalSupport`, which load an mmproj alongside the chat
+ *   target. The desktop bun:ffi shim does not yet bind those symbols —
+ *   the desktop FFI path returns `binding_missing_mtmd` until the shim
+ *   adds `mtmd_init_from_file` + the encode/decode helpers.
+ *
+ * What this module does today:
+ *   - Provides the WS2-shaped backend so plugin-vision / plugin-image-gen /
+ *     computer-use can compile against a stable contract.
+ *   - When the binding exposes the mtmd API, the backend dispatches
+ *     through it.
+ *   - Otherwise, the backend falls back to an injected
+ *     `VisionManagerLike` implementation when one is supplied (kept as a
+ *     pluggable seam for tests and out-of-tree integrations).
+ *   - When neither path is wired, `describe()` throws a structured
+ *     `VisionBackendUnavailableError` the arbiter surfaces upward.
+ *
+ * GPU validation status (this host has neither GPU):
+ *   The mtmd encode path is GPU-accelerated when the underlying llama.cpp
+ *   build dispatches `llama_image_t` through the model's batch path. We
+ *   document the on-device validation that's required for each GPU
+ *   family at the bottom of this file's tests (see
+ *   `__tests__/vision-describe.test.ts`). Until those run on real
+ *   hardware, GPU-backed vision is "implementation present, not
+ *   validated".
+ */
+import type { VisionDescribeBackend, VisionDescribeBackendOptions, VisionDescribeLoadArgs } from "./types";
+export declare class VisionBackendUnavailableError extends Error {
+    readonly backendId: string;
+    readonly reason: "binding_missing_mtmd" | "no_fallback_present" | "mmproj_missing";
+    readonly code = "VISION_BACKEND_UNAVAILABLE";
+    constructor(backendId: string, reason: "binding_missing_mtmd" | "no_fallback_present" | "mmproj_missing", message: string);
+}
+/**
+ * Optional shape the Capacitor-llama binding exposes once the mtmd typed
+ * wrappers land in the shared adapter. The backend only consumes
+ * `describeWithMmproj`, which wraps `LlamaContext.initMultimodal` +
+ * `completion(...)` with `media_paths`. Backends that don't satisfy this
+ * shape are treated as "binding without mtmd support" and the fallback
+ * path is used.
+ */
+export interface CapacitorLlamaMtmdBinding {
+    loadVisionModel(args: {
+        modelPath: string;
+        mmprojPath: string;
+        gpuLayers?: number | "auto" | "max";
+        contextSize?: number;
+    }): Promise<CapacitorLlamaMtmdHandle>;
+}
+export interface CapacitorLlamaMtmdHandle {
+    describeWithMmproj(args: {
+        imageBytes: Uint8Array;
+        mimeType?: string;
+        prompt: string;
+        maxTokens?: number;
+        temperature?: number;
+        signal?: AbortSignal;
+        projectedTokens?: VisionDescribeBackendOptions["projectedTokens"];
+    }): Promise<{
+        text: string;
+        projectorMs?: number;
+        decodeMs?: number;
+    }>;
+    dispose(): Promise<void>;
+}
+/**
+ * Optional VisionManager-shape fallback. Kept available as a pluggable
+ * injection point for tests and out-of-tree integrations that want to
+ * supply their own image captioning implementation.
+ */
+export interface VisionManagerLike {
+    processImage(dataUrl: string): Promise<{
+        title: string;
+        description: string;
+    }>;
+}
+export interface CapacitorLlamaVisionBackendOptions {
+    loadArgs: VisionDescribeLoadArgs;
+    /**
+     * Injected by tests and by the shared mtmd typed wrappers. When
+     * provided the backend uses the mtmd path.
+     */
+    mtmd?: CapacitorLlamaMtmdBinding;
+    /**
+     * Caption-only fallback. Optional — when present the backend uses it
+     * as last resort, after mtmd. Backends that have neither throw a
+     * structured `VisionBackendUnavailableError`.
+     */
+    visionManager?: VisionManagerLike;
+}
+export declare function loadCapacitorLlamaVisionBackend(opts: CapacitorLlamaVisionBackendOptions): Promise<VisionDescribeBackend>;
+//# sourceMappingURL=capacitor-llama.d.ts.map

package/dist/services/vision/capacitor-llama.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"capacitor-llama.d.ts","sourceRoot":"","sources":["../../../src/services/vision/capacitor-llama.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAIH,OAAO,KAAK,EACX,qBAAqB,EACrB,4BAA4B,EAC5B,sBAAsB,EAGtB,MAAM,SAAS,CAAC;AAEjB,qBAAa,6BAA8B,SAAQ,KAAK;IAGtD,QAAQ,CAAC,SAAS,EAAE,MAAM;IAC1B,QAAQ,CAAC,MAAM,EACZ,sBAAsB,GACtB,qBAAqB,GACrB,gBAAgB;IANpB,QAAQ,CAAC,IAAI,gCAAgC;gBAEnC,SAAS,EAAE,MAAM,EACjB,MAAM,EACZ,sBAAsB,GACtB,qBAAqB,GACrB,gBAAgB,EACnB,OAAO,EAAE,MAAM;CAKhB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,yBAAyB;IACzC,eAAe,CAAC,IAAI,EAAE;QACrB,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC;QACpC,WAAW,CAAC,EAAE,MAAM,CAAC;KACrB,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;CACtC;AAED,MAAM,WAAW,wBAAwB;IACxC,kBAAkB,CAAC,IAAI,EAAE;QACxB,UAAU,EAAE,UAAU,CAAC;QACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,eAAe,CAAC,EAAE,4BAA4B,CAAC,iBAAiB,CAAC,CAAC;KAClE,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACvE,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB;AAED;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IACjC,YAAY,CACX,OAAO,EAAE,MAAM,GACb,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACnD;AAED,MAAM,WAAW,kCAAkC;IAClD,QAAQ,EAAE,sBAAsB,CAAC;IACjC;;;OAGG;IACH,IAAI,CAAC,EAAE,yBAAyB,CAAC;IACjC;;;;OAIG;IACH,aAAa,CAAC,EAAE,iBAAiB,CAAC;CAClC;AAID,wBAAsB,+BAA+B,CACpD,IAAI,EAAE,kCAAkC,GACtC,OAAO,CAAC,qBAAqB,CAAC,CA2EhC"}

package/dist/services/vision/cloud-fallback.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Soft cloud fallback wrapper for local IMAGE_DESCRIPTION handlers.
+ *
+ * The local vision path can report recoverable unavailability without forcing
+ * callers to know which remote provider is paired. The wrapper keeps that
+ * state explicit: handlers return a normal image description or a typed
+ * fallback outcome that the next layer can handle.
+ */
+import type { ImageDescriptionParams, ImageDescriptionResult } from "@elizaos/core";
+export type VisionFallbackReason = "local-unavailable" | "local-overloaded" | "local-error" | "local-aborted-pre-completion" | "local-not-registered" | "cloud-unavailable" | "cloud-error" | "vast-unavailable" | "vast-error";
+export type LocalVisionOutcome = ImageDescriptionResult | string | {
+    kind: "ok";
+    result: ImageDescriptionResult | string;
+} | {
+    kind: "fallback";
+    reason: VisionFallbackReason;
+    cause?: Error;
+};
+export type LocalVisionResult = Exclude<LocalVisionOutcome, {
+    kind: "fallback";
+    reason: VisionFallbackReason;
+    cause?: Error;
+}>;
+export type LocalImageDescriptionHandler = (params: ImageDescriptionParams | string) => Promise<LocalVisionOutcome>;
+export type WrappedImageDescriptionHandler = LocalImageDescriptionHandler;
+export interface VisionCloudFallbackOptions {
+    enabled?: boolean;
+    token?: string;
+    apiKey?: string;
+    baseUrl?: string;
+    fetch?: typeof fetch;
+    handler?: (params: ImageDescriptionParams | string, reason: VisionFallbackReason) => Promise<LocalVisionOutcome>;
+    log?: (message: string, detail?: Record<string, unknown>) => void;
+}
+export declare function classifyLocalVisionError(error: unknown): {
+    fallback: boolean;
+    reason: VisionFallbackReason;
+    cause?: Error;
+};
+export declare function isVisionFallbackOutcome(outcome: LocalVisionOutcome): outcome is {
+    kind: "fallback";
+    reason: VisionFallbackReason;
+    cause?: Error;
+};
+export declare function normalizeVisionDescription(result: LocalVisionResult): ImageDescriptionResult;
+export declare function wrapImageDescriptionHandlerWithCloudFallback(local: LocalImageDescriptionHandler, options?: VisionCloudFallbackOptions): WrappedImageDescriptionHandler;
+//# sourceMappingURL=cloud-fallback.d.ts.map

package/dist/services/vision/cloud-fallback.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cloud-fallback.d.ts","sourceRoot":"","sources":["../../../src/services/vision/cloud-fallback.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EACX,sBAAsB,EACtB,sBAAsB,EACtB,MAAM,eAAe,CAAC;AAEvB,MAAM,MAAM,oBAAoB,GAC7B,mBAAmB,GACnB,kBAAkB,GAClB,aAAa,GACb,8BAA8B,GAC9B,sBAAsB,GACtB,mBAAmB,GACnB,aAAa,GACb,kBAAkB,GAClB,YAAY,CAAC;AAEhB,MAAM,MAAM,kBAAkB,GAC3B,sBAAsB,GACtB,MAAM,GACN;IAAE,IAAI,EAAE,IAAI,CAAC;IAAC,MAAM,EAAE,sBAAsB,GAAG,MAAM,CAAA;CAAE,GACvD;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,MAAM,EAAE,oBAAoB,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,CAAC;AAErE,MAAM,MAAM,iBAAiB,GAAG,OAAO,CACtC,kBAAkB,EAClB;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,MAAM,EAAE,oBAAoB,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,CACjE,CAAC;AAEF,MAAM,MAAM,4BAA4B,GAAG,CAC1C,MAAM,EAAE,sBAAsB,GAAG,MAAM,KACnC,OAAO,CAAC,kBAAkB,CAAC,CAAC;AAEjC,MAAM,MAAM,8BAA8B,GAAG,4BAA4B,CAAC;AAE1E,MAAM,WAAW,0BAA0B;IAC1C,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;IACrB,OAAO,CAAC,EAAE,CACT,MAAM,EAAE,sBAAsB,GAAG,MAAM,EACvC,MAAM,EAAE,oBAAoB,KACxB,OAAO,CAAC,kBAAkB,CAAC,CAAC;IACjC,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CAClE;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,OAAO,GAAG;IACzD,QAAQ,EAAE,OAAO,CAAC;IAClB,MAAM,EAAE,oBAAoB,CAAC;IAC7B,KAAK,CAAC,EAAE,KAAK,CAAC;CACd,CA2CA;AAED,wBAAgB,uBAAuB,CACtC,OAAO,EAAE,kBAAkB,GACzB,OAAO,IAAI;IACb,IAAI,EAAE,UAAU,CAAC;IACjB,MAAM,EAAE,oBAAoB,CAAC;IAC7B,KAAK,CAAC,EAAE,KAAK,CAAC;CACd,CAOA;AAED,wBAAgB,0BAA0B,CACzC,MAAM,EAAE,iBAAiB,GACvB,sBAAsB,CAiCxB;AAqED,wBAAgB,4CAA4C,CAC3D,KAAK,EAAE,4BAA4B,EACnC,OAAO,GAAE,0BAA+B,GACtC,8BAA8B,CAyChC"}

package/dist/services/vision/hash.d.ts

@@ -0,0 +1,71 @@
+/**
+ * Vision-embedding cache key derivation (WS2).
+ *
+ * The arbiter's vision-embedding cache (WS1, `vision-embedding-cache.ts`)
+ * is keyed by SHA-256 of a *normalized* representation of the input
+ * image. The normalization step is what makes the cache useful across
+ * platforms: two JPEG encodings of the same screenshot, or an RGBA vs
+ * RGB frame captured by different platforms, must hash to the same key
+ * or the cache hit rate collapses.
+ *
+ * Normalization is deliberately minimal:
+ *
+ *   1. Resolve the input to raw bytes (decoding base64/data-url wrappers).
+ *   2. Hash with the model-family prefix so the cache can hold tokens
+ *      for multiple VL families without collision.
+ *
+ * What we DO NOT do here:
+ *
+ *   - Resize the image. The backend's projector enforces its own input
+ *     resolution; the bytes the projector sees are what gets projected.
+ *      Re-encoding here would add work without changing the hit rate
+ *      (the platform-provided buffer is already at the camera's native
+ *      resolution).
+ *   - Strip JPEG/PNG headers. They contribute to the hash; two
+ *      reencodings of the same pixel array land in different cache
+ *      slots intentionally. Reuse only the exact same byte stream.
+ *
+ * If a downstream caller wants finer-grained cache hits (e.g. dedupe
+ * across re-encodings of the same screen frame), it should decode to
+ * RGBA pixels itself and call `hashRawPixels`. The default
+ * `hashVisionInput` path is the conservative, byte-stream-only path.
+ */
+import type { VisionImageInput } from "./types";
+/**
+ * Resolve a `VisionImageInput` to its raw bytes. Returns the decoded
+ * payload plus an optional MIME type the caller can forward to the
+ * backend. Throws on `url:` inputs — those must be fetched by the
+ * caller; the hash step does not own HTTP.
+ */
+export declare function resolveImageBytes(input: VisionImageInput): {
+    bytes: Uint8Array;
+    mimeType?: string;
+};
+/**
+ * Hash an opaque byte stream with the model-family prefix. The result
+ * is stable across processes and platforms (Node, Bun, and the
+ * Capacitor JS bridge all return the same hex string for the same
+ * input).
+ */
+export declare function hashImageBytes(bytes: Uint8Array, modelFamily?: string): string;
+/**
+ * Hash a raw pixel buffer (RGBA / RGB / BGRA / BGR). The channel order
+ * is folded into the prefix so the same image captured on two different
+ * platforms (Android = RGBA, macOS screenshot = BGRA) produces the same
+ * key when normalized. Width / height are also included so the cache
+ * doesn't conflate two scaled versions of the same source.
+ */
+export declare function hashRawPixels(args: {
+    bytes: Uint8Array;
+    width: number;
+    height: number;
+    channelOrder: "rgba" | "rgb" | "bgra" | "bgr";
+    modelFamily?: string;
+}): string;
+/**
+ * Convenience wrapper used by the provider: takes a `VisionImageInput`
+ * and a model family, returns the cache key. URL inputs throw —
+ * callers must fetch first.
+ */
+export declare function hashVisionInput(input: VisionImageInput, modelFamily?: string): string;
+//# sourceMappingURL=hash.d.ts.map

package/dist/services/vision/hash.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hash.d.ts","sourceRoot":"","sources":["../../../src/services/vision/hash.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAGH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAIhD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,gBAAgB,GAAG;IAC3D,KAAK,EAAE,UAAU,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB,CA4BA;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAC7B,KAAK,EAAE,UAAU,EACjB,WAAW,GAAE,MAAuB,GAClC,MAAM,CAWR;AAED;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE;IACnC,KAAK,EAAE,UAAU,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,GAAG,KAAK,GAAG,MAAM,GAAG,KAAK,CAAC;IAC9C,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB,GAAG,MAAM,CAeT;AAkBD;;;;GAIG;AACH,wBAAgB,eAAe,CAC9B,KAAK,EAAE,gBAAgB,EACvB,WAAW,GAAE,MAAuB,GAClC,MAAM,CAGR"}

package/dist/services/vision/index.d.ts

@@ -0,0 +1,95 @@
+/**
+ * Vision-describe capability (WS2) — public entry point.
+ *
+ * This module is what plugin-vision (WS4), the IMAGE_DESCRIPTION
+ * handler in `provider.ts`, and computer-use (WS9) import to register
+ * vision capability with the WS1 MemoryArbiter.
+ *
+ * Wiring:
+ *
+ *   const arbiter = service.getMemoryArbiter();
+ *   const registration = createVisionCapabilityRegistration({
+ *     loader: createDefaultVisionLoader({ ... }),
+ *     arbiterCache: arbiter,
+ *   });
+ *   arbiter.registerCapability(registration);
+ *
+ * `createVisionCapabilityRegistration` wraps the underlying backend so
+ * the arbiter's `run(request)` path:
+ *
+ *   1. Hashes the request's image bytes (model-family-scoped).
+ *   2. Checks the arbiter's vision-embedding cache.
+ *   3. On miss: calls `backend.describe(request)`, lets the backend
+ *      run its own projector + decoder. Backends that cannot expose projected
+ *      tokens return decoder text only, so the cache stays empty for this hash.
+ *      The decoder text is what the caller wanted anyway.
+ *   4. On hit: calls `backend.describe(request, { projectedTokens })`.
+ *      Backends that support pre-projected token reuse skip the
+ *      projector entirely. Backends that don't ignore the hint; the
+ *      result is still correct but the projector cost is paid again.
+ */
+export { type AospLlamaMtmdBinding, type AospMtmdHandle, type LoadAospVisionBackendOptions, loadAospVisionBackend, } from "./aosp-unavailable";
+export { type CapacitorLlamaMtmdBinding, type CapacitorLlamaMtmdHandle, type CapacitorLlamaVisionBackendOptions, loadCapacitorLlamaVisionBackend, VisionBackendUnavailableError, type VisionManagerLike, } from "./capacitor-llama";
+export { classifyLocalVisionError, type LocalImageDescriptionHandler, type LocalVisionOutcome, type VisionCloudFallbackOptions, type VisionFallbackReason, type WrappedImageDescriptionHandler, wrapImageDescriptionHandlerWithCloudFallback, } from "./cloud-fallback";
+export { hashImageBytes, hashRawPixels, hashVisionInput, resolveImageBytes, } from "./hash";
+export { createLlamaServerVisionBackend, type LlamaServerVisionBackendOptions, } from "./llama-server";
+export type { VisionDescribeBackend, VisionDescribeBackendLoader, VisionDescribeBackendOptions, VisionDescribeLoadArgs, VisionDescribeRequest, VisionDescribeResult, VisionImageChannelOrder, VisionImageInput, } from "./types";
+export { type VisionVastFallbackOptions, wrapImageDescriptionHandlerWithVastFallback, } from "./vast-fallback";
+import type { CapabilityRegistration } from "../memory-arbiter";
+import type { VisionDescribeBackend, VisionDescribeBackendLoader, VisionDescribeRequest, VisionDescribeResult } from "./types";
+/**
+ * Minimal arbiter shape we need from the cache. Lets tests inject a
+ * fake cache without pulling in the whole MemoryArbiter.
+ */
+export interface VisionEmbeddingCacheLike {
+    getCachedVisionEmbedding(hash: string): {
+        tokens: Float32Array;
+        tokenCount: number;
+        hiddenSize: number;
+        live?: boolean;
+    } | null;
+    setCachedVisionEmbedding(hash: string, entry: {
+        tokens: Float32Array;
+        tokenCount: number;
+        hiddenSize: number;
+    }, ttlMs?: number): void;
+}
+export interface CreateVisionCapabilityRegistrationOptions {
+    /**
+     * The arbiter (or any object with the cache passthroughs). When
+     * provided the wrapper performs hash → cache lookup before calling
+     * the backend's `describe`.
+     */
+    arbiterCache?: VisionEmbeddingCacheLike;
+    loader: VisionDescribeBackendLoader;
+    /** Default model family for the cache key. Defaults to `gemma-vl`. */
+    modelFamily?: string;
+    estimatedMb?: number;
+}
+/**
+ * Build a `CapabilityRegistration` ready to feed to
+ * `arbiter.registerCapability()`. The wrapper plumbs the cache hint
+ * into the backend's describe call so backends that support
+ * pre-projected tokens skip the projector.
+ */
+export declare function createVisionCapabilityRegistration(opts: CreateVisionCapabilityRegistrationOptions): CapabilityRegistration<VisionDescribeBackend, VisionDescribeRequest, VisionDescribeResult>;
+import type { IAgentRuntime, ImageDescriptionParams, ImageDescriptionResult } from "@elizaos/core";
+import { type LocalImageDescriptionHandler, type VisionCloudFallbackOptions } from "./cloud-fallback";
+import { type VisionVastFallbackOptions } from "./vast-fallback";
+/**
+ * Compose the full local → cloud → vast IMAGE_DESCRIPTION chain and
+ * terminate it as a runtime-shaped `ImageDescriptionHandler`. When all
+ * three paths return `{ kind: "fallback" }`, the terminator throws the
+ * underlying cause (or a structured upstream-fail message) so the runtime
+ * surfaces the failure cleanly rather than serving a sentinel result.
+ *
+ * This is the single entry point `ensure-local-inference-handler.ts`
+ * uses at the IMAGE_DESCRIPTION model registration site. Tests
+ * exercise the composition via the individual `wrap*` helpers; this
+ * function is the production wiring.
+ */
+export declare function withVisionFallbackChain(local: LocalImageDescriptionHandler, options?: {
+    cloud?: VisionCloudFallbackOptions;
+    vast?: VisionVastFallbackOptions;
+}): (runtime: IAgentRuntime, params: ImageDescriptionParams | string) => Promise<ImageDescriptionResult>;
+//# sourceMappingURL=index.d.ts.map

package/dist/services/vision/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/services/vision/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,OAAO,EACN,KAAK,oBAAoB,EACzB,KAAK,cAAc,EACnB,KAAK,4BAA4B,EACjC,qBAAqB,GACrB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACN,KAAK,yBAAyB,EAC9B,KAAK,wBAAwB,EAC7B,KAAK,kCAAkC,EACvC,+BAA+B,EAC/B,6BAA6B,EAC7B,KAAK,iBAAiB,GACtB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACN,wBAAwB,EACxB,KAAK,4BAA4B,EACjC,KAAK,kBAAkB,EACvB,KAAK,0BAA0B,EAC/B,KAAK,oBAAoB,EACzB,KAAK,8BAA8B,EACnC,4CAA4C,GAC5C,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACN,cAAc,EACd,aAAa,EACb,eAAe,EACf,iBAAiB,GACjB,MAAM,QAAQ,CAAC;AAChB,OAAO,EACN,8BAA8B,EAC9B,KAAK,+BAA+B,GACpC,MAAM,gBAAgB,CAAC;AACxB,YAAY,EACX,qBAAqB,EACrB,2BAA2B,EAC3B,4BAA4B,EAC5B,sBAAsB,EACtB,qBAAqB,EACrB,oBAAoB,EACpB,uBAAuB,EACvB,gBAAgB,GAChB,MAAM,SAAS,CAAC;AACjB,OAAO,EACN,KAAK,yBAAyB,EAC9B,2CAA2C,GAC3C,MAAM,iBAAiB,CAAC;AAEzB,OAAO,KAAK,EAEX,sBAAsB,EACtB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,KAAK,EACX,qBAAqB,EACrB,2BAA2B,EAC3B,qBAAqB,EACrB,oBAAoB,EACpB,MAAM,SAAS,CAAC;AAEjB;;;GAGG;AACH,MAAM,WAAW,wBAAwB;IACxC,wBAAwB,CAAC,IAAI,EAAE,MAAM,GAAG;QACvC,MAAM,EAAE,YAAY,CAAC;QACrB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;QACnB,IAAI,CAAC,EAAE,OAAO,CAAC;KACf,GAAG,IAAI,CAAC;IACT,wBAAwB,CACvB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE;QACN,MAAM,EAAE,YAAY,CAAC;QACrB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;KACnB,EACD,KAAK,CAAC,EAAE,MAAM,GACZ,IAAI,CAAC;CACR;AAED,MAAM,WAAW,yCAAyC;IACzD;;;;OAIG;IACH,YAAY,CAAC,EAAE,wBAAwB,CAAC;IACxC,MAAM,EAAE,2BAA2B,CAAC;IACpC,sEAAsE;IACtE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;;GAKG;AACH,wBAAgB,kCAAkC,CACjD,IAAI,EAAE,yCAAyC,GAC7C,sBAAsB,CACxB,qBAAqB,EACrB,qBAAqB,EACrB,oBAAoB,CACpB,CAkDA;AAED,OAAO,KAAK,EACX,aAAa,EACb,sBAAsB,EACtB,sBAAsB,EACtB,MAAM,eAAe,CAAC;AACvB,OAAO,EACN,KAAK,4BAA4B,EACjC,KAAK,0BAA0B,EAE/B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACN,KAAK,yBAAyB,EAE9B,MAAM,iBAAiB,CAAC;AAEzB;;;;;;;;;;;GAWG;AACH,wBAAgB,uBAAuB,CACtC,KAAK,EAAE,4BAA4B,EACnC,OAAO,GAAE;IACR,KAAK,CAAC,EAAE,0BAA0B,CAAC;IACnC,IAAI,CAAC,EAAE,yBAAyB,CAAC;CAC5B,GACJ,CACF,OAAO,EAAE,aAAa,EACtB,MAAM,EAAE,sBAAsB,GAAG,MAAM,KACnC,OAAO,CAAC,sBAAsB,CAAC,CAwBnC"}

package/dist/services/vision/llama-server.d.ts

@@ -0,0 +1,73 @@
+/**
+ * llama-server vision-describe backend (WS2).
+ *
+ * Wraps the out-of-process llama-server's `/completion` endpoint with
+ * the `image_data` array (base64-encoded payloads) and shapes the
+ * response to the WS2 `VisionDescribeBackend` contract.
+ *
+ * llama-server image-data API recap (verified against llama.cpp commit
+ * b8198+, May 2026):
+ *
+ *   POST /completion
+ *   { "prompt": "<...>USER: [img-12] What's in this image?\nASSISTANT:",
+ *     "image_data": [
+ *       { "data": "<base64 png/jpeg>", "id": 12 }
+ *     ],
+ *     "n_predict": 256,
+ *     "temperature": 0.2,
+ *     "stream": false }
+ *
+ *   Response:
+ *   { "content": "A photo of a cat.", "stop": true,
+ *     "timings": { "prompt_ms": 180.4, "predicted_ms": 423.1 } }
+ *
+ * Server-side mmproj is loaded via the `--mmproj <path>` flag on
+ * llama-server startup. The FFI runtime wrapper passes this flag
+ * already for tiers with vision enabled; this backend assumes the
+ * server has been started with the right mmproj for the active model.
+ *
+ * Backend responsibility:
+ *   - Encode the image as base64 (when not already).
+ *   - Build the prompt with the `[img-N]` placeholder convention.
+ *   - POST to `/completion`, parse the text + timings.
+ *   - Honour AbortSignal by passing it through to the fetch call.
+ *
+ * Backend explicitly does NOT:
+ *   - Start / stop the server. That's the FFI runtime wrapper's job.
+ *   - Resolve the mmproj path — the server already has it. The arbiter's
+ *     `--mmproj` was set when the text model loaded.
+ *   - Implement projector-token reuse. llama-server has no API to
+ *     accept pre-projected tokens; if the WS1 cache hit happens, this
+ *     backend ignores the hint and re-runs the projector. The cache
+ *     is more useful with the in-process node-llama-cpp backend.
+ *
+ * Metal / CUDA validation:
+ *   The llama-server build embeds the same mtmd_encode path the
+ *   in-process binding will eventually expose. On a Metal build the
+ *   image encode dispatches through the Metal compute encoder; on a
+ *   CUDA build through cuBLAS. We have no GPU on this host — see the
+ *   `__tests__/vision-describe.test.ts` notes for the GPU smoke check.
+ */
+import type { VisionDescribeBackend } from "./types";
+export interface LlamaServerVisionBackendOptions {
+    /**
+     * Base URL of the llama-server. The FFI runtime wrapper exposes
+     * this via `currentBaseUrl()`; pass the resolved URL here at load
+     * time. The backend keeps it as-is across calls.
+     */
+    baseUrl: string;
+    /**
+     * Optional fetch override. Tests inject a fake fetch; production
+     * uses global fetch. The signature mirrors `fetch` so the test
+     * surface is the same as the real one.
+     */
+    fetch?: typeof fetch;
+    /**
+     * Default `n_predict` budget when the caller doesn't specify
+     * `maxTokens`. 256 matches the description-length budget the
+     * Florence-2 / VisionManager path uses today.
+     */
+    defaultMaxTokens?: number;
+}
+export declare function createLlamaServerVisionBackend(opts: LlamaServerVisionBackendOptions): VisionDescribeBackend;
+//# sourceMappingURL=llama-server.d.ts.map

package/dist/services/vision/llama-server.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llama-server.d.ts","sourceRoot":"","sources":["../../../src/services/vision/llama-server.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiDG;AAGH,OAAO,KAAK,EACX,qBAAqB,EAGrB,MAAM,SAAS,CAAC;AAEjB,MAAM,WAAW,+BAA+B;IAC/C;;;;OAIG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;IACrB;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,wBAAgB,8BAA8B,CAC7C,IAAI,EAAE,+BAA+B,GACnC,qBAAqB,CAsEvB"}