@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/dist/services/backend-selector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"backend-selector.d.ts","sourceRoot":"","sources":["../../src/services/backend-selector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAMH,MAAM,MAAM,qBAAqB,GAAG,eAAe,CAAC;AAEpD,MAAM,MAAM,sBAAsB,GAAG,SAAS,GAAG,QAAQ,CAAC;AAE1D,MAAM,WAAW,kBAAkB;IAClC,iCAAiC;IACjC,QAAQ,EAAE,sBAAsB,CAAC;IACjC;;;;OAIG;IACH,YAAY,EAAE,OAAO,CAAC;IACtB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAED,0EAA0E;AAC1E,wBAAgB,sBAAsB,CACrC,GAAG,GAAE,MAAM,CAAC,UAAwB,GAClC,KAAK,GAAG,MAAM,GAAG,IAAI,CAKvB;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,CAC5B,KAAK,EAAE,kBAAkB,GACvB,qBAAqB,CAuBvB"}

package/dist/services/backend.d.ts

@@ -0,0 +1,440 @@
+/**
+ * Local-inference backend interface and dispatcher.
+ *
+ * Both shipping backends are served by the SAME in-process FFI
+ * `libelizainference` library behind the SAME streaming symbols — the
+ * difference is which in-process runtime the FFI's `llm_backend_select` drives:
+ *
+ *   - `llama-cpp`  → the optimized in-process FFI llama.cpp path (the default).
+ *     MTP, n-gram drafter, lookahead, `-ot` MoE offload, TurboQuant KV
+ *     cache, mlock/no-mmap/mmproj, etc. all live here. Serves the `.gguf`.
+ *   - `litert-lm`  → the in-process LiteRT-LM backend (Android NPU / GPU
+ *     delegate, gated `-DELIZA_ENABLE_LITERT`). Serves a `.litertlm` text
+ *     artifact staged under `<bundleRoot>/text/`. The dispatcher passes
+ *     `ELIZA_LLM_BACKEND=litert-lm` through the load; the C-side
+ *     `llm_backend_select` reads it (and probes `text/*.litertlm`) and routes
+ *     to the LiteRT factory. See `tools/omnivoice/src/llm-backend.h`.
+ *
+ * The dispatcher decides which one to use per-load based on:
+ *
+ *   1. `ELIZA_INFERENCE_BACKEND` env override — `llama-cpp` / `litert-lm` /
+ *      `auto`. A `litert-lm` force is honoured only when the build/platform
+ *      supports LiteRT and the bundle ships a `.litertlm` (else hard error).
+ *   2. A `.litertlm` text artifact in the bundle AND LiteRT support on this
+ *      build/platform → `litert-lm`. GGUF stays the default whenever the
+ *      LiteRT artifact or the build support is absent.
+ *   3. Catalog `runtime.optimizations.requiresKernel` — if any specialised
+ *      llama.cpp kernel is required (e.g. `turbo3`), the
+ *      dispatcher picks `llama-cpp`. Legacy bindings cannot
+ *      provide these kernels at all.
+ *   4. Default: optimized llama.cpp FFI.
+ *
+ * The dispatcher does NOT own backend internals. It owns selection only,
+ * plus a small load-state
+ * cache so callers can swap models without touching either backend
+ * directly.
+ */
+import type { StructuredGenerateParams } from "./structured-output";
+import type { CatalogModel, LocalRuntimeKernel } from "./types";
+import type { VerifierStreamEvent } from "./voice/types";
+/**
+ * Per-load runtime overrides forwarded by the dispatcher to whichever
+ * backend handles the load. Mirror of the relevant fields on
+ * `LocalInferenceLoadArgs` from `active-model.ts` — kept inline here so
+ * `backend.ts` stays free of cross-file circular imports (active-model
+ * imports engine, engine imports backend).
+ */
+export interface BackendLoadOverrides {
+    contextSize?: number;
+    cacheTypeK?: string;
+    cacheTypeV?: string;
+    gpuLayers?: number | "auto" | "max";
+    kvOffload?: "cpu" | "gpu" | "split" | {
+        gpuLayers: number;
+    };
+    flashAttention?: boolean;
+    mmap?: boolean;
+    mlock?: boolean;
+    useGpu?: boolean;
+    /** Absolute path to a multimodal projector GGUF passed to the FFI runtime. */
+    mmprojPath?: string;
+    /** Absolute path to the MTP drafter GGUF passed to the FFI runtime. */
+    draftModelPath?: string;
+    /** Eliza-1 bundle root for direct bundle loads not present in the registry. */
+    bundleRoot?: string;
+    /** Manifest path for direct bundle loads not present in the registry. */
+    manifestPath?: string;
+    /**
+     * Absolute path to a `.litertlm` LiteRT-LM text artifact staged under
+     * `<bundleRoot>/text/`, when the bundle ships one. Presence (plus LiteRT
+     * build/platform support) routes the load to the `litert-lm` backend; the
+     * `.gguf` `modelPath` stays the GGUF default otherwise.
+     */
+    litertModelPath?: string;
+}
+export interface BackendPlan {
+    /** Absolute path to the GGUF on disk. */
+    modelPath: string;
+    /**
+     * Catalog model id, when known. The dispatcher uses this to pull
+     * `runtime.optimizations` and `runtime.mtp` — without it, we can
+     * only honour the env override and fall back to `capacitor-llama`.
+     */
+    modelId?: string;
+    /** Catalog entry, when the caller already resolved it. */
+    catalog?: CatalogModel;
+    /**
+     * Per-load runtime overrides resolved by the active-model coordinator.
+     * The dispatcher passes these through verbatim to the chosen backend
+     * so the in-process binding can honour cache-type and contextSize
+     * requests instead of silently dropping them.
+     */
+    overrides?: BackendLoadOverrides;
+}
+export interface GenerateArgs extends StructuredGenerateParams {
+    prompt: string;
+    stopSequences?: string[];
+    /** Upper bound on output tokens; defaults to 2048. */
+    maxTokens?: number;
+    /** 0..1; 0.7 default. */
+    temperature?: number;
+    /** Nucleus sampling; defaults to 0.9. */
+    topP?: number;
+    /**
+     * Optional cache key from the runtime's `ProviderCachePlan`. Identical
+     * keys reuse the same KV cache prefix: the `llama-cpp` FFI backend derives
+     * a deterministic slot so requests with the same key land on the same
+     * persisted KV state. Empty / absent keys fall through to the historical
+     * stateless path.
+     */
+    cacheKey?: string;
+    /**
+     * Per-request abort signal. The `llama-cpp` FFI backend honours it
+     * cooperatively by cancelling the active FFI stream. Callers that want
+     * hard cancel for things like app pause / kill-switch pass the same signal
+     * here that they pass into `runtime.useModel`.
+     */
+    signal?: AbortSignal;
+    /**
+     * Optional per-request backend transport budget. This should be at least as
+     * long as the caller's user-visible generation timeout; shorter inner
+     * timeouts abort long local-prefill turns before the chat route can make the
+     * user-facing decision.
+     */
+    requestTimeoutMs?: number;
+    /**
+     * Incremental accepted text from the backend. The `llama-cpp` FFI backend
+     * calls this as accepted chunks arrive, per `llmStreamNext` step (it
+     * streams even when a `grammar` is set).
+     */
+    onTextChunk?: (chunk: string) => void | Promise<void>;
+    /**
+     * Max tokens the FFI backend decodes per `llmStreamNext` step — i.e. the
+     * granularity of `onTextChunk` emission. Smaller ⇒ smoother token-by-token
+     * streaming to the UI at the cost of more FFI round-trips per response.
+     * Unset ⇒ the backend default (coarse, throughput-tuned). The text/chat
+     * handler sets a small value for smooth streaming; voice leaves it unset.
+     */
+    maxTokensPerStep?: number;
+    /**
+     * Whether this generation is user-visible text and therefore eligible for
+     * voice-mode TTS. Internal JSON / planner calls must not be spoken.
+     */
+    voiceOutput?: "user-visible" | "internal";
+    /**
+     * Native verifier stream from speculative MTP. Exact accept/reject token
+     * ranges let voice TTS rollback avoid inferring state from text chunks.
+     */
+    onVerifierEvent?: (event: VerifierStreamEvent) => void | Promise<void>;
+}
+export type GenerateResult = string;
+export interface LocalGenerateWithUsageResult {
+    text: string;
+    usage?: {
+        prompt_tokens?: number;
+        completion_tokens?: number;
+        total_tokens?: number;
+        [key: string]: unknown;
+    };
+    slotId?: number;
+    firstTokenMs?: number | null;
+    mtpStats?: {
+        drafted: number;
+        accepted: number;
+        acceptanceRate: number | null;
+    };
+}
+/**
+ * The in-process runtime the FFI streaming pipe drives for a given load.
+ * `llama-cpp` is the default GGUF path; `litert-lm` is the LiteRT-LM
+ * `.litertlm` path (same FFI symbols, selected via `ELIZA_LLM_BACKEND` +
+ * the C-side `llm_backend_select`). This is the dispatcher's *selection*,
+ * distinct from `LocalInferenceBackend.id` (the implementation surface, which
+ * stays the single fused FFI backend regardless of the runtime it drives).
+ */
+export type BackendId = "llama-cpp" | "litert-lm";
+export interface LocalRuntimeLoadConfig {
+    modelId: string | null;
+    modelPath: string | null;
+    contextSize: number | null;
+    cacheTypeK: string | null;
+    cacheTypeV: string | null;
+    gpuLayers: number | null;
+    parallel: number;
+    binaryPath: string | null;
+    backend: BackendId | null;
+    mtp: {
+        specType: "draft-mtp";
+        draftMin: number;
+        draftMax: number;
+    } | null;
+}
+/**
+ * The backend contract every local-inference implementation satisfies.
+ *
+ * `available()` is a soft probe — it should NOT spawn anything; it just
+ * reports whether the backend can be used at all (e.g. is the binding
+ * loadable, is the binary on disk). Loading a specific model is `load()`.
+ */
+export interface LocalInferenceBackend {
+    /** Identifier for the concrete backend implementation. */
+    readonly id: "llama-cpp";
+    available(): Promise<boolean>;
+    load(plan: BackendPlan): Promise<void>;
+    unload(): Promise<void>;
+    generate(args: GenerateArgs): Promise<GenerateResult>;
+    hasLoadedModel(): boolean;
+    currentModelPath(): string | null;
+    /**
+     * Usage-instrumented variant of `generate`. Returns Anthropic-shape
+     * usage block plus per-turn MTP stats when available.
+     */
+    generateWithUsage?(args: GenerateArgs & {
+        slotId?: number;
+    }): Promise<LocalGenerateWithUsageResult>;
+    /** Vision describe via mmproj. Requires an mmproj-loaded backend. */
+    describeImage?(args: {
+        bytes: Uint8Array;
+        mimeType?: string;
+        prompt?: string;
+        maxTokens?: number;
+        temperature?: number;
+        signal?: AbortSignal;
+        /** Per-token callback for streaming vision describe (ABI v13). When set and
+         * the backend supports streaming, the description is decoded token-by-token
+         * through the same pipe as chat text; otherwise the backend returns the
+         * full description and ignores it. */
+        onTextChunk?: (chunk: string) => void | Promise<void>;
+        maxTokensPerStep?: number;
+    }): Promise<{
+        text: string;
+        projectorMs?: number;
+        decodeMs?: number;
+    }>;
+    /** Persist a slot's KV cache to disk under the conversation directory. */
+    persistConversationKv?(conversationId: string, slotId: number): Promise<void>;
+    /** Restore a slot's KV cache from disk into the running backend. */
+    restoreConversationKv?(conversationId: string, slotId: number): Promise<boolean>;
+    /**
+     * Pre-decode `promptPrefix` into the named slot/cache key so the next
+     * `generate` against the same key skips re-prefill. Returns false when
+     * no warmup happened (already cached, no model loaded, etc).
+     */
+    prewarmConversation?(promptPrefix: string, opts: {
+        slotId: number;
+        cacheKey: string;
+    }): Promise<boolean>;
+    /**
+     * Resize the backend's parallel slot pool. Returns true on a real
+     * restart/resize, false when no resize was needed (target ≤ current, etc).
+     */
+    resizeParallel?(target: number): Promise<boolean>;
+    /** Active parallel slot count. Default `1` on backends without pooling. */
+    parallelSlots?(): number;
+    /** True when native MTP speculative decoding is enabled. */
+    mtpEnabled?(): boolean;
+    /** Absolute path to the loaded mmproj (vision) GGUF, or null. */
+    currentMmprojPath?(): string | null;
+    /**
+     * Snapshot of the backend's current load configuration (ctx, cache
+     * types, parallel, binary path). Used by engine introspection +
+     * /api/local-inference/active.
+     */
+    currentRuntimeLoadConfig?(): LocalRuntimeLoadConfig | null;
+}
+export type BackendOverride = "auto" | "llama-cpp" | "litert-lm";
+/**
+ * The env name the C-side `llm_backend_select` reads to HARD-select an
+ * in-process runtime. The dispatcher sets it to `litert-lm` for a LiteRT load
+ * and clears it for a llama.cpp load so a prior LiteRT select never leaks into
+ * the next GGUF load. Mirrors `tools/omnivoice/src/llm-backend.h`.
+ */
+export declare const ELIZA_LLM_BACKEND_ENV: "ELIZA_LLM_BACKEND";
+export declare function readBackendOverride(): BackendOverride;
+/**
+ * Whether the LiteRT-LM in-process backend is usable on THIS build/platform.
+ * The C-side `LlmBackendFactory::available()` is the runtime authority (it is
+ * compiled in only under `-DELIZA_ENABLE_LITERT` and reports false when the
+ * NPU/GPU delegate is absent), but the TS dispatcher must decide *before* the
+ * FFI load whether to route there at all, so we gate on the same signals the
+ * build/launcher exports:
+ *
+ *   - `ELIZA_ENABLE_LITERT=1` — the explicit opt-in the LiteRT-enabled build
+ *     sets (matches the `-DELIZA_ENABLE_LITERT` CMake gate).
+ *   - `ELIZA_PLATFORM=android` — the NPU/GPU-delegate target where a LiteRT
+ *     `.litertlm` bundle is the on-device path.
+ *
+ * A bundle that ships a `.litertlm` but runs on a build without LiteRT support
+ * loads the GGUF (`llama-cpp`) instead — the artifact is additive, never a
+ * requirement. Returns false unless one of the signals is present, so GGUF
+ * stays the default everywhere LiteRT is not wired.
+ */
+export declare function litertBackendSupported(env?: NodeJS.ProcessEnv): boolean;
+/**
+ * Opt-in "reduced-optimization local mode" (the cross-platform escape hatch
+ * documented in `docs/voice-interactive.md` and `packages/inference/AGENTS.md`
+ * §4): when the installed llama.cpp runtime does not advertise the
+ * custom Eliza-1 KV kernels (`turbo3`/`qjl_full`/`polarquant`/…) — i.e. the
+ * fork hasn't been built with those kernels dispatched on this backend yet —
+ * setting `ELIZA_LOCAL_ALLOW_STOCK_KV=1` lets the model load anyway with
+ * stock `f16` KV cache instead of hard-refusing. The voice pipeline runs;
+ * it just runs without the KV-compression speedups on that backend. A loud
+ * one-time warning is emitted (see `warnReducedOptimizationLocalMode`).
+ *
+ * §3-vs-"works everywhere" reconciliation: AGENTS.md §3 says these kernels
+ * are *mandatory* and there is *no* "fallback to unoptimized" path. The
+ * user's directive for SA-1 is "works everywhere regardless of GPU". The
+ * reconciliation: the kernels DO build on every backend where they can be
+ * dispatched (Metal, CUDA, Vulkan-source-patched, CPU SIMD TUs), and this
+ * fallback is the *opt-in*, *loudly-warned*, *non-publishable* mode for the
+ * backends where dispatch isn't wired yet — it is not a silent downgrade,
+ * and `defaultEligible` bundles still require the verified kernels.
+ */
+export declare function localAllowStockKv(): boolean;
+export declare function warnReducedOptimizationLocalMode(detail: string): void;
+/** Reset the one-time warning latch (tests only). */
+export declare function __resetReducedModeWarnedForTests(): void;
+export interface BackendDecision {
+    /**
+     * In-process runtime the dispatcher routes this load to. `llama-cpp` (the
+     * GGUF path) is the default; `litert-lm` is selected only when the bundle
+     * ships a `.litertlm` AND the build/platform supports LiteRT (or it was
+     * forced via `ELIZA_INFERENCE_BACKEND=litert-lm`). Both run through the same
+     * fused `libelizainference` FFI — the selection only changes the env the
+     * C-side `llm_backend_select` reads.
+     */
+    backend: BackendId;
+    /** Why this backend was chosen — for diagnostics and warnings. */
+    reason: "env-override" | "kernel-required" | "preferred-backend" | "litert-artifact" | "default";
+    /**
+     * Absolute path to the selected `.litertlm` artifact when `backend ===
+     * "litert-lm"`, else undefined. The dispatcher exports
+     * `ELIZA_LLM_BACKEND=litert-lm` for this load so the FFI picks the LiteRT
+     * factory; the path is surfaced for diagnostics.
+     */
+    litertModelPath?: string;
+    /** Required kernels declared by the catalog, when any. */
+    kernels: LocalRuntimeKernel[];
+    /**
+     * Set when the dispatcher detected a kernel mismatch — the catalog model
+     * declares `requiresKernel: [...]` but CAPABILITIES.json next to the
+     * installed binary reports those kernels as unavailable. The dispatcher
+     * still routes to optimized llama.cpp (the only backend that could satisfy
+     * those kernels), but the load is expected to fail; the caller should
+     * surface this to the operator with a clear "rebuild your binary"
+     * message instead of letting the model silently misbehave.
+     */
+    unsatisfiedKernels?: LocalRuntimeKernel[];
+}
+/**
+ * Pure decision function. Easy to unit-test without spawning anything.
+ *
+ * Inputs are deliberately explicit — the caller resolves the catalog entry,
+ * the binary availability, the env override, and (for LiteRT) the staged
+ * `.litertlm` path + the build/platform support flag before calling us.
+ *
+ * `binaryKernels`, when present, is the parsed CAPABILITIES.json kernels
+ * map from the installed llama.cpp FFI runtime. The dispatcher uses it to
+ * compute `unsatisfiedKernels`; null means the binary is older / has no
+ * capabilities probe, in which case we trust the model's declaration and
+ * let the load attempt clarify.
+ *
+ * `litertModelPath` is the absolute path to a `.litertlm` text artifact when
+ * the bundle ships one (else undefined); `litertSupported` is whether this
+ * build/platform can run LiteRT ({@link litertBackendSupported}). LiteRT is
+ * selected only when BOTH hold, or when forced via
+ * `ELIZA_INFERENCE_BACKEND=litert-lm` (a forced LiteRT select with no
+ * `.litertlm` or no support throws — no silent downgrade to GGUF). GGUF stays
+ * the default in every other case.
+ */
+export declare function decideBackend(input: {
+    override: BackendOverride;
+    catalog: CatalogModel | undefined;
+    llamaCppAvailable: boolean;
+    binaryKernels?: Partial<Record<LocalRuntimeKernel | string, boolean>> | null;
+    litertModelPath?: string | null;
+    litertSupported?: boolean;
+}): BackendDecision;
+/**
+ * Resolve the catalog entry for a `BackendPlan`. Plans may carry the entry
+ * already (when the caller has it on hand), reference it by id, or carry
+ * neither — in which case the dispatcher falls back to the default backend.
+ */
+export declare function resolveCatalogForPlan(plan: BackendPlan): CatalogModel | undefined;
+/**
+ * Dispatcher that fronts the in-process FFI llama.cpp backend behind the
+ * `LocalInferenceBackend` contract. Holds at most one active backend at a
+ * time — load() unloads the previous backend before loading the new one if
+ * they differ.
+ */
+export declare class BackendDispatcher implements LocalInferenceBackend {
+    private readonly ffiStreaming;
+    private readonly probeFfiAvailable;
+    /**
+     * Optional capabilities probe that returns the kernels map from the
+     * installed llama.cpp FFI runtime, or null when no probe is available.
+     * Used to flag `unsatisfiedKernels`
+     * in the BackendDecision before load() so callers can give a clean
+     * "rebuild your fork binary" error instead of a kernel SIGSEGV at
+     * generation time.
+     */
+    private readonly probeBinaryKernels?;
+    readonly id: "llama-cpp";
+    private active;
+    constructor(ffiStreaming: LocalInferenceBackend, probeFfiAvailable: () => boolean, 
+    /**
+     * Optional capabilities probe that returns the kernels map from the
+     * installed llama.cpp FFI runtime, or null when no probe is available.
+     * Used to flag `unsatisfiedKernels`
+     * in the BackendDecision before load() so callers can give a clean
+     * "rebuild your fork binary" error instead of a kernel SIGSEGV at
+     * generation time.
+     */
+    probeBinaryKernels?: (() => Partial<Record<string, boolean>> | null) | undefined);
+    available(): Promise<boolean>;
+    activeBackendId(): "llama-cpp" | null;
+    hasLoadedModel(): boolean;
+    currentModelPath(): string | null;
+    decide(plan: BackendPlan): BackendDecision;
+    load(plan: BackendPlan): Promise<void>;
+    unload(): Promise<void>;
+    generate(args: GenerateArgs): Promise<GenerateResult>;
+    generateWithUsage(args: GenerateArgs & {
+        slotId?: number;
+    }): Promise<LocalGenerateWithUsageResult>;
+    describeImage(args: Parameters<NonNullable<LocalInferenceBackend["describeImage"]>>[0]): ReturnType<NonNullable<LocalInferenceBackend["describeImage"]>>;
+    persistConversationKv(conversationId: string, slotId: number): Promise<void>;
+    restoreConversationKv(conversationId: string, slotId: number): Promise<boolean>;
+    prewarmConversation(promptPrefix: string, opts: {
+        slotId: number;
+        cacheKey: string;
+    }): Promise<boolean>;
+    resizeParallel(target: number): Promise<boolean>;
+    parallelSlots(): number;
+    mtpEnabled(): boolean;
+    currentMmprojPath(): string | null;
+    currentRuntimeLoadConfig(): LocalRuntimeLoadConfig | null;
+    private ensureLoaded;
+    private notSupported;
+}
+//# sourceMappingURL=backend.d.ts.map

package/dist/services/backend.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"backend.d.ts","sourceRoot":"","sources":["../../src/services/backend.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAGH,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,KAAK,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAChE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAEzD;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC;IACpC,SAAS,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,OAAO,GAAG;QAAE,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5D,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,8EAA8E;IAC9E,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,uEAAuE;IACvE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,+EAA+E;IAC/E,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,yEAAyE;IACzE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;OAKG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,WAAW;IAC3B,yCAAyC;IACzC,SAAS,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,0DAA0D;IAC1D,OAAO,CAAC,EAAE,YAAY,CAAC;IACvB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,oBAAoB,CAAC;CACjC;AAED,MAAM,WAAW,YAAa,SAAQ,wBAAwB;IAC7D,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,sDAAsD;IACtD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yBAAyB;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yCAAyC;IACzC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;OAKG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;OAIG;IACH,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACtD;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;OAGG;IACH,WAAW,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC;IAC1C;;;OAGG;IACH,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACvE;AAED,MAAM,MAAM,cAAc,GAAG,MAAM,CAAC;AAEpC,MAAM,WAAW,4BAA4B;IAC5C,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE;QACP,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACvB,CAAC;IACF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,EAAE;QACV,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,EAAE,MAAM,CAAC;QACjB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;KAC9B,CAAC;CACF;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,SAAS,GAAG,WAAW,GAAG,WAAW,CAAC;AAElD,MAAM,WAAW,sBAAsB;IACtC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,OAAO,EAAE,SAAS,GAAG,IAAI,CAAC;IAC1B,GAAG,EAAE;QACJ,QAAQ,EAAE,WAAW,CAAC;QACtB,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;KACjB,GAAG,IAAI,CAAC;CACT;AAED;;;;;;GAMG;AACH,MAAM,WAAW,qBAAqB;IACrC,0DAA0D;IAC1D,QAAQ,CAAC,EAAE,EAAE,WAAW,CAAC;IACzB,SAAS,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAC9B,IAAI,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IACtD,cAAc,IAAI,OAAO,CAAC;IAC1B,gBAAgB,IAAI,MAAM,GAAG,IAAI,CAAC;IAUlC;;;OAGG;IACH,iBAAiB,CAAC,CACjB,IAAI,EAAE,YAAY,GAAG;QAAE,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GACtC,OAAO,CAAC,4BAA4B,CAAC,CAAC;IAEzC,qEAAqE;IACrE,aAAa,CAAC,CAAC,IAAI,EAAE;QACpB,KAAK,EAAE,UAAU,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB;;;8CAGsC;QACtC,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QACtD,gBAAgB,CAAC,EAAE,MAAM,CAAC;KAC1B,GAAG,OAAO,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC,CAAC;IAEH,0EAA0E;IAC1E,qBAAqB,CAAC,CAAC,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9E,oEAAoE;IACpE,qBAAqB,CAAC,CACrB,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,MAAM,GACZ,OAAO,CAAC,OAAO,CAAC,CAAC;IAEpB;;;;OAIG;IACH,mBAAmB,CAAC,CACnB,YAAY,EAAE,MAAM,EACpB,IAAI,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,GACxC,OAAO,CAAC,OAAO,CAAC,CAAC;IAEpB;;;OAGG;IACH,cAAc,CAAC,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAElD,2EAA2E;IAC3E,aAAa,CAAC,IAAI,MAAM,CAAC;IAEzB,4DAA4D;IAC5D,UAAU,CAAC,IAAI,OAAO,CAAC;IAEvB,iEAAiE;IACjE,iBAAiB,CAAC,IAAI,MAAM,GAAG,IAAI,CAAC;IAEpC;;;;OAIG;IACH,wBAAwB,CAAC,IAAI,sBAAsB,GAAG,IAAI,CAAC;CAC3D;AAED,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,WAAW,GAAG,WAAW,CAAC;AAEjE;;;;;GAKG;AACH,eAAO,MAAM,qBAAqB,EAAG,mBAA4B,CAAC;AAElE,wBAAgB,mBAAmB,IAAI,eAAe,CAUrD;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,sBAAsB,CACrC,GAAG,GAAE,MAAM,CAAC,UAAwB,GAClC,OAAO,CAGT;AAYD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,iBAAiB,IAAI,OAAO,CAE3C;AAGD,wBAAgB,gCAAgC,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAYrE;AAED,qDAAqD;AACrD,wBAAgB,gCAAgC,IAAI,IAAI,CAEvD;AAED,MAAM,WAAW,eAAe;IAC/B;;;;;;;OAOG;IACH,OAAO,EAAE,SAAS,CAAC;IACnB,kEAAkE;IAClE,MAAM,EACH,cAAc,GACd,iBAAiB,GACjB,mBAAmB,GACnB,iBAAiB,GACjB,SAAS,CAAC;IACb;;;;;OAKG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,0DAA0D;IAC1D,OAAO,EAAE,kBAAkB,EAAE,CAAC;IAC9B;;;;;;;;OAQG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,EAAE,CAAC;CAC1C;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE;IACpC,QAAQ,EAAE,eAAe,CAAC;IAC1B,OAAO,EAAE,YAAY,GAAG,SAAS,CAAC;IAClC,iBAAiB,EAAE,OAAO,CAAC;IAC3B,aAAa,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,kBAAkB,GAAG,MAAM,EAAE,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAC7E,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,eAAe,CAAC,EAAE,OAAO,CAAC;CAC1B,GAAG,eAAe,CA4ElB;AAgBD;;;;GAIG;AACH,wBAAgB,qBAAqB,CACpC,IAAI,EAAE,WAAW,GACf,YAAY,GAAG,SAAS,CAI1B;AAED;;;;;GAKG;AACH,qBAAa,iBAAkB,YAAW,qBAAqB;IAQ7D,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,iBAAiB;IAClC;;;;;;;OAOG;IACH,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IAjBrC,QAAQ,CAAC,EAAE,EAAG,WAAW,CAAU;IAInC,OAAO,CAAC,MAAM,CAAsC;gBAGlC,YAAY,EAAE,qBAAqB,EACnC,iBAAiB,EAAE,MAAM,OAAO;IACjD;;;;;;;OAOG;IACc,kBAAkB,CAAC,GAAE,MAAM,OAAO,CAClD,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CACvB,GAAG,IAAI,aAAA;IAGH,SAAS,IAAI,OAAO,CAAC,OAAO,CAAC;IAInC,eAAe,IAAI,WAAW,GAAG,IAAI;IAIrC,cAAc,IAAI,OAAO;IAIzB,gBAAgB,IAAI,MAAM,GAAG,IAAI;IAIjC,MAAM,CAAC,IAAI,EAAE,WAAW,GAAG,eAAe;IAYpC,IAAI,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IA6DtC,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAMvB,QAAQ,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,cAAc,CAAC;IAgBrD,iBAAiB,CACtB,IAAI,EAAE,YAAY,GAAG;QAAE,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GACtC,OAAO,CAAC,4BAA4B,CAAC;IAQlC,aAAa,CAClB,IAAI,EAAE,UAAU,CAAC,WAAW,CAAC,qBAAqB,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GACtE,UAAU,CAAC,WAAW,CAAC,qBAAqB,CAAC,eAAe,CAAC,CAAC,CAAC;IAW5D,qBAAqB,CAC1B,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,MAAM,GACZ,OAAO,CAAC,IAAI,CAAC;IAMV,qBAAqB,CAC1B,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,MAAM,GACZ,OAAO,CAAC,OAAO,CAAC;IAMb,mBAAmB,CACxB,YAAY,EAAE,MAAM,EACpB,IAAI,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,GACxC,OAAO,CAAC,OAAO,CAAC;IAMb,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAMtD,aAAa,IAAI,MAAM;IAIvB,UAAU,IAAI,OAAO;IAIrB,iBAAiB,IAAI,MAAM,GAAG,IAAI;IAIlC,wBAAwB,IAAI,sBAAsB,GAAG,IAAI;IAIzD,OAAO,CAAC,YAAY;IAQpB,OAAO,CAAC,YAAY;CAIpB"}

package/dist/services/bionic-host-loader.d.ts

@@ -0,0 +1,67 @@
+/**
+ * BionicHostLoader — the agent-side half of the on-device GPU delegation path.
+ *
+ * On Android the elizaOS agent runs as embedded bun under the musl loader, whose
+ * restricted linker namespace cannot load the bionic Android Vulkan driver (its
+ * HIDL/HAL closure) — so the musl agent can only run inference on the CPU. The
+ * GPU is reachable only from the normal bionic `ai.elizaos.app` process, where
+ * `ElizaBionicInferenceServer` (Java) has loaded `libelizainference.so` +
+ * `libggml-vulkan.so` and offloads the model to the Mali GPU.
+ *
+ * This loader implements the standard {@link LocalInferenceLoader} contract, so
+ * the TEXT_SMALL / TEXT_LARGE handlers in `ensure-local-inference-handler.ts`
+ * route through it transparently. `generate()` sends the prompt to the bionic
+ * host over an abstract-namespace `AF_UNIX` socket and gets the GPU completion
+ * back — the whole decode loop runs server-side, so there is no per-token
+ * two-process round trip.
+ *
+ * This is the buffered first slice (one GENERATE request → one full completion).
+ * Server-push per-step streaming, embed, and cancel are layered on later via the
+ * shared `LlmStreamingBinding`; the wire framing already carries an `op`
+ * discriminator for that.
+ */
+import type { LocalInferenceLoadArgs, LocalInferenceLoader } from "./active-model";
+export declare class BionicHostLoader implements LocalInferenceLoader {
+    private readonly socketName;
+    private modelPath;
+    private bundleDir;
+    /** @param socketName abstract-namespace socket name (no leading NUL). */
+    constructor(socketName: string);
+    loadModel(args: LocalInferenceLoadArgs): Promise<void>;
+    unloadModel(): Promise<void>;
+    currentModelPath(): string | null;
+    generate(args: {
+        prompt: string;
+        stopSequences?: string[];
+        maxTokens?: number;
+        temperature?: number;
+        cacheKey?: string;
+    }): Promise<string>;
+    /**
+     * On-device STT: transcribe mono fp32 PCM via the bionic host's fused
+     * Gemma ASR path (op="asr"). The musl agent can't load the fused lib, so
+     * the TRANSCRIPTION delegate routes the audio here over the UDS and gets
+     * the transcript back. `pcm` is little-endian fp32 already base64-encoded.
+     */
+    transcribe(args: {
+        pcmBase64: string;
+        sampleRate: number;
+    }): Promise<string>;
+    /**
+     * On-device vision / screen-recognition: describe a raw image (PNG/JPEG/WebP
+     * bytes, base64) via the bionic host's mmproj describe-image (op="image").
+     * `mmprojPath` may be empty — the host resolves the projector from the
+     * bundle's `vision/` dir.
+     */
+    describeImage(args: {
+        imageBase64: string;
+        mmprojPath?: string;
+        prompt?: string;
+    }): Promise<string>;
+    /**
+     * One request → one response over a fresh connection. Length-prefixed frames:
+     * `[int32 BE byte length][UTF-8 JSON]` in each direction.
+     */
+    private roundTrip;
+}
+//# sourceMappingURL=bionic-host-loader.d.ts.map

package/dist/services/bionic-host-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bionic-host-loader.d.ts","sourceRoot":"","sources":["../../src/services/bionic-host-loader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAKH,OAAO,KAAK,EACX,sBAAsB,EACtB,oBAAoB,EACpB,MAAM,gBAAgB,CAAC;AAwCxB,qBAAa,gBAAiB,YAAW,oBAAoB;IAKhD,OAAO,CAAC,QAAQ,CAAC,UAAU;IAJvC,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,SAAS,CAAM;IAEvB,yEAAyE;gBAC5C,UAAU,EAAE,MAAM;IAEzC,SAAS,CAAC,IAAI,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC;IAQtD,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAIlC,gBAAgB,IAAI,MAAM,GAAG,IAAI;IAI3B,QAAQ,CAAC,IAAI,EAAE;QACpB,MAAM,EAAE,MAAM,CAAC;QACf,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;QACzB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC,MAAM,CAAC;IAqBnB;;;;;OAKG;IACG,UAAU,CAAC,IAAI,EAAE;QACtB,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;KACnB,GAAG,OAAO,CAAC,MAAM,CAAC;IA0BnB;;;;;OAKG;IACG,aAAa,CAAC,IAAI,EAAE;QACzB,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,MAAM,CAAC,EAAE,MAAM,CAAC;KAChB,GAAG,OAAO,CAAC,MAAM,CAAC;IAgBnB;;;OAGG;IACH,OAAO,CAAC,SAAS;CAmEjB"}

package/dist/services/bundled-models.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Bundled-models bootstrap for AOSP / on-device installs.
+ *
+ * The AOSP build pipeline stages Eliza-1 models into the APK at
+ * `assets/agent/models/{file}.gguf` plus a
+ * `manifest.json` describing each one (id, role, sha256, sizeBytes).
+ * `ElizaAgentService.extractAssetsIfNeeded()` copies those files into
+ * `$ELIZA_STATE_DIR/local-inference/models/` on first launch.
+ *
+ * This module reads the manifest at runtime startup and registers each
+ * file as a eliza-owned model in the local-inference registry, so the
+ * auto-assign pass picks them up for TEXT_LARGE / TEXT_SMALL /
+ * TEXT_EMBEDDING slots without needing the user to download anything.
+ *
+ * Idempotent: re-running with the registry already populated is a
+ * no-op for unchanged entries (`upsertElizaModel` overwrites entries
+ * with the same id, so updated sha256s on a later re-bundle replace
+ * the old metadata cleanly).
+ *
+ * Source classification: the runtime treats bundled models as
+ * `source: "eliza-download"` because Eliza ships the file and Eliza
+ * owns it on disk — same lifecycle as a user-initiated download
+ * (uninstall removes the file, the registry tracks the install). The
+ * only difference is the file arrived via APK extraction rather than
+ * an HTTP transfer.
+ */
+/**
+ * Walk the manifest and register every bundled GGUF file in the
+ * local-inference registry. Returns the number of entries successfully
+ * registered. A missing manifest is normal on Capacitor / desktop /
+ * non-AOSP installs and returns 0 silently.
+ */
+export declare function registerBundledModels(): Promise<number>;
+//# sourceMappingURL=bundled-models.d.ts.map

package/dist/services/bundled-models.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bundled-models.d.ts","sourceRoot":"","sources":["../../src/services/bundled-models.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AA2DH;;;;;GAKG;AACH,wBAAsB,qBAAqB,IAAI,OAAO,CAAC,MAAM,CAAC,CAsC7D"}