@elizaos/app-core 2.0.0-alpha.85 → 2.0.0-beta.2

package/services/local-inference/backend.d.ts

@@ -0,0 +1,200 @@
+/**
+ * Unified local-inference backend interface and dispatcher.
+ *
+ * Two real implementations live behind this interface:
+ *
+ *   - `node-llama-cpp`  → in-process via the node-llama-cpp binding. Stock
+ *     GGUFs, no drafter, no MoE expert offload, no `--lookahead`. Fastest
+ *     to start, lowest IPC overhead, narrowest feature surface.
+ *   - `llama-server`    → out-of-process llama-server (the buun-llama-cpp
+ *     fork). Full optimization surface — DFlash, n-gram drafter, lookahead,
+ *     `-ot` MoE offload, TurboQuant KV cache, mlock/no-mmap/mmproj, etc.
+ *
+ * The dispatcher decides which one to use per-load based on:
+ *
+ *   1. `ELIZA_LOCAL_BACKEND=node-llama-cpp|llama-server|auto` — operator
+ *      override. `auto` (the default) hands the decision to the catalog.
+ *   2. Catalog `runtime.optimizations.requiresKernel` — if any specialised
+ *      llama-server kernel is required (e.g. `dflash`, `turbo3`), the
+ *      dispatcher MUST pick `llama-server`. The in-process binding cannot
+ *      provide these kernels at all.
+ *   3. Catalog `runtime.preferredBackend` — soft preference. We pick
+ *      `llama-server` when this is set AND the binary is available;
+ *      otherwise we fall back to `node-llama-cpp` unless DFlash is
+ *      explicitly required (`ELIZA_DFLASH_REQUIRED=1`).
+ *   4. Default: `node-llama-cpp` for stock GGUFs without runtime metadata.
+ *
+ * The dispatcher does NOT own the spawn body — `llama-server` and the
+ * node binding own that. It owns selection only, plus a small load-state
+ * cache so callers can swap models without touching either backend
+ * directly.
+ */
+import type { CatalogModel, LocalRuntimeKernel } from "./types";
+/**
+ * Per-load runtime overrides forwarded by the dispatcher to whichever
+ * backend handles the load. Mirror of the relevant fields on
+ * `LocalInferenceLoadArgs` from `active-model.ts` — kept inline here so
+ * `backend.ts` stays free of cross-file circular imports (active-model
+ * imports engine, engine imports backend).
+ */
+export interface BackendLoadOverrides {
+    contextSize?: number;
+    cacheTypeK?: string;
+    cacheTypeV?: string;
+    gpuLayers?: number | "auto" | "max";
+    flashAttention?: boolean;
+    mmap?: boolean;
+    mlock?: boolean;
+    useGpu?: boolean;
+}
+export interface BackendPlan {
+    /** Absolute path to the GGUF on disk. */
+    modelPath: string;
+    /**
+     * Catalog model id, when known. The dispatcher uses this to pull
+     * `runtime.optimizations` and `runtime.dflash` — without it, we can
+     * only honour the env override and fall back to `node-llama-cpp`.
+     */
+    modelId?: string;
+    /** Catalog entry, when the caller already resolved it. */
+    catalog?: CatalogModel;
+    /**
+     * Per-load runtime overrides resolved by the active-model coordinator.
+     * The dispatcher passes these through verbatim to the chosen backend
+     * so the in-process binding can honour cache-type and contextSize
+     * requests instead of silently dropping them.
+     */
+    overrides?: BackendLoadOverrides;
+}
+export interface GenerateArgs {
+    prompt: string;
+    stopSequences?: string[];
+    /** Upper bound on output tokens; defaults to 2048. */
+    maxTokens?: number;
+    /** 0..1; 0.7 default. */
+    temperature?: number;
+    /** Nucleus sampling; defaults to 0.9. */
+    topP?: number;
+    /**
+     * Optional cache key from the runtime's `ProviderCachePlan`. Identical
+     * keys reuse the same KV cache prefix in both backends:
+     *   - `node-llama-cpp` → routes to a pooled `LlamaChatSession` that
+     *     retains chat history (and therefore the KV) across calls.
+     *   - `llama-server`   → derives a deterministic `slot_id` so requests
+     *     with the same key land on the same persisted slot.
+     * Empty / absent keys fall through to the historical stateless path.
+     */
+    cacheKey?: string;
+}
+export type GenerateResult = string;
+export interface EmbedArgs {
+    input: string;
+}
+export interface EmbedResult {
+    embedding: number[];
+    tokens: number;
+}
+/**
+ * The backend contract every local-inference implementation satisfies.
+ *
+ * `available()` is a soft probe — it should NOT spawn anything; it just
+ * reports whether the backend can be used at all (e.g. is the binding
+ * loadable, is the binary on disk). Loading a specific model is `load()`.
+ */
+export interface LocalInferenceBackend {
+    /** Identifier — `"node-llama-cpp"` or `"llama-server"`. */
+    readonly id: "node-llama-cpp" | "llama-server";
+    available(): Promise<boolean>;
+    load(plan: BackendPlan): Promise<void>;
+    unload(): Promise<void>;
+    generate(args: GenerateArgs): Promise<GenerateResult>;
+    embed?(args: EmbedArgs): Promise<EmbedResult>;
+    hasLoadedModel(): boolean;
+    currentModelPath(): string | null;
+}
+export type BackendOverride = "auto" | "node-llama-cpp" | "llama-server";
+export declare function readBackendOverride(): BackendOverride;
+export interface BackendDecision {
+    backend: "node-llama-cpp" | "llama-server";
+    /** Why this backend was chosen — for diagnostics and warnings. */
+    reason: "env-override" | "kernel-required" | "dflash-required" | "preferred-backend" | "default";
+    /** Required kernels declared by the catalog, when any. */
+    kernels: LocalRuntimeKernel[];
+    /**
+     * Set when the dispatcher detected a kernel mismatch — the catalog model
+     * declares `requiresKernel: [...]` but CAPABILITIES.json next to the
+     * installed binary reports those kernels as unavailable. The dispatcher
+     * still routes to llama-server (the only backend that could ever satisfy
+     * those kernels), but the load is expected to fail; the caller should
+     * surface this to the operator with a clear "rebuild your binary"
+     * message instead of letting the model silently misbehave.
+     */
+    unsatisfiedKernels?: LocalRuntimeKernel[];
+}
+/**
+ * Pure decision function. Easy to unit-test without spawning anything.
+ *
+ * Inputs are deliberately explicit — the caller resolves the catalog entry,
+ * the binary availability, and the env override before calling us.
+ *
+ * `binaryKernels`, when present, is the parsed CAPABILITIES.json kernels
+ * map from the installed llama-server build. The dispatcher uses it to
+ * compute `unsatisfiedKernels`; null means the binary is older / has no
+ * capabilities probe, in which case we trust the model's declaration and
+ * let the load attempt clarify.
+ */
+export declare function decideBackend(input: {
+    override: BackendOverride;
+    catalog: CatalogModel | undefined;
+    llamaServerAvailable: boolean;
+    dflashRequired: boolean;
+    binaryKernels?: Partial<Record<LocalRuntimeKernel | string, boolean>> | null;
+}): BackendDecision;
+/**
+ * Resolve the catalog entry for a `BackendPlan`. Plans may carry the entry
+ * already (when the caller has it on hand), reference it by id, or carry
+ * neither — in which case the dispatcher falls back to the default backend.
+ */
+export declare function resolveCatalogForPlan(plan: BackendPlan): CatalogModel | undefined;
+/**
+ * Dispatcher that fronts both backends behind the `LocalInferenceBackend`
+ * contract. Holds at most one active backend at a time — load() unloads
+ * the previous backend before loading the new one if they differ.
+ */
+export declare class BackendDispatcher implements LocalInferenceBackend {
+    private readonly nodeLlamaCpp;
+    private readonly llamaServer;
+    private readonly probeLlamaServerAvailable;
+    private readonly probeDflashRequired;
+    /**
+     * Optional capabilities probe that returns the kernels map from
+     * CAPABILITIES.json next to the installed llama-server binary, or
+     * null when the file is absent. Used to flag `unsatisfiedKernels`
+     * in the BackendDecision before load() so callers can give a clean
+     * "rebuild your fork binary" error instead of a kernel SIGSEGV at
+     * generation time.
+     */
+    private readonly probeBinaryKernels?;
+    readonly id: "node-llama-cpp";
+    private active;
+    constructor(nodeLlamaCpp: LocalInferenceBackend, llamaServer: LocalInferenceBackend, probeLlamaServerAvailable: () => boolean, probeDflashRequired: () => boolean, 
+    /**
+     * Optional capabilities probe that returns the kernels map from
+     * CAPABILITIES.json next to the installed llama-server binary, or
+     * null when the file is absent. Used to flag `unsatisfiedKernels`
+     * in the BackendDecision before load() so callers can give a clean
+     * "rebuild your fork binary" error instead of a kernel SIGSEGV at
+     * generation time.
+     */
+    probeBinaryKernels?: (() => Partial<Record<string, boolean>> | null) | undefined);
+    available(): Promise<boolean>;
+    activeBackendId(): "node-llama-cpp" | "llama-server" | null;
+    hasLoadedModel(): boolean;
+    currentModelPath(): string | null;
+    decide(plan: BackendPlan): BackendDecision;
+    load(plan: BackendPlan): Promise<void>;
+    unload(): Promise<void>;
+    generate(args: GenerateArgs): Promise<GenerateResult>;
+    embed(args: EmbedArgs): Promise<EmbedResult>;
+}
+//# sourceMappingURL=backend.d.ts.map

package/services/local-inference/backend.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"backend.d.ts","sourceRoot":"","sources":["../../../../../../src/services/local-inference/backend.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAEhE;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC;IACpC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,WAAW;IAC1B,yCAAyC;IACzC,SAAS,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,0DAA0D;IAC1D,OAAO,CAAC,EAAE,YAAY,CAAC;IACvB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,oBAAoB,CAAC;CAClC;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,sDAAsD;IACtD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yBAAyB;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yCAAyC;IACzC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;;;;;OAQG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,cAAc,GAAG,MAAM,CAAC;AAEpC,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,qBAAqB;IACpC,2DAA2D;IAC3D,QAAQ,CAAC,EAAE,EAAE,gBAAgB,GAAG,cAAc,CAAC;IAC/C,SAAS,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAC9B,IAAI,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IACtD,KAAK,CAAC,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAC9C,cAAc,IAAI,OAAO,CAAC;IAC1B,gBAAgB,IAAI,MAAM,GAAG,IAAI,CAAC;CACnC;AAED,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,gBAAgB,GAAG,cAAc,CAAC;AAEzE,wBAAgB,mBAAmB,IAAI,eAAe,CAMrD;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,gBAAgB,GAAG,cAAc,CAAC;IAC3C,kEAAkE;IAClE,MAAM,EACF,cAAc,GACd,iBAAiB,GACjB,iBAAiB,GACjB,mBAAmB,GACnB,SAAS,CAAC;IACd,0DAA0D;IAC1D,OAAO,EAAE,kBAAkB,EAAE,CAAC;IAC9B;;;;;;;;OAQG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,EAAE,CAAC;CAC3C;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE;IACnC,QAAQ,EAAE,eAAe,CAAC;IAC1B,OAAO,EAAE,YAAY,GAAG,SAAS,CAAC;IAClC,oBAAoB,EAAE,OAAO,CAAC;IAC9B,cAAc,EAAE,OAAO,CAAC;IACxB,aAAa,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,kBAAkB,GAAG,MAAM,EAAE,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;CAC9E,GAAG,eAAe,CA8ElB;AAgBD;;;;GAIG;AACH,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,WAAW,GAChB,YAAY,GAAG,SAAS,CAI1B;AAED;;;;GAIG;AACH,qBAAa,iBAAkB,YAAW,qBAAqB;IAQ3D,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,yBAAyB;IAC1C,OAAO,CAAC,QAAQ,CAAC,mBAAmB;IACpC;;;;;;;OAOG;IACH,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IAnBtC,QAAQ,CAAC,EAAE,EAAG,gBAAgB,CAAU;IAIxC,OAAO,CAAC,MAAM,CAAsC;gBAGjC,YAAY,EAAE,qBAAqB,EACnC,WAAW,EAAE,qBAAqB,EAClC,yBAAyB,EAAE,MAAM,OAAO,EACxC,mBAAmB,EAAE,MAAM,OAAO;IACnD;;;;;;;OAOG;IACc,kBAAkB,CAAC,GAAE,MAAM,OAAO,CACjD,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CACxB,GAAG,IAAI,aAAA;IAGJ,SAAS,IAAI,OAAO,CAAC,OAAO,CAAC;IAMnC,eAAe,IAAI,gBAAgB,GAAG,cAAc,GAAG,IAAI;IAI3D,cAAc,IAAI,OAAO;IAIzB,gBAAgB,IAAI,MAAM,GAAG,IAAI;IAIjC,MAAM,CAAC,IAAI,EAAE,WAAW,GAAG,eAAe;IAWpC,IAAI,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAmBtC,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAMvB,QAAQ,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,cAAc,CAAC;IASrD,KAAK,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC;CAanD"}

package/services/local-inference/backend.js

@@ -0,0 +1,242 @@
+/**
+ * Unified local-inference backend interface and dispatcher.
+ *
+ * Two real implementations live behind this interface:
+ *
+ *   - `node-llama-cpp`  → in-process via the node-llama-cpp binding. Stock
+ *     GGUFs, no drafter, no MoE expert offload, no `--lookahead`. Fastest
+ *     to start, lowest IPC overhead, narrowest feature surface.
+ *   - `llama-server`    → out-of-process llama-server (the buun-llama-cpp
+ *     fork). Full optimization surface — DFlash, n-gram drafter, lookahead,
+ *     `-ot` MoE offload, TurboQuant KV cache, mlock/no-mmap/mmproj, etc.
+ *
+ * The dispatcher decides which one to use per-load based on:
+ *
+ *   1. `ELIZA_LOCAL_BACKEND=node-llama-cpp|llama-server|auto` — operator
+ *      override. `auto` (the default) hands the decision to the catalog.
+ *   2. Catalog `runtime.optimizations.requiresKernel` — if any specialised
+ *      llama-server kernel is required (e.g. `dflash`, `turbo3`), the
+ *      dispatcher MUST pick `llama-server`. The in-process binding cannot
+ *      provide these kernels at all.
+ *   3. Catalog `runtime.preferredBackend` — soft preference. We pick
+ *      `llama-server` when this is set AND the binary is available;
+ *      otherwise we fall back to `node-llama-cpp` unless DFlash is
+ *      explicitly required (`ELIZA_DFLASH_REQUIRED=1`).
+ *   4. Default: `node-llama-cpp` for stock GGUFs without runtime metadata.
+ *
+ * The dispatcher does NOT own the spawn body — `llama-server` and the
+ * node binding own that. It owns selection only, plus a small load-state
+ * cache so callers can swap models without touching either backend
+ * directly.
+ */
+import { findCatalogModel } from "./catalog";
+export function readBackendOverride() {
+    const raw = process.env.ELIZA_LOCAL_BACKEND?.trim().toLowerCase();
+    if (raw === "node-llama-cpp" || raw === "llama-server" || raw === "auto") {
+        return raw;
+    }
+    return "auto";
+}
+/**
+ * Pure decision function. Easy to unit-test without spawning anything.
+ *
+ * Inputs are deliberately explicit — the caller resolves the catalog entry,
+ * the binary availability, and the env override before calling us.
+ *
+ * `binaryKernels`, when present, is the parsed CAPABILITIES.json kernels
+ * map from the installed llama-server build. The dispatcher uses it to
+ * compute `unsatisfiedKernels`; null means the binary is older / has no
+ * capabilities probe, in which case we trust the model's declaration and
+ * let the load attempt clarify.
+ */
+export function decideBackend(input) {
+    const { override, catalog, llamaServerAvailable, dflashRequired } = input;
+    const optimizations = catalog?.runtime?.optimizations;
+    const kernels = optimizations?.requiresKernel ?? [];
+    const dflashConfigured = catalog?.runtime?.dflash;
+    const preferredBackend = catalog?.runtime?.preferredBackend;
+    const unsatisfiedKernels = computeUnsatisfiedKernels(kernels, input.binaryKernels ?? null);
+    if (override === "node-llama-cpp") {
+        if (kernels.length > 0 || dflashRequired) {
+            // The override conflicts with a hard requirement. Prefer the kernel
+            // requirement — silently honoring the override would silently break
+            // the model. Surface as a llama-server pick; the load itself will
+            // fail clearly if the binary really is missing.
+            return {
+                backend: "llama-server",
+                reason: "kernel-required",
+                kernels,
+                unsatisfiedKernels,
+            };
+        }
+        return {
+            backend: "node-llama-cpp",
+            reason: "env-override",
+            kernels,
+            unsatisfiedKernels,
+        };
+    }
+    if (override === "llama-server") {
+        return {
+            backend: "llama-server",
+            reason: "env-override",
+            kernels,
+            unsatisfiedKernels,
+        };
+    }
+    if (kernels.length > 0) {
+        return {
+            backend: "llama-server",
+            reason: "kernel-required",
+            kernels,
+            unsatisfiedKernels,
+        };
+    }
+    if (dflashConfigured && dflashRequired) {
+        return {
+            backend: "llama-server",
+            reason: "dflash-required",
+            kernels,
+            unsatisfiedKernels,
+        };
+    }
+    if (preferredBackend === "llama-server" && llamaServerAvailable) {
+        return {
+            backend: "llama-server",
+            reason: "preferred-backend",
+            kernels,
+            unsatisfiedKernels,
+        };
+    }
+    if (preferredBackend === "node-llama-cpp") {
+        return {
+            backend: "node-llama-cpp",
+            reason: "preferred-backend",
+            kernels,
+            unsatisfiedKernels,
+        };
+    }
+    return {
+        backend: "node-llama-cpp",
+        reason: "default",
+        kernels,
+        unsatisfiedKernels,
+    };
+}
+/**
+ * Returns the subset of `required` kernels that aren't reported as `true`
+ * in the binary's CAPABILITIES.json. Returns undefined when no probe is
+ * available; an empty array means "all required kernels are satisfied".
+ */
+function computeUnsatisfiedKernels(required, binaryKernels) {
+    if (required.length === 0)
+        return undefined;
+    if (!binaryKernels)
+        return undefined;
+    return required.filter((k) => binaryKernels[k] !== true);
+}
+/**
+ * Resolve the catalog entry for a `BackendPlan`. Plans may carry the entry
+ * already (when the caller has it on hand), reference it by id, or carry
+ * neither — in which case the dispatcher falls back to the default backend.
+ */
+export function resolveCatalogForPlan(plan) {
+    if (plan.catalog)
+        return plan.catalog;
+    if (plan.modelId)
+        return findCatalogModel(plan.modelId);
+    return undefined;
+}
+/**
+ * Dispatcher that fronts both backends behind the `LocalInferenceBackend`
+ * contract. Holds at most one active backend at a time — load() unloads
+ * the previous backend before loading the new one if they differ.
+ */
+export class BackendDispatcher {
+    nodeLlamaCpp;
+    llamaServer;
+    probeLlamaServerAvailable;
+    probeDflashRequired;
+    probeBinaryKernels;
+    id = "node-llama-cpp";
+    // The dispatcher's `id` is informational; the active backend's id is what
+    // matters for diagnostics. We expose `activeBackendId()` for that.
+    active = null;
+    constructor(nodeLlamaCpp, llamaServer, probeLlamaServerAvailable, probeDflashRequired, 
+    /**
+     * Optional capabilities probe that returns the kernels map from
+     * CAPABILITIES.json next to the installed llama-server binary, or
+     * null when the file is absent. Used to flag `unsatisfiedKernels`
+     * in the BackendDecision before load() so callers can give a clean
+     * "rebuild your fork binary" error instead of a kernel SIGSEGV at
+     * generation time.
+     */
+    probeBinaryKernels) {
+        this.nodeLlamaCpp = nodeLlamaCpp;
+        this.llamaServer = llamaServer;
+        this.probeLlamaServerAvailable = probeLlamaServerAvailable;
+        this.probeDflashRequired = probeDflashRequired;
+        this.probeBinaryKernels = probeBinaryKernels;
+    }
+    async available() {
+        const a = await this.nodeLlamaCpp.available();
+        if (a)
+            return true;
+        return this.llamaServer.available();
+    }
+    activeBackendId() {
+        return this.active ? this.active.id : null;
+    }
+    hasLoadedModel() {
+        return this.active?.hasLoadedModel() ?? false;
+    }
+    currentModelPath() {
+        return this.active?.currentModelPath() ?? null;
+    }
+    decide(plan) {
+        const catalog = resolveCatalogForPlan(plan);
+        return decideBackend({
+            override: readBackendOverride(),
+            catalog,
+            llamaServerAvailable: this.probeLlamaServerAvailable(),
+            dflashRequired: this.probeDflashRequired(),
+            binaryKernels: this.probeBinaryKernels?.() ?? null,
+        });
+    }
+    async load(plan) {
+        const decision = this.decide(plan);
+        if (decision.unsatisfiedKernels && decision.unsatisfiedKernels.length > 0) {
+            const missing = decision.unsatisfiedKernels.join(", ");
+            throw new Error(`[local-inference] Catalog model requires kernel(s) {${missing}}, but the installed llama-server binary does not advertise them in CAPABILITIES.json. Rebuild the fork with the matching backend (e.g. node packages/app-core/scripts/build-llama-cpp-dflash.mjs --target <triple>) or pick a different model.`);
+        }
+        const target = decision.backend === "llama-server"
+            ? this.llamaServer
+            : this.nodeLlamaCpp;
+        if (this.active && this.active !== target) {
+            await this.active.unload();
+        }
+        this.active = target;
+        await target.load(plan);
+    }
+    async unload() {
+        const active = this.active;
+        this.active = null;
+        if (active)
+            await active.unload();
+    }
+    async generate(args) {
+        if (!this.active) {
+            throw new Error("[local-inference] No backend loaded. Call load() before generate().");
+        }
+        return this.active.generate(args);
+    }
+    async embed(args) {
+        if (!this.active) {
+            throw new Error("[local-inference] No backend loaded. Call load() before embed().");
+        }
+        if (!this.active.embed) {
+            throw new Error(`[local-inference] Active backend (${this.active.id}) does not implement embed.`);
+        }
+        return this.active.embed(args);
+    }
+}

package/services/local-inference/bundled-models.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Bundled-models bootstrap for AOSP / on-device installs.
+ *
+ * The AOSP build pipeline stages Eliza-1 models into the APK at
+ * `assets/agent/models/{file}.gguf` plus a
+ * `manifest.json` describing each one (id, role, sha256, sizeBytes).
+ * `ElizaAgentService.extractAssetsIfNeeded()` copies those files into
+ * `$ELIZA_STATE_DIR/local-inference/models/` on first launch.
+ *
+ * This module reads the manifest at runtime startup and registers each
+ * file as a eliza-owned model in the local-inference registry, so the
+ * auto-assign pass picks them up for TEXT_LARGE / TEXT_SMALL /
+ * TEXT_EMBEDDING slots without needing the user to download anything.
+ *
+ * Idempotent: re-running with the registry already populated is a
+ * no-op for unchanged entries (`upsertElizaModel` overwrites entries
+ * with the same id, so updated sha256s on a future re-bundle replace
+ * the old metadata cleanly).
+ *
+ * Source classification: the runtime treats bundled models as
+ * `source: "eliza-download"` because Eliza ships the file and Eliza
+ * owns it on disk — same lifecycle as a user-initiated download
+ * (uninstall removes the file, the registry tracks the install). The
+ * only difference is the file arrived via APK extraction rather than
+ * an HTTP transfer.
+ */
+/**
+ * Walk the manifest and register every bundled GGUF file in the
+ * local-inference registry. Returns the number of entries successfully
+ * registered. A missing manifest is normal on Capacitor / desktop /
+ * non-AOSP installs and returns 0 silently.
+ */
+export declare function registerBundledModels(): Promise<number>;
+//# sourceMappingURL=bundled-models.d.ts.map

package/services/local-inference/bundled-models.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bundled-models.d.ts","sourceRoot":"","sources":["../../../../../../src/services/local-inference/bundled-models.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AA2DH;;;;;GAKG;AACH,wBAAsB,qBAAqB,IAAI,OAAO,CAAC,MAAM,CAAC,CAsC7D"}

package/services/local-inference/bundled-models.js

@@ -0,0 +1,104 @@
+/**
+ * Bundled-models bootstrap for AOSP / on-device installs.
+ *
+ * The AOSP build pipeline stages Eliza-1 models into the APK at
+ * `assets/agent/models/{file}.gguf` plus a
+ * `manifest.json` describing each one (id, role, sha256, sizeBytes).
+ * `ElizaAgentService.extractAssetsIfNeeded()` copies those files into
+ * `$ELIZA_STATE_DIR/local-inference/models/` on first launch.
+ *
+ * This module reads the manifest at runtime startup and registers each
+ * file as a eliza-owned model in the local-inference registry, so the
+ * auto-assign pass picks them up for TEXT_LARGE / TEXT_SMALL /
+ * TEXT_EMBEDDING slots without needing the user to download anything.
+ *
+ * Idempotent: re-running with the registry already populated is a
+ * no-op for unchanged entries (`upsertElizaModel` overwrites entries
+ * with the same id, so updated sha256s on a future re-bundle replace
+ * the old metadata cleanly).
+ *
+ * Source classification: the runtime treats bundled models as
+ * `source: "eliza-download"` because Eliza ships the file and Eliza
+ * owns it on disk — same lifecycle as a user-initiated download
+ * (uninstall removes the file, the registry tracks the install). The
+ * only difference is the file arrived via APK extraction rather than
+ * an HTTP transfer.
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import { ensureDefaultAssignment, readAssignments, writeAssignments, } from "./assignments";
+import { elizaModelsDir } from "./paths";
+import { upsertElizaModel } from "./registry";
+function manifestPath() {
+    return path.join(elizaModelsDir(), "manifest.json");
+}
+async function readManifest() {
+    try {
+        const raw = await fs.readFile(manifestPath(), "utf8");
+        const parsed = JSON.parse(raw);
+        if (parsed?.version !== 1 || !Array.isArray(parsed.models)) {
+            return null;
+        }
+        return parsed;
+    }
+    catch {
+        return null;
+    }
+}
+async function ensureBundledAssignment(modelId, role) {
+    if (role !== "embedding") {
+        await ensureDefaultAssignment(modelId);
+        return;
+    }
+    const current = await readAssignments();
+    if (current.TEXT_EMBEDDING)
+        return;
+    await writeAssignments({ ...current, TEXT_EMBEDDING: modelId });
+}
+/**
+ * Walk the manifest and register every bundled GGUF file in the
+ * local-inference registry. Returns the number of entries successfully
+ * registered. A missing manifest is normal on Capacitor / desktop /
+ * non-AOSP installs and returns 0 silently.
+ */
+export async function registerBundledModels() {
+    const manifest = await readManifest();
+    if (!manifest)
+        return 0;
+    const dir = elizaModelsDir();
+    let registered = 0;
+    for (const entry of manifest.models) {
+        const filePath = path.join(dir, entry.ggufFile);
+        let sizeBytes = entry.sizeBytes;
+        try {
+            const stat = await fs.stat(filePath);
+            sizeBytes = stat.size;
+        }
+        catch {
+            // File didn't extract — manifest references something the APK
+            // didn't ship. Skip this entry rather than registering a broken
+            // path. AOSP build's stage-default-models.mjs is the source of
+            // truth; if a file is missing the build is broken upstream.
+            continue;
+        }
+        const installed = {
+            id: entry.id,
+            displayName: entry.displayName,
+            path: filePath,
+            sizeBytes,
+            hfRepo: entry.hfRepo,
+            installedAt: new Date().toISOString(),
+            lastUsedAt: null,
+            source: "eliza-download",
+            sha256: entry.sha256 ?? undefined,
+        };
+        await upsertElizaModel(installed);
+        // Auto-assign each bundled model to its manifest role if the user
+        // hasn't already assigned that slot. This keeps Eliza-1 chat models
+        // eligible for TEXT_EMBEDDING when the bundle explicitly marks them
+        // as the local embedding bootstrap model.
+        await ensureBundledAssignment(entry.id, entry.role);
+        registered += 1;
+    }
+    return registered;
+}