@elizaos/agent 2.0.0-alpha.441 → 2.0.0-alpha.501

package/packages/agent/src/runtime/aosp-llama-adapter.js

@@ -0,0 +1,938 @@
+/**
+ * AOSP-only loader for native llama.cpp via `bun:ffi`.
+ *
+ * Targets the apothic/llama.cpp-1bit-turboquant fork (commit
+ *   https://github.com/Apothic-AI/llama.cpp-1bit-turboquant
+ *   tag: main-b8198-b2b5273
+ *   sha: b2b5273e8b275bb96362fe844a5202632eb3e52b
+ * — the matching libllama.so is compiled by the AOSP build pipeline
+ * against this same SHA via `scripts/miladyos/compile-libllama.mjs`).
+ *
+ * Why this fork (was stock llama.cpp b4500 before):
+ *   apothic's fork adds two GGML quant types (TBQ3_0 = 43, TBQ4_0 = 44)
+ *   for KV-cache compression, with CPU implementations under
+ *   `ggml/src/ggml-cpu/quants.c` + `ggml/src/ggml-cpu/ggml-cpu.c`.
+ *   block_tbq3_0 packs 32 floats into 14 bytes (vs 64 bytes for fp16) —
+ *   ~4.6x KV-cache memory reduction. KV cache dominates phone-RAM
+ *   pressure at long contexts, so this is the difference between Bonsai
+ *   "loads but OOMs after 1k tokens" and "loads and chats". The matching
+ *   Bonsai-8B-1bit GGUF on Hugging Face is trained against this fork.
+ *
+ *   The fork is based on llama.cpp b8198 (much newer than b4500), so it
+ *   inherits the post-2024 sampler-chain API
+ *   (`llama_sampler_chain_init`, `llama_sampler_init_greedy`, etc.) and
+ *   the renamed model/vocab API (`llama_model_load_from_file`,
+ *   `llama_init_from_model`, `llama_model_get_vocab`, `llama_vocab_eos`,
+ *   `llama_vocab_is_eog`) AND the embedding helpers
+ *   (`llama_set_embeddings`, `llama_get_embeddings_seq`,
+ *   `llama_model_n_embd`).
+ *
+ *   Drift since the b4500 pin handled in the shim:
+ *     - llama_context_params.flash_attn (bool) → flash_attn_type (enum);
+ *       shim removed the bool setter (the adapter never called it).
+ *     - llama_context_params adds type_k / type_v / samplers / kv_unified;
+ *       shim now exposes set_type_k / set_type_v for TBQ KV-cache wiring
+ *       (driven by `kvCacheType` in the adapter LoadOptions, with
+ *       Bonsai-by-filename auto-detection as a default).
+ *
+ * Symbols pinned for reference:
+ *   libllama.so (dlopen'd first):
+ *     - llama_backend_init / llama_backend_free
+ *     - llama_model_free / llama_free
+ *     - llama_model_get_vocab / llama_vocab_eos / llama_vocab_is_eog
+ *     - llama_tokenize / llama_token_to_piece
+ *     - llama_sampler_chain_add / llama_sampler_init_temp /
+ *       llama_sampler_init_top_p / llama_sampler_init_dist /
+ *       llama_sampler_init_greedy / llama_sampler_sample /
+ *       llama_sampler_accept / llama_sampler_free
+ *     - llama_get_model / llama_n_ctx / llama_model_n_embd
+ *     - llama_set_embeddings / llama_get_embeddings_seq / llama_get_embeddings
+ *   libmilady-llama-shim.so (dlopen'd second; NEEDED libllama.so):
+ *     - milady_llama_model_params_default / *_free + per-field setters
+ *     - milady_llama_model_load_from_file
+ *     - milady_llama_context_params_default / *_free + per-field setters
+ *     - milady_llama_init_from_model
+ *     - milady_llama_sampler_chain_params_default / *_free
+ *     - milady_llama_sampler_chain_init
+ *     - milady_llama_batch_get_one / milady_llama_batch_free
+ *     - milady_llama_decode
+ *
+ * Struct-by-value handled via libmilady-llama-shim.so (NEEDED-links
+ * libllama.so, ships in the same per-ABI asset dir). bun:ffi cannot pass
+ * llama.cpp's by-value param structs (model_params, context_params,
+ * sampler_chain_params) directly. The shim — built by
+ * `scripts/miladyos/compile-libllama.mjs` from
+ * `scripts/miladyos/llama-shim/milady_llama_shim.c` — exposes a
+ * pointer-style API: `milady_llama_model_params_default()` returns a
+ * malloc'd pointer initialized via `llama_model_default_params()`, then
+ * field-by-field setters override the few values the adapter cares about
+ * (n_gpu_layers, use_mmap, use_mlock, n_threads, n_ctx, etc.) before the
+ * pointer is handed to `milady_llama_model_load_from_file()` /
+ * `milady_llama_init_from_model()` / `milady_llama_sampler_chain_init()`,
+ * each of which dereferences once into the real struct-by-value entry
+ * point. This restores the canonical defaults — most importantly
+ * model_params.use_mmap = true (was clobbered to false by the previous
+ * zeroed-buffer workaround, which forced the loader to read entire
+ * weights files into RAM on phones).
+ *
+ * Wired in via `ensure-local-inference-handler.ts`:
+ *   - Trigger: `MILADY_LOCAL_LLAMA=1` in the AOSP agent process env.
+ *   - Slot:    `localInferenceLoader` runtime service (LocalInferenceLoader contract).
+ *   - Selection precedence: this loader is registered BEFORE the Capacitor
+ *     adapter so AOSP builds always pick the in-process FFI path.
+ *
+ * On a non-AOSP build that accidentally sets the env, this module logs and
+ * returns false from `registerAospLlamaLoader`. It does not throw at module
+ * load — bundlers must be able to statically import it on every platform.
+ */
+import { existsSync } from "node:fs";
+import path from "node:path";
+import { logger } from "@elizaos/core";
+/**
+ * Pooling type values from llama.h b4500. We always materialize the AOSP
+ * context with `MEAN` pooling so `llama_get_embeddings_seq(ctx, 0)` returns
+ * exactly `n_embd` floats — the sequence buffer is sized by pooling type,
+ * and `NONE` would shape it as `n_outputs * n_embd` where `n_outputs <
+ * written` for output-pruning models, leading to a read-OOB on the
+ * mean-pool fallback path. By forcing MEAN at init we collapse two code
+ * paths into one and remove the OOB risk entirely.
+ */
+const LLAMA_POOLING_TYPE_MEAN = 1;
+/**
+ * GGML type ids used for KV-cache configuration. The base set comes from
+ * ggml.h; TBQ3_0 / TBQ4_0 are the apothic/llama.cpp-1bit-turboquant fork
+ * additions and only valid against the fork-built libllama.so + matching
+ * Bonsai-8B-1bit GGUF model.
+ *
+ * Verified against
+ *   ~/.cache/milady-android-agent/llama-cpp-main-b8198-b2b5273/ggml/include/ggml.h
+ * (lines 420-435 — Q1_0 = 42 sits next to TBQ3_0 = 43, TBQ4_0 = 44).
+ */
+const GGML_TYPE_F16 = 1;
+const GGML_TYPE_TBQ3_0 = 43;
+const GGML_TYPE_TBQ4_0 = 44;
+export function kvCacheTypeNameToEnum(name) {
+    switch (name) {
+        case "f16":
+            return GGML_TYPE_F16;
+        case "tbq3_0":
+            return GGML_TYPE_TBQ3_0;
+        case "tbq4_0":
+            return GGML_TYPE_TBQ4_0;
+        default: {
+            // Exhaustive switch — fall here only if a future type is added without
+            // updating the map. Throw with the offending name so a future caller
+            // doesn't silently get f16.
+            const exhaustive = name;
+            throw new Error(`[aosp-llama] Unknown KV cache type: ${exhaustive}`);
+        }
+    }
+}
+/**
+ * Auto-detect when a model path indicates a Bonsai 1-bit TurboQuant build,
+ * which is the only model in the curated catalog that's trained against
+ * the fork's TBQ KV-cache codec. Match is intentionally loose
+ * (case-insensitive substring) because users may rename downloaded GGUFs.
+ *
+ * The Hugging Face repo is `apothic/bonsai-8B-1bit-turboquant` and ships
+ * `models/gguf/8B/Bonsai-8B.gguf`; downloads pass that filename through
+ * verbatim by default, so a "Bonsai" basename match is the right hook.
+ *
+ * Exported for unit tests.
+ */
+export function looksLikeBonsai(modelPath) {
+    const base = modelPath.split(/[/\\]/).pop() ?? modelPath;
+    return /bonsai/i.test(base);
+}
+/**
+ * Read a `KvCacheTypeName` from an env var, returning undefined when the var
+ * is unset, blank, or not a recognised type name. Recognised values are
+ * exactly `"f16"`, `"tbq3_0"`, `"tbq4_0"` (case-insensitive). An unrecognised
+ * value logs a warning and returns undefined rather than throwing — env-var
+ * typos shouldn't crash the loader.
+ *
+ * Exported for unit tests.
+ */
+export function readEnvKvCacheType(name, env = process.env) {
+    const raw = env[name]?.trim().toLowerCase();
+    if (!raw)
+        return undefined;
+    if (raw === "f16" || raw === "tbq3_0" || raw === "tbq4_0") {
+        return raw;
+    }
+    logger.warn(`[aosp-llama] ${name}=${raw} is not a recognised KV cache type; ignoring (use f16 / tbq3_0 / tbq4_0).`);
+    return undefined;
+}
+/**
+ * Resolve the KV-cache type to use for a given load. Precedence:
+ *   1. Explicit `LoadOptions.kvCacheType.{k,v}` (highest priority).
+ *   2. `MILADY_LLAMA_CACHE_TYPE_K` / `MILADY_LLAMA_CACHE_TYPE_V` env vars.
+ *   3. Auto-detection: Bonsai-by-filename → `{ k: "tbq4_0", v: "tbq3_0" }`
+ *      (matches the model card recommendation).
+ *   4. Otherwise undefined — the shim leaves the cache at llama.cpp's fp16
+ *      default, which is the safe choice for any non-Bonsai GGUF.
+ *
+ * Returns `undefined` when no override applies, so the caller can skip the
+ * shim setters entirely (smaller diff to upstream behaviour, easier to
+ * reason about in logs).
+ *
+ * Exported for unit tests.
+ */
+export function resolveKvCacheType(modelPath, override, env = process.env) {
+    const explicitK = override?.k;
+    const explicitV = override?.v;
+    const envK = readEnvKvCacheType("MILADY_LLAMA_CACHE_TYPE_K", env);
+    const envV = readEnvKvCacheType("MILADY_LLAMA_CACHE_TYPE_V", env);
+    // Auto-detection only kicks in when neither an explicit override nor an
+    // env override is set. Catalog blurb references this contract directly —
+    // change here = update catalog.ts blurb in the same commit.
+    const auto = looksLikeBonsai(modelPath)
+        ? { k: "tbq4_0", v: "tbq3_0" }
+        : undefined;
+    const k = explicitK ?? envK ?? auto?.k;
+    const v = explicitV ?? envV ?? auto?.v;
+    if (k === undefined && v === undefined)
+        return undefined;
+    return { k, v };
+}
+const SERVICE_NAME = "localInferenceLoader";
+function isAospEnabled(env = process.env) {
+    return env.MILADY_LOCAL_LLAMA?.trim() === "1";
+}
+/**
+ * Read a non-negative integer env override, falling back to `fallback`
+ * when the variable is unset, blank, or not parseable. Negative values
+ * are clamped to the fallback to avoid passing an int32-min into the
+ * shim setters.
+ */
+function readEnvInt(name, fallback) {
+    const raw = process.env[name]?.trim();
+    if (!raw)
+        return fallback;
+    const parsed = Number.parseInt(raw, 10);
+    if (!Number.isFinite(parsed) || parsed < 0)
+        return fallback;
+    return parsed;
+}
+/**
+ * Resolve the libllama.so path for the current ABI. The AOSP agent process
+ * runs with `cwd = <agent_root>`; the Java side unpacks `agent/{abi}/libllama.so`
+ * alongside the bun runtime and matching shared libraries.
+ *
+ * Exported for unit tests so we can verify ABI mapping without dlopen.
+ */
+export function resolveLibllamaPath(arch = process.arch, cwd = process.cwd()) {
+    return path.join(resolveAbiDir(arch, cwd), "libllama.so");
+}
+/**
+ * Resolve the libmilady-llama-shim.so path for the current ABI. Lives in
+ * the same per-ABI dir as libllama.so; the dynamic linker resolves the
+ * shim's NEEDED libllama.so via LD_LIBRARY_PATH.
+ *
+ * Exported for unit tests.
+ */
+export function resolveLlamaShimPath(arch = process.arch, cwd = process.cwd()) {
+    return path.join(resolveAbiDir(arch, cwd), "libmilady-llama-shim.so");
+}
+function resolveAbiDir(arch, cwd) {
+    const abiDir = arch === "arm64" ? "arm64-v8a" : arch === "x64" ? "x86_64" : null;
+    if (abiDir === null) {
+        throw new Error(`[aosp-llama] Unsupported process.arch for AOSP build: ${arch}`);
+    }
+    return path.join(cwd, abiDir);
+}
+async function loadBunFfi() {
+    // Dynamic import keeps non-Bun bundlers from failing on the bare specifier.
+    // The AOSP runtime is Bun, so this resolves; on Vitest/Node it throws and
+    // the adapter degrades to a logged failure rather than crashing the boot.
+    // We surface the real error so AOSP-only debugging on Android can see the
+    // root cause instead of the generic "bun:ffi unavailable" message.
+    try {
+        /* Deliberate boundary cast: the real bun:ffi typings define dlopen
+         * with a generic `Fns extends Record<string, FFIFunction>` constraint
+         * we don't want leaking into adapter types; we only consume the
+         * weakly-typed runtime shape. */
+        const mod = (await import("bun:ffi"));
+        return { ok: true, mod };
+    }
+    catch (err) {
+        return {
+            ok: false,
+            error: err instanceof Error ? err : new Error(String(err)),
+        };
+    }
+}
+function dlopenLlama(ffi, libPath) {
+    const T = ffi.FFIType;
+    const handle = ffi.dlopen(libPath, {
+        llama_backend_init: { args: [], returns: T.void },
+        llama_backend_free: { args: [], returns: T.void },
+        llama_model_free: { args: [T.ptr], returns: T.void },
+        llama_free: { args: [T.ptr], returns: T.void },
+        llama_get_model: { args: [T.ptr], returns: T.ptr },
+        llama_model_get_vocab: { args: [T.ptr], returns: T.ptr },
+        llama_model_n_embd: { args: [T.ptr], returns: T.i32 },
+        llama_n_ctx: { args: [T.ptr], returns: T.u32 },
+        llama_vocab_eos: { args: [T.ptr], returns: T.i32 },
+        llama_vocab_is_eog: { args: [T.ptr, T.i32], returns: T.bool },
+        llama_set_embeddings: { args: [T.ptr, T.bool], returns: T.void },
+        llama_get_embeddings_seq: { args: [T.ptr, T.i32], returns: T.ptr },
+        llama_get_embeddings: { args: [T.ptr], returns: T.ptr },
+        llama_get_memory: { args: [T.ptr], returns: T.ptr },
+        llama_memory_clear: { args: [T.ptr, T.bool], returns: T.void },
+        llama_tokenize: {
+            args: [T.ptr, T.ptr, T.i32, T.ptr, T.i32, T.bool, T.bool],
+            returns: T.i32,
+        },
+        llama_token_to_piece: {
+            args: [T.ptr, T.i32, T.ptr, T.i32, T.i32, T.bool],
+            returns: T.i32,
+        },
+        // Skip llama_batch_get_one / llama_decode — see LlamaSymbols comment.
+        // The pointer-style wrappers in ShimSymbols are bound below.
+        llama_sampler_chain_add: { args: [T.ptr, T.ptr], returns: T.void },
+        llama_sampler_init_temp: { args: [T.f32], returns: T.ptr },
+        llama_sampler_init_top_p: { args: [T.f32, T.u32], returns: T.ptr },
+        llama_sampler_init_dist: { args: [T.u32], returns: T.ptr },
+        llama_sampler_init_greedy: { args: [], returns: T.ptr },
+        llama_sampler_sample: { args: [T.ptr, T.ptr, T.i32], returns: T.i32 },
+        llama_sampler_accept: { args: [T.ptr, T.i32], returns: T.void },
+        llama_sampler_free: { args: [T.ptr], returns: T.void },
+    });
+    /* Deliberate boundary cast: bun:ffi.dlopen returns weakly-typed callable map */
+    return handle.symbols;
+}
+/**
+ * dlopen libmilady-llama-shim.so and bind the pointer-style wrappers
+ * around llama.cpp's struct-by-value entry points. The shim NEEDED-links
+ * libllama.so, so libllama.so MUST already be loaded (via the earlier
+ * `dlopenLlama` call) or resolvable through LD_LIBRARY_PATH before this
+ * runs. On Android both conditions are satisfied — MiladyAgentService.java
+ * sets LD_LIBRARY_PATH to the per-ABI asset dir, and we always dlopen
+ * libllama.so first.
+ */
+function dlopenShim(ffi, shimPath) {
+    const T = ffi.FFIType;
+    const handle = ffi.dlopen(shimPath, {
+        milady_llama_model_params_default: { args: [], returns: T.ptr },
+        milady_llama_model_params_free: { args: [T.ptr], returns: T.void },
+        milady_llama_model_params_set_n_gpu_layers: {
+            args: [T.ptr, T.i32],
+            returns: T.void,
+        },
+        milady_llama_model_load_from_file: {
+            args: [T.ptr, T.ptr],
+            returns: T.ptr,
+        },
+        milady_llama_context_params_default: { args: [], returns: T.ptr },
+        milady_llama_context_params_free: { args: [T.ptr], returns: T.void },
+        milady_llama_context_params_set_n_ctx: {
+            args: [T.ptr, T.u32],
+            returns: T.void,
+        },
+        milady_llama_context_params_set_n_batch: {
+            args: [T.ptr, T.u32],
+            returns: T.void,
+        },
+        milady_llama_context_params_set_n_ubatch: {
+            args: [T.ptr, T.u32],
+            returns: T.void,
+        },
+        milady_llama_context_params_set_n_threads: {
+            args: [T.ptr, T.i32],
+            returns: T.void,
+        },
+        milady_llama_context_params_set_n_threads_batch: {
+            args: [T.ptr, T.i32],
+            returns: T.void,
+        },
+        milady_llama_context_params_set_embeddings: {
+            args: [T.ptr, T.bool],
+            returns: T.void,
+        },
+        milady_llama_context_params_set_pooling_type: {
+            args: [T.ptr, T.i32],
+            returns: T.void,
+        },
+        milady_llama_context_params_set_type_k: {
+            args: [T.ptr, T.i32],
+            returns: T.void,
+        },
+        milady_llama_context_params_set_type_v: {
+            args: [T.ptr, T.i32],
+            returns: T.void,
+        },
+        milady_llama_init_from_model: { args: [T.ptr, T.ptr], returns: T.ptr },
+        milady_llama_sampler_chain_params_default: { args: [], returns: T.ptr },
+        milady_llama_sampler_chain_params_free: {
+            args: [T.ptr],
+            returns: T.void,
+        },
+        milady_llama_sampler_chain_init: { args: [T.ptr], returns: T.ptr },
+        milady_llama_batch_get_one: {
+            args: [T.ptr, T.i32],
+            returns: T.ptr,
+        },
+        milady_llama_batch_free: { args: [T.ptr], returns: T.void },
+        milady_llama_decode: { args: [T.ptr, T.ptr], returns: T.i32 },
+    });
+    /* Deliberate boundary cast: bun:ffi.dlopen returns weakly-typed callable map */
+    return handle.symbols;
+}
+function encodeCString(text) {
+    const enc = new TextEncoder().encode(text);
+    const buf = new Uint8Array(enc.length + 1);
+    buf.set(enc, 0);
+    buf[enc.length] = 0;
+    return buf;
+}
+class AospLlamaAdapter {
+    ffi;
+    sym;
+    shim;
+    model = null;
+    ctx = null;
+    vocab = null;
+    nCtx = 0;
+    loadedPath = null;
+    backendInitialized = false;
+    /**
+     * Tracks whether the current ctx has had at least one successful
+     * `llama_decode` call. `llama_memory_clear` segfaults on cuttlefish
+     * x86_64 when called against a freshly-initialized ctx with no
+     * decoded positions, so we only invoke it once we know the KV cache
+     * has live state to wipe. Reset to `false` on every `loadModel` /
+     * `unloadModel`.
+     */
+    hasDecoded = false;
+    constructor(ffi, sym, shim) {
+        this.ffi = ffi;
+        this.sym = sym;
+        this.shim = shim;
+    }
+    ensureBackend() {
+        if (this.backendInitialized)
+            return;
+        this.sym.llama_backend_init();
+        this.backendInitialized = true;
+    }
+    currentModelPath() {
+        return this.loadedPath;
+    }
+    async loadModel(args) {
+        this.ensureBackend();
+        if (this.loadedPath === args.modelPath && this.ctx !== null)
+            return;
+        if (this.ctx !== null || this.model !== null) {
+            await this.unloadModel();
+        }
+        // Resolve runtime tunables. The active-model coordinator only forwards
+        // `{ modelPath }` today, so we backfill from env so AOSP doesn't run at
+        // upstream defaults that under-use phone CPU cores.
+        const contextSize = args.contextSize ?? readEnvInt("MILADY_LLAMA_N_CTX", 8192);
+        const maxThreads = args.maxThreads ?? readEnvInt("MILADY_LLAMA_THREADS", 0);
+        const useGpu = args.useGpu ?? false;
+        const kvCacheType = resolveKvCacheType(args.modelPath, args.kvCacheType);
+        // Materialize llama_model_params via the shim. The shim runs
+        // llama_model_default_params() under the hood, so use_mmap=true,
+        // use_mlock=false, n_gpu_layers=999 (or whatever upstream's defaults
+        // are at the pinned tag) all land correctly. We pin n_gpu_layers=0
+        // explicitly when the caller opts out of GPU so the value is
+        // self-documenting in logs even though it matches the AOSP default.
+        const modelParamsPtr = this.shim.milady_llama_model_params_default();
+        if (!modelParamsPtr) {
+            throw new Error("[aosp-llama] milady_llama_model_params_default returned NULL (malloc failure?)");
+        }
+        let modelPtr = 0;
+        try {
+            if (!useGpu) {
+                this.shim.milady_llama_model_params_set_n_gpu_layers(modelParamsPtr, 0);
+            }
+            const pathBuf = encodeCString(args.modelPath);
+            modelPtr = this.shim.milady_llama_model_load_from_file(this.ffi.ptr(pathBuf), modelParamsPtr);
+        }
+        finally {
+            this.shim.milady_llama_model_params_free(modelParamsPtr);
+        }
+        if (!modelPtr) {
+            throw new Error(`[aosp-llama] llama_model_load_from_file returned NULL for ${args.modelPath}`);
+        }
+        const ctxParamsPtr = this.shim.milady_llama_context_params_default();
+        if (!ctxParamsPtr) {
+            this.sym.llama_model_free(modelPtr);
+            throw new Error("[aosp-llama] milady_llama_context_params_default returned NULL (malloc failure?)");
+        }
+        let ctxPtr = 0;
+        try {
+            // Override the canonical defaults for the few fields that actually
+            // matter on phones:
+            //   - n_ctx: cap the context window (defaults to 0 = "from model"
+            //     which can be huge on Llama-3-8B GGUFs and OOMs the device).
+            //   - n_threads / n_threads_batch: parallelize generation + batch
+            //     decode across the user's CPU cores. n_threads is on
+            //     context_params (verified against b4500 llama.h:319-320),
+            //     NOT model_params.
+            //   - embeddings: leave the runtime toggle (`llama_set_embeddings`)
+            //     to flip this per-call, but pre-allocate the buffers at init
+            //     so the first embed() call doesn't pay an allocation tax.
+            //   - pooling_type: pin to MEAN so `llama_get_embeddings_seq(ctx, 0)`
+            //     always returns exactly `n_embd` floats. NONE would shape the
+            //     ctx buffer as `n_outputs * n_embd` where n_outputs can be
+            //     less than the input token count for output-pruning models —
+            //     we'd read OOB on the mean-pool fallback. By forcing MEAN at
+            //     init we collapse the embed() path to a single read.
+            this.shim.milady_llama_context_params_set_n_ctx(ctxParamsPtr, contextSize);
+            // n_batch = 2048: the per-decode token cap. We chunk longer prompts
+            // in the decode loop. Setting this to n_ctx makes the worst-case
+            // compute graph allocate ~1.85 GiB (8192 ubatch on a 360M model)
+            // which both wedges the bun event loop long enough to trip the
+            // service watchdog and pushes phone RAM over budget.
+            // n_ubatch = 512: upstream default, matches phone CPU cache.
+            const nBatchParam = readEnvInt("MILADY_LLAMA_N_BATCH", 2048);
+            const nUBatchParam = readEnvInt("MILADY_LLAMA_N_UBATCH", 512);
+            this.shim.milady_llama_context_params_set_n_batch(ctxParamsPtr, nBatchParam);
+            this.shim.milady_llama_context_params_set_n_ubatch(ctxParamsPtr, nUBatchParam);
+            this.shim.milady_llama_context_params_set_n_threads(ctxParamsPtr, maxThreads);
+            this.shim.milady_llama_context_params_set_n_threads_batch(ctxParamsPtr, maxThreads);
+            this.shim.milady_llama_context_params_set_embeddings(ctxParamsPtr, true);
+            this.shim.milady_llama_context_params_set_pooling_type(ctxParamsPtr, LLAMA_POOLING_TYPE_MEAN);
+            // KV-cache type override (TBQ for Bonsai, fp16 default for everything
+            // else). When kvCacheType.k / .v are set we forward the resolved
+            // ggml_type enum to the shim setters; otherwise we leave the cache at
+            // llama.cpp's canonical default. Only the apothic fork-built libllama.so
+            // understands TBQ3_0 / TBQ4_0 — using these against stock llama.cpp
+            // would crash inside type_traits lookup.
+            if (kvCacheType?.k !== undefined) {
+                this.shim.milady_llama_context_params_set_type_k(ctxParamsPtr, kvCacheTypeNameToEnum(kvCacheType.k));
+            }
+            if (kvCacheType?.v !== undefined) {
+                this.shim.milady_llama_context_params_set_type_v(ctxParamsPtr, kvCacheTypeNameToEnum(kvCacheType.v));
+            }
+            ctxPtr = this.shim.milady_llama_init_from_model(modelPtr, ctxParamsPtr);
+        }
+        finally {
+            this.shim.milady_llama_context_params_free(ctxParamsPtr);
+        }
+        if (!ctxPtr) {
+            this.sym.llama_model_free(modelPtr);
+            throw new Error(`[aosp-llama] llama_init_from_model returned NULL for ${args.modelPath}`);
+        }
+        this.model = modelPtr;
+        this.ctx = ctxPtr;
+        this.vocab = this.sym.llama_model_get_vocab(modelPtr);
+        this.nCtx = this.sym.llama_n_ctx(ctxPtr);
+        this.loadedPath = args.modelPath;
+        this.hasDecoded = false;
+        logger.info(`[aosp-llama] Loaded ${args.modelPath} (n_ctx=${this.nCtx}, n_threads=${maxThreads}, gpu=${useGpu}, kv_k=${kvCacheType?.k ?? "f16"}, kv_v=${kvCacheType?.v ?? "f16"})`);
+    }
+    async unloadModel() {
+        if (this.ctx !== null) {
+            this.sym.llama_free(this.ctx);
+            this.ctx = null;
+        }
+        if (this.model !== null) {
+            this.sym.llama_model_free(this.model);
+            this.model = null;
+        }
+        this.vocab = null;
+        this.nCtx = 0;
+        this.loadedPath = null;
+        this.hasDecoded = false;
+    }
+    async generate(args) {
+        if (this.ctx === null || this.model === null || this.vocab === null) {
+            throw new Error("[aosp-llama] generate called before loadModel");
+        }
+        const ctx = this.ctx;
+        const vocab = this.vocab;
+        // 0. Reset KV cache for this turn. The b8198 cuttlefish build
+        // segfaults when llama_memory_clear runs on a freshly-initialized
+        // ctx (no positions yet), so we only wipe once we've decoded at
+        // least one batch. The first generate() / embed() against a fresh
+        // ctx skips the clear; subsequent calls always wipe before the
+        // first chunk so prompts can land cleanly without stacking on top
+        // of stale KV state.
+        if (this.hasDecoded) {
+            const memHandle = this.sym.llama_get_memory(ctx);
+            if (memHandle) {
+                this.sym.llama_memory_clear(memHandle, false);
+            }
+        }
+        // 1. Tokenize the prompt. Two-pass: ask for length (n_tokens_max=0,
+        // single-slot buffer — llama_tokenize never reads or writes through
+        // the pointer when the cap is zero, but bun:ffi's ptr() helper
+        // rejects zero-length TypedArrays with
+        // `ArrayBufferView must have a length > 0`. A length-1 probe is the
+        // smallest legal allocation that round-trips through ptr() without
+        // a runtime exception. Then alloc and fill on the second pass.
+        const promptBuf = encodeCString(args.prompt);
+        const promptByteLen = promptBuf.length - 1; // exclude NUL
+        const probe = new Int32Array(1);
+        const requested = this.sym.llama_tokenize(vocab, this.ffi.ptr(promptBuf), promptByteLen, this.ffi.ptr(probe), 0, true, false);
+        // llama_tokenize returns the negative of required length when n_tokens_max
+        // is too small. With n_tokens_max=0 we always get a negative number.
+        const required = requested < 0 ? -requested : requested;
+        if (required <= 0) {
+            throw new Error("[aosp-llama] llama_tokenize returned zero tokens");
+        }
+        const tokens = new Int32Array(required);
+        const written = this.sym.llama_tokenize(vocab, this.ffi.ptr(promptBuf), promptByteLen, this.ffi.ptr(tokens), required, true, false);
+        if (written < 0) {
+            throw new Error(`[aosp-llama] llama_tokenize second pass failed: ${written}`);
+        }
+        // 2. Build a sampler chain: temp → top_p → dist (or greedy). The
+        // sampler_chain_params struct is single-field (no_perf bool); the
+        // shim materializes it with llama.cpp's default and we don't
+        // override.
+        const samplerParamsPtr = this.shim.milady_llama_sampler_chain_params_default();
+        if (!samplerParamsPtr) {
+            throw new Error("[aosp-llama] milady_llama_sampler_chain_params_default returned NULL (malloc failure?)");
+        }
+        let chain = 0;
+        try {
+            chain = this.shim.milady_llama_sampler_chain_init(samplerParamsPtr);
+        }
+        finally {
+            this.shim.milady_llama_sampler_chain_params_free(samplerParamsPtr);
+        }
+        if (!chain) {
+            throw new Error("[aosp-llama] llama_sampler_chain_init returned NULL");
+        }
+        const temperature = args.temperature ?? 0.7;
+        if (temperature <= 0) {
+            this.sym.llama_sampler_chain_add(chain, this.sym.llama_sampler_init_greedy());
+        }
+        else {
+            this.sym.llama_sampler_chain_add(chain, this.sym.llama_sampler_init_temp(temperature));
+            this.sym.llama_sampler_chain_add(chain, this.sym.llama_sampler_init_top_p(0.9, 1));
+            this.sym.llama_sampler_chain_add(chain, this.sym.llama_sampler_init_dist(0xffffffff));
+        }
+        try {
+            // 3. Decode the prompt batch.
+            // llama.cpp's llama_decode rejects token-only batches when the
+            // context is in embedding mode — the per-call assert is
+            //   GGML_ASSERT((!batch_inp.token && batch_inp.embd) ||
+            //               (batch_inp.token && !batch_inp.embd))
+            // and a previous embed() call may have flipped the flag on the
+            // shared context. Reset to OFF before every chat decode so the
+            // batch shape that llama_batch_get_one produces (token-only)
+            // matches what the decoder accepts, regardless of prior calls.
+            this.sym.llama_set_embeddings(ctx, false);
+            // Chunk the prompt into n_batch-sized pieces and feed them to
+            // llama_decode one at a time. llama.cpp asserts
+            //   GGML_ASSERT(n_tokens_all <= cparams.n_batch)
+            // on the first decode if the prompt exceeds the configured n_batch
+            // — and even with n_batch == n_ctx the planner routinely hands us
+            // prompts that exceed n_ctx (system prompt + tools + history +
+            // user msg). When that happens we keep the TAIL of the prompt
+            // (the user's most recent message + closest context), reserving
+            // headroom for the model to generate output. Truncating from the
+            // tail would silently drop the user's question; truncating from
+            // the head preserves the question at the cost of dropping the
+            // earliest tools/history.
+            // bun:ffi struct-by-value workaround: route llama_batch_get_one +
+            // llama_decode through the shim. See ShimSymbols comment.
+            // Decode chunk size is bounded by n_batch (set in loadModel).
+            // Reading it here mirrors the parameter that loadModel committed
+            // to via milady_llama_context_params_set_n_batch.
+            const nBatch = readEnvInt("MILADY_LLAMA_N_BATCH", 2048);
+            const maxOutputReserve = args.maxTokens ?? 512;
+            // Reserve maxOutputReserve + n_batch (one ubatch slack) + an
+            // empirical 25 % safety margin. llama.cpp's Flash-Attention sliding
+            // memory allocator on the b8198 build returns
+            //   decode: failed to find a memory slot for batch of size N
+            // when the per-sequence KV slots get fragmented by repeated
+            // back-to-back chunks even with positions still nominally free,
+            // so we leave generous headroom rather than push to the limit.
+            const promptCapacity = Math.max(1, Math.floor((this.nCtx - maxOutputReserve - nBatch) * 0.75));
+            let promptTokens = tokens;
+            let promptLen = written;
+            if (written > promptCapacity) {
+                const head = written - promptCapacity;
+                promptTokens = tokens.subarray(head);
+                promptLen = promptCapacity;
+                logger.warn(`[aosp-llama] prompt ${written} tokens > capacity ${promptCapacity} (n_ctx=${this.nCtx} - reserve ${maxOutputReserve}); dropping ${head} head tokens`);
+            }
+            for (let offset = 0; offset < promptLen; offset += nBatch) {
+                const chunkLen = Math.min(nBatch, promptLen - offset);
+                const chunk = promptTokens.subarray(offset, offset + chunkLen);
+                const promptBatchPtr = this.shim.milady_llama_batch_get_one(this.ffi.ptr(chunk), chunkLen);
+                if (!promptBatchPtr) {
+                    throw new Error("[aosp-llama] milady_llama_batch_get_one returned NULL (malloc failure?)");
+                }
+                let decodeRc;
+                try {
+                    decodeRc = this.shim.milady_llama_decode(ctx, promptBatchPtr);
+                }
+                finally {
+                    this.shim.milady_llama_batch_free(promptBatchPtr);
+                }
+                if (decodeRc !== 0) {
+                    throw new Error(`[aosp-llama] llama_decode (prompt chunk @${offset}/${promptLen}) returned ${decodeRc}`);
+                }
+                // Mark the ctx as decoded so subsequent generate()/embed() calls
+                // will issue the leading llama_memory_clear safely.
+                this.hasDecoded = true;
+                // Yield to the event loop between chunks so the on-device
+                // watchdog's /api/health probe (3s timeout × 3 strikes = ~6 min
+                // grace window with WATCHDOG_INTERVAL_MS=120s) can complete.
+                // Without this yield bun's single-threaded loop sits inside
+                // FFI for the entire prompt decode (minutes on cuttlefish CPU)
+                // and the service kills it for missed health checks.
+                await new Promise((resolve) => setImmediate(resolve));
+            }
+            // 4. Token loop.
+            const maxTokens = args.maxTokens ?? 512;
+            const stopSequences = args.stopSequences ?? [];
+            const pieceBuf = new Uint8Array(256);
+            const singleToken = new Int32Array(1);
+            let output = "";
+            for (let i = 0; i < maxTokens; i++) {
+                const next = this.sym.llama_sampler_sample(chain, ctx, -1);
+                if (this.sym.llama_vocab_is_eog(vocab, next))
+                    break;
+                this.sym.llama_sampler_accept(chain, next);
+                const wrote = this.sym.llama_token_to_piece(vocab, next, this.ffi.ptr(pieceBuf), pieceBuf.length, 0, false);
+                if (wrote > 0) {
+                    const piece = new TextDecoder().decode(pieceBuf.subarray(0, wrote));
+                    output += piece;
+                    if (stopSequences.some((s) => s.length > 0 && output.endsWith(s))) {
+                        for (const stop of stopSequences) {
+                            if (stop.length > 0 && output.endsWith(stop)) {
+                                output = output.slice(0, -stop.length);
+                                break;
+                            }
+                        }
+                        break;
+                    }
+                }
+                singleToken[0] = next;
+                const stepBatchPtr = this.shim.milady_llama_batch_get_one(this.ffi.ptr(singleToken), 1);
+                if (!stepBatchPtr) {
+                    throw new Error("[aosp-llama] milady_llama_batch_get_one returned NULL (malloc failure?)");
+                }
+                let stepRc;
+                try {
+                    stepRc = this.shim.milady_llama_decode(ctx, stepBatchPtr);
+                }
+                finally {
+                    this.shim.milady_llama_batch_free(stepBatchPtr);
+                }
+                if (stepRc !== 0) {
+                    throw new Error(`[aosp-llama] llama_decode (step) returned ${stepRc}`);
+                }
+                // Yield every 16 generated tokens. setImmediate every step
+                // would cut sampling throughput by ~30 %; a stride of 16 keeps
+                // the loop responsive to /api/health while staying close to
+                // peak generation rate. SmolLM2-360M on cuttlefish CPU lands
+                // roughly 5 tok/s, so 16 tokens = ~3 s per yield, well under
+                // the 3-strike × 120 s health-probe budget.
+                if ((i & 15) === 15) {
+                    await new Promise((resolve) => setImmediate(resolve));
+                }
+            }
+            return output;
+        }
+        finally {
+            this.sym.llama_sampler_free(chain);
+        }
+    }
+    /**
+     * Compute a sentence-level embedding for `input`. Single-context loader:
+     * we toggle the loaded ctx into embeddings mode via `llama_set_embeddings`,
+     * decode the tokenized input as one sequence, then read the per-sequence
+     * pooled embedding.
+     *
+     * Pooling contract: `loadModel()` initialises the context with
+     * `pooling_type = MEAN` (verified against b4500 llama.h enum), so
+     * `llama_get_embeddings_seq(ctx, 0)` returns exactly `n_embd` floats and
+     * we never need the per-token fallback path. If a future change ever
+     * sets `pooling_type = NONE`, this method must reject — reading
+     * `llama_get_embeddings(ctx)` for `written * n_embd` floats races with
+     * llama.cpp's per-decode `n_outputs` and would over-read for
+     * output-pruning models.
+     *
+     * Trade-off: the same context is used for generation and embeddings;
+     * toggling between modes flushes the KV cache implicitly on the next
+     * `llama_decode`, so repeated mode-switching is slow. Acceptable for a
+     * mobile-first runtime where embeddings are infrequent (memory + RAG
+     * indexing) compared to chat turns.
+     */
+    async embed(args) {
+        if (this.ctx === null || this.model === null || this.vocab === null) {
+            throw new Error("[aosp-llama] embed called before loadModel");
+        }
+        const ctx = this.ctx;
+        const model = this.model;
+        const vocab = this.vocab;
+        // 0. Reset KV cache. See generate() for the hasDecoded gating
+        // rationale (cuttlefish x86_64 segfault on a freshly-initialized
+        // ctx).
+        if (this.hasDecoded) {
+            const memHandle = this.sym.llama_get_memory(ctx);
+            if (memHandle) {
+                this.sym.llama_memory_clear(memHandle, false);
+            }
+        }
+        // 1. Tokenize the input. Embedding pipelines typically include the BOS
+        //    token; we mirror generate() and pass add_special=true. Probe pass
+        //    needs only a length, not storage, but bun:ffi's ptr() rejects
+        //    zero-length TypedArrays — use a single-slot Int32Array, the
+        //    smallest legal allocation that round-trips through ptr() without
+        //    `ArrayBufferView must have a length > 0`.
+        const inputBuf = encodeCString(args.input);
+        const inputByteLen = inputBuf.length - 1;
+        const probeOut = new Int32Array(1);
+        const requested = this.sym.llama_tokenize(vocab, this.ffi.ptr(inputBuf), inputByteLen, this.ffi.ptr(probeOut), 0, true, false);
+        const required = requested < 0 ? -requested : requested;
+        if (required <= 0) {
+            throw new Error("[aosp-llama] llama_tokenize returned zero tokens for embed input");
+        }
+        const tokens = new Int32Array(required);
+        const written = this.sym.llama_tokenize(vocab, this.ffi.ptr(inputBuf), inputByteLen, this.ffi.ptr(tokens), required, true, false);
+        if (written < 0) {
+            throw new Error(`[aosp-llama] llama_tokenize embed second pass failed: ${written}`);
+        }
+        // 2. Switch ctx into embeddings mode, decode, then switch back. The
+        //    next decode() implicitly clears KV cache state when the embeddings
+        //    flag flips — `generate()` callers that ran before `embed()` see a
+        //    fresh prompt anyway, so this is safe to do unconditionally.
+        this.sym.llama_set_embeddings(ctx, true);
+        try {
+            // bun:ffi struct-by-value workaround: route through the shim's
+            // pointer-style wrappers. See ShimSymbols.milady_llama_batch_get_one
+            // / milady_llama_decode for the rationale.
+            const batchPtr = this.shim.milady_llama_batch_get_one(this.ffi.ptr(tokens), written);
+            if (!batchPtr) {
+                throw new Error("[aosp-llama] milady_llama_batch_get_one returned NULL (malloc failure?)");
+            }
+            let decodeRc;
+            try {
+                decodeRc = this.shim.milady_llama_decode(ctx, batchPtr);
+            }
+            finally {
+                this.shim.milady_llama_batch_free(batchPtr);
+            }
+            if (decodeRc !== 0) {
+                throw new Error(`[aosp-llama] llama_decode (embed) returned ${decodeRc}`);
+            }
+            this.hasDecoded = true;
+            const nEmbd = this.sym.llama_model_n_embd(model);
+            if (nEmbd <= 0) {
+                throw new Error(`[aosp-llama] llama_model_n_embd returned non-positive ${nEmbd}`);
+            }
+            const byteLength = nEmbd * 4; // float32
+            // Read the pooled per-sequence buffer. `loadModel` pinned
+            // pooling_type = MEAN, so llama.cpp produces exactly `n_embd`
+            // floats here. A NULL return means either pooling was disabled
+            // (contract violation) or the model emitted no output for
+            // sequence 0 — both cases are unrecoverable, so fail loudly.
+            const pooledPtr = this.sym.llama_get_embeddings_seq(ctx, 0);
+            if (!pooledPtr) {
+                throw new Error("[aosp-llama] llama_get_embeddings_seq returned NULL — pooling_type contract violated");
+            }
+            const buf = this.ffi.toArrayBuffer(pooledPtr, 0, byteLength);
+            // Copy off the ctx-owned buffer so the result outlives the next
+            // llama_decode() call.
+            const view = new Float32Array(buf.slice(0));
+            return { embedding: Array.from(view), tokens: written };
+        }
+        finally {
+            // Restore generation mode so the next `generate()` call doesn't get
+            // hit with a reload-KV-cache stall on its first decode.
+            this.sym.llama_set_embeddings(ctx, false);
+        }
+    }
+}
+let cachedAdapter = null;
+/**
+ * Build (or return cached) AOSP loader. Returns null if the env opt-in is not
+ * set, libllama.so / libmilady-llama-shim.so cannot be located, or `bun:ffi`
+ * is unavailable. Each failure is logged once. Failures while
+ * `MILADY_LOCAL_LLAMA=1` is set are elevated to `error` because the user
+ * explicitly opted in.
+ */
+async function buildAdapter() {
+    if (cachedAdapter)
+        return cachedAdapter;
+    if (!isAospEnabled())
+        return null;
+    let libPath;
+    let shimPath;
+    try {
+        libPath = resolveLibllamaPath();
+        shimPath = resolveLlamaShimPath();
+    }
+    catch (err) {
+        logger.error("[aosp-llama] Cannot resolve native library paths:", err instanceof Error ? err.message : String(err));
+        return null;
+    }
+    if (!existsSync(libPath)) {
+        logger.error(`[aosp-llama] MILADY_LOCAL_LLAMA=1 but libllama.so missing at ${libPath}`);
+        return null;
+    }
+    if (!existsSync(shimPath)) {
+        logger.error(`[aosp-llama] MILADY_LOCAL_LLAMA=1 but libmilady-llama-shim.so missing at ${shimPath}. ` +
+            `Re-run scripts/miladyos/compile-libllama.mjs to produce the bun:ffi struct-by-value shim.`);
+        return null;
+    }
+    const ffiResult = await loadBunFfi();
+    if (ffiResult.ok === false) {
+        logger.error(`[aosp-llama] MILADY_LOCAL_LLAMA=1 but bun:ffi is unavailable on this runtime: ${ffiResult.error.message}`);
+        return null;
+    }
+    const ffi = ffiResult.mod;
+    let symbols;
+    try {
+        // Order matters: libllama.so must be loaded first so the shim's
+        // NEEDED entry resolves at dlopen time. (LD_LIBRARY_PATH is the
+        // runtime fallback, but loading libllama.so first guarantees the
+        // symbols are already in the global namespace.)
+        symbols = dlopenLlama(ffi, libPath);
+    }
+    catch (err) {
+        logger.error(`[aosp-llama] dlopen failed for ${libPath}:`, err instanceof Error ? err.message : String(err));
+        return null;
+    }
+    let shim;
+    try {
+        shim = dlopenShim(ffi, shimPath);
+    }
+    catch (err) {
+        logger.error(`[aosp-llama] dlopen failed for ${shimPath}:`, err instanceof Error ? err.message : String(err));
+        return null;
+    }
+    cachedAdapter = new AospLlamaAdapter(ffi, symbols, shim);
+    return cachedAdapter;
+}
+/**
+ * Register the AOSP llama.cpp FFI loader on the runtime. No-op on non-AOSP
+ * builds (when `MILADY_LOCAL_LLAMA !== "1"`). Returns true on successful
+ * registration so the caller can confirm precedence.
+ */
+export async function registerAospLlamaLoader(runtime) {
+    if (!isAospEnabled())
+        return false;
+    if (typeof runtime.registerService !== "function")
+        return false;
+    const adapter = await buildAdapter();
+    if (!adapter)
+        return false;
+    runtime.registerService(SERVICE_NAME, {
+        // Accept the shared LocalInferenceLoader shape (`{ modelPath }`) AND the
+        // AOSP-specific extension (`{ modelPath, kvCacheType?, … }`) — callers
+        // that don't know about TBQ pass the slim shape and the adapter
+        // auto-detects from the filename.
+        loadModel: (a) => adapter.loadModel(a),
+        unloadModel: () => adapter.unloadModel(),
+        currentModelPath: () => adapter.currentModelPath(),
+        generate: (a) => adapter.generate(a),
+        embed: (a) => adapter.embed(a),
+    });
+    logger.info("[aosp-llama] Registered native libllama.so loader (MILADY_LOCAL_LLAMA=1)");
+    return true;
+}
+/** Test-only: drop the cached adapter so a fresh build can run. */
+export function __resetForTests() {
+    cachedAdapter = null;
+}