@elizaos/app-core 2.0.3-beta.2 → 2.0.3-beta.4

package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaBionicInferenceServer.java

@@ -2,6 +2,7 @@ package ai.elizaos.app;
 
 import android.net.LocalServerSocket;
 import android.net.LocalSocket;
+import android.system.Os;
 import android.util.Base64;
 import android.util.Log;
 
@@ -60,6 +61,12 @@ final class ElizaBionicInferenceServer {
     private long residentCtx = 0L;
     private long residentStream = 0L;
     private String residentBundle = null;
+    private String residentDrafterPath = "";
+    /** The previous turn's prompt tokens — used to find the longest common prefix
+     *  with the next turn's prompt so its KV can be reused (only the delta is
+     *  re-prefilled). null when the stream has no reusable KV (first turn / after
+     *  a reset/reopen). */
+    private int[] residentPrevTokens = null;
     private final Object residentLock = new Object();
     /** Hard decode ceiling for the resident stream (per-call cap is applied below). */
     private static final int RESIDENT_STREAM_MAX_TOKENS = 2048;
@@ -70,10 +77,62 @@ final class ElizaBionicInferenceServer {
     }
 
     /** Bind the abstract-namespace socket and start accepting. Idempotent. */
+    /**
+     * The bionic host runs the LLM in THIS (app) process via JNI, so the fused
+     * native lib reads its tuning from the app-process environment — NOT the bun
+     * agent subprocess env (which only carries the ELIZA_LLAMA_* names). On
+     * Mali-class 8 GB phones the 2B's non-flash-attn compute + logits buffers at
+     * the upstream n_batch=512 default push peak RSS past what the device can
+     * allocate ("llm_stream_open: failed to init llama context"). FA is disabled
+     * on Android (the scalar-FA race), so the non-FA attention buffer is the
+     * dominant cost and it scales with n_batch; capping n_batch shrinks both that
+     * and the n_vocab×n_batch logits buffer ~4x, which is what lets the context
+     * fit. n_ctx is left at the model-capped default (KV is only ~0.4 GB at 8k).
+     * Only sets a value when it is not already present, so any explicit override
+     * still wins.
+     */
+    private static void applyBionicInferenceMemoryDefaults() {
+        setEnvIfAbsent("ELIZA_LLM_N_BATCH", "128");
+        setEnvIfAbsent("ELIZA_LLM_N_CTX", "8192");
+        // The JNI bridge can default the KV cache to a fused QJL/TBQ quant
+        // (cache_type_k="qjl1_256"), but that path is a head_dim=128 sketch and
+        // is RETIRED for the shipped tiers (elizaOS/eliza#8848 / #9033 Gemma-4
+        // cutover): eliza-1 now ships Gemma-4-arch GGUFs whose KV is already
+        // minimal (MQA + windowed SWA + shared-KV; dual head dims 512 global /
+        // 256 SWA) and which run on stock f16/q8_0 KV. The head_dim=128 QJL/TBQ
+        // kernels are dimensionally inapplicable to Gemma, so keep KV quant OFF
+        // (stock f16, only ~0.4 GB at 8k ctx for the 2B). This is the intended
+        // Gemma KV path, not a fallback.
+        setEnvIfAbsent("ELIZA_BIONIC_KV_QUANT", "0");
+        // Allow MTP speculative decode when a Gemma-4 SEPARATE drafter
+        // (mtp/drafter-<tier>.gguf, loaded via "-md … --spec-type draft-mtp") —
+        // NOT a same-file NextN head embedded in the text GGUF; the JNI threads
+        // cfg.mtp_drafter_path when the TS bridge has staged one. MTP stays
+        // dormant when no drafter path is supplied; the retired same-file path
+        // is intentionally not used for Gemma bundles. The resident path uses
+        // MTP + full reset;
+        // prefix-KV reuse via reset_keep (resetAndPrefillResident →
+        // nativeLlmStreamResetKeep) stays wired for caches that support bounded
+        // partial removal (llama_memory_seq_rm).
+        setEnvIfAbsent("ELIZA_BIONIC_MTP", "1");
+    }
+
+    private static void setEnvIfAbsent(String key, String value) {
+        try {
+            if (System.getenv(key) == null) {
+                Os.setenv(key, value, true);
+                Log.i(TAG, "set " + key + "=" + value + " for in-process bionic inference");
+            }
+        } catch (Throwable t) {
+            Log.w(TAG, "could not set " + key, t);
+        }
+    }
+
     synchronized void start() {
         if (running.get()) {
             return;
         }
+        applyBionicInferenceMemoryDefaults();
         // Load the fused native engine up front so the first request doesn't pay
         // the dlopen + Vulkan-device init; also fail fast + loud if the GPU host
         // isn't actually usable, so the agent's refuse-and-fallback can engage.
@@ -150,6 +209,14 @@ final class ElizaBionicInferenceServer {
                 if (requestJson == null) {
                     break;
                 }
+                // op="generateStream" server-pushes one frame per decode step on
+                // this same connection (handled inline so it can write many
+                // frames); every other op is one-request/one-response.
+                if ("generateStream".equals(opOf(requestJson))) {
+                    generateStreamRequest(requestJson, out);
+                    out.flush();
+                    continue;
+                }
                 String responseJson = handleRequest(requestJson);
                 writeFrame(out, responseJson);
                 out.flush();
@@ -176,13 +243,23 @@ final class ElizaBionicInferenceServer {
                 return tts(bundleDir, req.optString("text", ""),
                     (float) req.optDouble("speed", 1.0));
             }
+            if ("asr".equals(op)) {
+                return asr(bundleDir, req.optString("pcmBase64", ""),
+                    req.optInt("sampleRate", 16000));
+            }
+            if ("image".equals(op)) {
+                return describeImage(bundleDir, req.optString("imageBase64", ""),
+                    req.optString("mmprojPath", ""), req.optString("prompt", ""));
+            }
             if (!"generate".equals(op)) {
                 return errorJson("unsupported op: " + op);
             }
             String prompt = req.optString("prompt", "");
+            String drafterPath = req.optString("drafterPath", "");
             int maxTokens = req.optInt("maxTokens", 256);
             Log.i(TAG, "GENERATE from agent: " + prompt.length() + " prompt chars,"
-                + " maxTokens=" + maxTokens + ", bundle=" + bundleDir);
+                + " maxTokens=" + maxTokens + ", bundle=" + bundleDir
+                + ", drafter=" + (drafterPath.isEmpty() ? "(none)" : drafterPath));
             // RESIDENT path (default): the model + context stay loaded across turns;
             // only the KV cache + sampler are reset and the prompt re-prefilled per
             // turn, so we skip the ~7-8s model RELOAD that nativeLlmSelfTest paid every
@@ -194,7 +271,7 @@ final class ElizaBionicInferenceServer {
             // ELIZA_BIONIC_RESIDENT=0 to force the old path).
             if (!"0".equals(System.getenv("ELIZA_BIONIC_RESIDENT"))) {
                 try {
-                    String r = generateResident(bundleDir, prompt, maxTokens);
+                    String r = generateResident(bundleDir, drafterPath, prompt, maxTokens);
                     Log.i(TAG, "GENERATE result (resident): "
                         + (r.length() > 200 ? r.substring(0, 200) + "…" : r));
                     return r;
@@ -218,29 +295,12 @@ final class ElizaBionicInferenceServer {
      * we skip the ~7-8s model reload. Greedy decode (temp=0, top_k=1), all-GPU.
      * Returns the same {ok,tokens,ms,tokS,text} JSON as nativeLlmSelfTest.
      */
-    private String generateResident(String bundleDir, String prompt, int maxTokens)
+    private String generateResident(String bundleDir, String drafterPath, String prompt, int maxTokens)
             throws org.json.JSONException {
         synchronized (residentLock) {
-            if (residentCtx == 0L || !bundleDir.equals(residentBundle)) {
-                resetResident();
-                residentCtx = ElizaVoiceNative.nativeContextCreate(bundleDir);
-                if (residentCtx == 0L) {
-                    throw new IllegalStateException("resident contextCreate failed: " + bundleDir);
-                }
-                residentBundle = bundleDir;
-            }
-            if (residentStream == 0L) {
-                residentStream = ElizaVoiceNative.nativeLlmStreamOpen(
-                    residentCtx, RESIDENT_STREAM_MAX_TOKENS, 0.0f, 1.0f, 1, -1, "");
-                if (residentStream == 0L) {
-                    throw new IllegalStateException("resident streamOpen failed");
-                }
-            }
-            // Drop the previous turn's KV + sampler state for a clean re-prefill.
-            ElizaVoiceNative.nativeLlmStreamReset(residentStream);
-            int[] toks = ElizaVoiceNative.nativeTokenize(residentCtx, prompt, true, true);
+            ensureResidentCtx(bundleDir);
             final long t0 = android.os.SystemClock.elapsedRealtime();
-            ElizaVoiceNative.nativeLlmStreamPrefill(residentStream, toks);
+            resetAndPrefillResident(prompt, drafterPath);
             final StringBuilder sb = new StringBuilder();
             int produced = 0;
             final int cap = maxTokens > 0 ? maxTokens : 32;
@@ -266,6 +326,118 @@ final class ElizaBionicInferenceServer {
         }
     }
 
+    /** Cheap op discriminator without fully consuming the request. */
+    private static String opOf(String requestJson) {
+        try {
+            return new JSONObject(requestJson).optString("op", "generate");
+        } catch (org.json.JSONException e) {
+            return "generate";
+        }
+    }
+
+    /** Parse an op="generateStream" request and run the streaming decode. */
+    private void generateStreamRequest(String requestJson, DataOutputStream out)
+            throws IOException {
+        String bundleDir = defaultBundleDir;
+        String drafterPath = "";
+        String prompt = "";
+        int maxTokens = 256;
+        try {
+            JSONObject req = new JSONObject(requestJson);
+            bundleDir = req.optString("bundleDir", "");
+            if (bundleDir.isEmpty()) {
+                bundleDir = defaultBundleDir;
+            }
+            drafterPath = req.optString("drafterPath", "");
+            prompt = req.optString("prompt", "");
+            maxTokens = req.optInt("maxTokens", 256);
+        } catch (org.json.JSONException e) {
+            writeFrame(out, errorJson(e.getMessage() == null ? e.toString() : e.getMessage()));
+            return;
+        }
+        generateStream(bundleDir, drafterPath, prompt, maxTokens, out);
+    }
+
+    /**
+     * Streaming variant of {@link #generateResident}: the identical warm decode,
+     * but it writes one length-prefixed {type:"token",text} frame per decode step
+     * to {@code out} as tokens are produced, then a terminal
+     * {type:"done",ok,tokens,ms,tokS,text} frame. This lets the agent render
+     * tokens as they decode (first paint at the first token instead of after the
+     * whole reply) and unblocks phrase-chunked LLM→TTS. The buffered op="generate"
+     * is unchanged for non-streaming callers (embed/tts/self-test).
+     */
+    private void generateStream(String bundleDir, String drafterPath, String prompt, int maxTokens,
+                                DataOutputStream out) throws IOException {
+        Log.i(TAG, "GENERATE_STREAM from agent: " + prompt.length() + " prompt chars,"
+            + " maxTokens=" + maxTokens + ", bundle=" + bundleDir
+            + ", drafter=" + (drafterPath.isEmpty() ? "(none)" : drafterPath));
+        final StringBuilder sb = new StringBuilder();
+        try {
+            synchronized (residentLock) {
+                ensureResidentCtx(bundleDir);
+                final long t0 = android.os.SystemClock.elapsedRealtime();
+                resetAndPrefillResident(prompt, drafterPath);
+                int produced = 0;
+                final int cap = maxTokens > 0 ? maxTokens : 32;
+                while (produced < cap) {
+                    String stepJson = ElizaVoiceNative.nativeLlmStreamNext(residentStream);
+                    if (stepJson == null) break;
+                    JSONObject step = new JSONObject(stepJson);
+                    String t = step.optString("text", "");
+                    if (!t.isEmpty()) {
+                        sb.append(t);
+                        writeFrame(out, new JSONObject()
+                            .put("type", "token").put("text", t).toString());
+                        out.flush();
+                    }
+                    int nout = step.optInt("nout", 1);
+                    produced += nout > 0 ? nout : 1;
+                    if (step.optBoolean("done", false)) break;
+                }
+                final long ms = android.os.SystemClock.elapsedRealtime() - t0;
+                final double tokS = ms > 0 ? produced * 1000.0 / ms : 0.0;
+                writeFrame(out, new JSONObject()
+                    .put("type", "done").put("ok", true)
+                    .put("tokens", produced).put("ms", ms).put("tokS", tokS)
+                    .put("text", sb.toString()).put("resident", true).toString());
+                out.flush();
+                Log.i(TAG, "GENERATE_STREAM done (resident): " + produced + " tok @ "
+                    + String.format(java.util.Locale.US, "%.2f", tokS) + " tok/s");
+            }
+        } catch (Throwable t) {
+            Log.w(TAG, "generate_stream failed", t);
+            resetResident();
+            try {
+                writeFrame(out, new JSONObject()
+                    .put("type", "done").put("ok", false)
+                    .put("error", t.getMessage() == null ? t.toString() : t.getMessage())
+                    .toString());
+                out.flush();
+            } catch (Throwable ignored) {
+            }
+        }
+    }
+
+    /**
+     * Get-or-create the shared resident inference context. ONE model load is
+     * reused by both generation (via residentStream) and embeddings (the native
+     * EliInferenceContext caches a separate non-causal embed_ctx + the causal
+     * stream within the same shared model weights), so embeds no longer reload
+     * the 1.27 GB model per call. Caller must hold residentLock.
+     */
+    private long ensureResidentCtx(String bundleDir) {
+        if (residentCtx == 0L || !bundleDir.equals(residentBundle)) {
+            resetResident();
+            residentCtx = ElizaVoiceNative.nativeContextCreate(bundleDir);
+            if (residentCtx == 0L) {
+                throw new IllegalStateException("resident contextCreate failed: " + bundleDir);
+            }
+            residentBundle = bundleDir;
+        }
+        return residentCtx;
+    }
+
     /** Tear down the resident model/context/stream (on bundle change, failure, or stop). */
     private void resetResident() {
         synchronized (residentLock) {
@@ -278,35 +450,111 @@ final class ElizaBionicInferenceServer {
                 residentCtx = 0L;
             }
             residentBundle = null;
+            residentDrafterPath = "";
+            residentPrevTokens = null;
+        }
+    }
+
+    /**
+     * Reset the resident stream for a new turn and prefill the prompt, REUSING
+     * the KV of the longest common token prefix with the previous turn (the
+     * system + tool-schema block is identical turn-to-turn) so only the per-turn
+     * delta is decoded. On Mali's scalar-matmul prefill the prefix is the
+     * dominant per-turn cost, so this is the single biggest latency win. Falls
+     * back to a full reset (close+reopen on failure) when there is no reusable
+     * prefix or the stream can't be trimmed (e.g. an MTP stream). Caller holds
+     * residentLock.
+     */
+    private void resetAndPrefillResident(String prompt, String drafterPath) {
+        final String effectiveDrafterPath = drafterPath == null ? "" : drafterPath;
+        if (residentStream != 0L && !effectiveDrafterPath.equals(residentDrafterPath)) {
+            ElizaVoiceNative.nativeLlmStreamClose(residentStream);
+            residentStream = 0L;
+            residentPrevTokens = null;
+        }
+        if (residentStream == 0L) {
+            residentStream = ElizaVoiceNative.nativeLlmStreamOpen(
+                residentCtx, RESIDENT_STREAM_MAX_TOKENS, 0.0f, 1.0f, 1, -1,
+                effectiveDrafterPath);
+            if (residentStream == 0L) {
+                throw new IllegalStateException("resident streamOpen failed");
+            }
+            residentDrafterPath = effectiveDrafterPath;
+            residentPrevTokens = null;
+        }
+        final int[] toks = ElizaVoiceNative.nativeTokenize(residentCtx, prompt, true, true);
+        // Longest common token prefix with the previous turn, capped so at least
+        // one new token is prefilled (the decode samples from the last prefilled
+        // position's logits, so the suffix must be non-empty).
+        int lcp = 0;
+        if (residentPrevTokens != null) {
+            final int max = Math.min(residentPrevTokens.length, toks.length);
+            while (lcp < max && residentPrevTokens[lcp] == toks[lcp]) {
+                lcp++;
+            }
+            if (lcp >= toks.length) {
+                lcp = toks.length - 1;
+            }
+        }
+        int applied = lcp > 0
+            ? ElizaVoiceNative.nativeLlmStreamResetKeep(residentStream, lcp)
+            : -1;
+        if (applied < 0) {
+            // No reusable prefix (first turn / MTP / trim failure): full reset,
+            // close+reopen on failure.
+            if (ElizaVoiceNative.nativeLlmStreamReset(residentStream) != 1) {
+                ElizaVoiceNative.nativeLlmStreamClose(residentStream);
+                residentStream = ElizaVoiceNative.nativeLlmStreamOpen(
+                    residentCtx, RESIDENT_STREAM_MAX_TOKENS, 0.0f, 1.0f, 1, -1,
+                    effectiveDrafterPath);
+                if (residentStream == 0L) {
+                    throw new IllegalStateException("resident streamReopen failed");
+                }
+                residentDrafterPath = effectiveDrafterPath;
+            }
+            applied = 0;
+        }
+        final int[] suffix = (applied <= 0)
+            ? toks
+            : java.util.Arrays.copyOfRange(toks, applied, toks.length);
+        ElizaVoiceNative.nativeLlmStreamPrefill(residentStream, suffix);
+        residentPrevTokens = toks;
+        if (applied > 0) {
+            Log.i(TAG, "resident prefill reuse: kept " + applied + "/" + toks.length
+                + " prefix tokens, prefilled " + suffix.length + " delta");
         }
     }
 
     /**
-     * Embed text on the GPU via the fused model (--pooling last). Fresh context
-     * per call (single forward pass, no autoregressive decode — fast + clean).
-     * Returns {ok, embedding:[...], dim}. This is what lets on-device memory /
-     * doc-seeding run locally instead of failing over to cloud BatchEmbeddings.
+     * Embed text on the GPU via the fused model (--pooling last). Reuses the
+     * shared resident context (the native side caches a non-causal embed_ctx
+     * inside it) so the 1.27 GB model is NOT reloaded per call — previously every
+     * embed did contextCreate→embed→contextDestroy (~15 s + a full model copy of
+     * memory churn each), which starved the LLM context on 8 GB devices. Single
+     * forward pass, no autoregressive decode. Returns {ok, embedding:[...], dim}.
      */
     private String embed(String bundleDir, String text) throws org.json.JSONException {
         final int POOLING_LAST = 3;
-        long ctx = ElizaVoiceNative.nativeContextCreate(bundleDir);
-        if (ctx == 0L) {
-            return errorJson("embed: failed to create context for " + bundleDir);
-        }
-        try {
-            float[] emb = ElizaVoiceNative.nativeEmbed(ctx, text, POOLING_LAST);
-            org.json.JSONArray arr = new org.json.JSONArray();
-            for (float v : emb) {
-                arr.put((double) v);
+        synchronized (residentLock) {
+            final long ctx = ensureResidentCtx(bundleDir);
+            try {
+                float[] emb = ElizaVoiceNative.nativeEmbed(ctx, text, POOLING_LAST);
+                org.json.JSONArray arr = new org.json.JSONArray();
+                for (float v : emb) {
+                    arr.put((double) v);
+                }
+                Log.i(TAG, "EMBED from agent: " + text.length() + " chars -> dim " + emb.length);
+                return new JSONObject()
+                    .put("ok", true)
+                    .put("embedding", arr)
+                    .put("dim", emb.length)
+                    .toString();
+            } catch (Throwable t) {
+                // A failed embed may leave the shared context in an unknown state;
+                // drop it so the next generate/embed rebuilds cleanly.
+                resetResident();
+                throw t;
             }
-            Log.i(TAG, "EMBED from agent: " + text.length() + " chars -> dim " + emb.length);
-            return new JSONObject()
-                .put("ok", true)
-                .put("embedding", arr)
-                .put("dim", emb.length)
-                .toString();
-        } finally {
-            ElizaVoiceNative.nativeContextDestroy(ctx);
         }
     }
 
@@ -329,30 +577,122 @@ final class ElizaBionicInferenceServer {
         if (gguf == null || voiceBin == null) {
             return errorJson("tts: Kokoro GGUF + voice .bin not found under " + kokoroDir);
         }
-        long ctx = ElizaVoiceNative.nativeContextCreate(bundleDir);
-        if (ctx == 0L) {
-            return errorJson("tts: failed to create context for " + bundleDir);
+        // Reuse the ONE resident context (the 1.27 GB model is already loaded for
+        // generation/embeddings) instead of contextCreate/Destroy per call — a
+        // fresh context reloaded the whole model every utterance. Kokoro itself
+        // is loaded once and cached on the ctx (idempotent kokoro_load), so a
+        // multi-clause reply synthesizes each clause without any reload.
+        synchronized (residentLock) {
+            final long ctx = ensureResidentCtx(bundleDir);
+            try {
+                float[] pcm = ElizaVoiceNative.nativeKokoroSynthesize(
+                    ctx, gguf, voiceBin, text, speed <= 0f ? 1.0f : speed);
+                int sampleRate = ElizaVoiceNative.nativeKokoroSampleRate(ctx);
+                // Pack fp32 PCM little-endian and base64 it for the JSON frame.
+                ByteBuffer buf = ByteBuffer.allocate(pcm.length * 4).order(ByteOrder.LITTLE_ENDIAN);
+                for (float v : pcm) {
+                    buf.putFloat(v);
+                }
+                String b64 = Base64.encodeToString(buf.array(), Base64.NO_WRAP);
+                Log.i(TAG, "TTS (kokoro) from agent: " + text.length() + " chars -> "
+                    + pcm.length + " samples @ " + sampleRate + " Hz");
+                return new JSONObject()
+                    .put("ok", true)
+                    .put("sampleRate", sampleRate)
+                    .put("samples", pcm.length)
+                    .put("pcmBase64", b64)
+                    .toString();
+            } catch (Throwable t) {
+                // A failed synth may leave the shared ctx in an unknown state;
+                // drop it so the next generate/embed/tts rebuilds cleanly.
+                resetResident();
+                throw t;
+            }
         }
-        try {
-            float[] pcm = ElizaVoiceNative.nativeKokoroSynthesize(
-                ctx, gguf, voiceBin, text, speed <= 0f ? 1.0f : speed);
-            int sampleRate = ElizaVoiceNative.nativeKokoroSampleRate(ctx);
-            // Pack fp32 PCM little-endian and base64 it for the JSON frame.
-            ByteBuffer buf = ByteBuffer.allocate(pcm.length * 4).order(ByteOrder.LITTLE_ENDIAN);
-            for (float v : pcm) {
-                buf.putFloat(v);
+    }
+
+    /**
+     * On-device STT: decode the base64 little-endian fp32 PCM, run the fused
+     * local ASR batch transcribe on the resident context (it mmap-acquires the
+     * {@code asr/} weights on first use), and return {ok, text}. The agent's
+     * TRANSCRIPTION delegate routes here over UDS (op="asr"); the musl agent
+     * can't reach the fused lib itself. Reuses the ONE resident context like
+     * embed/tts so the model is not reloaded per call.
+     */
+    private String asr(String bundleDir, String pcmBase64, int sampleRate)
+            throws org.json.JSONException {
+        if (pcmBase64.isEmpty()) {
+            return errorJson("asr: empty pcmBase64");
+        }
+        byte[] raw = Base64.decode(pcmBase64, Base64.DEFAULT);
+        final int n = raw.length / 4;
+        if (n <= 0) {
+            return errorJson("asr: pcmBase64 decoded to " + raw.length + " bytes (need fp32 PCM)");
+        }
+        float[] pcm = new float[n];
+        ByteBuffer bb = ByteBuffer.wrap(raw).order(ByteOrder.LITTLE_ENDIAN);
+        for (int i = 0; i < n; i++) {
+            pcm[i] = bb.getFloat();
+        }
+        final int sr = sampleRate > 0 ? sampleRate : 16000;
+        synchronized (residentLock) {
+            final long ctx = ensureResidentCtx(bundleDir);
+            try {
+                String text = ElizaVoiceNative.nativeAsrTranscribe(ctx, pcm, sr);
+                Log.i(TAG, "ASR from agent: " + n + " samples @ " + sr + " Hz -> \""
+                    + (text.length() > 200 ? text.substring(0, 200) + "…" : text) + "\"");
+                return new JSONObject()
+                    .put("ok", true)
+                    .put("text", text)
+                    .toString();
+            } catch (Throwable t) {
+                // A failed transcribe may leave the shared ctx in an unknown state;
+                // drop it so the next generate/embed/tts/asr rebuilds cleanly.
+                resetResident();
+                throw t;
+            }
+        }
+    }
+
+    /**
+     * On-device vision / screen-recognition: decode the base64 image bytes and
+     * run the fused mmproj describe-image on the resident TEXT model. When the
+     * caller doesn't pass an explicit {@code mmprojPath}, resolve the projector
+     * GGUF from the bundle's {@code vision/} dir. Returns {ok, text}. The agent's
+     * IMAGE_DESCRIPTION delegate routes here over UDS (op="image").
+     */
+    private String describeImage(String bundleDir, String imageBase64,
+            String mmprojPath, String prompt) throws org.json.JSONException {
+        if (imageBase64.isEmpty()) {
+            return errorJson("image: empty imageBase64");
+        }
+        byte[] img = Base64.decode(imageBase64, Base64.DEFAULT);
+        if (img.length == 0) {
+            return errorJson("image: imageBase64 decoded to 0 bytes");
+        }
+        String mmproj = mmprojPath;
+        if (mmproj == null || mmproj.isEmpty()) {
+            File visionDir = new File(bundleDir, "vision");
+            mmproj = firstMatch(visionDir, ".gguf");
+            if (mmproj == null) {
+                return errorJson("image: mmproj GGUF not found under " + visionDir
+                    + " (stage a vision projector or pass mmprojPath)");
+            }
+        }
+        synchronized (residentLock) {
+            final long ctx = ensureResidentCtx(bundleDir);
+            try {
+                String desc = ElizaVoiceNative.nativeDescribeImage(ctx, img, mmproj, prompt);
+                Log.i(TAG, "IMAGE from agent: " + img.length + " bytes (mmproj " + mmproj + ") -> \""
+                    + (desc.length() > 200 ? desc.substring(0, 200) + "…" : desc) + "\"");
+                return new JSONObject()
+                    .put("ok", true)
+                    .put("text", desc)
+                    .toString();
+            } catch (Throwable t) {
+                resetResident();
+                throw t;
             }
-            String b64 = Base64.encodeToString(buf.array(), Base64.NO_WRAP);
-            Log.i(TAG, "TTS (kokoro) from agent: " + text.length() + " chars -> "
-                + pcm.length + " samples @ " + sampleRate + " Hz");
-            return new JSONObject()
-                .put("ok", true)
-                .put("sampleRate", sampleRate)
-                .put("samples", pcm.length)
-                .put("pcmBase64", b64)
-                .toString();
-        } finally {
-            ElizaVoiceNative.nativeContextDestroy(ctx);
         }
     }
 

package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaVoiceNative.java

@@ -144,6 +144,9 @@ final class ElizaVoiceNative {
     /** Read the diariz int8 frame labels for the i-th turn of the last call. */
     static native byte[] nativePipelineTurnLabels(long handle, int index);
 
+    /** Read the segmented fp32 PCM for the i-th turn of the last call. */
+    static native float[] nativePipelineTurnPcm(long handle, int index);
+
     static native void nativePipelineReset(long handle);
 
     static native void nativePipelineClose(long handle);
@@ -198,6 +201,14 @@ final class ElizaVoiceNative {
     /** Reset a persistent stream (clear KV + sampler) for warm reuse. 1=ok, 0=no. */
     static native int nativeLlmStreamReset(long streamHandle);
 
+    /**
+     * Prefix-preserving reset: keep the first {@code nKeep} tokens of KV cache
+     * resident and drop the rest, so the next prefill only decodes the per-turn
+     * delta. Returns the n_keep actually applied ({@code >= 0}), or a negative
+     * code on a null/MTP/unopened stream (caller falls back to a full reset).
+     */
+    static native int nativeLlmStreamResetKeep(long streamHandle, int nKeep);
+
     /**
      * KEYSTONE proof: run a whole greedy text generation in one native call,
      * in the bionic app process. With the dynamic-Vulkan lib staged, ggml-vulkan
@@ -217,4 +228,28 @@ final class ElizaVoiceNative {
      */
     static native float[] nativeKokoroSynthesize(
             long ctxHandle, String ggufPath, String voiceBinPath, String text, float speed);
+
+    // ── Batch ASR + mmproj vision (the agent's STT / screen-recognition path) ──
+
+    /**
+     * Transcribe {@code pcm} (16 kHz mono fp32) → UTF-8 transcript via the fused
+     * local ASR head ({@code eliza_inference_asr_transcribe}). VAD-free: the
+     * resident context mmap-acquires the {@code asr/} weights on first use. The
+     * bionic host's op="asr" calls this so the agent TRANSCRIPTION delegate gets
+     * a real on-device transcript without the full attribution pipeline.
+     */
+    static native String nativeAsrTranscribe(long ctxHandle, float[] pcm, int sampleRate);
+
+    /** {@code eliza_inference_vision_supported()} (ABI v9): 1 if mmproj vision is built. */
+    static native int nativeVisionSupported();
+
+    /**
+     * Describe a raw PNG/JPEG/WebP image with the resident TEXT model + the
+     * mmproj projector at {@code mmprojPath} ({@code eliza_inference_describe_image}).
+     * {@code prompt} may be empty (a default describe prompt is used). The bionic
+     * host's op="image" calls this so the agent IMAGE_DESCRIPTION delegate runs
+     * screen/vision recognition fully on-device.
+     */
+    static native String nativeDescribeImage(
+            long ctxHandle, byte[] imageBytes, String mmprojPath, String prompt);
 }