@elizaos/app-core 2.0.3-beta.2 → 2.0.3-beta.4

package/platforms/android/app/src/main/elizavoice-jni/elizavoice-jni.cpp

@@ -226,6 +226,7 @@ struct PipelineSession {
     // read them without re-parsing big JSON.
     std::vector<std::vector<float>> turnEmbeddings;
     std::vector<std::vector<int8_t>> turnLabels;
+    std::vector<std::vector<float>> turnPcms;
 };
 
 // Append `n` samples to a bounded pre-roll ring, dropping oldest beyond cap.
@@ -321,6 +322,7 @@ bool finalize_turn(PipelineSession* s, char** outError) {
                                               : std::vector<float>{});
     s->turnLabels.push_back(haveLabels ? std::move(labels)
                                        : std::vector<int8_t>{});
+    s->turnPcms.push_back(std::move(s->turnPcm));
     s->turnPcm.clear();
     return true;
 }
@@ -405,49 +407,60 @@ bool bionic_bool_env_or_default(const char* name, bool fallback) {
     return fallback;
 }
 
-// Arm the eliza-1 text-model bionic config: the fused QJL K-cache /
-// TBQ3 V-cache KV-quant (so the device actually uses the fused QJL attention
-// route verified on lavapipe) and the same-file MTP draft window.
+int bionic_int_env_or_default(const char* name, int fallback) {
+    const char* v = std::getenv(name);
+    if (!v || v[0] == '\0') return fallback;
+    int parsed = std::atoi(v);
+    return parsed > 0 ? parsed : fallback;
+}
+
+// Arm the eliza-1 text-model bionic config for runtime optimizations that are
+// correct for the shipped model.
 //
-// KV-quant: ON by default (ELIZA_BIONIC_KV_QUANT != 0). Sets
-//   cache_type_k = "qjl1_256" (256-dim JL sketch matching quants-qjl.c's
-//                              process-global default Π, seed=42, head_dim=128)
-//   cache_type_v = "tbq3_0"   (TurboQuant-3 V-cache)
-// matching CatalogModel.runtime.kvCache (catalog.ts). When set to 0 the fields
-// stay NULL → the FFI leaves the llama.cpp f16 default (safe fallback).
+// KV-quant stays OFF by default. The shipped Eliza-1 tiers are Gemma 4, whose
+// KV is already minimal by construction (MQA + windowed-SWA + shared-KV, dual
+// head dims 512 global / 256 SWA) and runs on stock f16/q8_0 KV. The legacy
+// QJL1_256 / fused QJL-TBQ kernels are head_dim=128 and dimensionally
+// inapplicable to Gemma, so cache_type_k=qjl1_256/cache_type_v=tbq3_0 is not a
+// shipping path; F16 KV is correct. ELIZA_BIONIC_KV_QUANT=1 is left as an
+// explicit lab override for head_dim=128 test bundles only.
 //
-// MTP: OFF by default (no NextN draft on the keystone path). When
-//   ELIZA_BIONIC_MTP=1 the same-file MTP draft window is armed
-//   (draft_min/draft_max from ELIZA_BIONIC_MTP_DRAFT_MIN/MAX, default 1/2 =
-//   the single-NextN-head throughput peak). The 0.8b tier has no NextN head,
-//   so the FFI's same-file MTP path falls back to the plain decode loop there —
-//   tier gating is the bootstrap's job (ELIZA_1_LOAD_METADATA); this is the
-//   native override seam.
+// MTP is enabled only when the caller supplies a Gemma separate-drafter GGUF.
+// Omitting cfg.mtp_drafter_path would select the retired same-file NextN path,
+// so shipped Gemma bundles without a staged drafter run plain decode even if
+// ELIZA_BIONIC_MTP is set in the app process.
 //
 // The two static names below outlive the cfg they're attached to (cfg is a
 // stack struct consumed synchronously by eliza_inference_llm_stream_open).
-void arm_bionic_text_cfg(eliza_llm_stream_config_t& cfg) {
+void arm_bionic_text_cfg(eliza_llm_stream_config_t& cfg,
+                         const char* bundle_dir = nullptr) {
     static const char* kKvTypeK = "qjl1_256";
     static const char* kKvTypeV = "tbq3_0";
-    if (bionic_bool_env_or_default("ELIZA_BIONIC_KV_QUANT", true)) {
+    if (bionic_bool_env_or_default("ELIZA_BIONIC_KV_QUANT", false)) {
         cfg.cache_type_k = kKvTypeK;
         cfg.cache_type_v = kKvTypeV;
-        LOGI("bionic text cfg: KV-quant ON (k=%s v=%s)", kKvTypeK, kKvTypeV);
+        LOGI("bionic text cfg: KV-quant ON by explicit override (k=%s v=%s)",
+             kKvTypeK, kKvTypeV);
     } else {
-        LOGI("bionic text cfg: KV-quant OFF (f16 fallback)");
+        LOGI("bionic text cfg: KV-quant OFF (stock f16/q8_0 KV; Gemma 4 KV is "
+             "already minimal)");
     }
 
-    if (bionic_bool_env_or_default("ELIZA_BIONIC_MTP", false)) {
-        const char* dmin = std::getenv("ELIZA_BIONIC_MTP_DRAFT_MIN");
-        const char* dmax = std::getenv("ELIZA_BIONIC_MTP_DRAFT_MAX");
-        int draft_min = dmin && dmin[0] ? std::atoi(dmin) : 1;
-        int draft_max = dmax && dmax[0] ? std::atoi(dmax) : 2;
+    (void)bundle_dir;
+    const bool has_drafter =
+        cfg.mtp_drafter_path && cfg.mtp_drafter_path[0] != '\0';
+    if (has_drafter && bionic_bool_env_or_default("ELIZA_BIONIC_MTP", true)) {
+        int draft_min = bionic_int_env_or_default("ELIZA_BIONIC_MTP_DRAFT_MIN", 1);
+        int draft_max = bionic_int_env_or_default("ELIZA_BIONIC_MTP_DRAFT_MAX", 1);
         if (draft_min < 1) draft_min = 1;
         if (draft_max < draft_min) draft_max = draft_min;
         cfg.draft_min = draft_min;
         cfg.draft_max = draft_max;
-        LOGI("bionic text cfg: MTP ON (draft_min=%d draft_max=%d, same-file)",
-             draft_min, draft_max);
+        LOGI("bionic text cfg: MTP ON (draft_min=%d draft_max=%d, drafter=%s)",
+             draft_min, draft_max, cfg.mtp_drafter_path);
+    } else {
+        LOGI("bionic text cfg: MTP OFF (%s)",
+             has_drafter ? "disabled by ELIZA_BIONIC_MTP" : "no drafter");
     }
 }
 
@@ -797,6 +810,7 @@ Java_ai_elizaos_app_ElizaVoiceNative_nativePipelineProcess(JNIEnv* env, jclass,
     s->turns.clear();
     s->turnEmbeddings.clear();
     s->turnLabels.clear();
+    s->turnPcms.clear();
 
     const std::vector<float> pcm = read_float_array(env, jPcm);
     s->pending.insert(s->pending.end(), pcm.begin(), pcm.end());
@@ -825,6 +839,7 @@ Java_ai_elizaos_app_ElizaVoiceNative_nativePipelineFlush(JNIEnv* env, jclass,
     s->turns.clear();
     s->turnEmbeddings.clear();
     s->turnLabels.clear();
+    s->turnPcms.clear();
     if (s->seg.forceEnd()) {
         s->capturing = false;
         char* outError = nullptr;
@@ -880,6 +895,26 @@ Java_ai_elizaos_app_ElizaVoiceNative_nativePipelineTurnLabels(JNIEnv* env,
     return out;
 }
 
+// Read the segmented turn PCM for the i-th turn. Empty array when unavailable.
+JNIEXPORT jfloatArray JNICALL
+Java_ai_elizaos_app_ElizaVoiceNative_nativePipelineTurnPcm(JNIEnv* env,
+                                                           jclass,
+                                                           jlong handle,
+                                                           jint index) {
+    auto* s = reinterpret_cast<PipelineSession*>(handle);
+    if (!s || index < 0 ||
+        static_cast<size_t>(index) >= s->turnPcms.size()) {
+        return env->NewFloatArray(0);
+    }
+    const auto& pcm = s->turnPcms[static_cast<size_t>(index)];
+    jfloatArray out = env->NewFloatArray(static_cast<jsize>(pcm.size()));
+    if (out && !pcm.empty()) {
+        env->SetFloatArrayRegion(out, 0, static_cast<jsize>(pcm.size()),
+                                 pcm.data());
+    }
+    return out;
+}
+
 JNIEXPORT void JNICALL
 Java_ai_elizaos_app_ElizaVoiceNative_nativePipelineReset(JNIEnv*, jclass,
                                                          jlong handle) {
@@ -888,6 +923,7 @@ Java_ai_elizaos_app_ElizaVoiceNative_nativePipelineReset(JNIEnv*, jclass,
     s->seg.reset();
     s->pending.clear();
     s->turnPcm.clear();
+    s->turnPcms.clear();
     s->preRoll.clear();
     s->capturing = false;
     char* outError = nullptr;
@@ -1222,16 +1258,14 @@ Java_ai_elizaos_app_ElizaVoiceNative_nativeLlmStreamOpen(
     cfg.repeat_penalty = 1.0f;
     cfg.n_gpu_layers = nGpuLayers;
     cfg.mtp_drafter_path = drafter.empty() ? nullptr : drafter.c_str();
-    // Arm the fused QJL/TBQ3 KV-quant + (env-gated) same-file MTP draft window.
-    // A caller-supplied drafter path is separate-drafter MTP and needs its own
-    // draft window — only arm the same-file MTP env override when no drafter
-    // was passed.
+    // Arm safe runtime optimizations: stock f16/q8_0 KV for the shipped Gemma 4
+    // tiers, and separate-drafter MTP only when the caller passes a drafter.
     arm_bionic_text_cfg(cfg);
     if (!drafter.empty() && cfg.draft_min <= 0) {
         // Separate drafter supplied but env left the window 0; give it the
         // single-head default so the drafter actually drives speculation.
         cfg.draft_min = 1;
-        cfg.draft_max = 2;
+        cfg.draft_max = 1;
     }
     char* outError = nullptr;
     EliLlmStream* s = eliza_inference_llm_stream_open(ctx, &cfg, &outError);
@@ -1321,6 +1355,18 @@ Java_ai_elizaos_app_ElizaVoiceNative_nativeLlmStreamReset(JNIEnv*, jclass,
     return static_cast<jint>(rc == ELIZA_OK ? 1 : 0);
 }
 
+// Prefix-preserving reset: keep the first nKeep tokens of KV resident, drop the
+// rest. Returns the n_keep actually applied (>= 0), or a negative ELIZA_* on a
+// NULL / MTP / unopened stream (caller falls back to a full reset + prefill).
+JNIEXPORT jint JNICALL
+Java_ai_elizaos_app_ElizaVoiceNative_nativeLlmStreamResetKeep(JNIEnv*, jclass,
+                                                              jlong streamHandle,
+                                                              jint nKeep) {
+    const int rc = eliza_inference_llm_stream_reset_keep(
+        reinterpret_cast<EliLlmStream*>(streamHandle), static_cast<int32_t>(nKeep));
+    return static_cast<jint>(rc);
+}
+
 // ── LLM self-test (one native call: ctx→tokenize→stream→generate) ─────────
 //
 // THE KEYSTONE PROOF: runs a whole greedy text generation in ONE native call,
@@ -1358,9 +1404,9 @@ Java_ai_elizaos_app_ElizaVoiceNative_nativeLlmSelfTest(JNIEnv* env, jclass,
     cfg.top_p = 1.0f;
     cfg.repeat_penalty = 1.0f;
     cfg.n_gpu_layers = -1;   // all-GPU when the vulkan lib is staged
-    // Arm the fused QJL/TBQ3 KV-quant + (env-gated) same-file MTP draft window
-    // so the keystone generation path exercises the fused QJL attention route.
-    arm_bionic_text_cfg(cfg);
+    // Arm safe runtime optimizations (stock f16/q8_0 KV for shipped Gemma 4
+    // tiers; no MTP here because nativeLlmSelfTest has no drafter argument).
+    arm_bionic_text_cfg(cfg, bundleDir.c_str());
     EliLlmStream* s = eliza_inference_llm_stream_open(ctx, &cfg, &outError);
     if (!s) {
         if (tok) eliza_inference_free_tokens(tok);
@@ -1482,4 +1528,102 @@ Java_ai_elizaos_app_ElizaVoiceNative_nativeKokoroSynthesize(JNIEnv* env, jclass,
     return out;
 }
 
+// ── Batch ASR (synchronous, VAD-free) ────────────────────────────────────
+//
+// The streaming pipeline (nativePipelineProcess) is VAD-gated and needs the
+// VAD/diariz/speaker GGUFs staged; this is the DIRECT audio-in/text-out
+// transcribe the fused lib exposes (eliza_inference_asr_transcribe), which
+// only mmap-acquires the `asr/` weights on the resident context. The bionic
+// host's op="asr" calls this so the agent's TRANSCRIPTION delegate gets a
+// real on-device transcript without the full attribution pipeline.
+JNIEXPORT jstring JNICALL
+Java_ai_elizaos_app_ElizaVoiceNative_nativeAsrTranscribe(JNIEnv* env, jclass,
+                                                         jlong ctxHandle,
+                                                         jfloatArray jPcm,
+                                                         jint sampleRate) {
+    auto* ctx = reinterpret_cast<EliInferenceContext*>(ctxHandle);
+    if (!ctx) {
+        throw_runtime(env, "asrTranscribe: null context", nullptr);
+        return nullptr;
+    }
+    const jsize n = env->GetArrayLength(jPcm);
+    jfloat* pcm = env->GetFloatArrayElements(jPcm, nullptr);
+    if (!pcm) {
+        throw_runtime(env, "asrTranscribe: null PCM", nullptr);
+        return nullptr;
+    }
+    // The ASR weights are a voice-only mmap region that must be armed before
+    // transcribe (VoiceLifecycle normally does this on voice-on). The agent's
+    // one-shot transcribe path has no lifecycle, so arm it here (idempotent if
+    // already acquired).
+    char* acqErr = nullptr;
+    if (eliza_inference_mmap_acquire(ctx, "asr", &acqErr) != 0) {
+        env->ReleaseFloatArrayElements(jPcm, pcm, JNI_ABORT);
+        throw_runtime(env, "asr mmap_acquire", acqErr);
+        return nullptr;
+    }
+    // 64 KiB transcript cap — far longer than any single utterance.
+    std::vector<char> out(65536, 0);
+    char* err = nullptr;
+    const int rc = eliza_inference_asr_transcribe(
+        ctx, reinterpret_cast<const float*>(pcm), static_cast<size_t>(n),
+        sampleRate > 0 ? sampleRate : kSampleRate, out.data(), out.size(),
+        &err);
+    env->ReleaseFloatArrayElements(jPcm, pcm, JNI_ABORT);
+    if (rc < 0) {
+        throw_runtime(env, "asr_transcribe", err);
+        return nullptr;
+    }
+    LOGI("nativeAsrTranscribe: %d samples @ %d Hz -> %d transcript bytes",
+         (int)n, (int)sampleRate, rc);
+    return to_jstring(env, std::string(out.data()));
+}
+
+// ── mmproj vision (ABI v9) ───────────────────────────────────────────────
+JNIEXPORT jint JNICALL
+Java_ai_elizaos_app_ElizaVoiceNative_nativeVisionSupported(JNIEnv*, jclass) {
+    return static_cast<jint>(eliza_inference_vision_supported());
+}
+
+// Describe a raw PNG/JPEG/WebP image with the resident TEXT model + an mmproj
+// projector (eliza_inference_describe_image). The bionic host's op="image"
+// calls this so the agent's IMAGE_DESCRIPTION delegate runs screen/vision
+// recognition fully on-device.
+JNIEXPORT jstring JNICALL
+Java_ai_elizaos_app_ElizaVoiceNative_nativeDescribeImage(JNIEnv* env, jclass,
+                                                         jlong ctxHandle,
+                                                         jbyteArray jImage,
+                                                         jstring jMmproj,
+                                                         jstring jPrompt) {
+    auto* ctx = reinterpret_cast<EliInferenceContext*>(ctxHandle);
+    if (!ctx) {
+        throw_runtime(env, "describeImage: null context", nullptr);
+        return nullptr;
+    }
+    const std::string mmproj = from_jstring(env, jMmproj);
+    const std::string prompt = from_jstring(env, jPrompt);
+    const jsize n = env->GetArrayLength(jImage);
+    jbyte* img = env->GetByteArrayElements(jImage, nullptr);
+    if (!img) {
+        throw_runtime(env, "describeImage: null image bytes", nullptr);
+        return nullptr;
+    }
+    std::vector<char> out(16384, 0);
+    char* err = nullptr;
+    const int rc = eliza_inference_describe_image(
+        ctx, reinterpret_cast<const unsigned char*>(img),
+        static_cast<size_t>(n),
+        mmproj.empty() ? nullptr : mmproj.c_str(),
+        prompt.empty() ? nullptr : prompt.c_str(), out.data(), out.size(),
+        &err);
+    env->ReleaseByteArrayElements(jImage, img, JNI_ABORT);
+    if (rc < 0) {
+        throw_runtime(env, "describe_image", err);
+        return nullptr;
+    }
+    LOGI("nativeDescribeImage: %d image bytes -> %d description bytes", (int)n,
+         rc);
+    return to_jstring(env, std::string(out.data()));
+}
+
 }  // extern "C"

package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaAgentService.java

@@ -1115,7 +1115,56 @@ public class ElizaAgentService extends Service {
 
     // ── Process lifecycle ────────────────────────────────────────────────
 
+    /**
+     * Start the in-process bionic inference host whenever the fused native lib
+     * AND a local voice bundle (kokoro) are present — independent of the LLM
+     * runtime mode and of whether the agent process spawns fresh or adopts a
+     * surviving one.
+     *
+     * <p>Previously the host was started ONLY inside {@link #startAgentProcess()}'s
+     * fresh-spawn {@code delegateToBionicHost} block, so after a process restart
+     * that adopts a surviving agent (or when the agent runs cloud-routed) the
+     * host never bound its abstract UDS, and {@code TalkMode}'s local Kokoro TTS
+     * silently fell back to the system (Google) TTS. The voice host serves Kokoro
+     * from {@code libelizainference} + the staged bundle and needs no LLM, so its
+     * lifecycle is decoupled from agent inference here.
+     *
+     * <p>{@link ElizaBionicInferenceServer#start()} is idempotent and never throws
+     * (it logs + returns on lib-load / socket-bind failure), so this is purely
+     * additive: worst case the host stays down and TalkMode falls back to system
+     * TTS exactly as before — it cannot affect agent startup.
+     */
+    private synchronized void ensureBionicVoiceHost() {
+        if (bionicInferenceServer != null) {
+            return;
+        }
+        File fusedLib = new File(nativeLibraryDir(), "libelizainference.so");
+        if (!fusedLib.isFile()) {
+            return;
+        }
+        File kokoroVoice = new File(getFilesDir(), "eliza-1/bundle/tts/kokoro");
+        if (!kokoroVoice.isDirectory()) {
+            return;
+        }
+        try {
+            String defaultBundleDir =
+                new File(getFilesDir(), "eliza-1/bundle").getAbsolutePath();
+            ElizaBionicInferenceServer host =
+                new ElizaBionicInferenceServer(BIONIC_INFERENCE_SOCKET_NAME, defaultBundleDir);
+            host.start();
+            bionicInferenceServer = host;
+            Log.i(TAG, "ensureBionicVoiceHost: local voice host started"
+                + " (kokoro bundle present, independent of LLM mode)");
+        } catch (Throwable t) {
+            Log.w(TAG, "ensureBionicVoiceHost: could not start local voice host", t);
+        }
+    }
+
     private void requestAgentStart(boolean restartFirst) {
+        // Bring up the local-voice (Kokoro TTS) bionic host on every service
+        // start, before the agent-already-running guards below — so it binds
+        // even when the agent is adopted rather than freshly spawned.
+        ensureBionicVoiceHost();
         synchronized (processLock) {
             if (!restartFirst && agentProcess != null && agentProcess.isAlive()) {
                 return;
@@ -1393,10 +1442,11 @@ public class ElizaAgentService extends Service {
             // (tryBuildAospFusedTextLoader) and aosp-llama-paths.ts
             // (isAospEnabled reads ELIZA_LOCAL_LLAMA).
             // This is required, not optional: the eliza-1 model tiers are
-            // qwen35-arch + QJL/PolarQuant/TBQ-quantized, and the stock
-            // llama-cpp-capacitor JNI lib cannot load those GGUFs at all
-            // (`context->loadModel() returned false`). The fused
-            // libelizainference.so carries the kernels + qwen35 arch and is
+            // Gemma-4-arch GGUFs (TurboQuant-quantized; stock f16/q8_0 KV — the
+            // legacy QJL/PolarQuant/TBQ KV kernels are retired post-#9033), and
+            // the stock llama-cpp-capacitor JNI lib cannot load those GGUFs at
+            // all (`context->loadModel() returned false`). The fused
+            // libelizainference.so carries the kernels + Gemma arch and is
             // the SOLE text/voice native library the bun agent loads.
             //
             // This was previously gated on `BuildConfig.AOSP_BUILD &&
@@ -1417,7 +1467,6 @@ public class ElizaAgentService extends Service {
             File abiFusedInference = new File(abiDir, "libelizainference.so");
             File abiLibllama = new File(abiDir, "libllama.so");
             File abiLlamaShim = new File(abiDir, "libeliza-llama-shim.so");
-            File abiSpeculativeShim = new File(abiDir, "libeliza-llama-speculative-shim.so");
             File abiGgmlVulkan = new File(abiDir, "libggml-vulkan.so");
             boolean fusedInferenceBundled = abiFusedInference.isFile();
             boolean legacyLibllamaBundled = abiLibllama.isFile() && abiLlamaShim.isFile();
@@ -1501,33 +1550,12 @@ public class ElizaAgentService extends Service {
                     // cache priming.
                     agentEnv.put("ELIZA_KOKORO_PREWARM_DELAY_MS", "60000");
                 }
-                if (abiSpeculativeShim.isFile()
-                        && !env.containsKey("ELIZA_SPEC_TYPE")
-                        && !env.containsKey("ELIZA_SPECULATIVE_TYPE")) {
-                    // Prefer the in-process MTP path when the bundled
-                    // llama.cpp fork supports it. This is intentionally not
-                    // required: current mobile Eliza-1 bundles ship MTP
-                    // drafter GGUFs, and the adapter falls back to MTP when
-                    // the target model lacks NextN/MTP tensors.
-                    agentEnv.put("ELIZA_SPEC_TYPE", "mtp");
-                    Log.i(TAG, "agent/" + abiDir.getName()
-                        + "/libeliza-llama-speculative-shim.so present; preferring MTP speculative decode with MTP fallback");
-                }
-                if (abiSpeculativeShim.isFile() && brandedAospBuild && !env.containsKey("ELIZA_MTP")) {
-                    agentEnv.put("ELIZA_MTP", "1");
-                    Log.i(TAG, "agent/" + abiDir.getName()
-                        + "/libeliza-llama-speculative-shim.so present on branded AOSP build; enabling in-process MTP (ELIZA_MTP=1)");
-                } else if (abiSpeculativeShim.isFile() && !brandedAospBuild) {
-                    // Ship the speculative shim in stock APKs so explicit
-                    // ELIZA_MTP diagnostics still work, but do not make
-                    // MTP the default merely because the .so is present.
-                    // Pixel APK validation showed the current 2B drafter path
-                    // is functional but slower than target-only decode for
-                    // short chat turns, so stock Android stays target-only
-                    // until benchmark gating or a faster drafter path lands.
-                    Log.i(TAG, "agent/" + abiDir.getName()
-                        + "/libeliza-llama-speculative-shim.so present; leaving MTP opt-in for stock APK");
-                }
+                // MTP (multi-token / speculative decode) is compiled into the
+                // single fused libelizainference.so (the eliza_mtp::Engine over
+                // common/speculative.cpp, exported as eliza_inference_llm_mtp_supported).
+                // It is controlled in-process by ELIZA_BIONIC_MTP and the FFI
+                // capability probe — NOT by a separate speculative-shim .so (that
+                // shim is retired and never staged). No env wiring needed here.
                 if (BuildConfig.DEBUG && !env.containsKey("ELIZA_AOSP_LLAMA_DEBUG_LOG")) {
                     File debugLog = new File(agentStateDir(), "aosp-llama-debug.log");
                     agentEnv.put("ELIZA_AOSP_LLAMA_DEBUG_LOG", debugLog.getAbsolutePath());