llama-cpp-pro 0.2.2

package/android/src/main/jni-multimodal.cpp

@@ -0,0 +1,167 @@
+// Multimodal Methods for Android JNI
+
+JNIEXPORT jboolean JNICALL
+Java_ai_annadata_plugin_capacitor_LlamaCpp_initMultimodalNative(
+    JNIEnv* env, jobject thiz, jlong contextId, jstring mmproj_path, jboolean use_gpu) {
+    
+    try {
+        LOGI("Initializing multimodal for context ID: %ld", contextId);
+        
+        std::string path = jni_utils::jstring_to_string(env, mmproj_path);
+        LOGI("Multimodal projection path: %s", path.c_str());
+        
+        auto it = contexts.find(contextId);
+        if (it == contexts.end()) {
+            LOGE("Context not found: %ld", contextId);
+            throw_java_exception(env, "java/lang/RuntimeException", "Context not found");
+            return JNI_FALSE;
+        }
+        
+        auto& ctx = it->second;
+        if (!ctx || !ctx->ctx || !ctx->model) {
+            LOGE("Invalid context, llama context, or model is null");
+            throw_java_exception(env, "java/lang/RuntimeException", "Invalid context or model not loaded");
+            return JNI_FALSE;
+        }
+        
+        // Check if model file exists
+        std::ifstream mmproj_file(path, std::ios::binary);
+        if (!mmproj_file.good()) {
+            LOGE("Multimodal projection file not found: %s", path.c_str());
+            throw_java_exception(env, "java/lang/RuntimeException", "Multimodal projection file not found");
+            return JNI_FALSE;
+        }
+        mmproj_file.close();
+        
+        // Initialize multimodal
+        bool result = ctx->initMultimodal(path, jni_utils::jboolean_to_bool(use_gpu));
+        
+        if (!result) {
+            LOGE("Failed to initialize multimodal");
+            throw_java_exception(env, "java/lang/RuntimeException", "Failed to initialize multimodal");
+            return JNI_FALSE;
+        }
+        
+        LOGI("Multimodal initialized successfully");
+        return JNI_TRUE;
+        
+    } catch (const std::exception& e) {
+        LOGE("Exception in initMultimodal: %s", e.what());
+        throw_java_exception(env, "java/lang/RuntimeException", e.what());
+        return JNI_FALSE;
+    }
+}
+
+JNIEXPORT jboolean JNICALL
+Java_ai_annadata_plugin_capacitor_LlamaCpp_isMultimodalEnabledNative(
+    JNIEnv* env, jobject thiz, jlong contextId) {
+    
+    try {
+        auto it = contexts.find(contextId);
+        if (it == contexts.end()) {
+            LOGE("Context not found: %ld", contextId);
+            throw_java_exception(env, "java/lang/RuntimeException", "Context not found");
+            return JNI_FALSE;
+        }
+        
+        auto& ctx = it->second;
+        if (!ctx) {
+            LOGE("Invalid context");
+            throw_java_exception(env, "java/lang/RuntimeException", "Invalid context");
+            return JNI_FALSE;
+        }
+        
+        bool enabled = ctx->isMultimodalEnabled();
+        LOGI("Multimodal enabled: %s", enabled ? "true" : "false");
+        return enabled ? JNI_TRUE : JNI_FALSE;
+        
+    } catch (const std::exception& e) {
+        LOGE("Exception in isMultimodalEnabled: %s", e.what());
+        throw_java_exception(env, "java/lang/RuntimeException", e.what());
+        return JNI_FALSE;
+    }
+}
+
+JNIEXPORT jobject JNICALL
+Java_ai_annadata_plugin_capacitor_LlamaCpp_getMultimodalSupportNative(
+    JNIEnv* env, jobject thiz, jlong contextId) {
+    
+    try {
+        auto it = contexts.find(contextId);
+        if (it == contexts.end()) {
+            LOGE("Context not found: %ld", contextId);
+            throw_java_exception(env, "java/lang/RuntimeException", "Context not found");
+            return nullptr;
+        }
+        
+        auto& ctx = it->second;
+        if (!ctx) {
+            LOGE("Invalid context");
+            throw_java_exception(env, "java/lang/RuntimeException", "Invalid context");
+            return nullptr;
+        }
+        
+        // Create result HashMap
+        jclass hashMapClass = env->FindClass("java/util/HashMap");
+        jmethodID hashMapConstructor = env->GetMethodID(hashMapClass, "<init>", "()V");
+        jmethodID putMethod = env->GetMethodID(hashMapClass, "put", "(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;");
+        
+        jobject resultMap = env->NewObject(hashMapClass, hashMapConstructor);
+        
+        bool supports_vision = ctx->isMultimodalSupportVision();
+        bool supports_audio = ctx->isMultimodalSupportAudio();
+        
+        env->CallObjectMethod(resultMap, putMethod,
+            jni_utils::string_to_jstring(env, "vision"),
+            env->NewObject(env->FindClass("java/lang/Boolean"),
+                env->GetMethodID(env->FindClass("java/lang/Boolean"), "<init>", "(Z)V"),
+                supports_vision ? JNI_TRUE : JNI_FALSE));
+        
+        env->CallObjectMethod(resultMap, putMethod,
+            jni_utils::string_to_jstring(env, "audio"),
+            env->NewObject(env->FindClass("java/lang/Boolean"),
+                env->GetMethodID(env->FindClass("java/lang/Boolean"), "<init>", "(Z)V"),
+                supports_audio ? JNI_TRUE : JNI_FALSE));
+        
+        LOGI("Multimodal support - vision: %s, audio: %s",
+             supports_vision ? "true" : "false",
+             supports_audio ? "true" : "false");
+        
+        return resultMap;
+        
+    } catch (const std::exception& e) {
+        LOGE("Exception in getMultimodalSupport: %s", e.what());
+        throw_java_exception(env, "java/lang/RuntimeException", e.what());
+        return nullptr;
+    }
+}
+
+JNIEXPORT void JNICALL
+Java_ai_annadata_plugin_capacitor_LlamaCpp_releaseMultimodalNative(
+    JNIEnv* env, jobject thiz, jlong contextId) {
+    
+    try {
+        LOGI("Releasing multimodal for context ID: %ld", contextId);
+        
+        auto it = contexts.find(contextId);
+        if (it == contexts.end()) {
+            LOGE("Context not found: %ld", contextId);
+            throw_java_exception(env, "java/lang/RuntimeException", "Context not found");
+            return;
+        }
+        
+        auto& ctx = it->second;
+        if (!ctx) {
+            LOGE("Invalid context");
+            throw_java_exception(env, "java/lang/RuntimeException", "Invalid context");
+            return;
+        }
+        
+        ctx->releaseMultimodal();
+        LOGI("Multimodal released successfully");
+        
+    } catch (const std::exception& e) {
+        LOGE("Exception in releaseMultimodal: %s", e.what());
+        throw_java_exception(env, "java/lang/RuntimeException", e.what());
+    }
+}

package/android/src/main/jni-tts.cpp

@@ -0,0 +1,290 @@
+// TTS/Vocoder Methods for Android JNI
+
+JNIEXPORT jboolean JNICALL
+Java_ai_annadata_plugin_capacitor_LlamaCpp_initVocoderNative(
+    JNIEnv* env, jobject thiz, jlong contextId, jstring vocoder_model_path, jint n_batch) {
+    
+    try {
+        LOGI("Initializing vocoder for context ID: %ld", contextId);
+        
+        std::string path = jni_utils::jstring_to_string(env, vocoder_model_path);
+        LOGI("Vocoder model path: %s", path.c_str());
+        
+        auto it = contexts.find(contextId);
+        if (it == contexts.end()) {
+            LOGE("Context not found: %ld", contextId);
+            throw_java_exception(env, "java/lang/RuntimeException", "Context not found");
+            return JNI_FALSE;
+        }
+        
+        auto& ctx = it->second;
+        if (!ctx || !ctx->ctx || !ctx->model) {
+            LOGE("Invalid context, llama context, or model is null");
+            throw_java_exception(env, "java/lang/RuntimeException", "Invalid context or model not loaded");
+            return JNI_FALSE;
+        }
+        
+        // Check if vocoder model file exists
+        std::ifstream vocoder_file(path, std::ios::binary);
+        if (!vocoder_file.good()) {
+            LOGE("Vocoder model file not found: %s", path.c_str());
+            throw_java_exception(env, "java/lang/RuntimeException", "Vocoder model file not found");
+            return JNI_FALSE;
+        }
+        vocoder_file.close();
+        
+        // Initialize vocoder with batch size (-1 means use default)
+        int batch_size = (n_batch > 0) ? static_cast<int>(n_batch) : -1;
+        bool result = ctx->initVocoder(path, batch_size);
+        
+        if (!result) {
+            LOGE("Failed to initialize vocoder");
+            throw_java_exception(env, "java/lang/RuntimeException", "Failed to initialize vocoder");
+            return JNI_FALSE;
+        }
+        
+        LOGI("Vocoder initialized successfully with batch size: %d", batch_size);
+        return JNI_TRUE;
+        
+    } catch (const std::exception& e) {
+        LOGE("Exception in initVocoder: %s", e.what());
+        throw_java_exception(env, "java/lang/RuntimeException", e.what());
+        return JNI_FALSE;
+    }
+}
+
+JNIEXPORT jboolean JNICALL
+Java_ai_annadata_plugin_capacitor_LlamaCpp_isVocoderEnabledNative(
+    JNIEnv* env, jobject thiz, jlong contextId) {
+    
+    try {
+        auto it = contexts.find(contextId);
+        if (it == contexts.end()) {
+            LOGE("Context not found: %ld", contextId);
+            throw_java_exception(env, "java/lang/RuntimeException", "Context not found");
+            return JNI_FALSE;
+        }
+        
+        auto& ctx = it->second;
+        if (!ctx) {
+            LOGE("Invalid context");
+            throw_java_exception(env, "java/lang/RuntimeException", "Invalid context");
+            return JNI_FALSE;
+        }
+        
+        bool enabled = ctx->isVocoderEnabled();
+        LOGI("Vocoder enabled: %s", enabled ? "true" : "false");
+        return enabled ? JNI_TRUE : JNI_FALSE;
+        
+    } catch (const std::exception& e) {
+        LOGE("Exception in isVocoderEnabled: %s", e.what());
+        throw_java_exception(env, "java/lang/RuntimeException", e.what());
+        return JNI_FALSE;
+    }
+}
+
+JNIEXPORT jobject JNICALL
+Java_ai_annadata_plugin_capacitor_LlamaCpp_getFormattedAudioCompletionNative(
+    JNIEnv* env, jobject thiz, jlong contextId, jstring speaker_json_str, jstring text_to_speak) {
+    
+    try {
+        LOGI("Getting formatted audio completion for context ID: %ld", contextId);
+        
+        auto it = contexts.find(contextId);
+        if (it == contexts.end()) {
+            LOGE("Context not found: %ld", contextId);
+            throw_java_exception(env, "java/lang/RuntimeException", "Context not found");
+            return nullptr;
+        }
+        
+        auto& ctx = it->second;
+        if (!ctx || !ctx->tts_wrapper) {
+            LOGE("Context or vocoder not initialized");
+            throw_java_exception(env, "java/lang/RuntimeException", "Vocoder not initialized");
+            return nullptr;
+        }
+        
+        std::string speaker_json = jni_utils::jstring_to_string(env, speaker_json_str);
+        std::string text = jni_utils::jstring_to_string(env, text_to_speak);
+        
+        LOGI("Text to synthesize: %s", text.c_str());
+        
+        // Get formatted audio completion
+        capllama::llama_cap_audio_completion_result result = 
+            ctx->tts_wrapper->getFormattedAudioCompletion(ctx.get(), speaker_json, text);
+        
+        // Create result HashMap
+        jclass hashMapClass = env->FindClass("java/util/HashMap");
+        jmethodID hashMapConstructor = env->GetMethodID(hashMapClass, "<init>", "()V");
+        jmethodID putMethod = env->GetMethodID(hashMapClass, "put", "(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;");
+        
+        jobject resultMap = env->NewObject(hashMapClass, hashMapConstructor);
+        
+        env->CallObjectMethod(resultMap, putMethod,
+            jni_utils::string_to_jstring(env, "prompt"),
+            jni_utils::string_to_jstring(env, result.prompt));
+        
+        if (result.grammar != nullptr) {
+            env->CallObjectMethod(resultMap, putMethod,
+                jni_utils::string_to_jstring(env, "grammar"),
+                jni_utils::string_to_jstring(env, result.grammar));
+        }
+        
+        LOGI("Audio completion formatted successfully");
+        return resultMap;
+        
+    } catch (const std::exception& e) {
+        LOGE("Exception in getFormattedAudioCompletion: %s", e.what());
+        throw_java_exception(env, "java/lang/RuntimeException", e.what());
+        return nullptr;
+    }
+}
+
+JNIEXPORT jobject JNICALL
+Java_ai_annadata_plugin_capacitor_LlamaCpp_getAudioCompletionGuideTokensNative(
+    JNIEnv* env, jobject thiz, jlong contextId, jstring text_to_speak) {
+    
+    try {
+        LOGI("Getting audio completion guide tokens for context ID: %ld", contextId);
+        
+        auto it = contexts.find(contextId);
+        if (it == contexts.end()) {
+            LOGE("Context not found: %ld", contextId);
+            throw_java_exception(env, "java/lang/RuntimeException", "Context not found");
+            return nullptr;
+        }
+        
+        auto& ctx = it->second;
+        if (!ctx || !ctx->tts_wrapper) {
+            LOGE("Context or vocoder not initialized");
+            throw_java_exception(env, "java/lang/RuntimeException", "Vocoder not initialized");
+            return nullptr;
+        }
+        
+        std::string text = jni_utils::jstring_to_string(env, text_to_speak);
+        LOGI("Text for guide tokens: %s", text.c_str());
+        
+        // Get guide tokens
+        std::vector<llama_token> guide_tokens = 
+            ctx->tts_wrapper->getAudioCompletionGuideTokens(ctx.get(), text);
+        
+        // Create Java ArrayList for tokens
+        jclass arrayListClass = env->FindClass("java/util/ArrayList");
+        jmethodID arrayListConstructor = env->GetMethodID(arrayListClass, "<init>", "()V");
+        jmethodID addMethod = env->GetMethodID(arrayListClass, "add", "(Ljava/lang/Object;)Z");
+        
+        jobject tokenList = env->NewObject(arrayListClass, arrayListConstructor);
+        
+        // Add tokens
+        jclass integerClass = env->FindClass("java/lang/Integer");
+        jmethodID integerConstructor = env->GetMethodID(integerClass, "<init>", "(I)V");
+        
+        for (llama_token token : guide_tokens) {
+            jobject jToken = env->NewObject(integerClass, integerConstructor, static_cast<jint>(token));
+            env->CallBooleanMethod(tokenList, addMethod, jToken);
+            env->DeleteLocalRef(jToken);
+        }
+        
+        LOGI("Retrieved %zu guide tokens", guide_tokens.size());
+        return tokenList;
+        
+    } catch (const std::exception& e) {
+        LOGE("Exception in getAudioCompletionGuideTokens: %s", e.what());
+        throw_java_exception(env, "java/lang/RuntimeException", e.what());
+        return nullptr;
+    }
+}
+
+JNIEXPORT jobject JNICALL
+Java_ai_annadata_plugin_capacitor_LlamaCpp_decodeAudioTokensNative(
+    JNIEnv* env, jobject thiz, jlong contextId, jintArray tokens_array) {
+    
+    try {
+        LOGI("Decoding audio tokens for context ID: %ld", contextId);
+        
+        auto it = contexts.find(contextId);
+        if (it == contexts.end()) {
+            LOGE("Context not found: %ld", contextId);
+            throw_java_exception(env, "java/lang/RuntimeException", "Context not found");
+            return nullptr;
+        }
+        
+        auto& ctx = it->second;
+        if (!ctx || !ctx->tts_wrapper) {
+            LOGE("Context or vocoder not initialized");
+            throw_java_exception(env, "java/lang/RuntimeException", "Vocoder not initialized");
+            return nullptr;
+        }
+        
+        // Convert Java int array to C++ vector
+        jsize length = env->GetArrayLength(tokens_array);
+        jint* tokenArray = env->GetIntArrayElements(tokens_array, nullptr);
+        
+        std::vector<llama_token> tokens;
+        for (jsize i = 0; i < length; i++) {
+            tokens.push_back(static_cast<llama_token>(tokenArray[i]));
+        }
+        
+        env->ReleaseIntArrayElements(tokens_array, tokenArray, JNI_ABORT);
+        
+        LOGI("Decoding %zu audio tokens", tokens.size());
+        
+        // Decode audio tokens
+        std::vector<float> audio_samples = ctx->tts_wrapper->decodeAudioTokens(ctx.get(), tokens);
+        
+        // Create Java ArrayList for audio samples
+        jclass arrayListClass = env->FindClass("java/util/ArrayList");
+        jmethodID arrayListConstructor = env->GetMethodID(arrayListClass, "<init>", "()V");
+        jmethodID addMethod = env->GetMethodID(arrayListClass, "add", "(Ljava/lang/Object;)Z");
+        
+        jobject audioList = env->NewObject(arrayListClass, arrayListConstructor);
+        
+        // Add audio samples
+        jclass floatClass = env->FindClass("java/lang/Float");
+        jmethodID floatConstructor = env->GetMethodID(floatClass, "<init>", "(F)V");
+        
+        for (float sample : audio_samples) {
+            jobject jSample = env->NewObject(floatClass, floatConstructor, sample);
+            env->CallBooleanMethod(audioList, addMethod, jSample);
+            env->DeleteLocalRef(jSample);
+        }
+        
+        LOGI("Decoded %zu audio samples", audio_samples.size());
+        return audioList;
+        
+    } catch (const std::exception& e) {
+        LOGE("Exception in decodeAudioTokens: %s", e.what());
+        throw_java_exception(env, "java/lang/RuntimeException", e.what());
+        return nullptr;
+    }
+}
+
+JNIEXPORT void JNICALL
+Java_ai_annadata_plugin_capacitor_LlamaCpp_releaseVocoderNative(
+    JNIEnv* env, jobject thiz, jlong contextId) {
+    
+    try {
+        LOGI("Releasing vocoder for context ID: %ld", contextId);
+        
+        auto it = contexts.find(contextId);
+        if (it == contexts.end()) {
+            LOGE("Context not found: %ld", contextId);
+            throw_java_exception(env, "java/lang/RuntimeException", "Context not found");
+            return;
+        }
+        
+        auto& ctx = it->second;
+        if (!ctx) {
+            LOGE("Invalid context");
+            throw_java_exception(env, "java/lang/RuntimeException", "Invalid context");
+            return;
+        }
+        
+        ctx->releaseVocoder();
+        LOGI("Vocoder released successfully");
+        
+    } catch (const std::exception& e) {
+        LOGE("Exception in releaseVocoder: %s", e.what());
+        throw_java_exception(env, "java/lang/RuntimeException", e.what());
+    }
+}

package/android/src/main/jni-utils.h

@@ -0,0 +1,148 @@
+#ifndef JNI_UTILS_H
+#define JNI_UTILS_H
+
+#include <jni.h>
+#include <string>
+#include <vector>
+#include "cap-llama.h"
+
+namespace jni_utils {
+
+// Convert Java string to C++ string
+std::string jstring_to_string(JNIEnv* env, jstring jstr);
+
+// Convert C++ string to Java string
+jstring string_to_jstring(JNIEnv* env, const std::string& str);
+
+// Convert Java string array to C++ string vector
+std::vector<std::string> jstring_array_to_string_vector(JNIEnv* env, jobjectArray jarray);
+
+// Convert C++ string vector to Java string array
+jobjectArray string_vector_to_jstring_array(JNIEnv* env, const std::vector<std::string>& vec);
+
+// Convert Java boolean to C++ bool
+bool jboolean_to_bool(jboolean jbool);
+
+// Convert C++ bool to Java boolean
+jboolean bool_to_jboolean(bool b);
+
+// Convert Java int to C++ int
+int jint_to_int(jint jint_val);
+
+// Convert C++ int to Java int
+jint int_to_jint(int val);
+
+// Convert Java float to C++ float
+float jfloat_to_float(jfloat jfloat_val);
+
+// Convert C++ float to Java float
+jfloat float_to_jfloat(float val);
+
+// Convert Java long to C++ long
+long jlong_to_long(jlong jlong_val);
+
+// Convert C++ long to Java long
+jlong long_to_jlong(long val);
+
+// Convert Java double to C++ double
+double jdouble_to_double(jdouble jdouble_val);
+
+// Convert C++ double to Java double
+jdouble double_to_jdouble(double val);
+
+// Throw Java exception
+void throw_java_exception(JNIEnv* env, const char* class_name, const char* message);
+
+// Check if exception occurred
+bool check_exception(JNIEnv* env);
+
+/** Capacitor 8+ JSObject has no getDouble(); read via inherited JSONObject.optDouble. */
+double jsobject_opt_double(JNIEnv* env, jobject jso, const char* key, double default_value);
+
+/** Capacitor 8+ JSObject has no getBoolean(String); read via inherited JSONObject.optBoolean. */
+bool jsobject_opt_bool(JNIEnv* env, jobject jso, const char* key, bool default_value);
+
+/** Read optional string via inherited JSONObject.optString. */
+std::string jsobject_opt_string(JNIEnv* env, jobject jso, const char* key, const std::string& default_value);
+
+// Get field ID safely
+jfieldID get_field_id(JNIEnv* env, jclass clazz, const char* name, const char* sig);
+
+// Get method ID safely
+jmethodID get_method_id(JNIEnv* env, jclass clazz, const char* name, const char* sig);
+
+// Find class safely
+jclass find_class(JNIEnv* env, const char* name);
+
+// Create object safely
+jobject create_object(JNIEnv* env, jclass clazz, jmethodID constructor, ...);
+
+// Call method safely
+jobject call_method(JNIEnv* env, jobject obj, jmethodID method, ...);
+
+// Call static method safely
+jobject call_static_method(JNIEnv* env, jclass clazz, jmethodID method, ...);
+
+// Set field safely
+void set_field(JNIEnv* env, jobject obj, jfieldID field, ...);
+
+// Get field safely
+jobject get_field(JNIEnv* env, jobject obj, jfieldID field);
+
+// Set static field safely
+void set_static_field(JNIEnv* env, jclass clazz, jfieldID field, ...);
+
+// Get static field safely
+jobject get_static_field(JNIEnv* env, jclass clazz, jfieldID field);
+
+// Convert llama_cap_context to jobject
+jobject llama_context_to_jobject(JNIEnv* env, const capllama::llama_cap_context* context);
+
+// Convert jobject to llama_cap_context
+capllama::llama_cap_context* jobject_to_llama_context(JNIEnv* env, jobject obj);
+
+// Convert completion result to jobject
+jobject completion_result_to_jobject(JNIEnv* env, const capllama::completion_token_output& result);
+
+// Convert jobject to completion parameters
+common_params jobject_to_completion_params(JNIEnv* env, jobject obj);
+
+// Convert chat parameters to jobject
+jobject chat_params_to_jobject(JNIEnv* env, const common_chat_params& params);
+
+// Convert jobject to chat parameters
+common_chat_params jobject_to_chat_params(JNIEnv* env, jobject obj);
+
+// Convert tokenize result to jobject
+jobject tokenize_result_to_jobject(JNIEnv* env, const capllama::llama_cap_tokenize_result& result);
+
+// Convert embedding result to jobject
+jobject embedding_result_to_jobject(JNIEnv* env, const std::vector<float>& embedding);
+
+// Convert rerank result to jobject
+jobject rerank_result_to_jobject(JNIEnv* env, const std::vector<std::pair<size_t, float>>& results);
+
+// Convert benchmark result to jobject
+jobject benchmark_result_to_jobject(JNIEnv* env, const std::vector<float>& timings);
+
+// Convert LoRA adapter info to jobject
+jobject lora_adapter_info_to_jobject(JNIEnv* env, const common_adapter_lora_info& info);
+
+// Convert jobject to LoRA adapter info
+common_adapter_lora_info jobject_to_lora_adapter_info(JNIEnv* env, jobject obj);
+
+// Convert multimodal support info to jobject
+jobject multimodal_support_to_jobject(JNIEnv* env, bool vision, bool audio);
+
+// Convert TTS result to jobject
+jobject tts_result_to_jobject(JNIEnv* env, const std::vector<float>& audio_data, int sample_rate);
+
+// Convert session data to jobject
+jobject session_data_to_jobject(JNIEnv* env, const std::string& data);
+
+// Convert jobject to session data
+std::string jobject_to_session_data(JNIEnv* env, jobject obj);
+
+} // namespace jni_utils
+
+#endif // JNI_UTILS_H