cui-llama.rn 1.3.0 → 1.3.4

package/android/src/main/CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.10)
 
 project(llama.rn)
 
-set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD 17)
 set(RNLLAMA_LIB_DIR ${CMAKE_SOURCE_DIR}/../../../cpp)
 
 include_directories(${RNLLAMA_LIB_DIR})
@@ -13,21 +13,25 @@ set(
     ${RNLLAMA_LIB_DIR}/llama-sampling.cpp
     ${RNLLAMA_LIB_DIR}/llama-vocab.cpp
     ${RNLLAMA_LIB_DIR}/log.cpp
+    
+    #${RNLLAMA_LIB_DIR}/amx/amx.cpp
+    #${RNLLAMA_LIB_DIR}/amx/mmq.cpp
 
-    ${RNLLAMA_LIB_DIR}/ggml-aarch64.c
     ${RNLLAMA_LIB_DIR}/llama-grammar.cpp
     ${RNLLAMA_LIB_DIR}/llama-sampling.cpp
     ${RNLLAMA_LIB_DIR}/llama-vocab.cpp
     ${RNLLAMA_LIB_DIR}/log.cpp
+    ${RNLLAMA_LIB_DIR}/json.hpp
+    ${RNLLAMA_LIB_DIR}/json-schema-to-grammar.cpp
 
-    ${RNLLAMA_LIB_DIR}/ggml-aarch64.c
     ${RNLLAMA_LIB_DIR}/ggml-alloc.c
     ${RNLLAMA_LIB_DIR}/ggml-backend.cpp
     ${RNLLAMA_LIB_DIR}/ggml-backend-reg.cpp
     ${RNLLAMA_LIB_DIR}/ggml.c
     ${RNLLAMA_LIB_DIR}/ggml-cpu.c
     ${RNLLAMA_LIB_DIR}/ggml-cpu.cpp
-    ${RNLLAMA_LIB_DIR}/ggml-cpu-aarch64.c
+    ${RNLLAMA_LIB_DIR}/ggml-cpu-aarch64.cpp
+    ${RNLLAMA_LIB_DIR}/ggml-cpu-traits.cpp
     ${RNLLAMA_LIB_DIR}/ggml-cpu-quants.c
     ${RNLLAMA_LIB_DIR}/ggml-threading.cpp
     ${RNLLAMA_LIB_DIR}/ggml-quants.c
@@ -37,7 +41,6 @@ set(
     ${RNLLAMA_LIB_DIR}/unicode.cpp
     ${RNLLAMA_LIB_DIR}/llama.cpp
     ${RNLLAMA_LIB_DIR}/sgemm.cpp
-    ${RNLLAMA_LIB_DIR}/ggml-aarch64.c
     ${RNLLAMA_LIB_DIR}/rn-llama.hpp
     ${CMAKE_SOURCE_DIR}/jni.cpp
 )
@@ -53,7 +56,7 @@ function(build_library target_name cpu_flags)
 
     target_link_libraries(${target_name} ${LOG_LIB} android)
 
-    target_compile_options(${target_name} PRIVATE -pthread ${cpu_flags})
+    target_compile_options(${target_name} PRIVATE -pthread ${cpu_flags} -DLM_GGML_USE_CPU -DLM_GGML_USE_CPU_AARCH64)
 
     if (${CMAKE_BUILD_TYPE} STREQUAL "Debug")
         target_compile_options(${target_name} PRIVATE -DRNLLAMA_ANDROID_ENABLE_LOGGING)

package/android/src/main/java/com/rnllama/LlamaContext.java

@@ -115,9 +115,9 @@ public class LlamaContext {
       // boolean flash_attn,
       params.hasKey("flash_attn") ? params.getBoolean("flash_attn") : false,
       // String cache_type_k,
-      params.hasKey("cache_type_k") ? params.getString("cache_type_k") : "f16",
+      params.hasKey("cache_type_k") ? params.getInt("cache_type_k") : 1,
       // String cache_type_v,
-      params.hasKey("cache_type_v") ? params.getString("cache_type_v") : "f16",
+      params.hasKey("cache_type_v") ? params.getInt("cache_type_v") : 1,
       // boolean use_mlock,
       params.hasKey("use_mlock") ? params.getBoolean("use_mlock") : true,
       // boolean use_mmap,
@@ -463,8 +463,8 @@ public class LlamaContext {
     int n_threads,
     int n_gpu_layers, // TODO: Support this
     boolean flash_attn,
-    String cache_type_k,
-    String cache_type_v,
+    int cache_type_k,
+    int cache_type_v,
     boolean use_mlock,
     boolean use_mmap,
     boolean vocab_only,

package/android/src/main/jni.cpp

@@ -236,8 +236,8 @@ Java_com_rnllama_LlamaContext_initContext(
     jint n_threads,
     jint n_gpu_layers, // TODO: Support this
     jboolean flash_attn,
-    jstring cache_type_k,
-    jstring cache_type_v,
+    jint cache_type_k,
+    jint cache_type_v,
     jboolean use_mlock,
     jboolean use_mmap,
     jboolean vocab_only,
@@ -259,7 +259,7 @@ Java_com_rnllama_LlamaContext_initContext(
 
     const char *model_path_chars = env->GetStringUTFChars(model_path_str, nullptr);
     defaultParams.model = model_path_chars;
-
+    
     defaultParams.n_ctx = n_ctx;
     defaultParams.n_batch = n_batch;
 
@@ -281,13 +281,13 @@ Java_com_rnllama_LlamaContext_initContext(
     int default_n_threads = max_threads == 4 ? 2 : min(4, max_threads);
     defaultParams.cpuparams.n_threads = n_threads > 0 ? n_threads : default_n_threads;
 
-    defaultParams.n_gpu_layers = n_gpu_layers;
+    // defaultParams.n_gpu_layers = n_gpu_layers;
     defaultParams.flash_attn = flash_attn;
 
-    const char *cache_type_k_chars = env->GetStringUTFChars(cache_type_k, nullptr);
-    const char *cache_type_v_chars = env->GetStringUTFChars(cache_type_v, nullptr);
-    defaultParams.cache_type_k = cache_type_k_chars;
-    defaultParams.cache_type_v = cache_type_v_chars;
+    // const char *cache_type_k_chars = env->GetStringUTFChars(cache_type_k, nullptr);
+    // const char *cache_type_v_chars = env->GetStringUTFChars(cache_type_v, nullptr);
+    defaultParams.cache_type_k = (lm_ggml_type) cache_type_k;
+    defaultParams.cache_type_v = (lm_ggml_type) cache_type_v;
 
     defaultParams.use_mlock = use_mlock;
     defaultParams.use_mmap = use_mmap;
@@ -331,8 +331,8 @@ Java_com_rnllama_LlamaContext_initContext(
 
     env->ReleaseStringUTFChars(model_path_str, model_path_chars);
     env->ReleaseStringUTFChars(lora_str, lora_chars);
-    env->ReleaseStringUTFChars(cache_type_k, cache_type_k_chars);
-    env->ReleaseStringUTFChars(cache_type_v, cache_type_v_chars);
+    // env->ReleaseStringUTFChars(cache_type_k, cache_type_k_chars);
+    // env->ReleaseStringUTFChars(cache_type_v, cache_type_v_chars);
 
     LOGI("[RNLlama] is_model_loaded %s", (is_model_loaded ? "true" : "false"));
     if (is_model_loaded) {
@@ -558,7 +558,7 @@ Java_com_rnllama_LlamaContext_doCompletion(
     //llama_reset_timings(llama->ctx);
 
     llama->params.prompt = env->GetStringUTFChars(prompt, nullptr);
-    llama->params.sparams.seed = (seed == -1) ? time(NULL) : seed;
+    llama->params.sampling.seed = (seed == -1) ? time(NULL) : seed;
 
     int max_threads = std::thread::hardware_concurrency();
     // Use 2 threads by default on 4-core devices, 4 threads on more cores
@@ -566,9 +566,9 @@ Java_com_rnllama_LlamaContext_doCompletion(
     llama->params.cpuparams.n_threads = n_threads > 0 ? n_threads : default_n_threads;
 
     llama->params.n_predict = n_predict;
-    llama->params.sparams.ignore_eos = ignore_eos;
+    llama->params.sampling.ignore_eos = ignore_eos;
 
-    auto & sparams = llama->params.sparams;
+    auto & sparams = llama->params.sampling;
     sparams.temp = temperature;
     sparams.penalty_last_n = penalty_last_n;
     sparams.penalty_repeat = penalty_repeat;
@@ -577,7 +577,7 @@ Java_com_rnllama_LlamaContext_doCompletion(
     sparams.mirostat = mirostat;
     sparams.mirostat_tau = mirostat_tau;
     sparams.mirostat_eta = mirostat_eta;
-    sparams.penalize_nl = penalize_nl;
+    // sparams.penalize_nl = penalize_nl;
     sparams.top_k = top_k;
     sparams.top_p = top_p;
     sparams.min_p = min_p;
@@ -693,7 +693,7 @@ Java_com_rnllama_LlamaContext_doCompletion(
             auto tokenResult = createWriteableMap(env);
             putString(env, tokenResult, "token", to_send.c_str());
 
-            if (llama->params.sparams.n_probs > 0) {
+            if (llama->params.sampling.n_probs > 0) {
               const std::vector<llama_token> to_send_toks = common_tokenize(llama->ctx, to_send, false);
               size_t probs_pos = std::min(sent_token_probs_index, llama->generated_token_probs.size());
               size_t probs_stop_pos = std::min(sent_token_probs_index + to_send_toks.size(), llama->generated_token_probs.size());