cui-llama.rn 1.4.0 → 1.4.2

package/README.md

@@ -36,6 +36,8 @@ npm install llama.rn
 
 Please re-run `npx pod-install` again.
 
+By default, `llama.rn` will use pre-built `rnllama.xcframework` for iOS. If you want to build from source, please set `RNLLAMA_BUILD_FROM_SOURCE` to `1` in your Podfile.
+
 #### Android
 
 Add proguard rule if it's enabled in project (android/app/proguard-rules.pro):
@@ -45,6 +47,8 @@ Add proguard rule if it's enabled in project (android/app/proguard-rules.pro):
 -keep class com.rnllama.** { *; }
 ```
 
+By default, `llama.rn` will use pre-built libraries for Android. If you want to build from source, please set `rnllamaBuildFromSource` to `true` in `android/gradle.properties`.
+
 ## Obtain the model
 
 You can search HuggingFace for available models (Keyword: [`GGUF`](https://huggingface.co/search/full-text?q=GGUF&type=model)).
@@ -137,29 +141,6 @@ Please visit the [Documentation](docs/API) for more details.
 
 You can also visit the [example](example) to see how to use it.
 
-Run the example:
-
-```bash
-yarn && yarn bootstrap
-
-# iOS
-yarn example ios
-# Use device
-yarn example ios --device "<device name>"
-# With release mode
-yarn example ios --mode Release
-
-# Android
-yarn example android
-# With release mode
-yarn example android --mode release
-```
-
-This example used [react-native-document-picker](https://github.com/rnmods/react-native-document-picker) for select model.
-
-- iOS: You can move the model to iOS Simulator, or iCloud for real device.
-- Android: Selected file will be copied or downloaded to cache directory so it may be slow.
-
 ## Grammar Sampling
 
 GBNF (GGML BNF) is a format for defining [formal grammars](https://en.wikipedia.org/wiki/Formal_grammar) to constrain model outputs in `llama.cpp`. For example, you can use it to force the model to generate valid JSON, or speak only in emojis.

package/android/build.gradle

@@ -54,9 +54,18 @@ android {
       }
     }
   }
-  externalNativeBuild {
-    cmake {
-      path = file('src/main/CMakeLists.txt')
+  def rnllamaBuildFromSource = project.properties["rnllamaBuildFromSource"]
+  if (rnllamaBuildFromSource == "true") {
+    externalNativeBuild {
+      cmake {
+        path = file('src/main/CMakeLists.txt')
+      }
+    }
+    // Exclude jniLibs
+    sourceSets {
+      main {
+        jniLibs.srcDirs = []
+      }
     }
   }
   buildTypes {

package/android/src/main/CMakeLists.txt

@@ -2,6 +2,12 @@ cmake_minimum_required(VERSION 3.10)
 
 project(llama.rn)
 
+find_program(CCACHE_FOUND ccache)
+if(CCACHE_FOUND)
+  set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
+  set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache)
+endif(CCACHE_FOUND)
+
 set(CMAKE_CXX_STANDARD 17)
 set(RNLLAMA_LIB_DIR ${CMAKE_SOURCE_DIR}/../../../cpp)
 
@@ -45,7 +51,7 @@ set(
     ${RNLLAMA_LIB_DIR}/unicode.cpp
     ${RNLLAMA_LIB_DIR}/sgemm.cpp
     ${RNLLAMA_LIB_DIR}/common.cpp
-    ${RNLLAMA_LIB_DIR}/rn-llama.hpp
+    ${RNLLAMA_LIB_DIR}/rn-llama.cpp
     ${CMAKE_SOURCE_DIR}/jni-utils.h
     ${CMAKE_SOURCE_DIR}/jni.cpp
 )
@@ -86,13 +92,13 @@ build_library("rnllama" "")
 
 if (${ANDROID_ABI} STREQUAL "arm64-v8a")
     # ARM64 targets
-    build_library("rnllama_v8_4_fp16_dotprod_sve" "-march=armv8.4-a+fp16+dotprod+sve")
-    build_library("rnllama_v8_4_fp16_dotprod_i8mm_sve" "-march=armv8.4-a+fp16+dotprod+i8mm+sve")
-    build_library("rnllama_v8_4_fp16_dotprod_i8mm" "-march=armv8.4-a+fp16+dotprod+i8mm")
-    build_library("rnllama_v8_4_fp16_dotprod" "-march=armv8.4-a+fp16+dotprod")
-    build_library("rnllama_v8_2_fp16_dotprod" "-march=armv8.2-a+fp16+dotprod")
-    build_library("rnllama_v8_2_fp16" "-march=armv8.2-a+fp16")
+    # Removing fp16 for now as it leads to issues with some models like deepseek r1 distills
+    # https://github.com/mybigday/llama.rn/pull/110#issuecomment-2609918310
     build_library("rnllama_v8" "-march=armv8-a")
+    build_library("rnllama_v8_2" "-march=armv8.2-a")
+    build_library("rnllama_v8_2_dotprod" "-march=armv8.2-a+dotprod")
+    build_library("rnllama_v8_2_i8mm" "-march=armv8.2-a+i8mm")
+    build_library("rnllama_v8_2_dotprod_i8mm" "-march=armv8.2-a+dotprod+i8mm")
 
     # https://github.com/ggerganov/llama.cpp/blob/master/docs/android.md#cross-compile-using-android-ndk
     # llama.cpp will deal with the cpu features

package/android/src/main/java/com/rnllama/LlamaContext.java

@@ -26,6 +26,8 @@ import java.io.FileInputStream;
 public class LlamaContext {
   public static final String NAME = "RNLlamaContext";
 
+  private static String loadedLibrary = "";
+
   private int id;
   private ReactApplicationContext reactContext;
   private long context;
@@ -160,6 +162,10 @@ public class LlamaContext {
     return modelDetails;
   }
 
+  public String getLoadedLibrary() {
+    return loadedLibrary;
+  }
+
   public String getFormattedChat(ReadableArray messages, String chatTemplate) {
     ReadableMap[] msgs = new ReadableMap[messages.size()];
     for (int i = 0; i < messages.size(); i++) {
@@ -401,36 +407,37 @@ public class LlamaContext {
 
     // TODO: Add runtime check for cpu features
     if (LlamaContext.isArm64V8a()) {
-      if (isAtLeastArmV84 && hasSve && hasI8mm && hasFp16 && hasDotProd) {
-        Log.d(NAME, "Loading librnllama_v8_4_fp16_dotprod_i8mm_sve.so");
-        System.loadLibrary("rnllama_v8_4_fp16_dotprod_i8mm_sve");
-      } else if (isAtLeastArmV84 && hasSve && hasFp16 && hasDotProd) {
-        Log.d(NAME, "Loading librnllama_v8_4_fp16_dotprod_sve.so");
-        System.loadLibrary("rnllama_v8_4_fp16_dotprod_sve");
-      } else if (isAtLeastArmV84 && hasI8mm && hasFp16 && hasDotProd) {
-        Log.d(NAME, "Loading librnllama_v8_4_fp16_dotprod_i8mm.so");
-        System.loadLibrary("rnllama_v8_4_fp16_dotprod_i8mm");
-      } else if (isAtLeastArmV84 && hasFp16 && hasDotProd) {
-        Log.d(NAME, "Loading librnllama_v8_4_fp16_dotprod.so");
-        System.loadLibrary("rnllama_v8_4_fp16_dotprod");
-      } else if (isAtLeastArmV82 && hasFp16 && hasDotProd) {
-        Log.d(NAME, "Loading librnllama_v8_2_fp16_dotprod.so");
-        System.loadLibrary("rnllama_v8_2_fp16_dotprod");
-      } else if (isAtLeastArmV82 && hasFp16) {
-        Log.d(NAME, "Loading librnllama_v8_2_fp16.so");
-        System.loadLibrary("rnllama_v8_2_fp16");
+      if (hasDotProd && hasI8mm) {
+        Log.d(NAME, "Loading librnllama_v8_2_dotprod_i8mm.so");
+        System.loadLibrary("rnllama_v8_2_dotprod_i8mm");
+        loadedLibrary = "rnllama_v8_2_dotprod_i8mm";
+      } else if (hasDotProd) {
+        Log.d(NAME, "Loading librnllama_v8_2_dotprod.so");
+        System.loadLibrary("rnllama_v8_2_dotprod");
+        loadedLibrary = "rnllama_v8_2_dotprod";
+      } else if (hasI8mm) {
+        Log.d(NAME, "Loading librnllama_v8_2_i8mm.so");
+        System.loadLibrary("rnllama_v8_2_i8mm");
+        loadedLibrary = "rnllama_v8_2_i8mm";
+      } else if (hasFp16) {
+        Log.d(NAME, "Loading librnllama_v8_2.so");
+        System.loadLibrary("rnllama_v8_2");
+        loadedLibrary = "rnllama_v8_2";
       } else {
-        Log.d(NAME, "Loading librnllama_v8.so");
+        Log.d(NAME, "Loading default librnllama_v8.so");
         System.loadLibrary("rnllama_v8");
+        loadedLibrary = "rnllama_v8";
       }
       //  Log.d(NAME, "Loading librnllama_v8_7.so with runtime feature detection");
       //  System.loadLibrary("rnllama_v8_7");
     } else if (LlamaContext.isX86_64()) {
         Log.d(NAME, "Loading librnllama_x86_64.so");
         System.loadLibrary("rnllama_x86_64");
+        loadedLibrary = "rnllama_x86_64";
     } else {
         Log.d(NAME, "Loading default librnllama.so");
         System.loadLibrary("rnllama");
+        loadedLibrary = "rnllama";
     }
 }
 
@@ -465,7 +472,7 @@ public class LlamaContext {
   public void emitModelProgressUpdate(int progress) {
     WritableMap event = Arguments.createMap();
     event.putInt("progress", progress);
-    eventEmitter.emit("@RNLlama_onModelProgress", event);
+    eventEmitter.emit("@RNLlama_onInitContextProgress", event);
   }
 
   protected static native WritableMap modelInfo(

package/android/src/main/java/com/rnllama/RNLlama.java

@@ -35,7 +35,7 @@ public class RNLlama implements LifecycleEventListener {
 
   private HashMap<Integer, LlamaContext> contexts = new HashMap<>();
 
-  private int llamaContextLimit = 1;
+  private int llamaContextLimit = -1;
 
   public void setContextLimit(double limit, Promise promise) {
     llamaContextLimit = (int) limit;
@@ -83,6 +83,9 @@ public class RNLlama implements LifecycleEventListener {
           if (context != null) {
             throw new Exception("Context already exists");
           }
+          if (llamaContextLimit > -1 && contexts.size() >= llamaContextLimit) {
+            throw new Exception("Context limit reached");
+          }
           LlamaContext llamaContext = new LlamaContext(contextId, reactContext, params);
           if (llamaContext.getContext() == 0) {
             throw new Exception("Failed to initialize context");
@@ -92,6 +95,7 @@ public class RNLlama implements LifecycleEventListener {
           result.putBoolean("gpu", false);
           result.putString("reasonNoGPU", "Currently not supported");
           result.putMap("model", llamaContext.getModelDetails());
+          result.putString("androidLib", llamaContext.getLoadedLibrary());
           return result;
         } catch (Exception e) {
           exception = e;

package/android/src/main/jni.cpp

@@ -11,9 +11,8 @@
 #include <unordered_map>
 #include "llama.h"
 #include "llama-impl.h"
-#include "llama-context.h"
-#include "gguf.h"
-#include "rn-llama.hpp"
+#include "ggml.h"
+#include "rn-llama.h"
 #include "jni-utils.h"
 
 #define UNUSED(x) (void)(x)
@@ -345,10 +344,10 @@ Java_com_rnllama_LlamaContext_initContext(
         llama_free(llama->ctx);
     }
 
-    std::vector<common_lora_adapter_info> lora;
+    std::vector<common_adapter_lora_info> lora;
     const char *lora_chars = env->GetStringUTFChars(lora_str, nullptr);
     if (lora_chars != nullptr && lora_chars[0] != '\0') {
-        common_lora_adapter_info la;
+        common_adapter_lora_info la;
         la.path = lora_chars;
         la.scale = lora_scaled;
         lora.push_back(la);
@@ -362,7 +361,7 @@ Java_com_rnllama_LlamaContext_initContext(
             jstring path = readablemap::getString(env, lora_adapter, "path", nullptr);
             if (path != nullptr) {
                 const char *path_chars = env->GetStringUTFChars(path, nullptr);
-                common_lora_adapter_info la;
+                common_adapter_lora_info la;
                 la.path = path_chars;
                 la.scale = readablemap::getFloat(env, lora_adapter, "scaled", 1.0f);
                 lora.push_back(la);
@@ -409,7 +408,7 @@ Java_com_rnllama_LlamaContext_loadModelDetails(
     for (int i = 0; i < count; i++) {
         char key[256];
         llama_model_meta_key_by_index(llama->model, i, key, sizeof(key));
-        char val[2048];
+        char val[4096];
         llama_model_meta_val_str_by_index(llama->model, i, val, sizeof(val));
 
         putString(env, meta, key, val);
@@ -421,6 +420,7 @@ Java_com_rnllama_LlamaContext_loadModelDetails(
     llama_model_desc(llama->model, desc, sizeof(desc));
     putString(env, result, "desc", desc);
     putDouble(env, result, "size", llama_model_size(llama->model));
+    putDouble(env, result, "nEmbd", llama_model_n_embd(llama->model));
     putDouble(env, result, "nParams", llama_model_n_params(llama->model));
     putBoolean(env, result, "isChatTemplateSupported", llama->validateModelChatTemplate());
     putMap(env, result, "metadata", meta);
@@ -621,9 +621,12 @@ Java_com_rnllama_LlamaContext_doCompletion(
     sparams.dry_allowed_length = dry_allowed_length;
     sparams.dry_penalty_last_n = dry_penalty_last_n;
 
+    const llama_model * model = llama_get_model(llama->ctx);
+    const llama_vocab * vocab = llama_model_get_vocab(model);
+
     sparams.logit_bias.clear();
     if (ignore_eos) {
-        sparams.logit_bias[llama_token_eos(llama->model)].bias = -INFINITY;
+        sparams.logit_bias[llama_vocab_eos(vocab)].bias = -INFINITY;
     }
 
     // dry break seq
@@ -642,7 +645,7 @@ Java_com_rnllama_LlamaContext_doCompletion(
     sparams.dry_sequence_breakers = dry_sequence_breakers_vector;
 
     // logit bias
-    const int n_vocab = llama_n_vocab(llama_get_model(llama->ctx));
+    const int n_vocab = llama_vocab_n_tokens(vocab);
     jsize logit_bias_len = env->GetArrayLength(logit_bias);
 
     for (jsize i = 0; i < logit_bias_len; i++) {
@@ -921,7 +924,7 @@ Java_com_rnllama_LlamaContext_applyLoraAdapters(
     auto llama = context_map[(long) context_ptr];
 
     // lora_adapters: ReadableArray<ReadableMap>
-    std::vector<common_lora_adapter_info> lora_adapters;
+    std::vector<common_adapter_lora_info> lora_adapters;
     int lora_adapters_size = readablearray::size(env, loraAdapters);
     for (int i = 0; i < lora_adapters_size; i++) {
         jobject lora_adapter = readablearray::getMap(env, loraAdapters, i);
@@ -930,7 +933,7 @@ Java_com_rnllama_LlamaContext_applyLoraAdapters(
           const char *path_chars = env->GetStringUTFChars(path, nullptr);
           env->ReleaseStringUTFChars(path, path_chars);
           float scaled = readablemap::getFloat(env, lora_adapter, "scaled", 1.0f);
-          common_lora_adapter_info la;
+          common_adapter_lora_info la;
           la.path = path_chars;
           la.scale = scaled;
           lora_adapters.push_back(la);
@@ -955,7 +958,7 @@ Java_com_rnllama_LlamaContext_getLoadedLoraAdapters(
     auto llama = context_map[(long) context_ptr];
     auto loaded_lora_adapters = llama->getLoadedLoraAdapters();
     auto result = createWritableArray(env);
-    for (common_lora_adapter_info &la : loaded_lora_adapters) {
+    for (common_adapter_lora_info &la : loaded_lora_adapters) {
         auto map = createWriteableMap(env);
         putString(env, map, "path", la.path.c_str());
         putDouble(env, map, "scaled", la.scale);

package/cpp/README.md

@@ -1,4 +1,4 @@
 # Note
 
-- Only `rn-llama.hpp` is the specific file for this project, others are sync from [llama.cpp](https://github.com/ggerganov/llama.cpp).
+- Only `rn-llama.h` and `rn-llama.cpp` are the specific files for this folder, others are sync from [llama.cpp](https://github.com/ggerganov/llama.cpp).
 - We can update the native source by using the [bootstrap](../scripts/bootstrap.sh) script.