cui-llama.rn 1.3.6 → 1.4.0

package/README.md

@@ -53,12 +53,23 @@ For get a GGUF model or quantize manually, see [`Prepare and Quantize`](https://
 
 ## Usage
 
+Load model info only:
+
+```js
+import { loadLlamaModelInfo } from 'llama.rn'
+
+const modelPath = 'file://<path to gguf model>'
+console.log('Model Info:', await loadLlamaModelInfo(modelPath))
+```
+
+Initialize a Llama context & do completion:
+
 ```js
 import { initLlama } from 'llama.rn'
 
 // Initial a Llama context with the model (may take a while)
 const context = await initLlama({
-  model: 'file://<path to gguf model>',
+  model: modelPath,
   use_mlock: true,
   n_ctx: 2048,
   n_gpu_layers: 1, // > 0: enable Metal on iOS
@@ -318,6 +329,16 @@ Android:
 
 See the [contributing guide](CONTRIBUTING.md) to learn how to contribute to the repository and the development workflow.
 
+## Apps using `llama.rn`
+
+- [BRICKS](https://bricks.tools): Our product for building interactive signage in simple way. We provide LLM functions as Generator LLM/Assistant.
+- [ChatterUI](https://github.com/Vali-98/ChatterUI): Simple frontend for LLMs built in react-native.
+- [PocketPal AI](https://github.com/a-ghorbani/pocketpal-ai): An app that brings language models directly to your phone.
+
+## Node.js binding
+
+- [llama.node](https://github.com/mybigday/llama.node): An another Node.js binding of `llama.cpp` but made API same as `llama.rn`.
+
 ## License
 
 MIT

package/android/src/main/CMakeLists.txt

@@ -9,45 +9,44 @@ include_directories(${RNLLAMA_LIB_DIR})
 
 set(
     SOURCE_FILES
-
-    ${RNLLAMA_LIB_DIR}/common.cpp
-    ${RNLLAMA_LIB_DIR}/llama-grammar.cpp
-    ${RNLLAMA_LIB_DIR}/llama-sampling.cpp
-    ${RNLLAMA_LIB_DIR}/llama-vocab.cpp
-    ${RNLLAMA_LIB_DIR}/llama-chat.cpp
-    ${RNLLAMA_LIB_DIR}/llama-mmap.cpp
-    ${RNLLAMA_LIB_DIR}/llama-context.cpp
-    ${RNLLAMA_LIB_DIR}/llama-kv-cache.cpp
-    ${RNLLAMA_LIB_DIR}/llama-model-loader.cpp
-    ${RNLLAMA_LIB_DIR}/llama-model.cpp
-    ${RNLLAMA_LIB_DIR}/llama-batch.cpp
-    ${RNLLAMA_LIB_DIR}/llama-arch.cpp
-    ${RNLLAMA_LIB_DIR}/llama-cparams.cpp
-    ${RNLLAMA_LIB_DIR}/llama-hparams.cpp
-    ${RNLLAMA_LIB_DIR}/llama-adapter.cpp
-    ${RNLLAMA_LIB_DIR}/llama-impl.cpp
-    ${RNLLAMA_LIB_DIR}/log.cpp
-    ${RNLLAMA_LIB_DIR}/json.hpp
-    ${RNLLAMA_LIB_DIR}/json-schema-to-grammar.cpp
-
+    ${RNLLAMA_LIB_DIR}/ggml.c
     ${RNLLAMA_LIB_DIR}/ggml-alloc.c
     ${RNLLAMA_LIB_DIR}/ggml-backend.cpp
     ${RNLLAMA_LIB_DIR}/ggml-backend-reg.cpp
-    ${RNLLAMA_LIB_DIR}/ggml.c
-    ${RNLLAMA_LIB_DIR}/gguf.cpp
     ${RNLLAMA_LIB_DIR}/ggml-cpu.c
     ${RNLLAMA_LIB_DIR}/ggml-cpu.cpp
     ${RNLLAMA_LIB_DIR}/ggml-cpu-aarch64.cpp
-    ${RNLLAMA_LIB_DIR}/ggml-cpu-traits.cpp
     ${RNLLAMA_LIB_DIR}/ggml-cpu-quants.c
+    ${RNLLAMA_LIB_DIR}/ggml-cpu-traits.cpp
+    ${RNLLAMA_LIB_DIR}/ggml-opt.cpp
     ${RNLLAMA_LIB_DIR}/ggml-threading.cpp
     ${RNLLAMA_LIB_DIR}/ggml-quants.c
+    ${RNLLAMA_LIB_DIR}/gguf.cpp
+    ${RNLLAMA_LIB_DIR}/log.cpp
+    ${RNLLAMA_LIB_DIR}/llama-impl.cpp
+    ${RNLLAMA_LIB_DIR}/llama-grammar.cpp
+    ${RNLLAMA_LIB_DIR}/llama-sampling.cpp
+    ${RNLLAMA_LIB_DIR}/llama-vocab.cpp
+    ${RNLLAMA_LIB_DIR}/llama-adapter.cpp
+    ${RNLLAMA_LIB_DIR}/llama-chat.cpp
+    ${RNLLAMA_LIB_DIR}/llama-context.cpp
+    ${RNLLAMA_LIB_DIR}/llama-kv-cache.cpp
+    ${RNLLAMA_LIB_DIR}/llama-arch.cpp
+    ${RNLLAMA_LIB_DIR}/llama-batch.cpp
+    ${RNLLAMA_LIB_DIR}/llama-cparams.cpp
+    ${RNLLAMA_LIB_DIR}/llama-hparams.cpp
+    ${RNLLAMA_LIB_DIR}/llama.cpp
+    ${RNLLAMA_LIB_DIR}/llama-model.cpp
+    ${RNLLAMA_LIB_DIR}/llama-model-loader.cpp
+    ${RNLLAMA_LIB_DIR}/llama-mmap.cpp
+    ${RNLLAMA_LIB_DIR}/llama-vocab.cpp
     ${RNLLAMA_LIB_DIR}/sampling.cpp
     ${RNLLAMA_LIB_DIR}/unicode-data.cpp
     ${RNLLAMA_LIB_DIR}/unicode.cpp
-    ${RNLLAMA_LIB_DIR}/llama.cpp
     ${RNLLAMA_LIB_DIR}/sgemm.cpp
+    ${RNLLAMA_LIB_DIR}/common.cpp
     ${RNLLAMA_LIB_DIR}/rn-llama.hpp
+    ${CMAKE_SOURCE_DIR}/jni-utils.h
     ${CMAKE_SOURCE_DIR}/jni.cpp
 )
 
@@ -62,7 +61,7 @@ function(build_library target_name cpu_flags)
 
     target_link_libraries(${target_name} ${LOG_LIB} android)
 
-    target_compile_options(${target_name} PRIVATE -pthread ${cpu_flags} -DLM_GGML_USE_CPU -DLM_GGML_USE_CPU_AARCH64)
+    target_compile_options(${target_name} PRIVATE -DLM_GGML_USE_CPU -DLM_GGML_USE_CPU_AARCH64 -pthread ${cpu_flags})
 
     if (${CMAKE_BUILD_TYPE} STREQUAL "Debug")
         target_compile_options(${target_name} PRIVATE -DRNLLAMA_ANDROID_ENABLE_LOGGING)

package/android/src/main/java/com/rnllama/LlamaContext.java

@@ -108,6 +108,8 @@ public class LlamaContext {
       params.hasKey("n_ctx") ? params.getInt("n_ctx") : 512,
       // int n_batch,
       params.hasKey("n_batch") ? params.getInt("n_batch") : 512,
+      // int n_ubatch,
+      params.hasKey("n_ubatch") ? params.getInt("n_ubatch") : 512,
       // int n_threads,
       params.hasKey("n_threads") ? params.getInt("n_threads") : 0,
       // int n_gpu_layers, // TODO: Support this
@@ -115,9 +117,9 @@ public class LlamaContext {
       // boolean flash_attn,
       params.hasKey("flash_attn") ? params.getBoolean("flash_attn") : false,
       // String cache_type_k,
-      params.hasKey("cache_type_k") ? params.getInt("cache_type_k") : 1,
+      params.hasKey("cache_type_k") ? params.getString("cache_type_k") : "f16",
       // String cache_type_v,
-      params.hasKey("cache_type_v") ? params.getInt("cache_type_v") : 1,
+      params.hasKey("cache_type_v") ? params.getString("cache_type_v") : "f16",
       // boolean use_mlock,
       params.hasKey("use_mlock") ? params.getBoolean("use_mlock") : true,
       // boolean use_mmap,
@@ -128,6 +130,8 @@ public class LlamaContext {
       params.hasKey("lora") ? params.getString("lora") : "",
       // float lora_scaled,
       params.hasKey("lora_scaled") ? (float) params.getDouble("lora_scaled") : 1.0f,
+      // ReadableArray lora_adapters,
+      params.hasKey("lora_list") ? params.getArray("lora_list") : null,
       // float rope_freq_base,
       params.hasKey("rope_freq_base") ? (float) params.getDouble("rope_freq_base") : 0.0f,
       // float rope_freq_scale
@@ -168,7 +172,7 @@ public class LlamaContext {
     WritableMap event = Arguments.createMap();
     event.putInt("contextId", LlamaContext.this.id);
     event.putInt("progress", progress);
-    eventEmitter.emit("@RNLlama_onInitContextProgress", event);
+    eventEmitter.emit("@RNLlama_onContextProgress", event);
   }
 
   private static class LoadProgressCallback {
@@ -273,8 +277,6 @@ public class LlamaContext {
       params.hasKey("mirostat_tau") ? (float) params.getDouble("mirostat_tau") : 5.00f,
       // float mirostat_eta,
       params.hasKey("mirostat_eta") ? (float) params.getDouble("mirostat_eta") : 0.10f,
-      // boolean penalize_nl,
-      params.hasKey("penalize_nl") ? params.getBoolean("penalize_nl") : false,
       // int top_k,
       params.hasKey("top_k") ? params.getInt("top_k") : 40,
       // float top_p,
@@ -359,6 +361,22 @@ public class LlamaContext {
     return bench(this.context, pp, tg, pl, nr);
   }
 
+  public int applyLoraAdapters(ReadableArray loraAdapters) {
+    int result = applyLoraAdapters(this.context, loraAdapters);
+    if (result != 0) {
+      throw new IllegalStateException("Failed to apply lora adapters");
+    }
+    return result;
+  }
+
+  public void removeLoraAdapters() {
+    removeLoraAdapters(this.context);
+  }
+
+  public WritableArray getLoadedLoraAdapters() {
+    return getLoadedLoraAdapters(this.context);
+  }
+
   public void release() {
     freeContext(context);
   }
@@ -460,16 +478,18 @@ public class LlamaContext {
     int embd_normalize,
     int n_ctx,
     int n_batch,
+    int n_ubatch,
     int n_threads,
     int n_gpu_layers, // TODO: Support this
     boolean flash_attn,
-    int cache_type_k,
-    int cache_type_v,
+    String cache_type_k,
+    String cache_type_v,
     boolean use_mlock,
     boolean use_mmap,
     boolean vocab_only,
     String lora,
     float lora_scaled,
+    ReadableArray lora_list,
     float rope_freq_base,
     float rope_freq_scale,
     int pooling_type,
@@ -508,7 +528,6 @@ public class LlamaContext {
     float mirostat,
     float mirostat_tau,
     float mirostat_eta,
-    boolean penalize_nl,
     int top_k,
     float top_p,
     float min_p,
@@ -521,7 +540,7 @@ public class LlamaContext {
     double[][] logit_bias,
     float   dry_multiplier,
     float   dry_base,
-    int dry_allowed_length,    
+    int dry_allowed_length,
     int dry_penalty_last_n,
     String[] dry_sequence_breakers,
     PartialCompletionCallback partial_completion_callback
@@ -537,6 +556,9 @@ public class LlamaContext {
     int embd_normalize
   );
   protected static native String bench(long contextPtr, int pp, int tg, int pl, int nr);
+  protected static native int applyLoraAdapters(long contextPtr, ReadableArray loraAdapters);
+  protected static native void removeLoraAdapters(long contextPtr);
+  protected static native WritableArray getLoadedLoraAdapters(long contextPtr);
   protected static native void freeContext(long contextPtr);
   protected static native void logToAndroid();
 }

package/android/src/main/java/com/rnllama/RNLlama.java

@@ -462,6 +462,104 @@ public class RNLlama implements LifecycleEventListener {
     tasks.put(task, "bench-" + contextId);
   }
 
+  public void applyLoraAdapters(double id, final ReadableArray loraAdapters, final Promise promise) {
+    final int contextId = (int) id;
+    AsyncTask task = new AsyncTask<Void, Void, Void>() {
+      private Exception exception;
+
+      @Override
+      protected Void doInBackground(Void... voids) {
+        try {
+          LlamaContext context = contexts.get(contextId);
+          if (context == null) {
+            throw new Exception("Context not found");
+          }
+          if (context.isPredicting()) {
+            throw new Exception("Context is busy");
+          }
+          context.applyLoraAdapters(loraAdapters);
+        } catch (Exception e) {
+          exception = e;
+        }
+        return null;
+      }
+
+      @Override
+      protected void onPostExecute(Void result) {
+        if (exception != null) {
+          promise.reject(exception);
+          return;
+        }
+      }
+    }.executeOnExecutor(AsyncTask.THREAD_POOL_EXECUTOR);
+    tasks.put(task, "applyLoraAdapters-" + contextId);
+  }
+
+  public void removeLoraAdapters(double id, final Promise promise) {
+    final int contextId = (int) id;
+    AsyncTask task = new AsyncTask<Void, Void, Void>() {
+      private Exception exception;
+
+      @Override
+      protected Void doInBackground(Void... voids) {
+        try {
+          LlamaContext context = contexts.get(contextId);
+          if (context == null) {
+            throw new Exception("Context not found");
+          }
+          if (context.isPredicting()) {
+            throw new Exception("Context is busy");
+          }
+          context.removeLoraAdapters();
+        } catch (Exception e) {
+          exception = e;
+        }
+        return null;
+      }
+
+      @Override
+      protected void onPostExecute(Void result) {
+        if (exception != null) {
+          promise.reject(exception);
+          return;
+        }
+        promise.resolve(null);
+      }
+    }.executeOnExecutor(AsyncTask.THREAD_POOL_EXECUTOR);
+    tasks.put(task, "removeLoraAdapters-" + contextId);
+  }
+
+  public void getLoadedLoraAdapters(double id, final Promise promise) {
+    final int contextId = (int) id;
+    AsyncTask task = new AsyncTask<Void, Void, ReadableArray>() {
+      private Exception exception;
+
+      @Override
+      protected ReadableArray doInBackground(Void... voids) {
+        try {
+          LlamaContext context = contexts.get(contextId);
+          if (context == null) {
+            throw new Exception("Context not found");
+          }
+          return context.getLoadedLoraAdapters();
+        } catch (Exception e) {
+          exception = e;
+        }
+        return null;
+      }
+
+      @Override
+      protected void onPostExecute(ReadableArray result) {
+        if (exception != null) {
+          promise.reject(exception);
+          return;
+        }
+        promise.resolve(result);
+      }
+    }.executeOnExecutor(AsyncTask.THREAD_POOL_EXECUTOR);
+    tasks.put(task, "getLoadedLoraAdapters-" + contextId);
+  }
+
   public void releaseContext(double id, Promise promise) {
     final int contextId = (int) id;
     AsyncTask task = new AsyncTask<Void, Void, Void>() {

package/android/src/main/jni-utils.h

@@ -0,0 +1,94 @@
+#include <jni.h>
+
+// ReadableMap utils
+
+namespace readablearray {
+
+int size(JNIEnv *env, jobject readableArray) {
+    jclass arrayClass = env->GetObjectClass(readableArray);
+    jmethodID sizeMethod = env->GetMethodID(arrayClass, "size", "()I");
+    return env->CallIntMethod(readableArray, sizeMethod);
+}
+
+jobject getMap(JNIEnv *env, jobject readableArray, int index) {
+    jclass arrayClass = env->GetObjectClass(readableArray);
+    jmethodID getMapMethod = env->GetMethodID(arrayClass, "getMap", "(I)Lcom/facebook/react/bridge/ReadableMap;");
+    return env->CallObjectMethod(readableArray, getMapMethod, index);
+}
+
+// Other methods not used yet
+
+}
+
+namespace readablemap {
+
+bool hasKey(JNIEnv *env, jobject readableMap, const char *key) {
+    jclass mapClass = env->GetObjectClass(readableMap);
+    jmethodID hasKeyMethod = env->GetMethodID(mapClass, "hasKey", "(Ljava/lang/String;)Z");
+    jstring jKey = env->NewStringUTF(key);
+    jboolean result = env->CallBooleanMethod(readableMap, hasKeyMethod, jKey);
+    env->DeleteLocalRef(jKey);
+    return result;
+}
+
+int getInt(JNIEnv *env, jobject readableMap, const char *key, jint defaultValue) {
+    if (!hasKey(env, readableMap, key)) {
+        return defaultValue;
+    }
+    jclass mapClass = env->GetObjectClass(readableMap);
+    jmethodID getIntMethod = env->GetMethodID(mapClass, "getInt", "(Ljava/lang/String;)I");
+    jstring jKey = env->NewStringUTF(key);
+    jint result = env->CallIntMethod(readableMap, getIntMethod, jKey);
+    env->DeleteLocalRef(jKey);
+    return result;
+}
+
+bool getBool(JNIEnv *env, jobject readableMap, const char *key, jboolean defaultValue) {
+    if (!hasKey(env, readableMap, key)) {
+        return defaultValue;
+    }
+    jclass mapClass = env->GetObjectClass(readableMap);
+    jmethodID getBoolMethod = env->GetMethodID(mapClass, "getBoolean", "(Ljava/lang/String;)Z");
+    jstring jKey = env->NewStringUTF(key);
+    jboolean result = env->CallBooleanMethod(readableMap, getBoolMethod, jKey);
+    env->DeleteLocalRef(jKey);
+    return result;
+}
+
+long getLong(JNIEnv *env, jobject readableMap, const char *key, jlong defaultValue) {
+    if (!hasKey(env, readableMap, key)) {
+        return defaultValue;
+    }
+    jclass mapClass = env->GetObjectClass(readableMap);
+    jmethodID getLongMethod = env->GetMethodID(mapClass, "getLong", "(Ljava/lang/String;)J");
+    jstring jKey = env->NewStringUTF(key);
+    jlong result = env->CallLongMethod(readableMap, getLongMethod, jKey);
+    env->DeleteLocalRef(jKey);
+    return result;
+}
+
+float getFloat(JNIEnv *env, jobject readableMap, const char *key, jfloat defaultValue) {
+    if (!hasKey(env, readableMap, key)) {
+        return defaultValue;
+    }
+    jclass mapClass = env->GetObjectClass(readableMap);
+    jmethodID getFloatMethod = env->GetMethodID(mapClass, "getDouble", "(Ljava/lang/String;)D");
+    jstring jKey = env->NewStringUTF(key);
+    jfloat result = env->CallDoubleMethod(readableMap, getFloatMethod, jKey);
+    env->DeleteLocalRef(jKey);
+    return result;
+}
+
+jstring getString(JNIEnv *env, jobject readableMap, const char *key, jstring defaultValue) {
+    if (!hasKey(env, readableMap, key)) {
+        return defaultValue;
+    }
+    jclass mapClass = env->GetObjectClass(readableMap);
+    jmethodID getStringMethod = env->GetMethodID(mapClass, "getString", "(Ljava/lang/String;)Ljava/lang/String;");
+    jstring jKey = env->NewStringUTF(key);
+    jstring result = (jstring) env->CallObjectMethod(readableMap, getStringMethod, jKey);
+    env->DeleteLocalRef(jKey);
+    return result;
+}
+
+}