cui-llama.rn 1.1.7 → 1.2.1

package/README.md

@@ -12,6 +12,8 @@ The following features have been added for Android:
 - tokenizeSync: non-blocking, synchronous tokenizer function
 - Context Shift taken from [kobold.cpp](https://github.com/LostRuins/koboldcpp)
 - XTC sampling
+- Progress callback
+- Retrieving CPU Features to check for i8mm and dotprod flags
 
 Original repo README.md below.
 

package/android/src/main/java/com/rnllama/LlamaContext.java

@@ -71,6 +71,7 @@ public class LlamaContext {
     }
 
     this.id = id;
+    eventEmitter = reactContext.getJSModule(DeviceEventManagerModule.RCTDeviceEventEmitter.class);
     this.context = initContext(
       // String model,
       params.getString("model"),
@@ -97,11 +98,11 @@ public class LlamaContext {
       // float rope_freq_base,
       params.hasKey("rope_freq_base") ? (float) params.getDouble("rope_freq_base") : 0.0f,
       // float rope_freq_scale
-      params.hasKey("rope_freq_scale") ? (float) params.getDouble("rope_freq_scale") : 0.0f
+      params.hasKey("rope_freq_scale") ? (float) params.getDouble("rope_freq_scale") : 0.0f,
+      this
     );
     this.modelDetails = loadModelDetails(this.context);
     this.reactContext = reactContext;
-    eventEmitter = reactContext.getJSModule(DeviceEventManagerModule.RCTDeviceEventEmitter.class);
   }
 
   public long getContext() {
@@ -324,7 +325,7 @@ public class LlamaContext {
     }
   }
 
-  private static boolean isArm64V8a() {
+  public static boolean isArm64V8a() {
     return Build.SUPPORTED_ABIS[0].equals("arm64-v8a");
   }
 
@@ -332,7 +333,7 @@ public class LlamaContext {
     return Build.SUPPORTED_ABIS[0].equals("x86_64");
   }
 
-  private static String getCpuFeatures() {
+  public static String getCpuFeatures() {
     File file = new File("/proc/cpuinfo");
     StringBuilder stringBuilder = new StringBuilder();
     try {
@@ -352,6 +353,12 @@ public class LlamaContext {
     }
   }
 
+  public void emitModelProgressUpdate(int progress) {
+    WritableMap event = Arguments.createMap();
+    event.putInt("progress", progress);
+    eventEmitter.emit("@RNLlama_onModelProgress", event);
+  }
+
   protected static native long initContext(
     String model,
     boolean embedding,
@@ -365,7 +372,8 @@ public class LlamaContext {
     String lora,
     float lora_scaled,
     float rope_freq_base,
-    float rope_freq_scale
+    float rope_freq_scale,
+    LlamaContext javaLlamaContext
   );
   protected static native WritableMap loadModelDetails(
     long contextPtr

package/android/src/main/java/com/rnllama/RNLlama.java

@@ -294,7 +294,46 @@ public class RNLlama implements LifecycleEventListener {
     return context.tokenize(text);
   }
 
+  public void getCpuFeatures(Promise promise) {
+    AsyncTask task = new AsyncTask<Void, Void, WritableMap>() {
+      private Exception exception;
+      @Override
+      protected WritableMap doInBackground(Void... voids) {
+        try {
+          WritableMap result = Arguments.createMap();
+          boolean isV8 = LlamaContext.isArm64V8a();
+          result.putBoolean("armv8", isV8);
+          
+          if(isV8) {
+            String cpuFeatures = LlamaContext.getCpuFeatures();
+            boolean hasDotProd = cpuFeatures.contains("dotprod") || cpuFeatures.contains("asimddp");
+            boolean hasInt8Matmul = cpuFeatures.contains("i8mm");
+            result.putBoolean("i8mm", hasInt8Matmul);
+            result.putBoolean("dotprod", hasDotProd);
+          } else {
+            result.putBoolean("i8mm", false);
+            result.putBoolean("dotprod", false);
+          }
+          return result;
+        } catch (Exception e) {
+          exception = e;
+          return null;
+        }
+      }
 
+      @Override
+      protected void onPostExecute(WritableMap result) {
+        if (exception != null) {
+          promise.reject(exception);
+          return;
+        }
+        promise.resolve(result);
+        tasks.remove(this);
+      }
+    }.executeOnExecutor(AsyncTask.THREAD_POOL_EXECUTOR);
+    tasks.put(task, "getCPUFeatures");
+  }
+  
   public void detokenize(double id, final ReadableArray tokens, final Promise promise) {
     final int contextId = (int) id;
     AsyncTask task = new AsyncTask<Void, Void, String>() {

package/android/src/main/jni.cpp

@@ -128,6 +128,13 @@ static inline void putArray(JNIEnv *env, jobject map, const char *key, jobject v
 
 std::unordered_map<long, rnllama::llama_rn_context *> context_map;
 
+struct CallbackContext {
+    JNIEnv * env;
+    jobject  thiz;
+    jmethodID sendProgressMethod;
+    unsigned current;
+};
+
 JNIEXPORT jlong JNICALL
 Java_com_rnllama_LlamaContext_initContext(
     JNIEnv *env,
@@ -144,7 +151,8 @@ Java_com_rnllama_LlamaContext_initContext(
     jstring lora_str,
     jfloat lora_scaled,
     jfloat rope_freq_base,
-    jfloat rope_freq_scale
+    jfloat rope_freq_scale,
+    jobject javaLlamaContext
 ) {
     UNUSED(thiz);
 
@@ -169,7 +177,7 @@ Java_com_rnllama_LlamaContext_initContext(
     defaultParams.cpuparams.n_threads = n_threads > 0 ? n_threads : default_n_threads;
 
     defaultParams.n_gpu_layers = n_gpu_layers;
-
+    
     defaultParams.use_mlock = use_mlock;
     defaultParams.use_mmap = use_mmap;
 
@@ -182,6 +190,24 @@ Java_com_rnllama_LlamaContext_initContext(
     defaultParams.rope_freq_base = rope_freq_base;
     defaultParams.rope_freq_scale = rope_freq_scale;
 
+    // progress callback when loading
+    jclass llamaContextClass = env->GetObjectClass(javaLlamaContext);
+    jmethodID sendProgressMethod = env->GetMethodID(llamaContextClass, "emitModelProgressUpdate", "(I)V");
+
+    CallbackContext callbackctx = {env, javaLlamaContext, sendProgressMethod, 0};
+
+    defaultParams.progress_callback_user_data = &callbackctx;
+    defaultParams.progress_callback = [](float progress, void * ctx) {
+        unsigned percentage = (unsigned) (100 * progress);
+        CallbackContext * cbctx = static_cast<CallbackContext*>(ctx);
+        // reduce call frequency by only calling method when value changes
+        if (percentage <= cbctx->current) return true; 
+        cbctx->current = percentage;
+        cbctx->env->CallVoidMethod(cbctx->thiz, cbctx->sendProgressMethod, percentage);
+        return true;
+    };
+    
+
     auto llama = new rnllama::llama_rn_context();
     bool is_model_loaded = llama->loadModel(defaultParams);
 

package/android/src/newarch/java/com/rnllama/RNLlamaModule.java

@@ -78,6 +78,11 @@ public class RNLlamaModule extends NativeRNLlamaSpec {
     return rnllama.tokenizeSync(id, text);
   }
 
+  @ReactMethod
+  public void getCpuFeatures(final Promise promise) {
+    rnllama.getCpuFeatures(promise);
+  }
+
   @ReactMethod
   public void detokenize(double id, final ReadableArray tokens, final Promise promise) {
     rnllama.detokenize(id, tokens, promise);

package/android/src/oldarch/java/com/rnllama/RNLlamaModule.java

@@ -79,6 +79,11 @@ public class RNLlamaModule extends ReactContextBaseJavaModule {
     return rnllama.tokenizeSync(id, text);
   }
 
+  @ReactMethod
+  public void getCpuFeatures(final Promise promise) {
+    rnllama.getCpuFeatures(promise);
+  }
+
   @ReactMethod
   public void detokenize(double id, final ReadableArray tokens, final Promise promise) {  
     rnllama.detokenize(id, tokens, promise);

package/cpp/common.cpp

@@ -954,6 +954,9 @@ struct llama_model_params llama_model_params_from_gpt_params(const gpt_params &
     if (params.n_gpu_layers != -1) {
         mparams.n_gpu_layers = params.n_gpu_layers;
     }
+    
+    mparams.progress_callback_user_data = params.progress_callback_user_data;
+    mparams.progress_callback = params.progress_callback;
     mparams.vocab_only      = params.vocab_only;
     mparams.rpc_servers     = params.rpc_servers.c_str();
     mparams.main_gpu        = params.main_gpu;

package/cpp/common.h

@@ -158,6 +158,8 @@ struct gpt_sampler_params {
 
 struct gpt_params {
 
+    void * progress_callback_user_data        = nullptr;
+    llama_progress_callback progress_callback = nullptr;
     bool vocab_only               = false;
     int32_t n_predict             =    -1; // new tokens to predict
     int32_t n_ctx                 =     0; // context size