cui-llama.rn 1.2.6 → 1.3.0

package/README.md

@@ -6,13 +6,14 @@ This fork exists to update llama.cpp on a more frequent basis, plus adding usefu
 
 The following features have been added for Android:
 
-- Updated sync for llama.cpp
 - Added stopping prompt processing between batches, vital for mobile devices with very slow prompt processing
 - `vocab_only` mode: utilize the llama.cpp tokenizer
 - tokenizeSync: non-blocking, synchronous tokenizer function
 - Context Shift taken from [kobold.cpp](https://github.com/LostRuins/koboldcpp)
 - Retrieving CPU Features to check for i8mm and dotprod flags
 
+There is no IOS implementation for these features.
+
 Original repo README.md below.
 
 # llama.rn
@@ -305,7 +306,7 @@ iOS:
 
 - The [Extended Virtual Addressing](https://developer.apple.com/documentation/bundleresources/entitlements/com_apple_developer_kernel_extended-virtual-addressing) capability is recommended to enable on iOS project.
 - Metal:
-  - We have tested to know some devices is not able to use Metal ('params.n_gpu_layers > 0') due to llama.cpp used SIMD-scoped operation, you can check if your device is supported in [Metal feature set tables](https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf), Apple7 GPU will be the minimum requirement.
+  - We have tested to know some devices is not able to use Metal (GPU) due to llama.cpp used SIMD-scoped operation, you can check if your device is supported in [Metal feature set tables](https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf), Apple7 GPU will be the minimum requirement.
   - It's also not supported in iOS simulator due to [this limitation](https://developer.apple.com/documentation/metal/developing_metal_apps_that_run_in_simulator#3241609), we used constant buffers more than 14.
 
 Android:

package/android/src/main/CMakeLists.txt

@@ -14,22 +14,28 @@ set(
     ${RNLLAMA_LIB_DIR}/llama-vocab.cpp
     ${RNLLAMA_LIB_DIR}/log.cpp
 
+    ${RNLLAMA_LIB_DIR}/ggml-aarch64.c
+    ${RNLLAMA_LIB_DIR}/llama-grammar.cpp
+    ${RNLLAMA_LIB_DIR}/llama-sampling.cpp
+    ${RNLLAMA_LIB_DIR}/llama-vocab.cpp
+    ${RNLLAMA_LIB_DIR}/log.cpp
+
     ${RNLLAMA_LIB_DIR}/ggml-aarch64.c
     ${RNLLAMA_LIB_DIR}/ggml-alloc.c
     ${RNLLAMA_LIB_DIR}/ggml-backend.cpp
+    ${RNLLAMA_LIB_DIR}/ggml-backend-reg.cpp
     ${RNLLAMA_LIB_DIR}/ggml.c
     ${RNLLAMA_LIB_DIR}/ggml-cpu.c
+    ${RNLLAMA_LIB_DIR}/ggml-cpu.cpp
+    ${RNLLAMA_LIB_DIR}/ggml-cpu-aarch64.c
+    ${RNLLAMA_LIB_DIR}/ggml-cpu-quants.c
+    ${RNLLAMA_LIB_DIR}/ggml-threading.cpp
     ${RNLLAMA_LIB_DIR}/ggml-quants.c
     ${RNLLAMA_LIB_DIR}/common.cpp
-    ${RNLLAMA_LIB_DIR}/json.hpp
-    ${RNLLAMA_LIB_DIR}/json-schema-to-grammar.cpp
     ${RNLLAMA_LIB_DIR}/sampling.cpp
     ${RNLLAMA_LIB_DIR}/unicode-data.cpp
     ${RNLLAMA_LIB_DIR}/unicode.cpp
     ${RNLLAMA_LIB_DIR}/llama.cpp
-    ${RNLLAMA_LIB_DIR}/llama-vocab.cpp
-    ${RNLLAMA_LIB_DIR}/llama-sampling.cpp
-    ${RNLLAMA_LIB_DIR}/llama-grammar.cpp
     ${RNLLAMA_LIB_DIR}/sgemm.cpp
     ${RNLLAMA_LIB_DIR}/ggml-aarch64.c
     ${RNLLAMA_LIB_DIR}/rn-llama.hpp
@@ -72,11 +78,20 @@ build_library("rnllama" "")
 
 if (${ANDROID_ABI} STREQUAL "arm64-v8a")
     # ARM64 targets
+    build_library("rnllama_v8_4_fp16_dotprod_sve" "-march=armv8.4-a+fp16+dotprod+sve")
+    build_library("rnllama_v8_4_fp16_dotprod_i8mm_sve" "-march=armv8.4-a+fp16+dotprod+i8mm+sve")
     build_library("rnllama_v8_4_fp16_dotprod_i8mm" "-march=armv8.4-a+fp16+dotprod+i8mm")
     build_library("rnllama_v8_4_fp16_dotprod" "-march=armv8.4-a+fp16+dotprod")
     build_library("rnllama_v8_2_fp16_dotprod" "-march=armv8.2-a+fp16+dotprod")
     build_library("rnllama_v8_2_fp16" "-march=armv8.2-a+fp16")
     build_library("rnllama_v8" "-march=armv8-a")
+
+    # https://github.com/ggerganov/llama.cpp/blob/master/docs/android.md#cross-compile-using-android-ndk
+    # llama.cpp will deal with the cpu features
+    # build_library("rnllama_v8_7" "-march=armv8.7-a")
+    # TODO: Add support runtime check for cpu features
+    # At the moment runtime check is failing.
+
 elseif (${ANDROID_ABI} STREQUAL "x86_64")
     # x86_64 target
     build_library("rnllama_x86_64" "-march=x86-64" "-mtune=intel" "-msse4.2" "-mpopcnt")

package/android/src/main/java/com/rnllama/LlamaContext.java

@@ -93,7 +93,7 @@ public class LlamaContext {
         Log.e(NAME, "Failed to convert to FD!");
       }
     }
-    
+    logToAndroid();
     // Check if file has GGUF magic numbers
     this.id = id;
     eventEmitter = reactContext.getJSModule(DeviceEventManagerModule.RCTDeviceEventEmitter.class);
@@ -102,6 +102,8 @@ public class LlamaContext {
       modelName,
       // boolean embedding,
       params.hasKey("embedding") ? params.getBoolean("embedding") : false,
+      // int embd_normalize,
+      params.hasKey("embd_normalize") ? params.getInt("embd_normalize") : -1,
       // int n_ctx,
       params.hasKey("n_ctx") ? params.getInt("n_ctx") : 512,
       // int n_batch,
@@ -110,6 +112,12 @@ public class LlamaContext {
       params.hasKey("n_threads") ? params.getInt("n_threads") : 0,
       // int n_gpu_layers, // TODO: Support this
       params.hasKey("n_gpu_layers") ? params.getInt("n_gpu_layers") : 0,
+      // boolean flash_attn,
+      params.hasKey("flash_attn") ? params.getBoolean("flash_attn") : false,
+      // String cache_type_k,
+      params.hasKey("cache_type_k") ? params.getString("cache_type_k") : "f16",
+      // String cache_type_v,
+      params.hasKey("cache_type_v") ? params.getString("cache_type_v") : "f16",
       // boolean use_mlock,
       params.hasKey("use_mlock") ? params.getBoolean("use_mlock") : true,
       // boolean use_mmap,
@@ -124,12 +132,22 @@ public class LlamaContext {
       params.hasKey("rope_freq_base") ? (float) params.getDouble("rope_freq_base") : 0.0f,
       // float rope_freq_scale
       params.hasKey("rope_freq_scale") ? (float) params.getDouble("rope_freq_scale") : 0.0f,
-      this
+      // int pooling_type,
+      params.hasKey("pooling_type") ? params.getInt("pooling_type") : -1,
+      // LoadProgressCallback load_progress_callback
+      params.hasKey("use_progress_callback") ? new LoadProgressCallback(this) : null
     );
+    if (this.context == -1) {
+      throw new IllegalStateException("Failed to initialize context");
+    }
     this.modelDetails = loadModelDetails(this.context);
     this.reactContext = reactContext;
   }
 
+  public void interruptLoad() {
+    interruptLoad(this.context);
+  }
+
   public long getContext() {
     return context;
   }
@@ -146,6 +164,25 @@ public class LlamaContext {
     return getFormattedChat(this.context, msgs, chatTemplate == null ? "" : chatTemplate);
   }
 
+  private void emitLoadProgress(int progress) {
+    WritableMap event = Arguments.createMap();
+    event.putInt("contextId", LlamaContext.this.id);
+    event.putInt("progress", progress);
+    eventEmitter.emit("@RNLlama_onInitContextProgress", event);
+  }
+
+  private static class LoadProgressCallback {
+    LlamaContext context;
+
+    public LoadProgressCallback(LlamaContext context) {
+      this.context = context;
+    }
+
+    void onLoadProgress(int progress) {
+      context.emitLoadProgress(progress);
+    }
+  }
+
   private void emitPartialCompletion(WritableMap tokenResult) {
     WritableMap event = Arguments.createMap();
     event.putInt("contextId", LlamaContext.this.id);
@@ -244,10 +281,10 @@ public class LlamaContext {
       params.hasKey("top_p") ? (float) params.getDouble("top_p") : 0.95f,
       // float min_p,
       params.hasKey("min_p") ? (float) params.getDouble("min_p") : 0.05f,
-      // float xtc_t,
-      params.hasKey("xtc_t") ? (float) params.getDouble("xtc_t") : 0.00f,
-      // float xtc_p,
-      params.hasKey("xtc_p") ? (float) params.getDouble("xtc_p") : 0.00f,
+      // float xtc_threshold,
+      params.hasKey("xtc_threshold") ? (float) params.getDouble("xtc_threshold") : 0.00f,
+      // float xtc_probability,
+      params.hasKey("xtc_probability") ? (float) params.getDouble("xtc_probability") : 0.00f,
       // float typical_p,
       params.hasKey("typical_p") ? (float) params.getDouble("typical_p") : 1.00f,
       // int seed,
@@ -258,6 +295,16 @@ public class LlamaContext {
       params.hasKey("ignore_eos") ? params.getBoolean("ignore_eos") : false,
       // double[][] logit_bias,
       logit_bias,
+      // float dry_multiplier,
+      params.hasKey("dry_multiplier") ? (float) params.getDouble("dry_multiplier") : 0.00f,
+      // float dry_base,
+      params.hasKey("dry_base") ? (float) params.getDouble("dry_base") : 1.75f,
+      // int dry_allowed_length,
+      params.hasKey("dry_allowed_length") ? params.getInt("dry_allowed_length") : 2,
+      // int dry_penalty_last_n,
+      params.hasKey("dry_penalty_last_n") ? params.getInt("dry_penalty_last_n") : -1,
+      // String[] dry_sequence_breakers, when undef, we use the default definition from common.h
+      params.hasKey("dry_sequence_breakers") ? params.getArray("dry_sequence_breakers").toArrayList().toArray(new String[0]) : new String[]{"\n", ":", "\"", "*"},
       // PartialCompletionCallback partial_completion_callback
       new PartialCompletionCallback(
         this,
@@ -292,11 +339,16 @@ public class LlamaContext {
     return detokenize(this.context, toks);
   }
 
-  public WritableMap getEmbedding(String text) {
+  public WritableMap getEmbedding(String text, ReadableMap params) {
     if (isEmbeddingEnabled(this.context) == false) {
       throw new IllegalStateException("Embedding is not enabled");
     }
-    WritableMap result = embedding(this.context, text);
+    WritableMap result = embedding(
+      this.context,
+      text,
+      // int embd_normalize,
+      params.hasKey("embd_normalize") ? params.getInt("embd_normalize") : -1
+    );
     if (result.hasKey("error")) {
       throw new IllegalStateException(result.getString("error"));
     }
@@ -313,17 +365,31 @@ public class LlamaContext {
 
   static {
     Log.d(NAME, "Primary ABI: " + Build.SUPPORTED_ABIS[0]);
-    if (LlamaContext.isArm64V8a()) {
-      String cpuFeatures = LlamaContext.getCpuFeatures();
-      Log.d(NAME, "CPU features: " + cpuFeatures);
-
-      boolean hasFp16 = cpuFeatures.contains("fp16") || cpuFeatures.contains("fphp");
-      boolean hasDotProd = cpuFeatures.contains("dotprod") || cpuFeatures.contains("asimddp");
-      boolean isAtLeastArmV82 = cpuFeatures.contains("asimd") && cpuFeatures.contains("crc32") && cpuFeatures.contains("aes");
-      boolean isAtLeastArmV84 = cpuFeatures.contains("dcpop") && cpuFeatures.contains("uscat");
-      boolean hasInt8Matmul = cpuFeatures.contains("i8mm");
 
-      if (isAtLeastArmV84 && hasFp16 && hasDotProd && hasInt8Matmul) {
+    String cpuFeatures = LlamaContext.getCpuFeatures();
+    Log.d(NAME, "CPU features: " + cpuFeatures);
+    boolean hasFp16 = cpuFeatures.contains("fp16") || cpuFeatures.contains("fphp");
+    boolean hasDotProd = cpuFeatures.contains("dotprod") || cpuFeatures.contains("asimddp");
+    boolean hasSve = cpuFeatures.contains("sve");
+    boolean hasI8mm = cpuFeatures.contains("i8mm");
+    boolean isAtLeastArmV82 = cpuFeatures.contains("asimd") && cpuFeatures.contains("crc32") && cpuFeatures.contains("aes");
+    boolean isAtLeastArmV84 = cpuFeatures.contains("dcpop") && cpuFeatures.contains("uscat");
+    Log.d(NAME, "- hasFp16: " + hasFp16);
+    Log.d(NAME, "- hasDotProd: " + hasDotProd);
+    Log.d(NAME, "- hasSve: " + hasSve);
+    Log.d(NAME, "- hasI8mm: " + hasI8mm);
+    Log.d(NAME, "- isAtLeastArmV82: " + isAtLeastArmV82);
+    Log.d(NAME, "- isAtLeastArmV84: " + isAtLeastArmV84);
+
+    // TODO: Add runtime check for cpu features
+    if (LlamaContext.isArm64V8a()) {
+      if (isAtLeastArmV84 && hasSve && hasI8mm && hasFp16 && hasDotProd) {
+        Log.d(NAME, "Loading librnllama_v8_4_fp16_dotprod_i8mm_sve.so");
+        System.loadLibrary("rnllama_v8_4_fp16_dotprod_i8mm_sve");
+      } else if (isAtLeastArmV84 && hasSve && hasFp16 && hasDotProd) {
+        Log.d(NAME, "Loading librnllama_v8_4_fp16_dotprod_sve.so");
+        System.loadLibrary("rnllama_v8_4_fp16_dotprod_sve");
+      } else if (isAtLeastArmV84 && hasI8mm && hasFp16 && hasDotProd) {
         Log.d(NAME, "Loading librnllama_v8_4_fp16_dotprod_i8mm.so");
         System.loadLibrary("rnllama_v8_4_fp16_dotprod_i8mm");
       } else if (isAtLeastArmV84 && hasFp16 && hasDotProd) {
@@ -339,14 +405,16 @@ public class LlamaContext {
         Log.d(NAME, "Loading librnllama_v8.so");
         System.loadLibrary("rnllama_v8");
       }
+      //  Log.d(NAME, "Loading librnllama_v8_7.so with runtime feature detection");
+      //  System.loadLibrary("rnllama_v8_7");
     } else if (LlamaContext.isX86_64()) {
-      Log.d(NAME, "Loading librnllama_x86_64.so");
-      System.loadLibrary("rnllama_x86_64");
+        Log.d(NAME, "Loading librnllama_x86_64.so");
+        System.loadLibrary("rnllama_x86_64");
     } else {
-      Log.d(NAME, "Loading default librnllama.so");
-      System.loadLibrary("rnllama");
+        Log.d(NAME, "Loading default librnllama.so");
+        System.loadLibrary("rnllama");
     }
-  }
+}
 
   public static boolean isArm64V8a() {
     return Build.SUPPORTED_ABIS[0].equals("arm64-v8a");
@@ -382,13 +450,21 @@ public class LlamaContext {
     eventEmitter.emit("@RNLlama_onModelProgress", event);
   }
 
+  protected static native WritableMap modelInfo(
+    String model,
+    String[] skip
+  );
   protected static native long initContext(
     String model,
     boolean embedding,
+    int embd_normalize,
     int n_ctx,
     int n_batch,
     int n_threads,
     int n_gpu_layers, // TODO: Support this
+    boolean flash_attn,
+    String cache_type_k,
+    String cache_type_v,
     boolean use_mlock,
     boolean use_mmap,
     boolean vocab_only,
@@ -396,8 +472,10 @@ public class LlamaContext {
     float lora_scaled,
     float rope_freq_base,
     float rope_freq_scale,
-    LlamaContext javaLlamaContext
+    int pooling_type,
+    LoadProgressCallback load_progress_callback
   );
+  protected static native void interruptLoad(long contextPtr);
   protected static native WritableMap loadModelDetails(
     long contextPtr
   );
@@ -434,13 +512,18 @@ public class LlamaContext {
     int top_k,
     float top_p,
     float min_p,
-    float xtc_t,
-    float xtc_p,
+    float xtc_threshold,
+    float xtc_probability,
     float typical_p,
     int seed,
     String[] stop,
     boolean ignore_eos,
     double[][] logit_bias,
+    float   dry_multiplier,
+    float   dry_base,
+    int dry_allowed_length,    
+    int dry_penalty_last_n,
+    String[] dry_sequence_breakers,
     PartialCompletionCallback partial_completion_callback
   );
   protected static native void stopCompletion(long contextPtr);
@@ -448,7 +531,12 @@ public class LlamaContext {
   protected static native WritableArray tokenize(long contextPtr, String text);
   protected static native String detokenize(long contextPtr, int[] tokens);
   protected static native boolean isEmbeddingEnabled(long contextPtr);
-  protected static native WritableMap embedding(long contextPtr, String text);
+  protected static native WritableMap embedding(
+    long contextPtr,
+    String text,
+    int embd_normalize
+  );
   protected static native String bench(long contextPtr, int pp, int tg, int pl, int nr);
   protected static native void freeContext(long contextPtr);
+  protected static native void logToAndroid();
 }

package/android/src/main/java/com/rnllama/RNLlama.java

@@ -42,21 +42,53 @@ public class RNLlama implements LifecycleEventListener {
     promise.resolve(null);
   }
 
-  public void initContext(final ReadableMap params, final Promise promise) {
+  public void modelInfo(final String model, final ReadableArray skip, final Promise promise) {
+    new AsyncTask<Void, Void, WritableMap>() {
+      private Exception exception;
+
+      @Override
+      protected WritableMap doInBackground(Void... voids) {
+        try {
+          String[] skipArray = new String[skip.size()];
+          for (int i = 0; i < skip.size(); i++) {
+            skipArray[i] = skip.getString(i);
+          }
+          return LlamaContext.modelInfo(model, skipArray);
+        } catch (Exception e) {
+          exception = e;
+        }
+        return null;
+      }
+
+      @Override
+      protected void onPostExecute(WritableMap result) {
+        if (exception != null) {
+          promise.reject(exception);
+          return;
+        }
+        promise.resolve(result);
+      }
+    }.executeOnExecutor(AsyncTask.THREAD_POOL_EXECUTOR);
+  }
+
+  public void initContext(double id, final ReadableMap params, final Promise promise) {
+    final int contextId = (int) id;
     AsyncTask task = new AsyncTask<Void, Void, WritableMap>() {
       private Exception exception;
 
       @Override
       protected WritableMap doInBackground(Void... voids) {
         try {
-          int id = Math.abs(new Random().nextInt());
-          LlamaContext llamaContext = new LlamaContext(id, reactContext, params);
+          LlamaContext context = contexts.get(contextId);
+          if (context != null) {
+            throw new Exception("Context already exists");
+          }
+          LlamaContext llamaContext = new LlamaContext(contextId, reactContext, params);
           if (llamaContext.getContext() == 0) {
             throw new Exception("Failed to initialize context");
           }
-          contexts.put(id, llamaContext);
+          contexts.put(contextId, llamaContext);
           WritableMap result = Arguments.createMap();
-          result.putInt("contextId", id);
           result.putBoolean("gpu", false);
           result.putString("reasonNoGPU", "Currently not supported");
           result.putMap("model", llamaContext.getModelDetails());
@@ -366,7 +398,7 @@ public class RNLlama implements LifecycleEventListener {
     tasks.put(task, "detokenize-" + contextId);
   }
 
-  public void embedding(double id, final String text, final Promise promise) {
+  public void embedding(double id, final String text, final ReadableMap params, final Promise promise) {
     final int contextId = (int) id;
     AsyncTask task = new AsyncTask<Void, Void, WritableMap>() {
       private Exception exception;
@@ -378,7 +410,7 @@ public class RNLlama implements LifecycleEventListener {
           if (context == null) {
             throw new Exception("Context not found");
           }
-          return context.getEmbedding(text);
+          return context.getEmbedding(text, params);
         } catch (Exception e) {
           exception = e;
         }
@@ -442,6 +474,7 @@ public class RNLlama implements LifecycleEventListener {
           if (context == null) {
             throw new Exception("Context " + id + " not found");
           }
+          context.interruptLoad();
           context.stopCompletion();
           AsyncTask completionTask = null;
           for (AsyncTask task : tasks.keySet()) {