cui-llama.rn 1.4.3 → 1.4.6

package/README.md

@@ -76,7 +76,7 @@ const context = await initLlama({
   model: modelPath,
   use_mlock: true,
   n_ctx: 2048,
-  n_gpu_layers: 1, // > 0: enable Metal on iOS
+  n_gpu_layers: 99, // number of layers to store in VRAM (Currently only for iOS)
   // embedding: true, // use embedding
 })
 
@@ -141,148 +141,127 @@ Please visit the [Documentation](docs/API) for more details.
 
 You can also visit the [example](example) to see how to use it.
 
-## Grammar Sampling
+## Tool Calling
 
-GBNF (GGML BNF) is a format for defining [formal grammars](https://en.wikipedia.org/wiki/Formal_grammar) to constrain model outputs in `llama.cpp`. For example, you can use it to force the model to generate valid JSON, or speak only in emojis.
+`llama.rn` has universal tool call support by using [minja](https://github.com/google/minja) (as Jinja template parser) and [chat.cpp](https://github.com/ggerganov/llama.cpp/blob/master/common/chat.cpp) in llama.cpp.
 
-You can see [GBNF Guide](https://github.com/ggerganov/llama.cpp/tree/master/grammars) for more details.
-
-`llama.rn` provided a built-in function to convert JSON Schema to GBNF:
+Example:
 
 ```js
-import { initLlama, convertJsonSchemaToGrammar } from 'llama.rn'
-
-const schema = {
-  /* JSON Schema, see below */
-}
+import { initLlama } from 'llama.rn'
 
 const context = await initLlama({
-  model: 'file://<path to gguf model>',
-  use_mlock: true,
-  n_ctx: 2048,
-  n_gpu_layers: 1, // > 0: enable Metal on iOS
-  // embedding: true, // use embedding
-  grammar: convertJsonSchemaToGrammar({
-    schema,
-    propOrder: { function: 0, arguments: 1 },
-  }),
-})
-
-const { text } = await context.completion({
-  prompt: 'Schedule a birthday party on Aug 14th 2023 at 8pm.',
+  // ...params
 })
-console.log('Result:', text)
-// Example output:
-// {"function": "create_event","arguments":{"date": "Aug 14th 2023", "time": "8pm", "title": "Birthday Party"}}
-```
 
-<details>
-<summary>JSON Schema example (Define function get_current_weather / create_event / image_search)</summary>
-
-```json5
-{
-  oneOf: [
+const { text, tool_calls } = await context.completion({
+  // ...params
+  jinja: true, // Enable Jinja template parser
+  tool_choice: 'auto',
+  tools: [
     {
-      type: 'object',
-      name: 'get_current_weather',
-      description: 'Get the current weather in a given location',
-      properties: {
-        function: {
-          const: 'get_current_weather',
-        },
-        arguments: {
+      type: 'function',
+      function: {
+        name: 'ipython',
+        description:
+          'Runs code in an ipython interpreter and returns the result of the execution after 60 seconds.',
+        parameters: {
           type: 'object',
           properties: {
-            location: {
+            code: {
               type: 'string',
-              description: 'The city and state, e.g. San Francisco, CA',
-            },
-            unit: {
-              type: 'string',
-              enum: ['celsius', 'fahrenheit'],
+              description: 'The code to run in the ipython interpreter.',
             },
           },
-          required: ['location'],
+          required: ['code'],
         },
       },
     },
+  ],
+  messages: [
     {
-      type: 'object',
-      name: 'create_event',
-      description: 'Create a calendar event',
-      properties: {
-        function: {
-          const: 'create_event',
-        },
-        arguments: {
-          type: 'object',
-          properties: {
-            title: {
-              type: 'string',
-              description: 'The title of the event',
-            },
-            date: {
-              type: 'string',
-              description: 'The date of the event',
-            },
-            time: {
-              type: 'string',
-              description: 'The time of the event',
-            },
-          },
-          required: ['title', 'date', 'time'],
-        },
-      },
+      role: 'system',
+      content: 'You are a helpful assistant that can answer questions and help with tasks.',
     },
     {
-      type: 'object',
-      name: 'image_search',
-      description: 'Search for an image',
-      properties: {
-        function: {
-          const: 'image_search',
-        },
-        arguments: {
-          type: 'object',
-          properties: {
-            query: {
-              type: 'string',
-              description: 'The search query',
-            },
-          },
-          required: ['query'],
-        },
-      },
+      role: 'user',
+      content: 'Test',
     },
   ],
-}
+})
+console.log('Result:', text)
+// If tool_calls is not empty, it means the model has called the tool
+if (tool_calls) console.log('Tool Calls:', tool_calls)
 ```
 
-</details>
+You can check [chat.cpp](https://github.com/ggerganov/llama.cpp/blob/6eecde3cc8fda44da7794042e3668de4af3c32c6/common/chat.cpp#L7-L23) for models has native tool calling support, or it will fallback to `GENERIC` type tool call.
 
-<details>
-<summary>Converted GBNF looks like</summary>
+The generic tool call will be always JSON object as output, the output will be like `{"response": "..."}` when it not decided to use tool call.
 
+## Grammar Sampling
+
+GBNF (GGML BNF) is a format for defining [formal grammars](https://en.wikipedia.org/wiki/Formal_grammar) to constrain model outputs in `llama.cpp`. For example, you can use it to force the model to generate valid JSON, or speak only in emojis.
+
+You can see [GBNF Guide](https://github.com/ggerganov/llama.cpp/tree/master/grammars) for more details.
+
+`llama.rn` provided a built-in function to convert JSON Schema to GBNF:
+
+Example gbnf grammar:
 ```bnf
-space ::= " "?
-0-function ::= "\"get_current_weather\""
-string ::=  "\"" (
-        [^"\\] |
-        "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F])
-      )* "\"" space
-0-arguments-unit ::= "\"celsius\"" | "\"fahrenheit\""
-0-arguments ::= "{" space "\"location\"" space ":" space string "," space "\"unit\"" space ":" space 0-arguments-unit "}" space
-0 ::= "{" space "\"function\"" space ":" space 0-function "," space "\"arguments\"" space ":" space 0-arguments "}" space
-1-function ::= "\"create_event\""
-1-arguments ::= "{" space "\"date\"" space ":" space string "," space "\"time\"" space ":" space string "," space "\"title\"" space ":" space string "}" space
-1 ::= "{" space "\"function\"" space ":" space 1-function "," space "\"arguments\"" space ":" space 1-arguments "}" space
-2-function ::= "\"image_search\""
-2-arguments ::= "{" space "\"query\"" space ":" space string "}" space
-2 ::= "{" space "\"function\"" space ":" space 2-function "," space "\"arguments\"" space ":" space 2-arguments "}" space
-root ::= 0 | 1 | 2
+root   ::= object
+value  ::= object | array | string | number | ("true" | "false" | "null") ws
+
+object ::=
+  "{" ws (
+            string ":" ws value
+    ("," ws string ":" ws value)*
+  )? "}" ws
+
+array  ::=
+  "[" ws (
+            value
+    ("," ws value)*
+  )? "]" ws
+
+string ::=
+  "\"" (
+    [^"\\\x7F\x00-\x1F] |
+    "\\" (["\\bfnrt] | "u" [0-9a-fA-F]{4}) # escapes
+  )* "\"" ws
+
+number ::= ("-"? ([0-9] | [1-9] [0-9]{0,15})) ("." [0-9]+)? ([eE] [-+]? [0-9] [1-9]{0,15})? ws
+
+# Optional space: by convention, applied in this grammar after literal chars when allowed
+ws ::= | " " | "\n" [ \t]{0,20}
+```
+
+```js
+import { initLlama } from 'llama.rn'
+
+const gbnf = '...'
+
+const context = await initLlama({
+  // ...params
+  grammar: gbnf,
+})
+
+const { text } = await context.completion({
+  // ...params
+  messages: [
+    {
+      role: 'system',
+      content: 'You are a helpful assistant that can answer questions and help with tasks.',
+    },
+    {
+      role: 'user',
+      content: 'Test',
+    },
+  ],
+})
+console.log('Result:', text)
 ```
 
-</details>
+Also, this is how `json_schema` works in `response_format` during completion, it converts the json_schema to gbnf grammar.
 
 ## Mock `llama.rn`
 

package/android/src/main/CMakeLists.txt

@@ -51,6 +51,11 @@ set(
     ${RNLLAMA_LIB_DIR}/unicode.cpp
     ${RNLLAMA_LIB_DIR}/sgemm.cpp
     ${RNLLAMA_LIB_DIR}/common.cpp
+    ${RNLLAMA_LIB_DIR}/chat.cpp
+    ${RNLLAMA_LIB_DIR}/minja/chat-template.hpp
+    ${RNLLAMA_LIB_DIR}/json-schema-to-grammar.cpp
+    ${RNLLAMA_LIB_DIR}/minja/minja.hpp
+    ${RNLLAMA_LIB_DIR}/json.hpp
     ${RNLLAMA_LIB_DIR}/rn-llama.cpp
     ${CMAKE_SOURCE_DIR}/jni-utils.h
     ${CMAKE_SOURCE_DIR}/jni.cpp

package/android/src/main/java/com/rnllama/LlamaContext.java

@@ -28,6 +28,32 @@ public class LlamaContext {
 
   private static String loadedLibrary = "";
 
+  private static class NativeLogCallback {
+    DeviceEventManagerModule.RCTDeviceEventEmitter eventEmitter;
+
+    public NativeLogCallback(ReactApplicationContext reactContext) {
+      this.eventEmitter = reactContext.getJSModule(DeviceEventManagerModule.RCTDeviceEventEmitter.class);
+    }
+
+    void emitNativeLog(String level, String text) {
+      WritableMap event = Arguments.createMap();
+      event.putString("level", level);
+      event.putString("text", text);
+      eventEmitter.emit("@RNLlama_onNativeLog", event);
+    }
+  }
+
+  static void toggleNativeLog(ReactApplicationContext reactContext, boolean enabled) {
+    if (LlamaContext.isArchNotSupported()) {
+      throw new IllegalStateException("Only 64-bit architectures are supported");
+    }
+    if (enabled) {
+      setupLog(new NativeLogCallback(reactContext));
+    } else {
+      unsetLog();
+    }
+  }
+
   private int id;
   private ReactApplicationContext reactContext;
   private long context;
@@ -73,7 +99,7 @@ public class LlamaContext {
   }
 
   public LlamaContext(int id, ReactApplicationContext reactContext, ReadableMap params) {
-    if (LlamaContext.isArm64V8a() == false && LlamaContext.isX86_64() == false) {
+    if (LlamaContext.isArchNotSupported()) {
       throw new IllegalStateException("Only 64-bit architectures are supported");
     }
     if (!params.hasKey("model")) {
@@ -95,13 +121,17 @@ public class LlamaContext {
         Log.e(NAME, "Failed to convert to FD!");
       }
     }
-    logToAndroid();
+    
     // Check if file has GGUF magic numbers
     this.id = id;
     eventEmitter = reactContext.getJSModule(DeviceEventManagerModule.RCTDeviceEventEmitter.class);
     this.context = initContext(
       // String model,
       modelName,
+      // String chat_template,
+      params.hasKey("chat_template") ? params.getString("chat_template") : "",
+      // String reasoning_format,
+      params.hasKey("reasoning_format") ? params.getString("reasoning_format") : "none",
       // boolean embedding,
       params.hasKey("embedding") ? params.getBoolean("embedding") : false,
       // int embd_normalize,
@@ -166,12 +196,24 @@ public class LlamaContext {
     return loadedLibrary;
   }
 
-  public String getFormattedChat(ReadableArray messages, String chatTemplate) {
-    ReadableMap[] msgs = new ReadableMap[messages.size()];
-    for (int i = 0; i < messages.size(); i++) {
-      msgs[i] = messages.getMap(i);
-    }
-    return getFormattedChat(this.context, msgs, chatTemplate == null ? "" : chatTemplate);
+  public WritableMap getFormattedChatWithJinja(String messages, String chatTemplate, ReadableMap params) {
+    String jsonSchema = params.hasKey("json_schema") ? params.getString("json_schema") : "";
+    String tools = params.hasKey("tools") ? params.getString("tools") : "";
+    Boolean parallelToolCalls = params.hasKey("parallel_tool_calls") ? params.getBoolean("parallel_tool_calls") : false;
+    String toolChoice = params.hasKey("tool_choice") ? params.getString("tool_choice") : "";
+    return getFormattedChatWithJinja(
+      this.context,
+      messages,
+      chatTemplate == null ? "" : chatTemplate,
+      jsonSchema,
+      tools,
+      parallelToolCalls,
+      toolChoice
+    );
+  }
+
+  public String getFormattedChat(String messages, String chatTemplate) {
+    return getFormattedChat(this.context, messages, chatTemplate == null ? "" : chatTemplate);
   }
 
   private void emitLoadProgress(int progress) {
@@ -259,8 +301,18 @@ public class LlamaContext {
       this.context,
       // String prompt,
       params.getString("prompt"),
+      // int chat_format,
+      params.hasKey("chat_format") ? params.getInt("chat_format") : 0,
       // String grammar,
       params.hasKey("grammar") ? params.getString("grammar") : "",
+      // String json_schema,
+      params.hasKey("json_schema") ? params.getString("json_schema") : "",
+      // boolean grammar_lazy,
+      params.hasKey("grammar_lazy") ? params.getBoolean("grammar_lazy") : false,
+      // ReadableArray grammar_triggers,
+      params.hasKey("grammar_triggers") ? params.getArray("grammar_triggers") : null,
+      // ReadableArray preserved_tokens,
+      params.hasKey("preserved_tokens") ? params.getArray("preserved_tokens") : null,
       // float temperature,
       params.hasKey("temperature") ? (float) params.getDouble("temperature") : 0.7f,
       // int n_threads,
@@ -311,6 +363,8 @@ public class LlamaContext {
       params.hasKey("dry_allowed_length") ? params.getInt("dry_allowed_length") : 2,
       // int dry_penalty_last_n,
       params.hasKey("dry_penalty_last_n") ? params.getInt("dry_penalty_last_n") : -1,
+      // float top_n_sigma,
+      params.hasKey("top_n_sigma") ? (float) params.getDouble("top_n_sigma") : -1.0f,
       // String[] dry_sequence_breakers, when undef, we use the default definition from common.h
       params.hasKey("dry_sequence_breakers") ? params.getArray("dry_sequence_breakers").toArrayList().toArray(new String[0]) : new String[]{"\n", ":", "\"", "*"},
       // PartialCompletionCallback partial_completion_callback
@@ -431,15 +485,13 @@ public class LlamaContext {
       //  Log.d(NAME, "Loading librnllama_v8_7.so with runtime feature detection");
       //  System.loadLibrary("rnllama_v8_7");
     } else if (LlamaContext.isX86_64()) {
-        Log.d(NAME, "Loading librnllama_x86_64.so");
-        System.loadLibrary("rnllama_x86_64");
-        loadedLibrary = "rnllama_x86_64";
+      Log.d(NAME, "Loading librnllama_x86_64.so");
+      System.loadLibrary("rnllama_x86_64");
+      loadedLibrary = "rnllama_x86_64";
     } else {
-        Log.d(NAME, "Loading default librnllama.so");
-        System.loadLibrary("rnllama");
-        loadedLibrary = "rnllama";
+      Log.d(NAME, "ARM32 is not supported, skipping loading library");
     }
-}
+  }
 
   public static boolean isArm64V8a() {
     return Build.SUPPORTED_ABIS[0].equals("arm64-v8a");
@@ -449,6 +501,10 @@ public class LlamaContext {
     return Build.SUPPORTED_ABIS[0].equals("x86_64");
   }
 
+  private static boolean isArchNotSupported() {
+    return isArm64V8a() == false && isX86_64() == false;
+  }
+
   public static String getCpuFeatures() {
     File file = new File("/proc/cpuinfo");
     StringBuilder stringBuilder = new StringBuilder();
@@ -481,6 +537,8 @@ public class LlamaContext {
   );
   protected static native long initContext(
     String model,
+    String chat_template,
+    String reasoning_format,
     boolean embedding,
     int embd_normalize,
     int n_ctx,
@@ -506,9 +564,18 @@ public class LlamaContext {
   protected static native WritableMap loadModelDetails(
     long contextPtr
   );
+  protected static native WritableMap getFormattedChatWithJinja(
+    long contextPtr,
+    String messages,
+    String chatTemplate,
+    String jsonSchema,
+    String tools,
+    boolean parallelToolCalls,
+    String toolChoice
+  );
   protected static native String getFormattedChat(
     long contextPtr,
-    ReadableMap[] messages,
+    String messages,
     String chatTemplate
   );
   protected static native WritableMap loadSession(
@@ -523,7 +590,12 @@ public class LlamaContext {
   protected static native WritableMap doCompletion(
     long context_ptr,
     String prompt,
+    int chat_format,
     String grammar,
+    String json_schema,
+    boolean grammar_lazy,
+    ReadableArray grammar_triggers,
+    ReadableArray preserved_tokens,
     float temperature,
     int n_threads,
     int n_predict,
@@ -549,6 +621,7 @@ public class LlamaContext {
     float   dry_base,
     int dry_allowed_length,
     int dry_penalty_last_n,
+    float top_n_sigma,
     String[] dry_sequence_breakers,
     PartialCompletionCallback partial_completion_callback
   );
@@ -567,5 +640,6 @@ public class LlamaContext {
   protected static native void removeLoraAdapters(long contextPtr);
   protected static native WritableArray getLoadedLoraAdapters(long contextPtr);
   protected static native void freeContext(long contextPtr);
-  protected static native void logToAndroid();
+  protected static native void setupLog(NativeLogCallback logCallback);
+  protected static native void unsetLog();
 }

package/android/src/main/java/com/rnllama/RNLlama.java

@@ -35,6 +35,32 @@ public class RNLlama implements LifecycleEventListener {
 
   private HashMap<Integer, LlamaContext> contexts = new HashMap<>();
 
+  public void toggleNativeLog(boolean enabled, Promise promise) {
+    new AsyncTask<Void, Void, Boolean>() {
+      private Exception exception;
+
+      @Override
+      protected Boolean doInBackground(Void... voids) {
+        try {
+          LlamaContext.toggleNativeLog(reactContext, enabled);
+          return true;
+        } catch (Exception e) {
+          exception = e;
+        }
+        return null;
+      }
+
+      @Override
+      protected void onPostExecute(Boolean result) {
+        if (exception != null) {
+          promise.reject(exception);
+          return;
+        }
+        promise.resolve(result);
+      }
+    }.executeOnExecutor(AsyncTask.THREAD_POOL_EXECUTOR);
+  }
+
   private int llamaContextLimit = -1;
 
   public void setContextLimit(double limit, Promise promise) {
@@ -116,18 +142,25 @@ public class RNLlama implements LifecycleEventListener {
     tasks.put(task, "initContext");
   }
 
-  public void getFormattedChat(double id, final ReadableArray messages, final String chatTemplate, Promise promise) {
+  public void getFormattedChat(double id, final String messages, final String chatTemplate, final ReadableMap params, Promise promise) {
     final int contextId = (int) id;
-    AsyncTask task = new AsyncTask<Void, Void, String>() {
+    AsyncTask task = new AsyncTask<Void, Void, Object>() {
       private Exception exception;
 
       @Override
-      protected String doInBackground(Void... voids) {
+      protected Object doInBackground(Void... voids) {
         try {
           LlamaContext context = contexts.get(contextId);
           if (context == null) {
             throw new Exception("Context not found");
           }
+          if (params.hasKey("jinja") && params.getBoolean("jinja")) {
+            ReadableMap result = context.getFormattedChatWithJinja(messages, chatTemplate, params);
+            if (result.hasKey("_error")) {
+              throw new Exception(result.getString("_error"));
+            }
+            return result;
+          }
           return context.getFormattedChat(messages, chatTemplate);
         } catch (Exception e) {
           exception = e;
@@ -136,7 +169,7 @@ public class RNLlama implements LifecycleEventListener {
       }
 
       @Override
-      protected void onPostExecute(String result) {
+      protected void onPostExecute(Object result) {
         if (exception != null) {
           promise.reject(exception);
           return;

package/android/src/main/jni-utils.h

@@ -16,6 +16,12 @@ jobject getMap(JNIEnv *env, jobject readableArray, int index) {
     return env->CallObjectMethod(readableArray, getMapMethod, index);
 }
 
+jstring getString(JNIEnv *env, jobject readableArray, int index) {
+    jclass arrayClass = env->GetObjectClass(readableArray);
+    jmethodID getStringMethod = env->GetMethodID(arrayClass, "getString", "(I)Ljava/lang/String;");
+    return (jstring) env->CallObjectMethod(readableArray, getStringMethod, index);
+}
+
 // Other methods not used yet
 
 }