cui-llama.rn 1.4.6 → 1.5.0

package/ios/CMakeLists.txt

@@ -62,15 +62,18 @@ add_library(rnllama SHARED
     ${SOURCE_DIR}/llama-model-loader.cpp
     ${SOURCE_DIR}/llama-mmap.cpp
     ${SOURCE_DIR}/llama-vocab.cpp
+    ${SOURCE_DIR}/llama-memory.cpp
+    ${SOURCE_DIR}/llama-io.cpp
+    ${SOURCE_DIR}/llama-graph.cpp
     ${SOURCE_DIR}/sampling.cpp
     ${SOURCE_DIR}/unicode-data.cpp
     ${SOURCE_DIR}/unicode.cpp
     ${SOURCE_DIR}/sgemm.cpp
     ${SOURCE_DIR}/common.cpp
     ${SOURCE_DIR}/chat.cpp
-    ${SOURCE_DIR}/chat-template.hpp
     ${SOURCE_DIR}/json-schema-to-grammar.cpp
-    ${SOURCE_DIR}/minja.hpp
+    ${SOURCE_DIR}/minja/minja.hpp
+    ${SOURCE_DIR}/minja/chat-template.hpp
     ${SOURCE_DIR}/json.hpp
     ${SOURCE_DIR}/amx/amx.cpp
     ${SOURCE_DIR}/amx/mmq.cpp

package/ios/RNLlama.mm

@@ -102,8 +102,8 @@ RCT_EXPORT_METHOD(getFormattedChat:(double)contextId
         if ([params[@"jinja"] boolValue]) {
             NSString *jsonSchema = params[@"json_schema"];
             NSString *tools = params[@"tools"];
-            NSString *parallelToolCalls = params[@"parallel_tool_calls"];
-            NSString *toolChoice = params[@"tool_choice"];\
+            bool parallelToolCalls = [params[@"parallel_tool_calls"] boolValue];
+            NSString *toolChoice = params[@"tool_choice"];
             resolve([context getFormattedChatWithJinja:messages withChatTemplate:chatTemplate withJsonSchema:jsonSchema withTools:tools withParallelToolCalls:parallelToolCalls withToolChoice:toolChoice]);
         } else {
             resolve([context getFormattedChat:messages withChatTemplate:chatTemplate]);

package/ios/RNLlamaContext.mm

@@ -285,7 +285,7 @@
         [meta setValue:valStr forKey:keyStr];
     }
 
-    auto template_tool_use = llama->templates.template_tool_use.get();
+    auto template_tool_use = llama->templates.get()->template_tool_use.get();
     NSDictionary *tool_use_caps_dir = nil;
     if (template_tool_use) {
         auto tool_use_caps = template_tool_use->original_caps();
@@ -299,7 +299,7 @@
         };
     }
 
-    auto default_tmpl = llama->templates.template_default.get();
+    auto default_tmpl = llama->templates.get()->template_default.get();
     auto default_tmpl_caps = default_tmpl->original_caps();
 
     return @{
@@ -356,15 +356,16 @@
         parallelToolCalls,
         toolChoice == nil ? "" : [toolChoice UTF8String]
     );
-    result[@"prompt"] = [NSString stringWithUTF8String:chatParams.prompt.get<std::string>().c_str()];
+    result[@"prompt"] = [NSString stringWithUTF8String:chatParams.prompt.c_str()];
     result[@"chat_format"] = @(static_cast<int>(chatParams.format));
     result[@"grammar"] = [NSString stringWithUTF8String:chatParams.grammar.c_str()];
     result[@"grammar_lazy"] = @(chatParams.grammar_lazy);
     NSMutableArray *grammar_triggers = [[NSMutableArray alloc] init];
     for (const auto & trigger : chatParams.grammar_triggers) {
         [grammar_triggers addObject:@{
-            @"word": [NSString stringWithUTF8String:trigger.word.c_str()],
-            @"at_start": @(trigger.at_start),
+            @"type": @(trigger.type),
+            @"value": [NSString stringWithUTF8String:trigger.value.c_str()],
+            @"token": @(trigger.token),
         }];
     }
     result[@"grammar_triggers"] = grammar_triggers;
@@ -483,25 +484,6 @@
         sparams.grammar_lazy = [params[@"grammar_lazy"] boolValue];
     }
 
-    if (params[@"grammar_triggers"] && [params[@"grammar_triggers"] isKindOfClass:[NSArray class]]) {
-        NSArray *grammar_triggers = params[@"grammar_triggers"];
-        for (NSDictionary *grammar_trigger in grammar_triggers) {
-            common_grammar_trigger trigger;
-            trigger.word = [grammar_trigger[@"word"] UTF8String];
-            trigger.at_start = [grammar_trigger[@"at_start"] boolValue];
-
-            auto ids = common_tokenize(llama->ctx, trigger.word, /* add_special= */ false, /* parse_special= */ true);
-            if (ids.size() == 1) {
-                // LOG_DBG("Grammar trigger token: %d (`%s`)\n", ids[0], trigger.word.c_str());
-                sparams.grammar_trigger_tokens.push_back(ids[0]);
-                sparams.preserved_tokens.insert(ids[0]);
-                continue;
-            }
-            // LOG_DBG("Grammar trigger word: `%s`\n", trigger.word.c_str());
-            sparams.grammar_trigger_words.push_back(trigger);
-        }
-    }
-
     if (params[@"preserved_tokens"] && [params[@"preserved_tokens"] isKindOfClass:[NSArray class]]) {
         NSArray *preserved_tokens = params[@"preserved_tokens"];
         for (NSString *token in preserved_tokens) {
@@ -514,6 +496,40 @@
         }
     }
 
+    if (params[@"grammar_triggers"] && [params[@"grammar_triggers"] isKindOfClass:[NSArray class]]) {
+        NSArray *grammar_triggers = params[@"grammar_triggers"];
+        for (NSDictionary *grammar_trigger in grammar_triggers) {
+            const auto type = static_cast<common_grammar_trigger_type>([grammar_trigger[@"type"] intValue]);
+            const auto & word = [grammar_trigger[@"value"] UTF8String];
+
+            if (type == COMMON_GRAMMAR_TRIGGER_TYPE_WORD) {
+              auto ids = common_tokenize(llama->ctx, word, /* add_special= */ false, /* parse_special= */ true);
+              if (ids.size() == 1) {
+                  auto token = ids[0];
+                  if (std::find(sparams.preserved_tokens.begin(), sparams.preserved_tokens.end(), (llama_token) token) == sparams.preserved_tokens.end()) {
+                      throw std::runtime_error("Grammar trigger word should be marked as preserved token");
+                  }
+                  common_grammar_trigger trigger;
+                  trigger.type = COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN;
+                  trigger.value = word;
+                  trigger.token = token;
+                  sparams.grammar_triggers.push_back(std::move(trigger));
+              } else {
+                  sparams.grammar_triggers.push_back({COMMON_GRAMMAR_TRIGGER_TYPE_WORD, word});
+              }
+            } else {
+                common_grammar_trigger trigger;
+                trigger.type = type;
+                trigger.value = word;
+                if (type == COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN) {
+                    const auto token = (llama_token) [grammar_trigger[@"token"] intValue];
+                    trigger.token = token;
+                }
+                sparams.grammar_triggers.push_back(std::move(trigger));
+            }
+        }
+    }
+
     llama->params.antiprompt.clear();
     if (params[@"stop"]) {
         NSArray *stop = params[@"stop"];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cui-llama.rn",
-  "version": "1.4.6",
+  "version": "1.5.0",
   "description": "Fork of llama.rn for ChatterUI",
   "main": "lib/commonjs/index",
   "module": "lib/module/index",

package/src/NativeRNLlama.ts

@@ -93,8 +93,9 @@ export type NativeCompletionParams = {
    * Lazy grammar triggers. Default: []
    */
   grammar_triggers?: Array<{
-    at_start: boolean
-    word: string
+    type: number
+    value: string
+    token: number
   }>
   preserved_tokens?: Array<string>
   chat_format?: number
@@ -347,8 +348,9 @@ export type JinjaFormattedChatResult = {
   grammar?: string
   grammar_lazy?: boolean
   grammar_triggers?: Array<{
-    at_start: boolean
-    word: string
+    type: number
+    value: string
+    token: number
   }>
   preserved_tokens?: Array<string>
   additional_stops?: Array<string>

package/src/index.ts

@@ -70,7 +70,8 @@ if (EventEmitter) {
       logListeners.forEach((listener) => listener(evt.level, evt.text))
     },
   )
-  RNLlama?.toggleNativeLog?.(false) // Trigger unset to use default log callback
+  // Trigger unset to use default log callback
+  RNLlama?.toggleNativeLog?.(false)?.catch?.(() => {})
 }
 
 export type TokenData = {
@@ -398,6 +399,7 @@ const modelInfoSkip = [
   'tokenizer.ggml.tokens',
   'tokenizer.ggml.token_type',
   'tokenizer.ggml.merges',
+  'tokenizer.ggml.scores'
 ]
 export async function loadLlamaModelInfo(model: string): Promise<Object> {
   let path = model