cui-llama.rn 1.4.3 → 1.4.6

package/cpp/rn-llama.cpp

@@ -194,6 +194,8 @@ bool llama_rn_context::loadModel(common_params &params_)
         LOG_ERROR("unable to load model: %s", params_.model.c_str());
         return false;
     }
+    
+    templates = common_chat_templates_init(model, params.chat_template);
     n_ctx = llama_n_ctx(ctx);
 
     // We can uncomment for debugging or after this fix: https://github.com/ggerganov/llama.cpp/pull/11101
@@ -202,11 +204,70 @@ bool llama_rn_context::loadModel(common_params &params_)
     return true;
 }
 
-bool llama_rn_context::validateModelChatTemplate() const {
-    const char * tmpl = llama_model_chat_template(model);
-    llama_chat_message chat[] = {{"user", "test"}};
-    int32_t chat_res = llama_chat_apply_template(tmpl, chat, 1, true, nullptr, 0);
-    return chat_res > 0;
+bool llama_rn_context::validateModelChatTemplate(bool use_jinja, const char *name) const {
+    const char * tmpl = llama_model_chat_template(model, name);
+    if (tmpl == nullptr) {
+      return false;
+    }
+    return common_chat_verify_template(tmpl, use_jinja);
+}
+
+common_chat_params llama_rn_context::getFormattedChatWithJinja(
+  const std::string &messages,
+  const std::string &chat_template,
+  const std::string &json_schema,
+  const std::string &tools,
+  const bool &parallel_tool_calls,
+  const std::string &tool_choice
+) const {
+  common_chat_templates_inputs inputs;
+  inputs.messages = common_chat_msgs_parse_oaicompat(json::parse(messages));
+  auto useTools = !tools.empty();
+  if (useTools) {
+      inputs.tools = common_chat_tools_parse_oaicompat(json::parse(tools));
+  }
+  inputs.parallel_tool_calls = parallel_tool_calls;
+  if (!tool_choice.empty()) {
+      inputs.tool_choice = common_chat_tool_choice_parse_oaicompat(tool_choice);
+  }
+  
+  if (!json_schema.empty()) {
+      inputs.json_schema = json_schema;
+  }
+  inputs.extract_reasoning = params.reasoning_format != COMMON_REASONING_FORMAT_NONE;
+ 
+  // If chat_template is provided, create new one and use it (probably slow)
+  if (!chat_template.empty()) {
+      auto tmp = common_chat_templates_init(model, chat_template);
+      return common_chat_templates_apply(tmp.get(), inputs);
+  } else {
+      return common_chat_templates_apply(templates.get(), inputs);
+  }
+}
+
+std::string llama_rn_context::getFormattedChat(
+  const std::string &messages,
+  const std::string &chat_template
+) const {
+  auto chat_json = json::parse(messages);    
+  common_chat_templates_inputs inputs;
+  // Handle regular chat without tools
+  std::vector<common_chat_msg> chat_msgs;
+  for (const auto &msg : chat_json) {
+      chat_msgs.push_back({
+          msg["role"].get(),
+          msg["content"].get()
+      });
+  }
+  inputs.messages = chat_msgs;
+
+  // If chat_template is provided, create new one and use it (probably slow)
+  if (!chat_template.empty()) {
+      auto tmp = common_chat_templates_init(model, chat_template);
+      return common_chat_templates_apply(tmp.get(), inputs).prompt;
+  } else {
+      return common_chat_templates_apply(templates.get(), inputs).prompt;
+  }
 }
 
 void llama_rn_context::truncatePrompt(std::vector<llama_token> &prompt_tokens) {
@@ -518,7 +579,6 @@ std::vector<float> llama_rn_context::getEmbedding(common_params &embd_params)
     float *data;
 
     const enum llama_pooling_type pooling_type = llama_pooling_type(ctx);
-    printf("pooling_type: %d\n", pooling_type);
     if (pooling_type == LLAMA_POOLING_TYPE_NONE) {
         data = llama_get_embeddings(ctx);
     } else {

package/cpp/rn-llama.h

@@ -3,7 +3,9 @@
 
 #include <sstream>
 #include <iostream>
+#include "chat-template.hpp"
 #include "common.h"
+#include "chat.h"
 #include "ggml.h"
 #include "gguf.h"
 #include "llama.h"
@@ -13,8 +15,18 @@
 #include <android/log.h>
 #endif
 
+using json = nlohmann::ordered_json;
+typedef minja::chat_template common_chat_template;
+
+struct common_chat_templates {
+    bool has_explicit_template;
+    std::unique_ptr<common_chat_template> template_default;
+    std::unique_ptr<common_chat_template> template_tool_use;
+};
+
 namespace rnllama {
 
+
 std::string tokens_to_output_formatted_string(const llama_context *ctx, const llama_token token);
 
 std::string tokens_to_str(llama_context *ctx, const std::vector<llama_token>::const_iterator begin, const std::vector<llama_token>::const_iterator end);
@@ -63,6 +75,7 @@ struct llama_rn_context {
 
     llama_context *ctx = nullptr;
     common_sampler *ctx_sampling = nullptr;
+    common_chat_templates_ptr templates = nullptr;
 
     int n_ctx;
 
@@ -80,7 +93,19 @@ struct llama_rn_context {
     void rewind();
     bool initSampling();
     bool loadModel(common_params &params_);
-    bool validateModelChatTemplate() const;
+    bool validateModelChatTemplate(bool use_jinja, const char *name) const;
+    common_chat_params getFormattedChatWithJinja(
+      const std::string &messages,
+      const std::string &chat_template,
+      const std::string &json_schema,
+      const std::string &tools,
+      const bool &parallel_tool_calls,
+      const std::string &tool_choice
+    ) const;
+    std::string getFormattedChat(
+      const std::string &messages,
+      const std::string &chat_template
+    ) const;
     void truncatePrompt(std::vector<llama_token> &prompt_tokens);
     void loadPrompt();
     void beginCompletion();