@fugood/llama.node 1.0.0-beta.6 → 1.0.0-beta.7

package/lib/binding.ts

@@ -79,6 +79,8 @@ export type LlamaCompletionOptions = {
   tools?: object
   parallel_tool_calls?: boolean
   tool_choice?: string
+  enable_thinking?: boolean
+  thinking_forced_open?: boolean
   prompt?: string
   temperature?: number
   top_k?: number

package/lib/index.js

@@ -131,6 +131,7 @@ class LlamaContextWrapper {
         };
     }
     getFormattedChat(messages, template, params) {
+        var _a;
         const { messages: chat, has_media, media_paths, } = this._formatMediaChat(messages);
         const useJinja = this.isJinjaSupported() && (params === null || params === void 0 ? void 0 : params.jinja);
         let tmpl;
@@ -143,6 +144,7 @@ class LlamaContextWrapper {
             tools: params === null || params === void 0 ? void 0 : params.tools,
             parallel_tool_calls: params === null || params === void 0 ? void 0 : params.parallel_tool_calls,
             tool_choice: params === null || params === void 0 ? void 0 : params.tool_choice,
+            enable_thinking: (_a = params === null || params === void 0 ? void 0 : params.enable_thinking) !== null && _a !== void 0 ? _a : true,
         });
         if (!useJinja) {
             return {

package/lib/index.ts

@@ -158,7 +158,8 @@ class LlamaContextWrapper {
       response_format?: CompletionResponseFormat
       tools?: object
       parallel_tool_calls?: object
-      tool_choice?: string
+      tool_choice?: string,
+      enable_thinking?: boolean,
     },
   ): FormattedChatResult {
     const {
@@ -178,6 +179,7 @@ class LlamaContextWrapper {
       tools: params?.tools,
       parallel_tool_calls: params?.parallel_tool_calls,
       tool_choice: params?.tool_choice,
+      enable_thinking: params?.enable_thinking ?? true,
     })
 
     if (!useJinja) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "1.0.0-beta.6",
+  "version": "1.0.0-beta.7",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {
@@ -70,19 +70,19 @@
     "CMakeLists.txt"
   ],
   "optionalDependencies": {
-    "@fugood/node-llama-linux-x64": "1.0.0-beta.6",
-    "@fugood/node-llama-linux-x64-vulkan": "1.0.0-beta.6",
-    "@fugood/node-llama-linux-x64-cuda": "1.0.0-beta.6",
-    "@fugood/node-llama-linux-arm64": "1.0.0-beta.6",
-    "@fugood/node-llama-linux-arm64-vulkan": "1.0.0-beta.6",
-    "@fugood/node-llama-linux-arm64-cuda": "1.0.0-beta.6",
-    "@fugood/node-llama-win32-x64": "1.0.0-beta.6",
-    "@fugood/node-llama-win32-x64-vulkan": "1.0.0-beta.6",
-    "@fugood/node-llama-win32-x64-cuda": "1.0.0-beta.6",
-    "@fugood/node-llama-win32-arm64": "1.0.0-beta.6",
-    "@fugood/node-llama-win32-arm64-vulkan": "1.0.0-beta.6",
-    "@fugood/node-llama-darwin-x64": "1.0.0-beta.6",
-    "@fugood/node-llama-darwin-arm64": "1.0.0-beta.6"
+    "@fugood/node-llama-linux-x64": "1.0.0-beta.7",
+    "@fugood/node-llama-linux-x64-vulkan": "1.0.0-beta.7",
+    "@fugood/node-llama-linux-x64-cuda": "1.0.0-beta.7",
+    "@fugood/node-llama-linux-arm64": "1.0.0-beta.7",
+    "@fugood/node-llama-linux-arm64-vulkan": "1.0.0-beta.7",
+    "@fugood/node-llama-linux-arm64-cuda": "1.0.0-beta.7",
+    "@fugood/node-llama-win32-x64": "1.0.0-beta.7",
+    "@fugood/node-llama-win32-x64-vulkan": "1.0.0-beta.7",
+    "@fugood/node-llama-win32-x64-cuda": "1.0.0-beta.7",
+    "@fugood/node-llama-win32-arm64": "1.0.0-beta.7",
+    "@fugood/node-llama-win32-arm64-vulkan": "1.0.0-beta.7",
+    "@fugood/node-llama-darwin-x64": "1.0.0-beta.7",
+    "@fugood/node-llama-darwin-arm64": "1.0.0-beta.7"
   },
   "devDependencies": {
     "@babel/preset-env": "^7.24.4",

package/src/EmbeddingWorker.cpp

@@ -8,7 +8,7 @@ EmbeddingWorker::EmbeddingWorker(const Napi::CallbackInfo &info,
       _params(params) {}
 
 void EmbeddingWorker::Execute() {
-  llama_kv_self_clear(_sess->context());
+  llama_memory_clear(llama_get_memory(_sess->context()), true);
   auto tokens = ::common_tokenize(_sess->context(), _text, true);
   // add SEP if not present
   auto vocab = llama_model_get_vocab(_sess->model());

package/src/LlamaCompletionWorker.cpp

@@ -29,11 +29,13 @@ LlamaCompletionWorker::LlamaCompletionWorker(
     common_params params,
     std::vector<std::string> stop_words,
     int32_t chat_format,
+    bool thinking_forced_open,
     std::string reasoning_format,
     const std::vector<std::string> &media_paths,
     const std::vector<llama_token> &guide_tokens)
     : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess),
       _params(params), _stop_words(stop_words), _chat_format(chat_format),
+      _thinking_forced_open(thinking_forced_open),
       _reasoning_format(reasoning_format),
       _media_paths(media_paths), _guide_tokens(guide_tokens) {
   if (!callback.IsEmpty()) {
@@ -113,7 +115,7 @@ void LlamaCompletionWorker::Execute() {
         --n_cur;
       }
       n_input -= n_cur;
-      llama_kv_self_seq_rm(ctx, 0, n_cur, -1);
+      llama_memory_seq_rm(llama_get_memory(ctx), 0, n_cur, -1);
     }
     // Set the tokens
     _sess->set_tokens(std::move(prompt_tokens));
@@ -135,8 +137,9 @@ void LlamaCompletionWorker::Execute() {
       const int n_left = n_cur - n_keep - 1;
       const int n_discard = n_left / 2;
 
-      llama_kv_self_seq_rm(ctx, 0, n_keep + 1, n_keep + n_discard + 1);
-      llama_kv_self_seq_add(ctx, 0, n_keep + 1 + n_discard, n_cur, -n_discard);
+      auto mem = llama_get_memory(ctx);
+      llama_memory_seq_rm(mem, 0, n_keep + 1, n_keep + n_discard + 1);
+      llama_memory_seq_add(mem, 0, n_keep + 1 + n_discard, n_cur, -n_discard);
 
       // shift the tokens
       embd->insert(embd->begin() + n_keep + 1,
@@ -240,6 +243,7 @@ void LlamaCompletionWorker::OnOK() {
     try {
       common_chat_syntax chat_syntax;
       chat_syntax.format = static_cast<common_chat_format>(_chat_format);
+      chat_syntax.thinking_forced_open = _thinking_forced_open;
 
       if (_reasoning_format == "deepseek") {
           chat_syntax.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;

package/src/LlamaCompletionWorker.h

@@ -20,6 +20,7 @@ public:
                         Napi::Function callback, common_params params,
                         std::vector<std::string> stop_words,
                         int32_t chat_format,
+                        bool thinking_forced_open,
                         std::string reasoning_format,
                         const std::vector<std::string> &media_paths = {},
                         const std::vector<llama_token> &guide_tokens = {});
@@ -42,6 +43,7 @@ private:
   common_params _params;
   std::vector<std::string> _stop_words;
   int32_t _chat_format;
+  bool _thinking_forced_open;
   std::string _reasoning_format;
   std::vector<std::string> _media_paths;
   std::vector<llama_token> _guide_tokens;

package/src/LlamaContext.cpp

@@ -499,7 +499,9 @@ common_chat_params getFormattedChatWithJinja(
     const common_chat_templates_ptr &templates, const std::string &messages,
     const std::string &chat_template, const std::string &json_schema,
     const std::string &tools, const bool &parallel_tool_calls,
-    const std::string &tool_choice) {
+    const std::string &tool_choice,
+    const bool &enable_thinking
+) {
   common_chat_templates_inputs inputs;
   inputs.messages = common_chat_msgs_parse_oaicompat(json::parse(messages));
   auto useTools = !tools.empty();
@@ -513,6 +515,7 @@ common_chat_params getFormattedChatWithJinja(
   if (!json_schema.empty()) {
     inputs.json_schema = json::parse(json_schema);
   }
+  inputs.enable_thinking = enable_thinking;
 
   // If chat_template is provided, create new one and use it (probably slow)
   if (!chat_template.empty()) {
@@ -586,12 +589,11 @@ Napi::Value LlamaContext::GetFormattedChat(const Napi::CallbackInfo &info) {
     auto parallel_tool_calls =
         get_option<bool>(params, "parallel_tool_calls", false);
     auto tool_choice = get_option<std::string>(params, "tool_choice", "");
+    auto enable_thinking = get_option<bool>(params, "enable_thinking", false);
 
     auto chatParams = getFormattedChatWithJinja(
         _sess, _templates, messages, chat_template, json_schema_str, tools_str,
-        parallel_tool_calls, tool_choice);
-
-    console_log(env, std::string("format: ") + std::to_string(chatParams.format));
+        parallel_tool_calls, tool_choice, enable_thinking);
 
     Napi::Object result = Napi::Object::New(env);
     result.Set("prompt", chatParams.prompt);
@@ -612,6 +614,7 @@ Napi::Value LlamaContext::GetFormattedChat(const Napi::CallbackInfo &info) {
       grammar_triggers.Set(i, triggerObj);
     }
     result.Set("grammar_triggers", grammar_triggers);
+    result.Set("thinking_forced_open", chatParams.thinking_forced_open);
     // preserved_tokens: string[]
     Napi::Array preserved_tokens = Napi::Array::New(env);
     for (size_t i = 0; i < chatParams.preserved_tokens.size(); i++) {
@@ -685,6 +688,7 @@ Napi::Value LlamaContext::Completion(const Napi::CallbackInfo &info) {
   }
 
   int32_t chat_format = get_option<int32_t>(options, "chat_format", 0);
+  bool thinking_forced_open = get_option<bool>(options, "thinking_forced_open", false);
   std::string reasoning_format = get_option<std::string>(options, "reasoning_format", "none");
 
   common_params params = _sess->params();
@@ -793,14 +797,16 @@ Napi::Value LlamaContext::Completion(const Napi::CallbackInfo &info) {
           get_option<bool>(options, "parallel_tool_calls", false);
       auto tool_choice =
           get_option<std::string>(options, "tool_choice", "none");
+      auto enable_thinking = get_option<bool>(options, "enable_thinking", true);
 
       auto chatParams = getFormattedChatWithJinja(
           _sess, _templates, json_stringify(messages), chat_template,
-          json_schema_str, tools_str, parallel_tool_calls, tool_choice);
+          json_schema_str, tools_str, parallel_tool_calls, tool_choice, enable_thinking);
 
       params.prompt = chatParams.prompt;
 
       chat_format = chatParams.format;
+      thinking_forced_open = chatParams.thinking_forced_open;
 
       for (const auto &token : chatParams.preserved_tokens) {
         auto ids =
@@ -895,7 +901,7 @@ Napi::Value LlamaContext::Completion(const Napi::CallbackInfo &info) {
 
   auto *worker =
       new LlamaCompletionWorker(info, _sess, callback, params, stop_words,
-                                chat_format, reasoning_format, media_paths, guide_tokens);
+                                chat_format, thinking_forced_open, reasoning_format, media_paths, guide_tokens);
   worker->Queue();
   _wip = worker;
   worker->OnComplete([this]() { _wip = nullptr; });

package/src/common.hpp

@@ -461,7 +461,7 @@ processMediaPrompt(llama_context *ctx, mtmd_context *mtmd_ctx,
   }
 
   // Clear all KV cache entries after position n_past
-  llama_kv_self_seq_rm(ctx, 0, n_past, -1);
+  llama_memory_seq_rm(llama_get_memory(ctx), 0, n_past, -1);
 
   size_t num_chunks = mtmd_input_chunks_size(chunks);