@fugood/llama.node 1.4.15 → 1.6.0-rc.0

package/lib/binding.ts

@@ -374,24 +374,20 @@ export type ModelInfo = {
   is_hybrid: boolean
   chatTemplates: {
     llamaChat: boolean
-    minja: {
+    jinja: {
       default: boolean
       defaultCaps: {
         tools: boolean
         toolCalls: boolean
-        toolResponses: boolean
         systemRole: boolean
         parallelToolCalls: boolean
-        toolCallId: boolean
       }
       toolUse: boolean
       toolUseCaps?: {
         tools: boolean
         toolCalls: boolean
-        toolResponses: boolean
         systemRole: boolean
         parallelToolCalls: boolean
-        toolCallId: boolean
       }
     }
   }

package/lib/index.js

@@ -80,8 +80,8 @@ class LlamaContextWrapper {
         return this.ctx.getUsedDevices();
     }
     isJinjaSupported() {
-        const { minja } = this.ctx.getModelInfo().chatTemplates;
-        return !!(minja === null || minja === void 0 ? void 0 : minja.toolUse) || !!(minja === null || minja === void 0 ? void 0 : minja.default);
+        const { jinja } = this.ctx.getModelInfo().chatTemplates;
+        return !!(jinja === null || jinja === void 0 ? void 0 : jinja.toolUse) || !!(jinja === null || jinja === void 0 ? void 0 : jinja.default);
     }
     isLlamaChatSupported() {
         return !!this.ctx.getModelInfo().chatTemplates.llamaChat;

package/lib/index.ts

@@ -100,8 +100,8 @@ class LlamaContextWrapper {
   }
 
   isJinjaSupported(): boolean {
-    const { minja } = this.ctx.getModelInfo().chatTemplates
-    return !!minja?.toolUse || !!minja?.default
+    const { jinja } = this.ctx.getModelInfo().chatTemplates
+    return !!jinja?.toolUse || !!jinja?.default
   }
 
   isLlamaChatSupported(): boolean {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "1.4.15",
+  "version": "1.6.0-rc.0",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {
@@ -72,20 +72,20 @@
     "CMakeLists.txt"
   ],
   "optionalDependencies": {
-    "@fugood/node-llama-darwin-arm64": "1.4.15",
-    "@fugood/node-llama-darwin-x64": "1.4.15",
-    "@fugood/node-llama-linux-arm64": "1.4.15",
-    "@fugood/node-llama-linux-arm64-cuda": "1.4.15",
-    "@fugood/node-llama-linux-arm64-snapdragon": "1.4.15",
-    "@fugood/node-llama-linux-arm64-vulkan": "1.4.15",
-    "@fugood/node-llama-linux-x64": "1.4.15",
-    "@fugood/node-llama-linux-x64-cuda": "1.4.15",
-    "@fugood/node-llama-linux-x64-vulkan": "1.4.15",
-    "@fugood/node-llama-win32-arm64": "1.4.15",
-    "@fugood/node-llama-win32-arm64-vulkan": "1.4.15",
-    "@fugood/node-llama-win32-x64": "1.4.15",
-    "@fugood/node-llama-win32-x64-cuda": "1.4.15",
-    "@fugood/node-llama-win32-x64-vulkan": "1.4.15"
+    "@fugood/node-llama-darwin-arm64": "1.6.0-rc.0",
+    "@fugood/node-llama-darwin-x64": "1.6.0-rc.0",
+    "@fugood/node-llama-linux-arm64": "1.6.0-rc.0",
+    "@fugood/node-llama-linux-arm64-cuda": "1.6.0-rc.0",
+    "@fugood/node-llama-linux-arm64-snapdragon": "1.6.0-rc.0",
+    "@fugood/node-llama-linux-arm64-vulkan": "1.6.0-rc.0",
+    "@fugood/node-llama-linux-x64": "1.6.0-rc.0",
+    "@fugood/node-llama-linux-x64-cuda": "1.6.0-rc.0",
+    "@fugood/node-llama-linux-x64-vulkan": "1.6.0-rc.0",
+    "@fugood/node-llama-win32-arm64": "1.6.0-rc.0",
+    "@fugood/node-llama-win32-arm64-vulkan": "1.6.0-rc.0",
+    "@fugood/node-llama-win32-x64": "1.6.0-rc.0",
+    "@fugood/node-llama-win32-x64-cuda": "1.6.0-rc.0",
+    "@fugood/node-llama-win32-x64-vulkan": "1.6.0-rc.0"
   },
   "devDependencies": {
     "@babel/preset-env": "^7.24.4",

package/scripts/llama.cpp.patch

@@ -1,8 +1,8 @@
 diff --git a/src/llama.cpp/common/CMakeLists.txt b/src/llama.cpp/common/CMakeLists.txt
-index 723973ed7..e4b2c6537 100644
+index ae02c0bd7..f74d8bb26 100644
 --- a/src/llama.cpp/common/CMakeLists.txt
 +++ b/src/llama.cpp/common/CMakeLists.txt
-@@ -146,4 +146,11 @@ if (LLAMA_LLGUIDANCE)
+@@ -158,4 +158,11 @@ if (LLAMA_LLGUIDANCE)
      set(LLAMA_COMMON_EXTRA_LIBS ${LLAMA_COMMON_EXTRA_LIBS} llguidance ${LLGUIDANCE_PLATFORM_LIBS})
  endif ()
  
@@ -29,74 +29,89 @@ index 1bcba9cd8..b7cd68734 100644
  static std::string_view trim_trailing_space(std::string_view sv, int max = -1) {
      int count = 0;
 diff --git a/src/llama.cpp/common/chat.cpp b/src/llama.cpp/common/chat.cpp
-index d531388bc..e6712b368 100644
+index b29544dac..5fa2c6c17 100644
 --- a/src/llama.cpp/common/chat.cpp
 +++ b/src/llama.cpp/common/chat.cpp
-@@ -7,9 +7,6 @@
- #include "log.h"
- #include "regex-partial.h"
- 
--#include <minja/chat-template.hpp>
--#include <minja/minja.hpp>
--
- #include <algorithm>
- #include <cstdio>
- #include <cctype>
-@@ -135,16 +132,6 @@ std::vector<common_chat_msg_diff> common_chat_msg_diff::compute_diffs(const comm
-     return diffs;
+@@ -615,6 +615,37 @@ std::string common_chat_templates_source(const struct common_chat_templates * tm
+     return tmpls->template_default->source();
  }
  
--typedef minja::chat_template common_chat_template;
--
--struct common_chat_templates {
--    bool add_bos;
--    bool add_eos;
--    bool has_explicit_template; // Model had builtin template or template overridde was specified.
--    std::unique_ptr<common_chat_template> template_default; // always set (defaults to chatml)
--    std::unique_ptr<common_chat_template> template_tool_use;
--};
--
- struct templates_params {
-     json messages;
-     json tools;
-@@ -753,7 +740,7 @@ static std::string apply(
-         tmpl_inputs.extra_context.merge_patch(*additional_context);
++common_chat_template_caps common_chat_templates_get_caps(const struct common_chat_templates * tmpls, const std::string & variant) {
++    common_chat_template_caps result;
++    const common_chat_template * tmpl = nullptr;
++
++    if (!variant.empty() && variant == "tool_use") {
++        tmpl = tmpls->template_tool_use.get();
++    } else {
++        tmpl = tmpls->template_default.get();
++    }
++
++    if (tmpl) {
++        auto caps = tmpl->original_caps();
++        result.supports_tools = caps.supports_tools;
++        result.supports_tool_calls = caps.supports_tool_calls;
++        result.supports_system_role = caps.supports_system_role;
++        result.supports_parallel_tool_calls = caps.supports_parallel_tool_calls;
++    }
++
++    return result;
++}
++
++bool common_chat_templates_has_variant(const struct common_chat_templates * tmpls, const std::string & variant) {
++    if (variant.empty() || variant == "default") {
++        return tmpls->template_default != nullptr;
++    }
++    if (variant == "tool_use") {
++        return tmpls->template_tool_use != nullptr;
++    }
++    return false;
++}
++
+ common_chat_templates_ptr common_chat_templates_init(
+     const struct llama_model * model,
+     const std::string & chat_template_override,
+@@ -831,8 +862,9 @@ static std::string apply(
+     if (inputs.add_generation_prompt) {
+         inp["add_generation_prompt"] = true;
+     }
+-    if (inp["tools"].is_null()) {
+-        inp["tools"] = json::array();
++    // Remove tools key when null, so templates can check "{% if tools is defined %}"
++    if (inp["tools"].is_null() || (inp["tools"].is_array() && inp["tools"].empty())) {
++        inp.erase("tools");
      }
-     // TODO: add flag to control date/time, if only for testing purposes.
--    // tmpl_inputs.now = std::chrono::system_clock::now();
-+    tmpl_inputs.now = inputs.now;
  
-     minja::chat_template_options tmpl_opts;
-     // To avoid double BOS / EOS tokens, we're manually removing begining / trailing tokens
+     jinja::global_from_json(ctx, inp, inputs.mark_input);
 diff --git a/src/llama.cpp/common/chat.h b/src/llama.cpp/common/chat.h
-index 454085e90..e01390cf9 100644
+index ac19348ec..f6f9f612f 100644
 --- a/src/llama.cpp/common/chat.h
 +++ b/src/llama.cpp/common/chat.h
-@@ -10,7 +10,18 @@
- #include <vector>
- #include <map>
+@@ -231,6 +231,20 @@ common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::strin
+ 
+ bool common_chat_templates_support_enable_thinking(const common_chat_templates * chat_templates);
  
--struct common_chat_templates;
-+#include "minja/chat-template.hpp"
-+#include "minja/minja.hpp"
++// Template capabilities structure (for exposing capabilities to external code)
++struct common_chat_template_caps {
++    bool supports_tools = true;
++    bool supports_tool_calls = true;
++    bool supports_system_role = true;
++    bool supports_parallel_tool_calls = true;
++};
 +
-+typedef minja::chat_template common_chat_template;
++// Get template capabilities for a specific variant ("" for default, "tool_use" for tool_use template)
++common_chat_template_caps common_chat_templates_get_caps(const struct common_chat_templates * tmpls, const std::string & variant = "");
 +
-+struct common_chat_templates {
-+    bool add_bos;
-+    bool add_eos;
-+    bool has_explicit_template; // Model had builtin template or template overridde was specified.
-+    std::unique_ptr<common_chat_template> template_default; // always set (defaults to chatml)
-+    std::unique_ptr<common_chat_template> template_tool_use;
-+};
- 
- struct common_chat_tool_call {
-     std::string name;
++// Check if a template variant exists
++bool common_chat_templates_has_variant(const struct common_chat_templates * tmpls, const std::string & variant);
++
+ // Parses a JSON array of messages in OpenAI's chat completion API format.
+ // T can be std::string containing JSON or nlohmann::ordered_json
+ template <class T> std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const T & messages);
 diff --git a/src/llama.cpp/common/common.cpp b/src/llama.cpp/common/common.cpp
-index 744f0b4ee..04fcebb9e 100644
+index 26250abb6..72ceddcc7 100644
 --- a/src/llama.cpp/common/common.cpp
 +++ b/src/llama.cpp/common/common.cpp
-@@ -1361,6 +1361,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
+@@ -1360,6 +1360,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
          mparams.devices = params.devices.data();
      }
  
@@ -105,10 +120,10 @@ index 744f0b4ee..04fcebb9e 100644
      mparams.main_gpu        = params.main_gpu;
      mparams.split_mode      = params.split_mode;
 diff --git a/src/llama.cpp/common/common.h b/src/llama.cpp/common/common.h
-index e60087dea..c21797cd8 100644
+index 96c990c05..c0b0b3093 100644
 --- a/src/llama.cpp/common/common.h
 +++ b/src/llama.cpp/common/common.h
-@@ -311,6 +311,7 @@ struct lr_opt {
+@@ -317,6 +317,7 @@ struct lr_opt {
  struct ggml_opt_optimizer_params common_opt_lr_pars(void * userdata);
  
  struct common_params {
@@ -130,10 +145,10 @@ index 7622d0bf4..d2edcfddb 100644
              check_cxx_compiler_flag(-mfp16-format=ieee GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E)
              if (NOT "${GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E}" STREQUAL "")
 diff --git a/src/llama.cpp/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/src/llama.cpp/ggml/src/ggml-hexagon/ggml-hexagon.cpp
-index 365a24b49..83bf4ee62 100644
+index 5b835c11c..681c00504 100644
 --- a/src/llama.cpp/ggml/src/ggml-hexagon/ggml-hexagon.cpp
 +++ b/src/llama.cpp/ggml/src/ggml-hexagon/ggml-hexagon.cpp
-@@ -2798,9 +2798,24 @@ static const char * ggml_backend_hexagon_device_get_description(ggml_backend_dev
+@@ -2819,9 +2819,24 @@ static const char * ggml_backend_hexagon_device_get_description(ggml_backend_dev
      GGML_UNUSED(dev);
  }
  
@@ -160,7 +175,7 @@ index 365a24b49..83bf4ee62 100644
      *total = *free;
  
      GGML_UNUSED(dev);
-@@ -3010,10 +3025,17 @@ ggml_hexagon_registry::ggml_hexagon_registry(ggml_backend_reg_t reg) {
+@@ -3056,10 +3071,17 @@ ggml_hexagon_registry::ggml_hexagon_registry(ggml_backend_reg_t reg) {
          }
      }
  
@@ -182,7 +197,7 @@ index 365a24b49..83bf4ee62 100644
  
      GGML_LOG_INFO("ggml-hex: Hexagon Arch version v%d\n", opt_arch);
  
-@@ -3026,6 +3048,8 @@ ggml_hexagon_registry::ggml_hexagon_registry(ggml_backend_reg_t reg) {
+@@ -3072,6 +3094,8 @@ ggml_hexagon_registry::ggml_hexagon_registry(ggml_backend_reg_t reg) {
          } catch (const std::exception & exc) {
              GGML_LOG_ERROR("ggml-hex: failed to create device/session %zu\n", i);
              devices[i].context = nullptr;

package/src/LlamaContext.cpp

@@ -10,6 +10,7 @@
 #include "DecodeAudioTokenWorker.h"
 #include "ggml.h"
 #include "gguf.h"
+#include "chat.h"
 #include "json-schema-to-grammar.h"
 #include <nlohmann/json.hpp>
 #include "llama-impl.h"
@@ -600,40 +601,27 @@ Napi::Value LlamaContext::GetModelInfo(const Napi::CallbackInfo &info) {
 
   Napi::Object chatTemplates = Napi::Object::New(info.Env());
   chatTemplates.Set("llamaChat", _rn_ctx->validateModelChatTemplate(false, nullptr));
-  Napi::Object minja = Napi::Object::New(info.Env());
-  minja.Set("default", _rn_ctx->validateModelChatTemplate(true, nullptr));
+  Napi::Object jinja = Napi::Object::New(info.Env());
+  jinja.Set("default", _rn_ctx->validateModelChatTemplate(true, nullptr));
   Napi::Object defaultCaps = Napi::Object::New(info.Env());
-  auto default_tmpl = _rn_ctx->templates.get()->template_default.get();
-  auto default_tmpl_caps = default_tmpl->original_caps();
-  defaultCaps.Set(
-      "tools",
-      default_tmpl_caps.supports_tools);
-  defaultCaps.Set(
-      "toolCalls",
-      default_tmpl_caps.supports_tool_calls);
-  defaultCaps.Set("toolResponses", default_tmpl_caps.supports_tool_responses);
-  defaultCaps.Set(
-      "systemRole",
-      default_tmpl_caps.supports_system_role);
-  defaultCaps.Set("parallelToolCalls", default_tmpl_caps.supports_parallel_tool_calls);
-  defaultCaps.Set("toolCallId", default_tmpl_caps.supports_tool_call_id);
-  minja.Set("defaultCaps", defaultCaps);
-  minja.Set("toolUse", _rn_ctx->validateModelChatTemplate(true, "tool_use"));
-  if (_rn_ctx->validateModelChatTemplate(true, "tool_use")) {
+  auto default_caps = common_chat_templates_get_caps(_rn_ctx->templates.get(), "");
+  defaultCaps.Set("tools", default_caps.supports_tools);
+  defaultCaps.Set("toolCalls", default_caps.supports_tool_calls);
+  defaultCaps.Set("systemRole", default_caps.supports_system_role);
+  defaultCaps.Set("parallelToolCalls", default_caps.supports_parallel_tool_calls);
+  jinja.Set("defaultCaps", defaultCaps);
+  bool hasToolUse = common_chat_templates_has_variant(_rn_ctx->templates.get(), "tool_use");
+  jinja.Set("toolUse", hasToolUse);
+  if (hasToolUse) {
     Napi::Object toolUseCaps = Napi::Object::New(info.Env());
-    auto tool_use_tmpl = _rn_ctx->templates.get()->template_tool_use.get();
-    auto tool_use_tmpl_caps = tool_use_tmpl->original_caps();
-    toolUseCaps.Set(
-        "tools",
-        tool_use_tmpl_caps.supports_tools);
-    toolUseCaps.Set("toolCalls", tool_use_tmpl_caps.supports_tool_calls);
-    toolUseCaps.Set("toolResponses", tool_use_tmpl_caps.supports_tool_responses);
-    toolUseCaps.Set("systemRole", tool_use_tmpl_caps.supports_system_role);
-    toolUseCaps.Set("parallelToolCalls", tool_use_tmpl_caps.supports_parallel_tool_calls);
-    toolUseCaps.Set("toolCallId", tool_use_tmpl_caps.supports_tool_call_id);
-    minja.Set("toolUseCaps", toolUseCaps);
-  }
-  chatTemplates.Set("minja", minja);
+    auto tool_use_caps = common_chat_templates_get_caps(_rn_ctx->templates.get(), "tool_use");
+    toolUseCaps.Set("tools", tool_use_caps.supports_tools);
+    toolUseCaps.Set("toolCalls", tool_use_caps.supports_tool_calls);
+    toolUseCaps.Set("systemRole", tool_use_caps.supports_system_role);
+    toolUseCaps.Set("parallelToolCalls", tool_use_caps.supports_parallel_tool_calls);
+    jinja.Set("toolUseCaps", toolUseCaps);
+  }
+  chatTemplates.Set("jinja", jinja);
   details.Set("chatTemplates", chatTemplates);
 
   details.Set("metadata", metadata);

package/src/llama.cpp/common/CMakeLists.txt

@@ -85,6 +85,18 @@ add_library(${TARGET} STATIC
     speculative.h
     unicode.cpp
     unicode.h
+    jinja/lexer.cpp
+    jinja/lexer.h
+    jinja/parser.cpp
+    jinja/parser.h
+    jinja/runtime.cpp
+    jinja/runtime.h
+    jinja/value.cpp
+    jinja/value.h
+    jinja/string.cpp
+    jinja/string.h
+    jinja/caps.cpp
+    jinja/caps.h
     )
 
 target_include_directories(${TARGET} PUBLIC . ../vendor)

package/src/llama.cpp/common/arg.cpp

@@ -1729,6 +1729,26 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             }
         }
     ).set_sparam());
+    add_opt(common_arg(
+        {"--adaptive-target"}, "N",
+        string_format("adaptive-p: select tokens near this probability (valid range 0.0 "
+                      "to 1.0; negative = disabled) (default: %.2f)\n"
+                      "[(more info)](https://github.com/ggml-org/llama.cpp/pull/17927)",
+                      (double)params.sampling.adaptive_target),
+        [](common_params & params, const std::string & value) {
+            params.sampling.adaptive_target = std::stof(value);
+        }
+    ).set_sparam());
+    add_opt(common_arg(
+        {"--adaptive-decay"}, "N",
+        string_format("adaptive-p: decay rate for target adaptation over time. lower values "
+                      "are more reactive, higher values are more stable.\n"
+                      "(valid range 0.0 to 0.99) (default: %.2f)",
+                      (double)params.sampling.adaptive_decay),
+        [](common_params & params, const std::string & value) {
+            params.sampling.adaptive_decay = std::stof(value);
+        }
+    ).set_sparam());
     add_opt(common_arg(
         {"--dynatemp-range"}, "N",
         string_format("dynamic temperature range (default: %.1f, 0.0 = disabled)", (double)params.sampling.dynatemp_range),

package/src/llama.cpp/common/chat-parser.cpp

@@ -129,7 +129,7 @@ static void parse_json_tool_calls(
     }
 }
 
-common_chat_msg_parser::common_chat_msg_parser(const std::string & input, bool is_partial, const common_chat_syntax & syntax)
+common_chat_msg_parser::common_chat_msg_parser(const std::string & input, bool is_partial, const common_chat_parser_params & syntax)
     : input_(input), is_partial_(is_partial), syntax_(syntax)
 {
     result_.role = "assistant";
@@ -1611,7 +1611,7 @@ static void common_chat_parse(common_chat_msg_parser & builder) {
     builder.finish();
 }
 
-common_chat_msg common_chat_parse(const std::string & input, bool is_partial, const common_chat_syntax & syntax) {
+common_chat_msg common_chat_parse(const std::string & input, bool is_partial, const common_chat_parser_params & syntax) {
     if (syntax.format == COMMON_CHAT_FORMAT_PEG_SIMPLE ||
         syntax.format == COMMON_CHAT_FORMAT_PEG_NATIVE ||
         syntax.format == COMMON_CHAT_FORMAT_PEG_CONSTRUCTED) {
@@ -1635,7 +1635,7 @@ common_chat_msg common_chat_parse(const std::string & input, bool is_partial, co
     return msg;
 }
 
-common_chat_msg common_chat_peg_parse(const common_peg_arena & parser, const std::string & input, bool is_partial, const common_chat_syntax & syntax) {
+common_chat_msg common_chat_peg_parse(const common_peg_arena & parser, const std::string & input, bool is_partial, const common_chat_parser_params & syntax) {
     if (parser.empty()) {
         throw std::runtime_error("Failed to parse due to missing parser definition.");
     }

package/src/llama.cpp/common/chat-parser.h

@@ -5,7 +5,7 @@
 #include "json-partial.h"
 #include "regex-partial.h"
 
-#include <nlohmann/json.hpp>
+#include <nlohmann/json_fwd.hpp>
 
 #include <optional>
 #include <string>
@@ -19,20 +19,20 @@ class common_chat_msg_partial_exception : public std::runtime_error {
 class common_chat_msg_parser {
     std::string input_;
     bool is_partial_;
-    common_chat_syntax syntax_;
+    common_chat_parser_params syntax_; // TODO: rename to params
     std::string healing_marker_;
 
     size_t pos_ = 0;
     common_chat_msg result_;
 
   public:
-    common_chat_msg_parser(const std::string & input, bool is_partial, const common_chat_syntax & syntax);
+    common_chat_msg_parser(const std::string & input, bool is_partial, const common_chat_parser_params & syntax);
     const std::string & input() const { return input_; }
     size_t pos() const { return pos_; }
     const std::string & healing_marker() const { return healing_marker_; }
     const bool & is_partial() const { return is_partial_; }
     const common_chat_msg & result() const { return result_; }
-    const common_chat_syntax & syntax() const { return syntax_; }
+    const common_chat_parser_params & syntax() const { return syntax_; }
 
     void move_to(size_t pos) {
         if (pos > input_.size()) {