@fugood/llama.node 1.0.0-beta.5 → 1.0.0-beta.7

package/lib/binding.ts

@@ -22,7 +22,6 @@ export type ChatMessage = {
 export type LlamaModelOptions = {
   model: string
   chat_template?: string
-  reasoning_format?: string
   embedding?: boolean
   embd_normalize?: number
   pooling_type?: 'none' | 'mean' | 'cls' | 'last' | 'rank'
@@ -74,11 +73,14 @@ export type CompletionResponseFormat = {
 export type LlamaCompletionOptions = {
   messages?: ChatMessage[]
   jinja?: boolean
+  reasoning_format?: string
   chat_template?: string
   response_format?: CompletionResponseFormat
   tools?: object
   parallel_tool_calls?: boolean
   tool_choice?: string
+  enable_thinking?: boolean
+  thinking_forced_open?: boolean
   prompt?: string
   temperature?: number
   top_k?: number

package/lib/index.js

@@ -131,6 +131,7 @@ class LlamaContextWrapper {
         };
     }
     getFormattedChat(messages, template, params) {
+        var _a;
         const { messages: chat, has_media, media_paths, } = this._formatMediaChat(messages);
         const useJinja = this.isJinjaSupported() && (params === null || params === void 0 ? void 0 : params.jinja);
         let tmpl;
@@ -143,6 +144,7 @@ class LlamaContextWrapper {
             tools: params === null || params === void 0 ? void 0 : params.tools,
             parallel_tool_calls: params === null || params === void 0 ? void 0 : params.parallel_tool_calls,
             tool_choice: params === null || params === void 0 ? void 0 : params.tool_choice,
+            enable_thinking: (_a = params === null || params === void 0 ? void 0 : params.enable_thinking) !== null && _a !== void 0 ? _a : true,
         });
         if (!useJinja) {
             return {

package/lib/index.ts

@@ -158,7 +158,8 @@ class LlamaContextWrapper {
       response_format?: CompletionResponseFormat
       tools?: object
       parallel_tool_calls?: object
-      tool_choice?: string
+      tool_choice?: string,
+      enable_thinking?: boolean,
     },
   ): FormattedChatResult {
     const {
@@ -178,6 +179,7 @@ class LlamaContextWrapper {
       tools: params?.tools,
       parallel_tool_calls: params?.parallel_tool_calls,
       tool_choice: params?.tool_choice,
+      enable_thinking: params?.enable_thinking ?? true,
     })
 
     if (!useJinja) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "1.0.0-beta.5",
+  "version": "1.0.0-beta.7",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {
@@ -70,19 +70,19 @@
     "CMakeLists.txt"
   ],
   "optionalDependencies": {
-    "@fugood/node-llama-linux-x64": "1.0.0-beta.5",
-    "@fugood/node-llama-linux-x64-vulkan": "1.0.0-beta.5",
-    "@fugood/node-llama-linux-x64-cuda": "1.0.0-beta.5",
-    "@fugood/node-llama-linux-arm64": "1.0.0-beta.5",
-    "@fugood/node-llama-linux-arm64-vulkan": "1.0.0-beta.5",
-    "@fugood/node-llama-linux-arm64-cuda": "1.0.0-beta.5",
-    "@fugood/node-llama-win32-x64": "1.0.0-beta.5",
-    "@fugood/node-llama-win32-x64-vulkan": "1.0.0-beta.5",
-    "@fugood/node-llama-win32-x64-cuda": "1.0.0-beta.5",
-    "@fugood/node-llama-win32-arm64": "1.0.0-beta.5",
-    "@fugood/node-llama-win32-arm64-vulkan": "1.0.0-beta.5",
-    "@fugood/node-llama-darwin-x64": "1.0.0-beta.5",
-    "@fugood/node-llama-darwin-arm64": "1.0.0-beta.5"
+    "@fugood/node-llama-linux-x64": "1.0.0-beta.7",
+    "@fugood/node-llama-linux-x64-vulkan": "1.0.0-beta.7",
+    "@fugood/node-llama-linux-x64-cuda": "1.0.0-beta.7",
+    "@fugood/node-llama-linux-arm64": "1.0.0-beta.7",
+    "@fugood/node-llama-linux-arm64-vulkan": "1.0.0-beta.7",
+    "@fugood/node-llama-linux-arm64-cuda": "1.0.0-beta.7",
+    "@fugood/node-llama-win32-x64": "1.0.0-beta.7",
+    "@fugood/node-llama-win32-x64-vulkan": "1.0.0-beta.7",
+    "@fugood/node-llama-win32-x64-cuda": "1.0.0-beta.7",
+    "@fugood/node-llama-win32-arm64": "1.0.0-beta.7",
+    "@fugood/node-llama-win32-arm64-vulkan": "1.0.0-beta.7",
+    "@fugood/node-llama-darwin-x64": "1.0.0-beta.7",
+    "@fugood/node-llama-darwin-arm64": "1.0.0-beta.7"
   },
   "devDependencies": {
     "@babel/preset-env": "^7.24.4",

package/scripts/llama.cpp.patch

@@ -1,18 +1,19 @@
 diff --git a/src/llama.cpp/common/chat.cpp b/src/llama.cpp/common/chat.cpp
-index f138c7bc..e177fe92 100644
+index 7d9aaeb1..a7b68d4a 100644
 --- a/src/llama.cpp/common/chat.cpp
 +++ b/src/llama.cpp/common/chat.cpp
-@@ -1,8 +1,6 @@
- #include "chat.h"
- #include "json-schema-to-grammar.h"
+@@ -6,9 +6,6 @@
  #include "log.h"
--#include "minja/chat-template.hpp"
--#include "minja/minja.hpp"
+ #include "regex-partial.h"
  
- #include <optional>
- 
-@@ -15,14 +13,6 @@ static std::string format_time(const std::chrono::system_clock::time_point & now
-     return res;
+-#include <minja/chat-template.hpp>
+-#include <minja/minja.hpp>
+-
+ #include <cstdio>
+ #include <exception>
+ #include <iostream>
+@@ -121,14 +118,6 @@ std::vector<common_chat_msg_diff> common_chat_msg_diff::compute_diffs(const comm
+     return diffs;
  }
  
 -typedef minja::chat_template common_chat_template;
@@ -27,17 +28,17 @@ index f138c7bc..e177fe92 100644
      json messages;
      json tools;
 diff --git a/src/llama.cpp/common/chat.h b/src/llama.cpp/common/chat.h
-index d26a09c2..cb92721a 100644
+index 9f59e6b0..9b7fe724 100644
 --- a/src/llama.cpp/common/chat.h
 +++ b/src/llama.cpp/common/chat.h
-@@ -6,8 +6,16 @@
- #include <chrono>
+@@ -8,7 +8,16 @@
  #include <string>
  #include <vector>
-+#include "minja/chat-template.hpp"
-+#include "minja/minja.hpp"
  
 -struct common_chat_templates;
++#include <minja/chat-template.hpp>
++#include <minja/minja.hpp>
++
 +typedef minja::chat_template common_chat_template;
 +
 +struct common_chat_templates {
@@ -49,10 +50,10 @@ index d26a09c2..cb92721a 100644
  struct common_chat_tool_call {
      std::string name;
 diff --git a/src/llama.cpp/common/common.cpp b/src/llama.cpp/common/common.cpp
-index 94f545f8..a55df8aa 100644
+index e4e71ad1..091ddda4 100644
 --- a/src/llama.cpp/common/common.cpp
 +++ b/src/llama.cpp/common/common.cpp
-@@ -1062,6 +1062,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
+@@ -1101,6 +1101,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
          mparams.n_gpu_layers = params.n_gpu_layers;
      }
  
@@ -61,10 +62,10 @@ index 94f545f8..a55df8aa 100644
      mparams.split_mode      = params.split_mode;
      mparams.tensor_split    = params.tensor_split;
 diff --git a/src/llama.cpp/common/common.h b/src/llama.cpp/common/common.h
-index 0a9dc059..996afcd8 100644
+index e08a59ea..d120b67d 100644
 --- a/src/llama.cpp/common/common.h
 +++ b/src/llama.cpp/common/common.h
-@@ -217,6 +217,7 @@ enum common_reasoning_format {
+@@ -223,6 +223,7 @@ enum common_reasoning_format {
  };
  
  struct common_params {
@@ -73,11 +74,11 @@ index 0a9dc059..996afcd8 100644
      int32_t n_ctx                 =  4096; // context size
      int32_t n_batch               =  2048; // logical batch size for prompt processing (must be >=32 to use BLAS)
 diff --git a/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt b/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt
-index 9a3085be..8218cc16 100644
+index 71b1d67b..093cd6f9 100644
 --- a/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt
 +++ b/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt
-@@ -90,7 +90,7 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
-         message(STATUS "ARM detected")
+@@ -104,7 +104,7 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
+             )
  
          if (MSVC AND NOT CMAKE_C_COMPILER_ID STREQUAL "Clang")
 -            message(FATAL_ERROR "MSVC is not supported for ARM, use clang")
@@ -86,10 +87,10 @@ index 9a3085be..8218cc16 100644
              check_cxx_compiler_flag(-mfp16-format=ieee GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E)
              if (NOT "${GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E}" STREQUAL "")
 diff --git a/src/llama.cpp/ggml/src/ggml-vulkan/CMakeLists.txt b/src/llama.cpp/ggml/src/ggml-vulkan/CMakeLists.txt
-index 662f1377..f9f99698 100644
+index 39f022f3..7ae9047e 100644
 --- a/src/llama.cpp/ggml/src/ggml-vulkan/CMakeLists.txt
 +++ b/src/llama.cpp/ggml/src/ggml-vulkan/CMakeLists.txt
-@@ -122,7 +122,7 @@ if (Vulkan_FOUND)
+@@ -110,7 +110,7 @@ if (Vulkan_FOUND)
      endif()
  
      # Set up toolchain for host compilation whether cross-compiling or not
@@ -98,10 +99,10 @@ index 662f1377..f9f99698 100644
          if (GGML_VULKAN_SHADERS_GEN_TOOLCHAIN)
              set(HOST_CMAKE_TOOLCHAIN_FILE ${GGML_VULKAN_SHADERS_GEN_TOOLCHAIN})
          else()
-@@ -144,7 +144,7 @@ if (Vulkan_FOUND)
+@@ -130,7 +130,7 @@ if (Vulkan_FOUND)
+ 
      include(ExternalProject)
  
-     # Add toolchain file if cross-compiling
 -    if (CMAKE_CROSSCOMPILING)
 +    if (CMAKE_CROSSCOMPILING OR NOT CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL CMAKE_SYSTEM_PROCESSOR)
          list(APPEND VULKAN_SHADER_GEN_CMAKE_ARGS -DCMAKE_TOOLCHAIN_FILE=${HOST_CMAKE_TOOLCHAIN_FILE})

package/src/EmbeddingWorker.cpp

@@ -8,7 +8,7 @@ EmbeddingWorker::EmbeddingWorker(const Napi::CallbackInfo &info,
       _params(params) {}
 
 void EmbeddingWorker::Execute() {
-  llama_kv_self_clear(_sess->context());
+  llama_memory_clear(llama_get_memory(_sess->context()), true);
   auto tokens = ::common_tokenize(_sess->context(), _text, true);
   // add SEP if not present
   auto vocab = llama_model_get_vocab(_sess->model());

package/src/LlamaCompletionWorker.cpp

@@ -25,12 +25,18 @@ size_t findStoppingStrings(const std::string &text,
 
 LlamaCompletionWorker::LlamaCompletionWorker(
     const Napi::CallbackInfo &info, LlamaSessionPtr &sess,
-    Napi::Function callback, common_params params,
-    std::vector<std::string> stop_words, int32_t chat_format,
+    Napi::Function callback,
+    common_params params,
+    std::vector<std::string> stop_words,
+    int32_t chat_format,
+    bool thinking_forced_open,
+    std::string reasoning_format,
     const std::vector<std::string> &media_paths,
     const std::vector<llama_token> &guide_tokens)
     : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess),
       _params(params), _stop_words(stop_words), _chat_format(chat_format),
+      _thinking_forced_open(thinking_forced_open),
+      _reasoning_format(reasoning_format),
       _media_paths(media_paths), _guide_tokens(guide_tokens) {
   if (!callback.IsEmpty()) {
     _tsfn = Napi::ThreadSafeFunction::New(info.Env(), callback,
@@ -65,7 +71,7 @@ void LlamaCompletionWorker::Execute() {
 
   // Process media if any are provided
   if (!_media_paths.empty()) {
-    const auto *mtmd_ctx = _sess->get_mtmd_ctx();
+    auto *mtmd_ctx = _sess->get_mtmd_ctx();
 
     if (mtmd_ctx != nullptr) {
       // Process the media and get the tokens
@@ -109,7 +115,7 @@ void LlamaCompletionWorker::Execute() {
         --n_cur;
       }
       n_input -= n_cur;
-      llama_kv_self_seq_rm(ctx, 0, n_cur, -1);
+      llama_memory_seq_rm(llama_get_memory(ctx), 0, n_cur, -1);
     }
     // Set the tokens
     _sess->set_tokens(std::move(prompt_tokens));
@@ -131,8 +137,9 @@ void LlamaCompletionWorker::Execute() {
       const int n_left = n_cur - n_keep - 1;
       const int n_discard = n_left / 2;
 
-      llama_kv_self_seq_rm(ctx, 0, n_keep + 1, n_keep + n_discard + 1);
-      llama_kv_self_seq_add(ctx, 0, n_keep + 1 + n_discard, n_cur, -n_discard);
+      auto mem = llama_get_memory(ctx);
+      llama_memory_seq_rm(mem, 0, n_keep + 1, n_keep + n_discard + 1);
+      llama_memory_seq_add(mem, 0, n_keep + 1 + n_discard, n_cur, -n_discard);
 
       // shift the tokens
       embd->insert(embd->begin() + n_keep + 1,
@@ -234,8 +241,22 @@ void LlamaCompletionWorker::OnOK() {
   std::string content;
   if (!_stop) {
     try {
+      common_chat_syntax chat_syntax;
+      chat_syntax.format = static_cast<common_chat_format>(_chat_format);
+      chat_syntax.thinking_forced_open = _thinking_forced_open;
+
+      if (_reasoning_format == "deepseek") {
+          chat_syntax.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
+      } else if (_reasoning_format == "deepseek-legacy") {
+          chat_syntax.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY;
+      } else {
+          chat_syntax.reasoning_format = COMMON_REASONING_FORMAT_NONE;
+      }
       common_chat_msg message = common_chat_parse(
-          _result.text, static_cast<common_chat_format>(_chat_format));
+          _result.text,
+          false,
+          chat_syntax
+      );
       if (!message.reasoning_content.empty()) {
         reasoning_content = message.reasoning_content;
       }

package/src/LlamaCompletionWorker.h

@@ -20,6 +20,8 @@ public:
                         Napi::Function callback, common_params params,
                         std::vector<std::string> stop_words,
                         int32_t chat_format,
+                        bool thinking_forced_open,
+                        std::string reasoning_format,
                         const std::vector<std::string> &media_paths = {},
                         const std::vector<llama_token> &guide_tokens = {});
 
@@ -41,6 +43,8 @@ private:
   common_params _params;
   std::vector<std::string> _stop_words;
   int32_t _chat_format;
+  bool _thinking_forced_open;
+  std::string _reasoning_format;
   std::vector<std::string> _media_paths;
   std::vector<llama_token> _guide_tokens;
   std::function<void()> _onComplete;

package/src/LlamaContext.cpp

@@ -10,7 +10,7 @@
 #include "ggml.h"
 #include "gguf.h"
 #include "json-schema-to-grammar.h"
-#include "json.hpp"
+#include <nlohmann/json.hpp>
 #include "llama-impl.h"
 
 #include <atomic>
@@ -223,14 +223,6 @@ LlamaContext::LlamaContext(const Napi::CallbackInfo &info)
 
   params.chat_template = get_option<std::string>(options, "chat_template", "");
 
-  std::string reasoning_format =
-      get_option<std::string>(options, "reasoning_format", "none");
-  if (reasoning_format == "deepseek") {
-    params.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
-  } else {
-    params.reasoning_format = COMMON_REASONING_FORMAT_NONE;
-  }
-
   params.n_ctx = get_option<int32_t>(options, "n_ctx", 512);
   params.n_batch = get_option<int32_t>(options, "n_batch", 2048);
   params.n_ubatch = get_option<int32_t>(options, "n_ubatch", 512);
@@ -507,7 +499,9 @@ common_chat_params getFormattedChatWithJinja(
     const common_chat_templates_ptr &templates, const std::string &messages,
     const std::string &chat_template, const std::string &json_schema,
     const std::string &tools, const bool &parallel_tool_calls,
-    const std::string &tool_choice) {
+    const std::string &tool_choice,
+    const bool &enable_thinking
+) {
   common_chat_templates_inputs inputs;
   inputs.messages = common_chat_msgs_parse_oaicompat(json::parse(messages));
   auto useTools = !tools.empty();
@@ -521,8 +515,7 @@ common_chat_params getFormattedChatWithJinja(
   if (!json_schema.empty()) {
     inputs.json_schema = json::parse(json_schema);
   }
-  inputs.extract_reasoning =
-      sess->params().reasoning_format != COMMON_REASONING_FORMAT_NONE;
+  inputs.enable_thinking = enable_thinking;
 
   // If chat_template is provided, create new one and use it (probably slow)
   if (!chat_template.empty()) {
@@ -596,12 +589,11 @@ Napi::Value LlamaContext::GetFormattedChat(const Napi::CallbackInfo &info) {
     auto parallel_tool_calls =
         get_option<bool>(params, "parallel_tool_calls", false);
     auto tool_choice = get_option<std::string>(params, "tool_choice", "");
+    auto enable_thinking = get_option<bool>(params, "enable_thinking", false);
 
     auto chatParams = getFormattedChatWithJinja(
         _sess, _templates, messages, chat_template, json_schema_str, tools_str,
-        parallel_tool_calls, tool_choice);
-
-    console_log(env, std::string("format: ") + std::to_string(chatParams.format));
+        parallel_tool_calls, tool_choice, enable_thinking);
 
     Napi::Object result = Napi::Object::New(env);
     result.Set("prompt", chatParams.prompt);
@@ -622,6 +614,7 @@ Napi::Value LlamaContext::GetFormattedChat(const Napi::CallbackInfo &info) {
       grammar_triggers.Set(i, triggerObj);
     }
     result.Set("grammar_triggers", grammar_triggers);
+    result.Set("thinking_forced_open", chatParams.thinking_forced_open);
     // preserved_tokens: string[]
     Napi::Array preserved_tokens = Napi::Array::New(env);
     for (size_t i = 0; i < chatParams.preserved_tokens.size(); i++) {
@@ -695,6 +688,8 @@ Napi::Value LlamaContext::Completion(const Napi::CallbackInfo &info) {
   }
 
   int32_t chat_format = get_option<int32_t>(options, "chat_format", 0);
+  bool thinking_forced_open = get_option<bool>(options, "thinking_forced_open", false);
+  std::string reasoning_format = get_option<std::string>(options, "reasoning_format", "none");
 
   common_params params = _sess->params();
   auto grammar_from_params = get_option<std::string>(options, "grammar", "");
@@ -802,14 +797,16 @@ Napi::Value LlamaContext::Completion(const Napi::CallbackInfo &info) {
           get_option<bool>(options, "parallel_tool_calls", false);
       auto tool_choice =
           get_option<std::string>(options, "tool_choice", "none");
+      auto enable_thinking = get_option<bool>(options, "enable_thinking", true);
 
       auto chatParams = getFormattedChatWithJinja(
           _sess, _templates, json_stringify(messages), chat_template,
-          json_schema_str, tools_str, parallel_tool_calls, tool_choice);
+          json_schema_str, tools_str, parallel_tool_calls, tool_choice, enable_thinking);
 
       params.prompt = chatParams.prompt;
 
       chat_format = chatParams.format;
+      thinking_forced_open = chatParams.thinking_forced_open;
 
       for (const auto &token : chatParams.preserved_tokens) {
         auto ids =
@@ -904,7 +901,7 @@ Napi::Value LlamaContext::Completion(const Napi::CallbackInfo &info) {
 
   auto *worker =
       new LlamaCompletionWorker(info, _sess, callback, params, stop_words,
-                                chat_format, media_paths, guide_tokens);
+                                chat_format, thinking_forced_open, reasoning_format, media_paths, guide_tokens);
   worker->Queue();
   _wip = worker;
   worker->OnComplete([this]() { _wip = nullptr; });

package/src/common.hpp

@@ -6,6 +6,7 @@
 #include "llama.h"
 #include "tools/mtmd/clip.h"
 #include "tools/mtmd/mtmd.h"
+#include "tools/mtmd/mtmd-helper.h"
 #include <memory>
 #include <mutex>
 #include <napi.h>
@@ -97,7 +98,7 @@ public:
   inline std::mutex &get_mutex() { return mutex; }
 
   // Getter for the multimodal context
-  inline const mtmd_context *get_mtmd_ctx() const { return _mtmd_ctx; }
+  inline mtmd_context *get_mtmd_ctx() { return _mtmd_ctx; }
 
   // Setter for the multimodal context
   inline void set_mtmd_ctx(mtmd_context *ctx) { _mtmd_ctx = ctx; }
@@ -219,7 +220,7 @@ struct TokenizeResult {
 };
 
 static TokenizeResult
-tokenizeWithMedia(const mtmd_context *mtmd_ctx, const std::string &prompt,
+tokenizeWithMedia(mtmd_context *mtmd_ctx, const std::string &prompt,
                   const std::vector<std::string> &media_paths) {
   if (mtmd_ctx == nullptr) {
     throw std::runtime_error("Multimodal context is not initialized");
@@ -263,7 +264,7 @@ tokenizeWithMedia(const mtmd_context *mtmd_ctx, const std::string &prompt,
         std::vector<uint8_t> media_data = base64_decode(base64_data);
 
         // Load bitmap from memory buffer using direct initialization
-        mtmd::bitmap bmp(mtmd_helper_bitmap_init_from_buf(media_data.data(),
+        mtmd::bitmap bmp(mtmd_helper_bitmap_init_from_buf(mtmd_ctx, media_data.data(),
                                                           media_data.size()));
         if (!bmp.ptr) {
           bitmaps.entries.clear();
@@ -300,7 +301,7 @@ tokenizeWithMedia(const mtmd_context *mtmd_ctx, const std::string &prompt,
       fclose(file);
 
       // Create bitmap directly
-      mtmd::bitmap bmp(mtmd_helper_bitmap_init_from_file(media_path.c_str()));
+      mtmd::bitmap bmp(mtmd_helper_bitmap_init_from_file(mtmd_ctx, media_path.c_str()));
       if (!bmp.ptr) {
         bitmaps.entries.clear();
         throw std::runtime_error("Failed to load media");
@@ -388,7 +389,7 @@ tokenizeWithMedia(const mtmd_context *mtmd_ctx, const std::string &prompt,
 
 // Process media and add them to the tokenized input
 static llama_pos
-processMediaPrompt(llama_context *ctx, const mtmd_context *mtmd_ctx,
+processMediaPrompt(llama_context *ctx, mtmd_context *mtmd_ctx,
                    LlamaSessionPtr sess, const common_params &params,
                    const std::vector<std::string> &media_paths) {
   if (mtmd_ctx == nullptr) {
@@ -460,7 +461,7 @@ processMediaPrompt(llama_context *ctx, const mtmd_context *mtmd_ctx,
   }
 
   // Clear all KV cache entries after position n_past
-  llama_kv_self_seq_rm(ctx, 0, n_past, -1);
+  llama_memory_seq_rm(llama_get_memory(ctx), 0, n_past, -1);
 
   size_t num_chunks = mtmd_input_chunks_size(chunks);
 

package/src/llama.cpp/CMakeLists.txt

@@ -89,6 +89,14 @@ option(LLAMA_LLGUIDANCE "llama-common: include LLGuidance library for structured
 include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/build-info.cmake)
 include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/common.cmake)
 
+if (NOT DEFINED LLAMA_BUILD_NUMBER)
+    set(LLAMA_BUILD_NUMBER        ${BUILD_NUMBER})
+endif()
+if (NOT DEFINED LLAMA_BUILD_COMMIT)
+    set(LLAMA_BUILD_COMMIT        ${BUILD_COMMIT})
+endif()
+set(LLAMA_INSTALL_VERSION 0.0.${BUILD_NUMBER})
+
 # override ggml options
 set(GGML_ALL_WARNINGS   ${LLAMA_ALL_WARNINGS})
 set(GGML_FATAL_WARNINGS ${LLAMA_FATAL_WARNINGS})
@@ -155,10 +163,17 @@ if (LLAMA_USE_SYSTEM_GGML)
 endif()
 
 if (NOT TARGET ggml AND NOT LLAMA_USE_SYSTEM_GGML)
+    set(GGML_BUILD_NUMBER ${LLAMA_BUILD_NUMBER})
+    set(GGML_BUILD_COMMIT ${LLAMA_BUILD_COMMIT})
     add_subdirectory(ggml)
     # ... otherwise assume ggml is added by a parent CMakeLists.txt
 endif()
 
+if (MINGW)
+    # Target Windows 8 for PrefetchVirtualMemory
+    add_compile_definitions(_WIN32_WINNT=${GGML_WIN_VER})
+endif()
+
 #
 # build the library
 #
@@ -199,10 +214,6 @@ endif()
 include(GNUInstallDirs)
 include(CMakePackageConfigHelpers)
 
-set(LLAMA_BUILD_NUMBER        ${BUILD_NUMBER})
-set(LLAMA_BUILD_COMMIT        ${BUILD_COMMIT})
-set(LLAMA_INSTALL_VERSION 0.0.${BUILD_NUMBER})
-
 set(LLAMA_INCLUDE_INSTALL_DIR ${CMAKE_INSTALL_INCLUDEDIR} CACHE PATH "Location of header  files")
 set(LLAMA_LIB_INSTALL_DIR     ${CMAKE_INSTALL_LIBDIR}     CACHE PATH "Location of library files")
 set(LLAMA_BIN_INSTALL_DIR     ${CMAKE_INSTALL_BINDIR}     CACHE PATH "Location of binary  files")

package/src/llama.cpp/common/CMakeLists.txt

@@ -7,8 +7,8 @@ llama_add_compile_flags()
 # Build info header
 #
 
-if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../.git")
-    set(GIT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../.git")
+if(EXISTS "${PROJECT_SOURCE_DIR}/.git")
+    set(GIT_DIR "${PROJECT_SOURCE_DIR}/.git")
 
     # Is git submodule
     if(NOT IS_DIRECTORY "${GIT_DIR}")
@@ -18,36 +18,26 @@ if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../.git")
         if (SLASH_POS EQUAL 0)
             set(GIT_DIR "${REAL_GIT_DIR}")
         else()
-            set(GIT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../${REAL_GIT_DIR}")
+            set(GIT_DIR "${PROJECT_SOURCE_DIR}/${REAL_GIT_DIR}")
         endif()
     endif()
 
     if(EXISTS "${GIT_DIR}/index")
-        set(GIT_INDEX "${GIT_DIR}/index")
+        # For build-info.cpp below
+        set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${GIT_DIR}/index")
     else()
         message(WARNING "Git index not found in git repository.")
-        set(GIT_INDEX "")
     endif()
 else()
     message(WARNING "Git repository not found; to enable automatic generation of build info, make sure Git is installed and the project is a Git repository.")
-    set(GIT_INDEX "")
 endif()
 
-# Add a custom command to rebuild build-info.cpp when .git/index changes
-add_custom_command(
-    OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/build-info.cpp"
-    COMMENT "Generating build details from Git"
-    COMMAND ${CMAKE_COMMAND} -DMSVC=${MSVC} -DCMAKE_C_COMPILER_VERSION=${CMAKE_C_COMPILER_VERSION}
-            -DCMAKE_C_COMPILER_ID=${CMAKE_C_COMPILER_ID} -DCMAKE_VS_PLATFORM_NAME=${CMAKE_VS_PLATFORM_NAME}
-            -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-            -DCMAKE_SYSTEM_NAME=${CMAKE_SYSTEM_NAME} -DCMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}
-            -P "${CMAKE_CURRENT_SOURCE_DIR}/cmake/build-info-gen-cpp.cmake"
-    WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/.."
-    DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/build-info.cpp.in" ${GIT_INDEX}
-    VERBATIM
-)
+set(TEMPLATE_FILE "${CMAKE_CURRENT_SOURCE_DIR}/build-info.cpp.in")
+set(OUTPUT_FILE   "${CMAKE_CURRENT_BINARY_DIR}/build-info.cpp")
+configure_file(${TEMPLATE_FILE} ${OUTPUT_FILE})
+
 set(TARGET build_info)
-add_library(${TARGET} OBJECT build-info.cpp)
+add_library(${TARGET} OBJECT ${OUTPUT_FILE})
 if (BUILD_SHARED_LIBS)
     set_target_properties(${TARGET} PROPERTIES POSITION_INDEPENDENT_CODE ON)
 endif()
@@ -58,19 +48,20 @@ add_library(${TARGET} STATIC
     arg.cpp
     arg.h
     base64.hpp
+    chat-parser.cpp
+    chat-parser.h
     chat.cpp
     chat.h
     common.cpp
     common.h
     console.cpp
     console.h
+    json-partial.cpp
+    json-partial.h
     json-schema-to-grammar.cpp
-    json.hpp
     llguidance.cpp
     log.cpp
     log.h
-    minja/chat-template.hpp
-    minja/minja.hpp
     ngram-cache.cpp
     ngram-cache.h
     regex-partial.cpp
@@ -143,7 +134,7 @@ if (LLAMA_LLGUIDANCE)
     set(LLAMA_COMMON_EXTRA_LIBS ${LLAMA_COMMON_EXTRA_LIBS} llguidance ${LLGUIDANCE_PLATFORM_LIBS})
 endif ()
 
-target_include_directories(${TARGET} PUBLIC .)
+target_include_directories(${TARGET} PUBLIC . ../vendor)
 target_compile_features   (${TARGET} PUBLIC cxx_std_17)
 target_link_libraries     (${TARGET} PRIVATE ${LLAMA_COMMON_EXTRA_LIBS} PUBLIC llama Threads::Threads)