@fugood/llama.node 1.1.5 → 1.1.7

package/lib/binding.ts

@@ -167,6 +167,10 @@ export type LlamaCompletionResult = {
 
 export type LlamaCompletionToken = {
   token: string
+  content?: string
+  reasoning_content?: string
+  tool_calls?: ToolCall[]
+  accumulated_text?: string
 }
 
 export type TokenizeResult = {

package/lib/index.js

@@ -23,9 +23,10 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.loadLlamaModelInfo = exports.initLlama = exports.loadModel = exports.toggleNativeLog = exports.MTMD_DEFAULT_MEDIA_MARKER = void 0;
+exports.BuildInfo = exports.loadLlamaModelInfo = exports.initLlama = exports.loadModel = exports.toggleNativeLog = exports.MTMD_DEFAULT_MEDIA_MARKER = void 0;
 exports.addNativeLogListener = addNativeLogListener;
 const binding_1 = require("./binding");
+const version_1 = require("./version");
 __exportStar(require("./binding"), exports);
 exports.MTMD_DEFAULT_MEDIA_MARKER = '<__media__>';
 const mods = {};
@@ -259,3 +260,7 @@ const loadLlamaModelInfo = (path) => __awaiter(void 0, void 0, void 0, function*
     return mods[variant].LlamaContext.loadModelInfo(path, modelInfoSkip);
 });
 exports.loadLlamaModelInfo = loadLlamaModelInfo;
+exports.BuildInfo = {
+    number: version_1.BUILD_NUMBER,
+    commit: version_1.BUILD_COMMIT,
+};

package/lib/index.ts

@@ -17,6 +17,7 @@ import type {
   Tool,
   GGUFModelInfo,
 } from './binding'
+import { BUILD_NUMBER, BUILD_COMMIT } from './version'
 
 export * from './binding'
 
@@ -353,3 +354,8 @@ export const loadLlamaModelInfo = async (path: string): Promise<GGUFModelInfo> =
   refreshNativeLogSetup()
   return mods[variant].LlamaContext.loadModelInfo(path, modelInfoSkip)
 }
+
+export const BuildInfo = {
+  number: BUILD_NUMBER,
+  commit: BUILD_COMMIT,
+}

package/lib/version.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BUILD_COMMIT = exports.BUILD_NUMBER = void 0;
+exports.BUILD_NUMBER = '6096';
+exports.BUILD_COMMIT = 'fd1234cb';

package/lib/version.ts

@@ -0,0 +1,2 @@
+export const BUILD_NUMBER = '6096';
+export const BUILD_COMMIT = 'fd1234cb';

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "1.1.5",
+  "version": "1.1.7",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {
@@ -71,19 +71,19 @@
     "CMakeLists.txt"
   ],
   "optionalDependencies": {
-    "@fugood/node-llama-linux-x64": "1.1.5",
-    "@fugood/node-llama-linux-x64-vulkan": "1.1.5",
-    "@fugood/node-llama-linux-x64-cuda": "1.1.5",
-    "@fugood/node-llama-linux-arm64": "1.1.5",
-    "@fugood/node-llama-linux-arm64-vulkan": "1.1.5",
-    "@fugood/node-llama-linux-arm64-cuda": "1.1.5",
-    "@fugood/node-llama-win32-x64": "1.1.5",
-    "@fugood/node-llama-win32-x64-vulkan": "1.1.5",
-    "@fugood/node-llama-win32-x64-cuda": "1.1.5",
-    "@fugood/node-llama-win32-arm64": "1.1.5",
-    "@fugood/node-llama-win32-arm64-vulkan": "1.1.5",
-    "@fugood/node-llama-darwin-x64": "1.1.5",
-    "@fugood/node-llama-darwin-arm64": "1.1.5"
+    "@fugood/node-llama-linux-x64": "1.1.7",
+    "@fugood/node-llama-linux-x64-vulkan": "1.1.7",
+    "@fugood/node-llama-linux-x64-cuda": "1.1.7",
+    "@fugood/node-llama-linux-arm64": "1.1.7",
+    "@fugood/node-llama-linux-arm64-vulkan": "1.1.7",
+    "@fugood/node-llama-linux-arm64-cuda": "1.1.7",
+    "@fugood/node-llama-win32-x64": "1.1.7",
+    "@fugood/node-llama-win32-x64-vulkan": "1.1.7",
+    "@fugood/node-llama-win32-x64-cuda": "1.1.7",
+    "@fugood/node-llama-win32-arm64": "1.1.7",
+    "@fugood/node-llama-win32-arm64-vulkan": "1.1.7",
+    "@fugood/node-llama-darwin-x64": "1.1.7",
+    "@fugood/node-llama-darwin-arm64": "1.1.7"
   },
   "devDependencies": {
     "@babel/preset-env": "^7.24.4",

package/scripts/llama.cpp.patch

@@ -1,5 +1,5 @@
 diff --git a/src/llama.cpp/common/chat.cpp b/src/llama.cpp/common/chat.cpp
-index 114dbfcc..6771bd43 100644
+index 23d3828f9..ca48af00c 100644
 --- a/src/llama.cpp/common/chat.cpp
 +++ b/src/llama.cpp/common/chat.cpp
 @@ -6,9 +6,6 @@
@@ -12,13 +12,15 @@ index 114dbfcc..6771bd43 100644
  #include <cstdio>
  #include <exception>
  #include <iostream>
-@@ -123,14 +120,6 @@ std::vector<common_chat_msg_diff> common_chat_msg_diff::compute_diffs(const comm
+@@ -123,16 +120,6 @@ std::vector<common_chat_msg_diff> common_chat_msg_diff::compute_diffs(const comm
      return diffs;
  }
  
 -typedef minja::chat_template common_chat_template;
 -
 -struct common_chat_templates {
+-    bool add_bos;
+-    bool add_eos;
 -    bool has_explicit_template; // Model had builtin template or template overridde was specified.
 -    std::unique_ptr<common_chat_template> template_default; // always set (defaults to chatml)
 -    std::unique_ptr<common_chat_template> template_tool_use;
@@ -27,21 +29,23 @@ index 114dbfcc..6771bd43 100644
  struct templates_params {
      json messages;
      json tools;
-diff --git a/common/chat.h b/common/chat.h
-index ca807c14..56649863 100644
+diff --git a/src/llama.cpp/common/chat.h b/src/llama.cpp/common/chat.h
+index d1e480c91..437e64e29 100644
 --- a/src/llama.cpp/common/chat.h
 +++ b/src/llama.cpp/common/chat.h
-@@ -9,7 +9,16 @@
+@@ -9,7 +9,18 @@
  #include <vector>
  #include <map>
  
 -struct common_chat_templates;
-+#include <minja/chat-template.hpp>
-+#include <minja/minja.hpp>
++#include "minja/chat-template.hpp"
++#include "minja/minja.hpp"
 +
 +typedef minja::chat_template common_chat_template;
 +
 +struct common_chat_templates {
++    bool add_bos;
++    bool add_eos;
 +    bool has_explicit_template; // Model had builtin template or template overridde was specified.
 +    std::unique_ptr<common_chat_template> template_default; // always set (defaults to chatml)
 +    std::unique_ptr<common_chat_template> template_tool_use;
@@ -50,10 +54,10 @@ index ca807c14..56649863 100644
  struct common_chat_tool_call {
      std::string name;
 diff --git a/src/llama.cpp/common/common.cpp b/src/llama.cpp/common/common.cpp
-index e4e71ad1..091ddda4 100644
+index 67dd5404f..909a97c66 100644
 --- a/src/llama.cpp/common/common.cpp
 +++ b/src/llama.cpp/common/common.cpp
-@@ -1101,6 +1101,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
+@@ -1117,6 +1117,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
          mparams.n_gpu_layers = params.n_gpu_layers;
      }
  
@@ -62,11 +66,11 @@ index e4e71ad1..091ddda4 100644
      mparams.split_mode      = params.split_mode;
      mparams.tensor_split    = params.tensor_split;
 diff --git a/src/llama.cpp/common/common.h b/src/llama.cpp/common/common.h
-index 8922090e..3c2d1a6a 100644
+index 75596e6b3..0e04694c8 100644
 --- a/src/llama.cpp/common/common.h
 +++ b/src/llama.cpp/common/common.h
-@@ -224,6 +224,7 @@ enum common_reasoning_format {
- };
+@@ -267,6 +267,7 @@ struct lr_opt {
+ struct ggml_opt_optimizer_params common_opt_lr_pars(void * userdata);
  
  struct common_params {
 +    bool vocab_only               = false;
@@ -74,10 +78,10 @@ index 8922090e..3c2d1a6a 100644
      int32_t n_ctx                 =  4096; // context size
      int32_t n_batch               =  2048; // logical batch size for prompt processing (must be >=32 to use BLAS)
 diff --git a/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt b/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt
-index 671fad4d..93fc3cd7 100644
+index ce0a3e128..df9300224 100644
 --- a/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt
 +++ b/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt
-@@ -104,7 +104,7 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
+@@ -106,7 +106,7 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
              )
  
          if (MSVC AND NOT CMAKE_C_COMPILER_ID STREQUAL "Clang")
@@ -87,7 +91,7 @@ index 671fad4d..93fc3cd7 100644
              check_cxx_compiler_flag(-mfp16-format=ieee GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E)
              if (NOT "${GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E}" STREQUAL "")
 diff --git a/src/llama.cpp/ggml/src/ggml-vulkan/CMakeLists.txt b/src/llama.cpp/ggml/src/ggml-vulkan/CMakeLists.txt
-index b97e7bf9..c3eb9519 100644
+index b97e7bf99..c3eb9519f 100644
 --- a/src/llama.cpp/ggml/src/ggml-vulkan/CMakeLists.txt
 +++ b/src/llama.cpp/ggml/src/ggml-vulkan/CMakeLists.txt
 @@ -111,7 +111,7 @@ if (Vulkan_FOUND)

package/src/LlamaCompletionWorker.cpp

@@ -55,6 +55,32 @@ LlamaCompletionWorker::~LlamaCompletionWorker() {
   }
 }
 
+LlamaCompletionWorker::PartialOutput LlamaCompletionWorker::getPartialOutput(const std::string &generated_text) {
+  PartialOutput result;
+  
+  try {
+    common_chat_syntax chat_syntax;
+    chat_syntax.format = static_cast<common_chat_format>(_chat_format);
+    chat_syntax.thinking_forced_open = _thinking_forced_open;
+    
+    // Set reasoning format using the common function
+    chat_syntax.reasoning_format = common_reasoning_format_from_name(_reasoning_format);
+    
+    chat_syntax.parse_tool_calls = true;
+    
+    // Use is_partial=true for streaming partial output
+    common_chat_msg parsed_msg = common_chat_parse(generated_text, true, chat_syntax);
+    
+    result.content = parsed_msg.content;
+    result.reasoning_content = parsed_msg.reasoning_content;
+    result.tool_calls = parsed_msg.tool_calls;
+  } catch (const std::exception &e) {
+    // If parsing fails, leave content empty - this is expected for partial content
+  }
+  
+  return result;
+}
+
 void LlamaCompletionWorker::Execute() {
   _sess->get_mutex().lock();
   const auto t_main_start = ggml_time_us();
@@ -222,6 +248,13 @@ void LlamaCompletionWorker::Execute() {
 
     // sample the next token
     llama_token new_token_id = common_sampler_sample(sampling.get(), ctx, -1);
+
+    // is it an end of generation?
+    if (llama_vocab_is_eog(vocab, new_token_id)) {
+      _result.stopped_eos = true;
+      break;
+    }
+
     if (_next_token_uses_guide_token && !_guide_tokens.empty() &&
         !llama_vocab_is_control(vocab, new_token_id) &&
         !llama_vocab_is_eog(vocab, new_token_id)) {
@@ -250,21 +283,49 @@ void LlamaCompletionWorker::Execute() {
     if (_has_callback) {
       // TODO: When we got possible stop words (startsWith)
       // we should avoid calling the callback, wait for the next token
-      const char *c_token = strdup(token.c_str());
-      _tsfn.BlockingCall(c_token, [](Napi::Env env, Napi::Function jsCallback,
-                                     const char *value) {
+      struct TokenData {
+        std::string token;
+        std::string content;
+        std::string reasoning_content;
+        std::vector<common_chat_tool_call> tool_calls;
+        std::string accumulated_text;
+      };
+      
+      auto partial = getPartialOutput(_result.text);
+      TokenData *token_data = new TokenData{token, partial.content, partial.reasoning_content, partial.tool_calls, _result.text};
+      
+      _tsfn.BlockingCall(token_data, [](Napi::Env env, Napi::Function jsCallback,
+                                        TokenData *data) {
         auto obj = Napi::Object::New(env);
-        obj.Set("token", Napi::String::New(env, value));
-        delete value;
+        obj.Set("token", Napi::String::New(env, data->token));
+        if (!data->content.empty()) {
+          obj.Set("content", Napi::String::New(env, data->content));
+        }
+        if (!data->reasoning_content.empty()) {
+          obj.Set("reasoning_content", Napi::String::New(env, data->reasoning_content));
+        }
+        if (!data->tool_calls.empty()) {
+          Napi::Array tool_calls = Napi::Array::New(env);
+          for (size_t i = 0; i < data->tool_calls.size(); i++) {
+            const auto &tc = data->tool_calls[i];
+            Napi::Object tool_call = Napi::Object::New(env);
+            tool_call.Set("type", "function");
+            Napi::Object function = Napi::Object::New(env);
+            function.Set("name", tc.name);
+            function.Set("arguments", tc.arguments);
+            tool_call.Set("function", function);
+            if (!tc.id.empty()) {
+              tool_call.Set("id", tc.id);
+            }
+            tool_calls.Set(i, tool_call);
+          }
+          obj.Set("tool_calls", tool_calls);
+        }
+        obj.Set("accumulated_text", Napi::String::New(env, data->accumulated_text));
+        delete data;
         jsCallback.Call({obj});
       });
     }
-    // is it an end of generation?
-    if (llama_vocab_is_eog(vocab, new_token_id)) {
-      _result.stopped_eos = true;
-      // TODO: EOS token should be cut
-      break;
-    }
     // check for stop words
     if (!_stop_words.empty()) {
       const size_t stop_pos =
@@ -316,13 +377,7 @@ void LlamaCompletionWorker::OnOK() {
 
       chat_syntax.thinking_forced_open = _thinking_forced_open;
 
-      if (_reasoning_format == "deepseek") {
-          chat_syntax.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
-      } else if (_reasoning_format == "deepseek-legacy") {
-          chat_syntax.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY;
-      } else {
-          chat_syntax.reasoning_format = COMMON_REASONING_FORMAT_NONE;
-      }
+      chat_syntax.reasoning_format = common_reasoning_format_from_name(_reasoning_format);
       common_chat_msg message = common_chat_parse(
           _result.text,
           false,

package/src/LlamaCompletionWorker.h

@@ -42,6 +42,14 @@ protected:
   void OnError(const Napi::Error &err) override;
 
 private:
+  struct PartialOutput {
+    std::string content = "";
+    std::string reasoning_content = "";
+    std::vector<common_chat_tool_call> tool_calls;
+  };
+
+  PartialOutput getPartialOutput(const std::string &generated_text);
+
   LlamaSessionPtr _sess;
   common_params _params;
   std::vector<std::string> _stop_words;

package/src/llama.cpp/CMakeLists.txt

@@ -12,6 +12,8 @@ if (NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE)
     set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
 endif()
 
+message("CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}")
+
 # Add path to modules
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/")
 

package/src/llama.cpp/common/arg.cpp

@@ -24,6 +24,7 @@
 #include <cstdarg>
 #include <filesystem>
 #include <fstream>
+#include <list>
 #include <regex>
 #include <set>
 #include <string>
@@ -748,6 +749,39 @@ std::pair<long, std::vector<char>> common_remote_get_content(const std::string &
 // utils
 //
 
+// Helper function to parse tensor buffer override strings
+static void parse_tensor_buffer_overrides(const std::string & value, std::vector<llama_model_tensor_buft_override> & overrides) {
+    std::map<std::string, ggml_backend_buffer_type_t> buft_list;
+    for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
+        auto * dev = ggml_backend_dev_get(i);
+        auto * buft = ggml_backend_dev_buffer_type(dev);
+        if (buft) {
+            buft_list[ggml_backend_buft_name(buft)] = buft;
+        }
+    }
+
+    for (const auto & override : string_split<std::string>(value, ',')) {
+        std::string::size_type pos = override.find('=');
+        if (pos == std::string::npos) {
+            throw std::invalid_argument("invalid value");
+        }
+        std::string tensor_name = override.substr(0, pos);
+        std::string buffer_type = override.substr(pos + 1);
+
+        if (buft_list.find(buffer_type) == buft_list.end()) {
+            printf("Available buffer types:\n");
+            for (const auto & it : buft_list) {
+                printf("  %s\n", ggml_backend_buft_name(it.second));
+            }
+            throw std::invalid_argument("unknown buffer type");
+        }
+        // keep strings alive and avoid leaking memory by storing them in a static vector
+        static std::list<std::string> buft_overrides;
+        buft_overrides.push_back(tensor_name);
+        overrides.push_back({buft_overrides.back().c_str(), buft_list.at(buffer_type)});
+    }
+}
+
 struct handle_model_result {
     bool found_mmproj = false;
     common_params_model mmproj;
@@ -992,6 +1026,10 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
         params.tensor_buft_overrides.push_back({nullptr, nullptr});
     }
 
+    if (!params.speculative.tensor_buft_overrides.empty()) {
+        params.speculative.tensor_buft_overrides.push_back({nullptr, nullptr});
+    }
+
     if (!params.chat_template.empty() && !common_chat_verify_template(params.chat_template, params.use_jinja)) {
         throw std::runtime_error(string_format(
             "error: the supplied chat template is not supported: %s%s\n",
@@ -1200,6 +1238,7 @@ bool common_params_parse(int argc, char ** argv, common_params & params, llama_e
             common_params_print_completion(ctx_arg);
             exit(0);
         }
+        params.lr.init();
     } catch (const std::invalid_argument & ex) {
         fprintf(stderr, "%s\n", ex.what());
         ctx_arg.params = params_org;
@@ -1468,6 +1507,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.swa_full = true;
         }
     ).set_env("LLAMA_ARG_SWA_FULL"));
+    add_opt(common_arg(
+        {"--swa-checkpoints"}, "N",
+        string_format("max number of SWA checkpoints per slot to create (default: %d)\n"
+            "[(more info)](https://github.com/ggml-org/llama.cpp/pull/15293)", params.n_swa_checkpoints),
+        [](common_params & params, int value) {
+            params.n_swa_checkpoints = value;
+        }
+    ).set_env("LLAMA_ARG_SWA_CHECKPOINTS").set_examples({LLAMA_EXAMPLE_SERVER}));
     add_opt(common_arg(
         {"--kv-unified", "-kvu"},
         string_format("use single unified KV buffer for the KV cache of all sequences (default: %s)\n"
@@ -2348,47 +2395,58 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
     add_opt(common_arg(
         {"--override-tensor", "-ot"}, "<tensor name pattern>=<buffer type>,...",
         "override tensor buffer type", [](common_params & params, const std::string & value) {
-            /* static */ std::map<std::string, ggml_backend_buffer_type_t> buft_list;
-            if (buft_list.empty()) {
-                // enumerate all the devices and add their buffer types to the list
-                for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
-                    auto * dev = ggml_backend_dev_get(i);
-                    auto * buft = ggml_backend_dev_buffer_type(dev);
-                    if (buft) {
-                        buft_list[ggml_backend_buft_name(buft)] = buft;
-                    }
-                }
-            }
-
-            for (const auto & override : string_split<std::string>(value, ',')) {
-                std::string::size_type pos = override.find('=');
-                if (pos == std::string::npos) {
-                    throw std::invalid_argument("invalid value");
-                }
-                std::string tensor_name = override.substr(0, pos);
-                std::string buffer_type = override.substr(pos + 1);
-
-                if (buft_list.find(buffer_type) == buft_list.end()) {
-                    printf("Available buffer types:\n");
-                    for (const auto & it : buft_list) {
-                        printf("  %s\n", ggml_backend_buft_name(it.second));
-                    }
-                    throw std::invalid_argument("unknown buffer type");
-                }
-                // FIXME: this leaks memory
-                params.tensor_buft_overrides.push_back({strdup(tensor_name.c_str()), buft_list.at(buffer_type)});
-            }
+            parse_tensor_buffer_overrides(value, params.tensor_buft_overrides);
         }
     ));
     add_opt(common_arg(
-        {"--cpu-moe"},
-        "use CPU for Mixture of Experts (MoE) weights",
+        {"--override-tensor-draft", "-otd"}, "<tensor name pattern>=<buffer type>,...",
+        "override tensor buffer type for draft model", [](common_params & params, const std::string & value) {
+            parse_tensor_buffer_overrides(value, params.speculative.tensor_buft_overrides);
+        }
+    ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER}));
+    add_opt(common_arg(
+        {"--cpu-moe", "-cmoe"},
+        "keep all Mixture of Experts (MoE) weights in the CPU",
         [](common_params & params) {
-            params.tensor_buft_overrides.push_back({"\\.ffn_up_exps\\.weight$",   ggml_backend_cpu_buffer_type()});
-            params.tensor_buft_overrides.push_back({"\\.ffn_down_exps\\.weight$", ggml_backend_cpu_buffer_type()});
-            params.tensor_buft_overrides.push_back({"\\.ffn_gate_exps\\.weight$", ggml_backend_cpu_buffer_type()});
+            params.tensor_buft_overrides.push_back({"\\.ffn_(up|down|gate)_exps", ggml_backend_cpu_buffer_type()});
         }
     ).set_env("LLAMA_ARG_CPU_MOE"));
+    add_opt(common_arg(
+        {"--n-cpu-moe", "-ncmoe"}, "N",
+        "keep the Mixture of Experts (MoE) weights of the first N layers in the CPU",
+        [](common_params & params, int value) {
+            if (value < 0) {
+                throw std::invalid_argument("invalid value");
+            }
+            for (int i = 0; i < value; ++i) {
+                // keep strings alive and avoid leaking memory by storing them in a static vector
+                static std::list<std::string> buft_overrides;
+                buft_overrides.push_back(string_format("blk\\.%d\\.ffn_(up|down|gate)_exps", i));
+                params.tensor_buft_overrides.push_back({buft_overrides.back().c_str(), ggml_backend_cpu_buffer_type()});
+            }
+        }
+    ).set_env("LLAMA_ARG_N_CPU_MOE"));
+    add_opt(common_arg(
+        {"--cpu-moe-draft", "-cmoed"},
+        "keep all Mixture of Experts (MoE) weights in the CPU for the draft model",
+        [](common_params & params) {
+            params.speculative.tensor_buft_overrides.push_back({"\\.ffn_(up|down|gate)_exps", ggml_backend_cpu_buffer_type()});
+        }
+    ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_CPU_MOE_DRAFT"));
+    add_opt(common_arg(
+        {"--n-cpu-moe-draft", "-ncmoed"}, "N",
+        "keep the Mixture of Experts (MoE) weights of the first N layers in the CPU for the draft model",
+        [](common_params & params, int value) {
+            if (value < 0) {
+                throw std::invalid_argument("invalid value");
+            }
+            for (int i = 0; i < value; ++i) {
+                static std::list<std::string> buft_overrides_draft;
+                buft_overrides_draft.push_back(string_format("blk\\.%d\\.ffn_(up|down|gate)_exps", i));
+                params.speculative.tensor_buft_overrides.push_back({buft_overrides_draft.back().c_str(), ggml_backend_cpu_buffer_type()});
+            }
+        }
+    ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_N_CPU_MOE_DRAFT"));
     add_opt(common_arg(
         {"-ngl", "--gpu-layers", "--n-gpu-layers"}, "N",
         "number of layers to store in VRAM",
@@ -2639,7 +2697,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         [](common_params & params, const std::string & value) {
             params.out_file = value;
         }
-    ).set_examples({LLAMA_EXAMPLE_IMATRIX, LLAMA_EXAMPLE_CVECTOR_GENERATOR, LLAMA_EXAMPLE_EXPORT_LORA, LLAMA_EXAMPLE_TTS}));
+    ).set_examples({LLAMA_EXAMPLE_IMATRIX, LLAMA_EXAMPLE_CVECTOR_GENERATOR, LLAMA_EXAMPLE_EXPORT_LORA, LLAMA_EXAMPLE_TTS, LLAMA_EXAMPLE_FINETUNE}));
     add_opt(common_arg(
         {"-ofreq", "--output-frequency"}, "N",
         string_format("output the imatrix every N iterations (default: %d)", params.n_out_freq),
@@ -2649,10 +2707,10 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
     ).set_examples({LLAMA_EXAMPLE_IMATRIX}));
     add_opt(common_arg(
         {"--output-format"}, "{gguf,dat}",
-        string_format("output format for imatrix file (default: %s)", params.imat_dat ? "dat" : "gguf"),
+        string_format("output format for imatrix file (default: %s)", params.imat_dat > 0 ? "dat" : "gguf"),
         [](common_params & params, const std::string & value) {
-            /**/ if (value == "gguf") { params.imat_dat = false; }
-            else if (value == "dat")  { params.imat_dat = true;  }
+            /**/ if (value == "gguf") { params.imat_dat = -1; }
+            else if (value == "dat")  { params.imat_dat = 1;  }
             else { throw std::invalid_argument("invalid output format"); }
         }
     ).set_examples({LLAMA_EXAMPLE_IMATRIX}));
@@ -2931,12 +2989,9 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         "controls whether thought tags are allowed and/or extracted from the response, and in which format they're returned; one of:\n"
         "- none: leaves thoughts unparsed in `message.content`\n"
         "- deepseek: puts thoughts in `message.reasoning_content` (except in streaming mode, which behaves as `none`)\n"
-        "(default: deepseek)",
+        "(default: auto)",
         [](common_params & params, const std::string & value) {
-            /**/ if (value == "deepseek") { params.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK; }
-            else if (value == "deepseek-legacy") { params.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY; }
-            else if (value == "none") {     params.reasoning_format = COMMON_REASONING_FORMAT_NONE; }
-            else { throw std::invalid_argument("invalid value"); }
+            params.reasoning_format = common_reasoning_format_from_name(value);
         }
     ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_MAIN}).set_env("LLAMA_ARG_THINK"));
     add_opt(common_arg(
@@ -3117,7 +3172,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
                 params.speculative.cpuparams.n_threads = std::thread::hardware_concurrency();
             }
         }
-    ).set_examples({LLAMA_EXAMPLE_SPECULATIVE}));
+    ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER}));
     add_opt(common_arg(
         {"-tbd", "--threads-batch-draft"}, "N",
         "number of threads to use during batch and prompt processing (default: same as --threads-draft)",
@@ -3127,7 +3182,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
                 params.speculative.cpuparams_batch.n_threads = std::thread::hardware_concurrency();
             }
         }
-    ).set_examples({LLAMA_EXAMPLE_SPECULATIVE}));
+    ).set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER}));
     add_opt(common_arg(
         {"-Cd", "--cpu-mask-draft"}, "M",
         "Draft model CPU affinity mask. Complements cpu-range-draft (default: same as --cpu-mask)",
@@ -3520,5 +3575,51 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
     ).set_examples({ LLAMA_EXAMPLE_DIFFUSION }));
 
 
+    add_opt(
+        common_arg({ "-lr", "--learning-rate" }, "ALPHA",
+                   string_format(
+                       "adamw or sgd optimizer alpha (default: %.2g); note: sgd alpha recommended ~10x (no momentum)",
+                       (double) params.lr.lr0),
+                   [](common_params & params, const std::string & value) { params.lr.lr0 = std::stof(value); })
+            .set_examples({ LLAMA_EXAMPLE_FINETUNE }));
+    add_opt(
+        common_arg({ "-lr-min", "--learning-rate-min" }, "ALPHA",
+                   string_format(
+                       "(if >0) final learning rate after decay (if -decay-epochs is set, default=%.2g)",
+                       (double) params.lr.lr_min),
+                   [](common_params & params, const std::string & value) { params.lr.lr_min = std::stof(value); })
+            .set_examples({ LLAMA_EXAMPLE_FINETUNE }));
+    add_opt(
+        common_arg({ "-decay-epochs", "--learning-rate-decay-epochs" }, "ALPHA",
+                   string_format(
+                       "(if >0) decay learning rate to -lr-min after this many epochs (exponential decay, default=%.2g)",
+                       (double) params.lr.decay_epochs),
+                   [](common_params & params, const std::string & value) { params.lr.decay_epochs = std::stof(value); })
+            .set_examples({ LLAMA_EXAMPLE_FINETUNE }));
+    add_opt(common_arg(
+                { "-wd", "--weight-decay" }, "WD",
+                string_format(
+                    "adamw or sgd optimizer weight decay (0 is off; recommend very small e.g. 1e-9) (default: %.2g).",
+                    (double) params.lr.wd),
+                [](common_params & params, const std::string & value) { params.lr.wd = std::stof(value); })
+                .set_examples({ LLAMA_EXAMPLE_FINETUNE }));
+    add_opt(common_arg({ "-val-split", "--val-split" }, "FRACTION",
+                       string_format("fraction of data to use as validation set for training (default: %.2g).",
+                                     (double) params.val_split),
+                       [](common_params & params, const std::string & value) { params.val_split = std::stof(value); })
+                .set_examples({ LLAMA_EXAMPLE_FINETUNE }));
+    add_opt(common_arg({ "-epochs", "--epochs" }, "N",
+                       string_format("optimizer max # of epochs (default: %d)", params.lr.epochs),
+                       [](common_params & params, int epochs) { params.lr.epochs = epochs; })
+                .set_examples({ LLAMA_EXAMPLE_FINETUNE }));
+    add_opt(common_arg({ "-opt", "--optimizer" }, "sgd|adamw", "adamw or sgd",
+                       [](common_params & params, const std::string & name) {
+                           params.optimizer = common_opt_get_optimizer(name.c_str());
+                           if (params.optimizer == GGML_OPT_OPTIMIZER_TYPE_COUNT) {
+                               throw std::invalid_argument("invalid --optimizer, valid options: adamw, sgd");
+                           }
+                       })
+                .set_examples({ LLAMA_EXAMPLE_FINETUNE }));
+
     return ctx_arg;
 }

package/src/llama.cpp/common/chat-parser.cpp

@@ -55,7 +55,15 @@ bool common_chat_msg_parser::add_tool_call(const std::string & name, const std::
 bool common_chat_msg_parser::add_tool_call(const json & tool_call) {
     std::string name = tool_call.contains("name") ? tool_call.at("name") : "";
     std::string id = tool_call.contains("id") ? tool_call.at("id") : "";
-    std::string arguments = tool_call.contains("arguments") ? tool_call.at("arguments") : "";
+    std::string arguments = "";
+    if (tool_call.contains("arguments")) {
+        if (tool_call.at("arguments").is_object()) {
+            arguments = tool_call.at("arguments").dump();
+        } else {
+            arguments = tool_call.at("arguments");
+        }
+    }
+
     return add_tool_call(name, id, arguments);
 }