@fugood/llama.node 1.3.7 → 1.4.0

package/lib/binding.js

@@ -41,8 +41,12 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.isLibVariantAvailable = exports.loadModule = void 0;
+const path_1 = __importDefault(require("path"));
 const getPlatformPackageName = (variant) => {
     const platform = process.platform;
     const arch = process.arch;
@@ -58,7 +62,20 @@ const loadPlatformPackage = (packageName) => __awaiter(void 0, void 0, void 0, f
     }
 });
 const loadModule = (variant) => __awaiter(void 0, void 0, void 0, function* () {
-    let module = yield loadPlatformPackage(getPlatformPackageName(variant));
+    const packageName = getPlatformPackageName(variant);
+    // Set ADSP_LIBRARY_PATH for load HTP libs
+    if (variant === 'snapdragon') {
+        const adspLibraryPath = process.env.ADSP_LIBRARY_PATH;
+        if (!adspLibraryPath) {
+            try {
+                process.env.ADSP_LIBRARY_PATH = path_1.default.dirname(require.resolve(packageName));
+            }
+            catch (_a) {
+                /* no-op */
+            }
+        }
+    }
+    let module = yield loadPlatformPackage(packageName);
     if (module) {
         return module;
     }

package/lib/binding.ts

@@ -1,3 +1,5 @@
+import path from 'path'
+
 export type MessagePart = {
   type: string
   text?: string
@@ -600,7 +602,23 @@ const loadPlatformPackage = async (
 }
 
 export const loadModule = async (variant?: LibVariant): Promise<Module> => {
-  let module = await loadPlatformPackage(getPlatformPackageName(variant))
+  const packageName = getPlatformPackageName(variant)
+
+  // Set ADSP_LIBRARY_PATH for load HTP libs
+  if (variant === 'snapdragon') {
+    const adspLibraryPath = process.env.ADSP_LIBRARY_PATH
+    if (!adspLibraryPath) {
+      try {
+        process.env.ADSP_LIBRARY_PATH = path.dirname(
+          require.resolve(packageName),
+        )
+      } catch {
+        /* no-op */
+      }
+    }
+  }
+
+  let module = await loadPlatformPackage(packageName)
   if (module) {
     return module
   }

package/lib/index.js

@@ -87,9 +87,9 @@ class LlamaContextWrapper {
         return !!this.ctx.getModelInfo().chatTemplates.llamaChat;
     }
     getFormattedChat(messages, template, params) {
-        var _a;
+        var _a, _b;
         const { messages: chat, has_media, media_paths } = (0, utils_1.formatMediaChat)(messages);
-        const useJinja = this.isJinjaSupported() && (params === null || params === void 0 ? void 0 : params.jinja);
+        const useJinja = this.isJinjaSupported() && ((_a = params === null || params === void 0 ? void 0 : params.jinja) !== null && _a !== void 0 ? _a : true);
         let tmpl;
         if (template)
             tmpl = template; // Force replace if provided
@@ -99,7 +99,7 @@ class LlamaContextWrapper {
             tools: params === null || params === void 0 ? void 0 : params.tools,
             parallel_tool_calls: params === null || params === void 0 ? void 0 : params.parallel_tool_calls,
             tool_choice: params === null || params === void 0 ? void 0 : params.tool_choice,
-            enable_thinking: (_a = params === null || params === void 0 ? void 0 : params.enable_thinking) !== null && _a !== void 0 ? _a : true,
+            enable_thinking: (_b = params === null || params === void 0 ? void 0 : params.enable_thinking) !== null && _b !== void 0 ? _b : true,
             add_generation_prompt: params === null || params === void 0 ? void 0 : params.add_generation_prompt,
             now: params === null || params === void 0 ? void 0 : params.now,
             chat_template_kwargs: (params === null || params === void 0 ? void 0 : params.chat_template_kwargs)

package/lib/index.ts

@@ -124,7 +124,7 @@ class LlamaContextWrapper {
   ): FormattedChatResult {
     const { messages: chat, has_media, media_paths } = formatMediaChat(messages)
 
-    const useJinja = this.isJinjaSupported() && params?.jinja
+    const useJinja = this.isJinjaSupported() && (params?.jinja ?? true)
     let tmpl
     if (template) tmpl = template // Force replace if provided
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "1.3.7",
+  "version": "1.4.0",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {
@@ -72,20 +72,20 @@
     "CMakeLists.txt"
   ],
   "optionalDependencies": {
-    "@fugood/node-llama-linux-x64": "1.3.7",
-    "@fugood/node-llama-linux-x64-vulkan": "1.3.7",
-    "@fugood/node-llama-linux-x64-cuda": "1.3.7",
-    "@fugood/node-llama-linux-arm64-snapdragon": "1.3.7",
-    "@fugood/node-llama-linux-arm64": "1.3.7",
-    "@fugood/node-llama-linux-arm64-vulkan": "1.3.7",
-    "@fugood/node-llama-linux-arm64-cuda": "1.3.7",
-    "@fugood/node-llama-win32-x64": "1.3.7",
-    "@fugood/node-llama-win32-x64-vulkan": "1.3.7",
-    "@fugood/node-llama-win32-x64-cuda": "1.3.7",
-    "@fugood/node-llama-win32-arm64": "1.3.7",
-    "@fugood/node-llama-win32-arm64-vulkan": "1.3.7",
-    "@fugood/node-llama-darwin-x64": "1.3.7",
-    "@fugood/node-llama-darwin-arm64": "1.3.7"
+    "@fugood/node-llama-linux-x64": "1.4.0",
+    "@fugood/node-llama-linux-x64-vulkan": "1.4.0",
+    "@fugood/node-llama-linux-x64-cuda": "1.4.0",
+    "@fugood/node-llama-linux-arm64-snapdragon": "1.4.0",
+    "@fugood/node-llama-linux-arm64": "1.4.0",
+    "@fugood/node-llama-linux-arm64-vulkan": "1.4.0",
+    "@fugood/node-llama-linux-arm64-cuda": "1.4.0",
+    "@fugood/node-llama-win32-x64": "1.4.0",
+    "@fugood/node-llama-win32-x64-vulkan": "1.4.0",
+    "@fugood/node-llama-win32-x64-cuda": "1.4.0",
+    "@fugood/node-llama-win32-arm64": "1.4.0",
+    "@fugood/node-llama-win32-arm64-vulkan": "1.4.0",
+    "@fugood/node-llama-darwin-x64": "1.4.0",
+    "@fugood/node-llama-darwin-arm64": "1.4.0"
   },
   "devDependencies": {
     "@babel/preset-env": "^7.24.4",

package/scripts/llama.cpp.patch

@@ -21,7 +21,7 @@ index bb168e835..cfc0e2c2e 100644
  
  #
 diff --git a/src/llama.cpp/common/chat.cpp b/src/llama.cpp/common/chat.cpp
-index 6fa05a604..87dfa7a8b 100644
+index b4a0f985e..2383d2ea9 100644
 --- a/src/llama.cpp/common/chat.cpp
 +++ b/src/llama.cpp/common/chat.cpp
 @@ -6,9 +6,6 @@
@@ -51,7 +51,7 @@ index 6fa05a604..87dfa7a8b 100644
  struct templates_params {
      json messages;
      json tools;
-@@ -817,7 +804,7 @@ static std::string apply(
+@@ -709,7 +696,7 @@ static std::string apply(
          tmpl_inputs.extra_context.merge_patch(*additional_context);
      }
      // TODO: add flag to control date/time, if only for testing purposes.
@@ -85,10 +85,10 @@ index 754c411e2..71241a6cc 100644
  struct common_chat_tool_call {
      std::string name;
 diff --git a/src/llama.cpp/common/common.cpp b/src/llama.cpp/common/common.cpp
-index f3cc55247..65398844f 100644
+index 0d7fd9a93..6bf3cc7ab 100644
 --- a/src/llama.cpp/common/common.cpp
 +++ b/src/llama.cpp/common/common.cpp
-@@ -1162,6 +1162,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
+@@ -1217,6 +1217,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
          mparams.n_gpu_layers = params.n_gpu_layers;
      }
  
@@ -97,10 +97,10 @@ index f3cc55247..65398844f 100644
      mparams.split_mode      = params.split_mode;
      mparams.tensor_split    = params.tensor_split;
 diff --git a/src/llama.cpp/common/common.h b/src/llama.cpp/common/common.h
-index de5b404dd..d30d252c9 100644
+index 2f23d0baa..e4e6c795e 100644
 --- a/src/llama.cpp/common/common.h
 +++ b/src/llama.cpp/common/common.h
-@@ -281,6 +281,7 @@ struct lr_opt {
+@@ -299,6 +299,7 @@ struct lr_opt {
  struct ggml_opt_optimizer_params common_opt_lr_pars(void * userdata);
  
  struct common_params {
@@ -109,7 +109,7 @@ index de5b404dd..d30d252c9 100644
      int32_t n_ctx                 =  4096; // context size
      int32_t n_batch               =  2048; // logical batch size for prompt processing (must be >=32 to use BLAS)
 diff --git a/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt b/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt
-index d0cab0bcb..48d532838 100644
+index 7e53a57b7..a328d4db4 100644
 --- a/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt
 +++ b/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt
 @@ -106,7 +106,7 @@ function(ggml_add_cpu_backend_variant_impl tag_name)

package/src/LlamaCompletionWorker.cpp

@@ -10,14 +10,14 @@ Napi::Array TokenProbsToArray(Napi::Env env, llama_context* ctx, const std::vect
     const auto &prob = probs[i];
     Napi::Object token_obj = Napi::Object::New(env);
 
-    std::string token_str = common_token_to_piece(ctx, prob.tok);
+    std::string token_str = rnllama::tokens_to_output_formatted_string(ctx, prob.tok);
     token_obj.Set("content", Napi::String::New(env, token_str));
 
     Napi::Array token_probs = Napi::Array::New(env);
     for (size_t j = 0; j < prob.probs.size(); j++) {
       const auto &p = prob.probs[j];
       Napi::Object prob_obj = Napi::Object::New(env);
-      std::string tok_str = common_token_to_piece(ctx, p.tok);
+      std::string tok_str = rnllama::tokens_to_output_formatted_string(ctx, p.tok);
       prob_obj.Set("tok_str", Napi::String::New(env, tok_str));
       prob_obj.Set("prob", Napi::Number::New(env, p.prob));
       token_probs.Set(j, prob_obj);

package/src/llama.cpp/common/arg.cpp

@@ -694,6 +694,12 @@ static bool is_autoy(const std::string & value) {
 }
 
 common_params_context common_params_parser_init(common_params & params, llama_example ex, void(*print_usage)(int, char **)) {
+    // default values specific to example
+    // note: we place it here instead of inside server.cpp to allow llama-gen-docs to pick it up
+    if (ex == LLAMA_EXAMPLE_SERVER) {
+        params.use_jinja = true;
+    }
+
     // load dynamic backends
     ggml_backend_load_all();
 
@@ -974,7 +980,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         [](common_params & params) {
             params.kv_unified = true;
         }
-    ).set_env("LLAMA_ARG_KV_SPLIT"));
+    ).set_env("LLAMA_ARG_KV_UNIFIED"));
     add_opt(common_arg(
         {"--no-context-shift"},
         string_format("disables context shift on infinite text generation (default: %s)", params.ctx_shift ? "disabled" : "enabled"),
@@ -1232,6 +1238,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         [](common_params & params, const std::string & value) {
             const auto sampler_names = string_split<std::string>(value, ';');
             params.sampling.samplers = common_sampler_types_from_names(sampler_names, true);
+            params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_SAMPLERS;
         }
     ).set_sparam());
     add_opt(common_arg(
@@ -1261,6 +1268,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         [](common_params & params, const std::string & value) {
             params.sampling.temp = std::stof(value);
             params.sampling.temp = std::max(params.sampling.temp, 0.0f);
+            params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_TEMP;
         }
     ).set_sparam());
     add_opt(common_arg(
@@ -1268,6 +1276,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         string_format("top-k sampling (default: %d, 0 = disabled)", params.sampling.top_k),
         [](common_params & params, int value) {
             params.sampling.top_k = value;
+            params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_TOP_K;
         }
     ).set_sparam());
     add_opt(common_arg(
@@ -1275,6 +1284,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         string_format("top-p sampling (default: %.1f, 1.0 = disabled)", (double)params.sampling.top_p),
         [](common_params & params, const std::string & value) {
             params.sampling.top_p = std::stof(value);
+            params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_TOP_P;
         }
     ).set_sparam());
     add_opt(common_arg(
@@ -1282,6 +1292,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         string_format("min-p sampling (default: %.1f, 0.0 = disabled)", (double)params.sampling.min_p),
         [](common_params & params, const std::string & value) {
             params.sampling.min_p = std::stof(value);
+            params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_MIN_P;
         }
     ).set_sparam());
     add_opt(common_arg(
@@ -1296,6 +1307,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         string_format("xtc probability (default: %.1f, 0.0 = disabled)", (double)params.sampling.xtc_probability),
         [](common_params & params, const std::string & value) {
             params.sampling.xtc_probability = std::stof(value);
+            params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_XTC_PROBABILITY;
         }
     ).set_sparam());
     add_opt(common_arg(
@@ -1303,6 +1315,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         string_format("xtc threshold (default: %.1f, 1.0 = disabled)", (double)params.sampling.xtc_threshold),
         [](common_params & params, const std::string & value) {
             params.sampling.xtc_threshold = std::stof(value);
+            params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_XTC_THRESHOLD;
         }
     ).set_sparam());
     add_opt(common_arg(
@@ -1321,6 +1334,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             }
             params.sampling.penalty_last_n = value;
             params.sampling.n_prev = std::max(params.sampling.n_prev, params.sampling.penalty_last_n);
+            params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_PENALTY_LAST_N;
         }
     ).set_sparam());
     add_opt(common_arg(
@@ -1328,6 +1342,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         string_format("penalize repeat sequence of tokens (default: %.1f, 1.0 = disabled)", (double)params.sampling.penalty_repeat),
         [](common_params & params, const std::string & value) {
             params.sampling.penalty_repeat = std::stof(value);
+            params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_PENALTY_REPEAT;
         }
     ).set_sparam());
     add_opt(common_arg(
@@ -1425,6 +1440,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         "(default: %d, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0)", params.sampling.mirostat),
         [](common_params & params, int value) {
             params.sampling.mirostat = value;
+            params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_MIROSTAT;
         }
     ).set_sparam());
     add_opt(common_arg(
@@ -1432,6 +1448,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         string_format("Mirostat learning rate, parameter eta (default: %.1f)", (double)params.sampling.mirostat_eta),
         [](common_params & params, const std::string & value) {
             params.sampling.mirostat_eta = std::stof(value);
+            params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_MIROSTAT_ETA;
         }
     ).set_sparam());
     add_opt(common_arg(
@@ -1439,6 +1456,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         string_format("Mirostat target entropy, parameter tau (default: %.1f)", (double)params.sampling.mirostat_tau),
         [](common_params & params, const std::string & value) {
             params.sampling.mirostat_tau = std::stof(value);
+            params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_MIROSTAT_TAU;
         }
     ).set_sparam());
     add_opt(common_arg(
@@ -2476,11 +2494,18 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
     ).set_examples({LLAMA_EXAMPLE_SERVER}));
     add_opt(common_arg(
         {"--jinja"},
-        "use jinja template for chat (default: disabled)",
+        string_format("use jinja template for chat (default: %s)\n", params.use_jinja ? "enabled" : "disabled"),
         [](common_params & params) {
             params.use_jinja = true;
         }
     ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_MAIN, LLAMA_EXAMPLE_MTMD}).set_env("LLAMA_ARG_JINJA"));
+    add_opt(common_arg(
+        {"--no-jinja"},
+        string_format("disable jinja template for chat (default: %s)\n", params.use_jinja ? "enabled" : "disabled"),
+        [](common_params & params) {
+            params.use_jinja = false;
+        }
+    ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_MAIN, LLAMA_EXAMPLE_MTMD}).set_env("LLAMA_ARG_NO_JINJA"));
     add_opt(common_arg(
         {"--reasoning-format"}, "FORMAT",
         "controls whether thought tags are allowed and/or extracted from the response, and in which format they're returned; one of:\n"