@fugood/llama.node 1.3.8 → 1.4.0

package/lib/binding.js

@@ -41,8 +41,12 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.isLibVariantAvailable = exports.loadModule = void 0;
+const path_1 = __importDefault(require("path"));
 const getPlatformPackageName = (variant) => {
     const platform = process.platform;
     const arch = process.arch;
@@ -58,7 +62,20 @@ const loadPlatformPackage = (packageName) => __awaiter(void 0, void 0, void 0, f
     }
 });
 const loadModule = (variant) => __awaiter(void 0, void 0, void 0, function* () {
-    let module = yield loadPlatformPackage(getPlatformPackageName(variant));
+    const packageName = getPlatformPackageName(variant);
+    // Set ADSP_LIBRARY_PATH for load HTP libs
+    if (variant === 'snapdragon') {
+        const adspLibraryPath = process.env.ADSP_LIBRARY_PATH;
+        if (!adspLibraryPath) {
+            try {
+                process.env.ADSP_LIBRARY_PATH = path_1.default.dirname(require.resolve(packageName));
+            }
+            catch (_a) {
+                /* no-op */
+            }
+        }
+    }
+    let module = yield loadPlatformPackage(packageName);
     if (module) {
         return module;
     }

package/lib/binding.ts

@@ -1,3 +1,5 @@
+import path from 'path'
+
 export type MessagePart = {
   type: string
   text?: string
@@ -600,7 +602,23 @@ const loadPlatformPackage = async (
 }
 
 export const loadModule = async (variant?: LibVariant): Promise<Module> => {
-  let module = await loadPlatformPackage(getPlatformPackageName(variant))
+  const packageName = getPlatformPackageName(variant)
+
+  // Set ADSP_LIBRARY_PATH for load HTP libs
+  if (variant === 'snapdragon') {
+    const adspLibraryPath = process.env.ADSP_LIBRARY_PATH
+    if (!adspLibraryPath) {
+      try {
+        process.env.ADSP_LIBRARY_PATH = path.dirname(
+          require.resolve(packageName),
+        )
+      } catch {
+        /* no-op */
+      }
+    }
+  }
+
+  let module = await loadPlatformPackage(packageName)
   if (module) {
     return module
   }

package/lib/index.js

@@ -87,9 +87,9 @@ class LlamaContextWrapper {
         return !!this.ctx.getModelInfo().chatTemplates.llamaChat;
     }
     getFormattedChat(messages, template, params) {
-        var _a;
+        var _a, _b;
         const { messages: chat, has_media, media_paths } = (0, utils_1.formatMediaChat)(messages);
-        const useJinja = this.isJinjaSupported() && (params === null || params === void 0 ? void 0 : params.jinja);
+        const useJinja = this.isJinjaSupported() && ((_a = params === null || params === void 0 ? void 0 : params.jinja) !== null && _a !== void 0 ? _a : true);
         let tmpl;
         if (template)
             tmpl = template; // Force replace if provided
@@ -99,7 +99,7 @@ class LlamaContextWrapper {
             tools: params === null || params === void 0 ? void 0 : params.tools,
             parallel_tool_calls: params === null || params === void 0 ? void 0 : params.parallel_tool_calls,
             tool_choice: params === null || params === void 0 ? void 0 : params.tool_choice,
-            enable_thinking: (_a = params === null || params === void 0 ? void 0 : params.enable_thinking) !== null && _a !== void 0 ? _a : true,
+            enable_thinking: (_b = params === null || params === void 0 ? void 0 : params.enable_thinking) !== null && _b !== void 0 ? _b : true,
             add_generation_prompt: params === null || params === void 0 ? void 0 : params.add_generation_prompt,
             now: params === null || params === void 0 ? void 0 : params.now,
             chat_template_kwargs: (params === null || params === void 0 ? void 0 : params.chat_template_kwargs)

package/lib/index.ts

@@ -124,7 +124,7 @@ class LlamaContextWrapper {
   ): FormattedChatResult {
     const { messages: chat, has_media, media_paths } = formatMediaChat(messages)
 
-    const useJinja = this.isJinjaSupported() && params?.jinja
+    const useJinja = this.isJinjaSupported() && (params?.jinja ?? true)
     let tmpl
     if (template) tmpl = template // Force replace if provided
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "1.3.8",
+  "version": "1.4.0",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {
@@ -72,20 +72,20 @@
     "CMakeLists.txt"
   ],
   "optionalDependencies": {
-    "@fugood/node-llama-linux-x64": "1.3.8",
-    "@fugood/node-llama-linux-x64-vulkan": "1.3.8",
-    "@fugood/node-llama-linux-x64-cuda": "1.3.8",
-    "@fugood/node-llama-linux-arm64-snapdragon": "1.3.8",
-    "@fugood/node-llama-linux-arm64": "1.3.8",
-    "@fugood/node-llama-linux-arm64-vulkan": "1.3.8",
-    "@fugood/node-llama-linux-arm64-cuda": "1.3.8",
-    "@fugood/node-llama-win32-x64": "1.3.8",
-    "@fugood/node-llama-win32-x64-vulkan": "1.3.8",
-    "@fugood/node-llama-win32-x64-cuda": "1.3.8",
-    "@fugood/node-llama-win32-arm64": "1.3.8",
-    "@fugood/node-llama-win32-arm64-vulkan": "1.3.8",
-    "@fugood/node-llama-darwin-x64": "1.3.8",
-    "@fugood/node-llama-darwin-arm64": "1.3.8"
+    "@fugood/node-llama-linux-x64": "1.4.0",
+    "@fugood/node-llama-linux-x64-vulkan": "1.4.0",
+    "@fugood/node-llama-linux-x64-cuda": "1.4.0",
+    "@fugood/node-llama-linux-arm64-snapdragon": "1.4.0",
+    "@fugood/node-llama-linux-arm64": "1.4.0",
+    "@fugood/node-llama-linux-arm64-vulkan": "1.4.0",
+    "@fugood/node-llama-linux-arm64-cuda": "1.4.0",
+    "@fugood/node-llama-win32-x64": "1.4.0",
+    "@fugood/node-llama-win32-x64-vulkan": "1.4.0",
+    "@fugood/node-llama-win32-x64-cuda": "1.4.0",
+    "@fugood/node-llama-win32-arm64": "1.4.0",
+    "@fugood/node-llama-win32-arm64-vulkan": "1.4.0",
+    "@fugood/node-llama-darwin-x64": "1.4.0",
+    "@fugood/node-llama-darwin-arm64": "1.4.0"
   },
   "devDependencies": {
     "@babel/preset-env": "^7.24.4",

package/scripts/llama.cpp.patch

@@ -21,7 +21,7 @@ index bb168e835..cfc0e2c2e 100644
  
  #
 diff --git a/src/llama.cpp/common/chat.cpp b/src/llama.cpp/common/chat.cpp
-index 6fa05a604..87dfa7a8b 100644
+index b4a0f985e..2383d2ea9 100644
 --- a/src/llama.cpp/common/chat.cpp
 +++ b/src/llama.cpp/common/chat.cpp
 @@ -6,9 +6,6 @@
@@ -51,7 +51,7 @@ index 6fa05a604..87dfa7a8b 100644
  struct templates_params {
      json messages;
      json tools;
-@@ -817,7 +804,7 @@ static std::string apply(
+@@ -709,7 +696,7 @@ static std::string apply(
          tmpl_inputs.extra_context.merge_patch(*additional_context);
      }
      // TODO: add flag to control date/time, if only for testing purposes.

package/src/LlamaCompletionWorker.cpp

@@ -10,14 +10,14 @@ Napi::Array TokenProbsToArray(Napi::Env env, llama_context* ctx, const std::vect
     const auto &prob = probs[i];
     Napi::Object token_obj = Napi::Object::New(env);
 
-    std::string token_str = common_token_to_piece(ctx, prob.tok);
+    std::string token_str = rnllama::tokens_to_output_formatted_string(ctx, prob.tok);
     token_obj.Set("content", Napi::String::New(env, token_str));
 
     Napi::Array token_probs = Napi::Array::New(env);
     for (size_t j = 0; j < prob.probs.size(); j++) {
       const auto &p = prob.probs[j];
       Napi::Object prob_obj = Napi::Object::New(env);
-      std::string tok_str = common_token_to_piece(ctx, p.tok);
+      std::string tok_str = rnllama::tokens_to_output_formatted_string(ctx, p.tok);
       prob_obj.Set("tok_str", Napi::String::New(env, tok_str));
       prob_obj.Set("prob", Napi::Number::New(env, p.prob));
       token_probs.Set(j, prob_obj);

package/src/llama.cpp/common/arg.cpp

@@ -980,7 +980,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         [](common_params & params) {
             params.kv_unified = true;
         }
-    ).set_env("LLAMA_ARG_KV_SPLIT"));
+    ).set_env("LLAMA_ARG_KV_UNIFIED"));
     add_opt(common_arg(
         {"--no-context-shift"},
         string_format("disables context shift on infinite text generation (default: %s)", params.ctx_shift ? "disabled" : "enabled"),