@fugood/llama.node 1.4.0 → 1.4.2

package/lib/binding.js

@@ -15,23 +15,13 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
 }) : function(o, v) {
     o["default"] = v;
 });
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "1.4.0",
+  "version": "1.4.2",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {
@@ -72,20 +72,20 @@
     "CMakeLists.txt"
   ],
   "optionalDependencies": {
-    "@fugood/node-llama-linux-x64": "1.4.0",
-    "@fugood/node-llama-linux-x64-vulkan": "1.4.0",
-    "@fugood/node-llama-linux-x64-cuda": "1.4.0",
-    "@fugood/node-llama-linux-arm64-snapdragon": "1.4.0",
-    "@fugood/node-llama-linux-arm64": "1.4.0",
-    "@fugood/node-llama-linux-arm64-vulkan": "1.4.0",
-    "@fugood/node-llama-linux-arm64-cuda": "1.4.0",
-    "@fugood/node-llama-win32-x64": "1.4.0",
-    "@fugood/node-llama-win32-x64-vulkan": "1.4.0",
-    "@fugood/node-llama-win32-x64-cuda": "1.4.0",
-    "@fugood/node-llama-win32-arm64": "1.4.0",
-    "@fugood/node-llama-win32-arm64-vulkan": "1.4.0",
-    "@fugood/node-llama-darwin-x64": "1.4.0",
-    "@fugood/node-llama-darwin-arm64": "1.4.0"
+    "@fugood/node-llama-darwin-arm64": "1.4.2",
+    "@fugood/node-llama-darwin-x64": "1.4.2",
+    "@fugood/node-llama-linux-arm64": "1.4.2",
+    "@fugood/node-llama-linux-arm64-cuda": "1.4.2",
+    "@fugood/node-llama-linux-arm64-snapdragon": "1.4.2",
+    "@fugood/node-llama-linux-arm64-vulkan": "1.4.2",
+    "@fugood/node-llama-linux-x64": "1.4.2",
+    "@fugood/node-llama-linux-x64-cuda": "1.4.2",
+    "@fugood/node-llama-linux-x64-vulkan": "1.4.2",
+    "@fugood/node-llama-win32-arm64": "1.4.2",
+    "@fugood/node-llama-win32-arm64-vulkan": "1.4.2",
+    "@fugood/node-llama-win32-x64": "1.4.2",
+    "@fugood/node-llama-win32-x64-cuda": "1.4.2",
+    "@fugood/node-llama-win32-x64-vulkan": "1.4.2"
   },
   "devDependencies": {
     "@babel/preset-env": "^7.24.4",
@@ -100,7 +100,7 @@
     "jest": "^29.7.0",
     "node-addon-api": "^8.0.0",
     "node-wav": "^0.0.2",
-    "release-it": "^17.7.0",
+    "release-it": "^19.0.6",
     "rimraf": "^6.0.1",
     "typescript": "^5.4.5",
     "wait-for-expect": "^3.0.2"

package/scripts/llama.cpp.patch

@@ -122,10 +122,59 @@ index 7e53a57b7..a328d4db4 100644
              check_cxx_compiler_flag(-mfp16-format=ieee GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E)
              if (NOT "${GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E}" STREQUAL "")
 diff --git a/src/llama.cpp/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/src/llama.cpp/ggml/src/ggml-hexagon/ggml-hexagon.cpp
-index 72a82a891..7869ad323 100644
+index 72a82a891..1b681f4dd 100644
 --- a/src/llama.cpp/ggml/src/ggml-hexagon/ggml-hexagon.cpp
 +++ b/src/llama.cpp/ggml/src/ggml-hexagon/ggml-hexagon.cpp
-@@ -3417,6 +3417,8 @@ ggml_hexagon_registry::ggml_hexagon_registry(ggml_backend_reg_t reg) {
+@@ -3216,11 +3216,26 @@ static const char * ggml_backend_hexagon_device_get_description(ggml_backend_dev
+     GGML_UNUSED(dev);
+ }
+ 
++
++// ~2GB per session for now
++#define GGML_HEXAGON_SESSION_MEMORY_DEFAULT (2ULL * 1024 * 1024 * 1024)
++// Max to 3.5GB
++#define GGML_HEXAGON_SESSION_MEMORY_MAX (3ULL * 1024 * 1024 * 1024  +  512ULL * 1024 * 1024)
++
+ static void ggml_backend_hexagon_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {
+-    // ~2GB per session for now
+-    *free  = 2ULL * 1024 * 1024 * 1024;
+-    *total = *free;
++    const char * str_mem = getenv("GGML_HEXAGON_SESSION_MEMORY");
++    if (str_mem) {
++        *free = std::stoull(str_mem);
++        if (*free < GGML_HEXAGON_SESSION_MEMORY_DEFAULT) {
++            *free = GGML_HEXAGON_SESSION_MEMORY_DEFAULT;
++        } else if (*free > GGML_HEXAGON_SESSION_MEMORY_MAX) {
++            *free = GGML_HEXAGON_SESSION_MEMORY_MAX;
++        }
++    } else {
++        *free = GGML_HEXAGON_SESSION_MEMORY_DEFAULT;
++    }
+ 
++    *total = *free;
+     GGML_UNUSED(dev);
+ }
+ 
+@@ -3401,10 +3416,17 @@ ggml_hexagon_registry::ggml_hexagon_registry(ggml_backend_reg_t reg) {
+         }
+     }
+ 
++#if defined(__ANDROID__)
+     if(opt_arch < 75) {
+         opt_ndev = 1;
+-        GGML_LOG_WARN("ggml-hex: forcing ndev to 1 for SoCs archs lower than v75.\n");
++        GGML_LOG_WARN("ggml-hex: forcing ndev to 1 for SoCs archs lower than v75 for Android.\n");
++    }
++#else
++    if(opt_arch < 73) {
++        opt_ndev = 1;
++        GGML_LOG_WARN("ggml-hex: forcing ndev to 1 for SoCs archs lower than v73 for Linux and Windows.\n");
+     }
++#endif
+ 
+     GGML_LOG_INFO("ggml-hex: Hexagon Arch version v%d\n", opt_arch);
+ 
+@@ -3417,6 +3439,8 @@ ggml_hexagon_registry::ggml_hexagon_registry(ggml_backend_reg_t reg) {
          } catch (std::exception const &exc) {
              GGML_LOG_ERROR("ggml-hex: failed to create device/session %zu\n", i);
              devices[i].context = nullptr;

package/src/LlamaContext.cpp

@@ -321,15 +321,20 @@ LlamaContext::LlamaContext(const Napi::CallbackInfo &info)
 
   // Parse devices array
   if (options.Has("devices") && options.Get("devices").IsArray()) {
+    std::vector<ggml_backend_dev_t> devs;
     auto devices_array = options.Get("devices").As<Napi::Array>();
     for (size_t i = 0; i < devices_array.Length(); i++) {
       auto device_name = devices_array.Get(i).ToString().Utf8Value();
       auto * dev = ggml_backend_dev_by_name(device_name.c_str());
       if (dev) {
-        params.devices.push_back(dev);
+        devs.push_back(dev);
       }
       // Skip invalid device names silently
     }
+    if (!devs.empty()) {
+      params.devices = devs;
+      params.devices.push_back(nullptr); // nullptr terminator required by llama.cpp
+    }
   }
 
   std::vector<common_adapter_lora_info> lora;
@@ -652,7 +657,7 @@ Napi::Value LlamaContext::GetFormattedChat(const Napi::CallbackInfo &info) {
   auto params =
       has_params ? info[2].As<Napi::Object>() : Napi::Object::New(env);
 
-  if (get_option<bool>(params, "jinja", false)) {
+  if (get_option<bool>(params, "jinja", true)) {
     std::string json_schema_str = "";
     if (!is_nil(params.Get("response_format"))) {
       auto response_format = params.Get("response_format").As<Napi::Object>();
@@ -907,7 +912,7 @@ Napi::Value LlamaContext::Completion(const Napi::CallbackInfo &info) {
   if (options.Has("messages") && options.Get("messages").IsArray()) {
     auto messages = options.Get("messages").As<Napi::Array>();
     auto chat_template = get_option<std::string>(options, "chat_template", "");
-    auto jinja = get_option<bool>(options, "jinja", false);
+    auto jinja = get_option<bool>(options, "jinja", true);
     if (jinja) {
       auto tools_str =
           !is_nil(options.Get("tools"))