@fugood/llama.node 1.4.0 → 1.4.1

package/lib/binding.js

@@ -15,23 +15,13 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
 }) : function(o, v) {
     o["default"] = v;
 });
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "1.4.0",
+  "version": "1.4.1",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {
@@ -72,20 +72,20 @@
     "CMakeLists.txt"
   ],
   "optionalDependencies": {
-    "@fugood/node-llama-linux-x64": "1.4.0",
-    "@fugood/node-llama-linux-x64-vulkan": "1.4.0",
-    "@fugood/node-llama-linux-x64-cuda": "1.4.0",
-    "@fugood/node-llama-linux-arm64-snapdragon": "1.4.0",
-    "@fugood/node-llama-linux-arm64": "1.4.0",
-    "@fugood/node-llama-linux-arm64-vulkan": "1.4.0",
-    "@fugood/node-llama-linux-arm64-cuda": "1.4.0",
-    "@fugood/node-llama-win32-x64": "1.4.0",
-    "@fugood/node-llama-win32-x64-vulkan": "1.4.0",
-    "@fugood/node-llama-win32-x64-cuda": "1.4.0",
-    "@fugood/node-llama-win32-arm64": "1.4.0",
-    "@fugood/node-llama-win32-arm64-vulkan": "1.4.0",
-    "@fugood/node-llama-darwin-x64": "1.4.0",
-    "@fugood/node-llama-darwin-arm64": "1.4.0"
+    "@fugood/node-llama-darwin-arm64": "1.4.1",
+    "@fugood/node-llama-darwin-x64": "1.4.1",
+    "@fugood/node-llama-linux-arm64": "1.4.1",
+    "@fugood/node-llama-linux-arm64-cuda": "1.4.1",
+    "@fugood/node-llama-linux-arm64-snapdragon": "1.4.1",
+    "@fugood/node-llama-linux-arm64-vulkan": "1.4.1",
+    "@fugood/node-llama-linux-x64": "1.4.1",
+    "@fugood/node-llama-linux-x64-cuda": "1.4.1",
+    "@fugood/node-llama-linux-x64-vulkan": "1.4.1",
+    "@fugood/node-llama-win32-arm64": "1.4.1",
+    "@fugood/node-llama-win32-arm64-vulkan": "1.4.1",
+    "@fugood/node-llama-win32-x64": "1.4.1",
+    "@fugood/node-llama-win32-x64-cuda": "1.4.1",
+    "@fugood/node-llama-win32-x64-vulkan": "1.4.1"
   },
   "devDependencies": {
     "@babel/preset-env": "^7.24.4",
@@ -100,7 +100,7 @@
     "jest": "^29.7.0",
     "node-addon-api": "^8.0.0",
     "node-wav": "^0.0.2",
-    "release-it": "^17.7.0",
+    "release-it": "^19.0.6",
     "rimraf": "^6.0.1",
     "typescript": "^5.4.5",
     "wait-for-expect": "^3.0.2"
@@ -130,4 +130,4 @@
     "singleQuote": true,
     "printWidth": 80
   }
-}
+}

package/scripts/llama.cpp.patch

@@ -122,10 +122,59 @@ index 7e53a57b7..a328d4db4 100644
              check_cxx_compiler_flag(-mfp16-format=ieee GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E)
              if (NOT "${GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E}" STREQUAL "")
 diff --git a/src/llama.cpp/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/src/llama.cpp/ggml/src/ggml-hexagon/ggml-hexagon.cpp
-index 72a82a891..7869ad323 100644
+index 72a82a891..1b681f4dd 100644
 --- a/src/llama.cpp/ggml/src/ggml-hexagon/ggml-hexagon.cpp
 +++ b/src/llama.cpp/ggml/src/ggml-hexagon/ggml-hexagon.cpp
-@@ -3417,6 +3417,8 @@ ggml_hexagon_registry::ggml_hexagon_registry(ggml_backend_reg_t reg) {
+@@ -3216,11 +3216,26 @@ static const char * ggml_backend_hexagon_device_get_description(ggml_backend_dev
+     GGML_UNUSED(dev);
+ }
+ 
++
++// ~2GB per session for now
++#define GGML_HEXAGON_SESSION_MEMORY_DEFAULT (2ULL * 1024 * 1024 * 1024)
++// Max to 3.5GB
++#define GGML_HEXAGON_SESSION_MEMORY_MAX (3ULL * 1024 * 1024 * 1024  +  512ULL * 1024 * 1024)
++
+ static void ggml_backend_hexagon_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {
+-    // ~2GB per session for now
+-    *free  = 2ULL * 1024 * 1024 * 1024;
+-    *total = *free;
++    const char * str_mem = getenv("GGML_HEXAGON_SESSION_MEMORY");
++    if (str_mem) {
++        *free = std::stoull(str_mem);
++        if (*free < GGML_HEXAGON_SESSION_MEMORY_DEFAULT) {
++            *free = GGML_HEXAGON_SESSION_MEMORY_DEFAULT;
++        } else if (*free > GGML_HEXAGON_SESSION_MEMORY_MAX) {
++            *free = GGML_HEXAGON_SESSION_MEMORY_MAX;
++        }
++    } else {
++        *free = GGML_HEXAGON_SESSION_MEMORY_DEFAULT;
++    }
+ 
++    *total = *free;
+     GGML_UNUSED(dev);
+ }
+ 
+@@ -3401,10 +3416,17 @@ ggml_hexagon_registry::ggml_hexagon_registry(ggml_backend_reg_t reg) {
+         }
+     }
+ 
++#if defined(__ANDROID__)
+     if(opt_arch < 75) {
+         opt_ndev = 1;
+-        GGML_LOG_WARN("ggml-hex: forcing ndev to 1 for SoCs archs lower than v75.\n");
++        GGML_LOG_WARN("ggml-hex: forcing ndev to 1 for SoCs archs lower than v75 for Android.\n");
++    }
++#else
++    if(opt_arch < 73) {
++        opt_ndev = 1;
++        GGML_LOG_WARN("ggml-hex: forcing ndev to 1 for SoCs archs lower than v73 for Linux and Windows.\n");
+     }
++#endif
+ 
+     GGML_LOG_INFO("ggml-hex: Hexagon Arch version v%d\n", opt_arch);
+ 
+@@ -3417,6 +3439,8 @@ ggml_hexagon_registry::ggml_hexagon_registry(ggml_backend_reg_t reg) {
          } catch (std::exception const &exc) {
              GGML_LOG_ERROR("ggml-hex: failed to create device/session %zu\n", i);
              devices[i].context = nullptr;

package/src/LlamaContext.cpp

@@ -321,15 +321,20 @@ LlamaContext::LlamaContext(const Napi::CallbackInfo &info)
 
   // Parse devices array
   if (options.Has("devices") && options.Get("devices").IsArray()) {
+    std::vector<ggml_backend_dev_t> devs;
     auto devices_array = options.Get("devices").As<Napi::Array>();
     for (size_t i = 0; i < devices_array.Length(); i++) {
       auto device_name = devices_array.Get(i).ToString().Utf8Value();
       auto * dev = ggml_backend_dev_by_name(device_name.c_str());
       if (dev) {
-        params.devices.push_back(dev);
+        devs.push_back(dev);
       }
       // Skip invalid device names silently
     }
+    if (!devs.empty()) {
+      params.devices = devs;
+      params.devices.push_back(nullptr); // nullptr terminator required by llama.cpp
+    }
   }
 
   std::vector<common_adapter_lora_info> lora;