node-llama-cpp 3.1.1 → 3.2.0

package/llama/gpuInfo/metal-gpu-info.mm

@@ -3,15 +3,22 @@
 #include <string>
 #import <Metal/Metal.h>
 
-void getMetalGpuInfo(uint64_t * total, uint64_t * used) {
+void getMetalGpuInfo(uint64_t * total, uint64_t * used, uint64_t * unifiedMemorySize) {
     id<MTLDevice> device = MTLCreateSystemDefaultDevice();
 
     if (device) {
         *total = device.recommendedMaxWorkingSetSize;
         *used = device.currentAllocatedSize;
+
+        if (device.hasUnifiedMemory) {
+            *unifiedMemorySize = device.recommendedMaxWorkingSetSize;
+        } else {
+            *unifiedMemorySize = 0;
+        }
     } else {
         *total = 0;
         *used = 0;
+        *unifiedMemorySize = 0;
     }
 
     [device release];

package/llama/gpuInfo/vulkan-gpu-info.cpp

@@ -5,7 +5,7 @@
 
 typedef void (*gpuInfoVulkanWarningLogCallback_t)(const char* message);
 
-static bool enumerateVulkanDevices(size_t* total, size_t* used, bool addDeviceNames, std::vector<std::string> * deviceNames, gpuInfoVulkanWarningLogCallback_t warningLogCallback) {
+static bool enumerateVulkanDevices(size_t* total, size_t* used, size_t* unifiedMemorySize, bool addDeviceNames, std::vector<std::string> * deviceNames, gpuInfoVulkanWarningLogCallback_t warningLogCallback) {
     vk::ApplicationInfo appInfo("node-llama-cpp GPU info", 1, "llama.cpp", 1, VK_API_VERSION_1_2);
     vk::InstanceCreateInfo createInfo(vk::InstanceCreateFlags(), &appInfo, {}, {});
     vk::Instance instance = vk::createInstance(createInfo);
@@ -14,6 +14,7 @@ static bool enumerateVulkanDevices(size_t* total, size_t* used, bool addDeviceNa
 
     size_t usedMem = 0;
     size_t totalMem = 0;
+    size_t totalUnifiedMemorySize = 0;
 
     for (size_t i = 0; i < physicalDevices.size(); i++) {
         vk::PhysicalDevice physicalDevice = physicalDevices[i];
@@ -41,16 +42,20 @@ static bool enumerateVulkanDevices(size_t* total, size_t* used, bool addDeviceNa
             physicalDevice.getMemoryProperties2(&memProps2);
 
             for (uint32_t i = 0; i < memProps.memoryHeapCount; ++i) {
-                if (memProps.memoryHeaps[i].flags & vk::MemoryHeapFlagBits::eDeviceLocal) {
+                const auto flags = memProps.memoryHeaps[i].flags;
+
+                if (flags & vk::MemoryHeapFlagBits::eDeviceLocal) {
                     const auto size = memProps.memoryHeaps[i].size;
                     totalMem += size;
                     usedMem += memoryBudgetProperties.heapUsage[i];
 
+                    if (flags & vk::MemoryHeapFlagBits::eMultiInstance) {
+                        totalUnifiedMemorySize += size;
+                    }
+
                     if (size > 0 && addDeviceNames) {
                         (*deviceNames).push_back(std::string(deviceProps.deviceName.data()));
                     }
-
-                    break;
                 }
             }
         } else {
@@ -58,9 +63,8 @@ static bool enumerateVulkanDevices(size_t* total, size_t* used, bool addDeviceNa
             warningLogCallback(
                 (
                     "Vulkan VK_EXT_memory_budget extension not supported for device \"" +
-                    std::string(deviceProps.deviceName.data()) + "\", so VRAM info cannot be determained for it"
-                )
-                    .c_str()
+                    std::string(deviceProps.deviceName.data()) + "\", so VRAM info cannot be determined for it"
+                ).c_str()
             );
             return false;
         }
@@ -68,16 +72,19 @@ static bool enumerateVulkanDevices(size_t* total, size_t* used, bool addDeviceNa
 
     *total = totalMem;
     *used = usedMem;
+    *unifiedMemorySize = totalUnifiedMemorySize;
+
     return true;
 }
 
-bool gpuInfoGetTotalVulkanDevicesInfo(size_t* total, size_t* used, gpuInfoVulkanWarningLogCallback_t warningLogCallback) {
-    return enumerateVulkanDevices(total, used, false, nullptr, warningLogCallback);
+bool gpuInfoGetTotalVulkanDevicesInfo(size_t* total, size_t* used, size_t* unifiedMemorySize, gpuInfoVulkanWarningLogCallback_t warningLogCallback) {
+    return enumerateVulkanDevices(total, used, unifiedMemorySize, false, nullptr, warningLogCallback);
 }
 
 bool gpuInfoGetVulkanDeviceNames(std::vector<std::string> * deviceNames, gpuInfoVulkanWarningLogCallback_t warningLogCallback) {
     size_t vulkanDeviceTotal = 0;
     size_t vulkanDeviceUsed = 0;
+    size_t unifiedMemorySize = 0;
 
-    return enumerateVulkanDevices(&vulkanDeviceTotal, &vulkanDeviceUsed, true, deviceNames, warningLogCallback);
+    return enumerateVulkanDevices(&vulkanDeviceTotal, &vulkanDeviceUsed, &unifiedMemorySize, true, deviceNames, warningLogCallback);
 }

package/llama/gpuInfo/vulkan-gpu-info.h

@@ -5,5 +5,5 @@
 
 typedef void (*gpuInfoVulkanWarningLogCallback_t)(const char* message);
 
-bool gpuInfoGetTotalVulkanDevicesInfo(size_t* total, size_t* used, gpuInfoVulkanWarningLogCallback_t warningLogCallback);
+bool gpuInfoGetTotalVulkanDevicesInfo(size_t* total, size_t* used, size_t* unifiedMemorySize, gpuInfoVulkanWarningLogCallback_t warningLogCallback);
 bool gpuInfoGetVulkanDeviceNames(std::vector<std::string> * deviceNames, gpuInfoVulkanWarningLogCallback_t warningLogCallback);

package/llama/llama.cpp.info.json

@@ -1,4 +1,4 @@
 {
-    "tag": "b3889",
+    "tag": "b3995",
     "llamaCppGithubRepo": "ggerganov/llama.cpp"
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "node-llama-cpp",
-  "version": "3.1.1",
+  "version": "3.2.0",
   "description": "Run AI models locally on your machine with node.js bindings for llama.cpp. Enforce a JSON schema on the model output on the generation level",
   "main": "./dist/index.js",
   "type": "module",
@@ -60,7 +60,7 @@
     "test:modelDependent:interactive": "vitest watch ./test/modelDependent",
     "test:typescript": "tsc --noEmit --project tsconfig.json",
     "lint": "npm run lint:eslint",
-    "lint:eslint": "eslint --ext .js --ext .ts --report-unused-disable-directives .",
+    "lint:eslint": "eslint --report-unused-disable-directives .",
     "format": "npm run lint:eslint -- --fix",
     "dev:setup:downloadAllTestModels": "vite-node test/utils/scripts/downloadAllTestModels.ts",
     "dev:setup": "npm run build && node ./dist/cli/cli.js source download --noUsageExample && npm run docs:generateTypedoc && npm run dev:setup:downloadAllTestModels",
@@ -126,51 +126,53 @@
   "devDependencies": {
     "@commitlint/cli": "^19.5.0",
     "@commitlint/config-conventional": "^19.5.0",
+    "@eslint/compat": "^1.2.2",
     "@fontsource/inter": "^5.1.0",
-    "@nolebase/vitepress-plugin-git-changelog": "^2.5.0",
-    "@nolebase/vitepress-plugin-og-image": "^2.5.0",
+    "@nolebase/vitepress-plugin-git-changelog": "^2.6.1",
+    "@nolebase/vitepress-plugin-og-image": "^2.6.1",
     "@resvg/resvg-js": "^2.6.2",
     "@semantic-release/exec": "^6.0.3",
     "@semantic-release/github": "11.0.0",
     "@semantic-release/npm": "12.0.1",
-    "@shikijs/vitepress-twoslash": "^1.18.0",
-    "@types/async-retry": "^1.4.8",
+    "@shikijs/vitepress-twoslash": "^1.22.2",
+    "@stylistic/eslint-plugin": "^2.9.0",
+    "@types/async-retry": "^1.4.9",
     "@types/bytes": "^3.1.4",
     "@types/cross-spawn": "^6.0.2",
     "@types/fs-extra": "^11.0.4",
-    "@types/node": "^22.7.0",
+    "@types/node": "^22.8.4",
     "@types/proper-lockfile": "^4.1.4",
     "@types/semver": "^7.5.8",
     "@types/validate-npm-package-name": "^4.0.2",
     "@types/which": "^3.0.4",
     "@types/yargs": "^17.0.33",
-    "@typescript-eslint/eslint-plugin": "^7.15.0",
-    "@typescript-eslint/parser": "^7.15.0",
-    "@vitest/coverage-v8": "^2.1.1",
-    "@vitest/ui": "^2.1.1",
-    "eslint": "^8.46.0",
-    "eslint-plugin-import": "^2.30.0",
-    "eslint-plugin-jsdoc": "^50.2.3",
-    "eslint-plugin-n": "^17.10.2",
+    "@vitest/coverage-v8": "^2.1.4",
+    "@vitest/ui": "^2.1.4",
+    "eslint": "^9.13.0",
+    "eslint-import-resolver-typescript": "^3.6.3",
+    "eslint-plugin-import": "^2.31.0",
+    "eslint-plugin-jsdoc": "^50.4.3",
+    "eslint-plugin-n": "^17.12.0",
     "feed": "^4.2.2",
     "husky": "^9.1.6",
     "rehype": "^13.0.1",
     "rimraf": "^6.0.1",
-    "semantic-release": "24.1.1",
+    "semantic-release": "^24.2.0",
     "sharp": "^0.33.5",
-    "tslib": "^2.7.0",
-    "typedoc": "^0.26.7",
-    "typedoc-plugin-markdown": "^4.2.8",
-    "typedoc-plugin-mdn-links": "^3.3.1",
-    "typedoc-vitepress-theme": "^1.0.1",
-    "typescript": "^5.6.2",
-    "vite-node": "^2.1.1",
-    "vitepress": "1.3.4",
-    "vitest": "^2.1.1",
-    "zx": "^8.1.8"
+    "tslib": "^2.8.0",
+    "typedoc": "^0.26.10",
+    "typedoc-plugin-markdown": "^4.2.9",
+    "typedoc-plugin-mdn-links": "^3.3.5",
+    "typedoc-vitepress-theme": "^1.0.2",
+    "typescript": "^5.6.3",
+    "typescript-eslint": "^8.12.2",
+    "vite-node": "^2.1.4",
+    "vitepress": "^1.4.2",
+    "vitest": "^2.1.4",
+    "zx": "^8.1.9"
   },
   "dependencies": {
-    "@huggingface/jinja": "^0.3.1",
+    "@huggingface/jinja": "^0.3.2",
     "async-retry": "^1.3.3",
     "bytes": "^3.1.2",
     "chalk": "^5.3.0",
@@ -182,12 +184,12 @@
     "filenamify": "^6.0.0",
     "fs-extra": "^11.2.0",
     "ignore": "^5.3.2",
-    "ipull": "^3.7.4",
+    "ipull": "^3.9.0",
     "is-unicode-supported": "^2.1.0",
     "lifecycle-utils": "^1.7.0",
     "log-symbols": "^7.0.0",
-    "nanoid": "^5.0.7",
-    "node-addon-api": "^8.1.0",
+    "nanoid": "^5.0.8",
+    "node-addon-api": "^8.2.1",
     "octokit": "^4.0.2",
     "ora": "^8.1.0",
     "pretty-ms": "^9.1.0",
@@ -197,8 +199,8 @@
     "slice-ansi": "^7.1.0",
     "stdout-update": "^4.0.1",
     "strip-ansi": "^7.1.0",
-    "validate-npm-package-name": "^5.0.1",
-    "which": "^4.0.0",
+    "validate-npm-package-name": "^6.0.0",
+    "which": "^5.0.0",
     "yargs": "^17.7.2"
   },
   "peerDependencies": {
@@ -210,16 +212,16 @@
     }
   },
   "optionalDependencies": {
-    "@node-llama-cpp/linux-arm64": "3.1.1",
-    "@node-llama-cpp/linux-armv7l": "3.1.1",
-    "@node-llama-cpp/linux-x64": "3.1.1",
-    "@node-llama-cpp/linux-x64-cuda": "3.1.1",
-    "@node-llama-cpp/linux-x64-vulkan": "3.1.1",
-    "@node-llama-cpp/mac-arm64-metal": "3.1.1",
-    "@node-llama-cpp/mac-x64": "3.1.1",
-    "@node-llama-cpp/win-arm64": "3.1.1",
-    "@node-llama-cpp/win-x64": "3.1.1",
-    "@node-llama-cpp/win-x64-cuda": "3.1.1",
-    "@node-llama-cpp/win-x64-vulkan": "3.1.1"
+    "@node-llama-cpp/linux-arm64": "3.2.0",
+    "@node-llama-cpp/linux-armv7l": "3.2.0",
+    "@node-llama-cpp/linux-x64": "3.2.0",
+    "@node-llama-cpp/linux-x64-cuda": "3.2.0",
+    "@node-llama-cpp/linux-x64-vulkan": "3.2.0",
+    "@node-llama-cpp/mac-arm64-metal": "3.2.0",
+    "@node-llama-cpp/mac-x64": "3.2.0",
+    "@node-llama-cpp/win-arm64": "3.2.0",
+    "@node-llama-cpp/win-x64": "3.2.0",
+    "@node-llama-cpp/win-x64-cuda": "3.2.0",
+    "@node-llama-cpp/win-x64-vulkan": "3.2.0"
   }
 }