node-llama-cpp 3.1.0 → 3.2.0

package/llama/addon/globals/getGpuInfo.cpp

@@ -26,6 +26,7 @@ void logVulkanWarning(const char* message) {
 Napi::Value getGpuVramInfo(const Napi::CallbackInfo& info) {
     uint64_t total = 0;
     uint64_t used = 0;
+    uint64_t unifiedVramSize = 0;
 
 #ifdef GPU_INFO_USE_CUDA
     size_t cudaDeviceTotal = 0;
@@ -41,26 +42,31 @@ Napi::Value getGpuVramInfo(const Napi::CallbackInfo& info) {
 #ifdef GPU_INFO_USE_VULKAN
     uint64_t vulkanDeviceTotal = 0;
     uint64_t vulkanDeviceUsed = 0;
-    const bool vulkanDeviceSupportsMemoryBudgetExtension = gpuInfoGetTotalVulkanDevicesInfo(&vulkanDeviceTotal, &vulkanDeviceUsed, logVulkanWarning);
+    uint64_t vulkanDeviceUnifiedVramSize = 0;
+    const bool vulkanDeviceSupportsMemoryBudgetExtension = gpuInfoGetTotalVulkanDevicesInfo(&vulkanDeviceTotal, &vulkanDeviceUsed, &vulkanDeviceUnifiedVramSize, logVulkanWarning);
 
     if (vulkanDeviceSupportsMemoryBudgetExtension) {
         total += vulkanDeviceTotal;
         used += vulkanDeviceUsed;
+        unifiedVramSize += vulkanDeviceUnifiedVramSize;
     }
 #endif
 
 #ifdef GPU_INFO_USE_METAL
     uint64_t metalDeviceTotal = 0;
     uint64_t metalDeviceUsed = 0;
-    getMetalGpuInfo(&metalDeviceTotal, &metalDeviceUsed);
+    uint64_t metalDeviceUnifiedVramSize = 0;
+    getMetalGpuInfo(&metalDeviceTotal, &metalDeviceUsed, &metalDeviceUnifiedVramSize);
 
     total += metalDeviceTotal;
     used += metalDeviceUsed;
+    unifiedVramSize += metalDeviceUnifiedVramSize;
 #endif
 
     Napi::Object result = Napi::Object::New(info.Env());
     result.Set("total", Napi::Number::From(info.Env(), total));
     result.Set("used", Napi::Number::From(info.Env(), used));
+    result.Set("unifiedSize", Napi::Number::From(info.Env(), unifiedVramSize));
 
     return result;
 }

package/llama/addon/globals/getSwapInfo.cpp

@@ -0,0 +1,69 @@
+#include "getSwapInfo.h"
+#include "addonLog.h"
+
+#ifdef __APPLE__
+#include <iostream>
+#include <mach/mach.h>
+#include <sys/sysctl.h>
+#elif __linux__
+#include <iostream>
+#include <sys/sysinfo.h>
+#elif _WIN32
+#include <iostream>
+#include <windows.h>
+#include <psapi.h>
+#endif
+
+
+Napi::Value getSwapInfo(const Napi::CallbackInfo& info) {
+    uint64_t totalSwap = 0;
+    uint64_t freeSwap = 0;
+    uint64_t maxSize = 0;
+    bool maxSizeSet = true;
+
+#ifdef __APPLE__
+    struct xsw_usage swapInfo;
+    size_t size = sizeof(swapInfo);
+
+    if (sysctlbyname("vm.swapusage", &swapInfo, &size, NULL, 0) == 0) {
+        totalSwap = swapInfo.xsu_total;
+        freeSwap = swapInfo.xsu_avail;
+        maxSizeSet = false;
+    } else {
+        addonLlamaCppLogCallback(GGML_LOG_LEVEL_ERROR, std::string("Failed to get swap info").c_str(), nullptr);
+    }
+#elif __linux__
+    struct sysinfo sysInfo;
+
+    if (sysinfo(&sysInfo) == 0) {
+        totalSwap = sysInfo.totalswap;
+        freeSwap = sysInfo.freeswap;
+        maxSize = sysInfo.totalswap;
+    } else {
+        addonLlamaCppLogCallback(GGML_LOG_LEVEL_ERROR, std::string("Failed to get swap info").c_str(), nullptr);
+    }
+#elif _WIN32
+    MEMORYSTATUSEX memInfo;
+    memInfo.dwLength = sizeof(MEMORYSTATUSEX);
+
+    if (GlobalMemoryStatusEx(&memInfo)) {
+        PERFORMANCE_INFORMATION perfInfo;
+        perfInfo.cb = sizeof(PERFORMANCE_INFORMATION);
+        if (GetPerformanceInfo(&perfInfo, sizeof(perfInfo))) {
+            totalSwap = memInfo.ullTotalPageFile;
+            freeSwap = memInfo.ullAvailPageFile;
+            maxSize = perfInfo.CommitLimit * perfInfo.PageSize;
+        } else {
+            addonLlamaCppLogCallback(GGML_LOG_LEVEL_ERROR, std::string("Failed to get max pagefile size").c_str(), nullptr);
+        }
+    } else {
+        addonLlamaCppLogCallback(GGML_LOG_LEVEL_ERROR, std::string("Failed to get pagefile info").c_str(), nullptr);
+    }
+#endif
+    
+    Napi::Object obj = Napi::Object::New(info.Env());
+    obj.Set("total", Napi::Number::New(info.Env(), totalSwap));
+    obj.Set("free", Napi::Number::New(info.Env(), freeSwap));
+    obj.Set("maxSize", maxSizeSet ? Napi::Number::New(info.Env(), maxSize) : Napi::Number::New(info.Env(), -1));
+    return obj;
+}

package/llama/addon/globals/getSwapInfo.h

@@ -0,0 +1,4 @@
+#pragma once
+#include "napi.h"
+
+Napi::Value getSwapInfo(const Napi::CallbackInfo& info);

package/llama/binariesGithubRelease.json

@@ -1,3 +1,3 @@
 {
-    "release": "b3887"
+    "release": "b3995"
 }

package/llama/gpuInfo/metal-gpu-info.h

@@ -4,5 +4,5 @@
 #include <string>
 #include <vector>
 
-void getMetalGpuInfo(uint64_t * total, uint64_t * used);
+void getMetalGpuInfo(uint64_t * total, uint64_t * used, uint64_t * unifiedMemorySize);
 void getMetalGpuDeviceNames(std::vector<std::string> * deviceNames);

package/llama/gpuInfo/metal-gpu-info.mm

@@ -3,15 +3,22 @@
 #include <string>
 #import <Metal/Metal.h>
 
-void getMetalGpuInfo(uint64_t * total, uint64_t * used) {
+void getMetalGpuInfo(uint64_t * total, uint64_t * used, uint64_t * unifiedMemorySize) {
     id<MTLDevice> device = MTLCreateSystemDefaultDevice();
 
     if (device) {
         *total = device.recommendedMaxWorkingSetSize;
         *used = device.currentAllocatedSize;
+
+        if (device.hasUnifiedMemory) {
+            *unifiedMemorySize = device.recommendedMaxWorkingSetSize;
+        } else {
+            *unifiedMemorySize = 0;
+        }
     } else {
         *total = 0;
         *used = 0;
+        *unifiedMemorySize = 0;
     }
 
     [device release];

package/llama/gpuInfo/vulkan-gpu-info.cpp

@@ -5,7 +5,7 @@
 
 typedef void (*gpuInfoVulkanWarningLogCallback_t)(const char* message);
 
-static bool enumerateVulkanDevices(size_t* total, size_t* used, bool addDeviceNames, std::vector<std::string> * deviceNames, gpuInfoVulkanWarningLogCallback_t warningLogCallback) {
+static bool enumerateVulkanDevices(size_t* total, size_t* used, size_t* unifiedMemorySize, bool addDeviceNames, std::vector<std::string> * deviceNames, gpuInfoVulkanWarningLogCallback_t warningLogCallback) {
     vk::ApplicationInfo appInfo("node-llama-cpp GPU info", 1, "llama.cpp", 1, VK_API_VERSION_1_2);
     vk::InstanceCreateInfo createInfo(vk::InstanceCreateFlags(), &appInfo, {}, {});
     vk::Instance instance = vk::createInstance(createInfo);
@@ -14,6 +14,7 @@ static bool enumerateVulkanDevices(size_t* total, size_t* used, bool addDeviceNa
 
     size_t usedMem = 0;
     size_t totalMem = 0;
+    size_t totalUnifiedMemorySize = 0;
 
     for (size_t i = 0; i < physicalDevices.size(); i++) {
         vk::PhysicalDevice physicalDevice = physicalDevices[i];
@@ -41,16 +42,20 @@ static bool enumerateVulkanDevices(size_t* total, size_t* used, bool addDeviceNa
             physicalDevice.getMemoryProperties2(&memProps2);
 
             for (uint32_t i = 0; i < memProps.memoryHeapCount; ++i) {
-                if (memProps.memoryHeaps[i].flags & vk::MemoryHeapFlagBits::eDeviceLocal) {
+                const auto flags = memProps.memoryHeaps[i].flags;
+
+                if (flags & vk::MemoryHeapFlagBits::eDeviceLocal) {
                     const auto size = memProps.memoryHeaps[i].size;
                     totalMem += size;
                     usedMem += memoryBudgetProperties.heapUsage[i];
 
+                    if (flags & vk::MemoryHeapFlagBits::eMultiInstance) {
+                        totalUnifiedMemorySize += size;
+                    }
+
                     if (size > 0 && addDeviceNames) {
                         (*deviceNames).push_back(std::string(deviceProps.deviceName.data()));
                     }
-
-                    break;
                 }
             }
         } else {
@@ -58,9 +63,8 @@ static bool enumerateVulkanDevices(size_t* total, size_t* used, bool addDeviceNa
             warningLogCallback(
                 (
                     "Vulkan VK_EXT_memory_budget extension not supported for device \"" +
-                    std::string(deviceProps.deviceName.data()) + "\", so VRAM info cannot be determained for it"
-                )
-                    .c_str()
+                    std::string(deviceProps.deviceName.data()) + "\", so VRAM info cannot be determined for it"
+                ).c_str()
             );
             return false;
         }
@@ -68,16 +72,19 @@ static bool enumerateVulkanDevices(size_t* total, size_t* used, bool addDeviceNa
 
     *total = totalMem;
     *used = usedMem;
+    *unifiedMemorySize = totalUnifiedMemorySize;
+
     return true;
 }
 
-bool gpuInfoGetTotalVulkanDevicesInfo(size_t* total, size_t* used, gpuInfoVulkanWarningLogCallback_t warningLogCallback) {
-    return enumerateVulkanDevices(total, used, false, nullptr, warningLogCallback);
+bool gpuInfoGetTotalVulkanDevicesInfo(size_t* total, size_t* used, size_t* unifiedMemorySize, gpuInfoVulkanWarningLogCallback_t warningLogCallback) {
+    return enumerateVulkanDevices(total, used, unifiedMemorySize, false, nullptr, warningLogCallback);
 }
 
 bool gpuInfoGetVulkanDeviceNames(std::vector<std::string> * deviceNames, gpuInfoVulkanWarningLogCallback_t warningLogCallback) {
     size_t vulkanDeviceTotal = 0;
     size_t vulkanDeviceUsed = 0;
+    size_t unifiedMemorySize = 0;
 
-    return enumerateVulkanDevices(&vulkanDeviceTotal, &vulkanDeviceUsed, true, deviceNames, warningLogCallback);
+    return enumerateVulkanDevices(&vulkanDeviceTotal, &vulkanDeviceUsed, &unifiedMemorySize, true, deviceNames, warningLogCallback);
 }

package/llama/gpuInfo/vulkan-gpu-info.h

@@ -5,5 +5,5 @@
 
 typedef void (*gpuInfoVulkanWarningLogCallback_t)(const char* message);
 
-bool gpuInfoGetTotalVulkanDevicesInfo(size_t* total, size_t* used, gpuInfoVulkanWarningLogCallback_t warningLogCallback);
+bool gpuInfoGetTotalVulkanDevicesInfo(size_t* total, size_t* used, size_t* unifiedMemorySize, gpuInfoVulkanWarningLogCallback_t warningLogCallback);
 bool gpuInfoGetVulkanDeviceNames(std::vector<std::string> * deviceNames, gpuInfoVulkanWarningLogCallback_t warningLogCallback);

package/llama/llama.cpp.info.json

@@ -1,4 +1,4 @@
 {
-    "tag": "b3887",
+    "tag": "b3995",
     "llamaCppGithubRepo": "ggerganov/llama.cpp"
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "node-llama-cpp",
-  "version": "3.1.0",
+  "version": "3.2.0",
   "description": "Run AI models locally on your machine with node.js bindings for llama.cpp. Enforce a JSON schema on the model output on the generation level",
   "main": "./dist/index.js",
   "type": "module",
@@ -60,7 +60,7 @@
     "test:modelDependent:interactive": "vitest watch ./test/modelDependent",
     "test:typescript": "tsc --noEmit --project tsconfig.json",
     "lint": "npm run lint:eslint",
-    "lint:eslint": "eslint --ext .js --ext .ts --report-unused-disable-directives .",
+    "lint:eslint": "eslint --report-unused-disable-directives .",
     "format": "npm run lint:eslint -- --fix",
     "dev:setup:downloadAllTestModels": "vite-node test/utils/scripts/downloadAllTestModels.ts",
     "dev:setup": "npm run build && node ./dist/cli/cli.js source download --noUsageExample && npm run docs:generateTypedoc && npm run dev:setup:downloadAllTestModels",
@@ -126,51 +126,53 @@
   "devDependencies": {
     "@commitlint/cli": "^19.5.0",
     "@commitlint/config-conventional": "^19.5.0",
+    "@eslint/compat": "^1.2.2",
     "@fontsource/inter": "^5.1.0",
-    "@nolebase/vitepress-plugin-git-changelog": "^2.5.0",
-    "@nolebase/vitepress-plugin-og-image": "^2.5.0",
+    "@nolebase/vitepress-plugin-git-changelog": "^2.6.1",
+    "@nolebase/vitepress-plugin-og-image": "^2.6.1",
     "@resvg/resvg-js": "^2.6.2",
     "@semantic-release/exec": "^6.0.3",
     "@semantic-release/github": "11.0.0",
     "@semantic-release/npm": "12.0.1",
-    "@shikijs/vitepress-twoslash": "^1.18.0",
-    "@types/async-retry": "^1.4.8",
+    "@shikijs/vitepress-twoslash": "^1.22.2",
+    "@stylistic/eslint-plugin": "^2.9.0",
+    "@types/async-retry": "^1.4.9",
     "@types/bytes": "^3.1.4",
     "@types/cross-spawn": "^6.0.2",
     "@types/fs-extra": "^11.0.4",
-    "@types/node": "^22.7.0",
+    "@types/node": "^22.8.4",
     "@types/proper-lockfile": "^4.1.4",
     "@types/semver": "^7.5.8",
     "@types/validate-npm-package-name": "^4.0.2",
     "@types/which": "^3.0.4",
     "@types/yargs": "^17.0.33",
-    "@typescript-eslint/eslint-plugin": "^7.15.0",
-    "@typescript-eslint/parser": "^7.15.0",
-    "@vitest/coverage-v8": "^2.1.1",
-    "@vitest/ui": "^2.1.1",
-    "eslint": "^8.46.0",
-    "eslint-plugin-import": "^2.30.0",
-    "eslint-plugin-jsdoc": "^50.2.3",
-    "eslint-plugin-n": "^17.10.2",
+    "@vitest/coverage-v8": "^2.1.4",
+    "@vitest/ui": "^2.1.4",
+    "eslint": "^9.13.0",
+    "eslint-import-resolver-typescript": "^3.6.3",
+    "eslint-plugin-import": "^2.31.0",
+    "eslint-plugin-jsdoc": "^50.4.3",
+    "eslint-plugin-n": "^17.12.0",
     "feed": "^4.2.2",
     "husky": "^9.1.6",
     "rehype": "^13.0.1",
     "rimraf": "^6.0.1",
-    "semantic-release": "24.1.1",
+    "semantic-release": "^24.2.0",
     "sharp": "^0.33.5",
-    "tslib": "^2.7.0",
-    "typedoc": "^0.26.7",
-    "typedoc-plugin-markdown": "^4.2.8",
-    "typedoc-plugin-mdn-links": "^3.3.1",
-    "typedoc-vitepress-theme": "^1.0.1",
-    "typescript": "^5.6.2",
-    "vite-node": "^2.1.1",
-    "vitepress": "1.3.4",
-    "vitest": "^2.1.1",
-    "zx": "^8.1.8"
+    "tslib": "^2.8.0",
+    "typedoc": "^0.26.10",
+    "typedoc-plugin-markdown": "^4.2.9",
+    "typedoc-plugin-mdn-links": "^3.3.5",
+    "typedoc-vitepress-theme": "^1.0.2",
+    "typescript": "^5.6.3",
+    "typescript-eslint": "^8.12.2",
+    "vite-node": "^2.1.4",
+    "vitepress": "^1.4.2",
+    "vitest": "^2.1.4",
+    "zx": "^8.1.9"
   },
   "dependencies": {
-    "@huggingface/jinja": "^0.3.1",
+    "@huggingface/jinja": "^0.3.2",
     "async-retry": "^1.3.3",
     "bytes": "^3.1.2",
     "chalk": "^5.3.0",
@@ -182,12 +184,12 @@
     "filenamify": "^6.0.0",
     "fs-extra": "^11.2.0",
     "ignore": "^5.3.2",
-    "ipull": "^3.7.4",
+    "ipull": "^3.9.0",
     "is-unicode-supported": "^2.1.0",
     "lifecycle-utils": "^1.7.0",
     "log-symbols": "^7.0.0",
-    "nanoid": "^5.0.7",
-    "node-addon-api": "^8.1.0",
+    "nanoid": "^5.0.8",
+    "node-addon-api": "^8.2.1",
     "octokit": "^4.0.2",
     "ora": "^8.1.0",
     "pretty-ms": "^9.1.0",
@@ -197,8 +199,8 @@
     "slice-ansi": "^7.1.0",
     "stdout-update": "^4.0.1",
     "strip-ansi": "^7.1.0",
-    "validate-npm-package-name": "^5.0.1",
-    "which": "^4.0.0",
+    "validate-npm-package-name": "^6.0.0",
+    "which": "^5.0.0",
     "yargs": "^17.7.2"
   },
   "peerDependencies": {
@@ -210,16 +212,16 @@
     }
   },
   "optionalDependencies": {
-    "@node-llama-cpp/linux-arm64": "3.1.0",
-    "@node-llama-cpp/linux-armv7l": "3.1.0",
-    "@node-llama-cpp/linux-x64": "3.1.0",
-    "@node-llama-cpp/linux-x64-cuda": "3.1.0",
-    "@node-llama-cpp/linux-x64-vulkan": "3.1.0",
-    "@node-llama-cpp/mac-arm64-metal": "3.1.0",
-    "@node-llama-cpp/mac-x64": "3.1.0",
-    "@node-llama-cpp/win-arm64": "3.1.0",
-    "@node-llama-cpp/win-x64": "3.1.0",
-    "@node-llama-cpp/win-x64-cuda": "3.1.0",
-    "@node-llama-cpp/win-x64-vulkan": "3.1.0"
+    "@node-llama-cpp/linux-arm64": "3.2.0",
+    "@node-llama-cpp/linux-armv7l": "3.2.0",
+    "@node-llama-cpp/linux-x64": "3.2.0",
+    "@node-llama-cpp/linux-x64-cuda": "3.2.0",
+    "@node-llama-cpp/linux-x64-vulkan": "3.2.0",
+    "@node-llama-cpp/mac-arm64-metal": "3.2.0",
+    "@node-llama-cpp/mac-x64": "3.2.0",
+    "@node-llama-cpp/win-arm64": "3.2.0",
+    "@node-llama-cpp/win-x64": "3.2.0",
+    "@node-llama-cpp/win-x64-cuda": "3.2.0",
+    "@node-llama-cpp/win-x64-vulkan": "3.2.0"
   }
 }