node-llama-cpp 3.0.0-beta.4 → 3.0.0-beta.40

package/llama/addon/globals/getGpuInfo.cpp

@@ -0,0 +1,108 @@
+#include "getGpuInfo.h"
+#include "addonLog.h"
+
+#ifdef GPU_INFO_USE_CUDA
+#  include "../../gpuInfo/cuda-gpu-info.h"
+#endif
+#ifdef GPU_INFO_USE_VULKAN
+#  include "../../gpuInfo/vulkan-gpu-info.h"
+#endif
+#ifdef GPU_INFO_USE_METAL
+#  include "../../gpuInfo/metal-gpu-info.h"
+#endif
+
+
+#ifdef GPU_INFO_USE_CUDA
+void logCudaError(const char* message) {
+    addonLlamaCppLogCallback(GGML_LOG_LEVEL_ERROR, (std::string("CUDA error: ") + std::string(message)).c_str(), nullptr);
+}
+#endif
+#ifdef GPU_INFO_USE_VULKAN
+void logVulkanWarning(const char* message) {
+    addonLlamaCppLogCallback(GGML_LOG_LEVEL_WARN, (std::string("Vulkan warning: ") + std::string(message)).c_str(), nullptr);
+}
+#endif
+
+Napi::Value getGpuVramInfo(const Napi::CallbackInfo& info) {
+    uint64_t total = 0;
+    uint64_t used = 0;
+
+#ifdef GPU_INFO_USE_CUDA
+    size_t cudaDeviceTotal = 0;
+    size_t cudaDeviceUsed = 0;
+    bool cudeGetInfoSuccess = gpuInfoGetTotalCudaDevicesInfo(&cudaDeviceTotal, &cudaDeviceUsed, logCudaError);
+
+    if (cudeGetInfoSuccess) {
+        total += cudaDeviceTotal;
+        used += cudaDeviceUsed;
+    }
+#endif
+
+#ifdef GPU_INFO_USE_VULKAN
+    uint64_t vulkanDeviceTotal = 0;
+    uint64_t vulkanDeviceUsed = 0;
+    const bool vulkanDeviceSupportsMemoryBudgetExtension = gpuInfoGetTotalVulkanDevicesInfo(&vulkanDeviceTotal, &vulkanDeviceUsed, logVulkanWarning);
+
+    if (vulkanDeviceSupportsMemoryBudgetExtension) {
+        total += vulkanDeviceTotal;
+        used += vulkanDeviceUsed;
+    }
+#endif
+
+#ifdef GPU_INFO_USE_METAL
+    uint64_t metalDeviceTotal = 0;
+    uint64_t metalDeviceUsed = 0;
+    getMetalGpuInfo(&metalDeviceTotal, &metalDeviceUsed);
+
+    total += metalDeviceTotal;
+    used += metalDeviceUsed;
+#endif
+
+    Napi::Object result = Napi::Object::New(info.Env());
+    result.Set("total", Napi::Number::From(info.Env(), total));
+    result.Set("used", Napi::Number::From(info.Env(), used));
+
+    return result;
+}
+
+Napi::Value getGpuDeviceInfo(const Napi::CallbackInfo& info) {
+    std::vector<std::string> deviceNames;
+
+#ifdef GPU_INFO_USE_CUDA
+    gpuInfoGetCudaDeviceNames(&deviceNames, logCudaError);
+#endif
+
+#ifdef GPU_INFO_USE_VULKAN
+    gpuInfoGetVulkanDeviceNames(&deviceNames, logVulkanWarning);
+#endif
+
+#ifdef GPU_INFO_USE_METAL
+    getMetalGpuDeviceNames(&deviceNames);
+#endif
+
+    Napi::Object result = Napi::Object::New(info.Env());
+
+    Napi::Array deviceNamesNapiArray = Napi::Array::New(info.Env(), deviceNames.size());
+    for (size_t i = 0; i < deviceNames.size(); ++i) {
+        deviceNamesNapiArray[i] = Napi::String::New(info.Env(), deviceNames[i]);
+    }
+    result.Set("deviceNames", deviceNamesNapiArray);
+
+    return result;
+}
+
+Napi::Value getGpuType(const Napi::CallbackInfo& info) {
+#ifdef GPU_INFO_USE_CUDA
+    return Napi::String::New(info.Env(), "cuda");
+#endif
+
+#ifdef GPU_INFO_USE_VULKAN
+    return Napi::String::New(info.Env(), "vulkan");
+#endif
+
+#ifdef GPU_INFO_USE_METAL
+    return Napi::String::New(info.Env(), "metal");
+#endif
+
+    return info.Env().Undefined();
+}

package/llama/addon/globals/getGpuInfo.h

@@ -0,0 +1,6 @@
+#pragma once
+#include "napi.h"
+
+Napi::Value getGpuVramInfo(const Napi::CallbackInfo& info);
+Napi::Value getGpuDeviceInfo(const Napi::CallbackInfo& info);
+Napi::Value getGpuType(const Napi::CallbackInfo& info);

package/llama/binariesGithubRelease.json

@@ -1,3 +1,3 @@
 {
-    "release": "b1892"
+    "release": "b3488"
 }

package/llama/gpuInfo/cuda-gpu-info.cu

@@ -0,0 +1,120 @@
+#include <stddef.h>
+#include <vector>
+#include <string>
+
+#if defined(GPU_INFO_USE_HIPBLAS)
+#include <hip/hip_runtime.h>
+#include <hipblas/hipblas.h>
+#define cudaGetDevice hipGetDevice
+#define cudaGetDeviceCount hipGetDeviceCount
+#define cudaGetErrorString hipGetErrorString
+#define cudaMemGetInfo hipMemGetInfo
+#define cudaSetDevice hipSetDevice
+#define cudaSuccess hipSuccess
+#else
+#include <cuda_runtime.h>
+#include <cuda.h>
+#endif
+
+
+typedef void (*gpuInfoCudaErrorLogCallback_t)(const char* message);
+
+bool gpuInfoSetCudaDevice(const int device, gpuInfoCudaErrorLogCallback_t errorLogCallback) {
+    int current_device;
+    auto getDeviceResult = cudaGetDevice(&current_device);
+
+    if (getDeviceResult != cudaSuccess) {
+        errorLogCallback(cudaGetErrorString(getDeviceResult));
+        return false;
+    }
+
+    if (device == current_device) {
+        return true;
+    }
+
+    const auto setDeviceResult = cudaSetDevice(device);
+
+    if (setDeviceResult != cudaSuccess) {
+        errorLogCallback(cudaGetErrorString(setDeviceResult));
+        return false;
+    }
+
+    return true;
+}
+
+bool gpuInfoGetCudaDeviceInfo(int device, size_t * total, size_t * used, gpuInfoCudaErrorLogCallback_t errorLogCallback) {
+    gpuInfoSetCudaDevice(device, errorLogCallback);
+
+    size_t freeMem;
+    size_t totalMem;
+    auto getMemInfoResult = cudaMemGetInfo(&freeMem, &totalMem);
+
+    if (getMemInfoResult != cudaSuccess) {
+        errorLogCallback(cudaGetErrorString(getMemInfoResult));
+        return false;
+    }
+
+    *total = totalMem;
+    *used = totalMem - freeMem;
+
+    return true;
+}
+
+int gpuInfoGetCudaDeviceCount(gpuInfoCudaErrorLogCallback_t errorLogCallback) {
+    int deviceCount;
+    auto getDeviceCountResult = cudaGetDeviceCount(&deviceCount);
+
+    if (getDeviceCountResult != cudaSuccess) {
+        errorLogCallback(cudaGetErrorString(getDeviceCountResult));
+        return -1;
+    }
+
+    return deviceCount;
+}
+
+bool gpuInfoGetTotalCudaDevicesInfo(size_t * total, size_t * used, gpuInfoCudaErrorLogCallback_t errorLogCallback) {
+    int deviceCount = gpuInfoGetCudaDeviceCount(errorLogCallback);
+
+    if (deviceCount < 0) {
+        return false;
+    }
+
+    size_t usedMem = 0;
+    size_t totalMem = 0;
+
+    for (int i = 0; i < deviceCount; i++) {
+        size_t deviceUsedMem;
+        size_t deviceTotalMem;
+
+        if (!gpuInfoGetCudaDeviceInfo(i, &deviceTotalMem, &deviceUsedMem, errorLogCallback)) {
+            return false;
+        }
+
+        usedMem += deviceUsedMem;
+        totalMem += deviceTotalMem;
+    }
+
+    *total = totalMem;
+    *used = usedMem;
+
+    return true;
+}
+
+void gpuInfoGetCudaDeviceNames(std::vector<std::string> * deviceNames, gpuInfoCudaErrorLogCallback_t errorLogCallback) {
+    int deviceCount = gpuInfoGetCudaDeviceCount(errorLogCallback);
+
+    if (deviceCount < 0) {
+        return;
+    }
+
+    for (int i = 0; i < deviceCount; i++) {
+        cudaDeviceProp prop;
+        auto getDevicePropertiesResult = cudaGetDeviceProperties(&prop, i);
+
+        if (getDevicePropertiesResult != cudaSuccess) {
+            errorLogCallback(cudaGetErrorString(getDevicePropertiesResult));
+        } else {
+            (*deviceNames).push_back(std::string(prop.name));
+        }
+    }
+}

package/llama/gpuInfo/cuda-gpu-info.h

@@ -0,0 +1,10 @@
+#pragma once
+
+#include <stddef.h>
+#include <vector>
+#include <string>
+
+typedef void (*gpuInfoCudaErrorLogCallback_t)(const char* message);
+
+bool gpuInfoGetTotalCudaDevicesInfo(size_t * total, size_t * used, gpuInfoCudaErrorLogCallback_t errorLogCallback);
+void gpuInfoGetCudaDeviceNames(std::vector<std::string> * deviceNames, gpuInfoCudaErrorLogCallback_t errorLogCallback);

package/llama/gpuInfo/metal-gpu-info.h

@@ -0,0 +1,8 @@
+#pragma once
+
+#include <stdint.h>
+#include <string>
+#include <vector>
+
+void getMetalGpuInfo(uint64_t * total, uint64_t * used);
+void getMetalGpuDeviceNames(std::vector<std::string> * deviceNames);

package/llama/gpuInfo/metal-gpu-info.mm

@@ -0,0 +1,30 @@
+#include <stdint.h>
+#include <vector>
+#include <string>
+#import <Metal/Metal.h>
+
+void getMetalGpuInfo(uint64_t * total, uint64_t * used) {
+    id<MTLDevice> device = MTLCreateSystemDefaultDevice();
+
+    if (device) {
+        *total = device.recommendedMaxWorkingSetSize;
+        *used = device.currentAllocatedSize;
+    } else {
+        *total = 0;
+        *used = 0;
+    }
+
+    [device release];
+    device = nil;
+}
+
+void getMetalGpuDeviceNames(std::vector<std::string> * deviceNames) {
+    NSArray<id<MTLDevice>> *devices = MTLCopyAllDevices();
+
+    for (id<MTLDevice> device in devices) {
+        (*deviceNames).push_back(std::string(([NSString stringWithUTF8String:device.name.UTF8String]).UTF8String));
+    }
+
+    [devices release];
+    devices = nil;
+}

package/llama/gpuInfo/vulkan-gpu-info.cpp

@@ -0,0 +1,83 @@
+#include <stddef.h>
+#include <vector>
+
+#include <vulkan/vulkan.hpp>
+
+typedef void (*gpuInfoVulkanWarningLogCallback_t)(const char* message);
+
+static bool enumerateVulkanDevices(size_t* total, size_t* used, bool addDeviceNames, std::vector<std::string> * deviceNames, gpuInfoVulkanWarningLogCallback_t warningLogCallback) {
+    vk::ApplicationInfo appInfo("node-llama-cpp GPU info", 1, "llama.cpp", 1, VK_API_VERSION_1_2);
+    vk::InstanceCreateInfo createInfo(vk::InstanceCreateFlags(), &appInfo, {}, {});
+    vk::Instance instance = vk::createInstance(createInfo);
+
+    auto physicalDevices = instance.enumeratePhysicalDevices();
+
+    size_t usedMem = 0;
+    size_t totalMem = 0;
+
+    for (size_t i = 0; i < physicalDevices.size(); i++) {
+        vk::PhysicalDevice physicalDevice = physicalDevices[i];
+        vk::PhysicalDeviceMemoryProperties memProps = physicalDevice.getMemoryProperties();
+        vk::PhysicalDeviceProperties deviceProps = physicalDevice.getProperties();
+
+        if (deviceProps.deviceType == vk::PhysicalDeviceType::eCpu) {
+            // ignore CPU devices, as we don't want to count RAM from the CPU as VRAM
+            continue;
+        }
+
+        std::vector<vk::ExtensionProperties> extensionProperties = physicalDevice.enumerateDeviceExtensionProperties();
+        bool memoryBudgetExtensionSupported =
+            std::any_of(
+                extensionProperties.begin(),
+                extensionProperties.end(),
+                [](const vk::ExtensionProperties& ext) { return std::string(ext.extensionName.data()) == VK_EXT_MEMORY_BUDGET_EXTENSION_NAME;}
+            );
+
+        if (memoryBudgetExtensionSupported) {
+            vk::PhysicalDeviceMemoryBudgetPropertiesEXT memoryBudgetProperties;
+            vk::PhysicalDeviceMemoryProperties2 memProps2 = {};
+            memProps2.pNext = &memoryBudgetProperties;
+
+            physicalDevice.getMemoryProperties2(&memProps2);
+
+            for (uint32_t i = 0; i < memProps.memoryHeapCount; ++i) {
+                if (memProps.memoryHeaps[i].flags & vk::MemoryHeapFlagBits::eDeviceLocal) {
+                    const auto size = memProps.memoryHeaps[i].size;
+                    totalMem += size;
+                    usedMem += memoryBudgetProperties.heapUsage[i];
+
+                    if (size > 0 && addDeviceNames) {
+                        (*deviceNames).push_back(std::string(deviceProps.deviceName.data()));
+                    }
+
+                    break;
+                }
+            }
+        } else {
+            // VK_EXT_memory_budget extension is not supported, so we cannot determine used memory
+            warningLogCallback(
+                (
+                    "Vulkan VK_EXT_memory_budget extension not supported for device \"" +
+                    std::string(deviceProps.deviceName.data()) + "\", so VRAM info cannot be determained for it"
+                )
+                    .c_str()
+            );
+            return false;
+        }
+    }
+
+    *total = totalMem;
+    *used = usedMem;
+    return true;
+}
+
+bool gpuInfoGetTotalVulkanDevicesInfo(size_t* total, size_t* used, gpuInfoVulkanWarningLogCallback_t warningLogCallback) {
+    return enumerateVulkanDevices(total, used, false, nullptr, warningLogCallback);
+}
+
+bool gpuInfoGetVulkanDeviceNames(std::vector<std::string> * deviceNames, gpuInfoVulkanWarningLogCallback_t warningLogCallback) {
+    size_t vulkanDeviceTotal = 0;
+    size_t vulkanDeviceUsed = 0;
+
+    return enumerateVulkanDevices(&vulkanDeviceTotal, &vulkanDeviceUsed, true, deviceNames, warningLogCallback);
+}

package/llama/gpuInfo/vulkan-gpu-info.h

@@ -0,0 +1,9 @@
+#pragma once
+
+#include <stddef.h>
+#include <vector>
+
+typedef void (*gpuInfoVulkanWarningLogCallback_t)(const char* message);
+
+bool gpuInfoGetTotalVulkanDevicesInfo(size_t* total, size_t* used, gpuInfoVulkanWarningLogCallback_t warningLogCallback);
+bool gpuInfoGetVulkanDeviceNames(std::vector<std::string> * deviceNames, gpuInfoVulkanWarningLogCallback_t warningLogCallback);