node-llama-cpp 3.0.0-beta.10 → 3.0.0-beta.12

package/llama/addon.cpp

@@ -12,6 +12,9 @@
 #ifdef GPU_INFO_USE_CUBLAS
 #  include "gpuInfo/cuda-gpu-info.h"
 #endif
+#ifdef GPU_INFO_USE_VULKAN
+#  include "gpuInfo/vulkan-gpu-info.h"
+#endif
 #ifdef GPU_INFO_USE_METAL
 #  include "gpuInfo/metal-gpu-info.h"
 #endif
@@ -35,6 +38,7 @@ using AddonThreadSafeLogCallbackFunction =
 AddonThreadSafeLogCallbackFunction addonThreadSafeLoggerCallback;
 bool addonJsLoggerCallbackSet = false;
 int addonLoggerLogLevel = 5;
+bool backendInitialized = false;
 
 std::string addon_model_token_to_piece(const struct llama_model* model, llama_token token) {
     std::vector<char> result(8, 0);
@@ -51,10 +55,15 @@ std::string addon_model_token_to_piece(const struct llama_model* model, llama_to
 }
 
 #ifdef GPU_INFO_USE_CUBLAS
-void lodCudaError(const char* message) {
+void logCudaError(const char* message) {
     addonLlamaCppLogCallback(GGML_LOG_LEVEL_ERROR, (std::string("CUDA error: ") + std::string(message)).c_str(), nullptr);
 }
 #endif
+#ifdef GPU_INFO_USE_VULKAN
+void logVulkanWarning(const char* message) {
+    addonLlamaCppLogCallback(GGML_LOG_LEVEL_WARN, (std::string("Vulkan warning: ") + std::string(message)).c_str(), nullptr);
+}
+#endif
 
 Napi::Value getGpuVramInfo(const Napi::CallbackInfo& info) {
     uint64_t total = 0;
@@ -63,7 +72,7 @@ Napi::Value getGpuVramInfo(const Napi::CallbackInfo& info) {
 #ifdef GPU_INFO_USE_CUBLAS
     size_t cudaDeviceTotal = 0;
     size_t cudaDeviceUsed = 0;
-    bool cudeGetInfoSuccess = gpuInfoGetTotalCudaDevicesInfo(&cudaDeviceTotal, &cudaDeviceUsed, lodCudaError);
+    bool cudeGetInfoSuccess = gpuInfoGetTotalCudaDevicesInfo(&cudaDeviceTotal, &cudaDeviceUsed, logCudaError);
 
     if (cudeGetInfoSuccess) {
         total += cudaDeviceTotal;
@@ -71,6 +80,17 @@ Napi::Value getGpuVramInfo(const Napi::CallbackInfo& info) {
     }
 #endif
 
+#ifdef GPU_INFO_USE_VULKAN
+    uint64_t vulkanDeviceTotal = 0;
+    uint64_t vulkanDeviceUsed = 0;
+    const bool vulkanDeviceSupportsMemoryBudgetExtension = gpuInfoGetTotalVulkanDevicesInfo(&vulkanDeviceTotal, &vulkanDeviceUsed, logVulkanWarning);
+
+    if (vulkanDeviceSupportsMemoryBudgetExtension) {
+        total += vulkanDeviceTotal;
+        used += vulkanDeviceUsed;
+    }
+#endif
+
 #ifdef GPU_INFO_USE_METAL
     uint64_t metalDeviceTotal = 0;
     uint64_t metalDeviceUsed = 0;
@@ -87,6 +107,26 @@ Napi::Value getGpuVramInfo(const Napi::CallbackInfo& info) {
     return result;
 }
 
+static Napi::Value getNapiToken(const Napi::CallbackInfo& info, llama_model* model, llama_token token) {
+    auto tokenType = llama_token_get_type(model, token);
+
+    if (tokenType == LLAMA_TOKEN_TYPE_UNDEFINED || tokenType == LLAMA_TOKEN_TYPE_UNKNOWN) {
+        return Napi::Number::From(info.Env(), -1);
+    }
+
+    return Napi::Number::From(info.Env(), token);
+}
+
+static Napi::Value getNapiControlToken(const Napi::CallbackInfo& info, llama_model* model, llama_token token) {
+    auto tokenType = llama_token_get_type(model, token);
+
+    if (tokenType != LLAMA_TOKEN_TYPE_CONTROL) {
+        return Napi::Number::From(info.Env(), -1);
+    }
+
+    return Napi::Number::From(info.Env(), token);
+}
+
 class AddonModel : public Napi::ObjectWrap<AddonModel> {
     public:
         llama_model_params model_params;
@@ -119,7 +159,6 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
                 }
             }
 
-            llama_backend_init(false);
             model = llama_load_model_from_file(modelPath.c_str(), model_params);
 
             if (model == NULL) {
@@ -203,6 +242,15 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
             return Napi::Number::From(info.Env(), llama_n_ctx_train(model));
         }
 
+        Napi::Value GetEmbeddingVectorSize(const Napi::CallbackInfo& info) {
+            if (disposed) {
+                Napi::Error::New(info.Env(), "Context is disposed").ThrowAsJavaScriptException();
+                return info.Env().Undefined();
+            }
+
+            return Napi::Number::From(info.Env(), llama_n_embd(model));
+        }
+
         Napi::Value GetTotalSize(const Napi::CallbackInfo& info) {
             if (disposed) {
                 Napi::Error::New(info.Env(), "Context is disposed").ThrowAsJavaScriptException();
@@ -239,7 +287,7 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
                 return info.Env().Undefined();
             }
 
-            return Napi::Number::From(info.Env(), llama_token_bos(model));
+            return getNapiControlToken(info, model, llama_token_bos(model));
         }
         Napi::Value TokenEos(const Napi::CallbackInfo& info) {
             if (disposed) {
@@ -247,7 +295,7 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
                 return info.Env().Undefined();
             }
 
-            return Napi::Number::From(info.Env(), llama_token_eos(model));
+            return getNapiControlToken(info, model, llama_token_eos(model));
         }
         Napi::Value TokenNl(const Napi::CallbackInfo& info) {
             if (disposed) {
@@ -255,7 +303,7 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
                 return info.Env().Undefined();
             }
 
-            return Napi::Number::From(info.Env(), llama_token_nl(model));
+            return getNapiToken(info, model, llama_token_nl(model));
         }
         Napi::Value PrefixToken(const Napi::CallbackInfo& info) {
             if (disposed) {
@@ -263,7 +311,7 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
                 return info.Env().Undefined();
             }
 
-            return Napi::Number::From(info.Env(), llama_token_prefix(model));
+            return getNapiControlToken(info, model, llama_token_prefix(model));
         }
         Napi::Value MiddleToken(const Napi::CallbackInfo& info) {
             if (disposed) {
@@ -271,7 +319,7 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
                 return info.Env().Undefined();
             }
 
-            return Napi::Number::From(info.Env(), llama_token_middle(model));
+            return getNapiControlToken(info, model, llama_token_middle(model));
         }
         Napi::Value SuffixToken(const Napi::CallbackInfo& info) {
             if (disposed) {
@@ -279,7 +327,7 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
                 return info.Env().Undefined();
             }
 
-            return Napi::Number::From(info.Env(), llama_token_suffix(model));
+            return getNapiControlToken(info, model, llama_token_suffix(model));
         }
         Napi::Value EotToken(const Napi::CallbackInfo& info) {
             if (disposed) {
@@ -287,7 +335,7 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
                 return info.Env().Undefined();
             }
 
-            return Napi::Number::From(info.Env(), llama_token_eot(model));
+            return getNapiControlToken(info, model, llama_token_eot(model));
         }
         Napi::Value GetTokenString(const Napi::CallbackInfo& info) {
             if (disposed) {
@@ -308,6 +356,29 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
             return Napi::String::New(info.Env(), ss.str());
         }
 
+        Napi::Value GetTokenType(const Napi::CallbackInfo& info) {
+            if (disposed) {
+                Napi::Error::New(info.Env(), "Context is disposed").ThrowAsJavaScriptException();
+                return info.Env().Undefined();
+            }
+
+            if (info[0].IsNumber() == false) {
+                return Napi::Number::From(info.Env(), int32_t(LLAMA_TOKEN_TYPE_UNDEFINED));
+            }
+
+            int token = info[0].As<Napi::Number>().Int32Value();
+            auto tokenType = llama_token_get_type(model, token);
+
+            return Napi::Number::From(info.Env(), int32_t(tokenType));
+        }
+        Napi::Value ShouldPrependBosToken(const Napi::CallbackInfo& info) {
+            const int addBos = llama_add_bos_token(model);
+
+            bool shouldPrependBos = addBos != -1 ? bool(addBos) : (llama_vocab_type(model) == LLAMA_VOCAB_TYPE_SPM);
+
+            return Napi::Boolean::New(info.Env(), shouldPrependBos);
+        }
+
         static void init(Napi::Object exports) {
             exports.Set(
                 "AddonModel",
@@ -318,6 +389,7 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
                         InstanceMethod("tokenize", &AddonModel::Tokenize),
                         InstanceMethod("detokenize", &AddonModel::Detokenize),
                         InstanceMethod("getTrainContextSize", &AddonModel::GetTrainContextSize),
+                        InstanceMethod("getEmbeddingVectorSize", &AddonModel::GetEmbeddingVectorSize),
                         InstanceMethod("getTotalSize", &AddonModel::GetTotalSize),
                         InstanceMethod("getTotalParameters", &AddonModel::GetTotalParameters),
                         InstanceMethod("getModelDescription", &AddonModel::GetModelDescription),
@@ -329,6 +401,8 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
                         InstanceMethod("suffixToken", &AddonModel::SuffixToken),
                         InstanceMethod("eotToken", &AddonModel::EotToken),
                         InstanceMethod("getTokenString", &AddonModel::GetTokenString),
+                        InstanceMethod("getTokenType", &AddonModel::GetTokenType),
+                        InstanceMethod("shouldPrependBosToken", &AddonModel::ShouldPrependBosToken),
                         InstanceMethod("dispose", &AddonModel::Dispose),
                     }
                 )
@@ -896,7 +970,7 @@ void addonCallJsLogCallback(
             called = false;
         }
     }
-    
+
     if (!called && data != nullptr) {
         if (data->logLevelNumber == 2) {
             fputs(data->stringStream->str().c_str(), stderr);
@@ -992,8 +1066,17 @@ Napi::Value setLoggerLogLevel(const Napi::CallbackInfo& info) {
     return info.Env().Undefined();
 }
 
+static void addonFreeLlamaBackend(Napi::Env env, int* data) {
+    if (backendInitialized) {
+        llama_backend_free();
+        backendInitialized = false;
+    }
+}
+
 Napi::Object registerCallback(Napi::Env env, Napi::Object exports) {
-    llama_backend_init(false);
+    llama_backend_init();
+    backendInitialized = true;
+
     exports.DefineProperties({
         Napi::PropertyDescriptor::Function("systemInfo", systemInfo),
         Napi::PropertyDescriptor::Function("setLogger", setLogger),
@@ -1007,6 +1090,8 @@ Napi::Object registerCallback(Napi::Env env, Napi::Object exports) {
 
     llama_log_set(addonLlamaCppLogCallback, nullptr);
 
+    exports.AddFinalizer(addonFreeLlamaBackend, static_cast<int*>(nullptr));
+
     return exports;
 }
 

package/llama/binariesGithubRelease.json

@@ -1,3 +1,3 @@
 {
-    "release": "b2127"
+    "release": "b2254"
 }

package/llama/gpuInfo/cuda-gpu-info.cu

@@ -15,9 +15,9 @@
 #endif
 
 
-typedef void (*gpuInfoErrorLogCallback_t)(const char* message);
+typedef void (*gpuInfoCudaErrorLogCallback_t)(const char* message);
 
-bool gpuInfoSetCudaDevice(const int device, gpuInfoErrorLogCallback_t errorLogCallback) {
+bool gpuInfoSetCudaDevice(const int device, gpuInfoCudaErrorLogCallback_t errorLogCallback) {
     int current_device;
     auto getDeviceResult = cudaGetDevice(&current_device);
 
@@ -40,7 +40,7 @@ bool gpuInfoSetCudaDevice(const int device, gpuInfoErrorLogCallback_t errorLogCa
     return true;
 }
 
-bool gpuInfoGetCudaDeviceInfo(int device, size_t * total, size_t * used, gpuInfoErrorLogCallback_t errorLogCallback) {
+bool gpuInfoGetCudaDeviceInfo(int device, size_t * total, size_t * used, gpuInfoCudaErrorLogCallback_t errorLogCallback) {
     gpuInfoSetCudaDevice(device, errorLogCallback);
 
     size_t freeMem;
@@ -58,7 +58,7 @@ bool gpuInfoGetCudaDeviceInfo(int device, size_t * total, size_t * used, gpuInfo
     return true;
 }
 
-int gpuInfoGetCudaDeviceCount(gpuInfoErrorLogCallback_t errorLogCallback) {
+int gpuInfoGetCudaDeviceCount(gpuInfoCudaErrorLogCallback_t errorLogCallback) {
     int deviceCount;
     auto getDeviceCountResult = cudaGetDeviceCount(&deviceCount);
 
@@ -70,7 +70,7 @@ int gpuInfoGetCudaDeviceCount(gpuInfoErrorLogCallback_t errorLogCallback) {
     return deviceCount;
 }
 
-bool gpuInfoGetTotalCudaDevicesInfo(size_t * total, size_t * used, gpuInfoErrorLogCallback_t errorLogCallback) {
+bool gpuInfoGetTotalCudaDevicesInfo(size_t * total, size_t * used, gpuInfoCudaErrorLogCallback_t errorLogCallback) {
     int deviceCount = gpuInfoGetCudaDeviceCount(errorLogCallback);
 
     if (deviceCount < 0) {

package/llama/gpuInfo/cuda-gpu-info.h

@@ -2,6 +2,6 @@
 
 #include <stddef.h>
 
-typedef void (*gpuInfoErrorLogCallback_t)(const char* message);
+typedef void (*gpuInfoCudaErrorLogCallback_t)(const char* message);
 
-bool gpuInfoGetTotalCudaDevicesInfo(size_t * total, size_t * used, gpuInfoErrorLogCallback_t errorLogCallback);
+bool gpuInfoGetTotalCudaDevicesInfo(size_t * total, size_t * used, gpuInfoCudaErrorLogCallback_t errorLogCallback);

package/llama/gpuInfo/vulkan-gpu-info.cpp

@@ -0,0 +1,65 @@
+#include <stddef.h>
+
+#include <vulkan/vulkan.hpp>
+
+typedef void (*gpuInfoVulkanWarningLogCallback_t)(const char* message);
+
+bool gpuInfoGetTotalVulkanDevicesInfo(size_t* total, size_t* used, gpuInfoVulkanWarningLogCallback_t warningLogCallback) {
+    vk::ApplicationInfo appInfo("node-llama-cpp GPU info", 1, "llama.cpp", 1, VK_API_VERSION_1_2);
+    vk::InstanceCreateInfo createInfo(vk::InstanceCreateFlags(), &appInfo, {}, {});
+    vk::Instance instance = vk::createInstance(createInfo);
+
+    auto physicalDevices = instance.enumeratePhysicalDevices();
+
+    size_t usedMem = 0;
+    size_t totalMem = 0;
+
+    for (size_t i = 0; i < physicalDevices.size(); i++) {
+        vk::PhysicalDevice physicalDevice = physicalDevices[i];
+        vk::PhysicalDeviceMemoryProperties memProps = physicalDevice.getMemoryProperties();
+        vk::PhysicalDeviceProperties deviceProps = physicalDevice.getProperties();
+
+        if (deviceProps.deviceType == vk::PhysicalDeviceType::eCpu) {
+            // ignore CPU devices, as we don't want to count RAM from the CPU as VRAM
+            continue;
+        }
+
+        std::vector<vk::ExtensionProperties> extensionProperties = physicalDevice.enumerateDeviceExtensionProperties();
+        bool memoryBudgetExtensionSupported =
+            std::any_of(
+                extensionProperties.begin(),
+                extensionProperties.end(),
+                [](const vk::ExtensionProperties& ext) { return std::string(ext.extensionName.data()) == VK_EXT_MEMORY_BUDGET_EXTENSION_NAME;}
+            );
+
+        if (memoryBudgetExtensionSupported) {
+            vk::PhysicalDeviceMemoryBudgetPropertiesEXT memoryBudgetProperties;
+            vk::PhysicalDeviceMemoryProperties2 memProps2 = {};
+            memProps2.pNext = &memoryBudgetProperties;
+
+            physicalDevice.getMemoryProperties2(&memProps2);
+
+            for (uint32_t i = 0; i < memProps.memoryHeapCount; ++i) {
+                if (memProps.memoryHeaps[i].flags & vk::MemoryHeapFlagBits::eDeviceLocal) {
+                    totalMem += memProps.memoryHeaps[i].size;
+                    usedMem += memoryBudgetProperties.heapUsage[i];
+                    break;
+                }
+            }
+        } else {
+            // VK_EXT_memory_budget extension is not supported, so we cannot determine used memory
+            warningLogCallback(
+                (
+                    "Vulkan VK_EXT_memory_budget extension not supported for device \"" +
+                    std::string(deviceProps.deviceName.data()) + "\", so VRAM info cannot be determained for it"
+                )
+                    .c_str()
+            );
+            return false;
+        }
+    }
+
+    *total = totalMem;
+    *used = usedMem;
+    return true;
+}

package/llama/gpuInfo/vulkan-gpu-info.h

@@ -0,0 +1,7 @@
+#pragma once
+
+#include <stddef.h>
+
+typedef void (*gpuInfoVulkanWarningLogCallback_t)(const char* message);
+
+bool gpuInfoGetTotalVulkanDevicesInfo(size_t* total, size_t* used, gpuInfoVulkanWarningLogCallback_t warningLogCallback);

package/llama/llama.cpp.info.json

@@ -1,4 +1,4 @@
 {
-    "tag": "b2127",
+    "tag": "b2254",
     "llamaCppGithubRepo": "ggerganov/llama.cpp"
 }

package/llamaBins/linux-arm64/.buildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","arch":"arm64","computeLayers":{"metal":false,"cuda":false},"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2127"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","arch":"arm64","computeLayers":{"metal":false,"cuda":false,"vulkan":false},"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2254"}}}

package/llamaBins/linux-armv7l/.buildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","arch":"armv7l","computeLayers":{"metal":false,"cuda":false},"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2127"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","arch":"armv7l","computeLayers":{"metal":false,"cuda":false,"vulkan":false},"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2254"}}}

package/llamaBins/linux-x64/.buildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","arch":"x64","computeLayers":{"metal":false,"cuda":false},"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2127"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","arch":"x64","computeLayers":{"metal":false,"cuda":false,"vulkan":false},"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2254"}}}

package/llamaBins/linux-x64-cuda/.buildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","arch":"x64","computeLayers":{"metal":false,"cuda":true},"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2127"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","arch":"x64","computeLayers":{"metal":false,"cuda":true,"vulkan":false},"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2254"}}}

package/llamaBins/linux-x64-vulkan/.buildMetadata.json

@@ -0,0 +1 @@
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","arch":"x64","computeLayers":{"metal":false,"cuda":false,"vulkan":true},"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2254"}}}

package/llamaBins/mac-arm64-metal/.buildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"mac","arch":"arm64","computeLayers":{"metal":true,"cuda":false},"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2127"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"mac","arch":"arm64","computeLayers":{"metal":true,"cuda":false,"vulkan":false},"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2254"}}}