node-llama-cpp 3.0.0-beta.11 → 3.0.0-beta.13

package/dist/utils/parseTextTemplate.js

@@ -0,0 +1,116 @@
+import { splitText } from "lifecycle-utils";
+/**
+ * Parses a text template into a map of parts and their prefixes and suffixes.
+ * This parser assumes each part occurs exactly once in the template, and that all parts must occur in the order they are defined.
+ * For example,
+ * ```typescript
+ * const res = parseTextTemplate(
+ *     "Hello, {{name}}! What is the {{thing}}?",
+ *     [{
+ *         key: "name",
+ *         text: "{{name}}"
+ *     }, {
+ *         key: "thing",
+ *         text: "{{thing}}"
+ *     }]
+ * );
+ * expect(res).to.eql({
+ *     name: {
+ *         prefix: "Hello, ",
+ *         suffix: "! What is the "
+ *     },
+ *     thing: {
+ *         prefix: "What is the ",
+ *         suffix: "?"
+ *     }
+ * });
+ * ```
+ *
+ * ```typescript
+ * const res2 = parseTextTemplate(
+ *     "What is the {{thing}}?",
+ *     [{
+ *         key: "name",
+ *         text: "{{name}}",
+ *         optional: true
+ *     }, {
+ *         key: "thing",
+ *         text: "{{thing}}"
+ *     }]
+ * );
+ * expect(res2).to.eql({
+ *     thing: {
+ *         prefix: "What is the ",
+ *         suffix: "?"
+ *     }
+ * });
+ * ```
+ */
+export function parseTextTemplate(template, parts) {
+    const result = {};
+    const templateParts = splitText(template, parts.map((part) => part.text));
+    let partIndex = 0;
+    for (let i = 0; i < templateParts.length; i++) {
+        const textPart = templateParts[i];
+        if (typeof textPart === "string")
+            continue;
+        for (; partIndex < parts.length; partIndex++) {
+            const part = parts[partIndex];
+            if (textPart.separator === part.text) {
+                const previousItem = i > 0
+                    ? templateParts[i - 1]
+                    : null;
+                const nextItem = i < templateParts.length - 1
+                    ? templateParts[i + 1]
+                    : null;
+                result[part.key] = {
+                    prefix: typeof previousItem === "string"
+                        ? previousItem
+                        : "",
+                    suffix: typeof nextItem === "string"
+                        ? nextItem
+                        : ""
+                };
+                partIndex++;
+                break;
+            }
+            if (part.optional != true) {
+                if (result[part.key] != null)
+                    throw new Error(`Template must contain exactly one "${part.text}"`);
+                else if (partIndex > 0) {
+                    const previousNonOptionalOrFoundPart = parts
+                        .slice(0, partIndex)
+                        .reverse()
+                        .find((p) => (p.optional != true || result[p.key] != null));
+                    if (previousNonOptionalOrFoundPart != null)
+                        throw new Error(`Template must contain "${part.text}" after "${previousNonOptionalOrFoundPart.text}"`);
+                    throw new Error(`Template must contain "${part.text}" at the beginning`);
+                }
+                else
+                    throw new Error(`Template must contain "${part.text}" at the beginning`);
+            }
+            else
+                result[part.key] = undefined;
+        }
+    }
+    for (; partIndex < parts.length; partIndex++) {
+        const part = parts[partIndex];
+        if (part.optional == true) {
+            result[part.key] = undefined;
+            continue;
+        }
+        if (partIndex > 0) {
+            const previousNonOptionalOrFoundPart = parts
+                .slice(0, partIndex)
+                .reverse()
+                .find((p) => (p.optional != true || result[p.key] != null));
+            if (previousNonOptionalOrFoundPart != null)
+                throw new Error(`Template must contain "${part.text}" after "${previousNonOptionalOrFoundPart.text}"`);
+            throw new Error(`Template must contain "${part.text}" at the beginning`);
+        }
+        else
+            throw new Error(`Template must contain "${part.text}" at the beginning`);
+    }
+    return result;
+}
+//# sourceMappingURL=parseTextTemplate.js.map

package/dist/utils/parseTextTemplate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parseTextTemplate.js","sourceRoot":"","sources":["../../src/utils/parseTextTemplate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAG1C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,MAAM,UAAU,iBAAiB,CAC7B,QAAgB,EAAE,KAAY;IAE9B,MAAM,MAAM,GAKR,EAAE,CAAC;IAEP,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAE1E,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QAC3C,MAAM,QAAQ,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;QAElC,IAAI,OAAO,QAAQ,KAAK,QAAQ;YAC5B,SAAS;QAEb,OAAO,SAAS,GAAG,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE;YAC1C,MAAM,IAAI,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;YAE9B,IAAI,QAAQ,CAAC,SAAS,KAAK,IAAI,CAAC,IAAI,EAAE;gBAClC,MAAM,YAAY,GAAG,CAAC,GAAG,CAAC;oBACtB,CAAC,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,CAAC;oBACtB,CAAC,CAAC,IAAI,CAAC;gBACX,MAAM,QAAQ,GAAG,CAAC,GAAG,aAAa,CAAC,MAAM,GAAG,CAAC;oBACzC,CAAC,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,CAAC;oBACtB,CAAC,CAAC,IAAI,CAAC;gBAEX,MAAM,CAAC,IAAI,CAAC,GAA2B,CAAC,GAAG;oBACvC,MAAM,EAAE,OAAO,YAAY,KAAK,QAAQ;wBACpC,CAAC,CAAC,YAAY;wBACd,CAAC,CAAC,EAAE;oBACR,MAAM,EAAE,OAAO,QAAQ,KAAK,QAAQ;wBAChC,CAAC,CAAC,QAAQ;wBACV,CAAC,CAAC,EAAE;iBACX,CAAC;gBACF,SAAS,EAAE,CAAC;gBACZ,MAAM;aACT;YAED,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE;gBACvB,IAAI,MAAM,CAAC,IAAI,CAAC,GAA2B,CAAC,IAAI,IAAI;oBAChD,MAAM,IAAI,KAAK,CAAC,sCAAsC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;qBACnE,IAAI,SAAS,GAAG,CAAC,EAAE;oBACpB,MAAM,8BAA8B,GAAG,KAAK;yBACvC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC;yBACnB,OAAO,EAAE;yBACT,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC,GAA2B,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;oBAExF,IAAI,8BAA8B,IAAI,IAAI;wBACtC,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,CAAC,IAAI,YAAY,8BAA8B,CAAC,IAAI,GAAG,CAAC,CAAC;oBAE3G,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,CAAC,IAAI,oBAAoB,CAAC,CAAC;iBAC5E;;oBACG,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,CAAC,IAAI,oBAAoB,CAAC,CAAC;aAChF;;gBACG,MAAM,CAAC,IAAI,CAAC,GAA2B,CAAC,GAAG,SAAS,CAAC;SAC5D;KACJ;IAED,OAAO,SAAS,GAAG,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE;QAC1C,MAAM,IAAI,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;QAE9B,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE;YACvB,MAAM,CAAC,IAAI,CAAC,GAA2B,CAAC,GAAG,SAAS,CAAC;YACrD,SAAS;SACZ;QAED,IAAI,SAAS,GAAG,CAAC,EAAE;YACf,MAAM,8BAA8B,GAAG,KAAK;iBACvC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC;iBACnB,OAAO,EAAE;iBACT,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC,GAA2B,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;YAExF,IAAI,8BAA8B,IAAI,IAAI;gBACtC,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,CAAC,IAAI,YAAY,8BAA8B,CAAC,IAAI,GAAG,CAAC,CAAC;YAE3G,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,CAAC,IAAI,oBAAoB,CAAC,CAAC;SAC5E;;YACG,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,CAAC,IAAI,oBAAoB,CAAC,CAAC;KAChF;IAED,OAAO,MAA0C,CAAC;AACtD,CAAC"}

package/llama/CMakeLists.txt

@@ -3,9 +3,15 @@ cmake_minimum_required(VERSION 3.13)
 project("llama-addon" C CXX)
 
 if (MSVC)
-  # add_compile_options(/EHsc)
+    if (LLAMA_STATIC)
+        add_link_options(-static)
+        if (MINGW)
+            add_link_options(-static-libgcc -static-libstdc++)
+        endif()
+    endif()
+    # add_compile_options(/EHsc)
 else()
-  add_compile_options(-fexceptions)
+    add_compile_options(-fexceptions)
 endif()
 
 add_definitions(-DNAPI_VERSION=7)
@@ -54,7 +60,27 @@ if (LLAMA_CUBLAS)
             endif()
         endif()
     else()
-        message(WARNING "cuBLAS not found. Not using it for GPU info")
+        message(FATAL_ERROR "cuBLAS was not found")
+    endif()
+endif()
+
+if (LLAMA_VULKAN OR LLAMA_KOMPUTE)
+    find_package(Vulkan)
+    if (Vulkan_FOUND)
+        if (LLAMA_VULKAN)
+            message(STATUS "Using Vulkan for GPU info")
+        elseif (LLAMA_KOMPUTE)
+            message(STATUS "Using Vulkan for GPU info because Kompute is enabled")
+        endif()
+
+        set(GPU_INFO_HEADERS ${GPU_INFO_HEADERS} gpuInfo/vulkan-gpu-info.h)
+        set(GPU_INFO_SOURCES ${GPU_INFO_SOURCES} gpuInfo/vulkan-gpu-info.cpp)
+
+        add_compile_definitions(GPU_INFO_USE_VULKAN)
+
+        set(GPU_INFO_EXTRA_LIBS ${GPU_INFO_EXTRA_LIBS} Vulkan::Vulkan)
+    else()
+        message(FATAL_ERROR "Vulkan was not found")
     endif()
 endif()
 
@@ -81,7 +107,7 @@ if (LLAMA_HIPBLAS)
 
         set(GPU_INFO_EXTRA_LIBS ${GPU_INFO_EXTRA_LIBS} gpu-info-rocm)
     else()
-        message(WARNING "hipBLAS or HIP not found. Try setting CMAKE_PREFIX_PATH=/opt/rocm")
+        message(FATAL_ERROR "hipBLAS or HIP was not found. Try setting CMAKE_PREFIX_PATH=/opt/rocm")
     endif()
 endif()
 

package/llama/addon.cpp

@@ -12,6 +12,9 @@
 #ifdef GPU_INFO_USE_CUBLAS
 #  include "gpuInfo/cuda-gpu-info.h"
 #endif
+#ifdef GPU_INFO_USE_VULKAN
+#  include "gpuInfo/vulkan-gpu-info.h"
+#endif
 #ifdef GPU_INFO_USE_METAL
 #  include "gpuInfo/metal-gpu-info.h"
 #endif
@@ -35,6 +38,7 @@ using AddonThreadSafeLogCallbackFunction =
 AddonThreadSafeLogCallbackFunction addonThreadSafeLoggerCallback;
 bool addonJsLoggerCallbackSet = false;
 int addonLoggerLogLevel = 5;
+bool backendInitialized = false;
 
 std::string addon_model_token_to_piece(const struct llama_model* model, llama_token token) {
     std::vector<char> result(8, 0);
@@ -51,10 +55,15 @@ std::string addon_model_token_to_piece(const struct llama_model* model, llama_to
 }
 
 #ifdef GPU_INFO_USE_CUBLAS
-void lodCudaError(const char* message) {
+void logCudaError(const char* message) {
     addonLlamaCppLogCallback(GGML_LOG_LEVEL_ERROR, (std::string("CUDA error: ") + std::string(message)).c_str(), nullptr);
 }
 #endif
+#ifdef GPU_INFO_USE_VULKAN
+void logVulkanWarning(const char* message) {
+    addonLlamaCppLogCallback(GGML_LOG_LEVEL_WARN, (std::string("Vulkan warning: ") + std::string(message)).c_str(), nullptr);
+}
+#endif
 
 Napi::Value getGpuVramInfo(const Napi::CallbackInfo& info) {
     uint64_t total = 0;
@@ -63,7 +72,7 @@ Napi::Value getGpuVramInfo(const Napi::CallbackInfo& info) {
 #ifdef GPU_INFO_USE_CUBLAS
     size_t cudaDeviceTotal = 0;
     size_t cudaDeviceUsed = 0;
-    bool cudeGetInfoSuccess = gpuInfoGetTotalCudaDevicesInfo(&cudaDeviceTotal, &cudaDeviceUsed, lodCudaError);
+    bool cudeGetInfoSuccess = gpuInfoGetTotalCudaDevicesInfo(&cudaDeviceTotal, &cudaDeviceUsed, logCudaError);
 
     if (cudeGetInfoSuccess) {
         total += cudaDeviceTotal;
@@ -71,6 +80,17 @@ Napi::Value getGpuVramInfo(const Napi::CallbackInfo& info) {
     }
 #endif
 
+#ifdef GPU_INFO_USE_VULKAN
+    uint64_t vulkanDeviceTotal = 0;
+    uint64_t vulkanDeviceUsed = 0;
+    const bool vulkanDeviceSupportsMemoryBudgetExtension = gpuInfoGetTotalVulkanDevicesInfo(&vulkanDeviceTotal, &vulkanDeviceUsed, logVulkanWarning);
+
+    if (vulkanDeviceSupportsMemoryBudgetExtension) {
+        total += vulkanDeviceTotal;
+        used += vulkanDeviceUsed;
+    }
+#endif
+
 #ifdef GPU_INFO_USE_METAL
     uint64_t metalDeviceTotal = 0;
     uint64_t metalDeviceUsed = 0;
@@ -87,6 +107,22 @@ Napi::Value getGpuVramInfo(const Napi::CallbackInfo& info) {
     return result;
 }
 
+Napi::Value getGpuType(const Napi::CallbackInfo& info) {
+#ifdef GPU_INFO_USE_CUBLAS
+    return Napi::String::New(info.Env(), "cuda");
+#endif
+
+#ifdef GPU_INFO_USE_VULKAN
+    return Napi::String::New(info.Env(), "vulkan");
+#endif
+
+#ifdef GPU_INFO_USE_METAL
+    return Napi::String::New(info.Env(), "metal");
+#endif
+
+    return info.Env().Undefined();
+}
+
 static Napi::Value getNapiToken(const Napi::CallbackInfo& info, llama_model* model, llama_token token) {
     auto tokenType = llama_token_get_type(model, token);
 
@@ -100,7 +136,7 @@ static Napi::Value getNapiToken(const Napi::CallbackInfo& info, llama_model* mod
 static Napi::Value getNapiControlToken(const Napi::CallbackInfo& info, llama_model* model, llama_token token) {
     auto tokenType = llama_token_get_type(model, token);
 
-    if (tokenType != LLAMA_TOKEN_TYPE_CONTROL) {
+    if (tokenType != LLAMA_TOKEN_TYPE_CONTROL && tokenType != LLAMA_TOKEN_TYPE_USER_DEFINED) {
         return Napi::Number::From(info.Env(), -1);
     }
 
@@ -641,7 +677,7 @@ class AddonContext : public Napi::ObjectWrap<AddonContext> {
             int32_t endPos = info[2].As<Napi::Number>().Int32Value();
             int32_t shiftDelta = info[3].As<Napi::Number>().Int32Value();
 
-            llama_kv_cache_seq_shift(ctx, sequenceId, startPos, endPos, shiftDelta);
+            llama_kv_cache_seq_add(ctx, sequenceId, startPos, endPos, shiftDelta);
 
             return info.Env().Undefined();
         }
@@ -950,7 +986,7 @@ void addonCallJsLogCallback(
             called = false;
         }
     }
-    
+
     if (!called && data != nullptr) {
         if (data->logLevelNumber == 2) {
             fputs(data->stringStream->str().c_str(), stderr);
@@ -1046,20 +1082,39 @@ Napi::Value setLoggerLogLevel(const Napi::CallbackInfo& info) {
     return info.Env().Undefined();
 }
 
+Napi::Value addonInit(const Napi::CallbackInfo& info) {
+    if (!backendInitialized) {
+        llama_backend_init();
+        backendInitialized = true;
+    }
+
+    llama_log_set(addonLlamaCppLogCallback, nullptr);
+
+    return info.Env().Undefined();
+}
+
+static void addonFreeLlamaBackend(Napi::Env env, int* data) {
+    if (backendInitialized) {
+        llama_backend_free();
+        backendInitialized = false;
+    }
+}
+
 Napi::Object registerCallback(Napi::Env env, Napi::Object exports) {
-    llama_backend_init();
     exports.DefineProperties({
         Napi::PropertyDescriptor::Function("systemInfo", systemInfo),
         Napi::PropertyDescriptor::Function("setLogger", setLogger),
         Napi::PropertyDescriptor::Function("setLoggerLogLevel", setLoggerLogLevel),
         Napi::PropertyDescriptor::Function("getGpuVramInfo", getGpuVramInfo),
+        Napi::PropertyDescriptor::Function("getGpuType", getGpuType),
+        Napi::PropertyDescriptor::Function("init", addonInit),
     });
     AddonModel::init(exports);
     AddonGrammar::init(exports);
     AddonGrammarEvaluationState::init(exports);
     AddonContext::init(exports);
 
-    llama_log_set(addonLlamaCppLogCallback, nullptr);
+    exports.AddFinalizer(addonFreeLlamaBackend, static_cast<int*>(nullptr));
 
     return exports;
 }

package/llama/binariesGithubRelease.json

@@ -1,3 +1,3 @@
 {
-    "release": "b2174"
+    "release": "b2329"
 }

package/llama/gpuInfo/cuda-gpu-info.cu

@@ -15,9 +15,9 @@
 #endif
 
 
-typedef void (*gpuInfoErrorLogCallback_t)(const char* message);
+typedef void (*gpuInfoCudaErrorLogCallback_t)(const char* message);
 
-bool gpuInfoSetCudaDevice(const int device, gpuInfoErrorLogCallback_t errorLogCallback) {
+bool gpuInfoSetCudaDevice(const int device, gpuInfoCudaErrorLogCallback_t errorLogCallback) {
     int current_device;
     auto getDeviceResult = cudaGetDevice(&current_device);
 
@@ -40,7 +40,7 @@ bool gpuInfoSetCudaDevice(const int device, gpuInfoErrorLogCallback_t errorLogCa
     return true;
 }
 
-bool gpuInfoGetCudaDeviceInfo(int device, size_t * total, size_t * used, gpuInfoErrorLogCallback_t errorLogCallback) {
+bool gpuInfoGetCudaDeviceInfo(int device, size_t * total, size_t * used, gpuInfoCudaErrorLogCallback_t errorLogCallback) {
     gpuInfoSetCudaDevice(device, errorLogCallback);
 
     size_t freeMem;
@@ -58,7 +58,7 @@ bool gpuInfoGetCudaDeviceInfo(int device, size_t * total, size_t * used, gpuInfo
     return true;
 }
 
-int gpuInfoGetCudaDeviceCount(gpuInfoErrorLogCallback_t errorLogCallback) {
+int gpuInfoGetCudaDeviceCount(gpuInfoCudaErrorLogCallback_t errorLogCallback) {
     int deviceCount;
     auto getDeviceCountResult = cudaGetDeviceCount(&deviceCount);
 
@@ -70,7 +70,7 @@ int gpuInfoGetCudaDeviceCount(gpuInfoErrorLogCallback_t errorLogCallback) {
     return deviceCount;
 }
 
-bool gpuInfoGetTotalCudaDevicesInfo(size_t * total, size_t * used, gpuInfoErrorLogCallback_t errorLogCallback) {
+bool gpuInfoGetTotalCudaDevicesInfo(size_t * total, size_t * used, gpuInfoCudaErrorLogCallback_t errorLogCallback) {
     int deviceCount = gpuInfoGetCudaDeviceCount(errorLogCallback);
 
     if (deviceCount < 0) {

package/llama/gpuInfo/cuda-gpu-info.h

@@ -2,6 +2,6 @@
 
 #include <stddef.h>
 
-typedef void (*gpuInfoErrorLogCallback_t)(const char* message);
+typedef void (*gpuInfoCudaErrorLogCallback_t)(const char* message);
 
-bool gpuInfoGetTotalCudaDevicesInfo(size_t * total, size_t * used, gpuInfoErrorLogCallback_t errorLogCallback);
+bool gpuInfoGetTotalCudaDevicesInfo(size_t * total, size_t * used, gpuInfoCudaErrorLogCallback_t errorLogCallback);

package/llama/gpuInfo/vulkan-gpu-info.cpp

@@ -0,0 +1,65 @@
+#include <stddef.h>
+
+#include <vulkan/vulkan.hpp>
+
+typedef void (*gpuInfoVulkanWarningLogCallback_t)(const char* message);
+
+bool gpuInfoGetTotalVulkanDevicesInfo(size_t* total, size_t* used, gpuInfoVulkanWarningLogCallback_t warningLogCallback) {
+    vk::ApplicationInfo appInfo("node-llama-cpp GPU info", 1, "llama.cpp", 1, VK_API_VERSION_1_2);
+    vk::InstanceCreateInfo createInfo(vk::InstanceCreateFlags(), &appInfo, {}, {});
+    vk::Instance instance = vk::createInstance(createInfo);
+
+    auto physicalDevices = instance.enumeratePhysicalDevices();
+
+    size_t usedMem = 0;
+    size_t totalMem = 0;
+
+    for (size_t i = 0; i < physicalDevices.size(); i++) {
+        vk::PhysicalDevice physicalDevice = physicalDevices[i];
+        vk::PhysicalDeviceMemoryProperties memProps = physicalDevice.getMemoryProperties();
+        vk::PhysicalDeviceProperties deviceProps = physicalDevice.getProperties();
+
+        if (deviceProps.deviceType == vk::PhysicalDeviceType::eCpu) {
+            // ignore CPU devices, as we don't want to count RAM from the CPU as VRAM
+            continue;
+        }
+
+        std::vector<vk::ExtensionProperties> extensionProperties = physicalDevice.enumerateDeviceExtensionProperties();
+        bool memoryBudgetExtensionSupported =
+            std::any_of(
+                extensionProperties.begin(),
+                extensionProperties.end(),
+                [](const vk::ExtensionProperties& ext) { return std::string(ext.extensionName.data()) == VK_EXT_MEMORY_BUDGET_EXTENSION_NAME;}
+            );
+
+        if (memoryBudgetExtensionSupported) {
+            vk::PhysicalDeviceMemoryBudgetPropertiesEXT memoryBudgetProperties;
+            vk::PhysicalDeviceMemoryProperties2 memProps2 = {};
+            memProps2.pNext = &memoryBudgetProperties;
+
+            physicalDevice.getMemoryProperties2(&memProps2);
+
+            for (uint32_t i = 0; i < memProps.memoryHeapCount; ++i) {
+                if (memProps.memoryHeaps[i].flags & vk::MemoryHeapFlagBits::eDeviceLocal) {
+                    totalMem += memProps.memoryHeaps[i].size;
+                    usedMem += memoryBudgetProperties.heapUsage[i];
+                    break;
+                }
+            }
+        } else {
+            // VK_EXT_memory_budget extension is not supported, so we cannot determine used memory
+            warningLogCallback(
+                (
+                    "Vulkan VK_EXT_memory_budget extension not supported for device \"" +
+                    std::string(deviceProps.deviceName.data()) + "\", so VRAM info cannot be determained for it"
+                )
+                    .c_str()
+            );
+            return false;
+        }
+    }
+
+    *total = totalMem;
+    *used = usedMem;
+    return true;
+}

package/llama/gpuInfo/vulkan-gpu-info.h

@@ -0,0 +1,7 @@
+#pragma once
+
+#include <stddef.h>
+
+typedef void (*gpuInfoVulkanWarningLogCallback_t)(const char* message);
+
+bool gpuInfoGetTotalVulkanDevicesInfo(size_t* total, size_t* used, gpuInfoVulkanWarningLogCallback_t warningLogCallback);

package/llama/llama.cpp.info.json

@@ -1,4 +1,4 @@
 {
-    "tag": "b2174",
+    "tag": "b2329",
     "llamaCppGithubRepo": "ggerganov/llama.cpp"
 }

package/llamaBins/linux-arm64/.buildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","arch":"arm64","computeLayers":{"metal":false,"cuda":false},"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2174"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","platformInfo":{"name":"Ubuntu","version":"22.04"},"arch":"arm64","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2329"}}}

package/llamaBins/linux-armv7l/.buildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","arch":"armv7l","computeLayers":{"metal":false,"cuda":false},"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2174"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","platformInfo":{"name":"Ubuntu","version":"22.04"},"arch":"armv7l","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2329"}}}

package/llamaBins/linux-x64/.buildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","arch":"x64","computeLayers":{"metal":false,"cuda":false},"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2174"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","platformInfo":{"name":"Ubuntu","version":"22.04"},"arch":"x64","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2329"}}}

package/llamaBins/linux-x64-cuda/.buildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","arch":"x64","computeLayers":{"metal":false,"cuda":true},"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2174"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","platformInfo":{"name":"Ubuntu","version":"22.04"},"arch":"x64","gpu":"cuda","llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2329"}}}

package/llamaBins/linux-x64-vulkan/.buildMetadata.json

@@ -0,0 +1 @@
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","platformInfo":{"name":"Ubuntu","version":"22.04"},"arch":"x64","gpu":"vulkan","llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2329"}}}

package/llamaBins/mac-arm64-metal/.buildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"mac","arch":"arm64","computeLayers":{"metal":true,"cuda":false},"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2174"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"mac","platformInfo":{"name":"macOS","version":"21.6.0"},"arch":"arm64","gpu":"metal","llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2329"}}}