node-llama-cpp 3.3.2 → 3.4.1

package/llama/addon/AddonModelLora.cpp

@@ -30,7 +30,7 @@ class AddonModelLoraUnloadLoraWorker : public Napi::AsyncWorker {
             } catch (const std::exception& e) {
                 SetError(e.what());
             } catch(...) {
-                SetError("Unknown error when calling \"llama_lora_adapter_free\"");
+                SetError("Unknown error when calling \"llama_adapter_lora_free\"");
             }
         }
         void OnOK() {
@@ -55,7 +55,7 @@ void AddonModelLora::dispose(bool skipErase) {
     if (lora_adapter != nullptr) {
         auto loraAdapterToDispose = lora_adapter;
         lora_adapter = nullptr;
-        llama_lora_adapter_free(loraAdapterToDispose);
+        llama_adapter_lora_free(loraAdapterToDispose);
         
         if (!skipErase && model->data != nullptr) {
             model->data->removeLora(this);

package/llama/addon/AddonModelLora.h

@@ -6,7 +6,7 @@
 class AddonModelLora : public Napi::ObjectWrap<AddonModelLora> {
     public:
         AddonModel* model;
-        llama_lora_adapter * lora_adapter;
+        llama_adapter_lora * lora_adapter;
         std::string loraFilePath;
         uint32_t usages = 0;
 

package/llama/addon/AddonSampler.cpp

@@ -10,8 +10,8 @@ AddonSampler::AddonSampler(const Napi::CallbackInfo& info) : Napi::ObjectWrap<Ad
     model = Napi::ObjectWrap<AddonModel>::Unwrap(info[0].As<Napi::Object>());
     model->Ref();
 
-    tokenCandidates.resize(llama_n_vocab(model->model));
-    tokenCandidates.reserve(llama_n_vocab(model->model));
+    tokenCandidates.resize(llama_vocab_n_tokens(model->vocab));
+    tokenCandidates.reserve(llama_vocab_n_tokens(model->vocab));
 }
 AddonSampler::~AddonSampler() {
     dispose();
@@ -142,7 +142,7 @@ void AddonSampler::acceptToken(llama_token token) {
         repeatPenalty_lastTokens.push_back(token);
     }
 
-    if (grammarEvaluationState != nullptr && grammarEvaluationState->sampler != nullptr && !llama_token_is_eog(model->model, token)) {
+    if (grammarEvaluationState != nullptr && grammarEvaluationState->sampler != nullptr && !llama_vocab_is_eog(model->vocab, token)) {
         llama_sampler_accept(grammarEvaluationState->sampler, token);
     }
 }
@@ -231,8 +231,8 @@ Napi::Value AddonSampler::ApplyConfig(const Napi::CallbackInfo& info) {
             }
 
             const int32_t resolved_top_k = topKSampler_topK <= 0
-                ? llama_n_vocab(model->model)
-                : std::min(topKSampler_topK, llama_n_vocab(model->model));
+                ? llama_vocab_n_tokens(model->vocab)
+                : std::min(topKSampler_topK, llama_vocab_n_tokens(model->vocab));
 
             topKSampler = llama_sampler_init_top_k(resolved_top_k);
         }
@@ -350,15 +350,10 @@ Napi::Value AddonSampler::ApplyConfig(const Napi::CallbackInfo& info) {
 
         if (shouldCreateSampler) {
             repeatPenaltySampler = llama_sampler_init_penalties(
-                llama_n_vocab(model->model),
-                llama_token_eos(model->model),
-                llama_token_nl(model->model),
                 repeatPenaltyMaxTokens,
                 repeatPenalty,
                 repeatPenaltyFrequencyPenalty,
-                repeatPenaltyPresencePenalty,
-                true,
-                false
+                repeatPenaltyPresencePenalty
             );
             repeatPenalty_lastTokens = RingBuffer<llama_token>(repeatPenaltyMaxTokens);
 
@@ -409,7 +404,7 @@ Napi::Value AddonSampler::ApplyConfig(const Napi::CallbackInfo& info) {
                 }
 
                 tokenBiasSampler = llama_sampler_init_logit_bias(
-                    llama_n_vocab(model->model),
+                    llama_vocab_n_tokens(model->vocab),
                     tokenBiasSampler_biases.size(),
                     tokenBiasSampler_biases.data()
                 );

package/llama/addon/addon.cpp

@@ -9,6 +9,7 @@
 #include "globals/addonProgress.h"
 #include "globals/getGpuInfo.h"
 #include "globals/getSwapInfo.h"
+#include "globals/getMemoryInfo.h"
 
 bool backendInitialized = false;
 bool backendDisposed = false;
@@ -25,6 +26,21 @@ Napi::Value addonGetSupportsMmap(const Napi::CallbackInfo& info) {
     return Napi::Boolean::New(info.Env(), llama_supports_mmap());
 }
 
+Napi::Value addonGetGpuSupportsMmap(const Napi::CallbackInfo& info) {
+    const auto llamaSupportsMmap = llama_supports_mmap();
+    const auto gpuDevice = getGpuDevice().first;
+
+    if (gpuDevice == nullptr) {
+        return Napi::Boolean::New(info.Env(), false);
+    }
+
+    ggml_backend_dev_props props;
+    ggml_backend_dev_get_props(gpuDevice, &props);
+
+    const bool gpuSupportsMmap = llama_supports_mmap() && props.caps.buffer_from_host_ptr;
+    return Napi::Boolean::New(info.Env(), gpuSupportsMmap);
+}
+
 Napi::Value addonGetSupportsMlock(const Napi::CallbackInfo& info) {
     return Napi::Boolean::New(info.Env(), llama_supports_mlock());
 }
@@ -152,16 +168,16 @@ class AddonBackendUnloadWorker : public Napi::AsyncWorker {
 };
 
 Napi::Value addonLoadBackends(const Napi::CallbackInfo& info) {
-    const bool forceLoadLibraries = info.Length() == 0
-        ? false
-        : info[0].IsBoolean()
-            ? info[0].As<Napi::Boolean>().Value()
-            : false;
+    const std::string forceLoadLibrariesSearchPath = info.Length() == 0
+        ? ""
+        : info[0].IsString()
+            ? info[0].As<Napi::String>().Utf8Value()
+            : "";
 
     ggml_backend_reg_count();
 
-    if (forceLoadLibraries) {
-        ggml_backend_load_all();
+    if (forceLoadLibrariesSearchPath.length() > 0) {
+        ggml_backend_load_all_from_path(forceLoadLibrariesSearchPath.c_str());
     }
 
     return info.Env().Undefined();
@@ -210,6 +226,7 @@ Napi::Object registerCallback(Napi::Env env, Napi::Object exports) {
         Napi::PropertyDescriptor::Function("systemInfo", systemInfo),
         Napi::PropertyDescriptor::Function("getSupportsGpuOffloading", addonGetSupportsGpuOffloading),
         Napi::PropertyDescriptor::Function("getSupportsMmap", addonGetSupportsMmap),
+        Napi::PropertyDescriptor::Function("getGpuSupportsMmap", addonGetGpuSupportsMmap),
         Napi::PropertyDescriptor::Function("getSupportsMlock", addonGetSupportsMlock),
         Napi::PropertyDescriptor::Function("getMathCores", addonGetMathCores),
         Napi::PropertyDescriptor::Function("getBlockSizeForGgmlType", addonGetBlockSizeForGgmlType),
@@ -220,7 +237,9 @@ Napi::Object registerCallback(Napi::Env env, Napi::Object exports) {
         Napi::PropertyDescriptor::Function("getGpuVramInfo", getGpuVramInfo),
         Napi::PropertyDescriptor::Function("getGpuDeviceInfo", getGpuDeviceInfo),
         Napi::PropertyDescriptor::Function("getGpuType", getGpuType),
+        Napi::PropertyDescriptor::Function("ensureGpuDeviceIsSupported", ensureGpuDeviceIsSupported),
         Napi::PropertyDescriptor::Function("getSwapInfo", getSwapInfo),
+        Napi::PropertyDescriptor::Function("getMemoryInfo", getMemoryInfo),
         Napi::PropertyDescriptor::Function("loadBackends", addonLoadBackends),
         Napi::PropertyDescriptor::Function("init", addonInit),
         Napi::PropertyDescriptor::Function("dispose", addonDispose),

package/llama/addon/globals/getGpuInfo.cpp

@@ -89,17 +89,17 @@ Napi::Value getGpuDeviceInfo(const Napi::CallbackInfo& info) {
     return result;
 }
 
-Napi::Value getGpuType(const Napi::CallbackInfo& info) {
+std::pair<ggml_backend_dev_t, std::string> getGpuDevice() {
     for (size_t i = 0; i < ggml_backend_dev_count(); i++) {
         ggml_backend_dev_t device = ggml_backend_dev_get(i);
         const auto deviceName = std::string(ggml_backend_dev_name(device));
         
         if (deviceName == "Metal") {
-            return Napi::String::New(info.Env(), "metal");
+            return std::pair<ggml_backend_dev_t, std::string>(device, "metal");
         } else if (std::string(deviceName).find("Vulkan") == 0) {
-            return Napi::String::New(info.Env(), "vulkan");
+            return std::pair<ggml_backend_dev_t, std::string>(device, "vulkan");
         } else if (std::string(deviceName).find("CUDA") == 0 || std::string(deviceName).find("ROCm") == 0 || std::string(deviceName).find("MUSA") == 0) {
-            return Napi::String::New(info.Env(), "cuda");
+            return std::pair<ggml_backend_dev_t, std::string>(device, "cuda");
         }
     }
 
@@ -108,9 +108,34 @@ Napi::Value getGpuType(const Napi::CallbackInfo& info) {
         const auto deviceName = std::string(ggml_backend_dev_name(device));
         
         if (deviceName == "CPU") {
-            return Napi::Boolean::New(info.Env(), false);
+            return std::pair<ggml_backend_dev_t, std::string>(device, "cpu");
         }
     }
 
+    return std::pair<ggml_backend_dev_t, std::string>(nullptr, "");
+}
+
+Napi::Value getGpuType(const Napi::CallbackInfo& info) {
+    const auto gpuDeviceRes = getGpuDevice();
+    const auto device = gpuDeviceRes.first;
+    const auto deviceType = gpuDeviceRes.second;
+    
+    if (deviceType == "cpu") {
+        return Napi::Boolean::New(info.Env(), false);
+    } else if (device != nullptr && deviceType != "") {
+        return Napi::String::New(info.Env(), deviceType);
+    }
+
+    return info.Env().Undefined();
+}
+
+Napi::Value ensureGpuDeviceIsSupported(const Napi::CallbackInfo& info) {
+#ifdef GPU_INFO_USE_VULKAN
+    if (!checkIsVulkanEnvSupported(logVulkanWarning)) {
+        Napi::Error::New(info.Env(), "Vulkan device is not supported").ThrowAsJavaScriptException();
+        return info.Env().Undefined();
+    }
+#endif
+
     return info.Env().Undefined();
 }

package/llama/addon/globals/getGpuInfo.h

@@ -1,6 +1,11 @@
 #pragma once
+#include <utility>
+#include <string>
 #include "napi.h"
+#include "llama.h"
 
 Napi::Value getGpuVramInfo(const Napi::CallbackInfo& info);
 Napi::Value getGpuDeviceInfo(const Napi::CallbackInfo& info);
-Napi::Value getGpuType(const Napi::CallbackInfo& info);
+std::pair<ggml_backend_dev_t, std::string> getGpuDevice();
+Napi::Value getGpuType(const Napi::CallbackInfo& info);
+Napi::Value ensureGpuDeviceIsSupported(const Napi::CallbackInfo& info);

package/llama/addon/globals/getMemoryInfo.cpp

@@ -0,0 +1,63 @@
+#include "getMemoryInfo.h"
+#include "addonLog.h"
+
+#ifdef __APPLE__
+#include <iostream>
+#include <mach/mach.h>
+#include <sys/sysctl.h>
+#elif __linux__
+#include <fstream>
+#include <sstream>
+#include <string>
+#elif _WIN32
+#include <iostream>
+#include <windows.h>
+#include <psapi.h>
+#endif
+
+
+Napi::Value getMemoryInfo(const Napi::CallbackInfo& info) {
+    uint64_t totalMemoryUsage = 0;
+
+#ifdef __APPLE__
+    struct mach_task_basic_info taskInfo;
+    mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT;
+    if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&taskInfo, &infoCount) == KERN_SUCCESS) {
+        totalMemoryUsage = taskInfo.virtual_size;
+    } else {
+        addonLlamaCppLogCallback(GGML_LOG_LEVEL_ERROR, std::string("Failed to get memory usage info").c_str(), nullptr);
+    }
+#elif __linux__
+    std::ifstream procStatus("/proc/self/status");
+    std::string line;
+    bool foundMemoryUsage = false;
+    while (std::getline(procStatus, line)) {
+        if (line.rfind("VmSize:", 0) == 0) { // Resident Set Size (current memory usage)
+            std::istringstream iss(line);
+            std::string key, unit;
+            size_t value;
+            if (iss >> key >> value >> unit) {
+                totalMemoryUsage = value * 1024; // Convert from kB to bytes
+                foundMemoryUsage = true;
+            }
+            break;
+        }
+    }
+
+    if (!foundMemoryUsage) {
+        addonLlamaCppLogCallback(GGML_LOG_LEVEL_ERROR, std::string("Failed to get memory usage info").c_str(), nullptr);
+    }
+#elif _WIN32
+    PROCESS_MEMORY_COUNTERS_EX memCounters;
+
+    if (GetProcessMemoryInfo(GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS*)&memCounters, sizeof(memCounters))) {
+        totalMemoryUsage = memCounters.PrivateUsage;
+    } else {
+        addonLlamaCppLogCallback(GGML_LOG_LEVEL_ERROR, std::string("Failed to get memory usage info").c_str(), nullptr);
+    }
+#endif
+    
+    Napi::Object obj = Napi::Object::New(info.Env());
+    obj.Set("total", Napi::Number::New(info.Env(), totalMemoryUsage));
+    return obj;
+}

package/llama/addon/globals/getMemoryInfo.h

@@ -0,0 +1,4 @@
+#pragma once
+#include "napi.h"
+
+Napi::Value getMemoryInfo(const Napi::CallbackInfo& info);

package/llama/binariesGithubRelease.json

@@ -1,3 +1,3 @@
 {
-    "release": "b4291"
+    "release": "b4529"
 }

package/llama/cmake/win32.ensureNinjaPath.cmake

@@ -0,0 +1,68 @@
+function(ensureNinjaPath)
+    if ((NOT DEFINED CMAKE_MAKE_PROGRAM OR NOT EXISTS "${CMAKE_MAKE_PROGRAM}" OR NOT CMAKE_MAKE_PROGRAM) AND (CMAKE_GENERATOR STREQUAL "Ninja" OR CMAKE_GENERATOR STREQUAL "Ninja Multi-Config"))
+        find_program(NINJA_EXECUTABLE ninja)
+
+        set(CMAKE_MAKE_PROGRAM "")
+        set(CMAKE_MAKE_PROGRAM "" PARENT_SCOPE)
+
+        if(NINJA_EXECUTABLE AND EXISTS "${NINJA_EXECUTABLE}")
+            set(CMAKE_MAKE_PROGRAM "${NINJA_EXECUTABLE}")
+            set(CMAKE_MAKE_PROGRAM "${NINJA_EXECUTABLE}" CACHE FILEPATH "Make program")
+            set(CMAKE_MAKE_PROGRAM "${NINJA_EXECUTABLE}" PARENT_SCOPE)
+        endif()
+
+        if (NOT CMAKE_MAKE_PROGRAM OR NOT EXISTS "${CMAKE_MAKE_PROGRAM}")
+            set(PROGRAMDATA_PATH "$ENV{ProgramData}")
+            file(TO_CMAKE_PATH "${PROGRAMDATA_PATH}" PROGRAMDATA_PATH)
+
+            if (PROGRAMDATA_PATH AND EXISTS "${PROGRAMDATA_PATH}")
+                file(GLOB_RECURSE FOUND_NINJA_EXE "${PROGRAMDATA_PATH}/chocolatey/bin/ninja.exe")
+
+                if(FOUND_NINJA_EXE)
+                    list(GET FOUND_NINJA_EXE 0 FOUND_CMAKE_MAKE_PROGRAM)
+                    set(CMAKE_MAKE_PROGRAM "${FOUND_CMAKE_MAKE_PROGRAM}")
+                    set(CMAKE_MAKE_PROGRAM "${FOUND_CMAKE_MAKE_PROGRAM}" CACHE FILEPATH "Make program")
+                    set(CMAKE_MAKE_PROGRAM "${FOUND_CMAKE_MAKE_PROGRAM}" PARENT_SCOPE)
+                endif()
+            endif()
+        endif()
+
+        if (NOT CMAKE_MAKE_PROGRAM OR NOT EXISTS "${CMAKE_MAKE_PROGRAM}")
+            set(LOCALAPPDATA_PATH "$ENV{LOCALAPPDATA}")
+            file(TO_CMAKE_PATH "${LOCALAPPDATA_PATH}" LOCALAPPDATA_PATH)
+
+            if (LOCALAPPDATA_PATH AND EXISTS "${LOCALAPPDATA_PATH}")
+                file(GLOB_RECURSE FOUND_NINJA_EXE "${LOCALAPPDATA_PATH}/Microsoft/WinGet/Packages/Ninja-build.Ninja_Microsoft.Winget.*/ninja.exe")
+
+                if(FOUND_NINJA_EXE)
+                    list(GET FOUND_NINJA_EXE 0 FOUND_CMAKE_MAKE_PROGRAM)
+                    set(CMAKE_MAKE_PROGRAM "${FOUND_CMAKE_MAKE_PROGRAM}")
+                    set(CMAKE_MAKE_PROGRAM "${FOUND_CMAKE_MAKE_PROGRAM}" CACHE FILEPATH "Make program")
+                    set(CMAKE_MAKE_PROGRAM "${FOUND_CMAKE_MAKE_PROGRAM}" PARENT_SCOPE)
+                endif()
+            endif()
+        endif()
+
+        if (NOT CMAKE_MAKE_PROGRAM OR NOT EXISTS "${CMAKE_MAKE_PROGRAM}")
+            foreach(PATH IN LISTS PROGRAMFILES_PATHS)
+                file(GLOB_RECURSE FOUND_NINJA_EXE
+                    "${PATH}/Microsoft Visual Studio/*/CMake/Ninja/ninja.exe"
+                    "${PATH}/Microsoft Visual Studio/**/*/CMake/Ninja/ninja.exe"
+                    "${PATH}/Microsoft Visual Studio/*/Common7/IDE/CommonExtensions/Microsoft/CMake/Ninja/ninja.exe"
+                    "${PATH}/Microsoft Visual Studio/**/*/Common7/IDE/CommonExtensions/Microsoft/CMake/Ninja/ninja.exe")
+
+                if(FOUND_NINJA_EXE)
+                    list(GET FOUND_NINJA_EXE 0 FOUND_CMAKE_MAKE_PROGRAM)
+                    set(CMAKE_MAKE_PROGRAM "${FOUND_CMAKE_MAKE_PROGRAM}")
+                    set(CMAKE_MAKE_PROGRAM "${FOUND_CMAKE_MAKE_PROGRAM}" CACHE FILEPATH "Make program")
+                    set(CMAKE_MAKE_PROGRAM "${FOUND_CMAKE_MAKE_PROGRAM}" PARENT_SCOPE)
+                    break()
+                endif()
+            endforeach()
+        endif()
+
+        if (NOT CMAKE_MAKE_PROGRAM OR NOT EXISTS "${CMAKE_MAKE_PROGRAM}")
+            message(FATAL_ERROR "Ninja build system not found. Please install Ninja or Visual Studio Build Tools.")
+        endif()
+    endif()
+endfunction()

package/llama/cmake/win32.ensureNodeLib.cmake

@@ -0,0 +1,34 @@
+function(ensureNodeLib HOST_ARCH TARGET_ARCH)
+    if (CMAKE_JS_NODELIB_DEF AND CMAKE_JS_NODELIB_TARGET)
+        if (NOT DEFINED NODE_LIB_CMAKE_AR)
+            foreach(PATH IN LISTS PROGRAMFILES_PATHS)
+                if(NODE_LIB_CMAKE_AR)
+                    break()
+                endif()
+
+                file(GLOB_RECURSE FOUND_LIB_EXE
+                    "${PATH}/Microsoft Visual Studio/*/VC/Tools/MSVC/*/bin/Host${HOST_ARCH}/${TARGET_ARCH}/lib.exe"
+                    "${PATH}/Microsoft Visual Studio/**/*/VC/Tools/MSVC/*/bin/Host${HOST_ARCH}/${TARGET_ARCH}/lib.exe")
+
+                if(FOUND_LIB_EXE)
+                    list(GET FOUND_LIB_EXE 0 NODE_LIB_CMAKE_AR)
+                    break()
+                endif()
+            endforeach()
+        endif()
+
+        set(NODE_LIB_CMAKE_AR_MACHINE_FLAG "")
+        if (TARGET_ARCH STREQUAL "x64")
+            set(NODE_LIB_CMAKE_AR_MACHINE_FLAG "/MACHINE:X64")
+        elseif (TARGET_ARCH STREQUAL "arm64")
+            set(NODE_LIB_CMAKE_AR_MACHINE_FLAG "/MACHINE:ARM64")
+        endif()
+
+        if (EXISTS "${NODE_LIB_CMAKE_AR}")
+            # Generate node.lib
+            execute_process(COMMAND ${NODE_LIB_CMAKE_AR} /def:${CMAKE_JS_NODELIB_DEF} /out:${CMAKE_JS_NODELIB_TARGET} ${CMAKE_STATIC_LINKER_FLAGS} ${NODE_LIB_CMAKE_AR_MACHINE_FLAG} /nologo)
+        else()
+            message(FATAL_ERROR "Windows Resource Compiler (lib.exe) not found. Please install Visual Studio Build Tools.")
+        endif()
+    endif()
+endfunction()

package/llama/cmake/win32.llvmApplyGnuModeAdaptations.cmake

@@ -0,0 +1,12 @@
+function(llvmApplyGnuModeAdaptations)
+    # adapt cmake-js to work with llvm in GNU mode
+    if (NOT CMAKE_SHARED_LINKER_FLAGS MATCHES "-Xlinker /DELAYLOAD:NODE.EXE")
+        string(REPLACE "/DELAYLOAD:NODE.EXE" "-Xlinker /DELAYLOAD:NODE.EXE -Xlinker /defaultlib:delayimp"
+            UPDATED_CMAKE_SHARED_LINKER_FLAGS
+            "${CMAKE_SHARED_LINKER_FLAGS}")
+        set(CMAKE_SHARED_LINKER_FLAGS "${UPDATED_CMAKE_SHARED_LINKER_FLAGS}" PARENT_SCOPE)
+    endif()
+
+    set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -Xclang --dependent-lib=msvcrt" PARENT_SCOPE)
+    set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -Xclang --dependent-lib=msvcrt" PARENT_SCOPE)
+endfunction()

package/llama/cmake/win32.llvmEnsureCmakeAr.cmake

@@ -0,0 +1,37 @@
+function(llvmEnsureCmakeAr CURRENT_ARCH)
+    set (LLVM_DIR_ARCH_NAME "")
+    if (CURRENT_ARCH STREQUAL "x64")
+        set (LLVM_DIR_ARCH_NAME "x64")
+    elseif (CURRENT_ARCH STREQUAL "arm64")
+        set (LLVM_DIR_ARCH_NAME "ARM64")
+    endif()
+
+    if (NOT DEFINED CMAKE_AR OR NOT EXISTS "${CMAKE_AR}")
+        set(LLVM_INSTALL_PATHS "")
+        foreach(PATH IN LISTS PROGRAMFILES_PATHS)
+            list(APPEND LLVM_INSTALL_PATHS "${PATH}/LLVM")
+
+            file(GLOB_RECURSE FOUND_LLVM_ROOT LIST_DIRECTORIES true
+                "${PATH}/Microsoft Visual Studio/*/VC/Tools/Llvm/${LLVM_DIR_ARCH_NAME}"
+                "${PATH}/Microsoft Visual Studio/**/*/VC/Tools/Llvm/${LLVM_DIR_ARCH_NAME}")
+            list(FILTER FOUND_LLVM_ROOT INCLUDE REGEX "VC/Tools/Llvm/${LLVM_DIR_ARCH_NAME}$")
+
+            if(FOUND_LLVM_ROOT)
+                list(APPEND LLVM_INSTALL_PATHS ${FOUND_LLVM_ROOT})
+            endif()
+        endforeach()
+
+        if(DEFINED LLVM_ROOT AND EXISTS "${LLVM_ROOT}")
+            list(INSERT LLVM_INSTALL_PATHS 0 "${LLVM_ROOT}")
+        endif()
+
+        list(REMOVE_DUPLICATES LLVM_INSTALL_PATHS)
+
+        foreach(PATH IN LISTS LLVM_INSTALL_PATHS)
+            if(EXISTS "${PATH}/bin/llvm-ar.exe" AND EXISTS "${PATH}/bin/llvm-ar.exe")
+                set(CMAKE_AR "${PATH}/bin/llvm-ar.exe" PARENT_SCOPE)
+                break()
+            endif()
+        endforeach()
+    endif()
+endfunction()

package/llama/cmake/win32.llvmUseGnuModeCompilers.cmake

@@ -0,0 +1,87 @@
+function(llvmUseGnuModeCompilers CURRENT_ARCH)
+    set(LLVM_INSTALLATION_URL "https://github.com/llvm/llvm-project/releases/tag/llvmorg-19.1.5")
+
+    set(CMAKE_C_COMPILER clang)
+    set(CMAKE_C_COMPILER clang PARENT_SCOPE)
+    set(CMAKE_CXX_COMPILER clang++)
+    set(CMAKE_CXX_COMPILER clang++ PARENT_SCOPE)
+    set(CMAKE_RC_COMPILER llvm-rc)
+    set(CMAKE_RC_COMPILER llvm-rc PARENT_SCOPE)
+
+
+    set (LLVM_DIR_ARCH_NAME "")
+    if (CURRENT_ARCH STREQUAL "x64")
+        set (LLVM_DIR_ARCH_NAME "x64")
+    elseif (CURRENT_ARCH STREQUAL "arm64")
+        set (LLVM_DIR_ARCH_NAME "ARM64")
+    endif()
+
+    set(LLVM_INSTALL_PATHS "")
+    foreach(PATH IN LISTS PROGRAMFILES_PATHS)
+        list(APPEND LLVM_INSTALL_PATHS "${PATH}/LLVM")
+
+        file(GLOB_RECURSE FOUND_LLVM_ROOT LIST_DIRECTORIES true
+            "${PATH}/Microsoft Visual Studio/*/VC/Tools/Llvm/${LLVM_DIR_ARCH_NAME}"
+            "${PATH}/Microsoft Visual Studio/**/*/VC/Tools/Llvm/${LLVM_DIR_ARCH_NAME}")
+        list(FILTER FOUND_LLVM_ROOT INCLUDE REGEX "VC/Tools/Llvm/${LLVM_DIR_ARCH_NAME}$")
+
+        if(FOUND_LLVM_ROOT)
+            list(APPEND LLVM_INSTALL_PATHS ${FOUND_LLVM_ROOT})
+        endif()
+    endforeach()
+
+    if(DEFINED LLVM_ROOT AND EXISTS "${LLVM_ROOT}")
+        list(INSERT LLVM_INSTALL_PATHS 0 "${LLVM_ROOT}")
+    endif()
+
+    list(REMOVE_DUPLICATES LLVM_INSTALL_PATHS)
+
+    set(LLVM_ROOT "")
+    set(LLVM_ROOT "" PARENT_SCOPE)
+    foreach(PATH IN LISTS LLVM_INSTALL_PATHS)
+        if(EXISTS "${PATH}/bin/clang.exe" AND EXISTS "${PATH}/bin/clang++.exe" AND EXISTS "${PATH}/bin/llvm-rc.exe")
+            set(LLVM_ROOT "${PATH}")
+            set(LLVM_ROOT "${PATH}" PARENT_SCOPE)
+            break()
+        endif()
+    endforeach()
+
+    if(LLVM_ROOT STREQUAL "")
+        if (CURRENT_ARCH STREQUAL "arm64")
+            message(FATAL_ERROR "LLVM installation was not found. Please install LLVM for WoA (Windows on Arm): ${LLVM_INSTALLATION_URL}")
+        else()
+            message(FATAL_ERROR "LLVM installation was not found. Please install LLVM: ${LLVM_INSTALLATION_URL}")
+        endif()
+    endif()
+
+    if (NOT EXISTS "${CMAKE_C_COMPILER}" OR NOT EXISTS "${CMAKE_CXX_COMPILER}" OR NOT EXISTS "${CMAKE_RC_COMPILER}")
+        set(CMAKE_C_COMPILER "${LLVM_ROOT}/bin/clang.exe")
+        set(CMAKE_C_COMPILER "${LLVM_ROOT}/bin/clang.exe" PARENT_SCOPE)
+        set(CMAKE_CXX_COMPILER "${LLVM_ROOT}/bin/clang++.exe")
+        set(CMAKE_CXX_COMPILER "${LLVM_ROOT}/bin/clang++.exe" PARENT_SCOPE)
+        set(CMAKE_RC_COMPILER "${LLVM_ROOT}/bin/llvm-rc.exe")
+        set(CMAKE_RC_COMPILER "${LLVM_ROOT}/bin/llvm-rc.exe" PARENT_SCOPE)
+    endif()
+
+    if (NOT EXISTS "${CMAKE_C_COMPILER}")
+        if (CURRENT_ARCH STREQUAL "arm64")
+            message(FATAL_ERROR "Clang compiler not found at ${CMAKE_C_COMPILER}. Please reinstall LLVM for WoA (Windows on Arm): ${LLVM_INSTALLATION_URL}")
+        else()
+            message(FATAL_ERROR "Clang compiler not found at ${CMAKE_C_COMPILER}. Please reinstall LLVM: ${LLVM_INSTALLATION_URL}")
+        endif()
+    endif()
+    if (NOT EXISTS "${CMAKE_CXX_COMPILER}")
+        if (CURRENT_ARCH STREQUAL "arm64")
+            message(FATAL_ERROR "Clang++ compiler not found at ${CMAKE_CXX_COMPILER}. Please reinstall LLVM for WoA (Windows on Arm): ${LLVM_INSTALLATION_URL}")
+        else()
+            message(FATAL_ERROR "Clang++ compiler not found at ${CMAKE_CXX_COMPILER}. Please reinstall LLVM: ${LLVM_INSTALLATION_URL}")
+        endif()
+    endif()
+    if (NOT EXISTS "${CMAKE_RC_COMPILER}")
+        if (CURRENT_ARCH STREQUAL "arm64")
+            message(FATAL_ERROR "LLVM Resource Compiler not found at ${CMAKE_RC_COMPILER}. Please reinstall LLVM for WoA (Windows on Arm): ${LLVM_INSTALLATION_URL}")
+        else()
+            message(FATAL_ERROR "LLVM Resource Compiler not found at ${CMAKE_RC_COMPILER}. Please reinstall LLVM: ${LLVM_INSTALLATION_URL}")
+        endif()
+    endif()
+endfunction()

package/llama/cmake/win32.programFilesPaths.cmake

@@ -0,0 +1,35 @@
+function(setProgramFilesPaths CURRENT_ARCH)
+    set(PROGRAMFILES_X86_ENV_NAME "ProgramFiles(x86)")
+
+    set(PROGRAMFILES "$ENV{ProgramFiles}")
+    set(PROGRAMFILES_X86 "$ENV{${PROGRAMFILES_X86_ENV_NAME}}")
+    file(TO_CMAKE_PATH "${PROGRAMFILES}" PROGRAMFILES)
+    file(TO_CMAKE_PATH "${PROGRAMFILES_X86}" PROGRAMFILES_X86)
+
+    if(CURRENT_ARCH STREQUAL "arm64")
+        set(PROGRAMFILES_ARM64_ENV_NAME "ProgramFiles(Arm)")
+
+        set(PROGRAMFILES_ARM64 "$ENV{${PROGRAMFILES_ARM64_ENV_NAME}}")
+        file(TO_CMAKE_PATH "${PROGRAMFILES_ARM64}" PROGRAMFILES_ARM64)
+
+        set(PROGRAMFILES_PATHS_LIST
+            "${PROGRAMFILES_ARM64}"
+            "${PROGRAMFILES}"
+            "${PROGRAMFILES_X86}"
+            "C:/Program Files (Arm)"
+            "C:/Program Files"
+            "C:/Program Files (x86)"
+        )
+        list(REMOVE_DUPLICATES PROGRAMFILES_PATHS_LIST)
+        set(PROGRAMFILES_PATHS ${PROGRAMFILES_PATHS_LIST} PARENT_SCOPE)
+    else()
+        set(PROGRAMFILES_PATHS_LIST
+            "${PROGRAMFILES}"
+            "${PROGRAMFILES_X86}"
+            "C:/Program Files"
+            "C:/Program Files (x86)"
+        )
+        list(REMOVE_DUPLICATES PROGRAMFILES_PATHS_LIST)
+        set(PROGRAMFILES_PATHS ${PROGRAMFILES_PATHS_LIST} PARENT_SCOPE)
+    endif()
+endfunction()

package/llama/gpuInfo/vulkan-gpu-info.cpp

@@ -5,7 +5,7 @@
 
 typedef void (*gpuInfoVulkanWarningLogCallback_t)(const char* message);
 
-static bool enumerateVulkanDevices(size_t* total, size_t* used, size_t* unifiedMemorySize, bool addDeviceNames, std::vector<std::string> * deviceNames, gpuInfoVulkanWarningLogCallback_t warningLogCallback) {
+static bool enumerateVulkanDevices(size_t* total, size_t* used, size_t* unifiedMemorySize, bool addDeviceNames, std::vector<std::string> * deviceNames, gpuInfoVulkanWarningLogCallback_t warningLogCallback, bool * checkSupported) {
     vk::ApplicationInfo appInfo("node-llama-cpp GPU info", 1, "llama.cpp", 1, VK_API_VERSION_1_2);
     vk::InstanceCreateInfo createInfo(vk::InstanceCreateFlags(), &appInfo, {}, {});
     vk::Instance instance = vk::createInstance(createInfo);
@@ -56,6 +56,22 @@ static bool enumerateVulkanDevices(size_t* total, size_t* used, size_t* unifiedM
                     if (size > 0 && addDeviceNames) {
                         (*deviceNames).push_back(std::string(deviceProps.deviceName.data()));
                     }
+
+                    if (checkSupported != nullptr && checkSupported) {
+                        VkPhysicalDeviceFeatures2 features2 = {};
+                        features2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2;
+
+                        VkPhysicalDeviceVulkan11Features vk11Features = {};
+                        vk11Features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES;
+                        features2.pNext = &vk11Features;
+
+                        vkGetPhysicalDeviceFeatures2(physicalDevice, &features2);
+                        VkPhysicalDeviceFeatures2 device_features2;
+
+                        if (!vk11Features.storageBuffer16BitAccess) {
+                            *checkSupported = false;
+                        }
+                    }
                 }
             }
         } else {
@@ -78,5 +94,16 @@ static bool enumerateVulkanDevices(size_t* total, size_t* used, size_t* unifiedM
 }
 
 bool gpuInfoGetTotalVulkanDevicesInfo(size_t* total, size_t* used, size_t* unifiedMemorySize, gpuInfoVulkanWarningLogCallback_t warningLogCallback) {
-    return enumerateVulkanDevices(total, used, unifiedMemorySize, false, nullptr, warningLogCallback);
+    return enumerateVulkanDevices(total, used, unifiedMemorySize, false, nullptr, warningLogCallback, nullptr);
+}
+
+bool checkIsVulkanEnvSupported(gpuInfoVulkanWarningLogCallback_t warningLogCallback) {
+    size_t total = 0;
+    size_t used = 0;
+    size_t unifiedMemorySize = 0;
+
+    bool isSupported = true;
+    enumerateVulkanDevices(&total, &used, &unifiedMemorySize, false, nullptr, warningLogCallback, &isSupported);
+
+    return isSupported;
 }

package/llama/gpuInfo/vulkan-gpu-info.h

@@ -6,3 +6,4 @@
 typedef void (*gpuInfoVulkanWarningLogCallback_t)(const char* message);
 
 bool gpuInfoGetTotalVulkanDevicesInfo(size_t* total, size_t* used, size_t* unifiedMemorySize, gpuInfoVulkanWarningLogCallback_t warningLogCallback);
+bool checkIsVulkanEnvSupported(gpuInfoVulkanWarningLogCallback_t warningLogCallback);

package/llama/llama.cpp.info.json

@@ -1,4 +1,4 @@
 {
-    "tag": "b4291",
+    "tag": "b4529",
     "llamaCppGithubRepo": "ggerganov/llama.cpp"
 }