node-llama-cpp 3.2.0 → 3.3.1

package/llama/CMakeLists.txt

@@ -28,6 +28,16 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "Ap
     add_compile_options(-Wno-c++17-extensions)
 endif()
 
+if(APPLE)
+    set(CMAKE_SKIP_BUILD_RPATH  FALSE)
+    set(CMAKE_INSTALL_RPATH_USE_LINK_PATH FALSE)
+    set(CMAKE_BUILD_RPATH "@loader_path")
+    set(CMAKE_INSTALL_RPATH "@loader_path")
+    set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+else()
+    set(CMAKE_BUILD_RPATH_USE_ORIGIN ON)
+endif()
+
 include_directories(${NODE_ADDON_API_DIR} ${CMAKE_JS_INC})
 
 add_subdirectory("llama.cpp")
@@ -39,41 +49,6 @@ unset(GPU_INFO_HEADERS)
 unset(GPU_INFO_SOURCES)
 unset(GPU_INFO_EXTRA_LIBS)
 
-if (GGML_CUDA)
-    cmake_minimum_required(VERSION 3.17)
-
-    find_package(CUDAToolkit)
-    if (CUDAToolkit_FOUND)
-        message(STATUS "Using CUDA for GPU info")
-
-        enable_language(CUDA)
-
-        list(APPEND GPU_INFO_HEADERS gpuInfo/cuda-gpu-info.h)
-        list(APPEND GPU_INFO_SOURCES gpuInfo/cuda-gpu-info.cu)
-
-        add_compile_definitions(GPU_INFO_USE_CUDA)
-
-        if (GGML_STATIC)
-            list(APPEND GPU_INFO_EXTRA_LIBS CUDA::cudart_static)
-        else()
-            list(APPEND GPU_INFO_EXTRA_LIBS CUDA::cudart)
-        endif()
-
-        list(APPEND GPU_INFO_EXTRA_LIBS CUDA::cuda_driver)
-
-        if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
-            # copied from llama.cpp/CMakLists.txt under "if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES)"
-            if (GGML_CUDA_F16 OR GGML_CUDA_DMMV_F16)
-                set(CMAKE_CUDA_ARCHITECTURES "60;61;70;75")
-            else()
-                set(CMAKE_CUDA_ARCHITECTURES "52;61;70;75")
-            endif()
-        endif()
-    else()
-        message(FATAL_ERROR "CUDA was not found")
-    endif()
-endif()
-
 if (GGML_VULKAN OR GGML_KOMPUTE)
     find_package(Vulkan)
     if (Vulkan_FOUND)
@@ -94,67 +69,12 @@ if (GGML_VULKAN OR GGML_KOMPUTE)
     endif()
 endif()
 
-if (GGML_HIPBLAS)
-    list(APPEND CMAKE_PREFIX_PATH /opt/rocm)
-
-    if (NOT ${CMAKE_C_COMPILER_ID} MATCHES "Clang")
-        message(WARNING "Only LLVM is supported for HIP, hint: CC=/opt/rocm/llvm/bin/clang")
-    endif()
-    if (NOT ${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
-        message(WARNING "Only LLVM is supported for HIP, hint: CXX=/opt/rocm/llvm/bin/clang++")
-    endif()
-
-    find_package(hip)
-    find_package(hipblas)
-    find_package(rocblas)
-
-    if (${hipblas_FOUND} AND ${hip_FOUND})
-        message(STATUS "Using HIP and hipBLAS for GPU info")
-        add_compile_definitions(GPU_INFO_USE_HIPBLAS GPU_INFO_USE_CUDA)
-        add_library(gpu-info-rocm OBJECT gpuInfo/cuda-gpu-info.cu gpuInfo/cuda-gpu-info.h)
-        set_source_files_properties(gpuInfo/cuda-gpu-info.cu PROPERTIES LANGUAGE CXX)
-        target_link_libraries(gpu-info-rocm PRIVATE hip::device PUBLIC hip::host roc::rocblas roc::hipblas)
-
-        list(APPEND GPU_INFO_EXTRA_LIBS gpu-info-rocm)
-    else()
-        message(FATAL_ERROR "hipBLAS or HIP was not found. Try setting CMAKE_PREFIX_PATH=/opt/rocm")
-    endif()
-endif()
-
-if (GGML_METAL)
-    find_library(FOUNDATION_LIBRARY Foundation REQUIRED)
-    find_library(METAL_FRAMEWORK    Metal      REQUIRED)
-    find_library(METALKIT_FRAMEWORK MetalKit   REQUIRED)
-
-    message(STATUS "Using Metal for GPU info")
-    list(APPEND GPU_INFO_HEADERS gpuInfo/metal-gpu-info.h)
-    list(APPEND GPU_INFO_SOURCES gpuInfo/metal-gpu-info.mm)
-
-    add_compile_definitions(GPU_INFO_USE_METAL)
-
-    list(APPEND GPU_INFO_EXTRA_LIBS
-        ${FOUNDATION_LIBRARY}
-        ${METAL_FRAMEWORK}
-        ${METALKIT_FRAMEWORK}
-        )
-endif()
-
 list(REMOVE_DUPLICATES GPU_INFO_HEADERS)
 list(REMOVE_DUPLICATES GPU_INFO_SOURCES)
 list(REMOVE_DUPLICATES GPU_INFO_EXTRA_LIBS)
 
 file(GLOB SOURCE_FILES "addon/*.cpp" "addon/**/*.cpp" ${GPU_INFO_SOURCES})
 
-if(APPLE)
-    set(CMAKE_SKIP_BUILD_RPATH  FALSE)
-    set(CMAKE_INSTALL_RPATH_USE_LINK_PATH FALSE)
-    set(CMAKE_BUILD_RPATH "@loader_path")
-    set(CMAKE_INSTALL_RPATH "@loader_path")
-    set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
-else()
-    set(CMAKE_BUILD_RPATH_USE_ORIGIN ON)
-endif()
-
 add_library(${PROJECT_NAME} SHARED ${SOURCE_FILES} ${CMAKE_JS_SRC} ${GPU_INFO_HEADERS})
 set_target_properties(${PROJECT_NAME} PROPERTIES PREFIX "" SUFFIX ".node")
 target_link_libraries(${PROJECT_NAME} ${CMAKE_JS_LIB})

package/llama/addon/AddonContext.cpp

@@ -531,7 +531,8 @@ Napi::Value AddonContext::GetEmbedding(const Napi::CallbackInfo& info) {
     }
 
     const int n_embd = llama_n_embd(model->model);
-    const auto* embeddings = llama_get_embeddings_seq(ctx, 0);
+    const enum llama_pooling_type pooling_type = llama_pooling_type(ctx);
+    const auto* embeddings = pooling_type == LLAMA_POOLING_TYPE_NONE ? NULL : llama_get_embeddings_seq(ctx, 0);
     if (embeddings == NULL) {
         embeddings = llama_get_embeddings_ith(ctx, inputTokensLength - 1);
 

package/llama/addon/AddonGrammar.cpp

@@ -34,6 +34,54 @@ AddonGrammar::~AddonGrammar() {
     }
 }
 
+Napi::Value AddonGrammar::isTextCompatible(const Napi::CallbackInfo& info) {
+    const std::string testText = info[0].As<Napi::String>().Utf8Value();
+
+    auto parsed_grammar = llama_grammar_init_impl(nullptr, grammarCode.c_str(), rootRuleName.c_str());
+    
+    // will be empty if there are parse errors
+    if (parsed_grammar == nullptr) {
+        Napi::Error::New(info.Env(), "Failed to parse grammar").ThrowAsJavaScriptException();
+        return Napi::Boolean::New(info.Env(), false);
+    }
+
+    const auto cpts = unicode_cpts_from_utf8(testText);
+    const llama_grammar_rules  & rules = llama_grammar_get_rules(parsed_grammar);
+    llama_grammar_stacks & stacks_cur = llama_grammar_get_stacks(parsed_grammar);
+
+    for (const auto & cpt : cpts) {
+        const llama_grammar_stacks stacks_prev = llama_grammar_get_stacks(parsed_grammar);
+
+        llama_grammar_accept(rules, stacks_prev, cpt, stacks_cur);
+
+        if (stacks_cur.empty()) {
+            // no stacks means that the grammar failed to match at this point
+            llama_grammar_free_impl(parsed_grammar);
+            return Napi::Boolean::New(info.Env(), false);
+        }
+    }
+
+    for (const auto & stack : stacks_cur) {
+        if (stack.empty()) {
+            // an empty stack means that the grammar has been completed
+            llama_grammar_free_impl(parsed_grammar);
+            return Napi::Boolean::New(info.Env(), true);
+        }
+    }
+
+    llama_grammar_free_impl(parsed_grammar);
+    return Napi::Boolean::New(info.Env(), false);
+}
+
 void AddonGrammar::init(Napi::Object exports) {
-    exports.Set("AddonGrammar", DefineClass(exports.Env(), "AddonGrammar", {}));
+    exports.Set(
+        "AddonGrammar",
+        DefineClass(
+            exports.Env(),
+            "AddonGrammar",
+            {
+                InstanceMethod("isTextCompatible", &AddonGrammar::isTextCompatible),
+            }
+        )
+    );
 }

package/llama/addon/AddonGrammar.h

@@ -2,6 +2,7 @@
 #include "llama.h"
 #include "common/common.h"
 #include "llama-grammar.h"
+#include "unicode.h"
 #include "napi.h"
 #include "addonGlobals.h"
 
@@ -15,5 +16,7 @@ class AddonGrammar : public Napi::ObjectWrap<AddonGrammar> {
         AddonGrammar(const Napi::CallbackInfo& info);
         ~AddonGrammar();
 
+        Napi::Value isTextCompatible(const Napi::CallbackInfo& info);
+
         static void init(Napi::Object exports);
 };

package/llama/addon/AddonModel.cpp

@@ -9,7 +9,7 @@
 #include "AddonModelLora.h"
 
 static Napi::Value getNapiToken(const Napi::CallbackInfo& info, llama_model* model, llama_token token) {
-    if (token < 0) {
+    if (token < 0 || token == LLAMA_TOKEN_NULL) {
         return Napi::Number::From(info.Env(), -1);
     }
 
@@ -565,6 +565,22 @@ Napi::Value AddonModel::EotToken(const Napi::CallbackInfo& info) {
 
     return getNapiToken(info, model, llama_token_eot(model));
 }
+Napi::Value AddonModel::ClsToken(const Napi::CallbackInfo& info) {
+    if (disposed) {
+        Napi::Error::New(info.Env(), "Model is disposed").ThrowAsJavaScriptException();
+        return info.Env().Undefined();
+    }
+
+    return getNapiToken(info, model, llama_token_cls(model));
+}
+Napi::Value AddonModel::SepToken(const Napi::CallbackInfo& info) {
+    if (disposed) {
+        Napi::Error::New(info.Env(), "Model is disposed").ThrowAsJavaScriptException();
+        return info.Env().Undefined();
+    }
+
+    return getNapiToken(info, model, llama_token_sep(model));
+}
 Napi::Value AddonModel::GetTokenString(const Napi::CallbackInfo& info) {
     if (disposed) {
         Napi::Error::New(info.Env(), "Model is disposed").ThrowAsJavaScriptException();
@@ -624,11 +640,14 @@ Napi::Value AddonModel::GetVocabularyType(const Napi::CallbackInfo& info) {
     return Napi::Number::From(info.Env(), int32_t(vocabularyType));
 }
 Napi::Value AddonModel::ShouldPrependBosToken(const Napi::CallbackInfo& info) {
-    const int addBos = llama_add_bos_token(model);
+    const bool addBos = llama_add_bos_token(model);
 
-    bool shouldPrependBos = addBos != -1 ? bool(addBos) : (llama_vocab_type(model) == LLAMA_VOCAB_TYPE_SPM);
+    return Napi::Boolean::New(info.Env(), addBos);
+}
+Napi::Value AddonModel::ShouldAppendEosToken(const Napi::CallbackInfo& info) {
+    const bool addEos = llama_add_eos_token(model);
 
-    return Napi::Boolean::New(info.Env(), shouldPrependBos);
+    return Napi::Boolean::New(info.Env(), addEos);
 }
 
 Napi::Value AddonModel::GetModelSize(const Napi::CallbackInfo& info) {
@@ -659,11 +678,14 @@ void AddonModel::init(Napi::Object exports) {
                 InstanceMethod("middleToken", &AddonModel::MiddleToken),
                 InstanceMethod("suffixToken", &AddonModel::SuffixToken),
                 InstanceMethod("eotToken", &AddonModel::EotToken),
+                InstanceMethod("clsToken", &AddonModel::ClsToken),
+                InstanceMethod("sepToken", &AddonModel::SepToken),
                 InstanceMethod("getTokenString", &AddonModel::GetTokenString),
                 InstanceMethod("getTokenAttributes", &AddonModel::GetTokenAttributes),
                 InstanceMethod("isEogToken", &AddonModel::IsEogToken),
                 InstanceMethod("getVocabularyType", &AddonModel::GetVocabularyType),
                 InstanceMethod("shouldPrependBosToken", &AddonModel::ShouldPrependBosToken),
+                InstanceMethod("shouldAppendEosToken", &AddonModel::ShouldAppendEosToken),
                 InstanceMethod("getModelSize", &AddonModel::GetModelSize),
                 InstanceMethod("dispose", &AddonModel::Dispose),
             }

package/llama/addon/AddonModel.h

@@ -49,12 +49,15 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
         Napi::Value MiddleToken(const Napi::CallbackInfo& info);
         Napi::Value SuffixToken(const Napi::CallbackInfo& info);
         Napi::Value EotToken(const Napi::CallbackInfo& info);
+        Napi::Value ClsToken(const Napi::CallbackInfo& info);
+        Napi::Value SepToken(const Napi::CallbackInfo& info);
         Napi::Value GetTokenString(const Napi::CallbackInfo& info);
 
         Napi::Value GetTokenAttributes(const Napi::CallbackInfo& info);
         Napi::Value IsEogToken(const Napi::CallbackInfo& info);
         Napi::Value GetVocabularyType(const Napi::CallbackInfo& info);
         Napi::Value ShouldPrependBosToken(const Napi::CallbackInfo& info);
+        Napi::Value ShouldAppendEosToken(const Napi::CallbackInfo& info);
         Napi::Value GetModelSize(const Napi::CallbackInfo& info);
 
         static void init(Napi::Object exports);

package/llama/addon/addon.cpp

@@ -151,6 +151,22 @@ class AddonBackendUnloadWorker : public Napi::AsyncWorker {
         }
 };
 
+Napi::Value addonLoadBackends(const Napi::CallbackInfo& info) {
+    const bool forceLoadLibraries = info.Length() == 0
+        ? false
+        : info[0].IsBoolean()
+            ? info[0].As<Napi::Boolean>().Value()
+            : false;
+
+    ggml_backend_reg_count();
+
+    if (forceLoadLibraries) {
+        ggml_backend_load_all();
+    }
+
+    return info.Env().Undefined();
+}
+
 Napi::Value addonInit(const Napi::CallbackInfo& info) {
     if (backendInitialized) {
         Napi::Promise::Deferred deferred = Napi::Promise::Deferred::New(info.Env());
@@ -205,6 +221,7 @@ Napi::Object registerCallback(Napi::Env env, Napi::Object exports) {
         Napi::PropertyDescriptor::Function("getGpuDeviceInfo", getGpuDeviceInfo),
         Napi::PropertyDescriptor::Function("getGpuType", getGpuType),
         Napi::PropertyDescriptor::Function("getSwapInfo", getSwapInfo),
+        Napi::PropertyDescriptor::Function("loadBackends", addonLoadBackends),
         Napi::PropertyDescriptor::Function("init", addonInit),
         Napi::PropertyDescriptor::Function("dispose", addonDispose),
     });

package/llama/addon/globals/getGpuInfo.cpp

@@ -1,22 +1,15 @@
 #include "getGpuInfo.h"
 #include "addonLog.h"
 
-#ifdef GPU_INFO_USE_CUDA
-#  include "../../gpuInfo/cuda-gpu-info.h"
+#ifdef __APPLE__
+    #include <TargetConditionals.h>
 #endif
+
 #ifdef GPU_INFO_USE_VULKAN
 #  include "../../gpuInfo/vulkan-gpu-info.h"
 #endif
-#ifdef GPU_INFO_USE_METAL
-#  include "../../gpuInfo/metal-gpu-info.h"
-#endif
 
 
-#ifdef GPU_INFO_USE_CUDA
-void logCudaError(const char* message) {
-    addonLlamaCppLogCallback(GGML_LOG_LEVEL_ERROR, (std::string("CUDA error: ") + std::string(message)).c_str(), nullptr);
-}
-#endif
 #ifdef GPU_INFO_USE_VULKAN
 void logVulkanWarning(const char* message) {
     addonLlamaCppLogCallback(GGML_LOG_LEVEL_WARN, (std::string("Vulkan warning: ") + std::string(message)).c_str(), nullptr);
@@ -24,20 +17,31 @@ void logVulkanWarning(const char* message) {
 #endif
 
 Napi::Value getGpuVramInfo(const Napi::CallbackInfo& info) {
+    ggml_backend_dev_t device = NULL;
+    size_t deviceTotal = 0;
+    size_t deviceFree = 0;
+
     uint64_t total = 0;
     uint64_t used = 0;
     uint64_t unifiedVramSize = 0;
 
-#ifdef GPU_INFO_USE_CUDA
-    size_t cudaDeviceTotal = 0;
-    size_t cudaDeviceUsed = 0;
-    bool cudeGetInfoSuccess = gpuInfoGetTotalCudaDevicesInfo(&cudaDeviceTotal, &cudaDeviceUsed, logCudaError);
+    for (size_t i = 0; i < ggml_backend_dev_count(); i++) {
+        device = ggml_backend_dev_get(i);
+        if (ggml_backend_dev_type(device) == GGML_BACKEND_DEVICE_TYPE_GPU) {
+            deviceTotal = 0;
+            deviceFree = 0;
+            ggml_backend_dev_memory(device, &deviceFree, &deviceTotal);
 
-    if (cudeGetInfoSuccess) {
-        total += cudaDeviceTotal;
-        used += cudaDeviceUsed;
-    }
+            total += deviceTotal;
+            used += deviceTotal - deviceFree;
+
+#if defined(__arm64__) || defined(__aarch64__)
+            if (std::string(ggml_backend_dev_name(device)) == "Metal") {
+                unifiedVramSize += deviceTotal;
+            }
 #endif
+        }
+    }
 
 #ifdef GPU_INFO_USE_VULKAN
     uint64_t vulkanDeviceTotal = 0;
@@ -46,23 +50,15 @@ Napi::Value getGpuVramInfo(const Napi::CallbackInfo& info) {
     const bool vulkanDeviceSupportsMemoryBudgetExtension = gpuInfoGetTotalVulkanDevicesInfo(&vulkanDeviceTotal, &vulkanDeviceUsed, &vulkanDeviceUnifiedVramSize, logVulkanWarning);
 
     if (vulkanDeviceSupportsMemoryBudgetExtension) {
-        total += vulkanDeviceTotal;
-        used += vulkanDeviceUsed;
+        if (vulkanDeviceUnifiedVramSize > total) {
+            // this means that we counted memory from devices that aren't used by llama.cpp
+            vulkanDeviceUnifiedVramSize = 0;
+        }
+        
         unifiedVramSize += vulkanDeviceUnifiedVramSize;
     }
 #endif
 
-#ifdef GPU_INFO_USE_METAL
-    uint64_t metalDeviceTotal = 0;
-    uint64_t metalDeviceUsed = 0;
-    uint64_t metalDeviceUnifiedVramSize = 0;
-    getMetalGpuInfo(&metalDeviceTotal, &metalDeviceUsed, &metalDeviceUnifiedVramSize);
-
-    total += metalDeviceTotal;
-    used += metalDeviceUsed;
-    unifiedVramSize += metalDeviceUnifiedVramSize;
-#endif
-
     Napi::Object result = Napi::Object::New(info.Env());
     result.Set("total", Napi::Number::From(info.Env(), total));
     result.Set("used", Napi::Number::From(info.Env(), used));
@@ -74,17 +70,13 @@ Napi::Value getGpuVramInfo(const Napi::CallbackInfo& info) {
 Napi::Value getGpuDeviceInfo(const Napi::CallbackInfo& info) {
     std::vector<std::string> deviceNames;
 
-#ifdef GPU_INFO_USE_CUDA
-    gpuInfoGetCudaDeviceNames(&deviceNames, logCudaError);
-#endif
+    for (size_t i = 0; i < ggml_backend_dev_count(); i++) {
+        ggml_backend_dev_t device = ggml_backend_dev_get(i);
+        if (ggml_backend_dev_type(device) == GGML_BACKEND_DEVICE_TYPE_GPU) {
 
-#ifdef GPU_INFO_USE_VULKAN
-    gpuInfoGetVulkanDeviceNames(&deviceNames, logVulkanWarning);
-#endif
-
-#ifdef GPU_INFO_USE_METAL
-    getMetalGpuDeviceNames(&deviceNames);
-#endif
+            deviceNames.push_back(std::string(ggml_backend_dev_description(device)));
+        }
+    }
 
     Napi::Object result = Napi::Object::New(info.Env());
 
@@ -98,17 +90,27 @@ Napi::Value getGpuDeviceInfo(const Napi::CallbackInfo& info) {
 }
 
 Napi::Value getGpuType(const Napi::CallbackInfo& info) {
-#ifdef GPU_INFO_USE_CUDA
-    return Napi::String::New(info.Env(), "cuda");
-#endif
-
-#ifdef GPU_INFO_USE_VULKAN
-    return Napi::String::New(info.Env(), "vulkan");
-#endif
+    for (size_t i = 0; i < ggml_backend_dev_count(); i++) {
+        ggml_backend_dev_t device = ggml_backend_dev_get(i);
+        const auto deviceName = std::string(ggml_backend_dev_name(device));
+        
+        if (deviceName == "Metal") {
+            return Napi::String::New(info.Env(), "metal");
+        } else if (std::string(deviceName).find("Vulkan") == 0) {
+            return Napi::String::New(info.Env(), "vulkan");
+        } else if (std::string(deviceName).find("CUDA") == 0 || std::string(deviceName).find("ROCm") == 0 || std::string(deviceName).find("MUSA") == 0) {
+            return Napi::String::New(info.Env(), "cuda");
+        }
+    }
 
-#ifdef GPU_INFO_USE_METAL
-    return Napi::String::New(info.Env(), "metal");
-#endif
+    for (size_t i = 0; i < ggml_backend_dev_count(); i++) {
+        ggml_backend_dev_t device = ggml_backend_dev_get(i);
+        const auto deviceName = std::string(ggml_backend_dev_name(device));
+        
+        if (deviceName == "CPU") {
+            return Napi::Boolean::New(info.Env(), false);
+        }
+    }
 
     return info.Env().Undefined();
-}
+}

package/llama/binariesGithubRelease.json

@@ -1,3 +1,3 @@
 {
-    "release": "b3995"
+    "release": "b4291"
 }

package/llama/gpuInfo/vulkan-gpu-info.cpp

@@ -80,11 +80,3 @@ static bool enumerateVulkanDevices(size_t* total, size_t* used, size_t* unifiedM
 bool gpuInfoGetTotalVulkanDevicesInfo(size_t* total, size_t* used, size_t* unifiedMemorySize, gpuInfoVulkanWarningLogCallback_t warningLogCallback) {
     return enumerateVulkanDevices(total, used, unifiedMemorySize, false, nullptr, warningLogCallback);
 }
-
-bool gpuInfoGetVulkanDeviceNames(std::vector<std::string> * deviceNames, gpuInfoVulkanWarningLogCallback_t warningLogCallback) {
-    size_t vulkanDeviceTotal = 0;
-    size_t vulkanDeviceUsed = 0;
-    size_t unifiedMemorySize = 0;
-
-    return enumerateVulkanDevices(&vulkanDeviceTotal, &vulkanDeviceUsed, &unifiedMemorySize, true, deviceNames, warningLogCallback);
-}

package/llama/gpuInfo/vulkan-gpu-info.h

@@ -6,4 +6,3 @@
 typedef void (*gpuInfoVulkanWarningLogCallback_t)(const char* message);
 
 bool gpuInfoGetTotalVulkanDevicesInfo(size_t* total, size_t* used, size_t* unifiedMemorySize, gpuInfoVulkanWarningLogCallback_t warningLogCallback);
-bool gpuInfoGetVulkanDeviceNames(std::vector<std::string> * deviceNames, gpuInfoVulkanWarningLogCallback_t warningLogCallback);

package/llama/grammars/README.md

@@ -46,7 +46,7 @@ Terminals support the full range of Unicode. Unicode characters can be specified
 
 Character ranges can be negated with `^`:
 ```
-single-line ::= [^\n]+ "\n"`
+single-line ::= [^\n]+ "\n"
 ```
 
 ## Sequences and Alternatives
@@ -124,7 +124,7 @@ You can use GBNF grammars:
 - In [llama-cli](../examples/main), passed as the `--json` / `-j` flag
 - To convert to a grammar ahead of time:
     - in CLI, with [examples/json_schema_to_grammar.py](../examples/json_schema_to_grammar.py)
-    - in JavaScript with [json-schema-to-grammar.mjs](../examples/server/public/json-schema-to-grammar.mjs) (this is used by the [server](../examples/server)'s Web UI)
+    - in JavaScript with [json-schema-to-grammar.mjs](../examples/server/public_legacy/json-schema-to-grammar.mjs) (this is used by the [server](../examples/server)'s Web UI)
 
 Take a look at [tests](../tests/test-json-schema-to-grammar.cpp) to see which features are likely supported (you'll also find usage examples in https://github.com/ggerganov/llama.cpp/pull/5978, https://github.com/ggerganov/llama.cpp/pull/6659 & https://github.com/ggerganov/llama.cpp/pull/6555).
 

package/llama/grammars/english.gbnf

@@ -0,0 +1,6 @@
+# note: this might be incomplete, mostly an example
+root        ::= en-char+ ([ \t\n] en-char+)*
+en-char     ::= letter | digit | punctuation
+letter      ::= [a-zA-Z]
+digit       ::= [0-9]
+punctuation ::= [!"#$%&'()*+,-./:;<=>?@[\\\]^_`{|}~]

package/llama/llama.cpp.info.json

@@ -1,4 +1,4 @@
 {
-    "tag": "b3995",
+    "tag": "b4291",
     "llamaCppGithubRepo": "ggerganov/llama.cpp"
 }