node-llama-cpp 3.0.0-beta.14 → 3.0.0-beta.15

package/llama/addon.cpp

@@ -9,7 +9,7 @@
 #include "llama.h"
 #include "napi.h"
 
-#ifdef GPU_INFO_USE_CUBLAS
+#ifdef GPU_INFO_USE_CUDA
 #  include "gpuInfo/cuda-gpu-info.h"
 #endif
 #ifdef GPU_INFO_USE_VULKAN
@@ -121,7 +121,7 @@ std::string addon_model_token_to_piece(const struct llama_model* model, llama_to
     return std::string(result.data(), result.size());
 }
 
-#ifdef GPU_INFO_USE_CUBLAS
+#ifdef GPU_INFO_USE_CUDA
 void logCudaError(const char* message) {
     addonLlamaCppLogCallback(GGML_LOG_LEVEL_ERROR, (std::string("CUDA error: ") + std::string(message)).c_str(), nullptr);
 }
@@ -136,7 +136,7 @@ Napi::Value getGpuVramInfo(const Napi::CallbackInfo& info) {
     uint64_t total = 0;
     uint64_t used = 0;
 
-#ifdef GPU_INFO_USE_CUBLAS
+#ifdef GPU_INFO_USE_CUDA
     size_t cudaDeviceTotal = 0;
     size_t cudaDeviceUsed = 0;
     bool cudeGetInfoSuccess = gpuInfoGetTotalCudaDevicesInfo(&cudaDeviceTotal, &cudaDeviceUsed, logCudaError);
@@ -161,7 +161,7 @@ Napi::Value getGpuVramInfo(const Napi::CallbackInfo& info) {
 #ifdef GPU_INFO_USE_METAL
     uint64_t metalDeviceTotal = 0;
     uint64_t metalDeviceUsed = 0;
-    get_metal_gpu_info(&metalDeviceTotal, &metalDeviceUsed);
+    getMetalGpuInfo(&metalDeviceTotal, &metalDeviceUsed);
 
     total += metalDeviceTotal;
     used += metalDeviceUsed;
@@ -174,8 +174,34 @@ Napi::Value getGpuVramInfo(const Napi::CallbackInfo& info) {
     return result;
 }
 
+Napi::Value getGpuDeviceInfo(const Napi::CallbackInfo& info) {
+    std::vector<std::string> deviceNames;
+
+#ifdef GPU_INFO_USE_CUDA
+    gpuInfoGetCudaDeviceNames(&deviceNames, logCudaError);
+#endif
+
+#ifdef GPU_INFO_USE_VULKAN
+    gpuInfoGetVulkanDeviceNames(&deviceNames, logVulkanWarning);
+#endif
+
+#ifdef GPU_INFO_USE_METAL
+    getMetalGpuDeviceNames(&deviceNames);
+#endif
+
+    Napi::Object result = Napi::Object::New(info.Env());
+
+    Napi::Array deviceNamesNapiArray = Napi::Array::New(info.Env(), deviceNames.size());
+    for (size_t i = 0; i < deviceNames.size(); ++i) {
+        deviceNamesNapiArray[i] = Napi::String::New(info.Env(), deviceNames[i]);
+    }
+    result.Set("deviceNames", deviceNamesNapiArray);
+
+    return result;
+}
+
 Napi::Value getGpuType(const Napi::CallbackInfo& info) {
-#ifdef GPU_INFO_USE_CUBLAS
+#ifdef GPU_INFO_USE_CUDA
     return Napi::String::New(info.Env(), "cuda");
 #endif
 
@@ -507,6 +533,16 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
 
             return Napi::Number::From(info.Env(), int32_t(tokenType));
         }
+        Napi::Value GetVocabularyType(const Napi::CallbackInfo& info) {
+            if (disposed) {
+                Napi::Error::New(info.Env(), "Model is disposed").ThrowAsJavaScriptException();
+                return info.Env().Undefined();
+            }
+
+            auto vocabularyType = llama_vocab_type(model);
+
+            return Napi::Number::From(info.Env(), int32_t(vocabularyType));
+        }
         Napi::Value ShouldPrependBosToken(const Napi::CallbackInfo& info) {
             const int addBos = llama_add_bos_token(model);
 
@@ -515,6 +551,10 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
             return Napi::Boolean::New(info.Env(), shouldPrependBos);
         }
 
+        Napi::Value GetModelSize(const Napi::CallbackInfo& info) {
+            return Napi::Number::From(info.Env(), llama_model_size(model));
+        }
+
         static void init(Napi::Object exports) {
             exports.Set(
                 "AddonModel",
@@ -540,7 +580,9 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
                         InstanceMethod("eotToken", &AddonModel::EotToken),
                         InstanceMethod("getTokenString", &AddonModel::GetTokenString),
                         InstanceMethod("getTokenType", &AddonModel::GetTokenType),
+                        InstanceMethod("getVocabularyType", &AddonModel::GetVocabularyType),
                         InstanceMethod("shouldPrependBosToken", &AddonModel::ShouldPrependBosToken),
+                        InstanceMethod("getModelSize", &AddonModel::GetModelSize),
                         InstanceMethod("dispose", &AddonModel::Dispose),
                     }
                 )
@@ -822,6 +864,10 @@ class AddonContext : public Napi::ObjectWrap<AddonContext> {
                     context_params.n_ubatch = context_params.n_batch; // the batch queue is managed in the JS side, so there's no need for managing it on the C++ side
                 }
 
+                if (options.Has("sequences")) {
+                    context_params.n_seq_max = options.Get("sequences").As<Napi::Number>().Uint32Value();
+                }
+
                 if (options.Has("embeddings")) {
                     context_params.embeddings = options.Get("embeddings").As<Napi::Boolean>().Value();
                 }
@@ -1039,6 +1085,15 @@ class AddonContext : public Napi::ObjectWrap<AddonContext> {
             return result;
         }
 
+        Napi::Value GetStateSize(const Napi::CallbackInfo& info) {
+            if (disposed) {
+                Napi::Error::New(info.Env(), "Context is disposed").ThrowAsJavaScriptException();
+                return info.Env().Undefined();
+            }
+
+            return Napi::Number::From(info.Env(), llama_get_state_size(ctx));
+        }
+
         Napi::Value PrintTimings(const Napi::CallbackInfo& info) {
             llama_print_timings(ctx);
             llama_reset_timings(ctx);
@@ -1063,6 +1118,7 @@ class AddonContext : public Napi::ObjectWrap<AddonContext> {
                         InstanceMethod("sampleToken", &AddonContext::SampleToken),
                         InstanceMethod("acceptGrammarEvaluationStateToken", &AddonContext::AcceptGrammarEvaluationStateToken),
                         InstanceMethod("getEmbedding", &AddonContext::GetEmbedding),
+                        InstanceMethod("getStateSize", &AddonContext::GetStateSize),
                         InstanceMethod("printTimings", &AddonContext::PrintTimings),
                         InstanceMethod("dispose", &AddonContext::Dispose),
                     }
@@ -1444,6 +1500,55 @@ Napi::Value systemInfo(const Napi::CallbackInfo& info) {
     return Napi::String::From(info.Env(), llama_print_system_info());
 }
 
+Napi::Value addonGetSupportsGpuOffloading(const Napi::CallbackInfo& info) {
+    return Napi::Boolean::New(info.Env(), llama_supports_gpu_offload());
+}
+
+Napi::Value addonGetSupportsMmap(const Napi::CallbackInfo& info) {
+    return Napi::Boolean::New(info.Env(), llama_supports_mmap());
+}
+
+Napi::Value addonGetSupportsMlock(const Napi::CallbackInfo& info) {
+    return Napi::Boolean::New(info.Env(), llama_supports_mlock());
+}
+
+Napi::Value addonGetBlockSizeForGgmlType(const Napi::CallbackInfo& info) {
+    const int ggmlType = info[0].As<Napi::Number>().Int32Value();
+
+    if (ggmlType < 0 || ggmlType > GGML_TYPE_COUNT) {
+        return info.Env().Undefined();
+    }
+
+    const auto blockSize = ggml_blck_size(static_cast<ggml_type>(ggmlType));
+
+    return Napi::Number::New(info.Env(), blockSize);
+}
+
+Napi::Value addonGetTypeSizeForGgmlType(const Napi::CallbackInfo& info) {
+    const int ggmlType = info[0].As<Napi::Number>().Int32Value();
+
+    if (ggmlType < 0 || ggmlType > GGML_TYPE_COUNT) {
+        return info.Env().Undefined();
+    }
+
+    const auto typeSize = ggml_type_size(static_cast<ggml_type>(ggmlType));
+
+    return Napi::Number::New(info.Env(), typeSize);
+}
+
+Napi::Value addonGetConsts(const Napi::CallbackInfo& info) {
+    Napi::Object consts = Napi::Object::New(info.Env());
+    consts.Set("ggmlMaxDims", Napi::Number::New(info.Env(), GGML_MAX_DIMS));
+    consts.Set("ggmlTypeF16Size", Napi::Number::New(info.Env(), ggml_type_size(GGML_TYPE_F16)));
+    consts.Set("ggmlTypeF32Size", Napi::Number::New(info.Env(), ggml_type_size(GGML_TYPE_F32)));
+    consts.Set("ggmlTensorOverhead", Napi::Number::New(info.Env(), ggml_tensor_overhead()));
+    consts.Set("llamaMaxRngState", Napi::Number::New(info.Env(), LLAMA_MAX_RNG_STATE));
+    consts.Set("llamaPosSize", Napi::Number::New(info.Env(), sizeof(llama_pos)));
+    consts.Set("llamaSeqIdSize", Napi::Number::New(info.Env(), sizeof(llama_seq_id)));
+
+    return consts;
+}
+
 int addonGetGgmlLogLevelNumber(ggml_log_level level) {
     switch (level) {
         case GGML_LOG_LEVEL_ERROR: return 2;
@@ -1693,9 +1798,16 @@ static void addonFreeLlamaBackend(Napi::Env env, int* data) {
 Napi::Object registerCallback(Napi::Env env, Napi::Object exports) {
     exports.DefineProperties({
         Napi::PropertyDescriptor::Function("systemInfo", systemInfo),
+        Napi::PropertyDescriptor::Function("getSupportsGpuOffloading", addonGetSupportsGpuOffloading),
+        Napi::PropertyDescriptor::Function("getSupportsMmap", addonGetSupportsMmap),
+        Napi::PropertyDescriptor::Function("getSupportsMlock", addonGetSupportsMlock),
+        Napi::PropertyDescriptor::Function("getBlockSizeForGgmlType", addonGetBlockSizeForGgmlType),
+        Napi::PropertyDescriptor::Function("getTypeSizeForGgmlType", addonGetTypeSizeForGgmlType),
+        Napi::PropertyDescriptor::Function("getConsts", addonGetConsts),
         Napi::PropertyDescriptor::Function("setLogger", setLogger),
         Napi::PropertyDescriptor::Function("setLoggerLogLevel", setLoggerLogLevel),
         Napi::PropertyDescriptor::Function("getGpuVramInfo", getGpuVramInfo),
+        Napi::PropertyDescriptor::Function("getGpuDeviceInfo", getGpuDeviceInfo),
         Napi::PropertyDescriptor::Function("getGpuType", getGpuType),
         Napi::PropertyDescriptor::Function("init", addonInit),
         Napi::PropertyDescriptor::Function("dispose", addonDispose),

package/llama/binariesGithubRelease.json

@@ -1,3 +1,3 @@
 {
-    "release": "b2440"
+    "release": "b2608"
 }

package/llama/gpuInfo/cuda-gpu-info.cu

@@ -1,4 +1,6 @@
 #include <stddef.h>
+#include <vector>
+#include <string>
 
 #if defined(GPU_INFO_USE_HIPBLAS)
 #include <hip/hip_runtime.h>
@@ -97,3 +99,22 @@ bool gpuInfoGetTotalCudaDevicesInfo(size_t * total, size_t * used, gpuInfoCudaEr
 
     return true;
 }
+
+void gpuInfoGetCudaDeviceNames(std::vector<std::string> * deviceNames, gpuInfoCudaErrorLogCallback_t errorLogCallback) {
+    int deviceCount = gpuInfoGetCudaDeviceCount(errorLogCallback);
+
+    if (deviceCount < 0) {
+        return;
+    }
+
+    for (int i = 0; i < deviceCount; i++) {
+        cudaDeviceProp prop;
+        auto getDevicePropertiesResult = cudaGetDeviceProperties(&prop, i);
+
+        if (getDevicePropertiesResult != cudaSuccess) {
+            errorLogCallback(cudaGetErrorString(getDevicePropertiesResult));
+        } else {
+            (*deviceNames).push_back(std::string(prop.name));
+        }
+    }
+}

package/llama/gpuInfo/cuda-gpu-info.h

@@ -1,7 +1,10 @@
 #pragma once
 
 #include <stddef.h>
+#include <vector>
+#include <string>
 
 typedef void (*gpuInfoCudaErrorLogCallback_t)(const char* message);
 
 bool gpuInfoGetTotalCudaDevicesInfo(size_t * total, size_t * used, gpuInfoCudaErrorLogCallback_t errorLogCallback);
+void gpuInfoGetCudaDeviceNames(std::vector<std::string> * deviceNames, gpuInfoCudaErrorLogCallback_t errorLogCallback);

package/llama/gpuInfo/metal-gpu-info.h

@@ -1,5 +1,8 @@
 #pragma once
 
 #include <stdint.h>
+#include <string>
+#include <vector>
 
-void get_metal_gpu_info(uint64_t * total, uint64_t * used);
+void getMetalGpuInfo(uint64_t * total, uint64_t * used);
+void getMetalGpuDeviceNames(std::vector<std::string> * deviceNames);

package/llama/gpuInfo/metal-gpu-info.mm

@@ -1,7 +1,9 @@
 #include <stdint.h>
+#include <vector>
+#include <string>
 #import <Metal/Metal.h>
 
-void get_metal_gpu_info(uint64_t * total, uint64_t * used) {
+void getMetalGpuInfo(uint64_t * total, uint64_t * used) {
     id<MTLDevice> device = MTLCreateSystemDefaultDevice();
 
     if (device) {
@@ -15,3 +17,14 @@ void get_metal_gpu_info(uint64_t * total, uint64_t * used) {
     [device release];
     device = nil;
 }
+
+void getMetalGpuDeviceNames(std::vector<std::string> * deviceNames) {
+    NSArray<id<MTLDevice>> *devices = MTLCopyAllDevices();
+
+    for (id<MTLDevice> device in devices) {
+        (*deviceNames).push_back(std::string(([NSString stringWithUTF8String:device.name.UTF8String]).UTF8String));
+    }
+
+    [devices release];
+    devices = nil;
+}

package/llama/gpuInfo/vulkan-gpu-info.cpp

@@ -1,10 +1,11 @@
 #include <stddef.h>
+#include <vector>
 
 #include <vulkan/vulkan.hpp>
 
 typedef void (*gpuInfoVulkanWarningLogCallback_t)(const char* message);
 
-bool gpuInfoGetTotalVulkanDevicesInfo(size_t* total, size_t* used, gpuInfoVulkanWarningLogCallback_t warningLogCallback) {
+static bool enumerateVulkanDevices(size_t* total, size_t* used, bool addDeviceNames, std::vector<std::string> * deviceNames, gpuInfoVulkanWarningLogCallback_t warningLogCallback) {
     vk::ApplicationInfo appInfo("node-llama-cpp GPU info", 1, "llama.cpp", 1, VK_API_VERSION_1_2);
     vk::InstanceCreateInfo createInfo(vk::InstanceCreateFlags(), &appInfo, {}, {});
     vk::Instance instance = vk::createInstance(createInfo);
@@ -41,8 +42,14 @@ bool gpuInfoGetTotalVulkanDevicesInfo(size_t* total, size_t* used, gpuInfoVulkan
 
             for (uint32_t i = 0; i < memProps.memoryHeapCount; ++i) {
                 if (memProps.memoryHeaps[i].flags & vk::MemoryHeapFlagBits::eDeviceLocal) {
-                    totalMem += memProps.memoryHeaps[i].size;
+                    const auto size = memProps.memoryHeaps[i].size;
+                    totalMem += size;
                     usedMem += memoryBudgetProperties.heapUsage[i];
+
+                    if (size > 0 && addDeviceNames) {
+                        (*deviceNames).push_back(std::string(deviceProps.deviceName.data()));
+                    }
+
                     break;
                 }
             }
@@ -63,3 +70,14 @@ bool gpuInfoGetTotalVulkanDevicesInfo(size_t* total, size_t* used, gpuInfoVulkan
     *used = usedMem;
     return true;
 }
+
+bool gpuInfoGetTotalVulkanDevicesInfo(size_t* total, size_t* used, gpuInfoVulkanWarningLogCallback_t warningLogCallback) {
+    return enumerateVulkanDevices(total, used, false, nullptr, warningLogCallback);
+}
+
+bool gpuInfoGetVulkanDeviceNames(std::vector<std::string> * deviceNames, gpuInfoVulkanWarningLogCallback_t warningLogCallback) {
+    size_t vulkanDeviceTotal = 0;
+    size_t vulkanDeviceUsed = 0;
+
+    return enumerateVulkanDevices(&vulkanDeviceTotal, &vulkanDeviceUsed, true, deviceNames, warningLogCallback);
+}

package/llama/gpuInfo/vulkan-gpu-info.h

@@ -1,7 +1,9 @@
 #pragma once
 
 #include <stddef.h>
+#include <vector>
 
 typedef void (*gpuInfoVulkanWarningLogCallback_t)(const char* message);
 
 bool gpuInfoGetTotalVulkanDevicesInfo(size_t* total, size_t* used, gpuInfoVulkanWarningLogCallback_t warningLogCallback);
+bool gpuInfoGetVulkanDeviceNames(std::vector<std::string> * deviceNames, gpuInfoVulkanWarningLogCallback_t warningLogCallback);

package/llama/llama.cpp.info.json

@@ -1,4 +1,4 @@
 {
-    "tag": "b2440",
+    "tag": "b2608",
     "llamaCppGithubRepo": "ggerganov/llama.cpp"
 }

package/llama/toolchains/win32.host-x64.target-arm64.cmake

@@ -0,0 +1,41 @@
+set(CMAKE_SYSTEM_NAME Windows)
+set(CMAKE_SYSTEM_PROCESSOR ARM64)
+
+# Look for cl.exe in the Visual Studio installation directories
+set(PROGRAMFILES "$ENV{ProgramFiles}")
+set(PROGRAMFILES_X86 "$ENV{ProgramFiles\(x86\)}")
+
+set(VS_INSTALL_PATHS
+    "${PROGRAMFILES_X86}/Microsoft Visual Studio"
+    "${PROGRAMFILES}/Microsoft Visual Studio"
+    "C:/Program Files (x86)/Microsoft Visual Studio"
+    "C:/Program Files/Microsoft Visual Studio"
+)
+foreach(PATH IN LISTS VS_INSTALL_PATHS)
+    if(CL_EXE_PATH)
+        break()
+    endif()
+
+    file(GLOB_RECURSE FOUND_CL_EXE "${PATH}/*/VC/Tools/MSVC/*/bin/Hostx64/arm64/cl.exe")
+    if(FOUND_CL_EXE)
+        list(GET FOUND_CL_EXE 0 CL_EXE_PATH)
+        break()
+    endif()
+
+    if(CL_EXE_PATH)
+        break()
+    endif()
+
+    file(GLOB_RECURSE FOUND_CL_EXE "${PATH}/**/*/VC/Tools/MSVC/*/bin/Hostx64/arm64/cl.exe")
+    if(FOUND_CL_EXE)
+        list(GET FOUND_CL_EXE 0 CL_EXE_PATH)
+        break()
+    endif()
+endforeach()
+
+if(NOT CL_EXE_PATH)
+    message(FATAL_ERROR "cl.exe not found for ARM architecture.")
+else()
+    set(CMAKE_C_COMPILER "${CL_EXE_PATH}")
+    set(CMAKE_CXX_COMPILER "${CL_EXE_PATH}")
+endif()

package/llamaBins/linux-arm64/_nlcBuildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","platformInfo":{"name":"Ubuntu","version":"22.04"},"arch":"arm64","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2440"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","platformInfo":{"name":"Ubuntu","version":"22.04"},"arch":"arm64","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2608"}}}

package/llamaBins/linux-armv7l/_nlcBuildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","platformInfo":{"name":"Ubuntu","version":"22.04"},"arch":"armv7l","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2440"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","platformInfo":{"name":"Ubuntu","version":"22.04"},"arch":"armv7l","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2608"}}}

package/llamaBins/linux-x64/_nlcBuildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","platformInfo":{"name":"Ubuntu","version":"22.04"},"arch":"x64","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2440"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","platformInfo":{"name":"Ubuntu","version":"22.04"},"arch":"x64","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2608"}}}

package/llamaBins/linux-x64-cuda/_nlcBuildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","platformInfo":{"name":"Ubuntu","version":"22.04"},"arch":"x64","gpu":"cuda","llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2440"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","platformInfo":{"name":"Ubuntu","version":"22.04"},"arch":"x64","gpu":"cuda","llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2608"}}}

package/llamaBins/linux-x64-vulkan/_nlcBuildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","platformInfo":{"name":"Ubuntu","version":"22.04"},"arch":"x64","gpu":"vulkan","llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2440"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","platformInfo":{"name":"Ubuntu","version":"22.04"},"arch":"x64","gpu":"vulkan","llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2608"}}}

package/llamaBins/mac-arm64-metal/_nlcBuildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"mac","platformInfo":{"name":"macOS","version":"21.6.0"},"arch":"arm64","gpu":"metal","llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2440"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"mac","platformInfo":{"name":"macOS","version":"21.6.0"},"arch":"arm64","gpu":"metal","llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2608"}}}

package/llamaBins/mac-x64/_nlcBuildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"mac","platformInfo":{"name":"macOS","version":"21.6.0"},"arch":"x64","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2440"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"mac","platformInfo":{"name":"macOS","version":"21.6.0"},"arch":"x64","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2608"}}}

package/llamaBins/win-arm64/_nlcBuildMetadata.json

@@ -0,0 +1 @@
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"win","platformInfo":{"name":"Windows","version":"10.0.20348"},"arch":"arm64","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2608"}}}

package/llamaBins/win-x64/_nlcBuildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"win","platformInfo":{"name":"Windows","version":"10.0.20348"},"arch":"x64","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2440"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"win","platformInfo":{"name":"Windows","version":"10.0.20348"},"arch":"x64","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2608"}}}

package/llamaBins/win-x64-cuda/_nlcBuildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"win","platformInfo":{"name":"Windows","version":"10.0.20348"},"arch":"x64","gpu":"cuda","llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2440"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"win","platformInfo":{"name":"Windows","version":"10.0.20348"},"arch":"x64","gpu":"cuda","llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2608"}}}

package/llamaBins/win-x64-vulkan/_nlcBuildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"win","platformInfo":{"name":"Windows","version":"10.0.20348"},"arch":"x64","gpu":"vulkan","llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2440"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"win","platformInfo":{"name":"Windows","version":"10.0.20348"},"arch":"x64","gpu":"vulkan","llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2608"}}}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "node-llama-cpp",
-  "version": "3.0.0-beta.14",
+  "version": "3.0.0-beta.15",
   "description": "Run AI models locally on your machine with node.js bindings for llama.cpp. Force a JSON schema on the model output on the generation level",
   "main": "dist/index.js",
   "type": "module",
@@ -120,7 +120,7 @@
     "@types/cli-progress": "^3.11.0",
     "@types/cross-spawn": "^6.0.2",
     "@types/fs-extra": "^11.0.4",
-    "@types/node": "^20.8.4",
+    "@types/node": "^20.11.29",
     "@types/proper-lockfile": "^4.1.4",
     "@types/semver": "^7.5.8",
     "@types/uuid": "^9.0.2",
@@ -128,8 +128,8 @@
     "@types/yargs": "^17.0.24",
     "@typescript-eslint/eslint-plugin": "^6.3.0",
     "@typescript-eslint/parser": "^6.3.0",
-    "@vitest/coverage-v8": "^1.2.2",
-    "@vitest/ui": "^1.2.2",
+    "@vitest/coverage-v8": "^1.4.0",
+    "@vitest/ui": "^1.4.0",
     "eslint": "^8.46.0",
     "eslint-plugin-import": "^2.28.0",
     "eslint-plugin-jsdoc": "^46.9.0",
@@ -146,10 +146,11 @@
     "typescript": "^5.2.2",
     "vite-node": "^1.4.0",
     "vitepress": "1.0.0-rc.22",
-    "vitest": "^1.2.2",
+    "vitest": "^1.4.0",
     "zx": "^7.2.3"
   },
   "dependencies": {
+    "@huggingface/jinja": "^0.2.2",
     "async-retry": "^1.3.3",
     "bytes": "^3.1.2",
     "chalk": "^5.3.0",
@@ -160,7 +161,7 @@
     "cross-spawn": "^7.0.3",
     "env-var": "^7.3.1",
     "fs-extra": "^11.2.0",
-    "lifecycle-utils": "^1.4.0",
+    "lifecycle-utils": "^1.4.1",
     "log-symbols": "^5.1.0",
     "node-addon-api": "^7.0.0",
     "octokit": "^3.1.0",
@@ -168,6 +169,7 @@
     "proper-lockfile": "^4.1.2",
     "semver": "^7.6.0",
     "simple-git": "^3.19.1",
+    "slice-ansi": "^7.1.0",
     "strip-ansi": "^7.1.0",
     "uuid": "^9.0.0",
     "which": "^4.0.0",

package/dist/TemplateChatWrapper.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"TemplateChatWrapper.js","sourceRoot":"","sources":["../src/TemplateChatWrapper.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,mBAAmB,EAAE,SAAS,EAAkB,YAAY,EAAC,MAAM,sBAAsB,CAAC;AAClG,OAAO,EAAC,WAAW,EAAsB,MAAM,kBAAkB,CAAC;AAClE,OAAO,EAAC,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AAY/D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,OAAO,mBAAoB,SAAQ,WAAW;IAChC,WAAW,GAAG,UAAU,CAAC;IAChB,QAAQ,CAAsB;IAEvC,QAAQ,CAAe;IACvB,eAAe,CAAsB;IACrC,aAAa,CAAS;IACtB,YAAY,CAAS;IACrB,cAAc,CAAS;IACvB,iCAAiC,CAAU;IAE3D,gBAAgB,CAAkB,mBAAmB,CAAuC;IAC5F,gBAAgB,CAAkB,0BAA0B,CAA8C;IAE1G,YAAmB,EACf,QAAQ,EACR,eAAe,EACf,aAAa,EACb,YAAY,EACZ,cAAc,GAAG,QAAQ,EACzB,2BAA2B,EAC3B,iCAAiC,GAAG,IAAI,EACf;QACzB,KAAK,EAAE,CAAC;QAER,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,iCAAiC,GAAG,iCAAiC,CAAC;QAE3E,IAAI,CAAC,mBAAmB,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QACvD,IAAI,CAAC,0BAA0B,GAAG,wBAAwB,CAAC,eAAe,CAAC,CAAC;QAE5E,IAAI,CAAC,QAAQ,GAAG;YACZ,GAAG,KAAK,CAAC,QAAQ;YACjB,SAAS,EAAE,gCAAgC,CAAC,2BAA2B,CAAC,IAAI,WAAW,CAAC,cAAc,CAAC,SAAS;SACnH,CAAC;IACN,CAAC;IAEe,mBAAmB,CAAC,OAAmC,EAAE,EAAC,kBAAkB,EAAE,sBAAsB,KAGhH,EAAE;QAIF,MAAM,oBAAoB,GAAG,IAAI,CAAC,2CAA2C,CAAC,OAAO,EAAE,kBAAkB,EAAE;YACvG,cAAc,EAAE,sBAAsB;SACzC,CAAC,CAAC;QAEH,MAAM,WAAW,GAIZ,EAAE,CAAC;QAER,IAAI,WAAW,GAAa,EAAE,CAAC;QAC/B,IAAI,SAAS,GAAa,EAAE,CAAC;QAC7B,IAAI,UAAU,GAAa,EAAE,CAAC;QAC9B,IAAI,qBAAqB,GAAuC,IAAI,CAAC;QAErE,SAAS,KAAK;YACV,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;gBACvE,WAAW,CAAC,IAAI,CAAC;oBACb,MAAM,EAAE,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC;oBAChC,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC;oBAC5B,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC;iBACjC,CAAC,CAAC;YAEP,WAAW,GAAG,EAAE,CAAC;YACjB,SAAS,GAAG,EAAE,CAAC;YACf,UAAU,GAAG,EAAE,CAAC;QACpB,CAAC;QAED,MAAM,cAAc,GAAG,CAAC,IAAiC,EAAE,IAAY,EAAE,MAAsB,EAAE,EAAE;YAC/F,MAAM,EAAC,cAAc,EAAE,aAAa,EAAE,aAAa,EAAC,GAAG,IAAI,CAAC,0BAA0B,CAAC;YACvF,OAAO,SAAS,CAAC;gBACb,IAAI,YAAY,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,GAAG,cAAc,GAAG,IAAI,GAAG,aAAa,CAAC;gBACxE,IAAI;gBACJ,IAAI,YAAY,CAAC,aAAa,CAAC;aAClC,CAAC,CAAC;QACP,CAAC,CAAC;QAEF,KAAK,MAAM,IAAI,IAAI,oBAAoB,EAAE;YACrC,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE;gBACxB,IAAI,CAAC,IAAI,CAAC,iCAAiC,IAAI,qBAAqB,KAAK,QAAQ;oBAC7E,KAAK,EAAE,CAAC;gBAEZ,qBAAqB,GAAG,QAAQ,CAAC;gBACjC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aAC/B;iBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE;gBAC7B,IAAI,CAAC,IAAI,CAAC,iCAAiC,IAAI,CAAC,qBAAqB,KAAK,QAAQ,IAAI,qBAAqB,KAAK,MAAM,CAAC;oBACnH,KAAK,EAAE,CAAC;gBAEZ,qBAAqB,GAAG,MAAM,CAAC;gBAC/B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aAC7B;iBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE;gBAC9B,IAAI,CAAC,IAAI,CAAC,iCAAiC;oBACvC,KAAK,EAAE,CAAC;gBAEZ,qBAAqB,GAAG,OAAO,CAAC;gBAChC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,yBAAyB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;aAClE;SACJ;QAED,KAAK,EAAE,CAAC;QAER,MAAM,WAAW,GAAG,SAAS,CACzB,WAAW,CAAC,GAAG,CAAC,CAAC,EAAC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAC,EAAE,KAAK,EAAE,EAAE;YAC7C,MAAM,WAAW,GAAG,KAAK,KAAK,CAAC,CAAC;YAChC,MAAM,UAAU,GAAG,KAAK,KAAK,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC;YAEpD,MAAM,GAAG,GAAG,SAAS,CAAC;gBAClB,WAAW;oBACP,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC;wBACjB,CAAC,CAAC,IAAI,YAAY,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,kBAAkB,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC;wBAChH,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,kBAAkB,IAAI,IAAI;4BACjD,CAAC,CAAC,SAAS,CAAC;gCACR,IAAI,YAAY,CAAC,IAAI,CAAC,mBAAmB,CAAC,kBAAkB,CAAC;gCAC7D,MAAM;gCACN,IAAI,YAAY,CAAC,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC;6BAC3D,CAAC;4BACF,CAAC,CAAC,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC;oBAClF,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC;wBACjB,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC;wBACf,CAAC,CAAC,cAAc,CAAC,QAAQ,EAAE,MAAM,CAAC;gBAG1C,IAAI,CAAC,MAAM,KAAK,CAAC;oBACb,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC;oBACf,CAAC,CAAC,cAAc,CAAC,MAAM,EAAE,IAAI,CAAC;gBAElC,KAAK,CAAC,MAAM,KAAK,CAAC;oBACd,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC;oBACf,CAAC,CAAC,CAAC,UAAU;wBACT,CAAC,CAAC,cAAc,CAAC,OAAO,EAAE,KAAK,CAAC;wBAChC,CAAC,CAAC,SAAS,CAAC;4BACR,IAAI,YAAY,CAAC,IAAI,CAAC,mBAAmB,CAAC,gBAAgB,CAAC;4BAC3D,KAAK;yBACR,CAAC;aACb,CAAC,CAAC;YAEH,OAAO,SAAS,CACZ,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;gBAC7B,IAAI,KAAK,YAAY,YAAY,EAAE;oBAC/B,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBAErC,IAAI,QAAQ,IAAI,IAAI,IAAI,CAAC,CAAC,QAAQ,YAAY,YAAY,CAAC;wBACvD,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;oBAE/B,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;wBAC3B,IAAI,YAAY,CAAC,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;qBACjD,CAAC,CAAC;iBACN;gBAED,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;YAC/B,CAAC,EAAE,EAAsB,CAAC,CAC7B,CAAC;QACN,CAAC,CAAC,CACL,CAAC;QAEF,OAAO;YACH,WAAW;YACX,sBAAsB,EAAE;gBACpB,SAAS,CAAC,IAAI,mBAAmB,CAAC,KAAK,CAAC,CAAC;gBACzC,SAAS,CAAC,IAAI,CAAC,mBAAmB,CAAC,gBAAgB,CAAC;gBACpD,SAAS,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,mBAAmB,CAAC,gBAAgB,CAAC,CAAC;aACzE;SACJ,CAAC;IACN,CAAC;CACJ;AAUD,SAAS,gCAAgC,CAAC,QAAiD;IACvF,IAAI,QAAQ,IAAI,IAAI;QAChB,OAAO,IAAI,CAAC;IAEhB,MAAM,CAAC,oBAAoB,EAAE,0BAA0B,CAAC,GAAG,QAAQ,CAAC;IAEpE,IAAI,oBAAoB,IAAI,IAAI,IAAI,0BAA0B,IAAI,IAAI;QAClE,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;IAE1F,MAAM,0BAA0B,GAAG,iBAAiB,CAAC,oBAAoB,EAAE,CAAC;YACxE,IAAI,EAAE,kBAAkB;YACxB,GAAG,EAAE,cAAc;SACtB,EAAE;YACC,IAAI,EAAE,oBAAoB;YAC1B,GAAG,EAAE,gBAAgB;SACxB,CAAC,CAAC,CAAC;IACJ,MAAM,gCAAgC,GAAG,iBAAiB,CAAC,0BAA0B,EAAE,CAAC;YACpF,IAAI,EAAE,wBAAwB;YAC9B,GAAG,EAAE,oBAAoB;SAC5B,CAAC,CAAC,CAAC;IAEJ,MAAM,UAAU,GAAG,0BAA0B,CAAC,YAAY,CAAC,MAAM,CAAC;IAClE,MAAM,gBAAgB,GAAG,0BAA0B,CAAC,cAAc,CAAC,MAAM,CAAC;IAC1E,MAAM,UAAU,GAAG,0BAA0B,CAAC,cAAc,CAAC,MAAM,CAAC;IAEpE,MAAM,YAAY,GAAG,gCAAgC,CAAC,kBAAkB,CAAC,MAAM,CAAC;IAChF,MAAM,YAAY,GAAG,gCAAgC,CAAC,kBAAkB,CAAC,MAAM,CAAC;IAEhF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,iEAAiE,CAAC,CAAC;IAEvF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;IAExF,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,8EAA8E,CAAC,CAAC;IAEpG,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,6EAA6E,CAAC,CAAC;IAEnG,OAAO;QACH,IAAI,EAAE;YACF,mBAAmB,EAAE,IAAI;YACzB,MAAM,EAAE,UAAU;YAClB,YAAY,EAAE,gBAAgB;YAC9B,MAAM,EAAE,UAAU;SACrB;QACD,MAAM,EAAE;YACJ,MAAM,EAAE,YAAY;YACpB,MAAM,EAAE,YAAY;SACvB;KACJ,CAAC;AACN,CAAC;AAED,SAAS,iBAAiB,CAAC,QAAsB;IAM7C,MAAM,cAAc,GAAG,iBAAiB,CAAC,QAAQ,EAAE,CAAC;YAChD,IAAI,EAAE,kBAAkB;YACxB,GAAG,EAAE,cAAc;YACnB,QAAQ,EAAE,IAAI;SACjB,EAAE;YACC,IAAI,EAAE,aAAa;YACnB,GAAG,EAAE,SAAS;SACjB,EAAE;YACC,IAAI,EAAE,gBAAgB;YACtB,GAAG,EAAE,YAAY;SACpB,CAAC,CAAC,CAAC;IAEJ,IAAI,cAAc,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IAE3E,OAAO;QACH,kBAAkB,EAAE,cAAc,CAAC,YAAY,EAAE,MAAM,IAAI,IAAI;QAC/D,aAAa,EAAE,cAAc,CAAC,OAAO,CAAC,MAAM;QAC5C,gBAAgB,EAAE,cAAc,CAAC,UAAU,CAAC,MAAM;QAClD,gBAAgB,EAAE,cAAc,CAAC,UAAU,CAAC,MAAM;KACrD,CAAC;AACN,CAAC;AAED,SAAS,wBAAwB,CAAC,QAA6B;IAK3D,MAAM,cAAc,GAAG,iBAAiB,CAAC,QAAQ,EAAE,CAAC;YAChD,IAAI,EAAE,cAAc;YACpB,GAAG,EAAE,UAAU;SAClB,EAAE;YACC,IAAI,EAAE,aAAa;YACnB,GAAG,EAAE,SAAS;SACjB,CAAC,CAAC,CAAC;IAEJ,OAAO;QACH,cAAc,EAAE,cAAc,CAAC,QAAQ,CAAC,MAAM;QAC9C,aAAa,EAAE,cAAc,CAAC,OAAO,CAAC,MAAM;QAC5C,aAAa,EAAE,cAAc,CAAC,OAAO,CAAC,MAAM;KAC/C,CAAC;AACN,CAAC"}

package/dist/bindings/utils/resolveChatWrapperBasedOnWrapperTypeName.d.ts

@@ -1,33 +0,0 @@
-import { ModelTypeDescription } from "../AddonTypes.js";
-import { GeneralChatWrapper } from "../../chatWrappers/GeneralChatWrapper.js";
-import { LlamaChatWrapper } from "../../chatWrappers/LlamaChatWrapper.js";
-import { AlpacaChatWrapper } from "../../chatWrappers/AlpacaChatWrapper.js";
-import { FunctionaryChatWrapper } from "../../chatWrappers/FunctionaryChatWrapper.js";
-import { ChatMLChatWrapper } from "../../chatWrappers/ChatMLChatWrapper.js";
-import { FalconChatWrapper } from "../../chatWrappers/FalconChatWrapper.js";
-import { GemmaChatWrapper } from "../../chatWrappers/GemmaChatWrapper.js";
-export declare const chatWrapperTypeNames: readonly ["auto", "general", "llamaChat", "alpacaChat", "functionary", "chatML", "falconChat", "gemma"];
-export type ChatWrapperTypeName = (typeof chatWrapperTypeNames)[number];
-declare const chatWrappers: {
-    readonly general: typeof GeneralChatWrapper;
-    readonly llamaChat: typeof LlamaChatWrapper;
-    readonly alpacaChat: typeof AlpacaChatWrapper;
-    readonly functionary: typeof FunctionaryChatWrapper;
-    readonly chatML: typeof ChatMLChatWrapper;
-    readonly falconChat: typeof FalconChatWrapper;
-    readonly gemma: typeof GemmaChatWrapper;
-};
-/**
- * @param configType
- * @param options
- */
-export declare function resolveChatWrapperBasedOnWrapperTypeName(configType: ChatWrapperTypeName, { bosString, filename, typeDescription, customWrapperSettings }?: {
-    bosString?: string | null;
-    filename?: string;
-    /** @hidden this type alias is too long in the documentation */
-    typeDescription?: ModelTypeDescription;
-    customWrapperSettings?: {
-        [wrapper in keyof typeof chatWrappers]?: ConstructorParameters<(typeof chatWrappers)[wrapper]>[0];
-    };
-}): LlamaChatWrapper | ChatMLChatWrapper | GeneralChatWrapper | FalconChatWrapper | FunctionaryChatWrapper | GemmaChatWrapper;
-export {};