node-llama-cpp 3.0.0-beta.3 → 3.0.0-beta.30

package/llama/binariesGithubRelease.json

@@ -1,3 +1,3 @@
 {
-    "release": "b1892"
+    "release": "b3166"
 }

package/llama/gpuInfo/cuda-gpu-info.cu

@@ -0,0 +1,120 @@
+#include <stddef.h>
+#include <vector>
+#include <string>
+
+#if defined(GPU_INFO_USE_HIPBLAS)
+#include <hip/hip_runtime.h>
+#include <hipblas/hipblas.h>
+#define cudaGetDevice hipGetDevice
+#define cudaGetDeviceCount hipGetDeviceCount
+#define cudaGetErrorString hipGetErrorString
+#define cudaMemGetInfo hipMemGetInfo
+#define cudaSetDevice hipSetDevice
+#define cudaSuccess hipSuccess
+#else
+#include <cuda_runtime.h>
+#include <cuda.h>
+#endif
+
+
+typedef void (*gpuInfoCudaErrorLogCallback_t)(const char* message);
+
+bool gpuInfoSetCudaDevice(const int device, gpuInfoCudaErrorLogCallback_t errorLogCallback) {
+    int current_device;
+    auto getDeviceResult = cudaGetDevice(&current_device);
+
+    if (getDeviceResult != cudaSuccess) {
+        errorLogCallback(cudaGetErrorString(getDeviceResult));
+        return false;
+    }
+
+    if (device == current_device) {
+        return true;
+    }
+
+    const auto setDeviceResult = cudaSetDevice(device);
+
+    if (setDeviceResult != cudaSuccess) {
+        errorLogCallback(cudaGetErrorString(setDeviceResult));
+        return false;
+    }
+
+    return true;
+}
+
+bool gpuInfoGetCudaDeviceInfo(int device, size_t * total, size_t * used, gpuInfoCudaErrorLogCallback_t errorLogCallback) {
+    gpuInfoSetCudaDevice(device, errorLogCallback);
+
+    size_t freeMem;
+    size_t totalMem;
+    auto getMemInfoResult = cudaMemGetInfo(&freeMem, &totalMem);
+
+    if (getMemInfoResult != cudaSuccess) {
+        errorLogCallback(cudaGetErrorString(getMemInfoResult));
+        return false;
+    }
+
+    *total = totalMem;
+    *used = totalMem - freeMem;
+
+    return true;
+}
+
+int gpuInfoGetCudaDeviceCount(gpuInfoCudaErrorLogCallback_t errorLogCallback) {
+    int deviceCount;
+    auto getDeviceCountResult = cudaGetDeviceCount(&deviceCount);
+
+    if (getDeviceCountResult != cudaSuccess) {
+        errorLogCallback(cudaGetErrorString(getDeviceCountResult));
+        return -1;
+    }
+
+    return deviceCount;
+}
+
+bool gpuInfoGetTotalCudaDevicesInfo(size_t * total, size_t * used, gpuInfoCudaErrorLogCallback_t errorLogCallback) {
+    int deviceCount = gpuInfoGetCudaDeviceCount(errorLogCallback);
+
+    if (deviceCount < 0) {
+        return false;
+    }
+
+    size_t usedMem = 0;
+    size_t totalMem = 0;
+
+    for (int i = 0; i < deviceCount; i++) {
+        size_t deviceUsedMem;
+        size_t deviceTotalMem;
+
+        if (!gpuInfoGetCudaDeviceInfo(i, &deviceTotalMem, &deviceUsedMem, errorLogCallback)) {
+            return false;
+        }
+
+        usedMem += deviceUsedMem;
+        totalMem += deviceTotalMem;
+    }
+
+    *total = totalMem;
+    *used = usedMem;
+
+    return true;
+}
+
+void gpuInfoGetCudaDeviceNames(std::vector<std::string> * deviceNames, gpuInfoCudaErrorLogCallback_t errorLogCallback) {
+    int deviceCount = gpuInfoGetCudaDeviceCount(errorLogCallback);
+
+    if (deviceCount < 0) {
+        return;
+    }
+
+    for (int i = 0; i < deviceCount; i++) {
+        cudaDeviceProp prop;
+        auto getDevicePropertiesResult = cudaGetDeviceProperties(&prop, i);
+
+        if (getDevicePropertiesResult != cudaSuccess) {
+            errorLogCallback(cudaGetErrorString(getDevicePropertiesResult));
+        } else {
+            (*deviceNames).push_back(std::string(prop.name));
+        }
+    }
+}

package/llama/gpuInfo/cuda-gpu-info.h

@@ -0,0 +1,10 @@
+#pragma once
+
+#include <stddef.h>
+#include <vector>
+#include <string>
+
+typedef void (*gpuInfoCudaErrorLogCallback_t)(const char* message);
+
+bool gpuInfoGetTotalCudaDevicesInfo(size_t * total, size_t * used, gpuInfoCudaErrorLogCallback_t errorLogCallback);
+void gpuInfoGetCudaDeviceNames(std::vector<std::string> * deviceNames, gpuInfoCudaErrorLogCallback_t errorLogCallback);

package/llama/gpuInfo/metal-gpu-info.h

@@ -0,0 +1,8 @@
+#pragma once
+
+#include <stdint.h>
+#include <string>
+#include <vector>
+
+void getMetalGpuInfo(uint64_t * total, uint64_t * used);
+void getMetalGpuDeviceNames(std::vector<std::string> * deviceNames);

package/llama/gpuInfo/metal-gpu-info.mm

@@ -0,0 +1,30 @@
+#include <stdint.h>
+#include <vector>
+#include <string>
+#import <Metal/Metal.h>
+
+void getMetalGpuInfo(uint64_t * total, uint64_t * used) {
+    id<MTLDevice> device = MTLCreateSystemDefaultDevice();
+
+    if (device) {
+        *total = device.recommendedMaxWorkingSetSize;
+        *used = device.currentAllocatedSize;
+    } else {
+        *total = 0;
+        *used = 0;
+    }
+
+    [device release];
+    device = nil;
+}
+
+void getMetalGpuDeviceNames(std::vector<std::string> * deviceNames) {
+    NSArray<id<MTLDevice>> *devices = MTLCopyAllDevices();
+
+    for (id<MTLDevice> device in devices) {
+        (*deviceNames).push_back(std::string(([NSString stringWithUTF8String:device.name.UTF8String]).UTF8String));
+    }
+
+    [devices release];
+    devices = nil;
+}

package/llama/gpuInfo/vulkan-gpu-info.cpp

@@ -0,0 +1,83 @@
+#include <stddef.h>
+#include <vector>
+
+#include <vulkan/vulkan.hpp>
+
+typedef void (*gpuInfoVulkanWarningLogCallback_t)(const char* message);
+
+static bool enumerateVulkanDevices(size_t* total, size_t* used, bool addDeviceNames, std::vector<std::string> * deviceNames, gpuInfoVulkanWarningLogCallback_t warningLogCallback) {
+    vk::ApplicationInfo appInfo("node-llama-cpp GPU info", 1, "llama.cpp", 1, VK_API_VERSION_1_2);
+    vk::InstanceCreateInfo createInfo(vk::InstanceCreateFlags(), &appInfo, {}, {});
+    vk::Instance instance = vk::createInstance(createInfo);
+
+    auto physicalDevices = instance.enumeratePhysicalDevices();
+
+    size_t usedMem = 0;
+    size_t totalMem = 0;
+
+    for (size_t i = 0; i < physicalDevices.size(); i++) {
+        vk::PhysicalDevice physicalDevice = physicalDevices[i];
+        vk::PhysicalDeviceMemoryProperties memProps = physicalDevice.getMemoryProperties();
+        vk::PhysicalDeviceProperties deviceProps = physicalDevice.getProperties();
+
+        if (deviceProps.deviceType == vk::PhysicalDeviceType::eCpu) {
+            // ignore CPU devices, as we don't want to count RAM from the CPU as VRAM
+            continue;
+        }
+
+        std::vector<vk::ExtensionProperties> extensionProperties = physicalDevice.enumerateDeviceExtensionProperties();
+        bool memoryBudgetExtensionSupported =
+            std::any_of(
+                extensionProperties.begin(),
+                extensionProperties.end(),
+                [](const vk::ExtensionProperties& ext) { return std::string(ext.extensionName.data()) == VK_EXT_MEMORY_BUDGET_EXTENSION_NAME;}
+            );
+
+        if (memoryBudgetExtensionSupported) {
+            vk::PhysicalDeviceMemoryBudgetPropertiesEXT memoryBudgetProperties;
+            vk::PhysicalDeviceMemoryProperties2 memProps2 = {};
+            memProps2.pNext = &memoryBudgetProperties;
+
+            physicalDevice.getMemoryProperties2(&memProps2);
+
+            for (uint32_t i = 0; i < memProps.memoryHeapCount; ++i) {
+                if (memProps.memoryHeaps[i].flags & vk::MemoryHeapFlagBits::eDeviceLocal) {
+                    const auto size = memProps.memoryHeaps[i].size;
+                    totalMem += size;
+                    usedMem += memoryBudgetProperties.heapUsage[i];
+
+                    if (size > 0 && addDeviceNames) {
+                        (*deviceNames).push_back(std::string(deviceProps.deviceName.data()));
+                    }
+
+                    break;
+                }
+            }
+        } else {
+            // VK_EXT_memory_budget extension is not supported, so we cannot determine used memory
+            warningLogCallback(
+                (
+                    "Vulkan VK_EXT_memory_budget extension not supported for device \"" +
+                    std::string(deviceProps.deviceName.data()) + "\", so VRAM info cannot be determained for it"
+                )
+                    .c_str()
+            );
+            return false;
+        }
+    }
+
+    *total = totalMem;
+    *used = usedMem;
+    return true;
+}
+
+bool gpuInfoGetTotalVulkanDevicesInfo(size_t* total, size_t* used, gpuInfoVulkanWarningLogCallback_t warningLogCallback) {
+    return enumerateVulkanDevices(total, used, false, nullptr, warningLogCallback);
+}
+
+bool gpuInfoGetVulkanDeviceNames(std::vector<std::string> * deviceNames, gpuInfoVulkanWarningLogCallback_t warningLogCallback) {
+    size_t vulkanDeviceTotal = 0;
+    size_t vulkanDeviceUsed = 0;
+
+    return enumerateVulkanDevices(&vulkanDeviceTotal, &vulkanDeviceUsed, true, deviceNames, warningLogCallback);
+}

package/llama/gpuInfo/vulkan-gpu-info.h

@@ -0,0 +1,9 @@
+#pragma once
+
+#include <stddef.h>
+#include <vector>
+
+typedef void (*gpuInfoVulkanWarningLogCallback_t)(const char* message);
+
+bool gpuInfoGetTotalVulkanDevicesInfo(size_t* total, size_t* used, gpuInfoVulkanWarningLogCallback_t warningLogCallback);
+bool gpuInfoGetVulkanDeviceNames(std::vector<std::string> * deviceNames, gpuInfoVulkanWarningLogCallback_t warningLogCallback);

package/llama/grammars/README.md

@@ -51,7 +51,7 @@ single-line ::= [^\n]+ "\n"`
 
 ## Sequences and Alternatives
 
-The order of symbols in a sequence matter. For example, in `"1. " move " " move "\n"`, the `"1. "` must come before the first `move`, etc.
+The order of symbols in a sequence matters. For example, in `"1. " move " " move "\n"`, the `"1. "` must come before the first `move`, etc.
 
 Alternatives, denoted by `|`, give different sequences that are acceptable. For example, in `move ::= pawn | nonpawn | castle`, `move` can be a `pawn` move, a `nonpawn` move, or a `castle`.
 
@@ -59,9 +59,13 @@ Parentheses `()` can be used to group sequences, which allows for embedding alte
 
 ## Repetition and Optional Symbols
 
-- `*` after a symbol or sequence means that it can be repeated zero or more times.
-- `+` denotes that the symbol or sequence should appear one or more times.
-- `?` makes the preceding symbol or sequence optional.
+- `*` after a symbol or sequence means that it can be repeated zero or more times (equivalent to `{0,}`).
+- `+` denotes that the symbol or sequence should appear one or more times (equivalent to `{1,}`).
+- `?` makes the preceding symbol or sequence optional (equivalent to `{0,1}`).
+- `{m}` repeats the precedent symbol or sequence exactly `m` times
+- `{m,}` repeats the precedent symbol or sequence at least `m` times
+- `{m,n}` repeats the precedent symbol or sequence at between `m` and `n` times (included)
+- `{0,n}` repeats the precedent symbol or sequence at most `n` times (included)
 
 ## Comments and newlines
 
@@ -87,5 +91,54 @@ item ::= [^\n]+ "\n"
 
 This guide provides a brief overview. Check out the GBNF files in this directory (`grammars/`) for examples of full grammars. You can try them out with:
 ```
-./main -m <model> --grammar-file grammars/some-grammar.gbnf -p 'Some prompt'
+./llama-cli -m <model> --grammar-file grammars/some-grammar.gbnf -p 'Some prompt'
 ```
+
+`llama.cpp` can also convert JSON schemas to grammars either ahead of time or at each request, see below.
+
+## Troubleshooting
+
+Grammars currently have performance gotchas (see https://github.com/ggerganov/llama.cpp/issues/4218).
+
+### Efficient optional repetitions
+
+A common pattern is to allow repetitions of a pattern `x` up to N times.
+
+While semantically correct, the syntax `x? x? x?.... x?` (with N repetitions) may result in extremely slow sampling. Instead, you can write `x{0,N}` (or `(x (x (x ... (x)?...)?)?)?` w/ N-deep nesting in earlier llama.cpp versions).
+
+## Using GBNF grammars
+
+You can use GBNF grammars:
+
+- In [llama-server](../examples/server)'s completion endpoints, passed as the `grammar` body field
+- In [llama-cli](../examples/main), passed as the `--grammar` & `--grammar-file` flags
+- With [llama-gbnf-validator](../examples/gbnf-validator) tool, to test them against strings.
+
+## JSON Schemas → GBNF
+
+`llama.cpp` supports converting a subset of https://json-schema.org/ to GBNF grammars:
+
+- In [llama-server](../examples/server):
+    - For any completion endpoints, passed as the `json_schema` body field
+    - For the `/chat/completions` endpoint, passed inside the `result_format` body field (e.g. `{"type", "json_object", "schema": {"items": {}}}`)
+- In [llama-cli](../examples/main), passed as the `--json` / `-j` flag
+- To convert to a grammar ahead of time:
+    - in CLI, with [examples/json_schema_to_grammar.py](../examples/json_schema_to_grammar.py)
+    - in JavaScript with [json-schema-to-grammar.mjs](../examples/server/public/json-schema-to-grammar.mjs) (this is used by the [server](../examples/server)'s Web UI)
+
+Take a look at [tests](../../tests/test-json-schema-to-grammar.cpp) to see which features are likely supported (you'll also find usage examples in https://github.com/ggerganov/llama.cpp/pull/5978, https://github.com/ggerganov/llama.cpp/pull/6659 & https://github.com/ggerganov/llama.cpp/pull/6555).
+
+Here is also a non-exhaustive list of **unsupported** features:
+
+- `additionalProperties`: to be fixed in https://github.com/ggerganov/llama.cpp/pull/7840
+- `minimum`, `exclusiveMinimum`, `maximum`, `exclusiveMaximum`
+    - `integer` constraints to be implemented in https://github.com/ggerganov/llama.cpp/pull/7797
+- Remote `$ref`s in the C++ version (Python & JavaScript versions fetch https refs)
+- Mixing `properties` w/ `anyOf` / `oneOf` in the same type (https://github.com/ggerganov/llama.cpp/issues/7703)
+- `string` formats `uri`, `email`
+- [`contains`](https://json-schema.org/draft/2020-12/json-schema-core#name-contains) / `minContains`
+- `uniqueItems`
+- `$anchor` (cf. [dereferencing](https://json-schema.org/draft/2020-12/json-schema-core#name-dereferencing))
+- [`not`](https://json-schema.org/draft/2020-12/json-schema-core#name-not)
+- [Conditionals](https://json-schema.org/draft/2020-12/json-schema-core#name-keywords-for-applying-subsche) `if` / `then` / `else` / `dependentSchemas`
+- [`patternProperties`](https://json-schema.org/draft/2020-12/json-schema-core#name-patternproperties)

package/llama/grammars/json.gbnf

@@ -15,11 +15,11 @@ array  ::=
 
 string ::=
   "\"" (
-    [^"\\] |
-    "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes
+    [^"\\\x7F\x00-\x1F] |
+    "\\" (["\\bfnrt] | "u" [0-9a-fA-F]{4}) # escapes
   )* "\"" ws
 
-number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws
+number ::= ("-"? ([0-9] | [1-9] [0-9]{0,15})) ("." [0-9]+)? ([eE] [-+]? [0-9] [1-9]{0,15})? ws
 
 # Optional space: by convention, applied in this grammar after literal chars when allowed
-ws ::= ([ \t\n] ws)?
+ws ::= | " " | "\n" [ \t]{0,20}

package/llama/grammars/json_arr.gbnf

@@ -24,11 +24,11 @@ array  ::=
 
 string ::=
   "\"" (
-    [^"\\] |
-    "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes
+    [^"\\\x7F\x00-\x1F] |
+    "\\" (["\\bfnrt] | "u" [0-9a-fA-F]{4}) # escapes
   )* "\"" ws
 
-number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws
+number ::= ("-"? ([0-9] | [1-9] [0-9]{0,15})) ("." [0-9]+)? ([eE] [-+]? [1-9] [0-9]{0,15})? ws
 
 # Optional space: by convention, applied in this grammar after literal chars when allowed
-ws ::= ([ \t\n] ws)?
+ws ::= | " " | "\n" [ \t]{0,20}

package/llama/llama.cpp.info.json

@@ -0,0 +1,4 @@
+{
+    "tag": "b3166",
+    "llamaCppGithubRepo": "ggerganov/llama.cpp"
+}

package/llama/toolchains/win32.host-x64.target-arm64.cmake

@@ -0,0 +1,41 @@
+set(CMAKE_SYSTEM_NAME Windows)
+set(CMAKE_SYSTEM_PROCESSOR ARM64)
+
+# Look for cl.exe in the Visual Studio installation directories
+set(PROGRAMFILES "$ENV{ProgramFiles}")
+set(PROGRAMFILES_X86 "$ENV{ProgramFiles\(x86\)}")
+
+set(VS_INSTALL_PATHS
+    "${PROGRAMFILES_X86}/Microsoft Visual Studio"
+    "${PROGRAMFILES}/Microsoft Visual Studio"
+    "C:/Program Files (x86)/Microsoft Visual Studio"
+    "C:/Program Files/Microsoft Visual Studio"
+)
+foreach(PATH IN LISTS VS_INSTALL_PATHS)
+    if(CL_EXE_PATH)
+        break()
+    endif()
+
+    file(GLOB_RECURSE FOUND_CL_EXE "${PATH}/*/VC/Tools/MSVC/*/bin/Hostx64/arm64/cl.exe")
+    if(FOUND_CL_EXE)
+        list(GET FOUND_CL_EXE 0 CL_EXE_PATH)
+        break()
+    endif()
+
+    if(CL_EXE_PATH)
+        break()
+    endif()
+
+    file(GLOB_RECURSE FOUND_CL_EXE "${PATH}/**/*/VC/Tools/MSVC/*/bin/Hostx64/arm64/cl.exe")
+    if(FOUND_CL_EXE)
+        list(GET FOUND_CL_EXE 0 CL_EXE_PATH)
+        break()
+    endif()
+endforeach()
+
+if(NOT CL_EXE_PATH)
+    message(FATAL_ERROR "cl.exe not found for ARM architecture.")
+else()
+    set(CMAKE_C_COMPILER "${CL_EXE_PATH}")
+    set(CMAKE_CXX_COMPILER "${CL_EXE_PATH}")
+endif()

package/package.json

@@ -1,17 +1,18 @@
 {
   "name": "node-llama-cpp",
-  "version": "3.0.0-beta.3",
+  "version": "3.0.0-beta.30",
   "description": "Run AI models locally on your machine with node.js bindings for llama.cpp. Force a JSON schema on the model output on the generation level",
-  "main": "dist/index.js",
+  "main": "./dist/index.js",
   "type": "module",
   "types": "./dist/index.d.ts",
   "bin": {
-    "node-llama-cpp": "./dist/cli/cli.js"
+    "node-llama-cpp": "dist/cli/cli.js"
   },
   "files": [
     "dist/",
+    "templates/packed/",
     "llama/",
-    "llamaBins/",
+    "bins/",
     "package.json",
     "README.md",
     "LICENSE"
@@ -40,31 +41,33 @@
     "node": ">=18.0.0"
   },
   "scripts": {
-    "prepare": "[ \"$CI\" = true ] || [ -d '.husky/_' ] || husky install",
+    "prepare": "[ \"$CI\" = true ] || [ -d '.husky/_' ] || husky",
+    "postinstall": "node ./dist/cli/cli.js postinstall",
     "prebuild": "rimraf ./dist ./tsconfig.tsbuildinfo",
-    "build": "tsc --build tsconfig.json --force",
+    "build": "tsc --build tsconfig.json --force && npm run build:packTemplates",
+    "build:packTemplates": "vite-node scripts/packTemplates.ts",
     "addPostinstallScript": "npm pkg set scripts.postinstall=\"node ./dist/cli/cli.js postinstall\"",
     "prewatch": "rimraf ./dist ./tsconfig.tsbuildinfo",
     "watch": "tsc --build tsconfig.json --watch --force",
     "cmake-js-llama": "cd llama && cmake-js",
-    "test": "npm run test:typescript && npm run lint:eslint && npm run test:standalone && npm run test:modelDependent",
+    "test": "npm run test:typescript && npm run lint:eslint && npm run test:vitest",
+    "test:vitest": "vitest run ./test",
     "test:standalone": "vitest run ./test/standalone",
     "test:standalone:interactive": "vitest watch ./test/standalone",
     "test:modelDependent": "vitest run ./test/modelDependent",
     "test:modelDependent:interactive": "vitest watch ./test/modelDependent",
     "test:typescript": "tsc --build tsconfig.json --dry --force",
     "lint": "npm run lint:eslint",
-    "lint:eslint": "eslint --ext .js --ext .ts .",
+    "lint:eslint": "eslint --ext .js --ext .ts --report-unused-disable-directives .",
     "format": "npm run lint:eslint -- --fix",
-    "dev:setup:downloadAllTestModels": "node --loader ts-node/esm test/utils/scripts/downloadAllTestModels.ts",
-    "dev:setup": "npm run build && node ./dist/cli/cli.js download && npm run docs:generateTypedoc && npm run dev:setup:downloadAllTestModels",
-    "dev:build": "npm run build && node ./dist/cli/cli.js build",
-    "clean": "rm -rf ./node_modules ./dist ./tsconfig.tsbuildinfo ./test/.models",
-    "docs:generateTypedoc": "typedoc && rimraf ./docs/api/index.md ./docs/api/exports.md",
+    "dev:setup:downloadAllTestModels": "vite-node test/utils/scripts/downloadAllTestModels.ts",
+    "dev:setup": "npm run build && node ./dist/cli/cli.js download --noUsageExample && npm run docs:generateTypedoc && npm run dev:setup:downloadAllTestModels",
+    "dev:build": "npm run build && node ./dist/cli/cli.js build --noUsageExample",
+    "clean": "rm -rf ./node_modules ./dist ./tsconfig.tsbuildinfo ./test/.models ./docs/api ./docs/api-overrides ./templates/packed",
+    "docs:generateTypedoc": "typedoc --entryPoints ./src/apiDocsIndex.ts && rimraf ./docs/api/index.md ./docs/api/globals.md ./docs/api/functions/LlamaText.md",
     "docs:dev": "npm run docs:generateTypedoc && vitepress dev",
     "docs:build": "npm run docs:generateTypedoc && vitepress build",
-    "docs:preview": "npm run docs:generateTypedoc && vitepress preview",
-    "postinstall": "node ./dist/cli/cli.js postinstall"
+    "docs:preview": "npm run docs:generateTypedoc && vitepress preview"
   },
   "repository": {
     "type": "git",
@@ -83,10 +86,15 @@
     "gguf",
     "metal",
     "cuda",
+    "vulkan",
     "grammar",
     "json-grammar",
     "json-schema-grammar",
+    "functions",
+    "function-calling",
+    "embedding",
     "temperature",
+    "minP",
     "topK",
     "topP",
     "json-schema",
@@ -106,54 +114,71 @@
   },
   "homepage": "https://withcatai.github.io/node-llama-cpp/",
   "devDependencies": {
-    "@commitlint/cli": "^17.7.1",
-    "@commitlint/config-conventional": "^17.7.0",
+    "@commitlint/cli": "^19.3.0",
+    "@commitlint/config-conventional": "^19.2.2",
     "@semantic-release/exec": "^6.0.3",
-    "@types/cli-progress": "^3.11.0",
+    "@shikijs/vitepress-twoslash": "^1.6.3",
+    "@types/async-retry": "^1.4.8",
+    "@types/bytes": "^3.1.4",
     "@types/cross-spawn": "^6.0.2",
-    "@types/fs-extra": "^11.0.1",
-    "@types/node": "^20.8.4",
-    "@types/uuid": "^9.0.2",
-    "@types/which": "^3.0.0",
+    "@types/fs-extra": "^11.0.4",
+    "@types/node": "^20.14.2",
+    "@types/proper-lockfile": "^4.1.4",
+    "@types/semver": "^7.5.8",
+    "@types/validate-npm-package-name": "^4.0.2",
+    "@types/which": "^3.0.4",
     "@types/yargs": "^17.0.24",
-    "@typescript-eslint/eslint-plugin": "^6.3.0",
-    "@typescript-eslint/parser": "^6.3.0",
-    "@vitest/coverage-v8": "^0.34.6",
+    "@typescript-eslint/eslint-plugin": "^7.12.0",
+    "@typescript-eslint/parser": "^7.12.0",
+    "@vitest/coverage-v8": "^1.6.0",
+    "@vitest/ui": "^1.6.0",
     "eslint": "^8.46.0",
-    "eslint-plugin-import": "^2.28.0",
-    "eslint-plugin-jsdoc": "^46.9.0",
-    "eslint-plugin-n": "^16.3.1",
-    "husky": "^8.0.3",
-    "ipull": "^1.2.1",
-    "rimraf": "^5.0.1",
-    "semantic-release": "^22.0.8",
-    "ts-node": "^10.9.1",
-    "tslib": "^2.6.1",
-    "typedoc": "^0.25.3",
-    "typedoc-plugin-markdown": "4.0.0-next.41",
-    "typedoc-plugin-mdn-links": "^3.1.5",
-    "typedoc-vitepress-theme": "^1.0.0-next.7",
-    "typescript": "^5.2.2",
-    "vitepress": "1.0.0-rc.22",
-    "vitest": "^0.34.6",
-    "zx": "^7.2.3"
+    "eslint-plugin-import": "^2.29.1",
+    "eslint-plugin-jsdoc": "^46.10.1",
+    "eslint-plugin-n": "^17.8.1",
+    "husky": "^9.0.11",
+    "rimraf": "^5.0.7",
+    "semantic-release": "^23.1.1",
+    "tslib": "^2.6.3",
+    "typedoc": "^0.25.13",
+    "typedoc-plugin-markdown": "^4.0.3",
+    "typedoc-plugin-mdn-links": "^3.1.28",
+    "typedoc-vitepress-theme": "^1.0.0",
+    "typescript": "^5.4.5",
+    "vite-node": "^1.6.0",
+    "vitepress": "^1.2.3",
+    "vitest": "^1.6.0",
+    "zx": "^8.1.2"
   },
   "dependencies": {
+    "@huggingface/jinja": "^0.2.2",
+    "async-retry": "^1.3.3",
+    "bytes": "^3.1.2",
     "chalk": "^5.3.0",
     "chmodrp": "^1.0.2",
-    "cli-progress": "^3.12.0",
-    "cmake-js": "^7.2.1",
+    "cmake-js": "^7.3.0",
     "cross-env": "^7.0.3",
     "cross-spawn": "^7.0.3",
-    "env-var": "^7.3.1",
-    "fs-extra": "^11.1.1",
-    "lifecycle-utils": "^1.1.3",
-    "log-symbols": "^5.1.0",
-    "node-addon-api": "^7.0.0",
-    "octokit": "^3.1.0",
-    "ora": "^7.0.1",
-    "simple-git": "^3.19.1",
-    "uuid": "^9.0.0",
+    "env-var": "^7.5.0",
+    "filenamify": "^6.0.0",
+    "fs-extra": "^11.2.0",
+    "ignore": "^5.3.1",
+    "ipull": "^3.3.0",
+    "is-unicode-supported": "^2.0.0",
+    "lifecycle-utils": "^1.4.1",
+    "log-symbols": "^6.0.0",
+    "nanoid": "^5.0.7",
+    "node-addon-api": "^8.0.0",
+    "octokit": "^4.0.2",
+    "ora": "^8.0.1",
+    "pretty-ms": "^9.0.0",
+    "proper-lockfile": "^4.1.2",
+    "semver": "^7.6.2",
+    "simple-git": "^3.24.0",
+    "slice-ansi": "^7.1.0",
+    "stdout-update": "^4.0.1",
+    "strip-ansi": "^7.1.0",
+    "validate-npm-package-name": "^5.0.1",
     "which": "^4.0.0",
     "yargs": "^17.7.2"
   },