node-llama-cpp 3.0.0-beta.43 → 3.0.0-beta.45

package/llama/addon/addon.cpp

@@ -3,6 +3,7 @@
 #include "AddonModelLora.h"
 #include "AddonGrammar.h"
 #include "AddonGrammarEvaluationState.h"
+#include "AddonSampler.h"
 #include "AddonContext.h"
 #include "globals/addonLog.h"
 #include "globals/addonProgress.h"
@@ -27,6 +28,10 @@ Napi::Value addonGetSupportsMlock(const Napi::CallbackInfo& info) {
     return Napi::Boolean::New(info.Env(), llama_supports_mlock());
 }
 
+Napi::Value addonGetMathCores(const Napi::CallbackInfo& info) {
+    return Napi::Number::New(info.Env(), cpu_get_num_math());
+}
+
 Napi::Value addonGetBlockSizeForGgmlType(const Napi::CallbackInfo& info) {
     const int ggmlType = info[0].As<Napi::Number>().Int32Value();
 
@@ -189,6 +194,7 @@ Napi::Object registerCallback(Napi::Env env, Napi::Object exports) {
         Napi::PropertyDescriptor::Function("getSupportsGpuOffloading", addonGetSupportsGpuOffloading),
         Napi::PropertyDescriptor::Function("getSupportsMmap", addonGetSupportsMmap),
         Napi::PropertyDescriptor::Function("getSupportsMlock", addonGetSupportsMlock),
+        Napi::PropertyDescriptor::Function("getMathCores", addonGetMathCores),
         Napi::PropertyDescriptor::Function("getBlockSizeForGgmlType", addonGetBlockSizeForGgmlType),
         Napi::PropertyDescriptor::Function("getTypeSizeForGgmlType", addonGetTypeSizeForGgmlType),
         Napi::PropertyDescriptor::Function("getConsts", addonGetConsts),
@@ -205,6 +211,7 @@ Napi::Object registerCallback(Napi::Env env, Napi::Object exports) {
     AddonGrammar::init(exports);
     AddonGrammarEvaluationState::init(exports);
     AddonContext::init(exports);
+    AddonSampler::init(exports);
 
     llama_log_set(addonLlamaCppLogCallback, nullptr);
 

package/llama/addon/globals/addonLog.cpp

@@ -11,7 +11,8 @@ static int addonGetGgmlLogLevelNumber(ggml_log_level level) {
         case GGML_LOG_LEVEL_ERROR: return 2;
         case GGML_LOG_LEVEL_WARN: return 3;
         case GGML_LOG_LEVEL_INFO: return 4;
-        case GGML_LOG_LEVEL_DEBUG: return 5;
+        case GGML_LOG_LEVEL_NONE: return 5;
+        case GGML_LOG_LEVEL_DEBUG: return 6;
     }
 
     return 1;

package/llama/binariesGithubRelease.json

@@ -1,3 +1,3 @@
 {
-    "release": "b3560"
+    "release": "b3785"
 }

package/llama/grammars/README.md

@@ -120,7 +120,7 @@ You can use GBNF grammars:
 
 - In [llama-server](../examples/server):
     - For any completion endpoints, passed as the `json_schema` body field
-    - For the `/chat/completions` endpoint, passed inside the `result_format` body field (e.g. `{"type", "json_object", "schema": {"items": {}}}`)
+    - For the `/chat/completions` endpoint, passed inside the `response_format` body field (e.g. `{"type", "json_object", "schema": {"items": {}}}` or `{ type: "json_schema", json_schema: {"schema": ...} }`)
 - In [llama-cli](../examples/main), passed as the `--json` / `-j` flag
 - To convert to a grammar ahead of time:
     - in CLI, with [examples/json_schema_to_grammar.py](../examples/json_schema_to_grammar.py)

package/llama/llama.cpp.info.json

@@ -1,4 +1,4 @@
 {
-    "tag": "b3560",
+    "tag": "b3785",
     "llamaCppGithubRepo": "ggerganov/llama.cpp"
 }

package/package.json

@@ -1,12 +1,13 @@
 {
   "name": "node-llama-cpp",
-  "version": "3.0.0-beta.43",
+  "version": "3.0.0-beta.45",
   "description": "Run AI models locally on your machine with node.js bindings for llama.cpp. Force a JSON schema on the model output on the generation level",
   "main": "./dist/index.js",
   "type": "module",
   "types": "./dist/index.d.ts",
   "bin": {
-    "node-llama-cpp": "dist/cli/cli.js"
+    "node-llama-cpp": "dist/cli/cli.js",
+    "nlc": "dist/cli/cli.js"
   },
   "files": [
     "dist/",
@@ -57,15 +58,15 @@
     "test:standalone:interactive": "vitest watch ./test/standalone",
     "test:modelDependent": "vitest run ./test/modelDependent",
     "test:modelDependent:interactive": "vitest watch ./test/modelDependent",
-    "test:typescript": "tsc --build tsconfig.json --dry --force",
+    "test:typescript": "tsc --noEmit --project tsconfig.json",
     "lint": "npm run lint:eslint",
     "lint:eslint": "eslint --ext .js --ext .ts --report-unused-disable-directives .",
     "format": "npm run lint:eslint -- --fix",
     "dev:setup:downloadAllTestModels": "vite-node test/utils/scripts/downloadAllTestModels.ts",
-    "dev:setup": "npm run build && node ./dist/cli/cli.js download --noUsageExample && npm run docs:generateTypedoc && npm run dev:setup:downloadAllTestModels",
-    "dev:build": "npm run build && node ./dist/cli/cli.js build --noUsageExample",
+    "dev:setup": "npm run build && node ./dist/cli/cli.js source download --noUsageExample && npm run docs:generateTypedoc && npm run dev:setup:downloadAllTestModels",
+    "dev:build": "npm run build && node ./dist/cli/cli.js source build --noUsageExample",
     "clean": "rm -rf ./node_modules ./dist ./tsconfig.tsbuildinfo ./test/.models ./docs/api ./docs/api-overrides ./templates/packed",
-    "docs:generateTypedoc": "typedoc --entryPoints ./src/apiDocsIndex.ts && rimraf ./docs/api/index.md ./docs/api/globals.md ./docs/api/functions/LlamaText.md",
+    "docs:generateTypedoc": "typedoc && rimraf ./docs/api/index.md ./docs/api/globals.md ./docs/api/functions/LlamaText.md",
     "docs:dev": "npm run docs:generateTypedoc && vitepress dev",
     "docs:build": "npm run docs:generateTypedoc && vitepress build",
     "docs:preview": "npm run docs:generateTypedoc && vitepress preview"
@@ -89,6 +90,7 @@
     "cuda",
     "vulkan",
     "grammar",
+    "embedding",
     "json-grammar",
     "json-schema-grammar",
     "functions",
@@ -98,11 +100,17 @@
     "minP",
     "topK",
     "topP",
+    "seed",
     "json-schema",
     "raspberry-pi",
     "self-hosted",
     "local",
-    "catai"
+    "catai",
+    "mistral",
+    "typescript",
+    "lora",
+    "batching",
+    "gpu"
   ],
   "author": "Gilad S.",
   "license": "MIT",
@@ -114,46 +122,54 @@
     "type": "github",
     "url": "https://github.com/sponsors/giladgd"
   },
-  "homepage": "https://withcatai.github.io/node-llama-cpp/",
+  "homepage": "https://node-llama-cpp.withcat.ai",
   "devDependencies": {
-    "@commitlint/cli": "^19.3.0",
-    "@commitlint/config-conventional": "^19.2.2",
+    "@commitlint/cli": "^19.5.0",
+    "@commitlint/config-conventional": "^19.5.0",
+    "@fontsource/inter": "^5.1.0",
+    "@nolebase/vitepress-plugin-git-changelog": "^2.5.0",
+    "@nolebase/vitepress-plugin-og-image": "^2.5.0",
+    "@resvg/resvg-js": "^2.6.2",
     "@semantic-release/exec": "^6.0.3",
-    "@shikijs/vitepress-twoslash": "^1.6.3",
+    "@semantic-release/npm": "12.0.1",
+    "@shikijs/vitepress-twoslash": "^1.17.7",
     "@types/async-retry": "^1.4.8",
     "@types/bytes": "^3.1.4",
     "@types/cross-spawn": "^6.0.2",
     "@types/fs-extra": "^11.0.4",
-    "@types/node": "^20.14.2",
+    "@types/node": "^22.5.5",
     "@types/proper-lockfile": "^4.1.4",
     "@types/semver": "^7.5.8",
     "@types/validate-npm-package-name": "^4.0.2",
     "@types/which": "^3.0.4",
-    "@types/yargs": "^17.0.24",
-    "@typescript-eslint/eslint-plugin": "^7.12.0",
-    "@typescript-eslint/parser": "^7.12.0",
-    "@vitest/coverage-v8": "^1.6.0",
-    "@vitest/ui": "^1.6.0",
+    "@types/yargs": "^17.0.33",
+    "@typescript-eslint/eslint-plugin": "^7.15.0",
+    "@typescript-eslint/parser": "^7.15.0",
+    "@vitest/coverage-v8": "^2.1.1",
+    "@vitest/ui": "^2.1.1",
     "eslint": "^8.46.0",
-    "eslint-plugin-import": "^2.29.1",
-    "eslint-plugin-jsdoc": "^46.10.1",
-    "eslint-plugin-n": "^17.8.1",
-    "husky": "^9.0.11",
-    "rimraf": "^5.0.7",
-    "semantic-release": "^23.1.1",
-    "tslib": "^2.6.3",
-    "typedoc": "^0.25.13",
-    "typedoc-plugin-markdown": "^4.0.3",
-    "typedoc-plugin-mdn-links": "^3.1.28",
-    "typedoc-vitepress-theme": "^1.0.0",
-    "typescript": "^5.4.5",
-    "vite-node": "^1.6.0",
-    "vitepress": "^1.2.3",
-    "vitest": "^1.6.0",
-    "zx": "^8.1.2"
+    "eslint-plugin-import": "^2.30.0",
+    "eslint-plugin-jsdoc": "^50.2.3",
+    "eslint-plugin-n": "^17.10.2",
+    "feed": "^4.2.2",
+    "husky": "^9.1.6",
+    "rehype": "^13.0.1",
+    "rimraf": "^6.0.1",
+    "semantic-release": "24.1.1",
+    "sharp": "^0.33.5",
+    "tslib": "^2.7.0",
+    "typedoc": "^0.26.7",
+    "typedoc-plugin-markdown": "^4.2.7",
+    "typedoc-plugin-mdn-links": "^3.2.12",
+    "typedoc-vitepress-theme": "^1.0.1",
+    "typescript": "^5.6.2",
+    "vite-node": "^2.1.1",
+    "vitepress": "^1.3.4",
+    "vitest": "^2.1.1",
+    "zx": "^8.1.7"
   },
   "dependencies": {
-    "@huggingface/jinja": "^0.2.2",
+    "@huggingface/jinja": "^0.3.1",
     "async-retry": "^1.3.3",
     "bytes": "^3.1.2",
     "chalk": "^5.3.0",
@@ -164,19 +180,19 @@
     "env-var": "^7.5.0",
     "filenamify": "^6.0.0",
     "fs-extra": "^11.2.0",
-    "ignore": "^5.3.1",
-    "ipull": "^3.6.0",
-    "is-unicode-supported": "^2.0.0",
-    "lifecycle-utils": "^1.4.1",
-    "log-symbols": "^6.0.0",
+    "ignore": "^5.3.2",
+    "ipull": "^3.6.2",
+    "is-unicode-supported": "^2.1.0",
+    "lifecycle-utils": "^1.7.0",
+    "log-symbols": "^7.0.0",
     "nanoid": "^5.0.7",
-    "node-addon-api": "^8.0.0",
+    "node-addon-api": "^8.1.0",
     "octokit": "^4.0.2",
-    "ora": "^8.0.1",
-    "pretty-ms": "^9.0.0",
+    "ora": "^8.1.0",
+    "pretty-ms": "^9.1.0",
     "proper-lockfile": "^4.1.2",
-    "semver": "^7.6.2",
-    "simple-git": "^3.24.0",
+    "semver": "^7.6.3",
+    "simple-git": "^3.26.0",
     "slice-ansi": "^7.1.0",
     "stdout-update": "^4.0.1",
     "strip-ansi": "^7.1.0",
@@ -193,7 +209,16 @@
     }
   },
   "optionalDependencies": {
-    "@node-llama-cpp/linux-x64-cuda": "3.0.0-beta.43",
-    "@node-llama-cpp/win-x64-cuda": "3.0.0-beta.43"
+    "@node-llama-cpp/linux-arm64": "3.0.0-beta.45",
+    "@node-llama-cpp/linux-armv7l": "3.0.0-beta.45",
+    "@node-llama-cpp/linux-x64": "3.0.0-beta.45",
+    "@node-llama-cpp/linux-x64-cuda": "3.0.0-beta.45",
+    "@node-llama-cpp/linux-x64-vulkan": "3.0.0-beta.45",
+    "@node-llama-cpp/mac-arm64-metal": "3.0.0-beta.45",
+    "@node-llama-cpp/mac-x64": "3.0.0-beta.45",
+    "@node-llama-cpp/win-arm64": "3.0.0-beta.45",
+    "@node-llama-cpp/win-x64": "3.0.0-beta.45",
+    "@node-llama-cpp/win-x64-cuda": "3.0.0-beta.45",
+    "@node-llama-cpp/win-x64-vulkan": "3.0.0-beta.45"
   }
 }