node-llama-cpp 3.0.0-beta.19 → 3.0.0-beta.20

package/llama/addon.cpp

@@ -218,6 +218,10 @@ Napi::Value getGpuType(const Napi::CallbackInfo& info) {
 }
 
 static Napi::Value getNapiToken(const Napi::CallbackInfo& info, llama_model* model, llama_token token) {
+    if (token < 0) {
+        return Napi::Number::From(info.Env(), -1);
+    }
+
     auto tokenType = llama_token_get_type(model, token);
 
     if (tokenType == LLAMA_TOKEN_TYPE_UNDEFINED || tokenType == LLAMA_TOKEN_TYPE_UNKNOWN) {
@@ -228,6 +232,10 @@ static Napi::Value getNapiToken(const Napi::CallbackInfo& info, llama_model* mod
 }
 
 static Napi::Value getNapiControlToken(const Napi::CallbackInfo& info, llama_model* model, llama_token token) {
+    if (token < 0) {
+        return Napi::Number::From(info.Env(), -1);
+    }
+    
     auto tokenType = llama_token_get_type(model, token);
 
     if (tokenType != LLAMA_TOKEN_TYPE_CONTROL && tokenType != LLAMA_TOKEN_TYPE_USER_DEFINED) {
@@ -351,6 +359,7 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
         }
 
         Napi::Value Init(const Napi::CallbackInfo& info);
+        Napi::Value LoadLora(const Napi::CallbackInfo& info);
         Napi::Value AbortActiveModelLoad(const Napi::CallbackInfo& info) {
             abortModelLoad = true;
             return info.Env().Undefined();
@@ -585,6 +594,7 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
                     "AddonModel",
                     {
                         InstanceMethod("init", &AddonModel::Init),
+                        InstanceMethod("loadLora", &AddonModel::LoadLora),
                         InstanceMethod("abortActiveModelLoad", &AddonModel::AbortActiveModelLoad),
                         InstanceMethod("tokenize", &AddonModel::Tokenize),
                         InstanceMethod("detokenize", &AddonModel::Detokenize),
@@ -737,6 +747,76 @@ class AddonModelUnloadModelWorker : public Napi::AsyncWorker {
             deferred.Reject(err.Value());
         }
 };
+class AddonModelLoadLoraWorker : public Napi::AsyncWorker {
+    public:
+        AddonModel* model;
+        std::string loraFilePath;
+        float loraScale;
+        int32_t loraThreads;
+        std::string baseModelPath;
+
+        AddonModelLoadLoraWorker(
+            const Napi::Env& env,
+            AddonModel* model,
+            std::string loraFilePath,
+            float loraScale,
+            int32_t loraThreads,
+            std::string baseModelPath
+        )
+            : Napi::AsyncWorker(env, "AddonModelLoadLoraWorker"),
+              model(model),
+              loraFilePath(loraFilePath),
+              loraScale(loraScale),
+              loraThreads(loraThreads),
+              baseModelPath(baseModelPath),
+              deferred(Napi::Promise::Deferred::New(env)) {
+            model->Ref();
+        }
+        ~AddonModelLoadLoraWorker() {
+            model->Unref();
+        }
+
+        Napi::Promise GetPromise() {
+            return deferred.Promise();
+        }
+
+    protected:
+        Napi::Promise::Deferred deferred;
+
+        void Execute() {
+            try {
+                const auto res = llama_model_apply_lora_from_file(
+                    model->model,
+                    loraFilePath.c_str(),
+                    loraScale,
+                    baseModelPath.empty() ? NULL : baseModelPath.c_str(),
+                    loraThreads
+                );
+
+                if (res != 0) {
+                    SetError(
+                        std::string(
+                            std::string("Failed to apply LoRA \"") + loraFilePath + std::string("\"") + (
+                                baseModelPath.empty()
+                                    ? std::string("")
+                                    : (std::string(" with base model \"") + baseModelPath + std::string("\""))
+                            )
+                        )
+                    );
+                }
+            } catch (const std::exception& e) {
+                SetError(e.what());
+            } catch(...) {
+                SetError("Unknown error when calling \"llama_model_apply_lora_from_file\"");
+            }
+        }
+        void OnOK() {
+            deferred.Resolve(Env().Undefined());
+        }
+        void OnError(const Napi::Error& err) {
+            deferred.Reject(err.Value());
+        }
+};
 
 Napi::Value AddonModel::Init(const Napi::CallbackInfo& info) {
     if (disposed) {
@@ -748,6 +828,18 @@ Napi::Value AddonModel::Init(const Napi::CallbackInfo& info) {
     worker->Queue();
     return worker->GetPromise();
 }
+Napi::Value AddonModel::LoadLora(const Napi::CallbackInfo& info) {
+    std::string loraFilePath = info[0].As<Napi::String>().Utf8Value();
+    float scale = info[1].As<Napi::Number>().FloatValue();
+    int32_t threads = info[2].As<Napi::Number>().Int32Value();
+    std::string baseModelPath = (info.Length() > 3 && info[3].IsString()) ? info[3].As<Napi::String>().Utf8Value() : std::string("");
+
+    int32_t resolvedThreads = threads == 0 ? std::thread::hardware_concurrency() : threads;
+
+    AddonModelLoadLoraWorker* worker = new AddonModelLoadLoraWorker(this->Env(), this, loraFilePath, scale, threads, baseModelPath);
+    worker->Queue();
+    return worker->GetPromise();
+}
 Napi::Value AddonModel::Dispose(const Napi::CallbackInfo& info) {
     if (disposed) {
         return info.Env().Undefined();

package/llama/binariesGithubRelease.json

@@ -1,3 +1,3 @@
 {
-    "release": "b2861"
+    "release": "b2928"
 }

package/llama/llama.cpp.info.json

@@ -1,4 +1,4 @@
 {
-    "tag": "b2861",
+    "tag": "b2928",
     "llamaCppGithubRepo": "ggerganov/llama.cpp"
 }

package/llamaBins/linux-arm64/_nlcBuildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","platformInfo":{"name":"Ubuntu","version":"22.04"},"arch":"arm64","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2861"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","platformInfo":{"name":"Ubuntu","version":"22.04"},"arch":"arm64","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2928"}}}

package/llamaBins/linux-armv7l/_nlcBuildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","platformInfo":{"name":"Ubuntu","version":"22.04"},"arch":"armv7l","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2861"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","platformInfo":{"name":"Ubuntu","version":"22.04"},"arch":"armv7l","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2928"}}}

package/llamaBins/linux-x64/_nlcBuildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","platformInfo":{"name":"Ubuntu","version":"22.04"},"arch":"x64","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2861"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","platformInfo":{"name":"Ubuntu","version":"22.04"},"arch":"x64","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2928"}}}

package/llamaBins/linux-x64-cuda/_nlcBuildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","platformInfo":{"name":"Ubuntu","version":"22.04"},"arch":"x64","gpu":"cuda","llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2861"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","platformInfo":{"name":"Ubuntu","version":"22.04"},"arch":"x64","gpu":"cuda","llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2928"}}}

package/llamaBins/linux-x64-vulkan/_nlcBuildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","platformInfo":{"name":"Ubuntu","version":"22.04"},"arch":"x64","gpu":"vulkan","llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2861"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","platformInfo":{"name":"Ubuntu","version":"22.04"},"arch":"x64","gpu":"vulkan","llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2928"}}}

package/llamaBins/mac-arm64-metal/_nlcBuildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"mac","platformInfo":{"name":"macOS","version":"21.6.0"},"arch":"arm64","gpu":"metal","llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2861"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"mac","platformInfo":{"name":"macOS","version":"21.6.0"},"arch":"arm64","gpu":"metal","llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2928"}}}

package/llamaBins/mac-x64/_nlcBuildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"mac","platformInfo":{"name":"macOS","version":"21.6.0"},"arch":"x64","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2861"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"mac","platformInfo":{"name":"macOS","version":"21.6.0"},"arch":"x64","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2928"}}}

package/llamaBins/win-arm64/_nlcBuildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"win","platformInfo":{"name":"Windows","version":"10.0.20348"},"arch":"arm64","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2861"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"win","platformInfo":{"name":"Windows","version":"10.0.20348"},"arch":"arm64","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2928"}}}

package/llamaBins/win-x64/_nlcBuildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"win","platformInfo":{"name":"Windows","version":"10.0.20348"},"arch":"x64","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2861"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"win","platformInfo":{"name":"Windows","version":"10.0.20348"},"arch":"x64","gpu":false,"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2928"}}}

package/llamaBins/win-x64-cuda/_nlcBuildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"win","platformInfo":{"name":"Windows","version":"10.0.20348"},"arch":"x64","gpu":"cuda","llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2861"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"win","platformInfo":{"name":"Windows","version":"10.0.20348"},"arch":"x64","gpu":"cuda","llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2928"}}}

package/llamaBins/win-x64-vulkan/_nlcBuildMetadata.json

@@ -1 +1 @@
-{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"win","platformInfo":{"name":"Windows","version":"10.0.20348"},"arch":"x64","gpu":"vulkan","llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2861"}}}
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"win","platformInfo":{"name":"Windows","version":"10.0.20348"},"arch":"x64","gpu":"vulkan","llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2928"}}}

package/package.json

@@ -1,15 +1,16 @@
 {
   "name": "node-llama-cpp",
-  "version": "3.0.0-beta.19",
+  "version": "3.0.0-beta.20",
   "description": "Run AI models locally on your machine with node.js bindings for llama.cpp. Force a JSON schema on the model output on the generation level",
-  "main": "dist/index.js",
+  "main": "./dist/index.js",
   "type": "module",
   "types": "./dist/index.d.ts",
   "bin": {
-    "node-llama-cpp": "./dist/cli/cli.js"
+    "node-llama-cpp": "dist/cli/cli.js"
   },
   "files": [
     "dist/",
+    "templates/packed/",
     "llama/",
     "llamaBins/",
     "package.json",
@@ -41,8 +42,10 @@
   },
   "scripts": {
     "prepare": "[ \"$CI\" = true ] || [ -d '.husky/_' ] || husky install",
+    "postinstall": "node ./dist/cli/cli.js postinstall",
     "prebuild": "rimraf ./dist ./tsconfig.tsbuildinfo",
-    "build": "tsc --build tsconfig.json --force",
+    "build": "tsc --build tsconfig.json --force && npm run build:packTemplates",
+    "build:packTemplates": "vite-node scripts/packTemplates.ts",
     "addPostinstallScript": "npm pkg set scripts.postinstall=\"node ./dist/cli/cli.js postinstall\"",
     "prewatch": "rimraf ./dist ./tsconfig.tsbuildinfo",
     "watch": "tsc --build tsconfig.json --watch --force",
@@ -55,18 +58,17 @@
     "test:modelDependent:interactive": "vitest watch ./test/modelDependent",
     "test:typescript": "tsc --build tsconfig.json --dry --force",
     "lint": "npm run lint:eslint",
-    "lint:eslint": "eslint --ext .js --ext .ts .",
+    "lint:eslint": "eslint --ext .js --ext .ts --report-unused-disable-directives .",
     "format": "npm run lint:eslint -- --fix",
     "dev:setup:downloadAllTestModels": "vite-node test/utils/scripts/downloadAllTestModels.ts",
     "dev:setup": "npm run build && node ./dist/cli/cli.js download --noUsageExample && npm run docs:generateTypedoc && npm run dev:setup:downloadAllTestModels",
     "dev:build": "npm run build && node ./dist/cli/cli.js build --noUsageExample",
-    "clean": "rm -rf ./node_modules ./dist ./tsconfig.tsbuildinfo ./test/.models ./docs/api ./docs/api-overrides",
+    "clean": "rm -rf ./node_modules ./dist ./tsconfig.tsbuildinfo ./test/.models ./docs/api ./docs/api-overrides ./templates/packed",
     "docs:generateTypedoc": "typedoc && rimraf ./docs/api/index.md ./docs/api/globals.md ./docs/api/functions/LlamaText.md && npm run docs:generateTypedoc:overrides",
     "docs:generateTypedoc:overrides": "typedoc --entryPoints ./src/apiDocsOverrides.ts --out ./docs/api-overrides && copyfiles --flat \"./docs/api-overrides/classes/LlamaText.md\" ./docs/api/classes && rimraf ./docs/api-overrides",
     "docs:dev": "npm run docs:generateTypedoc && vitepress dev",
     "docs:build": "npm run docs:generateTypedoc && vitepress build",
-    "docs:preview": "npm run docs:generateTypedoc && vitepress preview",
-    "postinstall": "node ./dist/cli/cli.js postinstall"
+    "docs:preview": "npm run docs:generateTypedoc && vitepress preview"
   },
   "repository": {
     "type": "git",
@@ -113,8 +115,8 @@
   },
   "homepage": "https://withcatai.github.io/node-llama-cpp/",
   "devDependencies": {
-    "@commitlint/cli": "^17.7.1",
-    "@commitlint/config-conventional": "^17.7.0",
+    "@commitlint/cli": "^19.3.0",
+    "@commitlint/config-conventional": "^19.2.2",
     "@semantic-release/exec": "^6.0.3",
     "@shikijs/vitepress-twoslash": "^1.3.0",
     "@types/async-retry": "^1.4.8",
@@ -124,7 +126,7 @@
     "@types/node": "^20.11.29",
     "@types/proper-lockfile": "^4.1.4",
     "@types/semver": "^7.5.8",
-    "@types/uuid": "^9.0.2",
+    "@types/validate-npm-package-name": "^4.0.2",
     "@types/which": "^3.0.0",
     "@types/yargs": "^17.0.24",
     "@typescript-eslint/eslint-plugin": "^6.3.0",
@@ -138,7 +140,7 @@
     "eslint-plugin-n": "^16.3.1",
     "husky": "^8.0.3",
     "rimraf": "^5.0.1",
-    "semantic-release": "^22.0.8",
+    "semantic-release": "^23.1.1",
     "tslib": "^2.6.1",
     "typedoc": "^0.25.13",
     "typedoc-plugin-markdown": "^4.0.0-next.55",
@@ -162,10 +164,12 @@
     "env-var": "^7.3.1",
     "filenamify": "^6.0.0",
     "fs-extra": "^11.2.0",
+    "ignore": "^5.3.1",
     "ipull": "^3.1.1",
     "is-unicode-supported": "^2.0.0",
     "lifecycle-utils": "^1.4.1",
     "log-symbols": "^5.1.0",
+    "nanoid": "^5.0.7",
     "node-addon-api": "^7.0.0",
     "octokit": "^3.1.0",
     "ora": "^7.0.1",
@@ -176,7 +180,7 @@
     "slice-ansi": "^7.1.0",
     "stdout-update": "^4.0.1",
     "strip-ansi": "^7.1.0",
-    "uuid": "^9.0.0",
+    "validate-npm-package-name": "^5.0.1",
     "which": "^4.0.0",
     "yargs": "^17.7.2"
   },

package/templates/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "eslint-dependencies-for-ci",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "devDependencies": {
+    "eslint-plugin-react-hooks": "^4.6.0",
+    "eslint-plugin-react-refresh": "^0.4.5"
+  }
+}