@fugood/llama.node 1.0.5 → 1.0.6

package/lib/binding.ts

@@ -231,7 +231,7 @@ export interface LlamaContext {
    * @param path Path to the vocoder model
    * @returns Promise resolving to true if loading was successful
    */
-  initVocoder(options: { path: string }): Promise<boolean>
+  initVocoder(options: { path: string, n_batch?: number }): Promise<boolean>
 
   /**
    * Unload the vocoder model

package/lib/index.ts

@@ -286,7 +286,7 @@ class LlamaContextWrapper {
     return this.ctx.getMultimodalSupport()
   }
 
-  initVocoder(options: { path: string }): Promise<boolean> {
+  initVocoder(options: { path: string, n_batch?: number }): Promise<boolean> {
     return this.ctx.initVocoder(options)
   }
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "1.0.5",
+  "version": "1.0.6",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {
@@ -70,19 +70,19 @@
     "CMakeLists.txt"
   ],
   "optionalDependencies": {
-    "@fugood/node-llama-linux-x64": "1.0.5",
-    "@fugood/node-llama-linux-x64-vulkan": "1.0.5",
-    "@fugood/node-llama-linux-x64-cuda": "1.0.5",
-    "@fugood/node-llama-linux-arm64": "1.0.5",
-    "@fugood/node-llama-linux-arm64-vulkan": "1.0.5",
-    "@fugood/node-llama-linux-arm64-cuda": "1.0.5",
-    "@fugood/node-llama-win32-x64": "1.0.5",
-    "@fugood/node-llama-win32-x64-vulkan": "1.0.5",
-    "@fugood/node-llama-win32-x64-cuda": "1.0.5",
-    "@fugood/node-llama-win32-arm64": "1.0.5",
-    "@fugood/node-llama-win32-arm64-vulkan": "1.0.5",
-    "@fugood/node-llama-darwin-x64": "1.0.5",
-    "@fugood/node-llama-darwin-arm64": "1.0.5"
+    "@fugood/node-llama-linux-x64": "1.0.6",
+    "@fugood/node-llama-linux-x64-vulkan": "1.0.6",
+    "@fugood/node-llama-linux-x64-cuda": "1.0.6",
+    "@fugood/node-llama-linux-arm64": "1.0.6",
+    "@fugood/node-llama-linux-arm64-vulkan": "1.0.6",
+    "@fugood/node-llama-linux-arm64-cuda": "1.0.6",
+    "@fugood/node-llama-win32-x64": "1.0.6",
+    "@fugood/node-llama-win32-x64-vulkan": "1.0.6",
+    "@fugood/node-llama-win32-x64-cuda": "1.0.6",
+    "@fugood/node-llama-win32-arm64": "1.0.6",
+    "@fugood/node-llama-win32-arm64-vulkan": "1.0.6",
+    "@fugood/node-llama-darwin-x64": "1.0.6",
+    "@fugood/node-llama-darwin-arm64": "1.0.6"
   },
   "devDependencies": {
     "@babel/preset-env": "^7.24.4",

package/src/LlamaCompletionWorker.cpp

@@ -1,5 +1,6 @@
 #include "LlamaCompletionWorker.h"
 #include "LlamaContext.h"
+#include <limits>
 
 size_t findStoppingStrings(const std::string &text,
                            const size_t last_token_size,
@@ -124,11 +125,11 @@ void LlamaCompletionWorker::Execute() {
     _sess->set_tokens(std::move(prompt_tokens));
   }
 
-  const int max_len = _params.n_predict < 0 ? 0 : _params.n_predict;
+  const int max_len = _params.n_predict < 0 ? std::numeric_limits<int>::max() : _params.n_predict;
   _sess->tokens_ptr()->reserve(_sess->tokens_ptr()->size() + max_len);
 
   auto embd = _sess->tokens_ptr();
-  for (int i = 0; i < max_len || _stop; i++) {
+  for (int i = 0; (i < max_len || _stop) && !_params.vocab_only; i++) {
     // check if we need to remove some tokens
     if (embd->size() >= _params.n_ctx) {
       if (!_params.ctx_shift) {

package/src/LlamaContext.cpp

@@ -1291,14 +1291,16 @@ tts_type LlamaContext::getTTSType(Napi::Env env, nlohmann::json speaker) {
   return OUTETTS_V0_2;
 }
 
-// initVocoder(path: string): boolean
+// initVocoder(params?: object): boolean
 Napi::Value LlamaContext::InitVocoder(const Napi::CallbackInfo &info) {
   Napi::Env env = info.Env();
   if (info.Length() < 1 || !info[0].IsObject()) {
-    Napi::TypeError::New(env, "Object is expected for vocoder path")
+    Napi::TypeError::New(env, "Object is expected for vocoder options")
         .ThrowAsJavaScriptException();
   }
-  auto vocoder_path = info[0].As<Napi::Object>().Get("path").ToString().Utf8Value();
+  auto options = info[0].As<Napi::Object>();
+  auto vocoder_path = options.Get("path").ToString().Utf8Value();
+  auto n_batch = get_option<int32_t>(options, "n_batch", _sess->params().n_batch);
   if (vocoder_path.empty()) {
     Napi::TypeError::New(env, "vocoder path is required")
         .ThrowAsJavaScriptException();
@@ -1314,6 +1316,7 @@ Napi::Value LlamaContext::InitVocoder(const Napi::CallbackInfo &info) {
   _vocoder.params.model.path = vocoder_path;
   _vocoder.params.embedding = true;
   _vocoder.params.ctx_shift = false;
+  _vocoder.params.n_batch = n_batch;
   _vocoder.params.n_ubatch = _vocoder.params.n_batch;
   common_init_result result = common_init_from_params(_vocoder.params);
   if (result.model == nullptr || result.context == nullptr) {