node-llama-cpp 2.5.1 → 2.6.1

package/llama/addon.cpp

@@ -10,21 +10,11 @@
 
 class LLAMAModel : public Napi::ObjectWrap<LLAMAModel> {
   public:
-    llama_context_params params;
+    llama_model_params model_params;
     llama_model* model;
-    float temperature;
-    int threads;
-    int32_t top_k;
-    float top_p;
 
     LLAMAModel(const Napi::CallbackInfo& info) : Napi::ObjectWrap<LLAMAModel>(info) {
-        params = llama_context_default_params();
-        params.seed = -1;
-        params.n_ctx = 4096;
-        temperature = 0.0f;
-        threads = 6;
-        top_k = 40;
-        top_p = 0.95f;
+        model_params = llama_model_default_params();
 
         // Get the model path
         std::string modelPath = info[0].As<Napi::String>().Utf8Value();
@@ -32,69 +22,25 @@ class LLAMAModel : public Napi::ObjectWrap<LLAMAModel> {
         if (info.Length() > 1 && info[1].IsObject()) {
             Napi::Object options = info[1].As<Napi::Object>();
 
-            if (options.Has("seed")) {
-                params.seed = options.Get("seed").As<Napi::Number>().Int32Value();
-            }
-
-            if (options.Has("contextSize")) {
-                params.n_ctx = options.Get("contextSize").As<Napi::Number>().Int32Value();
-            }
-
-            if (options.Has("batchSize")) {
-                params.n_batch = options.Get("batchSize").As<Napi::Number>().Int32Value();
-            }
-
             if (options.Has("gpuLayers")) {
-                params.n_gpu_layers = options.Get("gpuLayers").As<Napi::Number>().Int32Value();
-            }
-
-            if (options.Has("lowVram")) {
-                params.low_vram = options.Get("lowVram").As<Napi::Boolean>().Value();
-            }
-
-            if (options.Has("f16Kv")) {
-                params.f16_kv = options.Get("f16Kv").As<Napi::Boolean>().Value();
-            }
-
-            if (options.Has("logitsAll")) {
-                params.logits_all = options.Get("logitsAll").As<Napi::Boolean>().Value();
+                model_params.n_gpu_layers = options.Get("gpuLayers").As<Napi::Number>().Int32Value();
             }
 
             if (options.Has("vocabOnly")) {
-                params.vocab_only = options.Get("vocabOnly").As<Napi::Boolean>().Value();
+                model_params.vocab_only = options.Get("vocabOnly").As<Napi::Boolean>().Value();
             }
 
             if (options.Has("useMmap")) {
-                params.use_mmap = options.Get("useMmap").As<Napi::Boolean>().Value();
+                model_params.use_mmap = options.Get("useMmap").As<Napi::Boolean>().Value();
             }
 
             if (options.Has("useMlock")) {
-                params.use_mlock = options.Get("useMlock").As<Napi::Boolean>().Value();
-            }
-
-            if (options.Has("embedding")) {
-                params.embedding = options.Get("embedding").As<Napi::Boolean>().Value();
-            }
-
-            if (options.Has("threads")) {
-                threads = options.Get("threads").As<Napi::Number>().Int32Value();
-            }
-
-            if (options.Has("temperature")) {
-                temperature = options.Get("temperature").As<Napi::Number>().FloatValue();
-            }
-
-            if (options.Has("topK")) {
-                top_k = options.Get("topK").As<Napi::Number>().Int32Value();
-            }
-
-            if (options.Has("topP")) {
-                top_p = options.Get("topP").As<Napi::Number>().FloatValue();
+                model_params.use_mlock = options.Get("useMlock").As<Napi::Boolean>().Value();
             }
         }
 
         llama_backend_init(false);
-        model = llama_load_model_from_file(modelPath.c_str(), params);
+        model = llama_load_model_from_file(modelPath.c_str(), model_params);
 
         if (model == NULL) {
             Napi::Error::New(info.Env(), "Failed to load model").ThrowAsJavaScriptException();
@@ -114,7 +60,6 @@ class LLAMAModel : public Napi::ObjectWrap<LLAMAModel> {
 class LLAMAGrammar : public Napi::ObjectWrap<LLAMAGrammar> {
   public:
     grammar_parser::parse_state parsed_grammar;
-    llama_grammar *grammar = nullptr;
 
     LLAMAGrammar(const Napi::CallbackInfo& info) : Napi::ObjectWrap<LLAMAGrammar>(info) {
         // Get the model path
@@ -139,13 +84,31 @@ class LLAMAGrammar : public Napi::ObjectWrap<LLAMAGrammar> {
         if (should_print_grammar) {
             grammar_parser::print_grammar(stderr, parsed_grammar);
         }
+    }
 
-        std::vector<const llama_grammar_element *> grammar_rules(parsed_grammar.c_rules());
+    static void init(Napi::Object exports) {
+        exports.Set("LLAMAGrammar", DefineClass(exports.Env(), "LLAMAGrammar", {}));
+    }
+};
+
+class LLAMAGrammarEvaluationState : public Napi::ObjectWrap<LLAMAGrammarEvaluationState> {
+  public:
+    LLAMAGrammar* grammarDef;
+    llama_grammar *grammar = nullptr;
+
+    LLAMAGrammarEvaluationState(const Napi::CallbackInfo& info) : Napi::ObjectWrap<LLAMAGrammarEvaluationState>(info) {
+        grammarDef = Napi::ObjectWrap<LLAMAGrammar>::Unwrap(info[0].As<Napi::Object>());
+        grammarDef->Ref();
+
+        std::vector<const llama_grammar_element *> grammar_rules(grammarDef->parsed_grammar.c_rules());
         grammar = llama_grammar_init(
-            grammar_rules.data(), grammar_rules.size(), parsed_grammar.symbol_ids.at("root"));
+            grammar_rules.data(), grammar_rules.size(), grammarDef->parsed_grammar.symbol_ids.at("root")
+        );
     }
 
-    ~LLAMAGrammar() {
+    ~LLAMAGrammarEvaluationState() {
+      grammarDef->Unref();
+
         if (grammar != nullptr) {
             llama_grammar_free(grammar);
             grammar = nullptr;
@@ -153,42 +116,67 @@ class LLAMAGrammar : public Napi::ObjectWrap<LLAMAGrammar> {
     }
 
     static void init(Napi::Object exports) {
-        exports.Set("LLAMAGrammar", DefineClass(exports.Env(), "LLAMAGrammar", {}));
+        exports.Set("LLAMAGrammarEvaluationState", DefineClass(exports.Env(), "LLAMAGrammarEvaluationState", {}));
     }
 };
 
 class LLAMAContext : public Napi::ObjectWrap<LLAMAContext> {
   public:
   LLAMAModel* model;
+  llama_context_params context_params;
   llama_context* ctx;
-  LLAMAGrammar* grammar;
-  bool use_grammar = false;
+  int n_cur = 0;
 
   LLAMAContext(const Napi::CallbackInfo& info) : Napi::ObjectWrap<LLAMAContext>(info) {
     model = Napi::ObjectWrap<LLAMAModel>::Unwrap(info[0].As<Napi::Object>());
     model->Ref();
-    ctx = llama_new_context_with_model(model->model, model->params);
-    Napi::MemoryManagement::AdjustExternalMemory(Env(), llama_get_state_size(ctx));
+
+    context_params = llama_context_default_params();
+    context_params.seed = -1;
+    context_params.n_ctx = 4096;
+    context_params.n_threads = 6;
+    context_params.n_threads_batch == -1 ? context_params.n_threads : context_params.n_threads_batch;
 
     if (info.Length() > 1 && info[1].IsObject()) {
-        Napi::Object options = info[1].As<Napi::Object>();
+      Napi::Object options = info[1].As<Napi::Object>();
 
-        if (options.Has("grammar")) {
-            grammar = Napi::ObjectWrap<LLAMAGrammar>::Unwrap(options.Get("grammar").As<Napi::Object>());
-            grammar->Ref();
-            use_grammar = true;
-        }
+      if (options.Has("seed")) {
+        context_params.seed = options.Get("seed").As<Napi::Number>().Int32Value();
+      }
+
+      if (options.Has("contextSize")) {
+        context_params.n_ctx = options.Get("contextSize").As<Napi::Number>().Int32Value();
+      }
+
+      if (options.Has("batchSize")) {
+        context_params.n_batch = options.Get("batchSize").As<Napi::Number>().Int32Value();
+      }
+
+      if (options.Has("f16Kv")) {
+          context_params.f16_kv = options.Get("f16Kv").As<Napi::Boolean>().Value();
+      }
+
+      if (options.Has("logitsAll")) {
+          context_params.logits_all = options.Get("logitsAll").As<Napi::Boolean>().Value();
+      }
+
+      if (options.Has("embedding")) {
+        context_params.embedding = options.Get("embedding").As<Napi::Boolean>().Value();
+      }
+
+      if (options.Has("threads")) {
+        context_params.n_threads = options.Get("threads").As<Napi::Number>().Int32Value();
+        context_params.n_threads_batch == -1 ? context_params.n_threads : context_params.n_threads_batch;
+      }
     }
+
+    ctx = llama_new_context_with_model(model->model, context_params);
+    Napi::MemoryManagement::AdjustExternalMemory(Env(), llama_get_state_size(ctx));
   }
   ~LLAMAContext() {
     Napi::MemoryManagement::AdjustExternalMemory(Env(), -(int64_t)llama_get_state_size(ctx));
     llama_free(ctx);
     model->Unref();
-
-    if (use_grammar) {
-        grammar->Unref();
-        use_grammar = false;
-    }
   }
   Napi::Value Encode(const Napi::CallbackInfo& info) {
     std::string text = info[0].As<Napi::String>().Utf8Value();
@@ -265,34 +253,124 @@ class LLAMAContext : public Napi::ObjectWrap<LLAMAContext> {
 
 class LLAMAContextEvalWorker : Napi::AsyncWorker, Napi::Promise::Deferred {
   LLAMAContext* ctx;
+  LLAMAGrammarEvaluationState* grammar_evaluation_state;
+  bool use_grammar = false;
   std::vector<llama_token> tokens;
   llama_token result;
+  float temperature;
+  int32_t top_k;
+  float top_p;
+  float repeat_penalty = 1.10f; // 1.0 = disabled
+  float repeat_penalty_presence_penalty = 0.00f; // 0.0 = disabled
+  float repeat_penalty_frequency_penalty = 0.00f; // 0.0 = disabled
+  std::vector<llama_token> repeat_penalty_tokens;
+  bool use_repeat_penalty = false;
 
   public:
   LLAMAContextEvalWorker(const Napi::CallbackInfo& info, LLAMAContext* ctx) : Napi::AsyncWorker(info.Env(), "LLAMAContextEvalWorker"), ctx(ctx), Napi::Promise::Deferred(info.Env()) {
     ctx->Ref();
     Napi::Uint32Array tokens = info[0].As<Napi::Uint32Array>();
+
+    temperature = 0.0f;
+    top_k = 40;
+    top_p = 0.95f;
+
+    if (info.Length() > 1 && info[1].IsObject()) {
+      Napi::Object options = info[1].As<Napi::Object>();
+
+      if (options.Has("temperature")) {
+          temperature = options.Get("temperature").As<Napi::Number>().FloatValue();
+      }
+
+      if (options.Has("topK")) {
+          top_k = options.Get("topK").As<Napi::Number>().Int32Value();
+      }
+
+      if (options.Has("topP")) {
+          top_p = options.Get("topP").As<Napi::Number>().FloatValue();
+      }
+
+      if (options.Has("repeatPenalty")) {
+          repeat_penalty = options.Get("repeatPenalty").As<Napi::Number>().FloatValue();
+      }
+
+      if (options.Has("repeatPenaltyTokens")) {
+          Napi::Uint32Array repeat_penalty_tokens_uint32_array = options.Get("repeatPenaltyTokens").As<Napi::Uint32Array>();
+
+          repeat_penalty_tokens.reserve(repeat_penalty_tokens_uint32_array.ElementLength());
+          for (size_t i = 0; i < repeat_penalty_tokens_uint32_array.ElementLength(); i++) {
+            repeat_penalty_tokens.push_back(static_cast<llama_token>(repeat_penalty_tokens_uint32_array[i]));
+          }
+
+          use_repeat_penalty = true;
+      }
+
+      if (options.Has("repeatPenaltyPresencePenalty")) {
+          repeat_penalty_presence_penalty = options.Get("repeatPenaltyPresencePenalty").As<Napi::Number>().FloatValue();
+      }
+
+      if (options.Has("repeatPenaltyFrequencyPenalty")) {
+          repeat_penalty_frequency_penalty = options.Get("repeatPenaltyFrequencyPenalty").As<Napi::Number>().FloatValue();
+      }
+
+      if (options.Has("grammarEvaluationState")) {
+          grammar_evaluation_state = Napi::ObjectWrap<LLAMAGrammarEvaluationState>::Unwrap(options.Get("grammarEvaluationState").As<Napi::Object>());
+          grammar_evaluation_state->Ref();
+          use_grammar = true;
+      }
+    }
+
     this->tokens.reserve(tokens.ElementLength());
     for (size_t i = 0; i < tokens.ElementLength(); i++) { this->tokens.push_back(static_cast<llama_token>(tokens[i])); }
   }
-  ~LLAMAContextEvalWorker() { ctx->Unref(); }
+  ~LLAMAContextEvalWorker() {
+    ctx->Unref();
+
+    if (use_grammar) {
+        grammar_evaluation_state->Unref();
+        use_grammar = false;
+    }
+  }
   using Napi::AsyncWorker::Queue;
   using Napi::Promise::Deferred::Promise;
 
   protected:
   void Execute() {
-    // Perform the evaluation using llama_eval.
-    int r = llama_eval(ctx->ctx, tokens.data(), int(tokens.size()), llama_get_kv_cache_token_count(ctx->ctx), (ctx->model)->threads);
+    llama_batch batch = llama_batch_init(tokens.size(), 0);
+
+    batch.n_tokens = tokens.size();
+
+    for (int32_t i = 0; i < batch.n_tokens; i++) {
+        batch.token[i]  = tokens[i];
+        batch.pos[i]    = ctx->n_cur;
+        batch.seq_id[i] = 0;
+        batch.logits[i] = false;
+
+        ctx->n_cur++;
+    }
+
+    batch.logits[batch.n_tokens - 1] = true;
+
+    // Perform the evaluation using llama_decode.
+    int r = llama_decode(ctx->ctx, batch);
+
+    llama_batch_free(batch);
+
     if (r != 0) {
-      SetError("Eval has failed");
+      if (r == 1) {
+        SetError("could not find a KV slot for the batch (try reducing the size of the batch or increase the context)");
+      } else {
+        SetError("Eval has failed");
+      }
+
       return;
     }
 
     llama_token new_token_id = 0;
 
     // Select the best prediction.
-    auto logits = llama_get_logits(ctx->ctx);
-    auto n_vocab = llama_n_vocab(ctx->ctx);
+    auto logits = llama_get_logits_ith(ctx->ctx, batch.n_tokens - 1);
+    auto n_vocab = llama_n_vocab(ctx->model->model);
 
     std::vector<llama_token_data> candidates;
     candidates.reserve(n_vocab);
@@ -303,48 +381,43 @@ class LLAMAContextEvalWorker : Napi::AsyncWorker, Napi::Promise::Deferred {
 
     llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false };
 
-    float originalEosLogit = 0;
     auto eos_token = llama_token_eos(ctx->ctx);
 
-    for (auto& candidate : candidates) {
-      if (candidate.id == eos_token) {
-        originalEosLogit = candidate.logit;
-        break;
-      }
-    }
-
-    if (ctx->use_grammar) {
-        llama_sample_grammar(ctx->ctx, &candidates_p, (ctx->grammar)->grammar);
+    if (use_repeat_penalty && !repeat_penalty_tokens.empty()) {
+      llama_sample_repetition_penalty(
+        ctx->ctx, &candidates_p, repeat_penalty_tokens.data(), repeat_penalty_tokens.size(), repeat_penalty
+      );
+      llama_sample_frequency_and_presence_penalties(
+        ctx->ctx, &candidates_p, repeat_penalty_tokens.data(), repeat_penalty_tokens.size(),
+        repeat_penalty_frequency_penalty, repeat_penalty_presence_penalty
+      );
     }
 
-    for (auto& candidate : candidates) {
-      if (candidate.id == eos_token) {
-        candidate.logit = originalEosLogit;
-        break;
-      }
+    if (use_grammar && (grammar_evaluation_state)->grammar != nullptr) {
+        llama_sample_grammar(ctx->ctx, &candidates_p, (grammar_evaluation_state)->grammar);
     }
 
-    if ((ctx->model)->temperature <= 0) {
+    if (temperature <= 0) {
         new_token_id = llama_sample_token_greedy(ctx->ctx , &candidates_p);
     } else {
-        const int32_t top_k = (ctx->model)->top_k <= 0 ? llama_n_vocab(ctx->ctx) : (ctx->model)->top_k;
+        const int32_t resolved_top_k = top_k <= 0 ? llama_n_vocab(ctx->model->model) : std::min(top_k, llama_n_vocab(ctx->model->model));
         const int32_t n_probs = 0; // Number of probabilities to keep - 0 = disabled
         const float tfs_z = 1.00f; // Tail free sampling - 1.0 = disabled
         const float typical_p = 1.00f; // Typical probability - 1.0 = disabled
-        const float top_p = (ctx->model)->top_p; // Top p sampling - 1.0 = disabled
+        const float resolved_top_p = top_p; // Top p sampling - 1.0 = disabled
 
         // Temperature sampling
         size_t min_keep = std::max(1, n_probs);
-        llama_sample_top_k(ctx->ctx, &candidates_p, top_k, min_keep);
+        llama_sample_top_k(ctx->ctx, &candidates_p, resolved_top_k, min_keep);
         llama_sample_tail_free(ctx->ctx, &candidates_p, tfs_z, min_keep);
         llama_sample_typical(ctx->ctx, &candidates_p, typical_p, min_keep);
-        llama_sample_top_p(ctx->ctx, &candidates_p, top_p, min_keep);
-        llama_sample_temperature(ctx->ctx, &candidates_p, (ctx->model)->temperature);;
+        llama_sample_top_p(ctx->ctx, &candidates_p, resolved_top_p, min_keep);
+        llama_sample_temperature(ctx->ctx, &candidates_p, temperature);
         new_token_id = llama_sample_token(ctx->ctx, &candidates_p);
     }
 
-    if (new_token_id != eos_token && ctx->use_grammar) {
-        llama_grammar_accept_token(ctx->ctx, (ctx->grammar)->grammar, new_token_id);
+    if (new_token_id != eos_token && use_grammar && (grammar_evaluation_state)->grammar != nullptr) {
+        llama_grammar_accept_token(ctx->ctx, (grammar_evaluation_state)->grammar, new_token_id);
     }
 
     result = new_token_id;
@@ -372,6 +445,7 @@ Napi::Object registerCallback(Napi::Env env, Napi::Object exports) {
   });
   LLAMAModel::init(exports);
   LLAMAGrammar::init(exports);
+  LLAMAGrammarEvaluationState::init(exports);
   LLAMAContext::init(exports);
   return exports;
 }

package/llama/binariesGithubRelease.json

@@ -1,3 +1,3 @@
 {
-    "release": "b1277"
+    "release": "b1357"
 }

package/llama/toolchains/darwin.host-x64.target-arm64.cmake

@@ -0,0 +1,8 @@
+set(CMAKE_SYSTEM_NAME Darwin) # macOS
+set(CMAKE_SYSTEM_PROCESSOR arm64)
+
+set(CMAKE_C_COMPILER clang)
+set(CMAKE_CXX_COMPILER clang++)
+
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch arm64")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch arm64")

package/llama/toolchains/linux.host-arm64.target-x64.cmake

@@ -0,0 +1,5 @@
+set(CMAKE_SYSTEM_NAME Linux)
+set(CMAKE_SYSTEM_PROCESSOR x86_64)
+
+set(CMAKE_C_COMPILER x86_64-linux-gnu-gcc)
+set(CMAKE_CXX_COMPILER x86_64-linux-gnu-g++)

package/llama/toolchains/linux.host-x64.target-arm64.cmake

@@ -0,0 +1,5 @@
+set(CMAKE_SYSTEM_NAME Linux)
+set(CMAKE_SYSTEM_PROCESSOR aarch64)
+
+set(CMAKE_C_COMPILER aarch64-linux-gnu-gcc)
+set(CMAKE_CXX_COMPILER aarch64-linux-gnu-g++)

package/llama/toolchains/linux.host-x64.target-arm71.cmake

@@ -0,0 +1,5 @@
+set(CMAKE_SYSTEM_NAME Linux)
+set(CMAKE_SYSTEM_PROCESSOR arm)
+
+set(CMAKE_C_COMPILER arm-linux-gnueabihf-gcc)
+set(CMAKE_CXX_COMPILER arm-linux-gnueabihf-g++)