node-llama-cpp 2.8.9 → 2.8.11

package/llama/addon.cpp

@@ -8,6 +8,7 @@
 #include "common/grammar-parser.h"
 #include "napi.h"
 
+
 class LLAMAModel : public Napi::ObjectWrap<LLAMAModel> {
   public:
     llama_model_params model_params;
@@ -167,10 +168,10 @@ class LLAMAContext : public Napi::ObjectWrap<LLAMAContext> {
     }
 
     ctx = llama_new_context_with_model(model->model, context_params);
-    Napi::MemoryManagement::AdjustExternalMemory(Env(), llama_get_state_size(ctx));
+    Napi::MemoryManagement::AdjustExternalMemory(Env(), llama_state_get_size(ctx));
   }
   ~LLAMAContext() {
-    Napi::MemoryManagement::AdjustExternalMemory(Env(), -(int64_t)llama_get_state_size(ctx));
+    Napi::MemoryManagement::AdjustExternalMemory(Env(), -(int64_t)llama_state_get_size(ctx));
     llama_free(ctx);
     model->Unref();
   }

package/llama/binariesGithubRelease.json

@@ -1,3 +1,3 @@
 {
-    "release": "b2487"
+    "release": "b2985"
 }

package/llama/grammars/README.md

@@ -51,7 +51,7 @@ single-line ::= [^\n]+ "\n"`
 
 ## Sequences and Alternatives
 
-The order of symbols in a sequence matter. For example, in `"1. " move " " move "\n"`, the `"1. "` must come before the first `move`, etc.
+The order of symbols in a sequence matters. For example, in `"1. " move " " move "\n"`, the `"1. "` must come before the first `move`, etc.
 
 Alternatives, denoted by `|`, give different sequences that are acceptable. For example, in `move ::= pawn | nonpawn | castle`, `move` can be a `pawn` move, a `nonpawn` move, or a `castle`.
 
@@ -89,3 +89,13 @@ This guide provides a brief overview. Check out the GBNF files in this directory
 ```
 ./main -m <model> --grammar-file grammars/some-grammar.gbnf -p 'Some prompt'
 ```
+
+## Troubleshooting
+
+Grammars currently have performance gotchas (see https://github.com/ggerganov/llama.cpp/issues/4218).
+
+### Efficient optional repetitions
+
+A common pattern is to allow repetitions of a pattern `x` up to N times.
+
+While semantically correct, the syntax `x? x? x?.... x?` (with N repetitions) will result in extremely slow inference. Instead, you can write `(x (x (x ... (x)?...)?)?)?` (w/ N-deep nesting)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "node-llama-cpp",
-  "version": "2.8.9",
+  "version": "2.8.11",
   "description": "Run AI models locally on your machine with node.js bindings for llama.cpp. Force a JSON schema on the model output on the generation level",
   "main": "dist/index.js",
   "type": "module",