node-llama-cpp 2.2.0 → 2.3.1

package/README.md

@@ -1,9 +1,12 @@
 <div align="center">
+    <img alt="node-llama-cpp Logo" src="assets/logo.roundEdges.png" width="360px" />
+    <h1>Node Llama.cpp</h1>
+    <p>Node.js bindings for llama.cpp</p>
+    <sub>Pre-built bindings are provided with a fallback to building from source with node-gyp</sub>
+    <p></p>
+</div>
 
-# Node Llama.cpp
-Node.js bindings for llama.cpp.
-
-<sub>Pre-built bindings are provided with a fallback to building from source with `node-gyp`.<sub>
+<div align="center">
 
 [![Build](https://github.com/withcatai/node-llama-cpp/actions/workflows/build.yml/badge.svg)](https://github.com/withcatai/node-llama-cpp/actions/workflows/build.yml)
 [![License](https://badgen.net/badge/color/MIT/green?label=license)](https://www.npmjs.com/package/node-llama-cpp)
@@ -287,6 +290,7 @@ Optional:
   -c, --contextSize      Context size to use for the model                  [number] [default: 4096]
   -g, --grammar          Restrict the model response to a specific grammar, like JSON for example
      [string] [choices: "text", "json", "list", "arithmetic", "japanese", "chess"] [default: "text"]
+      --threads          Number of threads to use for the evaluation of tokens [number] [default: 6]
   -t, --temperature      Temperature is a hyperparameter that controls the randomness of the generat
                          ed text. It affects the probability distribution of the model's output toke
                          ns. A higher temperature (e.g., 1.5) makes the output more random and creat
@@ -311,3 +315,17 @@ Options:
   -h, --help     Show help                                                                 [boolean]
   -v, --version  Show version number                                                       [boolean]
 ```
+
+## Acknowledgements
+* llama.cpp: [ggerganov/llama.cpp](https://github.com/ggerganov/llama.cpp)
+
+
+<br />
+
+<div align="center" width="360">
+    <img alt="Star please" src="assets/star.please.roundEdges.png" width="360px" margin="auto" />
+    <br/>
+    <p align="right">
+        <i>If you like this repo, star it ✨</i>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
+    </p>
+</div>

package/dist/cli/commands/ChatCommand.d.ts

@@ -7,6 +7,7 @@ type ChatCommand = {
     wrapper: "auto" | "general" | "llamaChat" | "chatML";
     contextSize: number;
     grammar: "text" | Parameters<typeof LlamaGrammar.getFor>[0];
+    threads: number;
     temperature: number;
     topK: number;
     topP: number;

package/dist/cli/commands/ChatCommand.js

@@ -57,6 +57,12 @@ export const ChatCommand = {
             choices: ["text", "json", "list", "arithmetic", "japanese", "chess"],
             description: "Restrict the model response to a specific grammar, like JSON for example",
             group: "Optional:"
+        })
+            .option("threads", {
+            type: "number",
+            default: 6,
+            description: "Number of threads to use for the evaluation of tokens",
+            group: "Optional:"
         })
             .option("temperature", {
             alias: "t",
@@ -87,9 +93,9 @@ export const ChatCommand = {
             group: "Optional:"
         });
     },
-    async handler({ model, systemInfo, systemPrompt, wrapper, contextSize, grammar, temperature, topK, topP, maxTokens }) {
+    async handler({ model, systemInfo, systemPrompt, wrapper, contextSize, grammar, threads, temperature, topK, topP, maxTokens }) {
         try {
-            await RunChat({ model, systemInfo, systemPrompt, wrapper, contextSize, grammar, temperature, topK, topP, maxTokens });
+            await RunChat({ model, systemInfo, systemPrompt, wrapper, contextSize, grammar, threads, temperature, topK, topP, maxTokens });
         }
         catch (err) {
             console.error(err);
@@ -97,7 +103,7 @@ export const ChatCommand = {
         }
     }
 };
-async function RunChat({ model: modelArg, systemInfo, systemPrompt, wrapper, contextSize, grammar: grammarArg, temperature, topK, topP, maxTokens }) {
+async function RunChat({ model: modelArg, systemInfo, systemPrompt, wrapper, contextSize, grammar: grammarArg, threads, temperature, topK, topP, maxTokens }) {
     const { LlamaChatSession } = await import("../../llamaEvaluator/LlamaChatSession.js");
     const { LlamaModel } = await import("../../llamaEvaluator/LlamaModel.js");
     const { LlamaContext } = await import("../../llamaEvaluator/LlamaContext.js");
@@ -105,6 +111,7 @@ async function RunChat({ model: modelArg, systemInfo, systemPrompt, wrapper, con
     const model = new LlamaModel({
         modelPath: modelArg,
         contextSize,
+        threads,
         temperature,
         topK,
         topP

package/dist/cli/commands/ChatCommand.js.map

@@ -1 +1 @@
-{"version":3,"file":"ChatCommand.js","sourceRoot":"","sources":["../../../src/cli/commands/ChatCommand.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,mBAAmB,CAAC;AAC9C,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,OAAO,MAAM,wBAAwB,CAAC;AAC7C,OAAO,EAAC,uBAAuB,EAAC,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAC,sBAAsB,EAAC,MAAM,8CAA8C,CAAC;AACpF,OAAO,EAAC,wBAAwB,EAAC,MAAM,gDAAgD,CAAC;AACxF,OAAO,EAAC,mBAAmB,EAAC,MAAM,2CAA2C,CAAC;AAC9E,OAAO,EAAC,mBAAmB,EAAC,MAAM,8CAA8C,CAAC;AAiBjF,MAAM,CAAC,MAAM,WAAW,GAAuC;IAC3D,OAAO,EAAE,MAAM;IACf,QAAQ,EAAE,yBAAyB;IACnC,OAAO,CAAC,KAAK;QACT,OAAO,KAAK;aACP,MAAM,CAAC,OAAO,EAAE;YACb,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,YAAY,EAAE,IAAI;YAClB,WAAW,EAAE,sCAAsC;YACnD,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,YAAY,EAAE;YAClB,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,KAAK;YACd,WAAW,EAAE,6BAA6B;YAC1C,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,cAAc,EAAE;YACpB,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,uBAAuB;YAChC,kBAAkB,EAAE,EAAE;YACtB,WAAW,EACP,0CAA0C;gBAC1C,kBAAkB,GAAG,uBAAuB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG;YAC5E,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,SAAS,EAAE;YACf,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,SAAmC;YAC5C,OAAO,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,QAAQ,CAAoC;YACtF,WAAW,EAAE,kGAAkG;YAC/G,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,aAAa,EAAE;YACnB,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,IAAI,GAAG,CAAC;YACjB,WAAW,EAAE,mCAAmC;YAChD,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,SAAS,EAAE;YACf,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,MAAgC;YACzC,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,OAAO,CAAoC;YACvG,WAAW,EAAE,0EAA0E;YACvF,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,aAAa,EAAE;YACnB,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,CAAC;YACV,WAAW,EAAE,6jBAA6jB;YAC1kB,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,MAAM,EAAE;YACZ,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,EAAE;YACX,WAAW,EAAE,4SAA4S;YACzT,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,MAAM,EAAE;YACZ,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,IAAI;YACb,WAAW,EAAE,qRAAqR;YAClS,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,WAAW,EAAE;YACjB,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,CAAC;YACV,WAAW,EAAE,kHAAkH;YAC/H,KAAK,EAAE,WAAW;SACrB,CAAC,CAAC;IACX,CAAC;IACD,KAAK,CAAC,OAAO,CAAC,EACV,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAC9D,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EACrC;QACG,IAAI;YACA,MAAM,OAAO,CAAC,EAAC,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAC,CAAC,CAAC;SACvH;QAAC,OAAO,GAAG,EAAE;YACV,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACnB;IACL,CAAC;CACJ,CAAC;AAGF,KAAK,UAAU,OAAO,CAAC,EACnB,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAC9G;IACV,MAAM,EAAC,gBAAgB,EAAC,GAAG,MAAM,MAAM,CAAC,0CAA0C,CAAC,CAAC;IACpF,MAAM,EAAC,UAAU,EAAC,GAAG,MAAM,MAAM,CAAC,oCAAoC,CAAC,CAAC;IACxE,MAAM,EAAC,YAAY,EAAC,GAAG,MAAM,MAAM,CAAC,sCAAsC,CAAC,CAAC;IAC5E,MAAM,EAAC,YAAY,EAAC,GAAG,MAAM,MAAM,CAAC,sCAAsC,CAAC,CAAC;IAE5E,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC;QACzB,SAAS,EAAE,QAAQ;QACnB,WAAW;QACX,WAAW;QACX,IAAI;QACJ,IAAI;KACP,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC;QAC7B,KAAK;QACL,OAAO,EAAE,UAAU,KAAK,MAAM;YAC1B,CAAC,CAAC,MAAM,YAAY,CAAC,MAAM,CAAC,UAAU,CAAC;YACvC,CAAC,CAAC,SAAS;KAClB,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,8BAA8B;IAClE,MAAM,GAAG,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,wBAAwB;IAC5D,MAAM,aAAa,GAAG,cAAc,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACnD,MAAM,OAAO,GAAG,IAAI,gBAAgB,CAAC;QACjC,OAAO;QACP,oBAAoB,EAAE,UAAU;QAChC,YAAY;QACZ,aAAa;KAChB,CAAC,CAAC;IAEH,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC;IAC/C,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC;IAC/C,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,aAAa,CAAC,WAAW,EAAE,CAAC,CAAC;IAE9E,MAAM,OAAO,CAAC;QACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC;QACpC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC;QACnC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC;KAC3C,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,iDAAiD;IACjD,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;IAErD,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;QAChC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM,EAAE,OAAO,CAAC,MAAM;KACzB,CAAC,CAAC;IAEH,iDAAiD;IACjD,OAAO,IAAI,EAAE;QACT,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAEpD,IAAI,KAAK,KAAK,OAAO;YACjB,MAAM;QAEV,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAE3C,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAEpE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACjC,MAAM,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE;YACxB,SAAS,EAAE,SAAS,KAAK,CAAC,CAAC;gBACvB,CAAC,CAAC,OAAO,CAAC,cAAc,EAAE;gBAC1B,CAAC,CAAC,SAAS,IAAI,CAAC;oBACZ,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,SAAS;YACnB,OAAO,CAAC,KAAK;gBACT,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC1E,CAAC;SACJ,CAAC,CAAC;QACH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC/B,OAAO,CAAC,GAAG,EAAE,CAAC;KACjB;AACL,CAAC;AAED,SAAS,cAAc,CAAC,OAA+B,EAAE,GAAkB;IACvE,QAAQ,OAAO,EAAE;QACb,KAAK,SAAS;YACV,OAAO,IAAI,wBAAwB,EAAE,CAAC;QAC1C,KAAK,WAAW;YACZ,OAAO,IAAI,sBAAsB,EAAE,CAAC;QACxC,KAAK,QAAQ;YACT,OAAO,IAAI,mBAAmB,EAAE,CAAC;QACrC,QAAQ;KACX;IAED,IAAI,OAAO,KAAK,MAAM,EAAE;QACpB,MAAM,WAAW,GAAG,mBAAmB,CAAC,GAAG,CAAC,CAAC;QAE7C,IAAI,WAAW,IAAI,IAAI;YACnB,OAAO,IAAI,WAAW,EAAE,CAAC;QAE7B,OAAO,IAAI,wBAAwB,EAAE,CAAC;KACzC;IAED,KAAK,CAAC,OAAuB,CAAC,CAAC;IAE/B,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,OAAO,CAAC,CAAC;AACnD,CAAC"}
+{"version":3,"file":"ChatCommand.js","sourceRoot":"","sources":["../../../src/cli/commands/ChatCommand.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,mBAAmB,CAAC;AAC9C,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,OAAO,MAAM,wBAAwB,CAAC;AAC7C,OAAO,EAAC,uBAAuB,EAAC,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAC,sBAAsB,EAAC,MAAM,8CAA8C,CAAC;AACpF,OAAO,EAAC,wBAAwB,EAAC,MAAM,gDAAgD,CAAC;AACxF,OAAO,EAAC,mBAAmB,EAAC,MAAM,2CAA2C,CAAC;AAC9E,OAAO,EAAC,mBAAmB,EAAC,MAAM,8CAA8C,CAAC;AAkBjF,MAAM,CAAC,MAAM,WAAW,GAAuC;IAC3D,OAAO,EAAE,MAAM;IACf,QAAQ,EAAE,yBAAyB;IACnC,OAAO,CAAC,KAAK;QACT,OAAO,KAAK;aACP,MAAM,CAAC,OAAO,EAAE;YACb,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,YAAY,EAAE,IAAI;YAClB,WAAW,EAAE,sCAAsC;YACnD,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,YAAY,EAAE;YAClB,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,KAAK;YACd,WAAW,EAAE,6BAA6B;YAC1C,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,cAAc,EAAE;YACpB,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,uBAAuB;YAChC,kBAAkB,EAAE,EAAE;YACtB,WAAW,EACP,0CAA0C;gBAC1C,kBAAkB,GAAG,uBAAuB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG;YAC5E,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,SAAS,EAAE;YACf,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,SAAmC;YAC5C,OAAO,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,QAAQ,CAAoC;YACtF,WAAW,EAAE,kGAAkG;YAC/G,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,aAAa,EAAE;YACnB,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,IAAI,GAAG,CAAC;YACjB,WAAW,EAAE,mCAAmC;YAChD,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,SAAS,EAAE;YACf,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,MAAgC;YACzC,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,OAAO,CAAoC;YACvG,WAAW,EAAE,0EAA0E;YACvF,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,SAAS,EAAE;YACf,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,CAAC;YACV,WAAW,EAAE,uDAAuD;YACpE,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,aAAa,EAAE;YACnB,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,CAAC;YACV,WAAW,EAAE,6jBAA6jB;YAC1kB,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,MAAM,EAAE;YACZ,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,EAAE;YACX,WAAW,EAAE,4SAA4S;YACzT,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,MAAM,EAAE;YACZ,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,IAAI;YACb,WAAW,EAAE,qRAAqR;YAClS,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,WAAW,EAAE;YACjB,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,CAAC;YACV,WAAW,EAAE,kHAAkH;YAC/H,KAAK,EAAE,WAAW;SACrB,CAAC,CAAC;IACX,CAAC;IACD,KAAK,CAAC,OAAO,CAAC,EACV,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAC9D,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAC9C;QACG,IAAI;YACA,MAAM,OAAO,CAAC,EAAC,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAC,CAAC,CAAC;SAChI;QAAC,OAAO,GAAG,EAAE;YACV,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACnB;IACL,CAAC;CACJ,CAAC;AAGF,KAAK,UAAU,OAAO,CAAC,EACnB,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EACvH;IACV,MAAM,EAAC,gBAAgB,EAAC,GAAG,MAAM,MAAM,CAAC,0CAA0C,CAAC,CAAC;IACpF,MAAM,EAAC,UAAU,EAAC,GAAG,MAAM,MAAM,CAAC,oCAAoC,CAAC,CAAC;IACxE,MAAM,EAAC,YAAY,EAAC,GAAG,MAAM,MAAM,CAAC,sCAAsC,CAAC,CAAC;IAC5E,MAAM,EAAC,YAAY,EAAC,GAAG,MAAM,MAAM,CAAC,sCAAsC,CAAC,CAAC;IAE5E,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC;QACzB,SAAS,EAAE,QAAQ;QACnB,WAAW;QACX,OAAO;QACP,WAAW;QACX,IAAI;QACJ,IAAI;KACP,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC;QAC7B,KAAK;QACL,OAAO,EAAE,UAAU,KAAK,MAAM;YAC1B,CAAC,CAAC,MAAM,YAAY,CAAC,MAAM,CAAC,UAAU,CAAC;YACvC,CAAC,CAAC,SAAS;KAClB,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,8BAA8B;IAClE,MAAM,GAAG,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,wBAAwB;IAC5D,MAAM,aAAa,GAAG,cAAc,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACnD,MAAM,OAAO,GAAG,IAAI,gBAAgB,CAAC;QACjC,OAAO;QACP,oBAAoB,EAAE,UAAU;QAChC,YAAY;QACZ,aAAa;KAChB,CAAC,CAAC;IAEH,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC;IAC/C,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC;IAC/C,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,aAAa,CAAC,WAAW,EAAE,CAAC,CAAC;IAE9E,MAAM,OAAO,CAAC;QACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC;QACpC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC;QACnC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC;KAC3C,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,iDAAiD;IACjD,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;IAErD,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;QAChC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM,EAAE,OAAO,CAAC,MAAM;KACzB,CAAC,CAAC;IAEH,iDAAiD;IACjD,OAAO,IAAI,EAAE;QACT,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAEpD,IAAI,KAAK,KAAK,OAAO;YACjB,MAAM;QAEV,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAE3C,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAEpE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACjC,MAAM,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE;YACxB,SAAS,EAAE,SAAS,KAAK,CAAC,CAAC;gBACvB,CAAC,CAAC,OAAO,CAAC,cAAc,EAAE;gBAC1B,CAAC,CAAC,SAAS,IAAI,CAAC;oBACZ,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,SAAS;YACnB,OAAO,CAAC,KAAK;gBACT,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC1E,CAAC;SACJ,CAAC,CAAC;QACH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC/B,OAAO,CAAC,GAAG,EAAE,CAAC;KACjB;AACL,CAAC;AAED,SAAS,cAAc,CAAC,OAA+B,EAAE,GAAkB;IACvE,QAAQ,OAAO,EAAE;QACb,KAAK,SAAS;YACV,OAAO,IAAI,wBAAwB,EAAE,CAAC;QAC1C,KAAK,WAAW;YACZ,OAAO,IAAI,sBAAsB,EAAE,CAAC;QACxC,KAAK,QAAQ;YACT,OAAO,IAAI,mBAAmB,EAAE,CAAC;QACrC,QAAQ;KACX;IAED,IAAI,OAAO,KAAK,MAAM,EAAE;QACpB,MAAM,WAAW,GAAG,mBAAmB,CAAC,GAAG,CAAC,CAAC;QAE7C,IAAI,WAAW,IAAI,IAAI;YACnB,OAAO,IAAI,WAAW,EAAE,CAAC;QAE7B,OAAO,IAAI,wBAAwB,EAAE,CAAC;KACzC;IAED,KAAK,CAAC,OAAuB,CAAC,CAAC;IAE/B,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,OAAO,CAAC,CAAC;AACnD,CAAC"}

package/dist/llamaEvaluator/LlamaChatSession.js

@@ -75,13 +75,25 @@ export class LlamaChatSession {
             if (signal?.aborted)
                 throw new AbortError();
             const tokenStr = this._ctx.decode(Uint32Array.from([chunk]));
-            const { shouldReturn, skipTokenEvent, stopString, stopStringSuffix } = this._checkStopString(tokenStr, stopStringIndexes);
-            if (shouldReturn)
+            const { shouldReturn, skipTokenEvent, stopString, stopStringSuffix } = this._checkStopString(tokenStr, stopStrings, stopStringIndexes);
+            if (shouldReturn) {
+                skippedChunksQueue.push(chunk);
+                const skippedChunksText = skippedChunksQueue.length > 0
+                    ? this._ctx.decode(Uint32Array.from(skippedChunksQueue))
+                    : "";
+                const [queuedTextBeforeStopString] = skippedChunksText.split(stopString);
+                if (queuedTextBeforeStopString.length > 0) {
+                    const beforeStopStringTokens = Array.from(this._ctx.encode(queuedTextBeforeStopString));
+                    res.push(...beforeStopStringTokens);
+                    onToken?.(beforeStopStringTokens);
+                    skippedChunksQueue.length = 0;
+                }
                 return {
                     text: this._ctx.decode(Uint32Array.from(res)),
                     stopString,
                     stopStringSuffix
                 };
+            }
             // if the token is unknown, it means it's not complete character
             if (tokenStr === UNKNOWN_UNICODE_CHAR || skipTokenEvent) {
                 skippedChunksQueue.push(chunk);
@@ -103,13 +115,13 @@ export class LlamaChatSession {
             stopStringSuffix: null
         };
     }
-    _checkStopString(tokenStr, stopStringIndexes) {
-        const stopStrings = this._promptWrapper.getStopStrings();
+    _checkStopString(tokenStr, stopStrings, stopStringIndexes) {
         let skipTokenEvent = false;
         for (let stopStringIndex = 0; stopStringIndex < stopStrings.length; stopStringIndex++) {
             const stopString = stopStrings[stopStringIndex];
             let localShouldSkipTokenEvent = false;
-            for (let i = 0; i < tokenStr.length && stopStringIndexes[stopStringIndex] !== stopString.length; i++) {
+            let i = 0;
+            for (; i < tokenStr.length && stopStringIndexes[stopStringIndex] !== stopString.length; i++) {
                 if (tokenStr[i] === stopString[stopStringIndexes[stopStringIndex]]) {
                     stopStringIndexes[stopStringIndex]++;
                     localShouldSkipTokenEvent = true;
@@ -117,16 +129,15 @@ export class LlamaChatSession {
                 else {
                     stopStringIndexes[stopStringIndex] = 0;
                     localShouldSkipTokenEvent = false;
-                    break;
                 }
             }
             if (stopStringIndexes[stopStringIndex] === stopString.length) {
                 return {
                     shouldReturn: true,
                     stopString,
-                    stopStringSuffix: tokenStr.length === stopString.length
+                    stopStringSuffix: tokenStr.length === i
                         ? null
-                        : tokenStr.slice(stopString.length)
+                        : tokenStr.slice(i)
                 };
             }
             skipTokenEvent ||= localShouldSkipTokenEvent;

package/dist/llamaEvaluator/LlamaChatSession.js.map

@@ -1 +1 @@
-{"version":3,"file":"LlamaChatSession.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaChatSession.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,uBAAuB,EAAC,MAAM,cAAc,CAAC;AACrD,OAAO,EAAC,QAAQ,EAAC,MAAM,sBAAsB,CAAC;AAE9C,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAC,wBAAwB,EAAC,MAAM,6CAA6C,CAAC;AACrF,OAAO,EAAC,mBAAmB,EAAC,MAAM,2CAA2C,CAAC;AAE9E,OAAO,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAG3C,MAAM,oBAAoB,GAAG,QAAQ,CAAC;AAUtC,MAAM,OAAO,gBAAgB;IACR,aAAa,CAAS;IACtB,qBAAqB,CAAU;IAC/B,cAAc,CAAoB;IAC3C,YAAY,GAAW,CAAC,CAAC;IACzB,YAAY,GAAY,KAAK,CAAC;IAC9B,eAAe,GAAkB,IAAI,CAAC;IACtC,qBAAqB,GAAkB,IAAI,CAAC;IACnC,IAAI,CAAe;IAEpC,YAAmB,EACf,OAAO,EACP,oBAAoB,GAAG,KAAK,EAC5B,aAAa,GAAG,IAAI,wBAAwB,EAAE,EAC9C,YAAY,GAAG,uBAAuB,EAChB;QACtB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC;QACpB,IAAI,CAAC,qBAAqB,GAAG,oBAAoB,CAAC;QAClD,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;QAElC,IAAI,aAAa,KAAK,MAAM,EAAE;YAC1B,MAAM,WAAW,GAAG,mBAAmB,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;YAEhE,IAAI,WAAW,IAAI,IAAI;gBACnB,IAAI,CAAC,cAAc,GAAG,IAAI,WAAW,EAAE,CAAC;;gBAExC,IAAI,CAAC,cAAc,GAAG,IAAI,wBAAwB,EAAE,CAAC;SAC5D;;YACG,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;IAC5C,CAAC;IAED,IAAW,WAAW;QAClB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED,IAAW,OAAO;QACd,OAAO,IAAI,CAAC,IAAI,CAAC;IACrB,CAAC;IAEM,KAAK,CAAC,IAAI;QACb,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE;YACpC,IAAI,IAAI,CAAC,YAAY;gBACjB,OAAO;YAEX,IAAI,IAAI,CAAC,qBAAqB;gBAC1B,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;YAE5D,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC7B,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,KAAK,CAAC,MAAM,CAAC,MAAc,EAAE,EAChC,OAAO,EAAE,MAAM,EAAE,SAAS,KACqD,EAAE;QACjF,IAAI,CAAC,IAAI,CAAC,WAAW;YACjB,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAEtB,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,IAAI,EAAE;YAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,MAAM,EAAE;gBACtD,YAAY,EAAE,IAAI,CAAC,aAAa;gBAChC,WAAW,EAAE,IAAI,CAAC,YAAY;gBAC9B,cAAc,EAAE,IAAI,CAAC,eAAe;gBACpC,oBAAoB,EAAE,IAAI,CAAC,YAAY,IAAI,CAAC;oBACxC,CAAC,CAAC,CACE,IAAI,CAAC,IAAI,CAAC,UAAU;wBAChB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;wBAC1B,CAAC,CAAC,IAAI,CACb;oBACD,CAAC,CAAC,IAAI,CAAC,qBAAqB;aACnC,CAAC,CAAC;YACH,IAAI,CAAC,YAAY,EAAE,CAAC;YACpB,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;YAC5B,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC;YAElC,MAAM,EAAC,IAAI,EAAE,UAAU,EAAE,gBAAgB,EAAC,GACtC,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,EAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAC,CAAC,CAAC;YACvF,IAAI,CAAC,eAAe,GAAG,UAAU,CAAC;YAClC,IAAI,CAAC,qBAAqB,GAAG,gBAAgB,CAAC;YAE9C,OAAO,IAAI,CAAC;QAChB,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,MAAmB,EAAE,EAC3C,OAAO,EAAE,MAAM,EAAE,SAAS,KACqD,EAAE;QACjF,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,CAAC;QACzD,MAAM,iBAAiB,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5D,MAAM,kBAAkB,GAAY,EAAE,CAAC;QACvC,MAAM,GAAG,GAAY,EAAE,CAAC;QAExB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;YAClD,IAAI,MAAM,EAAE,OAAO;gBACf,MAAM,IAAI,UAAU,EAAE,CAAC;YAE3B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC7D,MAAM,EAAC,YAAY,EAAE,cAAc,EAAE,UAAU,EAAE,gBAAgB,EAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC;YAExH,IAAI,YAAY;gBACZ,OAAO;oBACH,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAC7C,UAAU;oBACV,gBAAgB;iBACnB,CAAC;YAEN,gEAAgE;YAChE,IAAI,QAAQ,KAAK,oBAAoB,IAAI,cAAc,EAAE;gBACrD,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC/B,SAAS;aACZ;YAED,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE;gBAC/B,GAAG,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,CAAC;gBAChC,OAAO,EAAE,CAAC,kBAAkB,CAAC,CAAC;gBAC9B,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC;aACjC;YAED,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChB,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;YAEnB,IAAI,SAAS,IAAI,IAAI,IAAI,SAAS,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,IAAI,SAAS;gBAC7D,MAAM;SACb;QAED,OAAO;YACH,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC7C,UAAU,EAAE,IAAI;YAChB,gBAAgB,EAAE,IAAI;SACzB,CAAC;IACN,CAAC;IAEO,gBAAgB,CAAC,QAAgB,EAAE,iBAA2B;QAClE,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,CAAC;QACzD,IAAI,cAAc,GAAG,KAAK,CAAC;QAE3B,KAAK,IAAI,eAAe,GAAG,CAAC,EAAE,eAAe,GAAG,WAAW,CAAC,MAAM,EAAE,eAAe,EAAE,EAAE;YACnF,MAAM,UAAU,GAAG,WAAW,CAAC,eAAe,CAAC,CAAC;YAEhD,IAAI,yBAAyB,GAAG,KAAK,CAAC;YACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,IAAI,iBAAiB,CAAC,eAAe,CAAC,KAAK,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBAClG,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC,EAAE;oBAChE,iBAAiB,CAAC,eAAe,CAAC,EAAE,CAAC;oBACrC,yBAAyB,GAAG,IAAI,CAAC;iBACpC;qBAAM;oBACH,iBAAiB,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;oBACvC,yBAAyB,GAAG,KAAK,CAAC;oBAClC,MAAM;iBACT;aACJ;YAED,IAAI,iBAAiB,CAAC,eAAe,CAAC,KAAK,UAAU,CAAC,MAAM,EAAE;gBAC1D,OAAO;oBACH,YAAY,EAAE,IAAI;oBAClB,UAAU;oBACV,gBAAgB,EAAE,QAAQ,CAAC,MAAM,KAAK,UAAU,CAAC,MAAM;wBACnD,CAAC,CAAC,IAAI;wBACN,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC;iBAC1C,CAAC;aACL;YAED,cAAc,KAAK,yBAAyB,CAAC;SAChD;QAED,OAAO,EAAC,cAAc,EAAC,CAAC;IAC5B,CAAC;CACJ"}
+{"version":3,"file":"LlamaChatSession.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaChatSession.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,uBAAuB,EAAC,MAAM,cAAc,CAAC;AACrD,OAAO,EAAC,QAAQ,EAAC,MAAM,sBAAsB,CAAC;AAE9C,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAC,wBAAwB,EAAC,MAAM,6CAA6C,CAAC;AACrF,OAAO,EAAC,mBAAmB,EAAC,MAAM,2CAA2C,CAAC;AAE9E,OAAO,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAG3C,MAAM,oBAAoB,GAAG,QAAQ,CAAC;AAUtC,MAAM,OAAO,gBAAgB;IACR,aAAa,CAAS;IACtB,qBAAqB,CAAU;IAC/B,cAAc,CAAoB;IAC3C,YAAY,GAAW,CAAC,CAAC;IACzB,YAAY,GAAY,KAAK,CAAC;IAC9B,eAAe,GAAkB,IAAI,CAAC;IACtC,qBAAqB,GAAkB,IAAI,CAAC;IACnC,IAAI,CAAe;IAEpC,YAAmB,EACf,OAAO,EACP,oBAAoB,GAAG,KAAK,EAC5B,aAAa,GAAG,IAAI,wBAAwB,EAAE,EAC9C,YAAY,GAAG,uBAAuB,EAChB;QACtB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC;QACpB,IAAI,CAAC,qBAAqB,GAAG,oBAAoB,CAAC;QAClD,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;QAElC,IAAI,aAAa,KAAK,MAAM,EAAE;YAC1B,MAAM,WAAW,GAAG,mBAAmB,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;YAEhE,IAAI,WAAW,IAAI,IAAI;gBACnB,IAAI,CAAC,cAAc,GAAG,IAAI,WAAW,EAAE,CAAC;;gBAExC,IAAI,CAAC,cAAc,GAAG,IAAI,wBAAwB,EAAE,CAAC;SAC5D;;YACG,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;IAC5C,CAAC;IAED,IAAW,WAAW;QAClB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED,IAAW,OAAO;QACd,OAAO,IAAI,CAAC,IAAI,CAAC;IACrB,CAAC;IAEM,KAAK,CAAC,IAAI;QACb,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE;YACpC,IAAI,IAAI,CAAC,YAAY;gBACjB,OAAO;YAEX,IAAI,IAAI,CAAC,qBAAqB;gBAC1B,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;YAE5D,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC7B,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,KAAK,CAAC,MAAM,CAAC,MAAc,EAAE,EAChC,OAAO,EAAE,MAAM,EAAE,SAAS,KACqD,EAAE;QACjF,IAAI,CAAC,IAAI,CAAC,WAAW;YACjB,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAEtB,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,IAAI,EAAE;YAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,MAAM,EAAE;gBACtD,YAAY,EAAE,IAAI,CAAC,aAAa;gBAChC,WAAW,EAAE,IAAI,CAAC,YAAY;gBAC9B,cAAc,EAAE,IAAI,CAAC,eAAe;gBACpC,oBAAoB,EAAE,IAAI,CAAC,YAAY,IAAI,CAAC;oBACxC,CAAC,CAAC,CACE,IAAI,CAAC,IAAI,CAAC,UAAU;wBAChB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;wBAC1B,CAAC,CAAC,IAAI,CACb;oBACD,CAAC,CAAC,IAAI,CAAC,qBAAqB;aACnC,CAAC,CAAC;YACH,IAAI,CAAC,YAAY,EAAE,CAAC;YACpB,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;YAC5B,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC;YAElC,MAAM,EAAC,IAAI,EAAE,UAAU,EAAE,gBAAgB,EAAC,GACtC,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,EAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAC,CAAC,CAAC;YACvF,IAAI,CAAC,eAAe,GAAG,UAAU,CAAC;YAClC,IAAI,CAAC,qBAAqB,GAAG,gBAAgB,CAAC;YAE9C,OAAO,IAAI,CAAC;QAChB,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,MAAmB,EAAE,EAC3C,OAAO,EAAE,MAAM,EAAE,SAAS,KACqD,EAAE;QACjF,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,CAAC;QACzD,MAAM,iBAAiB,GAAa,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACtE,MAAM,kBAAkB,GAAY,EAAE,CAAC;QACvC,MAAM,GAAG,GAAY,EAAE,CAAC;QAExB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;YAClD,IAAI,MAAM,EAAE,OAAO;gBACf,MAAM,IAAI,UAAU,EAAE,CAAC;YAE3B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC7D,MAAM,EACF,YAAY,EAAE,cAAc,EAAE,UAAU,EAAE,gBAAgB,EAC7D,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,WAAW,EAAE,iBAAiB,CAAC,CAAC;YAEpE,IAAI,YAAY,EAAE;gBACd,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC/B,MAAM,iBAAiB,GAAG,kBAAkB,CAAC,MAAM,GAAG,CAAC;oBACnD,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;oBACxD,CAAC,CAAC,EAAE,CAAC;gBAET,MAAM,CAAC,0BAA0B,CAAC,GAAG,iBAAiB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBAEzE,IAAI,0BAA0B,CAAC,MAAM,GAAG,CAAC,EAAE;oBACvC,MAAM,sBAAsB,GAAY,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,0BAA0B,CAAC,CAAC,CAAC;oBAEjG,GAAG,CAAC,IAAI,CAAC,GAAG,sBAAsB,CAAC,CAAC;oBACpC,OAAO,EAAE,CAAC,sBAAsB,CAAC,CAAC;oBAClC,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC;iBACjC;gBAED,OAAO;oBACH,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAC7C,UAAU;oBACV,gBAAgB;iBACnB,CAAC;aACL;YAED,gEAAgE;YAChE,IAAI,QAAQ,KAAK,oBAAoB,IAAI,cAAc,EAAE;gBACrD,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC/B,SAAS;aACZ;YAED,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE;gBAC/B,GAAG,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,CAAC;gBAChC,OAAO,EAAE,CAAC,kBAAkB,CAAC,CAAC;gBAC9B,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC;aACjC;YAED,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChB,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;YAEnB,IAAI,SAAS,IAAI,IAAI,IAAI,SAAS,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,IAAI,SAAS;gBAC7D,MAAM;SACb;QAED,OAAO;YACH,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC7C,UAAU,EAAE,IAAI;YAChB,gBAAgB,EAAE,IAAI;SACzB,CAAC;IACN,CAAC;IAEO,gBAAgB,CAAC,QAAgB,EAAE,WAAqB,EAAE,iBAA2B;QACzF,IAAI,cAAc,GAAG,KAAK,CAAC;QAE3B,KAAK,IAAI,eAAe,GAAG,CAAC,EAAE,eAAe,GAAG,WAAW,CAAC,MAAM,EAAE,eAAe,EAAE,EAAE;YACnF,MAAM,UAAU,GAAG,WAAW,CAAC,eAAe,CAAC,CAAC;YAEhD,IAAI,yBAAyB,GAAG,KAAK,CAAC;YACtC,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,OAAO,CAAC,GAAG,QAAQ,CAAC,MAAM,IAAI,iBAAiB,CAAC,eAAe,CAAC,KAAK,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACzF,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC,EAAE;oBAChE,iBAAiB,CAAC,eAAe,CAAC,EAAE,CAAC;oBACrC,yBAAyB,GAAG,IAAI,CAAC;iBACpC;qBAAM;oBACH,iBAAiB,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;oBACvC,yBAAyB,GAAG,KAAK,CAAC;iBACrC;aACJ;YAED,IAAI,iBAAiB,CAAC,eAAe,CAAC,KAAK,UAAU,CAAC,MAAM,EAAE;gBAC1D,OAAO;oBACH,YAAY,EAAE,IAAI;oBAClB,UAAU;oBACV,gBAAgB,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC;wBACnC,CAAC,CAAC,IAAI;wBACN,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;iBAC1B,CAAC;aACL;YAED,cAAc,KAAK,yBAAyB,CAAC;SAChD;QAED,OAAO,EAAC,cAAc,EAAC,CAAC;IAC5B,CAAC;CACJ"}

package/dist/llamaEvaluator/LlamaContext.d.ts

@@ -11,7 +11,7 @@ export declare class LlamaContext {
     private _prependBos;
     constructor({ model, grammar, prependBos }: LlamaContextOptions);
     encode(text: string): Uint32Array;
-    decode(tokens: Uint32Array): string;
+    decode(tokens: Uint32Array | Token[]): string;
     get prependBos(): boolean;
     /**
      * @returns {Token | null} The BOS (Beginning Of Sequence) token.

package/dist/llamaEvaluator/LlamaContext.js

@@ -17,7 +17,9 @@ export class LlamaContext {
     decode(tokens) {
         if (tokens.length === 0)
             return "";
-        return this._ctx.decode(tokens);
+        if (tokens instanceof Uint32Array)
+            return this._ctx.decode(tokens);
+        return this._ctx.decode(Uint32Array.from(tokens));
     }
     get prependBos() {
         return this._prependBos;

package/dist/llamaEvaluator/LlamaContext.js.map

@@ -1 +1 @@
-{"version":3,"file":"LlamaContext.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaContext.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,gBAAgB,EAAC,MAAM,8BAA8B,CAAC;AAE9D,OAAO,EAAC,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAW5C,MAAM,OAAO,YAAY;IACJ,IAAI,CAAe;IAC5B,WAAW,CAAU;IAE7B,YAAmB,EAAC,KAAK,EAAE,OAAO,EAAE,UAAU,GAAG,IAAI,EAAsB;QACvE,IAAI,CAAC,IAAI,GAAG,IAAI,YAAY,CAAC,KAAK,CAAC,MAAM,EAAE,gBAAgB,CAAC;YACxD,OAAO,EAAE,OAAO,EAAE,QAAQ;SAC7B,CAAC,CAAC,CAAC;QACJ,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,IAAY;QACtB,IAAI,IAAI,KAAK,EAAE;YACX,OAAO,IAAI,WAAW,EAAE,CAAC;QAE7B,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,MAAmB;QAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YACnB,OAAO,EAAE,CAAC;QAEd,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAED,IAAW,UAAU;QACjB,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED;;OAEG;IACI,WAAW;QACd,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;QAEtC,IAAI,QAAQ,KAAK,CAAC,CAAC;YACf,OAAO,IAAI,CAAC;QAEhB,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED;;OAEG;IACI,WAAW;QACd,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;QAEtC,IAAI,QAAQ,KAAK,CAAC,CAAC;YACf,OAAO,IAAI,CAAC;QAEhB,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED;;OAEG;IACI,UAAU;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QAEpC,IAAI,OAAO,KAAK,CAAC,CAAC;YACd,OAAO,IAAI,CAAC;QAEhB,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;OAEG;IACI,YAAY;QACf,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAEpC,IAAI,QAAQ,IAAI,IAAI;YAChB,OAAO,IAAI,CAAC;QAEhB,OAAO,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IAC9C,CAAC;IAED;;OAEG;IACI,YAAY;QACf,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAEpC,IAAI,QAAQ,IAAI,IAAI;YAChB,OAAO,IAAI,CAAC;QAEhB,OAAO,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IAC9C,CAAC;IAED;;OAEG;IACI,WAAW;QACd,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QAElC,IAAI,OAAO,IAAI,IAAI;YACf,OAAO,IAAI,CAAC;QAEhB,OAAO,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;IAC7C,CAAC;IAEM,cAAc;QACjB,OAAO,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;IACtC,CAAC;IAEM,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAmB;QACtC,IAAI,UAAU,GAAG,MAAM,CAAC;QAExB,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,MAAM,UAAU,GAAY,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC/C,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEzC,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC1C,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;SAC5B;QAED,iDAAiD;QACjD,OAAO,IAAI,EAAE;YACT,kCAAkC;YAClC,MAAM,SAAS,GAAU,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAE1D,mCAAmC;YACnC,IAAI,SAAS,KAAK,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE;gBAClC,MAAM;YAEV,MAAM,SAAS,CAAC;YAEhB,mCAAmC;YACnC,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;SAC9C;IACL,CAAC;CAEJ"}
+{"version":3,"file":"LlamaContext.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaContext.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,gBAAgB,EAAC,MAAM,8BAA8B,CAAC;AAE9D,OAAO,EAAC,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAW5C,MAAM,OAAO,YAAY;IACJ,IAAI,CAAe;IAC5B,WAAW,CAAU;IAE7B,YAAmB,EAAC,KAAK,EAAE,OAAO,EAAE,UAAU,GAAG,IAAI,EAAsB;QACvE,IAAI,CAAC,IAAI,GAAG,IAAI,YAAY,CAAC,KAAK,CAAC,MAAM,EAAE,gBAAgB,CAAC;YACxD,OAAO,EAAE,OAAO,EAAE,QAAQ;SAC7B,CAAC,CAAC,CAAC;QACJ,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,IAAY;QACtB,IAAI,IAAI,KAAK,EAAE;YACX,OAAO,IAAI,WAAW,EAAE,CAAC;QAE7B,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,MAA6B;QACvC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YACnB,OAAO,EAAE,CAAC;QAEd,IAAI,MAAM,YAAY,WAAW;YAC7B,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAEpC,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;IACtD,CAAC;IAED,IAAW,UAAU;QACjB,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED;;OAEG;IACI,WAAW;QACd,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;QAEtC,IAAI,QAAQ,KAAK,CAAC,CAAC;YACf,OAAO,IAAI,CAAC;QAEhB,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED;;OAEG;IACI,WAAW;QACd,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;QAEtC,IAAI,QAAQ,KAAK,CAAC,CAAC;YACf,OAAO,IAAI,CAAC;QAEhB,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED;;OAEG;IACI,UAAU;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QAEpC,IAAI,OAAO,KAAK,CAAC,CAAC;YACd,OAAO,IAAI,CAAC;QAEhB,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;OAEG;IACI,YAAY;QACf,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAEpC,IAAI,QAAQ,IAAI,IAAI;YAChB,OAAO,IAAI,CAAC;QAEhB,OAAO,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IAC9C,CAAC;IAED;;OAEG;IACI,YAAY;QACf,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAEpC,IAAI,QAAQ,IAAI,IAAI;YAChB,OAAO,IAAI,CAAC;QAEhB,OAAO,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IAC9C,CAAC;IAED;;OAEG;IACI,WAAW;QACd,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QAElC,IAAI,OAAO,IAAI,IAAI;YACf,OAAO,IAAI,CAAC;QAEhB,OAAO,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;IAC7C,CAAC;IAEM,cAAc;QACjB,OAAO,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;IACtC,CAAC;IAEM,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAmB;QACtC,IAAI,UAAU,GAAG,MAAM,CAAC;QAExB,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,MAAM,UAAU,GAAY,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC/C,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEzC,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC1C,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;SAC5B;QAED,iDAAiD;QACjD,OAAO,IAAI,EAAE;YACT,kCAAkC;YAClC,MAAM,SAAS,GAAU,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAE1D,mCAAmC;YACnC,IAAI,SAAS,KAAK,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE;gBAClC,MAAM;YAEV,MAAM,SAAS,CAAC;YAEhB,mCAAmC;YACnC,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;SAC9C;IACL,CAAC;CAEJ"}

package/dist/llamaEvaluator/LlamaModel.d.ts

@@ -11,6 +11,8 @@ export type LlamaModelOptions = {
     gpuLayers?: number;
     /** if true, reduce VRAM usage at the cost of performance */
     lowVram?: boolean;
+    /** number of threads to use to evaluate tokens */
+    threads?: number;
     /**
      * Temperature is a hyperparameter that controls the randomness of the generated text.
      * It affects the probability distribution of the model's output tokens.
@@ -63,6 +65,7 @@ export declare class LlamaModel {
      * @param {number} [options.batchSize] - prompt processing batch size
      * @param {number} [options.gpuLayers] - number of layers to store in VRAM
      * @param {boolean} [options.lowVram] - if true, reduce VRAM usage at the cost of performance
+     * @param {number} [options.threads] - number of threads to use to evaluate tokens
      * @param {number} [options.temperature] - Temperature is a hyperparameter that controls the randomness of the generated text.
      * It affects the probability distribution of the model's output tokens.
      * A higher temperature (e.g., 1.5) makes the output more random and creative,
@@ -90,6 +93,6 @@ export declare class LlamaModel {
      * @param {boolean} [options.useMlock] - force system to keep model in RAM
      * @param {boolean} [options.embedding] - embedding mode only
      */
-    constructor({ modelPath, seed, contextSize, batchSize, gpuLayers, lowVram, temperature, topK, topP, f16Kv, logitsAll, vocabOnly, useMmap, useMlock, embedding }: LlamaModelOptions);
+    constructor({ modelPath, seed, contextSize, batchSize, gpuLayers, lowVram, threads, temperature, topK, topP, f16Kv, logitsAll, vocabOnly, useMmap, useMlock, embedding }: LlamaModelOptions);
     static get systemInfo(): string;
 }

package/dist/llamaEvaluator/LlamaModel.js

@@ -13,6 +13,7 @@ export class LlamaModel {
      * @param {number} [options.batchSize] - prompt processing batch size
      * @param {number} [options.gpuLayers] - number of layers to store in VRAM
      * @param {boolean} [options.lowVram] - if true, reduce VRAM usage at the cost of performance
+     * @param {number} [options.threads] - number of threads to use to evaluate tokens
      * @param {number} [options.temperature] - Temperature is a hyperparameter that controls the randomness of the generated text.
      * It affects the probability distribution of the model's output tokens.
      * A higher temperature (e.g., 1.5) makes the output more random and creative,
@@ -40,13 +41,14 @@ export class LlamaModel {
      * @param {boolean} [options.useMlock] - force system to keep model in RAM
      * @param {boolean} [options.embedding] - embedding mode only
      */
-    constructor({ modelPath, seed = null, contextSize = 1024 * 4, batchSize, gpuLayers, lowVram, temperature = 0, topK = 40, topP = 0.95, f16Kv, logitsAll, vocabOnly, useMmap, useMlock, embedding }) {
+    constructor({ modelPath, seed = null, contextSize = 1024 * 4, batchSize, gpuLayers, lowVram, threads = 6, temperature = 0, topK = 40, topP = 0.95, f16Kv, logitsAll, vocabOnly, useMmap, useMlock, embedding }) {
         this._model = new LLAMAModel(modelPath, removeNullFields({
             seed: seed != null ? Math.max(-1, seed) : undefined,
             contextSize,
             batchSize,
             gpuLayers,
             lowVram,
+            threads,
             temperature,
             topK,
             topP,

package/dist/llamaEvaluator/LlamaModel.js.map

@@ -1 +1 @@
-{"version":3,"file":"LlamaModel.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaModel.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,gBAAgB,EAAC,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAC,YAAY,EAAE,UAAU,EAAC,MAAM,gBAAgB,CAAC;AAwExD,MAAM,OAAO,UAAU;IACnB,gBAAgB;IACA,MAAM,CAAa;IAEnC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoCG;IACH,YAAmB,EACf,SAAS,EAAE,IAAI,GAAG,IAAI,EAAE,WAAW,GAAG,IAAI,GAAG,CAAC,EAAE,SAAS,EAAE,SAAS,EACpE,OAAO,EAAE,WAAW,GAAG,CAAC,EAAE,IAAI,GAAG,EAAE,EAAE,IAAI,GAAG,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAC3F;QAChB,IAAI,CAAC,MAAM,GAAG,IAAI,UAAU,CAAC,SAAS,EAAE,gBAAgB,CAAC;YACrD,IAAI,EAAE,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS;YACnD,WAAW;YACX,SAAS;YACT,SAAS;YACT,OAAO;YACP,WAAW;YACX,IAAI;YACJ,IAAI;YACJ,KAAK;YACL,SAAS;YACT,SAAS;YACT,OAAO;YACP,QAAQ;YACR,SAAS;SACZ,CAAC,CAAC,CAAC;IACR,CAAC;IAEM,MAAM,KAAK,UAAU;QACxB,OAAO,YAAY,CAAC,UAAU,EAAE,CAAC;IACrC,CAAC;CACJ"}
+{"version":3,"file":"LlamaModel.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaModel.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,gBAAgB,EAAC,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAC,YAAY,EAAE,UAAU,EAAC,MAAM,gBAAgB,CAAC;AA2ExD,MAAM,OAAO,UAAU;IACnB,gBAAgB;IACA,MAAM,CAAa;IAEnC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAqCG;IACH,YAAmB,EACf,SAAS,EAAE,IAAI,GAAG,IAAI,EAAE,WAAW,GAAG,IAAI,GAAG,CAAC,EAAE,SAAS,EAAE,SAAS,EACpE,OAAO,EAAE,OAAO,GAAG,CAAC,EAAE,WAAW,GAAG,CAAC,EAAE,IAAI,GAAG,EAAE,EAAE,IAAI,GAAG,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EACxG;QAChB,IAAI,CAAC,MAAM,GAAG,IAAI,UAAU,CAAC,SAAS,EAAE,gBAAgB,CAAC;YACrD,IAAI,EAAE,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS;YACnD,WAAW;YACX,SAAS;YACT,SAAS;YACT,OAAO;YACP,OAAO;YACP,WAAW;YACX,IAAI;YACJ,IAAI;YACJ,KAAK;YACL,SAAS;YACT,SAAS;YACT,OAAO;YACP,QAAQ;YACR,SAAS;SACZ,CAAC,CAAC,CAAC;IACR,CAAC;IAEM,MAAM,KAAK,UAAU;QACxB,OAAO,YAAY,CAAC,UAAU,EAAE,CAAC;IACrC,CAAC;CACJ"}

package/dist/utils/getBin.d.ts

@@ -19,6 +19,7 @@ export type LLAMAModel = {
         useMmap?: boolean;
         useMlock?: boolean;
         embedding?: boolean;
+        threads?: number;
         temperature?: number;
         topK?: number;
         topP?: number;

package/llama/addon.cpp

@@ -13,6 +13,7 @@ class LLAMAModel : public Napi::ObjectWrap<LLAMAModel> {
     llama_context_params params;
     llama_model* model;
     float temperature;
+    int threads;
     int32_t top_k;
     float top_p;
 
@@ -21,6 +22,7 @@ class LLAMAModel : public Napi::ObjectWrap<LLAMAModel> {
         params.seed = -1;
         params.n_ctx = 4096;
         temperature = 0.0f;
+        threads = 6;
         top_k = 40;
         top_p = 0.95f;
 
@@ -74,6 +76,10 @@ class LLAMAModel : public Napi::ObjectWrap<LLAMAModel> {
                 params.embedding = options.Get("embedding").As<Napi::Boolean>().Value();
             }
 
+            if (options.Has("threads")) {
+                threads = options.Get("threads").As<Napi::Number>().Int32Value();
+            }
+
             if (options.Has("temperature")) {
                 temperature = options.Get("temperature").As<Napi::Number>().FloatValue();
             }
@@ -283,7 +289,7 @@ class LLAMAContextEvalWorker : Napi::AsyncWorker, Napi::Promise::Deferred {
   protected:
   void Execute() {
     // Perform the evaluation using llama_eval.
-    int r = llama_eval(ctx->ctx, tokens.data(), int(tokens.size()), llama_get_kv_cache_token_count(ctx->ctx), 6);
+    int r = llama_eval(ctx->ctx, tokens.data(), int(tokens.size()), llama_get_kv_cache_token_count(ctx->ctx), (ctx->model)->threads);
     if (r != 0) {
       SetError("Eval has failed");
       return;

package/llama/binariesGithubRelease.json

@@ -1,3 +1,3 @@
 {
-    "release": "b1147"
+    "release": "b1154"
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "node-llama-cpp",
-  "version": "2.2.0",
+  "version": "2.3.1",
   "description": "node.js bindings for llama.cpp",
   "main": "dist/index.js",
   "type": "module",
@@ -18,22 +18,22 @@
   ],
   "exports": {
     ".": {
-      "default": "./dist/index.js",
       "import": "./dist/index.js",
       "node": "./dist/index.js",
-      "types": "./dist/index.d.ts"
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
     },
     "./commands": {
-      "default": "./dist/commands.js",
       "import": "./dist/commands.js",
       "node": "./dist/commands.js",
-      "types": "./dist/commands.d.ts"
+      "types": "./dist/commands.d.ts",
+      "default": "./dist/commands.js"
     },
     "./commands.js": {
-      "default": "./dist/commands.js",
       "import": "./dist/commands.js",
       "node": "./dist/commands.js",
-      "types": "./dist/commands.d.ts"
+      "types": "./dist/commands.d.ts",
+      "default": "./dist/commands.js"
     }
   },
   "engines": {
@@ -44,7 +44,7 @@
     "prebuild": "rm -rf ./dist ./tsconfig.tsbuildinfo",
     "build": "tsc --build tsconfig.json --force",
     "addPostinstallScript": "npm pkg set scripts.postinstall=\"node ./dist/cli/cli.js postinstall\"",
-    "generate-docs": "typedoc",
+    "generate-docs": "typedoc && cp -r ./assets/*.png ./docs/assets/",
     "prewatch": "rm -rf ./dist ./tsconfig.tsbuildinfo",
     "watch": "tsc --build tsconfig.json --watch --force",
     "node-gyp-llama": "cd llama && node-gyp",