node-llama-cpp 1.2.1 → 1.3.1

package/README.md

@@ -18,19 +18,22 @@ If binaries are not available for your platform, it'll fallback to download the
 To disable this behavior set the environment variable `NODE_LLAMA_CPP_SKIP_DOWNLOAD` to `true`.
 
 ## Documentation
+### [API reference](https://withcatai.github.io/node-llama-cpp/modules.html)
+
 ### Usage
 #### As a chatbot
 ```typescript
 import {fileURLToPath} from "url";
 import path from "path";
-import {LlamaModel, LlamaChatSession} from "node-llama-cpp";
+import {LlamaModel, LlamaContext, LlamaChatSession} from "node-llama-cpp";
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 const model = new LlamaModel({
     modelPath: path.join(__dirname, "models", "vicuna-13b-v1.5-16k.ggmlv3.q5_1.bin")
 });
-const session = new LlamaChatSession({context: model.createContext()});
+const context = new LlamaContext({model});
+const session = new LlamaChatSession({context});
 
 
 const q1 = "Hi there, how are you?";
@@ -51,7 +54,7 @@ console.log("AI: " + a2);
 ```typescript
 import {fileURLToPath} from "url";
 import path from "path";
-import {LlamaModel, LlamaChatSession, ChatPromptWrapper} from "node-llama-cpp";
+import {LlamaModel, LlamaContext, LlamaChatSession, ChatPromptWrapper} from "node-llama-cpp";
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
@@ -73,7 +76,8 @@ const model = new LlamaModel({
     modelPath: path.join(__dirname, "models", "vicuna-13b-v1.5-16k.ggmlv3.q5_1.bin"),
     promptWrapper: new MyCustomChatPromptWrapper() // by default, LlamaChatPromptWrapper is used
 })
-const session = new LlamaChatSession({context: model.createContext()});
+const context = new LlamaContext({model});
+const session = new LlamaChatSession({context});
 
 
 const q1 = "Hi there, how are you?";
@@ -94,7 +98,7 @@ console.log("AI: " + a2);
 ```typescript
 import {fileURLToPath} from "url";
 import path from "path";
-import {LlamaModel, LlamaChatSession} from "node-llama-cpp";
+import {LlamaModel, LlamaContext, LlamaChatSession} from "node-llama-cpp";
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
@@ -102,7 +106,7 @@ const model = new LlamaModel({
     modelPath: path.join(__dirname, "models", "vicuna-13b-v1.5-16k.ggmlv3.q5_1.bin")
 });
 
-const context = model.createContext();
+const context = new LlamaContext({model});
 
 const q1 = "Hi there, how are you?";
 console.log("AI: " + q1);
@@ -127,7 +131,7 @@ console.log("AI: " + a1);
 ```
 
 ### CLI
-```txt
+```
 Usage: node-llama-cpp <command> [options]
 
 Commands:
@@ -138,11 +142,11 @@ Commands:
 
 Options:
   -h, --help     Show help                                                                 [boolean]
-  -v, --version  Show version number                                                       [boolean]                                                [boolean]
+  -v, --version  Show version number                                                       [boolean]
 ```
 
 #### `download` command
-```txt
+```
 node-llama-cpp download
 
 Download a release of llama.cpp and compile it
@@ -160,7 +164,7 @@ Options:
 ```
 
 #### `build` command
-```txt
+```
 node-llama-cpp build
 
 Compile the currently downloaded llama.cpp
@@ -173,7 +177,7 @@ Options:
 ```
 
 #### `clear` command
-```txt
+```
 node-llama-cpp clear [type]
 
 Clear files created by llama-cli
@@ -185,7 +189,7 @@ Options:
 ```
 
 #### `chat` command
-```txt
+```
 node-llama-cpp chat
 
 Chat with a LLama model

package/dist/cli/commands/ChatCommand.d.ts

@@ -4,6 +4,7 @@ type ChatCommand = {
     systemInfo: boolean;
     systemPrompt: string;
     wrapper: string;
+    contextSize: number;
 };
 export declare const ChatCommand: CommandModule<object, ChatCommand>;
 export {};

package/dist/cli/commands/ChatCommand.js

@@ -36,11 +36,17 @@ export const ChatCommand = {
             choices: ["general", "llama"],
             description: "Chat wrapper to use",
             group: "Optional:"
+        })
+            .option("contextSize", {
+            type: "number",
+            default: 1024 * 4,
+            description: "Context size to use for the model",
+            group: "Optional:"
         });
     },
-    async handler({ model, systemInfo, systemPrompt, wrapper }) {
+    async handler({ model, systemInfo, systemPrompt, wrapper, contextSize }) {
         try {
-            await RunChat({ model, systemInfo, systemPrompt, wrapper });
+            await RunChat({ model, systemInfo, systemPrompt, wrapper, contextSize });
         }
         catch (err) {
             console.error(err);
@@ -48,14 +54,17 @@ export const ChatCommand = {
         }
     }
 };
-async function RunChat({ model: modelArg, systemInfo, systemPrompt, wrapper }) {
+async function RunChat({ model: modelArg, systemInfo, systemPrompt, wrapper, contextSize }) {
     const { LlamaChatSession } = await import("../../llamaEvaluator/LlamaChatSession.js");
     const { LlamaModel } = await import("../../llamaEvaluator/LlamaModel.js");
+    const { LlamaContext } = await import("../../llamaEvaluator/LlamaContext.js");
     const model = new LlamaModel({
-        modelPath: modelArg
+        modelPath: modelArg,
+        contextSize
     });
+    const context = new LlamaContext({ model });
     const session = new LlamaChatSession({
-        context: model.createContext(),
+        context,
         printLLamaSystemInfo: systemInfo,
         systemPrompt,
         promptWrapper: createChatWrapper(wrapper)

package/dist/cli/commands/ChatCommand.js.map

@@ -1 +1 @@
-{"version":3,"file":"ChatCommand.js","sourceRoot":"","sources":["../../../src/cli/commands/ChatCommand.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,mBAAmB,CAAC;AAC9C,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,OAAO,MAAM,wBAAwB,CAAC;AAC7C,OAAO,EAAC,uBAAuB,EAAC,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAC,sBAAsB,EAAC,MAAM,8CAA8C,CAAC;AACpF,OAAO,EAAC,wBAAwB,EAAC,MAAM,gDAAgD,CAAC;AASxF,MAAM,CAAC,MAAM,WAAW,GAAuC;IAC3D,OAAO,EAAE,MAAM;IACf,QAAQ,EAAE,yBAAyB;IACnC,OAAO,CAAC,KAAK;QACT,OAAO,KAAK;aACP,MAAM,CAAC,OAAO,EAAE;YACb,IAAI,EAAE,QAAQ;YACd,YAAY,EAAE,IAAI;YAClB,WAAW,EAAE,sCAAsC;YACnD,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,YAAY,EAAE;YAClB,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,KAAK;YACd,WAAW,EAAE,6BAA6B;YAC1C,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,cAAc,EAAE;YACpB,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,uBAAuB;YAChC,kBAAkB,EAAE,EAAE;YACtB,WAAW,EACP,0CAA0C;gBAC1C,kBAAkB,GAAG,uBAAuB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG;YAC5E,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,SAAS,EAAE;YACf,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,SAAS;YAClB,OAAO,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC;YAC7B,WAAW,EAAE,qBAAqB;YAClC,KAAK,EAAE,WAAW;SACrB,CAAC,CAAC;IACX,CAAC;IACD,KAAK,CAAC,OAAO,CAAC,EAAC,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,OAAO,EAAC;QACpD,IAAI;YACA,MAAM,OAAO,CAAC,EAAC,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,OAAO,EAAC,CAAC,CAAC;SAC7D;QAAC,OAAO,GAAG,EAAE;YACV,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACnB;IACL,CAAC;CACJ,CAAC;AAGF,KAAK,UAAU,OAAO,CAAC,EAAC,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,OAAO,EAAc;IACpF,MAAM,EAAC,gBAAgB,EAAC,GAAG,MAAM,MAAM,CAAC,0CAA0C,CAAC,CAAC;IACpF,MAAM,EAAC,UAAU,EAAC,GAAG,MAAM,MAAM,CAAC,oCAAoC,CAAC,CAAC;IAExE,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC;QACzB,SAAS,EAAE,QAAQ;KACtB,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,IAAI,gBAAgB,CAAC;QACjC,OAAO,EAAE,KAAK,CAAC,aAAa,EAAE;QAC9B,oBAAoB,EAAE,UAAU;QAChC,YAAY;QACZ,aAAa,EAAE,iBAAiB,CAAC,OAAO,CAAC;KAC5C,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC;QACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC;QACpC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC;QACnC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC;KAC3C,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,iDAAiD;IACjD,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;IAErD,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;QAChC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM,EAAE,OAAO,CAAC,MAAM;KACzB,CAAC,CAAC;IAEH,iDAAiD;IACjD,OAAO,IAAI,EAAE;QACT,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAEpD,IAAI,KAAK,KAAK,OAAO;YACjB,MAAM;QAEV,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAE3C,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAEpE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACjC,MAAM,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE;YAClC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;QACH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC/B,OAAO,CAAC,GAAG,EAAE,CAAC;KACjB;AACL,CAAC;AAED,SAAS,iBAAiB,CAAC,OAAe;IACtC,QAAQ,OAAO,EAAE;QACb,KAAK,SAAS;YACV,OAAO,IAAI,wBAAwB,EAAE,CAAC;QAC1C,KAAK,OAAO;YACR,OAAO,IAAI,sBAAsB,EAAE,CAAC;KAC3C;IACD,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,OAAO,CAAC,CAAC;AACnD,CAAC"}
+{"version":3,"file":"ChatCommand.js","sourceRoot":"","sources":["../../../src/cli/commands/ChatCommand.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,mBAAmB,CAAC;AAC9C,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,OAAO,MAAM,wBAAwB,CAAC;AAC7C,OAAO,EAAC,uBAAuB,EAAC,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAC,sBAAsB,EAAC,MAAM,8CAA8C,CAAC;AACpF,OAAO,EAAC,wBAAwB,EAAC,MAAM,gDAAgD,CAAC;AAUxF,MAAM,CAAC,MAAM,WAAW,GAAuC;IAC3D,OAAO,EAAE,MAAM;IACf,QAAQ,EAAE,yBAAyB;IACnC,OAAO,CAAC,KAAK;QACT,OAAO,KAAK;aACP,MAAM,CAAC,OAAO,EAAE;YACb,IAAI,EAAE,QAAQ;YACd,YAAY,EAAE,IAAI;YAClB,WAAW,EAAE,sCAAsC;YACnD,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,YAAY,EAAE;YAClB,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,KAAK;YACd,WAAW,EAAE,6BAA6B;YAC1C,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,cAAc,EAAE;YACpB,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,uBAAuB;YAChC,kBAAkB,EAAE,EAAE;YACtB,WAAW,EACP,0CAA0C;gBAC1C,kBAAkB,GAAG,uBAAuB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG;YAC5E,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,SAAS,EAAE;YACf,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,SAAS;YAClB,OAAO,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC;YAC7B,WAAW,EAAE,qBAAqB;YAClC,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,aAAa,EAAE;YACnB,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,IAAI,GAAG,CAAC;YACjB,WAAW,EAAE,mCAAmC;YAChD,KAAK,EAAE,WAAW;SACrB,CAAC,CAAC;IACX,CAAC;IACD,KAAK,CAAC,OAAO,CAAC,EAAC,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,OAAO,EAAE,WAAW,EAAC;QACjE,IAAI;YACA,MAAM,OAAO,CAAC,EAAC,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,OAAO,EAAE,WAAW,EAAC,CAAC,CAAC;SAC1E;QAAC,OAAO,GAAG,EAAE;YACV,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACnB;IACL,CAAC;CACJ,CAAC;AAGF,KAAK,UAAU,OAAO,CAAC,EAAC,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,OAAO,EAAE,WAAW,EAAc;IACjG,MAAM,EAAC,gBAAgB,EAAC,GAAG,MAAM,MAAM,CAAC,0CAA0C,CAAC,CAAC;IACpF,MAAM,EAAC,UAAU,EAAC,GAAG,MAAM,MAAM,CAAC,oCAAoC,CAAC,CAAC;IACxE,MAAM,EAAC,YAAY,EAAC,GAAG,MAAM,MAAM,CAAC,sCAAsC,CAAC,CAAC;IAE5E,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC;QACzB,SAAS,EAAE,QAAQ;QACnB,WAAW;KACd,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,EAAC,KAAK,EAAC,CAAC,CAAC;IAC1C,MAAM,OAAO,GAAG,IAAI,gBAAgB,CAAC;QACjC,OAAO;QACP,oBAAoB,EAAE,UAAU;QAChC,YAAY;QACZ,aAAa,EAAE,iBAAiB,CAAC,OAAO,CAAC;KAC5C,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC;QACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC;QACpC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC;QACnC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC;KAC3C,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,iDAAiD;IACjD,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;IAErD,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;QAChC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM,EAAE,OAAO,CAAC,MAAM;KACzB,CAAC,CAAC;IAEH,iDAAiD;IACjD,OAAO,IAAI,EAAE;QACT,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAEpD,IAAI,KAAK,KAAK,OAAO;YACjB,MAAM;QAEV,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAE3C,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAEpE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACjC,MAAM,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE;YAClC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;QACH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC/B,OAAO,CAAC,GAAG,EAAE,CAAC;KACjB;AACL,CAAC;AAED,SAAS,iBAAiB,CAAC,OAAe;IACtC,QAAQ,OAAO,EAAE;QACb,KAAK,SAAS;YACV,OAAO,IAAI,wBAAwB,EAAE,CAAC;QAC1C,KAAK,OAAO;YACR,OAAO,IAAI,sBAAsB,EAAE,CAAC;KAC3C;IACD,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,OAAO,CAAC,CAAC;AACnD,CAAC"}

package/dist/llamaEvaluator/LlamaChatSession.js

@@ -3,7 +3,7 @@ import { withLock } from "../utils/withLock.js";
 import { AbortError } from "../AbortError.js";
 import { GeneralChatPromptWrapper } from "../chatWrappers/GeneralChatPromptWrapper.js";
 import { LlamaModel } from "./LlamaModel.js";
-const UNKNOWN_UNICODE_CHAR = "�";
+const UNKNOWN_UNICODE_CHAR = "\ufffd";
 export class LlamaChatSession {
     _systemPrompt;
     _printLLamaSystemInfo;

package/dist/llamaEvaluator/LlamaChatSession.js.map

@@ -1 +1 @@
-{"version":3,"file":"LlamaChatSession.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaChatSession.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,uBAAuB,EAAC,MAAM,cAAc,CAAC;AACrD,OAAO,EAAC,QAAQ,EAAC,MAAM,sBAAsB,CAAC;AAE9C,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAC,wBAAwB,EAAC,MAAM,6CAA6C,CAAC;AACrF,OAAO,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAG3C,MAAM,oBAAoB,GAAG,GAAG,CAAC;AAEjC,MAAM,OAAO,gBAAgB;IACR,aAAa,CAAS;IACtB,qBAAqB,CAAU;IAC/B,cAAc,CAAoB;IAC3C,YAAY,GAAW,CAAC,CAAC;IACzB,YAAY,GAAY,KAAK,CAAC;IACrB,IAAI,CAAe;IAEpC,YAAmB,EACf,OAAO,EACP,oBAAoB,GAAG,KAAK,EAC5B,aAAa,GAAG,IAAI,wBAAwB,EAAE,EAC9C,YAAY,GAAG,uBAAuB,EAMzC;QACG,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC;QACpB,IAAI,CAAC,qBAAqB,GAAG,oBAAoB,CAAC;QAClD,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QAEpC,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;IACtC,CAAC;IAED,IAAW,WAAW;QAClB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED,IAAW,OAAO;QACd,OAAO,IAAI,CAAC,IAAI,CAAC;IACrB,CAAC;IAEM,KAAK,CAAC,IAAI;QACb,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE;YACpC,IAAI,IAAI,CAAC,YAAY;gBACjB,OAAO;YAEX,IAAI,IAAI,CAAC,qBAAqB;gBAC1B,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;YAE5D,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC7B,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,KAAK,CAAC,MAAM,CAAC,MAAc,EAAE,OAAoC,EAAE,EAAC,MAAM,KAA8B,EAAE;QAC7G,IAAI,CAAC,IAAI,CAAC,WAAW;YACjB,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAEtB,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,IAAI,EAAE;YAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,MAAM,EAAE,EAAC,YAAY,EAAE,IAAI,CAAC,aAAa,EAAE,WAAW,EAAE,IAAI,CAAC,YAAY,EAAC,CAAC,CAAC;YAC9H,IAAI,CAAC,YAAY,EAAE,CAAC;YAEpB,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,OAAO,EAAE,EAAC,MAAM,EAAC,CAAC,CAAC;QACnF,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,MAAmB,EAAE,OAAoC,EAAE,EAAC,MAAM,KAA8B,EAAE;QACxH,MAAM,YAAY,GAAG,CAAC,MAAgB,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QAEtF,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,CAAC;QACzD,MAAM,iBAAiB,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5D,MAAM,kBAAkB,GAAa,EAAE,CAAC;QACxC,MAAM,GAAG,GAAa,EAAE,CAAC;QAGzB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;YAClD,IAAI,MAAM,EAAE,OAAO;gBACf,MAAM,IAAI,UAAU,EAAE,CAAC;YAE3B,MAAM,QAAQ,GAAG,YAAY,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;YACvC,MAAM,EAAC,YAAY,EAAE,cAAc,EAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC;YAE1F,IAAI,YAAY;gBACZ,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC;YAE7B,gEAAgE;YAChE,IAAI,QAAQ,KAAK,oBAAoB,IAAI,cAAc,EAAE;gBACrD,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC/B,SAAS;aACZ;YAED,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE;gBAC/B,GAAG,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,CAAC;gBAChC,OAAO,EAAE,CAAC,kBAAkB,CAAC,CAAC;gBAC9B,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC;aACjC;YAED,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChB,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;SACtB;QAED,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAEO,gBAAgB,CAAC,QAAgB,EAAE,iBAA2B;QAClE,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,CAAC;QACzD,IAAI,cAAc,GAAG,KAAK,CAAC;QAE3B,KAAK,IAAI,eAAe,GAAG,CAAC,EAAE,eAAe,GAAG,WAAW,CAAC,MAAM,EAAE,eAAe,EAAE,EAAE;YACnF,MAAM,UAAU,GAAG,WAAW,CAAC,eAAe,CAAC,CAAC;YAEhD,IAAI,yBAAyB,GAAG,KAAK,CAAC;YACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,IAAI,iBAAiB,CAAC,eAAe,CAAC,KAAK,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBAClG,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC,EAAE;oBAChE,iBAAiB,CAAC,eAAe,CAAC,EAAE,CAAC;oBACrC,yBAAyB,GAAG,IAAI,CAAC;iBACpC;qBAAM;oBACH,iBAAiB,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;oBACvC,yBAAyB,GAAG,KAAK,CAAC;oBAClC,MAAM;iBACT;aACJ;YAED,IAAI,iBAAiB,CAAC,eAAe,CAAC,KAAK,UAAU,CAAC,MAAM,EAAE;gBAC1D,OAAO,EAAC,YAAY,EAAE,IAAI,EAAC,CAAC;aAC/B;YAED,cAAc,KAAK,yBAAyB,CAAC;SAChD;QAED,OAAO,EAAC,cAAc,EAAC,CAAC;IAC5B,CAAC;CACJ"}
+{"version":3,"file":"LlamaChatSession.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaChatSession.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,uBAAuB,EAAC,MAAM,cAAc,CAAC;AACrD,OAAO,EAAC,QAAQ,EAAC,MAAM,sBAAsB,CAAC;AAE9C,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAC,wBAAwB,EAAC,MAAM,6CAA6C,CAAC;AACrF,OAAO,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAG3C,MAAM,oBAAoB,GAAG,QAAQ,CAAC;AAEtC,MAAM,OAAO,gBAAgB;IACR,aAAa,CAAS;IACtB,qBAAqB,CAAU;IAC/B,cAAc,CAAoB;IAC3C,YAAY,GAAW,CAAC,CAAC;IACzB,YAAY,GAAY,KAAK,CAAC;IACrB,IAAI,CAAe;IAEpC,YAAmB,EACf,OAAO,EACP,oBAAoB,GAAG,KAAK,EAC5B,aAAa,GAAG,IAAI,wBAAwB,EAAE,EAC9C,YAAY,GAAG,uBAAuB,EAMzC;QACG,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC;QACpB,IAAI,CAAC,qBAAqB,GAAG,oBAAoB,CAAC;QAClD,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QAEpC,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;IACtC,CAAC;IAED,IAAW,WAAW;QAClB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED,IAAW,OAAO;QACd,OAAO,IAAI,CAAC,IAAI,CAAC;IACrB,CAAC;IAEM,KAAK,CAAC,IAAI;QACb,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE;YACpC,IAAI,IAAI,CAAC,YAAY;gBACjB,OAAO;YAEX,IAAI,IAAI,CAAC,qBAAqB;gBAC1B,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;YAE5D,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC7B,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,KAAK,CAAC,MAAM,CAAC,MAAc,EAAE,OAAoC,EAAE,EAAC,MAAM,KAA8B,EAAE;QAC7G,IAAI,CAAC,IAAI,CAAC,WAAW;YACjB,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAEtB,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,IAAI,EAAE;YAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,MAAM,EAAE,EAAC,YAAY,EAAE,IAAI,CAAC,aAAa,EAAE,WAAW,EAAE,IAAI,CAAC,YAAY,EAAC,CAAC,CAAC;YAC9H,IAAI,CAAC,YAAY,EAAE,CAAC;YAEpB,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,OAAO,EAAE,EAAC,MAAM,EAAC,CAAC,CAAC;QACnF,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,MAAmB,EAAE,OAAoC,EAAE,EAAC,MAAM,KAA8B,EAAE;QACxH,MAAM,YAAY,GAAG,CAAC,MAAgB,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QAEtF,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,CAAC;QACzD,MAAM,iBAAiB,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5D,MAAM,kBAAkB,GAAa,EAAE,CAAC;QACxC,MAAM,GAAG,GAAa,EAAE,CAAC;QAGzB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;YAClD,IAAI,MAAM,EAAE,OAAO;gBACf,MAAM,IAAI,UAAU,EAAE,CAAC;YAE3B,MAAM,QAAQ,GAAG,YAAY,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;YACvC,MAAM,EAAC,YAAY,EAAE,cAAc,EAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC;YAE1F,IAAI,YAAY;gBACZ,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC;YAE7B,gEAAgE;YAChE,IAAI,QAAQ,KAAK,oBAAoB,IAAI,cAAc,EAAE;gBACrD,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC/B,SAAS;aACZ;YAED,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE;gBAC/B,GAAG,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,CAAC;gBAChC,OAAO,EAAE,CAAC,kBAAkB,CAAC,CAAC;gBAC9B,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC;aACjC;YAED,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChB,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;SACtB;QAED,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAEO,gBAAgB,CAAC,QAAgB,EAAE,iBAA2B;QAClE,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,CAAC;QACzD,IAAI,cAAc,GAAG,KAAK,CAAC;QAE3B,KAAK,IAAI,eAAe,GAAG,CAAC,EAAE,eAAe,GAAG,WAAW,CAAC,MAAM,EAAE,eAAe,EAAE,EAAE;YACnF,MAAM,UAAU,GAAG,WAAW,CAAC,eAAe,CAAC,CAAC;YAEhD,IAAI,yBAAyB,GAAG,KAAK,CAAC;YACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,IAAI,iBAAiB,CAAC,eAAe,CAAC,KAAK,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBAClG,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC,EAAE;oBAChE,iBAAiB,CAAC,eAAe,CAAC,EAAE,CAAC;oBACrC,yBAAyB,GAAG,IAAI,CAAC;iBACpC;qBAAM;oBACH,iBAAiB,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;oBACvC,yBAAyB,GAAG,KAAK,CAAC;oBAClC,MAAM;iBACT;aACJ;YAED,IAAI,iBAAiB,CAAC,eAAe,CAAC,KAAK,UAAU,CAAC,MAAM,EAAE;gBAC1D,OAAO,EAAC,YAAY,EAAE,IAAI,EAAC,CAAC;aAC/B;YAED,cAAc,KAAK,yBAAyB,CAAC;SAChD;QAED,OAAO,EAAC,cAAc,EAAC,CAAC;IAC5B,CAAC;CACJ"}

package/dist/llamaEvaluator/LlamaContext.d.ts

@@ -1,6 +1,11 @@
+import { LlamaModel } from "./LlamaModel.js";
 export declare class LlamaContext {
     private readonly _ctx;
     private _prependBos;
+    constructor({ model, prependBos }: {
+        model: LlamaModel;
+        prependBos?: boolean;
+    });
     encode(text: string): Uint32Array;
     decode(tokens: Uint32Array): string;
     evaluate(tokens: Uint32Array, getRestrictions?: () => Uint32Array): AsyncGenerator<number, void, unknown>;

package/dist/llamaEvaluator/LlamaContext.js

@@ -1,10 +1,9 @@
-import { llamaCppNode } from "./LlamaBins.js";
+import { LLAMAContext, llamaCppNode } from "./LlamaBins.js";
 export class LlamaContext {
     _ctx;
     _prependBos;
-    /** @internal */
-    constructor({ ctx, prependBos }) {
-        this._ctx = ctx;
+    constructor({ model, prependBos = true }) {
+        this._ctx = new LLAMAContext(model._model);
         this._prependBos = prependBos;
     }
     encode(text) {

package/dist/llamaEvaluator/LlamaContext.js.map

@@ -1 +1 @@
-{"version":3,"file":"LlamaContext.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaContext.ts"],"names":[],"mappings":"AAAA,OAAO,EAAe,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAG1D,MAAM,OAAO,YAAY;IACJ,IAAI,CAAe;IAC5B,WAAW,CAAU;IAE7B,gBAAgB;IAChB,YAAoB,EAAC,GAAG,EAAE,UAAU,EAAoC;QACpE,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC;QAChB,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,IAAY;QACtB,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,MAAmB;QAC7B,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAEM,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAmB,EAAE,eAAmC;QAC3E,IAAI,UAAU,GAAG,MAAM,CAAC;QAExB,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtC,UAAU,CAAC,OAAO,CAAC,YAAY,CAAC,QAAQ,EAAE,CAAC,CAAC;YAE5C,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC1C,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;SAC5B;QAED,iDAAiD;QACjD,OAAO,IAAI,EAAE;YACT,kCAAkC;YAClC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;YAExE,mCAAmC;YACnC,IAAI,SAAS,KAAK,YAAY,CAAC,QAAQ,EAAE;gBACrC,MAAM;YAEV,MAAM,SAAS,CAAC;YAEhB,mCAAmC;YACnC,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;SAC9C;IACL,CAAC;CAEJ"}
+{"version":3,"file":"LlamaContext.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaContext.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAE,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAG1D,MAAM,OAAO,YAAY;IACJ,IAAI,CAAe;IAC5B,WAAW,CAAU;IAE7B,YAAmB,EAAC,KAAK,EAAE,UAAU,GAAG,IAAI,EAA4C;QACpF,IAAI,CAAC,IAAI,GAAG,IAAI,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC3C,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,IAAY;QACtB,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,MAAmB;QAC7B,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAEM,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAmB,EAAE,eAAmC;QAC3E,IAAI,UAAU,GAAG,MAAM,CAAC;QAExB,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtC,UAAU,CAAC,OAAO,CAAC,YAAY,CAAC,QAAQ,EAAE,CAAC,CAAC;YAE5C,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC1C,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;SAC5B;QAED,iDAAiD;QACjD,OAAO,IAAI,EAAE;YACT,kCAAkC;YAClC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;YAExE,mCAAmC;YACnC,IAAI,SAAS,KAAK,YAAY,CAAC,QAAQ,EAAE;gBACrC,MAAM;YAEV,MAAM,SAAS,CAAC;YAEhB,mCAAmC;YACnC,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;SAC9C;IACL,CAAC;CAEJ"}

package/dist/llamaEvaluator/LlamaModel.d.ts

@@ -1,11 +1,46 @@
-import { LlamaContext } from "./LlamaContext.js";
 export declare class LlamaModel {
-    private readonly _model;
-    private readonly _prependBos;
-    constructor({ modelPath, prependBos }: {
+    /**
+     * options source:
+     * https://github.com/ggerganov/llama.cpp/blob/b5ffb2849d23afe73647f68eec7b68187af09be6/llama.h#L102 (struct llama_context_params)
+     * @param {object} options
+     * @param {string} options.modelPath - path to the model on the filesystem
+     * @param {number | null} [options.seed] - If null, a random seed will be used
+     * @param {number} [options.contextSize] - text context size
+     * @param {number} [options.batchSize] - prompt processing batch size
+     * @param {number} [options.gpuLayers] - number of layers to store in VRAM
+     * @param {boolean} [options.lowVram] - if true, reduce VRAM usage at the cost of performance
+     * @param {boolean} [options.f16Kv] - use fp16 for KV cache
+     * @param {boolean} [options.logitsAll] - the llama_eval() call computes all logits, not just the last one
+     * @param {boolean} [options.vocabOnly] - only load the vocabulary, no weights
+     * @param {boolean} [options.useMmap] - use mmap if possible
+     * @param {boolean} [options.useMlock] - force system to keep model in RAM
+     * @param {boolean} [options.embedding] - embedding mode only
+     */
+    constructor({ modelPath, seed, contextSize, batchSize, gpuLayers, lowVram, f16Kv, logitsAll, vocabOnly, useMmap, useMlock, embedding }: {
+        /** path to the model on the filesystem */
         modelPath: string;
-        prependBos?: boolean;
+        /** If null, a random seed will be used */
+        seed?: number | null;
+        /** text context size */
+        contextSize?: number;
+        /** prompt processing batch size */
+        batchSize?: number;
+        /** number of layers to store in VRAM */
+        gpuLayers?: number;
+        /** if true, reduce VRAM usage at the cost of performance */
+        lowVram?: boolean;
+        /** use fp16 for KV cache */
+        f16Kv?: boolean;
+        /** the llama_eval() call computes all logits, not just the last one */
+        logitsAll?: boolean;
+        /** only load the vocabulary, no weights */
+        vocabOnly?: boolean;
+        /** use mmap if possible */
+        useMmap?: boolean;
+        /** force system to keep model in RAM */
+        useMlock?: boolean;
+        /** embedding mode only */
+        embedding?: boolean;
     });
-    createContext(): LlamaContext;
     static get systemInfo(): string;
 }

package/dist/llamaEvaluator/LlamaModel.js

@@ -1,20 +1,49 @@
-import { LlamaContext } from "./LlamaContext.js";
-import { LLAMAContext, llamaCppNode, LLAMAModel } from "./LlamaBins.js";
+import { llamaCppNode, LLAMAModel } from "./LlamaBins.js";
 export class LlamaModel {
+    /** @internal */
     _model;
-    _prependBos;
-    constructor({ modelPath, prependBos = true }) {
-        this._model = new LLAMAModel(modelPath);
-        this._prependBos = prependBos;
-    }
-    createContext() {
-        return new LlamaContext({
-            ctx: new LLAMAContext(this._model),
-            prependBos: this._prependBos
-        });
+    /**
+     * options source:
+     * https://github.com/ggerganov/llama.cpp/blob/b5ffb2849d23afe73647f68eec7b68187af09be6/llama.h#L102 (struct llama_context_params)
+     * @param {object} options
+     * @param {string} options.modelPath - path to the model on the filesystem
+     * @param {number | null} [options.seed] - If null, a random seed will be used
+     * @param {number} [options.contextSize] - text context size
+     * @param {number} [options.batchSize] - prompt processing batch size
+     * @param {number} [options.gpuLayers] - number of layers to store in VRAM
+     * @param {boolean} [options.lowVram] - if true, reduce VRAM usage at the cost of performance
+     * @param {boolean} [options.f16Kv] - use fp16 for KV cache
+     * @param {boolean} [options.logitsAll] - the llama_eval() call computes all logits, not just the last one
+     * @param {boolean} [options.vocabOnly] - only load the vocabulary, no weights
+     * @param {boolean} [options.useMmap] - use mmap if possible
+     * @param {boolean} [options.useMlock] - force system to keep model in RAM
+     * @param {boolean} [options.embedding] - embedding mode only
+     */
+    constructor({ modelPath, seed = null, contextSize = 1024 * 4, batchSize, gpuLayers, lowVram, f16Kv, logitsAll, vocabOnly, useMmap, useMlock, embedding }) {
+        this._model = new LLAMAModel(modelPath, removeNullFields({
+            seed: seed != null ? Math.max(-1, seed) : undefined,
+            contextSize,
+            batchSize,
+            gpuLayers,
+            lowVram,
+            f16Kv,
+            logitsAll,
+            vocabOnly,
+            useMmap,
+            useMlock,
+            embedding
+        }));
     }
     static get systemInfo() {
         return llamaCppNode.systemInfo();
     }
 }
+function removeNullFields(obj) {
+    const newObj = Object.assign({}, obj);
+    for (const key in obj) {
+        if (newObj[key] == null)
+            delete newObj[key];
+    }
+    return newObj;
+}
 //# sourceMappingURL=LlamaModel.js.map

package/dist/llamaEvaluator/LlamaModel.js.map

@@ -1 +1 @@
-{"version":3,"file":"LlamaModel.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaModel.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAC,YAAY,EAAE,YAAY,EAAE,UAAU,EAAC,MAAM,gBAAgB,CAAC;AAGtE,MAAM,OAAO,UAAU;IACF,MAAM,CAAa;IACnB,WAAW,CAAU;IAEtC,YAAmB,EAAC,SAAS,EAAE,UAAU,GAAG,IAAI,EAA8C;QAC1F,IAAI,CAAC,MAAM,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC;QACxC,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAClC,CAAC;IAEM,aAAa;QAChB,OAAO,IAAI,YAAY,CAAC;YACpB,GAAG,EAAE,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC;YAClC,UAAU,EAAE,IAAI,CAAC,WAAW;SAC/B,CAAC,CAAC;IACP,CAAC;IAEM,MAAM,KAAK,UAAU;QACxB,OAAO,YAAY,CAAC,UAAU,EAAE,CAAC;IACrC,CAAC;CACJ"}
+{"version":3,"file":"LlamaModel.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaModel.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAE,UAAU,EAAC,MAAM,gBAAgB,CAAC;AAGxD,MAAM,OAAO,UAAU;IACnB,gBAAgB;IACA,MAAM,CAAa;IAEnC;;;;;;;;;;;;;;;;OAgBG;IACH,YAAmB,EACf,SAAS,EAAE,IAAI,GAAG,IAAI,EAAE,WAAW,GAAG,IAAI,GAAG,CAAC,EAAE,SAAS,EAAE,SAAS,EACpE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAqCrE;QACG,IAAI,CAAC,MAAM,GAAG,IAAI,UAAU,CAAC,SAAS,EAAE,gBAAgB,CAAC;YACrD,IAAI,EAAE,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS;YACnD,WAAW;YACX,SAAS;YACT,SAAS;YACT,OAAO;YACP,KAAK;YACL,SAAS;YACT,SAAS;YACT,OAAO;YACP,QAAQ;YACR,SAAS;SACZ,CAAC,CAAC,CAAC;IACR,CAAC;IAEM,MAAM,KAAK,UAAU;QACxB,OAAO,YAAY,CAAC,UAAU,EAAE,CAAC;IACrC,CAAC;CACJ;AAED,SAAS,gBAAgB,CAAmB,GAAM;IAC9C,MAAM,MAAM,GAAM,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IAEzC,KAAK,MAAM,GAAG,IAAI,GAAG,EAAE;QACnB,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI;YACnB,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;KAC1B;IAED,OAAO,MAAM,CAAC;AAClB,CAAC"}

package/dist/utils/getBin.d.ts

@@ -8,7 +8,19 @@ export type LlamaCppNodeModule = {
     tokenEos(): number;
 };
 export type LLAMAModel = {
-    new (modelPath: string): LLAMAModel;
+    new (modelPath: string, params: {
+        seed?: number;
+        contextSize?: number;
+        batchSize?: number;
+        gpuCores?: number;
+        lowVram?: boolean;
+        f16Kv?: boolean;
+        logitsAll?: boolean;
+        vocabOnly?: boolean;
+        useMmap?: boolean;
+        useMlock?: boolean;
+        embedding?: boolean;
+    }): LLAMAModel;
 };
 export type LLAMAContext = {
     new (model: LLAMAModel): LLAMAContext;

package/llama/addon.cpp

@@ -8,21 +8,80 @@
 
 class LLAMAModel : public Napi::ObjectWrap<LLAMAModel> {
   public:
-  llama_context_params params;
-  llama_model* model;
-  LLAMAModel(const Napi::CallbackInfo& info) : Napi::ObjectWrap<LLAMAModel>(info) {
-    params = llama_context_default_params();
-    params.seed = -1;
-    params.n_ctx = 4096;
-    model = llama_load_model_from_file(info[0].As<Napi::String>().Utf8Value().c_str(), params);
-
-    if (model == NULL) {
-      Napi::Error::New(info.Env(), "Failed to load model").ThrowAsJavaScriptException();
-      return;
+    llama_context_params params;
+    llama_model* model;
+
+    LLAMAModel(const Napi::CallbackInfo& info) : Napi::ObjectWrap<LLAMAModel>(info) {
+        params = llama_context_default_params();
+        params.seed = -1;
+        params.n_ctx = 4096;
+
+        // Get the model path
+        std::string modelPath = info[0].As<Napi::String>().Utf8Value();
+
+        if (info.Length() > 1 && info[1].IsObject()) {
+            Napi::Object options = info[1].As<Napi::Object>();
+
+            if (options.Has("seed")) {
+                params.seed = options.Get("seed").As<Napi::Number>().Int32Value();
+            }
+
+            if (options.Has("contextSize")) {
+                params.n_ctx = options.Get("contextSize").As<Napi::Number>().Int32Value();
+            }
+
+            if (options.Has("batchSize")) {
+                params.n_batch = options.Get("batchSize").As<Napi::Number>().Int32Value();
+            }
+
+            if (options.Has("gpuLayers")) {
+                params.n_gpu_layers = options.Get("gpuLayers").As<Napi::Number>().Int32Value();
+            }
+
+            if (options.Has("lowVram")) {
+                params.low_vram = options.Get("lowVram").As<Napi::Boolean>().Value();
+            }
+
+            if (options.Has("f16Kv")) {
+                params.f16_kv = options.Get("f16Kv").As<Napi::Boolean>().Value();
+            }
+
+            if (options.Has("logitsAll")) {
+                params.logits_all = options.Get("logitsAll").As<Napi::Boolean>().Value();
+            }
+
+            if (options.Has("vocabOnly")) {
+                params.vocab_only = options.Get("vocabOnly").As<Napi::Boolean>().Value();
+            }
+
+            if (options.Has("useMmap")) {
+                params.use_mmap = options.Get("useMmap").As<Napi::Boolean>().Value();
+            }
+
+            if (options.Has("useMlock")) {
+                params.use_mlock = options.Get("useMlock").As<Napi::Boolean>().Value();
+            }
+
+            if (options.Has("embedding")) {
+                params.embedding = options.Get("embedding").As<Napi::Boolean>().Value();
+            }
+        }
+
+        model = llama_load_model_from_file(modelPath.c_str(), params);
+
+        if (model == NULL) {
+            Napi::Error::New(info.Env(), "Failed to load model").ThrowAsJavaScriptException();
+            return;
+        }
+    }
+
+    ~LLAMAModel() {
+        llama_free_model(model);
+    }
+
+    static void init(Napi::Object exports) {
+        exports.Set("LLAMAModel", DefineClass(exports.Env(), "LLAMAModel", {}));
     }
-  }
-  ~LLAMAModel() { llama_free_model(model); }
-  static void init(Napi::Object exports) { exports.Set("LLAMAModel", DefineClass(exports.Env(), "LLAMAModel", {})); }
 };
 
 class LLAMAContext : public Napi::ObjectWrap<LLAMAContext> {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "node-llama-cpp",
-  "version": "1.2.1",
+  "version": "1.3.1",
   "description": "node.js bindings for llama.cpp",
   "main": "dist/index.js",
   "type": "module",
@@ -44,6 +44,7 @@
     "prepare": "[ $CI = true ] || [ -d '.husky/_' ] || husky install",
     "prebuild": "rm -rf ./dist ./tsconfig.tsbuildinfo",
     "build": "tsc --build tsconfig.json --force",
+    "generate-docs": "typedoc",
     "prewatch": "rm -rf ./dist ./tsconfig.tsbuildinfo",
     "watch": "tsc --build tsconfig.json --watch --force",
     "node-gyp-llama": "cd llama && node-gyp",
@@ -79,10 +80,11 @@
   "bugs": {
     "url": "https://github.com/withcatai/node-llama-cpp/issues"
   },
-  "homepage": "https://github.com/withcatai/node-llama-cpp#readme",
+  "homepage": "https://withcatai.github.io/node-llama-cpp/",
   "devDependencies": {
     "@commitlint/cli": "^17.7.1",
     "@commitlint/config-conventional": "^17.7.0",
+    "@semantic-release/exec": "^6.0.3",
     "@types/bytes": "^3.1.1",
     "@types/cli-progress": "^3.11.0",
     "@types/cross-spawn": "^6.0.2",
@@ -99,6 +101,7 @@
     "semantic-release": "^21.0.7",
     "ts-node": "^10.9.1",
     "tslib": "^2.6.1",
+    "typedoc": "^0.24.8",
     "typescript": "^5.1.6",
     "zx": "^7.2.3"
   },