node-llama-cpp 1.1.1 → 1.2.1

package/README.md

@@ -29,8 +29,8 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 const model = new LlamaModel({
     modelPath: path.join(__dirname, "models", "vicuna-13b-v1.5-16k.ggmlv3.q5_1.bin")
-})
-const session = new LlamaChatSession({model});
+});
+const session = new LlamaChatSession({context: model.createContext()});
 
 
 const q1 = "Hi there, how are you?";
@@ -73,7 +73,7 @@ const model = new LlamaModel({
     modelPath: path.join(__dirname, "models", "vicuna-13b-v1.5-16k.ggmlv3.q5_1.bin"),
     promptWrapper: new MyCustomChatPromptWrapper() // by default, LlamaChatPromptWrapper is used
 })
-const session = new LlamaChatSession({model});
+const session = new LlamaChatSession({context: model.createContext()});
 
 
 const q1 = "Hi there, how are you?";
@@ -98,29 +98,31 @@ import {LlamaModel, LlamaChatSession} from "node-llama-cpp";
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
-const model = new LlamaChatSession({
+const model = new LlamaModel({
     modelPath: path.join(__dirname, "models", "vicuna-13b-v1.5-16k.ggmlv3.q5_1.bin")
 });
 
+const context = model.createContext();
+
 const q1 = "Hi there, how are you?";
 console.log("AI: " + q1);
 
-const tokens = model.encode(q1);
+const tokens = context.encode(q1);
 const res: number[] = [];
-for await (const chunk of model.evaluate(tokens)) {
+for await (const chunk of context.evaluate(tokens)) {
     res.push(chunk);
     
     // it's important to not concatinate the results as strings,
     // as doing so will break some characters (like some emojis) that are made of multiple tokens.
     // by using an array of tokens, we can decode them correctly together.
-    const resString: string = model.decode(Uint32Array.from(res));
+    const resString: string = context.decode(Uint32Array.from(res));
     
     const lastPart = resString.split("ASSISTANT:").reverse()[0];
     if (lastPart.includes("USER:"))
         break;
 }
 
-const a1 = model.decode(Uint32Array.from(res)).split("USER:")[0];
+const a1 = context.decode(Uint32Array.from(res)).split("USER:")[0];
 console.log("AI: " + a1);
 ```
 

package/dist/AbortError.d.ts

@@ -1,3 +1,2 @@
 export declare class AbortError extends Error {
-    constructor();
 }

package/dist/AbortError.js

@@ -1,4 +1,5 @@
 export class AbortError extends Error {
+    /** @internal */
     constructor() {
         super("AbortError");
     }

package/dist/AbortError.js.map

@@ -1 +1 @@
-{"version":3,"file":"AbortError.js","sourceRoot":"","sources":["../src/AbortError.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,UAAW,SAAQ,KAAK;IACjC;QACI,KAAK,CAAC,YAAY,CAAC,CAAC;IACxB,CAAC;CACJ"}
+{"version":3,"file":"AbortError.js","sourceRoot":"","sources":["../src/AbortError.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,UAAW,SAAQ,KAAK;IACjC,gBAAgB;IAChB;QACI,KAAK,CAAC,YAAY,CAAC,CAAC;IACxB,CAAC;CACJ"}

package/dist/chatWrappers/GeneralChatPromptWrapper.d.ts

@@ -0,0 +1,8 @@
+import { ChatPromptWrapper } from "../ChatPromptWrapper.js";
+export declare class GeneralChatPromptWrapper extends ChatPromptWrapper {
+    wrapPrompt(prompt: string, { systemPrompt, promptIndex }: {
+        systemPrompt: string;
+        promptIndex: number;
+    }): string;
+    getStopStrings(): string[];
+}

package/dist/chatWrappers/GeneralChatPromptWrapper.js

@@ -0,0 +1,11 @@
+import { ChatPromptWrapper } from "../ChatPromptWrapper.js";
+export class GeneralChatPromptWrapper extends ChatPromptWrapper {
+    wrapPrompt(prompt, { systemPrompt, promptIndex }) {
+        const conversationPrompt = "\n\n### Human:\n\n" + prompt + "\n\n### Assistant:\n\n";
+        return promptIndex === 0 ? systemPrompt + conversationPrompt : conversationPrompt;
+    }
+    getStopStrings() {
+        return ["### Human:", "Human:", "### Assistant:", "Assistant:", "<end>"];
+    }
+}
+//# sourceMappingURL=GeneralChatPromptWrapper.js.map

package/dist/chatWrappers/GeneralChatPromptWrapper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GeneralChatPromptWrapper.js","sourceRoot":"","sources":["../../src/chatWrappers/GeneralChatPromptWrapper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,iBAAiB,EAAC,MAAM,yBAAyB,CAAC;AAE1D,MAAM,OAAO,wBAAyB,SAAQ,iBAAiB;IAC3C,UAAU,CAAC,MAAc,EAAE,EAAC,YAAY,EAAE,WAAW,EAAgD;QACjH,MAAM,kBAAkB,GAAG,oBAAoB,GAAG,MAAM,GAAG,wBAAwB,CAAC;QAEpF,OAAO,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,kBAAkB,CAAC,CAAC,CAAC,kBAAkB,CAAC;IACtF,CAAC;IAEe,cAAc;QAC1B,OAAO,CAAC,YAAY,EAAE,QAAQ,EAAE,gBAAgB,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;IAC7E,CAAC;CACJ"}

package/dist/cli/commands/ChatCommand.d.ts

@@ -3,6 +3,7 @@ type ChatCommand = {
     model: string;
     systemInfo: boolean;
     systemPrompt: string;
+    wrapper: string;
 };
 export declare const ChatCommand: CommandModule<object, ChatCommand>;
 export {};

package/dist/cli/commands/ChatCommand.js

@@ -4,6 +4,7 @@ import chalk from "chalk";
 import withOra from "../../utils/withOra.js";
 import { defaultChatSystemPrompt } from "../../config.js";
 import { LlamaChatPromptWrapper } from "../../chatWrappers/LlamaChatPromptWrapper.js";
+import { GeneralChatPromptWrapper } from "../../chatWrappers/GeneralChatPromptWrapper.js";
 export const ChatCommand = {
     command: "chat",
     describe: "Chat with a LLama model",
@@ -28,11 +29,18 @@ export const ChatCommand = {
             description: "System prompt to use against the model. " +
                 "[default value: " + defaultChatSystemPrompt.split("\n").join(" ") + "]",
             group: "Optional:"
+        })
+            .option("wrapper", {
+            type: "string",
+            default: "general",
+            choices: ["general", "llama"],
+            description: "Chat wrapper to use",
+            group: "Optional:"
         });
     },
-    async handler({ model, systemInfo, systemPrompt }) {
+    async handler({ model, systemInfo, systemPrompt, wrapper }) {
         try {
-            await RunChat({ model, systemInfo, systemPrompt });
+            await RunChat({ model, systemInfo, systemPrompt, wrapper });
         }
         catch (err) {
             console.error(err);
@@ -40,17 +48,17 @@ export const ChatCommand = {
         }
     }
 };
-async function RunChat({ model: modelArg, systemInfo, systemPrompt }) {
-    const { LlamaChatSession } = await import("../../LlamaChatSession.js");
-    const { LlamaModel } = await import("../../LlamaModel.js");
+async function RunChat({ model: modelArg, systemInfo, systemPrompt, wrapper }) {
+    const { LlamaChatSession } = await import("../../llamaEvaluator/LlamaChatSession.js");
+    const { LlamaModel } = await import("../../llamaEvaluator/LlamaModel.js");
     const model = new LlamaModel({
         modelPath: modelArg
     });
     const session = new LlamaChatSession({
-        model,
+        context: model.createContext(),
         printLLamaSystemInfo: systemInfo,
         systemPrompt,
-        promptWrapper: new LlamaChatPromptWrapper()
+        promptWrapper: createChatWrapper(wrapper)
     });
     await withOra({
         loading: chalk.blue("Loading model"),
@@ -74,10 +82,19 @@ async function RunChat({ model: modelArg, systemInfo, systemPrompt }) {
         const [startColor, endColor] = chalk.blue("MIDDLE").split("MIDDLE");
         process.stdout.write(startColor);
         await session.prompt(input, (chunk) => {
-            process.stdout.write(model.decode(Uint32Array.from([chunk])));
+            process.stdout.write(session.context.decode(Uint32Array.from(chunk)));
         });
         process.stdout.write(endColor);
         console.log();
     }
 }
+function createChatWrapper(wrapper) {
+    switch (wrapper) {
+        case "general":
+            return new GeneralChatPromptWrapper();
+        case "llama":
+            return new LlamaChatPromptWrapper();
+    }
+    throw new Error("Unknown wrapper: " + wrapper);
+}
 //# sourceMappingURL=ChatCommand.js.map

package/dist/cli/commands/ChatCommand.js.map

@@ -1 +1 @@
-{"version":3,"file":"ChatCommand.js","sourceRoot":"","sources":["../../../src/cli/commands/ChatCommand.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,mBAAmB,CAAC;AAC9C,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,OAAO,MAAM,wBAAwB,CAAC;AAC7C,OAAO,EAAC,uBAAuB,EAAC,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAC,sBAAsB,EAAC,MAAM,8CAA8C,CAAC;AAQpF,MAAM,CAAC,MAAM,WAAW,GAAuC;IAC3D,OAAO,EAAE,MAAM;IACf,QAAQ,EAAE,yBAAyB;IACnC,OAAO,CAAC,KAAK;QACT,OAAO,KAAK;aACP,MAAM,CAAC,OAAO,EAAE;YACb,IAAI,EAAE,QAAQ;YACd,YAAY,EAAE,IAAI;YAClB,WAAW,EAAE,sCAAsC;YACnD,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,YAAY,EAAE;YAClB,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,KAAK;YACd,WAAW,EAAE,6BAA6B;YAC1C,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,cAAc,EAAE;YACpB,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,uBAAuB;YAChC,kBAAkB,EAAE,EAAE;YACtB,WAAW,EACP,0CAA0C;gBAC1C,kBAAkB,GAAG,uBAAuB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG;YAC5E,KAAK,EAAE,WAAW;SACrB,CAAC,CAAC;IACX,CAAC;IACD,KAAK,CAAC,OAAO,CAAC,EAAC,KAAK,EAAE,UAAU,EAAE,YAAY,EAAC;QAC3C,IAAI;YACA,MAAM,OAAO,CAAC,EAAC,KAAK,EAAE,UAAU,EAAE,YAAY,EAAC,CAAC,CAAC;SACpD;QAAC,OAAO,GAAG,EAAE;YACV,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACnB;IACL,CAAC;CACJ,CAAC;AAGF,KAAK,UAAU,OAAO,CAAC,EAAC,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAc;IAC3E,MAAM,EAAC,gBAAgB,EAAC,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;IACrE,MAAM,EAAC,UAAU,EAAC,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAC;IAEzD,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC;QACzB,SAAS,EAAE,QAAQ;KACtB,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,IAAI,gBAAgB,CAAC;QACjC,KAAK;QACL,oBAAoB,EAAE,UAAU;QAChC,YAAY;QACZ,aAAa,EAAE,IAAI,sBAAsB,EAAE;KAC9C,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC;QACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC;QACpC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC;QACnC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC;KAC3C,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,iDAAiD;IACjD,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;IAErD,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;QAChC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM,EAAE,OAAO,CAAC,MAAM;KACzB,CAAC,CAAC;IAEH,iDAAiD;IACjD,OAAO,IAAI,EAAE;QACT,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAEpD,IAAI,KAAK,KAAK,OAAO;YACjB,MAAM;QAEV,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAE3C,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAEpE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACjC,MAAM,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE;YAClC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAClE,CAAC,CAAC,CAAC;QACH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC/B,OAAO,CAAC,GAAG,EAAE,CAAC;KACjB;AACL,CAAC"}
+{"version":3,"file":"ChatCommand.js","sourceRoot":"","sources":["../../../src/cli/commands/ChatCommand.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,mBAAmB,CAAC;AAC9C,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,OAAO,MAAM,wBAAwB,CAAC;AAC7C,OAAO,EAAC,uBAAuB,EAAC,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAC,sBAAsB,EAAC,MAAM,8CAA8C,CAAC;AACpF,OAAO,EAAC,wBAAwB,EAAC,MAAM,gDAAgD,CAAC;AASxF,MAAM,CAAC,MAAM,WAAW,GAAuC;IAC3D,OAAO,EAAE,MAAM;IACf,QAAQ,EAAE,yBAAyB;IACnC,OAAO,CAAC,KAAK;QACT,OAAO,KAAK;aACP,MAAM,CAAC,OAAO,EAAE;YACb,IAAI,EAAE,QAAQ;YACd,YAAY,EAAE,IAAI;YAClB,WAAW,EAAE,sCAAsC;YACnD,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,YAAY,EAAE;YAClB,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,KAAK;YACd,WAAW,EAAE,6BAA6B;YAC1C,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,cAAc,EAAE;YACpB,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,uBAAuB;YAChC,kBAAkB,EAAE,EAAE;YACtB,WAAW,EACP,0CAA0C;gBAC1C,kBAAkB,GAAG,uBAAuB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG;YAC5E,KAAK,EAAE,WAAW;SACrB,CAAC;aACD,MAAM,CAAC,SAAS,EAAE;YACf,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,SAAS;YAClB,OAAO,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC;YAC7B,WAAW,EAAE,qBAAqB;YAClC,KAAK,EAAE,WAAW;SACrB,CAAC,CAAC;IACX,CAAC;IACD,KAAK,CAAC,OAAO,CAAC,EAAC,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,OAAO,EAAC;QACpD,IAAI;YACA,MAAM,OAAO,CAAC,EAAC,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,OAAO,EAAC,CAAC,CAAC;SAC7D;QAAC,OAAO,GAAG,EAAE;YACV,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACnB;IACL,CAAC;CACJ,CAAC;AAGF,KAAK,UAAU,OAAO,CAAC,EAAC,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,OAAO,EAAc;IACpF,MAAM,EAAC,gBAAgB,EAAC,GAAG,MAAM,MAAM,CAAC,0CAA0C,CAAC,CAAC;IACpF,MAAM,EAAC,UAAU,EAAC,GAAG,MAAM,MAAM,CAAC,oCAAoC,CAAC,CAAC;IAExE,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC;QACzB,SAAS,EAAE,QAAQ;KACtB,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,IAAI,gBAAgB,CAAC;QACjC,OAAO,EAAE,KAAK,CAAC,aAAa,EAAE;QAC9B,oBAAoB,EAAE,UAAU;QAChC,YAAY;QACZ,aAAa,EAAE,iBAAiB,CAAC,OAAO,CAAC;KAC5C,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC;QACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC;QACpC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC;QACnC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC;KAC3C,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,iDAAiD;IACjD,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;IAErD,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;QAChC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM,EAAE,OAAO,CAAC,MAAM;KACzB,CAAC,CAAC;IAEH,iDAAiD;IACjD,OAAO,IAAI,EAAE;QACT,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAEpD,IAAI,KAAK,KAAK,OAAO;YACjB,MAAM;QAEV,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAE3C,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAEpE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACjC,MAAM,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE;YAClC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;QACH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC/B,OAAO,CAAC,GAAG,EAAE,CAAC;KACjB;AACL,CAAC;AAED,SAAS,iBAAiB,CAAC,OAAe;IACtC,QAAQ,OAAO,EAAE;QACb,KAAK,SAAS;YACV,OAAO,IAAI,wBAAwB,EAAE,CAAC;QAC1C,KAAK,OAAO;YACR,OAAO,IAAI,sBAAsB,EAAE,CAAC;KAC3C;IACD,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,OAAO,CAAC,CAAC;AACnD,CAAC"}

package/dist/index.d.ts

@@ -1,7 +1,9 @@
-import { LlamaChatSession } from "./LlamaChatSession.js";
-import { LlamaModel } from "./LlamaModel.js";
+import { LlamaChatSession } from "./llamaEvaluator/LlamaChatSession.js";
+import { LlamaModel } from "./llamaEvaluator/LlamaModel.js";
 import { AbortError } from "./AbortError.js";
 import { ChatPromptWrapper } from "./ChatPromptWrapper.js";
 import { EmptyChatPromptWrapper } from "./chatWrappers/EmptyChatPromptWrapper.js";
 import { LlamaChatPromptWrapper } from "./chatWrappers/LlamaChatPromptWrapper.js";
-export { LlamaModel, LlamaChatSession, AbortError, ChatPromptWrapper, EmptyChatPromptWrapper, LlamaChatPromptWrapper };
+import { GeneralChatPromptWrapper } from "./chatWrappers/GeneralChatPromptWrapper.js";
+import { LlamaContext } from "./llamaEvaluator/LlamaContext.js";
+export { LlamaModel, LlamaContext, LlamaChatSession, AbortError, ChatPromptWrapper, EmptyChatPromptWrapper, LlamaChatPromptWrapper, GeneralChatPromptWrapper };

package/dist/index.js

@@ -1,8 +1,10 @@
-import { LlamaChatSession } from "./LlamaChatSession.js";
-import { LlamaModel } from "./LlamaModel.js";
+import { LlamaChatSession } from "./llamaEvaluator/LlamaChatSession.js";
+import { LlamaModel } from "./llamaEvaluator/LlamaModel.js";
 import { AbortError } from "./AbortError.js";
 import { ChatPromptWrapper } from "./ChatPromptWrapper.js";
 import { EmptyChatPromptWrapper } from "./chatWrappers/EmptyChatPromptWrapper.js";
 import { LlamaChatPromptWrapper } from "./chatWrappers/LlamaChatPromptWrapper.js";
-export { LlamaModel, LlamaChatSession, AbortError, ChatPromptWrapper, EmptyChatPromptWrapper, LlamaChatPromptWrapper };
+import { GeneralChatPromptWrapper } from "./chatWrappers/GeneralChatPromptWrapper.js";
+import { LlamaContext } from "./llamaEvaluator/LlamaContext.js";
+export { LlamaModel, LlamaContext, LlamaChatSession, AbortError, ChatPromptWrapper, EmptyChatPromptWrapper, LlamaChatPromptWrapper, GeneralChatPromptWrapper };
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,gBAAgB,EAAC,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAC,iBAAiB,EAAC,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAC,sBAAsB,EAAC,MAAM,0CAA0C,CAAC;AAChF,OAAO,EAAC,sBAAsB,EAAC,MAAM,0CAA0C,CAAC;AAEhF,OAAO,EACH,UAAU,EACV,gBAAgB,EAChB,UAAU,EACV,iBAAiB,EACjB,sBAAsB,EACtB,sBAAsB,EACzB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,gBAAgB,EAAC,MAAM,sCAAsC,CAAC;AACtE,OAAO,EAAC,UAAU,EAAC,MAAM,gCAAgC,CAAC;AAC1D,OAAO,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAC,iBAAiB,EAAC,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAC,sBAAsB,EAAC,MAAM,0CAA0C,CAAC;AAChF,OAAO,EAAC,sBAAsB,EAAC,MAAM,0CAA0C,CAAC;AAChF,OAAO,EAAC,wBAAwB,EAAC,MAAM,4CAA4C,CAAC;AACpF,OAAO,EAAC,YAAY,EAAC,MAAM,kCAAkC,CAAC;AAG9D,OAAO,EACH,UAAU,EACV,YAAY,EACZ,gBAAgB,EAChB,UAAU,EACV,iBAAiB,EACjB,sBAAsB,EACtB,sBAAsB,EACtB,wBAAwB,EAC3B,CAAC"}

package/dist/llamaEvaluator/LlamaBins.d.ts

@@ -0,0 +1,4 @@
+import { type LLAMAModel, type LLAMAContext } from "../utils/getBin.js";
+export declare const llamaCppNode: import("../utils/getBin.js").LlamaCppNodeModule;
+declare const LLAMAModel: LLAMAModel, LLAMAContext: LLAMAContext;
+export { LLAMAModel, LLAMAContext };

package/dist/llamaEvaluator/LlamaBins.js

@@ -0,0 +1,5 @@
+import { loadBin } from "../utils/getBin.js";
+export const llamaCppNode = await loadBin();
+const { LLAMAModel, LLAMAContext } = llamaCppNode;
+export { LLAMAModel, LLAMAContext };
+//# sourceMappingURL=LlamaBins.js.map

package/dist/llamaEvaluator/LlamaBins.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LlamaBins.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaBins.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,OAAO,EAAqC,MAAM,oBAAoB,CAAC;AAE/E,MAAM,CAAC,MAAM,YAAY,GAAG,MAAM,OAAO,EAAE,CAAC;AAC5C,MAAM,EAAC,UAAU,EAAE,YAAY,EAAC,GAAG,YAAY,CAAC;AAEhD,OAAO,EAAC,UAAU,EAAE,YAAY,EAAC,CAAC"}

package/dist/{LlamaChatSession.d.ts → llamaEvaluator/LlamaChatSession.d.ts}

@@ -1,24 +1,25 @@
 /// <reference types="node" />
-import { LlamaModel } from "./LlamaModel.js";
-import { ChatPromptWrapper } from "./ChatPromptWrapper.js";
+import { ChatPromptWrapper } from "../ChatPromptWrapper.js";
+import { LlamaContext } from "./LlamaContext.js";
 export declare class LlamaChatSession {
-    private readonly _model;
     private readonly _systemPrompt;
     private readonly _printLLamaSystemInfo;
     private readonly _promptWrapper;
     private _promptIndex;
     private _initialized;
-    constructor({ model, printLLamaSystemInfo, promptWrapper, systemPrompt }: {
-        model: LlamaModel;
+    private readonly _ctx;
+    constructor({ context, printLLamaSystemInfo, promptWrapper, systemPrompt }: {
+        context: LlamaContext;
         printLLamaSystemInfo?: boolean;
         promptWrapper?: ChatPromptWrapper;
         systemPrompt?: string;
     });
     get initialized(): boolean;
-    get model(): LlamaModel;
+    get context(): LlamaContext;
     init(): Promise<void>;
-    prompt(prompt: string, onToken?: (token: number) => void, { signal }?: {
+    prompt(prompt: string, onToken?: (tokens: number[]) => void, { signal }?: {
         signal?: AbortSignal;
     }): Promise<string>;
     private _evalTokens;
+    private _checkStopString;
 }

package/dist/llamaEvaluator/LlamaChatSession.js

@@ -0,0 +1,97 @@
+import { defaultChatSystemPrompt } from "../config.js";
+import { withLock } from "../utils/withLock.js";
+import { AbortError } from "../AbortError.js";
+import { GeneralChatPromptWrapper } from "../chatWrappers/GeneralChatPromptWrapper.js";
+import { LlamaModel } from "./LlamaModel.js";
+const UNKNOWN_UNICODE_CHAR = "�";
+export class LlamaChatSession {
+    _systemPrompt;
+    _printLLamaSystemInfo;
+    _promptWrapper;
+    _promptIndex = 0;
+    _initialized = false;
+    _ctx;
+    constructor({ context, printLLamaSystemInfo = false, promptWrapper = new GeneralChatPromptWrapper(), systemPrompt = defaultChatSystemPrompt }) {
+        this._ctx = context;
+        this._printLLamaSystemInfo = printLLamaSystemInfo;
+        this._promptWrapper = promptWrapper;
+        this._systemPrompt = systemPrompt;
+    }
+    get initialized() {
+        return this._initialized;
+    }
+    get context() {
+        return this._ctx;
+    }
+    async init() {
+        await withLock(this, "init", async () => {
+            if (this._initialized)
+                return;
+            if (this._printLLamaSystemInfo)
+                console.log("Llama system info", LlamaModel.systemInfo);
+            this._initialized = true;
+        });
+    }
+    async prompt(prompt, onToken, { signal } = {}) {
+        if (!this.initialized)
+            await this.init();
+        return await withLock(this, "prompt", async () => {
+            const promptText = this._promptWrapper.wrapPrompt(prompt, { systemPrompt: this._systemPrompt, promptIndex: this._promptIndex });
+            this._promptIndex++;
+            return await this._evalTokens(this._ctx.encode(promptText), onToken, { signal });
+        });
+    }
+    async _evalTokens(tokens, onToken, { signal } = {}) {
+        const decodeTokens = (tokens) => this._ctx.decode(Uint32Array.from(tokens));
+        const stopStrings = this._promptWrapper.getStopStrings();
+        const stopStringIndexes = Array(stopStrings.length).fill(0);
+        const skippedChunksQueue = [];
+        const res = [];
+        for await (const chunk of this._ctx.evaluate(tokens)) {
+            if (signal?.aborted)
+                throw new AbortError();
+            const tokenStr = decodeTokens([chunk]);
+            const { shouldReturn, skipTokenEvent } = this._checkStopString(tokenStr, stopStringIndexes);
+            if (shouldReturn)
+                return decodeTokens(res);
+            // if the token is unknown, it means it's not complete character
+            if (tokenStr === UNKNOWN_UNICODE_CHAR || skipTokenEvent) {
+                skippedChunksQueue.push(chunk);
+                continue;
+            }
+            if (skippedChunksQueue.length > 0) {
+                res.push(...skippedChunksQueue);
+                onToken?.(skippedChunksQueue);
+                skippedChunksQueue.length = 0;
+            }
+            res.push(chunk);
+            onToken?.([chunk]);
+        }
+        return decodeTokens(res);
+    }
+    _checkStopString(tokenStr, stopStringIndexes) {
+        const stopStrings = this._promptWrapper.getStopStrings();
+        let skipTokenEvent = false;
+        for (let stopStringIndex = 0; stopStringIndex < stopStrings.length; stopStringIndex++) {
+            const stopString = stopStrings[stopStringIndex];
+            let localShouldSkipTokenEvent = false;
+            for (let i = 0; i < tokenStr.length && stopStringIndexes[stopStringIndex] !== stopString.length; i++) {
+                if (tokenStr[i] === stopString[stopStringIndexes[stopStringIndex]]) {
+                    stopStringIndexes[stopStringIndex]++;
+                    localShouldSkipTokenEvent = true;
+                }
+                else {
+                    stopStringIndexes[stopStringIndex] = 0;
+                    localShouldSkipTokenEvent = false;
+                    break;
+                }
+            }
+            if (stopStringIndexes[stopStringIndex] === stopString.length) {
+                return { shouldReturn: true };
+            }
+            skipTokenEvent ||= localShouldSkipTokenEvent;
+        }
+        return { skipTokenEvent };
+    }
+}
+//# sourceMappingURL=LlamaChatSession.js.map

package/dist/llamaEvaluator/LlamaChatSession.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LlamaChatSession.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaChatSession.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,uBAAuB,EAAC,MAAM,cAAc,CAAC;AACrD,OAAO,EAAC,QAAQ,EAAC,MAAM,sBAAsB,CAAC;AAE9C,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAC,wBAAwB,EAAC,MAAM,6CAA6C,CAAC;AACrF,OAAO,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAG3C,MAAM,oBAAoB,GAAG,GAAG,CAAC;AAEjC,MAAM,OAAO,gBAAgB;IACR,aAAa,CAAS;IACtB,qBAAqB,CAAU;IAC/B,cAAc,CAAoB;IAC3C,YAAY,GAAW,CAAC,CAAC;IACzB,YAAY,GAAY,KAAK,CAAC;IACrB,IAAI,CAAe;IAEpC,YAAmB,EACf,OAAO,EACP,oBAAoB,GAAG,KAAK,EAC5B,aAAa,GAAG,IAAI,wBAAwB,EAAE,EAC9C,YAAY,GAAG,uBAAuB,EAMzC;QACG,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC;QACpB,IAAI,CAAC,qBAAqB,GAAG,oBAAoB,CAAC;QAClD,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QAEpC,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;IACtC,CAAC;IAED,IAAW,WAAW;QAClB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED,IAAW,OAAO;QACd,OAAO,IAAI,CAAC,IAAI,CAAC;IACrB,CAAC;IAEM,KAAK,CAAC,IAAI;QACb,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE;YACpC,IAAI,IAAI,CAAC,YAAY;gBACjB,OAAO;YAEX,IAAI,IAAI,CAAC,qBAAqB;gBAC1B,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;YAE5D,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC7B,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,KAAK,CAAC,MAAM,CAAC,MAAc,EAAE,OAAoC,EAAE,EAAC,MAAM,KAA8B,EAAE;QAC7G,IAAI,CAAC,IAAI,CAAC,WAAW;YACjB,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAEtB,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,IAAI,EAAE;YAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,MAAM,EAAE,EAAC,YAAY,EAAE,IAAI,CAAC,aAAa,EAAE,WAAW,EAAE,IAAI,CAAC,YAAY,EAAC,CAAC,CAAC;YAC9H,IAAI,CAAC,YAAY,EAAE,CAAC;YAEpB,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,OAAO,EAAE,EAAC,MAAM,EAAC,CAAC,CAAC;QACnF,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,MAAmB,EAAE,OAAoC,EAAE,EAAC,MAAM,KAA8B,EAAE;QACxH,MAAM,YAAY,GAAG,CAAC,MAAgB,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QAEtF,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,CAAC;QACzD,MAAM,iBAAiB,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5D,MAAM,kBAAkB,GAAa,EAAE,CAAC;QACxC,MAAM,GAAG,GAAa,EAAE,CAAC;QAGzB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;YAClD,IAAI,MAAM,EAAE,OAAO;gBACf,MAAM,IAAI,UAAU,EAAE,CAAC;YAE3B,MAAM,QAAQ,GAAG,YAAY,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;YACvC,MAAM,EAAC,YAAY,EAAE,cAAc,EAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC;YAE1F,IAAI,YAAY;gBACZ,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC;YAE7B,gEAAgE;YAChE,IAAI,QAAQ,KAAK,oBAAoB,IAAI,cAAc,EAAE;gBACrD,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC/B,SAAS;aACZ;YAED,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE;gBAC/B,GAAG,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,CAAC;gBAChC,OAAO,EAAE,CAAC,kBAAkB,CAAC,CAAC;gBAC9B,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC;aACjC;YAED,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChB,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;SACtB;QAED,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAEO,gBAAgB,CAAC,QAAgB,EAAE,iBAA2B;QAClE,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,CAAC;QACzD,IAAI,cAAc,GAAG,KAAK,CAAC;QAE3B,KAAK,IAAI,eAAe,GAAG,CAAC,EAAE,eAAe,GAAG,WAAW,CAAC,MAAM,EAAE,eAAe,EAAE,EAAE;YACnF,MAAM,UAAU,GAAG,WAAW,CAAC,eAAe,CAAC,CAAC;YAEhD,IAAI,yBAAyB,GAAG,KAAK,CAAC;YACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,IAAI,iBAAiB,CAAC,eAAe,CAAC,KAAK,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBAClG,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC,EAAE;oBAChE,iBAAiB,CAAC,eAAe,CAAC,EAAE,CAAC;oBACrC,yBAAyB,GAAG,IAAI,CAAC;iBACpC;qBAAM;oBACH,iBAAiB,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;oBACvC,yBAAyB,GAAG,KAAK,CAAC;oBAClC,MAAM;iBACT;aACJ;YAED,IAAI,iBAAiB,CAAC,eAAe,CAAC,KAAK,UAAU,CAAC,MAAM,EAAE;gBAC1D,OAAO,EAAC,YAAY,EAAE,IAAI,EAAC,CAAC;aAC/B;YAED,cAAc,KAAK,yBAAyB,CAAC;SAChD;QAED,OAAO,EAAC,cAAc,EAAC,CAAC;IAC5B,CAAC;CACJ"}

package/dist/{LlamaModel.d.ts → llamaEvaluator/LlamaContext.d.ts}

@@ -1,12 +1,6 @@
-export declare class LlamaModel {
-    private readonly _model;
+export declare class LlamaContext {
     private readonly _ctx;
     private _prependBos;
-    constructor({ modelPath, prependBos }: {
-        modelPath: string;
-        prependBos?: boolean;
-    });
-    get systemInfo(): string;
     encode(text: string): Uint32Array;
     decode(tokens: Uint32Array): string;
     evaluate(tokens: Uint32Array, getRestrictions?: () => Uint32Array): AsyncGenerator<number, void, unknown>;

package/dist/{LlamaModel.js → llamaEvaluator/LlamaContext.js}

@@ -1,18 +1,12 @@
-import { loadBin } from "./utils/getBin.js";
-const llamaCppNode = await loadBin();
-const { LLAMAModel, LLAMAContext } = llamaCppNode;
-export class LlamaModel {
-    _model;
+import { llamaCppNode } from "./LlamaBins.js";
+export class LlamaContext {
     _ctx;
     _prependBos;
-    constructor({ modelPath, prependBos = true }) {
-        this._model = new LLAMAModel(modelPath);
-        this._ctx = new LLAMAContext(this._model);
+    /** @internal */
+    constructor({ ctx, prependBos }) {
+        this._ctx = ctx;
         this._prependBos = prependBos;
     }
-    get systemInfo() {
-        return llamaCppNode.systemInfo();
-    }
     encode(text) {
         return this._ctx.encode(text);
     }
@@ -22,9 +16,9 @@ export class LlamaModel {
     async *evaluate(tokens, getRestrictions) {
         let evalTokens = tokens;
         if (this._prependBos) {
-            const tokensArray = Array.from(tokens);
-            tokensArray.unshift(llamaCppNode.tokenBos());
-            evalTokens = Uint32Array.from(tokensArray);
+            const tokenArray = Array.from(tokens);
+            tokenArray.unshift(llamaCppNode.tokenBos());
+            evalTokens = Uint32Array.from(tokenArray);
             this._prependBos = false;
         }
         // eslint-disable-next-line no-constant-condition
@@ -40,4 +34,4 @@ export class LlamaModel {
         }
     }
 }
-//# sourceMappingURL=LlamaModel.js.map
+//# sourceMappingURL=LlamaContext.js.map

package/dist/llamaEvaluator/LlamaContext.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LlamaContext.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaContext.ts"],"names":[],"mappings":"AAAA,OAAO,EAAe,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAG1D,MAAM,OAAO,YAAY;IACJ,IAAI,CAAe;IAC5B,WAAW,CAAU;IAE7B,gBAAgB;IAChB,YAAoB,EAAC,GAAG,EAAE,UAAU,EAAoC;QACpE,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC;QAChB,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,IAAY;QACtB,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,MAAmB;QAC7B,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAEM,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAmB,EAAE,eAAmC;QAC3E,IAAI,UAAU,GAAG,MAAM,CAAC;QAExB,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtC,UAAU,CAAC,OAAO,CAAC,YAAY,CAAC,QAAQ,EAAE,CAAC,CAAC;YAE5C,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC1C,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;SAC5B;QAED,iDAAiD;QACjD,OAAO,IAAI,EAAE;YACT,kCAAkC;YAClC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;YAExE,mCAAmC;YACnC,IAAI,SAAS,KAAK,YAAY,CAAC,QAAQ,EAAE;gBACrC,MAAM;YAEV,MAAM,SAAS,CAAC;YAEhB,mCAAmC;YACnC,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;SAC9C;IACL,CAAC;CAEJ"}

package/dist/llamaEvaluator/LlamaModel.d.ts

@@ -0,0 +1,11 @@
+import { LlamaContext } from "./LlamaContext.js";
+export declare class LlamaModel {
+    private readonly _model;
+    private readonly _prependBos;
+    constructor({ modelPath, prependBos }: {
+        modelPath: string;
+        prependBos?: boolean;
+    });
+    createContext(): LlamaContext;
+    static get systemInfo(): string;
+}

package/dist/llamaEvaluator/LlamaModel.js

@@ -0,0 +1,20 @@
+import { LlamaContext } from "./LlamaContext.js";
+import { LLAMAContext, llamaCppNode, LLAMAModel } from "./LlamaBins.js";
+export class LlamaModel {
+    _model;
+    _prependBos;
+    constructor({ modelPath, prependBos = true }) {
+        this._model = new LLAMAModel(modelPath);
+        this._prependBos = prependBos;
+    }
+    createContext() {
+        return new LlamaContext({
+            ctx: new LLAMAContext(this._model),
+            prependBos: this._prependBos
+        });
+    }
+    static get systemInfo() {
+        return llamaCppNode.systemInfo();
+    }
+}
+//# sourceMappingURL=LlamaModel.js.map

package/dist/llamaEvaluator/LlamaModel.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LlamaModel.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaModel.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAC,YAAY,EAAE,YAAY,EAAE,UAAU,EAAC,MAAM,gBAAgB,CAAC;AAGtE,MAAM,OAAO,UAAU;IACF,MAAM,CAAa;IACnB,WAAW,CAAU;IAEtC,YAAmB,EAAC,SAAS,EAAE,UAAU,GAAG,IAAI,EAA8C;QAC1F,IAAI,CAAC,MAAM,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC;QACxC,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAClC,CAAC;IAEM,aAAa;QAChB,OAAO,IAAI,YAAY,CAAC;YACpB,GAAG,EAAE,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC;YAClC,UAAU,EAAE,IAAI,CAAC,WAAW;SAC/B,CAAC,CAAC;IACP,CAAC;IAEM,MAAM,KAAK,UAAU;QACxB,OAAO,YAAY,CAAC,UAAU,EAAE,CAAC;IACrC,CAAC;CACJ"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "node-llama-cpp",
-  "version": "1.1.1",
+  "version": "1.2.1",
   "description": "node.js bindings for llama.cpp",
   "main": "dist/index.js",
   "type": "module",

package/dist/LlamaChatSession.js

@@ -1,86 +0,0 @@
-import { defaultChatSystemPrompt } from "./config.js";
-import { withLock } from "./utils/withLock.js";
-import { LlamaChatPromptWrapper } from "./chatWrappers/LlamaChatPromptWrapper.js";
-import { AbortError } from "./AbortError.js";
-export class LlamaChatSession {
-    _model;
-    _systemPrompt;
-    _printLLamaSystemInfo;
-    _promptWrapper;
-    _promptIndex = 0;
-    _initialized = false;
-    constructor({ model, printLLamaSystemInfo = false, promptWrapper = new LlamaChatPromptWrapper(), systemPrompt = defaultChatSystemPrompt }) {
-        this._model = model;
-        this._printLLamaSystemInfo = printLLamaSystemInfo;
-        this._promptWrapper = promptWrapper;
-        this._systemPrompt = systemPrompt;
-    }
-    get initialized() {
-        return this._initialized;
-    }
-    get model() {
-        return this._model;
-    }
-    async init() {
-        await withLock(this, "init", async () => {
-            if (this._initialized)
-                return;
-            if (this._printLLamaSystemInfo)
-                console.log("Llama system info", this._model.systemInfo);
-            this._initialized = true;
-        });
-    }
-    async prompt(prompt, onToken, { signal } = {}) {
-        if (!this.initialized)
-            await this.init();
-        return await withLock(this, "prompt", async () => {
-            const promptText = this._promptWrapper.wrapPrompt(prompt, { systemPrompt: this._systemPrompt, promptIndex: this._promptIndex });
-            this._promptIndex++;
-            return await this._evalTokens(this._model.encode(promptText), onToken, { signal });
-        });
-    }
-    async _evalTokens(tokens, onToken, { signal } = {}) {
-        const stopStrings = this._promptWrapper.getStopStrings();
-        const stopStringIndexes = Array(stopStrings.length).fill(0);
-        const skippedChunksQueue = [];
-        let res = "";
-        for await (const chunk of this._model.evaluate(tokens)) {
-            if (signal?.aborted)
-                throw new AbortError();
-            const tokenStr = this._model.decode(Uint32Array.from([chunk]));
-            let skipTokenEvent = false;
-            for (let stopStringIndex = 0; stopStringIndex < stopStrings.length; stopStringIndex++) {
-                const stopString = stopStrings[stopStringIndex];
-                let localShouldSkipTokenEvent = false;
-                for (let i = 0; i < tokenStr.length && stopStringIndexes[stopStringIndex] !== stopString.length; i++) {
-                    if (tokenStr[i] === stopString[stopStringIndexes[stopStringIndex]]) {
-                        stopStringIndexes[stopStringIndex]++;
-                        localShouldSkipTokenEvent = true;
-                    }
-                    else {
-                        stopStringIndexes[stopStringIndex] = 0;
-                        localShouldSkipTokenEvent = false;
-                        break;
-                    }
-                }
-                if (stopStringIndexes[stopStringIndex] === stopString.length) {
-                    return res;
-                }
-                skipTokenEvent ||= localShouldSkipTokenEvent;
-            }
-            if (skipTokenEvent) {
-                skippedChunksQueue.push(chunk);
-                continue;
-            }
-            while (skippedChunksQueue.length > 0) {
-                const token = skippedChunksQueue.shift();
-                res += this._model.decode(Uint32Array.from([token]));
-                onToken?.(token);
-            }
-            res += tokenStr;
-            onToken?.(chunk);
-        }
-        return res;
-    }
-}
-//# sourceMappingURL=LlamaChatSession.js.map

package/dist/LlamaChatSession.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"LlamaChatSession.js","sourceRoot":"","sources":["../src/LlamaChatSession.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,uBAAuB,EAAC,MAAM,aAAa,CAAC;AACpD,OAAO,EAAC,QAAQ,EAAC,MAAM,qBAAqB,CAAC;AAG7C,OAAO,EAAC,sBAAsB,EAAC,MAAM,0CAA0C,CAAC;AAChF,OAAO,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAG3C,MAAM,OAAO,gBAAgB;IACR,MAAM,CAAa;IACnB,aAAa,CAAS;IACtB,qBAAqB,CAAU;IAC/B,cAAc,CAAoB;IAC3C,YAAY,GAAW,CAAC,CAAC;IACzB,YAAY,GAAY,KAAK,CAAC;IAEtC,YAAmB,EACf,KAAK,EACL,oBAAoB,GAAG,KAAK,EAC5B,aAAa,GAAG,IAAI,sBAAsB,EAAE,EAC5C,YAAY,GAAG,uBAAuB,EAMzC;QACG,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,qBAAqB,GAAG,oBAAoB,CAAC;QAClD,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QAEpC,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;IACtC,CAAC;IAED,IAAW,WAAW;QAClB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED,IAAW,KAAK;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAEM,KAAK,CAAC,IAAI;QACb,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE;YACpC,IAAI,IAAI,CAAC,YAAY;gBACjB,OAAO;YAEX,IAAI,IAAI,CAAC,qBAAqB;gBAC1B,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YAE7D,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC7B,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,KAAK,CAAC,MAAM,CAAC,MAAc,EAAE,OAAiC,EAAE,EAAC,MAAM,KAA4B,EAAE;QACxG,IAAI,CAAC,IAAI,CAAC,WAAW;YACjB,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAEtB,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,IAAI,EAAE;YAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,MAAM,EAAE,EAAC,YAAY,EAAE,IAAI,CAAC,aAAa,EAAE,WAAW,EAAE,IAAI,CAAC,YAAY,EAAC,CAAC,CAAC;YAC9H,IAAI,CAAC,YAAY,EAAE,CAAC;YAEpB,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,OAAO,EAAE,EAAC,MAAM,EAAC,CAAC,CAAC;QACrF,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,MAAmB,EAAE,OAAiC,EAAE,EAAC,MAAM,KAA4B,EAAE;QACnH,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,CAAC;QACzD,MAAM,iBAAiB,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5D,MAAM,kBAAkB,GAAa,EAAE,CAAC;QACxC,IAAI,GAAG,GAAG,EAAE,CAAC;QAEb,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;YACpD,IAAI,MAAM,EAAE,OAAO;gBACf,MAAM,IAAI,UAAU,EAAE,CAAC;YAE3B,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC/D,IAAI,cAAc,GAAG,KAAK,CAAC;YAE3B,KAAK,IAAI,eAAe,GAAG,CAAC,EAAE,eAAe,GAAG,WAAW,CAAC,MAAM,EAAE,eAAe,EAAE,EAAE;gBACnF,MAAM,UAAU,GAAG,WAAW,CAAC,eAAe,CAAC,CAAC;gBAEhD,IAAI,yBAAyB,GAAG,KAAK,CAAC;gBACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,IAAI,iBAAiB,CAAC,eAAe,CAAC,KAAK,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBAClG,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC,EAAE;wBAChE,iBAAiB,CAAC,eAAe,CAAC,EAAE,CAAC;wBACrC,yBAAyB,GAAG,IAAI,CAAC;qBACpC;yBAAM;wBACH,iBAAiB,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;wBACvC,yBAAyB,GAAG,KAAK,CAAC;wBAClC,MAAM;qBACT;iBACJ;gBAED,IAAI,iBAAiB,CAAC,eAAe,CAAC,KAAK,UAAU,CAAC,MAAM,EAAE;oBAC1D,OAAO,GAAG,CAAC;iBACd;gBAED,cAAc,KAAK,yBAAyB,CAAC;aAChD;YAED,IAAI,cAAc,EAAE;gBAChB,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC/B,SAAS;aACZ;YAED,OAAO,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE;gBAClC,MAAM,KAAK,GAAG,kBAAkB,CAAC,KAAK,EAAG,CAAC;gBAC1C,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACrD,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;aACpB;YAED,GAAG,IAAI,QAAQ,CAAC;YAChB,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;SACpB;QAED,OAAO,GAAG,CAAC;IACf,CAAC;CACJ"}

package/dist/LlamaModel.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"LlamaModel.js","sourceRoot":"","sources":["../src/LlamaModel.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,OAAO,EAAqC,MAAM,mBAAmB,CAAC;AAE9E,MAAM,YAAY,GAAG,MAAM,OAAO,EAAE,CAAC;AACrC,MAAM,EAAC,UAAU,EAAE,YAAY,EAAC,GAAG,YAAY,CAAC;AAEhD,MAAM,OAAO,UAAU;IACF,MAAM,CAAa;IACnB,IAAI,CAAe;IAC5B,WAAW,CAAU;IAE7B,YAAmB,EACf,SAAS,EAAE,UAAU,GAAG,IAAI,EAG/B;QACG,IAAI,CAAC,MAAM,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAClC,CAAC;IAED,IAAW,UAAU;QACjB,OAAO,YAAY,CAAC,UAAU,EAAE,CAAC;IACrC,CAAC;IAEM,MAAM,CAAC,IAAY;QACtB,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,MAAmB;QAC7B,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAEM,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAmB,EAAE,eAAmC;QAC3E,IAAI,UAAU,GAAG,MAAM,CAAC;QAExB,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvC,WAAW,CAAC,OAAO,CAAC,YAAY,CAAC,QAAQ,EAAE,CAAC,CAAC;YAE7C,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC3C,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;SAC5B;QAED,iDAAiD;QACjD,OAAO,IAAI,EAAE;YACT,kCAAkC;YAClC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;YAExE,mCAAmC;YACnC,IAAI,SAAS,KAAK,YAAY,CAAC,QAAQ,EAAE;gBACrC,MAAM;YAEV,MAAM,SAAS,CAAC;YAEhB,mCAAmC;YACnC,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;SAC9C;IACL,CAAC;CACJ"}