node-llama-cpp 0.0.1 → 1.1.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Gilad S.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,210 @@
+# Node Llama.cpp
+Node.js bindings for llama.cpp.
+
+Pre-built bindings are provided with a fallback to building from source with `node-gyp`.
+
+[![Build](https://github.com/withcatai/node-llama-cpp/actions/workflows/build.yml/badge.svg)](https://github.com/withcatai/node-llama-cpp/actions/workflows/build.yml)
+[![Version](https://badgen.net/npm/v/node-llama-cpp)](https://www.npmjs.com/package/node-llama-cpp)
+
+
+## Installation
+```bash
+npm install --save node-llama-cpp
+```
+
+This package comes with pre-built binaries for macOS, Linux and Windows.
+
+If binaries are not available for your platform, it'll fallback to download the latest version of `llama.cpp` and build it from source with `node-gyp`.
+To disable this behavior set the environment variable `NODE_LLAMA_CPP_SKIP_DOWNLOAD` to `true`.
+
+## Documentation
+### Usage
+#### As a chatbot
+```typescript
+import {fileURLToPath} from "url";
+import path from "path";
+import {LlamaModel, LlamaChatSession} from "node-llama-cpp";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const model = new LlamaModel({
+    modelPath: path.join(__dirname, "models", "vicuna-13b-v1.5-16k.ggmlv3.q5_1.bin")
+})
+const session = new LlamaChatSession({model});
+
+
+const q1 = "Hi there, how are you?";
+console.log("User: " + q1);
+
+const a1 = await session.prompt(q1);
+console.log("AI: " + a1);
+
+
+const q2 = "Summerize what you said";
+console.log("User: " + q2);
+
+const a2 = await session.prompt(q2);
+console.log("AI: " + a2);
+```
+
+##### Custom prompt handling against the model
+```typescript
+import {fileURLToPath} from "url";
+import path from "path";
+import {LlamaModel, LlamaChatSession, ChatPromptWrapper} from "node-llama-cpp";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+export class MyCustomChatPromptWrapper extends ChatPromptWrapper {
+    public override wrapPrompt(prompt: string, {systemPrompt, promptIndex}: {systemPrompt: string, promptIndex: number}) {
+        if (promptIndex === 0) {
+            return "SYSTEM: " + systemPrompt + "\nUSER: " + prompt + "\nASSISTANT:";
+        } else {
+            return "USER: " + prompt + "\nASSISTANT:";
+        }
+    }
+
+    public override getStopStrings(): string[] {
+        return ["USER:"];
+    }
+}
+
+const model = new LlamaModel({
+    modelPath: path.join(__dirname, "models", "vicuna-13b-v1.5-16k.ggmlv3.q5_1.bin"),
+    promptWrapper: new MyCustomChatPromptWrapper() // by default, LlamaChatPromptWrapper is used
+})
+const session = new LlamaChatSession({model});
+
+
+const q1 = "Hi there, how are you?";
+console.log("User: " + q1);
+
+const a1 = await session.prompt(q1);
+console.log("AI: " + a1);
+
+
+const q2 = "Summerize what you said";
+console.log("User: " + q2);
+
+const a2 = await session.prompt(q2);
+console.log("AI: " + a2);
+```
+
+#### Raw
+```typescript
+import {fileURLToPath} from "url";
+import path from "path";
+import {LlamaModel, LlamaChatSession} from "node-llama-cpp";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const model = new LlamaChatSession({
+    modelPath: path.join(__dirname, "models", "vicuna-13b-v1.5-16k.ggmlv3.q5_1.bin")
+});
+
+const q1 = "Hi there, how are you?";
+console.log("AI: " + q1);
+
+const tokens = model.encode(q1);
+const res: number[] = [];
+for await (const chunk of model.evaluate(tokens)) {
+    res.push(chunk);
+    
+    // it's important to not concatinate the results as strings,
+    // as doing so will break some characters (like some emojis) that are made of multiple tokens.
+    // by using an array of tokens, we can decode them correctly together.
+    const resString: string = model.decode(Uint32Array.from(res));
+    
+    const lastPart = resString.split("ASSISTANT:").reverse()[0];
+    if (lastPart.includes("USER:"))
+        break;
+}
+
+const a1 = model.decode(Uint32Array.from(res)).split("USER:")[0];
+console.log("AI: " + a1);
+```
+
+### CLI
+```txt
+Usage: node-llama-cpp <command> [options]
+
+Commands:
+  node-llama-cpp download           Download a release of llama.cpp and compile it
+  node-llama-cpp build              Compile the currently downloaded llama.cpp
+  node-llama-cpp clear [type]       Clear files created by llama-cli
+  node-llama-cpp chat               Chat with a LLama model
+
+Options:
+  -h, --help     Show help                                                                 [boolean]
+  -v, --version  Show version number                                                       [boolean]                                                [boolean]
+```
+
+#### `download` command
+```txt
+node-llama-cpp download
+
+Download a release of llama.cpp and compile it
+
+Options:
+  -h, --help        Show help                                                              [boolean]
+      --repo        The GitHub repository to download a release of llama.cpp from. Can also be set v
+                    ia the NODE_LLAMA_CPP_REPO environment variable
+                                                           [string] [default: "ggerganov/llama.cpp"]
+      --release     The tag of the llama.cpp release to download. Can also be set via the NODE_LLAMA
+                    _CPP_REPO_RELEASE environment variable              [string] [default: "latest"]
+      --arch        The architecture to compile llama.cpp for                               [string]
+      --nodeTarget  The Node.js version to compile llama.cpp for. Example: v18.0.0          [string]
+  -v, --version     Show version number                                                    [boolean]
+```
+
+#### `build` command
+```txt
+node-llama-cpp build
+
+Compile the currently downloaded llama.cpp
+
+Options:
+  -h, --help        Show help                                                              [boolean]
+      --arch        The architecture to compile llama.cpp for                               [string]
+      --nodeTarget  The Node.js version to compile llama.cpp for. Example: v18.0.0          [string]
+  -v, --version     Show version number                                                    [boolean]
+```
+
+#### `clear` command
+```txt
+node-llama-cpp clear [type]
+
+Clear files created by llama-cli
+
+Options:
+  -h, --help     Show help                                                                 [boolean]
+      --type     Files to clear        [string] [choices: "source", "build", "all"] [default: "all"]
+  -v, --version  Show version number                                                       [boolean]
+```
+
+#### `chat` command
+```txt
+node-llama-cpp chat
+
+Chat with a LLama model
+
+Required:
+      --model  LLama model file to use for the chat                              [string] [required]
+
+Optional:
+      --systemInfo    Print llama.cpp system info                         [boolean] [default: false]
+      --systemPrompt  System prompt to use against the model. [default value: You are a helpful, res
+                      pectful and honest assistant. Always answer as helpfully as possible. If a que
+                      stion does not make any sense, or is not factually coherent, explain why inste
+                      ad of answering something not correct. If you don't know the answer to a quest
+                      ion, please don't share false information.]
+  [string] [default: "You are a helpful, respectful and honest assistant. Always answer as helpfully
+                                                                                        as possible.
+                If a question does not make any sense, or is not factually coherent, explain why ins
+   tead of answering something not correct. If you don't know the answer to a question, please don't
+                                                                          share false information."]
+
+Options:
+  -h, --help     Show help                                                                 [boolean]
+  -v, --version  Show version number                                                       [boolean]
+```

package/dist/AbortError.d.ts

@@ -0,0 +1,3 @@
+export declare class AbortError extends Error {
+    constructor();
+}

package/dist/AbortError.js

@@ -0,0 +1,6 @@
+export class AbortError extends Error {
+    constructor() {
+        super("AbortError");
+    }
+}
+//# sourceMappingURL=AbortError.js.map

package/dist/AbortError.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AbortError.js","sourceRoot":"","sources":["../src/AbortError.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,UAAW,SAAQ,KAAK;IACjC;QACI,KAAK,CAAC,YAAY,CAAC,CAAC;IACxB,CAAC;CACJ"}

package/dist/ChatPromptWrapper.d.ts

@@ -0,0 +1,7 @@
+export declare abstract class ChatPromptWrapper {
+    wrapPrompt(prompt: string, { systemPrompt, promptIndex }: {
+        systemPrompt: string;
+        promptIndex: number;
+    }): string;
+    getStopStrings(): string[];
+}

package/dist/ChatPromptWrapper.js

@@ -0,0 +1,14 @@
+export class ChatPromptWrapper {
+    wrapPrompt(prompt, { systemPrompt, promptIndex }) {
+        if (promptIndex === 0) {
+            return systemPrompt + "\n" + prompt;
+        }
+        else {
+            return prompt;
+        }
+    }
+    getStopStrings() {
+        return [];
+    }
+}
+//# sourceMappingURL=ChatPromptWrapper.js.map

package/dist/ChatPromptWrapper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ChatPromptWrapper.js","sourceRoot":"","sources":["../src/ChatPromptWrapper.ts"],"names":[],"mappings":"AAAA,MAAM,OAAgB,iBAAiB;IAC5B,UAAU,CAAC,MAAc,EAAE,EAAC,YAAY,EAAE,WAAW,EAA8C;QACtG,IAAI,WAAW,KAAK,CAAC,EAAE;YACnB,OAAO,YAAY,GAAG,IAAI,GAAG,MAAM,CAAC;SACvC;aAAM;YACH,OAAO,MAAM,CAAC;SACjB;IACL,CAAC;IAEM,cAAc;QACjB,OAAO,EAAE,CAAC;IACd,CAAC;CACJ"}

package/dist/LlamaChatSession.d.ts

@@ -0,0 +1,24 @@
+/// <reference types="node" />
+import { LlamaModel } from "./LlamaModel.js";
+import { ChatPromptWrapper } from "./ChatPromptWrapper.js";
+export declare class LlamaChatSession {
+    private readonly _model;
+    private readonly _systemPrompt;
+    private readonly _printLLamaSystemInfo;
+    private readonly _promptWrapper;
+    private _promptIndex;
+    private _initialized;
+    constructor({ model, printLLamaSystemInfo, promptWrapper, systemPrompt }: {
+        model: LlamaModel;
+        printLLamaSystemInfo?: boolean;
+        promptWrapper?: ChatPromptWrapper;
+        systemPrompt?: string;
+    });
+    get initialized(): boolean;
+    get model(): LlamaModel;
+    init(): Promise<void>;
+    prompt(prompt: string, onToken?: (token: number) => void, { signal }?: {
+        signal?: AbortSignal;
+    }): Promise<string>;
+    private _evalTokens;
+}

package/dist/LlamaChatSession.js

@@ -0,0 +1,86 @@
+import { defaultChatSystemPrompt } from "./config.js";
+import { withLock } from "./utils/withLock.js";
+import { LlamaChatPromptWrapper } from "./chatWrappers/LlamaChatPromptWrapper.js";
+import { AbortError } from "./AbortError.js";
+export class LlamaChatSession {
+    _model;
+    _systemPrompt;
+    _printLLamaSystemInfo;
+    _promptWrapper;
+    _promptIndex = 0;
+    _initialized = false;
+    constructor({ model, printLLamaSystemInfo = false, promptWrapper = new LlamaChatPromptWrapper(), systemPrompt = defaultChatSystemPrompt }) {
+        this._model = model;
+        this._printLLamaSystemInfo = printLLamaSystemInfo;
+        this._promptWrapper = promptWrapper;
+        this._systemPrompt = systemPrompt;
+    }
+    get initialized() {
+        return this._initialized;
+    }
+    get model() {
+        return this._model;
+    }
+    async init() {
+        await withLock(this, "init", async () => {
+            if (this._initialized)
+                return;
+            if (this._printLLamaSystemInfo)
+                console.log("Llama system info", this._model.systemInfo);
+            this._initialized = true;
+        });
+    }
+    async prompt(prompt, onToken, { signal } = {}) {
+        if (!this.initialized)
+            await this.init();
+        return await withLock(this, "prompt", async () => {
+            const promptText = this._promptWrapper.wrapPrompt(prompt, { systemPrompt: this._systemPrompt, promptIndex: this._promptIndex });
+            this._promptIndex++;
+            return await this._evalTokens(this._model.encode(promptText), onToken, { signal });
+        });
+    }
+    async _evalTokens(tokens, onToken, { signal } = {}) {
+        const stopStrings = this._promptWrapper.getStopStrings();
+        const stopStringIndexes = Array(stopStrings.length).fill(0);
+        const skippedChunksQueue = [];
+        let res = "";
+        for await (const chunk of this._model.evaluate(tokens)) {
+            if (signal?.aborted)
+                throw new AbortError();
+            const tokenStr = this._model.decode(Uint32Array.from([chunk]));
+            let skipTokenEvent = false;
+            for (let stopStringIndex = 0; stopStringIndex < stopStrings.length; stopStringIndex++) {
+                const stopString = stopStrings[stopStringIndex];
+                let localShouldSkipTokenEvent = false;
+                for (let i = 0; i < tokenStr.length && stopStringIndexes[stopStringIndex] !== stopString.length; i++) {
+                    if (tokenStr[i] === stopString[stopStringIndexes[stopStringIndex]]) {
+                        stopStringIndexes[stopStringIndex]++;
+                        localShouldSkipTokenEvent = true;
+                    }
+                    else {
+                        stopStringIndexes[stopStringIndex] = 0;
+                        localShouldSkipTokenEvent = false;
+                        break;
+                    }
+                }
+                if (stopStringIndexes[stopStringIndex] === stopString.length) {
+                    return res;
+                }
+                skipTokenEvent ||= localShouldSkipTokenEvent;
+            }
+            if (skipTokenEvent) {
+                skippedChunksQueue.push(chunk);
+                continue;
+            }
+            while (skippedChunksQueue.length > 0) {
+                const token = skippedChunksQueue.shift();
+                res += this._model.decode(Uint32Array.from([token]));
+                onToken?.(token);
+            }
+            res += tokenStr;
+            onToken?.(chunk);
+        }
+        return res;
+    }
+}
+//# sourceMappingURL=LlamaChatSession.js.map

package/dist/LlamaChatSession.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LlamaChatSession.js","sourceRoot":"","sources":["../src/LlamaChatSession.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,uBAAuB,EAAC,MAAM,aAAa,CAAC;AACpD,OAAO,EAAC,QAAQ,EAAC,MAAM,qBAAqB,CAAC;AAG7C,OAAO,EAAC,sBAAsB,EAAC,MAAM,0CAA0C,CAAC;AAChF,OAAO,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAG3C,MAAM,OAAO,gBAAgB;IACR,MAAM,CAAa;IACnB,aAAa,CAAS;IACtB,qBAAqB,CAAU;IAC/B,cAAc,CAAoB;IAC3C,YAAY,GAAW,CAAC,CAAC;IACzB,YAAY,GAAY,KAAK,CAAC;IAEtC,YAAmB,EACf,KAAK,EACL,oBAAoB,GAAG,KAAK,EAC5B,aAAa,GAAG,IAAI,sBAAsB,EAAE,EAC5C,YAAY,GAAG,uBAAuB,EAMzC;QACG,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,qBAAqB,GAAG,oBAAoB,CAAC;QAClD,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QAEpC,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;IACtC,CAAC;IAED,IAAW,WAAW;QAClB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED,IAAW,KAAK;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAEM,KAAK,CAAC,IAAI;QACb,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE;YACpC,IAAI,IAAI,CAAC,YAAY;gBACjB,OAAO;YAEX,IAAI,IAAI,CAAC,qBAAqB;gBAC1B,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YAE7D,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC7B,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,KAAK,CAAC,MAAM,CAAC,MAAc,EAAE,OAAiC,EAAE,EAAC,MAAM,KAA4B,EAAE;QACxG,IAAI,CAAC,IAAI,CAAC,WAAW;YACjB,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAEtB,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,IAAI,EAAE;YAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,MAAM,EAAE,EAAC,YAAY,EAAE,IAAI,CAAC,aAAa,EAAE,WAAW,EAAE,IAAI,CAAC,YAAY,EAAC,CAAC,CAAC;YAC9H,IAAI,CAAC,YAAY,EAAE,CAAC;YAEpB,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,OAAO,EAAE,EAAC,MAAM,EAAC,CAAC,CAAC;QACrF,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,MAAmB,EAAE,OAAiC,EAAE,EAAC,MAAM,KAA4B,EAAE;QACnH,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,CAAC;QACzD,MAAM,iBAAiB,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5D,MAAM,kBAAkB,GAAa,EAAE,CAAC;QACxC,IAAI,GAAG,GAAG,EAAE,CAAC;QAEb,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;YACpD,IAAI,MAAM,EAAE,OAAO;gBACf,MAAM,IAAI,UAAU,EAAE,CAAC;YAE3B,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC/D,IAAI,cAAc,GAAG,KAAK,CAAC;YAE3B,KAAK,IAAI,eAAe,GAAG,CAAC,EAAE,eAAe,GAAG,WAAW,CAAC,MAAM,EAAE,eAAe,EAAE,EAAE;gBACnF,MAAM,UAAU,GAAG,WAAW,CAAC,eAAe,CAAC,CAAC;gBAEhD,IAAI,yBAAyB,GAAG,KAAK,CAAC;gBACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,IAAI,iBAAiB,CAAC,eAAe,CAAC,KAAK,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBAClG,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC,EAAE;wBAChE,iBAAiB,CAAC,eAAe,CAAC,EAAE,CAAC;wBACrC,yBAAyB,GAAG,IAAI,CAAC;qBACpC;yBAAM;wBACH,iBAAiB,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;wBACvC,yBAAyB,GAAG,KAAK,CAAC;wBAClC,MAAM;qBACT;iBACJ;gBAED,IAAI,iBAAiB,CAAC,eAAe,CAAC,KAAK,UAAU,CAAC,MAAM,EAAE;oBAC1D,OAAO,GAAG,CAAC;iBACd;gBAED,cAAc,KAAK,yBAAyB,CAAC;aAChD;YAED,IAAI,cAAc,EAAE;gBAChB,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC/B,SAAS;aACZ;YAED,OAAO,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE;gBAClC,MAAM,KAAK,GAAG,kBAAkB,CAAC,KAAK,EAAG,CAAC;gBAC1C,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACrD,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;aACpB;YAED,GAAG,IAAI,QAAQ,CAAC;YAChB,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;SACpB;QAED,OAAO,GAAG,CAAC;IACf,CAAC;CACJ"}

package/dist/LlamaModel.d.ts

@@ -0,0 +1,13 @@
+export declare class LlamaModel {
+    private readonly _model;
+    private readonly _ctx;
+    private _prependBos;
+    constructor({ modelPath, prependBos }: {
+        modelPath: string;
+        prependBos?: boolean;
+    });
+    get systemInfo(): string;
+    encode(text: string): Uint32Array;
+    decode(tokens: Uint32Array): string;
+    evaluate(tokens: Uint32Array, getRestrictions?: () => Uint32Array): AsyncGenerator<number, void, unknown>;
+}

package/dist/LlamaModel.js

@@ -0,0 +1,43 @@
+import { loadBin } from "./utils/getBin.js";
+const llamaCppNode = await loadBin();
+const { LLAMAModel, LLAMAContext } = llamaCppNode;
+export class LlamaModel {
+    _model;
+    _ctx;
+    _prependBos;
+    constructor({ modelPath, prependBos = true }) {
+        this._model = new LLAMAModel(modelPath);
+        this._ctx = new LLAMAContext(this._model);
+        this._prependBos = prependBos;
+    }
+    get systemInfo() {
+        return llamaCppNode.systemInfo();
+    }
+    encode(text) {
+        return this._ctx.encode(text);
+    }
+    decode(tokens) {
+        return this._ctx.decode(tokens);
+    }
+    async *evaluate(tokens, getRestrictions) {
+        let evalTokens = tokens;
+        if (this._prependBos) {
+            const tokensArray = Array.from(tokens);
+            tokensArray.unshift(llamaCppNode.tokenBos());
+            evalTokens = Uint32Array.from(tokensArray);
+            this._prependBos = false;
+        }
+        // eslint-disable-next-line no-constant-condition
+        while (true) {
+            // Evaluate to get the next token.
+            const nextToken = await this._ctx.eval(evalTokens, getRestrictions?.());
+            // the assistant finished answering
+            if (nextToken === llamaCppNode.tokenEos())
+                break;
+            yield nextToken;
+            // Create tokens for the next eval.
+            evalTokens = Uint32Array.from([nextToken]);
+        }
+    }
+}
+//# sourceMappingURL=LlamaModel.js.map

package/dist/LlamaModel.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LlamaModel.js","sourceRoot":"","sources":["../src/LlamaModel.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,OAAO,EAAqC,MAAM,mBAAmB,CAAC;AAE9E,MAAM,YAAY,GAAG,MAAM,OAAO,EAAE,CAAC;AACrC,MAAM,EAAC,UAAU,EAAE,YAAY,EAAC,GAAG,YAAY,CAAC;AAEhD,MAAM,OAAO,UAAU;IACF,MAAM,CAAa;IACnB,IAAI,CAAe;IAC5B,WAAW,CAAU;IAE7B,YAAmB,EACf,SAAS,EAAE,UAAU,GAAG,IAAI,EAG/B;QACG,IAAI,CAAC,MAAM,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAClC,CAAC;IAED,IAAW,UAAU;QACjB,OAAO,YAAY,CAAC,UAAU,EAAE,CAAC;IACrC,CAAC;IAEM,MAAM,CAAC,IAAY;QACtB,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,MAAmB;QAC7B,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAEM,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAmB,EAAE,eAAmC;QAC3E,IAAI,UAAU,GAAG,MAAM,CAAC;QAExB,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvC,WAAW,CAAC,OAAO,CAAC,YAAY,CAAC,QAAQ,EAAE,CAAC,CAAC;YAE7C,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC3C,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;SAC5B;QAED,iDAAiD;QACjD,OAAO,IAAI,EAAE;YACT,kCAAkC;YAClC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;YAExE,mCAAmC;YACnC,IAAI,SAAS,KAAK,YAAY,CAAC,QAAQ,EAAE;gBACrC,MAAM;YAEV,MAAM,SAAS,CAAC;YAEhB,mCAAmC;YACnC,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;SAC9C;IACL,CAAC;CACJ"}

package/dist/chatWrappers/EmptyChatPromptWrapper.d.ts

@@ -0,0 +1,3 @@
+import { ChatPromptWrapper } from "../ChatPromptWrapper.js";
+export declare class EmptyChatPromptWrapper extends ChatPromptWrapper {
+}

package/dist/chatWrappers/EmptyChatPromptWrapper.js

@@ -0,0 +1,4 @@
+import { ChatPromptWrapper } from "../ChatPromptWrapper.js";
+export class EmptyChatPromptWrapper extends ChatPromptWrapper {
+}
+//# sourceMappingURL=EmptyChatPromptWrapper.js.map

package/dist/chatWrappers/EmptyChatPromptWrapper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"EmptyChatPromptWrapper.js","sourceRoot":"","sources":["../../src/chatWrappers/EmptyChatPromptWrapper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,iBAAiB,EAAC,MAAM,yBAAyB,CAAC;AAE1D,MAAM,OAAO,sBAAuB,SAAQ,iBAAiB;CAAG"}

package/dist/chatWrappers/LlamaChatPromptWrapper.d.ts

@@ -0,0 +1,8 @@
+import { ChatPromptWrapper } from "../ChatPromptWrapper.js";
+export declare class LlamaChatPromptWrapper extends ChatPromptWrapper {
+    wrapPrompt(prompt: string, { systemPrompt, promptIndex }: {
+        systemPrompt: string;
+        promptIndex: number;
+    }): string;
+    getStopStrings(): string[];
+}

package/dist/chatWrappers/LlamaChatPromptWrapper.js

@@ -0,0 +1,16 @@
+import { ChatPromptWrapper } from "../ChatPromptWrapper.js";
+// source: https://huggingface.co/blog/llama2#how-to-prompt-llama-2
+export class LlamaChatPromptWrapper extends ChatPromptWrapper {
+    wrapPrompt(prompt, { systemPrompt, promptIndex }) {
+        if (promptIndex === 0) {
+            return "<s>[INST] <<SYS>>\n" + systemPrompt + "\n<</SYS>>\n\n" + prompt + " [/INST]\n\n";
+        }
+        else {
+            return "<s>[INST] " + prompt + " [/INST]\n\n";
+        }
+    }
+    getStopStrings() {
+        return ["</s><s>[INST]"];
+    }
+}
+//# sourceMappingURL=LlamaChatPromptWrapper.js.map

package/dist/chatWrappers/LlamaChatPromptWrapper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LlamaChatPromptWrapper.js","sourceRoot":"","sources":["../../src/chatWrappers/LlamaChatPromptWrapper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,iBAAiB,EAAC,MAAM,yBAAyB,CAAC;AAE1D,mEAAmE;AACnE,MAAM,OAAO,sBAAuB,SAAQ,iBAAiB;IACzC,UAAU,CAAC,MAAc,EAAE,EAAC,YAAY,EAAE,WAAW,EAA8C;QAC/G,IAAI,WAAW,KAAK,CAAC,EAAE;YACnB,OAAO,qBAAqB,GAAG,YAAY,GAAG,gBAAgB,GAAG,MAAM,GAAG,cAAc,CAAC;SAC5F;aAAM;YACH,OAAO,YAAY,GAAG,MAAM,GAAG,cAAc,CAAC;SACjD;IACL,CAAC;IAEe,cAAc;QAC1B,OAAO,CAAC,eAAe,CAAC,CAAC;IAC7B,CAAC;CACJ"}

package/dist/cli/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/cli/cli.js

@@ -0,0 +1,33 @@
+#!/usr/bin/env node
+import { fileURLToPath } from "url";
+import path from "path";
+import yargs from "yargs";
+// eslint-disable-next-line node/file-extension-in-import
+import { hideBin } from "yargs/helpers";
+import fs from "fs-extra";
+import { DownloadCommand } from "./commands/DownloadCommand.js";
+import { BuildCommand } from "./commands/BuildCommand.js";
+import { OnPostInstallCommand } from "./commands/OnPostInstallCommand.js";
+import { ClearCommand } from "./commands/ClearCommand.js";
+import { ChatCommand } from "./commands/ChatCommand.js";
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const packageJson = fs.readJSONSync(path.join(__dirname, "..", "..", "package.json"));
+const yarg = yargs(hideBin(process.argv));
+yarg
+    .usage("Usage: $0 <command> [options]")
+    .command(DownloadCommand)
+    .command(BuildCommand)
+    .command(ClearCommand)
+    .command(ChatCommand)
+    .command(OnPostInstallCommand)
+    .recommendCommands()
+    .demandCommand(1)
+    .strict()
+    .strictCommands()
+    .alias("v", "version")
+    .help("h")
+    .alias("h", "help")
+    .version(packageJson.version)
+    .wrap(Math.min(100, yarg.terminalWidth()))
+    .parse();
+//# sourceMappingURL=cli.js.map

package/dist/cli/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/cli/cli.ts"],"names":[],"mappings":";AAEA,OAAO,EAAC,aAAa,EAAC,MAAM,KAAK,CAAC;AAClC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,yDAAyD;AACzD,OAAO,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,eAAe,EAAC,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAC,YAAY,EAAC,MAAM,4BAA4B,CAAC;AACxD,OAAO,EAAC,oBAAoB,EAAC,MAAM,oCAAoC,CAAC;AACxE,OAAO,EAAC,YAAY,EAAC,MAAM,4BAA4B,CAAC;AACxD,OAAO,EAAC,WAAW,EAAC,MAAM,2BAA2B,CAAC;AAEtD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE/D,MAAM,WAAW,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC;AAEtF,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;AAE1C,IAAI;KACC,KAAK,CAAC,+BAA+B,CAAC;KACtC,OAAO,CAAC,eAAe,CAAC;KACxB,OAAO,CAAC,YAAY,CAAC;KACrB,OAAO,CAAC,YAAY,CAAC;KACrB,OAAO,CAAC,WAAW,CAAC;KACpB,OAAO,CAAC,oBAAoB,CAAC;KAC7B,iBAAiB,EAAE;KACnB,aAAa,CAAC,CAAC,CAAC;KAChB,MAAM,EAAE;KACR,cAAc,EAAE;KAChB,KAAK,CAAC,GAAG,EAAE,SAAS,CAAC;KACrB,IAAI,CAAC,GAAG,CAAC;KACT,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC;KAClB,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC;KAC5B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;KACzC,KAAK,EAAE,CAAC"}

package/dist/cli/commands/BuildCommand.d.ts

@@ -0,0 +1,8 @@
+import { CommandModule } from "yargs";
+type BuildCommand = {
+    arch?: string;
+    nodeTarget?: string;
+};
+export declare const BuildCommand: CommandModule<object, BuildCommand>;
+export declare function BuildLlamaCppCommand({ arch, nodeTarget }: BuildCommand): Promise<void>;
+export {};

package/dist/cli/commands/BuildCommand.js

@@ -0,0 +1,41 @@
+import chalk from "chalk";
+import { compileLlamaCpp } from "../../utils/compileLLamaCpp.js";
+import withOra from "../../utils/withOra.js";
+import { clearTempFolder } from "../../utils/clearTempFolder.js";
+export const BuildCommand = {
+    command: "build",
+    describe: "Compile the currently downloaded llama.cpp",
+    builder(yargs) {
+        return yargs
+            .option("arch", {
+            type: "string",
+            description: "The architecture to compile llama.cpp for"
+        })
+            .option("nodeTarget", {
+            type: "string",
+            description: "The Node.js version to compile llama.cpp for. Example: v18.0.0"
+        });
+    },
+    handler: BuildLlamaCppCommand
+};
+export async function BuildLlamaCppCommand({ arch, nodeTarget }) {
+    await withOra({
+        loading: chalk.blue("Compiling llama.cpp"),
+        success: chalk.blue("Compiled llama.cpp"),
+        fail: chalk.blue("Failed to compile llama.cpp")
+    }, async () => {
+        await compileLlamaCpp({
+            arch: arch ? arch : undefined,
+            nodeTarget: nodeTarget ? nodeTarget : undefined,
+            setUsedBingFlag: true
+        });
+    });
+    await withOra({
+        loading: chalk.blue("Removing temporary files"),
+        success: chalk.blue("Removed temporary files"),
+        fail: chalk.blue("Failed to remove temporary files")
+    }, async () => {
+        await clearTempFolder();
+    });
+}
+//# sourceMappingURL=BuildCommand.js.map

package/dist/cli/commands/BuildCommand.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"BuildCommand.js","sourceRoot":"","sources":["../../../src/cli/commands/BuildCommand.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAC,eAAe,EAAC,MAAM,gCAAgC,CAAC;AAC/D,OAAO,OAAO,MAAM,wBAAwB,CAAC;AAC7C,OAAO,EAAC,eAAe,EAAC,MAAM,gCAAgC,CAAC;AAO/D,MAAM,CAAC,MAAM,YAAY,GAAwC;IAC7D,OAAO,EAAE,OAAO;IAChB,QAAQ,EAAE,4CAA4C;IACtD,OAAO,CAAC,KAAK;QACT,OAAO,KAAK;aACP,MAAM,CAAC,MAAM,EAAE;YACZ,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,2CAA2C;SAC3D,CAAC;aACD,MAAM,CAAC,YAAY,EAAE;YAClB,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,gEAAgE;SAChF,CAAC,CAAC;IACX,CAAC;IACD,OAAO,EAAE,oBAAoB;CAChC,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,EAAC,IAAI,EAAE,UAAU,EAAe;IACvE,MAAM,OAAO,CAAC;QACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC;QAC1C,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC;QACzC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,6BAA6B,CAAC;KAClD,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,eAAe,CAAC;YAClB,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;YAC7B,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;YAC/C,eAAe,EAAE,IAAI;SACxB,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC;QACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC;QAC/C,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC;QAC9C,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,kCAAkC,CAAC;KACvD,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,eAAe,EAAE,CAAC;IAC5B,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/cli/commands/ChatCommand.d.ts

@@ -0,0 +1,8 @@
+import { CommandModule } from "yargs";
+type ChatCommand = {
+    model: string;
+    systemInfo: boolean;
+    systemPrompt: string;
+};
+export declare const ChatCommand: CommandModule<object, ChatCommand>;
+export {};