node-llama-cpp 1.3.1 → 2.1.0

package/README.md

@@ -1,11 +1,16 @@
+<div align="center">
+
 # Node Llama.cpp
 Node.js bindings for llama.cpp.
 
-Pre-built bindings are provided with a fallback to building from source with `node-gyp`.
+<sub>Pre-built bindings are provided with a fallback to building from source with `node-gyp`.<sub>
 
 [![Build](https://github.com/withcatai/node-llama-cpp/actions/workflows/build.yml/badge.svg)](https://github.com/withcatai/node-llama-cpp/actions/workflows/build.yml)
+[![License](https://badgen.net/badge/color/MIT/green?label=license)](https://www.npmjs.com/package/node-llama-cpp)
+[![License](https://badgen.net/badge/color/TypeScript/blue?label=types)](https://www.npmjs.com/package/node-llama-cpp)
 [![Version](https://badgen.net/npm/v/node-llama-cpp)](https://www.npmjs.com/package/node-llama-cpp)
 
+</div>
 
 ## Installation
 ```bash
@@ -30,7 +35,7 @@ import {LlamaModel, LlamaContext, LlamaChatSession} from "node-llama-cpp";
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 const model = new LlamaModel({
-    modelPath: path.join(__dirname, "models", "vicuna-13b-v1.5-16k.ggmlv3.q5_1.bin")
+    modelPath: path.join(__dirname, "models", "codellama-13b.Q3_K_M.gguf")
 });
 const context = new LlamaContext({model});
 const session = new LlamaChatSession({context});
@@ -73,7 +78,7 @@ export class MyCustomChatPromptWrapper extends ChatPromptWrapper {
 }
 
 const model = new LlamaModel({
-    modelPath: path.join(__dirname, "models", "vicuna-13b-v1.5-16k.ggmlv3.q5_1.bin"),
+    modelPath: path.join(__dirname, "models", "codellama-13b.Q3_K_M.gguf"),
     promptWrapper: new MyCustomChatPromptWrapper() // by default, LlamaChatPromptWrapper is used
 })
 const context = new LlamaContext({model});
@@ -103,7 +108,7 @@ import {LlamaModel, LlamaContext, LlamaChatSession} from "node-llama-cpp";
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 const model = new LlamaModel({
-    modelPath: path.join(__dirname, "models", "vicuna-13b-v1.5-16k.ggmlv3.q5_1.bin")
+    modelPath: path.join(__dirname, "models", "codellama-13b.Q3_K_M.gguf")
 });
 
 const context = new LlamaContext({model});
@@ -113,8 +118,8 @@ console.log("AI: " + q1);
 
 const tokens = context.encode(q1);
 const res: number[] = [];
-for await (const chunk of context.evaluate(tokens)) {
-    res.push(chunk);
+for await (const modelToken of context.evaluate(tokens)) {
+    res.push(modelToken);
     
     // it's important to not concatinate the results as strings,
     // as doing so will break some characters (like some emojis) that are made of multiple tokens.
@@ -130,15 +135,55 @@ const a1 = context.decode(Uint32Array.from(res)).split("USER:")[0];
 console.log("AI: " + a1);
 ```
 
+#### With grammar
+Use this to direct the model to generate a specific format of text, like `JSON` for example.
+
+> **Note:** there's an issue with some grammars where the model won't stop generating output,
+> so it's advised to use it together with `maxTokens` set to the context size of the model
+
+```typescript
+import {fileURLToPath} from "url";
+import path from "path";
+import {LlamaModel, LlamaGrammar, LlamaContext, LlamaChatSession} from "node-llama-cpp";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const model = new LlamaModel({
+    modelPath: path.join(__dirname, "models", "codellama-13b.Q3_K_M.gguf")
+})
+const grammar = await LlamaGrammar.getFor("json");
+const context = new LlamaContext({
+    model,
+    grammar
+});
+const session = new LlamaChatSession({context});
+
+
+const q1 = 'Create a JSON that contains a message saying "hi there"';
+console.log("User: " + q1);
+
+const a1 = await session.prompt(q1, {maxTokens: context.getContextSize()});
+console.log("AI: " + a1);
+console.log(JSON.parse(a1));
+
+
+const q2 = 'Add another field to the JSON with the key being "author" and the value being "LLama"';
+console.log("User: " + q2);
+
+const a2 = await session.prompt(q2, {maxTokens: context.getContextSize()});
+console.log("AI: " + a2);
+console.log(JSON.parse(a2));
+```
+
 ### CLI
 ```
 Usage: node-llama-cpp <command> [options]
 
 Commands:
-  node-llama-cpp download           Download a release of llama.cpp and compile it
-  node-llama-cpp build              Compile the currently downloaded llama.cpp
-  node-llama-cpp clear [type]       Clear files created by llama-cli
-  node-llama-cpp chat               Chat with a LLama model
+  node-llama-cpp download      Download a release of llama.cpp and compile it
+  node-llama-cpp build         Compile the currently downloaded llama.cpp
+  node-llama-cpp clear [type]  Clear files created by node-llama-cpp
+  node-llama-cpp chat          Chat with a LLama model
 
 Options:
   -h, --help     Show help                                                                 [boolean]
@@ -152,15 +197,17 @@ node-llama-cpp download
 Download a release of llama.cpp and compile it
 
 Options:
-  -h, --help        Show help                                                              [boolean]
-      --repo        The GitHub repository to download a release of llama.cpp from. Can also be set v
-                    ia the NODE_LLAMA_CPP_REPO environment variable
+  -h, --help             Show help                                                         [boolean]
+      --repo             The GitHub repository to download a release of llama.cpp from. Can also be
+                         set via the NODE_LLAMA_CPP_REPO environment variable
                                                            [string] [default: "ggerganov/llama.cpp"]
-      --release     The tag of the llama.cpp release to download. Can also be set via the NODE_LLAMA
-                    _CPP_REPO_RELEASE environment variable              [string] [default: "latest"]
-      --arch        The architecture to compile llama.cpp for                               [string]
-      --nodeTarget  The Node.js version to compile llama.cpp for. Example: v18.0.0          [string]
-  -v, --version     Show version number                                                    [boolean]
+      --release          The tag of the llama.cpp release to download. Set to "latest" to download t
+                         he latest release. Can also be set via the NODE_LLAMA_CPP_REPO_RELEASE envi
+                         ronment variable                               [string] [default: "latest"]
+  -a, --arch             The architecture to compile llama.cpp for                          [string]
+  -t, --nodeTarget       The Node.js version to compile llama.cpp for. Example: v18.0.0     [string]
+      --skipBuild, --sb  Skip building llama.cpp after downloading it     [boolean] [default: false]
+  -v, --version          Show version number                                               [boolean]
 ```
 
 #### `build` command
@@ -171,8 +218,8 @@ Compile the currently downloaded llama.cpp
 
 Options:
   -h, --help        Show help                                                              [boolean]
-      --arch        The architecture to compile llama.cpp for                               [string]
-      --nodeTarget  The Node.js version to compile llama.cpp for. Example: v18.0.0          [string]
+  -a, --arch        The architecture to compile llama.cpp for                               [string]
+  -t, --nodeTarget  The Node.js version to compile llama.cpp for. Example: v18.0.0          [string]
   -v, --version     Show version number                                                    [boolean]
 ```
 
@@ -180,7 +227,7 @@ Options:
 ```
 node-llama-cpp clear [type]
 
-Clear files created by llama-cli
+Clear files created by node-llama-cpp
 
 Options:
   -h, --help     Show help                                                                 [boolean]
@@ -195,20 +242,45 @@ node-llama-cpp chat
 Chat with a LLama model
 
 Required:
-      --model  LLama model file to use for the chat                              [string] [required]
+  -m, --model  LLama model file to use for the chat                              [string] [required]
 
 Optional:
-      --systemInfo    Print llama.cpp system info                         [boolean] [default: false]
-      --systemPrompt  System prompt to use against the model. [default value: You are a helpful, res
-                      pectful and honest assistant. Always answer as helpfully as possible. If a que
-                      stion does not make any sense, or is not factually coherent, explain why inste
-                      ad of answering something not correct. If you don't know the answer to a quest
-                      ion, please don't share false information.]
+  -i, --systemInfo       Print llama.cpp system info                      [boolean] [default: false]
+  -s, --systemPrompt     System prompt to use against the model. [default value: You are a helpful,
+                         respectful and honest assistant. Always answer as helpfully as possible. If
+                          a question does not make any sense, or is not factually coherent, explain
+                         why instead of answering something not correct. If you don't know the answe
+                         r to a question, please don't share false information.]
   [string] [default: "You are a helpful, respectful and honest assistant. Always answer as helpfully
                                                                                         as possible.
                 If a question does not make any sense, or is not factually coherent, explain why ins
    tead of answering something not correct. If you don't know the answer to a question, please don't
                                                                           share false information."]
+  -w, --wrapper          Chat wrapper to use. Use `auto` to automatically select a wrapper based on
+                         the model's BOS token
+                   [string] [choices: "auto", "general", "llamaChat", "chatML"] [default: "general"]
+  -c, --contextSize      Context size to use for the model                  [number] [default: 4096]
+  -g, --grammar          Restrict the model response to a specific grammar, like JSON for example
+     [string] [choices: "text", "json", "list", "arithmetic", "japanese", "chess"] [default: "text"]
+  -t, --temperature      Temperature is a hyperparameter that controls the randomness of the generat
+                         ed text. It affects the probability distribution of the model's output toke
+                         ns. A higher temperature (e.g., 1.5) makes the output more random and creat
+                         ive, while a lower temperature (e.g., 0.5) makes the output more focused, d
+                         eterministic, and conservative. The suggested temperature is 0.8, which pro
+                         vides a balance between randomness and determinism. At the extreme, a tempe
+                         rature of 0 will always pick the most likely next token, leading to identic
+                         al outputs in each run. Set to `0` to disable.        [number] [default: 0]
+  -k, --topK             Limits the model to consider only the K most likely next tokens for samplin
+                         g at each step of sequence generation. An integer number between `1` and th
+                         e size of the vocabulary. Set to `0` to disable (which uses the full vocabu
+                         lary). Only relevant when `temperature` is set to a value greater than 0.
+                                                                              [number] [default: 40]
+  -p, --topP             Dynamically selects the smallest set of tokens whose cumulative probability
+                          exceeds the threshold P, and samples the next token only from this set. A
+                         float number between `0` and `1`. Set to `1` to disable. Only relevant when
+                          `temperature` is set to a value greater than `0`. [number] [default: 0.95]
+      --maxTokens, --mt  Maximum number of tokens to generate in responses. Set to `0` to disable. S
+                         et to `-1` to set to the context size                 [number] [default: 0]
 
 Options:
   -h, --help     Show help                                                                 [boolean]

package/dist/ChatPromptWrapper.d.ts

@@ -1,7 +1,10 @@
 export declare abstract class ChatPromptWrapper {
+    abstract readonly wrapperName: string;
     wrapPrompt(prompt: string, { systemPrompt, promptIndex }: {
         systemPrompt: string;
         promptIndex: number;
+        lastStopString: string | null;
+        lastStopStringSuffix: string | null;
     }): string;
     getStopStrings(): string[];
 }

package/dist/ChatPromptWrapper.js.map

@@ -1 +1 @@
-{"version":3,"file":"ChatPromptWrapper.js","sourceRoot":"","sources":["../src/ChatPromptWrapper.ts"],"names":[],"mappings":"AAAA,MAAM,OAAgB,iBAAiB;IAC5B,UAAU,CAAC,MAAc,EAAE,EAAC,YAAY,EAAE,WAAW,EAA8C;QACtG,IAAI,WAAW,KAAK,CAAC,EAAE;YACnB,OAAO,YAAY,GAAG,IAAI,GAAG,MAAM,CAAC;SACvC;aAAM;YACH,OAAO,MAAM,CAAC;SACjB;IACL,CAAC;IAEM,cAAc;QACjB,OAAO,EAAE,CAAC;IACd,CAAC;CACJ"}
+{"version":3,"file":"ChatPromptWrapper.js","sourceRoot":"","sources":["../src/ChatPromptWrapper.ts"],"names":[],"mappings":"AAAA,MAAM,OAAgB,iBAAiB;IAG5B,UAAU,CAAC,MAAc,EAAE,EAAC,YAAY,EAAE,WAAW,EAE3D;QACG,IAAI,WAAW,KAAK,CAAC,EAAE;YACnB,OAAO,YAAY,GAAG,IAAI,GAAG,MAAM,CAAC;SACvC;aAAM;YACH,OAAO,MAAM,CAAC;SACjB;IACL,CAAC;IAEM,cAAc;QACjB,OAAO,EAAE,CAAC;IACd,CAAC;CACJ"}

package/dist/chatWrappers/ChatMLPromptWrapper.d.ts

@@ -0,0 +1,11 @@
+import { ChatPromptWrapper } from "../ChatPromptWrapper.js";
+export declare class ChatMLPromptWrapper extends ChatPromptWrapper {
+    readonly wrapperName: string;
+    wrapPrompt(prompt: string, { systemPrompt, promptIndex, lastStopString, lastStopStringSuffix }: {
+        systemPrompt: string;
+        promptIndex: number;
+        lastStopString: string | null;
+        lastStopStringSuffix: string | null;
+    }): string;
+    getStopStrings(): string[];
+}

package/dist/chatWrappers/ChatMLPromptWrapper.js

@@ -0,0 +1,19 @@
+import { ChatPromptWrapper } from "../ChatPromptWrapper.js";
+import { getTextCompletion } from "../utils/getTextCompletion.js";
+// source: https://github.com/openai/openai-python/blob/120d225b91a8453e15240a49fb1c6794d8119326/chatml.md
+export class ChatMLPromptWrapper extends ChatPromptWrapper {
+    wrapperName = "ChatML";
+    wrapPrompt(prompt, { systemPrompt, promptIndex, lastStopString, lastStopStringSuffix }) {
+        const previousCompletionEnd = (lastStopString ?? "") + (lastStopStringSuffix ?? "");
+        if (promptIndex === 0 && systemPrompt != "")
+            return (getTextCompletion(previousCompletionEnd, "<|im_start|>system\n") ?? "<|im_start|>system\n") +
+                systemPrompt + "<|im_end|>\n<|im_start|>user\n" + prompt + "<|im_end|>\n<|im_start|>assistant\n";
+        else
+            return (getTextCompletion(previousCompletionEnd, "<|im_end|>\n<|im_start|>user\n") ?? "<|im_end|>\n<|im_start|>user\n") +
+                prompt + "<|im_end|>\n<|im_start|>assistant\n";
+    }
+    getStopStrings() {
+        return ["<|im_end|>"];
+    }
+}
+//# sourceMappingURL=ChatMLPromptWrapper.js.map

package/dist/chatWrappers/ChatMLPromptWrapper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ChatMLPromptWrapper.js","sourceRoot":"","sources":["../../src/chatWrappers/ChatMLPromptWrapper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,iBAAiB,EAAC,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAC,iBAAiB,EAAC,MAAM,+BAA+B,CAAC;AAEhE,0GAA0G;AAC1G,MAAM,OAAO,mBAAoB,SAAQ,iBAAiB;IACtC,WAAW,GAAW,QAAQ,CAAC;IAE/B,UAAU,CAAC,MAAc,EAAE,EAAC,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,oBAAoB,EAE1G;QACG,MAAM,qBAAqB,GAAG,CAAC,cAAc,IAAI,EAAE,CAAC,GAAG,CAAC,oBAAoB,IAAI,EAAE,CAAC,CAAC;QAEpF,IAAI,WAAW,KAAK,CAAC,IAAI,YAAY,IAAI,EAAE;YACvC,OAAO,CAAC,iBAAiB,CAAC,qBAAqB,EAAE,sBAAsB,CAAC,IAAI,sBAAsB,CAAC;gBAC/F,YAAY,GAAG,gCAAgC,GAAG,MAAM,GAAG,qCAAqC,CAAC;;YAErG,OAAO,CAAC,iBAAiB,CAAC,qBAAqB,EAAE,gCAAgC,CAAC,IAAI,gCAAgC,CAAC;gBACnH,MAAM,GAAG,qCAAqC,CAAC;IAC3D,CAAC;IAEe,cAAc;QAC1B,OAAO,CAAC,YAAY,CAAC,CAAC;IAC1B,CAAC;CACJ"}

package/dist/chatWrappers/EmptyChatPromptWrapper.d.ts

@@ -1,3 +1,4 @@
 import { ChatPromptWrapper } from "../ChatPromptWrapper.js";
 export declare class EmptyChatPromptWrapper extends ChatPromptWrapper {
+    readonly wrapperName: string;
 }

package/dist/chatWrappers/EmptyChatPromptWrapper.js

@@ -1,4 +1,5 @@
 import { ChatPromptWrapper } from "../ChatPromptWrapper.js";
 export class EmptyChatPromptWrapper extends ChatPromptWrapper {
+    wrapperName = "Empty";
 }
 //# sourceMappingURL=EmptyChatPromptWrapper.js.map

package/dist/chatWrappers/EmptyChatPromptWrapper.js.map

@@ -1 +1 @@
-{"version":3,"file":"EmptyChatPromptWrapper.js","sourceRoot":"","sources":["../../src/chatWrappers/EmptyChatPromptWrapper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,iBAAiB,EAAC,MAAM,yBAAyB,CAAC;AAE1D,MAAM,OAAO,sBAAuB,SAAQ,iBAAiB;CAAG"}
+{"version":3,"file":"EmptyChatPromptWrapper.js","sourceRoot":"","sources":["../../src/chatWrappers/EmptyChatPromptWrapper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,iBAAiB,EAAC,MAAM,yBAAyB,CAAC;AAE1D,MAAM,OAAO,sBAAuB,SAAQ,iBAAiB;IACzC,WAAW,GAAW,OAAO,CAAC;CACjD"}

package/dist/chatWrappers/GeneralChatPromptWrapper.d.ts

@@ -1,8 +1,18 @@
 import { ChatPromptWrapper } from "../ChatPromptWrapper.js";
 export declare class GeneralChatPromptWrapper extends ChatPromptWrapper {
-    wrapPrompt(prompt: string, { systemPrompt, promptIndex }: {
+    readonly wrapperName: string;
+    private readonly _instructionName;
+    private readonly _responseName;
+    constructor({ instructionName, responseName }?: {
+        instructionName?: string;
+        responseName?: string;
+    });
+    wrapPrompt(prompt: string, { systemPrompt, promptIndex, lastStopString, lastStopStringSuffix }: {
         systemPrompt: string;
         promptIndex: number;
+        lastStopString: string | null;
+        lastStopStringSuffix: string | null;
     }): string;
     getStopStrings(): string[];
+    private _getPromptPrefix;
 }

package/dist/chatWrappers/GeneralChatPromptWrapper.js

@@ -1,11 +1,35 @@
 import { ChatPromptWrapper } from "../ChatPromptWrapper.js";
+import { getTextCompletion } from "../utils/getTextCompletion.js";
 export class GeneralChatPromptWrapper extends ChatPromptWrapper {
-    wrapPrompt(prompt, { systemPrompt, promptIndex }) {
-        const conversationPrompt = "\n\n### Human:\n\n" + prompt + "\n\n### Assistant:\n\n";
-        return promptIndex === 0 ? systemPrompt + conversationPrompt : conversationPrompt;
+    wrapperName = "General";
+    _instructionName;
+    _responseName;
+    constructor({ instructionName = "Human", responseName = "Assistant" } = {}) {
+        super();
+        this._instructionName = instructionName;
+        this._responseName = responseName;
+    }
+    wrapPrompt(prompt, { systemPrompt, promptIndex, lastStopString, lastStopStringSuffix }) {
+        if (promptIndex === 0)
+            return systemPrompt + `\n\n### ${this._instructionName}:\n\n` + prompt + `\n\n### ${this._responseName}:\n\n`;
+        return this._getPromptPrefix(lastStopString, lastStopStringSuffix) + prompt + `\n\n### ${this._responseName}:\n\n`;
     }
     getStopStrings() {
-        return ["### Human:", "Human:", "### Assistant:", "Assistant:", "<end>"];
+        return [
+            `\n\n### ${this._instructionName}`,
+            `### ${this._instructionName}`,
+            `\n\n### ${this._responseName}`,
+            `### ${this._responseName}`,
+            "<end>"
+        ];
+    }
+    _getPromptPrefix(lastStopString, lastStopStringSuffix) {
+        return getTextCompletion(lastStopString === "<end>"
+            ? lastStopStringSuffix
+            : (lastStopString + (lastStopStringSuffix ?? "")), [
+            `\n\n### ${this._instructionName}:\n\n`,
+            `### ${this._instructionName}:\n\n`
+        ]) ?? `\n\n### ${this._instructionName}:\n\n`;
     }
 }
 //# sourceMappingURL=GeneralChatPromptWrapper.js.map

package/dist/chatWrappers/GeneralChatPromptWrapper.js.map

@@ -1 +1 @@
-{"version":3,"file":"GeneralChatPromptWrapper.js","sourceRoot":"","sources":["../../src/chatWrappers/GeneralChatPromptWrapper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,iBAAiB,EAAC,MAAM,yBAAyB,CAAC;AAE1D,MAAM,OAAO,wBAAyB,SAAQ,iBAAiB;IAC3C,UAAU,CAAC,MAAc,EAAE,EAAC,YAAY,EAAE,WAAW,EAAgD;QACjH,MAAM,kBAAkB,GAAG,oBAAoB,GAAG,MAAM,GAAG,wBAAwB,CAAC;QAEpF,OAAO,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,kBAAkB,CAAC,CAAC,CAAC,kBAAkB,CAAC;IACtF,CAAC;IAEe,cAAc;QAC1B,OAAO,CAAC,YAAY,EAAE,QAAQ,EAAE,gBAAgB,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;IAC7E,CAAC;CACJ"}
+{"version":3,"file":"GeneralChatPromptWrapper.js","sourceRoot":"","sources":["../../src/chatWrappers/GeneralChatPromptWrapper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,iBAAiB,EAAC,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAC,iBAAiB,EAAC,MAAM,+BAA+B,CAAC;AAEhE,MAAM,OAAO,wBAAyB,SAAQ,iBAAiB;IAC3C,WAAW,GAAW,SAAS,CAAC;IAC/B,gBAAgB,CAAS;IACzB,aAAa,CAAS;IAEvC,YAAmB,EAAC,eAAe,GAAG,OAAO,EAAE,YAAY,GAAG,WAAW,KAAuD,EAAE;QAC9H,KAAK,EAAE,CAAC;QAER,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC;QACxC,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;IACtC,CAAC;IAEe,UAAU,CAAC,MAAc,EAAE,EAAC,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,oBAAoB,EAE1G;QACG,IAAI,WAAW,KAAK,CAAC;YACjB,OAAO,YAAY,GAAG,WAAW,IAAI,CAAC,gBAAgB,OAAO,GAAG,MAAM,GAAG,WAAW,IAAI,CAAC,aAAa,OAAO,CAAC;QAElH,OAAO,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,oBAAoB,CAAC,GAAG,MAAM,GAAG,WAAW,IAAI,CAAC,aAAa,OAAO,CAAC;IACvH,CAAC;IAEe,cAAc;QAC1B,OAAO;YACH,WAAW,IAAI,CAAC,gBAAgB,EAAE;YAClC,OAAO,IAAI,CAAC,gBAAgB,EAAE;YAC9B,WAAW,IAAI,CAAC,aAAa,EAAE;YAC/B,OAAO,IAAI,CAAC,aAAa,EAAE;YAC3B,OAAO;SACV,CAAC;IACN,CAAC;IAEO,gBAAgB,CAAC,cAA6B,EAAE,oBAAmC;QACvF,OAAO,iBAAiB,CACpB,cAAc,KAAK,OAAO;YACtB,CAAC,CAAC,oBAAoB;YACtB,CAAC,CAAC,CAAC,cAAc,GAAG,CAAC,oBAAoB,IAAI,EAAE,CAAC,CAAC,EACrD;YACI,WAAW,IAAI,CAAC,gBAAgB,OAAO;YACvC,OAAO,IAAI,CAAC,gBAAgB,OAAO;SACtC,CACJ,IAAI,WAAW,IAAI,CAAC,gBAAgB,OAAO,CAAC;IACjD,CAAC;CACJ"}

package/dist/chatWrappers/LlamaChatPromptWrapper.d.ts

@@ -1,8 +1,11 @@
 import { ChatPromptWrapper } from "../ChatPromptWrapper.js";
 export declare class LlamaChatPromptWrapper extends ChatPromptWrapper {
-    wrapPrompt(prompt: string, { systemPrompt, promptIndex }: {
+    readonly wrapperName: string;
+    wrapPrompt(prompt: string, { systemPrompt, promptIndex, lastStopString, lastStopStringSuffix }: {
         systemPrompt: string;
         promptIndex: number;
+        lastStopString: string | null;
+        lastStopStringSuffix: string | null;
     }): string;
     getStopStrings(): string[];
 }

package/dist/chatWrappers/LlamaChatPromptWrapper.js

@@ -1,16 +1,20 @@
 import { ChatPromptWrapper } from "../ChatPromptWrapper.js";
+import { getTextCompletion } from "../utils/getTextCompletion.js";
 // source: https://huggingface.co/blog/llama2#how-to-prompt-llama-2
 export class LlamaChatPromptWrapper extends ChatPromptWrapper {
-    wrapPrompt(prompt, { systemPrompt, promptIndex }) {
-        if (promptIndex === 0) {
-            return "<s>[INST] <<SYS>>\n" + systemPrompt + "\n<</SYS>>\n\n" + prompt + " [/INST]\n\n";
+    wrapperName = "LlamaChat";
+    wrapPrompt(prompt, { systemPrompt, promptIndex, lastStopString, lastStopStringSuffix }) {
+        const previousCompletionEnd = (lastStopString ?? "") + (lastStopStringSuffix ?? "");
+        if (promptIndex === 0 && systemPrompt != "") {
+            return (getTextCompletion(previousCompletionEnd, "<s>[INST] <<SYS>>\n") ?? "<s>[INST] <<SYS>>\n") + systemPrompt +
+                "\n<</SYS>>\n\n" + prompt + " [/INST]\n\n";
         }
         else {
-            return "<s>[INST] " + prompt + " [/INST]\n\n";
+            return (getTextCompletion(previousCompletionEnd, "</s><s>[INST] ") ?? "<s>[INST] ") + prompt + " [/INST]\n\n";
         }
     }
     getStopStrings() {
-        return ["</s><s>[INST]"];
+        return ["</s>"];
     }
 }
 //# sourceMappingURL=LlamaChatPromptWrapper.js.map

package/dist/chatWrappers/LlamaChatPromptWrapper.js.map

@@ -1 +1 @@
-{"version":3,"file":"LlamaChatPromptWrapper.js","sourceRoot":"","sources":["../../src/chatWrappers/LlamaChatPromptWrapper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,iBAAiB,EAAC,MAAM,yBAAyB,CAAC;AAE1D,mEAAmE;AACnE,MAAM,OAAO,sBAAuB,SAAQ,iBAAiB;IACzC,UAAU,CAAC,MAAc,EAAE,EAAC,YAAY,EAAE,WAAW,EAA8C;QAC/G,IAAI,WAAW,KAAK,CAAC,EAAE;YACnB,OAAO,qBAAqB,GAAG,YAAY,GAAG,gBAAgB,GAAG,MAAM,GAAG,cAAc,CAAC;SAC5F;aAAM;YACH,OAAO,YAAY,GAAG,MAAM,GAAG,cAAc,CAAC;SACjD;IACL,CAAC;IAEe,cAAc;QAC1B,OAAO,CAAC,eAAe,CAAC,CAAC;IAC7B,CAAC;CACJ"}
+{"version":3,"file":"LlamaChatPromptWrapper.js","sourceRoot":"","sources":["../../src/chatWrappers/LlamaChatPromptWrapper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,iBAAiB,EAAC,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAC,iBAAiB,EAAC,MAAM,+BAA+B,CAAC;AAEhE,mEAAmE;AACnE,MAAM,OAAO,sBAAuB,SAAQ,iBAAiB;IACzC,WAAW,GAAW,WAAW,CAAC;IAElC,UAAU,CAAC,MAAc,EAAE,EAAC,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,oBAAoB,EAE1G;QACG,MAAM,qBAAqB,GAAG,CAAC,cAAc,IAAI,EAAE,CAAC,GAAG,CAAC,oBAAoB,IAAI,EAAE,CAAC,CAAC;QAEpF,IAAI,WAAW,KAAK,CAAC,IAAI,YAAY,IAAI,EAAE,EAAE;YACzC,OAAO,CAAC,iBAAiB,CAAC,qBAAqB,EAAE,qBAAqB,CAAC,IAAI,qBAAqB,CAAC,GAAG,YAAY;gBAC5G,gBAAgB,GAAG,MAAM,GAAG,cAAc,CAAC;SAClD;aAAM;YACH,OAAO,CAAC,iBAAiB,CAAC,qBAAqB,EAAE,gBAAgB,CAAC,IAAI,YAAY,CAAC,GAAG,MAAM,GAAG,cAAc,CAAC;SACjH;IACL,CAAC;IAEe,cAAc;QAC1B,OAAO,CAAC,MAAM,CAAC,CAAC;IACpB,CAAC;CACJ"}

package/dist/chatWrappers/createChatWrapperByBos.d.ts

@@ -0,0 +1,2 @@
+import { LlamaChatPromptWrapper } from "./LlamaChatPromptWrapper.js";
+export declare function getChatWrapperByBos(bos: string | undefined | null): typeof LlamaChatPromptWrapper | null;

package/dist/chatWrappers/createChatWrapperByBos.js

@@ -0,0 +1,14 @@
+import { LlamaChatPromptWrapper } from "./LlamaChatPromptWrapper.js";
+import { ChatMLPromptWrapper } from "./ChatMLPromptWrapper.js";
+export function getChatWrapperByBos(bos) {
+    if (bos === "" || bos == null)
+        return null;
+    if ("<s>[INST] <<SYS>>\n".startsWith(bos)) {
+        return LlamaChatPromptWrapper;
+    }
+    else if ("<|im_start|>system\n".startsWith(bos)) {
+        return ChatMLPromptWrapper;
+    }
+    return null;
+}
+//# sourceMappingURL=createChatWrapperByBos.js.map

package/dist/chatWrappers/createChatWrapperByBos.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"createChatWrapperByBos.js","sourceRoot":"","sources":["../../src/chatWrappers/createChatWrapperByBos.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,sBAAsB,EAAC,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAC,mBAAmB,EAAC,MAAM,0BAA0B,CAAC;AAE7D,MAAM,UAAU,mBAAmB,CAAC,GAA8B;IAC9D,IAAI,GAAG,KAAK,EAAE,IAAI,GAAG,IAAI,IAAI;QACzB,OAAO,IAAI,CAAC;IAEhB,IAAI,qBAAqB,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;QACvC,OAAO,sBAAsB,CAAC;KACjC;SAAM,IAAI,sBAAsB,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;QAC/C,OAAO,mBAAmB,CAAC;KAC9B;IAED,OAAO,IAAI,CAAC;AAChB,CAAC"}

package/dist/cli/commands/BuildCommand.d.ts

@@ -2,7 +2,9 @@ import { CommandModule } from "yargs";
 type BuildCommand = {
     arch?: string;
     nodeTarget?: string;
+    metal: boolean;
+    cuda: boolean;
 };
 export declare const BuildCommand: CommandModule<object, BuildCommand>;
-export declare function BuildLlamaCppCommand({ arch, nodeTarget }: BuildCommand): Promise<void>;
+export declare function BuildLlamaCppCommand({ arch, nodeTarget, metal, cuda }: BuildCommand): Promise<void>;
 export {};

package/dist/cli/commands/BuildCommand.js

@@ -1,24 +1,44 @@
+import process from "process";
 import chalk from "chalk";
 import { compileLlamaCpp } from "../../utils/compileLLamaCpp.js";
 import withOra from "../../utils/withOra.js";
 import { clearTempFolder } from "../../utils/clearTempFolder.js";
+import { defaultLlamaCppCudaSupport, defaultLlamaCppMetalSupport } from "../../config.js";
 export const BuildCommand = {
     command: "build",
     describe: "Compile the currently downloaded llama.cpp",
     builder(yargs) {
         return yargs
             .option("arch", {
+            alias: "a",
             type: "string",
             description: "The architecture to compile llama.cpp for"
         })
             .option("nodeTarget", {
+            alias: "t",
             type: "string",
             description: "The Node.js version to compile llama.cpp for. Example: v18.0.0"
+        })
+            .option("metal", {
+            type: "boolean",
+            default: defaultLlamaCppMetalSupport,
+            description: "Compile llama.cpp with Metal support. Can also be set via the NODE_LLAMA_CPP_METAL environment variable"
+        })
+            .option("cuda", {
+            type: "boolean",
+            default: defaultLlamaCppCudaSupport,
+            description: "Compile llama.cpp with CUDA support. Can also be set via the NODE_LLAMA_CPP_CUDA environment variable"
         });
     },
     handler: BuildLlamaCppCommand
 };
-export async function BuildLlamaCppCommand({ arch, nodeTarget }) {
+export async function BuildLlamaCppCommand({ arch, nodeTarget, metal, cuda }) {
+    if (metal && process.platform === "darwin") {
+        console.log(`${chalk.yellow("Metal:")} enabled`);
+    }
+    if (cuda) {
+        console.log(`${chalk.yellow("CUDA:")} enabled`);
+    }
     await withOra({
         loading: chalk.blue("Compiling llama.cpp"),
         success: chalk.blue("Compiled llama.cpp"),
@@ -27,7 +47,9 @@ export async function BuildLlamaCppCommand({ arch, nodeTarget }) {
         await compileLlamaCpp({
             arch: arch ? arch : undefined,
             nodeTarget: nodeTarget ? nodeTarget : undefined,
-            setUsedBingFlag: true
+            setUsedBingFlag: true,
+            metal,
+            cuda
         });
     });
     await withOra({

package/dist/cli/commands/BuildCommand.js.map

@@ -1 +1 @@
-{"version":3,"file":"BuildCommand.js","sourceRoot":"","sources":["../../../src/cli/commands/BuildCommand.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAC,eAAe,EAAC,MAAM,gCAAgC,CAAC;AAC/D,OAAO,OAAO,MAAM,wBAAwB,CAAC;AAC7C,OAAO,EAAC,eAAe,EAAC,MAAM,gCAAgC,CAAC;AAO/D,MAAM,CAAC,MAAM,YAAY,GAAwC;IAC7D,OAAO,EAAE,OAAO;IAChB,QAAQ,EAAE,4CAA4C;IACtD,OAAO,CAAC,KAAK;QACT,OAAO,KAAK;aACP,MAAM,CAAC,MAAM,EAAE;YACZ,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,2CAA2C;SAC3D,CAAC;aACD,MAAM,CAAC,YAAY,EAAE;YAClB,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,gEAAgE;SAChF,CAAC,CAAC;IACX,CAAC;IACD,OAAO,EAAE,oBAAoB;CAChC,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,EAAC,IAAI,EAAE,UAAU,EAAe;IACvE,MAAM,OAAO,CAAC;QACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC;QAC1C,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC;QACzC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,6BAA6B,CAAC;KAClD,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,eAAe,CAAC;YAClB,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;YAC7B,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;YAC/C,eAAe,EAAE,IAAI;SACxB,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC;QACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC;QAC/C,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC;QAC9C,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,kCAAkC,CAAC;KACvD,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,eAAe,EAAE,CAAC;IAC5B,CAAC,CAAC,CAAC;AACP,CAAC"}
+{"version":3,"file":"BuildCommand.js","sourceRoot":"","sources":["../../../src/cli/commands/BuildCommand.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAC,eAAe,EAAC,MAAM,gCAAgC,CAAC;AAC/D,OAAO,OAAO,MAAM,wBAAwB,CAAC;AAC7C,OAAO,EAAC,eAAe,EAAC,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAC,0BAA0B,EAAE,2BAA2B,EAAC,MAAM,iBAAiB,CAAC;AASxF,MAAM,CAAC,MAAM,YAAY,GAAwC;IAC7D,OAAO,EAAE,OAAO;IAChB,QAAQ,EAAE,4CAA4C;IACtD,OAAO,CAAC,KAAK;QACT,OAAO,KAAK;aACP,MAAM,CAAC,MAAM,EAAE;YACZ,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,2CAA2C;SAC3D,CAAC;aACD,MAAM,CAAC,YAAY,EAAE;YAClB,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,gEAAgE;SAChF,CAAC;aACD,MAAM,CAAC,OAAO,EAAE;YACb,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,2BAA2B;YACpC,WAAW,EAAE,yGAAyG;SACzH,CAAC;aACD,MAAM,CAAC,MAAM,EAAE;YACZ,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,0BAA0B;YACnC,WAAW,EAAE,uGAAuG;SACvH,CAAC,CAAC;IACX,CAAC;IACD,OAAO,EAAE,oBAAoB;CAChC,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,EAAC,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,IAAI,EAAe;IACpF,IAAI,KAAK,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE;QACxC,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;KACpD;IAED,IAAI,IAAI,EAAE;QACN,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;KACnD;IAED,MAAM,OAAO,CAAC;QACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC;QAC1C,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC;QACzC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,6BAA6B,CAAC;KAClD,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,eAAe,CAAC;YAClB,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;YAC7B,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;YAC/C,eAAe,EAAE,IAAI;YACrB,KAAK;YACL,IAAI;SACP,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC;QACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC;QAC/C,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC;QAC9C,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,kCAAkC,CAAC;KACvD,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,eAAe,EAAE,CAAC;IAC5B,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/cli/commands/ChatCommand.d.ts

@@ -1,10 +1,16 @@
 import { CommandModule } from "yargs";
+import type { LlamaGrammar } from "../../llamaEvaluator/LlamaGrammar.js";
 type ChatCommand = {
     model: string;
     systemInfo: boolean;
     systemPrompt: string;
-    wrapper: string;
+    wrapper: "auto" | "general" | "llamaChat" | "chatML";
     contextSize: number;
+    grammar: "text" | Parameters<typeof LlamaGrammar.getFor>[0];
+    temperature: number;
+    topK: number;
+    topP: number;
+    maxTokens: number;
 };
 export declare const ChatCommand: CommandModule<object, ChatCommand>;
 export {};