node-llama-cpp 1.3.1 → 2.0.0

package/README.md

@@ -30,7 +30,7 @@ import {LlamaModel, LlamaContext, LlamaChatSession} from "node-llama-cpp";
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 const model = new LlamaModel({
-    modelPath: path.join(__dirname, "models", "vicuna-13b-v1.5-16k.ggmlv3.q5_1.bin")
+    modelPath: path.join(__dirname, "models", "codellama-13b.Q3_K_M.gguf")
 });
 const context = new LlamaContext({model});
 const session = new LlamaChatSession({context});
@@ -73,7 +73,7 @@ export class MyCustomChatPromptWrapper extends ChatPromptWrapper {
 }
 
 const model = new LlamaModel({
-    modelPath: path.join(__dirname, "models", "vicuna-13b-v1.5-16k.ggmlv3.q5_1.bin"),
+    modelPath: path.join(__dirname, "models", "codellama-13b.Q3_K_M.gguf"),
     promptWrapper: new MyCustomChatPromptWrapper() // by default, LlamaChatPromptWrapper is used
 })
 const context = new LlamaContext({model});
@@ -103,7 +103,7 @@ import {LlamaModel, LlamaContext, LlamaChatSession} from "node-llama-cpp";
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 const model = new LlamaModel({
-    modelPath: path.join(__dirname, "models", "vicuna-13b-v1.5-16k.ggmlv3.q5_1.bin")
+    modelPath: path.join(__dirname, "models", "codellama-13b.Q3_K_M.gguf")
 });
 
 const context = new LlamaContext({model});

package/dist/chatWrappers/LlamaChatPromptWrapper.js

@@ -2,7 +2,7 @@ import { ChatPromptWrapper } from "../ChatPromptWrapper.js";
 // source: https://huggingface.co/blog/llama2#how-to-prompt-llama-2
 export class LlamaChatPromptWrapper extends ChatPromptWrapper {
     wrapPrompt(prompt, { systemPrompt, promptIndex }) {
-        if (promptIndex === 0) {
+        if (promptIndex === 0 && systemPrompt != "") {
             return "<s>[INST] <<SYS>>\n" + systemPrompt + "\n<</SYS>>\n\n" + prompt + " [/INST]\n\n";
         }
         else {

package/dist/chatWrappers/LlamaChatPromptWrapper.js.map

@@ -1 +1 @@
-{"version":3,"file":"LlamaChatPromptWrapper.js","sourceRoot":"","sources":["../../src/chatWrappers/LlamaChatPromptWrapper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,iBAAiB,EAAC,MAAM,yBAAyB,CAAC;AAE1D,mEAAmE;AACnE,MAAM,OAAO,sBAAuB,SAAQ,iBAAiB;IACzC,UAAU,CAAC,MAAc,EAAE,EAAC,YAAY,EAAE,WAAW,EAA8C;QAC/G,IAAI,WAAW,KAAK,CAAC,EAAE;YACnB,OAAO,qBAAqB,GAAG,YAAY,GAAG,gBAAgB,GAAG,MAAM,GAAG,cAAc,CAAC;SAC5F;aAAM;YACH,OAAO,YAAY,GAAG,MAAM,GAAG,cAAc,CAAC;SACjD;IACL,CAAC;IAEe,cAAc;QAC1B,OAAO,CAAC,eAAe,CAAC,CAAC;IAC7B,CAAC;CACJ"}
+{"version":3,"file":"LlamaChatPromptWrapper.js","sourceRoot":"","sources":["../../src/chatWrappers/LlamaChatPromptWrapper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,iBAAiB,EAAC,MAAM,yBAAyB,CAAC;AAE1D,mEAAmE;AACnE,MAAM,OAAO,sBAAuB,SAAQ,iBAAiB;IACzC,UAAU,CAAC,MAAc,EAAE,EAAC,YAAY,EAAE,WAAW,EAA8C;QAC/G,IAAI,WAAW,KAAK,CAAC,IAAI,YAAY,IAAI,EAAE,EAAE;YACzC,OAAO,qBAAqB,GAAG,YAAY,GAAG,gBAAgB,GAAG,MAAM,GAAG,cAAc,CAAC;SAC5F;aAAM;YACH,OAAO,YAAY,GAAG,MAAM,GAAG,cAAc,CAAC;SACjD;IACL,CAAC;IAEe,cAAc;QAC1B,OAAO,CAAC,eAAe,CAAC,CAAC;IAC7B,CAAC;CACJ"}

package/dist/cli/commands/DownloadCommand.d.ts

@@ -5,7 +5,8 @@ type DownloadCommandArgs = {
     arch?: string;
     nodeTarget?: string;
     skipBuild?: boolean;
+    updateBinariesReleaseMetadata?: boolean;
 };
 export declare const DownloadCommand: CommandModule<object, DownloadCommandArgs>;
-export declare function DownloadLlamaCppCommand({ repo, release, arch, nodeTarget, skipBuild }: DownloadCommandArgs): Promise<void>;
+export declare function DownloadLlamaCppCommand({ repo, release, arch, nodeTarget, skipBuild, updateBinariesReleaseMetadata }: DownloadCommandArgs): Promise<void>;
 export {};

package/dist/cli/commands/DownloadCommand.js

@@ -10,6 +10,7 @@ import { defaultLlamaCppGitHubRepo, defaultLlamaCppRelease, llamaCppDirectory, t
 import { compileLlamaCpp } from "../../utils/compileLLamaCpp.js";
 import withOra from "../../utils/withOra.js";
 import { clearTempFolder } from "../../utils/clearTempFolder.js";
+import { setBinariesGithubRelease } from "../../utils/binariesGithubRelease.js";
 export const DownloadCommand = {
     command: "download",
     describe: "Download a release of llama.cpp and compile it",
@@ -23,7 +24,7 @@ export const DownloadCommand = {
             .option("release", {
             type: "string",
             default: defaultLlamaCppRelease,
-            description: "The tag of the llama.cpp release to download. Can also be set via the NODE_LLAMA_CPP_REPO_RELEASE environment variable"
+            description: "The tag of the llama.cpp release to download. Set to \"latest\" to download the latest release. Can also be set via the NODE_LLAMA_CPP_REPO_RELEASE environment variable"
         })
             .option("arch", {
             type: "string",
@@ -37,11 +38,17 @@ export const DownloadCommand = {
             type: "boolean",
             default: false,
             description: "Skip building llama.cpp after downloading it"
+        })
+            .option("updateBinariesReleaseMetadata", {
+            type: "boolean",
+            hidden: true,
+            default: false,
+            description: "Update the binariesGithubRelease.json file with the release of llama.cpp that was downloaded"
         });
     },
     handler: DownloadLlamaCppCommand
 };
-export async function DownloadLlamaCppCommand({ repo, release, arch, nodeTarget, skipBuild }) {
+export async function DownloadLlamaCppCommand({ repo, release, arch, nodeTarget, skipBuild, updateBinariesReleaseMetadata }) {
     const octokit = new Octokit();
     const [githubOwner, githubRepo] = repo.split("/");
     console.log(`${chalk.yellow("Repo:")} ${repo}`);
@@ -120,6 +127,9 @@ export async function DownloadLlamaCppCommand({ repo, release, arch, nodeTarget,
             setUsedBingFlag: true
         });
     }
+    if (updateBinariesReleaseMetadata) {
+        await setBinariesGithubRelease(githubRelease.data.tag_name);
+    }
     console.log();
     console.log();
     console.log(`${chalk.yellow("Repo:")} ${repo}`);

package/dist/cli/commands/DownloadCommand.js.map

@@ -1 +1 @@
-{"version":3,"file":"DownloadCommand.js","sourceRoot":"","sources":["../../../src/cli/commands/DownloadCommand.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAC,OAAO,EAAC,MAAM,SAAS,CAAC;AAChC,OAAO,KAAK,EAAE,MAAM,UAAU,CAAC;AAC/B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAC,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AACxD,OAAO,WAAW,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,SAAS,MAAM,iBAAiB,CAAC;AACxC,OAAO,EAAC,yBAAyB,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,qBAAqB,EAAC,MAAM,iBAAiB,CAAC;AAC5H,OAAO,EAAC,eAAe,EAAC,MAAM,gCAAgC,CAAC;AAC/D,OAAO,OAAO,MAAM,wBAAwB,CAAC;AAC7C,OAAO,EAAC,eAAe,EAAC,MAAM,gCAAgC,CAAC;AAU/D,MAAM,CAAC,MAAM,eAAe,GAA+C;IACvE,OAAO,EAAE,UAAU;IACnB,QAAQ,EAAE,gDAAgD;IAC1D,OAAO,CAAC,KAAK;QACT,OAAO,KAAK;aACP,MAAM,CAAC,MAAM,EAAE;YACZ,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,yBAAyB;YAClC,WAAW,EAAE,iIAAiI;SACjJ,CAAC;aACD,MAAM,CAAC,SAAS,EAAE;YACf,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,sBAAsB;YAC/B,WAAW,EAAE,wHAAwH;SACxI,CAAC;aACD,MAAM,CAAC,MAAM,EAAE;YACZ,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,2CAA2C;SAC3D,CAAC;aACD,MAAM,CAAC,YAAY,EAAE;YAClB,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,gEAAgE;SAChF,CAAC;aACD,MAAM,CAAC,WAAW,EAAE;YACjB,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,KAAK;YACd,WAAW,EAAE,8CAA8C;SAC9D,CAAC,CAAC;IACX,CAAC;IACD,OAAO,EAAE,uBAAuB;CACnC,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,EAAC,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAsB;IAC3G,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;IAC9B,MAAM,CAAC,WAAW,EAAE,UAAU,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAElD,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,OAAO,EAAE,CAAC,CAAC;IACtD,OAAO,CAAC,GAAG,EAAE,CAAC;IAKd,IAAI,aAAa,GAA6B,IAAI,CAAC;IACnD,IAAI,MAAc,CAAC;IACnB,MAAM,OAAO,CAAC;QACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC;QAC9C,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC;QAC7C,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,gCAAgC,CAAC;KACrD,EAAE,KAAK,IAAI,EAAE;QACV,IAAI;YACA,IAAI,OAAO,KAAK,QAAQ,EAAE;gBACtB,aAAa,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC;oBACtD,KAAK,EAAE,WAAW;oBAClB,IAAI,EAAE,UAAU;iBACnB,CAAC,CAAC;aACN;iBAAM;gBACH,aAAa,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC;oBACrD,KAAK,EAAE,WAAW;oBAClB,IAAI,EAAE,UAAU;oBAChB,GAAG,EAAE,OAAO;iBACf,CAAC,CAAC;aACN;SACJ;QAAC,OAAO,GAAG,EAAE;YACV,OAAO,CAAC,KAAK,CAAC,wCAAwC,EAAE,GAAG,CAAC,CAAC;SAChE;QAED,IAAI,aAAa,IAAI,IAAI,EAAE;YACvB,MAAM,IAAI,KAAK,CAAC,2BAA2B,OAAO,SAAS,IAAI,GAAG,CAAC,CAAC;SACvE;QAED,IAAI,aAAa,CAAC,IAAI,EAAE,WAAW,IAAI,IAAI,EAAE;YACzC,MAAM,IAAI,KAAK,CAAC,6CAA6C,OAAO,SAAS,IAAI,GAAG,CAAC,CAAC;SACzF;QAED,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC;YACnE,KAAK,EAAE,WAAW;YAClB,IAAI,EAAE,UAAU;YAChB,GAAG,EAAE,aAAc,CAAC,IAAI,CAAC,gBAAgB;SAC5C,CAAC,CAAC;QAEH,IAAI,cAAc,CAAC,GAAG,IAAI,IAAI;YAC1B,MAAM,IAAI,KAAK,CAAC,8CAA8C,OAAO,SAAS,IAAI,GAAG,CAAC,CAAC;QAE3F,MAAM,GAAG,cAAc,CAAC,GAAG,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,MAAM,eAAe,EAAE,CAAC;IAGxB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC,CAAC;IAChD,MAAM,EAAE,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC;IAC1C,MAAM,YAAY,CAAC,MAAO,EAAE,eAAe,EAAE,qBAAqB,CAAC,CAAC;IAEpE,MAAM,OAAO,CAAC;QACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,uCAAuC,CAAC;QAC5D,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,sCAAsC,CAAC;QAC3D,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,+CAA+C,CAAC;KACpE,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,EAAE,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC;QACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,+BAA+B,CAAC;QACpD,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,8BAA8B,CAAC;QACnD,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,sCAAsC,CAAC;KAC3D,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,eAAe,CAAC,EAAE,iBAAiB,CAAC,CAAC;IACzG,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC;QACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC;QAC/C,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC;QAC9C,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,kCAAkC,CAAC;KACvD,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,eAAe,EAAE,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,SAAS,EAAE;QACZ,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC;QAC/C,MAAM,eAAe,CAAC;YAClB,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;YAC7B,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;YAC/C,eAAe,EAAE,IAAI;SACxB,CAAC,CAAC;KACN;IAED,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,OAAO,EAAE,CAAC,CAAC;IACtD,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;AACrC,CAAC;AAGD,KAAK,UAAU,YAAY,CAAC,GAAW,EAAE,QAAgB,EAAE,SAAiB;IACxE,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,GAAG,EAAE,SAAS,EAAE;QAClD,QAAQ,EAAE,QAAQ;QAClB,KAAK,EAAE;YACH,UAAU,EAAE,EAAE;YACd,KAAK,EAAE,IAAI,GAAG,CAAC;SAClB;KACJ,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC;QACpC,eAAe,EAAE,KAAK;QACtB,UAAU,EAAE,IAAI;QAChB,WAAW,EAAE,IAAI;QACjB,MAAM,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,KAAK,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE;KACzI,EAAE,WAAW,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IACvC,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE;QACtB,KAAK,EAAE,EAAE;QACT,QAAQ,EAAE,QAAQ;KACrB,CAAC,CAAC;IAEH,QAAQ,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,KAAK,EAAE,EAAE;QAC9B,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,GAAG,GAAG,EAAE;YAC3E,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE;SACpH,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IACH,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QACpB,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACxB,WAAW,CAAC,IAAI,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,4CAA4C;IAC5C,gEAAgE;IAChE,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAE/B,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC;AAC3B,CAAC;AAED,KAAK,UAAU,wBAAwB,CAAC,WAAmB,EAAE,SAAiB;IAC1E,MAAM,GAAG,GAAG,IAAI,SAAS,CAAC,KAAK,CAAC,EAAC,IAAI,EAAE,WAAW,EAAC,CAAC,CAAC;IAErD,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,OAAO,EAAE,CAAC;IACpC,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEpD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE;QACxC,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,cAAc,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QACpC,MAAM,oBAAoB,GAAG,iBAAiB,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QACxE,iBAAiB,CAAC,GAAG,CAAC,cAAc,EAAE,oBAAoB,GAAG,CAAC,CAAC,CAAC;KACnE;IAED,MAAM,sBAAsB,GAAG,CAAC,GAAG,iBAAiB,CAAC,IAAI,EAAE,CAAC;SACvD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAE,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC;SACrE,KAAK,EAAE,CAAC;IAEb,IAAI,sBAAsB,IAAI,IAAI;QAC9B,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;IAExF,MAAM,GAAG,CAAC,OAAO,CAAC,sBAAsB,EAAE,SAAS,CAAC,CAAC;AACzD,CAAC"}
+{"version":3,"file":"DownloadCommand.js","sourceRoot":"","sources":["../../../src/cli/commands/DownloadCommand.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAC,OAAO,EAAC,MAAM,SAAS,CAAC;AAChC,OAAO,KAAK,EAAE,MAAM,UAAU,CAAC;AAC/B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAC,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AACxD,OAAO,WAAW,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,SAAS,MAAM,iBAAiB,CAAC;AACxC,OAAO,EAAC,yBAAyB,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,qBAAqB,EAAC,MAAM,iBAAiB,CAAC;AAC5H,OAAO,EAAC,eAAe,EAAC,MAAM,gCAAgC,CAAC;AAC/D,OAAO,OAAO,MAAM,wBAAwB,CAAC;AAC7C,OAAO,EAAC,eAAe,EAAC,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAC,wBAAwB,EAAC,MAAM,sCAAsC,CAAC;AAW9E,MAAM,CAAC,MAAM,eAAe,GAA+C;IACvE,OAAO,EAAE,UAAU;IACnB,QAAQ,EAAE,gDAAgD;IAC1D,OAAO,CAAC,KAAK;QACT,OAAO,KAAK;aACP,MAAM,CAAC,MAAM,EAAE;YACZ,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,yBAAyB;YAClC,WAAW,EAAE,iIAAiI;SACjJ,CAAC;aACD,MAAM,CAAC,SAAS,EAAE;YACf,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,sBAAsB;YAC/B,WAAW,EAAE,0KAA0K;SAC1L,CAAC;aACD,MAAM,CAAC,MAAM,EAAE;YACZ,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,2CAA2C;SAC3D,CAAC;aACD,MAAM,CAAC,YAAY,EAAE;YAClB,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,gEAAgE;SAChF,CAAC;aACD,MAAM,CAAC,WAAW,EAAE;YACjB,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,KAAK;YACd,WAAW,EAAE,8CAA8C;SAC9D,CAAC;aACD,MAAM,CAAC,+BAA+B,EAAE;YACrC,IAAI,EAAE,SAAS;YACf,MAAM,EAAE,IAAI;YACZ,OAAO,EAAE,KAAK;YACd,WAAW,EAAE,8FAA8F;SAC9G,CAAC,CAAC;IACX,CAAC;IACD,OAAO,EAAE,uBAAuB;CACnC,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,EAAC,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,6BAA6B,EAAsB;IAC1I,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;IAC9B,MAAM,CAAC,WAAW,EAAE,UAAU,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAElD,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,OAAO,EAAE,CAAC,CAAC;IACtD,OAAO,CAAC,GAAG,EAAE,CAAC;IAKd,IAAI,aAAa,GAA6B,IAAI,CAAC;IACnD,IAAI,MAAc,CAAC;IACnB,MAAM,OAAO,CAAC;QACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC;QAC9C,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC;QAC7C,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,gCAAgC,CAAC;KACrD,EAAE,KAAK,IAAI,EAAE;QACV,IAAI;YACA,IAAI,OAAO,KAAK,QAAQ,EAAE;gBACtB,aAAa,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC;oBACtD,KAAK,EAAE,WAAW;oBAClB,IAAI,EAAE,UAAU;iBACnB,CAAC,CAAC;aACN;iBAAM;gBACH,aAAa,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC;oBACrD,KAAK,EAAE,WAAW;oBAClB,IAAI,EAAE,UAAU;oBAChB,GAAG,EAAE,OAAO;iBACf,CAAC,CAAC;aACN;SACJ;QAAC,OAAO,GAAG,EAAE;YACV,OAAO,CAAC,KAAK,CAAC,wCAAwC,EAAE,GAAG,CAAC,CAAC;SAChE;QAED,IAAI,aAAa,IAAI,IAAI,EAAE;YACvB,MAAM,IAAI,KAAK,CAAC,2BAA2B,OAAO,SAAS,IAAI,GAAG,CAAC,CAAC;SACvE;QAED,IAAI,aAAa,CAAC,IAAI,EAAE,WAAW,IAAI,IAAI,EAAE;YACzC,MAAM,IAAI,KAAK,CAAC,6CAA6C,OAAO,SAAS,IAAI,GAAG,CAAC,CAAC;SACzF;QAED,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC;YACnE,KAAK,EAAE,WAAW;YAClB,IAAI,EAAE,UAAU;YAChB,GAAG,EAAE,aAAc,CAAC,IAAI,CAAC,gBAAgB;SAC5C,CAAC,CAAC;QAEH,IAAI,cAAc,CAAC,GAAG,IAAI,IAAI;YAC1B,MAAM,IAAI,KAAK,CAAC,8CAA8C,OAAO,SAAS,IAAI,GAAG,CAAC,CAAC;QAE3F,MAAM,GAAG,cAAc,CAAC,GAAG,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,MAAM,eAAe,EAAE,CAAC;IAGxB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC,CAAC;IAChD,MAAM,EAAE,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC;IAC1C,MAAM,YAAY,CAAC,MAAO,EAAE,eAAe,EAAE,qBAAqB,CAAC,CAAC;IAEpE,MAAM,OAAO,CAAC;QACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,uCAAuC,CAAC;QAC5D,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,sCAAsC,CAAC;QAC3D,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,+CAA+C,CAAC;KACpE,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,EAAE,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC;QACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,+BAA+B,CAAC;QACpD,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,8BAA8B,CAAC;QACnD,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,sCAAsC,CAAC;KAC3D,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,eAAe,CAAC,EAAE,iBAAiB,CAAC,CAAC;IACzG,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC;QACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC;QAC/C,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC;QAC9C,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,kCAAkC,CAAC;KACvD,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,eAAe,EAAE,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,SAAS,EAAE;QACZ,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC;QAC/C,MAAM,eAAe,CAAC;YAClB,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;YAC7B,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;YAC/C,eAAe,EAAE,IAAI;SACxB,CAAC,CAAC;KACN;IAED,IAAI,6BAA6B,EAAE;QAC/B,MAAM,wBAAwB,CAAC,aAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;KAChE;IAED,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,OAAO,EAAE,CAAC,CAAC;IACtD,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;AACrC,CAAC;AAGD,KAAK,UAAU,YAAY,CAAC,GAAW,EAAE,QAAgB,EAAE,SAAiB;IACxE,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,GAAG,EAAE,SAAS,EAAE;QAClD,QAAQ,EAAE,QAAQ;QAClB,KAAK,EAAE;YACH,UAAU,EAAE,EAAE;YACd,KAAK,EAAE,IAAI,GAAG,CAAC;SAClB;KACJ,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC;QACpC,eAAe,EAAE,KAAK;QACtB,UAAU,EAAE,IAAI;QAChB,WAAW,EAAE,IAAI;QACjB,MAAM,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,KAAK,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE;KACzI,EAAE,WAAW,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IACvC,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE;QACtB,KAAK,EAAE,EAAE;QACT,QAAQ,EAAE,QAAQ;KACrB,CAAC,CAAC;IAEH,QAAQ,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,KAAK,EAAE,EAAE;QAC9B,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,GAAG,GAAG,EAAE;YAC3E,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE;SACpH,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IACH,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QACpB,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACxB,WAAW,CAAC,IAAI,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,4CAA4C;IAC5C,gEAAgE;IAChE,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAE/B,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC;AAC3B,CAAC;AAED,KAAK,UAAU,wBAAwB,CAAC,WAAmB,EAAE,SAAiB;IAC1E,MAAM,GAAG,GAAG,IAAI,SAAS,CAAC,KAAK,CAAC,EAAC,IAAI,EAAE,WAAW,EAAC,CAAC,CAAC;IAErD,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,OAAO,EAAE,CAAC;IACpC,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEpD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE;QACxC,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,cAAc,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QACpC,MAAM,oBAAoB,GAAG,iBAAiB,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QACxE,iBAAiB,CAAC,GAAG,CAAC,cAAc,EAAE,oBAAoB,GAAG,CAAC,CAAC,CAAC;KACnE;IAED,MAAM,sBAAsB,GAAG,CAAC,GAAG,iBAAiB,CAAC,IAAI,EAAE,CAAC;SACvD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAE,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC;SACrE,KAAK,EAAE,CAAC;IAEb,IAAI,sBAAsB,IAAI,IAAI;QAC9B,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;IAExF,MAAM,GAAG,CAAC,OAAO,CAAC,sBAAsB,EAAE,SAAS,CAAC,CAAC;AACzD,CAAC"}

package/dist/config.d.ts

@@ -3,6 +3,7 @@ export declare const llamaBinsDirectory: string;
 export declare const llamaCppDirectory: string;
 export declare const tempDownloadDirectory: string;
 export declare const usedBinFlagJsonPath: string;
+export declare const binariesGithubReleasePath: string;
 export declare const defaultLlamaCppGitHubRepo: string;
 export declare const defaultLlamaCppRelease: string;
 export declare const defaultSkipDownload: boolean;

package/dist/config.js

@@ -3,6 +3,7 @@ import * as path from "path";
 import * as os from "os";
 import envVar from "env-var";
 import * as uuid from "uuid";
+import { getBinariesGithubRelease } from "./utils/binariesGithubRelease.js";
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const env = envVar.from(process.env);
 export const llamaDirectory = path.join(__dirname, "..", "llama");
@@ -10,11 +11,12 @@ export const llamaBinsDirectory = path.join(__dirname, "..", "llamaBins");
 export const llamaCppDirectory = path.join(llamaDirectory, "llama.cpp");
 export const tempDownloadDirectory = path.join(os.tmpdir(), "node-llama-cpp", uuid.v4());
 export const usedBinFlagJsonPath = path.join(llamaDirectory, "usedBin.json");
+export const binariesGithubReleasePath = path.join(llamaDirectory, "binariesGithubRelease.json");
 export const defaultLlamaCppGitHubRepo = env.get("NODE_LLAMA_CPP_REPO")
     .default("ggerganov/llama.cpp")
     .asString();
 export const defaultLlamaCppRelease = env.get("NODE_LLAMA_CPP_REPO_RELEASE")
-    .default("latest")
+    .default(await getBinariesGithubRelease())
     .asString();
 export const defaultSkipDownload = env.get("NODE_LLAMA_CPP_SKIP_DOWNLOAD")
     .default("false")

package/dist/config.js.map

@@ -1 +1 @@
-{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,KAAK,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,MAAM,MAAM,SAAS,CAAC;AAC7B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE/D,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;AAGrC,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;AAClE,MAAM,CAAC,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;AAC1E,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;AACxE,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,gBAAgB,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;AACzF,MAAM,CAAC,MAAM,mBAAmB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,cAAc,CAAC,CAAC;AAE7E,MAAM,CAAC,MAAM,yBAAyB,GAAG,GAAG,CAAC,GAAG,CAAC,qBAAqB,CAAC;KAClE,OAAO,CAAC,qBAAqB,CAAC;KAC9B,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC,GAAG,CAAC,6BAA6B,CAAC;KACvE,OAAO,CAAC,QAAQ,CAAC;KACjB,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,mBAAmB,GAAG,GAAG,CAAC,GAAG,CAAC,8BAA8B,CAAC;KACrE,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,uBAAuB,GAAG,+FAA+F;IAClI,+HAA+H;IAC/H,mFAAmF,CAAC"}
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,KAAK,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,MAAM,MAAM,SAAS,CAAC;AAC7B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAC,wBAAwB,EAAC,MAAM,kCAAkC,CAAC;AAE1E,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE/D,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;AAGrC,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;AAClE,MAAM,CAAC,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;AAC1E,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;AACxE,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,gBAAgB,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;AACzF,MAAM,CAAC,MAAM,mBAAmB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,cAAc,CAAC,CAAC;AAC7E,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,4BAA4B,CAAC,CAAC;AAEjG,MAAM,CAAC,MAAM,yBAAyB,GAAG,GAAG,CAAC,GAAG,CAAC,qBAAqB,CAAC;KAClE,OAAO,CAAC,qBAAqB,CAAC;KAC9B,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC,GAAG,CAAC,6BAA6B,CAAC;KACvE,OAAO,CAAC,MAAM,wBAAwB,EAAE,CAAC;KACzC,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,mBAAmB,GAAG,GAAG,CAAC,GAAG,CAAC,8BAA8B,CAAC;KACrE,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,uBAAuB,GAAG,+FAA+F;IAClI,+HAA+H;IAC/H,mFAAmF,CAAC"}

package/dist/llamaEvaluator/LlamaContext.d.ts

@@ -8,5 +8,5 @@ export declare class LlamaContext {
     });
     encode(text: string): Uint32Array;
     decode(tokens: Uint32Array): string;
-    evaluate(tokens: Uint32Array, getRestrictions?: () => Uint32Array): AsyncGenerator<number, void, unknown>;
+    evaluate(tokens: Uint32Array): AsyncGenerator<number, void, unknown>;
 }

package/dist/llamaEvaluator/LlamaContext.js

@@ -1,4 +1,4 @@
-import { LLAMAContext, llamaCppNode } from "./LlamaBins.js";
+import { LLAMAContext } from "./LlamaBins.js";
 export class LlamaContext {
     _ctx;
     _prependBos;
@@ -12,20 +12,20 @@ export class LlamaContext {
     decode(tokens) {
         return this._ctx.decode(tokens);
     }
-    async *evaluate(tokens, getRestrictions) {
+    async *evaluate(tokens) {
         let evalTokens = tokens;
         if (this._prependBos) {
             const tokenArray = Array.from(tokens);
-            tokenArray.unshift(llamaCppNode.tokenBos());
+            tokenArray.unshift(this._ctx.tokenBos());
             evalTokens = Uint32Array.from(tokenArray);
             this._prependBos = false;
         }
         // eslint-disable-next-line no-constant-condition
         while (true) {
             // Evaluate to get the next token.
-            const nextToken = await this._ctx.eval(evalTokens, getRestrictions?.());
+            const nextToken = await this._ctx.eval(evalTokens);
             // the assistant finished answering
-            if (nextToken === llamaCppNode.tokenEos())
+            if (nextToken === this._ctx.tokenEos())
                 break;
             yield nextToken;
             // Create tokens for the next eval.

package/dist/llamaEvaluator/LlamaContext.js.map

@@ -1 +1 @@
-{"version":3,"file":"LlamaContext.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaContext.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAE,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAG1D,MAAM,OAAO,YAAY;IACJ,IAAI,CAAe;IAC5B,WAAW,CAAU;IAE7B,YAAmB,EAAC,KAAK,EAAE,UAAU,GAAG,IAAI,EAA4C;QACpF,IAAI,CAAC,IAAI,GAAG,IAAI,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC3C,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,IAAY;QACtB,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,MAAmB;QAC7B,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAEM,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAmB,EAAE,eAAmC;QAC3E,IAAI,UAAU,GAAG,MAAM,CAAC;QAExB,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtC,UAAU,CAAC,OAAO,CAAC,YAAY,CAAC,QAAQ,EAAE,CAAC,CAAC;YAE5C,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC1C,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;SAC5B;QAED,iDAAiD;QACjD,OAAO,IAAI,EAAE;YACT,kCAAkC;YAClC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;YAExE,mCAAmC;YACnC,IAAI,SAAS,KAAK,YAAY,CAAC,QAAQ,EAAE;gBACrC,MAAM;YAEV,MAAM,SAAS,CAAC;YAEhB,mCAAmC;YACnC,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;SAC9C;IACL,CAAC;CAEJ"}
+{"version":3,"file":"LlamaContext.js","sourceRoot":"","sources":["../../src/llamaEvaluator/LlamaContext.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAG5C,MAAM,OAAO,YAAY;IACJ,IAAI,CAAe;IAC5B,WAAW,CAAU;IAE7B,YAAmB,EAAC,KAAK,EAAE,UAAU,GAAG,IAAI,EAA4C;QACpF,IAAI,CAAC,IAAI,GAAG,IAAI,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC3C,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,IAAY;QACtB,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,MAAmB;QAC7B,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAEM,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAmB;QACtC,IAAI,UAAU,GAAG,MAAM,CAAC;QAExB,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEzC,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC1C,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;SAC5B;QAED,iDAAiD;QACjD,OAAO,IAAI,EAAE;YACT,kCAAkC;YAClC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAEnD,mCAAmC;YACnC,IAAI,SAAS,KAAK,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE;gBAClC,MAAM;YAEV,MAAM,SAAS,CAAC;YAEhB,mCAAmC;YACnC,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;SAC9C;IACL,CAAC;CAEJ"}

package/dist/utils/binariesGithubRelease.d.ts

@@ -0,0 +1,6 @@
+type BinariesGithubReleaseFile = {
+    release: "latest" | string;
+};
+export declare function getBinariesGithubRelease(): Promise<string>;
+export declare function setBinariesGithubRelease(release: BinariesGithubReleaseFile["release"]): Promise<void>;
+export {};

package/dist/utils/binariesGithubRelease.js

@@ -0,0 +1,15 @@
+import fs from "fs-extra";
+import { binariesGithubReleasePath } from "../config.js";
+export async function getBinariesGithubRelease() {
+    const binariesGithubRelease = await fs.readJson(binariesGithubReleasePath);
+    return binariesGithubRelease.release;
+}
+export async function setBinariesGithubRelease(release) {
+    const binariesGithubReleaseJson = {
+        release: release
+    };
+    await fs.writeJson(binariesGithubReleasePath, binariesGithubReleaseJson, {
+        spaces: 4
+    });
+}
+//# sourceMappingURL=binariesGithubRelease.js.map

package/dist/utils/binariesGithubRelease.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"binariesGithubRelease.js","sourceRoot":"","sources":["../../src/utils/binariesGithubRelease.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,yBAAyB,EAAC,MAAM,cAAc,CAAC;AAMvD,MAAM,CAAC,KAAK,UAAU,wBAAwB;IAC1C,MAAM,qBAAqB,GAA8B,MAAM,EAAE,CAAC,QAAQ,CAAC,yBAAyB,CAAC,CAAC;IAEtG,OAAO,qBAAqB,CAAC,OAAO,CAAC;AACzC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,OAA6C;IACxF,MAAM,yBAAyB,GAA8B;QACzD,OAAO,EAAE,OAAO;KACnB,CAAC;IAEF,MAAM,EAAE,CAAC,SAAS,CAAC,yBAAyB,EAAE,yBAAyB,EAAE;QACrE,MAAM,EAAE,CAAC;KACZ,CAAC,CAAC;AACP,CAAC"}

package/dist/utils/getBin.d.ts

@@ -3,9 +3,7 @@ export declare function loadBin(): Promise<LlamaCppNodeModule>;
 export type LlamaCppNodeModule = {
     LLAMAModel: LLAMAModel;
     LLAMAContext: LLAMAContext;
-    tokenBos(): number;
     systemInfo(): string;
-    tokenEos(): number;
 };
 export type LLAMAModel = {
     new (modelPath: string, params: {
@@ -25,6 +23,9 @@ export type LLAMAModel = {
 export type LLAMAContext = {
     new (model: LLAMAModel): LLAMAContext;
     encode(text: string): Uint32Array;
-    eval(tokens: Uint32Array, restrictions?: Uint32Array): Promise<number>;
+    eval(tokens: Uint32Array): Promise<number>;
     decode(tokens: Uint32Array): string;
+    tokenBos(): number;
+    tokenEos(): number;
+    getMaxContextSize(): number;
 };

package/llama/addon.cpp

@@ -67,6 +67,7 @@ class LLAMAModel : public Napi::ObjectWrap<LLAMAModel> {
             }
         }
 
+        llama_backend_init(false);
         model = llama_load_model_from_file(modelPath.c_str(), params);
 
         if (model == NULL) {
@@ -124,7 +125,18 @@ class LLAMAContext : public Napi::ObjectWrap<LLAMAContext> {
 
     // Decode each token and accumulate the result.
     for (size_t i = 0; i < tokens.ElementLength(); i++) {
-      const char* str = llama_token_to_str(ctx, (llama_token)tokens[i]);
+      // source: https://github.com/ggerganov/llama.cpp/blob/232caf3c1581a6cb023571780ff41dc2d66d1ca0/llama.cpp#L799-L811
+      std::vector<char> result(8, 0);
+      const int n_tokens = llama_token_to_str(ctx, (llama_token)tokens[i], result.data(), result.size());
+      if (n_tokens < 0) {
+          result.resize(-n_tokens);
+          int check = llama_token_to_str(ctx, (llama_token)tokens[i], result.data(), result.size());
+          GGML_ASSERT(check == -n_tokens);
+      } else {
+          result.resize(n_tokens);
+      }
+
+      const char* str = result.data();
       if (str == nullptr) {
         Napi::Error::New(info.Env(), "Invalid token").ThrowAsJavaScriptException();
         return info.Env().Undefined();
@@ -134,6 +146,15 @@ class LLAMAContext : public Napi::ObjectWrap<LLAMAContext> {
 
     return Napi::String::New(info.Env(), ss.str());
   }
+  Napi::Value TokenBos(const Napi::CallbackInfo& info) {
+    return Napi::Number::From(info.Env(), llama_token_bos(ctx));
+  }
+  Napi::Value TokenEos(const Napi::CallbackInfo& info) {
+    return Napi::Number::From(info.Env(), llama_token_eos(ctx));
+  }
+  Napi::Value GetMaxContextSize(const Napi::CallbackInfo& info) {
+    return Napi::Number::From(info.Env(), llama_n_ctx(ctx));
+  }
   Napi::Value Eval(const Napi::CallbackInfo& info);
   static void init(Napi::Object exports) {
     exports.Set("LLAMAContext",
@@ -142,6 +163,9 @@ class LLAMAContext : public Napi::ObjectWrap<LLAMAContext> {
             {
                 InstanceMethod("encode", &LLAMAContext::Encode),
                 InstanceMethod("decode", &LLAMAContext::Decode),
+                InstanceMethod("tokenBos", &LLAMAContext::TokenBos),
+                InstanceMethod("tokenEos", &LLAMAContext::TokenEos),
+                InstanceMethod("getMaxContextSize", &LLAMAContext::GetMaxContextSize),
                 InstanceMethod("eval", &LLAMAContext::Eval),
             }));
   }
@@ -151,7 +175,6 @@ class LLAMAContext : public Napi::ObjectWrap<LLAMAContext> {
 class LLAMAContextEvalWorker : Napi::AsyncWorker, Napi::Promise::Deferred {
   LLAMAContext* ctx;
   std::vector<llama_token> tokens;
-  std::vector<llama_token> restriction;
   llama_token result;
 
   public:
@@ -160,13 +183,6 @@ class LLAMAContextEvalWorker : Napi::AsyncWorker, Napi::Promise::Deferred {
     Napi::Uint32Array tokens = info[0].As<Napi::Uint32Array>();
     this->tokens.reserve(tokens.ElementLength());
     for (size_t i = 0; i < tokens.ElementLength(); i++) { this->tokens.push_back(static_cast<llama_token>(tokens[i])); }
-
-    if (info.Length() > 1 && info[1].IsTypedArray()) {
-      Napi::Uint32Array restriction = info[1].As<Napi::Uint32Array>();
-      this->restriction.reserve(restriction.ElementLength());
-      for (size_t i = 0; i < restriction.ElementLength(); i++) { this->restriction.push_back(static_cast<llama_token>(restriction[i])); }
-      std::sort(this->restriction.begin(), this->restriction.end());
-    }
   }
   ~LLAMAContextEvalWorker() { ctx->Unref(); }
   using Napi::AsyncWorker::Queue;
@@ -175,39 +191,30 @@ class LLAMAContextEvalWorker : Napi::AsyncWorker, Napi::Promise::Deferred {
   protected:
   void Execute() {
     // Perform the evaluation using llama_eval.
-    int r = llama_eval(ctx->ctx, tokens.data(), tokens.size(), llama_get_kv_cache_token_count(ctx->ctx), 6);
+    int r = llama_eval(ctx->ctx, tokens.data(), int(tokens.size()), llama_get_kv_cache_token_count(ctx->ctx), 6);
     if (r != 0) {
       SetError("Eval has failed");
       return;
     }
 
+    llama_token new_token_id = 0;
+
     // Select the best prediction.
-    float* logits = llama_get_logits(ctx->ctx);
-    int n_vocab = llama_n_vocab(ctx->ctx);
-    llama_token re;
-    if (restriction.empty()) {
-      float max = logits[0];
-      re = 0;
-      for (llama_token id = 1; id < n_vocab; id++) {
-        float logit = logits[id];
-        if (logit > max) {
-          max = logit;
-          re = id;
-        }
-      }
-    } else {
-      float max = logits[restriction[0]];
-      re = 0;
-      for (size_t i = 1; i < restriction.size(); i++) {
-        llama_token id = restriction[i];
-        float logit = logits[id];
-        if (logit > max) {
-          max = logit;
-          re = id;
-        }
-      }
+    auto logits = llama_get_logits(ctx->ctx);
+    auto n_vocab = llama_n_vocab(ctx->ctx);
+
+    std::vector<llama_token_data> candidates;
+    candidates.reserve(n_vocab);
+
+    for (llama_token token_id = 0; token_id < n_vocab; token_id++) {
+      candidates.emplace_back(llama_token_data{ token_id, logits[token_id], 0.0f });
     }
-    result = re;
+
+    llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false };
+
+    new_token_id = llama_sample_token_greedy(ctx->ctx , &candidates_p);
+
+    result = new_token_id;
   }
   void OnOK() {
     Napi::Env env = Napi::AsyncWorker::Env();
@@ -223,15 +230,11 @@ Napi::Value LLAMAContext::Eval(const Napi::CallbackInfo& info) {
   return worker->Promise();
 }
 
-Napi::Value tokenBos(const Napi::CallbackInfo& info) { return Napi::Number::From(info.Env(), llama_token_bos()); }
-Napi::Value tokenEos(const Napi::CallbackInfo& info) { return Napi::Number::From(info.Env(), llama_token_eos()); }
 Napi::Value systemInfo(const Napi::CallbackInfo& info) { return Napi::String::From(info.Env(), llama_print_system_info()); }
 
 Napi::Object registerCallback(Napi::Env env, Napi::Object exports) {
   llama_backend_init(false);
   exports.DefineProperties({
-      Napi::PropertyDescriptor::Function("tokenBos", tokenBos),
-      Napi::PropertyDescriptor::Function("tokenEos", tokenEos),
       Napi::PropertyDescriptor::Function("systemInfo", systemInfo),
   });
   LLAMAModel::init(exports);

package/llama/binariesGithubRelease.json

@@ -0,0 +1,3 @@
+{
+    "release": "b1069"
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "node-llama-cpp",
-  "version": "1.3.1",
+  "version": "2.0.0",
   "description": "node.js bindings for llama.cpp",
   "main": "dist/index.js",
   "type": "module",
@@ -68,15 +68,14 @@
     "node-gyp",
     "prebuilt-binaries",
     "llm",
-    "ggml",
-    "ggmlv3",
+    "gguf",
     "raspberry-pi",
     "self-hosted",
     "local",
     "catai"
   ],
   "author": "Gilad S.",
-  "license": "ISC",
+  "license": "MIT",
   "bugs": {
     "url": "https://github.com/withcatai/node-llama-cpp/issues"
   },