node-llama-cpp 2.0.0 → 2.1.0

package/dist/utils/getBin.js

@@ -3,7 +3,7 @@ import * as console from "console";
 import path from "path";
 import process from "process";
 import fs from "fs-extra";
-import { defaultLlamaCppGitHubRepo, defaultLlamaCppRelease, defaultSkipDownload, llamaBinsDirectory } from "../config.js";
+import { defaultLlamaCppCudaSupport, defaultLlamaCppGitHubRepo, defaultLlamaCppMetalSupport, defaultLlamaCppRelease, defaultSkipDownload, llamaBinsDirectory } from "../config.js";
 import { DownloadLlamaCppCommand } from "../cli/commands/DownloadCommand.js";
 import { getUsedBinFlag } from "./usedBinFlag.js";
 import { getCompiledLlamaCppBinaryPath } from "./compileLLamaCpp.js";
@@ -56,7 +56,9 @@ export async function loadBin() {
         else {
             await DownloadLlamaCppCommand({
                 repo: defaultLlamaCppGitHubRepo,
-                release: defaultLlamaCppRelease
+                release: defaultLlamaCppRelease,
+                metal: defaultLlamaCppMetalSupport,
+                cuda: defaultLlamaCppCudaSupport
             });
             const modulePath = await getCompiledLlamaCppBinaryPath();
             if (modulePath == null) {

package/dist/utils/getBin.js.map

@@ -1 +1 @@
-{"version":3,"file":"getBin.js","sourceRoot":"","sources":["../../src/utils/getBin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,QAAQ,CAAC;AACrC,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,yBAAyB,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,kBAAkB,EAAC,MAAM,cAAc,CAAC;AACxH,OAAO,EAAC,uBAAuB,EAAC,MAAM,oCAAoC,CAAC;AAC3E,OAAO,EAAC,cAAc,EAAC,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAC,6BAA6B,EAAC,MAAM,sBAAsB,CAAC;AAEnE,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAE/C,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACpC,MAAM,gBAAgB,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IACrE,MAAM,iBAAiB,GAAG,CAAC,gBAAgB,EAAE,gBAAgB,GAAG,CAAC,CAAC,CAAC;IAEnE,SAAS,UAAU,CAAC,QAAgB,EAAE,IAAY,EAAE,WAAmB;QACnE,OAAO,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,QAAQ,IAAI,IAAI,IAAI,WAAW,OAAO,CAAC,CAAC;IACpF,CAAC;IAED,KAAK,UAAU,WAAW,CAAC,QAAgB,EAAE,IAAY,EAAE,YAAsB;QAC7E,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE;YACpC,MAAM,OAAO,GAAG,UAAU,CAAC,QAAQ,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;YAExD,IAAI,MAAM,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC;gBACxB,OAAO,OAAO,CAAC;SACtB;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,KAAK,UAAU,OAAO;QAClB,QAAQ,OAAO,CAAC,QAAQ,EAAE;YACtB,KAAK,OAAO,CAAC;YACb,KAAK,QAAQ;gBACT,OAAO,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;YAE/D,KAAK,OAAO,CAAC;YACb,KAAK,SAAS;gBACV,OAAO,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;YAEjE,KAAK,QAAQ;gBACT,OAAO,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;SAClE;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,OAAO,MAAM,OAAO,EAAE,CAAC;AAC3B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO;IACzB,MAAM,YAAY,GAAG,MAAM,cAAc,EAAE,CAAC;IAE5C,IAAI,YAAY,KAAK,kBAAkB,EAAE;QACrC,MAAM,eAAe,GAAG,MAAM,kBAAkB,EAAE,CAAC;QAEnD,IAAI,eAAe,IAAI,IAAI,EAAE;YACzB,OAAO,CAAC,IAAI,CAAC,wEAAwE,CAAC,CAAC;SAC1F;aAAM;YACH,OAAO,OAAO,CAAC,eAAe,CAAC,CAAC;SACnC;KACJ;IAED,MAAM,UAAU,GAAG,MAAM,6BAA6B,EAAE,CAAC;IAEzD,IAAI,UAAU,IAAI,IAAI,EAAE;QACpB,IAAI,mBAAmB,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,oFAAoF,CAAC,CAAC;SACzG;aAAM;YACH,MAAM,uBAAuB,CAAC;gBAC1B,IAAI,EAAE,yBAAyB;gBAC/B,OAAO,EAAE,sBAAsB;aAClC,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,MAAM,6BAA6B,EAAE,CAAC;YAEzD,IAAI,UAAU,IAAI,IAAI,EAAE;gBACpB,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;aAC/D;YAED,OAAO,OAAO,CAAC,UAAU,CAAC,CAAC;SAC9B;KACJ;IAED,OAAO,OAAO,CAAC,UAAU,CAAC,CAAC;AAC/B,CAAC"}
+{"version":3,"file":"getBin.js","sourceRoot":"","sources":["../../src/utils/getBin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,QAAQ,CAAC;AACrC,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EACH,0BAA0B,EAAE,yBAAyB,EAAE,2BAA2B,EAAE,sBAAsB,EAAE,mBAAmB,EAC/H,kBAAkB,EACrB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAC,uBAAuB,EAAC,MAAM,oCAAoC,CAAC;AAC3E,OAAO,EAAC,cAAc,EAAC,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAC,6BAA6B,EAAC,MAAM,sBAAsB,CAAC;AAEnE,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAE/C,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACpC,MAAM,gBAAgB,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IACrE,MAAM,iBAAiB,GAAG,CAAC,gBAAgB,EAAE,gBAAgB,GAAG,CAAC,CAAC,CAAC;IAEnE,SAAS,UAAU,CAAC,QAAgB,EAAE,IAAY,EAAE,WAAmB;QACnE,OAAO,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,QAAQ,IAAI,IAAI,IAAI,WAAW,OAAO,CAAC,CAAC;IACpF,CAAC;IAED,KAAK,UAAU,WAAW,CAAC,QAAgB,EAAE,IAAY,EAAE,YAAsB;QAC7E,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE;YACpC,MAAM,OAAO,GAAG,UAAU,CAAC,QAAQ,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;YAExD,IAAI,MAAM,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC;gBACxB,OAAO,OAAO,CAAC;SACtB;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,KAAK,UAAU,OAAO;QAClB,QAAQ,OAAO,CAAC,QAAQ,EAAE;YACtB,KAAK,OAAO,CAAC;YACb,KAAK,QAAQ;gBACT,OAAO,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;YAE/D,KAAK,OAAO,CAAC;YACb,KAAK,SAAS;gBACV,OAAO,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;YAEjE,KAAK,QAAQ;gBACT,OAAO,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;SAClE;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,OAAO,MAAM,OAAO,EAAE,CAAC;AAC3B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO;IACzB,MAAM,YAAY,GAAG,MAAM,cAAc,EAAE,CAAC;IAE5C,IAAI,YAAY,KAAK,kBAAkB,EAAE;QACrC,MAAM,eAAe,GAAG,MAAM,kBAAkB,EAAE,CAAC;QAEnD,IAAI,eAAe,IAAI,IAAI,EAAE;YACzB,OAAO,CAAC,IAAI,CAAC,wEAAwE,CAAC,CAAC;SAC1F;aAAM;YACH,OAAO,OAAO,CAAC,eAAe,CAAC,CAAC;SACnC;KACJ;IAED,MAAM,UAAU,GAAG,MAAM,6BAA6B,EAAE,CAAC;IAEzD,IAAI,UAAU,IAAI,IAAI,EAAE;QACpB,IAAI,mBAAmB,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,oFAAoF,CAAC,CAAC;SACzG;aAAM;YACH,MAAM,uBAAuB,CAAC;gBAC1B,IAAI,EAAE,yBAAyB;gBAC/B,OAAO,EAAE,sBAAsB;gBAC/B,KAAK,EAAE,2BAA2B;gBAClC,IAAI,EAAE,0BAA0B;aACnC,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,MAAM,6BAA6B,EAAE,CAAC;YAEzD,IAAI,UAAU,IAAI,IAAI,EAAE;gBACpB,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;aAC/D;YAED,OAAO,OAAO,CAAC,UAAU,CAAC,CAAC;SAC9B;KACJ;IAED,OAAO,OAAO,CAAC,UAAU,CAAC,CAAC;AAC/B,CAAC"}

package/dist/utils/getGrammarsFolder.d.ts

@@ -0,0 +1 @@
+export declare function getGrammarsFolder(): Promise<string>;

package/dist/utils/getGrammarsFolder.js

@@ -0,0 +1,18 @@
+import fs from "fs-extra";
+import { llamaBinsGrammarsDirectory, llamaCppGrammarsDirectory } from "../config.js";
+import { getUsedBinFlag } from "./usedBinFlag.js";
+export async function getGrammarsFolder() {
+    const usedBingFlag = await getUsedBinFlag();
+    if (usedBingFlag === "localBuildFromSource") {
+        if (await fs.exists(llamaCppGrammarsDirectory))
+            return llamaCppGrammarsDirectory;
+    }
+    else if (usedBingFlag === "prebuiltBinaries") {
+        if (await fs.exists(llamaBinsGrammarsDirectory))
+            return llamaBinsGrammarsDirectory;
+        else if (await fs.exists(llamaCppGrammarsDirectory))
+            return llamaCppGrammarsDirectory;
+    }
+    throw new Error("Grammars folder not found");
+}
+//# sourceMappingURL=getGrammarsFolder.js.map

package/dist/utils/getGrammarsFolder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"getGrammarsFolder.js","sourceRoot":"","sources":["../../src/utils/getGrammarsFolder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,0BAA0B,EAAE,yBAAyB,EAAC,MAAM,cAAc,CAAC;AACnF,OAAO,EAAC,cAAc,EAAC,MAAM,kBAAkB,CAAC;AAEhD,MAAM,CAAC,KAAK,UAAU,iBAAiB;IACnC,MAAM,YAAY,GAAG,MAAM,cAAc,EAAE,CAAC;IAE5C,IAAI,YAAY,KAAK,sBAAsB,EAAE;QACzC,IAAI,MAAM,EAAE,CAAC,MAAM,CAAC,yBAAyB,CAAC;YAC1C,OAAO,yBAAyB,CAAC;KACxC;SAAM,IAAI,YAAY,KAAK,kBAAkB,EAAE;QAC5C,IAAI,MAAM,EAAE,CAAC,MAAM,CAAC,0BAA0B,CAAC;YAC3C,OAAO,0BAA0B,CAAC;aACjC,IAAI,MAAM,EAAE,CAAC,MAAM,CAAC,yBAAyB,CAAC;YAC/C,OAAO,yBAAyB,CAAC;KACxC;IAED,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;AACjD,CAAC"}

package/dist/utils/getTextCompletion.d.ts

@@ -0,0 +1,3 @@
+export declare function getTextCompletion(text: null, fullText: string | string[]): null;
+export declare function getTextCompletion(text: string, fullText: string | string[]): string | null;
+export declare function getTextCompletion(text: string | null, fullText: string | string[]): string | null;

package/dist/utils/getTextCompletion.js

@@ -0,0 +1,12 @@
+export function getTextCompletion(text, fullText) {
+    if (text == null) {
+        return null;
+    }
+    const fullTexts = typeof fullText === "string" ? [fullText] : fullText;
+    for (const fullText of fullTexts) {
+        if (fullText.startsWith(text))
+            return fullText.slice(text.length);
+    }
+    return null;
+}
+//# sourceMappingURL=getTextCompletion.js.map

package/dist/utils/getTextCompletion.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"getTextCompletion.js","sourceRoot":"","sources":["../../src/utils/getTextCompletion.ts"],"names":[],"mappings":"AAGA,MAAM,UAAU,iBAAiB,CAAC,IAAmB,EAAE,QAA2B;IAC9E,IAAI,IAAI,IAAI,IAAI,EAAE;QACd,OAAO,IAAI,CAAC;KACf;IAED,MAAM,SAAS,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IAEvE,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE;QAC9B,IAAI,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC;YACzB,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;KAC1C;IAED,OAAO,IAAI,CAAC;AAChB,CAAC"}

package/dist/utils/removeNullFields.d.ts

@@ -0,0 +1 @@
+export declare function removeNullFields<T extends object>(obj: T): T;

package/dist/utils/removeNullFields.js

@@ -0,0 +1,9 @@
+export function removeNullFields(obj) {
+    const newObj = Object.assign({}, obj);
+    for (const key in obj) {
+        if (newObj[key] == null)
+            delete newObj[key];
+    }
+    return newObj;
+}
+//# sourceMappingURL=removeNullFields.js.map

package/dist/utils/removeNullFields.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"removeNullFields.js","sourceRoot":"","sources":["../../src/utils/removeNullFields.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,gBAAgB,CAAmB,GAAM;IACrD,MAAM,MAAM,GAAM,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IAEzC,KAAK,MAAM,GAAG,IAAI,GAAG,EAAE;QACnB,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI;YACnB,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;KAC1B;IAED,OAAO,MAAM,CAAC;AAClB,CAAC"}

package/dist/utils/spawnCommand.d.ts

@@ -1 +1,2 @@
-export declare function spawnCommand(command: string, args: string[], cwd: string): Promise<void>;
+/// <reference types="node" />
+export declare function spawnCommand(command: string, args: string[], cwd: string, env?: NodeJS.ProcessEnv): Promise<void>;

package/dist/utils/spawnCommand.js

@@ -1,5 +1,5 @@
 import spawn from "cross-spawn";
-export function spawnCommand(command, args, cwd) {
+export function spawnCommand(command, args, cwd, env = process.env) {
     function getCommandString() {
         let res = command;
         for (const arg of args) {
@@ -16,7 +16,7 @@ export function spawnCommand(command, args, cwd) {
         const child = spawn(command, args, {
             stdio: "inherit",
             cwd,
-            env: process.env,
+            env,
             detached: false,
             windowsHide: true
         });

package/dist/utils/spawnCommand.js.map

@@ -1 +1 @@
-{"version":3,"file":"spawnCommand.js","sourceRoot":"","sources":["../../src/utils/spawnCommand.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,aAAa,CAAC;AAEhC,MAAM,UAAU,YAAY,CAAC,OAAe,EAAE,IAAc,EAAE,GAAW;IACrE,SAAS,gBAAgB;QACrB,IAAI,GAAG,GAAG,OAAO,CAAC;QAElB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACpB,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;gBACnB,GAAG,IAAI,KAAK,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;aAC7C;iBAAM;gBACH,GAAG,IAAI,IAAI,GAAG,EAAE,CAAC;aACpB;SACJ;QAED,OAAO,GAAG,CAAC;IACf,CAAC;IAED,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACzC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE;YAC/B,KAAK,EAAE,SAAS;YAChB,GAAG;YACH,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,QAAQ,EAAE,KAAK;YACf,WAAW,EAAE,IAAI;SACpB,CAAC,CAAC;QAEH,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACtB,IAAI,IAAI,IAAI,CAAC;gBACT,OAAO,EAAE,CAAC;;gBAEV,MAAM,CAAC,IAAI,KAAK,CAAC,WAAW,gBAAgB,EAAE,qBAAqB,IAAI,EAAE,CAAC,CAAC,CAAC;QACpF,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC1B,KAAK,CAAC,EAAE,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,WAAW,gBAAgB,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC;QAC9F,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE;YACrB,IAAI,IAAI,IAAI,CAAC;gBACT,OAAO,EAAE,CAAC;;gBAEV,MAAM,CAAC,IAAI,KAAK,CAAC,WAAW,gBAAgB,EAAE,qBAAqB,IAAI,EAAE,CAAC,CAAC,CAAC;QACpF,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACP,CAAC"}
+{"version":3,"file":"spawnCommand.js","sourceRoot":"","sources":["../../src/utils/spawnCommand.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,aAAa,CAAC;AAEhC,MAAM,UAAU,YAAY,CAAC,OAAe,EAAE,IAAc,EAAE,GAAW,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG;IACxF,SAAS,gBAAgB;QACrB,IAAI,GAAG,GAAG,OAAO,CAAC;QAElB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACpB,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;gBACnB,GAAG,IAAI,KAAK,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;aAC7C;iBAAM;gBACH,GAAG,IAAI,IAAI,GAAG,EAAE,CAAC;aACpB;SACJ;QAED,OAAO,GAAG,CAAC;IACf,CAAC;IAED,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACzC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE;YAC/B,KAAK,EAAE,SAAS;YAChB,GAAG;YACH,GAAG;YACH,QAAQ,EAAE,KAAK;YACf,WAAW,EAAE,IAAI;SACpB,CAAC,CAAC;QAEH,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACtB,IAAI,IAAI,IAAI,CAAC;gBACT,OAAO,EAAE,CAAC;;gBAEV,MAAM,CAAC,IAAI,KAAK,CAAC,WAAW,gBAAgB,EAAE,qBAAqB,IAAI,EAAE,CAAC,CAAC,CAAC;QACpF,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC1B,KAAK,CAAC,EAAE,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,WAAW,gBAAgB,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC;QAC9F,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE;YACrB,IAAI,IAAI,IAAI,CAAC;gBACT,OAAO,EAAE,CAAC;;gBAEV,MAAM,CAAC,IAAI,KAAK,CAAC,WAAW,gBAAgB,EAAE,qBAAqB,IAAI,EAAE,CAAC,CAAC,CAAC;QACpF,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACP,CAAC"}

package/llama/addon.cpp

@@ -3,18 +3,26 @@
 #include <sstream>
 #include <vector>
 
+#include "common.h"
 #include "llama.h"
+#include "common/grammar-parser.h"
 #include "napi.h"
 
 class LLAMAModel : public Napi::ObjectWrap<LLAMAModel> {
   public:
     llama_context_params params;
     llama_model* model;
+    float temperature;
+    int32_t top_k;
+    float top_p;
 
     LLAMAModel(const Napi::CallbackInfo& info) : Napi::ObjectWrap<LLAMAModel>(info) {
         params = llama_context_default_params();
         params.seed = -1;
         params.n_ctx = 4096;
+        temperature = 0.0f;
+        top_k = 40;
+        top_p = 0.95f;
 
         // Get the model path
         std::string modelPath = info[0].As<Napi::String>().Utf8Value();
@@ -65,6 +73,18 @@ class LLAMAModel : public Napi::ObjectWrap<LLAMAModel> {
             if (options.Has("embedding")) {
                 params.embedding = options.Get("embedding").As<Napi::Boolean>().Value();
             }
+
+            if (options.Has("temperature")) {
+                temperature = options.Get("temperature").As<Napi::Number>().FloatValue();
+            }
+
+            if (options.Has("topK")) {
+                top_k = options.Get("topK").As<Napi::Number>().Int32Value();
+            }
+
+            if (options.Has("topP")) {
+                top_p = options.Get("topP").As<Napi::Number>().FloatValue();
+            }
         }
 
         llama_backend_init(false);
@@ -85,20 +105,84 @@ class LLAMAModel : public Napi::ObjectWrap<LLAMAModel> {
     }
 };
 
+class LLAMAGrammar : public Napi::ObjectWrap<LLAMAGrammar> {
+  public:
+    grammar_parser::parse_state parsed_grammar;
+    llama_grammar *grammar = nullptr;
+
+    LLAMAGrammar(const Napi::CallbackInfo& info) : Napi::ObjectWrap<LLAMAGrammar>(info) {
+        // Get the model path
+        std::string grammarCode = info[0].As<Napi::String>().Utf8Value();
+        bool should_print_grammar = false;
+
+        if (info.Length() > 1 && info[1].IsObject()) {
+            Napi::Object options = info[1].As<Napi::Object>();
+
+            if (options.Has("printGrammar")) {
+                should_print_grammar = options.Get("printGrammar").As<Napi::Boolean>().Value();
+            }
+        }
+
+        parsed_grammar = grammar_parser::parse(grammarCode.c_str());
+        // will be empty (default) if there are parse errors
+        if (parsed_grammar.rules.empty()) {
+            Napi::Error::New(info.Env(), "Failed to parse grammar").ThrowAsJavaScriptException();
+            return;
+        }
+
+        if (should_print_grammar) {
+            grammar_parser::print_grammar(stderr, parsed_grammar);
+        }
+
+        std::vector<const llama_grammar_element *> grammar_rules(parsed_grammar.c_rules());
+        grammar = llama_grammar_init(
+            grammar_rules.data(), grammar_rules.size(), parsed_grammar.symbol_ids.at("root"));
+    }
+
+    ~LLAMAGrammar() {
+        if (grammar != nullptr) {
+            llama_grammar_free(grammar);
+            grammar = nullptr;
+        }
+    }
+
+    static void init(Napi::Object exports) {
+        exports.Set("LLAMAGrammar", DefineClass(exports.Env(), "LLAMAGrammar", {}));
+    }
+};
+
 class LLAMAContext : public Napi::ObjectWrap<LLAMAContext> {
   public:
   LLAMAModel* model;
   llama_context* ctx;
+  LLAMAGrammar* grammar;
+  bool use_grammar = false;
+
   LLAMAContext(const Napi::CallbackInfo& info) : Napi::ObjectWrap<LLAMAContext>(info) {
     model = Napi::ObjectWrap<LLAMAModel>::Unwrap(info[0].As<Napi::Object>());
     model->Ref();
     ctx = llama_new_context_with_model(model->model, model->params);
     Napi::MemoryManagement::AdjustExternalMemory(Env(), llama_get_state_size(ctx));
+
+    if (info.Length() > 1 && info[1].IsObject()) {
+        Napi::Object options = info[1].As<Napi::Object>();
+
+        if (options.Has("grammar")) {
+            grammar = Napi::ObjectWrap<LLAMAGrammar>::Unwrap(options.Get("grammar").As<Napi::Object>());
+            grammar->Ref();
+            use_grammar = true;
+        }
+    }
   }
   ~LLAMAContext() {
     Napi::MemoryManagement::AdjustExternalMemory(Env(), -(int64_t)llama_get_state_size(ctx));
     llama_free(ctx);
     model->Unref();
+
+    if (use_grammar) {
+        grammar->Unref();
+        use_grammar = false;
+    }
   }
   Napi::Value Encode(const Napi::CallbackInfo& info) {
     std::string text = info[0].As<Napi::String>().Utf8Value();
@@ -125,23 +209,13 @@ class LLAMAContext : public Napi::ObjectWrap<LLAMAContext> {
 
     // Decode each token and accumulate the result.
     for (size_t i = 0; i < tokens.ElementLength(); i++) {
-      // source: https://github.com/ggerganov/llama.cpp/blob/232caf3c1581a6cb023571780ff41dc2d66d1ca0/llama.cpp#L799-L811
-      std::vector<char> result(8, 0);
-      const int n_tokens = llama_token_to_str(ctx, (llama_token)tokens[i], result.data(), result.size());
-      if (n_tokens < 0) {
-          result.resize(-n_tokens);
-          int check = llama_token_to_str(ctx, (llama_token)tokens[i], result.data(), result.size());
-          GGML_ASSERT(check == -n_tokens);
-      } else {
-          result.resize(n_tokens);
-      }
+        const std::string piece = llama_token_to_piece(ctx, (llama_token)tokens[i]);
 
-      const char* str = result.data();
-      if (str == nullptr) {
-        Napi::Error::New(info.Env(), "Invalid token").ThrowAsJavaScriptException();
-        return info.Env().Undefined();
-      }
-      ss << str;
+        if (piece.empty()) {
+            continue;
+        }
+
+        ss << piece;
     }
 
     return Napi::String::New(info.Env(), ss.str());
@@ -152,9 +226,25 @@ class LLAMAContext : public Napi::ObjectWrap<LLAMAContext> {
   Napi::Value TokenEos(const Napi::CallbackInfo& info) {
     return Napi::Number::From(info.Env(), llama_token_eos(ctx));
   }
-  Napi::Value GetMaxContextSize(const Napi::CallbackInfo& info) {
+  Napi::Value TokenNl(const Napi::CallbackInfo& info) {
+    return Napi::Number::From(info.Env(), llama_token_nl(ctx));
+  }
+  Napi::Value GetContextSize(const Napi::CallbackInfo& info) {
     return Napi::Number::From(info.Env(), llama_n_ctx(ctx));
   }
+  Napi::Value GetTokenString(const Napi::CallbackInfo& info) {
+    int token = info[0].As<Napi::Number>().Int32Value();
+    std::stringstream ss;
+
+    const char* str = llama_token_get_text(ctx, token);
+    if (str == nullptr) {
+      return info.Env().Undefined();
+    }
+
+    ss << str;
+
+    return Napi::String::New(info.Env(), ss.str());
+  }
   Napi::Value Eval(const Napi::CallbackInfo& info);
   static void init(Napi::Object exports) {
     exports.Set("LLAMAContext",
@@ -165,7 +255,9 @@ class LLAMAContext : public Napi::ObjectWrap<LLAMAContext> {
                 InstanceMethod("decode", &LLAMAContext::Decode),
                 InstanceMethod("tokenBos", &LLAMAContext::TokenBos),
                 InstanceMethod("tokenEos", &LLAMAContext::TokenEos),
-                InstanceMethod("getMaxContextSize", &LLAMAContext::GetMaxContextSize),
+                InstanceMethod("tokenNl", &LLAMAContext::TokenNl),
+                InstanceMethod("getContextSize", &LLAMAContext::GetContextSize),
+                InstanceMethod("getTokenString", &LLAMAContext::GetTokenString),
                 InstanceMethod("eval", &LLAMAContext::Eval),
             }));
   }
@@ -212,7 +304,49 @@ class LLAMAContextEvalWorker : Napi::AsyncWorker, Napi::Promise::Deferred {
 
     llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false };
 
-    new_token_id = llama_sample_token_greedy(ctx->ctx , &candidates_p);
+    float originalEosLogit = 0;
+    auto eos_token = llama_token_eos(ctx->ctx);
+
+    for (auto& candidate : candidates) {
+      if (candidate.id == eos_token) {
+        originalEosLogit = candidate.logit;
+        break;
+      }
+    }
+
+    if (ctx->use_grammar) {
+        llama_sample_grammar(ctx->ctx, &candidates_p, (ctx->grammar)->grammar);
+    }
+
+    for (auto& candidate : candidates) {
+      if (candidate.id == eos_token) {
+        candidate.logit = originalEosLogit;
+        break;
+      }
+    }
+
+    if ((ctx->model)->temperature <= 0) {
+        new_token_id = llama_sample_token_greedy(ctx->ctx , &candidates_p);
+    } else {
+        const int32_t top_k = (ctx->model)->top_k <= 0 ? llama_n_vocab(ctx->ctx) : (ctx->model)->top_k;
+        const int32_t n_probs = 0; // Number of probabilities to keep - 0 = disabled
+        const float tfs_z = 1.00f; // Tail free sampling - 1.0 = disabled
+        const float typical_p = 1.00f; // Typical probability - 1.0 = disabled
+        const float top_p = (ctx->model)->top_p; // Top p sampling - 1.0 = disabled
+
+        // Temperature sampling
+        size_t min_keep = std::max(1, n_probs);
+        llama_sample_top_k(ctx->ctx, &candidates_p, top_k, min_keep);
+        llama_sample_tail_free(ctx->ctx, &candidates_p, tfs_z, min_keep);
+        llama_sample_typical(ctx->ctx, &candidates_p, typical_p, min_keep);
+        llama_sample_top_p(ctx->ctx, &candidates_p, top_p, min_keep);
+        llama_sample_temperature(ctx->ctx, &candidates_p, (ctx->model)->temperature);;
+        new_token_id = llama_sample_token(ctx->ctx, &candidates_p);
+    }
+
+    if (new_token_id != eos_token && ctx->use_grammar) {
+        llama_grammar_accept_token(ctx->ctx, (ctx->grammar)->grammar, new_token_id);
+    }
 
     result = new_token_id;
   }
@@ -238,6 +372,7 @@ Napi::Object registerCallback(Napi::Env env, Napi::Object exports) {
       Napi::PropertyDescriptor::Function("systemInfo", systemInfo),
   });
   LLAMAModel::init(exports);
+  LLAMAGrammar::init(exports);
   LLAMAContext::init(exports);
   return exports;
 }

package/llama/binariesGithubRelease.json

@@ -1,3 +1,3 @@
 {
-    "release": "b1069"
+    "release": "b1107"
 }

package/llama/binding.gyp

@@ -7,15 +7,18 @@
         "llama.cpp/ggml.c",
         "llama.cpp/ggml-alloc.c",
         "llama.cpp/k_quants.c",
-        "llama.cpp/llama.cpp"
+        "llama.cpp/llama.cpp",
+        "llama.cpp/common/common.cpp",
+        "llama.cpp/common/grammar-parser.cpp"
       ],
       "include_dirs": [
         "<!@(node -p \"require('node-addon-api').include\")",
-        "llama.cpp"
+        "llama.cpp",
+        "llama.cpp/common"
       ],
       "cflags": ["-fexceptions"],
       "cflags_cc": ["-fexceptions"],
-      "defines": [ "GGML_USE_K_QUANTS", "NAPI_CPP_EXCEPTIONS" ],
+      "defines": ["GGML_USE_K_QUANTS", "NAPI_CPP_EXCEPTIONS"],
       "msvs_settings": {
         "VCCLCompilerTool": { "AdditionalOptions": [ '/arch:AVX2', '/EHsc' ] }
       }

package/llama/grammars/README.md

@@ -0,0 +1,91 @@
+# GBNF Guide
+
+GBNF (GGML BNF) is a format for defining [formal grammars](https://en.wikipedia.org/wiki/Formal_grammar) to constrain model outputs in `llama.cpp`. For example, you can use it to force the model to generate valid JSON, or speak only in emojis. GBNF grammars are supported in various ways in `examples/main` and `examples/server`.
+
+## Background
+
+[Bakus-Naur Form (BNF)](https://en.wikipedia.org/wiki/Backus%E2%80%93Naur_form) is a notation for describing the syntax of formal languages like programming languages, file formats, and protocols. GBNF is an extension of BNF that primarily adds a few modern regex-like features.
+
+## Basics
+
+In GBNF, we define *production rules* that specify how a *non-terminal* (rule name) can be replaced with sequences of *terminals* (characters, specifically Unicode [code points](https://en.wikipedia.org/wiki/Code_point)) and other non-terminals. The basic format of a production rule is `nonterminal ::= sequence...`.
+
+## Example
+
+Before going deeper, let's look at some of the features demonstrated in `grammars/chess.gbnf`, a small chess notation grammar:
+```
+# `root` specifies the pattern for the overall output
+root ::= (
+    # it must start with the characters "1. " followed by a sequence
+    # of characters that match the `move` rule, followed by a space, followed
+    # by another move, and then a newline
+    "1. " move " " move "\n"
+
+    # it's followed by one or more subsequent moves, numbered with one or two digits
+    ([1-9] [0-9]? ". " move " " move "\n")+
+)
+
+# `move` is an abstract representation, which can be a pawn, nonpawn, or castle.
+# The `[+#]?` denotes the possibility of checking or mate signs after moves
+move ::= (pawn | nonpawn | castle) [+#]?
+
+pawn ::= ...
+nonpawn ::= ...
+castle ::= ...
+```
+
+## Non-Terminals and Terminals
+
+Non-terminal symbols (rule names) stand for a pattern of terminals and other non-terminals. They are required to be a dashed lowercase word, like `move`, `castle`, or `check-mate`.
+
+Terminals are actual characters ([code points](https://en.wikipedia.org/wiki/Code_point)). They can be specified as a sequence like `"1"` or `"O-O"` or as ranges like `[1-9]` or `[NBKQR]`.
+
+## Characters and character ranges
+
+Terminals support the full range of Unicode. Unicode characters can be specified directly in the grammar, for example `hiragana ::= [ぁ-ゟ]`, or with escapes: 8-bit (`\xXX`), 16-bit (`\uXXXX`) or 32-bit (`\UXXXXXXXX`).
+
+Character ranges can be negated with `^`:
+```
+single-line ::= [^\n]+ "\n"`
+```
+
+## Sequences and Alternatives
+
+The order of symbols in a sequence matter. For example, in `"1. " move " " move "\n"`, the `"1. "` must come before the first `move`, etc.
+
+Alternatives, denoted by `|`, give different sequences that are acceptable. For example, in `move ::= pawn | nonpawn | castle`, `move` can be a `pawn` move, a `nonpawn` move, or a `castle`.
+
+Parentheses `()` can be used to group sequences, which allows for embedding alternatives in a larger rule or applying repetition and optptional symbols (below) to a sequence.
+
+## Repetition and Optional Symbols
+
+- `*` after a symbol or sequence means that it can be repeated zero or more times.
+- `+` denotes that the symbol or sequence should appear one or more times.
+- `?` makes the preceding symbol or sequence optional.
+
+## Comments and newlines
+
+Comments can be specified with `#`:
+```
+# defines optional whitspace
+ws ::= [ \t\n]+
+```
+
+Newlines are allowed between rules and between symbols or sequences nested inside parentheses. Additionally, a newline after an alternate marker `|` will continue the current rule, even outside of parentheses.
+
+## The root rule
+
+In a full grammar, the `root` rule always defines the starting point of the grammar. In other words, it specifies what the entire output must match.
+
+```
+# a grammar for lists
+root ::= ("- " item)+
+item ::= [^\n]+ "\n"
+```
+
+## Next steps
+
+This guide provides a brief overview. Check out the GBNF files in this directory (`grammars/`) for examples of full grammars. You can try them out with:
+```
+./main -m <model> --grammar-file grammars/some-grammar.gbnf -p 'Some prompt'
+```

package/llama/grammars/arithmetic.gbnf

@@ -0,0 +1,6 @@
+root  ::= (expr "=" ws term "\n")+
+expr  ::= term ([-+*/] term)*
+term  ::= ident | num | "(" ws expr ")" ws
+ident ::= [a-z] [a-z0-9_]* ws
+num   ::= [0-9]+ ws
+ws    ::= [ \t\n]*

package/llama/grammars/chess.gbnf

@@ -0,0 +1,13 @@
+# Specifies chess moves as a list in algebraic notation, using PGN conventions
+
+# Force first move to "1. ", then any 1-2 digit number after, relying on model to follow the pattern
+root    ::= "1. " move " " move "\n" ([1-9] [0-9]? ". " move " " move "\n")+
+move    ::= (pawn | nonpawn | castle) [+#]?
+
+# piece type, optional file/rank, optional capture, dest file & rank
+nonpawn ::= [NBKQR] [a-h]? [1-8]? "x"? [a-h] [1-8]
+
+# optional file & capture, dest file & rank, optional promotion
+pawn    ::= ([a-h] "x")? [a-h] [1-8] ("=" [NBKQR])?
+
+castle  ::= "O-O" "-O"?

package/llama/grammars/japanese.gbnf

@@ -0,0 +1,7 @@
+# A probably incorrect grammar for Japanese
+root        ::= jp-char+ ([ \t\n] jp-char+)*
+jp-char     ::= hiragana | katakana | punctuation | cjk
+hiragana    ::= [ぁ-ゟ]
+katakana    ::= [ァ-ヿ]
+punctuation ::= [、-〾]
+cjk         ::= [一-鿿]

package/llama/grammars/json.gbnf

@@ -0,0 +1,25 @@
+root   ::= object
+value  ::= object | array | string | number | ("true" | "false" | "null") ws
+
+object ::=
+  "{" ws (
+            string ":" ws value
+    ("," ws string ":" ws value)*
+  )? "}" ws
+
+array  ::=
+  "[" ws (
+            value
+    ("," ws value)*
+  )? "]" ws
+
+string ::=
+  "\"" (
+    [^"\\] |
+    "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes
+  )* "\"" ws
+
+number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws
+
+# Optional space: by convention, applied in this grammar after literal chars when allowed
+ws ::= ([ \t\n] ws)?

package/llama/grammars/list.gbnf

@@ -0,0 +1,4 @@
+root ::= item+
+
+# Excludes various line break characters
+item ::= "- " [^\r\n\x0b\x0c\x85\u2028\u2029]+ "\n"