node-llama-cpp 1.3.1 → 2.1.0

package/dist/utils/compileLLamaCpp.js

@@ -7,14 +7,44 @@ import { clearLlamaBuild } from "./clearLlamaBuild.js";
 import { setUsedBinFlag } from "./usedBinFlag.js";
 import { spawnCommand } from "./spawnCommand.js";
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
-export async function compileLlamaCpp({ arch = process.arch, nodeTarget = process.version, setUsedBingFlag = true }) {
+export async function compileLlamaCpp({ arch = process.arch, nodeTarget = process.version, setUsedBingFlag = true, metal = false, cuda = false }) {
     try {
         if (!(await fs.exists(llamaCppDirectory))) {
             throw new Error(`"${llamaCppDirectory}" directory does not exist`);
         }
+        const gypDefines = ["GGML_USE_K_QUANTS", "NAPI_CPP_EXCEPTIONS"];
+        if ((metal && process.platform === "darwin") || process.env.LLAMA_METAL === "1")
+            gypDefines.push("LLAMA_METAL=1");
+        if (cuda || process.env.LLAMA_CUBLAS === "1")
+            gypDefines.push("LLAMA_CUBLAS=1");
+        if (process.env.LLAMA_MPI === "1")
+            gypDefines.push("LLAMA_MPI=1");
+        if (process.env.LLAMA_OPENBLAS === "1")
+            gypDefines.push("LLAMA_OPENBLAS=1");
+        if (process.env.LLAMA_BLAS_VENDOR != null)
+            gypDefines.push("LLAMA_BLAS_VENDOR=" + process.env.LLAMA_BLAS_VENDOR);
+        if (process.env.LLAMA_CUDA_FORCE_DMMV != null)
+            gypDefines.push("LLAMA_CUDA_FORCE_DMMV=" + process.env.LLAMA_CUDA_FORCE_DMMV);
+        if (process.env.LLAMA_CUDA_DMMV_X != null)
+            gypDefines.push("LLAMA_CUDA_DMMV_X=" + process.env.LLAMA_CUDA_DMMV_X);
+        if (process.env.LLAMA_CUDA_MMV_Y != null)
+            gypDefines.push("LLAMA_CUDA_MMV_Y=" + process.env.LLAMA_CUDA_MMV_Y);
+        if (process.env.LLAMA_CUDA_F16 != null)
+            gypDefines.push("LLAMA_CUDA_F16=" + process.env.LLAMA_CUDA_F16);
+        if (process.env.LLAMA_CUDA_KQUANTS_ITER != null)
+            gypDefines.push("LLAMA_CUDA_KQUANTS_ITER=" + process.env.LLAMA_CUDA_KQUANTS_ITER);
+        if (process.env.LLAMA_HIPBLAS === "1")
+            gypDefines.push("LLAMA_HIPBLAS=1");
+        if (process.env.LLAMA_CLBLAST === "1")
+            gypDefines.push("LLAMA_CLBLAST=1");
+        const nodeGypEnv = {
+            ...process.env,
+            "CMAKE_CURRENT_SOURCE_DIR": llamaCppDirectory,
+            "GYP_DEFINES": gypDefines.join(" ")
+        };
         await clearLlamaBuild();
-        await spawnCommand("npm", ["run", "-s", "node-gyp-llama", "--", "configure", "--arch=" + arch, "--target=" + nodeTarget], __dirname);
-        await spawnCommand("npm", ["run", "-s", "node-gyp-llama", "--", "configure", "--arch=" + arch, "--target=" + nodeTarget, "--", "-f", "compile_commands_json"], __dirname);
+        await spawnCommand("npm", ["run", "-s", "node-gyp-llama", "--", "configure", "--arch=" + arch, "--target=" + nodeTarget], __dirname, nodeGypEnv);
+        await spawnCommand("npm", ["run", "-s", "node-gyp-llama", "--", "configure", "--arch=" + arch, "--target=" + nodeTarget, "--", "-f", "compile_commands_json"], __dirname, nodeGypEnv);
         if (await fs.exists(path.join(llamaDirectory, "Release", "compile_commands.json"))) {
             await fs.move(path.join(llamaDirectory, "Release", "compile_commands.json"), path.join(llamaDirectory, "compile_commands.json"));
         }
@@ -23,7 +53,7 @@ export async function compileLlamaCpp({ arch = process.arch, nodeTarget = proces
         }
         await fs.remove(path.join(llamaDirectory, "Release"));
         await fs.remove(path.join(llamaDirectory, "Debug"));
-        await spawnCommand("npm", ["run", "-s", "node-gyp-llama-build", "--", "--arch=" + arch, "--target=" + nodeTarget], __dirname);
+        await spawnCommand("npm", ["run", "-s", "node-gyp-llama-build", "--", "--arch=" + arch, "--target=" + nodeTarget], __dirname, nodeGypEnv);
         if (setUsedBingFlag) {
             await setUsedBinFlag("localBuildFromSource");
         }

package/dist/utils/compileLLamaCpp.js.map

@@ -1 +1 @@
-{"version":3,"file":"compileLLamaCpp.js","sourceRoot":"","sources":["../../src/utils/compileLLamaCpp.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAC,aAAa,EAAC,MAAM,KAAK,CAAC;AAClC,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,iBAAiB,EAAE,cAAc,EAAC,MAAM,cAAc,CAAC;AAC/D,OAAO,EAAC,eAAe,EAAC,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAC,cAAc,EAAC,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAC,YAAY,EAAC,MAAM,mBAAmB,CAAC;AAE/C,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE/D,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,EAAC,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,OAAO,EAAE,eAAe,GAAG,IAAI,EAE/G;IACG,IAAI;QACA,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,EAAE;YACvC,MAAM,IAAI,KAAK,CAAC,IAAI,iBAAiB,4BAA4B,CAAC,CAAC;SACtE;QAED,MAAM,eAAe,EAAE,CAAC;QAExB,MAAM,YAAY,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,GAAG,IAAI,EAAE,WAAW,GAAG,UAAU,CAAC,EAAE,SAAS,CAAC,CAAC;QAErI,MAAM,YAAY,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,GAAG,IAAI,EAAE,WAAW,GAAG,UAAU,EAAE,IAAI,EAAE,IAAI,EAAE,uBAAuB,CAAC,EAAE,SAAS,CAAC,CAAC;QAE1K,IAAI,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,SAAS,EAAE,uBAAuB,CAAC,CAAC,EAAE;YAChF,MAAM,EAAE,CAAC,IAAI,CACT,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,SAAS,EAAE,uBAAuB,CAAC,EAC7D,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,uBAAuB,CAAC,CACrD,CAAC;SACL;aAAM,IAAI,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,EAAE,uBAAuB,CAAC,CAAC,EAAE;YACrF,MAAM,EAAE,CAAC,IAAI,CACT,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,EAAE,uBAAuB,CAAC,EAC3D,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,uBAAuB,CAAC,CACrD,CAAC;SACL;QAED,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC,CAAC;QACtD,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC,CAAC;QAGpD,MAAM,YAAY,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,sBAAsB,EAAE,IAAI,EAAE,SAAS,GAAG,IAAI,EAAE,WAAW,GAAG,UAAU,CAAC,EAAE,SAAS,CAAC,CAAC;QAE9H,IAAI,eAAe,EAAE;YACjB,MAAM,cAAc,CAAC,sBAAsB,CAAC,CAAC;SAChD;KACJ;IAAC,OAAO,GAAG,EAAE;QACV,IAAI,eAAe;YACf,MAAM,cAAc,CAAC,kBAAkB,CAAC,CAAC;QAE7C,MAAM,GAAG,CAAC;KACb;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,6BAA6B;IAC/C,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;IAE/F,IAAI,MAAM,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC3B,OAAO,UAAU,CAAC;IAEtB,OAAO,IAAI,CAAC;AAChB,CAAC"}
+{"version":3,"file":"compileLLamaCpp.js","sourceRoot":"","sources":["../../src/utils/compileLLamaCpp.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAC,aAAa,EAAC,MAAM,KAAK,CAAC;AAClC,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,iBAAiB,EAAE,cAAc,EAAC,MAAM,cAAc,CAAC;AAC/D,OAAO,EAAC,eAAe,EAAC,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAC,cAAc,EAAC,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAC,YAAY,EAAC,MAAM,mBAAmB,CAAC;AAE/C,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE/D,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,EAClC,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,OAAO,EAAE,eAAe,GAAG,IAAI,EAAE,KAAK,GAAG,KAAK,EAAE,IAAI,GAAG,KAAK,EAGzG;IACG,IAAI;QACA,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,EAAE;YACvC,MAAM,IAAI,KAAK,CAAC,IAAI,iBAAiB,4BAA4B,CAAC,CAAC;SACtE;QAED,MAAM,UAAU,GAAG,CAAC,mBAAmB,EAAE,qBAAqB,CAAC,CAAC;QAEhE,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW,KAAK,GAAG;YAAE,UAAU,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAClH,IAAI,IAAI,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,KAAK,GAAG;YAAE,UAAU,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAChF,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,KAAK,GAAG;YAAE,UAAU,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAClE,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,GAAG;YAAE,UAAU,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC5E,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,IAAI;YAAE,UAAU,CAAC,IAAI,CAAC,oBAAoB,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QACjH,IAAI,OAAO,CAAC,GAAG,CAAC,qBAAqB,IAAI,IAAI;YAAE,UAAU,CAAC,IAAI,CAAC,wBAAwB,GAAG,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;QAC7H,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,IAAI;YAAE,UAAU,CAAC,IAAI,CAAC,oBAAoB,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QACjH,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,IAAI;YAAE,UAAU,CAAC,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QAC9G,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,IAAI;YAAE,UAAU,CAAC,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QACxG,IAAI,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,IAAI;YAAE,UAAU,CAAC,IAAI,CAAC,0BAA0B,GAAG,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;QACnI,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,KAAK,GAAG;YAAE,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC1E,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,KAAK,GAAG;YAAE,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAE1E,MAAM,UAAU,GAAsB;YAClC,GAAG,OAAO,CAAC,GAAG;YACd,0BAA0B,EAAE,iBAAiB;YAC7C,aAAa,EAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC;SACtC,CAAC;QAEF,MAAM,eAAe,EAAE,CAAC;QAExB,MAAM,YAAY,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,GAAG,IAAI,EAAE,WAAW,GAAG,UAAU,CAAC,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;QAEjJ,MAAM,YAAY,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,GAAG,IAAI,EAAE,WAAW,GAAG,UAAU,EAAE,IAAI,EAAE,IAAI,EAAE,uBAAuB,CAAC,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;QAEtL,IAAI,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,SAAS,EAAE,uBAAuB,CAAC,CAAC,EAAE;YAChF,MAAM,EAAE,CAAC,IAAI,CACT,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,SAAS,EAAE,uBAAuB,CAAC,EAC7D,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,uBAAuB,CAAC,CACrD,CAAC;SACL;aAAM,IAAI,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,EAAE,uBAAuB,CAAC,CAAC,EAAE;YACrF,MAAM,EAAE,CAAC,IAAI,CACT,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,EAAE,uBAAuB,CAAC,EAC3D,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,uBAAuB,CAAC,CACrD,CAAC;SACL;QAED,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC,CAAC;QACtD,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC,CAAC;QAGpD,MAAM,YAAY,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,sBAAsB,EAAE,IAAI,EAAE,SAAS,GAAG,IAAI,EAAE,WAAW,GAAG,UAAU,CAAC,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;QAE1I,IAAI,eAAe,EAAE;YACjB,MAAM,cAAc,CAAC,sBAAsB,CAAC,CAAC;SAChD;KACJ;IAAC,OAAO,GAAG,EAAE;QACV,IAAI,eAAe;YACf,MAAM,cAAc,CAAC,kBAAkB,CAAC,CAAC;QAE7C,MAAM,GAAG,CAAC;KACb;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,6BAA6B;IAC/C,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;IAE/F,IAAI,MAAM,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC3B,OAAO,UAAU,CAAC;IAEtB,OAAO,IAAI,CAAC;AAChB,CAAC"}

package/dist/utils/getBin.d.ts

@@ -3,9 +3,8 @@ export declare function loadBin(): Promise<LlamaCppNodeModule>;
 export type LlamaCppNodeModule = {
     LLAMAModel: LLAMAModel;
     LLAMAContext: LLAMAContext;
-    tokenBos(): number;
+    LLAMAGrammar: LLAMAGrammar;
     systemInfo(): string;
-    tokenEos(): number;
 };
 export type LLAMAModel = {
     new (modelPath: string, params: {
@@ -20,11 +19,26 @@ export type LLAMAModel = {
         useMmap?: boolean;
         useMlock?: boolean;
         embedding?: boolean;
+        temperature?: number;
+        topK?: number;
+        topP?: number;
     }): LLAMAModel;
 };
 export type LLAMAContext = {
-    new (model: LLAMAModel): LLAMAContext;
+    new (model: LLAMAModel, params?: {
+        grammar?: LLAMAGrammar;
+    }): LLAMAContext;
     encode(text: string): Uint32Array;
-    eval(tokens: Uint32Array, restrictions?: Uint32Array): Promise<number>;
+    eval(tokens: Uint32Array): Promise<number>;
     decode(tokens: Uint32Array): string;
+    tokenBos(): number;
+    tokenEos(): number;
+    tokenNl(): number;
+    getContextSize(): number;
+    getTokenString(token: number): string;
+};
+export type LLAMAGrammar = {
+    new (grammarPath: string, params?: {
+        printGrammar?: boolean;
+    }): LLAMAGrammar;
 };

package/dist/utils/getBin.js

@@ -3,7 +3,7 @@ import * as console from "console";
 import path from "path";
 import process from "process";
 import fs from "fs-extra";
-import { defaultLlamaCppGitHubRepo, defaultLlamaCppRelease, defaultSkipDownload, llamaBinsDirectory } from "../config.js";
+import { defaultLlamaCppCudaSupport, defaultLlamaCppGitHubRepo, defaultLlamaCppMetalSupport, defaultLlamaCppRelease, defaultSkipDownload, llamaBinsDirectory } from "../config.js";
 import { DownloadLlamaCppCommand } from "../cli/commands/DownloadCommand.js";
 import { getUsedBinFlag } from "./usedBinFlag.js";
 import { getCompiledLlamaCppBinaryPath } from "./compileLLamaCpp.js";
@@ -56,7 +56,9 @@ export async function loadBin() {
         else {
             await DownloadLlamaCppCommand({
                 repo: defaultLlamaCppGitHubRepo,
-                release: defaultLlamaCppRelease
+                release: defaultLlamaCppRelease,
+                metal: defaultLlamaCppMetalSupport,
+                cuda: defaultLlamaCppCudaSupport
             });
             const modulePath = await getCompiledLlamaCppBinaryPath();
             if (modulePath == null) {

package/dist/utils/getBin.js.map

@@ -1 +1 @@
-{"version":3,"file":"getBin.js","sourceRoot":"","sources":["../../src/utils/getBin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,QAAQ,CAAC;AACrC,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,yBAAyB,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,kBAAkB,EAAC,MAAM,cAAc,CAAC;AACxH,OAAO,EAAC,uBAAuB,EAAC,MAAM,oCAAoC,CAAC;AAC3E,OAAO,EAAC,cAAc,EAAC,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAC,6BAA6B,EAAC,MAAM,sBAAsB,CAAC;AAEnE,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAE/C,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACpC,MAAM,gBAAgB,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IACrE,MAAM,iBAAiB,GAAG,CAAC,gBAAgB,EAAE,gBAAgB,GAAG,CAAC,CAAC,CAAC;IAEnE,SAAS,UAAU,CAAC,QAAgB,EAAE,IAAY,EAAE,WAAmB;QACnE,OAAO,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,QAAQ,IAAI,IAAI,IAAI,WAAW,OAAO,CAAC,CAAC;IACpF,CAAC;IAED,KAAK,UAAU,WAAW,CAAC,QAAgB,EAAE,IAAY,EAAE,YAAsB;QAC7E,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE;YACpC,MAAM,OAAO,GAAG,UAAU,CAAC,QAAQ,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;YAExD,IAAI,MAAM,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC;gBACxB,OAAO,OAAO,CAAC;SACtB;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,KAAK,UAAU,OAAO;QAClB,QAAQ,OAAO,CAAC,QAAQ,EAAE;YACtB,KAAK,OAAO,CAAC;YACb,KAAK,QAAQ;gBACT,OAAO,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;YAE/D,KAAK,OAAO,CAAC;YACb,KAAK,SAAS;gBACV,OAAO,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;YAEjE,KAAK,QAAQ;gBACT,OAAO,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;SAClE;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,OAAO,MAAM,OAAO,EAAE,CAAC;AAC3B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO;IACzB,MAAM,YAAY,GAAG,MAAM,cAAc,EAAE,CAAC;IAE5C,IAAI,YAAY,KAAK,kBAAkB,EAAE;QACrC,MAAM,eAAe,GAAG,MAAM,kBAAkB,EAAE,CAAC;QAEnD,IAAI,eAAe,IAAI,IAAI,EAAE;YACzB,OAAO,CAAC,IAAI,CAAC,wEAAwE,CAAC,CAAC;SAC1F;aAAM;YACH,OAAO,OAAO,CAAC,eAAe,CAAC,CAAC;SACnC;KACJ;IAED,MAAM,UAAU,GAAG,MAAM,6BAA6B,EAAE,CAAC;IAEzD,IAAI,UAAU,IAAI,IAAI,EAAE;QACpB,IAAI,mBAAmB,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,oFAAoF,CAAC,CAAC;SACzG;aAAM;YACH,MAAM,uBAAuB,CAAC;gBAC1B,IAAI,EAAE,yBAAyB;gBAC/B,OAAO,EAAE,sBAAsB;aAClC,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,MAAM,6BAA6B,EAAE,CAAC;YAEzD,IAAI,UAAU,IAAI,IAAI,EAAE;gBACpB,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;aAC/D;YAED,OAAO,OAAO,CAAC,UAAU,CAAC,CAAC;SAC9B;KACJ;IAED,OAAO,OAAO,CAAC,UAAU,CAAC,CAAC;AAC/B,CAAC"}
+{"version":3,"file":"getBin.js","sourceRoot":"","sources":["../../src/utils/getBin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,QAAQ,CAAC;AACrC,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EACH,0BAA0B,EAAE,yBAAyB,EAAE,2BAA2B,EAAE,sBAAsB,EAAE,mBAAmB,EAC/H,kBAAkB,EACrB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAC,uBAAuB,EAAC,MAAM,oCAAoC,CAAC;AAC3E,OAAO,EAAC,cAAc,EAAC,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAC,6BAA6B,EAAC,MAAM,sBAAsB,CAAC;AAEnE,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAE/C,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACpC,MAAM,gBAAgB,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IACrE,MAAM,iBAAiB,GAAG,CAAC,gBAAgB,EAAE,gBAAgB,GAAG,CAAC,CAAC,CAAC;IAEnE,SAAS,UAAU,CAAC,QAAgB,EAAE,IAAY,EAAE,WAAmB;QACnE,OAAO,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,QAAQ,IAAI,IAAI,IAAI,WAAW,OAAO,CAAC,CAAC;IACpF,CAAC;IAED,KAAK,UAAU,WAAW,CAAC,QAAgB,EAAE,IAAY,EAAE,YAAsB;QAC7E,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE;YACpC,MAAM,OAAO,GAAG,UAAU,CAAC,QAAQ,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;YAExD,IAAI,MAAM,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC;gBACxB,OAAO,OAAO,CAAC;SACtB;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,KAAK,UAAU,OAAO;QAClB,QAAQ,OAAO,CAAC,QAAQ,EAAE;YACtB,KAAK,OAAO,CAAC;YACb,KAAK,QAAQ;gBACT,OAAO,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;YAE/D,KAAK,OAAO,CAAC;YACb,KAAK,SAAS;gBACV,OAAO,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;YAEjE,KAAK,QAAQ;gBACT,OAAO,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;SAClE;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,OAAO,MAAM,OAAO,EAAE,CAAC;AAC3B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO;IACzB,MAAM,YAAY,GAAG,MAAM,cAAc,EAAE,CAAC;IAE5C,IAAI,YAAY,KAAK,kBAAkB,EAAE;QACrC,MAAM,eAAe,GAAG,MAAM,kBAAkB,EAAE,CAAC;QAEnD,IAAI,eAAe,IAAI,IAAI,EAAE;YACzB,OAAO,CAAC,IAAI,CAAC,wEAAwE,CAAC,CAAC;SAC1F;aAAM;YACH,OAAO,OAAO,CAAC,eAAe,CAAC,CAAC;SACnC;KACJ;IAED,MAAM,UAAU,GAAG,MAAM,6BAA6B,EAAE,CAAC;IAEzD,IAAI,UAAU,IAAI,IAAI,EAAE;QACpB,IAAI,mBAAmB,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,oFAAoF,CAAC,CAAC;SACzG;aAAM;YACH,MAAM,uBAAuB,CAAC;gBAC1B,IAAI,EAAE,yBAAyB;gBAC/B,OAAO,EAAE,sBAAsB;gBAC/B,KAAK,EAAE,2BAA2B;gBAClC,IAAI,EAAE,0BAA0B;aACnC,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,MAAM,6BAA6B,EAAE,CAAC;YAEzD,IAAI,UAAU,IAAI,IAAI,EAAE;gBACpB,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;aAC/D;YAED,OAAO,OAAO,CAAC,UAAU,CAAC,CAAC;SAC9B;KACJ;IAED,OAAO,OAAO,CAAC,UAAU,CAAC,CAAC;AAC/B,CAAC"}

package/dist/utils/getGrammarsFolder.d.ts

@@ -0,0 +1 @@
+export declare function getGrammarsFolder(): Promise<string>;

package/dist/utils/getGrammarsFolder.js

@@ -0,0 +1,18 @@
+import fs from "fs-extra";
+import { llamaBinsGrammarsDirectory, llamaCppGrammarsDirectory } from "../config.js";
+import { getUsedBinFlag } from "./usedBinFlag.js";
+export async function getGrammarsFolder() {
+    const usedBingFlag = await getUsedBinFlag();
+    if (usedBingFlag === "localBuildFromSource") {
+        if (await fs.exists(llamaCppGrammarsDirectory))
+            return llamaCppGrammarsDirectory;
+    }
+    else if (usedBingFlag === "prebuiltBinaries") {
+        if (await fs.exists(llamaBinsGrammarsDirectory))
+            return llamaBinsGrammarsDirectory;
+        else if (await fs.exists(llamaCppGrammarsDirectory))
+            return llamaCppGrammarsDirectory;
+    }
+    throw new Error("Grammars folder not found");
+}
+//# sourceMappingURL=getGrammarsFolder.js.map

package/dist/utils/getGrammarsFolder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"getGrammarsFolder.js","sourceRoot":"","sources":["../../src/utils/getGrammarsFolder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,0BAA0B,EAAE,yBAAyB,EAAC,MAAM,cAAc,CAAC;AACnF,OAAO,EAAC,cAAc,EAAC,MAAM,kBAAkB,CAAC;AAEhD,MAAM,CAAC,KAAK,UAAU,iBAAiB;IACnC,MAAM,YAAY,GAAG,MAAM,cAAc,EAAE,CAAC;IAE5C,IAAI,YAAY,KAAK,sBAAsB,EAAE;QACzC,IAAI,MAAM,EAAE,CAAC,MAAM,CAAC,yBAAyB,CAAC;YAC1C,OAAO,yBAAyB,CAAC;KACxC;SAAM,IAAI,YAAY,KAAK,kBAAkB,EAAE;QAC5C,IAAI,MAAM,EAAE,CAAC,MAAM,CAAC,0BAA0B,CAAC;YAC3C,OAAO,0BAA0B,CAAC;aACjC,IAAI,MAAM,EAAE,CAAC,MAAM,CAAC,yBAAyB,CAAC;YAC/C,OAAO,yBAAyB,CAAC;KACxC;IAED,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;AACjD,CAAC"}

package/dist/utils/getTextCompletion.d.ts

@@ -0,0 +1,3 @@
+export declare function getTextCompletion(text: null, fullText: string | string[]): null;
+export declare function getTextCompletion(text: string, fullText: string | string[]): string | null;
+export declare function getTextCompletion(text: string | null, fullText: string | string[]): string | null;

package/dist/utils/getTextCompletion.js

@@ -0,0 +1,12 @@
+export function getTextCompletion(text, fullText) {
+    if (text == null) {
+        return null;
+    }
+    const fullTexts = typeof fullText === "string" ? [fullText] : fullText;
+    for (const fullText of fullTexts) {
+        if (fullText.startsWith(text))
+            return fullText.slice(text.length);
+    }
+    return null;
+}
+//# sourceMappingURL=getTextCompletion.js.map

package/dist/utils/getTextCompletion.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"getTextCompletion.js","sourceRoot":"","sources":["../../src/utils/getTextCompletion.ts"],"names":[],"mappings":"AAGA,MAAM,UAAU,iBAAiB,CAAC,IAAmB,EAAE,QAA2B;IAC9E,IAAI,IAAI,IAAI,IAAI,EAAE;QACd,OAAO,IAAI,CAAC;KACf;IAED,MAAM,SAAS,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IAEvE,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE;QAC9B,IAAI,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC;YACzB,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;KAC1C;IAED,OAAO,IAAI,CAAC;AAChB,CAAC"}

package/dist/utils/removeNullFields.d.ts

@@ -0,0 +1 @@
+export declare function removeNullFields<T extends object>(obj: T): T;

package/dist/utils/removeNullFields.js

@@ -0,0 +1,9 @@
+export function removeNullFields(obj) {
+    const newObj = Object.assign({}, obj);
+    for (const key in obj) {
+        if (newObj[key] == null)
+            delete newObj[key];
+    }
+    return newObj;
+}
+//# sourceMappingURL=removeNullFields.js.map

package/dist/utils/removeNullFields.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"removeNullFields.js","sourceRoot":"","sources":["../../src/utils/removeNullFields.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,gBAAgB,CAAmB,GAAM;IACrD,MAAM,MAAM,GAAM,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IAEzC,KAAK,MAAM,GAAG,IAAI,GAAG,EAAE;QACnB,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI;YACnB,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;KAC1B;IAED,OAAO,MAAM,CAAC;AAClB,CAAC"}

package/dist/utils/spawnCommand.d.ts

@@ -1 +1,2 @@
-export declare function spawnCommand(command: string, args: string[], cwd: string): Promise<void>;
+/// <reference types="node" />
+export declare function spawnCommand(command: string, args: string[], cwd: string, env?: NodeJS.ProcessEnv): Promise<void>;

package/dist/utils/spawnCommand.js

@@ -1,5 +1,5 @@
 import spawn from "cross-spawn";
-export function spawnCommand(command, args, cwd) {
+export function spawnCommand(command, args, cwd, env = process.env) {
     function getCommandString() {
         let res = command;
         for (const arg of args) {
@@ -16,7 +16,7 @@ export function spawnCommand(command, args, cwd) {
         const child = spawn(command, args, {
             stdio: "inherit",
             cwd,
-            env: process.env,
+            env,
             detached: false,
             windowsHide: true
         });

package/dist/utils/spawnCommand.js.map

@@ -1 +1 @@
-{"version":3,"file":"spawnCommand.js","sourceRoot":"","sources":["../../src/utils/spawnCommand.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,aAAa,CAAC;AAEhC,MAAM,UAAU,YAAY,CAAC,OAAe,EAAE,IAAc,EAAE,GAAW;IACrE,SAAS,gBAAgB;QACrB,IAAI,GAAG,GAAG,OAAO,CAAC;QAElB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACpB,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;gBACnB,GAAG,IAAI,KAAK,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;aAC7C;iBAAM;gBACH,GAAG,IAAI,IAAI,GAAG,EAAE,CAAC;aACpB;SACJ;QAED,OAAO,GAAG,CAAC;IACf,CAAC;IAED,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACzC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE;YAC/B,KAAK,EAAE,SAAS;YAChB,GAAG;YACH,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,QAAQ,EAAE,KAAK;YACf,WAAW,EAAE,IAAI;SACpB,CAAC,CAAC;QAEH,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACtB,IAAI,IAAI,IAAI,CAAC;gBACT,OAAO,EAAE,CAAC;;gBAEV,MAAM,CAAC,IAAI,KAAK,CAAC,WAAW,gBAAgB,EAAE,qBAAqB,IAAI,EAAE,CAAC,CAAC,CAAC;QACpF,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC1B,KAAK,CAAC,EAAE,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,WAAW,gBAAgB,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC;QAC9F,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE;YACrB,IAAI,IAAI,IAAI,CAAC;gBACT,OAAO,EAAE,CAAC;;gBAEV,MAAM,CAAC,IAAI,KAAK,CAAC,WAAW,gBAAgB,EAAE,qBAAqB,IAAI,EAAE,CAAC,CAAC,CAAC;QACpF,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACP,CAAC"}
+{"version":3,"file":"spawnCommand.js","sourceRoot":"","sources":["../../src/utils/spawnCommand.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,aAAa,CAAC;AAEhC,MAAM,UAAU,YAAY,CAAC,OAAe,EAAE,IAAc,EAAE,GAAW,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG;IACxF,SAAS,gBAAgB;QACrB,IAAI,GAAG,GAAG,OAAO,CAAC;QAElB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACpB,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;gBACnB,GAAG,IAAI,KAAK,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;aAC7C;iBAAM;gBACH,GAAG,IAAI,IAAI,GAAG,EAAE,CAAC;aACpB;SACJ;QAED,OAAO,GAAG,CAAC;IACf,CAAC;IAED,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACzC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE;YAC/B,KAAK,EAAE,SAAS;YAChB,GAAG;YACH,GAAG;YACH,QAAQ,EAAE,KAAK;YACf,WAAW,EAAE,IAAI;SACpB,CAAC,CAAC;QAEH,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACtB,IAAI,IAAI,IAAI,CAAC;gBACT,OAAO,EAAE,CAAC;;gBAEV,MAAM,CAAC,IAAI,KAAK,CAAC,WAAW,gBAAgB,EAAE,qBAAqB,IAAI,EAAE,CAAC,CAAC,CAAC;QACpF,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC1B,KAAK,CAAC,EAAE,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,WAAW,gBAAgB,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC;QAC9F,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE;YACrB,IAAI,IAAI,IAAI,CAAC;gBACT,OAAO,EAAE,CAAC;;gBAEV,MAAM,CAAC,IAAI,KAAK,CAAC,WAAW,gBAAgB,EAAE,qBAAqB,IAAI,EAAE,CAAC,CAAC,CAAC;QACpF,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACP,CAAC"}

package/llama/addon.cpp

@@ -3,18 +3,26 @@
 #include <sstream>
 #include <vector>
 
+#include "common.h"
 #include "llama.h"
+#include "common/grammar-parser.h"
 #include "napi.h"
 
 class LLAMAModel : public Napi::ObjectWrap<LLAMAModel> {
   public:
     llama_context_params params;
     llama_model* model;
+    float temperature;
+    int32_t top_k;
+    float top_p;
 
     LLAMAModel(const Napi::CallbackInfo& info) : Napi::ObjectWrap<LLAMAModel>(info) {
         params = llama_context_default_params();
         params.seed = -1;
         params.n_ctx = 4096;
+        temperature = 0.0f;
+        top_k = 40;
+        top_p = 0.95f;
 
         // Get the model path
         std::string modelPath = info[0].As<Napi::String>().Utf8Value();
@@ -65,8 +73,21 @@ class LLAMAModel : public Napi::ObjectWrap<LLAMAModel> {
             if (options.Has("embedding")) {
                 params.embedding = options.Get("embedding").As<Napi::Boolean>().Value();
             }
+
+            if (options.Has("temperature")) {
+                temperature = options.Get("temperature").As<Napi::Number>().FloatValue();
+            }
+
+            if (options.Has("topK")) {
+                top_k = options.Get("topK").As<Napi::Number>().Int32Value();
+            }
+
+            if (options.Has("topP")) {
+                top_p = options.Get("topP").As<Napi::Number>().FloatValue();
+            }
         }
 
+        llama_backend_init(false);
         model = llama_load_model_from_file(modelPath.c_str(), params);
 
         if (model == NULL) {
@@ -84,20 +105,84 @@ class LLAMAModel : public Napi::ObjectWrap<LLAMAModel> {
     }
 };
 
+class LLAMAGrammar : public Napi::ObjectWrap<LLAMAGrammar> {
+  public:
+    grammar_parser::parse_state parsed_grammar;
+    llama_grammar *grammar = nullptr;
+
+    LLAMAGrammar(const Napi::CallbackInfo& info) : Napi::ObjectWrap<LLAMAGrammar>(info) {
+        // Get the model path
+        std::string grammarCode = info[0].As<Napi::String>().Utf8Value();
+        bool should_print_grammar = false;
+
+        if (info.Length() > 1 && info[1].IsObject()) {
+            Napi::Object options = info[1].As<Napi::Object>();
+
+            if (options.Has("printGrammar")) {
+                should_print_grammar = options.Get("printGrammar").As<Napi::Boolean>().Value();
+            }
+        }
+
+        parsed_grammar = grammar_parser::parse(grammarCode.c_str());
+        // will be empty (default) if there are parse errors
+        if (parsed_grammar.rules.empty()) {
+            Napi::Error::New(info.Env(), "Failed to parse grammar").ThrowAsJavaScriptException();
+            return;
+        }
+
+        if (should_print_grammar) {
+            grammar_parser::print_grammar(stderr, parsed_grammar);
+        }
+
+        std::vector<const llama_grammar_element *> grammar_rules(parsed_grammar.c_rules());
+        grammar = llama_grammar_init(
+            grammar_rules.data(), grammar_rules.size(), parsed_grammar.symbol_ids.at("root"));
+    }
+
+    ~LLAMAGrammar() {
+        if (grammar != nullptr) {
+            llama_grammar_free(grammar);
+            grammar = nullptr;
+        }
+    }
+
+    static void init(Napi::Object exports) {
+        exports.Set("LLAMAGrammar", DefineClass(exports.Env(), "LLAMAGrammar", {}));
+    }
+};
+
 class LLAMAContext : public Napi::ObjectWrap<LLAMAContext> {
   public:
   LLAMAModel* model;
   llama_context* ctx;
+  LLAMAGrammar* grammar;
+  bool use_grammar = false;
+
   LLAMAContext(const Napi::CallbackInfo& info) : Napi::ObjectWrap<LLAMAContext>(info) {
     model = Napi::ObjectWrap<LLAMAModel>::Unwrap(info[0].As<Napi::Object>());
     model->Ref();
     ctx = llama_new_context_with_model(model->model, model->params);
     Napi::MemoryManagement::AdjustExternalMemory(Env(), llama_get_state_size(ctx));
+
+    if (info.Length() > 1 && info[1].IsObject()) {
+        Napi::Object options = info[1].As<Napi::Object>();
+
+        if (options.Has("grammar")) {
+            grammar = Napi::ObjectWrap<LLAMAGrammar>::Unwrap(options.Get("grammar").As<Napi::Object>());
+            grammar->Ref();
+            use_grammar = true;
+        }
+    }
   }
   ~LLAMAContext() {
     Napi::MemoryManagement::AdjustExternalMemory(Env(), -(int64_t)llama_get_state_size(ctx));
     llama_free(ctx);
     model->Unref();
+
+    if (use_grammar) {
+        grammar->Unref();
+        use_grammar = false;
+    }
   }
   Napi::Value Encode(const Napi::CallbackInfo& info) {
     std::string text = info[0].As<Napi::String>().Utf8Value();
@@ -124,16 +209,42 @@ class LLAMAContext : public Napi::ObjectWrap<LLAMAContext> {
 
     // Decode each token and accumulate the result.
     for (size_t i = 0; i < tokens.ElementLength(); i++) {
-      const char* str = llama_token_to_str(ctx, (llama_token)tokens[i]);
-      if (str == nullptr) {
-        Napi::Error::New(info.Env(), "Invalid token").ThrowAsJavaScriptException();
-        return info.Env().Undefined();
-      }
-      ss << str;
+        const std::string piece = llama_token_to_piece(ctx, (llama_token)tokens[i]);
+
+        if (piece.empty()) {
+            continue;
+        }
+
+        ss << piece;
     }
 
     return Napi::String::New(info.Env(), ss.str());
   }
+  Napi::Value TokenBos(const Napi::CallbackInfo& info) {
+    return Napi::Number::From(info.Env(), llama_token_bos(ctx));
+  }
+  Napi::Value TokenEos(const Napi::CallbackInfo& info) {
+    return Napi::Number::From(info.Env(), llama_token_eos(ctx));
+  }
+  Napi::Value TokenNl(const Napi::CallbackInfo& info) {
+    return Napi::Number::From(info.Env(), llama_token_nl(ctx));
+  }
+  Napi::Value GetContextSize(const Napi::CallbackInfo& info) {
+    return Napi::Number::From(info.Env(), llama_n_ctx(ctx));
+  }
+  Napi::Value GetTokenString(const Napi::CallbackInfo& info) {
+    int token = info[0].As<Napi::Number>().Int32Value();
+    std::stringstream ss;
+
+    const char* str = llama_token_get_text(ctx, token);
+    if (str == nullptr) {
+      return info.Env().Undefined();
+    }
+
+    ss << str;
+
+    return Napi::String::New(info.Env(), ss.str());
+  }
   Napi::Value Eval(const Napi::CallbackInfo& info);
   static void init(Napi::Object exports) {
     exports.Set("LLAMAContext",
@@ -142,6 +253,11 @@ class LLAMAContext : public Napi::ObjectWrap<LLAMAContext> {
             {
                 InstanceMethod("encode", &LLAMAContext::Encode),
                 InstanceMethod("decode", &LLAMAContext::Decode),
+                InstanceMethod("tokenBos", &LLAMAContext::TokenBos),
+                InstanceMethod("tokenEos", &LLAMAContext::TokenEos),
+                InstanceMethod("tokenNl", &LLAMAContext::TokenNl),
+                InstanceMethod("getContextSize", &LLAMAContext::GetContextSize),
+                InstanceMethod("getTokenString", &LLAMAContext::GetTokenString),
                 InstanceMethod("eval", &LLAMAContext::Eval),
             }));
   }
@@ -151,7 +267,6 @@ class LLAMAContext : public Napi::ObjectWrap<LLAMAContext> {
 class LLAMAContextEvalWorker : Napi::AsyncWorker, Napi::Promise::Deferred {
   LLAMAContext* ctx;
   std::vector<llama_token> tokens;
-  std::vector<llama_token> restriction;
   llama_token result;
 
   public:
@@ -160,13 +275,6 @@ class LLAMAContextEvalWorker : Napi::AsyncWorker, Napi::Promise::Deferred {
     Napi::Uint32Array tokens = info[0].As<Napi::Uint32Array>();
     this->tokens.reserve(tokens.ElementLength());
     for (size_t i = 0; i < tokens.ElementLength(); i++) { this->tokens.push_back(static_cast<llama_token>(tokens[i])); }
-
-    if (info.Length() > 1 && info[1].IsTypedArray()) {
-      Napi::Uint32Array restriction = info[1].As<Napi::Uint32Array>();
-      this->restriction.reserve(restriction.ElementLength());
-      for (size_t i = 0; i < restriction.ElementLength(); i++) { this->restriction.push_back(static_cast<llama_token>(restriction[i])); }
-      std::sort(this->restriction.begin(), this->restriction.end());
-    }
   }
   ~LLAMAContextEvalWorker() { ctx->Unref(); }
   using Napi::AsyncWorker::Queue;
@@ -175,39 +283,72 @@ class LLAMAContextEvalWorker : Napi::AsyncWorker, Napi::Promise::Deferred {
   protected:
   void Execute() {
     // Perform the evaluation using llama_eval.
-    int r = llama_eval(ctx->ctx, tokens.data(), tokens.size(), llama_get_kv_cache_token_count(ctx->ctx), 6);
+    int r = llama_eval(ctx->ctx, tokens.data(), int(tokens.size()), llama_get_kv_cache_token_count(ctx->ctx), 6);
     if (r != 0) {
       SetError("Eval has failed");
       return;
     }
 
+    llama_token new_token_id = 0;
+
     // Select the best prediction.
-    float* logits = llama_get_logits(ctx->ctx);
-    int n_vocab = llama_n_vocab(ctx->ctx);
-    llama_token re;
-    if (restriction.empty()) {
-      float max = logits[0];
-      re = 0;
-      for (llama_token id = 1; id < n_vocab; id++) {
-        float logit = logits[id];
-        if (logit > max) {
-          max = logit;
-          re = id;
-        }
+    auto logits = llama_get_logits(ctx->ctx);
+    auto n_vocab = llama_n_vocab(ctx->ctx);
+
+    std::vector<llama_token_data> candidates;
+    candidates.reserve(n_vocab);
+
+    for (llama_token token_id = 0; token_id < n_vocab; token_id++) {
+      candidates.emplace_back(llama_token_data{ token_id, logits[token_id], 0.0f });
+    }
+
+    llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false };
+
+    float originalEosLogit = 0;
+    auto eos_token = llama_token_eos(ctx->ctx);
+
+    for (auto& candidate : candidates) {
+      if (candidate.id == eos_token) {
+        originalEosLogit = candidate.logit;
+        break;
       }
-    } else {
-      float max = logits[restriction[0]];
-      re = 0;
-      for (size_t i = 1; i < restriction.size(); i++) {
-        llama_token id = restriction[i];
-        float logit = logits[id];
-        if (logit > max) {
-          max = logit;
-          re = id;
-        }
+    }
+
+    if (ctx->use_grammar) {
+        llama_sample_grammar(ctx->ctx, &candidates_p, (ctx->grammar)->grammar);
+    }
+
+    for (auto& candidate : candidates) {
+      if (candidate.id == eos_token) {
+        candidate.logit = originalEosLogit;
+        break;
       }
     }
-    result = re;
+
+    if ((ctx->model)->temperature <= 0) {
+        new_token_id = llama_sample_token_greedy(ctx->ctx , &candidates_p);
+    } else {
+        const int32_t top_k = (ctx->model)->top_k <= 0 ? llama_n_vocab(ctx->ctx) : (ctx->model)->top_k;
+        const int32_t n_probs = 0; // Number of probabilities to keep - 0 = disabled
+        const float tfs_z = 1.00f; // Tail free sampling - 1.0 = disabled
+        const float typical_p = 1.00f; // Typical probability - 1.0 = disabled
+        const float top_p = (ctx->model)->top_p; // Top p sampling - 1.0 = disabled
+
+        // Temperature sampling
+        size_t min_keep = std::max(1, n_probs);
+        llama_sample_top_k(ctx->ctx, &candidates_p, top_k, min_keep);
+        llama_sample_tail_free(ctx->ctx, &candidates_p, tfs_z, min_keep);
+        llama_sample_typical(ctx->ctx, &candidates_p, typical_p, min_keep);
+        llama_sample_top_p(ctx->ctx, &candidates_p, top_p, min_keep);
+        llama_sample_temperature(ctx->ctx, &candidates_p, (ctx->model)->temperature);;
+        new_token_id = llama_sample_token(ctx->ctx, &candidates_p);
+    }
+
+    if (new_token_id != eos_token && ctx->use_grammar) {
+        llama_grammar_accept_token(ctx->ctx, (ctx->grammar)->grammar, new_token_id);
+    }
+
+    result = new_token_id;
   }
   void OnOK() {
     Napi::Env env = Napi::AsyncWorker::Env();
@@ -223,18 +364,15 @@ Napi::Value LLAMAContext::Eval(const Napi::CallbackInfo& info) {
   return worker->Promise();
 }
 
-Napi::Value tokenBos(const Napi::CallbackInfo& info) { return Napi::Number::From(info.Env(), llama_token_bos()); }
-Napi::Value tokenEos(const Napi::CallbackInfo& info) { return Napi::Number::From(info.Env(), llama_token_eos()); }
 Napi::Value systemInfo(const Napi::CallbackInfo& info) { return Napi::String::From(info.Env(), llama_print_system_info()); }
 
 Napi::Object registerCallback(Napi::Env env, Napi::Object exports) {
   llama_backend_init(false);
   exports.DefineProperties({
-      Napi::PropertyDescriptor::Function("tokenBos", tokenBos),
-      Napi::PropertyDescriptor::Function("tokenEos", tokenEos),
       Napi::PropertyDescriptor::Function("systemInfo", systemInfo),
   });
   LLAMAModel::init(exports);
+  LLAMAGrammar::init(exports);
   LLAMAContext::init(exports);
   return exports;
 }

package/llama/binariesGithubRelease.json

@@ -0,0 +1,3 @@
+{
+    "release": "b1107"
+}

package/llama/binding.gyp

@@ -7,15 +7,18 @@
         "llama.cpp/ggml.c",
         "llama.cpp/ggml-alloc.c",
         "llama.cpp/k_quants.c",
-        "llama.cpp/llama.cpp"
+        "llama.cpp/llama.cpp",
+        "llama.cpp/common/common.cpp",
+        "llama.cpp/common/grammar-parser.cpp"
       ],
       "include_dirs": [
         "<!@(node -p \"require('node-addon-api').include\")",
-        "llama.cpp"
+        "llama.cpp",
+        "llama.cpp/common"
       ],
       "cflags": ["-fexceptions"],
       "cflags_cc": ["-fexceptions"],
-      "defines": [ "GGML_USE_K_QUANTS", "NAPI_CPP_EXCEPTIONS" ],
+      "defines": ["GGML_USE_K_QUANTS", "NAPI_CPP_EXCEPTIONS"],
       "msvs_settings": {
         "VCCLCompilerTool": { "AdditionalOptions": [ '/arch:AVX2', '/EHsc' ] }
       }